Add support for automatic (nedb -> Postgres) migration to matrix-appservice-slack
This commit is contained in:
parent
9b95e1937c
commit
8675dedbdb
|
@ -106,7 +106,6 @@ matrix_docker_package_name: docker-ce
|
||||||
run_postgres_import: true
|
run_postgres_import: true
|
||||||
run_postgres_upgrade: true
|
run_postgres_upgrade: true
|
||||||
run_postgres_import_sqlite_db: true
|
run_postgres_import_sqlite_db: true
|
||||||
run_postgres_import_nedb: true
|
|
||||||
run_postgres_vacuum: true
|
run_postgres_vacuum: true
|
||||||
run_synapse_register_user: true
|
run_synapse_register_user: true
|
||||||
run_synapse_update_user_password: true
|
run_synapse_update_user_password: true
|
||||||
|
|
|
@ -51,3 +51,14 @@
|
||||||
with_items:
|
with_items:
|
||||||
- rooms.db
|
- rooms.db
|
||||||
- users.db
|
- users.db
|
||||||
|
|
||||||
|
- name: Inject result
|
||||||
|
set_fact:
|
||||||
|
matrix_playbook_runtime_results: |
|
||||||
|
{{
|
||||||
|
matrix_playbook_runtime_results|default([])
|
||||||
|
+
|
||||||
|
[
|
||||||
|
"NOTE: Your appservice-irc database files have been imported into Postgres. The original database files have been moved from `{{ matrix_appservice_irc_data_path }}/*.db` to `{{ matrix_appservice_irc_data_path }}/*.db.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete these files."
|
||||||
|
]
|
||||||
|
}}
|
||||||
|
|
|
@ -51,13 +51,18 @@ matrix_appservice_slack_database_password: ~
|
||||||
matrix_appservice_slack_database_hostname: 'matrix-postgres'
|
matrix_appservice_slack_database_hostname: 'matrix-postgres'
|
||||||
matrix_appservice_slack_database_port: 5432
|
matrix_appservice_slack_database_port: 5432
|
||||||
matrix_appservice_slack_database_name: matrix_appservice_slack
|
matrix_appservice_slack_database_name: matrix_appservice_slack
|
||||||
matrix_appservice_slack_database_file: /data
|
|
||||||
matrix_appservice_slack_database_connString: >-2
|
# This is just the Postgres connection string, if Postgres is used.
|
||||||
{%- if matrix_appservice_slack_database_engine == 'postgres' -%}
|
# Naming clashes with `matrix_appservice_slack_database_connectionString` somewhat.
|
||||||
postgresql://{{ matrix_appservice_slack_database_username }}:{{ matrix_appservice_slack_database_password }}@{{ matrix_appservice_slack_database_hostname }}:{{ matrix_appservice_slack_database_port }}/{{ matrix_appservice_slack_database_name }}?sslmode=disable
|
matrix_appservice_slack_database_connection_string: 'postgresql://{{ matrix_appservice_slack_database_username }}:{{ matrix_appservice_slack_database_password }}@{{ matrix_appservice_slack_database_hostname }}:{{ matrix_appservice_slack_database_port }}/{{ matrix_appservice_slack_database_name }}?sslmode=disable'
|
||||||
{%- elif matrix_appservice_slack_database_engine == 'nedb' -%}
|
|
||||||
{{ matrix_appservice_slack_database_engine }}://{{ matrix_appservice_slack_database_file }}
|
# This is what actually goes into `database.connectionString` for the bridge.
|
||||||
{%- endif -%}
|
matrix_appservice_slack_database_connectionString: "{{
|
||||||
|
{
|
||||||
|
'nedb': 'nedb:///data',
|
||||||
|
'postgres': matrix_appservice_slack_database_connection_string,
|
||||||
|
}[matrix_appservice_slack_database_engine]
|
||||||
|
}}"
|
||||||
|
|
||||||
|
|
||||||
matrix_appservice_slack_configuration_yaml: "{{ lookup('template', 'templates/config.yaml.j2') }}"
|
matrix_appservice_slack_configuration_yaml: "{{ lookup('template', 'templates/config.yaml.j2') }}"
|
||||||
|
|
|
@ -0,0 +1,66 @@
|
||||||
|
- name: Fail if Postgres not enabled
|
||||||
|
fail:
|
||||||
|
msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot migrate."
|
||||||
|
when: "not matrix_postgres_enabled|bool"
|
||||||
|
|
||||||
|
# Defaults
|
||||||
|
|
||||||
|
- name: Set postgres_start_wait_time, if not provided
|
||||||
|
set_fact:
|
||||||
|
postgres_start_wait_time: 15
|
||||||
|
when: "postgres_start_wait_time|default('') == ''"
|
||||||
|
|
||||||
|
# Actual import work
|
||||||
|
|
||||||
|
- name: Ensure matrix-postgres is started
|
||||||
|
service:
|
||||||
|
name: matrix-postgres
|
||||||
|
state: started
|
||||||
|
daemon_reload: yes
|
||||||
|
register: matrix_postgres_service_start_result
|
||||||
|
|
||||||
|
- name: Wait a bit, so that Postgres can start
|
||||||
|
wait_for:
|
||||||
|
timeout: "{{ postgres_start_wait_time }}"
|
||||||
|
delegate_to: 127.0.0.1
|
||||||
|
become: false
|
||||||
|
when: "matrix_postgres_service_start_result.changed|bool"
|
||||||
|
|
||||||
|
- name: Ensure matrix-appservice-slack is stopped
|
||||||
|
service:
|
||||||
|
name: matrix-appservice-slack
|
||||||
|
state: stopped
|
||||||
|
|
||||||
|
- name: Import appservice-slack NeDB database into Postgres
|
||||||
|
command:
|
||||||
|
cmd: >-
|
||||||
|
{{ matrix_host_command_docker }} run
|
||||||
|
--rm
|
||||||
|
--user={{ matrix_user_uid }}:{{ matrix_user_gid }}
|
||||||
|
--cap-drop=ALL
|
||||||
|
--network={{ matrix_docker_network }}
|
||||||
|
--mount type=bind,src={{ matrix_appservice_slack_data_path }},dst=/data
|
||||||
|
--entrypoint=/bin/sh
|
||||||
|
{{ matrix_appservice_slack_docker_image }}
|
||||||
|
-c
|
||||||
|
'/usr/local/bin/node /usr/src/app/lib/scripts/migrateToPostgres.js --dbdir /data --connectionString {{ matrix_appservice_slack_database_connection_string }}'
|
||||||
|
|
||||||
|
- name: Archive NeDB database files
|
||||||
|
command:
|
||||||
|
cmd: "mv {{ matrix_appservice_slack_data_path }}/{{ item }} {{ matrix_appservice_slack_data_path }}/{{ item }}.backup"
|
||||||
|
with_items:
|
||||||
|
- teams.db
|
||||||
|
- room-store.db
|
||||||
|
- user-store.db
|
||||||
|
- event-store.db
|
||||||
|
|
||||||
|
- name: Inject result
|
||||||
|
set_fact:
|
||||||
|
matrix_playbook_runtime_results: |
|
||||||
|
{{
|
||||||
|
matrix_playbook_runtime_results|default([])
|
||||||
|
+
|
||||||
|
[
|
||||||
|
"NOTE: Your appservice-slack database files have been imported into Postgres. The original database files have been moved from `{{ matrix_appservice_slack_data_path }}/*.db` to `{{ matrix_appservice_slack_data_path }}/*.db.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete these files."
|
||||||
|
]
|
||||||
|
}}
|
|
@ -1,12 +1,5 @@
|
||||||
---
|
---
|
||||||
|
|
||||||
- name: Ensure Appservice Slack image is pulled
|
|
||||||
docker_image:
|
|
||||||
name: "{{ matrix_appservice_slack_docker_image }}"
|
|
||||||
source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
|
|
||||||
force_source: "{{ matrix_appservice_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
|
|
||||||
force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_slack_docker_image_force_pull }}"
|
|
||||||
|
|
||||||
- name: Ensure AppService Slack paths exist
|
- name: Ensure AppService Slack paths exist
|
||||||
file:
|
file:
|
||||||
path: "{{ item }}"
|
path: "{{ item }}"
|
||||||
|
@ -19,6 +12,30 @@
|
||||||
- "{{ matrix_appservice_slack_config_path }}"
|
- "{{ matrix_appservice_slack_config_path }}"
|
||||||
- "{{ matrix_appservice_slack_data_path }}"
|
- "{{ matrix_appservice_slack_data_path }}"
|
||||||
|
|
||||||
|
- set_fact:
|
||||||
|
matrix_appservice_slack_requires_restart: false
|
||||||
|
|
||||||
|
- block:
|
||||||
|
- name: Check if a nedb database already exists
|
||||||
|
stat:
|
||||||
|
path: "{{ matrix_appservice_slack_data_path }}/teams.db"
|
||||||
|
register: matrix_appservice_slack_nedb_database_path_local_stat_result
|
||||||
|
|
||||||
|
- block:
|
||||||
|
- import_tasks: "{{ role_path }}/tasks/migrate_nedb_to_postgres.yml"
|
||||||
|
|
||||||
|
- set_fact:
|
||||||
|
matrix_appservice_slack_requires_restart: true
|
||||||
|
when: "matrix_appservice_slack_nedb_database_path_local_stat_result.stat.exists|bool"
|
||||||
|
when: "matrix_appservice_slack_database_engine == 'postgres'"
|
||||||
|
|
||||||
|
- name: Ensure Appservice Slack image is pulled
|
||||||
|
docker_image:
|
||||||
|
name: "{{ matrix_appservice_slack_docker_image }}"
|
||||||
|
source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
|
||||||
|
force_source: "{{ matrix_appservice_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
|
||||||
|
force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_slack_docker_image_force_pull }}"
|
||||||
|
|
||||||
- name: Ensure Matrix Appservice Slack config installed
|
- name: Ensure Matrix Appservice Slack config installed
|
||||||
copy:
|
copy:
|
||||||
content: "{{ matrix_appservice_slack_configuration|to_nice_yaml }}"
|
content: "{{ matrix_appservice_slack_configuration|to_nice_yaml }}"
|
||||||
|
@ -46,3 +63,9 @@
|
||||||
service:
|
service:
|
||||||
daemon_reload: yes
|
daemon_reload: yes
|
||||||
when: "matrix_appservice_slack_systemd_service_result.changed"
|
when: "matrix_appservice_slack_systemd_service_result.changed"
|
||||||
|
|
||||||
|
- name: Ensure matrix-appservice-slack.service restarted, if necessary
|
||||||
|
service:
|
||||||
|
name: "matrix-appservice-slack.service"
|
||||||
|
state: restarted
|
||||||
|
when: "matrix_appservice_slack_requires_restart|bool"
|
||||||
|
|
|
@ -10,23 +10,3 @@
|
||||||
- "matrix_appservice_slack_appservice_token"
|
- "matrix_appservice_slack_appservice_token"
|
||||||
- "matrix_appservice_slack_homeserver_token"
|
- "matrix_appservice_slack_homeserver_token"
|
||||||
- "matrix_appservice_slack_id_token"
|
- "matrix_appservice_slack_id_token"
|
||||||
|
|
||||||
- block:
|
|
||||||
- name: Check if a neDB database already exists
|
|
||||||
stat:
|
|
||||||
path: "{{ matrix_appservice_slack_data_path }}/"
|
|
||||||
register: matrix_appservice_slack_nedb_stat_result
|
|
||||||
|
|
||||||
- name: Fail if an neDB database already exists when using Postgres
|
|
||||||
fail:
|
|
||||||
msg: >-2
|
|
||||||
matrix_appservice_slack_database_engine has been set to `postgres` (which is our new default now).
|
|
||||||
However, we've discovered an existing neDB database in {{ matrix_appservice_slack_data_path }}/.
|
|
||||||
It appears that you've been using this bridge with the neDB engine until now.
|
|
||||||
To continue using neDB, opt into it explicitly: add `matrix_appservice_slack_database_engine: nedb` to your vars.yml file and re-run this same command.
|
|
||||||
Alternatively, to migrate your existing neDB database to Postgres:
|
|
||||||
1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
|
|
||||||
2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_slack_data_path }} postgres_connection_string_variable_name=matrix_appservice_slack_database_connString'`)
|
|
||||||
3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
|
|
||||||
when: "matrix_appservice_slack_nedb_stat_result.stat.exists"
|
|
||||||
when: "matrix_appservice_slack_database_engine == 'postgres'"
|
|
||||||
|
|
|
@ -11,9 +11,10 @@ homeserver:
|
||||||
|
|
||||||
{% if matrix_appservice_slack_database_engine == 'nedb' %}
|
{% if matrix_appservice_slack_database_engine == 'nedb' %}
|
||||||
dbdir: "/data"
|
dbdir: "/data"
|
||||||
{% endif %}
|
{% else %}
|
||||||
db:
|
db:
|
||||||
engine: "{{ matrix_appservice_slack_database_engine }}"
|
engine: {{ matrix_appservice_slack_database_engine|to_json }}
|
||||||
connectionString: {{ matrix_appservice_slack_database_connString | to_json }}
|
connectionString: {{ matrix_appservice_slack_database_connectionString|to_json }}
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
matrix_admin_room: "{{ matrix_appservice_slack_control_room_id }}"
|
matrix_admin_room: "{{ matrix_appservice_slack_control_room_id }}"
|
||||||
|
|
|
@ -1,98 +0,0 @@
|
||||||
---
|
|
||||||
|
|
||||||
# Pre-checks
|
|
||||||
|
|
||||||
- name: Fail if Postgres not enabled
|
|
||||||
fail:
|
|
||||||
msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot import."
|
|
||||||
when: "not matrix_postgres_enabled|bool"
|
|
||||||
|
|
||||||
- name: Fail if playbook called incorrectly
|
|
||||||
fail:
|
|
||||||
msg: "The `nedb_database_path` variable needs to be provided to this playbook, via --extra-vars"
|
|
||||||
when: "nedb_database_path is not defined or nedb_database_path.startswith('<')"
|
|
||||||
|
|
||||||
- name: Check if the provided nedb database file exists
|
|
||||||
stat:
|
|
||||||
path: "{{ nedb_database_path }}"
|
|
||||||
register: nedb_database_path_stat_result
|
|
||||||
|
|
||||||
- name: Fail if provided SQLite database file doesn't exist
|
|
||||||
fail:
|
|
||||||
msg: "File cannot be found on the server at {{ nedb_database_path }}"
|
|
||||||
when: "not nedb_database_path_stat_result.stat.exists"
|
|
||||||
|
|
||||||
# We either expect `postgres_db_connection_string` specifying a full Postgres database connection string,
|
|
||||||
# or `postgres_connection_string_variable_name`, specifying a name of a variable, which contains a valid connection string.
|
|
||||||
|
|
||||||
- block:
|
|
||||||
- name: Fail if postgres_connection_string_variable_name points to an undefined variable
|
|
||||||
fail: msg="postgres_connection_string_variable_name is defined, but there is no variable with the name `{{ postgres_connection_string_variable_name }}`"
|
|
||||||
when: "postgres_connection_string_variable_name not in vars"
|
|
||||||
|
|
||||||
- name: Get Postgres connection string from variable
|
|
||||||
set_fact:
|
|
||||||
postgres_db_connection_string: "{{ lookup('vars', postgres_connection_string_variable_name) }}"
|
|
||||||
when: 'postgres_connection_string_variable_name is defined'
|
|
||||||
|
|
||||||
- name: Fail if playbook called incorrectly
|
|
||||||
fail:
|
|
||||||
msg: >-
|
|
||||||
Either a `postgres_db_connection_string` variable or a `postgres_connection_string_variable_name` needs to be provided to this playbook, via `--extra-vars`.
|
|
||||||
Example: `--extra-vars="postgres_db_connection_string=postgresql://username:password@localhost:<port>/database_name"` or `--extra-vars="postgres_connection_string_variable_name=matrix_appservice_discord_database_connString"`
|
|
||||||
when: "postgres_db_connection_string is not defined or not postgres_db_connection_string.startswith('postgresql://')"
|
|
||||||
|
|
||||||
|
|
||||||
# Defaults
|
|
||||||
|
|
||||||
- name: Set postgres_start_wait_time, if not provided
|
|
||||||
set_fact:
|
|
||||||
postgres_start_wait_time: 15
|
|
||||||
when: "postgres_start_wait_time|default('') == ''"
|
|
||||||
|
|
||||||
|
|
||||||
# Actual import work
|
|
||||||
|
|
||||||
- name: Ensure matrix-postgres is started
|
|
||||||
service:
|
|
||||||
name: matrix-postgres
|
|
||||||
state: started
|
|
||||||
daemon_reload: yes
|
|
||||||
register: matrix_postgres_service_start_result
|
|
||||||
|
|
||||||
- name: Wait a bit, so that Postgres can start
|
|
||||||
wait_for:
|
|
||||||
timeout: "{{ postgres_start_wait_time }}"
|
|
||||||
delegate_to: 127.0.0.1
|
|
||||||
become: false
|
|
||||||
when: "matrix_postgres_service_start_result.changed|bool"
|
|
||||||
|
|
||||||
# No migration.sh available, but found this:
|
|
||||||
# https://github.com/matrix-org/matrix-appservice-slack/blob/develop/src/scripts/migrateToPostgres.ts
|
|
||||||
# Usage should be similar to appservice_irc
|
|
||||||
- name: Import appservice_slack NeDB database from {{ sqlite_database_path }} into Postgres
|
|
||||||
when: database == 'appservice_slack'
|
|
||||||
command:
|
|
||||||
cmd: >-
|
|
||||||
{{ matrix_host_command_docker }} run
|
|
||||||
--rm
|
|
||||||
--user={{ matrix_user_uid }}:{{ matrix_user_gid }}
|
|
||||||
--cap-drop=ALL
|
|
||||||
--network={{ matrix_docker_network }}
|
|
||||||
--mount type=bind,src={{ matrix_appservice_irc_data_path }},dst=/data,ro
|
|
||||||
--entrypoint=/bin/sh
|
|
||||||
{{ matrix_appservice_slack_docker_image }}
|
|
||||||
-c
|
|
||||||
'node /lib/scripts/migrate-db-to-pgres.js -d /data -p passkey.pem -c {{ postgres_db_connection_string }}'
|
|
||||||
|
|
||||||
|
|
||||||
- name: Inject result
|
|
||||||
set_fact:
|
|
||||||
matrix_playbook_runtime_results: |
|
|
||||||
{{
|
|
||||||
matrix_playbook_runtime_results|default([])
|
|
||||||
+
|
|
||||||
[
|
|
||||||
"NOTE: Your NeDB database file has been imported into Postgres. The original directory has been moved from `{{ nedb_database_path }}` to `{{ nedb_database_path }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file."
|
|
||||||
]
|
|
||||||
}}
|
|
|
@ -32,14 +32,6 @@
|
||||||
tags:
|
tags:
|
||||||
- import-generic-sqlite-db
|
- import-generic-sqlite-db
|
||||||
|
|
||||||
# Imports slacks neDB to postgres.
|
|
||||||
- import_tasks: "{{ role_path }}/tasks/import_nedb.yml"
|
|
||||||
vars:
|
|
||||||
database: appservice_slack
|
|
||||||
when: run_postgres_import_nedb|bool
|
|
||||||
tags:
|
|
||||||
- import-slack-nedb
|
|
||||||
|
|
||||||
- import_tasks: "{{ role_path }}/tasks/upgrade_postgres.yml"
|
- import_tasks: "{{ role_path }}/tasks/upgrade_postgres.yml"
|
||||||
when: run_postgres_upgrade|bool
|
when: run_postgres_upgrade|bool
|
||||||
tags:
|
tags:
|
||||||
|
|
Loading…
Reference in a new issue