From 1f988969a5e91d3b2e4421f35db1563ca4ac957d Mon Sep 17 00:00:00 2001 From: Scott Crossen Date: Tue, 13 Oct 2020 16:18:38 -0700 Subject: [PATCH 001/217] Added role for dynamic dns --- docs/configuring-playbook-budget-builds.md | 21 +++++++++++++ docs/configuring-playbook.md | 1 + roles/matrix-dynamic-dns/defaults/main.yml | 14 +++++++++ roles/matrix-dynamic-dns/tasks/init.yml | 4 +++ roles/matrix-dynamic-dns/tasks/install.yml | 24 ++++++++++++++ roles/matrix-dynamic-dns/tasks/main.yml | 28 +++++++++++++++++ .../tasks/platform/archlinux.yml | 16 ++++++++++ .../tasks/platform/centos.yml | 23 ++++++++++++++ .../tasks/platform/debian.yml | 18 +++++++++++ .../tasks/platform/main.yml | 11 +++++++ roles/matrix-dynamic-dns/tasks/uninstall.yml | 31 +++++++++++++++++++ .../tasks/validate_config.yml | 19 ++++++++++++ .../templates/ddclient.conf.j2 | 7 +++++ .../matrix-dynamic-dns/templates/ddclient.j2 | 4 +++ setup.yml | 1 + 15 files changed, 222 insertions(+) create mode 100644 docs/configuring-playbook-budget-builds.md create mode 100644 roles/matrix-dynamic-dns/defaults/main.yml create mode 100644 roles/matrix-dynamic-dns/tasks/init.yml create mode 100644 roles/matrix-dynamic-dns/tasks/install.yml create mode 100644 roles/matrix-dynamic-dns/tasks/main.yml create mode 100644 roles/matrix-dynamic-dns/tasks/platform/archlinux.yml create mode 100644 roles/matrix-dynamic-dns/tasks/platform/centos.yml create mode 100644 roles/matrix-dynamic-dns/tasks/platform/debian.yml create mode 100644 roles/matrix-dynamic-dns/tasks/platform/main.yml create mode 100644 roles/matrix-dynamic-dns/tasks/uninstall.yml create mode 100644 roles/matrix-dynamic-dns/tasks/validate_config.yml create mode 100644 roles/matrix-dynamic-dns/templates/ddclient.conf.j2 create mode 100644 roles/matrix-dynamic-dns/templates/ddclient.j2 diff --git a/docs/configuring-playbook-budget-builds.md b/docs/configuring-playbook-budget-builds.md new file mode 100644 index 00000000..0fb73835 --- /dev/null +++ b/docs/configuring-playbook-budget-builds.md @@ -0,0 +1,21 @@ +# Tips for deploying Matrix on a Budget + +## Dynamic DNS + +Most cloud providers / ISPs will charge you extra for a static IP address. If you're +not hosting a highly reliable homeserver you can workaround this via dynamic DNS. To +set this up, you'll need to get the username/password from your DNS provider. For +google domains, this process is described [here](https://support.google.com/domains/answer/6147083). +After you've gotten the proper credentials you can add the following config to your `host-vars`: + +``` +matrix_dynamic_dns_username: XXXXXXXXXXXXXXXX +matrix_dynamic_dns_password: XXXXXXXXXXXXXXXX +matrix_dynamic_dns_provider: 'domains.google.com' +``` + +## Additional Reading + +Additional resources: + +- https://matrix.org/docs/guides/free-small-matrix-server \ No newline at end of file diff --git a/docs/configuring-playbook.md b/docs/configuring-playbook.md index b174637e..8b0c5537 100644 --- a/docs/configuring-playbook.md +++ b/docs/configuring-playbook.md @@ -33,6 +33,7 @@ When you're done with all the configuration you'd like to do, continue with [Ins - [Setting up the Jitsi video-conferencing platform](configuring-playbook-jitsi.md) (optional) +- [Setting up budget builds or resource-constrained builds](configuring-playbook-budget-builds.md) (optional) ### Core service adjustments diff --git a/roles/matrix-dynamic-dns/defaults/main.yml b/roles/matrix-dynamic-dns/defaults/main.yml new file mode 100644 index 00000000..a0afc99e --- /dev/null +++ b/roles/matrix-dynamic-dns/defaults/main.yml @@ -0,0 +1,14 @@ +# Whether dynamic dns is enabled +matrix_dynamic_dns_enabled: true + +# Allowed values: 'daemon', 'dhcp' +matrix_dynamic_dns_mode: 'dhcp' + +# The DNS provider domain +matrix_dynamic_dns_provider: 'domains.google.com' + +# The dynamic dns protocol +matrix_dynamic_dns_protocol: 'dyndns2' + +# The dynamic dns daemon interval +matrix_dynamic_dns_daemon_interval: '300' diff --git a/roles/matrix-dynamic-dns/tasks/init.yml b/roles/matrix-dynamic-dns/tasks/init.yml new file mode 100644 index 00000000..4cddb8b5 --- /dev/null +++ b/roles/matrix-dynamic-dns/tasks/init.yml @@ -0,0 +1,4 @@ +- set_fact: + matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['ddclient'] }}" + when: "matrix_dynamic_dns_enabled|bool" + \ No newline at end of file diff --git a/roles/matrix-dynamic-dns/tasks/install.yml b/roles/matrix-dynamic-dns/tasks/install.yml new file mode 100644 index 00000000..edaf6864 --- /dev/null +++ b/roles/matrix-dynamic-dns/tasks/install.yml @@ -0,0 +1,24 @@ +--- + +- name: Ensure ddclient domain config exists + template: + src: "{{ role_path }}/templates/ddclient.conf.j2" + dest: "/etc/ddclient.conf" + mode: 0600 + register: matrix_dynamic_dns_ddclient_domain_config + +- name: Ensure ddclient client config directory exists + file: + path: "/etc/default" + state: directory + mode: 0700 + owner: "{{ user_username }}" + group: "{{ user_groupname }}" + +- name: Ensure ddclient client config exists + template: + src: "{{ role_path }}/templates/ddclient.j2" + dest: "/etc/default/ddclient" + mode: 0600 + register: matrix_dynamic_dns_ddclient_client_config + \ No newline at end of file diff --git a/roles/matrix-dynamic-dns/tasks/main.yml b/roles/matrix-dynamic-dns/tasks/main.yml new file mode 100644 index 00000000..727bf16e --- /dev/null +++ b/roles/matrix-dynamic-dns/tasks/main.yml @@ -0,0 +1,28 @@ +- import_tasks: "{{ role_path }}/tasks/init.yml" + tags: + - always + +- import_tasks: "{{ role_path }}/tasks/validate_config.yml" + when: "run_setup|bool and matrix_dynamic_dns_enabled|bool" + tags: + - setup-all + - setup-dynamic-dns + +- import_tasks: "{{ role_path }}/tasks/install.yml" + when: "run_setup|bool and matrix_dynamic_dns_enabled|bool" + tags: + - setup-all + - setup-dynamic-dns + +- import_tasks: "{{ role_path }}/tasks/platform/main.yml" + when: "run_setup|bool" + tags: + - setup-all + - setup-dynamic-dns + +- import_tasks: "{{ role_path }}/tasks/uninstall.yml" + when: "run_setup|bool and not matrix_dynamic_dns_enabled|bool" + tags: + - setup-all + - setup-dynamic-dns + \ No newline at end of file diff --git a/roles/matrix-dynamic-dns/tasks/platform/archlinux.yml b/roles/matrix-dynamic-dns/tasks/platform/archlinux.yml new file mode 100644 index 00000000..773cfce6 --- /dev/null +++ b/roles/matrix-dynamic-dns/tasks/platform/archlinux.yml @@ -0,0 +1,16 @@ +--- + +- name: Ensure ddclient is installed + pacman: + name: ddclient + state: latest + when: "run_setup|bool and matrix_dynamic_dns_enabled|bool" + +- name: Ensure ddclient is uninstalled + pacman: + name: ddclient + state: absent + update_cache: true + become: true + when: "run_setup|bool and not matrix_dynamic_dns_enabled|bool" + \ No newline at end of file diff --git a/roles/matrix-dynamic-dns/tasks/platform/centos.yml b/roles/matrix-dynamic-dns/tasks/platform/centos.yml new file mode 100644 index 00000000..5e84ef10 --- /dev/null +++ b/roles/matrix-dynamic-dns/tasks/platform/centos.yml @@ -0,0 +1,23 @@ +--- + +- name: Ensure yum packages are installed + yum: + name: epel-release + state: latest + update_cache: yes + when: "run_setup|bool and matrix_dynamic_dns_enabled|bool" + +- name: Ensure ddclient is installed + yum: + name: ddclient + state: latest + when: "run_setup|bool and matrix_dynamic_dns_enabled|bool" + +- name: Ensure ddclient is uninstalled + yum: + name: + - ddclient + - epel-release + state: absent + when: "run_setup|bool and not matrix_dynamic_dns_enabled|bool" + \ No newline at end of file diff --git a/roles/matrix-dynamic-dns/tasks/platform/debian.yml b/roles/matrix-dynamic-dns/tasks/platform/debian.yml new file mode 100644 index 00000000..f6d664fd --- /dev/null +++ b/roles/matrix-dynamic-dns/tasks/platform/debian.yml @@ -0,0 +1,18 @@ +--- + +- name: Ensure ddclient is installed + apt: + name: ddclient + state: present + update_cache: true + become: true + when: "run_setup|bool and matrix_dynamic_dns_enabled|bool" + +- name: Ensure ddclient is uninstalled + apt: + name: ddclient + state: absent + update_cache: true + become: true + when: "run_setup|bool and not matrix_dynamic_dns_enabled|bool" + \ No newline at end of file diff --git a/roles/matrix-dynamic-dns/tasks/platform/main.yml b/roles/matrix-dynamic-dns/tasks/platform/main.yml new file mode 100644 index 00000000..ca973749 --- /dev/null +++ b/roles/matrix-dynamic-dns/tasks/platform/main.yml @@ -0,0 +1,11 @@ +--- + +- include_tasks: "{{ role_path }}/tasks/platform/centos.yml" + when: ansible_distribution == 'CentOS' + +# The instructions are the same for Debian, Ubuntu, and Raspbian +- include_tasks: "{{ role_path }}/tasks/platform/debian.yml" + when: ansible_distribution == 'Debian' + +- include_tasks: "{{ role_path }}/tasks/platform/archlinux.yml" + when: ansible_distribution == 'Archlinux' diff --git a/roles/matrix-dynamic-dns/tasks/uninstall.yml b/roles/matrix-dynamic-dns/tasks/uninstall.yml new file mode 100644 index 00000000..27604585 --- /dev/null +++ b/roles/matrix-dynamic-dns/tasks/uninstall.yml @@ -0,0 +1,31 @@ +--- + +- name: Check existence of ddclient.service + stat: + path: "{{ systemd_path }}/ddclient.service" + register: matrix_dynamic_dns_ddclient_service_stat + +- name: Ensure ddclient.service is stopped + service: + name: dynamic-dns + state: stopped + daemon_reload: yes + when: "matrix_dynamic_dns_ddclient_service_stat.stat.exists" + +- name: Ensure systemd reloaded after ddclient.service removal + service: + daemon_reload: yes + when: "matrix_dynamic_dns_ddclient_service_stat.stat.exists" + +- name: Ensure ddclient.service doesn't exist + file: + path: "{{ systemd_path }}/ddclient.service" + state: absent + when: "matrix_dynamic_dns_ddclient_service_stat.stat.exists" + +- name: Ensure ddclient configuration files don't exist + file: + path: + - "etc/ddclient.conf" + - "etc/default/ddclient" + state: absent diff --git a/roles/matrix-dynamic-dns/tasks/validate_config.yml b/roles/matrix-dynamic-dns/tasks/validate_config.yml new file mode 100644 index 00000000..bb351e49 --- /dev/null +++ b/roles/matrix-dynamic-dns/tasks/validate_config.yml @@ -0,0 +1,19 @@ +--- + +- name: Fail if required settings not defined + fail: + msg: >- + You need to define a required configuration setting (`{{ item }}`). + when: "vars[item] == ''" + with_items: + - "matrix_dynamic_dns_username" + - "matrix_dynamic_dns_password" + - "matrix_domain" + - "matrix_dynamic_dns_provider" + - "matrix_dynamic_dns_mode" + +- name: Fail if dynamic dns mode is incorrect + fail: + msg: >- + matrix_dynamic_dns_mode needs to be set to 'daemon' or 'startup' + when: "matrix_dynamic_dns_enabled and matrix_dynamic_dns_mode != 'daemon' and matrix_dynamic_dns_mode != 'dhcp'" diff --git a/roles/matrix-dynamic-dns/templates/ddclient.conf.j2 b/roles/matrix-dynamic-dns/templates/ddclient.conf.j2 new file mode 100644 index 00000000..dcb5c1e4 --- /dev/null +++ b/roles/matrix-dynamic-dns/templates/ddclient.conf.j2 @@ -0,0 +1,7 @@ +protocol={{ matrix_dynamic_dns_protocol }} +use=web +ssl=yes +server={{ matrix_dynamic_dns_provider }} +login='{{ matrix_dynamic_dns_username }}' +password='{{ matrix_dynamic_dns_password }}' +{{ matrix_domain }} \ No newline at end of file diff --git a/roles/matrix-dynamic-dns/templates/ddclient.j2 b/roles/matrix-dynamic-dns/templates/ddclient.j2 new file mode 100644 index 00000000..eb5d053d --- /dev/null +++ b/roles/matrix-dynamic-dns/templates/ddclient.j2 @@ -0,0 +1,4 @@ +run_dhclient="{{ (matrix_dynamic_dns_mode == 'dhcp') | lower }}" +run_ipup="false" +run_daemon="{{ (matrix_dynamic_dns_mode == 'daemon') | lower }}" +daemon_interval="{{ matrix_dynamic_dns_daemon_interval }}" diff --git a/setup.yml b/setup.yml index 68c4e4ee..f07f0278 100755 --- a/setup.yml +++ b/setup.yml @@ -5,6 +5,7 @@ roles: - matrix-base + - dynamic-dns - matrix-mailer - matrix-postgres - matrix-corporal From 53bc7a77e1c851132e41df963c650d434418652a Mon Sep 17 00:00:00 2001 From: Scott Crossen Date: Tue, 13 Oct 2020 16:47:09 -0700 Subject: [PATCH 002/217] fixed EOF issues --- roles/matrix-dynamic-dns/tasks/init.yml | 1 - roles/matrix-dynamic-dns/tasks/install.yml | 1 - roles/matrix-dynamic-dns/tasks/main.yml | 1 - roles/matrix-dynamic-dns/tasks/platform/archlinux.yml | 1 - roles/matrix-dynamic-dns/tasks/platform/centos.yml | 1 - roles/matrix-dynamic-dns/tasks/platform/debian.yml | 1 - roles/matrix-dynamic-dns/templates/ddclient.conf.j2 | 2 +- 7 files changed, 1 insertion(+), 7 deletions(-) diff --git a/roles/matrix-dynamic-dns/tasks/init.yml b/roles/matrix-dynamic-dns/tasks/init.yml index 4cddb8b5..97f5eaec 100644 --- a/roles/matrix-dynamic-dns/tasks/init.yml +++ b/roles/matrix-dynamic-dns/tasks/init.yml @@ -1,4 +1,3 @@ - set_fact: matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['ddclient'] }}" when: "matrix_dynamic_dns_enabled|bool" - \ No newline at end of file diff --git a/roles/matrix-dynamic-dns/tasks/install.yml b/roles/matrix-dynamic-dns/tasks/install.yml index edaf6864..0948e79d 100644 --- a/roles/matrix-dynamic-dns/tasks/install.yml +++ b/roles/matrix-dynamic-dns/tasks/install.yml @@ -21,4 +21,3 @@ dest: "/etc/default/ddclient" mode: 0600 register: matrix_dynamic_dns_ddclient_client_config - \ No newline at end of file diff --git a/roles/matrix-dynamic-dns/tasks/main.yml b/roles/matrix-dynamic-dns/tasks/main.yml index 727bf16e..7646dd13 100644 --- a/roles/matrix-dynamic-dns/tasks/main.yml +++ b/roles/matrix-dynamic-dns/tasks/main.yml @@ -25,4 +25,3 @@ tags: - setup-all - setup-dynamic-dns - \ No newline at end of file diff --git a/roles/matrix-dynamic-dns/tasks/platform/archlinux.yml b/roles/matrix-dynamic-dns/tasks/platform/archlinux.yml index 773cfce6..10e20802 100644 --- a/roles/matrix-dynamic-dns/tasks/platform/archlinux.yml +++ b/roles/matrix-dynamic-dns/tasks/platform/archlinux.yml @@ -13,4 +13,3 @@ update_cache: true become: true when: "run_setup|bool and not matrix_dynamic_dns_enabled|bool" - \ No newline at end of file diff --git a/roles/matrix-dynamic-dns/tasks/platform/centos.yml b/roles/matrix-dynamic-dns/tasks/platform/centos.yml index 5e84ef10..7375ad32 100644 --- a/roles/matrix-dynamic-dns/tasks/platform/centos.yml +++ b/roles/matrix-dynamic-dns/tasks/platform/centos.yml @@ -20,4 +20,3 @@ - epel-release state: absent when: "run_setup|bool and not matrix_dynamic_dns_enabled|bool" - \ No newline at end of file diff --git a/roles/matrix-dynamic-dns/tasks/platform/debian.yml b/roles/matrix-dynamic-dns/tasks/platform/debian.yml index f6d664fd..c41dbaf9 100644 --- a/roles/matrix-dynamic-dns/tasks/platform/debian.yml +++ b/roles/matrix-dynamic-dns/tasks/platform/debian.yml @@ -15,4 +15,3 @@ update_cache: true become: true when: "run_setup|bool and not matrix_dynamic_dns_enabled|bool" - \ No newline at end of file diff --git a/roles/matrix-dynamic-dns/templates/ddclient.conf.j2 b/roles/matrix-dynamic-dns/templates/ddclient.conf.j2 index dcb5c1e4..6d873688 100644 --- a/roles/matrix-dynamic-dns/templates/ddclient.conf.j2 +++ b/roles/matrix-dynamic-dns/templates/ddclient.conf.j2 @@ -4,4 +4,4 @@ ssl=yes server={{ matrix_dynamic_dns_provider }} login='{{ matrix_dynamic_dns_username }}' password='{{ matrix_dynamic_dns_password }}' -{{ matrix_domain }} \ No newline at end of file +{{ matrix_domain }} From 51cca4c312b70ac09de5c423681c69e045e725cf Mon Sep 17 00:00:00 2001 From: Scott Crossen Date: Fri, 16 Oct 2020 21:21:58 -0700 Subject: [PATCH 003/217] Added containerization --- roles/matrix-dynamic-dns/defaults/main.yml | 45 +++++++++++--- roles/matrix-dynamic-dns/tasks/init.yml | 6 +- .../tasks/install_daemon.yml | 59 +++++++++++++++++++ .../tasks/{install.yml => install_dhcp.yml} | 14 +++-- roles/matrix-dynamic-dns/tasks/main.yml | 18 ++++-- .../tasks/uninstall_daemon.yml | 24 ++++++++ .../{uninstall.yml => uninstall_dhcp.yml} | 12 ++-- .../tasks/validate_config.yml | 4 +- .../templates/ddclient.conf.j2 | 22 +++++-- .../matrix-dynamic-dns/templates/ddclient.j2 | 4 -- .../systemd/matrix-dynamic-dns.service.j2 | 41 +++++++++++++ setup.yml | 2 +- 12 files changed, 211 insertions(+), 40 deletions(-) create mode 100644 roles/matrix-dynamic-dns/tasks/install_daemon.yml rename roles/matrix-dynamic-dns/tasks/{install.yml => install_dhcp.yml} (55%) create mode 100644 roles/matrix-dynamic-dns/tasks/uninstall_daemon.yml rename roles/matrix-dynamic-dns/tasks/{uninstall.yml => uninstall_dhcp.yml} (93%) delete mode 100644 roles/matrix-dynamic-dns/templates/ddclient.j2 create mode 100644 roles/matrix-dynamic-dns/templates/systemd/matrix-dynamic-dns.service.j2 diff --git a/roles/matrix-dynamic-dns/defaults/main.yml b/roles/matrix-dynamic-dns/defaults/main.yml index a0afc99e..c2490837 100644 --- a/roles/matrix-dynamic-dns/defaults/main.yml +++ b/roles/matrix-dynamic-dns/defaults/main.yml @@ -1,14 +1,43 @@ # Whether dynamic dns is enabled -matrix_dynamic_dns_enabled: true +matrix_dynamic_dns_enabled: false # Allowed values: 'daemon', 'dhcp' -matrix_dynamic_dns_mode: 'dhcp' - -# The DNS provider domain -matrix_dynamic_dns_provider: 'domains.google.com' - -# The dynamic dns protocol -matrix_dynamic_dns_protocol: 'dyndns2' +matrix_dynamic_dns_mode: 'daemon' # The dynamic dns daemon interval matrix_dynamic_dns_daemon_interval: '300' + +# The docker container to use when in daemon mode +matrix_dynamic_dns_docker_image: 'linuxserver/ddclient' + +# The image to force pull +matrix_dynamic_dns_docker_image_force_pull: "{{ matrix_dynamic_dns_docker_image.endswith(':latest') }}" + +# List of extra arguments to pass to the ontainer daemon mode +matrix_dynamic_dns_container_extra_arguments: [] + +# List of wanted services when running in daemon mode +matrix_dynamic_dns_systemd_wanted_services_list: [] + +# List of required services when running in daemon mode +matrix_dynamic_dns_systemd_required_services_list: ['docker.service'] + +# Build the container from source when running in daemon mode +matrix_dynamic_dns_container_image_self_build: false + +# Config paths +matrix_dynamic_dns_base_path: "{{ matrix_base_data_path }}/dynamic-dns" +matrix_dynamic_dns_config_path: "{{ matrix_dynamic_dns_base_path }}/config" +matrix_dynamic_dns_docker_src_files_path: "{{ matrix_dynamic_dns_base_path }}/docker-src" + +# Config options +matrix_dynamic_dns_use: "web" +matrix_dynamic_dns_static: false +matrix_dynamic_dns_custom: false +matrix_dynamic_dns_zone: "" +matrix_dynamic_dns_ttl: "" +matrix_dynamic_dns_mx: "" +matrix_dynamic_dns_wildcard: false +matrix_dynamic_dns_protocol: 'dyndns2' +matrix_dynamic_dns_provider: 'domains.google.com' +matrix_dynamic_dns_domain: '{{ matrix_domain }}' diff --git a/roles/matrix-dynamic-dns/tasks/init.yml b/roles/matrix-dynamic-dns/tasks/init.yml index 97f5eaec..2954ac9a 100644 --- a/roles/matrix-dynamic-dns/tasks/init.yml +++ b/roles/matrix-dynamic-dns/tasks/init.yml @@ -1,3 +1,7 @@ - set_fact: matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['ddclient'] }}" - when: "matrix_dynamic_dns_enabled|bool" + when: "matrix_dynamic_dns_enabled|bool and matrix_dynamic_dns_mode == 'dhcp'" + +- set_fact: + matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-dynamic-dns'] }}" + when: "matrix_dynamic_dns_enabled|bool and matrix_dynamic_dns_mode == 'daemon'" diff --git a/roles/matrix-dynamic-dns/tasks/install_daemon.yml b/roles/matrix-dynamic-dns/tasks/install_daemon.yml new file mode 100644 index 00000000..816dc7c3 --- /dev/null +++ b/roles/matrix-dynamic-dns/tasks/install_daemon.yml @@ -0,0 +1,59 @@ +--- + +- name: Ensure Dynamic DNS image is pulled + docker_image: + name: "{{ matrix_dynamic_dns_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_dynamic_dns_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_dynamic_dns_docker_image_force_pull }}" + when: matrix_dynamic_dns_enabled|bool and not matrix_dynamic_dns_container_image_self_build + +- name: Ensure Dynamic DNS paths exist + file: + path: "{{ item.path }}" + state: directory + mode: 0750 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + with_items: + - { path: "{{ matrix_dynamic_dns_base_path }}", when: true } + - { path: "{{ matrix_dynamic_dns_config_path }}", when: true } + - { path: "{{ matrix_dynamic_dns_docker_src_files_path }}", when: "{{ matrix_dynamic_dns_container_image_self_build }}" } + when: matrix_dynamic_dns_enabled|bool and item.when|bool + +- name: Ensure Dynamic DNS repository is present on self build + git: + repo: https://github.com/linuxserver/docker-ddclient.git + dest: "{{ matrix_dynamic_dns_docker_src_files_path }}" + force: "yes" + when: "matrix_dynamic_dns_enabled|bool and matrix_dynamic_dns_container_image_self_build" + +- name: Ensure Dynamic DNS Docker image is built + docker_image: + name: "{{ matrix_dynamic_dns_docker_image }}" + source: build + build: + dockerfile: Dockerfile + path: "{{ matrix_dynamic_dns_docker_src_files_path }}" + pull: yes + when: "matrix_dynamic_dns_enabled|bool and matrix_dynamic_dns_container_image_self_build" + +- name: Ensure Dynamic DNS ddclient.conf installed + template: + src: "{{ role_path }}/templates/ddclient.conf.j2" + dest: "{{ matrix_dynamic_dns_config_path }}/ddclient.conf" + mode: 0644 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + +- name: Ensure matrix-dynamic-dns.service installed + template: + src: "{{ role_path }}/templates/systemd/matrix-dynamic-dns.service.j2" + dest: "/etc/systemd/system/matrix-dynamic-dns.service" + mode: 0644 + register: matrix_dynamic_dns_systemd_service_result + +- name: Ensure systemd reloaded after matrix-dynamic-dns.service installation + service: + daemon_reload: yes + when: "matrix_dynamic_dns_systemd_service_result.changed" diff --git a/roles/matrix-dynamic-dns/tasks/install.yml b/roles/matrix-dynamic-dns/tasks/install_dhcp.yml similarity index 55% rename from roles/matrix-dynamic-dns/tasks/install.yml rename to roles/matrix-dynamic-dns/tasks/install_dhcp.yml index 0948e79d..cb340686 100644 --- a/roles/matrix-dynamic-dns/tasks/install.yml +++ b/roles/matrix-dynamic-dns/tasks/install_dhcp.yml @@ -5,19 +5,21 @@ src: "{{ role_path }}/templates/ddclient.conf.j2" dest: "/etc/ddclient.conf" mode: 0600 - register: matrix_dynamic_dns_ddclient_domain_config - name: Ensure ddclient client config directory exists file: path: "/etc/default" state: directory mode: 0700 - owner: "{{ user_username }}" - group: "{{ user_groupname }}" + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" - name: Ensure ddclient client config exists - template: - src: "{{ role_path }}/templates/ddclient.j2" + copy: + content: "run_dhclient=\"true\"\nrun_ipup=\"false\"\nrun_daemon=\"false\"" dest: "/etc/default/ddclient" mode: 0600 - register: matrix_dynamic_dns_ddclient_client_config + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + +- import_tasks: "{{ role_path }}/tasks/platform/main.yml" diff --git a/roles/matrix-dynamic-dns/tasks/main.yml b/roles/matrix-dynamic-dns/tasks/main.yml index 7646dd13..2d999568 100644 --- a/roles/matrix-dynamic-dns/tasks/main.yml +++ b/roles/matrix-dynamic-dns/tasks/main.yml @@ -8,20 +8,26 @@ - setup-all - setup-dynamic-dns -- import_tasks: "{{ role_path }}/tasks/install.yml" - when: "run_setup|bool and matrix_dynamic_dns_enabled|bool" +- import_tasks: "{{ role_path }}/tasks/install_dhcp.yml" + when: "run_setup|bool and matrix_dynamic_dns_enabled|bool and matrix_dynamic_dns_mode == 'dhcp'" tags: - setup-all - setup-dynamic-dns -- import_tasks: "{{ role_path }}/tasks/platform/main.yml" - when: "run_setup|bool" +- import_tasks: "{{ role_path }}/tasks/uninstall_dhcp.yml" + when: "run_setup|bool and not matrix_dynamic_dns_enabled|bool and matrix_dynamic_dns_mode == 'dhcp'" tags: - setup-all - setup-dynamic-dns -- import_tasks: "{{ role_path }}/tasks/uninstall.yml" - when: "run_setup|bool and not matrix_dynamic_dns_enabled|bool" +- import_tasks: "{{ role_path }}/tasks/install_daemon.yml" + when: "run_setup|bool and matrix_dynamic_dns_enabled|bool and matrix_dynamic_dns_mode == 'daemon'" + tags: + - setup-all + - setup-dynamic-dns + +- import_tasks: "{{ role_path }}/tasks/uninstall_daemon.yml" + when: "run_setup|bool and not matrix_dynamic_dns_enabled|bool and matrix_dynamic_dns_mode == 'daemon'" tags: - setup-all - setup-dynamic-dns diff --git a/roles/matrix-dynamic-dns/tasks/uninstall_daemon.yml b/roles/matrix-dynamic-dns/tasks/uninstall_daemon.yml new file mode 100644 index 00000000..8480f3c3 --- /dev/null +++ b/roles/matrix-dynamic-dns/tasks/uninstall_daemon.yml @@ -0,0 +1,24 @@ +--- + +- name: Check existence of matrix-dynamic-dns service + stat: + path: "{{ systemd_path }}m/matrix-dynamic-dns.service" + register: matrix_dynamic_dns_service_stat + +- name: Ensure matrix-dynamic-dns is stopped + service: + name: matrix-dynamic-dns + state: stopped + daemon_reload: yes + when: "matrix_dynamic_dns_service_stat.stat.exists" + +- name: Ensure matrix-dynamic-dns.service doesn't exist + file: + path: "{{ systemd_path }}m/matrix-dynamic-dns.service" + state: absent + when: "matrix_dynamic_dns_service_stat.stat.exists" + +- name: Ensure systemd reloaded after matrix-dynamic-dns.service removal + service: + daemon_reload: yes + when: "matrix_dynamic_dns_service_stat.stat.exists" diff --git a/roles/matrix-dynamic-dns/tasks/uninstall.yml b/roles/matrix-dynamic-dns/tasks/uninstall_dhcp.yml similarity index 93% rename from roles/matrix-dynamic-dns/tasks/uninstall.yml rename to roles/matrix-dynamic-dns/tasks/uninstall_dhcp.yml index 27604585..09018f68 100644 --- a/roles/matrix-dynamic-dns/tasks/uninstall.yml +++ b/roles/matrix-dynamic-dns/tasks/uninstall_dhcp.yml @@ -1,5 +1,7 @@ --- +- import_tasks: "{{ role_path }}/tasks/platform/main.yml" + - name: Check existence of ddclient.service stat: path: "{{ systemd_path }}/ddclient.service" @@ -12,11 +14,6 @@ daemon_reload: yes when: "matrix_dynamic_dns_ddclient_service_stat.stat.exists" -- name: Ensure systemd reloaded after ddclient.service removal - service: - daemon_reload: yes - when: "matrix_dynamic_dns_ddclient_service_stat.stat.exists" - - name: Ensure ddclient.service doesn't exist file: path: "{{ systemd_path }}/ddclient.service" @@ -29,3 +26,8 @@ - "etc/ddclient.conf" - "etc/default/ddclient" state: absent + +- name: Ensure systemd reloaded after ddclient.service removal + service: + daemon_reload: yes + when: "matrix_dynamic_dns_ddclient_service_stat.stat.exists" diff --git a/roles/matrix-dynamic-dns/tasks/validate_config.yml b/roles/matrix-dynamic-dns/tasks/validate_config.yml index bb351e49..650f9b70 100644 --- a/roles/matrix-dynamic-dns/tasks/validate_config.yml +++ b/roles/matrix-dynamic-dns/tasks/validate_config.yml @@ -6,8 +6,6 @@ You need to define a required configuration setting (`{{ item }}`). when: "vars[item] == ''" with_items: - - "matrix_dynamic_dns_username" - - "matrix_dynamic_dns_password" - "matrix_domain" - "matrix_dynamic_dns_provider" - "matrix_dynamic_dns_mode" @@ -15,5 +13,5 @@ - name: Fail if dynamic dns mode is incorrect fail: msg: >- - matrix_dynamic_dns_mode needs to be set to 'daemon' or 'startup' + matrix_dynamic_dns_mode needs to be set to 'daemon' or 'dhcp' when: "matrix_dynamic_dns_enabled and matrix_dynamic_dns_mode != 'daemon' and matrix_dynamic_dns_mode != 'dhcp'" diff --git a/roles/matrix-dynamic-dns/templates/ddclient.conf.j2 b/roles/matrix-dynamic-dns/templates/ddclient.conf.j2 index 6d873688..ffbf4c05 100644 --- a/roles/matrix-dynamic-dns/templates/ddclient.conf.j2 +++ b/roles/matrix-dynamic-dns/templates/ddclient.conf.j2 @@ -1,7 +1,17 @@ -protocol={{ matrix_dynamic_dns_protocol }} -use=web +{% if matrix_dynamic_dns_mode == "daemon" %}} +daemon={{ matrix_dynamic_dns_daemon_interval }} +syslog=no +pid=/var/run/ddclient/ddclient.pid {% endif %} ssl=yes -server={{ matrix_dynamic_dns_provider }} -login='{{ matrix_dynamic_dns_username }}' -password='{{ matrix_dynamic_dns_password }}' -{{ matrix_domain }} +use={{ matrix_dynamic_dns_use }} +protocol={{ matrix_dynamic_dns_protocol }} +server={{ matrix_dynamic_dns_provider }} {% if matrix_dynamic_dns_username %} +login='{{ matrix_dynamic_dns_username }}' {% endif %} {% if matrix_dynamic_dns_username %} +password='{{ matrix_dynamic_dns_password }}' {% endif %} {% if matrix_dynamic_dns_static %} +static=yes {% endif %} {% if matrix_dynamic_dns_custom %} +custom=yes {% endif %} {% if matrix_dynamic_dns_zone %} +zone={{ matrix_dynamic_dns_zone }} {% endif %} {% if matrix_dynamic_dns_ttl %} +ttl={{ matrix_dynamic_dns_ttl }} {% endif %} {% if matrix_dynamic_dns_mx %} +mx={{ matrix_dynamic_dns_mx }} {% endif %} {% if matrix_dynamic_dns_wildcard %} +wildcard=yes {% endif %} +{{ matrix_dynamic_dns_domain }} diff --git a/roles/matrix-dynamic-dns/templates/ddclient.j2 b/roles/matrix-dynamic-dns/templates/ddclient.j2 deleted file mode 100644 index eb5d053d..00000000 --- a/roles/matrix-dynamic-dns/templates/ddclient.j2 +++ /dev/null @@ -1,4 +0,0 @@ -run_dhclient="{{ (matrix_dynamic_dns_mode == 'dhcp') | lower }}" -run_ipup="false" -run_daemon="{{ (matrix_dynamic_dns_mode == 'daemon') | lower }}" -daemon_interval="{{ matrix_dynamic_dns_daemon_interval }}" diff --git a/roles/matrix-dynamic-dns/templates/systemd/matrix-dynamic-dns.service.j2 b/roles/matrix-dynamic-dns/templates/systemd/matrix-dynamic-dns.service.j2 new file mode 100644 index 00000000..9f866e1e --- /dev/null +++ b/roles/matrix-dynamic-dns/templates/systemd/matrix-dynamic-dns.service.j2 @@ -0,0 +1,41 @@ +#jinja2: lstrip_blocks: "True" +[Unit] +Description=Matrix Dynamic DNS +{% for service in matrix_dynamic_dns_systemd_required_services_list %} +Requires={{ service }} +After={{ service }} +{% endfor %} +{% for service in matrix_dynamic_dns_systemd_wanted_services_list %} +Wants={{ service }} +{% endfor %} + +[Service] +Type=simple +ExecStartPre=-{{ matrix_host_command_docker }} kill matrix-dynamic-dns +ExecStartPre=-{{ matrix_host_command_docker }} rm matrix-dynamic-dns + +# Intentional delay, so that the homeserver (we likely depend on) can manage to start. +ExecStartPre={{ matrix_host_command_sleep }} 5 + +ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-dynamic-dns \ + --log-driver=none \ + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + --cap-drop=ALL \ + --network={{ matrix_docker_network }} \ + -e PUID={{ matrix_user_uid }} \ + -e PGID={{ matrix_user_gid }} \ + -e CONFIG_PATH=/config/config.yaml \ + -v {{ matrix_dynamic_dns_config_path }}:/config:z \ + {% for arg in matrix_dynamic_dns_container_extra_arguments %} + {{ arg }} \ + {% endfor %} + {{ matrix_dynamic_dns_docker_image }} + +ExecStop=-{{ matrix_host_command_docker }} kill matrix-dynamic-dns +ExecStop=-{{ matrix_host_command_docker }} rm matrix-dynamic-dns +Restart=always +RestartSec=30 +SyslogIdentifier=matrix-dynamic-dns + +[Install] +WantedBy=multi-user.target diff --git a/setup.yml b/setup.yml index f07f0278..a1f4f9ab 100755 --- a/setup.yml +++ b/setup.yml @@ -5,7 +5,7 @@ roles: - matrix-base - - dynamic-dns + - matrix-dynamic-dns - matrix-mailer - matrix-postgres - matrix-corporal From 806f98447c771adaf4fa027346e5df8cd32c15c9 Mon Sep 17 00:00:00 2001 From: Scott Crossen Date: Fri, 16 Oct 2020 21:26:58 -0700 Subject: [PATCH 004/217] Removed directory creation --- roles/matrix-dynamic-dns/tasks/install_dhcp.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/roles/matrix-dynamic-dns/tasks/install_dhcp.yml b/roles/matrix-dynamic-dns/tasks/install_dhcp.yml index cb340686..31f88178 100644 --- a/roles/matrix-dynamic-dns/tasks/install_dhcp.yml +++ b/roles/matrix-dynamic-dns/tasks/install_dhcp.yml @@ -6,14 +6,6 @@ dest: "/etc/ddclient.conf" mode: 0600 -- name: Ensure ddclient client config directory exists - file: - path: "/etc/default" - state: directory - mode: 0700 - owner: "{{ matrix_user_username }}" - group: "{{ matrix_user_groupname }}" - - name: Ensure ddclient client config exists copy: content: "run_dhclient=\"true\"\nrun_ipup=\"false\"\nrun_daemon=\"false\"" From 8e6d7d9022711154929d538ca7b912f2d76b1fbb Mon Sep 17 00:00:00 2001 From: Scott Crossen Date: Fri, 16 Oct 2020 21:28:43 -0700 Subject: [PATCH 005/217] changed domain naming --- docs/configuring-playbook-budget-builds.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuring-playbook-budget-builds.md b/docs/configuring-playbook-budget-builds.md index 0fb73835..318c3c0c 100644 --- a/docs/configuring-playbook-budget-builds.md +++ b/docs/configuring-playbook-budget-builds.md @@ -6,7 +6,7 @@ Most cloud providers / ISPs will charge you extra for a static IP address. If yo not hosting a highly reliable homeserver you can workaround this via dynamic DNS. To set this up, you'll need to get the username/password from your DNS provider. For google domains, this process is described [here](https://support.google.com/domains/answer/6147083). -After you've gotten the proper credentials you can add the following config to your `host-vars`: +After you've gotten the proper credentials you can add the following config to your inventory/host_vars/matrix.DOMAIN/vars.yml: ``` matrix_dynamic_dns_username: XXXXXXXXXXXXXXXX From de1511b4bbf2f6ee1924a3796eee5d66933039c0 Mon Sep 17 00:00:00 2001 From: Scott Crossen Date: Fri, 16 Oct 2020 21:31:07 -0700 Subject: [PATCH 006/217] Fixed valdiation --- roles/matrix-dynamic-dns/tasks/validate_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-dynamic-dns/tasks/validate_config.yml b/roles/matrix-dynamic-dns/tasks/validate_config.yml index 650f9b70..61a8ea23 100644 --- a/roles/matrix-dynamic-dns/tasks/validate_config.yml +++ b/roles/matrix-dynamic-dns/tasks/validate_config.yml @@ -6,7 +6,7 @@ You need to define a required configuration setting (`{{ item }}`). when: "vars[item] == ''" with_items: - - "matrix_domain" + - "matrix_dynamic_dns_domain" - "matrix_dynamic_dns_provider" - "matrix_dynamic_dns_mode" From 19721be8b11ef55964cd2255617b869e949419a5 Mon Sep 17 00:00:00 2001 From: Scott Crossen Date: Sun, 18 Oct 2020 21:05:23 -0700 Subject: [PATCH 007/217] removed dhcp option --- roles/matrix-dynamic-dns/defaults/main.yml | 13 +++----- roles/matrix-dynamic-dns/tasks/init.yml | 6 +--- .../tasks/{install_daemon.yml => install.yml} | 0 .../matrix-dynamic-dns/tasks/install_dhcp.yml | 17 ---------- roles/matrix-dynamic-dns/tasks/main.yml | 20 +++-------- .../{uninstall_daemon.yml => uninstall.yml} | 0 .../tasks/uninstall_dhcp.yml | 33 ------------------- .../tasks/validate_config.yml | 7 ---- .../templates/ddclient.conf.j2 | 3 +- 9 files changed, 11 insertions(+), 88 deletions(-) rename roles/matrix-dynamic-dns/tasks/{install_daemon.yml => install.yml} (100%) delete mode 100644 roles/matrix-dynamic-dns/tasks/install_dhcp.yml rename roles/matrix-dynamic-dns/tasks/{uninstall_daemon.yml => uninstall.yml} (100%) delete mode 100644 roles/matrix-dynamic-dns/tasks/uninstall_dhcp.yml diff --git a/roles/matrix-dynamic-dns/defaults/main.yml b/roles/matrix-dynamic-dns/defaults/main.yml index c2490837..e57b47da 100644 --- a/roles/matrix-dynamic-dns/defaults/main.yml +++ b/roles/matrix-dynamic-dns/defaults/main.yml @@ -1,28 +1,25 @@ # Whether dynamic dns is enabled matrix_dynamic_dns_enabled: false -# Allowed values: 'daemon', 'dhcp' -matrix_dynamic_dns_mode: 'daemon' - # The dynamic dns daemon interval matrix_dynamic_dns_daemon_interval: '300' -# The docker container to use when in daemon mode +# The docker container to use when in mode matrix_dynamic_dns_docker_image: 'linuxserver/ddclient' # The image to force pull matrix_dynamic_dns_docker_image_force_pull: "{{ matrix_dynamic_dns_docker_image.endswith(':latest') }}" -# List of extra arguments to pass to the ontainer daemon mode +# List of extra arguments to pass to the ontainer mode matrix_dynamic_dns_container_extra_arguments: [] -# List of wanted services when running in daemon mode +# List of wanted services when running in mode matrix_dynamic_dns_systemd_wanted_services_list: [] -# List of required services when running in daemon mode +# List of required services when running in mode matrix_dynamic_dns_systemd_required_services_list: ['docker.service'] -# Build the container from source when running in daemon mode +# Build the container from source when running in mode matrix_dynamic_dns_container_image_self_build: false # Config paths diff --git a/roles/matrix-dynamic-dns/tasks/init.yml b/roles/matrix-dynamic-dns/tasks/init.yml index 2954ac9a..7b87fdb1 100644 --- a/roles/matrix-dynamic-dns/tasks/init.yml +++ b/roles/matrix-dynamic-dns/tasks/init.yml @@ -1,7 +1,3 @@ -- set_fact: - matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['ddclient'] }}" - when: "matrix_dynamic_dns_enabled|bool and matrix_dynamic_dns_mode == 'dhcp'" - - set_fact: matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-dynamic-dns'] }}" - when: "matrix_dynamic_dns_enabled|bool and matrix_dynamic_dns_mode == 'daemon'" + when: "matrix_dynamic_dns_enabled|bool" diff --git a/roles/matrix-dynamic-dns/tasks/install_daemon.yml b/roles/matrix-dynamic-dns/tasks/install.yml similarity index 100% rename from roles/matrix-dynamic-dns/tasks/install_daemon.yml rename to roles/matrix-dynamic-dns/tasks/install.yml diff --git a/roles/matrix-dynamic-dns/tasks/install_dhcp.yml b/roles/matrix-dynamic-dns/tasks/install_dhcp.yml deleted file mode 100644 index 31f88178..00000000 --- a/roles/matrix-dynamic-dns/tasks/install_dhcp.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- - -- name: Ensure ddclient domain config exists - template: - src: "{{ role_path }}/templates/ddclient.conf.j2" - dest: "/etc/ddclient.conf" - mode: 0600 - -- name: Ensure ddclient client config exists - copy: - content: "run_dhclient=\"true\"\nrun_ipup=\"false\"\nrun_daemon=\"false\"" - dest: "/etc/default/ddclient" - mode: 0600 - owner: "{{ matrix_user_username }}" - group: "{{ matrix_user_groupname }}" - -- import_tasks: "{{ role_path }}/tasks/platform/main.yml" diff --git a/roles/matrix-dynamic-dns/tasks/main.yml b/roles/matrix-dynamic-dns/tasks/main.yml index 2d999568..f9aaab8f 100644 --- a/roles/matrix-dynamic-dns/tasks/main.yml +++ b/roles/matrix-dynamic-dns/tasks/main.yml @@ -8,26 +8,14 @@ - setup-all - setup-dynamic-dns -- import_tasks: "{{ role_path }}/tasks/install_dhcp.yml" - when: "run_setup|bool and matrix_dynamic_dns_enabled|bool and matrix_dynamic_dns_mode == 'dhcp'" +- import_tasks: "{{ role_path }}/tasks/install.yml" + when: "run_setup|bool and matrix_dynamic_dns_enabled|bool" tags: - setup-all - setup-dynamic-dns -- import_tasks: "{{ role_path }}/tasks/uninstall_dhcp.yml" - when: "run_setup|bool and not matrix_dynamic_dns_enabled|bool and matrix_dynamic_dns_mode == 'dhcp'" - tags: - - setup-all - - setup-dynamic-dns - -- import_tasks: "{{ role_path }}/tasks/install_daemon.yml" - when: "run_setup|bool and matrix_dynamic_dns_enabled|bool and matrix_dynamic_dns_mode == 'daemon'" - tags: - - setup-all - - setup-dynamic-dns - -- import_tasks: "{{ role_path }}/tasks/uninstall_daemon.yml" - when: "run_setup|bool and not matrix_dynamic_dns_enabled|bool and matrix_dynamic_dns_mode == 'daemon'" +- import_tasks: "{{ role_path }}/tasks/uninstall.yml" + when: "run_setup|bool and not matrix_dynamic_dns_enabled|bool" tags: - setup-all - setup-dynamic-dns diff --git a/roles/matrix-dynamic-dns/tasks/uninstall_daemon.yml b/roles/matrix-dynamic-dns/tasks/uninstall.yml similarity index 100% rename from roles/matrix-dynamic-dns/tasks/uninstall_daemon.yml rename to roles/matrix-dynamic-dns/tasks/uninstall.yml diff --git a/roles/matrix-dynamic-dns/tasks/uninstall_dhcp.yml b/roles/matrix-dynamic-dns/tasks/uninstall_dhcp.yml deleted file mode 100644 index 09018f68..00000000 --- a/roles/matrix-dynamic-dns/tasks/uninstall_dhcp.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- - -- import_tasks: "{{ role_path }}/tasks/platform/main.yml" - -- name: Check existence of ddclient.service - stat: - path: "{{ systemd_path }}/ddclient.service" - register: matrix_dynamic_dns_ddclient_service_stat - -- name: Ensure ddclient.service is stopped - service: - name: dynamic-dns - state: stopped - daemon_reload: yes - when: "matrix_dynamic_dns_ddclient_service_stat.stat.exists" - -- name: Ensure ddclient.service doesn't exist - file: - path: "{{ systemd_path }}/ddclient.service" - state: absent - when: "matrix_dynamic_dns_ddclient_service_stat.stat.exists" - -- name: Ensure ddclient configuration files don't exist - file: - path: - - "etc/ddclient.conf" - - "etc/default/ddclient" - state: absent - -- name: Ensure systemd reloaded after ddclient.service removal - service: - daemon_reload: yes - when: "matrix_dynamic_dns_ddclient_service_stat.stat.exists" diff --git a/roles/matrix-dynamic-dns/tasks/validate_config.yml b/roles/matrix-dynamic-dns/tasks/validate_config.yml index 61a8ea23..2895f407 100644 --- a/roles/matrix-dynamic-dns/tasks/validate_config.yml +++ b/roles/matrix-dynamic-dns/tasks/validate_config.yml @@ -8,10 +8,3 @@ with_items: - "matrix_dynamic_dns_domain" - "matrix_dynamic_dns_provider" - - "matrix_dynamic_dns_mode" - -- name: Fail if dynamic dns mode is incorrect - fail: - msg: >- - matrix_dynamic_dns_mode needs to be set to 'daemon' or 'dhcp' - when: "matrix_dynamic_dns_enabled and matrix_dynamic_dns_mode != 'daemon' and matrix_dynamic_dns_mode != 'dhcp'" diff --git a/roles/matrix-dynamic-dns/templates/ddclient.conf.j2 b/roles/matrix-dynamic-dns/templates/ddclient.conf.j2 index ffbf4c05..651712db 100644 --- a/roles/matrix-dynamic-dns/templates/ddclient.conf.j2 +++ b/roles/matrix-dynamic-dns/templates/ddclient.conf.j2 @@ -1,7 +1,6 @@ -{% if matrix_dynamic_dns_mode == "daemon" %}} daemon={{ matrix_dynamic_dns_daemon_interval }} syslog=no -pid=/var/run/ddclient/ddclient.pid {% endif %} +pid=/var/run/ddclient/ddclient.pid ssl=yes use={{ matrix_dynamic_dns_use }} protocol={{ matrix_dynamic_dns_protocol }} From e7d79a95dc9a20948ea439cc308e185057fafd3a Mon Sep 17 00:00:00 2001 From: Scott Crossen Date: Mon, 19 Oct 2020 10:46:02 -0700 Subject: [PATCH 008/217] removed platform-specific stuff --- .../tasks/platform/archlinux.yml | 15 ------------- .../tasks/platform/centos.yml | 22 ------------------- .../tasks/platform/debian.yml | 17 -------------- .../tasks/platform/main.yml | 11 ---------- 4 files changed, 65 deletions(-) delete mode 100644 roles/matrix-dynamic-dns/tasks/platform/archlinux.yml delete mode 100644 roles/matrix-dynamic-dns/tasks/platform/centos.yml delete mode 100644 roles/matrix-dynamic-dns/tasks/platform/debian.yml delete mode 100644 roles/matrix-dynamic-dns/tasks/platform/main.yml diff --git a/roles/matrix-dynamic-dns/tasks/platform/archlinux.yml b/roles/matrix-dynamic-dns/tasks/platform/archlinux.yml deleted file mode 100644 index 10e20802..00000000 --- a/roles/matrix-dynamic-dns/tasks/platform/archlinux.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- - -- name: Ensure ddclient is installed - pacman: - name: ddclient - state: latest - when: "run_setup|bool and matrix_dynamic_dns_enabled|bool" - -- name: Ensure ddclient is uninstalled - pacman: - name: ddclient - state: absent - update_cache: true - become: true - when: "run_setup|bool and not matrix_dynamic_dns_enabled|bool" diff --git a/roles/matrix-dynamic-dns/tasks/platform/centos.yml b/roles/matrix-dynamic-dns/tasks/platform/centos.yml deleted file mode 100644 index 7375ad32..00000000 --- a/roles/matrix-dynamic-dns/tasks/platform/centos.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- - -- name: Ensure yum packages are installed - yum: - name: epel-release - state: latest - update_cache: yes - when: "run_setup|bool and matrix_dynamic_dns_enabled|bool" - -- name: Ensure ddclient is installed - yum: - name: ddclient - state: latest - when: "run_setup|bool and matrix_dynamic_dns_enabled|bool" - -- name: Ensure ddclient is uninstalled - yum: - name: - - ddclient - - epel-release - state: absent - when: "run_setup|bool and not matrix_dynamic_dns_enabled|bool" diff --git a/roles/matrix-dynamic-dns/tasks/platform/debian.yml b/roles/matrix-dynamic-dns/tasks/platform/debian.yml deleted file mode 100644 index c41dbaf9..00000000 --- a/roles/matrix-dynamic-dns/tasks/platform/debian.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- - -- name: Ensure ddclient is installed - apt: - name: ddclient - state: present - update_cache: true - become: true - when: "run_setup|bool and matrix_dynamic_dns_enabled|bool" - -- name: Ensure ddclient is uninstalled - apt: - name: ddclient - state: absent - update_cache: true - become: true - when: "run_setup|bool and not matrix_dynamic_dns_enabled|bool" diff --git a/roles/matrix-dynamic-dns/tasks/platform/main.yml b/roles/matrix-dynamic-dns/tasks/platform/main.yml deleted file mode 100644 index ca973749..00000000 --- a/roles/matrix-dynamic-dns/tasks/platform/main.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- - -- include_tasks: "{{ role_path }}/tasks/platform/centos.yml" - when: ansible_distribution == 'CentOS' - -# The instructions are the same for Debian, Ubuntu, and Raspbian -- include_tasks: "{{ role_path }}/tasks/platform/debian.yml" - when: ansible_distribution == 'Debian' - -- include_tasks: "{{ role_path }}/tasks/platform/archlinux.yml" - when: ansible_distribution == 'Archlinux' From efeb651789871128eaa6a09706b5cf1999000e3c Mon Sep 17 00:00:00 2001 From: Scott Crossen Date: Mon, 19 Oct 2020 11:25:01 -0700 Subject: [PATCH 009/217] Removed typo --- roles/matrix-dynamic-dns/tasks/uninstall.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/matrix-dynamic-dns/tasks/uninstall.yml b/roles/matrix-dynamic-dns/tasks/uninstall.yml index 8480f3c3..1be0e190 100644 --- a/roles/matrix-dynamic-dns/tasks/uninstall.yml +++ b/roles/matrix-dynamic-dns/tasks/uninstall.yml @@ -2,7 +2,7 @@ - name: Check existence of matrix-dynamic-dns service stat: - path: "{{ systemd_path }}m/matrix-dynamic-dns.service" + path: "{{ systemd_path }}/matrix-dynamic-dns.service" register: matrix_dynamic_dns_service_stat - name: Ensure matrix-dynamic-dns is stopped @@ -14,7 +14,7 @@ - name: Ensure matrix-dynamic-dns.service doesn't exist file: - path: "{{ systemd_path }}m/matrix-dynamic-dns.service" + path: "{{ systemd_path }}/matrix-dynamic-dns.service" state: absent when: "matrix_dynamic_dns_service_stat.stat.exists" From 94dcceb7b9c77ca5b1be02c53aa9a12e310df5e5 Mon Sep 17 00:00:00 2001 From: Scott Crossen Date: Mon, 19 Oct 2020 11:26:37 -0700 Subject: [PATCH 010/217] removed intentional delay --- .../templates/systemd/matrix-dynamic-dns.service.j2 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/roles/matrix-dynamic-dns/templates/systemd/matrix-dynamic-dns.service.j2 b/roles/matrix-dynamic-dns/templates/systemd/matrix-dynamic-dns.service.j2 index 9f866e1e..9c11e148 100644 --- a/roles/matrix-dynamic-dns/templates/systemd/matrix-dynamic-dns.service.j2 +++ b/roles/matrix-dynamic-dns/templates/systemd/matrix-dynamic-dns.service.j2 @@ -13,10 +13,6 @@ Wants={{ service }} Type=simple ExecStartPre=-{{ matrix_host_command_docker }} kill matrix-dynamic-dns ExecStartPre=-{{ matrix_host_command_docker }} rm matrix-dynamic-dns - -# Intentional delay, so that the homeserver (we likely depend on) can manage to start. -ExecStartPre={{ matrix_host_command_sleep }} 5 - ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-dynamic-dns \ --log-driver=none \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ From 1227349d0cd6671df2fbf30fac0193727ec0d928 Mon Sep 17 00:00:00 2001 From: Ivar Troost Date: Sat, 24 Oct 2020 17:03:24 +0200 Subject: [PATCH 011/217] Fix docker mount instruction in documentation The SSL certificates should be accessible to the nginx proxy container, not the synapse container. --- docs/howto-server-delegation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/howto-server-delegation.md b/docs/howto-server-delegation.md index 5235b843..1ea1091a 100644 --- a/docs/howto-server-delegation.md +++ b/docs/howto-server-delegation.md @@ -89,7 +89,7 @@ matrix_nginx_proxy_proxy_matrix_federation_api_ssl_certificate_key: /matrix/ssl/ If your files are not in `/matrix/ssl` but in some other location, you would need to mount them into the container: ```yaml -matrix_synapse_container_extra_arguments: +matrix_nginx_proxy_container_extra_arguments: - "--mount type=bind,src=/some/path/on/the/host,dst=/some/path/inside/the/container,ro" ``` From 63a49bb2dc7780e023b2801a7230cda529b2b3c1 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 26 Oct 2020 10:36:38 +0200 Subject: [PATCH 012/217] Do not expose /_synapse/admin publicly by default Fixes #685 (Github Issue). --- CHANGELOG.md | 16 ++++++++++ examples/apache/matrix-synapse.conf | 2 ++ examples/caddy/matrix-synapse | 8 +++-- roles/matrix-nginx-proxy/defaults/main.yml | 20 +++++++++++++ .../nginx/conf.d/matrix-synapse.conf.j2 | 29 ++++--------------- 5 files changed, 48 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 247e4e14..6b6dd8e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +# 2020-10-26 + +## (Compatibility Break) /_synapse/admin is no longer publicly exposed by default + +We used to expose the Synapse Admin APIs publicly (at `https://matrix.DOMAIN/_synapse/admin`). +These APIs require authentication with a valid access token, so it's not that big a deal to expose them. + +However, following [official Synapse's reverse-proxying recommendations](https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.md#synapse-administration-endpoints), we're no longer exposing `/_synapse/admin` by default. + +If you'd like to restore restore the old behavior and expose `/_synapse/admin` publicly, you can use the following configuration (in your `vars.yml`): + +```yaml +matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_admin_api_enabled: true +``` + + # 2020-10-02 ## Minimum Ansible version raised to v2.7.0 diff --git a/examples/apache/matrix-synapse.conf b/examples/apache/matrix-synapse.conf index 3e09121f..e4266b81 100644 --- a/examples/apache/matrix-synapse.conf +++ b/examples/apache/matrix-synapse.conf @@ -43,6 +43,8 @@ AllowEncodedSlashes NoDecode ProxyPass /_matrix http://127.0.0.1:8008/_matrix retry=0 nocanon ProxyPassReverse /_matrix http://127.0.0.1:8008/_matrix + ProxyPass /_synapse/client http://127.0.0.1:8008/_synapse/client retry=0 nocanon + ProxyPassReverse /_synapse/client http://127.0.0.1:8008/_synapse/client # Map /.well-known/matrix/client for client discovery Alias /.well-known/matrix/client /matrix/static-files/.well-known/matrix/client diff --git a/examples/caddy/matrix-synapse b/examples/caddy/matrix-synapse index 30f7c5a2..4202c759 100644 --- a/examples/caddy/matrix-synapse +++ b/examples/caddy/matrix-synapse @@ -21,9 +21,11 @@ https://matrix.DOMAIN { } # Synapse Client<>Server API - proxy / matrix-synapse:8008 { + proxy /_matrix matrix-synapse:8008 { + transparent + except /_matrix/identity/ /_matrix/client/r0/user_directory/search + } + proxy /_synapse/client matrix-synapse:8008 { transparent - except /.well-known/ /_matrix/identity/ /_matrix/client/r0/user_directory/search } - } diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 643d723e..89007a85 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -149,6 +149,26 @@ matrix_nginx_proxy_proxy_matrix_client_api_addr_sans_container: "127.0.0.1:8008" # This needs to be equal or higher than the maximum upload size accepted by Synapse. matrix_nginx_proxy_proxy_matrix_client_api_client_max_body_size_mb: 25 + +# Tells wheter `/_synapse/client` is forwarded to the Matrix Client API server. +matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_client_api_enabled: true + +# Tells wheter `/_synapse/admin` is forwarded to the Matrix Client API server. +# Following these recommendations (https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.md), by default, we don't. +matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_admin_api_enabled: false + +# `matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_prefixes` holds +# the location prefixes that get forwarded to the Matrix Client API server. +# These locations get combined into a regex like this `^(/_matrix|/_synapse/client)`. +matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_prefix_regexes: | + {{ + (['/_matrix']) + + + (['/_synapse/client'] if matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_client_api_enabled else []) + + + (['/_synapse/admin'] if matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_admin_api_enabled else []) + }} + # Controls whether proxying for the Matrix Federation API should be done. matrix_nginx_proxy_proxy_matrix_federation_api_enabled: false matrix_nginx_proxy_proxy_matrix_federation_api_addr_with_container: "matrix-synapse:8048" diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 8fd87958..8bcc9bc6 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -105,29 +105,6 @@ {{- configuration_block }} {% endfor %} - {# - This handles the Matrix Client API only. - The Matrix Federation API is handled by a separate vhost. - #} - location /_matrix { - {% if matrix_nginx_proxy_enabled %} - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "{{ matrix_nginx_proxy_proxy_matrix_client_api_addr_with_container }}"; - proxy_pass http://$backend; - {% else %} - {# Generic configuration for use outside of our container setup #} - proxy_pass http://{{ matrix_nginx_proxy_proxy_matrix_client_api_addr_sans_container }}; - {% endif %} - - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - - client_body_buffer_size 25M; - client_max_body_size {{ matrix_nginx_proxy_proxy_matrix_client_api_client_max_body_size_mb }}M; - proxy_max_temp_file_size 0; - } - {% if matrix_nginx_proxy_proxy_synapse_metrics %} location /_synapse/metrics { {% if matrix_nginx_proxy_enabled %} @@ -150,7 +127,11 @@ } {% endif %} - location /_synapse { + {# + This handles the Matrix Client API only. + The Matrix Federation API is handled by a separate vhost. + #} + location ~* ^({{ matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_prefix_regexes|join('|') }}) { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; From 70f0b97a0aa8adb84833480ec3c53aab51eec349 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 27 Oct 2020 14:24:02 +0200 Subject: [PATCH 013/217] Upgrade Synapse (v1.21.2 -> v1.22.0) --- group_vars/matrix_servers | 2 +- roles/matrix-synapse/defaults/main.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 00cdbdfb..a817f1e6 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -917,7 +917,7 @@ matrix_client_element_jitsi_preferredDomain: "{{ matrix_server_fqn_jitsi if matr # ###################################################################### -matrix_synapse_container_image_self_build: "{{ matrix_architecture != 'amd64'}}" +matrix_synapse_container_image_self_build: "{{ matrix_architecture not in ['arm32', 'arm64', 'amd64'] }}" # When ma1sd is enabled, we can use it to validate email addresses and phone numbers. # Synapse can validate email addresses by itself as well, but it's probably not what we want by default when we have an identity server. diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index fd8eaa52..450a26be 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -5,7 +5,7 @@ matrix_synapse_enabled: true matrix_synapse_container_image_self_build: false -matrix_synapse_docker_image: "matrixdotorg/synapse:v1.21.2" +matrix_synapse_docker_image: "matrixdotorg/synapse:v1.22.0" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse" @@ -73,7 +73,7 @@ matrix_synapse_registration_shared_secret: "{{ matrix_synapse_macaroon_secret_ke matrix_synapse_allow_guest_access: false matrix_synapse_form_secret: "{{ matrix_synapse_macaroon_secret_key }}" -matrix_synapse_max_upload_size_mb: 10 +matrix_synapse_max_upload_size_mb: 50 # The tmpfs at /tmp needs to be large enough to handle multiple concurrent file uploads. matrix_synapse_tmp_directory_size_mb: "{{ matrix_synapse_max_upload_size_mb * 50 }}" @@ -349,7 +349,7 @@ matrix_synapse_room_list_publication_rules: room_id: "*" action: allow -matrix_synapse_default_room_version: "5" +matrix_synapse_default_room_version: "6" # Controls the Synapse `spam_checker` setting. # From 7d211b71e2123ba0a62f2264497c6d4696243e61 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 27 Oct 2020 14:30:43 +0200 Subject: [PATCH 014/217] Restore self-building on ARM for Synapse for now While v1.22.0 supposedly has multi-arch Docker images (thanks to https://github.com/matrix-org/synapse/pull/7921), I can't them on Docker Hub yet, so I'm backing out of this change for now and letting people fall back to self-building there. --- group_vars/matrix_servers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index a817f1e6..817b4c8f 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -917,7 +917,7 @@ matrix_client_element_jitsi_preferredDomain: "{{ matrix_server_fqn_jitsi if matr # ###################################################################### -matrix_synapse_container_image_self_build: "{{ matrix_architecture not in ['arm32', 'arm64', 'amd64'] }}" +matrix_synapse_container_image_self_build: "{{ matrix_architecture not in ['amd64'] }}" # When ma1sd is enabled, we can use it to validate email addresses and phone numbers. # Synapse can validate email addresses by itself as well, but it's probably not what we want by default when we have an identity server. From 5a7d26599786cdd075610a130f85556248350035 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 27 Oct 2020 20:33:37 +0200 Subject: [PATCH 015/217] Fix Synapse Admin not working due to unexposed Synapse Admin APIs Regression since 63a49bb2dc7780. Related to #685 (Github Issue). We now automatically expose the APIs when Synapse Admin is enabled. --- docs/configuring-playbook-synapse-admin.md | 2 ++ group_vars/matrix_servers | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/configuring-playbook-synapse-admin.md b/docs/configuring-playbook-synapse-admin.md index b3eafaea..a2f5e646 100644 --- a/docs/configuring-playbook-synapse-admin.md +++ b/docs/configuring-playbook-synapse-admin.md @@ -15,6 +15,8 @@ Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars. matrix_synapse_admin_enabled: true ``` +**Note**: enabling Synapse Admin automatically exposes Synapse's Administration APIs (equivalent to `matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_admin_api_enabled: true`). + ## Installing diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 817b4c8f..71a6bc85 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -769,6 +769,8 @@ matrix_nginx_proxy_proxy_matrix_client_api_addr_with_container: "{{ 'matrix-corp matrix_nginx_proxy_proxy_matrix_client_api_addr_sans_container: "{{ '127.0.0.1:41080' if matrix_corporal_enabled else '127.0.0.1:8008' }}" matrix_nginx_proxy_proxy_matrix_client_api_client_max_body_size_mb: "{{ matrix_synapse_max_upload_size_mb }}" +matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_admin_api_enabled: "{{ matrix_synapse_admin_enabled }}" + matrix_nginx_proxy_proxy_matrix_enabled: true matrix_nginx_proxy_proxy_element_enabled: "{{ matrix_client_element_enabled }}" matrix_nginx_proxy_proxy_dimension_enabled: "{{ matrix_dimension_enabled }}" From 67d0f264636c5b7bb67f47698545beca9e85d8b0 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 27 Oct 2020 20:41:52 +0200 Subject: [PATCH 016/217] Improve wording a bit --- docs/configuring-playbook-synapse-admin.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuring-playbook-synapse-admin.md b/docs/configuring-playbook-synapse-admin.md index a2f5e646..68d70305 100644 --- a/docs/configuring-playbook-synapse-admin.md +++ b/docs/configuring-playbook-synapse-admin.md @@ -15,7 +15,7 @@ Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars. matrix_synapse_admin_enabled: true ``` -**Note**: enabling Synapse Admin automatically exposes Synapse's Administration APIs (equivalent to `matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_admin_api_enabled: true`). +**Note**: Synapse Admin requires Synapse's [Admin APIs](https://github.com/matrix-org/synapse/tree/master/docs/admin_api) to function. Access to them is restricted with a valid access token, so exposing them publicly should not be a real security concern. Still, for additional security, we normally leave them unexposed, following [official Synapse reverse-proxying recommendations](https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.md#synapse-administration-endpoints). Because Synapse Admin needs these APIs to function, when installing Synapse Admin, we **automatically** exposes them publicly for you (equivalent to `matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_admin_api_enabled: true`). ## Installing From ef07aa8e5df1e8708b6e3a5437570d7677f4cf18 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 28 Oct 2020 09:38:04 +0200 Subject: [PATCH 017/217] Prevent certain nginx location blocks from being ignored The regex introduced in 63a49bb2dc778 seems to take precedence over the bare location blocks, causing a regression. > It is important to understand that, by default, Nginx will serve regular expression matches in preference to prefix matches. > However, it evaluates prefix locations first, allowing for the administer to override this tendency by specifying locations using the = and ^~ modifiers. Source: https://www.digitalocean.com/community/tutorials/understanding-nginx-server-and-location-block-selection-algorithms --- .../templates/nginx/conf.d/matrix-synapse.conf.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 8bcc9bc6..463aec07 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -34,7 +34,7 @@ {% endif %} {% if matrix_nginx_proxy_proxy_matrix_corporal_api_enabled %} - location /_matrix/corporal { + location ^~ /_matrix/corporal { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; @@ -51,7 +51,7 @@ {% endif %} {% if matrix_nginx_proxy_proxy_matrix_identity_api_enabled %} - location /_matrix/identity { + location ^~ /_matrix/identity { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; @@ -68,7 +68,7 @@ {% endif %} {% if matrix_nginx_proxy_proxy_matrix_user_directory_search_enabled %} - location /_matrix/client/r0/user_directory/search { + location ^~ /_matrix/client/r0/user_directory/search { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; From 4700e803897d2ceb12e328e8dbfa688bb849e0b5 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 28 Oct 2020 10:02:47 +0200 Subject: [PATCH 018/217] Raise standalone default Matrix Client API client_max_body_size We do this to match Synapse's new default "max_upload_size" (50MB). This `matrix_nginx_proxy_proxy_matrix_client_api_client_max_body_size_mb` default value only affects standalone usage of the `matrix-nginx-proxy` role. When the role is used in the context of the playbook, the value is dynamically assigned from `group_vars/matrix_servers`. Somewhat related to #692 (Github Issue). --- roles/matrix-nginx-proxy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 89007a85..2ce8eb30 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -147,7 +147,7 @@ matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_key: "" matrix_nginx_proxy_proxy_matrix_client_api_addr_with_container: "matrix-synapse:8008" matrix_nginx_proxy_proxy_matrix_client_api_addr_sans_container: "127.0.0.1:8008" # This needs to be equal or higher than the maximum upload size accepted by Synapse. -matrix_nginx_proxy_proxy_matrix_client_api_client_max_body_size_mb: 25 +matrix_nginx_proxy_proxy_matrix_client_api_client_max_body_size_mb: 50 # Tells wheter `/_synapse/client` is forwarded to the Matrix Client API server. From 9a46647010989a0d8925f771240de9d49692b56b Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 28 Oct 2020 10:39:12 +0200 Subject: [PATCH 019/217] Make https://matrix.DOMAIN/ redirect to https://element.DOMAIN/ Fixes #696 (Github Issue) --- CHANGELOG.md | 14 ++++++++++++++ group_vars/matrix_servers | 2 ++ roles/matrix-nginx-proxy/defaults/main.yml | 6 ++++++ .../templates/nginx/conf.d/matrix-synapse.conf.j2 | 6 +++++- 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b6dd8e5..089d0650 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +# 2020-10-28 + +## (Compatibility Break) https://matrix.DOMAIN/ now redirects to https://element.DOMAIN/ + +Until now, we used to serve a static page coming from Synapse at `https://matrix.DOMAIN/`. This page was not very useful to anyone. + +Since `matrix.DOMAIN` may be accessed by regular users in certain conditions, it's probably better to redirect them to a better place (e.g. to the [Element](docs/configuring-playbook-client-element.md) client). + +If Element is installed (`matrix_client_element_enabled: true`, which it is by default), we now redirect people to it, instead of showing them a Synapse static page. + +If you'd like to control where the redirect goes, use the `matrix_nginx_proxy_proxy_matrix_client_redirect_root_uri_to_domain` variable. +To restore the old behavior of not redirecting anywhere and serving the Synapse static page, set it to an empty value (`matrix_nginx_proxy_proxy_matrix_client_redirect_root_uri_to_domain: ""`). + + # 2020-10-26 ## (Compatibility Break) /_synapse/admin is no longer publicly exposed by default diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 71a6bc85..eabce668 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -771,6 +771,8 @@ matrix_nginx_proxy_proxy_matrix_client_api_client_max_body_size_mb: "{{ matrix_s matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_admin_api_enabled: "{{ matrix_synapse_admin_enabled }}" +matrix_nginx_proxy_proxy_matrix_client_redirect_root_uri_to_domain: "{{ matrix_server_fqn_element if matrix_client_element_enabled else '' }}" + matrix_nginx_proxy_proxy_matrix_enabled: true matrix_nginx_proxy_proxy_element_enabled: "{{ matrix_client_element_enabled }}" matrix_nginx_proxy_proxy_dimension_enabled: "{{ matrix_dimension_enabled }}" diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 2ce8eb30..6d9ff33b 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -169,6 +169,12 @@ matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_prefix_regexes: | (['/_synapse/admin'] if matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_admin_api_enabled else []) }} +# Specifies where requests for the root URI (`/`) on the `matrix.` domain should be redirected. +# If this has an empty value, they're just passed to the homeserver, which serves a static page. +# If you'd like to make `https://matrix.DOMAIN` redirect to `https://element.DOMAIN` (or something of that sort), specify the domain name here. +# Example value: `element.DOMAIN` (or `{{ matrix_server_fqn_element }}`). +matrix_nginx_proxy_proxy_matrix_client_redirect_root_uri_to_domain: "" + # Controls whether proxying for the Matrix Federation API should be done. matrix_nginx_proxy_proxy_matrix_federation_api_enabled: false matrix_nginx_proxy_proxy_matrix_federation_api_addr_with_container: "matrix-synapse:8048" diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 463aec07..adbee18e 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -151,7 +151,11 @@ } location / { - rewrite ^/$ /_matrix/static/ last; + {% if matrix_nginx_proxy_proxy_matrix_client_redirect_root_uri_to_domain %} + return 302 $scheme://{{ matrix_nginx_proxy_proxy_matrix_client_redirect_root_uri_to_domain }}$request_uri; + {% else %} + rewrite ^/$ /_matrix/static/ last; + {% endif %} } {% endmacro %} From c1c6eaefff7eb40ef133b72870af8c77b569076c Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 28 Oct 2020 17:34:39 +0200 Subject: [PATCH 020/217] Upgrade Element (1.7.10 -> 1.7.12) --- roles/matrix-client-element/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index d2b9258f..100a23b4 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -2,7 +2,7 @@ matrix_client_element_enabled: true matrix_client_element_container_image_self_build: false -matrix_client_element_docker_image: "vectorim/riot-web:v1.7.10" +matrix_client_element_docker_image: "vectorim/riot-web:v1.7.12" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" matrix_client_element_data_path: "{{ matrix_base_data_path }}/client-element" From 56ebeb1ae26ecc58888979efd327c3c25c2ac0b4 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 29 Oct 2020 07:35:53 +0200 Subject: [PATCH 021/217] Revert "Restore self-building on ARM for Synapse for now" This reverts commit 7d211b71e2123ba0a62f2264497c6d4696243e61. The multi-arch images are now available, as discussed in #699 (Github Issue). --- group_vars/matrix_servers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index eabce668..ceaab812 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -921,7 +921,7 @@ matrix_client_element_jitsi_preferredDomain: "{{ matrix_server_fqn_jitsi if matr # ###################################################################### -matrix_synapse_container_image_self_build: "{{ matrix_architecture not in ['amd64'] }}" +matrix_synapse_container_image_self_build: "{{ matrix_architecture not in ['arm32', 'arm64', 'amd64'] }}" # When ma1sd is enabled, we can use it to validate email addresses and phone numbers. # Synapse can validate email addresses by itself as well, but it's probably not what we want by default when we have an identity server. From c33d007306761a56a9cf909d62088089c6a8cef1 Mon Sep 17 00:00:00 2001 From: Aaron Raimist Date: Thu, 29 Oct 2020 11:46:58 -0500 Subject: [PATCH 022/217] Switch to the new vectorim/element-web Docker image --- README.md | 2 +- docs/configuring-dns.md | 2 +- docs/configuring-playbook-client-element.md | 2 +- roles/matrix-client-element/defaults/main.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 72018115..1a207937 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ This playbook sets up your server using the following Docker images: - [instrumentisto/coturn](https://hub.docker.com/r/instrumentisto/coturn/) - the [Coturn](https://github.com/coturn/coturn) STUN/TURN server (optional) -- [vectorim/riot-web](https://hub.docker.com/r/vectorim/riot-web/) - the [Element](https://element.io/) web client (optional) +- [vectorim/element-web](https://hub.docker.com/r/vectorim/element-web/) - the [Element](https://element.io/) web client (optional) - [ma1uta/ma1sd](https://hub.docker.com/r/ma1uta/ma1sd/) - the [ma1sd](https://github.com/ma1uta/ma1sd) Matrix Identity server (optional) diff --git a/docs/configuring-dns.md b/docs/configuring-dns.md index 08bc905e..9d738477 100644 --- a/docs/configuring-dns.md +++ b/docs/configuring-dns.md @@ -34,7 +34,7 @@ DNS records marked with `(*)` above are optional. They refer to services that wi As the table above illustrates, you need to create 2 subdomains (`matrix.` and `element.`) and point both of them to your new server's IP address (DNS `A` record or `CNAME` record is fine). -The `element.` subdomain is necessary, because this playbook installs the [Element](https://github.com/vector-im/riot-web) web client for you. +The `element.` subdomain is necessary, because this playbook installs the [Element](https://github.com/vector-im/element-web) web client for you. If you'd rather instruct the playbook not to install Element (`matrix_client_element_enabled: false` when [Configuring the playbook](configuring-playbook.md) later), feel free to skip the `element.` DNS record. The `dimension.` subdomain may be necessary, because this playbook could install the [Dimension integrations manager](http://dimension.t2bot.io/) for you. Dimension installation is disabled by default, because it's only possible to install it after the other Matrix services are working (see [Setting up Dimension](configuring-playbook-dimension.md) later). If you do not wish to set up Dimension, feel free to skip the `dimension.` DNS record. diff --git a/docs/configuring-playbook-client-element.md b/docs/configuring-playbook-client-element.md index 8b0eb331..45299c55 100644 --- a/docs/configuring-playbook-client-element.md +++ b/docs/configuring-playbook-client-element.md @@ -1,6 +1,6 @@ # Configuring Element (optional) -By default, this playbook installs the [Element](https://github.com/vector-im/riot-web) Matrix client web application. +By default, this playbook installs the [Element](https://github.com/vector-im/element-web) Matrix client web application. If that's okay, you can skip this document. diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index 100a23b4..76bc7142 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -2,7 +2,7 @@ matrix_client_element_enabled: true matrix_client_element_container_image_self_build: false -matrix_client_element_docker_image: "vectorim/riot-web:v1.7.12" +matrix_client_element_docker_image: "vectorim/element-web:v1.7.12" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" matrix_client_element_data_path: "{{ matrix_base_data_path }}/client-element" From 5c91e56898cc0ded68afda1c892fc20edb01ebd5 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 30 Oct 2020 19:35:55 +0200 Subject: [PATCH 023/217] Upgrade Synapse (v1.22.0 -> v1.22.1) --- roles/matrix-synapse/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 450a26be..5777e802 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -5,7 +5,7 @@ matrix_synapse_enabled: true matrix_synapse_container_image_self_build: false -matrix_synapse_docker_image: "matrixdotorg/synapse:v1.22.0" +matrix_synapse_docker_image: "matrixdotorg/synapse:v1.22.1" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse" From ef68d3d2961c603d79e5fdda6d2b350ac694147e Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 2 Nov 2020 11:10:03 +0200 Subject: [PATCH 024/217] Add support for reverse-proxying /_synapse/oidc This broke in 63a49bb2dc7780e023b28. Proxying the OpenID Connect endpoints is now possible, but needs to be enabled explicitly now. Supersedes #702 (Github Pull Request). This patch builds up on the idea from that Pull Request, but does things in a cleaner way. --- docs/configuring-playbook-nginx.md | 9 +++++++++ docs/configuring-playbook-synapse.md | 5 +++++ roles/matrix-nginx-proxy/defaults/main.yml | 9 +++++++-- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/docs/configuring-playbook-nginx.md b/docs/configuring-playbook-nginx.md index e0b4911e..945864e9 100644 --- a/docs/configuring-playbook-nginx.md +++ b/docs/configuring-playbook-nginx.md @@ -23,3 +23,12 @@ matrix_nginx_proxy_proxy_matrix_nginx_status_allowed_addresses: - 8.8.8.8 - 1.1.1.1 ``` + + +## Synapse + OpenID Connect for Single-Sign-On + +If you want to use OpenID Connect as an SSO provider (as per the [Synapse OpenID docs](https://github.com/matrix-org/synapse/blob/develop/docs/openid.md)), you need to use the following configuration (in your `vars.yml` file) to instruct nginx to forward `/_synapse/oidc` to Synapse: + +```yaml +matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_oidc_api_enabled: true +``` diff --git a/docs/configuring-playbook-synapse.md b/docs/configuring-playbook-synapse.md index 019a651f..9ae1e903 100644 --- a/docs/configuring-playbook-synapse.md +++ b/docs/configuring-playbook-synapse.md @@ -21,3 +21,8 @@ Alternatively, **if there is no pre-defined variable** for a Synapse setting you ## Synapse Admin Certain Synapse administration tasks (managing users and rooms, etc.) can be performed via a web user-interace, if you install [Synapse Admin](configuring-playbook-synapse-admin.md). + + +## Synapse + OpenID Connect for Single-Sign-On + +If you'd like to use OpenID Connect authentication with Synapse, you'll need some additional reverse-proxy configuration (see [our nginx reverse-proxy doc page](configuring-playbook-nginx.md#synapse-openid-connect-for-single-sign-on)). diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 6d9ff33b..546f1f1d 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -150,10 +150,13 @@ matrix_nginx_proxy_proxy_matrix_client_api_addr_sans_container: "127.0.0.1:8008" matrix_nginx_proxy_proxy_matrix_client_api_client_max_body_size_mb: 50 -# Tells wheter `/_synapse/client` is forwarded to the Matrix Client API server. +# Tells whether `/_synapse/client` is forwarded to the Matrix Client API server. matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_client_api_enabled: true -# Tells wheter `/_synapse/admin` is forwarded to the Matrix Client API server. +# Tells whether `/_synapse/oidc` is forwarded to the Matrix Client API server. +matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_oidc_api_enabled: false + +# Tells whether `/_synapse/admin` is forwarded to the Matrix Client API server. # Following these recommendations (https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.md), by default, we don't. matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_admin_api_enabled: false @@ -166,6 +169,8 @@ matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_prefix_regexes: | + (['/_synapse/client'] if matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_client_api_enabled else []) + + (['/_synapse/oidc'] if matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_oidc_api_enabled else []) + + (['/_synapse/admin'] if matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_admin_api_enabled else []) }} From 350c39d7457f325bc814800934c5e843bccadd97 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 2 Nov 2020 11:13:25 +0200 Subject: [PATCH 025/217] Update comment --- roles/matrix-nginx-proxy/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 546f1f1d..dc25b792 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -154,6 +154,7 @@ matrix_nginx_proxy_proxy_matrix_client_api_client_max_body_size_mb: 50 matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_client_api_enabled: true # Tells whether `/_synapse/oidc` is forwarded to the Matrix Client API server. +# Enable this if you need OpenID Connect authentication support. matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_oidc_api_enabled: false # Tells whether `/_synapse/admin` is forwarded to the Matrix Client API server. From e894befd87dce731debad58e77fd0de33e303732 Mon Sep 17 00:00:00 2001 From: Scott Crossen Date: Sat, 7 Nov 2020 17:34:16 -0800 Subject: [PATCH 026/217] Updates to reviewer comments --- ...md => configuring-playbook-dynamic-dns.md} | 13 ++++---- docs/configuring-playbook.md | 2 +- group_vars/matrix_servers | 17 +++++++++++ roles/matrix-dynamic-dns/defaults/main.yml | 14 ++------- roles/matrix-dynamic-dns/tasks/install.yml | 2 +- .../tasks/validate_config.yml | 10 +++---- .../templates/ddclient.conf.j2 | 30 ++++++++++++------- .../systemd/matrix-dynamic-dns.service.j2 | 3 -- ...tup_ssl_lets_encrypt_obtain_for_domain.yml | 12 ++++++++ 9 files changed, 66 insertions(+), 37 deletions(-) rename docs/{configuring-playbook-budget-builds.md => configuring-playbook-dynamic-dns.md} (65%) diff --git a/docs/configuring-playbook-budget-builds.md b/docs/configuring-playbook-dynamic-dns.md similarity index 65% rename from docs/configuring-playbook-budget-builds.md rename to docs/configuring-playbook-dynamic-dns.md index 318c3c0c..1047eaf1 100644 --- a/docs/configuring-playbook-budget-builds.md +++ b/docs/configuring-playbook-dynamic-dns.md @@ -1,6 +1,6 @@ -# Tips for deploying Matrix on a Budget +# Dynamic DNS -## Dynamic DNS +## Setup Most cloud providers / ISPs will charge you extra for a static IP address. If you're not hosting a highly reliable homeserver you can workaround this via dynamic DNS. To @@ -9,13 +9,14 @@ google domains, this process is described [here](https://support.google.com/doma After you've gotten the proper credentials you can add the following config to your inventory/host_vars/matrix.DOMAIN/vars.yml: ``` -matrix_dynamic_dns_username: XXXXXXXXXXXXXXXX -matrix_dynamic_dns_password: XXXXXXXXXXXXXXXX -matrix_dynamic_dns_provider: 'domains.google.com' +matrix_dynamic_dns_domain_configurations: | + {{ + [{'provider': 'domains.google.com', 'protocol': 'dyndn2', 'username': 'XXXXXXXXXXXXXXXX', 'password': 'XXXXXXXXXXXXXXXX', 'domain': matrix_domain}] + }} ``` ## Additional Reading Additional resources: -- https://matrix.org/docs/guides/free-small-matrix-server \ No newline at end of file +- https://matrix.org/docs/guides/free-small-matrix-server diff --git a/docs/configuring-playbook.md b/docs/configuring-playbook.md index 8b0c5537..7ef58b2f 100644 --- a/docs/configuring-playbook.md +++ b/docs/configuring-playbook.md @@ -33,7 +33,7 @@ When you're done with all the configuration you'd like to do, continue with [Ins - [Setting up the Jitsi video-conferencing platform](configuring-playbook-jitsi.md) (optional) -- [Setting up budget builds or resource-constrained builds](configuring-playbook-budget-builds.md) (optional) +- [Setting Dynamic DNS](configuring-playbook-dynamic-dns.md) (optional) ### Core service adjustments diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 00cdbdfb..2be361c9 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -619,6 +619,23 @@ matrix_dimension_homeserver_federationUrl: "http://matrix-synapse:{{ 8048 if mat ###################################################################### + +###################################################################### +# +# matrix-dynamic-dns +# +###################################################################### + +matrix_dynamic_dns_domain_configurations: [] + +###################################################################### +# +# /matrix-dynamic-dns +# +###################################################################### + + + ###################################################################### # # matrix-email2matrix diff --git a/roles/matrix-dynamic-dns/defaults/main.yml b/roles/matrix-dynamic-dns/defaults/main.yml index e57b47da..8c86d35c 100644 --- a/roles/matrix-dynamic-dns/defaults/main.yml +++ b/roles/matrix-dynamic-dns/defaults/main.yml @@ -1,11 +1,11 @@ # Whether dynamic dns is enabled -matrix_dynamic_dns_enabled: false +matrix_dynamic_dns_enabled: "{{ matrix_dynamic_dns_domain_configurations is defined }}" # The dynamic dns daemon interval matrix_dynamic_dns_daemon_interval: '300' # The docker container to use when in mode -matrix_dynamic_dns_docker_image: 'linuxserver/ddclient' +matrix_dynamic_dns_docker_image: 'linuxserver/ddclient:v3.9.1-ls45' # The image to force pull matrix_dynamic_dns_docker_image_force_pull: "{{ matrix_dynamic_dns_docker_image.endswith(':latest') }}" @@ -28,13 +28,5 @@ matrix_dynamic_dns_config_path: "{{ matrix_dynamic_dns_base_path }}/config" matrix_dynamic_dns_docker_src_files_path: "{{ matrix_dynamic_dns_base_path }}/docker-src" # Config options +matrix_dynamic_dns_additional_configuration_blocks: [] matrix_dynamic_dns_use: "web" -matrix_dynamic_dns_static: false -matrix_dynamic_dns_custom: false -matrix_dynamic_dns_zone: "" -matrix_dynamic_dns_ttl: "" -matrix_dynamic_dns_mx: "" -matrix_dynamic_dns_wildcard: false -matrix_dynamic_dns_protocol: 'dyndns2' -matrix_dynamic_dns_provider: 'domains.google.com' -matrix_dynamic_dns_domain: '{{ matrix_domain }}' diff --git a/roles/matrix-dynamic-dns/tasks/install.yml b/roles/matrix-dynamic-dns/tasks/install.yml index 816dc7c3..225738bf 100644 --- a/roles/matrix-dynamic-dns/tasks/install.yml +++ b/roles/matrix-dynamic-dns/tasks/install.yml @@ -12,7 +12,7 @@ file: path: "{{ item.path }}" state: directory - mode: 0750 + mode: 0751 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" with_items: diff --git a/roles/matrix-dynamic-dns/tasks/validate_config.yml b/roles/matrix-dynamic-dns/tasks/validate_config.yml index 2895f407..c6e4c4a6 100644 --- a/roles/matrix-dynamic-dns/tasks/validate_config.yml +++ b/roles/matrix-dynamic-dns/tasks/validate_config.yml @@ -1,10 +1,10 @@ --- -- name: Fail if required settings not defined +- name: Fail if required settings not defined in configuration blocks fail: msg: >- You need to define a required configuration setting (`{{ item }}`). - when: "vars[item] == ''" - with_items: - - "matrix_dynamic_dns_domain" - - "matrix_dynamic_dns_provider" + when: "'domain' not in configuration == '' or 'provider' not in configuration == '' or 'protocol' not in configuration == ''" + with_items: "{{ matrix_dynamic_dns_domain_configurations }}" + loop_control: + loop_var: configuration diff --git a/roles/matrix-dynamic-dns/templates/ddclient.conf.j2 b/roles/matrix-dynamic-dns/templates/ddclient.conf.j2 index 651712db..1480d834 100644 --- a/roles/matrix-dynamic-dns/templates/ddclient.conf.j2 +++ b/roles/matrix-dynamic-dns/templates/ddclient.conf.j2 @@ -3,14 +3,24 @@ syslog=no pid=/var/run/ddclient/ddclient.pid ssl=yes use={{ matrix_dynamic_dns_use }} -protocol={{ matrix_dynamic_dns_protocol }} -server={{ matrix_dynamic_dns_provider }} {% if matrix_dynamic_dns_username %} -login='{{ matrix_dynamic_dns_username }}' {% endif %} {% if matrix_dynamic_dns_username %} -password='{{ matrix_dynamic_dns_password }}' {% endif %} {% if matrix_dynamic_dns_static %} -static=yes {% endif %} {% if matrix_dynamic_dns_custom %} -custom=yes {% endif %} {% if matrix_dynamic_dns_zone %} -zone={{ matrix_dynamic_dns_zone }} {% endif %} {% if matrix_dynamic_dns_ttl %} -ttl={{ matrix_dynamic_dns_ttl }} {% endif %} {% if matrix_dynamic_dns_mx %} -mx={{ matrix_dynamic_dns_mx }} {% endif %} {% if matrix_dynamic_dns_wildcard %} + +{% for dynamic_dns_domain_configuration in matrix_dynamic_dns_domain_configurations %} +protocol={{ dynamic_dns_domain_configuration.protocol }} +server={{ dynamic_dns_domain_configuration.provider }} {% if 'username' in dynamic_dns_domain_configuration %} +login='{{ dynamic_dns_domain_configuration.username }}' {% endif %} {% if 'password' in dynamic_dns_domain_configuration %} +password='{{ dynamic_dns_domain_configuration.password }}' {% endif %} {% if 'static' in dynamic_dns_domain_configuration %} +static=yes {% endif %} {% if 'custom' in dynamic_dns_domain_configuration %} +custom=yes {% endif %} {% if 'zone' in dynamic_dns_domain_configuration %} +zone={{ dynamic_dns_domain_configuration.zone }} {% endif %} {% if 'ttl' in dynamic_dns_domain_configuration %} +ttl={{ dynamic_dns_domain_configuration.ttl }} {% endif %} {% if 'mx' in dynamic_dns_domain_configuration %} +mx={{ dynamic_dns_domain_configuration.mx }} {% endif %} {% if 'wildcard' in dynamic_dns_domain_configuration %} wildcard=yes {% endif %} -{{ matrix_dynamic_dns_domain }} +{{ dynamic_dns_domain_configuration.domain }} + +{% endfor %} + + +{% for matrix_dynamic_dns_additional_configuration in matrix_dynamic_dns_additional_configuration_blocks %} +{{ matrix_dynamic_dns_additional_configuration }} + +{% endfor %} diff --git a/roles/matrix-dynamic-dns/templates/systemd/matrix-dynamic-dns.service.j2 b/roles/matrix-dynamic-dns/templates/systemd/matrix-dynamic-dns.service.j2 index 9c11e148..7e73b587 100644 --- a/roles/matrix-dynamic-dns/templates/systemd/matrix-dynamic-dns.service.j2 +++ b/roles/matrix-dynamic-dns/templates/systemd/matrix-dynamic-dns.service.j2 @@ -15,12 +15,9 @@ ExecStartPre=-{{ matrix_host_command_docker }} kill matrix-dynamic-dns ExecStartPre=-{{ matrix_host_command_docker }} rm matrix-dynamic-dns ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-dynamic-dns \ --log-driver=none \ - --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ - --cap-drop=ALL \ --network={{ matrix_docker_network }} \ -e PUID={{ matrix_user_uid }} \ -e PGID={{ matrix_user_gid }} \ - -e CONFIG_PATH=/config/config.yaml \ -v {{ matrix_dynamic_dns_config_path }}:/config:z \ {% for arg in matrix_dynamic_dns_container_extra_arguments %} {{ arg }} \ diff --git a/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt_obtain_for_domain.yml b/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt_obtain_for_domain.yml index e80b655d..0c12fa93 100644 --- a/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt_obtain_for_domain.yml +++ b/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt_obtain_for_domain.yml @@ -12,6 +12,18 @@ - set_fact: domain_name_needs_cert: "{{ not domain_name_certificate_path_stat.stat.exists }}" +- name: Ensure dynamic dns has ran + service: + name: "dynamic-dns" + state: started + register: dynamic_dns_service_update + when: "domain_name_needs_cert|bool and matrix_dynamic_dns_enabled|bool" + +- name: Sleep for 60 seconds so that DNS records can be updated + wait_for: + timeout: 60 + when: dynamic_dns_service_update.changed + # This will fail if there is something running on port 80 (like matrix-nginx-proxy). # We suppress the error, as we'll try another method below. - name: Attempt initial SSL certificate retrieval with standalone authenticator (directly) From c448aa54a1eda4fba76ce20aec0ef6909b1411dd Mon Sep 17 00:00:00 2001 From: Eneko Nieto Date: Sun, 8 Nov 2020 22:39:08 +0100 Subject: [PATCH 027/217] Added docker-compose example for launching Traefik. --- docs/configuring-playbook-own-webserver.md | 35 ++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/configuring-playbook-own-webserver.md b/docs/configuring-playbook-own-webserver.md index 2c4b43a2..54afce6d 100644 --- a/docs/configuring-playbook-own-webserver.md +++ b/docs/configuring-playbook-own-webserver.md @@ -193,3 +193,38 @@ Note that this configuration on its own does **not** redirect traffic on port 80 scheme = "https" permanent = true ``` + +You can use the following `docker-compose.yml` as example to launch Traefik. + +```yaml +version: "3.3" + +services: + + traefik: + image: "traefik:v2.3" + restart: always + container_name: "traefik" + networks: + - traefik + command: + - "--api.insecure=true" + - "--providers.docker=true" + - "--providers.docker.network=traefik" + - "--providers.docker.exposedbydefault=false" + - "--entrypoints.websecure.address=:443" + - "--entrypoints.synapse.address=:8448" + - "--certificatesresolvers.default.acme.tlschallenge=true" + - "--certificatesresolvers.default.acme.email=YOUR EMAIL" + - "--certificatesresolvers.default.acme.storage=/letsencrypt/acme.json" + ports: + - "443:443" + - "8080:8080" + volumes: + - "./letsencrypt:/letsencrypt" + - "/var/run/docker.sock:/var/run/docker.sock:ro" + +networks: + traefik: + external: true +``` From 2c33aa3ec54533f70e7558baec5c62a7e51fbbde Mon Sep 17 00:00:00 2001 From: Eneko Nieto Date: Mon, 9 Nov 2020 00:03:15 +0100 Subject: [PATCH 028/217] Fixed entrypoint name. --- docs/configuring-playbook-own-webserver.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuring-playbook-own-webserver.md b/docs/configuring-playbook-own-webserver.md index 54afce6d..dac78376 100644 --- a/docs/configuring-playbook-own-webserver.md +++ b/docs/configuring-playbook-own-webserver.md @@ -212,7 +212,7 @@ services: - "--providers.docker=true" - "--providers.docker.network=traefik" - "--providers.docker.exposedbydefault=false" - - "--entrypoints.websecure.address=:443" + - "--entrypoints.web-secure.address=:443" - "--entrypoints.synapse.address=:8448" - "--certificatesresolvers.default.acme.tlschallenge=true" - "--certificatesresolvers.default.acme.email=YOUR EMAIL" From 59bb6b297138dd586198a6aa463c55f6374d23bc Mon Sep 17 00:00:00 2001 From: Scott Crossen Date: Mon, 9 Nov 2020 13:32:58 -0800 Subject: [PATCH 029/217] responded to reviewer comments --- docs/configuring-playbook-dynamic-dns.md | 1 + group_vars/matrix_servers | 1 + roles/matrix-dynamic-dns/defaults/main.yml | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/configuring-playbook-dynamic-dns.md b/docs/configuring-playbook-dynamic-dns.md index 1047eaf1..e46d87c0 100644 --- a/docs/configuring-playbook-dynamic-dns.md +++ b/docs/configuring-playbook-dynamic-dns.md @@ -9,6 +9,7 @@ google domains, this process is described [here](https://support.google.com/doma After you've gotten the proper credentials you can add the following config to your inventory/host_vars/matrix.DOMAIN/vars.yml: ``` +matrix_dyanmic_dns_enabled: true matrix_dynamic_dns_domain_configurations: | {{ [{'provider': 'domains.google.com', 'protocol': 'dyndn2', 'username': 'XXXXXXXXXXXXXXXX', 'password': 'XXXXXXXXXXXXXXXX', 'domain': matrix_domain}] diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 2be361c9..86d91088 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -626,6 +626,7 @@ matrix_dimension_homeserver_federationUrl: "http://matrix-synapse:{{ 8048 if mat # ###################################################################### +matrix_dyanmic_dns_enabled: false matrix_dynamic_dns_domain_configurations: [] ###################################################################### diff --git a/roles/matrix-dynamic-dns/defaults/main.yml b/roles/matrix-dynamic-dns/defaults/main.yml index 8c86d35c..cb17c90c 100644 --- a/roles/matrix-dynamic-dns/defaults/main.yml +++ b/roles/matrix-dynamic-dns/defaults/main.yml @@ -1,5 +1,5 @@ # Whether dynamic dns is enabled -matrix_dynamic_dns_enabled: "{{ matrix_dynamic_dns_domain_configurations is defined }}" +matrix_dynamic_dns_enabled: true # The dynamic dns daemon interval matrix_dynamic_dns_daemon_interval: '300' From 235299939da49d5b7984a4deb66b626fa07504ce Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 10 Nov 2020 09:30:00 +0200 Subject: [PATCH 030/217] Upgrade nginx (1.19.3 -> 1.19.4) --- roles/matrix-nginx-proxy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index dc25b792..e36270b7 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -3,7 +3,7 @@ matrix_nginx_proxy_enabled: true # We use an official nginx image, which we fix-up to run unprivileged. # An alternative would be an `nginxinc/nginx-unprivileged` image, but # that is frequently out of date. -matrix_nginx_proxy_docker_image: "nginx:1.19.3-alpine" +matrix_nginx_proxy_docker_image: "nginx:1.19.4-alpine" matrix_nginx_proxy_docker_image_force_pull: "{{ matrix_nginx_proxy_docker_image.endswith(':latest') }}" matrix_nginx_proxy_base_path: "{{ matrix_base_data_path }}/nginx-proxy" From 31619e0968dc060107447da9de764a20cf64e5a6 Mon Sep 17 00:00:00 2001 From: Aaron Raimist Date: Tue, 10 Nov 2020 11:27:15 -0600 Subject: [PATCH 031/217] Upgrade Element (1.7.12 -> 1.7.13) --- roles/matrix-client-element/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index 76bc7142..c591b025 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -2,7 +2,7 @@ matrix_client_element_enabled: true matrix_client_element_container_image_self_build: false -matrix_client_element_docker_image: "vectorim/element-web:v1.7.12" +matrix_client_element_docker_image: "vectorim/element-web:v1.7.13" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" matrix_client_element_data_path: "{{ matrix_base_data_path }}/client-element" From 73d5faa1f9567548b54a7d9ff888cebdb93392f0 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 10 Nov 2020 23:30:38 +0200 Subject: [PATCH 032/217] Fix variable name typo --- docs/configuring-playbook-dynamic-dns.md | 2 +- group_vars/matrix_servers | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuring-playbook-dynamic-dns.md b/docs/configuring-playbook-dynamic-dns.md index e46d87c0..4251291b 100644 --- a/docs/configuring-playbook-dynamic-dns.md +++ b/docs/configuring-playbook-dynamic-dns.md @@ -9,7 +9,7 @@ google domains, this process is described [here](https://support.google.com/doma After you've gotten the proper credentials you can add the following config to your inventory/host_vars/matrix.DOMAIN/vars.yml: ``` -matrix_dyanmic_dns_enabled: true +matrix_dynamic_dns_enabled: true matrix_dynamic_dns_domain_configurations: | {{ [{'provider': 'domains.google.com', 'protocol': 'dyndn2', 'username': 'XXXXXXXXXXXXXXXX', 'password': 'XXXXXXXXXXXXXXXX', 'domain': matrix_domain}] diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 86d91088..e199ea86 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -626,7 +626,7 @@ matrix_dimension_homeserver_federationUrl: "http://matrix-synapse:{{ 8048 if mat # ###################################################################### -matrix_dyanmic_dns_enabled: false +matrix_dynamic_dns_enabled: false matrix_dynamic_dns_domain_configurations: [] ###################################################################### From 5c5f828405f5a279b9a1b1df14f7ecbf51d755e1 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 10 Nov 2020 23:33:27 +0200 Subject: [PATCH 033/217] Use more YAML-like syntax for the dynamic DNS configurations Related to #681 (Github Pull Request) --- docs/configuring-playbook-dynamic-dns.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/configuring-playbook-dynamic-dns.md b/docs/configuring-playbook-dynamic-dns.md index 4251291b..cec53e4a 100644 --- a/docs/configuring-playbook-dynamic-dns.md +++ b/docs/configuring-playbook-dynamic-dns.md @@ -8,12 +8,20 @@ set this up, you'll need to get the username/password from your DNS provider. Fo google domains, this process is described [here](https://support.google.com/domains/answer/6147083). After you've gotten the proper credentials you can add the following config to your inventory/host_vars/matrix.DOMAIN/vars.yml: -``` +```yaml matrix_dynamic_dns_enabled: true + matrix_dynamic_dns_domain_configurations: | {{ [{'provider': 'domains.google.com', 'protocol': 'dyndn2', 'username': 'XXXXXXXXXXXXXXXX', 'password': 'XXXXXXXXXXXXXXXX', 'domain': matrix_domain}] }} + +matrix_dynamic_dns_domain_configurations: + - provider: domains.google.com + protocol: dyndn2 + username: XXXXXXXXXXXXXXXX + password: XXXXXXXXXXXXXXXX + domain: "{{ matrix_domain }}" ``` ## Additional Reading From fef44b93d33d1e2c6835d50eef1f6c79f01d3ea2 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 10 Nov 2020 23:33:58 +0200 Subject: [PATCH 034/217] Define the matrix_dynamic_dns_domain_configurations variable in the role Having it unset in the role itself (while referencign it) is a little strange. Now people can look at the `roles/matrix-dynamic-dns/defaults/main.yml` file and figure out everything that's necessary to run the role. Related to #681 (Github Pull Request) --- roles/matrix-dynamic-dns/defaults/main.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/roles/matrix-dynamic-dns/defaults/main.yml b/roles/matrix-dynamic-dns/defaults/main.yml index cb17c90c..0d34ccea 100644 --- a/roles/matrix-dynamic-dns/defaults/main.yml +++ b/roles/matrix-dynamic-dns/defaults/main.yml @@ -27,6 +27,17 @@ matrix_dynamic_dns_base_path: "{{ matrix_base_data_path }}/dynamic-dns" matrix_dynamic_dns_config_path: "{{ matrix_dynamic_dns_base_path }}/config" matrix_dynamic_dns_docker_src_files_path: "{{ matrix_dynamic_dns_base_path }}/docker-src" +# Holds the configurations (the domains to update DNS for, the providers they use, etc.) +# +# Example: +# matrix_dynamic_dns_domain_configurations: +# - provider: domains.google.com +# protocol: dyndn2 +# username: XXXXXXXXXXXXXXXX +# password: XXXXXXXXXXXXXXXX +# domain: "{{ matrix_domain }}" +matrix_dynamic_dns_domain_configurations: [] + # Config options matrix_dynamic_dns_additional_configuration_blocks: [] matrix_dynamic_dns_use: "web" From 97a7c8b0f08d91ec68f946c5def9cea79dbfef5e Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 10 Nov 2020 23:43:15 +0200 Subject: [PATCH 035/217] Fix matrix_dynamic_dns_domain_configurations validation check - `item` was undefined - `'key' in configurations == ''` was doing the wrong thing Related to #681 (Github Pull Request) --- roles/matrix-dynamic-dns/tasks/validate_config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/matrix-dynamic-dns/tasks/validate_config.yml b/roles/matrix-dynamic-dns/tasks/validate_config.yml index c6e4c4a6..3976a4b0 100644 --- a/roles/matrix-dynamic-dns/tasks/validate_config.yml +++ b/roles/matrix-dynamic-dns/tasks/validate_config.yml @@ -3,8 +3,8 @@ - name: Fail if required settings not defined in configuration blocks fail: msg: >- - You need to define a required configuration setting (`{{ item }}`). - when: "'domain' not in configuration == '' or 'provider' not in configuration == '' or 'protocol' not in configuration == ''" + One of the configurations in matrix_dynamic_dns_domain_configurations is missing a required key (domain, provider, protocol). + when: "'domain' not in configuration or 'provider' not in configuration or 'protocol' not in configuration" with_items: "{{ matrix_dynamic_dns_domain_configurations }}" loop_control: loop_var: configuration From 8782919d8509ea53b5bdd16d7d672b053ff59cca Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 10 Nov 2020 23:44:09 +0200 Subject: [PATCH 036/217] Ensure matrix_dynamic_dns_domain_configurations contains configurations If `matrix_dynamic_dns_enabled`, we'd like to ensure there's at least one configuration defined. Related to #681 (Github Pull Request) --- roles/matrix-dynamic-dns/tasks/validate_config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/matrix-dynamic-dns/tasks/validate_config.yml b/roles/matrix-dynamic-dns/tasks/validate_config.yml index 3976a4b0..8f0001ea 100644 --- a/roles/matrix-dynamic-dns/tasks/validate_config.yml +++ b/roles/matrix-dynamic-dns/tasks/validate_config.yml @@ -1,5 +1,11 @@ --- +- name: Fail if no configurations specified + fail: + msg: >- + You need to define at least one configuration in `matrix_dynamic_dns_domain_configurations` for using matrix-dynamic-dns. + when: "matrix_dynamic_dns_domain_configurations|length == 0" + - name: Fail if required settings not defined in configuration blocks fail: msg: >- From 1427286cec29db8fb821dce963b2aa6c54c04845 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 10 Nov 2020 23:46:33 +0200 Subject: [PATCH 037/217] Integrate matrix-dynamic-dns with matrix-nginx-proxy without causing a dependency We'd like the roles to be self-contained (as much as possible). Thus, the `matrix-nginx-proxy` shouldn't reference any variables from other roles. Instead, we rely on injection via `group_vars/matrix_servers`. Related to #681 (Github Pull Request) --- group_vars/matrix_servers | 2 ++ roles/matrix-nginx-proxy/defaults/main.yml | 4 ++++ ...tup_ssl_lets_encrypt_obtain_for_domain.yml | 21 ++++++++++--------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index e199ea86..f38284e8 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -851,6 +851,8 @@ matrix_ssl_architecture: "{{ }[matrix_architecture] }}" +matrix_ssl_pre_obtaining_required_service_name: "{{ 'matrix-dynamic-dns' if matrix_dynamic_dns_enabled else '' }} + ###################################################################### # # /matrix-nginx-proxy diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 3b9dd60b..4bf57f61 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -243,6 +243,10 @@ matrix_ssl_base_path: "{{ matrix_base_data_path }}/ssl" matrix_ssl_config_dir_path: "{{ matrix_ssl_base_path }}/config" matrix_ssl_log_dir_path: "{{ matrix_ssl_base_path }}/log" +# If you'd like to start some service before a certificate is obtained, specify it here. +# This could be something like `matrix-dynamic-dns`, etc. +matrix_ssl_pre_obtaining_required_service_name: ~ +matrix_ssl_pre_obtaining_required_service_start_wait_time_seconds: 60 # nginx status page configurations. matrix_nginx_proxy_proxy_matrix_nginx_status_enabled: false diff --git a/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt_obtain_for_domain.yml b/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt_obtain_for_domain.yml index 0c12fa93..3dea71fb 100644 --- a/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt_obtain_for_domain.yml +++ b/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt_obtain_for_domain.yml @@ -12,17 +12,18 @@ - set_fact: domain_name_needs_cert: "{{ not domain_name_certificate_path_stat.stat.exists }}" -- name: Ensure dynamic dns has ran - service: - name: "dynamic-dns" - state: started - register: dynamic_dns_service_update - when: "domain_name_needs_cert|bool and matrix_dynamic_dns_enabled|bool" +- block: + - name: Ensure required service for obtaining is started + service: + name: "{{ matrix_ssl_pre_obtaining_required_service_name }}" + state: started + register: matrix_ssl_pre_obtaining_required_service_start_result -- name: Sleep for 60 seconds so that DNS records can be updated - wait_for: - timeout: 60 - when: dynamic_dns_service_update.changed + - name: Wait some time, so that the required service for obtaining can start + wait_for: + timeout: "{{ matrix_ssl_service_to_start_before_obtaining_start_wait_time_seconds }}" + when: "matrix_ssl_pre_obtaining_required_service_start_result.changed|bool" + when: "domain_name_needs_cert|bool and matrix_ssl_pre_obtaining_required_service_name != ''" # This will fail if there is something running on port 80 (like matrix-nginx-proxy). # We suppress the error, as we'll try another method below. From 2708f9aec71bb3f829979825bd8c2dbe66dcfc1f Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 10 Nov 2020 23:50:41 +0200 Subject: [PATCH 038/217] Remove duplicate configuration in docs page Related to #681 (Github Pull Request) --- docs/configuring-playbook-dynamic-dns.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/configuring-playbook-dynamic-dns.md b/docs/configuring-playbook-dynamic-dns.md index cec53e4a..c06bcf43 100644 --- a/docs/configuring-playbook-dynamic-dns.md +++ b/docs/configuring-playbook-dynamic-dns.md @@ -11,11 +11,6 @@ After you've gotten the proper credentials you can add the following config to y ```yaml matrix_dynamic_dns_enabled: true -matrix_dynamic_dns_domain_configurations: | - {{ - [{'provider': 'domains.google.com', 'protocol': 'dyndn2', 'username': 'XXXXXXXXXXXXXXXX', 'password': 'XXXXXXXXXXXXXXXX', 'domain': matrix_domain}] - }} - matrix_dynamic_dns_domain_configurations: - provider: domains.google.com protocol: dyndn2 From de0987e8064181c606efd4f2c6aaa8b6c97b7dbc Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 10 Nov 2020 23:51:38 +0200 Subject: [PATCH 039/217] Remove unnecessary variable definition Related to #681 (Github Pull Request) --- group_vars/matrix_servers | 1 - 1 file changed, 1 deletion(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index f38284e8..c2e40507 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -627,7 +627,6 @@ matrix_dimension_homeserver_federationUrl: "http://matrix-synapse:{{ 8048 if mat ###################################################################### matrix_dynamic_dns_enabled: false -matrix_dynamic_dns_domain_configurations: [] ###################################################################### # From d910df2b1cc1b8e497c7a643d22852a625492bb3 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 11 Nov 2020 00:00:52 +0200 Subject: [PATCH 040/217] Announce Dynamic DNS support Related to #681 (Github Pull Request) --- CHANGELOG.md | 9 +++++++++ docs/configuring-playbook-dynamic-dns.md | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 089d0650..aa378ae3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# 2020-11-10 + +## Dynamic DNS support + +Thanks to [Scott Crossen](https://github.com/scottcrossen), the playbook can now manage Dynamic DNS for you using [ddclient](https://ddclient.net/). + +To learn more, follow our [Dynamic DNS docs page](docs/configuring-playbook-dynamic-dns.md). + + # 2020-10-28 ## (Compatibility Break) https://matrix.DOMAIN/ now redirects to https://element.DOMAIN/ diff --git a/docs/configuring-playbook-dynamic-dns.md b/docs/configuring-playbook-dynamic-dns.md index c06bcf43..bc58c273 100644 --- a/docs/configuring-playbook-dynamic-dns.md +++ b/docs/configuring-playbook-dynamic-dns.md @@ -6,7 +6,7 @@ Most cloud providers / ISPs will charge you extra for a static IP address. If yo not hosting a highly reliable homeserver you can workaround this via dynamic DNS. To set this up, you'll need to get the username/password from your DNS provider. For google domains, this process is described [here](https://support.google.com/domains/answer/6147083). -After you've gotten the proper credentials you can add the following config to your inventory/host_vars/matrix.DOMAIN/vars.yml: +After you've gotten the proper credentials you can add the following config to your `inventory/host_vars/matrix.DOMAIN/vars.yml`: ```yaml matrix_dynamic_dns_enabled: true @@ -19,6 +19,7 @@ matrix_dynamic_dns_domain_configurations: domain: "{{ matrix_domain }}" ``` + ## Additional Reading Additional resources: From b5435db3386da670de8add53aa0278a0b99fd330 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 11 Nov 2020 00:22:09 +0200 Subject: [PATCH 041/217] Fix syntax error in group_vars/matrix_servers --- group_vars/matrix_servers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 8bda2bbe..742391c3 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -854,7 +854,7 @@ matrix_ssl_architecture: "{{ }[matrix_architecture] }}" -matrix_ssl_pre_obtaining_required_service_name: "{{ 'matrix-dynamic-dns' if matrix_dynamic_dns_enabled else '' }} +matrix_ssl_pre_obtaining_required_service_name: "{{ 'matrix-dynamic-dns' if matrix_dynamic_dns_enabled else '' }}" ###################################################################### # From 4dbec2470f3fb8d96af2de0c9db24a1d3baf3c44 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 11 Nov 2020 00:45:02 +0200 Subject: [PATCH 042/217] Fix systemd_path being undefined breakage Regression since #681 (Github Pull Request). Fixes #715 (Github Issue). --- roles/matrix-dynamic-dns/tasks/uninstall.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/matrix-dynamic-dns/tasks/uninstall.yml b/roles/matrix-dynamic-dns/tasks/uninstall.yml index 1be0e190..98dca0e8 100644 --- a/roles/matrix-dynamic-dns/tasks/uninstall.yml +++ b/roles/matrix-dynamic-dns/tasks/uninstall.yml @@ -2,7 +2,7 @@ - name: Check existence of matrix-dynamic-dns service stat: - path: "{{ systemd_path }}/matrix-dynamic-dns.service" + path: "{{ matrix_systemd_path }}/matrix-dynamic-dns.service" register: matrix_dynamic_dns_service_stat - name: Ensure matrix-dynamic-dns is stopped @@ -14,7 +14,7 @@ - name: Ensure matrix-dynamic-dns.service doesn't exist file: - path: "{{ systemd_path }}/matrix-dynamic-dns.service" + path: "{{ matrix_systemd_path }}/matrix-dynamic-dns.service" state: absent when: "matrix_dynamic_dns_service_stat.stat.exists" From fe7bed5df37dbe8bf575143dff095dde4286428f Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 10 Nov 2020 23:10:59 +0200 Subject: [PATCH 043/217] Upgrade appservice-discord --- .../defaults/main.yml | 8 +++++--- .../tasks/validate_config.yml | 1 + .../templates/config.yaml.j2 | 14 +++++++++----- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml index 97ad8e89..392f3028 100644 --- a/roles/matrix-bridge-appservice-discord/defaults/main.yml +++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml @@ -3,7 +3,7 @@ matrix_appservice_discord_enabled: true -matrix_appservice_discord_docker_image: "halfshot/matrix-appservice-discord:latest" +matrix_appservice_discord_docker_image: "halfshot/matrix-appservice-discord:v1.0.0-rc3" matrix_appservice_discord_docker_image_force_pull: "{{ matrix_appservice_discord_docker_image.endswith(':latest') }}" matrix_appservice_discord_base_path: "{{ matrix_base_data_path }}/appservice-discord" @@ -17,6 +17,8 @@ matrix_appservice_discord_bot_token: '' matrix_appservice_discord_appservice_token: '' matrix_appservice_discord_homeserver_token: '' +matrix_appservice_discord_homeserver_domain: "{{ matrix_domain }}" + # Controls whether the matrix-appservice-discord container exposes its HTTP port (tcp/9005 in the container). # # Takes an ":" or "" value (e.g. "127.0.0.1:9005"), or empty string to not expose. @@ -62,10 +64,10 @@ matrix_appservice_discord_registration_yaml: | namespaces: users: - exclusive: true - regex: '^@_discord_.*' + regex: '@_discord_.*:{{ matrix_appservice_discord_homeserver_domain|regex_escape }}' aliases: - exclusive: true - regex: '^#_discord_.*' + regex: '#_discord_.*:{{ matrix_appservice_discord_homeserver_domain|regex_escape }}' url: {{ matrix_appservice_discord_appservice_url }} sender_localpart: _discord_bot rate_limited: false diff --git a/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml b/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml index 46612ba5..b0713a43 100644 --- a/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml +++ b/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml @@ -10,6 +10,7 @@ - "matrix_appservice_discord_bot_token" - "matrix_appservice_discord_appservice_token" - "matrix_appservice_discord_homeserver_token" + - "matrix_appservice_discord_homeserver_domain" - name: (Deprecation) Catch and report renamed appservice-discord variables fail: diff --git a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 index e91c60de..0c2aab41 100644 --- a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 +++ b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 @@ -26,6 +26,10 @@ bridge: disableReadReceipts: false # Disable Join Leave echos from matrix disableJoinLeaveNotifications: false + # Disable Invite echos from matrix + disableInviteNotifications: false + # Auto-determine the language of code blocks (this can be CPU-intensive) + determineCodeLanguage: false # Authentication configuration for the discord bot. auth: clientID: {{ matrix_appservice_discord_client_id|string|to_json }} @@ -45,8 +49,6 @@ logging: # enable: # - "DiscordBot" database: - userStorePath: "/data/user-store.db" - roomStorePath: "/data/room-store.db" # You may either use SQLite or Postgresql for the bridge database, which contains # important mappings for events and user puppeting configurations. # Use the filename option for SQLite, or connString for Postgresql. @@ -82,10 +84,12 @@ channel: limits: # Delay in milliseconds between discord users joining a room. roomGhostJoinDelay: 6000 - # Delay in milliseconds before sending messages to discord to avoid echos. - # (Copies of a sent message may arrive from discord before we've + # Lock timeout in milliseconds before sending messages to discord to avoid + # echos. Default is rather high as the lock will most likely time out + # before anyways. + # echos = (Copies of a sent message may arrive from discord before we've # fininished handling it, causing us to echo it back to the room) - discordSendDelay: 750 + discordSendDelay: 1500 ghosts: # Pattern for the ghosts nick, available is :nick, :username, :tag and :id nickPattern: ":nick" From 6dbb90258e143ed1452eb381dfc793760b993fe7 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 13 Nov 2020 08:23:40 +0200 Subject: [PATCH 044/217] Mention and recommend enabling usePrivilegedIntents --- ...configuring-playbook-bridge-appservice-discord.md | 12 +++++++----- .../defaults/main.yml | 8 ++++++++ .../templates/config.yaml.j2 | 3 +++ 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/docs/configuring-playbook-bridge-appservice-discord.md b/docs/configuring-playbook-bridge-appservice-discord.md index 13a5174b..11be678e 100644 --- a/docs/configuring-playbook-bridge-appservice-discord.md +++ b/docs/configuring-playbook-bridge-appservice-discord.md @@ -14,18 +14,20 @@ Instructions loosely based on [this](https://github.com/Half-Shot/matrix-appserv 1. Create a Discord Application [here](https://discordapp.com/developers/applications). 2. Retrieve Client ID. 3. Create a bot from the Bot tab and retrieve the Bot token. -4. Enable the bridge with the following configuration in your `vars.yml` file: +4. From the Bot tab, enable all checkboxes related to Privileged Gateway Intents (you can skip this step if you're not using `matrix_appservice_discord_auth_usePrivilegedIntents: true` below) +5. Enable the bridge with the following configuration in your `vars.yml` file: ```yaml matrix_appservice_discord_enabled: true matrix_appservice_discord_client_id: "YOUR DISCORD APP CLIENT ID" matrix_appservice_discord_bot_token: "YOUR DISCORD APP BOT TOKEN" +matrix_appservice_discord_auth_usePrivilegedIntents: true ``` -4. If you've already installed Matrix services using the playbook before, you'll need to re-run it (`--tags=setup-all,start`). If not, proceed with [configuring other playbook services](configuring-playbook.md) and then with [Installing](installing.md). Get back to this guide once ready. -5. Retrieve Discord invite link from the `{{ matrix_appservice_discord_config_path }}/invite_link` file on the server (this defaults to `/matrix/appservice-discord/config/invite_link`). You need to peek at the file on the server via SSH, etc., because it's not available via HTTP(S). -6. Invite the Bot to Discord servers you wish to bridge. Administrator permission is recommended. -7. Room addresses follow this syntax: `#_discord_guildid_channelid`. You can easily find the guild and channel ids by logging into Discord in a browser and opening the desired channel. The URL will have this format: `discordapp.com/channels/guild_id/channel_id`. Once you have figured out the appropriate room addrss, you can join by doing `/join #_discord_guildid_channelid` in your Matrix client. +6. If you've already installed Matrix services using the playbook before, you'll need to re-run it (`--tags=setup-all,start`). If not, proceed with [configuring other playbook services](configuring-playbook.md) and then with [Installing](installing.md). Get back to this guide once ready. +7. Retrieve Discord invite link from the `{{ matrix_appservice_discord_config_path }}/invite_link` file on the server (this defaults to `/matrix/appservice-discord/config/invite_link`). You need to peek at the file on the server via SSH, etc., because it's not available via HTTP(S). +8. Invite the Bot to Discord servers you wish to bridge. Administrator permission is recommended. +9. Room addresses follow this syntax: `#_discord_guildid_channelid`. You can easily find the guild and channel ids by logging into Discord in a browser and opening the desired channel. The URL will have this format: `discordapp.com/channels/guild_id/channel_id`. Once you have figured out the appropriate room addrss, you can join by doing `/join #_discord_guildid_channelid` in your Matrix client. Other configuration options are available via the `matrix_appservice_discord_configuration_extension_yaml` variable. diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml index 392f3028..cb262166 100644 --- a/roles/matrix-bridge-appservice-discord/defaults/main.yml +++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml @@ -41,6 +41,14 @@ matrix_appservice_discord_bridge_homeserverUrl: "{{ matrix_homeserver_url }}" matrix_appservice_discord_bridge_disablePresence: false matrix_appservice_discord_bridge_enableSelfServiceBridging: false +# Tells whether the bot should make use of "Privileged Gateway Intents". +# +# Enabling this means that you need to enable it for the bot (Discord application) as well, +# by triggering all Intent checkboxes on a page like this: `https://discord.com/developers/applications/694448564151123988/bot` +# +# Learn more: https://gist.github.com/advaith1/e69bcc1cdd6d0087322734451f15aa2f +matrix_appservice_discord_auth_usePrivilegedIntents: false + matrix_appservice_discord_configuration_yaml: "{{ lookup('template', 'templates/config.yaml.j2') }}" matrix_appservice_discord_configuration_extension_yaml: | diff --git a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 index 0c2aab41..aca49228 100644 --- a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 +++ b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 @@ -34,6 +34,9 @@ bridge: auth: clientID: {{ matrix_appservice_discord_client_id|string|to_json }} botToken: {{ matrix_appservice_discord_bot_token }} + # You must enable "Privileged Gateway Intents" in your bot settings on discord.com (e.g. https://discord.com/developers/applications/12345/bot) + # for this to work + usePrivilegedIntents: false logging: # What level should the logger output to the console at. console: "warn" #silly, verbose, info, http, warn, error, silent From c985e17f18d0c3eba97febc546902039d9617ef8 Mon Sep 17 00:00:00 2001 From: benkuly <12199167+benkuly@users.noreply.github.com> Date: Fri, 13 Nov 2020 08:44:21 +0100 Subject: [PATCH 045/217] updated matrix-sms-bridge --- CHANGELOG.md | 13 +++++++ roles/matrix-bridge-sms/defaults/main.yml | 36 +++++++++---------- roles/matrix-bridge-sms/tasks/init.yml | 2 +- .../matrix-bridge-sms/tasks/setup_install.yml | 20 ++--------- .../tasks/setup_uninstall.yml | 25 +------------ .../tasks/validate_config.yml | 3 +- 6 files changed, 35 insertions(+), 64 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa378ae3..ae96ef36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +# 2020-11-13 + +## Breaking change matrix-sms-bridge + +The new version of [matrix-sms-bridge](https://github.com/benkuly/matrix-sms-bridge) changed its database from neo4j to h2. You need to sync the bridge at the first start. Note that this only will sync rooms where the @smsbot:yourServer is member. For rooms without @smsbot:yourServer you need to kick and invite the telephone number **or** invite @smsbot:yourServer. + +1. Add the following to your `vars.yml` file: `matrix_sms_bridge_container_extra_arguments=['--env SPRING_PROFILES_ACTIVE=initialsync']` +2. Login to your host shell and remove old systemd file from your host: `rm /etc/systemd/system/matrix-sms-bridge-database.service` +2. Run `ansible-playbook -i inventory/hosts setup.yml --tags=setup-matrix-sms-bridge,start` +3. Login to your host shell and check the logs with `journalctl -u matrix-sms-bridge` until the sync finished. +4. Remove the var from the first step. +5. Run `ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`. + # 2020-11-10 ## Dynamic DNS support diff --git a/roles/matrix-bridge-sms/defaults/main.yml b/roles/matrix-bridge-sms/defaults/main.yml index 25d2208c..0801fe5b 100644 --- a/roles/matrix-bridge-sms/defaults/main.yml +++ b/roles/matrix-bridge-sms/defaults/main.yml @@ -3,9 +3,7 @@ matrix_sms_bridge_enabled: true -matrix_sms_bridge_docker_image: "folivonet/matrix-sms-bridge:0.3.2.RELEASE" -matrix_sms_bridge_database_docker_image: "neo4j:latest" -matrix_sms_bridge_database_docker_image_force_pull: "{{ matrix_sms_bridge_docker_image.endswith(':latest') }}" +matrix_sms_bridge_docker_image: "folivonet/matrix-sms-bridge:0.4.1.RELEASE" matrix_sms_bridge_base_path: "{{ matrix_base_data_path }}/matrix-sms-bridge" matrix_sms_bridge_config_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/config" @@ -17,29 +15,22 @@ matrix_sms_bridge_data_spool_inbox_processed_path: "{{ matrix_base_data_path }}/ matrix_sms_bridge_data_spool_outbox_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/data/spool/outbox" matrix_sms_bridge_data_spool_sent_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/data/spool/sent" matrix_sms_bridge_data_spool_error_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/data/spool/error" -matrix_sms_bridge_database_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/database" matrix_sms_bridge_appservice_token: '' matrix_sms_bridge_homeserver_token: '' -matrix_sms_bridge_database_username: 'neo4j' -matrix_sms_bridge_database_password: '' - matrix_sms_bridge_container_http_host_bind_port: '' # A list of extra arguments to pass to the container matrix_sms_bridge_container_extra_arguments: [] # List of systemd services that service depends on. -matrix_sms_bridge_systemd_required_services_list: ['docker.service','matrix-sms-bridge-database.service'] -matrix_sms_bridge_database_systemd_required_services_list: ['docker.service'] +matrix_sms_bridge_systemd_required_services_list: ['docker.service'] # List of systemd services that service wants matrix_sms_bridge_systemd_wanted_services_list: [] -matrix_sms_bridge_database_systemd_wanted_services_list: [] matrix_sms_bridge_appservice_url: 'http://matrix-sms-bridge:8080' -matrix_sms_bridge_database_url: 'bolt://matrix-sms-bridge-database:7687' matrix_sms_bridge_homeserver_hostname: 'matrix-synapse' matrix_sms_bridge_homeserver_port: '8008' @@ -47,25 +38,19 @@ matrix_sms_bridge_homserver_domain: "{{ matrix_domain }}" matrix_sms_bridge_default_room: '' matrix_sms_bridge_default_region: '' matrix_sms_bridge_default_timezone: '' +matrix_sms_bridge_single_mode_enabled: false matrix_sms_bridge_gammu_modem: '' matrix_sms_bridge_modem_group: 'dialout' matrix_sms_bridge_gammu_reset_frequency: 0 matrix_sms_bridge_gammu_hard_reset_frequency: 0 +matrix_sms_bridge_gammu_smsc: '' matrix_sms_bridge_configuration_yaml: | #jinja2: lstrip_blocks: "True" # Database connection - org: - neo4j: - driver: - uri: {{ matrix_sms_bridge_database_url }} - authentication: - username: {{ matrix_sms_bridge_database_username }} - password: {{ matrix_sms_bridge_database_password }} - matrix: bridge: sms: @@ -74,6 +59,7 @@ matrix_sms_bridge_configuration_yaml: | defaultRoomId: "{{ matrix_sms_bridge_default_room }}" defaultRegion: "{{ matrix_sms_bridge_default_region }}" defaultTimeZone: "{{ matrix_sms_bridge_default_timezone }}" + singleModeEnabled: "{{ matrix_sms_bridge_single_mode_enabled }}" provider: gammu: # (optional) default is disabled @@ -85,6 +71,12 @@ matrix_sms_bridge_configuration_yaml: | bot: # The domain-part of matrix-ids. E. g. example.org when your userIds look like @unicorn:example.org serverName: {{ matrix_sms_bridge_homserver_domain }} + migration: + url: "jdbc:h2:file:/data/database/db" + username: sa + database: + url: "r2dbc:h2:file:////data/database/db" + username: sa client: homeServer: # The hostname of your Homeserver. @@ -128,6 +120,9 @@ matrix_sms_bridge_gammu_configuration: | InboxFormat = detail OutboxFormat = detail TransmitFormat = auto + {% if matrix_sms_bridge_gammu_smsc is defined and matrix_sms_bridge_gammu_smsc|length %} + SMSC = {{ matrix_sms_bridge_gammu_smsc }} + {% endif %} ResetFrequency = {{ matrix_sms_bridge_gammu_reset_frequency }} HardResetFrequency = {{ matrix_sms_bridge_gammu_hard_reset_frequency }} debugLevel = 1 @@ -145,6 +140,9 @@ matrix_sms_bridge_registration_yaml: | users: - exclusive: true regex: '^@sms_.+:{{ matrix_sms_bridge_homserver_domain|regex_escape }}$' + aliases: + - exclusive: true + regex: '^#sms_.+:{{ matrix_sms_bridge_homserver_domain|regex_escape }}$' url: {{ matrix_sms_bridge_appservice_url }} sender_localpart: smsbot rate_limited: false diff --git a/roles/matrix-bridge-sms/tasks/init.yml b/roles/matrix-bridge-sms/tasks/init.yml index 7a49e4b2..cca4d4c5 100644 --- a/roles/matrix-bridge-sms/tasks/init.yml +++ b/roles/matrix-bridge-sms/tasks/init.yml @@ -7,7 +7,7 @@ when: "matrix_sms_bridge_enabled and matrix_synapse_role_executed|default(False)" - set_fact: - matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-sms-bridge','matrix-sms-bridge-database'] }}" + matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-sms-bridge'] }}" when: matrix_sms_bridge_enabled|bool # If the matrix-synapse role is not used, these variables may not exist. diff --git a/roles/matrix-bridge-sms/tasks/setup_install.yml b/roles/matrix-bridge-sms/tasks/setup_install.yml index 254510a3..1d36ea8d 100644 --- a/roles/matrix-bridge-sms/tasks/setup_install.yml +++ b/roles/matrix-bridge-sms/tasks/setup_install.yml @@ -5,14 +5,6 @@ name: "{{ matrix_sms_bridge_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" -- name: Ensure matrix-sms-bridge databse image is pulled - docker_image: - name: "{{ matrix_sms_bridge_database_docker_image }}" - source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" - force_source: "{{ matrix_sms_bridge_database_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_sms_bridge_database_docker_image_force_pull }}" - - - name: Ensure matrix-sms-bridge paths exist file: path: "{{ item }}" @@ -31,7 +23,6 @@ - "{{ matrix_sms_bridge_data_spool_outbox_path }}" - "{{ matrix_sms_bridge_data_spool_sent_path }}" - "{{ matrix_sms_bridge_data_spool_error_path }}" - - "{{ matrix_sms_bridge_database_path }}" - name: Ensure matrix-sms-bridge application.yml installed copy: @@ -64,14 +55,7 @@ mode: 0644 register: matrix_sms_bridge_systemd_service_result -- name: Ensure matrix-sms-bridge-database.service installed - template: - src: "{{ role_path }}/templates/systemd/matrix-sms-bridge-database.service.j2" - dest: "{{ matrix_systemd_path }}/matrix-sms-bridge-database.service" - mode: 0644 - register: matrix_sms_bridge_database_systemd_service_result - -- name: Ensure systemd reloaded after matrix-sms-bridge.service or matrix-sms-bridge-database.service installation +- name: Ensure systemd reloaded after matrix-sms-bridge.service installation service: daemon_reload: yes - when: matrix_sms_bridge_systemd_service_result.changed or matrix_sms_bridge_database_systemd_service_result.changed + when: matrix_sms_bridge_systemd_service_result.changed \ No newline at end of file diff --git a/roles/matrix-bridge-sms/tasks/setup_uninstall.yml b/roles/matrix-bridge-sms/tasks/setup_uninstall.yml index 87436687..03ddaad0 100644 --- a/roles/matrix-bridge-sms/tasks/setup_uninstall.yml +++ b/roles/matrix-bridge-sms/tasks/setup_uninstall.yml @@ -5,11 +5,6 @@ path: "{{ matrix_systemd_path }}/matrix-sms-bridge.service" register: matrix_sms_bridge_service_stat -- name: Check existence of matrix-sms-bridge-database service - stat: - path: "{{ matrix_systemd_path }}/matrix-sms-bridge-database.service" - register: matrix_sms_bridge_database_service_stat - - name: Ensure matrix-sms-bridge is stopped service: name: matrix-sms-bridge @@ -17,26 +12,8 @@ daemon_reload: yes when: "matrix_sms_bridge_service_stat.stat.exists" -- name: Ensure matrix-sms-bridge-database is stopped - service: - name: matrix-sms-bridge-database - state: stopped - daemon_reload: yes - when: "matrix_sms_bridge_database_service_stat.stat.exists" - - name: Ensure matrix-sms-bridge.service doesn't exist file: path: "{{ matrix_systemd_path }}/matrix-sms-bridge.service" state: absent - when: "matrix_sms_bridge_service_stat.stat.exists" - -- name: Ensure matrix-sms-bridge-database.service doesn't exist - file: - path: "{{ matrix_systemd_path }}/matrix-sms-bridge-database.service" - state: absent - when: "matrix_sms_bridge_database_service_stat.stat.exists" - -- name: Ensure systemd reloaded after matrix-sms-bridge.service or matrix-sms-bridge-database.service removal - service: - daemon_reload: yes - when: matrix_sms_bridge_service_stat.stat.exists or matrix_sms_bridge_database_service_stat.stat.exists + when: "matrix_sms_bridge_service_stat.stat.exists" \ No newline at end of file diff --git a/roles/matrix-bridge-sms/tasks/validate_config.yml b/roles/matrix-bridge-sms/tasks/validate_config.yml index 0e40aefa..79f62fe1 100644 --- a/roles/matrix-bridge-sms/tasks/validate_config.yml +++ b/roles/matrix-bridge-sms/tasks/validate_config.yml @@ -8,7 +8,6 @@ with_items: - "matrix_sms_bridge_appservice_token" - "matrix_sms_bridge_homeserver_token" - - "matrix_sms_bridge_database_password" - "matrix_sms_bridge_gammu_modem" - "matrix_sms_bridge_default_region" - - "matrix_sms_bridge_default_timezone" + - "matrix_sms_bridge_default_timezone" \ No newline at end of file From 078592454ca0272563525dddcadedb578c4d5a4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Marques?= Date: Fri, 13 Nov 2020 18:13:45 +0000 Subject: [PATCH 046/217] Update jitsi web to stable-5142 Changelog https://github.com/jitsi/jitsi-meet/releases/tag/stable%2Fjitsi-meet_5142 --- roles/matrix-jitsi/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-jitsi/defaults/main.yml b/roles/matrix-jitsi/defaults/main.yml index 7486821d..d05d2c71 100644 --- a/roles/matrix-jitsi/defaults/main.yml +++ b/roles/matrix-jitsi/defaults/main.yml @@ -51,7 +51,7 @@ matrix_jitsi_jibri_recorder_password: '' matrix_jitsi_enable_lobby: false -matrix_jitsi_container_image_tag: "stable-4857" +matrix_jitsi_container_image_tag: "stable-5142" matrix_jitsi_web_docker_image: "jitsi/web:{{ matrix_jitsi_container_image_tag }}" matrix_jitsi_web_docker_image_force_pull: "{{ matrix_jitsi_web_docker_image.endswith(':latest') }}" From 5eed874199e51be851995efcb980c260866aceca Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sat, 14 Nov 2020 22:47:14 +0200 Subject: [PATCH 047/217] Improve self-building experience (avoid conflict with pullable images) Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/716 This patch makes us use more fully-qualified container image names (either prefixed with docker.io/ or with localhost/). The latter happens when self-building is enabled. We've recently had issues where if an image was removed manually and the service was restarted (making `docker run` fetch it from Docker Hub, etc.), we'd end up with a pulled image, even though we're aiming for a self-built one. Re-running the playbook would then not do a rebuild, because: - the image with that name already exists (even though it's something else) - we sometimes had conditional logic where we'd build only if the git repo changed By explicitly changing the name of the images (prefixing with localhost/), we avoid such confusion and the possibility that we'd automatically pul something which is not what we expect. Also, I've removed that condition where building would happen on git changes only. We now always build (unless an image with that name already exists). We just force-build when the git repo changes. --- roles/matrix-bridge-mautrix-facebook/defaults/main.yml | 3 ++- .../matrix-bridge-mautrix-facebook/tasks/setup_install.yml | 4 ++-- roles/matrix-bridge-mautrix-hangouts/defaults/main.yml | 3 ++- .../matrix-bridge-mautrix-hangouts/tasks/setup_install.yml | 6 +++--- roles/matrix-bridge-mx-puppet-discord/defaults/main.yml | 3 ++- .../matrix-bridge-mx-puppet-discord/tasks/setup_install.yml | 4 +++- roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml | 3 ++- .../tasks/setup_install.yml | 6 ++++-- roles/matrix-bridge-mx-puppet-skype/defaults/main.yml | 3 ++- roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml | 6 +++--- roles/matrix-bridge-mx-puppet-slack/defaults/main.yml | 3 ++- roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml | 2 ++ roles/matrix-bridge-mx-puppet-steam/defaults/main.yml | 3 ++- roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml | 2 ++ roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml | 3 ++- .../matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml | 2 ++ roles/matrix-client-element/defaults/main.yml | 3 ++- roles/matrix-client-element/tasks/setup.yml | 6 ++++-- roles/matrix-coturn/defaults/main.yml | 3 ++- roles/matrix-coturn/tasks/setup_coturn.yml | 4 +++- roles/matrix-dynamic-dns/defaults/main.yml | 4 +++- roles/matrix-dynamic-dns/tasks/install.yml | 6 ++++-- roles/matrix-ma1sd/defaults/main.yml | 3 ++- roles/matrix-ma1sd/tasks/setup_ma1sd.yml | 6 +++++- roles/matrix-mailer/defaults/main.yml | 3 ++- roles/matrix-mailer/tasks/setup_mailer.yml | 4 +++- roles/matrix-registration/defaults/main.yml | 3 ++- roles/matrix-registration/tasks/setup.yml | 4 ++-- roles/matrix-synapse-admin/defaults/main.yml | 3 ++- roles/matrix-synapse-admin/tasks/setup.yml | 4 ++-- roles/matrix-synapse/defaults/main.yml | 3 ++- roles/matrix-synapse/tasks/synapse/setup_install.yml | 6 ++++-- 32 files changed, 81 insertions(+), 40 deletions(-) diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml index 610bcc11..c34da88f 100644 --- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml @@ -6,7 +6,8 @@ matrix_mautrix_facebook_enabled: true matrix_mautrix_facebook_container_image_self_build: false # See: https://mau.dev/tulir/mautrix-facebook/container_registry -matrix_mautrix_facebook_docker_image: "dock.mau.dev/tulir/mautrix-facebook:latest" +matrix_mautrix_facebook_docker_image: "{{ matrix_mautrix_facebook_docker_image_name_prefix }}tulir/mautrix-facebook:latest" +matrix_mautrix_facebook_docker_image_name_prefix: "{{ 'localhost/' if matrix_mautrix_facebook_container_image_self_build else 'dock.mau.dev/' }}" matrix_mautrix_facebook_docker_image_force_pull: "{{ matrix_mautrix_facebook_docker_image.endswith(':latest') }}" matrix_mautrix_facebook_base_path: "{{ matrix_base_data_path }}/mautrix-facebook" diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml index a1131d8e..75ea7a6b 100644 --- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml @@ -43,12 +43,12 @@ docker_image: name: "{{ matrix_mautrix_facebook_docker_image }}" source: build - force_source: yes + force_source: "{{ matrix_mautrix_facebook_git_pull_results.changed }}" build: dockerfile: Dockerfile path: "{{ matrix_mautrix_facebook_docker_src_files_path }}" pull: yes - when: "matrix_mautrix_facebook_enabled|bool and matrix_mautrix_facebook_container_image_self_build and matrix_mautrix_facebook_git_pull_results.changed" + when: "matrix_mautrix_facebook_enabled|bool and matrix_mautrix_facebook_container_image_self_build|bool" - name: Check if an old database file already exists stat: diff --git a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml index 8430527a..13bc1a6c 100644 --- a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml @@ -6,7 +6,8 @@ matrix_mautrix_hangouts_enabled: true matrix_mautrix_hangouts_container_image_self_build: false # See: https://mau.dev/tulir/mautrix-hangouts/container_registry -matrix_mautrix_hangouts_docker_image: "dock.mau.dev/tulir/mautrix-hangouts:latest" +matrix_mautrix_hangouts_docker_image: "{{ matrix_mautrix_hangouts_docker_image_name_prefix }}tulir/mautrix-hangouts:latest" +matrix_mautrix_hangouts_docker_image_name_prefix: "{{ 'localhost/' if matrix_mautrix_hangouts_container_image_self_build else 'dock.mau.dev/' }}" matrix_mautrix_hangouts_docker_image_force_pull: "{{ matrix_mautrix_hangouts_docker_image.endswith(':latest') }}" matrix_mautrix_hangouts_base_path: "{{ matrix_base_data_path }}/mautrix-hangouts" diff --git a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml index 97893be7..5dc5f20a 100644 --- a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml @@ -36,18 +36,18 @@ dest: "{{ matrix_mautrix_hangouts_docker_src_files_path }}" force: "yes" register: matrix_mautrix_hangouts_git_pull_results - when: "matrix_mautrix_hangouts_enabled|bool and matrix_mautrix_hangouts_container_image_self_build" + when: "matrix_mautrix_hangouts_enabled|bool and matrix_mautrix_hangouts_container_image_self_build|bool" - name: Ensure Mautrix Hangouts Docker image is built docker_image: name: "{{ matrix_mautrix_hangouts_docker_image }}" source: build - force_source: yes + force_source: "{{ matrix_mautrix_hangouts_git_pull_results.changed }}" build: dockerfile: Dockerfile path: "{{ matrix_mautrix_hangouts_docker_src_files_path }}" pull: yes - when: "matrix_mautrix_hangouts_enabled|bool and matrix_mautrix_hangouts_container_image_self_build and matrix_mautrix_hangouts_git_pull_results.changed" + when: "matrix_mautrix_hangouts_enabled|bool and matrix_mautrix_hangouts_container_image_self_build|bool" - name: Check if an old database file already exists stat: diff --git a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml index 12c597d4..30732000 100644 --- a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml @@ -10,7 +10,8 @@ matrix_mx_puppet_discord_container_image_self_build: false # Takes an ":" or "" value (e.g. "127.0.0.1:8432"), or empty string to not expose. matrix_mx_puppet_discord_container_http_host_bind_port: '' -matrix_mx_puppet_discord_docker_image: "sorunome/mx-puppet-discord:latest" +matrix_mx_puppet_discord_docker_image: "{{ matrix_mx_puppet_discord_docker_image_name_prefix }}sorunome/mx-puppet-discord:latest" +matrix_mx_puppet_discord_docker_image_name_prefix: "{{ 'localhost/' if matrix_mx_puppet_discord_container_image_self_build else 'docker.io/' }}" matrix_mx_puppet_discord_docker_image_force_pull: "{{ matrix_mx_puppet_discord_docker_image.endswith(':latest') }}" matrix_mx_puppet_discord_base_path: "{{ matrix_base_data_path }}/mx-puppet-discord" diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml index e1aef147..e36950bd 100644 --- a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml @@ -35,17 +35,19 @@ repo: https://github.com/matrix-discord/mx-puppet-discord.git dest: "{{ matrix_mx_puppet_discord_docker_src_files_path }}" force: "yes" + register: matrix_mx_puppet_discord_git_pull_results when: "matrix_mx_puppet_discord_enabled|bool and matrix_mx_puppet_discord_container_image_self_build" - name: Ensure MX Puppet Discord Docker image is built docker_image: name: "{{ matrix_mx_puppet_discord_docker_image }}" source: build + force_source: "{{ matrix_mx_puppet_discord_git_pull_results.changed }}" build: dockerfile: Dockerfile path: "{{ matrix_mx_puppet_discord_docker_src_files_path }}" pull: yes - when: "matrix_mx_puppet_discord_enabled|bool and matrix_mx_puppet_discord_container_image_self_build" + when: "matrix_mx_puppet_discord_enabled|bool and matrix_mx_puppet_discord_container_image_self_build|bool" - name: Check if an old database file already exists stat: diff --git a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml index c7488cbb..108a0a34 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml @@ -5,7 +5,8 @@ matrix_mx_puppet_instagram_enabled: true matrix_mx_puppet_instagram_container_image_self_build: false -matrix_mx_puppet_instagram_docker_image: "docker.io/sorunome/mx-puppet-instagram:latest" +matrix_mx_puppet_instagram_docker_image: "{{ matrix_mx_puppet_instagram_docker_image_name_prefix }}sorunome/mx-puppet-instagram:latest" +matrix_mx_puppet_instagram_docker_image_name_prefix: "{{ 'localhost/' if matrix_mx_puppet_instagram_container_image_self_build else 'docker.io/' }}" matrix_mx_puppet_instagram_docker_image_force_pull: "{{ matrix_mx_puppet_instagram_docker_image.endswith(':latest') }}" matrix_mx_puppet_instagram_base_path: "{{ matrix_base_data_path }}/mx-puppet-instagram" diff --git a/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml index d60fbaf2..6b638626 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml @@ -35,17 +35,19 @@ repo: https://github.com/Sorunome/mx-puppet-instagram.git dest: "{{ matrix_mx_puppet_instagram_docker_src_files_path }}" force: "yes" - when: "matrix_mx_puppet_instagram_enabled|bool and matrix_mx_puppet_instagram_container_image_self_build" + register: matrix_mx_puppet_instagram_git_pull_results + when: "matrix_mx_puppet_instagram_enabled|bool and matrix_mx_puppet_instagram_container_image_self_build|bool" - name: Ensure mx-puppet-instagram Docker image is built docker_image: name: "{{ matrix_mx_puppet_instagram_docker_image }}" source: build + force_source: "{{ matrix_mx_puppet_instagram_git_pull_results.changed }}" build: dockerfile: Dockerfile path: "{{ matrix_mx_puppet_instagram_docker_src_files_path }}" pull: yes - when: "matrix_mx_puppet_instagram_enabled|bool and matrix_mx_puppet_instagram_container_image_self_build" + when: "matrix_mx_puppet_instagram_enabled|bool and matrix_mx_puppet_instagram_container_image_self_build|bool" - name: Ensure mx-puppet-instagram config.yaml installed copy: diff --git a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml index 8b977acf..9e1c71bb 100644 --- a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml @@ -5,7 +5,8 @@ matrix_mx_puppet_skype_enabled: true matrix_mx_puppet_skype_container_image_self_build: false -matrix_mx_puppet_skype_docker_image: "sorunome/mx-puppet-skype:latest" +matrix_mx_puppet_skype_docker_image: "{{ matrix_mx_puppet_skype_docker_image_name_prefix }}sorunome/mx-puppet-skype:latest" +matrix_mx_puppet_skype_docker_image_name_prefix: "{{ 'localhost/' if matrix_mx_puppet_skype_container_image_self_build else 'docker.io/' }}" matrix_mx_puppet_skype_docker_image_force_pull: "{{ matrix_mx_puppet_skype_docker_image.endswith(':latest') }}" matrix_mx_puppet_skype_base_path: "{{ matrix_base_data_path }}/mx-puppet-skype" diff --git a/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml index 1febca3e..9b04c290 100644 --- a/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml @@ -36,18 +36,18 @@ dest: "{{ matrix_mx_puppet_skype_docker_src_files_path }}" force: "yes" register: matrix_mx_puppet_skype_git_pull_results - when: "matrix_mx_puppet_skype_enabled|bool and matrix_mx_puppet_skype_container_image_self_build" + when: "matrix_mx_puppet_skype_enabled|bool and matrix_mx_puppet_skype_container_image_self_build|bool" - name: Ensure MX Puppet Skype Docker image is built docker_image: name: "{{ matrix_mx_puppet_skype_docker_image }}" source: build - force_source: yes + force_source: "{{ matrix_mx_puppet_skype_git_pull_results.changed }}" build: dockerfile: Dockerfile path: "{{ matrix_mx_puppet_skype_docker_src_files_path }}" pull: yes - when: "matrix_mx_puppet_skype_enabled|bool and matrix_mx_puppet_skype_container_image_self_build and matrix_mx_puppet_skype_git_pull_results.changed" + when: "matrix_mx_puppet_skype_enabled|bool and matrix_mx_puppet_skype_container_image_self_build|bool" - name: Check if an old database file already exists stat: diff --git a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml index 6acfb714..94b37041 100644 --- a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml @@ -10,7 +10,8 @@ matrix_mx_puppet_slack_container_image_self_build: false # Takes an ":" or "" value (e.g. "127.0.0.1:8432"), or empty string to not expose. matrix_mx_puppet_slack_container_http_host_bind_port: '' -matrix_mx_puppet_slack_docker_image: "sorunome/mx-puppet-slack:latest" +matrix_mx_puppet_slack_docker_image: "{{ matrix_mx_puppet_slack_docker_image_name_prefix }}sorunome/mx-puppet-slack:latest" +matrix_mx_puppet_slack_docker_image_name_prefix: "{{ 'localhost/' if matrix_mx_puppet_slack_container_image_self_build else 'docker.io/' }}" matrix_mx_puppet_slack_docker_image_force_pull: "{{ matrix_mx_puppet_slack_docker_image.endswith(':latest') }}" matrix_mx_puppet_slack_base_path: "{{ matrix_base_data_path }}/mx-puppet-slack" diff --git a/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml index ddaede72..b7cdc13c 100644 --- a/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml @@ -35,12 +35,14 @@ repo: https://github.com/Sorunome/mx-puppet-slack.git dest: "{{ matrix_mx_puppet_slack_docker_src_files_path }}" force: "yes" + register: matrix_mx_puppet_slack_git_pull_results when: "matrix_mx_puppet_slack_enabled|bool and matrix_mx_puppet_slack_container_image_self_build" - name: Ensure MX Puppet Slack Docker image is built docker_image: name: "{{ matrix_mx_puppet_slack_docker_image }}" source: build + force_source: "{{ matrix_mx_puppet_slack_git_pull_results.changed }}" build: dockerfile: Dockerfile path: "{{ matrix_mx_puppet_slack_docker_src_files_path }}" diff --git a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml index bea26d39..cb06bf7d 100644 --- a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml @@ -10,7 +10,8 @@ matrix_mx_puppet_steam_container_image_self_build: false # Takes an ":" or "" value (e.g. "127.0.0.1:8432"), or empty string to not expose. matrix_mx_puppet_steam_container_http_host_bind_port: '' -matrix_mx_puppet_steam_docker_image: "icewind1991/mx-puppet-steam:latest" +matrix_mx_puppet_steam_docker_image: "{{ matrix_mx_puppet_steam_docker_image_name_prefix }}icewind1991/mx-puppet-steam:latest" +matrix_mx_puppet_steam_docker_image_name_prefix: "{{ 'localhost/' if matrix_mx_puppet_steam_container_image_self_build else 'docker.io/' }}" matrix_mx_puppet_steam_docker_image_force_pull: "{{ matrix_mx_puppet_steam_docker_image.endswith(':latest') }}" matrix_mx_puppet_steam_base_path: "{{ matrix_base_data_path }}/mx-puppet-steam" diff --git a/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml index be41c645..d9679510 100644 --- a/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml @@ -35,12 +35,14 @@ repo: https://github.com/icewind1991/mx-puppet-steam.git dest: "{{ matrix_mx_puppet_steam_docker_src_files_path }}" force: "yes" + register: matrix_mx_puppet_steam_git_pull_results when: "matrix_mx_puppet_steam_enabled|bool and matrix_mx_puppet_steam_container_image_self_build" - name: Ensure MX Puppet Steam Docker image is built docker_image: name: "{{ matrix_mx_puppet_steam_docker_image }}" source: build + force_source: "{{ matrix_mx_puppet_steam_git_pull_results.changed }}" build: dockerfile: Dockerfile path: "{{ matrix_mx_puppet_steam_docker_src_files_path }}" diff --git a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml index 8299fd81..18c39826 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml @@ -10,7 +10,8 @@ matrix_mx_puppet_twitter_container_image_self_build: false # Takes an ":" or "" value (e.g. "127.0.0.1:8432"), or empty string to not expose. matrix_mx_puppet_twitter_container_http_host_bind_port: '' -matrix_mx_puppet_twitter_docker_image: "sorunome/mx-puppet-twitter:latest" +matrix_mx_puppet_twitter_docker_image: "{{ matrix_mx_puppet_twitter_docker_image_name_prefix }}sorunome/mx-puppet-twitter:latest" +matrix_mx_puppet_twitter_docker_image_name_prefix: "{{ 'localhost/' if matrix_mx_puppet_twitter_container_image_self_build else 'docker.io/' }}" matrix_mx_puppet_twitter_docker_image_force_pull: "{{ matrix_mx_puppet_twitter_docker_image.endswith(':latest') }}" matrix_mx_puppet_twitter_base_path: "{{ matrix_base_data_path }}/mx-puppet-twitter" diff --git a/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml index 4773055d..10e49572 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml @@ -35,12 +35,14 @@ repo: https://github.com/Sorunome/mx-puppet-twitter.git dest: "{{ matrix_mx_puppet_twitter_docker_src_files_path }}" force: "yes" + register: matrix_mx_puppet_twitter_git_pull_results when: "matrix_mx_puppet_twitter_enabled|bool and matrix_mx_puppet_twitter_container_image_self_build" - name: Ensure MX Puppet Twitter Docker image is built docker_image: name: "{{ matrix_mx_puppet_twitter_docker_image }}" source: build + force_source: "{{ matrix_mx_puppet_twitter_git_pull_results.changed }}" build: dockerfile: Dockerfile path: "{{ matrix_mx_puppet_twitter_docker_src_files_path }}" diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index c591b025..f1504c5b 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -2,7 +2,8 @@ matrix_client_element_enabled: true matrix_client_element_container_image_self_build: false -matrix_client_element_docker_image: "vectorim/element-web:v1.7.13" +matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:v1.7.13" +matrix_client_element_docker_image_name_prefix: "{{ 'localhost/' if matrix_client_element_container_image_self_build else 'docker.io/' }}" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" matrix_client_element_data_path: "{{ matrix_base_data_path }}/client-element" diff --git a/roles/matrix-client-element/tasks/setup.yml b/roles/matrix-client-element/tasks/setup.yml index 2e8071ca..81f7842f 100644 --- a/roles/matrix-client-element/tasks/setup.yml +++ b/roles/matrix-client-element/tasks/setup.yml @@ -30,17 +30,19 @@ dest: "{{ matrix_client_element_docker_src_files_path }}" version: "{{ matrix_client_element_docker_image.split(':')[1] }}" force: "yes" - when: "matrix_client_element_enabled|bool and matrix_client_element_container_image_self_build" + register: matrix_client_element_git_pull_results + when: "matrix_client_element_enabled|bool and matrix_client_element_container_image_self_build|bool" - name: Ensure Element Docker image is built docker_image: name: "{{ matrix_client_element_docker_image }}" source: build + force_source: "{{ matrix_client_element_git_pull_results.changed }}" build: dockerfile: Dockerfile path: "{{ matrix_client_element_docker_src_files_path }}" pull: yes - when: "matrix_client_element_enabled|bool and matrix_client_element_container_image_self_build" + when: "matrix_client_element_enabled|bool and matrix_client_element_container_image_self_build|bool" - name: Ensure Element configuration installed copy: diff --git a/roles/matrix-coturn/defaults/main.yml b/roles/matrix-coturn/defaults/main.yml index 104019a4..4aae3f06 100644 --- a/roles/matrix-coturn/defaults/main.yml +++ b/roles/matrix-coturn/defaults/main.yml @@ -2,7 +2,8 @@ matrix_coturn_enabled: true matrix_coturn_container_image_self_build: false -matrix_coturn_docker_image: "instrumentisto/coturn:4.5.1.3" +matrix_coturn_docker_image: "{{ matrix_coturn_docker_image_name_prefix }}instrumentisto/coturn:4.5.1.3" +matrix_coturn_docker_image_name_prefix: "{{ 'localhost/' if matrix_coturn_container_image_self_build else 'docker.io/' }}" matrix_coturn_docker_image_force_pull: "{{ matrix_coturn_docker_image.endswith(':latest') }}" # The Docker network that Coturn would be put into. diff --git a/roles/matrix-coturn/tasks/setup_coturn.yml b/roles/matrix-coturn/tasks/setup_coturn.yml index 17b6137d..fa622caf 100644 --- a/roles/matrix-coturn/tasks/setup_coturn.yml +++ b/roles/matrix-coturn/tasks/setup_coturn.yml @@ -29,17 +29,19 @@ dest: "{{ matrix_coturn_docker_src_files_path }}" version: "{{ matrix_coturn_docker_image.split(':')[1] }}" force: "yes" + register: matrix_coturn_git_pull_results when: "matrix_coturn_enabled|bool and matrix_coturn_container_image_self_build" - name: Ensure Coturn Docker image is built docker_image: name: "{{ matrix_coturn_docker_image }}" source: build + force_source: "{{ matrix_coturn_git_pull_results.changed }}" build: dockerfile: Dockerfile path: "{{ matrix_coturn_docker_src_files_path }}" pull: yes - when: "matrix_coturn_enabled|bool and matrix_coturn_container_image_self_build" + when: "matrix_coturn_enabled|bool and matrix_coturn_container_image_self_build|bool" - name: Ensure Coturn configuration path exists file: diff --git a/roles/matrix-dynamic-dns/defaults/main.yml b/roles/matrix-dynamic-dns/defaults/main.yml index 0d34ccea..17c57f2c 100644 --- a/roles/matrix-dynamic-dns/defaults/main.yml +++ b/roles/matrix-dynamic-dns/defaults/main.yml @@ -5,7 +5,9 @@ matrix_dynamic_dns_enabled: true matrix_dynamic_dns_daemon_interval: '300' # The docker container to use when in mode -matrix_dynamic_dns_docker_image: 'linuxserver/ddclient:v3.9.1-ls45' +matrix_dynamic_dns_docker_image: '{{ matrix_dynamic_dns_docker_image_name_prefix }}linuxserver/ddclient:v3.9.1-ls45' + +matrix_dynamic_dns_docker_image_name_prefix: "{{ 'localhost/' if matrix_dynamic_dns_container_image_self_build else 'docker.io/' }}" # The image to force pull matrix_dynamic_dns_docker_image_force_pull: "{{ matrix_dynamic_dns_docker_image.endswith(':latest') }}" diff --git a/roles/matrix-dynamic-dns/tasks/install.yml b/roles/matrix-dynamic-dns/tasks/install.yml index 225738bf..9ca1043f 100644 --- a/roles/matrix-dynamic-dns/tasks/install.yml +++ b/roles/matrix-dynamic-dns/tasks/install.yml @@ -26,17 +26,19 @@ repo: https://github.com/linuxserver/docker-ddclient.git dest: "{{ matrix_dynamic_dns_docker_src_files_path }}" force: "yes" - when: "matrix_dynamic_dns_enabled|bool and matrix_dynamic_dns_container_image_self_build" + register: matrix_dynamic_dns_git_pull_results + when: "matrix_dynamic_dns_enabled|bool and matrix_dynamic_dns_container_image_self_build|bool" - name: Ensure Dynamic DNS Docker image is built docker_image: name: "{{ matrix_dynamic_dns_docker_image }}" source: build + force_source: "{{ matrix_dynamic_dns_git_pull_results.changed }}" build: dockerfile: Dockerfile path: "{{ matrix_dynamic_dns_docker_src_files_path }}" pull: yes - when: "matrix_dynamic_dns_enabled|bool and matrix_dynamic_dns_container_image_self_build" + when: "matrix_dynamic_dns_enabled|bool and matrix_dynamic_dns_container_image_self_build|bool" - name: Ensure Dynamic DNS ddclient.conf installed template: diff --git a/roles/matrix-ma1sd/defaults/main.yml b/roles/matrix-ma1sd/defaults/main.yml index 63bc5a59..b86f7ee0 100644 --- a/roles/matrix-ma1sd/defaults/main.yml +++ b/roles/matrix-ma1sd/defaults/main.yml @@ -7,7 +7,8 @@ matrix_ma1sd_container_image_self_build: false matrix_ma1sd_architecture: "amd64" -matrix_ma1sd_docker_image: "ma1uta/ma1sd:2.4.0-{{ matrix_ma1sd_architecture }}" +matrix_ma1sd_docker_image: "{{ matrix_ma1sd_docker_image_name_prefix }}ma1uta/ma1sd:2.4.0-{{ matrix_ma1sd_architecture }}" +matrix_ma1sd_docker_image_name_prefix: "{{ 'localhost/' if matrix_ma1sd_container_image_self_build else 'docker.io/' }}" matrix_ma1sd_docker_image_force_pull: "{{ matrix_ma1sd_docker_image.endswith(':latest') }}" matrix_ma1sd_base_path: "{{ matrix_base_data_path }}/ma1sd" diff --git a/roles/matrix-ma1sd/tasks/setup_ma1sd.yml b/roles/matrix-ma1sd/tasks/setup_ma1sd.yml index 44fe6b19..8814e6b9 100644 --- a/roles/matrix-ma1sd/tasks/setup_ma1sd.yml +++ b/roles/matrix-ma1sd/tasks/setup_ma1sd.yml @@ -56,6 +56,7 @@ dest: "{{ matrix_ma1sd_docker_src_files_path }}" version: "{{ matrix_ma1sd_docker_image.split(':')[1].split('-')[0] }}" force: "yes" + register: matrix_ma1sd_git_pull_results - name: Ensure ma1sd Docker image is built shell: "./gradlew dockerBuild" @@ -64,7 +65,10 @@ - name: Ensure ma1sd Docker image is tagged correctly docker_image: - name: "{{ matrix_ma1sd_docker_image.split('-')[0] }}" + # The build script always tags the image with something like `ma1uta/ma1sd:2.4.0`. + # Remove the `-{{ matrix_ma1sd_architecture }}` suffix and our `localhost/` prefix (applied when self-building) + # to get to what has actually been built, so we can retag it as `{{ matrix_ma1sd_docker_image }}`. + name: "{{ matrix_ma1sd_docker_image.split('-')[0].replace('localhost/', '') }}" repository: "{{ matrix_ma1sd_docker_image }}" force_tag: yes source: local diff --git a/roles/matrix-mailer/defaults/main.yml b/roles/matrix-mailer/defaults/main.yml index 0a8fd13c..f396d8ee 100644 --- a/roles/matrix-mailer/defaults/main.yml +++ b/roles/matrix-mailer/defaults/main.yml @@ -7,7 +7,8 @@ matrix_mailer_container_image_self_build_repository_url: "https://github.com/dev matrix_mailer_container_image_self_build_src_files_path: "{{ matrix_mailer_base_path }}/docker-src" matrix_mailer_container_image_self_build_version: "{{ matrix_mailer_docker_image.split(':')[1] }}" -matrix_mailer_docker_image: "devture/exim-relay:4.93.1-r0" +matrix_mailer_docker_image: "{{ matrix_mailer_docker_image_name_prefix }}devture/exim-relay:4.93.1-r0" +matrix_mailer_docker_image_name_prefix: "{{ 'localhost/' if matrix_mailer_container_image_self_build else 'docker.io/' }}" matrix_mailer_docker_image_force_pull: "{{ matrix_mailer_docker_image.endswith(':latest') }}" # The user/group that the container runs with. diff --git a/roles/matrix-mailer/tasks/setup_mailer.yml b/roles/matrix-mailer/tasks/setup_mailer.yml index 99370638..cb979080 100644 --- a/roles/matrix-mailer/tasks/setup_mailer.yml +++ b/roles/matrix-mailer/tasks/setup_mailer.yml @@ -29,12 +29,14 @@ dest: "{{ matrix_mailer_container_image_self_build_src_files_path }}" version: "{{ matrix_mailer_container_image_self_build_version }}" force: "yes" - when: "matrix_mailer_container_image_self_build|bool" + register: matrix_mailer_git_pull_results + when: "matrix_mailer_enabled|bool and matrix_mailer_container_image_self_build|bool" - name: Ensure exim-relay Docker image is built docker_image: name: "{{ matrix_mailer_docker_image }}" source: build + force_source: "{{ matrix_mailer_git_pull_results.changed }}" build: dockerfile: Dockerfile path: "{{ matrix_mailer_container_image_self_build_src_files_path }}" diff --git a/roles/matrix-registration/defaults/main.yml b/roles/matrix-registration/defaults/main.yml index 7eb9340e..c2150488 100644 --- a/roles/matrix-registration/defaults/main.yml +++ b/roles/matrix-registration/defaults/main.yml @@ -12,7 +12,8 @@ matrix_registration_docker_src_files_path: "{{ matrix_registration_base_path }}/ matrix_registration_version: "v0.7.0" -matrix_registration_docker_image: "devture/zeratax-matrix-registration:{{ matrix_registration_version }}" +matrix_registration_docker_image: "{{ matrix_registration_docker_image_name_prefix }}devture/zeratax-matrix-registration:{{ matrix_registration_version }}" +matrix_registration_docker_image_name_prefix: "{{ 'localhost/' if matrix_registration_container_image_self_build else 'docker.io/' }}" matrix_registration_docker_image_force_pull: "{{ matrix_registration_docker_image.endswith(':latest') }}" matrix_registration_docker_repo: "https://github.com/ZerataX/matrix-registration" diff --git a/roles/matrix-registration/tasks/setup.yml b/roles/matrix-registration/tasks/setup.yml index 29b2347b..253f7aea 100644 --- a/roles/matrix-registration/tasks/setup.yml +++ b/roles/matrix-registration/tasks/setup.yml @@ -39,12 +39,12 @@ docker_image: name: "{{ matrix_registration_docker_image }}" source: build - force_source: yes + force_source: "{{ matrix_registration_git_pull_results.changed }}" build: dockerfile: Dockerfile path: "{{ matrix_registration_docker_src_files_path }}" pull: yes - when: "matrix_registration_enabled|bool and matrix_registration_container_image_self_build|bool and matrix_registration_git_pull_results.changed" + when: "matrix_registration_enabled|bool and matrix_registration_container_image_self_build|bool" - name: Ensure matrix-registration config installed copy: diff --git a/roles/matrix-synapse-admin/defaults/main.yml b/roles/matrix-synapse-admin/defaults/main.yml index 17de0636..6f1d219e 100644 --- a/roles/matrix-synapse-admin/defaults/main.yml +++ b/roles/matrix-synapse-admin/defaults/main.yml @@ -7,7 +7,8 @@ matrix_synapse_admin_container_self_build: false matrix_synapse_admin_docker_repo: "https://github.com/Awesome-Technologies/synapse-admin.git" matrix_synapse_admin_docker_src_files_path: "{{ matrix_base_data_path }}/synapse-admin/docker-src" -matrix_synapse_admin_docker_image: "awesometechnologies/synapse-admin:0.5.0" +matrix_synapse_admin_docker_image: "{{ matrix_synapse_admin_docker_image_name_prefix }}awesometechnologies/synapse-admin:0.5.0" +matrix_synapse_admin_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_admin_container_self_build else 'docker.io/' }}" matrix_synapse_admin_docker_image_force_pull: "{{ matrix_synapse_admin_docker_image.endswith(':latest') }}" # A list of extra arguments to pass to the container diff --git a/roles/matrix-synapse-admin/tasks/setup.yml b/roles/matrix-synapse-admin/tasks/setup.yml index 0ee5e8d2..6d2b8c20 100644 --- a/roles/matrix-synapse-admin/tasks/setup.yml +++ b/roles/matrix-synapse-admin/tasks/setup.yml @@ -24,12 +24,12 @@ docker_image: name: "{{ matrix_synapse_admin_docker_image }}" source: build - force_source: yes + force_source: "{{ matrix_synapse_admin_git_pull_results }}" build: dockerfile: Dockerfile path: "{{ matrix_synapse_admin_docker_src_files_path }}" pull: yes - when: "matrix_synapse_admin_enabled|bool and matrix_synapse_admin_container_self_build|bool and matrix_synapse_admin_git_pull_results.changed" + when: "matrix_synapse_admin_enabled|bool and matrix_synapse_admin_container_self_build|bool" - name: Ensure matrix-synapse-admin.service installed template: diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 5777e802..4cc819a6 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -5,7 +5,8 @@ matrix_synapse_enabled: true matrix_synapse_container_image_self_build: false -matrix_synapse_docker_image: "matrixdotorg/synapse:v1.22.1" +matrix_synapse_docker_image: "{{ matrix_synapse_docker_image_name_prefix }}matrixdotorg/synapse:v1.22.1" +matrix_synapse_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_container_image_self_build else 'docker.io/' }}" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse" diff --git a/roles/matrix-synapse/tasks/synapse/setup_install.yml b/roles/matrix-synapse/tasks/synapse/setup_install.yml index 58f4d31d..51f846e3 100644 --- a/roles/matrix-synapse/tasks/synapse/setup_install.yml +++ b/roles/matrix-synapse/tasks/synapse/setup_install.yml @@ -24,17 +24,19 @@ dest: "{{ matrix_synapse_docker_src_files_path }}" version: "{{ matrix_synapse_docker_image.split(':')[1] }}" force: "yes" - when: "matrix_synapse_container_image_self_build" + register: matrix_synapse_git_pull_results + when: "matrix_synapse_container_image_self_build|bool" - name: Ensure Synapse Docker image is built docker_image: name: "{{ matrix_synapse_docker_image }}" source: build + force_source: "{{ matrix_synapse_git_pull_results.changed }}" build: dockerfile: docker/Dockerfile path: "{{ matrix_synapse_docker_src_files_path }}" pull: yes - when: "matrix_synapse_container_image_self_build" + when: "matrix_synapse_container_image_self_build|bool" - name: Ensure Synapse Docker image is pulled docker_image: From ccabc82d4cd46f586fac9fc63c4f54abe35cd49c Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sat, 14 Nov 2020 22:53:23 +0200 Subject: [PATCH 048/217] Use more fully-qualified container images This is both for consistency with 93cc71cb69976c and for making things more obvious. --- roles/matrix-bot-matrix-reminder-bot/defaults/main.yml | 2 +- .../matrix-bridge-appservice-discord/defaults/main.yml | 2 +- roles/matrix-bridge-appservice-irc/defaults/main.yml | 2 +- roles/matrix-bridge-appservice-slack/defaults/main.yml | 2 +- .../defaults/main.yml | 2 +- roles/matrix-bridge-sms/defaults/main.yml | 2 +- roles/matrix-corporal/defaults/main.yml | 2 +- roles/matrix-dimension/defaults/main.yml | 2 +- roles/matrix-email2matrix/defaults/main.yml | 2 +- roles/matrix-jitsi/defaults/main.yml | 8 ++++---- roles/matrix-nginx-proxy/defaults/main.yml | 4 ++-- roles/matrix-postgres/defaults/main.yml | 10 +++++----- 12 files changed, 20 insertions(+), 20 deletions(-) diff --git a/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml b/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml index 8f4a2c2f..33028147 100644 --- a/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml +++ b/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml @@ -3,7 +3,7 @@ matrix_bot_matrix_reminder_bot_enabled: true -matrix_bot_matrix_reminder_bot_docker_image: "anoa/matrix-reminder-bot:release-v0.2.0" +matrix_bot_matrix_reminder_bot_docker_image: "docker.io/anoa/matrix-reminder-bot:release-v0.2.0" matrix_bot_matrix_reminder_bot_docker_image_force_pull: "{{ matrix_bot_matrix_reminder_bot_docker_image.endswith(':latest') }}" matrix_bot_matrix_reminder_bot_base_path: "{{ matrix_base_data_path }}/matrix-reminder-bot" diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml index 97ad8e89..d03ebb92 100644 --- a/roles/matrix-bridge-appservice-discord/defaults/main.yml +++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml @@ -3,7 +3,7 @@ matrix_appservice_discord_enabled: true -matrix_appservice_discord_docker_image: "halfshot/matrix-appservice-discord:latest" +matrix_appservice_discord_docker_image: "docker.io/halfshot/matrix-appservice-discord:latest" matrix_appservice_discord_docker_image_force_pull: "{{ matrix_appservice_discord_docker_image.endswith(':latest') }}" matrix_appservice_discord_base_path: "{{ matrix_base_data_path }}/appservice-discord" diff --git a/roles/matrix-bridge-appservice-irc/defaults/main.yml b/roles/matrix-bridge-appservice-irc/defaults/main.yml index 2cbb3e5f..7ff2d46a 100644 --- a/roles/matrix-bridge-appservice-irc/defaults/main.yml +++ b/roles/matrix-bridge-appservice-irc/defaults/main.yml @@ -3,7 +3,7 @@ matrix_appservice_irc_enabled: true -matrix_appservice_irc_docker_image: "matrixdotorg/matrix-appservice-irc:release-0.17.1" +matrix_appservice_irc_docker_image: "docker.io/matrixdotorg/matrix-appservice-irc:release-0.17.1" matrix_appservice_irc_docker_image_force_pull: "{{ matrix_appservice_irc_docker_image.endswith(':latest') }}" matrix_appservice_irc_base_path: "{{ matrix_base_data_path }}/appservice-irc" diff --git a/roles/matrix-bridge-appservice-slack/defaults/main.yml b/roles/matrix-bridge-appservice-slack/defaults/main.yml index eef7d070..f2cfd8f9 100644 --- a/roles/matrix-bridge-appservice-slack/defaults/main.yml +++ b/roles/matrix-bridge-appservice-slack/defaults/main.yml @@ -3,7 +3,7 @@ matrix_appservice_slack_enabled: true -matrix_appservice_slack_docker_image: "matrixdotorg/matrix-appservice-slack:release-1.5.0" +matrix_appservice_slack_docker_image: "docker.io/matrixdotorg/matrix-appservice-slack:release-1.5.0" matrix_appservice_slack_docker_image_force_pull: "{{ matrix_appservice_slack_docker_image.endswith(':latest') }}" matrix_appservice_slack_base_path: "{{ matrix_base_data_path }}/appservice-slack" diff --git a/roles/matrix-bridge-appservice-webhooks/defaults/main.yml b/roles/matrix-bridge-appservice-webhooks/defaults/main.yml index 0fb5abbc..e668f918 100644 --- a/roles/matrix-bridge-appservice-webhooks/defaults/main.yml +++ b/roles/matrix-bridge-appservice-webhooks/defaults/main.yml @@ -3,7 +3,7 @@ matrix_appservice_webhooks_enabled: true -matrix_appservice_webhooks_docker_image: "turt2live/matrix-appservice-webhooks:latest" +matrix_appservice_webhooks_docker_image: "docker.io/turt2live/matrix-appservice-webhooks:latest" matrix_appservice_webhooks_docker_image_force_pull: "{{ matrix_appservice_webhooks_docker_image.endswith(':latest') }}" matrix_appservice_webhooks_base_path: "{{ matrix_base_data_path }}/appservice-webhooks" diff --git a/roles/matrix-bridge-sms/defaults/main.yml b/roles/matrix-bridge-sms/defaults/main.yml index 0801fe5b..9f0f10f6 100644 --- a/roles/matrix-bridge-sms/defaults/main.yml +++ b/roles/matrix-bridge-sms/defaults/main.yml @@ -3,7 +3,7 @@ matrix_sms_bridge_enabled: true -matrix_sms_bridge_docker_image: "folivonet/matrix-sms-bridge:0.4.1.RELEASE" +matrix_sms_bridge_docker_image: "docker.io/folivonet/matrix-sms-bridge:0.4.1.RELEASE" matrix_sms_bridge_base_path: "{{ matrix_base_data_path }}/matrix-sms-bridge" matrix_sms_bridge_config_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/config" diff --git a/roles/matrix-corporal/defaults/main.yml b/roles/matrix-corporal/defaults/main.yml index 52681a30..def3fcc9 100644 --- a/roles/matrix-corporal/defaults/main.yml +++ b/roles/matrix-corporal/defaults/main.yml @@ -19,7 +19,7 @@ matrix_corporal_container_extra_arguments: [] # List of systemd services that matrix-corporal.service depends on matrix_corporal_systemd_required_services_list: ['docker.service'] -matrix_corporal_docker_image: "devture/matrix-corporal:1.11.0" +matrix_corporal_docker_image: "docker.io/devture/matrix-corporal:1.11.0" matrix_corporal_docker_image_force_pull: "{{ matrix_corporal_docker_image.endswith(':latest') }}" matrix_corporal_base_path: "{{ matrix_base_data_path }}/corporal" diff --git a/roles/matrix-dimension/defaults/main.yml b/roles/matrix-dimension/defaults/main.yml index 33e7212c..3b69227a 100644 --- a/roles/matrix-dimension/defaults/main.yml +++ b/roles/matrix-dimension/defaults/main.yml @@ -12,7 +12,7 @@ matrix_dimension_widgets_allow_self_signed_ssl_certificates: false matrix_dimension_base_path: "{{ matrix_base_data_path }}/dimension" -matrix_dimension_docker_image: "turt2live/matrix-dimension:latest" +matrix_dimension_docker_image: "docker.io/turt2live/matrix-dimension:latest" matrix_dimension_docker_image_force_pull: "{{ matrix_dimension_docker_image.endswith(':latest') }}" # The user and group id correspond to the node user in the `turt2live/matrix-dimension` image. diff --git a/roles/matrix-email2matrix/defaults/main.yml b/roles/matrix-email2matrix/defaults/main.yml index 68ff1990..7917decf 100644 --- a/roles/matrix-email2matrix/defaults/main.yml +++ b/roles/matrix-email2matrix/defaults/main.yml @@ -3,7 +3,7 @@ matrix_email2matrix_enabled: true matrix_email2matrix_base_path: "{{ matrix_base_data_path }}/email2matrix" matrix_email2matrix_config_dir_path: "{{ matrix_email2matrix_base_path }}/config" -matrix_email2matrix_docker_image: "devture/email2matrix:1.0.1" +matrix_email2matrix_docker_image: "docker.io/devture/email2matrix:1.0.1" matrix_email2matrix_docker_image_force_pull: "{{ matrix_email2matrix_docker_image.endswith(':latest') }}" # A list of extra arguments to pass to the container diff --git a/roles/matrix-jitsi/defaults/main.yml b/roles/matrix-jitsi/defaults/main.yml index d05d2c71..69e1ae54 100644 --- a/roles/matrix-jitsi/defaults/main.yml +++ b/roles/matrix-jitsi/defaults/main.yml @@ -53,7 +53,7 @@ matrix_jitsi_enable_lobby: false matrix_jitsi_container_image_tag: "stable-5142" -matrix_jitsi_web_docker_image: "jitsi/web:{{ matrix_jitsi_container_image_tag }}" +matrix_jitsi_web_docker_image: "docker.io/jitsi/web:{{ matrix_jitsi_container_image_tag }}" matrix_jitsi_web_docker_image_force_pull: "{{ matrix_jitsi_web_docker_image.endswith(':latest') }}" matrix_jitsi_web_base_path: "{{ matrix_base_data_path }}/jitsi/web" @@ -114,7 +114,7 @@ matrix_jitsi_web_config_constraints_video_height_ideal: 720 matrix_jitsi_web_config_constraints_video_height_max: 720 matrix_jitsi_web_config_constraints_video_height_min: 240 -matrix_jitsi_prosody_docker_image: "jitsi/prosody:{{ matrix_jitsi_container_image_tag }}" +matrix_jitsi_prosody_docker_image: "docker.io/jitsi/prosody:{{ matrix_jitsi_container_image_tag }}" matrix_jitsi_prosody_docker_image_force_pull: "{{ matrix_jitsi_prosody_docker_image.endswith(':latest') }}" matrix_jitsi_prosody_base_path: "{{ matrix_base_data_path }}/jitsi/prosody" @@ -128,7 +128,7 @@ matrix_jitsi_prosody_container_extra_arguments: [] matrix_jitsi_prosody_systemd_required_services_list: ['docker.service'] -matrix_jitsi_jicofo_docker_image: "jitsi/jicofo:{{ matrix_jitsi_container_image_tag }}" +matrix_jitsi_jicofo_docker_image: "docker.io/jitsi/jicofo:{{ matrix_jitsi_container_image_tag }}" matrix_jitsi_jicofo_docker_image_force_pull: "{{ matrix_jitsi_jicofo_docker_image.endswith(':latest') }}" matrix_jitsi_jicofo_base_path: "{{ matrix_base_data_path }}/jitsi/jicofo" @@ -145,7 +145,7 @@ matrix_jitsi_jicofo_auth_user: focus matrix_jitsi_jicofo_auth_password: '' -matrix_jitsi_jvb_docker_image: "jitsi/jvb:{{ matrix_jitsi_container_image_tag }}" +matrix_jitsi_jvb_docker_image: "docker.io/jitsi/jvb:{{ matrix_jitsi_container_image_tag }}" matrix_jitsi_jvb_docker_image_force_pull: "{{ matrix_jitsi_jvb_docker_image.endswith(':latest') }}" matrix_jitsi_jvb_base_path: "{{ matrix_base_data_path }}/jitsi/jvb" diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 67b32d89..ceba9ab6 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -3,7 +3,7 @@ matrix_nginx_proxy_enabled: true # We use an official nginx image, which we fix-up to run unprivileged. # An alternative would be an `nginxinc/nginx-unprivileged` image, but # that is frequently out of date. -matrix_nginx_proxy_docker_image: "nginx:1.19.4-alpine" +matrix_nginx_proxy_docker_image: "docker.io/nginx:1.19.4-alpine" matrix_nginx_proxy_docker_image_force_pull: "{{ matrix_nginx_proxy_docker_image.endswith(':latest') }}" matrix_nginx_proxy_base_path: "{{ matrix_base_data_path }}/nginx-proxy" @@ -259,7 +259,7 @@ matrix_ssl_domains_to_obtain_certificates_for: [] # Controls whether to obtain production or staging certificates from Let's Encrypt. matrix_ssl_lets_encrypt_staging: false -matrix_ssl_lets_encrypt_certbot_docker_image: "certbot/certbot:{{ matrix_ssl_architecture }}-v1.9.0" +matrix_ssl_lets_encrypt_certbot_docker_image: "docker.io/certbot/certbot:{{ matrix_ssl_architecture }}-v1.9.0" matrix_ssl_lets_encrypt_certbot_docker_image_force_pull: "{{ matrix_ssl_lets_encrypt_certbot_docker_image.endswith(':latest') }}" matrix_ssl_lets_encrypt_certbot_standalone_http_port: 2402 matrix_ssl_lets_encrypt_support_email: ~ diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index f4fc180e..8717fdb6 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -8,11 +8,11 @@ matrix_postgres_db_name: "" matrix_postgres_base_path: "{{ matrix_base_data_path }}/postgres" matrix_postgres_data_path: "{{ matrix_postgres_base_path }}/data" -matrix_postgres_docker_image_v9: "postgres:9.6.19-alpine" -matrix_postgres_docker_image_v10: "postgres:10.14-alpine" -matrix_postgres_docker_image_v11: "postgres:11.9-alpine" -matrix_postgres_docker_image_v12: "postgres:12.4-alpine" -matrix_postgres_docker_image_v13: "postgres:13.0-alpine" +matrix_postgres_docker_image_v9: "docker.io/postgres:9.6.19-alpine" +matrix_postgres_docker_image_v10: "docker.io/postgres:10.14-alpine" +matrix_postgres_docker_image_v11: "docker.io/postgres:11.9-alpine" +matrix_postgres_docker_image_v12: "docker.io/postgres:12.4-alpine" +matrix_postgres_docker_image_v13: "docker.io/postgres:13.0-alpine" matrix_postgres_docker_image_latest: "{{ matrix_postgres_docker_image_v13 }}" # This variable is assigned at runtime. Overriding its value has no effect. From 775b1ca7afdeddc91a04f37e2999696bc9cd2289 Mon Sep 17 00:00:00 2001 From: benkuly <12199167+benkuly@users.noreply.github.com> Date: Sun, 15 Nov 2020 12:12:44 +0100 Subject: [PATCH 049/217] updated matrix-sms-bridge image version --- roles/matrix-bridge-sms/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-sms/defaults/main.yml b/roles/matrix-bridge-sms/defaults/main.yml index 9f0f10f6..ca431abf 100644 --- a/roles/matrix-bridge-sms/defaults/main.yml +++ b/roles/matrix-bridge-sms/defaults/main.yml @@ -3,7 +3,7 @@ matrix_sms_bridge_enabled: true -matrix_sms_bridge_docker_image: "docker.io/folivonet/matrix-sms-bridge:0.4.1.RELEASE" +matrix_sms_bridge_docker_image: "docker.io/folivonet/matrix-sms-bridge:0.4.2.RELEASE" matrix_sms_bridge_base_path: "{{ matrix_base_data_path }}/matrix-sms-bridge" matrix_sms_bridge_config_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/config" From 787a9ef8ad2313b6623e8baf3a0dc2a6819eb448 Mon Sep 17 00:00:00 2001 From: benkuly <12199167+benkuly@users.noreply.github.com> Date: Mon, 16 Nov 2020 11:51:11 +0100 Subject: [PATCH 050/217] updated matrix-sms-bridge image --- roles/matrix-bridge-sms/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-sms/defaults/main.yml b/roles/matrix-bridge-sms/defaults/main.yml index ca431abf..882f44d4 100644 --- a/roles/matrix-bridge-sms/defaults/main.yml +++ b/roles/matrix-bridge-sms/defaults/main.yml @@ -3,7 +3,7 @@ matrix_sms_bridge_enabled: true -matrix_sms_bridge_docker_image: "docker.io/folivonet/matrix-sms-bridge:0.4.2.RELEASE" +matrix_sms_bridge_docker_image: "docker.io/folivonet/matrix-sms-bridge:0.4.3.RELEASE" matrix_sms_bridge_base_path: "{{ matrix_base_data_path }}/matrix-sms-bridge" matrix_sms_bridge_config_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/config" From 8153e25d2de5e3f33b3a49d64449fab81e108679 Mon Sep 17 00:00:00 2001 From: benkuly <12199167+benkuly@users.noreply.github.com> Date: Mon, 16 Nov 2020 13:59:03 +0100 Subject: [PATCH 051/217] updated matrix-sms-bridge image --- roles/matrix-bridge-sms/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-sms/defaults/main.yml b/roles/matrix-bridge-sms/defaults/main.yml index 882f44d4..7ad449bb 100644 --- a/roles/matrix-bridge-sms/defaults/main.yml +++ b/roles/matrix-bridge-sms/defaults/main.yml @@ -3,7 +3,7 @@ matrix_sms_bridge_enabled: true -matrix_sms_bridge_docker_image: "docker.io/folivonet/matrix-sms-bridge:0.4.3.RELEASE" +matrix_sms_bridge_docker_image: "docker.io/folivonet/matrix-sms-bridge:0.4.4.RELEASE" matrix_sms_bridge_base_path: "{{ matrix_base_data_path }}/matrix-sms-bridge" matrix_sms_bridge_config_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/config" From f7d7190bd0909f29005fab87ae636c284d14cdda Mon Sep 17 00:00:00 2001 From: transcaffeine Date: Tue, 17 Nov 2020 21:20:12 +0100 Subject: [PATCH 052/217] update mautrix-telegram to 0.9.0 --- roles/matrix-bridge-mautrix-telegram/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml index d2ab9909..b6f4ef5b 100644 --- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml @@ -4,7 +4,7 @@ matrix_mautrix_telegram_enabled: true # See: https://mau.dev/tulir/mautrix-telegram/container_registry -matrix_mautrix_telegram_docker_image: "dock.mau.dev/tulir/mautrix-telegram:v0.8.2" +matrix_mautrix_telegram_docker_image: "dock.mau.dev/tulir/mautrix-telegram:v0.9.0" matrix_mautrix_telegram_docker_image_force_pull: "{{ matrix_mautrix_telegram_docker_image.endswith(':latest') }}" matrix_mautrix_telegram_base_path: "{{ matrix_base_data_path }}/mautrix-telegram" From 41fa00edb412f31b264998df6b6750e1e479c15b Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 18 Nov 2020 12:53:58 +0200 Subject: [PATCH 053/217] Revert "Update jitsi web to stable-5142" This reverts commit 078592454ca0272563525dddcadedb578c4d5a4e due to reports of breakage both in the support chat room and in here https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/719 --- roles/matrix-jitsi/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-jitsi/defaults/main.yml b/roles/matrix-jitsi/defaults/main.yml index 69e1ae54..865424ed 100644 --- a/roles/matrix-jitsi/defaults/main.yml +++ b/roles/matrix-jitsi/defaults/main.yml @@ -51,7 +51,7 @@ matrix_jitsi_jibri_recorder_password: '' matrix_jitsi_enable_lobby: false -matrix_jitsi_container_image_tag: "stable-5142" +matrix_jitsi_container_image_tag: "stable-4857" matrix_jitsi_web_docker_image: "docker.io/jitsi/web:{{ matrix_jitsi_container_image_tag }}" matrix_jitsi_web_docker_image_force_pull: "{{ matrix_jitsi_web_docker_image.endswith(':latest') }}" From c58a7e03c72510bcdd30662a440775bc26226e1c Mon Sep 17 00:00:00 2001 From: transcaffeine Date: Wed, 18 Nov 2020 14:16:46 +0100 Subject: [PATCH 054/217] synapse: update to 1.23.0 --- roles/matrix-synapse/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 4cc819a6..aadb47ee 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -5,7 +5,7 @@ matrix_synapse_enabled: true matrix_synapse_container_image_self_build: false -matrix_synapse_docker_image: "{{ matrix_synapse_docker_image_name_prefix }}matrixdotorg/synapse:v1.22.1" +matrix_synapse_docker_image: "{{ matrix_synapse_docker_image_name_prefix }}matrixdotorg/synapse:v1.23.0" matrix_synapse_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_container_image_self_build else 'docker.io/' }}" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" @@ -59,7 +59,7 @@ matrix_synapse_systemd_required_services_list: ['docker.service'] # List of systemd services that matrix-synapse.service wants matrix_synapse_systemd_wanted_services_list: [] -matrix_synapse_in_container_python_packages_path: "/usr/local/lib/python3.7/site-packages" +matrix_synapse_in_container_python_packages_path: "/usr/local/lib/python3.8/site-packages" # Specifies which template files to use when configuring Synapse. # If you'd like to have your own different configuration, feel free to copy and paste From b627d93cdc78fc37191a0fcf5d0eeabcd37c5a35 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 18 Nov 2020 16:57:50 +0200 Subject: [PATCH 055/217] Update homeserver.yaml to keep up with Synapse v1.23.0 Related to #724 (Github Pull Request) --- .../templates/synapse/homeserver.yaml.j2 | 187 ++++++++++-------- 1 file changed, 109 insertions(+), 78 deletions(-) diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index f58f34f8..8c68189b 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -1482,10 +1482,8 @@ trusted_key_servers: {{ matrix_synapse_trusted_key_servers|to_json }} ## Single sign-on integration ## -# Enable SAML2 for registration and login. Uses pysaml2. -# -# At least one of `sp_config` or `config_path` must be set in this section to -# enable SAML login. +# The following settings can be used to make Synapse use a single sign-on +# provider for authentication, instead of its internal password database. # # You will probably also want to set the following options to `false` to # disable the regular login/registration flows: @@ -1494,6 +1492,11 @@ trusted_key_servers: {{ matrix_synapse_trusted_key_servers|to_json }} # # You will also want to investigate the settings under the "sso" configuration # section below. + +# Enable SAML2 for registration and login. Uses pysaml2. +# +# At least one of `sp_config` or `config_path` must be set in this section to +# enable SAML login. # # Once SAML support is enabled, a metadata file will be exposed at # https://:/_matrix/saml2/metadata.xml, which you may be able to @@ -1509,40 +1512,64 @@ saml2_config: # so it is not normally necessary to specify them unless you need to # override them. # - #sp_config: - # # point this to the IdP's metadata. You can use either a local file or - # # (preferably) a URL. - # metadata: - # #local: ["saml2/idp.xml"] - # remote: - # - url: https://our_idp/metadata.xml - # - # # By default, the user has to go to our login page first. If you'd like - # # to allow IdP-initiated login, set 'allow_unsolicited: true' in a - # # 'service.sp' section: - # # - # #service: - # # sp: - # # allow_unsolicited: true - # - # # The examples below are just used to generate our metadata xml, and you - # # may well not need them, depending on your setup. Alternatively you - # # may need a whole lot more detail - see the pysaml2 docs! - # - # description: ["My awesome SP", "en"] - # name: ["Test SP", "en"] - # - # organization: - # name: Example com - # display_name: - # - ["Example co", "en"] - # url: "http://example.com" - # - # contact_person: - # - given_name: Bob - # sur_name: "the Sysadmin" - # email_address": ["admin@example.com"] - # contact_type": technical + sp_config: + # Point this to the IdP's metadata. You must provide either a local + # file via the `local` attribute or (preferably) a URL via the + # `remote` attribute. + # + #metadata: + # local: ["saml2/idp.xml"] + # remote: + # - url: https://our_idp/metadata.xml + + # By default, the user has to go to our login page first. If you'd like + # to allow IdP-initiated login, set 'allow_unsolicited: true' in a + # 'service.sp' section: + # + #service: + # sp: + # allow_unsolicited: true + + # The examples below are just used to generate our metadata xml, and you + # may well not need them, depending on your setup. Alternatively you + # may need a whole lot more detail - see the pysaml2 docs! + + #description: ["My awesome SP", "en"] + #name: ["Test SP", "en"] + + #ui_info: + # display_name: + # - lang: en + # text: "Display Name is the descriptive name of your service." + # description: + # - lang: en + # text: "Description should be a short paragraph explaining the purpose of the service." + # information_url: + # - lang: en + # text: "https://example.com/terms-of-service" + # privacy_statement_url: + # - lang: en + # text: "https://example.com/privacy-policy" + # keywords: + # - lang: en + # text: ["Matrix", "Element"] + # logo: + # - lang: en + # text: "https://example.com/logo.svg" + # width: "200" + # height: "80" + + #organization: + # name: Example com + # display_name: + # - ["Example co", "en"] + # url: "http://example.com" + + #contact_person: + # - given_name: Bob + # sur_name: "the Sysadmin" + # email_address": ["admin@example.com"] + # contact_type": technical # Instead of putting the config inline as above, you can specify a # separate pysaml2 configuration file: @@ -1617,37 +1644,11 @@ saml2_config: # - attribute: department # value: "sales" - # Directory in which Synapse will try to find the template files below. - # If not set, default templates from within the Synapse package will be used. - # - # DO NOT UNCOMMENT THIS SETTING unless you want to customise the templates. - # If you *do* uncomment it, you will need to make sure that all the templates - # below are in the directory. - # - # Synapse will look for the following templates in this directory: - # - # * HTML page to display to users if something goes wrong during the - # authentication process: 'saml_error.html'. - # - # When rendering, this template is given the following variables: - # * code: an HTML error code corresponding to the error that is being - # returned (typically 400 or 500) - # - # * msg: a textual message describing the error. - # - # The variables will automatically be HTML-escaped. - # - # You can see the default templates at: - # https://github.com/matrix-org/synapse/tree/master/synapse/res/templates - # - #template_dir: "res/templates" - -# OpenID Connect integration. The following settings can be used to make Synapse -# use an OpenID Connect Provider for authentication, instead of its internal -# password database. +# Enable OpenID Connect (OIDC) / OAuth 2.0 for registration and login. # -# See https://github.com/matrix-org/synapse/blob/master/docs/openid.md. +# See https://github.com/matrix-org/synapse/blob/master/docs/openid.md +# for some example configurations. # oidc_config: # Uncomment the following to enable authorization against an OpenID Connect @@ -1770,17 +1771,47 @@ oidc_config: # #display_name_template: "{% raw %}{{ user.given_name }} {{ user.last_name }}{% endraw %}" + # Jinja2 templates for extra attributes to send back to the client during + # login. + # + # Note that these are non-standard and clients will ignore them without modifications. + # + #extra_attributes: + #birthdate: "{% raw %}{{ user.birthdate }}{% endraw %}" -# Enable CAS for registration and login. + +# Enable Central Authentication Service (CAS) for registration and login. # -#cas_config: -# enabled: true -# server_url: "https://cas-server.com" -# service_url: "https://homeserver.domain.com:8448" -# #displayname_attribute: name -# #required_attributes: -# # name: value +cas_config: + # Uncomment the following to enable authorization against a CAS server. + # Defaults to false. + # + #enabled: true + + # The URL of the CAS authorization endpoint. + # + #server_url: "https://cas-server.com" + + # The public URL of the homeserver. + # + #service_url: "https://homeserver.domain.com:8448" + + # The attribute of the CAS response to use as the display name. + # + # If unset, no displayname will be set. + # + #displayname_attribute: name + + # It is possible to configure Synapse to only allow logins if CAS attributes + # match particular values. All of the keys in the mapping below must exist + # and the values must match the given value. Alternately if the given value + # is None then any value is allowed (the attribute just must exist). + # All of the listed attributes must match for the login to be permitted. + # + #required_attributes: + # userGroup: "staff" + # department: None # Additional settings to use with single-sign on systems such as OpenID Connect, @@ -1880,7 +1911,7 @@ sso: # and issued at ("iat") claims are validated if present. # # Note that this is a non-standard login type and client support is -# expected to be non-existant. +# expected to be non-existent. # # See https://github.com/matrix-org/synapse/blob/master/docs/jwt.md. # @@ -2411,7 +2442,7 @@ alias_creation_rules: {{ matrix_synapse_alias_creation_rules|to_json }} # # Options for the rules include: # -# user_id: Matches agaisnt the creator of the alias +# user_id: Matches against the creator of the alias # room_id: Matches against the room ID being published # alias: Matches against any current local or canonical aliases # associated with the room @@ -2459,7 +2490,7 @@ opentracing: # This is a list of regexes which are matched against the server_name of the # homeserver. # - # By defult, it is empty, so no servers are matched. + # By default, it is empty, so no servers are matched. # #homeserver_whitelist: # - ".*" From 6c85b84c1e146bb100d18e01d0d123d1b299490d Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 18 Nov 2020 18:36:03 +0200 Subject: [PATCH 056/217] Fix self-building for synapse-admin --- roles/matrix-synapse-admin/tasks/setup.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse-admin/tasks/setup.yml b/roles/matrix-synapse-admin/tasks/setup.yml index 6d2b8c20..d30657d3 100644 --- a/roles/matrix-synapse-admin/tasks/setup.yml +++ b/roles/matrix-synapse-admin/tasks/setup.yml @@ -24,7 +24,7 @@ docker_image: name: "{{ matrix_synapse_admin_docker_image }}" source: build - force_source: "{{ matrix_synapse_admin_git_pull_results }}" + force_source: "{{ matrix_synapse_admin_git_pull_results.changed }}" build: dockerfile: Dockerfile path: "{{ matrix_synapse_admin_docker_src_files_path }}" From 4713e5d5f7a061ff8441810e52c2effd92524ab4 Mon Sep 17 00:00:00 2001 From: benkuly <12199167+benkuly@users.noreply.github.com> Date: Mon, 23 Nov 2020 12:30:39 +0100 Subject: [PATCH 057/217] updated matrix-sms-bridge to 0.5.0 --- ...uring-playbook-bridge-matrix-bridge-sms.md | 30 +++++---- roles/matrix-bridge-sms/defaults/main.yml | 66 ++++++------------- .../matrix-bridge-sms/tasks/setup_install.yml | 14 ++-- .../tasks/validate_config.yml | 6 +- .../matrix-sms-bridge-database.service.j2 | 36 ---------- .../systemd/matrix-sms-bridge.service.j2 | 4 -- 6 files changed, 46 insertions(+), 110 deletions(-) delete mode 100644 roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge-database.service.j2 diff --git a/docs/configuring-playbook-bridge-matrix-bridge-sms.md b/docs/configuring-playbook-bridge-matrix-bridge-sms.md index 99b4f6f0..59b21d61 100644 --- a/docs/configuring-playbook-bridge-matrix-bridge-sms.md +++ b/docs/configuring-playbook-bridge-matrix-bridge-sms.md @@ -1,11 +1,10 @@ # Setting up matrix-sms-bridge (optional) -The playbook can install and configure -[matrix-sms-bridge](https://github.com/benkuly/matrix-sms-bridge) for you. +The playbook can install and configure [matrix-sms-bridge](https://github.com/benkuly/matrix-sms-bridge) for you. See the project page to learn what it does and why it might be useful to you. -First you need to ensure, that the bridge has unix read and write rights to your modem. On debian based distributions there is nothing to do. On others distributions you either add a group `dialout` to your host and assign it to your modem or you give the matrix user or group access to your modem. +** The bridge uses [android-sms-gateway-server](https://github.com/RebekkaMa/android-sms-gateway-server). You need to configure it first. ** To enable the bridge just use the following playbook configuration: @@ -13,16 +12,23 @@ playbook configuration: ```yaml matrix_sms_bridge_enabled: true -matrix_sms_bridge_gammu_modem: "/dev/serial/by-id/myDeviceId" -# generate a secret passwort e.g. with pwgen -s 64 1 -matrix_sms_bridge_database_password: "" -# (optional) a room id to a default room + +# (optional but recommended) a room id to a default room matrix_sms_bridge_default_room: "" -# (optional) gammu reset frequencies (see https://wammu.eu/docs/manual/smsd/config.html#option-ResetFrequency) -matrix_sms_bridge_gammu_reset_frequency: 3600 -matrix_sms_bridge_gammu_hard_reset_frequency: 0 -# (optional) group with unix read and write rights to modem -matrix_sms_bridge_modem_group: 'dialout' + +# (optional but recommended) configure your server location +matrix_sms_bridge_default_region: DE +matrix_sms_bridge_default_timezone: Europe/Berlin + +# Settings to connect to android-sms-gateway-server +matrix_sms_bridge_provider_android_baseurl: https://192.168.24.24:9090 +matrix_sms_bridge_provider_android_username: admin +matrix_sms_bridge_provider_android_password: supeSecretPassword + +# (optional) ff your android-sms-gateway-server uses a self signed vertificate, the bridge needs a "truststore". This can be the certificate itself. +matrix_sms_bridge_provider_android_truststore_local_path: android-sms-gateway-server.p12 +matrix_sms_bridge_provider_android_truststore_password: 123 + ``` diff --git a/roles/matrix-bridge-sms/defaults/main.yml b/roles/matrix-bridge-sms/defaults/main.yml index 7ad449bb..74ec2ec7 100644 --- a/roles/matrix-bridge-sms/defaults/main.yml +++ b/roles/matrix-bridge-sms/defaults/main.yml @@ -3,18 +3,11 @@ matrix_sms_bridge_enabled: true -matrix_sms_bridge_docker_image: "docker.io/folivonet/matrix-sms-bridge:0.4.4.RELEASE" +matrix_sms_bridge_docker_image: "docker.io/folivonet/matrix-sms-bridge:0.5.0" matrix_sms_bridge_base_path: "{{ matrix_base_data_path }}/matrix-sms-bridge" matrix_sms_bridge_config_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/config" matrix_sms_bridge_data_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/data" -matrix_sms_bridge_data_log_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/data/log" -matrix_sms_bridge_data_spool_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/data/spool" -matrix_sms_bridge_data_spool_inbox_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/data/spool/inbox" -matrix_sms_bridge_data_spool_inbox_processed_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/data/spool/inbox_processed" -matrix_sms_bridge_data_spool_outbox_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/data/spool/outbox" -matrix_sms_bridge_data_spool_sent_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/data/spool/sent" -matrix_sms_bridge_data_spool_error_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/data/spool/error" matrix_sms_bridge_appservice_token: '' matrix_sms_bridge_homeserver_token: '' @@ -40,11 +33,11 @@ matrix_sms_bridge_default_region: '' matrix_sms_bridge_default_timezone: '' matrix_sms_bridge_single_mode_enabled: false -matrix_sms_bridge_gammu_modem: '' -matrix_sms_bridge_modem_group: 'dialout' -matrix_sms_bridge_gammu_reset_frequency: 0 -matrix_sms_bridge_gammu_hard_reset_frequency: 0 -matrix_sms_bridge_gammu_smsc: '' +matrix_sms_bridge_provider_android_baseurl: '' +matrix_sms_bridge_provider_android_username: '' +matrix_sms_bridge_provider_android_password: '' +matrix_sms_bridge_provider_android_truststore_local_path: '' +matrix_sms_bridge_provider_android_truststore_password: '' matrix_sms_bridge_configuration_yaml: | @@ -61,13 +54,22 @@ matrix_sms_bridge_configuration_yaml: | defaultTimeZone: "{{ matrix_sms_bridge_default_timezone }}" singleModeEnabled: "{{ matrix_sms_bridge_single_mode_enabled }}" provider: - gammu: + android: # (optional) default is disabled enabled: true - # (optional) Path to the Gammu-Inbox directory. - inboxPath: /data/spool/inbox - # (optional) Path to the directory, where to put processed messages. - inboxProcessedPath: /data/spool/inbox_processed + # The url to the android-sms-gateway-server + baseUrl: {{ matrix_sms_bridge_provider_android_baseurl }} + # The username of the gateway + username: {{ matrix_sms_bridge_provider_android_username }} + # The password of the gateway + password: {{ matrix_sms_bridge_provider_android_password }} + # (optional) if you use a self signed certificate, you can add the public key here + {% if matrix_sms_bridge_provider_android_truststore_path %} + trustStore: + path: /data/config/matrix-sms-gateway-server.p12 + password: {{ matrix_sms_bridge_provider_android_truststore_password }} + type: PKCS12 + {% endif %} bot: # The domain-part of matrix-ids. E. g. example.org when your userIds look like @unicorn:example.org serverName: {{ matrix_sms_bridge_homserver_domain }} @@ -104,34 +106,6 @@ matrix_sms_bridge_configuration_extension: "{{ matrix_sms_bridge_configuration_e matrix_sms_bridge_configuration: "{{ matrix_sms_bridge_configuration_yaml|from_yaml|combine(matrix_sms_bridge_configuration_extension, recursive=True) }}" -matrix_sms_bridge_gammu_configuration: | - [gammu] - Device = {{ matrix_sms_bridge_gammu_modem }} - LogFile = /data/log/gammu.log - debugLevel = 1 - - [smsd] - Service = files - LoopSleep = 2 - InboxPath = /data/spool/inbox/ - OutboxPath = /data/spool/outbox/ - SentSMSPath = /data/spool/sent/ - ErrorSMSPath = /data/spool/error/ - InboxFormat = detail - OutboxFormat = detail - TransmitFormat = auto - {% if matrix_sms_bridge_gammu_smsc is defined and matrix_sms_bridge_gammu_smsc|length %} - SMSC = {{ matrix_sms_bridge_gammu_smsc }} - {% endif %} - ResetFrequency = {{ matrix_sms_bridge_gammu_reset_frequency }} - HardResetFrequency = {{ matrix_sms_bridge_gammu_hard_reset_frequency }} - debugLevel = 1 - LogFile = /data/log/smsd.log - DeliveryReport = no - HangupCalls = 1 - CheckBattery = 0 - - matrix_sms_bridge_registration_yaml: | id: sms as_token: "{{ matrix_sms_bridge_appservice_token }}" diff --git a/roles/matrix-bridge-sms/tasks/setup_install.yml b/roles/matrix-bridge-sms/tasks/setup_install.yml index 1d36ea8d..1e5f6137 100644 --- a/roles/matrix-bridge-sms/tasks/setup_install.yml +++ b/roles/matrix-bridge-sms/tasks/setup_install.yml @@ -16,13 +16,6 @@ - "{{ matrix_sms_bridge_base_path }}" - "{{ matrix_sms_bridge_config_path }}" - "{{ matrix_sms_bridge_data_path }}" - - "{{ matrix_sms_bridge_data_log_path }}" - - "{{ matrix_sms_bridge_data_spool_path }}" - - "{{ matrix_sms_bridge_data_spool_inbox_path }}" - - "{{ matrix_sms_bridge_data_spool_inbox_processed_path }}" - - "{{ matrix_sms_bridge_data_spool_outbox_path }}" - - "{{ matrix_sms_bridge_data_spool_sent_path }}" - - "{{ matrix_sms_bridge_data_spool_error_path }}" - name: Ensure matrix-sms-bridge application.yml installed copy: @@ -40,13 +33,14 @@ owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" -- name: Ensure matrix-sms-bridge gammu-smsdrc installed +- name: Ensure android-sms-gateway-server cert installed copy: - content: "{{ matrix_sms_bridge_gammu_configuration }}" - dest: "{{ matrix_sms_bridge_config_path }}/gammu-smsdrc" + src: "{{ matrix_sms_bridge_provider_android_truststore_path }}" + dest: "{{ matrix_sms_bridge_config_path }}/matrix-sms-gateway-server.p12" mode: 0644 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" + when: matrix_sms_bridge_provider_android_truststore_path != "" - name: Ensure matrix-sms-bridge.service installed template: diff --git a/roles/matrix-bridge-sms/tasks/validate_config.yml b/roles/matrix-bridge-sms/tasks/validate_config.yml index 79f62fe1..6dc6ce9c 100644 --- a/roles/matrix-bridge-sms/tasks/validate_config.yml +++ b/roles/matrix-bridge-sms/tasks/validate_config.yml @@ -8,6 +8,8 @@ with_items: - "matrix_sms_bridge_appservice_token" - "matrix_sms_bridge_homeserver_token" - - "matrix_sms_bridge_gammu_modem" - "matrix_sms_bridge_default_region" - - "matrix_sms_bridge_default_timezone" \ No newline at end of file + - "matrix_sms_bridge_default_timezone" + - "matrix_sms_bridge_provider_android_baseurl" + - "matrix_sms_bridge_provider_android_username" + - "matrix_sms_bridge_provider_android_password" \ No newline at end of file diff --git a/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge-database.service.j2 b/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge-database.service.j2 deleted file mode 100644 index c9bb0343..00000000 --- a/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge-database.service.j2 +++ /dev/null @@ -1,36 +0,0 @@ -#jinja2: lstrip_blocks: "True" -[Unit] -Description=Matrix sms bridge database -{% for service in matrix_sms_bridge_database_systemd_required_services_list %} -Requires={{ service }} -After={{ service }} -{% endfor %} -{% for service in matrix_sms_bridge_database_systemd_wanted_services_list %} -Wants={{ service }} -{% endfor %} - -[Service] -Type=simple -ExecStartPre=-/usr/bin/docker kill matrix-sms-bridge-database -ExecStartPre=-/usr/bin/docker rm matrix-sms-bridge-database - -# Intentional delay, so that the homeserver (we likely depend on) can manage to start. -ExecStartPre=/bin/sleep 5 - -ExecStart=/usr/bin/docker run --rm --name matrix-sms-bridge-database \ - --log-driver=none \ - --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ - --cap-drop=ALL \ - --network={{ matrix_docker_network }} \ - -v {{ matrix_sms_bridge_database_path }}:/data:z \ - -e NEO4J_AUTH={{ matrix_sms_bridge_database_username }}/{{ matrix_sms_bridge_database_password }} \ - {{ matrix_sms_bridge_database_docker_image }} - -ExecStop=-/usr/bin/docker kill matrix-sms-bridge-database -ExecStop=-/usr/bin/docker rm matrix-sms-bridge-database -Restart=always -RestartSec=30 -SyslogIdentifier=matrix-sms-bridge - - [Install] -WantedBy=multi-user.target diff --git a/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge.service.j2 b/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge.service.j2 index 7cab3742..15464cc4 100644 --- a/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge.service.j2 +++ b/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge.service.j2 @@ -20,7 +20,6 @@ ExecStartPre=/bin/sleep 5 ExecStart=/usr/bin/docker run --rm --name matrix-sms-bridge \ --log-driver=none \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ - --group-add {{ matrix_sms_bridge_modem_group }} \ --cap-drop=ALL \ --network={{ matrix_docker_network }} \ {% if matrix_sms_bridge_container_http_host_bind_port %} @@ -28,9 +27,6 @@ ExecStart=/usr/bin/docker run --rm --name matrix-sms-bridge \ {% endif %} -v {{ matrix_sms_bridge_config_path }}:/config:z \ -v {{ matrix_sms_bridge_data_path }}:/data:z \ - -v {{ matrix_sms_bridge_config_path }}/gammu-smsdrc:/etc/gammu-smsdrc:z \ - --privileged \ - -v /dev:/dev:slave \ {% for arg in matrix_sms_bridge_container_extra_arguments %} {{ arg }} \ {% endfor %} From 2fb42dd7f1c6956087da5071839b596ce14fc91d Mon Sep 17 00:00:00 2001 From: benkuly <12199167+benkuly@users.noreply.github.com> Date: Mon, 23 Nov 2020 12:38:17 +0100 Subject: [PATCH 058/217] fixed typo in truststore path --- docs/configuring-playbook-bridge-matrix-bridge-sms.md | 2 +- roles/matrix-bridge-sms/defaults/main.yml | 2 +- roles/matrix-bridge-sms/tasks/setup_install.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/configuring-playbook-bridge-matrix-bridge-sms.md b/docs/configuring-playbook-bridge-matrix-bridge-sms.md index 59b21d61..3dd7398b 100644 --- a/docs/configuring-playbook-bridge-matrix-bridge-sms.md +++ b/docs/configuring-playbook-bridge-matrix-bridge-sms.md @@ -25,7 +25,7 @@ matrix_sms_bridge_provider_android_baseurl: https://192.168.24.24:9090 matrix_sms_bridge_provider_android_username: admin matrix_sms_bridge_provider_android_password: supeSecretPassword -# (optional) ff your android-sms-gateway-server uses a self signed vertificate, the bridge needs a "truststore". This can be the certificate itself. +# (optional) if your android-sms-gateway-server uses a self signed vertificate, the bridge needs a "truststore". This can be the certificate itself. matrix_sms_bridge_provider_android_truststore_local_path: android-sms-gateway-server.p12 matrix_sms_bridge_provider_android_truststore_password: 123 diff --git a/roles/matrix-bridge-sms/defaults/main.yml b/roles/matrix-bridge-sms/defaults/main.yml index 74ec2ec7..b551b43a 100644 --- a/roles/matrix-bridge-sms/defaults/main.yml +++ b/roles/matrix-bridge-sms/defaults/main.yml @@ -64,7 +64,7 @@ matrix_sms_bridge_configuration_yaml: | # The password of the gateway password: {{ matrix_sms_bridge_provider_android_password }} # (optional) if you use a self signed certificate, you can add the public key here - {% if matrix_sms_bridge_provider_android_truststore_path %} + {% if matrix_sms_bridge_provider_android_truststore_local_path %} trustStore: path: /data/config/matrix-sms-gateway-server.p12 password: {{ matrix_sms_bridge_provider_android_truststore_password }} diff --git a/roles/matrix-bridge-sms/tasks/setup_install.yml b/roles/matrix-bridge-sms/tasks/setup_install.yml index 1e5f6137..61de923f 100644 --- a/roles/matrix-bridge-sms/tasks/setup_install.yml +++ b/roles/matrix-bridge-sms/tasks/setup_install.yml @@ -35,12 +35,12 @@ - name: Ensure android-sms-gateway-server cert installed copy: - src: "{{ matrix_sms_bridge_provider_android_truststore_path }}" + src: "{{ matrix_sms_bridge_provider_android_truststore_local_path }}" dest: "{{ matrix_sms_bridge_config_path }}/matrix-sms-gateway-server.p12" mode: 0644 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" - when: matrix_sms_bridge_provider_android_truststore_path != "" + when: matrix_sms_bridge_provider_android_truststore_local_path != "" - name: Ensure matrix-sms-bridge.service installed template: From 76b0b9dc34d4cded3e30c29397d8a59b4d727875 Mon Sep 17 00:00:00 2001 From: benkuly <12199167+benkuly@users.noreply.github.com> Date: Mon, 23 Nov 2020 12:48:08 +0100 Subject: [PATCH 059/217] fixed application.yml loading --- .../templates/systemd/matrix-sms-bridge.service.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge.service.j2 b/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge.service.j2 index 15464cc4..2c7e0105 100644 --- a/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge.service.j2 +++ b/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge.service.j2 @@ -27,6 +27,7 @@ ExecStart=/usr/bin/docker run --rm --name matrix-sms-bridge \ {% endif %} -v {{ matrix_sms_bridge_config_path }}:/config:z \ -v {{ matrix_sms_bridge_data_path }}:/data:z \ + --env SPRING_CONFIG_ADDITIONAL_LOCATION=/config/application.yml {% for arg in matrix_sms_bridge_container_extra_arguments %} {{ arg }} \ {% endfor %} From ad1f0a01ce5ed11482046168ee7e6f1852bdc97a Mon Sep 17 00:00:00 2001 From: benkuly <12199167+benkuly@users.noreply.github.com> Date: Mon, 23 Nov 2020 12:49:28 +0100 Subject: [PATCH 060/217] fixed systemd service typo --- .../templates/systemd/matrix-sms-bridge.service.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge.service.j2 b/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge.service.j2 index 2c7e0105..7c96f9f4 100644 --- a/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge.service.j2 +++ b/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge.service.j2 @@ -27,7 +27,7 @@ ExecStart=/usr/bin/docker run --rm --name matrix-sms-bridge \ {% endif %} -v {{ matrix_sms_bridge_config_path }}:/config:z \ -v {{ matrix_sms_bridge_data_path }}:/data:z \ - --env SPRING_CONFIG_ADDITIONAL_LOCATION=/config/application.yml + --env SPRING_CONFIG_ADDITIONAL_LOCATION=/config/application.yml \ {% for arg in matrix_sms_bridge_container_extra_arguments %} {{ arg }} \ {% endfor %} From f1ceb49ae20b010dc271a15fe644e3d3ee5c02ee Mon Sep 17 00:00:00 2001 From: benkuly <12199167+benkuly@users.noreply.github.com> Date: Mon, 23 Nov 2020 12:52:16 +0100 Subject: [PATCH 061/217] fixed wrong path of truststore --- roles/matrix-bridge-sms/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-sms/defaults/main.yml b/roles/matrix-bridge-sms/defaults/main.yml index b551b43a..9df04de7 100644 --- a/roles/matrix-bridge-sms/defaults/main.yml +++ b/roles/matrix-bridge-sms/defaults/main.yml @@ -66,7 +66,7 @@ matrix_sms_bridge_configuration_yaml: | # (optional) if you use a self signed certificate, you can add the public key here {% if matrix_sms_bridge_provider_android_truststore_local_path %} trustStore: - path: /data/config/matrix-sms-gateway-server.p12 + path: /config/matrix-sms-gateway-server.p12 password: {{ matrix_sms_bridge_provider_android_truststore_password }} type: PKCS12 {% endif %} From 3a2e058f2e26c76c45fb287c2b80aa9d2dbb8185 Mon Sep 17 00:00:00 2001 From: benkuly <12199167+benkuly@users.noreply.github.com> Date: Mon, 23 Nov 2020 13:07:08 +0100 Subject: [PATCH 062/217] updated version of matrix-sms-bridge --- roles/matrix-bridge-sms/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-sms/defaults/main.yml b/roles/matrix-bridge-sms/defaults/main.yml index 9df04de7..6477f176 100644 --- a/roles/matrix-bridge-sms/defaults/main.yml +++ b/roles/matrix-bridge-sms/defaults/main.yml @@ -3,7 +3,7 @@ matrix_sms_bridge_enabled: true -matrix_sms_bridge_docker_image: "docker.io/folivonet/matrix-sms-bridge:0.5.0" +matrix_sms_bridge_docker_image: "docker.io/folivonet/matrix-sms-bridge:0.5.1" matrix_sms_bridge_base_path: "{{ matrix_base_data_path }}/matrix-sms-bridge" matrix_sms_bridge_config_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/config" From ff9a4e90c4f0cac1cb7d0471074d122123e9563b Mon Sep 17 00:00:00 2001 From: benkuly <12199167+benkuly@users.noreply.github.com> Date: Mon, 23 Nov 2020 13:43:04 +0100 Subject: [PATCH 063/217] updated matrix-sms-bridge --- roles/matrix-bridge-sms/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-sms/defaults/main.yml b/roles/matrix-bridge-sms/defaults/main.yml index 6477f176..9f1c1d9c 100644 --- a/roles/matrix-bridge-sms/defaults/main.yml +++ b/roles/matrix-bridge-sms/defaults/main.yml @@ -3,7 +3,7 @@ matrix_sms_bridge_enabled: true -matrix_sms_bridge_docker_image: "docker.io/folivonet/matrix-sms-bridge:0.5.1" +matrix_sms_bridge_docker_image: "docker.io/folivonet/matrix-sms-bridge:0.5.2" matrix_sms_bridge_base_path: "{{ matrix_base_data_path }}/matrix-sms-bridge" matrix_sms_bridge_config_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/config" From 8c8002f6396be4eea63f2895689725b528c09928 Mon Sep 17 00:00:00 2001 From: benkuly <12199167+benkuly@users.noreply.github.com> Date: Mon, 23 Nov 2020 14:48:04 +0100 Subject: [PATCH 064/217] added changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae96ef36..c350d235 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +# 2020-11-23 + +## Breaking change matrix-sms-bridge + +Because of many problems using gammu as SMS provider, matrix-sms-bridge now uses (https://github.com/RebekkaMa/android-sms-gateway-server) by default. See (the docs)[./docs/configuring-playbook-bridge-matrix-bridge-sms.md] which new vars you need to add. + +If you are using this playbook to deploy matrix-sms-bridge and still really want to use gammu as SMS provider, we could possibly add support for both android-sms-gateway-server and gammu. + # 2020-11-13 ## Breaking change matrix-sms-bridge From c9d2ef7981a54525c17c201b604efc764422c2c5 Mon Sep 17 00:00:00 2001 From: Aaron Raimist Date: Mon, 23 Nov 2020 23:05:54 -0600 Subject: [PATCH 065/217] Upgrade Element (1.7.13 -> 1.7.14) --- roles/matrix-client-element/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index f1504c5b..17a04465 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -2,7 +2,7 @@ matrix_client_element_enabled: true matrix_client_element_container_image_self_build: false -matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:v1.7.13" +matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:v1.7.14" matrix_client_element_docker_image_name_prefix: "{{ 'localhost/' if matrix_client_element_container_image_self_build else 'docker.io/' }}" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" From 3e2355282bffe214a3ca5cf98ae56a160e763c8e Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 24 Nov 2020 09:06:16 +0200 Subject: [PATCH 066/217] Upgrade Postgres minor versions Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/727 --- roles/matrix-postgres/defaults/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index 8717fdb6..0d2f50e3 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -8,11 +8,11 @@ matrix_postgres_db_name: "" matrix_postgres_base_path: "{{ matrix_base_data_path }}/postgres" matrix_postgres_data_path: "{{ matrix_postgres_base_path }}/data" -matrix_postgres_docker_image_v9: "docker.io/postgres:9.6.19-alpine" -matrix_postgres_docker_image_v10: "docker.io/postgres:10.14-alpine" -matrix_postgres_docker_image_v11: "docker.io/postgres:11.9-alpine" -matrix_postgres_docker_image_v12: "docker.io/postgres:12.4-alpine" -matrix_postgres_docker_image_v13: "docker.io/postgres:13.0-alpine" +matrix_postgres_docker_image_v9: "docker.io/postgres:9.6.20-alpine" +matrix_postgres_docker_image_v10: "docker.io/postgres:10.15-alpine" +matrix_postgres_docker_image_v11: "docker.io/postgres:11.10-alpine" +matrix_postgres_docker_image_v12: "docker.io/postgres:12.5-alpine" +matrix_postgres_docker_image_v13: "docker.io/postgres:13.1-alpine" matrix_postgres_docker_image_latest: "{{ matrix_postgres_docker_image_v13 }}" # This variable is assigned at runtime. Overriding its value has no effect. From 27c9014cb881fb5296af10b079fa67a39fcdc0f2 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 24 Nov 2020 09:38:17 +0200 Subject: [PATCH 067/217] Improve uninstallation instructions Also switches to using `docker system prune -a` for a less invasive cleanup of Docker images and related resources. --- docs/uninstalling.md | 30 ++++++++++++++----- .../usr-local-bin/matrix-remove-all.j2 | 6 ++-- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/docs/uninstalling.md b/docs/uninstalling.md index 7acb9bcc..cb55a7b5 100644 --- a/docs/uninstalling.md +++ b/docs/uninstalling.md @@ -1,24 +1,40 @@ # Uninstalling -**Note**: If you have some trouble with your installation configuration, you can just [re-run the playbook](installing.md) and it will try to set things up again. You don't need to uninstall and install fresh. +**Warnings**: -However, if you've installed this on some server where you have other stuff you wish to preserve, and now want get rid of Matrix, it's enough to do these: +- If your server federates with others, make sure to **leave any federated rooms before nuking your Matrix server's data**. Otherwise, the next time you set up a Matrix server for this domain (regardless of the installation method you use), you'll encounter trouble federating. -- ensure all Matrix services are stopped (`systemctl stop 'matrix*'`) +- If you have some trouble with your installation, you can just [re-run the playbook](installing.md) and it will try to set things up again. **Uninstalling and then installing anew rarely solves anything**. -- delete the Matrix-related systemd .service files (`rm -f /etc/systemd/system/matrix*`) and reload systemd (`systemctl daemon-reload`) + +----------------- + + +## Uninstalling using a script + +Installing places a `/usr/local/bin/matrix-remove-all` script on the server. + +You can run it to to have it uninstall things for you automatically (see below). **Use with caution!** + + +## Uninstalling manually + +If you prefer to uninstall manually, run these commands (most are meant to be executed on the Matrix server itself): + +- ensure all Matrix services are stopped: `ansible-playbook -i inventory/hosts setup.yml --tags=stop` (if you can't get Ansible working to run this command, you can run `systemctl stop 'matrix*'` manually on the server) + +- delete the Matrix-related systemd `.service` files (`rm -f /etc/systemd/system/matrix*.service`) and reload systemd (`systemctl daemon-reload`) - delete all Matrix-related cronjobs (`rm -f /etc/cron.d/matrix*`) - delete some helper scripts (`rm -f /usr/local/bin/matrix*`) -- delete some cached Docker images (or just delete them all: `docker rmi $(docker images -aq)`) +- delete some cached Docker images (`docker system prune -a`) or just delete them all (`docker rmi $(docker images -aq)`) -- delete the Docker network: `docker network rm matrix` +- delete the Docker network: `docker network rm matrix` (might have been deleted already if you ran the `docker system prune` command) - uninstall Docker itself, if necessary - delete the `/matrix` directory (`rm -rf /matrix`) -The script `/usr/local/bin/matrix-remove-all` performs all these steps (**use with caution!**). diff --git a/roles/matrix-base/templates/usr-local-bin/matrix-remove-all.j2 b/roles/matrix-base/templates/usr-local-bin/matrix-remove-all.j2 index 972919e3..2a2314a5 100644 --- a/roles/matrix-base/templates/usr-local-bin/matrix-remove-all.j2 +++ b/roles/matrix-base/templates/usr-local-bin/matrix-remove-all.j2 @@ -24,9 +24,9 @@ else find /etc/cron.d/ -name "matrix-*" -delete echo "Remove matrix scripts" find {{ matrix_local_bin_path }}/ -name "matrix-*" -delete - echo "Remove every docker images" - docker rmi $(docker images -aq) - echo "Remove docker matrix network" + echo "Remove unused Docker images and resources" + docker system prune -af + echo "Remove Docker matrix network (should be gone already, but ..)" docker network rm {{ matrix_docker_network }} echo "Remove {{ matrix_base_data_path }} directory" rm -fr "{{ matrix_base_data_path }}" From 1fca917ad13103845eade92be3f5a25616b92988 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 24 Nov 2020 10:15:12 +0200 Subject: [PATCH 068/217] Replace some -v instances with --mount `-v` magically creates the source destination as a directory, if it doesn't exist already. We'd like to avoid this magic and the potential breakage that it might cause. We'd rather fail while Docker tries to find things to `--mount` than have it automatically create directories and fail anyway, while having contaminated the filesystem. There's a lot more `-v` instances remaining to be fixed later on. This is just some start. Things like `matrix_synapse_container_additional_volumes` and `matrix_nginx_proxy_container_additional_volumes` were not changed to use `--mount`, as options for each one are passed differently (`ro` is `ro`, but `rw` doesn't exist and `slave` is `bind-propagation=slave`). To avoid breaking people's custom volume mounts, we keep it as it is for now. A deficiency with `--mount` is that it lacks the `z` option (SELinux ownership changes), and some of our `-v` instances use that. I'm not sure how supported SELinux is for us right now, but it might be, and breaking that would not be a good idea. --- .../templates/systemd/matrix-client-element.service.j2 | 10 +++++----- .../templates/systemd/matrix-dimension.service.j2 | 2 +- .../templates/jicofo/matrix-jitsi-jicofo.service.j2 | 2 +- .../templates/jvb/matrix-jitsi-jvb.service.j2 | 2 +- .../templates/prosody/matrix-jitsi-prosody.service.j2 | 4 ++-- .../templates/web/matrix-jitsi-web.service.j2 | 4 ++-- .../templates/systemd/matrix-ma1sd.service.j2 | 4 ++-- .../templates/systemd/matrix-nginx-proxy.service.j2 | 10 +++++----- .../templates/systemd/matrix-postgres.service.j2 | 4 ++-- .../templates/systemd/matrix-registration.service.j2 | 4 ++-- roles/matrix-synapse/tasks/synapse/setup_install.yml | 2 +- .../templates/goofys/systemd/matrix-goofys.service.j2 | 6 +++--- .../synapse/systemd/matrix-synapse.service.j2 | 4 ++-- 13 files changed, 29 insertions(+), 29 deletions(-) diff --git a/roles/matrix-client-element/templates/systemd/matrix-client-element.service.j2 b/roles/matrix-client-element/templates/systemd/matrix-client-element.service.j2 index 49b2f198..39b7aa52 100644 --- a/roles/matrix-client-element/templates/systemd/matrix-client-element.service.j2 +++ b/roles/matrix-client-element/templates/systemd/matrix-client-element.service.j2 @@ -21,13 +21,13 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-client-element -p {{ matrix_client_element_container_http_host_bind_port }}:8080 \ {% endif %} --tmpfs=/tmp:rw,noexec,nosuid,size=10m \ - -v {{ matrix_client_element_data_path }}/nginx.conf:/etc/nginx/nginx.conf:ro \ - -v {{ matrix_client_element_data_path }}/config.json:/app/config.json:ro \ - -v {{ matrix_client_element_data_path }}/config.json:/app/config.{{ matrix_server_fqn_element }}.json:ro \ + --mount type=bind,src={{ matrix_client_element_data_path }}/nginx.conf,dst=/etc/nginx/nginx.conf,ro \ + --mount type=bind,src={{ matrix_client_element_data_path }}/config.json,dst=/app/config.json,ro \ + --mount type=bind,src={{ matrix_client_element_data_path }}/config.json,dst=/app/config.{{ matrix_server_fqn_element }}.json,ro \ {% if matrix_client_element_embedded_pages_home_path is not none %} - -v {{ matrix_client_element_data_path }}/home.html:/app/home.html:ro \ + --mount type=bind,src={{ matrix_client_element_data_path }}/home.html,dst=/app/home.html,ro \ {% endif %} - -v {{ matrix_client_element_data_path }}/welcome.html:/app/welcome.html:ro \ + --mount type=bind,src={{ matrix_client_element_data_path }}/welcome.html,dst=/app/welcome.html,ro \ {% for arg in matrix_client_element_container_extra_arguments %} {{ arg }} \ {% endfor %} diff --git a/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 b/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 index db9d58a8..bad94f64 100644 --- a/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 +++ b/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 @@ -23,7 +23,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-dimension \ {% if matrix_dimension_container_http_host_bind_port %} -p {{ matrix_dimension_container_http_host_bind_port }}:8184 \ {% endif %} - -v {{ matrix_dimension_base_path }}:/data:rw \ + --mount type=bind,src={{ matrix_dimension_base_path }},dst=/data \ {% for arg in matrix_dimension_container_extra_arguments %} {{ arg }} \ {% endfor %} diff --git a/roles/matrix-jitsi/templates/jicofo/matrix-jitsi-jicofo.service.j2 b/roles/matrix-jitsi/templates/jicofo/matrix-jitsi-jicofo.service.j2 index 3512b3af..f19d08ff 100644 --- a/roles/matrix-jitsi/templates/jicofo/matrix-jitsi-jicofo.service.j2 +++ b/roles/matrix-jitsi/templates/jicofo/matrix-jitsi-jicofo.service.j2 @@ -15,7 +15,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-jitsi-jicofo \ --log-driver=none \ --network={{ matrix_docker_network }} \ --env-file={{ matrix_jitsi_jicofo_base_path }}/env \ - -v {{ matrix_jitsi_jicofo_config_path }}:/config \ + --mount type=bind,src={{ matrix_jitsi_jicofo_config_path }},dst=/config \ {% for arg in matrix_jitsi_jicofo_container_extra_arguments %} {{ arg }} \ {% endfor %} diff --git a/roles/matrix-jitsi/templates/jvb/matrix-jitsi-jvb.service.j2 b/roles/matrix-jitsi/templates/jvb/matrix-jitsi-jvb.service.j2 index a189df9c..6db6a850 100644 --- a/roles/matrix-jitsi/templates/jvb/matrix-jitsi-jvb.service.j2 +++ b/roles/matrix-jitsi/templates/jvb/matrix-jitsi-jvb.service.j2 @@ -21,7 +21,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-jitsi-jvb \ {% if matrix_jitsi_jvb_container_rtp_tcp_host_bind_port %} -p {{ matrix_jitsi_jvb_container_rtp_tcp_host_bind_port }}:{{ matrix_jitsi_jvb_rtp_tcp_port }} \ {% endif %} - -v {{ matrix_jitsi_jvb_config_path }}:/config \ + --mount type=bind,src={{ matrix_jitsi_jvb_config_path }},dst=/config \ {% for arg in matrix_jitsi_jvb_container_extra_arguments %} {{ arg }} \ {% endfor %} diff --git a/roles/matrix-jitsi/templates/prosody/matrix-jitsi-prosody.service.j2 b/roles/matrix-jitsi/templates/prosody/matrix-jitsi-prosody.service.j2 index 30801d77..74b13df3 100644 --- a/roles/matrix-jitsi/templates/prosody/matrix-jitsi-prosody.service.j2 +++ b/roles/matrix-jitsi/templates/prosody/matrix-jitsi-prosody.service.j2 @@ -15,8 +15,8 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-jitsi-prosody --log-driver=none \ --network={{ matrix_docker_network }} \ --env-file={{ matrix_jitsi_prosody_base_path }}/env \ - -v {{ matrix_jitsi_prosody_config_path }}:/config \ - -v {{ matrix_jitsi_prosody_plugins_path }}:/prosody-plugins-custom \ + --mount type=bind,src={{ matrix_jitsi_prosody_config_path }},dst=/config \ + --mount type=bind,src={{ matrix_jitsi_prosody_plugins_path }},dst=/prosody-plugins-custom \ {% for arg in matrix_jitsi_prosody_container_extra_arguments %} {{ arg }} \ {% endfor %} diff --git a/roles/matrix-jitsi/templates/web/matrix-jitsi-web.service.j2 b/roles/matrix-jitsi/templates/web/matrix-jitsi-web.service.j2 index 1f8a890b..e13f9633 100644 --- a/roles/matrix-jitsi/templates/web/matrix-jitsi-web.service.j2 +++ b/roles/matrix-jitsi/templates/web/matrix-jitsi-web.service.j2 @@ -18,8 +18,8 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-jitsi-web \ {% if matrix_jitsi_web_container_http_host_bind_port %} -p {{ matrix_jitsi_web_container_http_host_bind_port }}:80 \ {% endif %} - -v {{ matrix_jitsi_web_config_path }}:/config \ - -v {{ matrix_jitsi_web_transcripts_path }}:/usr/share/jitsi-meet/transcripts \ + --mount type=bind,src={{ matrix_jitsi_web_config_path }},dst=/config \ + --mount type=bind,src={{ matrix_jitsi_web_transcripts_path }},dst=/usr/share/jitsi-meet/transcripts \ {% for arg in matrix_jitsi_web_container_extra_arguments %} {{ arg }} \ {% endfor %} diff --git a/roles/matrix-ma1sd/templates/systemd/matrix-ma1sd.service.j2 b/roles/matrix-ma1sd/templates/systemd/matrix-ma1sd.service.j2 index 80370088..3c49ec75 100644 --- a/roles/matrix-ma1sd/templates/systemd/matrix-ma1sd.service.j2 +++ b/roles/matrix-ma1sd/templates/systemd/matrix-ma1sd.service.j2 @@ -29,8 +29,8 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-ma1sd \ {% if matrix_ma1sd_verbose_logging %} -e MA1SD_LOG_LEVEL=debug \ {% endif %} - -v {{ matrix_ma1sd_config_path }}:/etc/ma1sd:ro \ - -v {{ matrix_ma1sd_data_path }}:/var/ma1sd:rw \ + --mount type=bind,src={{ matrix_ma1sd_config_path }},dst=/etc/ma1sd,ro \ + --mount type=bind,src={{ matrix_ma1sd_data_path }},dst=/var/ma1sd \ {% for arg in matrix_ma1sd_container_extra_arguments %} {{ arg }} \ {% endfor %} diff --git a/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2 b/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2 index 7a385a64..e2182d28 100644 --- a/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2 +++ b/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2 @@ -30,11 +30,11 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-nginx-proxy \ {% if matrix_nginx_proxy_proxy_matrix_federation_api_enabled and matrix_nginx_proxy_container_federation_host_bind_port %} -p {{ matrix_nginx_proxy_container_federation_host_bind_port }}:8448 \ {% endif %} - -v {{ matrix_nginx_proxy_base_path }}/nginx.conf:/etc/nginx/nginx.conf:ro \ - -v {{ matrix_nginx_proxy_data_path }}:/nginx-data:ro \ - -v {{ matrix_nginx_proxy_confd_path }}:/etc/nginx/conf.d:ro \ - -v {{ matrix_ssl_config_dir_path }}:{{ matrix_ssl_config_dir_path }}:ro \ - -v {{ matrix_static_files_base_path }}:{{ matrix_static_files_base_path }}:ro \ + --mount type=bind,src={{ matrix_nginx_proxy_base_path }}/nginx.conf,dst=/etc/nginx/nginx.conf,ro \ + --mount type=bind,src={{ matrix_nginx_proxy_data_path }},dst=/nginx-data,ro \ + --mount type=bind,src={{ matrix_nginx_proxy_confd_path }},dst=/etc/nginx/conf.d,ro \ + --mount type=bind,src={{ matrix_ssl_config_dir_path }},dst={{ matrix_ssl_config_dir_path }},ro \ + --mount type=bind,src={{ matrix_static_files_base_path }},dst={{ matrix_static_files_base_path }},ro \ {% for volume in matrix_nginx_proxy_container_additional_volumes %} -v {{ volume.src }}:{{ volume.dst }}:{{ volume.options }} \ {% endfor %} diff --git a/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2 b/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2 index 8e6392c1..47f19e9b 100644 --- a/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2 +++ b/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2 @@ -21,8 +21,8 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-postgres \ -p {{ matrix_postgres_container_postgres_bind_port }}:5432 \ {% endif %} --env-file={{ matrix_postgres_base_path }}/env-postgres-server \ - -v {{ matrix_postgres_data_path }}:/var/lib/postgresql/data:rw \ - -v /etc/passwd:/etc/passwd:ro \ + --mount type=bind,src={{ matrix_postgres_data_path }},dst=/var/lib/postgresql/data \ + --mount type=bind,src=/etc/passwd,dst=/etc/passwd,ro \ {% for arg in matrix_postgres_container_extra_arguments %} {{ arg }} \ {% endfor %} diff --git a/roles/matrix-registration/templates/systemd/matrix-registration.service.j2 b/roles/matrix-registration/templates/systemd/matrix-registration.service.j2 index 38860729..052b7d95 100644 --- a/roles/matrix-registration/templates/systemd/matrix-registration.service.j2 +++ b/roles/matrix-registration/templates/systemd/matrix-registration.service.j2 @@ -22,8 +22,8 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-registration \ {% if matrix_registration_container_http_host_bind_port %} -p {{ matrix_registration_container_http_host_bind_port }}:5000 \ {% endif %} - -v {{ matrix_registration_config_path }}:/config:ro \ - -v {{ matrix_registration_data_path }}:/data \ + --mount type=bind,src={{ matrix_registration_config_path }},dst=/config,ro \ + --mount type=bind,src={{ matrix_registration_data_path }},dst=/data \ {% for arg in matrix_registration_container_extra_arguments %} {{ arg }} \ {% endfor %} diff --git a/roles/matrix-synapse/tasks/synapse/setup_install.yml b/roles/matrix-synapse/tasks/synapse/setup_install.yml index 51f846e3..3aa2c223 100644 --- a/roles/matrix-synapse/tasks/synapse/setup_install.yml +++ b/roles/matrix-synapse/tasks/synapse/setup_install.yml @@ -65,7 +65,7 @@ --name=matrix-config --user={{ matrix_user_uid }}:{{ matrix_user_gid }} --cap-drop=ALL - -v {{ matrix_synapse_config_dir_path }}:/data + --mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/data -e UID={{ matrix_user_uid }} -e GID={{ matrix_user_gid }} -e SYNAPSE_CONFIG_PATH=/data/homeserver.yaml diff --git a/roles/matrix-synapse/templates/goofys/systemd/matrix-goofys.service.j2 b/roles/matrix-synapse/templates/goofys/systemd/matrix-goofys.service.j2 index d9d752c2..c3ec9249 100644 --- a/roles/matrix-synapse/templates/goofys/systemd/matrix-goofys.service.j2 +++ b/roles/matrix-synapse/templates/goofys/systemd/matrix-goofys.service.j2 @@ -12,13 +12,13 @@ ExecStartPre=-{{ matrix_host_command_docker }} rm %n ExecStart={{ matrix_host_command_docker }} run --rm --name %n \ --log-driver=none \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ - -v /etc/passwd:/etc/passwd:ro \ - -v /etc/group:/etc/group:ro \ + --mount type=bind,src=/etc/passwd,dst=/etc/passwd,ro \ + --mount type=bind,src=/etc/group,dst=/etc/group,ro \ + --mount type=bind,src={{ matrix_synapse_media_store_path }},dst=/s3,bind-propagation=shared \ --security-opt apparmor:unconfined \ --cap-add mknod \ --cap-add sys_admin \ --device=/dev/fuse \ - -v {{ matrix_synapse_media_store_path }}:/s3:shared \ --env-file={{ matrix_synapse_config_dir_path }}/env-goofys \ --entrypoint /bin/sh \ {{ matrix_s3_goofys_docker_image }} \ diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 index 47786eee..791d324e 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 @@ -43,8 +43,8 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \ {% if matrix_synapse_manhole_enabled and matrix_synapse_container_manhole_api_host_bind_port %} -p {{ matrix_synapse_container_manhole_api_host_bind_port }}:9000 \ {% endif %} - -v {{ matrix_synapse_config_dir_path }}:/data:ro \ - -v {{ matrix_synapse_storage_path }}:/matrix-media-store-parent:slave \ + --mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/data,ro \ + --mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \ {% for volume in matrix_synapse_container_additional_volumes %} -v {{ volume.src }}:{{ volume.dst }}:{{ volume.options }} \ {% endfor %} From 75f9fde7a48c11e62d83a543432e583c8d790077 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 25 Nov 2020 10:49:59 +0200 Subject: [PATCH 069/217] Remove some more -v usage Continuation of 1fca917ad13103. Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/722 --- .../systemd/matrix-bot-matrix-reminder-bot.service.j2 | 4 ++-- .../tasks/setup_install.yml | 2 +- .../systemd/matrix-appservice-discord.service.j2 | 4 ++-- .../templates/systemd/matrix-corporal.service.j2 | 6 +++--- .../templates/systemd/matrix-coturn.service.j2 | 2 +- .../ssl/setup_ssl_lets_encrypt_obtain_for_domain.yml | 8 ++++---- .../matrix-ssl-lets-encrypt-certificates-renew.j2 | 4 ++-- roles/matrix-postgres/tasks/import_postgres.yml | 2 +- roles/matrix-postgres/tasks/import_sqlite_db.yml | 6 +++--- roles/matrix-postgres/tasks/upgrade_postgres.yml | 4 ++-- .../tasks/rust-synapse-compress-state/compress_room.yml | 4 ++-- 11 files changed, 23 insertions(+), 23 deletions(-) diff --git a/roles/matrix-bot-matrix-reminder-bot/templates/systemd/matrix-bot-matrix-reminder-bot.service.j2 b/roles/matrix-bot-matrix-reminder-bot/templates/systemd/matrix-bot-matrix-reminder-bot.service.j2 index 0decac02..23493c54 100644 --- a/roles/matrix-bot-matrix-reminder-bot/templates/systemd/matrix-bot-matrix-reminder-bot.service.j2 +++ b/roles/matrix-bot-matrix-reminder-bot/templates/systemd/matrix-bot-matrix-reminder-bot.service.j2 @@ -21,8 +21,8 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-bot-matrix-rem --read-only \ --network={{ matrix_docker_network }} \ -e 'TZ={{ matrix_bot_matrix_reminder_bot_reminders_timezone }}' \ - -v {{ matrix_bot_matrix_reminder_bot_config_path }}:/config:ro \ - -v {{ matrix_bot_matrix_reminder_bot_data_path }}:/data:rw \ + --mount type=bind,src={{ matrix_bot_matrix_reminder_bot_config_path }},dst=/config,ro \ + --mount type=bind,src={{ matrix_bot_matrix_reminder_bot_data_path }},dst=/data \ --entrypoint=/bin/sh \ {% for arg in matrix_bot_matrix_reminder_bot_container_extra_arguments %} {{ arg }} \ diff --git a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml index 3678b35b..935d07eb 100644 --- a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml @@ -63,7 +63,7 @@ {{ matrix_host_command_docker }} run --rm --name matrix-appservice-discord-link-gen --user={{ matrix_user_uid }}:{{ matrix_user_gid }} --cap-drop=ALL - -v {{ matrix_appservice_discord_config_path }}:/cfg + --mount type=bind,src={{ matrix_appservice_discord_config_path }},dst=/cfg -w /cfg {{ matrix_appservice_discord_docker_image }} /bin/sh -c "node /build/tools/addbot.js > /cfg/invite_link" diff --git a/roles/matrix-bridge-appservice-discord/templates/systemd/matrix-appservice-discord.service.j2 b/roles/matrix-bridge-appservice-discord/templates/systemd/matrix-appservice-discord.service.j2 index f2187ca3..6a44a3e6 100644 --- a/roles/matrix-bridge-appservice-discord/templates/systemd/matrix-appservice-discord.service.j2 +++ b/roles/matrix-bridge-appservice-discord/templates/systemd/matrix-appservice-discord.service.j2 @@ -25,8 +25,8 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-appservice-dis {% if matrix_appservice_discord_container_http_host_bind_port %} -p {{ matrix_appservice_discord_container_http_host_bind_port }}:9005 \ {% endif %} - -v {{ matrix_appservice_discord_config_path }}:/cfg \ - -v {{ matrix_appservice_discord_data_path }}:/data \ + --mount type=bind,src={{ matrix_appservice_discord_config_path }},dst=/cfg \ + --mount type=bind,src={{ matrix_appservice_discord_data_path }},dst=/data \ {% for arg in matrix_appservice_discord_container_extra_arguments %} {{ arg }} \ {% endfor %} diff --git a/roles/matrix-corporal/templates/systemd/matrix-corporal.service.j2 b/roles/matrix-corporal/templates/systemd/matrix-corporal.service.j2 index 533ece0a..97ec7d89 100644 --- a/roles/matrix-corporal/templates/systemd/matrix-corporal.service.j2 +++ b/roles/matrix-corporal/templates/systemd/matrix-corporal.service.j2 @@ -23,9 +23,9 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-corporal \ {% if matrix_corporal_container_http_api_host_bind_port %} -p {{ matrix_corporal_container_http_api_host_bind_port }}:41081 \ {% endif %} - -v {{ matrix_corporal_config_dir_path }}:/etc/matrix-corporal:ro \ - -v {{ matrix_corporal_cache_dir_path }}:/var/cache/matrix-corporal:rw \ - -v {{ matrix_corporal_var_dir_path }}:/var/matrix-corporal:rw \ + --mount type=bind,src={{ matrix_corporal_config_dir_path }},dst=/etc/matrix-corporal,ro \ + --mount type=bind,src={{ matrix_corporal_cache_dir_path }},dst=/var/cache/matrix-corporal \ + --mount type=bind,src={{ matrix_corporal_var_dir_path }},dst=/var/matrix-corporal \ {% for arg in matrix_corporal_container_extra_arguments %} {{ arg }} \ {% endfor %} diff --git a/roles/matrix-coturn/templates/systemd/matrix-coturn.service.j2 b/roles/matrix-coturn/templates/systemd/matrix-coturn.service.j2 index a6d19705..e484b59b 100644 --- a/roles/matrix-coturn/templates/systemd/matrix-coturn.service.j2 +++ b/roles/matrix-coturn/templates/systemd/matrix-coturn.service.j2 @@ -30,7 +30,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-coturn \ {% if matrix_coturn_container_turn_range_listen_interface is not none %} -p {{ matrix_coturn_container_turn_range_listen_interface }}{{ ':' if matrix_coturn_container_turn_range_listen_interface else '' }}{{ matrix_coturn_turn_udp_min_port }}-{{ matrix_coturn_turn_udp_max_port }}:{{ matrix_coturn_turn_udp_min_port }}-{{ matrix_coturn_turn_udp_max_port }}/udp \ {% endif %} - -v {{ matrix_coturn_config_path }}:/turnserver.conf:ro \ + --mount type=bind,src={{ matrix_coturn_config_path }},dst=/turnserver.conf,ro \ {% for volume in matrix_coturn_container_additional_volumes %} -v {{ volume.src }}:{{ volume.dst }}:{{ volume.options }} \ {% endfor %} diff --git a/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt_obtain_for_domain.yml b/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt_obtain_for_domain.yml index 3dea71fb..4639f122 100644 --- a/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt_obtain_for_domain.yml +++ b/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt_obtain_for_domain.yml @@ -35,8 +35,8 @@ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} --cap-drop=ALL -p {{ matrix_ssl_lets_encrypt_container_standalone_http_host_bind_port }}:8080 - -v {{ matrix_ssl_config_dir_path }}:/etc/letsencrypt - -v {{ matrix_ssl_log_dir_path }}:/var/log/letsencrypt + --mount type=bind,src={{ matrix_ssl_config_dir_path }},dst=/etc/letsencrypt + --mount type=bind,src={{ matrix_ssl_log_dir_path }},dst=/var/log/letsencrypt {{ matrix_ssl_lets_encrypt_certbot_docker_image }} certonly --non-interactive @@ -63,8 +63,8 @@ --cap-drop=ALL -p 127.0.0.1:{{ matrix_ssl_lets_encrypt_certbot_standalone_http_port }}:8080 --network={{ matrix_docker_network }} - -v {{ matrix_ssl_config_dir_path }}:/etc/letsencrypt - -v {{ matrix_ssl_log_dir_path }}:/var/log/letsencrypt + --mount type=bind,src={{ matrix_ssl_config_dir_path }},dst=/etc/letsencrypt + --mount type=bind,src={{ matrix_ssl_log_dir_path }},dst=/var/log/letsencrypt {{ matrix_ssl_lets_encrypt_certbot_docker_image }} certonly --non-interactive diff --git a/roles/matrix-nginx-proxy/templates/usr-local-bin/matrix-ssl-lets-encrypt-certificates-renew.j2 b/roles/matrix-nginx-proxy/templates/usr-local-bin/matrix-ssl-lets-encrypt-certificates-renew.j2 index f7c930c0..39366abf 100644 --- a/roles/matrix-nginx-proxy/templates/usr-local-bin/matrix-ssl-lets-encrypt-certificates-renew.j2 +++ b/roles/matrix-nginx-proxy/templates/usr-local-bin/matrix-ssl-lets-encrypt-certificates-renew.j2 @@ -14,8 +14,8 @@ docker run \ --cap-drop=ALL \ --network="{{ matrix_docker_network }}" \ -p 127.0.0.1:{{ matrix_ssl_lets_encrypt_certbot_standalone_http_port }}:8080 \ - -v {{ matrix_ssl_config_dir_path }}:/etc/letsencrypt \ - -v {{ matrix_ssl_log_dir_path }}:/var/log/letsencrypt \ + --mount type=bind,src={{ matrix_ssl_config_dir_path }},dst=/etc/letsencrypt \ + --mount type=bind,src={{ matrix_ssl_log_dir_path }},dst=/var/log/letsencrypt \ {{ matrix_ssl_lets_encrypt_certbot_docker_image }} \ renew \ --non-interactive \ diff --git a/roles/matrix-postgres/tasks/import_postgres.yml b/roles/matrix-postgres/tasks/import_postgres.yml index 0753c292..33d98691 100644 --- a/roles/matrix-postgres/tasks/import_postgres.yml +++ b/roles/matrix-postgres/tasks/import_postgres.yml @@ -69,7 +69,7 @@ --cap-drop=ALL --network={{ matrix_docker_network }} --env-file={{ matrix_postgres_base_path }}/env-postgres-psql - -v {{ server_path_postgres_dump }}:/{{ server_path_postgres_dump|basename }}:ro + --mount type=bind,src={{ server_path_postgres_dump }},dst=/{{ server_path_postgres_dump|basename }},ro --entrypoint=/bin/sh {{ matrix_postgres_docker_image_latest }} -c "cat /{{ server_path_postgres_dump|basename }} | diff --git a/roles/matrix-postgres/tasks/import_sqlite_db.yml b/roles/matrix-postgres/tasks/import_sqlite_db.yml index c877ead4..ea15c5a8 100644 --- a/roles/matrix-postgres/tasks/import_sqlite_db.yml +++ b/roles/matrix-postgres/tasks/import_sqlite_db.yml @@ -79,8 +79,8 @@ --cap-drop=ALL --network={{ matrix_docker_network }} --entrypoint=python - -v {{ matrix_synapse_config_dir_path }}:/data - -v {{ matrix_synapse_config_dir_path }}:/matrix-media-store-parent/media-store - -v {{ server_path_homeserver_db }}:/{{ server_path_homeserver_db|basename }} + --mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/data + --mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/matrix-media-store-parent/media-store + --mount type=bind,src={{ server_path_homeserver_db }},dst=/{{ server_path_homeserver_db|basename }} {{ matrix_synapse_docker_image }} /usr/local/bin/synapse_port_db --sqlite-database /{{ server_path_homeserver_db|basename }} --postgres-config /data/homeserver.yaml diff --git a/roles/matrix-postgres/tasks/upgrade_postgres.yml b/roles/matrix-postgres/tasks/upgrade_postgres.yml index c1a01d3f..72f327b3 100644 --- a/roles/matrix-postgres/tasks/upgrade_postgres.yml +++ b/roles/matrix-postgres/tasks/upgrade_postgres.yml @@ -85,7 +85,7 @@ --network={{ matrix_docker_network }} --env-file={{ matrix_postgres_base_path }}/env-postgres-psql --entrypoint=/bin/sh - -v {{ postgres_dump_dir }}:/out + --mount type=bind,src={{ postgres_dump_dir }},dst=/out {{ matrix_postgres_detected_version_corresponding_docker_image }} -c "pg_dumpall -h matrix-postgres {{ '| gzip -c ' if postgres_dump_name.endswith('.gz') else '' }} @@ -131,7 +131,7 @@ --network={{ matrix_docker_network }} --env-file={{ matrix_postgres_base_path }}/env-postgres-psql --entrypoint=/bin/sh - -v {{ postgres_dump_dir }}:/in:ro + --mount type=bind,src={{ postgres_dump_dir }},dst=/in,ro {{ matrix_postgres_docker_image_latest }} -c "cat /in/{{ postgres_dump_name }} | {{ 'gunzip |' if postgres_dump_name.endswith('.gz') else '' }} diff --git a/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml b/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml index ad7b91b4..8570411f 100644 --- a/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml +++ b/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml @@ -8,7 +8,7 @@ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} --cap-drop=ALL --network={{ matrix_docker_network }} - -v {{ matrix_synapse_rust_synapse_compress_state_base_path }}:/work + --mount type=bind,src={{ matrix_synapse_rust_synapse_compress_state_base_path }},dst=/work {{ matrix_synapse_rust_synapse_compress_state_docker_image }} /synapse-compress-state -t -o /work/state-compressor.sql -p "host={{ matrix_synapse_database_host }} user={{ matrix_synapse_database_user }} password={{ matrix_synapse_database_password }} dbname={{ matrix_synapse_database_database }}" @@ -30,7 +30,7 @@ --cap-drop=ALL --network={{ matrix_docker_network }} --env-file={{ matrix_postgres_base_path }}/env-postgres-psql - -v {{ matrix_synapse_rust_synapse_compress_state_base_path }}:/work:ro + --mount type=bind,src={{ matrix_synapse_rust_synapse_compress_state_base_path }},dst=/work,ro --entrypoint=/bin/sh {{ matrix_postgres_docker_image_latest }} -c "cat /work/state-compressor.sql | From 58baa0ac30ebe4759bed23433f0c1aee9cbc57db Mon Sep 17 00:00:00 2001 From: benkuly <12199167+benkuly@users.noreply.github.com> Date: Wed, 25 Nov 2020 15:39:14 +0100 Subject: [PATCH 070/217] Update configuring-playbook-bridge-matrix-bridge-sms.md --- docs/configuring-playbook-bridge-matrix-bridge-sms.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuring-playbook-bridge-matrix-bridge-sms.md b/docs/configuring-playbook-bridge-matrix-bridge-sms.md index 3dd7398b..86a95ab2 100644 --- a/docs/configuring-playbook-bridge-matrix-bridge-sms.md +++ b/docs/configuring-playbook-bridge-matrix-bridge-sms.md @@ -4,7 +4,7 @@ The playbook can install and configure [matrix-sms-bridge](https://github.com/be See the project page to learn what it does and why it might be useful to you. -** The bridge uses [android-sms-gateway-server](https://github.com/RebekkaMa/android-sms-gateway-server). You need to configure it first. ** +**The bridge uses [android-sms-gateway-server](https://github.com/RebekkaMa/android-sms-gateway-server). You need to configure it first.** To enable the bridge just use the following playbook configuration: From 732f57cbd915aa486522829f432b4b60f37fe49f Mon Sep 17 00:00:00 2001 From: anonym Date: Thu, 26 Nov 2020 01:24:43 +0100 Subject: [PATCH 071/217] Note about the max length of the postgres password --- group_vars/matrix_servers | 1 + 1 file changed, 1 insertion(+) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 742391c3..c3d42d74 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -874,6 +874,7 @@ matrix_postgres_enabled: true matrix_postgres_connection_hostname: "matrix-postgres" matrix_postgres_connection_username: "synapse" +# Please note that max length of the password is 99 characters matrix_postgres_connection_password: "synapse-password" matrix_postgres_db_name: "homeserver" From d4ef751052380632f491570c182fd5829d48a483 Mon Sep 17 00:00:00 2001 From: anonym Date: Thu, 26 Nov 2020 01:30:08 +0100 Subject: [PATCH 072/217] Update matrix_servers --- group_vars/matrix_servers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index c3d42d74..cdd01153 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -874,7 +874,7 @@ matrix_postgres_enabled: true matrix_postgres_connection_hostname: "matrix-postgres" matrix_postgres_connection_username: "synapse" -# Please note that max length of the password is 99 characters +# Please note that the max length of the password is 99 characters matrix_postgres_connection_password: "synapse-password" matrix_postgres_db_name: "homeserver" From 796c752b6028b0883011c757312f3e239fa3dd70 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 26 Nov 2020 09:51:48 +0200 Subject: [PATCH 073/217] Ensure Postgres passwords are not longer than 99 characters Complements https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/732 --- roles/matrix-postgres/tasks/validate_config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/matrix-postgres/tasks/validate_config.yml b/roles/matrix-postgres/tasks/validate_config.yml index 4985a4c7..6ff5adb0 100644 --- a/roles/matrix-postgres/tasks/validate_config.yml +++ b/roles/matrix-postgres/tasks/validate_config.yml @@ -20,3 +20,8 @@ - "matrix_postgres_connection_username" - "matrix_postgres_connection_password" - "matrix_postgres_db_name" + +- name: Fail if Postgres password length exceeded + fail: + msg: "The maximum `matrix_postgres_connection_password` length is 99 characters" + when: "matrix_postgres_connection_hostname|length > 99" From 12867e9f18a3321217083bfbcdfc9a79de0de9ba Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 26 Nov 2020 09:55:26 +0200 Subject: [PATCH 074/217] Do not try to mount /matrix/ssl when matrix_ssl_retrieval_method is 'none' Since the switch from `-v` to `--mount` (in 1fca917ad1310), we've regressed when `matrix_ssl_retrieval_method == 'none'`. In such a case, we don't create `/matrix/ssl` directories at all and shouldn't be trying to mount them into the `matrix-nginx-proxy` container. Previously, with `-v`, Docker would auto-create them, effectively hiding our mistake. Now that `--mount` doesn't do such auto-creation magic, the `matrix-nginx-proxy` container was failing to start. Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/734 --- .../templates/systemd/matrix-nginx-proxy.service.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2 b/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2 index e2182d28..6e8d31f4 100644 --- a/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2 +++ b/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2 @@ -33,8 +33,10 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-nginx-proxy \ --mount type=bind,src={{ matrix_nginx_proxy_base_path }}/nginx.conf,dst=/etc/nginx/nginx.conf,ro \ --mount type=bind,src={{ matrix_nginx_proxy_data_path }},dst=/nginx-data,ro \ --mount type=bind,src={{ matrix_nginx_proxy_confd_path }},dst=/etc/nginx/conf.d,ro \ + {% if matrix_ssl_retrieval_method != 'none' %} --mount type=bind,src={{ matrix_ssl_config_dir_path }},dst={{ matrix_ssl_config_dir_path }},ro \ --mount type=bind,src={{ matrix_static_files_base_path }},dst={{ matrix_static_files_base_path }},ro \ + {% endif %} {% for volume in matrix_nginx_proxy_container_additional_volumes %} -v {{ volume.src }}:{{ volume.dst }}:{{ volume.options }} \ {% endfor %} From d2353355077891e299c25b08da71faf8e69fa872 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 26 Nov 2020 10:08:52 +0200 Subject: [PATCH 075/217] Do not mount /matrix/ssl into matrix-coturn when matrix_ssl_retrieval_method is 'none' Related to https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/734 Similar to 12867e9f18a3321217083, but for `matrix-coturn` (not `matrix-nginx-proxy`). --- group_vars/matrix_servers | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index cdd01153..464cff78 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -578,13 +578,19 @@ matrix_coturn_container_image_self_build: "{{ matrix_architecture != 'amd64'}}" matrix_coturn_turn_external_ip_address: "{{ ansible_host }}" -matrix_coturn_tls_enabled: true +matrix_coturn_tls_enabled: "{{ matrix_ssl_retrieval_method != 'none' }}" matrix_coturn_tls_cert_path: "{{ matrix_ssl_config_dir_path }}/live/{{ matrix_server_fqn_matrix }}/fullchain.pem" matrix_coturn_tls_key_path: "{{ matrix_ssl_config_dir_path }}/live/{{ matrix_server_fqn_matrix }}/privkey.pem" -matrix_coturn_container_additional_volumes: - - src: "{{ matrix_ssl_config_dir_path }}" - dst: "{{ matrix_ssl_config_dir_path }}" - options: ro +matrix_coturn_container_additional_volumes: | + {{ + ([] if matrix_ssl_retrieval_method == 'none' else [ + { + 'src': matrix_ssl_config_dir_path, + 'dst': matrix_ssl_config_dir_path, + 'options': 'ro', + } + ]) + }} ###################################################################### # From d702e74079f917e1c017ac1476a13aaac6b3847e Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 26 Nov 2020 18:40:15 +0200 Subject: [PATCH 076/217] Fix matrix-nginx-proxy static files mounting when SSL retrieval is none Fixup for 12867e9f18a33212. This shouldn't have been caught in the `if`. Related to https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/734 --- .../templates/systemd/matrix-nginx-proxy.service.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2 b/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2 index 6e8d31f4..0945c25e 100644 --- a/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2 +++ b/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2 @@ -35,8 +35,8 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-nginx-proxy \ --mount type=bind,src={{ matrix_nginx_proxy_confd_path }},dst=/etc/nginx/conf.d,ro \ {% if matrix_ssl_retrieval_method != 'none' %} --mount type=bind,src={{ matrix_ssl_config_dir_path }},dst={{ matrix_ssl_config_dir_path }},ro \ - --mount type=bind,src={{ matrix_static_files_base_path }},dst={{ matrix_static_files_base_path }},ro \ {% endif %} + --mount type=bind,src={{ matrix_static_files_base_path }},dst={{ matrix_static_files_base_path }},ro \ {% for volume in matrix_nginx_proxy_container_additional_volumes %} -v {{ volume.src }}:{{ volume.dst }}:{{ volume.options }} \ {% endfor %} From 84c7c190682763e2e4994becbd9554bfda87da0a Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 27 Nov 2020 13:30:00 +0200 Subject: [PATCH 077/217] Add jitsi.DOMAIN to sample Traefik 2.0 config Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/735 --- docs/configuring-playbook-own-webserver.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/configuring-playbook-own-webserver.md b/docs/configuring-playbook-own-webserver.md index dac78376..8c671df4 100644 --- a/docs/configuring-playbook-own-webserver.md +++ b/docs/configuring-playbook-own-webserver.md @@ -113,7 +113,7 @@ With this, nginx would still be in use, but it would not bother with anything SS All services would be served locally on `127.0.0.1:81` and `127.0.0.1:8449` (as per the example configuration above). You can then set up another reverse-proxy server on ports 80/443/8448 for all of the expected domains and make traffic go to these local ports. -The expected domains vary depending on the services you have enabled (`matrix.DOMAIN` for sure; `element.DOMAIN` and `dimension.DOMAIN` are optional). +The expected domains vary depending on the services you have enabled (`matrix.DOMAIN` for sure; `element.DOMAIN`, `dimension.DOMAIN` and `jitsi.DOMAIN` are optional). ### Sample configuration for running behind Traefik 2.0 @@ -144,7 +144,7 @@ matrix_nginx_proxy_container_extra_arguments: - '--label "traefik.enable=true"' # The Nginx proxy container will receive traffic from these subdomains - - '--label "traefik.http.routers.matrix-nginx-proxy.rule=Host(`{{ matrix_server_fqn_matrix }}`,`{{ matrix_server_fqn_element }}`,`{{ matrix_server_fqn_dimension }}`)"' + - '--label "traefik.http.routers.matrix-nginx-proxy.rule=Host(`{{ matrix_server_fqn_matrix }}`,`{{ matrix_server_fqn_element }}`,`{{ matrix_server_fqn_dimension }},`{{ matrix_server_fqn_jitsi }}`)"' # (The 'web-secure' entrypoint must bind to port 443 in Traefik config) - '--label "traefik.http.routers.matrix-nginx-proxy.entrypoints=web-secure"' @@ -172,7 +172,7 @@ matrix_synapse_container_extra_arguments: - '--label "traefik.http.services.matrix-synapse.loadbalancer.server.port=8048"' ``` -This method uses labels attached to the Nginx and Synapse containers to provide the Traefik Docker provider with the information it needs to proxy `matrix.DOMAIN`, `element.DOMAIN`, and `dimension.DOMAIN`. Some [static configuration](https://docs.traefik.io/v2.0/reference/static-configuration/file/) is required in Traefik; namely, having endpoints on ports 443 and 8448 and having a certificate resolver. +This method uses labels attached to the Nginx and Synapse containers to provide the Traefik Docker provider with the information it needs to proxy `matrix.DOMAIN`, `element.DOMAIN`, `dimension.DOMAIN` and `jitsi.DOMAIN`. Some [static configuration](https://docs.traefik.io/v2.0/reference/static-configuration/file/) is required in Traefik; namely, having endpoints on ports 443 and 8448 and having a certificate resolver. Note that this configuration on its own does **not** redirect traffic on port 80 (plain HTTP) to port 443 for HTTPS, which may cause some issues, since the built-in Nginx proxy usually does this. If you are not already doing this in Traefik, it can be added to Traefik in a [file provider](https://docs.traefik.io/v2.0/providers/file/) as follows: @@ -205,7 +205,7 @@ services: image: "traefik:v2.3" restart: always container_name: "traefik" - networks: + networks: - traefik command: - "--api.insecure=true" From f93a4f6474ede88cf9aee7c35dc25ab527c9798d Mon Sep 17 00:00:00 2001 From: benkuly <12199167+benkuly@users.noreply.github.com> Date: Fri, 27 Nov 2020 16:01:24 +0100 Subject: [PATCH 078/217] updated matrix-sms-bridge --- roles/matrix-bridge-sms/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-sms/defaults/main.yml b/roles/matrix-bridge-sms/defaults/main.yml index 9f1c1d9c..82f9b841 100644 --- a/roles/matrix-bridge-sms/defaults/main.yml +++ b/roles/matrix-bridge-sms/defaults/main.yml @@ -3,7 +3,7 @@ matrix_sms_bridge_enabled: true -matrix_sms_bridge_docker_image: "docker.io/folivonet/matrix-sms-bridge:0.5.2" +matrix_sms_bridge_docker_image: "docker.io/folivonet/matrix-sms-bridge:0.5.4" matrix_sms_bridge_base_path: "{{ matrix_base_data_path }}/matrix-sms-bridge" matrix_sms_bridge_config_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/config" From fa76128fd84727f00fc6925afe70fb6f2d0fe97e Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 27 Nov 2020 17:28:24 +0200 Subject: [PATCH 079/217] Update Jitsi to build 5142 This supersedes/fixes-up this Pull Request: https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/719 The Jitsi Web and JVB containers now (in build 5142) always start by bulding their own default configuration (`config.js` and `sip-communicator.properties`, respectively). The fact that we were generating these files ourselves was no longer of use, because our configuration was thrown away in favor of the one created by the containers on startup. With this commit, we're completely redoing things. We no longer generate these configuration files. We try to pass the proper environment variables, so that Jitsi services can generate the configuration files themselves. Besides that, we try to use the "custom configuration" mechanism provided by Jitsi Web and Jitsi JVB (`custom-config.js` and `custom-sip-communicator.properties`, respectively), so that we and our users can inject additional configuration. Some configuration options we had are gone now. Others are no longer controllable via variables and need to be injected using the `_config_extension` variables that we provide. The validation logic that is part of the role should take care to inform people about how to upgrade (if they're using some custom configuration, which needs special care now). Most users should not have to do anything special though. --- docs/configuring-playbook-jitsi.md | 51 +- roles/matrix-jitsi/defaults/main.yml | 114 +++- roles/matrix-jitsi/tasks/setup_jitsi_jvb.yml | 2 +- roles/matrix-jitsi/tasks/setup_jitsi_web.yml | 2 +- roles/matrix-jitsi/tasks/validate_config.yml | 21 + .../jvb/custom-sip-communicator.properties.j2 | 7 + roles/matrix-jitsi/templates/jvb/env.j2 | 19 + .../jvb/sip-communicator.properties.j2 | 19 - roles/matrix-jitsi/templates/web/config.js.j2 | 491 ------------------ .../templates/web/custom-config.js.j2 | 15 + roles/matrix-jitsi/templates/web/env.j2 | 12 + .../templates/web/interface_config.js.j2 | 377 ++++++++------ 12 files changed, 408 insertions(+), 722 deletions(-) create mode 100644 roles/matrix-jitsi/templates/jvb/custom-sip-communicator.properties.j2 delete mode 100644 roles/matrix-jitsi/templates/jvb/sip-communicator.properties.j2 delete mode 100644 roles/matrix-jitsi/templates/web/config.js.j2 create mode 100644 roles/matrix-jitsi/templates/web/custom-config.js.j2 diff --git a/docs/configuring-playbook-jitsi.md b/docs/configuring-playbook-jitsi.md index 72402360..62edc578 100644 --- a/docs/configuring-playbook-jitsi.md +++ b/docs/configuring-playbook-jitsi.md @@ -91,44 +91,33 @@ matrix_jitsi_jvb_container_extra_arguments: ## (Optional) Fine tune Jitsi -You may want to suspend unused video layers until they are requested again, to save up resources on both server and clients. +Sample **additional** `inventory/host_vars/matrix.DOMAIN/vars.yml` configuration to save up resources (explained below): + +```yaml +matrix_jitsi_web_custom_config_extension: | + config.enableLayerSuspension = true; + + config.disableAudioLevels = true; + + # Limit the number of video feeds forwarded to each client + config.channelLastN = 4; + +matrix_jitsi_web_config_resolution_width_ideal_and_max: 480 +matrix_jitsi_web_config_resolution_height_ideal_and_max: 240 +``` + +You may want to **suspend unused video layers** until they are requested again, to save up resources on both server and clients. Read more on this feature [here](https://jitsi.org/blog/new-off-stage-layer-suppression-feature/) For this add this line to your `inventory/host_vars/matrix.DOMAIN/vars.yml` configuration: -```yaml -matrix_jitsi_web_config_enableLayerSuspension: true -``` +You may wish to **disable audio levels** to avoid excessive refresh of the client-side page and decrease the CPU consumption involved. -You may wish to disable audio levels to avoid excessive refresh of the client-side page and decrease the CPU consumption involved. -For this add this line to your `inventory/host_vars/matrix.DOMAIN/vars.yml` configuration: - -```yaml -matrix_jitsi_web_config_disableAudioLevels: true -``` - -You may want to limit the number of video feeds forwarded to each client, to save up resources on both server and clients. As clients’ bandwidth and CPU may not bear the load, use this setting to avoid lag and crashes. +You may want to **limit the number of video feeds forwarded to each client**, to save up resources on both server and clients. As clients’ bandwidth and CPU may not bear the load, use this setting to avoid lag and crashes. This feature is found by default in other webconference applications such as Office 365 Teams (limit is set to 4). -Read how it works [here](https://github.com/jitsi/jitsi-videobridge/blob/master/doc/last-n.md) and performance evaluation on this [study](https://jitsi.org/wp-content/uploads/2016/12/nossdav2015lastn.pdf) -For this add this line to your `inventory/host_vars/matrix.DOMAIN/vars.yml` configuration: +Read how it works [here](https://github.com/jitsi/jitsi-videobridge/blob/master/doc/last-n.md) and performance evaluation on this [study](https://jitsi.org/wp-content/uploads/2016/12/nossdav2015lastn.pdf). -```yaml -matrix_jitsi_web_config_channelLastN: 4 -``` +You may want to **limit the maximum video resolution**, to save up resources on both server and clients. -To enable the variables that allow you to manage the video configuration you must add the following line to your `inventory/host_vars/matrix.DOMAIN/vars.yml` configuration: - -```yaml -matrix_jitsi_web_config_constraints_enabled: true -``` - -You may want to limit the maximum video resolution, to save up resources on both server and clients. -For example, to set resolution to 480. -For this add this two lines to your `inventory/host_vars/matrix.DOMAIN/vars.yml` configuration: - -```yaml -matrix_jitsi_web_config_constraints_video_height_ideal: 480 -matrix_jitsi_web_config_constraints_video_height_max: 480 -``` ## Apply changes diff --git a/roles/matrix-jitsi/defaults/main.yml b/roles/matrix-jitsi/defaults/main.yml index 865424ed..c9c8745e 100644 --- a/roles/matrix-jitsi/defaults/main.yml +++ b/roles/matrix-jitsi/defaults/main.yml @@ -6,6 +6,7 @@ matrix_jitsi_enable_auth: false matrix_jitsi_enable_guests: false matrix_jitsi_enable_recording: true matrix_jitsi_enable_transcriptions: true +matrix_jitsi_enable_p2p: true # Authentication type, must be one of internal, jwt or ldap. Currently only # internal and ldap are supported by this playbook. @@ -51,7 +52,7 @@ matrix_jitsi_jibri_recorder_password: '' matrix_jitsi_enable_lobby: false -matrix_jitsi_container_image_tag: "stable-4857" +matrix_jitsi_container_image_tag: "stable-5142" matrix_jitsi_web_docker_image: "docker.io/jitsi/web:{{ matrix_jitsi_container_image_tag }}" matrix_jitsi_web_docker_image_force_pull: "{{ matrix_jitsi_web_docker_image.endswith(':latest') }}" @@ -77,42 +78,86 @@ matrix_jitsi_web_container_extra_arguments: [] # List of systemd services that matrix-jitsi-web.service depends on matrix_jitsi_web_systemd_required_services_list: ['docker.service'] -matrix_jitsi_web_config_defaultLanguage: 'en' -matrix_jitsi_web_config_start_with_audio_muted: false -matrix_jitsi_web_config_start_with_video_muted: false -matrix_jitsi_web_config_testing_enableFirefoxSimulcast: false -matrix_jitsi_web_config_testing_p2pTestMode: false +# Some variables controlling the interface of Jitsi Web. +# These get applied to `templates/web/interface_config.js.j2`. +# +# Besides this, you can also use `matrix_jitsi_web_custom_interface_config_extension` +# to define any other configuration option. matrix_jitsi_web_interface_config_lang_detection: false matrix_jitsi_web_interface_config_show_jitsi_watermark: true matrix_jitsi_web_interface_config_jitsi_watermark_link: "https://jitsi.org" matrix_jitsi_web_interface_config_show_brand_watermark: false matrix_jitsi_web_interface_config_brand_watermark_link: "" -matrix_jitsi_web_interface_config_show_watermark_for_guests: true matrix_jitsi_web_interface_config_generate_room_names_on_welcome_page: true matrix_jitsi_web_interface_config_display_welcome_page_content: true matrix_jitsi_web_interface_config_app_name: "Jitsi Meet" matrix_jitsi_web_interface_config_native_app_name: "Jitsi Meet" matrix_jitsi_web_interface_config_provider_name: "Jitsi" -matrix_jitsi_web_interface_config_invitation_powered_by: true matrix_jitsi_web_interface_config_show_powered_by: false matrix_jitsi_web_interface_config_disable_transcription_subtitles: false matrix_jisti_web_interface_config_show_deep_linking_image: false -# Jitsi_web Fine Tune default values. -# Useful to manage bandwidth and CPU consumption in server and client side -matrix_jitsi_web_config_disableAudioLevels: false -matrix_jitsi_web_config_enableLayerSuspension: false -matrix_jitsi_web_config_channelLastN: -1 -# If 'matrix_jitsi_web_config_constraints_enabled: false' -# the video constraints will be disabled and will take the default values of jitsi -matrix_jitsi_web_config_constraints_enabled: false -# This settings work if matrix_jitsi_web_config_constraints_enabled: true -# See their definitions in config.js.j2 (templates / web) -matrix_jitsi_web_config_constraints_video_aspectRatio: 16 / 9 -matrix_jitsi_web_config_constraints_video_height_ideal: 720 -matrix_jitsi_web_config_constraints_video_height_max: 720 -matrix_jitsi_web_config_constraints_video_height_min: 240 +# Custom configuration to be injected into `interface_config.js`, passed to Jitsi Web. +# This configuration gets appended to the final interface configuration that Jitsi Web uses. +# +# Note: not to be confused with `matrix_jitsi_web_custom_config_extension`. +# +# For interface configuration, the flow is like this: +# - the contents of `templates/web/interface_config.js.j2` is generated (based on various `matrix_jitsi_web_interface_config_*` variables you see in this file) +# - the contents of `matrix_jitsi_web_custom_interface_config_extension` is appended and can define new settings or override defaults. +# +# Example: +# matrix_jitsi_web_custom_interface_config_extension: | +# interfaceConfig.CONNECTION_INDICATOR_AUTO_HIDE_ENABLED = false; +# interfaceConfig.DISABLE_VIDEO_BACKGROUND = true; +matrix_jitsi_web_custom_interface_config_extension: '' + + +# Controls after which participant audio will be muted. If not specified, defaults to Jitsi's default value (likely 10) +matrix_jitsi_web_config_start_audio_muted_after_nth_participant: ~ +# Controls after which participant video will be muted. If not specified, defaults to Jitsi's default value (likely 10) +matrix_jitsi_web_config_start_video_muted_after_nth_participant: ~ + +matrix_jitsi_web_config_defaultLanguage: 'en' + +# Ideal and also maximum resolution width. If not specified, defaults to Jitsi's default value (likely 1280) +matrix_jitsi_web_config_resolution_width_ideal_and_max: ~ +# Minimum resolution width. If not specified, defaults to Jitsi's default value (likely 320) +matrix_jitsi_web_config_resolution_width_min: ~ +# Ideal and also maximum resolution height. If not specified, defaults to Jitsi's default value (likely 720) +matrix_jitsi_web_config_resolution_height_ideal_and_max: ~ +# Minimum resolution height. If not specified, defaults to Jitsi's default value (likely 180) +matrix_jitsi_web_config_resolution_height_min: ~ + +# Custom configuration to be injected into `custom-config.js`, passed to Jitsi Web. +# This configuration gets appended to the final configuration that Jitsi Web uses. +# +# Note: not to be confused with `matrix_jitsi_web_custom_interface_config_extension`. +# +# The flow is like this: +# - some default configuration is automatically generated based on the environment variables passed to the Jitsi Web container +# - the contents of `custom-config.js` is appended to it (see `templates/web/custom-config.js.j2`) +# - said `custom-config.js` contains your custom contents specified in `matrix_jitsi_web_custom_config_extension`. +# +# Example: +# matrix_jitsi_web_custom_config_extension: | +# if (!config.hasOwnProperty('testing')) config.testing = {}; +# config.testing.p2pTestMode = true +matrix_jitsi_web_custom_config_extension: '' + +# Additional environment variables to pass to the Jitsi Web container. +# You can use this to further influence the default configuration generated by the Jitsi Web container on every startup. +# Besides influencing the final configuration by passing environment variables, you can also inject custom configuration +# by using `matrix_jitsi_web_custom_config_extension`. +# +# Example: +# matrix_jitsi_web_environment_variables_extension: | +# ENABLE_FILE_RECORDING_SERVICE=1 +# DROPBOX_APPKEY=something +# DROPBOX_REDIRECT_URI=something +matrix_jitsi_web_environment_variables_extension: '' + matrix_jitsi_prosody_docker_image: "docker.io/jitsi/prosody:{{ matrix_jitsi_container_image_tag }}" matrix_jitsi_prosody_docker_image_force_pull: "{{ matrix_jitsi_prosody_docker_image.endswith(':latest') }}" @@ -168,6 +213,31 @@ matrix_jitsi_jvb_brewery_muc: jvbbrewery matrix_jitsi_jvb_rtp_udp_port: 10000 matrix_jitsi_jvb_rtp_tcp_port: 4443 +# Custom configuration to be injected into `custom-sip-communicator.properties`, passed to Jitsi JVB. +# This configuration gets appended to the final configuration that Jitsi JVB uses. +# +# The flow is like this: +# - some default configuration is automatically generated based on the environment variables passed to the Jitsi JVB container +# - the contents of `custom-sip-communicator.properties` is appended to it (see `templates/jvb/custom-sip-communicator.properties.j2`) +# - said `custom-sip-communicator.properties` contains your custom contents specified in `matrix_jitsi_jvb_custom_config_extension`. +# +# Example: +# matrix_jitsi_jvb_custom_config_extension: | +# org.jitsi.videobridge.xmpp.user.shard.DISABLE_CERTIFICATE_VERIFICATION=false +# org.jitsi.videobridge.ENABLE_STATISTICS=false +matrix_jitsi_jvb_custom_config_extension: '' + +# Additional environment variables to pass to the Jitsi JVB container. +# You can use this to further influence the default configuration generated by the Jitsi JVB container on every startup. +# Besides influencing the final configuration by passing environment variables, you can also inject custom configuration +# by using `matrix_jitsi_jvb_custom_config_extension`. +# +# Example: +# matrix_jitsi_jvb_environment_variables_extension: | +# SOME_VARIABLE=1 +# ANOTHER_VARIABLE=something +matrix_jitsi_jvb_environment_variables_extension: '' + # Controls whether the matrix-jitsi-jvb container exposes its RTP UDP port (udp/10000 in the container). # # Takes an ":" or "" value (e.g. "127.0.0.1:10000"), or empty string to not expose. diff --git a/roles/matrix-jitsi/tasks/setup_jitsi_jvb.yml b/roles/matrix-jitsi/tasks/setup_jitsi_jvb.yml index 09055b59..e4c7f277 100644 --- a/roles/matrix-jitsi/tasks/setup_jitsi_jvb.yml +++ b/roles/matrix-jitsi/tasks/setup_jitsi_jvb.yml @@ -30,7 +30,7 @@ dest: "{{ matrix_jitsi_jvb_config_path }}/{{ item }}" mode: 0644 with_items: - - sip-communicator.properties + - custom-sip-communicator.properties - logging.properties when: matrix_jitsi_enabled|bool diff --git a/roles/matrix-jitsi/tasks/setup_jitsi_web.yml b/roles/matrix-jitsi/tasks/setup_jitsi_web.yml index 6e5d20b0..3dd6f30c 100644 --- a/roles/matrix-jitsi/tasks/setup_jitsi_web.yml +++ b/roles/matrix-jitsi/tasks/setup_jitsi_web.yml @@ -38,7 +38,7 @@ dest: "{{ matrix_jitsi_web_config_path }}/{{ item }}" mode: 0644 with_items: - - config.js + - custom-config.js - interface_config.js when: matrix_jitsi_enabled|bool diff --git a/roles/matrix-jitsi/tasks/validate_config.yml b/roles/matrix-jitsi/tasks/validate_config.yml index ea92c914..bd939d3a 100644 --- a/roles/matrix-jitsi/tasks/validate_config.yml +++ b/roles/matrix-jitsi/tasks/validate_config.yml @@ -19,3 +19,24 @@ - "matrix_jitsi_jicofo_component_secret" - "matrix_jitsi_jicofo_auth_password" - "matrix_jitsi_jvb_auth_password" + +- name: (Deprecation) Catch and report renamed settings + fail: + msg: >- + Your configuration contains a variable, which now has a different name. + Please change your configuration to rename the variable (`{{ item.old }}` -> `{{ item.new }}`). + when: "item.old in vars" + with_items: + - {'old': 'matrix_jitsi_web_config_constraints_enabled', 'new': ''} + - {'old': 'matrix_jitsi_web_config_constraints_video_aspectRatio', 'new': ''} + - {'old': 'matrix_jitsi_web_config_constraints_video_height_ideal', 'new': 'matrix_jitsi_web_config_resolution_height_ideal_and_max'} + - {'old': 'matrix_jitsi_web_config_constraints_video_height_max', 'new': 'matrix_jitsi_web_config_resolution_height_ideal_and_max'} + - {'old': 'matrix_jitsi_web_config_constraints_video_height_min', 'new': 'matrix_jitsi_web_config_resolution_height_min'} + - {'old': 'matrix_jitsi_web_config_disableAudioLevels', 'new': ''} + - {'old': 'matrix_jitsi_web_config_enableLayerSuspension', 'new': ''} + - {'old': 'matrix_jitsi_web_config_channelLastN', 'new': ''} + - {'old': 'matrix_jitsi_web_config_testing_p2pTestMode', 'new': ''} + - {'old': 'matrix_jitsi_web_config_start_with_audio_muted', 'new': ''} + - {'old': 'matrix_jitsi_web_config_start_with_video_muted', 'new': ''} + - {'old': 'matrix_jitsi_web_interface_config_show_watermark_for_guests', 'new': ''} + - {'old': 'matrix_jitsi_web_interface_config_invitation_powered_by', 'new': ''} diff --git a/roles/matrix-jitsi/templates/jvb/custom-sip-communicator.properties.j2 b/roles/matrix-jitsi/templates/jvb/custom-sip-communicator.properties.j2 new file mode 100644 index 00000000..44b6b8c2 --- /dev/null +++ b/roles/matrix-jitsi/templates/jvb/custom-sip-communicator.properties.j2 @@ -0,0 +1,7 @@ +org.jitsi.videobridge.xmpp.user.shard.DISABLE_CERTIFICATE_VERIFICATION=true + +org.jitsi.videobridge.ENABLE_STATISTICS=true +org.jitsi.videobridge.STATISTICS_TRANSPORT=muc +org.jitsi.videobridge.STATISTICS_INTERVAL=5000 + +{{ matrix_jitsi_jvb_custom_config_extension }} diff --git a/roles/matrix-jitsi/templates/jvb/env.j2 b/roles/matrix-jitsi/templates/jvb/env.j2 index 423070db..f7dc9247 100644 --- a/roles/matrix-jitsi/templates/jvb/env.j2 +++ b/roles/matrix-jitsi/templates/jvb/env.j2 @@ -1 +1,20 @@ JVB_AUTH_PASSWORD={{ matrix_jitsi_jvb_auth_password }} +JVB_TCP_PORT={{ matrix_jitsi_jvb_rtp_tcp_port }} +JVB_PORT={{ matrix_jitsi_jvb_rtp_udp_port }} +JVB_AUTH_USER={{ matrix_jitsi_jvb_auth_user }} +JVB_AUTH_PASSWORD={{ matrix_jitsi_jvb_auth_password }} +JVB_BREWERY_MUC={{ matrix_jitsi_jvb_brewery_muc }} + +XMPP_SERVER={{ matrix_jitsi_xmpp_server }} +XMPP_AUTH_DOMAIN={{ matrix_jitsi_xmpp_auth_domain }} +XMPP_INTERNAL_MUC_DOMAIN={{ matrix_jitsi_xmpp_internal_muc_domain }} + +HOSTNAME=matrix-jitsi-jvb + +{% if matrix_jitsi_jvb_stun_servers|length > 0 %} +JVB_STUN_SERVERS={{ matrix_jitsi_jvb_stun_servers|join(',') }} +{% endif %} + +PUBLIC_URL={{ matrix_jitsi_web_public_url }} + +{{ matrix_jitsi_jvb_environment_variables_extension }} diff --git a/roles/matrix-jitsi/templates/jvb/sip-communicator.properties.j2 b/roles/matrix-jitsi/templates/jvb/sip-communicator.properties.j2 deleted file mode 100644 index 173af0b6..00000000 --- a/roles/matrix-jitsi/templates/jvb/sip-communicator.properties.j2 +++ /dev/null @@ -1,19 +0,0 @@ -org.jitsi.videobridge.SINGLE_PORT_HARVESTER_PORT={{ matrix_jitsi_jvb_rtp_udp_port }} -org.jitsi.videobridge.DISABLE_TCP_HARVESTER=false -org.jitsi.videobridge.TCP_HARVESTER_PORT={{ matrix_jitsi_jvb_rtp_tcp_port }} - -{% if matrix_jitsi_jvb_stun_servers|length > 0 %} -org.ice4j.ice.harvest.STUN_MAPPING_HARVESTER_ADDRESSES={{ matrix_jitsi_jvb_stun_servers|join(',') }} -{% endif %} - -org.jitsi.videobridge.xmpp.user.shard.HOSTNAME={{ matrix_jitsi_xmpp_server }} -org.jitsi.videobridge.xmpp.user.shard.DOMAIN={{ matrix_jitsi_xmpp_auth_domain }} -org.jitsi.videobridge.xmpp.user.shard.USERNAME={{ matrix_jitsi_jvb_auth_user }} -org.jitsi.videobridge.xmpp.user.shard.PASSWORD={{ matrix_jitsi_jvb_auth_password }} -org.jitsi.videobridge.xmpp.user.shard.MUC_JIDS={{ matrix_jitsi_jvb_brewery_muc }}@{{ matrix_jitsi_xmpp_internal_muc_domain }} -org.jitsi.videobridge.xmpp.user.shard.MUC_NICKNAME=matrix-jitsi-jvb -org.jitsi.videobridge.xmpp.user.shard.DISABLE_CERTIFICATE_VERIFICATION=true - -org.jitsi.videobridge.ENABLE_STATISTICS=true -org.jitsi.videobridge.STATISTICS_TRANSPORT=muc -org.jitsi.videobridge.STATISTICS_INTERVAL=5000 diff --git a/roles/matrix-jitsi/templates/web/config.js.j2 b/roles/matrix-jitsi/templates/web/config.js.j2 deleted file mode 100644 index ad0e5410..00000000 --- a/roles/matrix-jitsi/templates/web/config.js.j2 +++ /dev/null @@ -1,491 +0,0 @@ -/* eslint-disable no-unused-vars, no-var */ - -var config = { - // Configuration - // - - // Alternative location for the configuration. - // configLocation: './config.json', - - // Custom function which given the URL path should return a room name. - // getroomnode: function (path) { return 'someprefixpossiblybasedonpath'; }, - - - // Connection - // - - hosts: { - // XMPP domain. - domain: '{{ matrix_jitsi_xmpp_domain }}', - - {% if matrix_jitsi_enable_auth %} - {% if matrix_jitsi_enable_guests %} - // When using authentication, domain for guest users. - anonymousdomain: '{{ matrix_jitsi_xmpp_guest_domain }}', - {% endif %} - - // Domain for authenticated users. Defaults to . - authdomain: '{{ matrix_jitsi_xmpp_domain }}', - {% endif %} - - // Jirecon recording component domain. - // jirecon: 'jirecon.{{ matrix_jitsi_xmpp_domain }}', - - // Call control component (Jigasi). - // call_control: 'callcontrol.{{ matrix_jitsi_xmpp_domain }}', - - // Focus component domain. Defaults to focus.. - // focus: 'focus.{{ matrix_jitsi_xmpp_domain }}', - - // XMPP MUC domain. FIXME: use XEP-0030 to discover it. - muc: {{ matrix_jitsi_xmpp_muc_domain|to_json }}, - }, - - // BOSH URL. FIXME: use XEP-0156 to discover it. - bosh: '/http-bind', - - // The name of client node advertised in XEP-0115 'c' stanza - clientNode: 'http://jitsi.org/jitsimeet', - - // The real JID of focus participant - can be overridden here - focusUserJid: {{ matrix_jitsi_jicofo_auth_user|to_json }} + '@' + {{ matrix_jitsi_xmpp_auth_domain|to_json }}, - - - // Testing / experimental features. - // - - testing: { - // Enables experimental simulcast support on Firefox. - enableFirefoxSimulcast: {{ matrix_jitsi_web_config_testing_enableFirefoxSimulcast|to_json }}, - - // P2P test mode disables automatic switching to P2P when there are 2 - // participants in the conference. - p2pTestMode: {{ matrix_jitsi_web_config_testing_p2pTestMode|to_json }} - - // Enables the test specific features consumed by jitsi-meet-torture - // testMode: false - }, - - // Disables ICE/UDP by filtering out local and remote UDP candidates in - // signalling. - // webrtcIceUdpDisable: false, - - // Disables ICE/TCP by filtering out local and remote TCP candidates in - // signalling. - // webrtcIceTcpDisable: false, - - - // Media - // - - // Audio - - // Disable measuring of audio levels. - disableAudioLevels: {{ matrix_jitsi_web_config_disableAudioLevels|to_json }}, - - // Start the conference in audio only mode (no video is being received nor - // sent). - // startAudioOnly: false, - - // Every participant after the Nth will start audio muted. - // startAudioMuted: 10, - - // Start calls with audio muted. Unlike the option above, this one is only - // applied locally. FIXME: having these 2 options is confusing. - // startWithAudioMuted: false, - startWithAudioMuted: {{ matrix_jitsi_web_config_start_with_audio_muted|to_json }}, - - // Enabling it (with #params) will disable local audio output of remote - // participants and to enable it back a reload is needed. - // startSilent: false - - // Video - - // Sets the preferred resolution (height) for local video. Defaults to 720. - // resolution: 720, - - // w3c spec-compliant video constraints to use for video capture. Currently - // used by browsers that return true from lib-jitsi-meet's - // util#browser#usesNewGumFlow. The constraints are independency from - // this config's resolution value. Defaults to requesting an ideal aspect - // ratio of 16:9 with an ideal resolution of 720. - {% if matrix_jitsi_web_config_constraints_enabled %} - constraints: { - video: { - aspectRatio: {{ matrix_jitsi_web_config_constraints_video_aspectRatio }}, - height: { - ideal: {{ matrix_jitsi_web_config_constraints_video_height_ideal|to_json }}, - max: {{ matrix_jitsi_web_config_constraints_video_height_max|to_json }}, - min: {{ matrix_jitsi_web_config_constraints_video_height_min|to_json }} - } - } - }, - {% endif %} - // Enable / disable simulcast support. - // disableSimulcast: false, - - // Enable / disable layer suspension. If enabled, endpoints whose HD - // layers are not in use will be suspended (no longer sent) until they - // are requested again. - enableLayerSuspension: {{ matrix_jitsi_web_config_enableLayerSuspension|to_json }}, - - // Suspend sending video if bandwidth estimation is too low. This may cause - // problems with audio playback. Disabled until these are fixed. - disableSuspendVideo: true, - - // Every participant after the Nth will start video muted. - // startVideoMuted: 10, - - // Start calls with video muted. Unlike the option above, this one is only - // applied locally. FIXME: having these 2 options is confusing. - // startWithVideoMuted: false, - startWithVideoMuted: {{ matrix_jitsi_web_config_start_with_video_muted|to_json }}, - - // If set to true, prefer to use the H.264 video codec (if supported). - // Note that it's not recommended to do this because simulcast is not - // supported when using H.264. For 1-to-1 calls this setting is enabled by - // default and can be toggled in the p2p section. - // preferH264: true, - - // If set to true, disable H.264 video codec by stripping it out of the - // SDP. - // disableH264: false, - - // Desktop sharing - - // The ID of the jidesha extension for Chrome. - desktopSharingChromeExtId: null, - - // Whether desktop sharing should be disabled on Chrome. - // desktopSharingChromeDisabled: false, - - // The media sources to use when using screen sharing with the Chrome - // extension. - desktopSharingChromeSources: [ 'screen', 'window', 'tab' ], - - // Required version of Chrome extension - desktopSharingChromeMinExtVersion: '0.1', - - // Whether desktop sharing should be disabled on Firefox. - // desktopSharingFirefoxDisabled: false, - - // Optional desktop sharing frame rate options. Default value: min:5, max:5. - // desktopSharingFrameRate: { - // min: 5, - // max: 5 - // }, - - // Try to start calls with screen-sharing instead of camera video. - // startScreenSharing: false, - - // Recording -hiddenDomain: {{ matrix_jitsi_recorder_domain|to_json }}, - - // Whether to enable file recording or not. - fileRecordingsEnabled: {{ matrix_jitsi_enable_recording|to_json }}, - // Enable the dropbox integration. - // dropbox: { - // appKey: '' // Specify your app key here. - // // A URL to redirect the user to, after authenticating - // // by default uses: - // // 'https://{{ matrix_jitsi_xmpp_domain }}/static/oauth.html' - // redirectURI: - // 'https://{{ matrix_jitsi_xmpp_domain }}/subfolder/static/oauth.html' - // }, - // When integrations like dropbox are enabled only that will be shown, - // by enabling fileRecordingsServiceEnabled, we show both the integrations - // and the generic recording service (its configuration and storage type - // depends on jibri configuration) - // fileRecordingsServiceEnabled: false, - // Whether to show the possibility to share file recording with other people - // (e.g. meeting participants), based on the actual implementation - // on the backend. - // fileRecordingsServiceSharingEnabled: false, - - // Whether to enable live streaming or not. - liveStreamingEnabled: {{ matrix_jitsi_enable_recording|to_json }}, - - // Transcription (in interface_config, - // subtitles and buttons can be configured) - transcribingEnabled: {{ matrix_jitsi_enable_transcriptions|to_json }}, - - // Misc - - // Default value for the channel "last N" attribute. -1 for unlimited. - channelLastN: {{ matrix_jitsi_web_config_channelLastN|to_json }}, - - // Disables or enables RTX (RFC 4588) (defaults to false). - // disableRtx: false, - - // Disables or enables TCC (the default is in Jicofo and set to true) - // (draft-holmer-rmcat-transport-wide-cc-extensions-01). This setting - // affects congestion control, it practically enables send-side bandwidth - // estimations. - // enableTcc: true, - - // Disables or enables REMB (the default is in Jicofo and set to false) - // (draft-alvestrand-rmcat-remb-03). This setting affects congestion - // control, it practically enables recv-side bandwidth estimations. When - // both TCC and REMB are enabled, TCC takes precedence. When both are - // disabled, then bandwidth estimations are disabled. - // enableRemb: false, - - // Defines the minimum number of participants to start a call (the default - // is set in Jicofo and set to 2). - // minParticipants: 2, - - // Use XEP-0215 to fetch STUN and TURN servers. - // useStunTurn: true, - - // Enable IPv6 support. - // useIPv6: true, - - // Enables / disables a data communication channel with the Videobridge. - // Values can be 'datachannel', 'websocket', true (treat it as - // 'datachannel'), undefined (treat it as 'datachannel') and false (don't - // open any channel). - // openBridgeChannel: true, - - - // UI - // - - // Use display name as XMPP nickname. - // useNicks: false, - - // Require users to always specify a display name. - // requireDisplayName: true, - - // Whether to use a welcome page or not. In case it's false a random room - // will be joined when no room is specified. - enableWelcomePage: true, - - // Enabling the close page will ignore the welcome page redirection when - // a call is hangup. - // enableClosePage: false, - - // Disable hiding of remote thumbnails when in a 1-on-1 conference call. - // disable1On1Mode: false, - - // Default language for the user interface. - defaultLanguage: {{ matrix_jitsi_web_config_defaultLanguage|to_json }}, - - // If true all users without a token will be considered guests and all users - // with token will be considered non-guests. Only guests will be allowed to - // edit their profile. - enableUserRolesBasedOnToken: false, - - // Whether or not some features are checked based on token. - // enableFeaturesBasedOnToken: false, - - // Enable lock room for all moderators, even when userRolesBasedOnToken is enabled and participants are guests. - // lockRoomGuestEnabled: false, - - // When enabled the password used for locking a room is restricted to up to the number of digits specified - // roomPasswordNumberOfDigits: 10, - // default: roomPasswordNumberOfDigits: false, - - // Message to show the users. Example: 'The service will be down for - // maintenance at 01:00 AM GMT, - // noticeMessage: '', - - // Enables calendar integration, depends on googleApiApplicationClientID - // and microsoftApiApplicationClientID - // enableCalendarIntegration: false, - - // Stats - // - - // Whether to enable stats collection or not in the TraceablePeerConnection. - // This can be useful for debugging purposes (post-processing/analysis of - // the webrtc stats) as it is done in the jitsi-meet-torture bandwidth - // estimation tests. - // gatherStats: false, - - // To enable sending statistics to callstats.io you must provide the - // Application ID and Secret. - // callStatsID: '', - // callStatsSecret: '', - - // enables callstatsUsername to be reported as statsId and used - // by callstats as repoted remote id - // enableStatsID: false - - // enables sending participants display name to callstats - // enableDisplayNameInStats: false - - - // Privacy - // - - // If third party requests are disabled, no other server will be contacted. - // This means avatars will be locally generated and callstats integration - // will not function. - // disableThirdPartyRequests: false, - - - // Peer-To-Peer mode: used (if enabled) when there are just 2 participants. - // - - p2p: { - // Enables peer to peer mode. When enabled the system will try to - // establish a direct connection when there are exactly 2 participants - // in the room. If that succeeds the conference will stop sending data - // through the JVB and use the peer to peer connection instead. When a - // 3rd participant joins the conference will be moved back to the JVB - // connection. - enabled: true, - - // Use XEP-0215 to fetch STUN and TURN servers. - // useStunTurn: true, - - // The STUN servers that will be used in the peer to peer connections - {% if matrix_jitsi_web_stun_servers|length > 0 %} - stunServers: [ - {% for url in matrix_jitsi_web_stun_servers %} - { urls: {{ url|to_json }} }{% if not loop.last %},{% endif %} - {% endfor %} - ], - {% endif %} - - // Sets the ICE transport policy for the p2p connection. At the time - // of this writing the list of possible values are 'all' and 'relay', - // but that is subject to change in the future. The enum is defined in - // the WebRTC standard: - // https://www.w3.org/TR/webrtc/#rtcicetransportpolicy-enum. - // If not set, the effective value is 'all'. - // iceTransportPolicy: 'all', - - // If set to true, it will prefer to use H.264 for P2P calls (if H.264 - // is supported). - preferH264: true - - // If set to true, disable H.264 video codec by stripping it out of the - // SDP. - // disableH264: false, - - // How long we're going to wait, before going back to P2P after the 3rd - // participant has left the conference (to filter out page reload). - // backToP2PDelay: 5 - }, - - analytics: { - // The Google Analytics Tracking ID: - // googleAnalyticsTrackingId: 'your-tracking-id-UA-123456-1' - - // The Amplitude APP Key: - // amplitudeAPPKey: '' - - // Array of script URLs to load as lib-jitsi-meet "analytics handlers". - // scriptURLs: [ - // "libs/analytics-ga.min.js", // google-analytics - // "https://example.com/my-custom-analytics.js" - // ], - }, - - // Information about the jitsi-meet instance we are connecting to, including - // the user region as seen by the server. - deploymentInfo: { - // shard: "shard1", - // region: "europe", - // userRegion: "asia" - } - - // Local Recording - // - - // localRecording: { - // Enables local recording. - // Additionally, 'localrecording' (all lowercase) needs to be added to - // TOOLBAR_BUTTONS in interface_config.js for the Local Recording - // button to show up on the toolbar. - // - // enabled: true, - // - - // The recording format, can be one of 'ogg', 'flac' or 'wav'. - // format: 'flac' - // - - // } - - // Options related to end-to-end (participant to participant) ping. - // e2eping: { - // // The interval in milliseconds at which pings will be sent. - // // Defaults to 10000, set to <= 0 to disable. - // pingInterval: 10000, - // - // // The interval in milliseconds at which analytics events - // // with the measured RTT will be sent. Defaults to 60000, set - // // to <= 0 to disable. - // analyticsInterval: 60000, - // } - - // If set, will attempt to use the provided video input device label when - // triggering a screenshare, instead of proceeding through the normal flow - // for obtaining a desktop stream. - // NOTE: This option is experimental and is currently intended for internal - // use only. - // _desktopSharingSourceDevice: 'sample-id-or-label' - - // If true, any checks to handoff to another application will be prevented - // and instead the app will continue to display in the current browser. - // disableDeepLinking: false - - // A property to disable the right click context menu for localVideo - // the menu has option to flip the locally seen video for local presentations - // disableLocalVideoFlip: false - - // List of undocumented settings used in jitsi-meet - /** - _immediateReloadThreshold - autoRecord - autoRecordToken - debug - debugAudioLevels - deploymentInfo - dialInConfCodeUrl - dialInNumbersUrl - dialOutAuthUrl - dialOutCodesUrl - disableRemoteControl - displayJids - etherpad_base - externalConnectUrl - firefox_fake_device - googleApiApplicationClientID - iAmRecorder - iAmSipGateway - microsoftApiApplicationClientID - peopleSearchQueryTypes - peopleSearchUrl - requireDisplayName - tokenAuthUrl - */ - - // List of undocumented settings used in lib-jitsi-meet - /** - _peerConnStatusOutOfLastNTimeout - _peerConnStatusRtcMuteTimeout - abTesting - avgRtpStatsN - callStatsConfIDNamespace - callStatsCustomScriptUrl - desktopSharingSources - disableAEC - disableAGC - disableAP - disableHPF - disableNS - enableLipSync - enableTalkWhileMuted - forceJVB121Ratio - hiddenDomain - ignoreStartMuted - nick - startBitrate - */ - -}; - -/* eslint-enable no-unused-vars, no-var */ diff --git a/roles/matrix-jitsi/templates/web/custom-config.js.j2 b/roles/matrix-jitsi/templates/web/custom-config.js.j2 new file mode 100644 index 00000000..02316ca0 --- /dev/null +++ b/roles/matrix-jitsi/templates/web/custom-config.js.j2 @@ -0,0 +1,15 @@ +config.defaultLanguage = {{ matrix_jitsi_web_config_defaultLanguage|to_json }}; + + +if (!config.hasOwnProperty('p2p')) config.p2p = {% raw %}{}{% endraw %}; + +{% if matrix_jitsi_web_stun_servers|length > 0 %} +config.p2p.stunServers = [ + {% for url in matrix_jitsi_web_stun_servers %} + { urls: {{ url|to_json }} }{% if not loop.last %},{% endif %} + {% endfor %} +]; +{% endif %} + + +{{ matrix_jitsi_web_custom_config_extension }} diff --git a/roles/matrix-jitsi/templates/web/env.j2 b/roles/matrix-jitsi/templates/web/env.j2 index b85e9af5..353a3d14 100644 --- a/roles/matrix-jitsi/templates/web/env.j2 +++ b/roles/matrix-jitsi/templates/web/env.j2 @@ -3,6 +3,8 @@ ENABLE_GUESTS={{ 1 if matrix_jitsi_enable_guests else 0 }} ENABLE_TRANSCRIPTIONS={{ 1 if matrix_jitsi_enable_transcriptions else 0 }} +ENABLE_P2P={{ 1 if matrix_jitsi_enable_p2p else 0 }} + DISABLE_HTTPS=1 JICOFO_AUTH_USER={{ matrix_jitsi_jicofo_auth_user }} @@ -26,3 +28,13 @@ JIBRI_RECORDER_USER={{ matrix_jitsi_jibri_recorder_user }} JIBRI_RECORDER_PASSWORD={{ matrix_jitsi_jibri_recorder_password }} ENABLE_RECORDING={{ 1 if matrix_jitsi_enable_recording else 0 }} + +RESOLUTION={{ matrix_jitsi_web_config_resolution_height_ideal_and_max }} +RESOLUTION_MIN={{ matrix_jitsi_web_config_resolution_height_min }} +RESOLUTION_WIDTH={{ matrix_jitsi_web_config_resolution_width_ideal_and_max }} +RESOLUTION_WIDTH_MIN={{ matrix_jitsi_web_config_resolution_width_min }} + +START_AUDIO_MUTED={{ matrix_jitsi_web_config_start_audio_muted_after_nth_participant }} +START_VIDEO_MUTED={{ matrix_jitsi_web_config_start_video_muted_after_nth_participant }} + +{{ matrix_jitsi_web_environment_variables_extension }} diff --git a/roles/matrix-jitsi/templates/web/interface_config.js.j2 b/roles/matrix-jitsi/templates/web/interface_config.js.j2 index c56f8c8c..60fac4d5 100644 --- a/roles/matrix-jitsi/templates/web/interface_config.js.j2 +++ b/roles/matrix-jitsi/templates/web/interface_config.js.j2 @@ -1,137 +1,23 @@ /* eslint-disable no-unused-vars, no-var, max-len */ +/* eslint sort-keys: ["error", "asc", {"caseSensitive": false}] */ var interfaceConfig = { - // TO FIX: this needs to be handled from SASS variables. There are some - // methods allowing to use variables both in css and js. - DEFAULT_BACKGROUND: '#474747', - - /** - * Whether or not the blurred video background for large video should be - * displayed on browsers that can support it. - */ - DISABLE_VIDEO_BACKGROUND: false, - - INITIAL_TOOLBAR_TIMEOUT: 20000, - TOOLBAR_TIMEOUT: 4000, - TOOLBAR_ALWAYS_VISIBLE: false, - DEFAULT_REMOTE_DISPLAY_NAME: 'Fellow Jitster', - DEFAULT_LOCAL_DISPLAY_NAME: 'me', - SHOW_JITSI_WATERMARK: {{ matrix_jitsi_web_interface_config_show_jitsi_watermark|to_json }}, - JITSI_WATERMARK_LINK: {{ matrix_jitsi_web_interface_config_jitsi_watermark_link|to_json }}, - - // if watermark is disabled by default, it can be shown only for guests - SHOW_WATERMARK_FOR_GUESTS: {{ matrix_jitsi_web_interface_config_show_watermark_for_guests|to_json }}, - SHOW_BRAND_WATERMARK: {{ matrix_jitsi_web_interface_config_show_brand_watermark|to_json }}, - BRAND_WATERMARK_LINK: {{ matrix_jitsi_web_interface_config_brand_watermark_link|to_json }}, - SHOW_POWERED_BY: {{ matrix_jitsi_web_interface_config_show_powered_by|to_json }}, - SHOW_DEEP_LINKING_IMAGE: {{ matrix_jisti_web_interface_config_show_deep_linking_image|to_json }}, - GENERATE_ROOMNAMES_ON_WELCOME_PAGE: {{ matrix_jitsi_web_interface_config_generate_room_names_on_welcome_page|to_json }}, - DISPLAY_WELCOME_PAGE_CONTENT: {{ matrix_jitsi_web_interface_config_display_welcome_page_content|to_json }}, APP_NAME: {{ matrix_jitsi_web_interface_config_app_name|to_json }}, - NATIVE_APP_NAME: {{ matrix_jitsi_web_interface_config_native_app_name|to_json }}, - PROVIDER_NAME: {{ matrix_jitsi_web_interface_config_provider_name|to_json }}, - LANG_DETECTION: {{ matrix_jitsi_web_interface_config_lang_detection|to_json }}, // Allow i18n to detect the system language - INVITATION_POWERED_BY: {{ matrix_jitsi_web_interface_config_invitation_powered_by|to_json }}, - - /** - * If we should show authentication block in profile - */ - AUTHENTICATION_ENABLE: true, - - /** - * The name of the toolbar buttons to display in the toolbar. If present, - * the button will display. Exceptions are "livestreaming" and "recording" - * which also require being a moderator and some values in config.js to be - * enabled. Also, the "profile" button will not display for user's with a - * jwt. - */ - TOOLBAR_BUTTONS: [ - {% if matrix_jitsi_enable_transcriptions %} - 'closedcaptions', - {% endif %} - - 'microphone', 'camera', 'desktop', 'fullscreen', - 'fodeviceselection', 'hangup', 'profile', 'info', 'chat', 'recording', - 'livestreaming', 'etherpad', 'sharedvideo', 'settings', 'raisehand', - 'videoquality', 'filmstrip', 'invite', 'feedback', 'stats', 'shortcuts', - 'tileview', 'videobackgroundblur' - ], - - SETTINGS_SECTIONS: [ 'devices', 'language', 'moderator', 'profile', 'calendar' ], - - // Determines how the video would fit the screen. 'both' would fit the whole - // screen, 'height' would fit the original video height to the height of the - // screen, 'width' would fit the original video width to the width of the - // screen respecting ratio. - VIDEO_LAYOUT_FIT: 'both', - - /** - * Whether to only show the filmstrip (and hide the toolbar). - */ - filmStripOnly: false, - - /** - * Whether to show thumbnails in filmstrip as a column instead of as a row. - */ - VERTICAL_FILMSTRIP: true, - - // A html text to be shown to guests on the close page, false disables it - CLOSE_PAGE_GUEST_HINT: false, - RANDOM_AVATAR_URL_PREFIX: false, - RANDOM_AVATAR_URL_SUFFIX: false, - FILM_STRIP_MAX_HEIGHT: 120, - - // Enables feedback star animation. - ENABLE_FEEDBACK_ANIMATION: false, - DISABLE_FOCUS_INDICATOR: false, - DISABLE_DOMINANT_SPEAKER_INDICATOR: false, - - /** - * Whether the speech to text transcription subtitles panel is disabled. - * If {@code undefined}, defaults to {@code false}. - * - * @type {boolean} - */ - DISABLE_TRANSCRIPTION_SUBTITLES: {{ matrix_jitsi_web_interface_config_disable_transcription_subtitles|to_json }}, - - /** - * Whether the ringing sound in the call/ring overlay is disabled. If - * {@code undefined}, defaults to {@code false}. - * - * @type {boolean} - */ - DISABLE_RINGING: false, AUDIO_LEVEL_PRIMARY_COLOR: 'rgba(255,255,255,0.4)', AUDIO_LEVEL_SECONDARY_COLOR: 'rgba(255,255,255,0.2)', - POLICY_LOGO: null, - LOCAL_THUMBNAIL_RATIO: 16 / 9, // 16:9 - REMOTE_THUMBNAIL_RATIO: 1, // 1:1 - // Documentation reference for the live streaming feature. - LIVE_STREAMING_HELP_LINK: 'https://jitsi.org/live', /** - * Whether the mobile app Jitsi Meet is to be promoted to participants - * attempting to join a conference in a mobile Web browser. If - * {@code undefined}, defaults to {@code true}. + * A UX mode where the last screen share participant is automatically + * pinned. Valid values are the string "remote-only" so remote participants + * get pinned but not local, otherwise any truthy value for all participants, + * and any falsy value to disable the feature. * - * @type {boolean} + * Note: this mode is experimental and subject to breakage. */ - MOBILE_APP_PROMO: true, - - /** - * Maximum coeficient of the ratio of the large video to the visible area - * after the large video is scaled to fit the window. - * - * @type {number} - */ - MAXIMUM_ZOOMING_COEFFICIENT: 1.3, - - /* - * If indicated some of the error dialogs may point to the support URL for - * help. - */ - SUPPORT_URL: 'https://github.com/jitsi/jitsi-meet/issues/new', + AUTO_PIN_LATEST_SCREEN_SHARE: 'remote-only', + BRAND_WATERMARK_LINK: {{ matrix_jitsi_web_interface_config_brand_watermark_link|to_json }}, + CLOSE_PAGE_GUEST_HINT: false, // A html text to be shown to guests on the close page, false disables it /** * Whether the connection indicator icon should hide itself based on * connection strength. If true, the connection indicator will remain @@ -158,6 +44,192 @@ var interfaceConfig = { */ CONNECTION_INDICATOR_DISABLED: false, + DEFAULT_BACKGROUND: '#474747', + DEFAULT_LOCAL_DISPLAY_NAME: 'me', + DEFAULT_LOGO_URL: 'images/watermark.svg', + DEFAULT_REMOTE_DISPLAY_NAME: 'Fellow Jitster', + DEFAULT_WELCOME_PAGE_LOGO_URL: 'images/watermark.svg', + + DISABLE_DOMINANT_SPEAKER_INDICATOR: false, + + DISABLE_FOCUS_INDICATOR: false, + + /** + * If true, notifications regarding joining/leaving are no longer displayed. + */ + DISABLE_JOIN_LEAVE_NOTIFICATIONS: false, + + /** + * If true, presence status: busy, calling, connected etc. is not displayed. + */ + DISABLE_PRESENCE_STATUS: false, + + /** + * Whether the ringing sound in the call/ring overlay is disabled. If + * {@code undefined}, defaults to {@code false}. + * + * @type {boolean} + */ + DISABLE_RINGING: false, + + /** + * Whether the speech to text transcription subtitles panel is disabled. + * If {@code undefined}, defaults to {@code false}. + * + * @type {boolean} + */ + DISABLE_TRANSCRIPTION_SUBTITLES: {{ matrix_jitsi_web_interface_config_disable_transcription_subtitles|to_json }}, + + /** + * Whether or not the blurred video background for large video should be + * displayed on browsers that can support it. + */ + DISABLE_VIDEO_BACKGROUND: false, + + DISPLAY_WELCOME_FOOTER: true, + DISPLAY_WELCOME_PAGE_ADDITIONAL_CARD: false, + DISPLAY_WELCOME_PAGE_CONTENT: {{ matrix_jitsi_web_interface_config_display_welcome_page_content|to_json }}, + DISPLAY_WELCOME_PAGE_TOOLBAR_ADDITIONAL_CONTENT: false, + + ENABLE_DIAL_OUT: true, + + ENABLE_FEEDBACK_ANIMATION: false, // Enables feedback star animation. + + FILM_STRIP_MAX_HEIGHT: 120, + + GENERATE_ROOMNAMES_ON_WELCOME_PAGE: {{ matrix_jitsi_web_interface_config_generate_room_names_on_welcome_page|to_json }}, + + /** + * Hide the logo on the deep linking pages. + */ + HIDE_DEEP_LINKING_LOGO: false, + + /** + * Hide the invite prompt in the header when alone in the meeting. + */ + HIDE_INVITE_MORE_HEADER: false, + + INITIAL_TOOLBAR_TIMEOUT: 20000, + JITSI_WATERMARK_LINK: {{ matrix_jitsi_web_interface_config_jitsi_watermark_link|to_json }}, + + LANG_DETECTION: {{ matrix_jitsi_web_interface_config_lang_detection|to_json }}, // Allow i18n to detect the system language + LIVE_STREAMING_HELP_LINK: 'https://jitsi.org/live', // Documentation reference for the live streaming feature. + LOCAL_THUMBNAIL_RATIO: 16 / 9, // 16:9 + + /** + * Maximum coefficient of the ratio of the large video to the visible area + * after the large video is scaled to fit the window. + * + * @type {number} + */ + MAXIMUM_ZOOMING_COEFFICIENT: 1.3, + + /** + * Whether the mobile app Jitsi Meet is to be promoted to participants + * attempting to join a conference in a mobile Web browser. If + * {@code undefined}, defaults to {@code true}. + * + * @type {boolean} + */ + MOBILE_APP_PROMO: true, + + /** + * Specify custom URL for downloading android mobile app. + */ + MOBILE_DOWNLOAD_LINK_ANDROID: 'https://play.google.com/store/apps/details?id=org.jitsi.meet', + + /** + * Specify custom URL for downloading f droid app. + */ + MOBILE_DOWNLOAD_LINK_F_DROID: 'https://f-droid.org/en/packages/org.jitsi.meet/', + + /** + * Specify URL for downloading ios mobile app. + */ + MOBILE_DOWNLOAD_LINK_IOS: 'https://itunes.apple.com/us/app/jitsi-meet/id1165103905', + + NATIVE_APP_NAME: {{ matrix_jitsi_web_interface_config_native_app_name|to_json }}, + + // Names of browsers which should show a warning stating the current browser + // has a suboptimal experience. Browsers which are not listed as optimal or + // unsupported are considered suboptimal. Valid values are: + // chrome, chromium, edge, electron, firefox, nwjs, opera, safari + OPTIMAL_BROWSERS: [ 'chrome', 'chromium', 'firefox', 'nwjs', 'electron', 'safari' ], + + POLICY_LOGO: null, + PROVIDER_NAME: {{ matrix_jitsi_web_interface_config_provider_name|to_json }}, + + /** + * If true, will display recent list + * + * @type {boolean} + */ + RECENT_LIST_ENABLED: true, + REMOTE_THUMBNAIL_RATIO: 1, // 1:1 + + SETTINGS_SECTIONS: [ 'devices', 'language', 'moderator', 'profile', 'calendar' ], + SHOW_BRAND_WATERMARK: {{ matrix_jitsi_web_interface_config_show_brand_watermark|to_json }}, + + /** + * Decides whether the chrome extension banner should be rendered on the landing page and during the meeting. + * If this is set to false, the banner will not be rendered at all. If set to true, the check for extension(s) + * being already installed is done before rendering. + */ + SHOW_CHROME_EXTENSION_BANNER: false, + + SHOW_DEEP_LINKING_IMAGE: {{ matrix_jisti_web_interface_config_show_deep_linking_image|to_json }}, + SHOW_JITSI_WATERMARK: {{ matrix_jitsi_web_interface_config_show_jitsi_watermark|to_json }}, + SHOW_POWERED_BY: {{ matrix_jitsi_web_interface_config_show_powered_by|to_json }}, + SHOW_PROMOTIONAL_CLOSE_PAGE: false, + + /* + * If indicated some of the error dialogs may point to the support URL for + * help. + */ + SUPPORT_URL: 'https://community.jitsi.org/', + + TOOLBAR_ALWAYS_VISIBLE: false, + + /** + * The name of the toolbar buttons to display in the toolbar, including the + * "More actions" menu. If present, the button will display. Exceptions are + * "livestreaming" and "recording" which also require being a moderator and + * some values in config.js to be enabled. Also, the "profile" button will + * not display for users with a JWT. + * Notes: + * - it's impossible to choose which buttons go in the "More actions" menu + * - it's impossible to control the placement of buttons + * - 'desktop' controls the "Share your screen" button + */ + TOOLBAR_BUTTONS: [ + {% if matrix_jitsi_enable_transcriptions %} + 'closedcaptions', + {% endif %} + + 'microphone', 'camera', 'desktop', 'embedmeeting', 'fullscreen', + 'fodeviceselection', 'hangup', 'profile', 'chat', 'recording', + 'livestreaming', 'etherpad', 'sharedvideo', 'settings', 'raisehand', + 'videoquality', 'filmstrip', 'invite', 'feedback', 'stats', 'shortcuts', + 'tileview', 'videobackgroundblur', 'download', 'help', 'mute-everyone', 'security' + ], + + TOOLBAR_TIMEOUT: 4000, + + // Browsers, in addition to those which do not fully support WebRTC, that + // are not supported and should show the unsupported browser page. + UNSUPPORTED_BROWSERS: [], + + /** + * Whether to show thumbnails in filmstrip as a column instead of as a row. + */ + VERTICAL_FILMSTRIP: true, + + // Determines how the video would fit the screen. 'both' would fit the whole + // screen, 'height' would fit the original video height to the height of the + // screen, 'width' would fit the original video width to the width of the + // screen respecting ratio. + VIDEO_LAYOUT_FIT: 'both', + /** * If true, hides the video quality label indicating the resolution status * of the current large video. @@ -166,33 +238,6 @@ var interfaceConfig = { */ VIDEO_QUALITY_LABEL_DISABLED: false, - /** - * If true, will display recent list - * - * @type {boolean} - */ - RECENT_LIST_ENABLED: true, - - // Names of browsers which should show a warning stating the current browser - // has a suboptimal experience. Browsers which are not listed as optimal or - // unsupported are considered suboptimal. Valid values are: - // chrome, chromium, edge, electron, firefox, nwjs, opera, safari - OPTIMAL_BROWSERS: [ 'chrome', 'chromium', 'firefox', 'nwjs', 'electron' ], - - // Browsers, in addition to those which do not fully support WebRTC, that - // are not supported and should show the unsupported browser page. - UNSUPPORTED_BROWSERS: [], - - /** - * A UX mode where the last screen share participant is automatically - * pinned. Valid values are the string "remote-only" so remote participants - * get pinned but not local, otherwise any truthy value for all participants, - * and any falsy value to disable the feature. - * - * Note: this mode is experimental and subject to breakage. - */ - AUTO_PIN_LATEST_SCREEN_SHARE: 'remote-only' - /** * How many columns the tile view can expand to. The respected range is * between 1 and 5. @@ -200,14 +245,15 @@ var interfaceConfig = { // TILE_VIEW_MAX_COLUMNS: 5, /** - * Specify custom URL for downloading android mobile app. + * Specify Firebase dynamic link properties for the mobile apps. */ - // MOBILE_DOWNLOAD_LINK_ANDROID: 'https://play.google.com/store/apps/details?id=org.jitsi.meet', - - /** - * Specify URL for downloading ios mobile app. - */ - // MOBILE_DOWNLOAD_LINK_IOS: 'https://itunes.apple.com/us/app/jitsi-meet/id1165103905', + // MOBILE_DYNAMIC_LINK: { + // APN: 'org.jitsi.meet', + // APP_CODE: 'w2atb', + // CUSTOM_DOMAIN: undefined, + // IBI: 'com.atlassian.JitsiMeet.ios', + // ISI: '1165103905' + // }, /** * Specify mobile app scheme for opening the app from the mobile browser. @@ -225,6 +271,23 @@ var interfaceConfig = { * milliseconds, those notifications should remain displayed. */ // ENFORCE_NOTIFICATION_AUTO_DISMISS_TIMEOUT: 15000, + + // List of undocumented settings + /** + INDICATOR_FONT_SIZES + PHONE_NUMBER_REGEX + */ + + // Allow all above example options to include a trailing comma and + // prevent fear when commenting out the last value. + // eslint-disable-next-line sort-keys + makeJsonParserHappy: 'even if last key had a trailing comma' + + // No configuration value should follow this line. }; + +{{ matrix_jitsi_web_custom_interface_config_extension }} + + /* eslint-enable no-unused-vars, no-var, max-len */ From b354155d7cbca855dd979ae941668154f10afe18 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 27 Nov 2020 17:57:07 +0200 Subject: [PATCH 080/217] Make JVB websockets reverse-proxying work --- group_vars/matrix_servers | 2 ++ roles/matrix-jitsi/defaults/main.yml | 5 +++++ .../templates/jvb/matrix-jitsi-jvb.service.j2 | 3 +++ .../nginx/conf.d/matrix-jitsi.conf.j2 | 21 +++++++++++++++++++ 4 files changed, 31 insertions(+) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 464cff78..07f9a88c 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -671,6 +671,8 @@ matrix_jitsi_enabled: false # the Jitsi HTTP port to the local host. matrix_jitsi_web_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else '127.0.0.1:12080' }}" +matrix_jitsi_jvb_container_colibri_ws_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else '127.0.0.1:12090' }}" + matrix_jitsi_jibri_xmpp_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'jibri') | to_uuid }}" matrix_jitsi_jicofo_auth_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'jicofo') | to_uuid }}" matrix_jitsi_jvb_auth_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'jvb') | to_uuid }}" diff --git a/roles/matrix-jitsi/defaults/main.yml b/roles/matrix-jitsi/defaults/main.yml index c9c8745e..85123cf7 100644 --- a/roles/matrix-jitsi/defaults/main.yml +++ b/roles/matrix-jitsi/defaults/main.yml @@ -247,3 +247,8 @@ matrix_jitsi_jvb_container_rtp_udp_host_bind_port: "{{ matrix_jitsi_jvb_rtp_udp_ # # Takes an ":" or "" value (e.g. "127.0.0.1:4443"), or empty string to not expose. matrix_jitsi_jvb_container_rtp_tcp_host_bind_port: "{{ matrix_jitsi_jvb_rtp_tcp_port }}" + +# Controls whether the matrix-jitsi-jvb container exposes its Colibri WebSocket port (tcp/9090 in the container). +# +# Takes an ":" or "" value (e.g. "127.0.0.1:12090"), or empty string to not expose. +matrix_jitsi_jvb_container_colibri_ws_host_bind_port: '' diff --git a/roles/matrix-jitsi/templates/jvb/matrix-jitsi-jvb.service.j2 b/roles/matrix-jitsi/templates/jvb/matrix-jitsi-jvb.service.j2 index 6db6a850..7fcfeec6 100644 --- a/roles/matrix-jitsi/templates/jvb/matrix-jitsi-jvb.service.j2 +++ b/roles/matrix-jitsi/templates/jvb/matrix-jitsi-jvb.service.j2 @@ -21,6 +21,9 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-jitsi-jvb \ {% if matrix_jitsi_jvb_container_rtp_tcp_host_bind_port %} -p {{ matrix_jitsi_jvb_container_rtp_tcp_host_bind_port }}:{{ matrix_jitsi_jvb_rtp_tcp_port }} \ {% endif %} + {% if matrix_jitsi_jvb_container_colibri_ws_host_bind_port %} + -p {{ matrix_jitsi_jvb_container_colibri_ws_host_bind_port }}:9090 \ + {% endif %} --mount type=bind,src={{ matrix_jitsi_jvb_config_path }},dst=/config \ {% for arg in matrix_jitsi_jvb_container_extra_arguments %} {{ arg }} \ diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-jitsi.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-jitsi.conf.j2 index 4cacf1f1..500f1943 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-jitsi.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-jitsi.conf.j2 @@ -23,6 +23,27 @@ proxy_set_header Host $host; proxy_set_header X-Forwarded-For $remote_addr; } + + # colibri (JVB) websockets + location ~ ^/colibri-ws/([a-zA-Z0-9-\.]+)/(.*) { + {% if matrix_nginx_proxy_enabled %} + resolver 127.0.0.11 valid=5s; + set $backend "matrix-jitsi-jvb:9090"; + proxy_pass http://$backend; + {% else %} + {# Generic configuration for use outside of our container setup #} + proxy_pass http://127.0.0.1:12090; + {% endif %} + + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + proxy_http_version 1.1; + + tcp_nodelay on; + } {% endmacro %} server { From be5263f397eba7c3cc554f952344d0d7d36ec02a Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sat, 28 Nov 2020 21:34:14 +0200 Subject: [PATCH 081/217] Move self-building git repository URLs to variables (stop hardcoding) --- roles/matrix-bridge-mautrix-facebook/defaults/main.yml | 1 + .../tasks/setup_install.yml | 2 +- roles/matrix-bridge-mautrix-hangouts/defaults/main.yml | 1 + .../tasks/setup_install.yml | 2 +- .../matrix-bridge-mx-puppet-discord/defaults/main.yml | 1 + .../tasks/setup_install.yml | 2 +- .../defaults/main.yml | 1 + .../tasks/setup_install.yml | 2 +- roles/matrix-bridge-mx-puppet-skype/defaults/main.yml | 1 + .../tasks/setup_install.yml | 2 +- roles/matrix-bridge-mx-puppet-slack/defaults/main.yml | 1 + .../tasks/setup_install.yml | 2 +- roles/matrix-bridge-mx-puppet-steam/defaults/main.yml | 1 + .../tasks/setup_install.yml | 2 +- .../matrix-bridge-mx-puppet-twitter/defaults/main.yml | 1 + .../tasks/setup_install.yml | 2 +- roles/matrix-client-element/defaults/main.yml | 1 + roles/matrix-client-element/tasks/setup.yml | 2 +- roles/matrix-coturn/defaults/main.yml | 1 + roles/matrix-coturn/tasks/setup_coturn.yml | 2 +- roles/matrix-dynamic-dns/defaults/main.yml | 1 + roles/matrix-dynamic-dns/tasks/install.yml | 2 +- roles/matrix-ma1sd/defaults/main.yml | 1 + roles/matrix-ma1sd/tasks/setup_ma1sd.yml | 2 +- roles/matrix-registration/defaults/main.yml | 2 +- roles/matrix-registration/tasks/setup.yml | 2 +- roles/matrix-registration/tasks/validate_config.yml | 9 +++++++++ roles/matrix-synapse-admin/defaults/main.yml | 3 ++- roles/matrix-synapse-admin/tasks/main.yml | 6 ++++++ roles/matrix-synapse-admin/tasks/setup.yml | 2 +- roles/matrix-synapse-admin/tasks/validate_config.yml | 10 ++++++++++ roles/matrix-synapse/defaults/main.yml | 1 + roles/matrix-synapse/tasks/synapse/setup_install.yml | 2 +- 33 files changed, 56 insertions(+), 17 deletions(-) create mode 100644 roles/matrix-synapse-admin/tasks/validate_config.yml diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml index c34da88f..c14a3ace 100644 --- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml @@ -4,6 +4,7 @@ matrix_mautrix_facebook_enabled: true matrix_mautrix_facebook_container_image_self_build: false +matrix_mautrix_facebook_container_image_self_build_repo: "https://github.com/tulir/mautrix-facebook.git" # See: https://mau.dev/tulir/mautrix-facebook/container_registry matrix_mautrix_facebook_docker_image: "{{ matrix_mautrix_facebook_docker_image_name_prefix }}tulir/mautrix-facebook:latest" diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml index 75ea7a6b..638854e2 100644 --- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml @@ -32,7 +32,7 @@ - name: Ensure Mautrix Facebook repository is present on self-build git: - repo: https://github.com/tulir/mautrix-facebook.git + repo: "{{ matrix_mautrix_facebook_container_image_self_build_repo }}" dest: "{{ matrix_mautrix_facebook_docker_src_files_path }}" # version: "{{ matrix_coturn_docker_image.split(':')[1] }}" force: "yes" diff --git a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml index 13bc1a6c..ab670052 100644 --- a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml @@ -4,6 +4,7 @@ matrix_mautrix_hangouts_enabled: true matrix_mautrix_hangouts_container_image_self_build: false +matrix_mautrix_hangouts_container_image_self_build_repo: "https://github.com/tulir/mautrix-hangouts.git" # See: https://mau.dev/tulir/mautrix-hangouts/container_registry matrix_mautrix_hangouts_docker_image: "{{ matrix_mautrix_hangouts_docker_image_name_prefix }}tulir/mautrix-hangouts:latest" diff --git a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml index 5dc5f20a..eec5e006 100644 --- a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml @@ -32,7 +32,7 @@ - name: Ensure Mautrix Hangots repository is present on self build git: - repo: https://github.com/tulir/mautrix-hangouts.git + repo: "{{ matrix_mautrix_hangouts_container_image_self_build_repo }}" dest: "{{ matrix_mautrix_hangouts_docker_src_files_path }}" force: "yes" register: matrix_mautrix_hangouts_git_pull_results diff --git a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml index 30732000..268bbf8f 100644 --- a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml @@ -4,6 +4,7 @@ matrix_mx_puppet_discord_enabled: true matrix_mx_puppet_discord_container_image_self_build: false +matrix_mx_puppet_discord_container_image_self_build_repo: "https://github.com/matrix-discord/mx-puppet-discord.git" # Controls whether the mx-puppet-discord container exposes its HTTP port (tcp/8432 in the container). # diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml index e36950bd..c3f2b01f 100644 --- a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml @@ -32,7 +32,7 @@ - name: Ensure MX Puppet Discord repository is present on self build git: - repo: https://github.com/matrix-discord/mx-puppet-discord.git + repo: "{{ matrix_mx_puppet_discord_container_image_self_build_repo }}" dest: "{{ matrix_mx_puppet_discord_docker_src_files_path }}" force: "yes" register: matrix_mx_puppet_discord_git_pull_results diff --git a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml index 108a0a34..21cde53f 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml @@ -4,6 +4,7 @@ matrix_mx_puppet_instagram_enabled: true matrix_mx_puppet_instagram_container_image_self_build: false +matrix_mx_puppet_instagram_container_image_self_build_repo: "https://github.com/Sorunome/mx-puppet-instagram.git" matrix_mx_puppet_instagram_docker_image: "{{ matrix_mx_puppet_instagram_docker_image_name_prefix }}sorunome/mx-puppet-instagram:latest" matrix_mx_puppet_instagram_docker_image_name_prefix: "{{ 'localhost/' if matrix_mx_puppet_instagram_container_image_self_build else 'docker.io/' }}" diff --git a/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml index 6b638626..78bd78c1 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml @@ -32,7 +32,7 @@ - name: Ensure mx-puppet-instagram repository is present on self build git: - repo: https://github.com/Sorunome/mx-puppet-instagram.git + repo: "{{ matrix_mx_puppet_instagram_container_image_self_build_repo }}" dest: "{{ matrix_mx_puppet_instagram_docker_src_files_path }}" force: "yes" register: matrix_mx_puppet_instagram_git_pull_results diff --git a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml index 9e1c71bb..0c16a8ba 100644 --- a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml @@ -4,6 +4,7 @@ matrix_mx_puppet_skype_enabled: true matrix_mx_puppet_skype_container_image_self_build: false +matrix_mx_puppet_skype_container_image_self_build_repo: "https://github.com/Sorunome/mx-puppet-skype.git" matrix_mx_puppet_skype_docker_image: "{{ matrix_mx_puppet_skype_docker_image_name_prefix }}sorunome/mx-puppet-skype:latest" matrix_mx_puppet_skype_docker_image_name_prefix: "{{ 'localhost/' if matrix_mx_puppet_skype_container_image_self_build else 'docker.io/' }}" diff --git a/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml index 9b04c290..fb5185ed 100644 --- a/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml @@ -32,7 +32,7 @@ - name: Ensure MX Puppet Skype repository is present on self build git: - repo: https://github.com/Sorunome/mx-puppet-skype.git + repo: "{{ matrix_mx_puppet_skype_container_image_self_build_repo }}" dest: "{{ matrix_mx_puppet_skype_docker_src_files_path }}" force: "yes" register: matrix_mx_puppet_skype_git_pull_results diff --git a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml index 94b37041..96ab7833 100644 --- a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml @@ -4,6 +4,7 @@ matrix_mx_puppet_slack_enabled: true matrix_mx_puppet_slack_container_image_self_build: false +matrix_mx_puppet_slack_container_image_self_build_repo: "https://github.com/Sorunome/mx-puppet-slack.git" # Controls whether the mx-puppet-slack container exposes its HTTP port (tcp/8432 in the container). # diff --git a/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml index b7cdc13c..2e45ecf6 100644 --- a/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml @@ -32,7 +32,7 @@ - name: Ensure MX Puppet Slack repository is present on self build git: - repo: https://github.com/Sorunome/mx-puppet-slack.git + repo: "{{ matrix_mx_puppet_slack_container_image_self_build_repo }}" dest: "{{ matrix_mx_puppet_slack_docker_src_files_path }}" force: "yes" register: matrix_mx_puppet_slack_git_pull_results diff --git a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml index cb06bf7d..b5eb8473 100644 --- a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml @@ -4,6 +4,7 @@ matrix_mx_puppet_steam_enabled: true matrix_mx_puppet_steam_container_image_self_build: false +matrix_mx_puppet_steam_container_image_self_build_repo: "https://github.com/icewind1991/mx-puppet-steam.git" # Controls whether the mx-puppet-steam container exposes its HTTP port (tcp/8432 in the container). # diff --git a/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml index d9679510..7b7f8211 100644 --- a/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml @@ -32,7 +32,7 @@ - name: Ensure MX Puppet Steam repository is present on self build git: - repo: https://github.com/icewind1991/mx-puppet-steam.git + repo: "{{ matrix_mx_puppet_steam_container_image_self_build_repo }}" dest: "{{ matrix_mx_puppet_steam_docker_src_files_path }}" force: "yes" register: matrix_mx_puppet_steam_git_pull_results diff --git a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml index 18c39826..13438a21 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml @@ -4,6 +4,7 @@ matrix_mx_puppet_twitter_enabled: true matrix_mx_puppet_twitter_container_image_self_build: false +matrix_mx_puppet_twitter_container_image_self_build_repo: "https://github.com/Sorunome/mx-puppet-twitter.git" # Controls whether the mx-puppet-twitter container exposes its HTTP port (tcp/8432 in the container). # diff --git a/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml index 10e49572..663f822c 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml @@ -32,7 +32,7 @@ - name: Ensure MX Puppet Twitter repository is present on self build git: - repo: https://github.com/Sorunome/mx-puppet-twitter.git + repo: "{{ matrix_mx_puppet_twitter_container_image_self_build_repo }}" dest: "{{ matrix_mx_puppet_twitter_docker_src_files_path }}" force: "yes" register: matrix_mx_puppet_twitter_git_pull_results diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index 17a04465..cbe70503 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -1,6 +1,7 @@ matrix_client_element_enabled: true matrix_client_element_container_image_self_build: false +matrix_client_element_container_image_self_build_repo: "https://github.com/vector-im/riot-web.git" matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:v1.7.14" matrix_client_element_docker_image_name_prefix: "{{ 'localhost/' if matrix_client_element_container_image_self_build else 'docker.io/' }}" diff --git a/roles/matrix-client-element/tasks/setup.yml b/roles/matrix-client-element/tasks/setup.yml index 81f7842f..3b542b14 100644 --- a/roles/matrix-client-element/tasks/setup.yml +++ b/roles/matrix-client-element/tasks/setup.yml @@ -26,7 +26,7 @@ - name: Ensure Element repository is present on self-build git: - repo: https://github.com/vector-im/riot-web.git + repo: "{{ matrix_client_element_container_image_self_build_repo }}" dest: "{{ matrix_client_element_docker_src_files_path }}" version: "{{ matrix_client_element_docker_image.split(':')[1] }}" force: "yes" diff --git a/roles/matrix-coturn/defaults/main.yml b/roles/matrix-coturn/defaults/main.yml index 4aae3f06..8cbb3e3f 100644 --- a/roles/matrix-coturn/defaults/main.yml +++ b/roles/matrix-coturn/defaults/main.yml @@ -1,6 +1,7 @@ matrix_coturn_enabled: true matrix_coturn_container_image_self_build: false +matrix_coturn_container_image_self_build_repo: "https://github.com/instrumentisto/coturn-docker-image.git" matrix_coturn_docker_image: "{{ matrix_coturn_docker_image_name_prefix }}instrumentisto/coturn:4.5.1.3" matrix_coturn_docker_image_name_prefix: "{{ 'localhost/' if matrix_coturn_container_image_self_build else 'docker.io/' }}" diff --git a/roles/matrix-coturn/tasks/setup_coturn.yml b/roles/matrix-coturn/tasks/setup_coturn.yml index fa622caf..1f13da03 100644 --- a/roles/matrix-coturn/tasks/setup_coturn.yml +++ b/roles/matrix-coturn/tasks/setup_coturn.yml @@ -25,7 +25,7 @@ - name: Ensure Coturn repository is present on self-build git: - repo: https://github.com/instrumentisto/coturn-docker-image.git + repo: "{{ matrix_coturn_container_image_self_build_repo }}" dest: "{{ matrix_coturn_docker_src_files_path }}" version: "{{ matrix_coturn_docker_image.split(':')[1] }}" force: "yes" diff --git a/roles/matrix-dynamic-dns/defaults/main.yml b/roles/matrix-dynamic-dns/defaults/main.yml index 17c57f2c..b33961c1 100644 --- a/roles/matrix-dynamic-dns/defaults/main.yml +++ b/roles/matrix-dynamic-dns/defaults/main.yml @@ -23,6 +23,7 @@ matrix_dynamic_dns_systemd_required_services_list: ['docker.service'] # Build the container from source when running in mode matrix_dynamic_dns_container_image_self_build: false +matrix_dynamic_dns_container_image_self_build_repo: "https://github.com/linuxserver/docker-ddclient.git" # Config paths matrix_dynamic_dns_base_path: "{{ matrix_base_data_path }}/dynamic-dns" diff --git a/roles/matrix-dynamic-dns/tasks/install.yml b/roles/matrix-dynamic-dns/tasks/install.yml index 9ca1043f..e7b06d95 100644 --- a/roles/matrix-dynamic-dns/tasks/install.yml +++ b/roles/matrix-dynamic-dns/tasks/install.yml @@ -23,7 +23,7 @@ - name: Ensure Dynamic DNS repository is present on self build git: - repo: https://github.com/linuxserver/docker-ddclient.git + repo: "{{ matrix_dynamic_dns_container_image_self_build_repo }}" dest: "{{ matrix_dynamic_dns_docker_src_files_path }}" force: "yes" register: matrix_dynamic_dns_git_pull_results diff --git a/roles/matrix-ma1sd/defaults/main.yml b/roles/matrix-ma1sd/defaults/main.yml index b86f7ee0..f7a0782d 100644 --- a/roles/matrix-ma1sd/defaults/main.yml +++ b/roles/matrix-ma1sd/defaults/main.yml @@ -4,6 +4,7 @@ matrix_ma1sd_enabled: true matrix_ma1sd_container_image_self_build: false +matrix_ma1sd_container_image_self_build_repo: "https://github.com/ma1uta/ma1sd.git" matrix_ma1sd_architecture: "amd64" diff --git a/roles/matrix-ma1sd/tasks/setup_ma1sd.yml b/roles/matrix-ma1sd/tasks/setup_ma1sd.yml index 8814e6b9..46acb428 100644 --- a/roles/matrix-ma1sd/tasks/setup_ma1sd.yml +++ b/roles/matrix-ma1sd/tasks/setup_ma1sd.yml @@ -52,7 +52,7 @@ - name: Ensure ma1sd repository is present on self-build git: - repo: https://github.com/ma1uta/ma1sd.git + repo: "{{ matrix_ma1sd_container_image_self_build_repo }}" dest: "{{ matrix_ma1sd_docker_src_files_path }}" version: "{{ matrix_ma1sd_docker_image.split(':')[1].split('-')[0] }}" force: "yes" diff --git a/roles/matrix-registration/defaults/main.yml b/roles/matrix-registration/defaults/main.yml index c2150488..680e8bfe 100644 --- a/roles/matrix-registration/defaults/main.yml +++ b/roles/matrix-registration/defaults/main.yml @@ -4,6 +4,7 @@ matrix_registration_enabled: true matrix_registration_container_image_self_build: false +matrix_registration_container_image_self_build_repo: "https://github.com/ZerataX/matrix-registration" matrix_registration_base_path: "{{ matrix_base_data_path }}/matrix-registration" matrix_registration_config_path: "{{ matrix_registration_base_path }}/config" @@ -15,7 +16,6 @@ matrix_registration_version: "v0.7.0" matrix_registration_docker_image: "{{ matrix_registration_docker_image_name_prefix }}devture/zeratax-matrix-registration:{{ matrix_registration_version }}" matrix_registration_docker_image_name_prefix: "{{ 'localhost/' if matrix_registration_container_image_self_build else 'docker.io/' }}" matrix_registration_docker_image_force_pull: "{{ matrix_registration_docker_image.endswith(':latest') }}" -matrix_registration_docker_repo: "https://github.com/ZerataX/matrix-registration" # A list of extra arguments to pass to the container matrix_registration_container_extra_arguments: [] diff --git a/roles/matrix-registration/tasks/setup.yml b/roles/matrix-registration/tasks/setup.yml index 253f7aea..8427d950 100644 --- a/roles/matrix-registration/tasks/setup.yml +++ b/roles/matrix-registration/tasks/setup.yml @@ -28,7 +28,7 @@ - name: Ensure matrix-registration repository is present when self-building git: - repo: "{{ matrix_registration_docker_repo }}" + repo: "{{ matrix_registration_container_image_self_build_repo }}" dest: "{{ matrix_registration_docker_src_files_path }}" version: "{{ matrix_registration_version }}" force: "yes" diff --git a/roles/matrix-registration/tasks/validate_config.yml b/roles/matrix-registration/tasks/validate_config.yml index 6b2f0277..90466b46 100644 --- a/roles/matrix-registration/tasks/validate_config.yml +++ b/roles/matrix-registration/tasks/validate_config.yml @@ -9,3 +9,12 @@ - "matrix_registration_shared_secret" - "matrix_registration_admin_secret" - "matrix_registration_server_location" + +- name: (Deprecation) Catch and report renamed settings + fail: + msg: >- + Your configuration contains a variable, which now has a different name. + Please change your configuration to rename the variable (`{{ item.old }}` -> `{{ item.new }}`). + when: "item.old in vars" + with_items: + - {'old': 'matrix_registration_docker_repo', 'new': 'matrix_registration_container_image_self_build_repo'} diff --git a/roles/matrix-synapse-admin/defaults/main.yml b/roles/matrix-synapse-admin/defaults/main.yml index 6f1d219e..ce149dfd 100644 --- a/roles/matrix-synapse-admin/defaults/main.yml +++ b/roles/matrix-synapse-admin/defaults/main.yml @@ -4,7 +4,8 @@ matrix_synapse_admin_enabled: true matrix_synapse_admin_container_self_build: false -matrix_synapse_admin_docker_repo: "https://github.com/Awesome-Technologies/synapse-admin.git" +matrix_synapse_admin_container_self_build_repo: "https://github.com/Awesome-Technologies/synapse-admin.git" + matrix_synapse_admin_docker_src_files_path: "{{ matrix_base_data_path }}/synapse-admin/docker-src" matrix_synapse_admin_docker_image: "{{ matrix_synapse_admin_docker_image_name_prefix }}awesometechnologies/synapse-admin:0.5.0" diff --git a/roles/matrix-synapse-admin/tasks/main.yml b/roles/matrix-synapse-admin/tasks/main.yml index 3763ba28..b5cb1689 100644 --- a/roles/matrix-synapse-admin/tasks/main.yml +++ b/roles/matrix-synapse-admin/tasks/main.yml @@ -2,6 +2,12 @@ tags: - always +- import_tasks: "{{ role_path }}/tasks/validate_config.yml" + when: run_setup|bool + tags: + - setup-all + - setup-synapse-admin + - import_tasks: "{{ role_path }}/tasks/setup.yml" tags: - setup-all diff --git a/roles/matrix-synapse-admin/tasks/setup.yml b/roles/matrix-synapse-admin/tasks/setup.yml index d30657d3..a96ec10f 100644 --- a/roles/matrix-synapse-admin/tasks/setup.yml +++ b/roles/matrix-synapse-admin/tasks/setup.yml @@ -14,7 +14,7 @@ - name: Ensure matrix-synapse-admin repository is present when self-building git: - repo: "{{ matrix_synapse_admin_docker_repo }}" + repo: "{{ matrix_synapse_admin_container_self_build_repo }}" dest: "{{ matrix_synapse_admin_docker_src_files_path }}" force: "yes" register: matrix_synapse_admin_git_pull_results diff --git a/roles/matrix-synapse-admin/tasks/validate_config.yml b/roles/matrix-synapse-admin/tasks/validate_config.yml new file mode 100644 index 00000000..e08680e0 --- /dev/null +++ b/roles/matrix-synapse-admin/tasks/validate_config.yml @@ -0,0 +1,10 @@ +--- + +- name: (Deprecation) Catch and report renamed settings + fail: + msg: >- + Your configuration contains a variable, which now has a different name. + Please change your configuration to rename the variable (`{{ item.old }}` -> `{{ item.new }}`). + when: "item.old in vars" + with_items: + - {'old': 'matrix_synapse_admin_docker_repo', 'new': 'matrix_synapse_admin_container_self_build_repo'} diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index aadb47ee..f5c2c433 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -4,6 +4,7 @@ matrix_synapse_enabled: true matrix_synapse_container_image_self_build: false +matrix_synapse_container_image_self_build_repo: "https://github.com/matrix-org/synapse.git" matrix_synapse_docker_image: "{{ matrix_synapse_docker_image_name_prefix }}matrixdotorg/synapse:v1.23.0" matrix_synapse_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_container_image_self_build else 'docker.io/' }}" diff --git a/roles/matrix-synapse/tasks/synapse/setup_install.yml b/roles/matrix-synapse/tasks/synapse/setup_install.yml index 3aa2c223..f1abcbcc 100644 --- a/roles/matrix-synapse/tasks/synapse/setup_install.yml +++ b/roles/matrix-synapse/tasks/synapse/setup_install.yml @@ -20,7 +20,7 @@ - name: Ensure Synapse repository is present on self-build git: - repo: https://github.com/matrix-org/synapse.git + repo: "{{ matrix_synapse_container_image_self_build_repo }}" dest: "{{ matrix_synapse_docker_src_files_path }}" version: "{{ matrix_synapse_docker_image.split(':')[1] }}" force: "yes" From e0d7d5f0cac47731ec86b4e60e8ae91d388c38ba Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sat, 28 Nov 2020 22:11:35 +0200 Subject: [PATCH 082/217] Disable Jitsi recording/transcriptions by default It's not like it worked anyway, because we don't have the necessary services installed for transcription (Jigasi), nor recording (Jibri). Disabling these, should hopefully disable their related elements in the Jitsi Web UI. Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/726 --- roles/matrix-jitsi/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/matrix-jitsi/defaults/main.yml b/roles/matrix-jitsi/defaults/main.yml index 85123cf7..924198b4 100644 --- a/roles/matrix-jitsi/defaults/main.yml +++ b/roles/matrix-jitsi/defaults/main.yml @@ -4,8 +4,8 @@ matrix_jitsi_base_path: "{{ matrix_base_data_path }}/jitsi" matrix_jitsi_enable_auth: false matrix_jitsi_enable_guests: false -matrix_jitsi_enable_recording: true -matrix_jitsi_enable_transcriptions: true +matrix_jitsi_enable_recording: false +matrix_jitsi_enable_transcriptions: false matrix_jitsi_enable_p2p: true # Authentication type, must be one of internal, jwt or ldap. Currently only From d64ad91c25552e1263a73e5a18107c7ff6da1f16 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 30 Nov 2020 13:11:15 +0200 Subject: [PATCH 083/217] Add some notes about recent Jitsi changes A bit late, but still better than nothing. --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c350d235..27afb64e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,18 @@ +# 2020-11-27 + +## Recent Jitsi updates may require configuration changes + +We've recently [updated from Jitsi build 4857 to build 5142](https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/719), which brings a lot of configuration changes. + +**If you use our default Jitsi settings, you won't have to do anything.** + +People who have [fine-tuned Jitsi](docs/configuring-playbook-jitsi.md#optional-fine-tune-jitsi) may find that some options got renamed now, others are gone and yet others still need to be defined in another way. + +The next time you run the playbook [installation](docs/installing.md) command, our validation logic will tell you if you're using some variables like that and will recommend a migration path for each one. + +Additionally, we've recently disabled transcriptions (`matrix_jitsi_enable_transcriptions: false`) and recording (`matrix_jitsi_enable_recording: false`) by default. These features did not work anyway, because we don't install the required dependencies for them (Jigasi and Jibri, respectively). If you've been somehow pointing your Jitsi installation to some manually installed Jigasi/Jibri service, you may need to toggle these flags back to enabled to have transcriptions and recordings working. + + # 2020-11-23 ## Breaking change matrix-sms-bridge From 90078dd2966600c0fcaf8d3f6f415b69bc34f4ac Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 30 Nov 2020 20:58:21 +0200 Subject: [PATCH 084/217] Add matrix_services_autostart_enabled variable for preventing services autostart Some people requested that `--tags=start` not set up service autostart. One can now do `--tags=start --extra-vars="matrix_services_autostart_enabled=false"` to just start services ones and not set up autostarting. --- roles/matrix-common-after/tasks/start.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/roles/matrix-common-after/tasks/start.yml b/roles/matrix-common-after/tasks/start.yml index 4bb59bfe..bad84d92 100644 --- a/roles/matrix-common-after/tasks/start.yml +++ b/roles/matrix-common-after/tasks/start.yml @@ -1,5 +1,9 @@ --- +- name: Deterimne whether we should make services autostart + set_fact: + matrix_services_autostart_enabled_bool: "{{ true if matrix_services_autostart_enabled|default('') == '' else matrix_services_autostart_enabled|bool }}" + - name: Ensure systemd is reloaded service: daemon_reload: yes @@ -14,7 +18,7 @@ - name: Ensure Matrix services are started service: name: "{{ item }}" - enabled: yes + enabled: "{{ matrix_services_autostart_enabled_bool }}" state: started with_items: "{{ matrix_systemd_services_list }}" when: not ansible_check_mode From 04da1bddf7c6e87af7c4cfa8fe1365008814e1a5 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Dec 2020 11:55:18 +0200 Subject: [PATCH 085/217] Update matrix-mautrix-facebook config a bit This also disables presence if it's disabled for Synapse. --- group_vars/matrix_servers | 2 ++ .../defaults/main.yml | 4 +++ .../templates/config.yaml.j2 | 25 +++++++++++++------ 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 07f9a88c..8e46a48f 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -201,6 +201,8 @@ matrix_mautrix_facebook_homeserver_token: "{{ matrix_synapse_macaroon_secret_key matrix_mautrix_facebook_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}" +matrix_mautrix_facebook_bridge_presence: "{{ matrix_synapse_use_presence if matrix_synapse_enabled else true }}" + ###################################################################### # # /matrix-bridge-mautrix-facebook diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml index c14a3ace..1547ad71 100644 --- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml @@ -35,6 +35,10 @@ matrix_mautrix_facebook_homeserver_token: '' # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). matrix_mautrix_facebook_login_shared_secret: '' +matrix_mautrix_facebook_bridge_login_shared_secret_map: "{{ {matrix_mautrix_facebook_homeserver_domain: matrix_mautrix_facebook_login_shared_secret} if matrix_mautrix_facebook_login_shared_secret else {} }}" + +matrix_mautrix_facebook_bridge_presence: true + # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. # diff --git a/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 index 7da7b588..9d8de2d5 100644 --- a/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 @@ -81,23 +81,32 @@ bridge: command_prefix: "!fb" # Number of chats to sync (and create portals for) on startup/login. - # Maximum 20, set 0 to disable automatic syncing. + # Set 0 to disable automatic syncing. initial_chat_sync: 10 # Whether or not the Facebook users of logged in Matrix users should be # invited to private chats when the user sends a message from another client. invite_own_puppet_to_pm: false - # Whether or not to use /sync to get presence, read receipts and typing notifications when using - # your own Matrix account as the Matrix puppet for your Facebook account. + # Whether or not to use /sync to get presence, read receipts and typing notifications + # when double puppeting is enabled sync_with_custom_puppets: true - # Shared secret for https://github.com/devture/matrix-synapse-shared-secret-auth + # Whether or not to update the m.direct account data event when double puppeting is enabled. + # Note that updating the m.direct event is not atomic (except with mautrix-asmux) + # and is therefore prone to race conditions. + sync_direct_chat_list: false + # Servers to always allow double puppeting from + double_puppet_server_map: {} + # example.com: https://example.com + # Allow using double puppeting from any server with a valid client .well-known file. + double_puppet_allow_discovery: false + # Shared secrets for https://github.com/devture/matrix-synapse-shared-secret-auth # # If set, custom puppets will be enabled automatically for local users # instead of users having to find an access token and run `login-matrix` # manually. - login_shared_secret: {{ matrix_mautrix_facebook_login_shared_secret|to_json }} - # Whether or not to bridge presence in both directions. Facebook allows users not to broadcast - # presence, but then it won't send other users' presence to the client. - presence: true + # If using this for other servers than the bridge's server, + # you must also set the URL in the double_puppet_server_map. + login_shared_secret_map: {{ matrix_mautrix_facebook_bridge_login_shared_secret_map|to_json }} + presence: {{ matrix_mautrix_facebook_bridge_presence|to_json }} # Whether or not to update avatars when syncing all contacts at startup. update_avatar_initial_sync: true # End-to-bridge encryption support options. These require matrix-nio to be installed with pip From 694e6fe1414d0a2c897d0ae4f0495f8832648b48 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Dec 2020 15:11:37 +0200 Subject: [PATCH 086/217] Create FUNDING.yml --- .github/FUNDING.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..bc3c8e75 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,4 @@ +# These are supported funding model platforms + +# https://liberapay.com/s.pantaleev/ +liberapay: s.pantaleev From 06ade5e187aeb6931197bb061966ff8b30d9d055 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Dec 2020 15:24:07 +0200 Subject: [PATCH 087/217] Add donation and support room badges --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 1a207937..ac2b7ca9 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Support room on Matrix](https://img.shields.io/matrix/matrix-docker-ansible-deploy:devture.com.svg?label=%23matrix-docker-ansible-deploy%3Adevture.com&logo=matrix&style=for-the-badge&server_fqdn=matrix.devture.com)](https://matrix.to/#/#matrix-docker-ansible-deploy:devture.com) [![donate](https://liberapay.com/assets/widgets/donate.svg)](https://liberapay.com/s.pantaleev/donate) + # Matrix (An open network for secure, decentralized communication) server setup using Ansible and Docker ## Purpose From edd40811a59172b48d62d4c774f37c4854f213c2 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 3 Dec 2020 15:16:26 +0200 Subject: [PATCH 088/217] Update matrix-appservice-discord to v1.0.0 final --- roles/matrix-bridge-appservice-discord/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml index cb262166..9ca06b05 100644 --- a/roles/matrix-bridge-appservice-discord/defaults/main.yml +++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml @@ -3,7 +3,7 @@ matrix_appservice_discord_enabled: true -matrix_appservice_discord_docker_image: "halfshot/matrix-appservice-discord:v1.0.0-rc3" +matrix_appservice_discord_docker_image: "docker.io/halfshot/matrix-appservice-discord:v1.0.0" matrix_appservice_discord_docker_image_force_pull: "{{ matrix_appservice_discord_docker_image.endswith(':latest') }}" matrix_appservice_discord_base_path: "{{ matrix_base_data_path }}/appservice-discord" From b3d91ed4885be0672cdfb8045107215982fd2e3f Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 4 Dec 2020 01:06:42 +0200 Subject: [PATCH 089/217] Fix passing of matrix_appservice_discord_auth_usePrivilegedIntents --- roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 index aca49228..b99f522a 100644 --- a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 +++ b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 @@ -36,7 +36,7 @@ auth: botToken: {{ matrix_appservice_discord_bot_token }} # You must enable "Privileged Gateway Intents" in your bot settings on discord.com (e.g. https://discord.com/developers/applications/12345/bot) # for this to work - usePrivilegedIntents: false + usePrivilegedIntents: {{ matrix_appservice_discord_auth_usePrivilegedIntents|to_json }} logging: # What level should the logger output to the console at. console: "warn" #silly, verbose, info, http, warn, error, silent From a5ae7e9ef045c81c401da3cd5d84ac5677346aac Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 4 Dec 2020 01:48:08 +0200 Subject: [PATCH 090/217] Add self-building support to matrix-corporal --- docs/self-building.md | 1 + group_vars/matrix_servers | 2 ++ roles/matrix-corporal/defaults/main.yml | 8 ++++++- .../matrix-corporal/tasks/setup_corporal.yml | 22 ++++++++++++++++++- 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/self-building.md b/docs/self-building.md index 169e4aa5..fa4db222 100644 --- a/docs/self-building.md +++ b/docs/self-building.md @@ -15,6 +15,7 @@ List of roles where self-building the Docker image is currently possible: - `matrix-client-element` - `matrix-registration` - `matrix-coturn` +- `matrix-corporal` - `matrix-ma1sd` - `matrix-mailer` - `matrix-bridge-mautrix-facebook` diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 8e46a48f..d68db7ed 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -541,6 +541,8 @@ matrix_bot_matrix_reminder_bot_enabled: false matrix_corporal_enabled: false +matrix_corporal_container_image_self_build: "{{ matrix_architecture != 'amd64' }}" + # Normally, matrix-nginx-proxy is enabled and nginx can reach matrix-corporal over the container network. # If matrix-nginx-proxy is not enabled, or you otherwise have a need for it, you can expose # matrix-corporal's web-server ports to the local host. diff --git a/roles/matrix-corporal/defaults/main.yml b/roles/matrix-corporal/defaults/main.yml index def3fcc9..1cab3119 100644 --- a/roles/matrix-corporal/defaults/main.yml +++ b/roles/matrix-corporal/defaults/main.yml @@ -3,6 +3,9 @@ matrix_corporal_enabled: true +matrix_corporal_container_image_self_build: false +matrix_corporal_container_image_self_build_repo: "https://github.com/devture/matrix-corporal.git" + # Controls whether the matrix-corporal container exposes its gateway HTTP port (tcp/41080 in the container). # # Takes an ":" or "" value (e.g. "127.0.0.1:41080"), or empty string to not expose. @@ -19,10 +22,13 @@ matrix_corporal_container_extra_arguments: [] # List of systemd services that matrix-corporal.service depends on matrix_corporal_systemd_required_services_list: ['docker.service'] -matrix_corporal_docker_image: "docker.io/devture/matrix-corporal:1.11.0" +matrix_corporal_docker_image: "{{ matrix_corporal_docker_image_name_prefix }}devture/matrix-corporal:{{ matrix_corporal_docker_image_tag }}" +matrix_corporal_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_container_image_self_build else 'docker.io/' }}" +matrix_corporal_docker_image_tag: "1.11.0" matrix_corporal_docker_image_force_pull: "{{ matrix_corporal_docker_image.endswith(':latest') }}" matrix_corporal_base_path: "{{ matrix_base_data_path }}/corporal" +matrix_corporal_container_src_files_path: "{{ matrix_corporal_base_path }}/container-src" matrix_corporal_config_dir_path: "{{ matrix_corporal_base_path }}/config" matrix_corporal_cache_dir_path: "{{ matrix_corporal_base_path }}/cache" matrix_corporal_var_dir_path: "{{ matrix_corporal_base_path }}/var" diff --git a/roles/matrix-corporal/tasks/setup_corporal.yml b/roles/matrix-corporal/tasks/setup_corporal.yml index 72b6a5b2..188f09bf 100644 --- a/roles/matrix-corporal/tasks/setup_corporal.yml +++ b/roles/matrix-corporal/tasks/setup_corporal.yml @@ -17,13 +17,33 @@ - "{{ matrix_corporal_var_dir_path }}" when: matrix_corporal_enabled|bool +- name: Ensure Matrix Corporal repository is present on self-build + git: + repo: "{{ matrix_corporal_container_image_self_build_repo }}" + dest: "{{ matrix_corporal_container_src_files_path }}" + version: "{{ matrix_corporal_docker_image.split(':')[1] }}" + force: "yes" + register: matrix_corporal_git_pull_results + when: "matrix_corporal_enabled|bool and matrix_corporal_container_image_self_build|bool" + +- name: Ensure Matrix Corporal Docker image is built + docker_image: + name: "{{ matrix_corporal_docker_image }}" + source: build + force_source: "{{ matrix_corporal_git_pull_results.changed }}" + build: + dockerfile: etc/docker/Dockerfile + path: "{{ matrix_corporal_container_src_files_path }}" + pull: yes + when: "matrix_corporal_enabled|bool and matrix_corporal_container_image_self_build|bool" + - name: Ensure Matrix Corporal Docker image is pulled docker_image: name: "{{ matrix_corporal_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_corporal_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_corporal_docker_image_force_pull }}" - when: matrix_corporal_enabled|bool + when: "matrix_corporal_enabled|bool and not matrix_corporal_container_image_self_build|bool" - name: Ensure Matrix Corporal config installed copy: From ec2a9d4852c49029c08df9f3651bab94fb2057fd Mon Sep 17 00:00:00 2001 From: Hardy Erlinger Date: Sun, 6 Dec 2020 13:50:45 +0100 Subject: [PATCH 091/217] Remove the recording button from the Jitsi UI if recording is disabled. --- roles/matrix-jitsi/templates/web/interface_config.js.j2 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/roles/matrix-jitsi/templates/web/interface_config.js.j2 b/roles/matrix-jitsi/templates/web/interface_config.js.j2 index 60fac4d5..a12ca973 100644 --- a/roles/matrix-jitsi/templates/web/interface_config.js.j2 +++ b/roles/matrix-jitsi/templates/web/interface_config.js.j2 @@ -205,9 +205,11 @@ var interfaceConfig = { {% if matrix_jitsi_enable_transcriptions %} 'closedcaptions', {% endif %} - + {% if matrix_jitsi_enable_recording %} + 'recording', + {% endif %} 'microphone', 'camera', 'desktop', 'embedmeeting', 'fullscreen', - 'fodeviceselection', 'hangup', 'profile', 'chat', 'recording', + 'fodeviceselection', 'hangup', 'profile', 'chat', 'livestreaming', 'etherpad', 'sharedvideo', 'settings', 'raisehand', 'videoquality', 'filmstrip', 'invite', 'feedback', 'stats', 'shortcuts', 'tileview', 'videobackgroundblur', 'download', 'help', 'mute-everyone', 'security' From 7372480e95d9fa60c05270236cc6f4d6753c3e67 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 6 Dec 2020 23:59:58 +0200 Subject: [PATCH 092/217] Properly serialize some ma1sd configuration values We've had a report of the `connection` value getting cut off, supposedly because it contains something that breaks off the string. Using `|to_json` takes care of it. --- roles/matrix-ma1sd/templates/ma1sd.yaml.j2 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 index 8f3569b1..84585707 100644 --- a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 +++ b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 @@ -73,10 +73,10 @@ hashing: - none # the same as v1 bulk lookup - sha256 # hash the 3PID and pepper. delay: 2m # how often hashes will be updated if rotation policy = per_seconds (default is 10s) - requests: 10 + requests: 10 {% endif %} synapseSql: - enabled: {{ matrix_ma1sd_synapsesql_enabled }} - type: {{ matrix_ma1sd_synapsesql_type }} - connection: {{ matrix_ma1sd_synapsesql_connection }} + enabled: {{ matrix_ma1sd_synapsesql_enabled|to_json }} + type: {{ matrix_ma1sd_synapsesql_type|to_json }} + connection: {{ matrix_ma1sd_synapsesql_connection|to_json }} From 9713ac96177d71cb4e126a0442a134179f4af8cf Mon Sep 17 00:00:00 2001 From: Mike Date: Sun, 6 Dec 2020 23:42:44 +0100 Subject: [PATCH 093/217] fixes config.js comments in config.js must begin with two slashes --- docs/configuring-playbook-jitsi.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuring-playbook-jitsi.md b/docs/configuring-playbook-jitsi.md index 62edc578..f72241e1 100644 --- a/docs/configuring-playbook-jitsi.md +++ b/docs/configuring-playbook-jitsi.md @@ -99,7 +99,7 @@ matrix_jitsi_web_custom_config_extension: | config.disableAudioLevels = true; - # Limit the number of video feeds forwarded to each client + // Limit the number of video feeds forwarded to each client config.channelLastN = 4; matrix_jitsi_web_config_resolution_width_ideal_and_max: 480 From d556aa943fbd736c0401d3fec2c78d5f827a5e78 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 7 Dec 2020 06:33:35 +0200 Subject: [PATCH 094/217] Update docker-ce.repo to not hardcode $releasever=7 This keeps it in line with https://download.docker.com/linux/centos/docker-ce.repo Whether or not Docker works well on CentOS 8 for our purposes hasn't been verified yet. Related to https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/300 --- .../files/yum.repos.d/docker-ce.repo | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/roles/matrix-base/files/yum.repos.d/docker-ce.repo b/roles/matrix-base/files/yum.repos.d/docker-ce.repo index 56242d98..1abdbe36 100644 --- a/roles/matrix-base/files/yum.repos.d/docker-ce.repo +++ b/roles/matrix-base/files/yum.repos.d/docker-ce.repo @@ -1,62 +1,62 @@ [docker-ce-stable] name=Docker CE Stable - $basearch -baseurl=https://download.docker.com/linux/centos/7/$basearch/stable +baseurl=https://download.docker.com/linux/centos/$releasever/$basearch/stable enabled=1 gpgcheck=1 gpgkey=https://download.docker.com/linux/centos/gpg [docker-ce-stable-debuginfo] name=Docker CE Stable - Debuginfo $basearch -baseurl=https://download.docker.com/linux/centos/7/debug-$basearch/stable +baseurl=https://download.docker.com/linux/centos/$releasever/debug-$basearch/stable enabled=0 gpgcheck=1 gpgkey=https://download.docker.com/linux/centos/gpg [docker-ce-stable-source] name=Docker CE Stable - Sources -baseurl=https://download.docker.com/linux/centos/7/source/stable -enabled=0 -gpgcheck=1 -gpgkey=https://download.docker.com/linux/centos/gpg - -[docker-ce-edge] -name=Docker CE Edge - $basearch -baseurl=https://download.docker.com/linux/centos/7/$basearch/edge -enabled=0 -gpgcheck=1 -gpgkey=https://download.docker.com/linux/centos/gpg - -[docker-ce-edge-debuginfo] -name=Docker CE Edge - Debuginfo $basearch -baseurl=https://download.docker.com/linux/centos/7/debug-$basearch/edge -enabled=0 -gpgcheck=1 -gpgkey=https://download.docker.com/linux/centos/gpg - -[docker-ce-edge-source] -name=Docker CE Edge - Sources -baseurl=https://download.docker.com/linux/centos/7/source/edge +baseurl=https://download.docker.com/linux/centos/$releasever/source/stable enabled=0 gpgcheck=1 gpgkey=https://download.docker.com/linux/centos/gpg [docker-ce-test] name=Docker CE Test - $basearch -baseurl=https://download.docker.com/linux/centos/7/$basearch/test +baseurl=https://download.docker.com/linux/centos/$releasever/$basearch/test enabled=0 gpgcheck=1 gpgkey=https://download.docker.com/linux/centos/gpg [docker-ce-test-debuginfo] name=Docker CE Test - Debuginfo $basearch -baseurl=https://download.docker.com/linux/centos/7/debug-$basearch/test +baseurl=https://download.docker.com/linux/centos/$releasever/debug-$basearch/test enabled=0 gpgcheck=1 gpgkey=https://download.docker.com/linux/centos/gpg [docker-ce-test-source] name=Docker CE Test - Sources -baseurl=https://download.docker.com/linux/centos/7/source/test +baseurl=https://download.docker.com/linux/centos/$releasever/source/test +enabled=0 +gpgcheck=1 +gpgkey=https://download.docker.com/linux/centos/gpg + +[docker-ce-nightly] +name=Docker CE Nightly - $basearch +baseurl=https://download.docker.com/linux/centos/$releasever/$basearch/nightly +enabled=0 +gpgcheck=1 +gpgkey=https://download.docker.com/linux/centos/gpg + +[docker-ce-nightly-debuginfo] +name=Docker CE Nightly - Debuginfo $basearch +baseurl=https://download.docker.com/linux/centos/$releasever/debug-$basearch/nightly +enabled=0 +gpgcheck=1 +gpgkey=https://download.docker.com/linux/centos/gpg + +[docker-ce-nightly-source] +name=Docker CE Nightly - Sources +baseurl=https://download.docker.com/linux/centos/$releasever/source/nightly enabled=0 gpgcheck=1 gpgkey=https://download.docker.com/linux/centos/gpg From 8c02f7b79bfe752b42e9efef069e675eba5b3d6a Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 7 Dec 2020 15:18:03 +0200 Subject: [PATCH 095/217] Upgrade services --- roles/matrix-client-element/defaults/main.yml | 2 +- roles/matrix-nginx-proxy/defaults/main.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index cbe70503..8b032ac2 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -3,7 +3,7 @@ matrix_client_element_enabled: true matrix_client_element_container_image_self_build: false matrix_client_element_container_image_self_build_repo: "https://github.com/vector-im/riot-web.git" -matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:v1.7.14" +matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:v1.7.15" matrix_client_element_docker_image_name_prefix: "{{ 'localhost/' if matrix_client_element_container_image_self_build else 'docker.io/' }}" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index ceba9ab6..7331c589 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -3,7 +3,7 @@ matrix_nginx_proxy_enabled: true # We use an official nginx image, which we fix-up to run unprivileged. # An alternative would be an `nginxinc/nginx-unprivileged` image, but # that is frequently out of date. -matrix_nginx_proxy_docker_image: "docker.io/nginx:1.19.4-alpine" +matrix_nginx_proxy_docker_image: "docker.io/nginx:1.19.5-alpine" matrix_nginx_proxy_docker_image_force_pull: "{{ matrix_nginx_proxy_docker_image.endswith(':latest') }}" matrix_nginx_proxy_base_path: "{{ matrix_base_data_path }}/nginx-proxy" @@ -259,7 +259,7 @@ matrix_ssl_domains_to_obtain_certificates_for: [] # Controls whether to obtain production or staging certificates from Let's Encrypt. matrix_ssl_lets_encrypt_staging: false -matrix_ssl_lets_encrypt_certbot_docker_image: "docker.io/certbot/certbot:{{ matrix_ssl_architecture }}-v1.9.0" +matrix_ssl_lets_encrypt_certbot_docker_image: "docker.io/certbot/certbot:{{ matrix_ssl_architecture }}-v1.10.1" matrix_ssl_lets_encrypt_certbot_docker_image_force_pull: "{{ matrix_ssl_lets_encrypt_certbot_docker_image.endswith(':latest') }}" matrix_ssl_lets_encrypt_certbot_standalone_http_port: 2402 matrix_ssl_lets_encrypt_support_email: ~ From c07c927d9f2f37958300169ab27ecf6a4eeabd52 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 8 Dec 2020 16:48:25 +0200 Subject: [PATCH 096/217] Automatically enable openid listeners when ma1sd enabled ma1sd requires the openid endpoints for certain functionality. Example: https://github.com/ma1uta/ma1sd/blob/90b2b5301c34168346fdc5e7eccc09d6958e999f/src/main/java/io/kamax/mxisd/auth/AccountManager.java#L67-L99 If federation is disabled, we still need to expose these openid APIs on the federation port. Previously, we were doing similar magic for Dimension. As per its documentation, when running unfederated, one is to enable the openid listener as well. As per their recommendation, people are advised to do enable it on the Client-Server API port and use the `federationUrl` variable to override where the federation port is (making federation requests go to the Client-Server API). Because ma1sd always uses the federation port (unless you do some DNS overwriting magic using its configuration -- which we'd rather not do), it's better if we just default to putting the `openid` listener where it belongs - on the federation port. With this commit, we retain the "automatically enable openid APIs" thing we've been doing for Dimension, but move it to the federation port instead. We also now do the same thing when ma1sd is enabled. --- CHANGELOG.md | 12 ++++++++++ docs/configuring-playbook-dimension.md | 5 ++++- docs/configuring-playbook-federation.md | 10 +++++++++ docs/configuring-playbook-ma1sd.md | 7 +++++- docs/prerequisites.md | 13 ++++++++++- group_vars/matrix_servers | 12 +++------- roles/matrix-synapse/defaults/main.yml | 22 ++++++++++++++++++- .../templates/synapse/homeserver.yaml.j2 | 8 +++---- 8 files changed, 72 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27afb64e..5024d841 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +# 2020-12-08 + +## openid APIs exposed by default on the federation port when federation disabled + +We've changed some defaults. People running with our default configuration (federation enabled), are not affected at all. + +If you are running an unfederated server (`matrix_synapse_federation_enabled: false`), this may be of interest to you. + +When federation is disabled, but ma1sd or Dimension are enabled, we'll now expose the `openid` APIs on the federation port. +These APIs are necessary for some ma1sd features to work. If you'd like to prevent this, you can: `matrix_synapse_federation_port_openid_resource_required: false`. + + # 2020-11-27 ## Recent Jitsi updates may require configuration changes diff --git a/docs/configuring-playbook-dimension.md b/docs/configuring-playbook-dimension.md index 604e6aa6..d5f0a9e6 100644 --- a/docs/configuring-playbook-dimension.md +++ b/docs/configuring-playbook-dimension.md @@ -3,6 +3,9 @@ **[Dimension](https://dimension.t2bot.io) can only be installed after Matrix services are installed and running.** If you're just installing Matrix services for the first time, please continue with the [Configuration](configuring-playbook.md) / [Installation](installing.md) flow and come back here later. +**Note**: enabling Dimension, means that the `openid` API endpoints will be exposed on the Matrix Federation port (usually `8448`), even if [federation](configuring-playbook-federation.md) is disabled. It's something to be aware of, especially in terms of firewall whitelisting (make sure port `8448` is accessible). + + ## Prerequisites This playbook now supports running [Dimension](https://dimension.t2bot.io) in both a federated and an [unfederated](https://github.com/turt2live/matrix-dimension/blob/master/docs/unfederated.md) environment. This is handled automatically based on the value of `matrix_synapse_federation_enabled`. @@ -48,7 +51,7 @@ To get an access token for the Dimension user, you can follow one of two options 3. Copy the highlighted text to your configuration. 4. Close the private browsing session. **Do not log out**. Logging out will invalidate the token, making it not work. -*With CURL* +*With CURL* ``` curl -X POST --header 'Content-Type: application/json' -d '{ diff --git a/docs/configuring-playbook-federation.md b/docs/configuring-playbook-federation.md index 1e4ad61e..2e6410ec 100644 --- a/docs/configuring-playbook-federation.md +++ b/docs/configuring-playbook-federation.md @@ -37,3 +37,13 @@ matrix_synapse_federation_enabled: false ``` With that, your server's users will only be able to talk among themselves, but not to anyone who is on another server. + +**Disabling federation does not necessarily disable the federation port** (`8448`). Services like [Dimension](configuring-playbook-dimension.md) and [ma1sd](configuring-playbook-ma1sd.md) normally rely on `openid` APIs exposed on that port. Even if you disable federation and only if necessary, we may still be exposing the federation port and serving the `openid` APIs there. To override this and completely disable Synapse's federation port use: + +```yaml +# This stops the federation port on the Synapse side (normally `matrix-synapse:8048` on the container network). +matrix_synapse_federation_port_enabled: false + +# This removes the `8448` virtual host from the matrix-nginx-proxy reverse-proxy server. +matrix_nginx_proxy_proxy_matrix_federation_api_enabled: false +``` diff --git a/docs/configuring-playbook-ma1sd.md b/docs/configuring-playbook-ma1sd.md index 03208337..70c507cb 100644 --- a/docs/configuring-playbook-ma1sd.md +++ b/docs/configuring-playbook-ma1sd.md @@ -4,7 +4,9 @@ By default, this playbook configures an [ma1sd](https://github.com/ma1uta/ma1sd) This server is private by default, potentially at the expense of user discoverability. -ma1sd is a fork of [mxisd](https://github.com/kamax-io/mxisd) which was pronounced end of life 2019-06-21. +*ma1sd is a fork of [mxisd](https://github.com/kamax-io/mxisd) which was pronounced end of life 2019-06-21.* + +**Note**: enabling ma1sd (which is also the default), means that the `openid` API endpoints will be exposed on the Matrix Federation port (usually `8448`), even if [federation](configuring-playbook-federation.md) is disabled. It's something to be aware of, especially in terms of firewall whitelisting (make sure port `8448` is accessible). ## Disabling ma1sd @@ -50,6 +52,9 @@ To use the [Registration](https://github.com/ma1uta/ma1sd/blob/master/docs/featu - `matrix_ma1sd_configuration_extension_yaml` - to configure ma1sd as required. See the [Registration feature's docs](https://github.com/ma1uta/ma1sd/blob/master/docs/features/registration.md) for inspiration. Also see the [Additional features](#additional-features) section below to learn more about how to use `matrix_ma1sd_configuration_extension_yaml`. +**Note**: For this to work, either the homeserver needs to [federate](configuring-playbook-federation.md) or the `openid` APIs need to exposed on the federation port. When federation is disabled and ma1sd is enabled, we automatically expose the `openid` APIs (only!) on the federation port. Make sure the federation port (usually `https://matrix.DOMAIN:8448`) is whitelisted in your firewall (even if you don't actually use/need federation). + + ## Authentication [Authentication](https://github.com/ma1uta/ma1sd/blob/master/docs/features/authentication.md) provides the possibility to use your own [Identity Stores](https://github.com/ma1uta/ma1sd/blob/master/docs/stores/README.md) (for example LDAP) to authenticate users on your Homeserver. The following configuration can be used to authenticate against an LDAP server: diff --git a/docs/prerequisites.md b/docs/prerequisites.md index 4356081b..daf6ff80 100644 --- a/docs/prerequisites.md +++ b/docs/prerequisites.md @@ -22,6 +22,17 @@ This playbook doesn't support running on ARM (see [this issue](https://github.co - Properly configured DNS records for `` (details in [Configuring DNS](configuring-dns.md)). -- Some TCP/UDP ports open. This playbook configures the server's internal firewall for you. In most cases, you don't need to do anything special. But **if your server is running behind another firewall**, you'd need to open these ports: `80/tcp` (HTTP webserver), `443/tcp` (HTTPS webserver), `3478/tcp` (TURN over TCP), `3478/udp` (TURN over UDP), `5349/tcp` (TURN over TCP), `5349/udp` (TURN over UDP), `8448/tcp` (Matrix Federation API HTTPS webserver), the range `49152-49172/udp` (TURN over UDP), `4443/tcp` (Jitsi Harvester fallback), `10000/udp` (Jitsi video RTP). Depending on your firewall/NAT setup, incoming RTP packets on port 10000 may have the external IP of your firewall as destination address, due to the usage of STUN in JVB (see [`matrix_jitsi_jvb_stun_servers`](../roles/matrix-jitsi/defaults/main.yml)). +- Some TCP/UDP ports open. This playbook configures the server's internal firewall for you. In most cases, you don't need to do anything special. But **if your server is running behind another firewall**, you'd need to open these ports: + + - `80/tcp`: HTTP webserver + - `443/tcp`: HTTPS webserver + - `3478/tcp`: TURN over TCP (used by Coturn) + - `3478/udp`: TURN over UDP (used by Coturn) + - `5349/tcp`: TURN over TCP (used by Coturn) + - `5349/udp`: TURN over UDP (used by Coturn) + - `8448/tcp`: Matrix Federation API HTTPS webserver. In some cases, this **may necessary even with federation disabled**. Integration Servers (like Dimension) and Identity Servers (like ma1sd) may need to access `openid` APIs on the federation port. + - the range `49152-49172/udp`: TURN over UDP + - `4443/tcp`: Jitsi Harvester fallback + - `10000/udp`: Jitsi video RTP. Depending on your firewall/NAT setup, incoming RTP packets on port `10000` may have the external IP of your firewall as destination address, due to the usage of STUN in JVB (see [`matrix_jitsi_jvb_stun_servers`](../roles/matrix-jitsi/defaults/main.yml)). When ready to proceed, continue with [Configuring DNS](configuring-dns.md). diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index d68db7ed..78a03851 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -620,8 +620,6 @@ matrix_dimension_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_ena matrix_integration_manager_rest_url: "{{ matrix_dimension_integrations_rest_url if matrix_dimension_enabled else None }}" matrix_integration_manager_ui_url: "{{ matrix_dimension_integrations_ui_url if matrix_dimension_enabled else None }}" -matrix_dimension_homeserver_federationUrl: "http://matrix-synapse:{{ 8048 if matrix_synapse_federation_enabled|bool else 8008 }}" - ###################################################################### # # /matrix-dimension @@ -816,7 +814,8 @@ matrix_nginx_proxy_proxy_matrix_identity_api_addr_with_container: "matrix-ma1sd: matrix_nginx_proxy_proxy_matrix_identity_api_addr_sans_container: "127.0.0.1:8090" # By default, we do TLS termination for the Matrix Federation API (port 8448) at matrix-nginx-proxy. -matrix_nginx_proxy_proxy_matrix_federation_api_enabled: true +# Unless this is handled there OR Synapse's federation listener port is disabled, we'll reverse-proxy. +matrix_nginx_proxy_proxy_matrix_federation_api_enabled: "{{ matrix_synapse_federation_port_enabled and not matrix_synapse_tls_federation_listener_enabled }}" matrix_nginx_proxy_proxy_matrix_federation_api_addr_with_container: "matrix-synapse:8048" matrix_nginx_proxy_proxy_matrix_federation_api_addr_sans_container: "127.0.0.1:8048" @@ -990,12 +989,7 @@ matrix_synapse_tls_federation_listener_enabled: false matrix_synapse_tls_certificate_path: ~ matrix_synapse_tls_private_key_path: ~ -matrix_synapse_http_listener_resource_names: | - {{ - ["client"] - + - ( ["openid"] if matrix_dimension_enabled and not matrix_synapse_federation_enabled else [] ) - }} +matrix_synapse_federation_port_openid_resource_required: "{{ not matrix_synapse_federation_enabled and (matrix_dimension_enabled or matrix_ma1sd_enabled) }}" matrix_synapse_email_enabled: "{{ matrix_mailer_enabled }}" matrix_synapse_email_smtp_host: "matrix-mailer" diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index f5c2c433..35d40c7c 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -136,6 +136,11 @@ matrix_synapse_tls_private_key_path: "/data/{{ matrix_server_fqn_matrix }}.tls.k # names. matrix_synapse_http_listener_resource_names: ["client"] +# Resources served on Synapse's federation port. +# When disabling federation, we may wish to serve the `openid` resource here, +# so that services like Dimension and ma1sd can work. +matrix_synapse_federation_listener_resource_names: "{{ ['federation'] if matrix_synapse_federation_enabled else (['openid'] if matrix_synapse_federation_port_openid_resource_required else []) }}" + # Enable this to allow Synapse to report utilization statistics about your server to matrix.org # (things like number of users, number of messages sent, uptime, load, etc.) matrix_synapse_report_stats: false @@ -211,10 +216,25 @@ matrix_synapse_caches_global_factor: 0.5 # Controls whether Synapse will federate at all. # Disable this to completely isolate your server from the rest of the Matrix network. -# Also see: `matrix_synapse_tls_federation_listener_enabled` if you wish to keep federation enabled, +# +# Disabling this still keeps the federation port exposed, because it may be used for other services (`openid`). +# +# Also see: +# - `matrix_synapse_tls_federation_listener_enabled` if you wish to keep federation enabled, # but want to stop the TLS listener (port 8448). +# - `matrix_synapse_federation_port_enabled` to avoid exposing the federation ports matrix_synapse_federation_enabled: true +# Controls whether the federation ports are used at all. +# One may wish to disable federation (`matrix_synapse_federation_enabled: true`), +# but still run other resources (like `openid`) on the federation port +# by enabling them in `matrix_synapse_federation_listener_resource_names`. +matrix_synapse_federation_port_enabled: "{{ matrix_synapse_federation_enabled or matrix_synapse_federation_port_openid_resource_required }}" + +# Controls whether an `openid` listener is to be enabled. Useful when disabling federation, +# but needing the `openid` APIs for Dimension or an identity server like ma1sd. +matrix_synapse_federation_port_openid_resource_required: false + # A list of domain names that are allowed to federate with the given Synapse server. # An empty list value (`[]`) will also effectively stop federation, but if that's the desired # result, it's better to accomplish it by changing `matrix_synapse_federation_enabled`. diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index 8c68189b..c64ed9cc 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -188,7 +188,7 @@ listeners: - '0.0.0.0' {% endif %} -{% if matrix_synapse_federation_enabled and matrix_synapse_tls_federation_listener_enabled %} +{% if matrix_synapse_federation_port_enabled and matrix_synapse_tls_federation_listener_enabled %} # TLS-enabled listener: for when matrix traffic is sent directly to synapse. - port: 8448 tls: true @@ -197,7 +197,7 @@ listeners: x_forwarded: false resources: - - names: [federation] + - names: {{ matrix_synapse_federation_listener_resource_names|to_json }} compress: false {% endif %} @@ -213,7 +213,7 @@ listeners: - names: {{ matrix_synapse_http_listener_resource_names|to_json }} compress: false -{% if matrix_synapse_federation_enabled %} +{% if matrix_synapse_federation_port_enabled %} # Unsecure HTTP listener (Federation API): for when matrix traffic passes through a reverse proxy # that unwraps TLS. - port: 8048 @@ -223,7 +223,7 @@ listeners: x_forwarded: true resources: - - names: [federation] + - names: {{ matrix_synapse_federation_listener_resource_names|to_json }} compress: false {% endif %} From ad92c61fdddb21539f1286c65696902aad035a0c Mon Sep 17 00:00:00 2001 From: benkuly <12199167+benkuly@users.noreply.github.com> Date: Wed, 9 Dec 2020 09:45:44 +0100 Subject: [PATCH 097/217] updated matrix-sms-bridge --- roles/matrix-bridge-sms/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-sms/defaults/main.yml b/roles/matrix-bridge-sms/defaults/main.yml index 82f9b841..3c6b4c61 100644 --- a/roles/matrix-bridge-sms/defaults/main.yml +++ b/roles/matrix-bridge-sms/defaults/main.yml @@ -3,7 +3,7 @@ matrix_sms_bridge_enabled: true -matrix_sms_bridge_docker_image: "docker.io/folivonet/matrix-sms-bridge:0.5.4" +matrix_sms_bridge_docker_image: "docker.io/folivonet/matrix-sms-bridge:0.5.5" matrix_sms_bridge_base_path: "{{ matrix_base_data_path }}/matrix-sms-bridge" matrix_sms_bridge_config_path: "{{ matrix_base_data_path }}/matrix-sms-bridge/config" From aa86e0dac65df44c21820b6cc451617c830b4ae1 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 9 Dec 2020 13:30:28 +0200 Subject: [PATCH 098/217] Upgrade Synapse (v1.23.0 -> v1.24.0) Because the ARM images are not pushed yet, we hold back to v1.23.0 for now. --- roles/matrix-synapse/defaults/main.yml | 3 +- .../templates/synapse/homeserver.yaml.j2 | 59 ++++++++++++++----- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 35d40c7c..02d788a3 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -6,8 +6,9 @@ matrix_synapse_enabled: true matrix_synapse_container_image_self_build: false matrix_synapse_container_image_self_build_repo: "https://github.com/matrix-org/synapse.git" -matrix_synapse_docker_image: "{{ matrix_synapse_docker_image_name_prefix }}matrixdotorg/synapse:v1.23.0" +matrix_synapse_docker_image: "{{ matrix_synapse_docker_image_name_prefix }}matrixdotorg/synapse:{{ matrix_synapse_docker_image_tag }}" matrix_synapse_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_container_image_self_build else 'docker.io/' }}" +matrix_synapse_docker_image_tag: "{{ 'v1.24.0' if matrix_architecture == 'amd64' else 'v1.23.0' }}" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse" diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index c64ed9cc..41b28c88 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -1202,8 +1202,9 @@ allow_guest_access: {{ matrix_synapse_allow_guest_access|to_json }} # email will be globally disabled. # # Additionally, if `msisdn` is not set, registration and password resets via msisdn -# will be disabled regardless. This is due to Synapse currently not supporting any -# method of sending SMS messages on its own. +# will be disabled regardless, and users will not be able to associate an msisdn +# identifier to their account. This is due to Synapse currently not supporting +# any method of sending SMS messages on its own. # # To enable using an identity server for operations regarding a particular third-party # identifier type, set the value to the URL of that identity server as shown in the @@ -1522,6 +1523,12 @@ saml2_config: # remote: # - url: https://our_idp/metadata.xml + # Allowed clock difference in seconds between the homeserver and IdP. + # + # Uncomment the below to increase the accepted time difference from 0 to 3 seconds. + # + #accepted_time_diff: 3 + # By default, the user has to go to our login page first. If you'd like # to allow IdP-initiated login, set 'allow_unsolicited: true' in a # 'service.sp' section: @@ -1644,6 +1651,14 @@ saml2_config: # - attribute: department # value: "sales" + # If the metadata XML contains multiple IdP entities then the `idp_entityid` + # option must be set to the entity to redirect users to. + # + # Most deployments only have a single IdP entity and so should omit this + # option. + # + #idp_entityid: 'https://our_idp/entityid' + # Enable OpenID Connect (OIDC) / OAuth 2.0 for registration and login. # @@ -2222,21 +2237,35 @@ password_providers: {% endif %} +## Push ## -# Clients requesting push notifications can either have the body of -# the message sent in the notification poke along with other details -# like the sender, or just the event ID and room ID (`event_id_only`). -# If clients choose the former, this option controls whether the -# notification request includes the content of the event (other details -# like the sender are still included). For `event_id_only` push, it -# has no effect. -# -# For modern android devices the notification content will still appear -# because it is loaded by the app. iPhone, however will send a -# notification saying only that a message arrived and who it came from. -# push: - include_content: {{ matrix_synapse_push_include_content|to_json }} + # Clients requesting push notifications can either have the body of + # the message sent in the notification poke along with other details + # like the sender, or just the event ID and room ID (`event_id_only`). + # If clients choose the former, this option controls whether the + # notification request includes the content of the event (other details + # like the sender are still included). For `event_id_only` push, it + # has no effect. + # + # For modern android devices the notification content will still appear + # because it is loaded by the app. iPhone, however will send a + # notification saying only that a message arrived and who it came from. + # + # The default value is "true" to include message details. Uncomment to only + # include the event ID and room ID in push notification payloads. + # + include_content: {{ matrix_synapse_push_include_content|to_json }} + + # When a push notification is received, an unread count is also sent. + # This number can either be calculated as the number of unread messages + # for the user, or the number of *rooms* the user has unread messages in. + # + # The default value is "true", meaning push clients will see the number of + # rooms with unread messages in them. Uncomment to instead send the number + # of unread messages. + # + #group_unread_count_by_room: false # Spam checkers are third-party modules that can block specific actions From 245b749946a971e3b019d7be5500a59da80d1b24 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 9 Dec 2020 20:54:18 +0200 Subject: [PATCH 099/217] Upgrade Synapse for ARM (v1.23.0 -> v1.24.0) Continuation of aa86e0dac65, now that ARM images are out. --- roles/matrix-synapse/defaults/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 02d788a3..0dc71646 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -8,7 +8,10 @@ matrix_synapse_container_image_self_build_repo: "https://github.com/matrix-org/s matrix_synapse_docker_image: "{{ matrix_synapse_docker_image_name_prefix }}matrixdotorg/synapse:{{ matrix_synapse_docker_image_tag }}" matrix_synapse_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_container_image_self_build else 'docker.io/' }}" -matrix_synapse_docker_image_tag: "{{ 'v1.24.0' if matrix_architecture == 'amd64' else 'v1.23.0' }}" +# The if statement below may look silly at times (leading to the same version being returned), +# but ARM-compatible container images are only released 1-7 hours after a release, +# so we may often be on different versions for different architectures when new Synapse releases come out. +matrix_synapse_docker_image_tag: "{{ 'v1.24.0' if matrix_architecture == 'amd64' else 'v1.24.0' }}" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse" From 673e19f8307bdfc76dc4d1e63dcab40c84ebb37e Mon Sep 17 00:00:00 2001 From: John Goerzen Date: Wed, 9 Dec 2020 21:20:06 -0600 Subject: [PATCH 100/217] Correct inabillity for appservice-discord to connect After recently updating my matrix-docker-ansible-deploy installation, matrix-appservice-discord would refuse to start, logging ECONNREFUSED to https://matrix.[mydomain]:443, which was resolving to 172.18.0.2 due to the `--hostname` in mailer grabbing that hostname. Curious why the IRC bridge didn't have this issue, I looked into it, and it was connecting to `http://matrix-synapse:8008`. Correcting this one to that URL resolved the issue. --- roles/matrix-bridge-appservice-discord/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml index 9ca06b05..357b93d6 100644 --- a/roles/matrix-bridge-appservice-discord/defaults/main.yml +++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml @@ -37,7 +37,7 @@ matrix_appservice_discord_appservice_url: 'http://matrix-appservice-discord:9005 matrix_appservice_discord_bridge_domain: "{{ matrix_domain }}" # As of right now, the homeserver URL must be a public URL. See below. -matrix_appservice_discord_bridge_homeserverUrl: "{{ matrix_homeserver_url }}" +matrix_appservice_discord_bridge_homeserverUrl: "http://matrix-synapse:8008" matrix_appservice_discord_bridge_disablePresence: false matrix_appservice_discord_bridge_enableSelfServiceBridging: false From d08b27784f222effcbce2abf924bf07bbe0893be Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 10 Dec 2020 11:36:39 +0200 Subject: [PATCH 101/217] Fix systemd services autostart problem with Docker 20.10 The Docker 19.04 -> 20.10 upgrade contains the following change in `/usr/lib/systemd/system/docker.service`: ``` -BindsTo=containerd.service -After=network-online.target firewalld.service containerd.service +After=network-online.target firewalld.service containerd.service multi-user.target -Requires=docker.socket +Requires=docker.socket containerd.service Wants=network-online.target ``` The `multi-user.target` requirement in `After` seems to be in conflict with our `WantedBy=multi-user.target` and `After=docker.service` / `Requires=docker.service` definitions, causing the following error on startup for all of our systemd services: > Job matrix-synapse.service/start deleted to break ordering cycle starting with multi-user.target/start A workaround which appears to work is to add `DefaultDependencies=no` to all of our services. --- .../templates/systemd/matrix-bot-matrix-reminder-bot.service.j2 | 1 + .../templates/systemd/matrix-appservice-discord.service.j2 | 1 + .../templates/systemd/matrix-appservice-irc.service.j2 | 1 + .../templates/systemd/matrix-appservice-slack.service.j2 | 1 + .../templates/systemd/matrix-appservice-webhooks.service.j2 | 1 + .../templates/systemd/matrix-mautrix-facebook.service.j2 | 1 + .../templates/systemd/matrix-mautrix-hangouts.service.j2 | 1 + .../templates/systemd/matrix-mautrix-telegram.service.j2 | 1 + .../templates/systemd/matrix-mautrix-whatsapp.service.j2 | 1 + .../templates/systemd/matrix-mx-puppet-discord.service.j2 | 1 + .../templates/systemd/matrix-mx-puppet-instagram.service.j2 | 1 + .../templates/systemd/matrix-mx-puppet-skype.service.j2 | 1 + .../templates/systemd/matrix-mx-puppet-slack.service.j2 | 1 + .../templates/systemd/matrix-mx-puppet-steam.service.j2 | 1 + .../templates/systemd/matrix-mx-puppet-twitter.service.j2 | 1 + .../templates/systemd/matrix-sms-bridge.service.j2 | 1 + .../templates/systemd/matrix-client-element.service.j2 | 1 + .../matrix-corporal/templates/systemd/matrix-corporal.service.j2 | 1 + roles/matrix-coturn/templates/systemd/matrix-coturn.service.j2 | 1 + .../templates/systemd/matrix-dimension.service.j2 | 1 + .../templates/systemd/matrix-dynamic-dns.service.j2 | 1 + .../templates/systemd/matrix-email2matrix.service.j2 | 1 + .../matrix-jitsi/templates/jicofo/matrix-jitsi-jicofo.service.j2 | 1 + roles/matrix-jitsi/templates/jvb/matrix-jitsi-jvb.service.j2 | 1 + .../templates/prosody/matrix-jitsi-prosody.service.j2 | 1 + roles/matrix-jitsi/templates/web/matrix-jitsi-web.service.j2 | 1 + roles/matrix-ma1sd/templates/systemd/matrix-ma1sd.service.j2 | 1 + roles/matrix-mailer/templates/systemd/matrix-mailer.service.j2 | 1 + .../templates/systemd/matrix-nginx-proxy.service.j2 | 1 + .../matrix-postgres/templates/systemd/matrix-postgres.service.j2 | 1 + .../templates/systemd/matrix-registration.service.j2 | 1 + .../templates/systemd/matrix-synapse-admin.service.j2 | 1 + .../templates/goofys/systemd/matrix-goofys.service.j2 | 1 + .../templates/synapse/systemd/matrix-synapse.service.j2 | 1 + 34 files changed, 34 insertions(+) diff --git a/roles/matrix-bot-matrix-reminder-bot/templates/systemd/matrix-bot-matrix-reminder-bot.service.j2 b/roles/matrix-bot-matrix-reminder-bot/templates/systemd/matrix-bot-matrix-reminder-bot.service.j2 index 23493c54..950242e5 100644 --- a/roles/matrix-bot-matrix-reminder-bot/templates/systemd/matrix-bot-matrix-reminder-bot.service.j2 +++ b/roles/matrix-bot-matrix-reminder-bot/templates/systemd/matrix-bot-matrix-reminder-bot.service.j2 @@ -8,6 +8,7 @@ After={{ service }} {% for service in matrix_bot_matrix_reminder_bot_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-bridge-appservice-discord/templates/systemd/matrix-appservice-discord.service.j2 b/roles/matrix-bridge-appservice-discord/templates/systemd/matrix-appservice-discord.service.j2 index 6a44a3e6..7ec15752 100644 --- a/roles/matrix-bridge-appservice-discord/templates/systemd/matrix-appservice-discord.service.j2 +++ b/roles/matrix-bridge-appservice-discord/templates/systemd/matrix-appservice-discord.service.j2 @@ -8,6 +8,7 @@ After={{ service }} {% for service in matrix_appservice_discord_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-bridge-appservice-irc/templates/systemd/matrix-appservice-irc.service.j2 b/roles/matrix-bridge-appservice-irc/templates/systemd/matrix-appservice-irc.service.j2 index 95638612..c86eb835 100644 --- a/roles/matrix-bridge-appservice-irc/templates/systemd/matrix-appservice-irc.service.j2 +++ b/roles/matrix-bridge-appservice-irc/templates/systemd/matrix-appservice-irc.service.j2 @@ -8,6 +8,7 @@ After={{ service }} {% for service in matrix_appservice_irc_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-bridge-appservice-slack/templates/systemd/matrix-appservice-slack.service.j2 b/roles/matrix-bridge-appservice-slack/templates/systemd/matrix-appservice-slack.service.j2 index 1c68294f..b16b2fed 100644 --- a/roles/matrix-bridge-appservice-slack/templates/systemd/matrix-appservice-slack.service.j2 +++ b/roles/matrix-bridge-appservice-slack/templates/systemd/matrix-appservice-slack.service.j2 @@ -8,6 +8,7 @@ After={{ service }} {% for service in matrix_appservice_slack_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-bridge-appservice-webhooks/templates/systemd/matrix-appservice-webhooks.service.j2 b/roles/matrix-bridge-appservice-webhooks/templates/systemd/matrix-appservice-webhooks.service.j2 index 08f5813f..2049ee65 100644 --- a/roles/matrix-bridge-appservice-webhooks/templates/systemd/matrix-appservice-webhooks.service.j2 +++ b/roles/matrix-bridge-appservice-webhooks/templates/systemd/matrix-appservice-webhooks.service.j2 @@ -8,6 +8,7 @@ After={{ service }} {% for service in matrix_appservice_webhooks_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 b/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 index b593a2e3..18184650 100644 --- a/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 +++ b/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 @@ -8,6 +8,7 @@ After={{ service }} {% for service in matrix_mautrix_facebook_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2 b/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2 index 58433982..26280da3 100644 --- a/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2 +++ b/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2 @@ -8,6 +8,7 @@ After={{ service }} {% for service in matrix_mautrix_hangouts_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-bridge-mautrix-telegram/templates/systemd/matrix-mautrix-telegram.service.j2 b/roles/matrix-bridge-mautrix-telegram/templates/systemd/matrix-mautrix-telegram.service.j2 index c0fb8938..d61cdaa4 100644 --- a/roles/matrix-bridge-mautrix-telegram/templates/systemd/matrix-mautrix-telegram.service.j2 +++ b/roles/matrix-bridge-mautrix-telegram/templates/systemd/matrix-mautrix-telegram.service.j2 @@ -8,6 +8,7 @@ After={{ service }} {% for service in matrix_mautrix_telegram_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-bridge-mautrix-whatsapp/templates/systemd/matrix-mautrix-whatsapp.service.j2 b/roles/matrix-bridge-mautrix-whatsapp/templates/systemd/matrix-mautrix-whatsapp.service.j2 index ac2b961e..972d4e01 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/templates/systemd/matrix-mautrix-whatsapp.service.j2 +++ b/roles/matrix-bridge-mautrix-whatsapp/templates/systemd/matrix-mautrix-whatsapp.service.j2 @@ -8,6 +8,7 @@ After={{ service }} {% for service in matrix_mautrix_whatsapp_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-bridge-mx-puppet-discord/templates/systemd/matrix-mx-puppet-discord.service.j2 b/roles/matrix-bridge-mx-puppet-discord/templates/systemd/matrix-mx-puppet-discord.service.j2 index 78737a17..36a4da8b 100644 --- a/roles/matrix-bridge-mx-puppet-discord/templates/systemd/matrix-mx-puppet-discord.service.j2 +++ b/roles/matrix-bridge-mx-puppet-discord/templates/systemd/matrix-mx-puppet-discord.service.j2 @@ -8,6 +8,7 @@ After={{ service }} {% for service in matrix_mx_puppet_discord_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-bridge-mx-puppet-instagram/templates/systemd/matrix-mx-puppet-instagram.service.j2 b/roles/matrix-bridge-mx-puppet-instagram/templates/systemd/matrix-mx-puppet-instagram.service.j2 index 81e3e081..4c94c84a 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/templates/systemd/matrix-mx-puppet-instagram.service.j2 +++ b/roles/matrix-bridge-mx-puppet-instagram/templates/systemd/matrix-mx-puppet-instagram.service.j2 @@ -8,6 +8,7 @@ After={{ service }} {% for service in matrix_mx_puppet_instagram_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-bridge-mx-puppet-skype/templates/systemd/matrix-mx-puppet-skype.service.j2 b/roles/matrix-bridge-mx-puppet-skype/templates/systemd/matrix-mx-puppet-skype.service.j2 index 8a46cb25..cd958662 100644 --- a/roles/matrix-bridge-mx-puppet-skype/templates/systemd/matrix-mx-puppet-skype.service.j2 +++ b/roles/matrix-bridge-mx-puppet-skype/templates/systemd/matrix-mx-puppet-skype.service.j2 @@ -8,6 +8,7 @@ After={{ service }} {% for service in matrix_mx_puppet_skype_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-bridge-mx-puppet-slack/templates/systemd/matrix-mx-puppet-slack.service.j2 b/roles/matrix-bridge-mx-puppet-slack/templates/systemd/matrix-mx-puppet-slack.service.j2 index 23c2504f..2b1456f5 100644 --- a/roles/matrix-bridge-mx-puppet-slack/templates/systemd/matrix-mx-puppet-slack.service.j2 +++ b/roles/matrix-bridge-mx-puppet-slack/templates/systemd/matrix-mx-puppet-slack.service.j2 @@ -8,6 +8,7 @@ After={{ service }} {% for service in matrix_mx_puppet_slack_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-bridge-mx-puppet-steam/templates/systemd/matrix-mx-puppet-steam.service.j2 b/roles/matrix-bridge-mx-puppet-steam/templates/systemd/matrix-mx-puppet-steam.service.j2 index 986c5d2c..e263154b 100644 --- a/roles/matrix-bridge-mx-puppet-steam/templates/systemd/matrix-mx-puppet-steam.service.j2 +++ b/roles/matrix-bridge-mx-puppet-steam/templates/systemd/matrix-mx-puppet-steam.service.j2 @@ -8,6 +8,7 @@ After={{ service }} {% for service in matrix_mx_puppet_steam_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-bridge-mx-puppet-twitter/templates/systemd/matrix-mx-puppet-twitter.service.j2 b/roles/matrix-bridge-mx-puppet-twitter/templates/systemd/matrix-mx-puppet-twitter.service.j2 index 6afb6fb4..8d7898eb 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/templates/systemd/matrix-mx-puppet-twitter.service.j2 +++ b/roles/matrix-bridge-mx-puppet-twitter/templates/systemd/matrix-mx-puppet-twitter.service.j2 @@ -8,6 +8,7 @@ After={{ service }} {% for service in matrix_mx_puppet_twitter_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge.service.j2 b/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge.service.j2 index 7c96f9f4..0eb0eb58 100644 --- a/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge.service.j2 +++ b/roles/matrix-bridge-sms/templates/systemd/matrix-sms-bridge.service.j2 @@ -8,6 +8,7 @@ After={{ service }} {% for service in matrix_sms_bridge_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-client-element/templates/systemd/matrix-client-element.service.j2 b/roles/matrix-client-element/templates/systemd/matrix-client-element.service.j2 index 39b7aa52..e0dd2e7e 100644 --- a/roles/matrix-client-element/templates/systemd/matrix-client-element.service.j2 +++ b/roles/matrix-client-element/templates/systemd/matrix-client-element.service.j2 @@ -5,6 +5,7 @@ Description=Matrix Element server Requires={{ service }} After={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-corporal/templates/systemd/matrix-corporal.service.j2 b/roles/matrix-corporal/templates/systemd/matrix-corporal.service.j2 index 97ec7d89..e8ce8c0a 100644 --- a/roles/matrix-corporal/templates/systemd/matrix-corporal.service.j2 +++ b/roles/matrix-corporal/templates/systemd/matrix-corporal.service.j2 @@ -5,6 +5,7 @@ Description=Matrix Corporal Requires={{ service }} After={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-coturn/templates/systemd/matrix-coturn.service.j2 b/roles/matrix-coturn/templates/systemd/matrix-coturn.service.j2 index e484b59b..16ca5d2a 100644 --- a/roles/matrix-coturn/templates/systemd/matrix-coturn.service.j2 +++ b/roles/matrix-coturn/templates/systemd/matrix-coturn.service.j2 @@ -5,6 +5,7 @@ Description=Matrix Coturn server Requires={{ service }} After={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 b/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 index bad94f64..94c38491 100644 --- a/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 +++ b/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 @@ -3,6 +3,7 @@ Description=Matrix Dimension After=docker.service Requires=docker.service +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-dynamic-dns/templates/systemd/matrix-dynamic-dns.service.j2 b/roles/matrix-dynamic-dns/templates/systemd/matrix-dynamic-dns.service.j2 index 7e73b587..df7d810a 100644 --- a/roles/matrix-dynamic-dns/templates/systemd/matrix-dynamic-dns.service.j2 +++ b/roles/matrix-dynamic-dns/templates/systemd/matrix-dynamic-dns.service.j2 @@ -8,6 +8,7 @@ After={{ service }} {% for service in matrix_dynamic_dns_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-email2matrix/templates/systemd/matrix-email2matrix.service.j2 b/roles/matrix-email2matrix/templates/systemd/matrix-email2matrix.service.j2 index 52806f35..1577877b 100644 --- a/roles/matrix-email2matrix/templates/systemd/matrix-email2matrix.service.j2 +++ b/roles/matrix-email2matrix/templates/systemd/matrix-email2matrix.service.j2 @@ -3,6 +3,7 @@ Description=Email2Matrix After=docker.service Requires=docker.service +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-jitsi/templates/jicofo/matrix-jitsi-jicofo.service.j2 b/roles/matrix-jitsi/templates/jicofo/matrix-jitsi-jicofo.service.j2 index f19d08ff..6b5cc941 100644 --- a/roles/matrix-jitsi/templates/jicofo/matrix-jitsi-jicofo.service.j2 +++ b/roles/matrix-jitsi/templates/jicofo/matrix-jitsi-jicofo.service.j2 @@ -5,6 +5,7 @@ Description=Matrix jitsi-jicofo server Requires={{ service }} After={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-jitsi/templates/jvb/matrix-jitsi-jvb.service.j2 b/roles/matrix-jitsi/templates/jvb/matrix-jitsi-jvb.service.j2 index 7fcfeec6..2931133f 100644 --- a/roles/matrix-jitsi/templates/jvb/matrix-jitsi-jvb.service.j2 +++ b/roles/matrix-jitsi/templates/jvb/matrix-jitsi-jvb.service.j2 @@ -5,6 +5,7 @@ Description=Matrix jitsi-jvb server Requires={{ service }} After={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-jitsi/templates/prosody/matrix-jitsi-prosody.service.j2 b/roles/matrix-jitsi/templates/prosody/matrix-jitsi-prosody.service.j2 index 74b13df3..4f532d89 100644 --- a/roles/matrix-jitsi/templates/prosody/matrix-jitsi-prosody.service.j2 +++ b/roles/matrix-jitsi/templates/prosody/matrix-jitsi-prosody.service.j2 @@ -5,6 +5,7 @@ Description=Matrix jitsi-prosody server Requires={{ service }} After={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-jitsi/templates/web/matrix-jitsi-web.service.j2 b/roles/matrix-jitsi/templates/web/matrix-jitsi-web.service.j2 index e13f9633..1978fb0e 100644 --- a/roles/matrix-jitsi/templates/web/matrix-jitsi-web.service.j2 +++ b/roles/matrix-jitsi/templates/web/matrix-jitsi-web.service.j2 @@ -5,6 +5,7 @@ Description=Matrix jitsi-web server Requires={{ service }} After={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-ma1sd/templates/systemd/matrix-ma1sd.service.j2 b/roles/matrix-ma1sd/templates/systemd/matrix-ma1sd.service.j2 index 3c49ec75..95f15254 100644 --- a/roles/matrix-ma1sd/templates/systemd/matrix-ma1sd.service.j2 +++ b/roles/matrix-ma1sd/templates/systemd/matrix-ma1sd.service.j2 @@ -8,6 +8,7 @@ After={{ service }} {% for service in matrix_ma1sd_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-mailer/templates/systemd/matrix-mailer.service.j2 b/roles/matrix-mailer/templates/systemd/matrix-mailer.service.j2 index 1371a861..cab03128 100644 --- a/roles/matrix-mailer/templates/systemd/matrix-mailer.service.j2 +++ b/roles/matrix-mailer/templates/systemd/matrix-mailer.service.j2 @@ -3,6 +3,7 @@ Description=Matrix mailer After=docker.service Requires=docker.service +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2 b/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2 index 0945c25e..58f5c953 100644 --- a/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2 +++ b/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2 @@ -8,6 +8,7 @@ After={{ service }} {% for service in matrix_nginx_proxy_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2 b/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2 index 47f19e9b..0a935fb0 100644 --- a/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2 +++ b/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2 @@ -3,6 +3,7 @@ Description=Matrix Postgres server After=docker.service Requires=docker.service +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-registration/templates/systemd/matrix-registration.service.j2 b/roles/matrix-registration/templates/systemd/matrix-registration.service.j2 index 052b7d95..3744c2de 100644 --- a/roles/matrix-registration/templates/systemd/matrix-registration.service.j2 +++ b/roles/matrix-registration/templates/systemd/matrix-registration.service.j2 @@ -8,6 +8,7 @@ After={{ service }} {% for service in matrix_registration_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-synapse-admin/templates/systemd/matrix-synapse-admin.service.j2 b/roles/matrix-synapse-admin/templates/systemd/matrix-synapse-admin.service.j2 index c03c627b..7b1e40de 100644 --- a/roles/matrix-synapse-admin/templates/systemd/matrix-synapse-admin.service.j2 +++ b/roles/matrix-synapse-admin/templates/systemd/matrix-synapse-admin.service.j2 @@ -8,6 +8,7 @@ After={{ service }} {% for service in matrix_synapse_admin_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-synapse/templates/goofys/systemd/matrix-goofys.service.j2 b/roles/matrix-synapse/templates/goofys/systemd/matrix-goofys.service.j2 index c3ec9249..0bbfde99 100644 --- a/roles/matrix-synapse/templates/goofys/systemd/matrix-goofys.service.j2 +++ b/roles/matrix-synapse/templates/goofys/systemd/matrix-goofys.service.j2 @@ -3,6 +3,7 @@ Description=Matrix Goofys media store After=docker.service Requires=docker.service +DefaultDependencies=no [Service] Type=simple diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 index 791d324e..30c85b99 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 @@ -8,6 +8,7 @@ After={{ service }} {% for service in matrix_synapse_systemd_wanted_services_list %} Wants={{ service }} {% endfor %} +DefaultDependencies=no [Service] Type=simple From d9f4914e0d78fa45c7a76526a660faf94213b3df Mon Sep 17 00:00:00 2001 From: transcaffeine Date: Sun, 29 Nov 2020 15:20:22 +0100 Subject: [PATCH 102/217] WIP: postgres: create databases for all services If a service is enabled, a database for it is created in postgres with a uniqque password. The service can then use this database for data storage instead of relying on sqlite. --- group_vars/matrix_servers | 64 +++++++++++++++++++ .../templates/config.yaml.j2 | 4 +- .../templates/config.yaml.j2 | 4 +- .../templates/config.yaml.j2 | 3 + .../templates/config.yaml.j2 | 2 +- .../templates/config.yaml.j2 | 2 +- .../templates/config.yaml.j2 | 2 +- .../templates/config.yaml.j2 | 2 +- .../templates/config.yaml.j2 | 4 +- .../templates/config.yaml.j2 | 4 +- .../templates/config.yaml.j2 | 4 +- .../templates/config.yaml.j2 | 4 +- .../templates/config.yaml.j2 | 4 +- .../templates/config.yaml.j2 | 4 +- .../matrix-dimension/templates/config.yaml.j2 | 2 +- .../matrix-postgres/tasks/setup_postgres.yml | 30 +++++++++ 16 files changed, 118 insertions(+), 21 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 78a03851..8a099275 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -889,6 +889,70 @@ matrix_postgres_connection_username: "synapse" matrix_postgres_connection_password: "synapse-password" matrix_postgres_db_name: "homeserver" +matrix_postgres_additional_databases: | + {{ + ([{ + name: 'matrix_appservice_discord', + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string + }] if matrix_appservice_discord_enabled else []) + + ([{ + name: 'matrix_appservice_slack' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string + }] if matrix_appservice_slack_enabled else []) + + ([{ + name: 'matrix_appservice_irc' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string + }] if matrix_appservice_irc_enabled else []) + + ([{ + name: 'mautrix-bridge-facebook' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string + }] if matrix_mautrix_facebook_enabled else []) + + ([{ + name: 'mautrix_bridge_hangouts' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string + }] if matrix_mautrix_hangouts_enabled else []) + + ([{ + name: 'mautrix_bridge_telegram' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string + }] if matrix_mautrix_telegram_enabled else []) + + ([{ + name: 'mautrix_bridge_whatsapp' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string + }] if matrix_mautrix_whatsapp_enabled else []) + + ([{ + name: 'matrix_bridge_sms' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string + }] if matrix_sms_bridge_enabled else []) + + ([{ + name: 'matrix_puppet_skype' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string + }] if matrix_mx_puppet_skype_enabled else []) + + ([{ + name: 'matrix_puppet_slack' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string + }] if matrix_mx_puppet_slack_enabled else []) + + ([{ + name: 'matrix_puppet_twitter' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string + }] if matrix_mx_puppet_twitter_enabled else []) + + ([{ + name: 'matrix_puppet_instagram' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string + ] if matrix_mx_puppet_instagram_enabled else []) + + ([{ + name: 'matrix_puppet_discord' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string + }] if matrix_mx_puppet_discord_enabled else []) + + ([{ + name: 'matrix_puppet_steam' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string + }] if matrix_mx_puppet_steam_enabled else []) + + ([{ + name: 'matrix_dimension' + pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string + }] if matrix_dimension_enabled else []) + }} + ###################################################################### # # /matrix-postgres diff --git a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 index b99f522a..de4182d9 100644 --- a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 +++ b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 @@ -58,8 +58,8 @@ database: # If you are migrating, see https://github.com/Half-Shot/matrix-appservice-discord/blob/master/docs/howto.md#migrate-to-postgres-from-sqlite # WARNING: You will almost certainly be fine with sqlite unless your bridge # is in heavy demand and you suffer from IO slowness. - filename: "/data/discord.db" - # connString: "postgresql://user:password@localhost/database_name" + #filename: "/data/discord.db" + connString: "postgresql://matrix_appservice_discord:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_appservice_discord') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_discord" room: # Set the default visibility of alias rooms, defaults to "public". # One of: "public", "private" diff --git a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 index 3daa1898..83b71835 100644 --- a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 +++ b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 @@ -127,8 +127,8 @@ advanced: # Use an external database to store bridge state. database: # database engine (must be 'postgres' or 'nedb'). Default: nedb - engine: "nedb" + engine: "postgres" # Either a PostgreSQL connection string, or a path to the NeDB storage directory. # For postgres, it must start with postgres:// # For NeDB, it must start with nedb://. The path is relative to the project directory. - connectionString: "nedb:///data" + connectionString: "postgres://matrix_appservice_irc:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_appservice_irc') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_irc" diff --git a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 index 8f48d317..6c491134 100644 --- a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 +++ b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 @@ -10,5 +10,8 @@ homeserver: server_name: "{{ matrix_domain }}" dbdir: "/data" +db: + engine: "postgres" + connectionString: "postgresql://matrix_appservice_slack:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_appservice_slack') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_slack" matrix_admin_room: "{{ matrix_appservice_slack_control_room_id }}" diff --git a/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 index 9d8de2d5..304571c1 100644 --- a/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 @@ -27,7 +27,7 @@ appservice: # Format examples: # SQLite: sqlite:///filename.db # Postgres: postgres://username:password@hostname/dbname - database: sqlite:////data/mautrix-facebook.db + database: sqlite://matrix_bridge_facebook:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_bridge_facebook') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_bridge_facebook # Public part of web server for out-of-Matrix interaction with the bridge. public: diff --git a/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2 index f274b203..e4e59ad2 100644 --- a/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2 @@ -27,7 +27,7 @@ appservice: # Format examples: # SQLite: sqlite:///filename.db # Postgres: postgres://username:password@hostname/dbname - database: sqlite:////data/mautrix-hangouts.db + database: postgres://mautrix_bridge_hangouts:{{ matrix_additional_databases | selectattr('name', 'equalto', 'mautrix_bridge_hangouts') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/mautrix_bridge_hangouts # The unique ID of this appservice. id: hangouts diff --git a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 index 1a4ac43e..65f17d33 100644 --- a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 @@ -27,7 +27,7 @@ appservice: # Format examples: # SQLite: sqlite:///filename.db # Postgres: postgres://username:password@hostname/dbname - database: sqlite:////data/mautrix-telegram.db + database: postgres://mautrix_bridge_telegram:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_bridge_telegram') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/mautrix_bridge_telegram # Public part of web server for out-of-Matrix interaction with the bridge. # Used for things like login if the user wants to make sure the 2FA password isn't stored in diff --git a/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 index a527a188..93956049 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 @@ -23,7 +23,7 @@ appservice: # The database URI. # SQLite: File name is enough. https://github.com/mattn/go-sqlite3#connection-string # Postgres: Connection string. For example, postgres://user:password@host/database - uri: mautrix-whatsapp.db + uri: postgres://matrix_bridge_whatsapp@{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_bridge_whatsapp') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_bridge_whatsapp # Maximum number of connections. Mostly relevant for Postgres. max_open_conns: 20 max_idle_conns: 2 diff --git a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 index 2c703796..88c591e2 100644 --- a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 @@ -105,10 +105,10 @@ database: # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - #connString: "postgres://user:pass@localhost/dbname?sslmode=disable" + connString: "postgres://matrix_puppet_discord:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_discord') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_discord?sslmode=disable" # Use SQLite3 as a database backend # The name of the database file - filename: /data/database.db + #filename: /data/database.db logging: # Log level of console output diff --git a/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 index 634fbaec..2d189195 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 @@ -49,10 +49,10 @@ database: # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - #connString: "postgres://user:pass@localhost/dbname?sslmode=disable" + connString: "postgres://matrix_puppet_instagram:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_instagram') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_instagram?sslmode=disable" # Use SQLite3 as a database backend # The name of the database file - filename: /data/database.db + #filename: /data/database.db logging: # Log level of console output diff --git a/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 index c7b5c870..a54ca758 100644 --- a/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 @@ -73,10 +73,10 @@ database: # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - #connString: "postgres://user:pass@localhost/dbname?sslmode=disable" + connString: "postgres://matrix_puppet_skype:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_skype') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_skype?sslmode=disable" # Use SQLite3 as a database backend # The name of the database file - filename: /data/database.db + #filename: /data/database.db provisioning: # Regex of Matrix IDs allowed to use the puppet bridge diff --git a/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 index b6e88784..7f1b21d7 100644 --- a/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 @@ -63,10 +63,10 @@ database: # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - #connString: "postgres://user:pass@localhost/dbname?sslmode=disable" + connString: "postgres://matrix_puppet_slack:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_slack') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_slack?sslmode=disable" # Use SQLite3 as a database backend # The name of the database file - filename: /data/database.db + #filename: /data/database.db logging: # Log level of console output diff --git a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 index d08982ca..14f86319 100644 --- a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 @@ -66,10 +66,10 @@ database: # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - #connString: "postgres://user:pass@localhost/dbname?sslmode=disable" + connString: "postgres://matrix_puppet_steam:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_steam') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_steam?sslmode=disable" # Use SQLite3 as a database backend # The name of the database file - filename: /data/database.db + #filename: /data/database.db logging: # Log level of console output diff --git a/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 index 7d3033b3..853d23fc 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 @@ -59,10 +59,10 @@ database: # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - #connString: "postgres://user:pass@localhost/dbname?sslmode=disable" + connString: "postgres://matrix_puppet_twitter:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_twitter') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_twitter?sslmode=disable" # Use SQLite3 as a database backend # The name of the database file - filename: /data/database.db + #filename: /data/database.db logging: # Log level of console output diff --git a/roles/matrix-dimension/templates/config.yaml.j2 b/roles/matrix-dimension/templates/config.yaml.j2 index a05b6c35..b7ca1ee3 100644 --- a/roles/matrix-dimension/templates/config.yaml.j2 +++ b/roles/matrix-dimension/templates/config.yaml.j2 @@ -44,7 +44,7 @@ widgetBlacklist: # Where the database for Dimension is database: - file: "dimension.db" + uri: "postgres://matrix_dimension:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_dimension') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_dimension" # Display settings that apply to self-hosted go-neb instances goneb: diff --git a/roles/matrix-postgres/tasks/setup_postgres.yml b/roles/matrix-postgres/tasks/setup_postgres.yml index f186bdca..85a8604c 100644 --- a/roles/matrix-postgres/tasks/setup_postgres.yml +++ b/roles/matrix-postgres/tasks/setup_postgres.yml @@ -162,3 +162,33 @@ - matrix-change-user-admin-status - matrix-postgres-update-user-password-hash when: "not matrix_postgres_enabled|bool" + +# Create additional databases +- name: Retrieve IP of postgres container + shell: "docker inspect matrix-postgres | jq -r '.[0].NetworkSettings.Networks.{{ matrix_docker_network }}.IPAddress'" + register: matirx_postgres_container_ip + +- name: Create additional users in postgres + postgresql_user: + name: "{{ item.name }}" + password: "{{ item.pass }}" + login_host: "{{ matrx_postgres_container_ip.stdout }}" + login_port: 5432 + login_user: "{{ matrix_postgres_connection_username }}" + login_password: "{{ matrix_postgres_connection_password }}" + login_db: "{{ matrix_postgres_db_name }}" + loop: matrix_postgres_additional_databases + when: matrix_postgres_enabed|bool + +- name: Create additional users in postgres + postgresql_db: + name: "{{ item.name }}" + owner: "{{ item.name }}" + lc_ctype: 'C' + lc_collate: 'C' + login_host: "{{ matrx_postgres_container_ip.stdout }}" + login_port: 5432 + login_user: "{{ matrix_postgres_connection_username }}" + login_password: "{{ matrix_postgres_connection_password }}" + loop: matrix_postgres_additional_databases + when: matrix_postgres_enabled|bool From 7593d969e316cc0144bce378a5be58c76c2c37ee Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 10 Dec 2020 23:51:11 +0200 Subject: [PATCH 103/217] Make matrix-mailer not occupy matrix_server_fqn_matrix Starting with Docker 20.10, `--hostname` seems to have the side-effect of making Docker's internal DNS server resolve said hostname to the IP address of the container. Because we were giving the mailer service a hostname of `matrix.DOMAIN`, all requests destined for `matrix.DOMAIN` originating from other services on the container network were resolving to `matrix-mailer`. This is obviously wrong. Initially reported here: https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/748 We normally try to not use the public hostname (and IP address) on the container network and try to make services talk to one another locally, but it sometimes could happen. With this, we use a `matrix-mailer` hostname for the matrix-mailer container. My testing shows that it doesn't cause any trouble with email deliverability. --- roles/matrix-mailer/templates/systemd/matrix-mailer.service.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-mailer/templates/systemd/matrix-mailer.service.j2 b/roles/matrix-mailer/templates/systemd/matrix-mailer.service.j2 index cab03128..9345a1d6 100644 --- a/roles/matrix-mailer/templates/systemd/matrix-mailer.service.j2 +++ b/roles/matrix-mailer/templates/systemd/matrix-mailer.service.j2 @@ -19,7 +19,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-mailer \ --tmpfs=/var/spool/exim:rw,noexec,nosuid,size=100m \ --network={{ matrix_docker_network }} \ --env-file={{ matrix_mailer_base_path }}/env-mailer \ - --hostname={{ matrix_server_fqn_matrix }} \ + --hostname=matrix-mailer \ {% for arg in matrix_mailer_container_extra_arguments %} {{ arg }} \ {% endfor %} From eae4f674708939c6b4bfd68e596fecad61b939ca Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 11 Dec 2020 00:12:56 +0200 Subject: [PATCH 104/217] Mention Docker 20.10 in the changelog --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5024d841..85700b58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +# 2020-12-11 + +## Docker 20.10 is here + +(No need to do anything special in relation to this. Just something to keep in mind) + +Docker 20.10 got released recently and your server will likely get it the next time you update. + +This is the first major Docker update in a long time and it packs a lot of changes. +Some of them introduced some breakage for us initially (see [here](https://github.com/spantaleev/matrix-docker-ansible-deploy/commit/d08b27784f222effcbce2abf924bf07bbe0893be) and [here](https://github.com/spantaleev/matrix-docker-ansible-deploy/commit/7593d969e316cc0144bce378a5be58c76c2c37ee)), but it should be all good now. + + # 2020-12-08 ## openid APIs exposed by default on the federation port when federation disabled From 3c2a644e5c8b672344b5aeafe374bcf70432b469 Mon Sep 17 00:00:00 2001 From: Aaron Raimist Date: Thu, 10 Dec 2020 16:28:48 -0600 Subject: [PATCH 105/217] Upgrade synapse-admin (v0.5.0 -> 0.6.1) --- roles/matrix-synapse-admin/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse-admin/defaults/main.yml b/roles/matrix-synapse-admin/defaults/main.yml index ce149dfd..1dbf0ad2 100644 --- a/roles/matrix-synapse-admin/defaults/main.yml +++ b/roles/matrix-synapse-admin/defaults/main.yml @@ -8,7 +8,7 @@ matrix_synapse_admin_container_self_build_repo: "https://github.com/Awesome-Tech matrix_synapse_admin_docker_src_files_path: "{{ matrix_base_data_path }}/synapse-admin/docker-src" -matrix_synapse_admin_docker_image: "{{ matrix_synapse_admin_docker_image_name_prefix }}awesometechnologies/synapse-admin:0.5.0" +matrix_synapse_admin_docker_image: "{{ matrix_synapse_admin_docker_image_name_prefix }}awesometechnologies/synapse-admin:0.6.1" matrix_synapse_admin_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_admin_container_self_build else 'docker.io/' }}" matrix_synapse_admin_docker_image_force_pull: "{{ matrix_synapse_admin_docker_image.endswith(':latest') }}" From 0a9109771dd1d380fed00208fe1f98b1078265b6 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 11 Dec 2020 22:17:47 +0200 Subject: [PATCH 106/217] Use latest/master version of matrix-registration v0.7.0 is broken right now, because it calls `/_matrix/client/r0/admin/register`, which is now at `/_synapse/admin/v1/register`. This has been fixed here: https://github.com/ZerataX/matrix-registration/commit/6b26255feada4c4f0ec49bb16a60a12b85476a0f .. but it's not part of any release. Switching to `master` (`docker.io/devture/zeratax-matrix-registration:latest`) until it gets resolved. Reported upstream here: https://github.com/ZerataX/matrix-registration/issues/43 --- roles/matrix-registration/defaults/main.yml | 3 ++- roles/matrix-registration/tasks/setup.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/matrix-registration/defaults/main.yml b/roles/matrix-registration/defaults/main.yml index 680e8bfe..87779faf 100644 --- a/roles/matrix-registration/defaults/main.yml +++ b/roles/matrix-registration/defaults/main.yml @@ -5,13 +5,14 @@ matrix_registration_enabled: true matrix_registration_container_image_self_build: false matrix_registration_container_image_self_build_repo: "https://github.com/ZerataX/matrix-registration" +matrix_registration_container_image_self_build_branch: "{{ 'master' if matrix_registration_version == 'latest' else matrix_registration_version }}" matrix_registration_base_path: "{{ matrix_base_data_path }}/matrix-registration" matrix_registration_config_path: "{{ matrix_registration_base_path }}/config" matrix_registration_data_path: "{{ matrix_registration_base_path }}/data" matrix_registration_docker_src_files_path: "{{ matrix_registration_base_path }}/docker-src" -matrix_registration_version: "v0.7.0" +matrix_registration_version: "latest" matrix_registration_docker_image: "{{ matrix_registration_docker_image_name_prefix }}devture/zeratax-matrix-registration:{{ matrix_registration_version }}" matrix_registration_docker_image_name_prefix: "{{ 'localhost/' if matrix_registration_container_image_self_build else 'docker.io/' }}" diff --git a/roles/matrix-registration/tasks/setup.yml b/roles/matrix-registration/tasks/setup.yml index 8427d950..dfe1ba8b 100644 --- a/roles/matrix-registration/tasks/setup.yml +++ b/roles/matrix-registration/tasks/setup.yml @@ -30,7 +30,7 @@ git: repo: "{{ matrix_registration_container_image_self_build_repo }}" dest: "{{ matrix_registration_docker_src_files_path }}" - version: "{{ matrix_registration_version }}" + version: "{{ matrix_registration_container_image_self_build_branch }}" force: "yes" register: matrix_registration_git_pull_results when: "matrix_registration_enabled|bool and matrix_registration_container_image_self_build|bool" From 86988ae1805525ab7c48ce95b34559c7f0647228 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 11 Dec 2020 22:52:42 +0200 Subject: [PATCH 107/217] Switch matrix-registration to v0.7.1 Now that a new release has been made, we no longer need to use `latest` / `master`. Related to 0a9109771dd1d380f and https://github.com/ZerataX/matrix-registration/issues/43 --- roles/matrix-registration/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-registration/defaults/main.yml b/roles/matrix-registration/defaults/main.yml index 87779faf..95147665 100644 --- a/roles/matrix-registration/defaults/main.yml +++ b/roles/matrix-registration/defaults/main.yml @@ -12,7 +12,7 @@ matrix_registration_config_path: "{{ matrix_registration_base_path }}/config" matrix_registration_data_path: "{{ matrix_registration_base_path }}/data" matrix_registration_docker_src_files_path: "{{ matrix_registration_base_path }}/docker-src" -matrix_registration_version: "latest" +matrix_registration_version: "v0.7.1" matrix_registration_docker_image: "{{ matrix_registration_docker_image_name_prefix }}devture/zeratax-matrix-registration:{{ matrix_registration_version }}" matrix_registration_docker_image_name_prefix: "{{ 'localhost/' if matrix_registration_container_image_self_build else 'docker.io/' }}" From 47613e5a27ad226236c617eda86dbb61c692eb7a Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 11 Dec 2020 23:24:42 +0200 Subject: [PATCH 108/217] Remove synapse-janitor support Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/746 --- CHANGELOG.md | 7 ++ docs/maintenance-synapse.md | 24 ---- roles/matrix-base/defaults/main.yml | 1 - roles/matrix-postgres/defaults/main.yml | 2 - roles/matrix-postgres/tasks/main.yml | 5 - .../tasks/run_synapse_janitor.yml | 117 ------------------ 6 files changed, 7 insertions(+), 149 deletions(-) delete mode 100644 roles/matrix-postgres/tasks/run_synapse_janitor.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 85700b58..946ec5b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # 2020-12-11 +## synapse-janitor support removed + +We've removed support for the unmaintained [synapse-janitor](https://github.com/xwiki-labs/synapse_scripts) script. There's been past reports of it corrupting the Synapse database. Since there hasn't been any new development on it and it doesn't seem too useful nowadays, there's no point in including it in the playbook. + +If you need to clean up or compact your database, consider using the Synapse Admin APIs directly. See our [Synapse maintenance](docs/maintenance-synapse.md) and [Postgres maintenance](docs/maintenance-postgres.md) documentation pages for more details. + + ## Docker 20.10 is here (No need to do anything special in relation to this. Just something to keep in mind) diff --git a/docs/maintenance-synapse.md b/docs/maintenance-synapse.md index 79998299..143238c1 100644 --- a/docs/maintenance-synapse.md +++ b/docs/maintenance-synapse.md @@ -4,14 +4,11 @@ This document shows you how to perform various maintenance tasks related to the Table of contents: -- [Purging unused data with synapse-janitor](#purging-unused-data-with-synapse-janitor), for when you wish to delete unused data from the Synapse database - - [Purging old data with the Purge History API](#purging-old-data-with-the-purge-history-api), for when you wish to delete in-use (but old) data from the Synapse database - [Synapse maintenance](#synapse-maintenance) - [Purging old data with the Purge History API](#purging-old-data-with-the-purge-history-api) - [Compressing state with rust-synapse-compress-state](#compressing-state-with-rust-synapse-compress-state) - - [Purging unused data with synapse-janitor](#purging-unused-data-with-synapse-janitor) - [Browse and manipulate the database](#browse-and-manipulate-the-database) - [Browse and manipulate the database](#browse-and-manipulate-the-database), for when you really need to take matters into your own hands @@ -57,27 +54,6 @@ If you need to adjust this, pass: `--extra-vars='matrix_synapse_rust_synapse_com After state compression, you may wish to run a [`FULL` Postgres `VACUUM`](./maintenance-postgres.md#vacuuming-postgresql). -## Purging unused data with synapse-janitor - -**NOTE**: There are [reports](https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/465) that **synapse-janitor is dangerous to use and causes database corruption**. You may wish to refrain from using it. - -When you **leave** and **forget** a room, Synapse can clean up its data, but currently doesn't. -This **unused and unreachable data** remains in your database forever. - -There are external tools (like [synapse-janitor](https://github.com/xwiki-labs/synapse_scripts)), which are meant to solve this problem. - -To ask the playbook to run synapse-janitor, execute: - -```bash -ansible-playbook -i inventory/hosts setup.yml --tags=run-postgres-synapse-janitor,start -``` - -**Note**: this will automatically stop Synapse temporarily and restart it later. - -Running synapse-janitor potentially deletes a lot of data from the Postgres database. -You may wish to run a [`FULL` Postgres `VACUUM`](./maintenance-postgres.md#vacuuming-postgresql) after that. - - ## Browse and manipulate the database When the [matrix admin API](https://github.com/matrix-org/synapse/tree/master/docs/admin_api) and the other tools do not provide a more convenient way, having a look at synapse's postgresql database can satisfy a lot of admins' needs. diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index 164509b7..e0522ba8 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -106,7 +106,6 @@ matrix_docker_package_name: docker-ce run_postgres_import: true run_postgres_upgrade: true run_postgres_import_sqlite_db: true -run_postgres_synapse_janitor: true run_postgres_vacuum: true run_synapse_register_user: true run_synapse_update_user_password: true diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index 0d2f50e3..ec5cb3dc 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -30,5 +30,3 @@ matrix_postgres_container_extra_arguments: [] # # Takes an ":" or "" value (e.g. "127.0.0.1:5432"), or empty string to not expose. matrix_postgres_container_postgres_bind_port: "" - -matrix_postgres_tool_synapse_janitor: "https://raw.githubusercontent.com/xwiki-labs/synapse_scripts/a9188ff175ae581610f92d58ea6eac9a114d854b/synapse_janitor.sql" diff --git a/roles/matrix-postgres/tasks/main.yml b/roles/matrix-postgres/tasks/main.yml index 41b9c861..717b73e4 100644 --- a/roles/matrix-postgres/tasks/main.yml +++ b/roles/matrix-postgres/tasks/main.yml @@ -29,11 +29,6 @@ tags: - upgrade-postgres -- import_tasks: "{{ role_path }}/tasks/run_synapse_janitor.yml" - when: run_postgres_synapse_janitor|bool - tags: - - run-postgres-synapse-janitor - - import_tasks: "{{ role_path }}/tasks/run_vacuum.yml" when: run_postgres_vacuum|bool tags: diff --git a/roles/matrix-postgres/tasks/run_synapse_janitor.yml b/roles/matrix-postgres/tasks/run_synapse_janitor.yml deleted file mode 100644 index d7f283be..00000000 --- a/roles/matrix-postgres/tasks/run_synapse_janitor.yml +++ /dev/null @@ -1,117 +0,0 @@ ---- - -# Pre-checks - -- name: Fail if Postgres not enabled - fail: - msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot run synapse-janitor." - when: "not matrix_postgres_enabled|bool" - -- name: Fail if not aware of the risks - fail: - msg: >- - Using Synapse Janitor is considered dangerous and may break your database. - See https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/465. - If you'd like to run it anyway, add `--extra-vars='i_know_synapse_janitor_is_dangerous=1'` to your command. - when: "i_know_synapse_janitor_is_dangerous|default('') == ''" - -# Defaults - -- name: Set postgres_start_wait_time, if not provided - set_fact: - postgres_start_wait_time: 15 - when: "postgres_start_wait_time|default('') == ''" - -- name: Set postgres_synapse_janitor_wait_time, if not provided - set_fact: - postgres_synapse_janitor_wait_time: "{{ 7 * 86400 }}" - when: "postgres_synapse_janitor_wait_time|default('') == ''" - -- name: Set postgres_synapse_janitor_tool_path, if not provided - set_fact: - postgres_synapse_janitor_tool_path: "{{ matrix_postgres_base_path }}/synapse_janitor.sql" - when: "postgres_synapse_janitor_tool_path|default('') == ''" - - -# Actual janitor work - -- name: Download synapse-janitor tool - get_url: - url: "{{ matrix_postgres_tool_synapse_janitor }}" - dest: "{{ postgres_synapse_janitor_tool_path }}" - force: true - mode: 0550 - owner: "{{ matrix_user_username }}" - group: "{{ matrix_user_groupname }}" - -- name: Ensure matrix-postgres is started - service: - name: matrix-postgres - state: started - daemon_reload: yes - -- name: Wait a bit, so that Postgres can start - wait_for: - timeout: "{{ postgres_start_wait_time }}" - delegate_to: 127.0.0.1 - become: false - -- import_tasks: tasks/util/detect_existing_postgres_version.yml - -- name: Abort, if no existing Postgres version detected - fail: - msg: "Could not find existing Postgres installation" - when: "not matrix_postgres_detected_existing|bool" - -- name: Generate Postgres database synapse-janitor command - set_fact: - matrix_postgres_synapse_janitor_command: >- - {{ matrix_host_command_docker }} run --rm --name matrix-postgres-synapse-janitor - --user={{ matrix_user_uid }}:{{ matrix_user_gid }} - --cap-drop=ALL - --network={{ matrix_docker_network }} - --env-file={{ matrix_postgres_base_path }}/env-postgres-psql - --mount type=bind,src={{ postgres_synapse_janitor_tool_path }},dst=/synapse_janitor.sql,ro=true - {{ matrix_postgres_docker_image_latest }} - psql -v ON_ERROR_STOP=1 -h matrix-postgres {{ matrix_synapse_database_database }} -f /synapse_janitor.sql - -- name: Note about Postgres purging alternative - debug: - msg: >- - Running synapse-janitor with the following Postgres command: `{{ matrix_postgres_synapse_janitor_command }}`. - If this crashes, you can stop all processes (`systemctl stop matrix-*`), - start Postgres only (`systemctl start matrix-postgres`) - and manually run the above command directly on the server. - -- name: Populate service facts - service_facts: - -- set_fact: - matrix_postgres_synapse_was_running: "{{ ansible_facts.services['matrix-synapse.service']|default(none) is not none and ansible_facts.services['matrix-synapse.service'].state == 'running' }}" - -- name: Ensure matrix-synapse is stopped - service: - name: matrix-synapse - state: stopped - daemon_reload: yes - -- name: Run synapse-janitor - command: "{{ matrix_postgres_synapse_janitor_command }}" - async: "{{ postgres_synapse_janitor_wait_time }}" - poll: 10 - register: matrix_postgres_synapse_janitor_result - -# Intentionally show the results -- debug: var="matrix_postgres_synapse_janitor_result" - -- name: Ensure matrix-synapse is started, if it previously was - service: - name: matrix-synapse - state: started - daemon_reload: yes - when: "matrix_postgres_synapse_was_running|bool" - -- name: Delete synapse-janitor tool - file: - path: "{{ postgres_synapse_janitor_tool_path }}" - state: absent From d96d7f2a43306e9b74673c2d5f6930a87f44bd2a Mon Sep 17 00:00:00 2001 From: chagai95 <31655082+chagai95@users.noreply.github.com> Date: Sat, 12 Dec 2020 00:24:38 +0100 Subject: [PATCH 109/217] Adding jitsi as well --- docs/configuring-playbook-ssl-certificates.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/configuring-playbook-ssl-certificates.md b/docs/configuring-playbook-ssl-certificates.md index 5b5c7cdc..3dc65516 100644 --- a/docs/configuring-playbook-ssl-certificates.md +++ b/docs/configuring-playbook-ssl-certificates.md @@ -80,6 +80,7 @@ matrix_ssl_domains_to_obtain_certificates_for: - '{{ matrix_server_fqn_matrix }}' - '{{ matrix_server_fqn_element }}' - '{{ matrix_server_fqn_dimension }}' + - '{{ matrix_server_fqn_jitsi }}' - '{{ matrix_domain }}' ``` From bae411342204a42cab69710c33870fed02c33a4b Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sat, 12 Dec 2020 01:44:34 +0200 Subject: [PATCH 110/217] Mention that we possibly obtain a Jitsi certificate --- docs/configuring-playbook-ssl-certificates.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/configuring-playbook-ssl-certificates.md b/docs/configuring-playbook-ssl-certificates.md index 3dc65516..7f05a5b2 100644 --- a/docs/configuring-playbook-ssl-certificates.md +++ b/docs/configuring-playbook-ssl-certificates.md @@ -67,6 +67,7 @@ By default, it obtains certificates for: - possibly for `element.`, unless you have disabled the [Element client component](configuring-playbook-client-element.md) using `matrix_client_element_enabled: false` - possibly for `riot.`, if you have explicitly enabled Riot to Element redirection (for background compatibility) using `matrix_nginx_proxy_proxy_riot_compat_redirect_enabled: true` - possibly for `dimension.`, if you have explicitly [set up Dimension](configuring-playbook-dimension.md). +- possibly for `jitsi.`, if you have explicitly [set up Jitsi](configuring-playbook-jitsi.md). - possibly for your base domain (``), if you have explicitly configured [Serving the base domain](configuring-playbook-base-domain-serving.md) If you are hosting other domains on the Matrix machine, you can make the playbook obtain and renew certificates for those other domains too. From dac0d3a682d6a288dea3c22115c3d8385e79c720 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 13 Dec 2020 21:07:16 +0200 Subject: [PATCH 111/217] Add default matrix_postgres_additional_databases --- roles/matrix-postgres/defaults/main.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index ec5cb3dc..548cac15 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -30,3 +30,13 @@ matrix_postgres_container_extra_arguments: [] # # Takes an ":" or "" value (e.g. "127.0.0.1:5432"), or empty string to not expose. matrix_postgres_container_postgres_bind_port: "" + +# A list of additional (databases and their credentials) to create. +# +# Example: +# matrix_postgres_additional_databases: +# - name: matrix_appservice_discord +# pass: some-password +# - name: matrix_appservice_slack +# pass: some-password +matrix_postgres_additional_databases: [] From 527d5f57d58ad7c1feba743fc99fe8508cc90534 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 13 Dec 2020 21:40:32 +0200 Subject: [PATCH 112/217] Relocate Postgres additional database creation logic Moving it above the "uninstalling" set of tasks is better. Extracting it out to another file at the same time, for readability, especially given that it will probably have to become more complex in the future (potentially installing `jq`, etc.) --- .../matrix-postgres/tasks/setup_postgres.yml | 37 ++++--------------- .../util/create_additional_databases.yml | 28 ++++++++++++++ 2 files changed, 35 insertions(+), 30 deletions(-) create mode 100644 roles/matrix-postgres/tasks/util/create_additional_databases.yml diff --git a/roles/matrix-postgres/tasks/setup_postgres.yml b/roles/matrix-postgres/tasks/setup_postgres.yml index 85a8604c..6a0f9854 100644 --- a/roles/matrix-postgres/tasks/setup_postgres.yml +++ b/roles/matrix-postgres/tasks/setup_postgres.yml @@ -113,6 +113,13 @@ daemon_reload: yes when: "matrix_postgres_enabled|bool and matrix_postgres_systemd_service_result.changed" +- include_tasks: + file: "{{ role_path }}/tasks/util/create_additional_databases.yml" + apply: + tags: + - always + when: "matrix_postgres_enabled|bool" + # # Tasks related to getting rid of the internal postgres server (if it was previously enabled) # @@ -162,33 +169,3 @@ - matrix-change-user-admin-status - matrix-postgres-update-user-password-hash when: "not matrix_postgres_enabled|bool" - -# Create additional databases -- name: Retrieve IP of postgres container - shell: "docker inspect matrix-postgres | jq -r '.[0].NetworkSettings.Networks.{{ matrix_docker_network }}.IPAddress'" - register: matirx_postgres_container_ip - -- name: Create additional users in postgres - postgresql_user: - name: "{{ item.name }}" - password: "{{ item.pass }}" - login_host: "{{ matrx_postgres_container_ip.stdout }}" - login_port: 5432 - login_user: "{{ matrix_postgres_connection_username }}" - login_password: "{{ matrix_postgres_connection_password }}" - login_db: "{{ matrix_postgres_db_name }}" - loop: matrix_postgres_additional_databases - when: matrix_postgres_enabed|bool - -- name: Create additional users in postgres - postgresql_db: - name: "{{ item.name }}" - owner: "{{ item.name }}" - lc_ctype: 'C' - lc_collate: 'C' - login_host: "{{ matrx_postgres_container_ip.stdout }}" - login_port: 5432 - login_user: "{{ matrix_postgres_connection_username }}" - login_password: "{{ matrix_postgres_connection_password }}" - loop: matrix_postgres_additional_databases - when: matrix_postgres_enabled|bool diff --git a/roles/matrix-postgres/tasks/util/create_additional_databases.yml b/roles/matrix-postgres/tasks/util/create_additional_databases.yml new file mode 100644 index 00000000..ec37ea63 --- /dev/null +++ b/roles/matrix-postgres/tasks/util/create_additional_databases.yml @@ -0,0 +1,28 @@ +--- + +- name: Retrieve IP of postgres container + shell: "docker inspect matrix-postgres | jq -r '.[0].NetworkSettings.Networks.{{ matrix_docker_network }}.IPAddress'" + register: matrix_postgres_container_ip + +- name: Create additional users in postgres + postgresql_user: + name: "{{ item.name }}" + password: "{{ item.pass }}" + login_host: "{{ matrix_postgres_container_ip.stdout }}" + login_port: 5432 + login_user: "{{ matrix_postgres_connection_username }}" + login_password: "{{ matrix_postgres_connection_password }}" + login_db: "{{ matrix_postgres_db_name }}" + loop: matrix_postgres_additional_databases + +- name: Create additional users in postgres + postgresql_db: + name: "{{ item.name }}" + owner: "{{ item.name }}" + lc_ctype: 'C' + lc_collate: 'C' + login_host: "{{ matrix_postgres_container_ip.stdout }}" + login_port: 5432 + login_user: "{{ matrix_postgres_connection_username }}" + login_password: "{{ matrix_postgres_connection_password }}" + loop: matrix_postgres_additional_databases From 0641106370aed272e16cecaf2cf17754d30eb6c6 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 13 Dec 2020 21:43:53 +0200 Subject: [PATCH 113/217] Allow username of additional Postgres databases to be different We'll most likely use one that matches the database name, but it's better to have it configurable. --- group_vars/matrix_servers | 17 ++++++++++++++++- roles/matrix-postgres/defaults/main.yml | 6 ++++-- .../tasks/util/create_additional_databases.yml | 4 ++-- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 8a099275..f131ab9e 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -893,62 +893,77 @@ matrix_postgres_additional_databases: | {{ ([{ name: 'matrix_appservice_discord', + username: 'matrix_appservice_discord', pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string }] if matrix_appservice_discord_enabled else []) + ([{ name: 'matrix_appservice_slack' + username: 'matrix_appservice_slack', pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string }] if matrix_appservice_slack_enabled else []) + ([{ name: 'matrix_appservice_irc' + username: 'matrix_appservice_irc', pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string }] if matrix_appservice_irc_enabled else []) + ([{ - name: 'mautrix-bridge-facebook' + name: 'mautrix_bridge_facebook' + username: 'mautrix_bridge_facebook', pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string }] if matrix_mautrix_facebook_enabled else []) + ([{ name: 'mautrix_bridge_hangouts' + username: 'mautrix_bridge_hangouts', pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string }] if matrix_mautrix_hangouts_enabled else []) + ([{ name: 'mautrix_bridge_telegram' + username: 'mautrix_bridge_telegram', pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string }] if matrix_mautrix_telegram_enabled else []) + ([{ name: 'mautrix_bridge_whatsapp' + username: 'mautrix_bridge_whatsapp', pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string }] if matrix_mautrix_whatsapp_enabled else []) + ([{ name: 'matrix_bridge_sms' + username: 'matrix_bridge_sms', pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string }] if matrix_sms_bridge_enabled else []) + ([{ name: 'matrix_puppet_skype' + username: 'matrix_puppet_skype', pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string }] if matrix_mx_puppet_skype_enabled else []) + ([{ name: 'matrix_puppet_slack' + username: 'matrix_puppet_slack', pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string }] if matrix_mx_puppet_slack_enabled else []) + ([{ name: 'matrix_puppet_twitter' + username: 'matrix_puppet_twitter', pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string }] if matrix_mx_puppet_twitter_enabled else []) + ([{ name: 'matrix_puppet_instagram' + username: 'matrix_puppet_instagram', pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string ] if matrix_mx_puppet_instagram_enabled else []) + ([{ name: 'matrix_puppet_discord' + username: 'matrix_puppet_discord', pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string }] if matrix_mx_puppet_discord_enabled else []) + ([{ name: 'matrix_puppet_steam' + username: 'matrix_puppet_steam', pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string }] if matrix_mx_puppet_steam_enabled else []) + ([{ name: 'matrix_dimension' + username: 'matrix_dimension', pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string }] if matrix_dimension_enabled else []) }} diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index 548cac15..2645d4b0 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -36,7 +36,9 @@ matrix_postgres_container_postgres_bind_port: "" # Example: # matrix_postgres_additional_databases: # - name: matrix_appservice_discord -# pass: some-password +# username: matrix_appservice_discord +# pass: some_password # - name: matrix_appservice_slack -# pass: some-password +# username: matrix_appservice_slack +# pass: some_password matrix_postgres_additional_databases: [] diff --git a/roles/matrix-postgres/tasks/util/create_additional_databases.yml b/roles/matrix-postgres/tasks/util/create_additional_databases.yml index ec37ea63..0532c99d 100644 --- a/roles/matrix-postgres/tasks/util/create_additional_databases.yml +++ b/roles/matrix-postgres/tasks/util/create_additional_databases.yml @@ -6,7 +6,7 @@ - name: Create additional users in postgres postgresql_user: - name: "{{ item.name }}" + name: "{{ item.username }}" password: "{{ item.pass }}" login_host: "{{ matrix_postgres_container_ip.stdout }}" login_port: 5432 @@ -18,7 +18,7 @@ - name: Create additional users in postgres postgresql_db: name: "{{ item.name }}" - owner: "{{ item.name }}" + owner: "{{ item.username }}" lc_ctype: 'C' lc_collate: 'C' login_host: "{{ matrix_postgres_container_ip.stdout }}" From d251764c16303dea542326746a0e13a46d13fda2 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 13 Dec 2020 22:19:54 +0200 Subject: [PATCH 114/217] Fix syntax issues in matrix_postgres_additional_databases Quotes are necessary around dictionary field names. There was a missing `}` as well. --- group_vars/matrix_servers | 92 +++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index f131ab9e..11c87a64 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -892,79 +892,79 @@ matrix_postgres_db_name: "homeserver" matrix_postgres_additional_databases: | {{ ([{ - name: 'matrix_appservice_discord', - username: 'matrix_appservice_discord', - pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string + 'name': 'matrix_appservice_discord', + 'username': 'matrix_appservice_discord', + 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string }] if matrix_appservice_discord_enabled else []) + ([{ - name: 'matrix_appservice_slack' - username: 'matrix_appservice_slack', - pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string + 'name': 'matrix_appservice_slack', + 'username': 'matrix_appservice_slack', + 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string }] if matrix_appservice_slack_enabled else []) + ([{ - name: 'matrix_appservice_irc' - username: 'matrix_appservice_irc', - pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string + 'name': 'matrix_appservice_irc', + 'username': 'matrix_appservice_irc', + 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string }] if matrix_appservice_irc_enabled else []) + ([{ - name: 'mautrix_bridge_facebook' - username: 'mautrix_bridge_facebook', - pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string + 'name': 'mautrix_bridge_facebook', + 'username': 'mautrix_bridge_facebook', + 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string }] if matrix_mautrix_facebook_enabled else []) + ([{ - name: 'mautrix_bridge_hangouts' - username: 'mautrix_bridge_hangouts', - pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string + 'name': 'mautrix_bridge_hangouts', + 'username': 'mautrix_bridge_hangouts', + 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string }] if matrix_mautrix_hangouts_enabled else []) + ([{ - name: 'mautrix_bridge_telegram' - username: 'mautrix_bridge_telegram', - pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string + 'name': 'mautrix_bridge_telegram', + 'username': 'mautrix_bridge_telegram', + 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string }] if matrix_mautrix_telegram_enabled else []) + ([{ - name: 'mautrix_bridge_whatsapp' - username: 'mautrix_bridge_whatsapp', - pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string + 'name': 'mautrix_bridge_whatsapp', + 'username': 'mautrix_bridge_whatsapp', + 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string }] if matrix_mautrix_whatsapp_enabled else []) + ([{ - name: 'matrix_bridge_sms' - username: 'matrix_bridge_sms', - pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string + 'name': 'matrix_bridge_sms', + 'username': 'matrix_bridge_sms', + 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string }] if matrix_sms_bridge_enabled else []) + ([{ - name: 'matrix_puppet_skype' - username: 'matrix_puppet_skype', - pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string + 'name': 'matrix_puppet_skype', + 'username': 'matrix_puppet_skype', + 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string }] if matrix_mx_puppet_skype_enabled else []) + ([{ - name: 'matrix_puppet_slack' - username: 'matrix_puppet_slack', - pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string + 'name': 'matrix_puppet_slack', + 'username': 'matrix_puppet_slack', + 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string }] if matrix_mx_puppet_slack_enabled else []) + ([{ - name: 'matrix_puppet_twitter' - username: 'matrix_puppet_twitter', - pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string + 'name': 'matrix_puppet_twitter', + 'username': 'matrix_puppet_twitter', + 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string }] if matrix_mx_puppet_twitter_enabled else []) + ([{ - name: 'matrix_puppet_instagram' - username: 'matrix_puppet_instagram', - pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string - ] if matrix_mx_puppet_instagram_enabled else []) + 'name': 'matrix_puppet_instagram', + 'username': 'matrix_puppet_instagram', + 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string + }] if matrix_mx_puppet_instagram_enabled else []) + ([{ - name: 'matrix_puppet_discord' - username: 'matrix_puppet_discord', - pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string + 'name': 'matrix_puppet_discord', + 'username': 'matrix_puppet_discord', + 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string }] if matrix_mx_puppet_discord_enabled else []) + ([{ - name: 'matrix_puppet_steam' - username: 'matrix_puppet_steam', - pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string + 'name': 'matrix_puppet_steam', + 'username': 'matrix_puppet_steam', + 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string }] if matrix_mx_puppet_steam_enabled else []) + ([{ - name: 'matrix_dimension' - username: 'matrix_dimension', - pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string + 'name': 'matrix_dimension', + 'username': 'matrix_dimension', + 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string }] if matrix_dimension_enabled else []) }} From f47e8a97e6629da02a45d9d75bde3a82ef057bd6 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 13 Dec 2020 22:38:35 +0200 Subject: [PATCH 115/217] Make use of matrix_host_command_docker instead of hardcoding --- .../matrix-postgres/tasks/util/create_additional_databases.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/matrix-postgres/tasks/util/create_additional_databases.yml b/roles/matrix-postgres/tasks/util/create_additional_databases.yml index 0532c99d..424be626 100644 --- a/roles/matrix-postgres/tasks/util/create_additional_databases.yml +++ b/roles/matrix-postgres/tasks/util/create_additional_databases.yml @@ -1,7 +1,6 @@ --- - - name: Retrieve IP of postgres container - shell: "docker inspect matrix-postgres | jq -r '.[0].NetworkSettings.Networks.{{ matrix_docker_network }}.IPAddress'" + shell: "{{ matrix_host_command_docker }} inspect matrix-postgres | jq -r '.[0].NetworkSettings.Networks.{{ matrix_docker_network }}.IPAddress'" register: matrix_postgres_container_ip - name: Create additional users in postgres From e2952f16f7a97b2883ea4c00c6bc4451c71c785d Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 13 Dec 2020 22:45:48 +0200 Subject: [PATCH 116/217] Determine matrix-postgres IP address without relying on jq To avoid needing to have `jq` installed on the machine, we could: - try to run jq in a Docker container using some small image providing that - better yet, avoid `jq` altogether --- .../matrix-postgres/tasks/util/create_additional_databases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-postgres/tasks/util/create_additional_databases.yml b/roles/matrix-postgres/tasks/util/create_additional_databases.yml index 424be626..0864d8f8 100644 --- a/roles/matrix-postgres/tasks/util/create_additional_databases.yml +++ b/roles/matrix-postgres/tasks/util/create_additional_databases.yml @@ -1,6 +1,6 @@ --- - name: Retrieve IP of postgres container - shell: "{{ matrix_host_command_docker }} inspect matrix-postgres | jq -r '.[0].NetworkSettings.Networks.{{ matrix_docker_network }}.IPAddress'" + command: "{{ matrix_host_command_docker }} inspect matrix-postgres --format='{% raw %}{{ .NetworkSettings.Networks.{% endraw %}{{ matrix_docker_network }}{% raw %}.IPAddress }}{% endraw %}'" register: matrix_postgres_container_ip - name: Create additional users in postgres From c765ceb270b7377e799a32db383eef39257d43e8 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 13 Dec 2020 22:56:56 +0200 Subject: [PATCH 117/217] Prevent weird loop error > Invalid data passed to 'loop', it requires a list, got this instead: matrix_postgres_additional_databases. Hint: If you passed a list/dict of just one element, try adding wantlist=True to your lookup invocation or use q/query instead of lookup. Well, or working around it, as I've done in this commit (which seems more sane than `wantlist=True` stuff). --- .../tasks/util/create_additional_databases.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/matrix-postgres/tasks/util/create_additional_databases.yml b/roles/matrix-postgres/tasks/util/create_additional_databases.yml index 0864d8f8..0b40cabf 100644 --- a/roles/matrix-postgres/tasks/util/create_additional_databases.yml +++ b/roles/matrix-postgres/tasks/util/create_additional_databases.yml @@ -1,4 +1,5 @@ --- + - name: Retrieve IP of postgres container command: "{{ matrix_host_command_docker }} inspect matrix-postgres --format='{% raw %}{{ .NetworkSettings.Networks.{% endraw %}{{ matrix_docker_network }}{% raw %}.IPAddress }}{% endraw %}'" register: matrix_postgres_container_ip @@ -12,7 +13,7 @@ login_user: "{{ matrix_postgres_connection_username }}" login_password: "{{ matrix_postgres_connection_password }}" login_db: "{{ matrix_postgres_db_name }}" - loop: matrix_postgres_additional_databases + loop: "{{ matrix_postgres_additional_databases }}" - name: Create additional users in postgres postgresql_db: @@ -24,4 +25,4 @@ login_port: 5432 login_user: "{{ matrix_postgres_connection_username }}" login_password: "{{ matrix_postgres_connection_password }}" - loop: matrix_postgres_additional_databases + loop: "{{ matrix_postgres_additional_databases }}" From bbc09d013b8037e784004363b7bdfdcc6e8d13c7 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 13 Dec 2020 23:46:02 +0200 Subject: [PATCH 118/217] Do not execute additional databases creation code if not necessary The tasks in `create_additional_databases.yml` will likely ensure `matrix-postgres.service` is started, etc. If no additional databases are defined, we'd rather not execute that file and all these tasks that it may do in the future. --- roles/matrix-postgres/tasks/setup_postgres.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-postgres/tasks/setup_postgres.yml b/roles/matrix-postgres/tasks/setup_postgres.yml index 6a0f9854..518d1a5f 100644 --- a/roles/matrix-postgres/tasks/setup_postgres.yml +++ b/roles/matrix-postgres/tasks/setup_postgres.yml @@ -118,7 +118,7 @@ apply: tags: - always - when: "matrix_postgres_enabled|bool" + when: "matrix_postgres_enabled|bool and matrix_postgres_additional_databases|length > 0" # # Tasks related to getting rid of the internal postgres server (if it was previously enabled) From da4cb2f63998c2c994776b3520303e90b735157c Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 00:25:13 +0200 Subject: [PATCH 119/217] Do not use the postgresql_user/postgresql_db modules While these modules are really nice and helpful, we can't use them for at least 2 reasons: - for us, Postgres runs in a container on a private Docker network (`--network=matrix`) without usually being exposed to the host. These modules execute on the host so they won't be able to reach it. - these modules require `psycopg2`, so we need to install it before using it. This might or might not be its own can of worms. --- .../tasks/util/create_additional_database.yml | 34 ++++++++++++++++++ .../util/create_additional_databases.yml | 35 ++++++------------- .../init-additional-db-user-and-role.sql.j2 | 19 ++++++++++ 3 files changed, 63 insertions(+), 25 deletions(-) create mode 100644 roles/matrix-postgres/tasks/util/create_additional_database.yml create mode 100644 roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2 diff --git a/roles/matrix-postgres/tasks/util/create_additional_database.yml b/roles/matrix-postgres/tasks/util/create_additional_database.yml new file mode 100644 index 00000000..2da505eb --- /dev/null +++ b/roles/matrix-postgres/tasks/util/create_additional_database.yml @@ -0,0 +1,34 @@ +--- + +# TODO - ensure `additional_db` contains all keys that we expect + +# The SQL statements that we'll run against Postgres are stored in a file that others can't read. +# This file will be mounted into the container and fed to Postgres. +# This way, we avoid passing sensitive data around in CLI commands that other users on the system can see. +- name: Create additional database initialization SQL file for {{ additional_db.name }} + template: + src: "{{ role_path }}/templates/init-additional-db-user-and-role.sql.j2" + dest: "/tmp/matrix-postgres-init-additional-db-user-and-role.sql" + mode: 0600 + owner: "{{ matrix_user_uid }}" + group: "{{ matrix_user_gid }}" + +- name: Execute Postgres additional database initialization SQL file for {{ additional_db.name }} + command: + cmd: >- + {{ matrix_host_command_docker }} run + --rm + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --env-file={{ matrix_postgres_base_path }}/env-postgres-psql + --network {{ matrix_docker_network }} + --mount type=bind,src=/tmp/matrix-postgres-init-additional-db-user-and-role.sql,dst=/matrix-postgres-init-additional-db-user-and-role.sql,ro + --entrypoint=/bin/sh + {{ matrix_postgres_docker_image_to_use }} + -c + 'psql -h {{ matrix_postgres_connection_hostname }} --file=/matrix-postgres-init-additional-db-user-and-role.sql' + +- name: Delete additional database initialization SQL file for {{ additional_db.name }} + file: + path: /tmp/matrix-postgres-init-additional-db-user-and-role.sql + state: absent diff --git a/roles/matrix-postgres/tasks/util/create_additional_databases.yml b/roles/matrix-postgres/tasks/util/create_additional_databases.yml index 0b40cabf..51deb228 100644 --- a/roles/matrix-postgres/tasks/util/create_additional_databases.yml +++ b/roles/matrix-postgres/tasks/util/create_additional_databases.yml @@ -1,28 +1,13 @@ --- -- name: Retrieve IP of postgres container - command: "{{ matrix_host_command_docker }} inspect matrix-postgres --format='{% raw %}{{ .NetworkSettings.Networks.{% endraw %}{{ matrix_docker_network }}{% raw %}.IPAddress }}{% endraw %}'" - register: matrix_postgres_container_ip +# TODO - we should ensure matrix-postgres.service is started. +# .. and that if we had just started it, we've given it ample time to initialize, +# before we attempt to run queries against it. -- name: Create additional users in postgres - postgresql_user: - name: "{{ item.username }}" - password: "{{ item.pass }}" - login_host: "{{ matrix_postgres_container_ip.stdout }}" - login_port: 5432 - login_user: "{{ matrix_postgres_connection_username }}" - login_password: "{{ matrix_postgres_connection_password }}" - login_db: "{{ matrix_postgres_db_name }}" - loop: "{{ matrix_postgres_additional_databases }}" - -- name: Create additional users in postgres - postgresql_db: - name: "{{ item.name }}" - owner: "{{ item.username }}" - lc_ctype: 'C' - lc_collate: 'C' - login_host: "{{ matrix_postgres_container_ip.stdout }}" - login_port: 5432 - login_user: "{{ matrix_postgres_connection_username }}" - login_password: "{{ matrix_postgres_connection_password }}" - loop: "{{ matrix_postgres_additional_databases }}" +- name: Create additional Postgers user and database + include_tasks: "{{ role_path }}/tasks/util/create_additional_database.yml" + with_items: "{{ matrix_postgres_additional_databases }}" + loop_control: + loop_var: additional_db + # Suppress logging to avoid dumping the credentials to the shell + no_log: true diff --git a/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2 b/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2 new file mode 100644 index 00000000..732b0187 --- /dev/null +++ b/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2 @@ -0,0 +1,19 @@ +-- `CREATE USER` does not support `IF NOT EXISTS`, so we use this workaround to prevent an error and raise a notice instead. +-- Seen here: https://stackoverflow.com/a/49858797 +DO $$ +BEGIN + CREATE USER {{ additional_db.username }}; + EXCEPTION WHEN DUPLICATE_OBJECT THEN + RAISE NOTICE 'not creating role {{ additional_db.username }}, since it already exists'; +END +$$; + +-- This is useful for initial user creation (since we don't assign a password above) and for handling subsequent password changes +-- TODO - we should escape quotes in the password. +ALTER ROLE {{ additional_db.username }} PASSWORD '{{ additional_db.pass }}'; + +-- This will generate an error on subsequent execution +CREATE DATABASE {{ additional_db.name }} WITH LC_CTYPE 'C' LC_COLLATE 'C' OWNER {{ additional_db.username }}; + +-- This is useful for changing the database owner subsequently +ALTER DATABASE {{ additional_db.name }} OWNER TO {{ additional_db.username }}; From 3a037a59935002729dfe7da742721111956eaf0b Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 00:39:38 +0200 Subject: [PATCH 120/217] Ensure additional databases contain all the keys that we expect --- .../tasks/util/create_additional_database.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/roles/matrix-postgres/tasks/util/create_additional_database.yml b/roles/matrix-postgres/tasks/util/create_additional_database.yml index 2da505eb..a994cc26 100644 --- a/roles/matrix-postgres/tasks/util/create_additional_database.yml +++ b/roles/matrix-postgres/tasks/util/create_additional_database.yml @@ -1,6 +1,12 @@ --- -# TODO - ensure `additional_db` contains all keys that we expect +# It'd be better if this is belonged to `validate_config.yml`, but it would have to be some loop-within-a-loop there, +# and that's ugly. We also don't expect this to catch errors often. It's more of a defensive last-minute check. +- name: Fail if additional database data appears invalid + fail: + msg: "Additional database definition ({{ additional_db }} lacks a required key: {{ item }}" + when: "item not in additional_db" + with_items: "{{ ['name', 'username', 'pass'] }}" # The SQL statements that we'll run against Postgres are stored in a file that others can't read. # This file will be mounted into the container and fed to Postgres. From 46a4034d3ec0b8ae331119e249fc584f7be2b6ed Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 00:43:03 +0200 Subject: [PATCH 121/217] Use "password" for additional Postgres databases, not "pass" Being more explicit sounds better. --- group_vars/matrix_servers | 30 +++++++++---------- roles/matrix-postgres/defaults/main.yml | 4 +-- .../tasks/util/create_additional_database.yml | 2 +- .../init-additional-db-user-and-role.sql.j2 | 2 +- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 11c87a64..3169e18e 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -894,77 +894,77 @@ matrix_postgres_additional_databases: | ([{ 'name': 'matrix_appservice_discord', 'username': 'matrix_appservice_discord', - 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string, }] if matrix_appservice_discord_enabled else []) + ([{ 'name': 'matrix_appservice_slack', 'username': 'matrix_appservice_slack', - 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string, }] if matrix_appservice_slack_enabled else []) + ([{ 'name': 'matrix_appservice_irc', 'username': 'matrix_appservice_irc', - 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string, }] if matrix_appservice_irc_enabled else []) + ([{ 'name': 'mautrix_bridge_facebook', 'username': 'mautrix_bridge_facebook', - 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string, }] if matrix_mautrix_facebook_enabled else []) + ([{ 'name': 'mautrix_bridge_hangouts', 'username': 'mautrix_bridge_hangouts', - 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string, }] if matrix_mautrix_hangouts_enabled else []) + ([{ 'name': 'mautrix_bridge_telegram', 'username': 'mautrix_bridge_telegram', - 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string, }] if matrix_mautrix_telegram_enabled else []) + ([{ 'name': 'mautrix_bridge_whatsapp', 'username': 'mautrix_bridge_whatsapp', - 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string, }] if matrix_mautrix_whatsapp_enabled else []) + ([{ 'name': 'matrix_bridge_sms', 'username': 'matrix_bridge_sms', - 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string, }] if matrix_sms_bridge_enabled else []) + ([{ 'name': 'matrix_puppet_skype', 'username': 'matrix_puppet_skype', - 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string, }] if matrix_mx_puppet_skype_enabled else []) + ([{ 'name': 'matrix_puppet_slack', 'username': 'matrix_puppet_slack', - 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string, }] if matrix_mx_puppet_slack_enabled else []) + ([{ 'name': 'matrix_puppet_twitter', 'username': 'matrix_puppet_twitter', - 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string, }] if matrix_mx_puppet_twitter_enabled else []) + ([{ 'name': 'matrix_puppet_instagram', 'username': 'matrix_puppet_instagram', - 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string, }] if matrix_mx_puppet_instagram_enabled else []) + ([{ 'name': 'matrix_puppet_discord', 'username': 'matrix_puppet_discord', - 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string, }] if matrix_mx_puppet_discord_enabled else []) + ([{ 'name': 'matrix_puppet_steam', 'username': 'matrix_puppet_steam', - 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string, }] if matrix_mx_puppet_steam_enabled else []) + ([{ 'name': 'matrix_dimension', 'username': 'matrix_dimension', - 'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string, }] if matrix_dimension_enabled else []) }} diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index 2645d4b0..ad417e0b 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -37,8 +37,8 @@ matrix_postgres_container_postgres_bind_port: "" # matrix_postgres_additional_databases: # - name: matrix_appservice_discord # username: matrix_appservice_discord -# pass: some_password +# password: some_password # - name: matrix_appservice_slack # username: matrix_appservice_slack -# pass: some_password +# password: some_password matrix_postgres_additional_databases: [] diff --git a/roles/matrix-postgres/tasks/util/create_additional_database.yml b/roles/matrix-postgres/tasks/util/create_additional_database.yml index a994cc26..ce064d59 100644 --- a/roles/matrix-postgres/tasks/util/create_additional_database.yml +++ b/roles/matrix-postgres/tasks/util/create_additional_database.yml @@ -6,7 +6,7 @@ fail: msg: "Additional database definition ({{ additional_db }} lacks a required key: {{ item }}" when: "item not in additional_db" - with_items: "{{ ['name', 'username', 'pass'] }}" + with_items: "{{ ['name', 'username', 'password'] }}" # The SQL statements that we'll run against Postgres are stored in a file that others can't read. # This file will be mounted into the container and fed to Postgres. diff --git a/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2 b/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2 index 732b0187..af3d912c 100644 --- a/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2 +++ b/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2 @@ -10,7 +10,7 @@ $$; -- This is useful for initial user creation (since we don't assign a password above) and for handling subsequent password changes -- TODO - we should escape quotes in the password. -ALTER ROLE {{ additional_db.username }} PASSWORD '{{ additional_db.pass }}'; +ALTER ROLE {{ additional_db.username }} PASSWORD '{{ additional_db.password }}'; -- This will generate an error on subsequent execution CREATE DATABASE {{ additional_db.name }} WITH LC_CTYPE 'C' LC_COLLATE 'C' OWNER {{ additional_db.username }}; From a374d309c8f5d6038bfd5f94a66c3cd9a68c7e23 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 00:52:25 +0200 Subject: [PATCH 122/217] Make appservice-discord support both SQLite and Postgres People can toggle between them now. The playbook also defaults to using SQLite if an external Postgres server is used. Ideally, we'd be able to create databases/users in external Postgres servers as well, but our initialization logic (and `docker run` command, etc.) hardcode too many things right now. --- group_vars/matrix_servers | 13 +++++++++---- .../defaults/main.yml | 16 ++++++++++++++++ .../tasks/validate_config.yml | 4 ++++ .../templates/config.yaml.j2 | 7 +++++-- 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 3169e18e..717e71bc 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -67,6 +67,11 @@ matrix_appservice_discord_appservice_token: "{{ matrix_synapse_macaroon_secret_k matrix_appservice_discord_homeserver_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'discord.hs.token') | to_uuid }}" +# We only make this use Postgres if our own Postgres server is enabled. +# It's only then (for now) that we can automatically create the necessary database and user for this service. +matrix_appservice_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_appservice_discord_database_connString_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.discord.db') }}" + ###################################################################### # # /matrix-bridge-appservice-discord @@ -892,10 +897,10 @@ matrix_postgres_db_name: "homeserver" matrix_postgres_additional_databases: | {{ ([{ - 'name': 'matrix_appservice_discord', - 'username': 'matrix_appservice_discord', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string, - }] if matrix_appservice_discord_enabled else []) + 'name': matrix_appservice_discord_database_connString_db_name, + 'username': matrix_appservice_discord_database_connString_username, + 'password': matrix_appservice_discord_database_connString_password, + }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_database_connString_hostname == 'matrix-postgres') else []) + ([{ 'name': 'matrix_appservice_slack', 'username': 'matrix_appservice_slack', diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml index 357b93d6..36b39b8e 100644 --- a/roles/matrix-bridge-appservice-discord/defaults/main.yml +++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml @@ -41,6 +41,22 @@ matrix_appservice_discord_bridge_homeserverUrl: "http://matrix-synapse:8008" matrix_appservice_discord_bridge_disablePresence: false matrix_appservice_discord_bridge_enableSelfServiceBridging: false +# Database-related configuration fields. +# +# To use SQLite, stick to these defaults. +# +# To use Postgres: +# - change the engine (`matrix_appservice_discord_database_engine: 'postgres'`) +# - adjust your database credentials +matrix_appservice_discord_database_engine: 'sqlite' +matrix_appservice_discord_database_filename: "/data/discord.db" +matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_connString_username }}:{{ matrix_appservice_discord_database_connString_password }}@{{ matrix_appservice_discord_database_connString_hostname }}:{{ matrix_appservice_discord_database_connString_port }}/{{ matrix_appservice_discord_database_connString_db_name }}' +matrix_appservice_discord_database_connString_username: 'matrix_appservice_discord' +matrix_appservice_discord_database_connString_password: 'some-password' +matrix_appservice_discord_database_connString_hostname: 'matrix-postgres' +matrix_appservice_discord_database_connString_port: 5432 +matrix_appservice_discord_database_connString_db_name: 'matrix_appservice_discord' + # Tells whether the bot should make use of "Privileged Gateway Intents". # # Enabling this means that you need to enable it for the bot (Discord application) as well, diff --git a/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml b/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml index b0713a43..73253ba0 100644 --- a/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml +++ b/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml @@ -20,3 +20,7 @@ when: "item.old in vars" with_items: - {'old': 'matrix_appservice_discord_container_expose_client_server_api_port', 'new': ''} + +- name: Require a valid database engine + fail: msg="`matrix_appservice_discord_database_engine` needs to be either 'sqlite' or 'postgres'" + when: "matrix_appservice_discord_database_engine not in ['sqlite', 'postgres']" diff --git a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 index de4182d9..b2ecd198 100644 --- a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 +++ b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 @@ -58,8 +58,11 @@ database: # If you are migrating, see https://github.com/Half-Shot/matrix-appservice-discord/blob/master/docs/howto.md#migrate-to-postgres-from-sqlite # WARNING: You will almost certainly be fine with sqlite unless your bridge # is in heavy demand and you suffer from IO slowness. - #filename: "/data/discord.db" - connString: "postgresql://matrix_appservice_discord:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_appservice_discord') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_discord" + {% if matrix_appservice_discord_database_engine == 'sqlite' %} + filename: {{ matrix_appservice_discord_database_filename|to_json }} + {% else %} + connString: {{ matrix_appservice_discord_database_connString|to_json }} + {% endif %} room: # Set the default visibility of alias rooms, defaults to "public". # One of: "public", "private" From 183d2a10dbdfc142ace8e88e515e2d2f60ab84bc Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 00:59:59 +0200 Subject: [PATCH 123/217] Ensure matrix-postgres.service is started before creating additional users/databases --- roles/matrix-postgres/defaults/main.yml | 6 ++++++ .../tasks/util/create_additional_databases.yml | 18 ++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index ad417e0b..89559f5d 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -42,3 +42,9 @@ matrix_postgres_container_postgres_bind_port: "" # username: matrix_appservice_slack # password: some_password matrix_postgres_additional_databases: [] + +# The number of seconds to wait after starting `matrix-postgres.service` +# and before trying to run queries for creating additional databases/users against it. +# +# For most (subsequent) runs, Postgres would already be running, so no waiting will be happening at all. +matrix_postgres_additional_databases_postgres_start_wait_timeout_seconds: 15 diff --git a/roles/matrix-postgres/tasks/util/create_additional_databases.yml b/roles/matrix-postgres/tasks/util/create_additional_databases.yml index 51deb228..0ad460dd 100644 --- a/roles/matrix-postgres/tasks/util/create_additional_databases.yml +++ b/roles/matrix-postgres/tasks/util/create_additional_databases.yml @@ -1,10 +1,20 @@ --- -# TODO - we should ensure matrix-postgres.service is started. -# .. and that if we had just started it, we've given it ample time to initialize, -# before we attempt to run queries against it. +- name: Ensure matrix-postgres is started + service: + name: matrix-postgres + state: started + daemon_reload: yes + register: matrix_postgres_service_start_result -- name: Create additional Postgers user and database +- name: Wait a bit, so that Postgres can start + wait_for: + timeout: "{{ matrix_postgres_additional_databases_postgres_start_wait_timeout_seconds }}" + delegate_to: 127.0.0.1 + become: false + when: "matrix_postgres_service_start_result.changed|bool" + +- name: Create additional Postgres user and database include_tasks: "{{ role_path }}/tasks/util/create_additional_database.yml" with_items: "{{ matrix_postgres_additional_databases }}" loop_control: From dd994995bca0cb0bd162aec1f5ee4dae35848614 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 01:22:23 +0200 Subject: [PATCH 124/217] Simplify password for additional Postgres databases Using the result of `password_hash` works for creating them, but authentication seems to be failing with some tools like pgloader. It's possible that we're not escaping things properly somewhere. Ideally, it'd be nice to solve that. But the easier (and still relatively safe/good) solution is to just turn that password hash into a UUID that's safe for passing around without worrying about escaping. --- group_vars/matrix_servers | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 717e71bc..b9d0be26 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -70,7 +70,7 @@ matrix_appservice_discord_homeserver_token: "{{ matrix_synapse_macaroon_secret_k # We only make this use Postgres if our own Postgres server is enabled. # It's only then (for now) that we can automatically create the necessary database and user for this service. matrix_appservice_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_appservice_discord_database_connString_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.discord.db') }}" +matrix_appservice_discord_database_connString_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.discord.db') | to_uuid }}" ###################################################################### # @@ -904,72 +904,72 @@ matrix_postgres_additional_databases: | + ([{ 'name': 'matrix_appservice_slack', 'username': 'matrix_appservice_slack', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string, + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | to_uuid, }] if matrix_appservice_slack_enabled else []) + ([{ 'name': 'matrix_appservice_irc', 'username': 'matrix_appservice_irc', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string, + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | to_uuid, }] if matrix_appservice_irc_enabled else []) + ([{ 'name': 'mautrix_bridge_facebook', 'username': 'mautrix_bridge_facebook', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string, + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | to_uuid, }] if matrix_mautrix_facebook_enabled else []) + ([{ 'name': 'mautrix_bridge_hangouts', 'username': 'mautrix_bridge_hangouts', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string, + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | to_uuid, }] if matrix_mautrix_hangouts_enabled else []) + ([{ 'name': 'mautrix_bridge_telegram', 'username': 'mautrix_bridge_telegram', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string, + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | to_uuid, }] if matrix_mautrix_telegram_enabled else []) + ([{ 'name': 'mautrix_bridge_whatsapp', 'username': 'mautrix_bridge_whatsapp', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string, + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | to_uuid, }] if matrix_mautrix_whatsapp_enabled else []) + ([{ 'name': 'matrix_bridge_sms', 'username': 'matrix_bridge_sms', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string, + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid, }] if matrix_sms_bridge_enabled else []) + ([{ 'name': 'matrix_puppet_skype', 'username': 'matrix_puppet_skype', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string, + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | to_uuid, }] if matrix_mx_puppet_skype_enabled else []) + ([{ 'name': 'matrix_puppet_slack', 'username': 'matrix_puppet_slack', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string, + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | to_uuid, }] if matrix_mx_puppet_slack_enabled else []) + ([{ 'name': 'matrix_puppet_twitter', 'username': 'matrix_puppet_twitter', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string, + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | to_uuid, }] if matrix_mx_puppet_twitter_enabled else []) + ([{ 'name': 'matrix_puppet_instagram', 'username': 'matrix_puppet_instagram', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string, + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | to_uuid, }] if matrix_mx_puppet_instagram_enabled else []) + ([{ 'name': 'matrix_puppet_discord', 'username': 'matrix_puppet_discord', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string, + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | to_uuid, }] if matrix_mx_puppet_discord_enabled else []) + ([{ 'name': 'matrix_puppet_steam', 'username': 'matrix_puppet_steam', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string, + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | to_uuid, }] if matrix_mx_puppet_steam_enabled else []) + ([{ 'name': 'matrix_dimension', 'username': 'matrix_dimension', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string, + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | to_uuid, }] if matrix_dimension_enabled else []) }} From b87b75437232a8b3fa4c2ba2258382dfa761a62e Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 01:36:15 +0200 Subject: [PATCH 125/217] Fail if appservice-discord wants Postgres, but has leftover SQLite data --- .../defaults/main.yml | 3 ++- .../tasks/setup_install.yml | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml index 36b39b8e..53d7968d 100644 --- a/roles/matrix-bridge-appservice-discord/defaults/main.yml +++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml @@ -49,7 +49,8 @@ matrix_appservice_discord_bridge_enableSelfServiceBridging: false # - change the engine (`matrix_appservice_discord_database_engine: 'postgres'`) # - adjust your database credentials matrix_appservice_discord_database_engine: 'sqlite' -matrix_appservice_discord_database_filename: "/data/discord.db" +matrix_appservice_discord_database_filename: "/data/{{ matrix_appservice_discord_database_filename_name }}" +matrix_appservice_discord_database_filename_name: "discord.db" matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_connString_username }}:{{ matrix_appservice_discord_database_connString_password }}@{{ matrix_appservice_discord_database_connString_hostname }}:{{ matrix_appservice_discord_database_connString_port }}/{{ matrix_appservice_discord_database_connString_db_name }}' matrix_appservice_discord_database_connString_username: 'matrix_appservice_discord' matrix_appservice_discord_database_connString_password: 'some-password' diff --git a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml index 935d07eb..cea8fda8 100644 --- a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml @@ -1,5 +1,21 @@ --- +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_appservice_discord_data_path }}/{{ matrix_appservice_discord_database_filename_name }}" + register: matrix_appservice_discord_stat_sqlite_db + + - name: Fail if an SQLite database already exists when using Postgres + fail: + msg: >- + matrix_appservice_discord_database_engine has been set to `postgres` (which is our new default now). + However, we've discovered an existing SQLite database in {{ matrix_appservice_discord_data_path }}/{{ matrix_appservice_discord_database_filename_name }}. + It appears that you've been using this bridge with the SQLite engine until now. + To continue using SQLite, opt into it explicitly: add `matrix_appservice_discord_database_engine: sqlite` to your vars.yml file. + To migrate to Postgres: TODO - migration instructions here. + when: "matrix_appservice_discord_database_engine == 'postgres'" + - name: Ensure Appservice Discord image is pulled docker_image: name: "{{ matrix_appservice_discord_docker_image }}" From 6e1dfb62f0d371fdb82fc7a9e635dd32d0179e86 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 01:48:36 +0200 Subject: [PATCH 126/217] Rename some doc files and commands related to importing Since we'll likely have generic SQLite database importing via [pgloader](https://pgloader.io/) for migrating bridge databases from SQLite to Postgres, we'd rather avoid calling the "import Synapse SQLite database" command as just `--tags=import-sqlite-db`. Similarly, for the media store, we'd like to mention that it's related to Synapse as well. We'd like to be more explicit, so as to be less confusing, especially in light of other homeserver implementations coming in the future. --- docs/README.md | 6 +++--- docs/importing-postgres.md | 4 ++-- ...ting-media-store.md => importing-synapse-media-store.md} | 4 ++-- docs/{importing-sqlite.md => importing-synapse-sqlite.md} | 6 +++--- docs/installing.md | 4 ++-- .../{import_sqlite_db.yml => import_synapse_sqlite_db.yml} | 0 roles/matrix-postgres/tasks/main.yml | 6 ++++-- roles/matrix-synapse/tasks/main.yml | 2 +- 8 files changed, 17 insertions(+), 15 deletions(-) rename docs/{importing-media-store.md => importing-synapse-media-store.md} (92%) rename docs/{importing-sqlite.md => importing-synapse-sqlite.md} (86%) rename roles/matrix-postgres/tasks/{import_sqlite_db.yml => import_synapse_sqlite_db.yml} (100%) diff --git a/docs/README.md b/docs/README.md index 8d22477e..8c95eff3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,13 +10,13 @@ - [Installing](installing.md) -- **Importing data from another Synapse server installation** +- **Importing data from another server installation** - - [Importing an existing SQLite database (from another installation)](importing-sqlite.md) (optional) + - [Importing an existing SQLite database (from another Synapse installation)](importing-synapse-sqlite.md) (optional) - [Importing an existing Postgres database (from another installation)](importing-postgres.md) (optional) - - [Importing `media_store` data files from an existing installation](importing-media-store.md) (optional) + - [Importing `media_store` data files from an existing Synapse installation](importing-synapse-media-store.md) (optional) - [Registering users](registering-users.md) diff --git a/docs/importing-postgres.md b/docs/importing-postgres.md index a88067e1..0dd75cb2 100644 --- a/docs/importing-postgres.md +++ b/docs/importing-postgres.md @@ -1,7 +1,7 @@ # Importing an existing Postgres database from another installation (optional) -Run this if you'd like to import your database from a previous installation of Synapse. -(don't forget to import your `media_store` files as well - see [the importing-media-store guide](importing-media-store.md)). +Run this if you'd like to import your database from a previous installation. +(don't forget to import your Synapse `media_store` files as well - see [the importing-synape-media-store guide](importing-synapse-media-store.md)). ## Prerequisites diff --git a/docs/importing-media-store.md b/docs/importing-synapse-media-store.md similarity index 92% rename from docs/importing-media-store.md rename to docs/importing-synapse-media-store.md index 0d86370b..0ba7bacb 100644 --- a/docs/importing-media-store.md +++ b/docs/importing-synapse-media-store.md @@ -1,4 +1,4 @@ -# Importing `media_store` data files from an existing installation (optional) +# Importing `media_store` data files from an existing Synapse installation (optional) Run this if you'd like to import your `media_store` files from a previous installation of Synapse. @@ -17,6 +17,6 @@ As an alternative, you can perform a manual restore using the [AWS CLI tool](htt Run this command (make sure to replace `` with a path on your server): - ansible-playbook -i inventory/hosts setup.yml --extra-vars='server_path_media_store=' --tags=import-media-store + ansible-playbook -i inventory/hosts setup.yml --extra-vars='server_path_media_store=' --tags=import-synapse-media-store **Note**: `` must be a file path to a `media_store` directory on the server (not on your local machine!). diff --git a/docs/importing-sqlite.md b/docs/importing-synapse-sqlite.md similarity index 86% rename from docs/importing-sqlite.md rename to docs/importing-synapse-sqlite.md index 9e3a910d..aade2226 100644 --- a/docs/importing-sqlite.md +++ b/docs/importing-synapse-sqlite.md @@ -1,7 +1,7 @@ -# Importing an existing SQLite database from another installation (optional) +# Importing an existing SQLite database from another Synapse installation (optional) Run this if you'd like to import your database from a previous default installation of Synapse. -(don't forget to import your `media_store` files as well - see [the importing-media-store guide](importing-media-store.md)). +(don't forget to import your `media_store` files as well - see [the importing-synapse-media-store guide](importing-synapse-media-store.md)). While this playbook always sets up PostgreSQL, by default a Synapse installation would run using an SQLite database. @@ -18,7 +18,7 @@ Before doing the actual import, **you need to upload your SQLite database file t Run this command (make sure to replace `` with a file path on your server): - ansible-playbook -i inventory/hosts setup.yml --extra-vars='server_path_homeserver_db=' --tags=import-sqlite-db + ansible-playbook -i inventory/hosts setup.yml --extra-vars='server_path_homeserver_db=' --tags=import-synapse-sqlite-db **Notes**: diff --git a/docs/installing.md b/docs/installing.md index 43758ffb..a2ce1371 100644 --- a/docs/installing.md +++ b/docs/installing.md @@ -21,11 +21,11 @@ Feel free to **re-run this setup command any time** you think something is off w After installing, but before starting the services, you may want to do additional things like: -- [Importing an existing SQLite database (from another installation)](importing-sqlite.md) (optional) +- [Importing an existing SQLite database (from another Synapse installation)](importing-synapse-sqlite.md) (optional) - [Importing an existing Postgres database (from another installation)](importing-postgres.md) (optional) -- [Importing `media_store` data files from an existing installation](importing-media-store.md) (optional) +- [Importing `media_store` data files from an existing Synapse installation](importing-synapse-media-store.md) (optional) ## Starting the services diff --git a/roles/matrix-postgres/tasks/import_sqlite_db.yml b/roles/matrix-postgres/tasks/import_synapse_sqlite_db.yml similarity index 100% rename from roles/matrix-postgres/tasks/import_sqlite_db.yml rename to roles/matrix-postgres/tasks/import_synapse_sqlite_db.yml diff --git a/roles/matrix-postgres/tasks/main.yml b/roles/matrix-postgres/tasks/main.yml index 717b73e4..f4c752a0 100644 --- a/roles/matrix-postgres/tasks/main.yml +++ b/roles/matrix-postgres/tasks/main.yml @@ -19,10 +19,12 @@ tags: - import-postgres -- import_tasks: "{{ role_path }}/tasks/import_sqlite_db.yml" +# The `run_postgres_import_sqlite_db` variable had better be renamed to be consistent, +# but that's a breaking change which may cause trouble for people. +- import_tasks: "{{ role_path }}/tasks/import_synapse_sqlite_db.yml" when: run_postgres_import_sqlite_db|bool tags: - - import-sqlite-db + - import-synapse-sqlite-db - import_tasks: "{{ role_path }}/tasks/upgrade_postgres.yml" when: run_postgres_upgrade|bool diff --git a/roles/matrix-synapse/tasks/main.yml b/roles/matrix-synapse/tasks/main.yml index 48e72ace..e366a13f 100644 --- a/roles/matrix-synapse/tasks/main.yml +++ b/roles/matrix-synapse/tasks/main.yml @@ -17,7 +17,7 @@ - import_tasks: "{{ role_path }}/tasks/import_media_store.yml" when: run_synapse_import_media_store|bool tags: - - import-media-store + - import-synapse-media-store - import_tasks: "{{ role_path }}/tasks/register_user.yml" when: run_synapse_register_user|bool From cb969c6ca297b21216f896033ae86702be227812 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 02:23:29 +0200 Subject: [PATCH 127/217] Add --tags=import-generic-sqlite-db (pgloader import) This can be used by various bridges, etc., to import an SQLite (or some other supported) database into Postgres. --- roles/matrix-postgres/defaults/main.yml | 2 + .../tasks/import_generic_sqlite_db.yml | 84 +++++++++++++++++++ roles/matrix-postgres/tasks/main.yml | 6 ++ 3 files changed, 92 insertions(+) create mode 100644 roles/matrix-postgres/tasks/import_generic_sqlite_db.yml diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index 89559f5d..5755742f 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -48,3 +48,5 @@ matrix_postgres_additional_databases: [] # # For most (subsequent) runs, Postgres would already be running, so no waiting will be happening at all. matrix_postgres_additional_databases_postgres_start_wait_timeout_seconds: 15 + +matrix_postgres_pgloader_docker_image: "docker.io/illagrenan/pgloader:3.6.2" diff --git a/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml b/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml new file mode 100644 index 00000000..f2798a73 --- /dev/null +++ b/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml @@ -0,0 +1,84 @@ +--- + +# Pre-checks + +- name: Fail if Postgres not enabled + fail: + msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot import." + when: "not matrix_postgres_enabled|bool" + +- name: Fail if playbook called incorrectly + fail: + msg: "The `sqlite_database_path` variable needs to be provided to this playbook, via --extra-vars" + when: "sqlite_database_path is not defined or sqlite_database_path.startswith('<')" + +- name: Fail if playbook called incorrectly + fail: + msg: >- + The `postgres_db_connection_string` variable needs to be provided to this playbook, via `--extra-vars`. + Example: `--extra-vars="postgres_db_connection_string=postgresql://username:password@localhost:/database_name`" + when: "postgres_db_connection_string is not defined or not postgres_db_connection_string.startswith('postgresql://')" + +- name: Check if the provided SQLite database file exists + stat: + path: "{{ sqlite_database_path }}" + register: sqlite_database_path_stat_result + +- name: Fail if provided SQLite database file doesn't exist + fail: + msg: "File cannot be found on the server at {{ sqlite_database_path }}" + when: "not sqlite_database_path_stat_result.stat.exists" + + +# Defaults + +- name: Set postgres_start_wait_time, if not provided + set_fact: + postgres_start_wait_time: 15 + when: "postgres_start_wait_time|default('') == ''" + + +# Actual import work + +- name: Ensure matrix-postgres is started + service: + name: matrix-postgres + state: started + daemon_reload: yes + register: matrix_postgres_service_start_result + +- name: Wait a bit, so that Postgres can start + wait_for: + timeout: "{{ postgres_start_wait_time }}" + delegate_to: 127.0.0.1 + become: false + when: "matrix_postgres_service_start_result.changed|bool" + +- name: Import SQLite database from {{ sqlite_database_path }} into Postgres + command: + cmd: >- + {{ matrix_host_command_docker }} run + --rm + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --network={{ matrix_docker_network }} + --mount type=bind,src={{ sqlite_database_path }},dst=/in.db,ro + --entrypoint=/bin/sh + {{ matrix_postgres_pgloader_docker_image }} + -c + 'pgloader /in.db {{ postgres_db_connection_string }}' + +- name: Archive SQLite database ({{ sqlite_database_path }} -> {{ sqlite_database_path }}.backup) + command: + cmd: "mv {{ sqlite_database_path }} {{ sqlite_database_path }}.backup" + +- name: Inject result + set_fact: + matrix_playbook_runtime_results: | + {{ + matrix_playbook_runtime_results|default([]) + + + [ + "NOTE: Your SQLite database file has been imported into Postgres. The original file has been moved from `{{ sqlite_database_path }}` to `{{ sqlite_database_path }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file." + ] + }} diff --git a/roles/matrix-postgres/tasks/main.yml b/roles/matrix-postgres/tasks/main.yml index f4c752a0..b9c2ae7c 100644 --- a/roles/matrix-postgres/tasks/main.yml +++ b/roles/matrix-postgres/tasks/main.yml @@ -26,6 +26,12 @@ tags: - import-synapse-sqlite-db +# Perhaps we need a new variable here, instead of `run_postgres_import_sqlite_db`. +- import_tasks: "{{ role_path }}/tasks/import_generic_sqlite_db.yml" + when: run_postgres_import_sqlite_db|bool + tags: + - import-generic-sqlite-db + - import_tasks: "{{ role_path }}/tasks/upgrade_postgres.yml" when: run_postgres_upgrade|bool tags: From 4617984b9f998e032f426ec629e408a1f0c2ecf8 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 02:24:32 +0200 Subject: [PATCH 128/217] Add (SQLite -> Postgres) migration instructions --- .../tasks/setup_install.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml index cea8fda8..57e59df0 100644 --- a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml @@ -1,10 +1,13 @@ --- - block: + - set_fact: + matrix_appservice_discord_sqlite_db_path: "{{ matrix_appservice_discord_data_path }}/{{ matrix_appservice_discord_database_filename_name }}" + - name: Check if an SQLite database already exists stat: - path: "{{ matrix_appservice_discord_data_path }}/{{ matrix_appservice_discord_database_filename_name }}" - register: matrix_appservice_discord_stat_sqlite_db + path: "{{ matrix_appservice_discord_sqlite_db_path }}" + register: matrix_appservice_discord_sqlite_db_path_stat_result - name: Fail if an SQLite database already exists when using Postgres fail: @@ -12,8 +15,12 @@ matrix_appservice_discord_database_engine has been set to `postgres` (which is our new default now). However, we've discovered an existing SQLite database in {{ matrix_appservice_discord_data_path }}/{{ matrix_appservice_discord_database_filename_name }}. It appears that you've been using this bridge with the SQLite engine until now. - To continue using SQLite, opt into it explicitly: add `matrix_appservice_discord_database_engine: sqlite` to your vars.yml file. - To migrate to Postgres: TODO - migration instructions here. + To continue using SQLite, opt into it explicitly: add `matrix_appservice_discord_database_engine: sqlite` to your vars.yml file and re-run this same command. + Alternatively, to migrate your existing SQLite database to Postgres: + 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) + 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_discord_sqlite_db_path }} postgres_db_connection_string={{ matrix_appservice_discord_database_connString }}'`) + 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) + when: "matrix_appservice_discord_sqlite_db_path_stat_result.stat.exists" when: "matrix_appservice_discord_database_engine == 'postgres'" - name: Ensure Appservice Discord image is pulled From b217a2ed6907347200c57d66195dd780e4a1f24f Mon Sep 17 00:00:00 2001 From: transcaffeine Date: Mon, 14 Dec 2020 07:54:24 +0100 Subject: [PATCH 129/217] postgres: set default in playbook to postgres but fallback to sqlite --- group_vars/matrix_servers | 175 +++++++++++++++++++++++++------------- 1 file changed, 118 insertions(+), 57 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index b9d0be26..0cbc4587 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -140,6 +140,10 @@ matrix_appservice_slack_systemd_required_services_list: | (['matrix-synapse.service'] if matrix_synapse_enabled else []) }} +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_appservice_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_appservice_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.slack.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-appservice-slack @@ -175,6 +179,11 @@ matrix_appservice_irc_appservice_token: "{{ matrix_synapse_macaroon_secret_key | matrix_appservice_irc_homeserver_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'irc.hs.token') | to_uuid }}" +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_appservice_irc_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_appservice_irc_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.irc.db') | to_uuid }}" + + ###################################################################### # # /matrix-bridge-appservice-irc @@ -208,6 +217,10 @@ matrix_mautrix_facebook_login_shared_secret: "{{ matrix_synapse_ext_password_pro matrix_mautrix_facebook_bridge_presence: "{{ matrix_synapse_use_presence if matrix_synapse_enabled else true }}" +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_mautrix_facebook_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_mautrix_facebook_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_facebook.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-mautrix-facebook @@ -241,6 +254,10 @@ matrix_mautrix_hangouts_container_http_host_bind_port: "{{ '' if matrix_nginx_pr matrix_mautrix_hangouts_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}" +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_mautrix_hangouts_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_mautrix_hangouts_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_hangouts.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-mautrix-hangouts @@ -277,6 +294,10 @@ matrix_mautrix_telegram_container_http_host_bind_port: "{{ '' if matrix_nginx_pr matrix_mautrix_telegram_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}" +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_mautrix_telegram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_mautrix_telegram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_telegram.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-mautrix-telegram @@ -305,6 +326,10 @@ matrix_mautrix_whatsapp_homeserver_token: "{{ matrix_synapse_macaroon_secret_key matrix_mautrix_whatsapp_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}" +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_mautrix_whatsapp_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_mautrix_whatsapp_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_whatsapp.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-mautrix-whatsapp @@ -361,6 +386,10 @@ matrix_mx_puppet_skype_homeserver_token: "{{ matrix_synapse_macaroon_secret_key matrix_mx_puppet_skype_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}" +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_mx_puppet_skype_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_mx_puppet_skype_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_skype.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-mx-puppet-skype @@ -392,6 +421,10 @@ matrix_mx_puppet_slack_homeserver_token: "{{ matrix_synapse_macaroon_secret_key matrix_mx_puppet_slack_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}" +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_mx_puppet_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_mx_puppet_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_slack.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-mx-puppet-slack @@ -424,6 +457,10 @@ matrix_mx_puppet_twitter_login_shared_secret: "{{ matrix_synapse_ext_password_pr matrix_mx_puppet_twitter_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else ('127.0.0.1:' ~ matrix_mx_puppet_twitter_appservice_port) }}" +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_mx_puppet_twitter_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_mx_puppet_twitter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_twitter.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-mx-puppet-twitter @@ -455,6 +492,10 @@ matrix_mx_puppet_instagram_homeserver_token: "{{ matrix_synapse_macaroon_secret_ matrix_mx_puppet_instagram_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}" +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_mx_puppet_instagram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_mx_puppet_instagram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_instagram.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-mx-puppet-instagram @@ -485,6 +526,10 @@ matrix_mx_puppet_discord_homeserver_token: "{{ matrix_synapse_macaroon_secret_ke matrix_mx_puppet_discord_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}" +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_mx_puppet_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_mx_puppet_discord_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_discord.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-mx-puppet-discord @@ -515,6 +560,10 @@ matrix_mx_puppet_steam_homeserver_token: "{{ matrix_synapse_macaroon_secret_key matrix_mx_puppet_steam_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}" +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_mx_puppet_stream_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_steam.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-mx-puppet-steam @@ -625,6 +674,10 @@ matrix_dimension_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_ena matrix_integration_manager_rest_url: "{{ matrix_dimension_integrations_rest_url if matrix_dimension_enabled else None }}" matrix_integration_manager_ui_url: "{{ matrix_dimension_integrations_ui_url if matrix_dimension_enabled else None }}" +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_dimension_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_dimension_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.dimension.db') | to_uuid }}" + ###################################################################### # # /matrix-dimension @@ -902,75 +955,83 @@ matrix_postgres_additional_databases: | 'password': matrix_appservice_discord_database_connString_password, }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_database_connString_hostname == 'matrix-postgres') else []) + ([{ - 'name': 'matrix_appservice_slack', - 'username': 'matrix_appservice_slack', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | to_uuid, - }] if matrix_appservice_slack_enabled else []) + 'name': matrix_appservice_slack_database_db_name + 'username': matrix_appservice_slack_database_username + 'password': matrix_appservice_slack_database_password + }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres' else []) + ([{ - 'name': 'matrix_appservice_irc', - 'username': 'matrix_appservice_irc', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | to_uuid, - }] if matrix_appservice_irc_enabled else []) + 'name': matrix_appservice_irc_database_db_name + 'username': matrix_appservice_irc_database_username + 'password': matrix_appservice_irc_database_password + }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres' else []) + ([{ - 'name': 'mautrix_bridge_facebook', - 'username': 'mautrix_bridge_facebook', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | to_uuid, - }] if matrix_mautrix_facebook_enabled else []) + 'name': matrix_mautrix_facebook_database_db_name + 'username': matrix_mautrix_facebook_database_username + 'password': matrix_mautrix_facebook_database_password + }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_database_hostname == 'matrix-postgres' else []) + ([{ - 'name': 'mautrix_bridge_hangouts', - 'username': 'mautrix_bridge_hangouts', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | to_uuid, - }] if matrix_mautrix_hangouts_enabled else []) + 'name': matrix_mautrix_hangouts_database_db_name + 'username': matrix_mautrix_hangouts_database_username + 'password': matrix_mautrix_hangouts_database_password + }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres' else []) + ([{ - 'name': 'mautrix_bridge_telegram', - 'username': 'mautrix_bridge_telegram', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | to_uuid, - }] if matrix_mautrix_telegram_enabled else []) + 'name': matrix_mautrix_telegram_database_db_name + 'username': matrix_mautrix_telegram_database_username + 'password': matrix_mautrix_telegram_database_password + }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_telegram_facebook_database_hostname == 'matrix-postgres' else []) + ([{ - 'name': 'mautrix_bridge_whatsapp', - 'username': 'mautrix_bridge_whatsapp', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | to_uuid, - }] if matrix_mautrix_whatsapp_enabled else []) + 'name': matrix_mautrix_whatsapp_database_db_name + 'username': matrix_mautrix_whatsapp_database_username + 'password': matrix_mautrix_whatsapp_database_password + }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres' else []) + ([{ 'name': 'matrix_bridge_sms', 'username': 'matrix_bridge_sms', 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid, }] if matrix_sms_bridge_enabled else []) + ([{ - 'name': 'matrix_puppet_skype', - 'username': 'matrix_puppet_skype', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | to_uuid, - }] if matrix_mx_puppet_skype_enabled else []) + 'name': matrix_mx_puppet_skype_database_db_name + 'username': matrix_mx_puppet_skype_database_username + 'password': matrix_mx_puppet_skype_database_password + }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres' else []) + ([{ - 'name': 'matrix_puppet_slack', - 'username': 'matrix_puppet_slack', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | to_uuid, - }] if matrix_mx_puppet_slack_enabled else []) - + ([{ - 'name': 'matrix_puppet_twitter', - 'username': 'matrix_puppet_twitter', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | to_uuid, - }] if matrix_mx_puppet_twitter_enabled else []) - + ([{ - 'name': 'matrix_puppet_instagram', - 'username': 'matrix_puppet_instagram', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | to_uuid, - }] if matrix_mx_puppet_instagram_enabled else []) - + ([{ - 'name': 'matrix_puppet_discord', - 'username': 'matrix_puppet_discord', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | to_uuid, - }] if matrix_mx_puppet_discord_enabled else []) - + ([{ - 'name': 'matrix_puppet_steam', - 'username': 'matrix_puppet_steam', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | to_uuid, - }] if matrix_mx_puppet_steam_enabled else []) - + ([{ - 'name': 'matrix_dimension', - 'username': 'matrix_dimension', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | to_uuid, - }] if matrix_dimension_enabled else []) + 'name': matrix_mx_puppet_slack_database_db_name + 'username': matrix_mx_puppet_slack_database_username + 'password': matrix_mx_puppet_slack_database_password + }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres' else []) + + ([{ + 'name': matrix_mx_puppet_twitter_database_db_name + 'username': matrix_mx_puppet_twitter_database_username + 'password': matrix_mx_puppet_twitter_database_password + }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres' else []) + + ([{ + 'name': matrix_mx_puppet_instagram_database_db_name + 'username': matrix_mx_puppet_instagram_database_username + 'password': matrix_mx_puppet_instagram_database_password + }] if (matrix_mx_puppet_instagram_enabled + and matrix_mx_puppet_instagram_database_engine == 'postgres' + and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres' else []) + + ([{ + 'name': matrix_mx_puppet_discord_database_db_name + 'username': matrix_mx_puppet_discord_database_username + 'password': matrix_mx_puppet_discord_database_password + }] if (matrix_mx_puppet_discord_enabled + and matrix_mx_puppet_discord_database_engine == 'postgres' + and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres' else []) + + ([{ + 'name': matrix_mx_puppet_steam_database_db_name + 'username': matrix_mx_puppet_steam_database_username + 'password': matrix_mx_puppet_steam_database_password + }] if (matrix_mx_puppet_steam_enabled + and matrix_mx_puppet_steam_database_engine == 'postgres' + and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres' else []) + + ([{ + 'name': matrix_dimension_database_db_name + 'username': matrix_dimension_database_username + 'password': matrix_dimension_database_password + }] if (matrix_dimension_enabled + and matrix_dimension_database_engine == 'postgres' + and matrix_dimension_database_hostname == 'matrix-postgres' else []) }} ###################################################################### From f1e85f7112ffa9dd3309e8963c9a8e69723f48a0 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 10:04:37 +0200 Subject: [PATCH 130/217] Don't mention Postgres roles, just say users --- .../templates/init-additional-db-user-and-role.sql.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2 b/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2 index af3d912c..609a1344 100644 --- a/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2 +++ b/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2 @@ -4,7 +4,7 @@ DO $$ BEGIN CREATE USER {{ additional_db.username }}; EXCEPTION WHEN DUPLICATE_OBJECT THEN - RAISE NOTICE 'not creating role {{ additional_db.username }}, since it already exists'; + RAISE NOTICE 'not creating user {{ additional_db.username }}, since it already exists'; END $$; From d91aa5a060ab0742f25c15a951429104c1793a91 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 10:51:53 +0200 Subject: [PATCH 131/217] Do not introduce sub-variables exposing implementation details --- group_vars/matrix_servers | 10 ++++---- .../defaults/main.yml | 25 ++++++++++++------- .../tasks/setup_install.yml | 13 ++++------ 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 0cbc4587..efb29d4a 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -70,7 +70,7 @@ matrix_appservice_discord_homeserver_token: "{{ matrix_synapse_macaroon_secret_k # We only make this use Postgres if our own Postgres server is enabled. # It's only then (for now) that we can automatically create the necessary database and user for this service. matrix_appservice_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_appservice_discord_database_connString_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.discord.db') | to_uuid }}" +matrix_appservice_discord_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.discord.db') | to_uuid }}" ###################################################################### # @@ -950,10 +950,10 @@ matrix_postgres_db_name: "homeserver" matrix_postgres_additional_databases: | {{ ([{ - 'name': matrix_appservice_discord_database_connString_db_name, - 'username': matrix_appservice_discord_database_connString_username, - 'password': matrix_appservice_discord_database_connString_password, - }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_database_connString_hostname == 'matrix-postgres') else []) + 'name': matrix_appservice_discord_postgres_db_name, + 'username': matrix_appservice_discord_postgres_username, + 'password': matrix_appservice_discord_postgres_password, + }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_postgres_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_appservice_slack_database_db_name 'username': matrix_appservice_slack_database_username diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml index 53d7968d..fb517cd5 100644 --- a/roles/matrix-bridge-appservice-discord/defaults/main.yml +++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml @@ -47,16 +47,23 @@ matrix_appservice_discord_bridge_enableSelfServiceBridging: false # # To use Postgres: # - change the engine (`matrix_appservice_discord_database_engine: 'postgres'`) -# - adjust your database credentials +# - adjust your database credentials via the `matrix_appservice_discord_postgres_*` variables matrix_appservice_discord_database_engine: 'sqlite' -matrix_appservice_discord_database_filename: "/data/{{ matrix_appservice_discord_database_filename_name }}" -matrix_appservice_discord_database_filename_name: "discord.db" -matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_connString_username }}:{{ matrix_appservice_discord_database_connString_password }}@{{ matrix_appservice_discord_database_connString_hostname }}:{{ matrix_appservice_discord_database_connString_port }}/{{ matrix_appservice_discord_database_connString_db_name }}' -matrix_appservice_discord_database_connString_username: 'matrix_appservice_discord' -matrix_appservice_discord_database_connString_password: 'some-password' -matrix_appservice_discord_database_connString_hostname: 'matrix-postgres' -matrix_appservice_discord_database_connString_port: 5432 -matrix_appservice_discord_database_connString_db_name: 'matrix_appservice_discord' + +matrix_appservice_discord_sqlite_database_path_local: "{{ matrix_appservice_discord_data_path }}/discord.db" +matrix_appservice_discord_sqlite_database_path_in_container: "/data/discord.db" + +matrix_appservice_discord_postgres_username: 'matrix_appservice_discord' +matrix_appservice_discord_postgres_password: 'some-password' +matrix_appservice_discord_postgres_hostname: 'matrix-postgres' +matrix_appservice_discord_postgres_port: 5432 +matrix_appservice_discord_postgres_db_name: 'matrix_appservice_discord' + +# These 2 variables are what actually ends up in the bridge configuration. +# It's best if you don't change them directly, but rather redefine the sub-variables that constitute them. +matrix_appservice_discord_database_filename: "{{ matrix_appservice_discord_sqlite_database_path_in_container }}" +matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_postgres_username }}:{{ matrix_appservice_discord_postgres_password }}@{{ matrix_appservice_discord_postgres_hostname }}:{{ matrix_appservice_discord_postgres_port }}/{{ matrix_appservice_discord_postgres_db_name }}' + # Tells whether the bot should make use of "Privileged Gateway Intents". # diff --git a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml index 57e59df0..86a5fe51 100644 --- a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml @@ -1,26 +1,23 @@ --- - block: - - set_fact: - matrix_appservice_discord_sqlite_db_path: "{{ matrix_appservice_discord_data_path }}/{{ matrix_appservice_discord_database_filename_name }}" - - name: Check if an SQLite database already exists stat: - path: "{{ matrix_appservice_discord_sqlite_db_path }}" - register: matrix_appservice_discord_sqlite_db_path_stat_result + path: "{{ matrix_appservice_discord_sqlite_database_path_local }}" + register: matrix_appservice_discord_sqlite_database_path_local_stat_result - name: Fail if an SQLite database already exists when using Postgres fail: msg: >- matrix_appservice_discord_database_engine has been set to `postgres` (which is our new default now). - However, we've discovered an existing SQLite database in {{ matrix_appservice_discord_data_path }}/{{ matrix_appservice_discord_database_filename_name }}. + However, we've discovered an existing SQLite database in {{ matrix_appservice_discord_sqlite_database_path_local }}. It appears that you've been using this bridge with the SQLite engine until now. To continue using SQLite, opt into it explicitly: add `matrix_appservice_discord_database_engine: sqlite` to your vars.yml file and re-run this same command. Alternatively, to migrate your existing SQLite database to Postgres: 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_discord_sqlite_db_path }} postgres_db_connection_string={{ matrix_appservice_discord_database_connString }}'`) + 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_discord_sqlite_database_path_local }} postgres_db_connection_string={{ matrix_appservice_discord_database_connString }}'`) 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) - when: "matrix_appservice_discord_sqlite_db_path_stat_result.stat.exists" + when: "matrix_appservice_discord_sqlite_database_path_local_stat_result.stat.exists" when: "matrix_appservice_discord_database_engine == 'postgres'" - name: Ensure Appservice Discord image is pulled From c0edacbefd60d5bfd136c448ff3fdf76da645731 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 11:17:12 +0200 Subject: [PATCH 132/217] Fix matrix_postgres_additional_databases syntax problems --- group_vars/matrix_servers | 106 +++++++++++++++++++------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index efb29d4a..763fccd8 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -955,84 +955,84 @@ matrix_postgres_additional_databases: | 'password': matrix_appservice_discord_postgres_password, }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_postgres_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_appservice_slack_database_db_name - 'username': matrix_appservice_slack_database_username - 'password': matrix_appservice_slack_database_password - }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres' else []) + 'name': matrix_appservice_slack_database_db_name, + 'username': matrix_appservice_slack_database_username, + 'password': matrix_appservice_slack_database_password, + }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_appservice_irc_database_db_name - 'username': matrix_appservice_irc_database_username - 'password': matrix_appservice_irc_database_password - }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres' else []) + 'name': matrix_appservice_irc_database_db_name, + 'username': matrix_appservice_irc_database_username, + 'password': matrix_appservice_irc_database_password, + }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mautrix_facebook_database_db_name - 'username': matrix_mautrix_facebook_database_username - 'password': matrix_mautrix_facebook_database_password - }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_database_hostname == 'matrix-postgres' else []) + 'name': matrix_mautrix_facebook_postgres_db_name, + 'username': matrix_mautrix_facebook_postgres_username, + 'password': matrix_mautrix_facebook_postgres_password, + }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_postgres_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mautrix_hangouts_database_db_name - 'username': matrix_mautrix_hangouts_database_username - 'password': matrix_mautrix_hangouts_database_password - }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres' else []) + 'name': matrix_mautrix_hangouts_database_db_name, + 'username': matrix_mautrix_hangouts_database_username, + 'password': matrix_mautrix_hangouts_database_password, + }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mautrix_telegram_database_db_name - 'username': matrix_mautrix_telegram_database_username - 'password': matrix_mautrix_telegram_database_password - }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_telegram_facebook_database_hostname == 'matrix-postgres' else []) + 'name': matrix_mautrix_telegram_database_db_name, + 'username': matrix_mautrix_telegram_database_username, + 'password': matrix_mautrix_telegram_database_password, + }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_telegram_facebook_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mautrix_whatsapp_database_db_name - 'username': matrix_mautrix_whatsapp_database_username - 'password': matrix_mautrix_whatsapp_database_password - }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres' else []) + 'name': matrix_mautrix_whatsapp_database_db_name, + 'username': matrix_mautrix_whatsapp_database_username, + 'password': matrix_mautrix_whatsapp_database_password, + }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres') else []) + ([{ 'name': 'matrix_bridge_sms', 'username': 'matrix_bridge_sms', 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid, }] if matrix_sms_bridge_enabled else []) + ([{ - 'name': matrix_mx_puppet_skype_database_db_name - 'username': matrix_mx_puppet_skype_database_username - 'password': matrix_mx_puppet_skype_database_password - }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres' else []) + 'name': matrix_mx_puppet_skype_database_db_name, + 'username': matrix_mx_puppet_skype_database_username, + 'password': matrix_mx_puppet_skype_database_password, + }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mx_puppet_slack_database_db_name - 'username': matrix_mx_puppet_slack_database_username - 'password': matrix_mx_puppet_slack_database_password - }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres' else []) + 'name': matrix_mx_puppet_slack_database_db_name, + 'username': matrix_mx_puppet_slack_database_username, + 'password': matrix_mx_puppet_slack_database_password, + }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mx_puppet_twitter_database_db_name - 'username': matrix_mx_puppet_twitter_database_username - 'password': matrix_mx_puppet_twitter_database_password - }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres' else []) + 'name': matrix_mx_puppet_twitter_database_db_name, + 'username': matrix_mx_puppet_twitter_database_username, + 'password': matrix_mx_puppet_twitter_database_password, + }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mx_puppet_instagram_database_db_name - 'username': matrix_mx_puppet_instagram_database_username - 'password': matrix_mx_puppet_instagram_database_password + 'name': matrix_mx_puppet_instagram_database_db_name, + 'username': matrix_mx_puppet_instagram_database_username, + 'password': matrix_mx_puppet_instagram_database_password, }] if (matrix_mx_puppet_instagram_enabled and matrix_mx_puppet_instagram_database_engine == 'postgres' - and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres' else []) + and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mx_puppet_discord_database_db_name - 'username': matrix_mx_puppet_discord_database_username - 'password': matrix_mx_puppet_discord_database_password + 'name': matrix_mx_puppet_discord_database_db_name, + 'username': matrix_mx_puppet_discord_database_username, + 'password': matrix_mx_puppet_discord_database_password, }] if (matrix_mx_puppet_discord_enabled and matrix_mx_puppet_discord_database_engine == 'postgres' - and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres' else []) + and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mx_puppet_steam_database_db_name - 'username': matrix_mx_puppet_steam_database_username - 'password': matrix_mx_puppet_steam_database_password + 'name': matrix_mx_puppet_steam_database_db_name, + 'username': matrix_mx_puppet_steam_database_username, + 'password': matrix_mx_puppet_steam_database_password, }] if (matrix_mx_puppet_steam_enabled and matrix_mx_puppet_steam_database_engine == 'postgres' - and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres' else []) + and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_dimension_database_db_name - 'username': matrix_dimension_database_username - 'password': matrix_dimension_database_password + 'name': matrix_dimension_database_db_name, + 'username': matrix_dimension_database_username, + 'password': matrix_dimension_database_password, }] if (matrix_dimension_enabled and matrix_dimension_database_engine == 'postgres' - and matrix_dimension_database_hostname == 'matrix-postgres' else []) - }} + and matrix_dimension_database_hostname == 'matrix-postgres') else []) + }} ###################################################################### # From 2a99e84b5bfd9f3988566e498b2fe729baa81c75 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 11:19:18 +0200 Subject: [PATCH 133/217] Rename Postgres-related variables (`matrix*database_*` -> `matrix*postgres_*`) `matrix_SERVICE_database_engine` remains as it is - the only one with a `_database_` infix. Postgres-related ones move to `_postgres_`. --- group_vars/matrix_servers | 72 +++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 763fccd8..5e3695ca 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -955,14 +955,14 @@ matrix_postgres_additional_databases: | 'password': matrix_appservice_discord_postgres_password, }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_postgres_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_appservice_slack_database_db_name, - 'username': matrix_appservice_slack_database_username, - 'password': matrix_appservice_slack_database_password, + 'name': matrix_appservice_slack_postgres_db_name, + 'username': matrix_appservice_slack_postgres_username, + 'password': matrix_appservice_slack_postgres_password, }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_appservice_irc_database_db_name, - 'username': matrix_appservice_irc_database_username, - 'password': matrix_appservice_irc_database_password, + 'name': matrix_appservice_irc_postgres_db_name, + 'username': matrix_appservice_irc_postgres_username, + 'password': matrix_appservice_irc_postgres_password, }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_mautrix_facebook_postgres_db_name, @@ -970,19 +970,19 @@ matrix_postgres_additional_databases: | 'password': matrix_mautrix_facebook_postgres_password, }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_postgres_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mautrix_hangouts_database_db_name, - 'username': matrix_mautrix_hangouts_database_username, - 'password': matrix_mautrix_hangouts_database_password, + 'name': matrix_mautrix_hangouts_postgres_db_name, + 'username': matrix_mautrix_hangouts_postgres_username, + 'password': matrix_mautrix_hangouts_postgres_password, }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mautrix_telegram_database_db_name, - 'username': matrix_mautrix_telegram_database_username, - 'password': matrix_mautrix_telegram_database_password, + 'name': matrix_mautrix_telegram_postgres_db_name, + 'username': matrix_mautrix_telegram_postgres_username, + 'password': matrix_mautrix_telegram_postgres_password, }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_telegram_facebook_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mautrix_whatsapp_database_db_name, - 'username': matrix_mautrix_whatsapp_database_username, - 'password': matrix_mautrix_whatsapp_database_password, + 'name': matrix_mautrix_whatsapp_postgres_db_name, + 'username': matrix_mautrix_whatsapp_postgres_username, + 'password': matrix_mautrix_whatsapp_postgres_password, }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres') else []) + ([{ 'name': 'matrix_bridge_sms', @@ -990,45 +990,45 @@ matrix_postgres_additional_databases: | 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid, }] if matrix_sms_bridge_enabled else []) + ([{ - 'name': matrix_mx_puppet_skype_database_db_name, - 'username': matrix_mx_puppet_skype_database_username, - 'password': matrix_mx_puppet_skype_database_password, + 'name': matrix_mx_puppet_skype_postgres_db_name, + 'username': matrix_mx_puppet_skype_postgres_username, + 'password': matrix_mx_puppet_skype_postgres_password, }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mx_puppet_slack_database_db_name, - 'username': matrix_mx_puppet_slack_database_username, - 'password': matrix_mx_puppet_slack_database_password, + 'name': matrix_mx_puppet_slack_postgres_db_name, + 'username': matrix_mx_puppet_slack_postgres_username, + 'password': matrix_mx_puppet_slack_postgres_password, }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mx_puppet_twitter_database_db_name, - 'username': matrix_mx_puppet_twitter_database_username, - 'password': matrix_mx_puppet_twitter_database_password, + 'name': matrix_mx_puppet_twitter_postgres_db_name, + 'username': matrix_mx_puppet_twitter_postgres_username, + 'password': matrix_mx_puppet_twitter_postgres_password, }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mx_puppet_instagram_database_db_name, - 'username': matrix_mx_puppet_instagram_database_username, - 'password': matrix_mx_puppet_instagram_database_password, + 'name': matrix_mx_puppet_instagram_postgres_db_name, + 'username': matrix_mx_puppet_instagram_postgres_username, + 'password': matrix_mx_puppet_instagram_postgres_password, }] if (matrix_mx_puppet_instagram_enabled and matrix_mx_puppet_instagram_database_engine == 'postgres' and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mx_puppet_discord_database_db_name, - 'username': matrix_mx_puppet_discord_database_username, - 'password': matrix_mx_puppet_discord_database_password, + 'name': matrix_mx_puppet_discord_postgres_db_name, + 'username': matrix_mx_puppet_discord_postgres_username, + 'password': matrix_mx_puppet_discord_postgres_password, }] if (matrix_mx_puppet_discord_enabled and matrix_mx_puppet_discord_database_engine == 'postgres' and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mx_puppet_steam_database_db_name, - 'username': matrix_mx_puppet_steam_database_username, - 'password': matrix_mx_puppet_steam_database_password, + 'name': matrix_mx_puppet_steam_postgres_db_name, + 'username': matrix_mx_puppet_steam_postgres_username, + 'password': matrix_mx_puppet_steam_postgres_password, }] if (matrix_mx_puppet_steam_enabled and matrix_mx_puppet_steam_database_engine == 'postgres' and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_dimension_database_db_name, - 'username': matrix_dimension_database_username, - 'password': matrix_dimension_database_password, + 'name': matrix_dimension_postgres_db_name, + 'username': matrix_dimension_postgres_username, + 'password': matrix_dimension_postgres_password, }] if (matrix_dimension_enabled and matrix_dimension_database_engine == 'postgres' and matrix_dimension_database_hostname == 'matrix-postgres') else []) From 5dba0c038b6c19a9b430210cfaaa7538e383cef9 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 11:47:00 +0200 Subject: [PATCH 134/217] Make --tags=import-generic-sqlite-db commands not pass a sensitive connection string around Instead of passing the connection string, we can now pass a name of a variable, which contains a connection string. Both are supported for having extra flexibility. --- .../tasks/setup_install.yml | 2 +- .../tasks/import_generic_sqlite_db.yml | 27 ++++++++++++++----- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml index 86a5fe51..a9d8da4c 100644 --- a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml @@ -15,7 +15,7 @@ To continue using SQLite, opt into it explicitly: add `matrix_appservice_discord_database_engine: sqlite` to your vars.yml file and re-run this same command. Alternatively, to migrate your existing SQLite database to Postgres: 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_discord_sqlite_database_path_local }} postgres_db_connection_string={{ matrix_appservice_discord_database_connString }}'`) + 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_discord_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_appservice_discord_database_connString'`) 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) when: "matrix_appservice_discord_sqlite_database_path_local_stat_result.stat.exists" when: "matrix_appservice_discord_database_engine == 'postgres'" diff --git a/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml b/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml index f2798a73..a42c6f55 100644 --- a/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml +++ b/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml @@ -12,13 +12,6 @@ msg: "The `sqlite_database_path` variable needs to be provided to this playbook, via --extra-vars" when: "sqlite_database_path is not defined or sqlite_database_path.startswith('<')" -- name: Fail if playbook called incorrectly - fail: - msg: >- - The `postgres_db_connection_string` variable needs to be provided to this playbook, via `--extra-vars`. - Example: `--extra-vars="postgres_db_connection_string=postgresql://username:password@localhost:/database_name`" - when: "postgres_db_connection_string is not defined or not postgres_db_connection_string.startswith('postgresql://')" - - name: Check if the provided SQLite database file exists stat: path: "{{ sqlite_database_path }}" @@ -29,6 +22,26 @@ msg: "File cannot be found on the server at {{ sqlite_database_path }}" when: "not sqlite_database_path_stat_result.stat.exists" +# We either expect `postgres_db_connection_string` specifying a full Postgres database connection string, +# or `postgres_connection_string_variable_name`, specifying a name of a variable, which contains a valid connection string. + +- block: + - name: Fail if postgres_connection_string_variable_name points to an undefined variable + fail: msg="postgres_connection_string_variable_name is defined, but there is no variable with the name `{{ postgres_connection_string_variable_name }}`" + when: "postgres_connection_string_variable_name not in vars" + + - name: Get Postgres connection string from variable + set_fact: + postgres_db_connection_string: "{{ lookup('vars', postgres_connection_string_variable_name) }}" + when: 'postgres_connection_string_variable_name is defined' + +- name: Fail if playbook called incorrectly + fail: + msg: >- + Either a `postgres_db_connection_string` variable or a `postgres_connection_string_variable_name` needs to be provided to this playbook, via `--extra-vars`. + Example: `--extra-vars="postgres_db_connection_string=postgresql://username:password@localhost:/database_name"` or `--extra-vars="postgres_connection_string_variable_name=matrix_appservice_discord_database_connString"` + when: "postgres_db_connection_string is not defined or not postgres_db_connection_string.startswith('postgresql://')" + # Defaults From 2d99ade72f09c9f25b718f0cda690319d9bea526 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 11:50:42 +0200 Subject: [PATCH 135/217] Add (Postgres + SQLite) support to matrix-mautrix-facebook bridge --- .../defaults/main.yml | 29 +++++++++++++++++++ .../tasks/setup_install.yml | 21 ++++++++++++++ .../templates/config.yaml.j2 | 2 +- .../matrix-mautrix-facebook.service.j2 | 1 + 4 files changed, 52 insertions(+), 1 deletion(-) diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml index 1547ad71..95adf354 100644 --- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml @@ -32,6 +32,35 @@ matrix_mautrix_facebook_systemd_wanted_services_list: [] matrix_mautrix_facebook_appservice_token: '' matrix_mautrix_facebook_homeserver_token: '' + + +# Database-related configuration fields. +# +# To use SQLite, stick to these defaults. +# +# To use Postgres: +# - change the engine (`matrix_mautrix_facebook_database_engine: 'postgres'`) +# - adjust your database credentials via the `matrix_mautrix_facebook_postgres_*` variables +matrix_mautrix_facebook_database_engine: 'sqlite' + +matrix_mautrix_facebook_sqlite_database_path_local: "{{ matrix_mautrix_facebook_data_path }}/mautrix-facebook.db" +matrix_mautrix_facebook_sqlite_database_path_in_container: "/data/mautrix-facebook.db" + +matrix_mautrix_facebook_postgres_username: 'matrix_mautrix_facebook' +matrix_mautrix_facebook_postgres_password: 'some-password' +matrix_mautrix_facebook_postgres_hostname: 'matrix-postgres' +matrix_mautrix_facebook_postgres_port: 5432 +matrix_mautrix_facebook_postgres_db_name: 'matrix_mautrix_facebook' + +matrix_mautrix_facebook_postgres_connection_string: 'postgresql://{{ matrix_mautrix_facebook_postgres_username }}:{{ matrix_mautrix_facebook_postgres_password }}@{{ matrix_mautrix_facebook_postgres_hostname }}:{{ matrix_mautrix_facebook_postgres_port }}/{{ matrix_mautrix_facebook_postgres_db_name }}' + +matrix_mautrix_facebook_appservice_database: "{{ + { + 'sqlite': matrix_mautrix_facebook_sqlite_database_path_in_container, + 'postgres': matrix_mautrix_facebook_postgres_connection_string, + }[matrix_mautrix_facebook_database_engine] +}}" + # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). matrix_mautrix_facebook_login_shared_secret: '' diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml index 638854e2..ef814108 100644 --- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml @@ -1,5 +1,26 @@ --- +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mautrix_facebook_sqlite_database_path_local }}" + register: matrix_mautrix_facebook_sqlite_database_path_local_stat_result + + - name: Fail if an SQLite database already exists when using Postgres + fail: + msg: >- + matrix_mautrix_facebook_database_engine has been set to `postgres` (which is our new default now). + However, we've discovered an existing SQLite database in {{ matrix_mautrix_facebook_sqlite_database_path_local }}. + It appears that you've been using this bridge with the SQLite engine until now. + To continue using SQLite, opt into it explicitly: add `matrix_mautrix_facebook_database_engine: sqlite` to your vars.yml file and re-run this same command. + Alternatively, to migrate your existing SQLite database to Postgres: + 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) + 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_facebook_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_facebook_postgres_connection_string'`) + 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) + when: "matrix_mautrix_facebook_sqlite_database_path_local_stat_result.stat.exists" + when: "matrix_mautrix_facebook_database_engine == 'postgres'" + + # If the matrix-synapse role is not used, `matrix_synapse_role_executed` won't exist. # We don't want to fail in such cases. - name: Fail if matrix-synapse role already executed diff --git a/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 index 304571c1..09287362 100644 --- a/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 @@ -27,7 +27,7 @@ appservice: # Format examples: # SQLite: sqlite:///filename.db # Postgres: postgres://username:password@hostname/dbname - database: sqlite://matrix_bridge_facebook:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_bridge_facebook') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_bridge_facebook + database: {{ matrix_mautrix_facebook_appservice_database|to_json }} # Public part of web server for out-of-Matrix interaction with the bridge. public: diff --git a/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 b/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 index 18184650..caa52eb5 100644 --- a/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 +++ b/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 @@ -18,6 +18,7 @@ ExecStartPre={{ matrix_host_command_docker }} run --rm --name matrix-mautrix-fac --log-driver=none \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ --cap-drop=ALL \ + --network={{ matrix_docker_network }} \ -v {{ matrix_mautrix_facebook_config_path }}:/config:z \ -v {{ matrix_mautrix_facebook_data_path }}:/data:z \ {{ matrix_mautrix_facebook_docker_image }} \ From a3406a182bf9590d7ee018867ee4fbd02c6473b5 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 12:04:47 +0200 Subject: [PATCH 136/217] Move some things around --- .../defaults/main.yml | 2 +- .../tasks/setup_install.yml | 17 ++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml index 95adf354..d0b39682 100644 --- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml @@ -33,7 +33,6 @@ matrix_mautrix_facebook_appservice_token: '' matrix_mautrix_facebook_homeserver_token: '' - # Database-related configuration fields. # # To use SQLite, stick to these defaults. @@ -61,6 +60,7 @@ matrix_mautrix_facebook_appservice_database: "{{ }[matrix_mautrix_facebook_database_engine] }}" + # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). matrix_mautrix_facebook_login_shared_secret: '' diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml index ef814108..c3c8d61f 100644 --- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml @@ -1,5 +1,13 @@ --- +# If the matrix-synapse role is not used, `matrix_synapse_role_executed` won't exist. +# We don't want to fail in such cases. +- name: Fail if matrix-synapse role already executed + fail: + msg: >- + The matrix-bridge-mautrix-facebook role needs to execute before the matrix-synapse role. + when: "matrix_synapse_role_executed|default(False)" + - block: - name: Check if an SQLite database already exists stat: @@ -20,15 +28,6 @@ when: "matrix_mautrix_facebook_sqlite_database_path_local_stat_result.stat.exists" when: "matrix_mautrix_facebook_database_engine == 'postgres'" - -# If the matrix-synapse role is not used, `matrix_synapse_role_executed` won't exist. -# We don't want to fail in such cases. -- name: Fail if matrix-synapse role already executed - fail: - msg: >- - The matrix-bridge-mautrix-facebook role needs to execute before the matrix-synapse role. - when: "matrix_synapse_role_executed|default(False)" - - name: Ensure Mautrix Facebook image is pulled docker_image: name: "{{ matrix_mautrix_facebook_docker_image }}" From 9bf8ce878e9451f2f3aa24d200d9d10dbf08e8b3 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 12:06:23 +0200 Subject: [PATCH 137/217] Add (Postgres + SQLite) support to matrix-mautrix-telegram bridge I don't use this bridge, so this is completely untested. --- .../defaults/main.yml | 29 +++++++++++++++++++ .../tasks/setup_install.yml | 20 +++++++++++++ .../templates/config.yaml.j2 | 2 +- 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml index b6f4ef5b..d0df6a7d 100644 --- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml @@ -43,6 +43,35 @@ matrix_mautrix_telegram_systemd_wanted_services_list: [] matrix_mautrix_telegram_appservice_token: '' matrix_mautrix_telegram_homeserver_token: '' + +# Database-related configuration fields. +# +# To use SQLite, stick to these defaults. +# +# To use Postgres: +# - change the engine (`matrix_mautrix_telegram_database_engine: 'postgres'`) +# - adjust your database credentials via the `matrix_mautrix_telegram_postgres_*` variables +matrix_mautrix_telegram_database_engine: 'sqlite' + +matrix_mautrix_telegram_sqlite_database_path_local: "{{ matrix_mautrix_telegram_data_path }}/mautrix-telegram.db" +matrix_mautrix_telegram_sqlite_database_path_in_container: "/data/mautrix-telegram.db" + +matrix_mautrix_telegram_postgres_username: 'matrix_mautrix_telegram' +matrix_mautrix_telegram_postgres_password: 'some-password' +matrix_mautrix_telegram_postgres_hostname: 'matrix-postgres' +matrix_mautrix_telegram_postgres_port: 5432 +matrix_mautrix_telegram_postgres_db_name: 'matrix_mautrix_telegram' + +matrix_mautrix_telegram_postgres_connection_string: 'postgresql://{{ matrix_mautrix_telegram_postgres_username }}:{{ matrix_mautrix_telegram_postgres_password }}@{{ matrix_mautrix_telegram_postgres_hostname }}:{{ matrix_mautrix_telegram_postgres_port }}/{{ matrix_mautrix_telegram_postgres_db_name }}' + +matrix_mautrix_telegram_appservice_database: "{{ + { + 'sqlite': matrix_mautrix_telegram_sqlite_database_path_in_container, + 'postgres': matrix_mautrix_telegram_postgres_connection_string, + }[matrix_mautrix_telegram_database_engine] +}}" + + # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). matrix_mautrix_telegram_login_shared_secret: '' diff --git a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml index 9dc38ec2..db1a8f83 100644 --- a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml @@ -8,6 +8,26 @@ The matrix-bridge-mautrix-telegram role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mautrix_telegram_sqlite_database_path_local }}" + register: matrix_mautrix_telegram_sqlite_database_path_local_stat_result + + - name: Fail if an SQLite database already exists when using Postgres + fail: + msg: >- + matrix_mautrix_telegram_database_engine has been set to `postgres` (which is our new default now). + However, we've discovered an existing SQLite database in {{ matrix_mautrix_telegram_sqlite_database_path_local }}. + It appears that you've been using this bridge with the SQLite engine until now. + To continue using SQLite, opt into it explicitly: add `matrix_mautrix_telegram_database_engine: sqlite` to your vars.yml file and re-run this same command. + Alternatively, to migrate your existing SQLite database to Postgres: + 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) + 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_telegram_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_telegram_postgres_connection_string'`) + 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) + when: "matrix_mautrix_telegram_sqlite_database_path_local_stat_result.stat.exists" + when: "matrix_mautrix_telegram_database_engine == 'postgres'" + - name: Ensure Mautrix Telegram image is pulled docker_image: name: "{{ matrix_mautrix_telegram_docker_image }}" diff --git a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 index 65f17d33..52efba02 100644 --- a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 @@ -27,7 +27,7 @@ appservice: # Format examples: # SQLite: sqlite:///filename.db # Postgres: postgres://username:password@hostname/dbname - database: postgres://mautrix_bridge_telegram:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_bridge_telegram') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/mautrix_bridge_telegram + database: {{ matrix_mautrix_telegram_appservice_database|to_json }} # Public part of web server for out-of-Matrix interaction with the bridge. # Used for things like login if the user wants to make sure the 2FA password isn't stored in From 05dd091133ad7833ca544bd4ccda55311c0450c5 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 12:11:41 +0200 Subject: [PATCH 138/217] Rename `_database_hostname` to `_postgres_hostname` Should have been done during 2a99e84b5bfd9f39, but I've missed it. Because of this, 9bf8ce878e945 was also incomplete and would have caused an error. --- group_vars/matrix_servers | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 5e3695ca..0bc0c0c5 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -958,12 +958,12 @@ matrix_postgres_additional_databases: | 'name': matrix_appservice_slack_postgres_db_name, 'username': matrix_appservice_slack_postgres_username, 'password': matrix_appservice_slack_postgres_password, - }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres') else []) + }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_postgres_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_appservice_irc_postgres_db_name, 'username': matrix_appservice_irc_postgres_username, 'password': matrix_appservice_irc_postgres_password, - }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres') else []) + }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_postgres_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_mautrix_facebook_postgres_db_name, 'username': matrix_mautrix_facebook_postgres_username, @@ -973,17 +973,17 @@ matrix_postgres_additional_databases: | 'name': matrix_mautrix_hangouts_postgres_db_name, 'username': matrix_mautrix_hangouts_postgres_username, 'password': matrix_mautrix_hangouts_postgres_password, - }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres') else []) + }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_postgres_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_mautrix_telegram_postgres_db_name, 'username': matrix_mautrix_telegram_postgres_username, 'password': matrix_mautrix_telegram_postgres_password, - }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_telegram_facebook_database_hostname == 'matrix-postgres') else []) + }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_telegram_facebook_postgres_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_mautrix_whatsapp_postgres_db_name, 'username': matrix_mautrix_whatsapp_postgres_username, 'password': matrix_mautrix_whatsapp_postgres_password, - }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres') else []) + }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_postgres_hostname == 'matrix-postgres') else []) + ([{ 'name': 'matrix_bridge_sms', 'username': 'matrix_bridge_sms', @@ -993,45 +993,45 @@ matrix_postgres_additional_databases: | 'name': matrix_mx_puppet_skype_postgres_db_name, 'username': matrix_mx_puppet_skype_postgres_username, 'password': matrix_mx_puppet_skype_postgres_password, - }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres') else []) + }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_postgres_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_mx_puppet_slack_postgres_db_name, 'username': matrix_mx_puppet_slack_postgres_username, 'password': matrix_mx_puppet_slack_postgres_password, - }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres') else []) + }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_postgres_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_mx_puppet_twitter_postgres_db_name, 'username': matrix_mx_puppet_twitter_postgres_username, 'password': matrix_mx_puppet_twitter_postgres_password, - }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres') else []) + }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_postgres_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_mx_puppet_instagram_postgres_db_name, 'username': matrix_mx_puppet_instagram_postgres_username, 'password': matrix_mx_puppet_instagram_postgres_password, }] if (matrix_mx_puppet_instagram_enabled and matrix_mx_puppet_instagram_database_engine == 'postgres' - and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres') else []) + and matrix_mx_puppet_instagram_postgres_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_mx_puppet_discord_postgres_db_name, 'username': matrix_mx_puppet_discord_postgres_username, 'password': matrix_mx_puppet_discord_postgres_password, }] if (matrix_mx_puppet_discord_enabled and matrix_mx_puppet_discord_database_engine == 'postgres' - and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres') else []) + and matrix_mx_puppet_discord_postgres_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_mx_puppet_steam_postgres_db_name, 'username': matrix_mx_puppet_steam_postgres_username, 'password': matrix_mx_puppet_steam_postgres_password, }] if (matrix_mx_puppet_steam_enabled and matrix_mx_puppet_steam_database_engine == 'postgres' - and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else []) + and matrix_mx_puppet_steam_postgres_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_dimension_postgres_db_name, 'username': matrix_dimension_postgres_username, 'password': matrix_dimension_postgres_password, }] if (matrix_dimension_enabled and matrix_dimension_database_engine == 'postgres' - and matrix_dimension_database_hostname == 'matrix-postgres') else []) + and matrix_dimension_postgres_hostname == 'matrix-postgres') else []) }} ###################################################################### From 6a2dd8088cea10713ffd760d32cc789821ae5ae9 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 12:13:10 +0200 Subject: [PATCH 139/217] Fix typo (matrix_telegram_facebook_postgres_hostname -> matrix_mautrix_telegram_postgres_hostname) Related to 9bf8ce878e945. --- group_vars/matrix_servers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 0bc0c0c5..626498c0 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -978,7 +978,7 @@ matrix_postgres_additional_databases: | 'name': matrix_mautrix_telegram_postgres_db_name, 'username': matrix_mautrix_telegram_postgres_username, 'password': matrix_mautrix_telegram_postgres_password, - }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_telegram_facebook_postgres_hostname == 'matrix-postgres') else []) + }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_mautrix_telegram_postgres_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_mautrix_whatsapp_postgres_db_name, 'username': matrix_mautrix_whatsapp_postgres_username, From 2848322461e451d2fb1985ecf797d87e5765ea03 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 12:19:35 +0200 Subject: [PATCH 140/217] Remove some remaining `_database_password` usage in favor of `_postgres_pasword` The only one that remains is `matrix_synapse_database_password`, but that's something old and should be dealt with separately in the future (unless it remains as it is). --- group_vars/matrix_servers | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 626498c0..e3f16d3f 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -142,7 +142,7 @@ matrix_appservice_slack_systemd_required_services_list: | # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_appservice_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_appservice_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.slack.db') | to_uuid }}" +matrix_appservice_slack_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.slack.db') | to_uuid }}" ###################################################################### # @@ -181,7 +181,7 @@ matrix_appservice_irc_homeserver_token: "{{ matrix_synapse_macaroon_secret_key | # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_appservice_irc_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_appservice_irc_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.irc.db') | to_uuid }}" +matrix_appservice_irc_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.irc.db') | to_uuid }}" ###################################################################### @@ -219,7 +219,7 @@ matrix_mautrix_facebook_bridge_presence: "{{ matrix_synapse_use_presence if matr # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mautrix_facebook_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mautrix_facebook_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_facebook.db') | to_uuid }}" +matrix_mautrix_facebook_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_facebook.db') | to_uuid }}" ###################################################################### # @@ -256,7 +256,7 @@ matrix_mautrix_hangouts_login_shared_secret: "{{ matrix_synapse_ext_password_pro # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mautrix_hangouts_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mautrix_hangouts_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_hangouts.db') | to_uuid }}" +matrix_mautrix_hangouts_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_hangouts.db') | to_uuid }}" ###################################################################### # @@ -296,7 +296,7 @@ matrix_mautrix_telegram_login_shared_secret: "{{ matrix_synapse_ext_password_pro # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mautrix_telegram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mautrix_telegram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_telegram.db') | to_uuid }}" +matrix_mautrix_telegram_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_telegram.db') | to_uuid }}" ###################################################################### # @@ -328,7 +328,7 @@ matrix_mautrix_whatsapp_login_shared_secret: "{{ matrix_synapse_ext_password_pro # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mautrix_whatsapp_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mautrix_whatsapp_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_whatsapp.db') | to_uuid }}" +matrix_mautrix_whatsapp_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_whatsapp.db') | to_uuid }}" ###################################################################### # @@ -388,7 +388,7 @@ matrix_mx_puppet_skype_login_shared_secret: "{{ matrix_synapse_ext_password_prov # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_skype_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_skype_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_skype.db') | to_uuid }}" +matrix_mx_puppet_skype_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_skype.db') | to_uuid }}" ###################################################################### # @@ -423,7 +423,7 @@ matrix_mx_puppet_slack_login_shared_secret: "{{ matrix_synapse_ext_password_prov # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_slack.db') | to_uuid }}" +matrix_mx_puppet_slack_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_slack.db') | to_uuid }}" ###################################################################### # @@ -459,7 +459,7 @@ matrix_mx_puppet_twitter_container_http_host_bind_port: "{{ '' if matrix_nginx_p # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_twitter_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_twitter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_twitter.db') | to_uuid }}" +matrix_mx_puppet_twitter_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_twitter.db') | to_uuid }}" ###################################################################### # @@ -494,7 +494,7 @@ matrix_mx_puppet_instagram_login_shared_secret: "{{ matrix_synapse_ext_password_ # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_instagram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_instagram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_instagram.db') | to_uuid }}" +matrix_mx_puppet_instagram_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_instagram.db') | to_uuid }}" ###################################################################### # @@ -528,7 +528,7 @@ matrix_mx_puppet_discord_login_shared_secret: "{{ matrix_synapse_ext_password_pr # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_discord_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_discord.db') | to_uuid }}" +matrix_mx_puppet_discord_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_discord.db') | to_uuid }}" ###################################################################### # @@ -562,7 +562,7 @@ matrix_mx_puppet_steam_login_shared_secret: "{{ matrix_synapse_ext_password_prov # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_stream_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_steam.db') | to_uuid }}" +matrix_mx_puppet_steam_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_steam.db') | to_uuid }}" ###################################################################### # @@ -676,7 +676,7 @@ matrix_integration_manager_ui_url: "{{ matrix_dimension_integrations_ui_url if m # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_dimension_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_dimension_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.dimension.db') | to_uuid }}" +matrix_dimension_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.dimension.db') | to_uuid }}" ###################################################################### # From 6c77eae96935f6a78c6ad170134d0797b18520af Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 12:24:37 +0200 Subject: [PATCH 141/217] Add (Postgres + SQLite) support to matrix-mautrix-whatsapp bridge I don't use this bridge, so this is completely untested. --- .../defaults/main.yml | 36 +++++++++++++++++++ .../tasks/setup_install.yml | 26 ++++++++++++-- .../templates/config.yaml.j2 | 4 +-- 3 files changed, 61 insertions(+), 5 deletions(-) diff --git a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml index e9929f96..6cabea3e 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml @@ -27,6 +27,42 @@ matrix_mautrix_whatsapp_systemd_wanted_services_list: [] matrix_mautrix_whatsapp_appservice_token: '' matrix_mautrix_whatsapp_homeserver_token: '' + +# Database-related configuration fields. +# +# To use SQLite, stick to these defaults. +# +# To use Postgres: +# - change the engine (`matrix_mautrix_whatsapp_database_engine: 'postgres'`) +# - adjust your database credentials via the `matrix_mautrix_whatsapp_postgres_*` variables +matrix_mautrix_whatsapp_database_engine: 'sqlite' + +matrix_mautrix_whatsapp_sqlite_database_path_local: "{{ matrix_mautrix_whatsapp_data_path }}/mautrix-whatsapp.db" +matrix_mautrix_whatsapp_sqlite_database_path_in_container: "/data/mautrix-whatsapp.db" + +matrix_mautrix_whatsapp_postgres_username: 'matrix_mautrix_whatsapp' +matrix_mautrix_whatsapp_postgres_password: 'some-password' +matrix_mautrix_whatsapp_postgres_hostname: 'matrix-postgres' +matrix_mautrix_whatsapp_postgres_port: 5432 +matrix_mautrix_whatsapp_postgres_db_name: 'matrix_mautrix_whatsapp' + +matrix_mautrix_whatsapp_postgres_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_postgres_username }}:{{ matrix_mautrix_whatsapp_postgres_password }}@{{ matrix_mautrix_whatsapp_postgres_hostname }}:{{ matrix_mautrix_whatsapp_postgres_port }}/{{ matrix_mautrix_whatsapp_postgres_db_name }}' + +matrix_mautrix_whatsapp_appservice_database_type: "{{ + { + 'sqlite': 'sqlite3', + 'postgres':'postgres', + }[matrix_mautrix_whatsapp_database_engine] +}}" + +matrix_mautrix_whatsapp_appservice_database_uri: "{{ + { + 'sqlite': matrix_mautrix_whatsapp_sqlite_database_path_in_container, + 'postgres': matrix_mautrix_whatsapp_postgres_connection_string, + }[matrix_mautrix_whatsapp_database_engine] +}}" + + # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). matrix_mautrix_whatsapp_login_shared_secret: '' diff --git a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml index 6bd87bbe..15453e8c 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml @@ -8,6 +8,26 @@ The matrix-bridge-mautrix-whatsapp role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mautrix_whatsapp_sqlite_database_path_local }}" + register: matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result + + - name: Fail if an SQLite database already exists when using Postgres + fail: + msg: >- + matrix_mautrix_whatsapp_database_engine has been set to `postgres` (which is our new default now). + However, we've discovered an existing SQLite database in {{ matrix_mautrix_whatsapp_sqlite_database_path_local }}. + It appears that you've been using this bridge with the SQLite engine until now. + To continue using SQLite, opt into it explicitly: add `matrix_mautrix_whatsapp_database_engine: sqlite` to your vars.yml file and re-run this same command. + Alternatively, to migrate your existing SQLite database to Postgres: + 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) + 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_whatsapp_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_whatsapp_postgres_connection_string'`) + 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) + when: "matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result.stat.exists" + when: "matrix_mautrix_whatsapp_database_engine == 'postgres'" + - name: Ensure Mautrix Whatsapp image is pulled docker_image: name: "{{ matrix_mautrix_whatsapp_docker_image }}" @@ -26,12 +46,12 @@ - "{{ matrix_mautrix_whatsapp_base_path }}" - "{{ matrix_mautrix_whatsapp_config_path }}" - "{{ matrix_mautrix_whatsapp_data_path }}" - + - name: Check if an old database file exists stat: path: "{{ matrix_mautrix_whatsapp_base_path }}/mautrix-whatsapp.db" register: matrix_mautrix_whatsapp_stat_database - + - name: Check if an old matrix state file exists stat: path: "{{ matrix_mautrix_whatsapp_base_path }}/mx-state.json" @@ -48,7 +68,7 @@ - name: (Data relocation) Move mautrix-whatsapp database file to ./data directory command: "mv {{ matrix_mautrix_whatsapp_base_path }}/mautrix-whatsapp.db {{ matrix_mautrix_whatsapp_data_path }}/mautrix-whatsapp.db" when: "matrix_mautrix_whatsapp_stat_database.stat.exists" - + - name: (Data relocation) Move mautrix-whatsapp mx-state file to ./data directory command: "mv {{ matrix_mautrix_whatsapp_base_path }}/mx-state.json {{ matrix_mautrix_whatsapp_data_path }}/mx-state.json" when: "matrix_mautrix_whatsapp_stat_mx_state.stat.exists" diff --git a/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 index 93956049..89216695 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 @@ -19,11 +19,11 @@ appservice: # Database config. database: # The database type. "sqlite3" and "postgres" are supported. - type: sqlite3 + type: {{ matrix_mautrix_whatsapp_appservice_database_type|to_json }} # The database URI. # SQLite: File name is enough. https://github.com/mattn/go-sqlite3#connection-string # Postgres: Connection string. For example, postgres://user:password@host/database - uri: postgres://matrix_bridge_whatsapp@{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_bridge_whatsapp') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_bridge_whatsapp + uri: {{ matrix_mautrix_whatsapp_appservice_database_uri|to_json }} # Maximum number of connections. Mostly relevant for Postgres. max_open_conns: 20 max_idle_conns: 2 From 43d6ff2af83abf0610c73b507eb4d7acae045052 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 12:30:10 +0200 Subject: [PATCH 142/217] Fix sqlite usage for mautrix-facebook/mautrix-telegram Regression since 2d99ade72f and 9bf8ce878e94, respectively. When SQLite is to be used, these bridges expect an `sqlite://` connection string, and not a plain file name (path), like Appservice Discord and mautrix-whatsapp do. --- roles/matrix-bridge-mautrix-facebook/defaults/main.yml | 2 +- roles/matrix-bridge-mautrix-telegram/defaults/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml index d0b39682..97584997 100644 --- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml @@ -55,7 +55,7 @@ matrix_mautrix_facebook_postgres_connection_string: 'postgresql://{{ matrix_maut matrix_mautrix_facebook_appservice_database: "{{ { - 'sqlite': matrix_mautrix_facebook_sqlite_database_path_in_container, + 'sqlite': ('sqlite://' + matrix_mautrix_facebook_sqlite_database_path_in_container), 'postgres': matrix_mautrix_facebook_postgres_connection_string, }[matrix_mautrix_facebook_database_engine] }}" diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml index d0df6a7d..9ba9be09 100644 --- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml @@ -66,7 +66,7 @@ matrix_mautrix_telegram_postgres_connection_string: 'postgresql://{{ matrix_maut matrix_mautrix_telegram_appservice_database: "{{ { - 'sqlite': matrix_mautrix_telegram_sqlite_database_path_in_container, + 'sqlite': ('sqlite://' + matrix_mautrix_telegram_sqlite_database_path_in_container), 'postgres': matrix_mautrix_telegram_postgres_connection_string, }[matrix_mautrix_telegram_database_engine] }}" From ce21ea3640283889be6ed7c526116ca6c47ca921 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 12:34:59 +0200 Subject: [PATCH 143/217] Add (Postgres + SQLite) support to matrix-mautrix-hangouts bridge I don't use this bridge, so this is completely untested. --- .../defaults/main.yml | 29 +++++++++++++++++++ .../tasks/setup_install.yml | 20 +++++++++++++ .../templates/config.yaml.j2 | 2 +- .../matrix-mautrix-hangouts.service.j2 | 1 + 4 files changed, 51 insertions(+), 1 deletion(-) diff --git a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml index ab670052..af0fe061 100644 --- a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml @@ -39,6 +39,35 @@ matrix_mautrix_hangouts_systemd_wanted_services_list: [] matrix_mautrix_hangouts_appservice_token: '' matrix_mautrix_hangouts_homeserver_token: '' + +# Database-related configuration fields. +# +# To use SQLite, stick to these defaults. +# +# To use Postgres: +# - change the engine (`matrix_mautrix_hangouts_database_engine: 'postgres'`) +# - adjust your database credentials via the `matrix_mautrix_hangouts_postgres_*` variables +matrix_mautrix_hangouts_database_engine: 'sqlite' + +matrix_mautrix_hangouts_sqlite_database_path_local: "{{ matrix_mautrix_hangouts_data_path }}/mautrix-hangouts.db" +matrix_mautrix_hangouts_sqlite_database_path_in_container: "/data/mautrix-hangouts.db" + +matrix_mautrix_hangouts_postgres_username: 'matrix_mautrix_hangouts' +matrix_mautrix_hangouts_postgres_password: 'some-password' +matrix_mautrix_hangouts_postgres_hostname: 'matrix-postgres' +matrix_mautrix_hangouts_postgres_port: 5432 +matrix_mautrix_hangouts_postgres_db_name: 'matrix_mautrix_hangouts' + +matrix_mautrix_hangouts_postgres_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_postgres_username }}:{{ matrix_mautrix_hangouts_postgres_password }}@{{ matrix_mautrix_hangouts_postgres_hostname }}:{{ matrix_mautrix_hangouts_postgres_port }}/{{ matrix_mautrix_hangouts_postgres_db_name }}' + +matrix_mautrix_hangouts_appservice_database: "{{ + { + 'sqlite': ('sqlite://' + matrix_mautrix_hangouts_sqlite_database_path_in_container), + 'postgres': matrix_mautrix_hangouts_postgres_connection_string, + }[matrix_mautrix_hangouts_database_engine] +}}" + + # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). matrix_mautrix_hangouts_login_shared_secret: '' diff --git a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml index eec5e006..9e7bfbba 100644 --- a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml @@ -8,6 +8,26 @@ The matrix-bridge-mautrix-hangouts role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mautrix_hangouts_sqlite_database_path_local }}" + register: matrix_mautrix_hangouts_sqlite_database_path_local_stat_result + + - name: Fail if an SQLite database already exists when using Postgres + fail: + msg: >- + matrix_mautrix_hangouts_database_engine has been set to `postgres` (which is our new default now). + However, we've discovered an existing SQLite database in {{ matrix_mautrix_hangouts_sqlite_database_path_local }}. + It appears that you've been using this bridge with the SQLite engine until now. + To continue using SQLite, opt into it explicitly: add `matrix_mautrix_hangouts_database_engine: sqlite` to your vars.yml file and re-run this same command. + Alternatively, to migrate your existing SQLite database to Postgres: + 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) + 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_hangouts_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_hangouts_postgres_connection_string'`) + 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) + when: "matrix_mautrix_hangouts_sqlite_database_path_local_stat_result.stat.exists" + when: "matrix_mautrix_hangouts_database_engine == 'postgres'" + - name: Ensure Mautrix Hangouts image is pulled docker_image: name: "{{ matrix_mautrix_hangouts_docker_image }}" diff --git a/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2 index e4e59ad2..cc2ca90b 100644 --- a/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2 @@ -27,7 +27,7 @@ appservice: # Format examples: # SQLite: sqlite:///filename.db # Postgres: postgres://username:password@hostname/dbname - database: postgres://mautrix_bridge_hangouts:{{ matrix_additional_databases | selectattr('name', 'equalto', 'mautrix_bridge_hangouts') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/mautrix_bridge_hangouts + database: {{ matrix_mautrix_hangouts_appservice_database|to_json }} # The unique ID of this appservice. id: hangouts diff --git a/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2 b/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2 index 26280da3..39559190 100644 --- a/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2 +++ b/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2 @@ -18,6 +18,7 @@ ExecStartPre={{ matrix_host_command_docker }} run --rm --name matrix-mautrix-han --log-driver=none \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ --cap-drop=ALL \ + --network={{ matrix_docker_network }} \ -v {{ matrix_mautrix_hangouts_config_path }}:/config:z \ -v {{ matrix_mautrix_hangouts_data_path }}:/data:z \ {{ matrix_mautrix_hangouts_docker_image }} \ From 087dbe4ddc80ba6308e7ee98391ea475354e8860 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 13:02:47 +0200 Subject: [PATCH 144/217] Rename `matrix_*_postgres_*` back to `matrix_*_database_*` I was thinking that it makes sense to be more specific, and using `_postgres_` also separated these variables from the `_database_` variables that ended up in bridge configuration. However, @jdreichmann makes a good point (https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/740#discussion_r542281102) that we don't need to be so specific and can allow for other engines (like MySQL) to use these variables. --- group_vars/matrix_servers | 112 +++++++++--------- .../defaults/main.yml | 10 +- .../defaults/main.yml | 10 +- .../defaults/main.yml | 10 +- .../defaults/main.yml | 10 +- .../defaults/main.yml | 10 +- 6 files changed, 81 insertions(+), 81 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index e3f16d3f..acf133e1 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -70,7 +70,7 @@ matrix_appservice_discord_homeserver_token: "{{ matrix_synapse_macaroon_secret_k # We only make this use Postgres if our own Postgres server is enabled. # It's only then (for now) that we can automatically create the necessary database and user for this service. matrix_appservice_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_appservice_discord_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.discord.db') | to_uuid }}" +matrix_appservice_discord_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.discord.db') | to_uuid }}" ###################################################################### # @@ -142,7 +142,7 @@ matrix_appservice_slack_systemd_required_services_list: | # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_appservice_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_appservice_slack_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.slack.db') | to_uuid }}" +matrix_appservice_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.slack.db') | to_uuid }}" ###################################################################### # @@ -181,7 +181,7 @@ matrix_appservice_irc_homeserver_token: "{{ matrix_synapse_macaroon_secret_key | # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_appservice_irc_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_appservice_irc_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.irc.db') | to_uuid }}" +matrix_appservice_irc_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.irc.db') | to_uuid }}" ###################################################################### @@ -219,7 +219,7 @@ matrix_mautrix_facebook_bridge_presence: "{{ matrix_synapse_use_presence if matr # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mautrix_facebook_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mautrix_facebook_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_facebook.db') | to_uuid }}" +matrix_mautrix_facebook_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_facebook.db') | to_uuid }}" ###################################################################### # @@ -256,7 +256,7 @@ matrix_mautrix_hangouts_login_shared_secret: "{{ matrix_synapse_ext_password_pro # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mautrix_hangouts_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mautrix_hangouts_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_hangouts.db') | to_uuid }}" +matrix_mautrix_hangouts_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_hangouts.db') | to_uuid }}" ###################################################################### # @@ -296,7 +296,7 @@ matrix_mautrix_telegram_login_shared_secret: "{{ matrix_synapse_ext_password_pro # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mautrix_telegram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mautrix_telegram_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_telegram.db') | to_uuid }}" +matrix_mautrix_telegram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_telegram.db') | to_uuid }}" ###################################################################### # @@ -328,7 +328,7 @@ matrix_mautrix_whatsapp_login_shared_secret: "{{ matrix_synapse_ext_password_pro # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mautrix_whatsapp_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mautrix_whatsapp_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_whatsapp.db') | to_uuid }}" +matrix_mautrix_whatsapp_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_whatsapp.db') | to_uuid }}" ###################################################################### # @@ -388,7 +388,7 @@ matrix_mx_puppet_skype_login_shared_secret: "{{ matrix_synapse_ext_password_prov # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_skype_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_skype_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_skype.db') | to_uuid }}" +matrix_mx_puppet_skype_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_skype.db') | to_uuid }}" ###################################################################### # @@ -423,7 +423,7 @@ matrix_mx_puppet_slack_login_shared_secret: "{{ matrix_synapse_ext_password_prov # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_slack_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_slack.db') | to_uuid }}" +matrix_mx_puppet_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_slack.db') | to_uuid }}" ###################################################################### # @@ -459,7 +459,7 @@ matrix_mx_puppet_twitter_container_http_host_bind_port: "{{ '' if matrix_nginx_p # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_twitter_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_twitter_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_twitter.db') | to_uuid }}" +matrix_mx_puppet_twitter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_twitter.db') | to_uuid }}" ###################################################################### # @@ -494,7 +494,7 @@ matrix_mx_puppet_instagram_login_shared_secret: "{{ matrix_synapse_ext_password_ # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_instagram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_instagram_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_instagram.db') | to_uuid }}" +matrix_mx_puppet_instagram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_instagram.db') | to_uuid }}" ###################################################################### # @@ -528,7 +528,7 @@ matrix_mx_puppet_discord_login_shared_secret: "{{ matrix_synapse_ext_password_pr # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_discord_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_discord.db') | to_uuid }}" +matrix_mx_puppet_discord_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_discord.db') | to_uuid }}" ###################################################################### # @@ -562,7 +562,7 @@ matrix_mx_puppet_steam_login_shared_secret: "{{ matrix_synapse_ext_password_prov # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_stream_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_steam_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_steam.db') | to_uuid }}" +matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_steam.db') | to_uuid }}" ###################################################################### # @@ -676,7 +676,7 @@ matrix_integration_manager_ui_url: "{{ matrix_dimension_integrations_ui_url if m # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_dimension_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_dimension_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.dimension.db') | to_uuid }}" +matrix_dimension_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.dimension.db') | to_uuid }}" ###################################################################### # @@ -950,39 +950,39 @@ matrix_postgres_db_name: "homeserver" matrix_postgres_additional_databases: | {{ ([{ - 'name': matrix_appservice_discord_postgres_db_name, - 'username': matrix_appservice_discord_postgres_username, - 'password': matrix_appservice_discord_postgres_password, + 'name': matrix_appservice_discord_database_db_name, + 'username': matrix_appservice_discord_database_username, + 'password': matrix_appservice_discord_database_password, }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_postgres_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_appservice_slack_postgres_db_name, - 'username': matrix_appservice_slack_postgres_username, - 'password': matrix_appservice_slack_postgres_password, + 'name': matrix_appservice_slack_database_db_name, + 'username': matrix_appservice_slack_database_username, + 'password': matrix_appservice_slack_database_password, }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_postgres_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_appservice_irc_postgres_db_name, - 'username': matrix_appservice_irc_postgres_username, - 'password': matrix_appservice_irc_postgres_password, + 'name': matrix_appservice_irc_database_db_name, + 'username': matrix_appservice_irc_database_username, + 'password': matrix_appservice_irc_database_password, }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_postgres_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mautrix_facebook_postgres_db_name, - 'username': matrix_mautrix_facebook_postgres_username, - 'password': matrix_mautrix_facebook_postgres_password, + 'name': matrix_mautrix_facebook_database_db_name, + 'username': matrix_mautrix_facebook_database_username, + 'password': matrix_mautrix_facebook_database_password, }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_postgres_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mautrix_hangouts_postgres_db_name, - 'username': matrix_mautrix_hangouts_postgres_username, - 'password': matrix_mautrix_hangouts_postgres_password, + 'name': matrix_mautrix_hangouts_database_db_name, + 'username': matrix_mautrix_hangouts_database_username, + 'password': matrix_mautrix_hangouts_database_password, }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_postgres_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mautrix_telegram_postgres_db_name, - 'username': matrix_mautrix_telegram_postgres_username, - 'password': matrix_mautrix_telegram_postgres_password, + 'name': matrix_mautrix_telegram_database_db_name, + 'username': matrix_mautrix_telegram_database_username, + 'password': matrix_mautrix_telegram_database_password, }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_mautrix_telegram_postgres_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mautrix_whatsapp_postgres_db_name, - 'username': matrix_mautrix_whatsapp_postgres_username, - 'password': matrix_mautrix_whatsapp_postgres_password, + 'name': matrix_mautrix_whatsapp_database_db_name, + 'username': matrix_mautrix_whatsapp_database_username, + 'password': matrix_mautrix_whatsapp_database_password, }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_postgres_hostname == 'matrix-postgres') else []) + ([{ 'name': 'matrix_bridge_sms', @@ -990,45 +990,45 @@ matrix_postgres_additional_databases: | 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid, }] if matrix_sms_bridge_enabled else []) + ([{ - 'name': matrix_mx_puppet_skype_postgres_db_name, - 'username': matrix_mx_puppet_skype_postgres_username, - 'password': matrix_mx_puppet_skype_postgres_password, + 'name': matrix_mx_puppet_skype_database_db_name, + 'username': matrix_mx_puppet_skype_database_username, + 'password': matrix_mx_puppet_skype_database_password, }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_postgres_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mx_puppet_slack_postgres_db_name, - 'username': matrix_mx_puppet_slack_postgres_username, - 'password': matrix_mx_puppet_slack_postgres_password, + 'name': matrix_mx_puppet_slack_database_db_name, + 'username': matrix_mx_puppet_slack_database_username, + 'password': matrix_mx_puppet_slack_database_password, }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_postgres_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mx_puppet_twitter_postgres_db_name, - 'username': matrix_mx_puppet_twitter_postgres_username, - 'password': matrix_mx_puppet_twitter_postgres_password, + 'name': matrix_mx_puppet_twitter_database_db_name, + 'username': matrix_mx_puppet_twitter_database_username, + 'password': matrix_mx_puppet_twitter_database_password, }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_postgres_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mx_puppet_instagram_postgres_db_name, - 'username': matrix_mx_puppet_instagram_postgres_username, - 'password': matrix_mx_puppet_instagram_postgres_password, + 'name': matrix_mx_puppet_instagram_database_db_name, + 'username': matrix_mx_puppet_instagram_database_username, + 'password': matrix_mx_puppet_instagram_database_password, }] if (matrix_mx_puppet_instagram_enabled and matrix_mx_puppet_instagram_database_engine == 'postgres' and matrix_mx_puppet_instagram_postgres_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mx_puppet_discord_postgres_db_name, - 'username': matrix_mx_puppet_discord_postgres_username, - 'password': matrix_mx_puppet_discord_postgres_password, + 'name': matrix_mx_puppet_discord_database_db_name, + 'username': matrix_mx_puppet_discord_database_username, + 'password': matrix_mx_puppet_discord_database_password, }] if (matrix_mx_puppet_discord_enabled and matrix_mx_puppet_discord_database_engine == 'postgres' and matrix_mx_puppet_discord_postgres_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mx_puppet_steam_postgres_db_name, - 'username': matrix_mx_puppet_steam_postgres_username, - 'password': matrix_mx_puppet_steam_postgres_password, + 'name': matrix_mx_puppet_steam_database_db_name, + 'username': matrix_mx_puppet_steam_database_username, + 'password': matrix_mx_puppet_steam_database_password, }] if (matrix_mx_puppet_steam_enabled and matrix_mx_puppet_steam_database_engine == 'postgres' and matrix_mx_puppet_steam_postgres_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_dimension_postgres_db_name, - 'username': matrix_dimension_postgres_username, - 'password': matrix_dimension_postgres_password, + 'name': matrix_dimension_database_db_name, + 'username': matrix_dimension_database_username, + 'password': matrix_dimension_database_password, }] if (matrix_dimension_enabled and matrix_dimension_database_engine == 'postgres' and matrix_dimension_postgres_hostname == 'matrix-postgres') else []) diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml index fb517cd5..57a88867 100644 --- a/roles/matrix-bridge-appservice-discord/defaults/main.yml +++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml @@ -53,16 +53,16 @@ matrix_appservice_discord_database_engine: 'sqlite' matrix_appservice_discord_sqlite_database_path_local: "{{ matrix_appservice_discord_data_path }}/discord.db" matrix_appservice_discord_sqlite_database_path_in_container: "/data/discord.db" -matrix_appservice_discord_postgres_username: 'matrix_appservice_discord' -matrix_appservice_discord_postgres_password: 'some-password' +matrix_appservice_discord_database_username: 'matrix_appservice_discord' +matrix_appservice_discord_database_password: 'some-password' matrix_appservice_discord_postgres_hostname: 'matrix-postgres' -matrix_appservice_discord_postgres_port: 5432 -matrix_appservice_discord_postgres_db_name: 'matrix_appservice_discord' +matrix_appservice_discord_database_port: 5432 +matrix_appservice_discord_database_db_name: 'matrix_appservice_discord' # These 2 variables are what actually ends up in the bridge configuration. # It's best if you don't change them directly, but rather redefine the sub-variables that constitute them. matrix_appservice_discord_database_filename: "{{ matrix_appservice_discord_sqlite_database_path_in_container }}" -matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_postgres_username }}:{{ matrix_appservice_discord_postgres_password }}@{{ matrix_appservice_discord_postgres_hostname }}:{{ matrix_appservice_discord_postgres_port }}/{{ matrix_appservice_discord_postgres_db_name }}' +matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_username }}:{{ matrix_appservice_discord_database_password }}@{{ matrix_appservice_discord_postgres_hostname }}:{{ matrix_appservice_discord_database_port }}/{{ matrix_appservice_discord_database_db_name }}' # Tells whether the bot should make use of "Privileged Gateway Intents". diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml index 97584997..246c32fe 100644 --- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml @@ -45,13 +45,13 @@ matrix_mautrix_facebook_database_engine: 'sqlite' matrix_mautrix_facebook_sqlite_database_path_local: "{{ matrix_mautrix_facebook_data_path }}/mautrix-facebook.db" matrix_mautrix_facebook_sqlite_database_path_in_container: "/data/mautrix-facebook.db" -matrix_mautrix_facebook_postgres_username: 'matrix_mautrix_facebook' -matrix_mautrix_facebook_postgres_password: 'some-password' +matrix_mautrix_facebook_database_username: 'matrix_mautrix_facebook' +matrix_mautrix_facebook_database_password: 'some-password' matrix_mautrix_facebook_postgres_hostname: 'matrix-postgres' -matrix_mautrix_facebook_postgres_port: 5432 -matrix_mautrix_facebook_postgres_db_name: 'matrix_mautrix_facebook' +matrix_mautrix_facebook_database_port: 5432 +matrix_mautrix_facebook_database_db_name: 'matrix_mautrix_facebook' -matrix_mautrix_facebook_postgres_connection_string: 'postgresql://{{ matrix_mautrix_facebook_postgres_username }}:{{ matrix_mautrix_facebook_postgres_password }}@{{ matrix_mautrix_facebook_postgres_hostname }}:{{ matrix_mautrix_facebook_postgres_port }}/{{ matrix_mautrix_facebook_postgres_db_name }}' +matrix_mautrix_facebook_postgres_connection_string: 'postgresql://{{ matrix_mautrix_facebook_database_username }}:{{ matrix_mautrix_facebook_database_password }}@{{ matrix_mautrix_facebook_postgres_hostname }}:{{ matrix_mautrix_facebook_database_port }}/{{ matrix_mautrix_facebook_database_db_name }}' matrix_mautrix_facebook_appservice_database: "{{ { diff --git a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml index af0fe061..04280ac4 100644 --- a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml @@ -52,13 +52,13 @@ matrix_mautrix_hangouts_database_engine: 'sqlite' matrix_mautrix_hangouts_sqlite_database_path_local: "{{ matrix_mautrix_hangouts_data_path }}/mautrix-hangouts.db" matrix_mautrix_hangouts_sqlite_database_path_in_container: "/data/mautrix-hangouts.db" -matrix_mautrix_hangouts_postgres_username: 'matrix_mautrix_hangouts' -matrix_mautrix_hangouts_postgres_password: 'some-password' +matrix_mautrix_hangouts_database_username: 'matrix_mautrix_hangouts' +matrix_mautrix_hangouts_database_password: 'some-password' matrix_mautrix_hangouts_postgres_hostname: 'matrix-postgres' -matrix_mautrix_hangouts_postgres_port: 5432 -matrix_mautrix_hangouts_postgres_db_name: 'matrix_mautrix_hangouts' +matrix_mautrix_hangouts_database_port: 5432 +matrix_mautrix_hangouts_database_db_name: 'matrix_mautrix_hangouts' -matrix_mautrix_hangouts_postgres_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_postgres_username }}:{{ matrix_mautrix_hangouts_postgres_password }}@{{ matrix_mautrix_hangouts_postgres_hostname }}:{{ matrix_mautrix_hangouts_postgres_port }}/{{ matrix_mautrix_hangouts_postgres_db_name }}' +matrix_mautrix_hangouts_postgres_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_database_username }}:{{ matrix_mautrix_hangouts_database_password }}@{{ matrix_mautrix_hangouts_postgres_hostname }}:{{ matrix_mautrix_hangouts_database_port }}/{{ matrix_mautrix_hangouts_database_db_name }}' matrix_mautrix_hangouts_appservice_database: "{{ { diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml index 9ba9be09..4f584128 100644 --- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml @@ -56,13 +56,13 @@ matrix_mautrix_telegram_database_engine: 'sqlite' matrix_mautrix_telegram_sqlite_database_path_local: "{{ matrix_mautrix_telegram_data_path }}/mautrix-telegram.db" matrix_mautrix_telegram_sqlite_database_path_in_container: "/data/mautrix-telegram.db" -matrix_mautrix_telegram_postgres_username: 'matrix_mautrix_telegram' -matrix_mautrix_telegram_postgres_password: 'some-password' +matrix_mautrix_telegram_database_username: 'matrix_mautrix_telegram' +matrix_mautrix_telegram_database_password: 'some-password' matrix_mautrix_telegram_postgres_hostname: 'matrix-postgres' -matrix_mautrix_telegram_postgres_port: 5432 -matrix_mautrix_telegram_postgres_db_name: 'matrix_mautrix_telegram' +matrix_mautrix_telegram_database_port: 5432 +matrix_mautrix_telegram_database_db_name: 'matrix_mautrix_telegram' -matrix_mautrix_telegram_postgres_connection_string: 'postgresql://{{ matrix_mautrix_telegram_postgres_username }}:{{ matrix_mautrix_telegram_postgres_password }}@{{ matrix_mautrix_telegram_postgres_hostname }}:{{ matrix_mautrix_telegram_postgres_port }}/{{ matrix_mautrix_telegram_postgres_db_name }}' +matrix_mautrix_telegram_postgres_connection_string: 'postgresql://{{ matrix_mautrix_telegram_database_username }}:{{ matrix_mautrix_telegram_database_password }}@{{ matrix_mautrix_telegram_postgres_hostname }}:{{ matrix_mautrix_telegram_database_port }}/{{ matrix_mautrix_telegram_database_db_name }}' matrix_mautrix_telegram_appservice_database: "{{ { diff --git a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml index 6cabea3e..357f5892 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml @@ -40,13 +40,13 @@ matrix_mautrix_whatsapp_database_engine: 'sqlite' matrix_mautrix_whatsapp_sqlite_database_path_local: "{{ matrix_mautrix_whatsapp_data_path }}/mautrix-whatsapp.db" matrix_mautrix_whatsapp_sqlite_database_path_in_container: "/data/mautrix-whatsapp.db" -matrix_mautrix_whatsapp_postgres_username: 'matrix_mautrix_whatsapp' -matrix_mautrix_whatsapp_postgres_password: 'some-password' +matrix_mautrix_whatsapp_database_username: 'matrix_mautrix_whatsapp' +matrix_mautrix_whatsapp_database_password: 'some-password' matrix_mautrix_whatsapp_postgres_hostname: 'matrix-postgres' -matrix_mautrix_whatsapp_postgres_port: 5432 -matrix_mautrix_whatsapp_postgres_db_name: 'matrix_mautrix_whatsapp' +matrix_mautrix_whatsapp_database_port: 5432 +matrix_mautrix_whatsapp_database_db_name: 'matrix_mautrix_whatsapp' -matrix_mautrix_whatsapp_postgres_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_postgres_username }}:{{ matrix_mautrix_whatsapp_postgres_password }}@{{ matrix_mautrix_whatsapp_postgres_hostname }}:{{ matrix_mautrix_whatsapp_postgres_port }}/{{ matrix_mautrix_whatsapp_postgres_db_name }}' +matrix_mautrix_whatsapp_postgres_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_database_username }}:{{ matrix_mautrix_whatsapp_database_password }}@{{ matrix_mautrix_whatsapp_postgres_hostname }}:{{ matrix_mautrix_whatsapp_database_port }}/{{ matrix_mautrix_whatsapp_database_db_name }}' matrix_mautrix_whatsapp_appservice_database_type: "{{ { From 5d70bc1376f30c4453beffe0c23ccb4c3004635c Mon Sep 17 00:00:00 2001 From: transcaffeine Date: Mon, 14 Dec 2020 12:38:51 +0100 Subject: [PATCH 145/217] add postgres support for mx-puppet-* with fallback to sqlite in role and migration notice --- .../defaults/main.yml | 15 +++++++++++++ .../tasks/validate_config.yml | 20 +++++++++++++++++ .../templates/config.yaml.j2 | 11 ++++++---- .../defaults/main.yml | 14 ++++++++++++ .../tasks/validate_config.yml | 21 ++++++++++++++++++ .../templates/config.yaml.j2 | 7 ++++-- .../defaults/main.yml | 15 +++++++++++++ .../tasks/validate_config.yml | 20 +++++++++++++++++ .../templates/config.yaml.j2 | 7 ++++-- .../defaults/main.yml | 15 +++++++++++++ .../tasks/validate_config.yml | 20 +++++++++++++++++ .../templates/config.yaml.j2 | 7 ++++-- .../defaults/main.yml | 14 ++++++++++++ .../tasks/validate_config.yml | 22 +++++++++++++++++++ .../templates/config.yaml.j2 | 7 ++++-- .../defaults/main.yml | 15 +++++++++++++ .../tasks/validate_config.yml | 22 +++++++++++++++++++ .../templates/config.yaml.j2 | 7 ++++-- 18 files changed, 245 insertions(+), 14 deletions(-) diff --git a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml index 268bbf8f..5a65f33e 100644 --- a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml @@ -55,6 +55,21 @@ matrix_mx_puppet_discord_homeserver_token: '' # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). matrix_mx_puppet_discord_login_shared_secret: '' +# Database configuration +matrix_mx_puppet_discord_database_engine: 'sqlite' +matrix_mx_puppet_discord_database_username: matrix_mx_puppet_discord +matrix_mx_puppet_discord_database_password: ~ +matrix_mx_puppet_discord_database_hostname: 'matrix-postgres' +matrix_mx_puppet_discord_database_port: 5432 +matrix_mx_puppet_discord_database_name: matrix_mx_puppet_discord +matrix_mx_puppet_discord_database_file: /data/database.db +matrix_mx_puppet_discord_database_connString: >-2 + {%- if matrix_mx_puppet_discord_database_engine == 'postgres' -%} + postgresql://{{ matrix_mx_puppet_discord_database_username }}:{{ matrix_mx_puppet_discord_database_password }}@{{ matrix_mx_puppet_discord_database_hostname }}:{{ matrix_mx_puppet_discord_database_port }}/{{ matrix_mx_puppet_discord_database_name }}?sslmode=disable + {%- else if matrix_mx_puppet_discord_database_engine == 'sqlite' -%} + {{ matrix_mx_puppet_discord_database_engine }}://{{ matrix_mx_puppet_discord_database_file }} + {%- endif -%} + # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. # diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml index c253eda2..8db47ed1 100644 --- a/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml +++ b/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml @@ -8,3 +8,23 @@ with_items: - "matrix_mx_puppet_discord_appservice_token" - "matrix_mx_puppet_discord_homeserver_token" + +- block: + - name: Check if a SQLite database already exists + stat: + path: "{{ matrix_mx_puppet_discord_data_path }}/database.db" + register: matrix_mx_puppet_discord_sqlite_database_path_local_stat_result + + - name: Fail if an SQLite database already exists when using Postgres + fail: + msg: >-2 + matrix_mx_puppet_discord_database_engine has been set to `postgres` (which is our new default now). + However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_discord_data_path }}/database.db. + It appears that you've been using this bridge with the SQLite engine until now. + To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_discord_database_engine: sqlite` to your vars.yml file and re-run this same command. + Alternatively, to migrate your existing SQLite database to Postgres: + 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) + 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_discord_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_discord_database_connString'`) + 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) + when: "matrix_mx_puppet_discord_sqlite_database_path_local_stat_result.stat.exists" + when: "matrix_mx_puppet_discord_database_engine == 'postgres'" diff --git a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 index 88c591e2..c24c7e54 100644 --- a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 @@ -100,15 +100,18 @@ namePatterns: group: :name database: +{% if matrix_mx_puppet_slack_discord_database_engine == 'sqlite' %} + # Use SQLite3 as a database backend + # The name of the database file + filename: /data/database.db +{% else %} # Use Postgres as a database backend # If set, will be used instead of SQLite3 # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - connString: "postgres://matrix_puppet_discord:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_discord') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_discord?sslmode=disable" - # Use SQLite3 as a database backend - # The name of the database file - #filename: /data/database.db + connString: {{ matrix_mx_puppet_discord_database_connString|to_json }} +{% endif %} logging: # Log level of console output diff --git a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml index 21cde53f..c30f9b8b 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml @@ -46,6 +46,20 @@ matrix_mx_puppet_instagram_homeserver_token: '' # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). matrix_mx_puppet_instagram_login_shared_secret: '' +matrix_mx_puppet_instagram_database_engine: sqlite +matrix_mx_puppet_instagram_database_username: matrix_mx_puppet_instagram +matrix_mx_puppet_instagram_database_password: ~ +matrix_mx_puppet_instagram_database_hostname: 'matrix-postgres' +matrix_mx_puppet_instagram_database_port: 5432 +matrix_mx_puppet_instagram_database_name: matrix_mx_puppet_instagram +matrix_mx_puppet_instagram_database_file: /data/database.db +matrix_mx_puppet_instagram_database_connString: >-2 + {%- if matrix_mx_puppet_instagram_database_engine == 'postgres' -%} + postgresql://{{ matrix_mx_puppet_instagram_database_username }}:{{ matrix_mx_puppet_instagram_database_password }}@{{ matrix_mx_puppet_instagram_database_hostname }}:{{ matrix_mx_puppet_instagram_database_port }}/{{ matrix_mx_puppet_instagram_database_name }}?sslmode=disable + {%- else if matrix_mx_puppet_instagram_database_engine == 'sqlite' -%} + {{ matrix_mx_puppet_instagram_database_engine }}://{{ matrix_mx_puppet_instagram_database_file }} + {%- endif -%} + # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. # diff --git a/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml index b6d9d994..7000a8a1 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml +++ b/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml @@ -8,3 +8,24 @@ with_items: - "matrix_mx_puppet_instagram_appservice_token" - "matrix_mx_puppet_instagram_homeserver_token" + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mx_puppet_instagram_data_path }}/database.db" + register: matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result + + - name: Fail if an SQLite database already exists when using Postgres + fail: + msg: >- + matrix_mx_puppet_instagram_database_engine has been set to `postgres` (which is our new default now). + However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_instagram_data_path }}/database.db. + It appears that you've been using this bridge with the SQLite engine until now. + To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_instagram_database_engine: sqlite` to your vars.yml file and re-run this same command. + Alternatively, to migrate your existing SQLite database to Postgres: + 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) + 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_instagram_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_instagram_database_connString'`) + 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) + when: "matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result.stat.exists" + when: "matrix_mx_puppet_instagram_database_engine == 'postgres'" + diff --git a/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 index 2d189195..d89c5bdb 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 @@ -44,15 +44,18 @@ provisioning: apiPrefix: /_matrix/provision database: +{% if matrix_mx_puppet_instagram_database_engine == 'postgres' %} # Use Postgres as a database backend # If set, will be used instead of SQLite3 # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - connString: "postgres://matrix_puppet_instagram:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_instagram') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_instagram?sslmode=disable" + connString: {{ matrix_mx_puppet_instagram_database_connString | to_json }} +{% else %} # Use SQLite3 as a database backend # The name of the database file - #filename: /data/database.db + filename: {{ matrix_mx_puppet_instagram_database_file }} +{% endif %} logging: # Log level of console output diff --git a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml index 0c16a8ba..3b7448c6 100644 --- a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml @@ -53,6 +53,21 @@ matrix_mx_puppet_skype_homeserver_token: '' # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). matrix_mx_puppet_skype_login_shared_secret: '' +# Database configuration, role default is `sqlite` but playbook default is `postgres` +matrix_mx_puppet_skype_database_engine: sqlite +matrix_mx_puppet_skype_database_username: matrix_mx_puppet_skype +matrix_mx_puppet_skype_database_password: ~ +matrix_mx_puppet_skype_database_hostname: 'matrix-postgres' +matrix_mx_puppet_skype_database_port: 5432 +matrix_mx_puppet_skype_database_name: matrix_mx_puppet_skype +matrix_mx_puppet_skype_database_file: /data/database.db +matrix_mx_puppet_skype_database_connString: >-2 + {%- if matrix_mx_puppet_skype_database_engine == 'postgres' -%} + postgresql://{{ matrix_mx_puppet_skype_database_username }}:{{ matrix_mx_puppet_skype_database_password }}@{{ matrix_mx_puppet_skype_database_hostname }}:{{ matrix_mx_puppet_skype_database_port }}/{{ matrix_mx_puppet_skype_database_name }}?sslmode=disable + {%- else if matrix_mx_puppet_skype_database_engine == 'sqlite' -%} + {{ matrix_mx_puppet_skype_database_engine }}://{{ matrix_mx_puppet_skype_database_file }} + {%- endif -%} + # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. # diff --git a/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml index 7ed433b1..c7100e51 100644 --- a/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml +++ b/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml @@ -8,3 +8,23 @@ with_items: - "matrix_mx_puppet_skype_appservice_token" - "matrix_mx_puppet_skype_homeserver_token" + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mx_puppet_skype_data_path }}/database.db" + register: matrix_mx_puppet_skype_sqlite_database_path_local_stat_result + + - name: Fail if an SQLite database already exists when using Postgres + fail: + msg: >- + matrix_mx_puppet_skype_database_engine has been set to `postgres` (which is our new default now). + However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_skype_data_path }}/database.db. + It appears that you've been using this bridge with the SQLite engine until now. + To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_skype_database_engine: sqlite` to your vars.yml file and re-run this same command. + Alternatively, to migrate your existing SQLite database to Postgres: + 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) + 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_skype_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_skype_database_connString'`) + 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) + when: "matrix_mx_puppet_skype_sqlite_database_path_local_stat_result.stat.exists" + when: "matrix_mx_puppet_skype_database_engine == 'postgres'" diff --git a/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 index a54ca758..a32523b1 100644 --- a/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 @@ -68,15 +68,18 @@ logging: # - Store database: +{% if matrix_mx_puppet_skype_database_engine == 'postgres' %} # Use Postgres as a database backend # If set, will be used instead of SQLite3 # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - connString: "postgres://matrix_puppet_skype:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_skype') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_skype?sslmode=disable" + connString: {{ matrix_mx_puppet_skype_database_connString | to_json }} +{% else %} # Use SQLite3 as a database backend # The name of the database file - #filename: /data/database.db + filename: {{ matrix_mx_puppet_skype_database_file }} +{% endif %} provisioning: # Regex of Matrix IDs allowed to use the puppet bridge diff --git a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml index 96ab7833..5f91beb8 100644 --- a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml @@ -57,6 +57,21 @@ matrix_mx_puppet_slack_homeserver_token: '' # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). matrix_mx_puppet_slack_login_shared_secret: '' +# Database configuration, role uses 'sqlite' per default but playbook sets up postgres by default +matrix_mx_puppet_slack_database_engine: sqlite +matrix_mx_puppet_slack_database_username: matrix_mx_puppet_slack +matrix_mx_puppet_slack_database_password: ~ +matrix_mx_puppet_slack_database_hostname: 'matrix-postgres' +matrix_mx_puppet_slack_database_port: 5432 +matrix_mx_puppet_slack_database_name: matrix_mx_puppet_slack +matrix_mx_puppet_slack_database_file: /data/database.db +matrix_mx_puppet_slack_database_connString: >-2 + {%- if matrix_mx_puppet_slack_database_engine == 'postgres' -%} + postgresql://{{ matrix_mx_puppet_slack_database_username }}:{{ matrix_mx_puppet_slack_database_password }}@{{ matrix_mx_puppet_slack_database_hostname }}:{{ matrix_mx_puppet_slack_database_port }}/{{ matrix_mx_puppet_slack_database_name }}?sslmode=disable + {%- else if matrix_mx_puppet_slack_database_engine == 'sqlite' -%} + {{ matrix_mx_puppet_slack_database_engine }}://{{ matrix_mx_puppet_slack_database_file }} + {%- endif -%} + # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. # diff --git a/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml index 3a0bca11..bc722ee6 100644 --- a/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml +++ b/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml @@ -8,3 +8,23 @@ with_items: - "matrix_mx_puppet_slack_appservice_token" - "matrix_mx_puppet_slack_homeserver_token" + +- block: + - name: Check if sqlite database already exists + stat: + path: "{{ matrix_mx_puppet_slack_data_path }}/database.db" + register: matrix_mx_puppet_slack_sqlite_database_path_local_stat_result + + - name: Fail if an SQLite database already exists when using Postgres + fail: + msg: >- + matrix_mx_puppet_slack_database_engine has been set to `postgres` (which is our new default now). + However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_slack_data_path }}/database.db. + It appears that you've been using this bridge with the SQLite engine until now. + To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_slack_database_engine: sqlite` to your vars.yml file and re-run this same command. + Alternatively, to migrate your existing SQLite database to Postgres: + 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) + 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_slack_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_slack_database_connString'`) + 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) + when: "matrix_mx_puppet_slack_sqlite_database_path_local_stat_result.stat.exists" + when: "matrix_mx_puppet_slack_database_engine == 'postgres'" diff --git a/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 index 7f1b21d7..58c5c11a 100644 --- a/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 @@ -58,15 +58,18 @@ provisioning: apiPrefix: /_matrix/provision database: +{% if matrix_mx_puppet_slack_database_engine == 'postgres' %} # Use Postgres as a database backend # If set, will be used instead of SQLite3 # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - connString: "postgres://matrix_puppet_slack:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_slack') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_slack?sslmode=disable" + connString: {{ matrix_mx_puppet_slack_database_connString | to_json }} +{% else %} # Use SQLite3 as a database backend # The name of the database file - #filename: /data/database.db + filename: {{ matrix_mx_puppet_slack_database_file }} +{% endif %} logging: # Log level of console output diff --git a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml index b5eb8473..be343db4 100644 --- a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml @@ -55,6 +55,20 @@ matrix_mx_puppet_steam_homeserver_token: '' # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). matrix_mx_puppet_steam_login_shared_secret: '' +matrix_mx_puppet_steam_database_engine: sqlite +matrix_mx_puppet_steam_database_username: matrix_mx_puppet_steam +matrix_mx_puppet_steam_database_password: ~ +matrix_mx_puppet_steam_database_hostname: 'matrix-postgres' +matrix_mx_puppet_steam_database_port: 5432 +matrix_mx_puppet_steam_database_name: matrix_mx_puppet_steam +matrix_mx_puppet_steam_database_file: /data/database.db +matrix_mx_puppet_steam_database_connString: >-2 + {%- if matrix_mx_puppet_steam_database_engine == 'postgres' -%} + postgresql://{{ matrix_mx_puppet_steam_database_username }}:{{ matrix_mx_puppet_steam_database_password }}@{{ matrix_mx_puppet_steam_database_hostname }}:{{ matrix_mx_puppet_steam_database_port }}/{{ matrix_mx_puppet_steam_database_name }}?sslmode=disable + {%- else if matrix_mx_puppet_steam_database_engine == 'sqlite' -%} + {{ matrix_mx_puppet_steam_database_engine }}://{{ matrix_mx_puppet_steam_database_file }} + {%- endif -%} + # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. # diff --git a/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml index a8bc6a42..50ce15d9 100644 --- a/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml +++ b/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml @@ -8,3 +8,25 @@ with_items: - "matrix_mx_puppet_steam_appservice_token" - "matrix_mx_puppet_steam_homeserver_token" + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mx_puppet_steam_data_path }}" + register: matrix_mx_puppet_steam_sqlite_database_path_local_stat_result + + - name: Fail if an SQLite database already exists when using Postgres + fail: + msg: >- + matrix_mx_puppet_steam_database_engine has been set to `postgres` (which is our new default now). + However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_steam_data_path }}/database.db. + It appears that you've been using this bridge with the SQLite engine until now. + To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_steam_database_engine: sqlite` to your vars.yml file and re-run this same command. + Alternatively, to migrate your existing SQLite database to Postgres: + 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) + 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_steam_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_steam_database_connString'`) + 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) + when: "matrix_mx_puppet_steam_sqlite_database_path_local_stat_result.stat.exists" + when: "matrix_mx_puppet_steam_database_engine == 'postgres'" + + diff --git a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 index 14f86319..8eda278d 100644 --- a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 @@ -61,15 +61,18 @@ selfService: blacklist: {{ matrix_mx_puppet_steam_provisioning_blacklist|to_json }} database: +{% if matrix_mx_puppet_steam_database_engine == 'postgres' %} # Use Postgres as a database backend # If set, will be used instead of SQLite3 # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - connString: "postgres://matrix_puppet_steam:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_steam') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_steam?sslmode=disable" + connString: {{ matrix_mx_puppet_steam_database_connString | to_json }} +{% else %} # Use SQLite3 as a database backend # The name of the database file - #filename: /data/database.db + filename: {{ matrix_mx_puppet_steam_database_file }} +{% endif %} logging: # Log level of console output diff --git a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml index 13438a21..f536029c 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml @@ -60,6 +60,21 @@ matrix_mx_puppet_twitter_homeserver_token: '' # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). matrix_mx_puppet_twitter_login_shared_secret: '' +# Database configuration +matrix_mx_puppet_twitter_database_engine: sqlite +matrix_mx_puppet_twitter_database_username: matrix:mx_puppet_twitter +matrix_mx_puppet_twitter_database_password: ~ +matrix_mx_puppet_twitter_database_hostname: 'matrix-postgres' +matrix_mx_puppet_twitter_database_port: 5432 +matrix_mx_puppet_twitter_database_name: matrix_mx_puppet_twitter +matrix_mx_puppet_twitter_database_file: /data/database.db +matrix_mx_puppet_twitter_database_connString: >-2 + {%- if matrix_mx_puppet_twitter_database_engine == 'postgres' -%} + postgresql://{{ matrix_mx_puppet_twitter_database_username }}:{{ matrix_mx_puppet_twitter_database_password }}@{{ matrix_mx_puppet_twitter_database_hostname }}:{{ matrix_mx_puppet_twitter_database_port }}/{{ matrix_mx_puppet_twitter_database_name }}?sslmode=disable + {%- else if matrix_mx_puppet_twitter_database_engine == 'sqlite' -%} + {{ matrix_mx_puppet_twitter_database_engine }}://{{ matrix_mx_puppet_twitter_database_file }} + {%- endif -%} + # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. # diff --git a/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml index d13a39e1..0b3bd8f8 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml +++ b/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml @@ -8,3 +8,25 @@ with_items: - "matrix_mx_puppet_twitter_appservice_token" - "matrix_mx_puppet_twitter_homeserver_token" + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mx_puppet_twitter_sqlite_data_path }}/database.db" + register: matrix_mx_puppet_twitter_sqlite_database_path_local_stat_result + + - name: Fail if an SQLite database already exists when using Postgres + fail: + msg: >- + matrix_mx_puppet_twitter_database_engine has been set to `postgres` (which is our new default now). + However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_twitter_data_path }}/database.db. + It appears that you've been using this bridge with the SQLite engine until now. + To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_twitter_database_engine: sqlite` to your vars.yml file and re-run this same command. + Alternatively, to migrate your existing SQLite database to Postgres: + 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) + 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_twitter_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_twitter_database_connString'`) + 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) + when: "matrix_mx_puppet_twitter_sqlite_database_path_local_stat_result.stat.exists" + when: "matrix_mx_puppet_twitter_database_engine == 'postgres'" + + diff --git a/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 index 853d23fc..cecb171d 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 @@ -54,15 +54,18 @@ provisioning: apiPrefix: /_matrix/provision database: +{% if matrix_mx_puppet_twitter_database_engine == 'postgres' %} # Use Postgres as a database backend # If set, will be used instead of SQLite3 # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - connString: "postgres://matrix_puppet_twitter:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_twitter') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_twitter?sslmode=disable" + connString: {{ matrix_mx_puppet_twitter_database_connString | to_json }} +{% else %} # Use SQLite3 as a database backend # The name of the database file - #filename: /data/database.db + filename: {{ matrix_mx_puppet_twitter_database_file }} +{% endif %} logging: # Log level of console output From b9a04a7f953e504a3e6798eeebcc786f51bf9788 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 14:42:18 +0200 Subject: [PATCH 146/217] Rename some remaining `matrix_*_postgres_*` vars back to `matrix_*_database_*` Looks like there are some that I missed in 087dbe4ddc80ba6 --- group_vars/matrix_servers | 28 +++++++++---------- .../defaults/main.yml | 4 +-- .../defaults/main.yml | 6 ++-- .../tasks/setup_install.yml | 2 +- .../defaults/main.yml | 6 ++-- .../tasks/setup_install.yml | 2 +- .../defaults/main.yml | 6 ++-- .../tasks/setup_install.yml | 2 +- .../defaults/main.yml | 6 ++-- .../tasks/setup_install.yml | 2 +- 10 files changed, 32 insertions(+), 32 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index acf133e1..ad1e8ce3 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -953,37 +953,37 @@ matrix_postgres_additional_databases: | 'name': matrix_appservice_discord_database_db_name, 'username': matrix_appservice_discord_database_username, 'password': matrix_appservice_discord_database_password, - }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_postgres_hostname == 'matrix-postgres') else []) + }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_database_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_appservice_slack_database_db_name, 'username': matrix_appservice_slack_database_username, 'password': matrix_appservice_slack_database_password, - }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_postgres_hostname == 'matrix-postgres') else []) + }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_appservice_irc_database_db_name, 'username': matrix_appservice_irc_database_username, 'password': matrix_appservice_irc_database_password, - }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_postgres_hostname == 'matrix-postgres') else []) + }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_mautrix_facebook_database_db_name, 'username': matrix_mautrix_facebook_database_username, 'password': matrix_mautrix_facebook_database_password, - }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_postgres_hostname == 'matrix-postgres') else []) + }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_database_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_mautrix_hangouts_database_db_name, 'username': matrix_mautrix_hangouts_database_username, 'password': matrix_mautrix_hangouts_database_password, - }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_postgres_hostname == 'matrix-postgres') else []) + }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_mautrix_telegram_database_db_name, 'username': matrix_mautrix_telegram_database_username, 'password': matrix_mautrix_telegram_database_password, - }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_mautrix_telegram_postgres_hostname == 'matrix-postgres') else []) + }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_mautrix_telegram_database_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_mautrix_whatsapp_database_db_name, 'username': matrix_mautrix_whatsapp_database_username, 'password': matrix_mautrix_whatsapp_database_password, - }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_postgres_hostname == 'matrix-postgres') else []) + }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres') else []) + ([{ 'name': 'matrix_bridge_sms', 'username': 'matrix_bridge_sms', @@ -993,45 +993,45 @@ matrix_postgres_additional_databases: | 'name': matrix_mx_puppet_skype_database_db_name, 'username': matrix_mx_puppet_skype_database_username, 'password': matrix_mx_puppet_skype_database_password, - }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_postgres_hostname == 'matrix-postgres') else []) + }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_mx_puppet_slack_database_db_name, 'username': matrix_mx_puppet_slack_database_username, 'password': matrix_mx_puppet_slack_database_password, - }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_postgres_hostname == 'matrix-postgres') else []) + }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_mx_puppet_twitter_database_db_name, 'username': matrix_mx_puppet_twitter_database_username, 'password': matrix_mx_puppet_twitter_database_password, - }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_postgres_hostname == 'matrix-postgres') else []) + }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_mx_puppet_instagram_database_db_name, 'username': matrix_mx_puppet_instagram_database_username, 'password': matrix_mx_puppet_instagram_database_password, }] if (matrix_mx_puppet_instagram_enabled and matrix_mx_puppet_instagram_database_engine == 'postgres' - and matrix_mx_puppet_instagram_postgres_hostname == 'matrix-postgres') else []) + and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_mx_puppet_discord_database_db_name, 'username': matrix_mx_puppet_discord_database_username, 'password': matrix_mx_puppet_discord_database_password, }] if (matrix_mx_puppet_discord_enabled and matrix_mx_puppet_discord_database_engine == 'postgres' - and matrix_mx_puppet_discord_postgres_hostname == 'matrix-postgres') else []) + and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_mx_puppet_steam_database_db_name, 'username': matrix_mx_puppet_steam_database_username, 'password': matrix_mx_puppet_steam_database_password, }] if (matrix_mx_puppet_steam_enabled and matrix_mx_puppet_steam_database_engine == 'postgres' - and matrix_mx_puppet_steam_postgres_hostname == 'matrix-postgres') else []) + and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else []) + ([{ 'name': matrix_dimension_database_db_name, 'username': matrix_dimension_database_username, 'password': matrix_dimension_database_password, }] if (matrix_dimension_enabled and matrix_dimension_database_engine == 'postgres' - and matrix_dimension_postgres_hostname == 'matrix-postgres') else []) + and matrix_dimension_database_hostname == 'matrix-postgres') else []) }} ###################################################################### diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml index 57a88867..e2f97b85 100644 --- a/roles/matrix-bridge-appservice-discord/defaults/main.yml +++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml @@ -55,14 +55,14 @@ matrix_appservice_discord_sqlite_database_path_in_container: "/data/discord.db" matrix_appservice_discord_database_username: 'matrix_appservice_discord' matrix_appservice_discord_database_password: 'some-password' -matrix_appservice_discord_postgres_hostname: 'matrix-postgres' +matrix_appservice_discord_database_hostname: 'matrix-postgres' matrix_appservice_discord_database_port: 5432 matrix_appservice_discord_database_db_name: 'matrix_appservice_discord' # These 2 variables are what actually ends up in the bridge configuration. # It's best if you don't change them directly, but rather redefine the sub-variables that constitute them. matrix_appservice_discord_database_filename: "{{ matrix_appservice_discord_sqlite_database_path_in_container }}" -matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_username }}:{{ matrix_appservice_discord_database_password }}@{{ matrix_appservice_discord_postgres_hostname }}:{{ matrix_appservice_discord_database_port }}/{{ matrix_appservice_discord_database_db_name }}' +matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_username }}:{{ matrix_appservice_discord_database_password }}@{{ matrix_appservice_discord_database_hostname }}:{{ matrix_appservice_discord_database_port }}/{{ matrix_appservice_discord_database_db_name }}' # Tells whether the bot should make use of "Privileged Gateway Intents". diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml index 246c32fe..c02197a5 100644 --- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml @@ -47,16 +47,16 @@ matrix_mautrix_facebook_sqlite_database_path_in_container: "/data/mautrix-facebo matrix_mautrix_facebook_database_username: 'matrix_mautrix_facebook' matrix_mautrix_facebook_database_password: 'some-password' -matrix_mautrix_facebook_postgres_hostname: 'matrix-postgres' +matrix_mautrix_facebook_database_hostname: 'matrix-postgres' matrix_mautrix_facebook_database_port: 5432 matrix_mautrix_facebook_database_db_name: 'matrix_mautrix_facebook' -matrix_mautrix_facebook_postgres_connection_string: 'postgresql://{{ matrix_mautrix_facebook_database_username }}:{{ matrix_mautrix_facebook_database_password }}@{{ matrix_mautrix_facebook_postgres_hostname }}:{{ matrix_mautrix_facebook_database_port }}/{{ matrix_mautrix_facebook_database_db_name }}' +matrix_mautrix_facebook_database_connection_string: 'postgresql://{{ matrix_mautrix_facebook_database_username }}:{{ matrix_mautrix_facebook_database_password }}@{{ matrix_mautrix_facebook_database_hostname }}:{{ matrix_mautrix_facebook_database_port }}/{{ matrix_mautrix_facebook_database_db_name }}' matrix_mautrix_facebook_appservice_database: "{{ { 'sqlite': ('sqlite://' + matrix_mautrix_facebook_sqlite_database_path_in_container), - 'postgres': matrix_mautrix_facebook_postgres_connection_string, + 'postgres': matrix_mautrix_facebook_database_connection_string, }[matrix_mautrix_facebook_database_engine] }}" diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml index c3c8d61f..1a0ff70c 100644 --- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml @@ -23,7 +23,7 @@ To continue using SQLite, opt into it explicitly: add `matrix_mautrix_facebook_database_engine: sqlite` to your vars.yml file and re-run this same command. Alternatively, to migrate your existing SQLite database to Postgres: 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_facebook_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_facebook_postgres_connection_string'`) + 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_facebook_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_facebook_database_connection_string'`) 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) when: "matrix_mautrix_facebook_sqlite_database_path_local_stat_result.stat.exists" when: "matrix_mautrix_facebook_database_engine == 'postgres'" diff --git a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml index 04280ac4..5d38289b 100644 --- a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml @@ -54,16 +54,16 @@ matrix_mautrix_hangouts_sqlite_database_path_in_container: "/data/mautrix-hangou matrix_mautrix_hangouts_database_username: 'matrix_mautrix_hangouts' matrix_mautrix_hangouts_database_password: 'some-password' -matrix_mautrix_hangouts_postgres_hostname: 'matrix-postgres' +matrix_mautrix_hangouts_database_hostname: 'matrix-postgres' matrix_mautrix_hangouts_database_port: 5432 matrix_mautrix_hangouts_database_db_name: 'matrix_mautrix_hangouts' -matrix_mautrix_hangouts_postgres_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_database_username }}:{{ matrix_mautrix_hangouts_database_password }}@{{ matrix_mautrix_hangouts_postgres_hostname }}:{{ matrix_mautrix_hangouts_database_port }}/{{ matrix_mautrix_hangouts_database_db_name }}' +matrix_mautrix_hangouts_database_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_database_username }}:{{ matrix_mautrix_hangouts_database_password }}@{{ matrix_mautrix_hangouts_database_hostname }}:{{ matrix_mautrix_hangouts_database_port }}/{{ matrix_mautrix_hangouts_database_db_name }}' matrix_mautrix_hangouts_appservice_database: "{{ { 'sqlite': ('sqlite://' + matrix_mautrix_hangouts_sqlite_database_path_in_container), - 'postgres': matrix_mautrix_hangouts_postgres_connection_string, + 'postgres': matrix_mautrix_hangouts_database_connection_string, }[matrix_mautrix_hangouts_database_engine] }}" diff --git a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml index 9e7bfbba..02c34357 100644 --- a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml @@ -23,7 +23,7 @@ To continue using SQLite, opt into it explicitly: add `matrix_mautrix_hangouts_database_engine: sqlite` to your vars.yml file and re-run this same command. Alternatively, to migrate your existing SQLite database to Postgres: 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_hangouts_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_hangouts_postgres_connection_string'`) + 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_hangouts_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_hangouts_database_connection_string'`) 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) when: "matrix_mautrix_hangouts_sqlite_database_path_local_stat_result.stat.exists" when: "matrix_mautrix_hangouts_database_engine == 'postgres'" diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml index 4f584128..fcdf95db 100644 --- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml @@ -58,16 +58,16 @@ matrix_mautrix_telegram_sqlite_database_path_in_container: "/data/mautrix-telegr matrix_mautrix_telegram_database_username: 'matrix_mautrix_telegram' matrix_mautrix_telegram_database_password: 'some-password' -matrix_mautrix_telegram_postgres_hostname: 'matrix-postgres' +matrix_mautrix_telegram_database_hostname: 'matrix-postgres' matrix_mautrix_telegram_database_port: 5432 matrix_mautrix_telegram_database_db_name: 'matrix_mautrix_telegram' -matrix_mautrix_telegram_postgres_connection_string: 'postgresql://{{ matrix_mautrix_telegram_database_username }}:{{ matrix_mautrix_telegram_database_password }}@{{ matrix_mautrix_telegram_postgres_hostname }}:{{ matrix_mautrix_telegram_database_port }}/{{ matrix_mautrix_telegram_database_db_name }}' +matrix_mautrix_telegram_database_connection_string: 'postgresql://{{ matrix_mautrix_telegram_database_username }}:{{ matrix_mautrix_telegram_database_password }}@{{ matrix_mautrix_telegram_database_hostname }}:{{ matrix_mautrix_telegram_database_port }}/{{ matrix_mautrix_telegram_database_db_name }}' matrix_mautrix_telegram_appservice_database: "{{ { 'sqlite': ('sqlite://' + matrix_mautrix_telegram_sqlite_database_path_in_container), - 'postgres': matrix_mautrix_telegram_postgres_connection_string, + 'postgres': matrix_mautrix_telegram_database_connection_string, }[matrix_mautrix_telegram_database_engine] }}" diff --git a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml index db1a8f83..d898c590 100644 --- a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml @@ -23,7 +23,7 @@ To continue using SQLite, opt into it explicitly: add `matrix_mautrix_telegram_database_engine: sqlite` to your vars.yml file and re-run this same command. Alternatively, to migrate your existing SQLite database to Postgres: 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_telegram_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_telegram_postgres_connection_string'`) + 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_telegram_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_telegram_database_connection_string'`) 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) when: "matrix_mautrix_telegram_sqlite_database_path_local_stat_result.stat.exists" when: "matrix_mautrix_telegram_database_engine == 'postgres'" diff --git a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml index 357f5892..7e198b36 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml @@ -42,11 +42,11 @@ matrix_mautrix_whatsapp_sqlite_database_path_in_container: "/data/mautrix-whatsa matrix_mautrix_whatsapp_database_username: 'matrix_mautrix_whatsapp' matrix_mautrix_whatsapp_database_password: 'some-password' -matrix_mautrix_whatsapp_postgres_hostname: 'matrix-postgres' +matrix_mautrix_whatsapp_database_hostname: 'matrix-postgres' matrix_mautrix_whatsapp_database_port: 5432 matrix_mautrix_whatsapp_database_db_name: 'matrix_mautrix_whatsapp' -matrix_mautrix_whatsapp_postgres_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_database_username }}:{{ matrix_mautrix_whatsapp_database_password }}@{{ matrix_mautrix_whatsapp_postgres_hostname }}:{{ matrix_mautrix_whatsapp_database_port }}/{{ matrix_mautrix_whatsapp_database_db_name }}' +matrix_mautrix_whatsapp_database_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_database_username }}:{{ matrix_mautrix_whatsapp_database_password }}@{{ matrix_mautrix_whatsapp_database_hostname }}:{{ matrix_mautrix_whatsapp_database_port }}/{{ matrix_mautrix_whatsapp_database_db_name }}' matrix_mautrix_whatsapp_appservice_database_type: "{{ { @@ -58,7 +58,7 @@ matrix_mautrix_whatsapp_appservice_database_type: "{{ matrix_mautrix_whatsapp_appservice_database_uri: "{{ { 'sqlite': matrix_mautrix_whatsapp_sqlite_database_path_in_container, - 'postgres': matrix_mautrix_whatsapp_postgres_connection_string, + 'postgres': matrix_mautrix_whatsapp_database_connection_string, }[matrix_mautrix_whatsapp_database_engine] }}" diff --git a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml index 15453e8c..9521e0a5 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml @@ -23,7 +23,7 @@ To continue using SQLite, opt into it explicitly: add `matrix_mautrix_whatsapp_database_engine: sqlite` to your vars.yml file and re-run this same command. Alternatively, to migrate your existing SQLite database to Postgres: 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_whatsapp_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_whatsapp_postgres_connection_string'`) + 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_whatsapp_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_whatsapp_database_connection_string'`) 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) when: "matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result.stat.exists" when: "matrix_mautrix_whatsapp_database_engine == 'postgres'" From aa828ff9f64415a404b3d7bb2a70690fea968d02 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 14:50:04 +0200 Subject: [PATCH 147/217] Separate matrix-reminder-bot install/uninstall tasks --- .../tasks/main.yml | 9 +- .../tasks/setup.yml | 88 ------------------- .../tasks/setup_install.yml | 41 +++++++++ .../tasks/setup_uninstall.yml | 35 ++++++++ 4 files changed, 84 insertions(+), 89 deletions(-) delete mode 100644 roles/matrix-bot-matrix-reminder-bot/tasks/setup.yml create mode 100644 roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml create mode 100644 roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/main.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/main.yml index ade3d191..fc2afddb 100644 --- a/roles/matrix-bot-matrix-reminder-bot/tasks/main.yml +++ b/roles/matrix-bot-matrix-reminder-bot/tasks/main.yml @@ -8,7 +8,14 @@ - setup-all - setup-bot-matrix-reminder-bot -- import_tasks: "{{ role_path }}/tasks/setup.yml" +- import_tasks: "{{ role_path }}/tasks/setup_install.yml" + when: "run_setup|bool and matrix_bot_matrix_reminder_bot_enabled|bool" + tags: + - setup-all + - setup-bot-matrix-reminder-bot + +- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml" + when: "run_setup|bool and not matrix_bot_matrix_reminder_bot_enabled|bool" tags: - setup-all - setup-bot-matrix-reminder-bot diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup.yml deleted file mode 100644 index bc211e31..00000000 --- a/roles/matrix-bot-matrix-reminder-bot/tasks/setup.yml +++ /dev/null @@ -1,88 +0,0 @@ ---- - -# -# Tasks related to setting up matrix-reminder-bot -# - -- name: Ensure matrix-reminder-bot paths exist - file: - path: "{{ item.path }}" - state: directory - mode: 0750 - owner: "{{ matrix_user_username }}" - group: "{{ matrix_user_groupname }}" - with_items: - - { path: "{{ matrix_bot_matrix_reminder_bot_config_path }}", when: true } - - { path: "{{ matrix_bot_matrix_reminder_bot_data_path }}", when: true } - - { path: "{{ matrix_bot_matrix_reminder_bot_data_store_path }}", when: true } - when: matrix_bot_matrix_reminder_bot_enabled|bool and item.when - -- name: Ensure matrix-reminder-bot image is pulled - docker_image: - name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}" - source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" - force_source: "{{ matrix_bot_matrix_reminder_bot_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_bot_matrix_reminder_bot_docker_image_force_pull }}" - when: matrix_bot_matrix_reminder_bot_enabled|bool - -- name: Ensure matrix-reminder-bot config installed - copy: - content: "{{ matrix_bot_matrix_reminder_bot_configuration|to_nice_yaml }}" - dest: "{{ matrix_bot_matrix_reminder_bot_config_path }}/config.yaml" - mode: 0644 - owner: "{{ matrix_user_username }}" - group: "{{ matrix_user_groupname }}" - when: matrix_bot_matrix_reminder_bot_enabled|bool - -- name: Ensure matrix-matrix-reminder-bot.service installed - template: - src: "{{ role_path }}/templates/systemd/matrix-bot-matrix-reminder-bot.service.j2" - dest: "{{ matrix_systemd_path }}/matrix-bot-matrix-reminder-bot.service" - mode: 0644 - register: matrix_bot_matrix_reminder_bot_systemd_service_result - when: matrix_bot_matrix_reminder_bot_enabled|bool - -- name: Ensure systemd reloaded after matrix-matrix-reminder-bot.service installation - service: - daemon_reload: yes - when: "matrix_bot_matrix_reminder_bot_enabled|bool and matrix_bot_matrix_reminder_bot_systemd_service_result.changed" - -# -# Tasks related to getting rid of matrix-reminder-bot (if it was previously enabled) -# - -- name: Check existence of matrix-matrix-reminder-bot service - stat: - path: "{{ matrix_systemd_path }}/matrix-matrix-reminder-bot.service" - register: matrix_bot_matrix_reminder_bot_service_stat - -- name: Ensure matrix-matrix-reminder-bot is stopped - service: - name: matrix-matrix-reminder-bot - state: stopped - daemon_reload: yes - register: stopping_result - when: "not matrix_bot_matrix_reminder_bot_enabled|bool and matrix_bot_matrix_reminder_bot_service_stat.stat.exists" - -- name: Ensure matrix-matrix-reminder-bot.service doesn't exist - file: - path: "{{ matrix_systemd_path }}/matrix-matrix-reminder-bot.service" - state: absent - when: "not matrix_bot_matrix_reminder_bot_enabled|bool and matrix_bot_matrix_reminder_bot_service_stat.stat.exists" - -- name: Ensure systemd reloaded after matrix-matrix-reminder-bot.service removal - service: - daemon_reload: yes - when: "not matrix_bot_matrix_reminder_bot_enabled|bool and matrix_bot_matrix_reminder_bot_service_stat.stat.exists" - -- name: Ensure Matrix matrix-reminder-bot paths don't exist - file: - path: "{{ matrix_bot_matrix_reminder_bot_base_path }}" - state: absent - when: "not matrix_bot_matrix_reminder_bot_enabled|bool" - -- name: Ensure matrix-reminder-bot Docker image doesn't exist - docker_image: - name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}" - state: absent - when: "not matrix_bot_matrix_reminder_bot_enabled|bool" diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml new file mode 100644 index 00000000..366c39c3 --- /dev/null +++ b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml @@ -0,0 +1,41 @@ +--- + +- name: Ensure matrix-reminder-bot paths exist + file: + path: "{{ item.path }}" + state: directory + mode: 0750 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + with_items: + - { path: "{{ matrix_bot_matrix_reminder_bot_config_path }}", when: true } + - { path: "{{ matrix_bot_matrix_reminder_bot_data_path }}", when: true } + - { path: "{{ matrix_bot_matrix_reminder_bot_data_store_path }}", when: true } + when: "item.when|bool" + +- name: Ensure matrix-reminder-bot image is pulled + docker_image: + name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_bot_matrix_reminder_bot_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_bot_matrix_reminder_bot_docker_image_force_pull }}" + +- name: Ensure matrix-reminder-bot config installed + copy: + content: "{{ matrix_bot_matrix_reminder_bot_configuration|to_nice_yaml }}" + dest: "{{ matrix_bot_matrix_reminder_bot_config_path }}/config.yaml" + mode: 0644 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + +- name: Ensure matrix-bot-matrix-reminder-bot.service installed + template: + src: "{{ role_path }}/templates/systemd/matrix-bot-matrix-reminder-bot.service.j2" + dest: "{{ matrix_systemd_path }}/matrix-bot-matrix-reminder-bot.service" + mode: 0644 + register: matrix_bot_matrix_reminder_bot_systemd_service_result + +- name: Ensure systemd reloaded after matrix-bot-matrix-reminder-bot.service installation + service: + daemon_reload: yes + when: "matrix_bot_matrix_reminder_bot_systemd_service_result.changed|bool" diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml new file mode 100644 index 00000000..744f474d --- /dev/null +++ b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml @@ -0,0 +1,35 @@ +--- + +- name: Check existence of matrix-matrix-reminder-bot service + stat: + path: "{{ matrix_systemd_path }}/matrix-bot-matrix-reminder-bot.service" + register: matrix_bot_matrix_reminder_bot_service_stat + +- name: Ensure matrix-matrix-reminder-bot is stopped + service: + name: matrix-matrix-reminder-bot + state: stopped + daemon_reload: yes + register: stopping_result + when: "matrix_bot_matrix_reminder_bot_service_stat.stat.exists|bool" + +- name: Ensure matrix-bot-matrix-reminder-bot.service doesn't exist + file: + path: "{{ matrix_systemd_path }}/matrix-bot-matrix-reminder-bot.service" + state: absent + when: "matrix_bot_matrix_reminder_bot_service_stat.stat.exists|bool" + +- name: Ensure systemd reloaded after matrix-bot-matrix-reminder-bot.service removal + service: + daemon_reload: yes + when: "matrix_bot_matrix_reminder_bot_service_stat.stat.exists|bool" + +- name: Ensure Matrix matrix-reminder-bot paths don't exist + file: + path: "{{ matrix_bot_matrix_reminder_bot_base_path }}" + state: absent + +- name: Ensure matrix-reminder-bot Docker image doesn't exist + docker_image: + name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}" + state: absent From dde1c9f899230813ca70e657c4d7873bb890f6e4 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 14:53:35 +0200 Subject: [PATCH 148/217] Fix indentation causing YAML syntax error --- .../tasks/validate_config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml index 7000a8a1..c2115cbb 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml +++ b/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml @@ -15,7 +15,7 @@ path: "{{ matrix_mx_puppet_instagram_data_path }}/database.db" register: matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result - - name: Fail if an SQLite database already exists when using Postgres + - name: Fail if an SQLite database already exists when using Postgres fail: msg: >- matrix_mx_puppet_instagram_database_engine has been set to `postgres` (which is our new default now). @@ -28,4 +28,3 @@ 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) when: "matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result.stat.exists" when: "matrix_mx_puppet_instagram_database_engine == 'postgres'" - From e3a0c9addac73580da23f1e386a67ba14d7461bf Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 15:02:11 +0200 Subject: [PATCH 149/217] Add (Postgres + SQLite) support to matrix-reminder-bot This has been tested and appears to work. --- group_vars/matrix_servers | 10 +++++++ .../defaults/main.yml | 28 +++++++++++++++++++ .../tasks/setup_install.yml | 20 +++++++++++++ .../templates/config.yaml.j2 | 2 +- 4 files changed, 59 insertions(+), 1 deletion(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index ad1e8ce3..d6d6cc4a 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -580,6 +580,10 @@ matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key # We don't enable bots by default. matrix_bot_matrix_reminder_bot_enabled: false +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_bot_matrix_reminder_bot_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_bot_matrix_reminder_bot_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'reminder.bot.db') | to_uuid }}" + ###################################################################### # # /matrix-bot-matrix-reminder-bot @@ -949,6 +953,12 @@ matrix_postgres_db_name: "homeserver" matrix_postgres_additional_databases: | {{ + ([{ + 'name': matrix_bot_matrix_reminder_bot_database_db_name, + 'username': matrix_bot_matrix_reminder_bot_database_username, + 'password': matrix_bot_matrix_reminder_bot_database_password, + }] if (matrix_bot_matrix_reminder_bot_enabled and matrix_bot_matrix_reminder_bot_database_engine == 'postgres' and matrix_bot_matrix_reminder_bot_database_hostname == 'matrix-postgres') else []) + + ([{ 'name': matrix_appservice_discord_database_db_name, 'username': matrix_appservice_discord_database_username, diff --git a/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml b/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml index 33028147..70e7b2d2 100644 --- a/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml +++ b/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml @@ -21,6 +21,34 @@ matrix_bot_matrix_reminder_bot_systemd_required_services_list: ['docker.service' matrix_bot_matrix_reminder_bot_systemd_wanted_services_list: [] +# Database-related configuration fields. +# +# To use SQLite, stick to these defaults. +# +# To use Postgres: +# - change the engine (`matrix_bot_matrix_reminder_bot_database_engine: 'postgres'`) +# - adjust your database credentials via the `matrix_bot_matrix_reminder_bot_database_*` variables +matrix_bot_matrix_reminder_bot_database_engine: 'sqlite' + +matrix_bot_matrix_reminder_bot_sqlite_database_path_local: "{{ matrix_bot_matrix_reminder_bot_data_path }}/bot.db" +matrix_bot_matrix_reminder_bot_sqlite_database_path_in_container: "/data/bot.db" + +matrix_bot_matrix_reminder_bot_database_username: 'matrix_reminder_bot' +matrix_bot_matrix_reminder_bot_database_password: 'some-password' +matrix_bot_matrix_reminder_bot_database_hostname: 'matrix-postgres' +matrix_bot_matrix_reminder_bot_database_port: 5432 +matrix_bot_matrix_reminder_bot_database_db_name: 'matrix_reminder_bot' + +matrix_bot_matrix_reminder_bot_database_connection_string: 'postgres://{{ matrix_bot_matrix_reminder_bot_database_username }}:{{ matrix_bot_matrix_reminder_bot_database_password }}@{{ matrix_bot_matrix_reminder_bot_database_hostname }}:{{ matrix_bot_matrix_reminder_bot_database_port }}/{{ matrix_bot_matrix_reminder_bot_database_db_name }}' + +matrix_bot_matrix_reminder_bot_storage_database: "{{ + { + 'sqlite': ('sqlite://' + matrix_bot_matrix_reminder_bot_sqlite_database_path_in_container), + 'postgres': matrix_bot_matrix_reminder_bot_database_connection_string, + }[matrix_bot_matrix_reminder_bot_database_engine] +}}" + + # The bot's username. This user needs to be created manually beforehand. # Also see `matrix_bot_matrix_reminder_bot_user_password`. matrix_bot_matrix_reminder_bot_matrix_user_id_localpart: "bot.matrix-reminder-bot" diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml index 366c39c3..4fbafa3a 100644 --- a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml +++ b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml @@ -1,5 +1,25 @@ --- +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }}" + register: matrix_bot_matrix_reminder_bot_sqlite_database_path_local_stat_result + + - name: Fail if an SQLite database already exists when using Postgres + fail: + msg: >- + matrix_bot_matrix_reminder_bot_database_engine has been set to `postgres` (which is our new default now). + However, we've discovered an existing SQLite database in {{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }}. + It appears that you've been using this bridge with the SQLite engine until now. + To continue using SQLite, opt into it explicitly: add `matrix_bot_matrix_reminder_bot_database_engine: sqlite` to your vars.yml file and re-run this same command. + Alternatively, to migrate your existing SQLite database to Postgres: + 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) + 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_bot_matrix_reminder_bot_database_connection_string'`) + 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) + when: "matrix_bot_matrix_reminder_bot_sqlite_database_path_local_stat_result.stat.exists" + when: "matrix_bot_matrix_reminder_bot_database_engine == 'postgres'" + - name: Ensure matrix-reminder-bot paths exist file: path: "{{ item.path }}" diff --git a/roles/matrix-bot-matrix-reminder-bot/templates/config.yaml.j2 b/roles/matrix-bot-matrix-reminder-bot/templates/config.yaml.j2 index e5ba5651..59643958 100644 --- a/roles/matrix-bot-matrix-reminder-bot/templates/config.yaml.j2 +++ b/roles/matrix-bot-matrix-reminder-bot/templates/config.yaml.j2 @@ -23,7 +23,7 @@ storage: # For Postgres, this would look like: # database: "postgres://username:password@localhost/dbname?sslmode=disable" #database: "postgres://matrix-reminder-bot:remindme@localhost/matrix-reminder-bot?sslmode=disable" - database: "sqlite:///data/bot.db" + database: {{ matrix_bot_matrix_reminder_bot_storage_database|to_json }} # The path to a directory for internal bot storage # containing encryption keys, sync tokens, etc. store_path: "/data/store" From 54da61f81b56e4c5906d09cd7dbbf0fb35092948 Mon Sep 17 00:00:00 2001 From: transcaffeine Date: Mon, 14 Dec 2020 14:02:51 +0100 Subject: [PATCH 150/217] add postgres support mx-appservice-[slack|irc] with fallback to nedb in role and migration notice --- .../defaults/main.yml | 14 ++++++++++++ .../tasks/validate_config.yml | 20 +++++++++++++++++ .../templates/config.yaml.j2 | 4 ++-- .../defaults/main.yml | 15 +++++++++++++ .../tasks/validate_config.yml | 22 ++++++++++++++++++- .../templates/config.yaml.j2 | 6 +++-- 6 files changed, 76 insertions(+), 5 deletions(-) diff --git a/roles/matrix-bridge-appservice-irc/defaults/main.yml b/roles/matrix-bridge-appservice-irc/defaults/main.yml index 7ff2d46a..a1c76253 100644 --- a/roles/matrix-bridge-appservice-irc/defaults/main.yml +++ b/roles/matrix-bridge-appservice-irc/defaults/main.yml @@ -16,6 +16,20 @@ matrix_appservice_irc_homeserver_domain: '{{ matrix_domain }}' matrix_appservice_irc_homeserver_enablePresence: true matrix_appservice_irc_appservice_address: 'http://matrix-appservice-irc:9999' +matrix_appservice_irc_database_engine: nedb +matrix_appservice_irc_database_username: matrix_appservice_irc +matrix_appservice_irc_database_password: ~ +matrix_appservice_irc_database_hostname: 'matrix-postgres' +matrix_appservice_irc_database_port: 5432 +matrix_appservice_irc_database_name: matrix_appservice_irc +matrix_appservice_irc_databasa +matrix_appservice_irc_database_connString: >-2 + {%- if matrix_appservice_irc_database_engine == 'postgres' -%} + postgresql://{{ matrix_appservice_irc_database_username }}:{{ matrix_appservice_irc_database_password }}@{{ matrix_appservice_irc_database_hostname }}:{{ matrix_appservice_irc_database_port }}/{{ matrix_appservice_irc_database_name }}?sslmode=disable + {%- else if matrix_appservice_irc_database_engine == 'nedb' -%} + {{ matrix_appservice_irc_database_engine }}://{{ matrix_appservice_irc_database_file }} + {%- endif -%} + matrix_appservice_irc_ircService_servers: [] # Example of `matrix_appservice_irc_ircService_servers` with one server (and all its options): diff --git a/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml b/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml index bd08427c..e0e932bc 100644 --- a/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml +++ b/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml @@ -33,3 +33,23 @@ when: "item.old in vars" with_items: - {'old': 'matrix_appservice_irc_container_expose_client_server_api_port', 'new': ''} + +- block: + - name: Check if a neDB database already exists + stat: + path: "{{ matrix_appservice_irc_data_path }}/" + register: matrix_appservice_irc_nedb_stat_result + + - name: Fail if an neDB database already exists when using Postgres + fail: + msg: >-2 + matrix_appservice_irc_database_engine has been set to `postgres` (which is our new default now). + However, we've discovered an existing neDB database in {{ matrix_appservice_irc_data_path }}/. + It appears that you've been using this bridge with the neDB engine until now. + To continue using neDB, opt into it explicitly: add `matrix_appservice_irc_database_engine: nedb` to your vars.yml file and re-run this same command. + Alternatively, to migrate your existing neDB database to Postgres: + 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) + 2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_irc_data_path }} postgres_connection_string_variable_name=matrix_appservice_irc_database_connString'`) + 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) + when: "matrix_appservice_irc_nedb_stat_result.stat.exists" + when: "matrix_appservice_irc_database_engine == 'postgres'" diff --git a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 index 83b71835..0da28403 100644 --- a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 +++ b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 @@ -127,8 +127,8 @@ advanced: # Use an external database to store bridge state. database: # database engine (must be 'postgres' or 'nedb'). Default: nedb - engine: "postgres" + engine: {{ matrix_appservice_irc_database_engine }} # Either a PostgreSQL connection string, or a path to the NeDB storage directory. # For postgres, it must start with postgres:// # For NeDB, it must start with nedb://. The path is relative to the project directory. - connectionString: "postgres://matrix_appservice_irc:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_appservice_irc') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_irc" + connectionString: {{ matrix_appservice_irc_database_connString | to_json }} diff --git a/roles/matrix-bridge-appservice-slack/defaults/main.yml b/roles/matrix-bridge-appservice-slack/defaults/main.yml index f2cfd8f9..e2127c29 100644 --- a/roles/matrix-bridge-appservice-slack/defaults/main.yml +++ b/roles/matrix-bridge-appservice-slack/defaults/main.yml @@ -45,6 +45,21 @@ matrix_appservice_slack_appservice_token: '' matrix_appservice_slack_homeserver_token: '' matrix_appservice_slack_id_token: '' +matrix_appservice_slack_database_engine: nedb +matrix_appservice_slack_database_username: matrix_appservice_slack +matrix_appservice_slack_database_password: ~ +matrix_appservice_slack_database_hostname: 'matrix-postgres' +matrix_appservice_slack_database_port: 5432 +matrix_appservice_slack_database_name: matrix_appservice_slack +matrix_appservice_slack_database_file: /data +matrix_appservice_slack_database_connString: >-2 + {%- if matrix_appservice_slack_database_engine == 'postgres' -%} + postgresql://{{ matrix_appservice_slack_database_username }}:{{ matrix_appservice_slack_database_password }}@{{ matrix_appservice_slack_database_hostname }}:{{ matrix_appservice_slack_database_port }}/{{ matrix_appservice_slack_database_name }}?sslmode=disable + {%- else if matrix_appservice_slack_database_engine == 'nedb' -%} + {{ matrix_appservice_slack_database_engine }}://{{ matrix_appservice_slack_database_file }} + {%- endif -%} + + matrix_appservice_slack_configuration_yaml: "{{ lookup('template', 'templates/config.yaml.j2') }}" matrix_appservice_slack_configuration_extension_yaml: | diff --git a/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml b/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml index 5da5f947..5421b112 100644 --- a/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml +++ b/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml @@ -9,4 +9,24 @@ - "matrix_appservice_slack_control_room_id" - "matrix_appservice_slack_appservice_token" - "matrix_appservice_slack_homeserver_token" - - "matrix_appservice_slack_id_token" \ No newline at end of file + - "matrix_appservice_slack_id_token" + +- block: + - name: Check if a neDB database already exists + stat: + path: "{{ matrix_appservice_slack_data_path }}/" + register: matrix_appservice_slack_nedb_stat_result + + - name: Fail if an neDB database already exists when using Postgres + fail: + msg: >-2 + matrix_appservice_slack_database_engine has been set to `postgres` (which is our new default now). + However, we've discovered an existing neDB database in {{ matrix_appservice_slack_data_path }}/. + It appears that you've been using this bridge with the neDB engine until now. + To continue using neDB, opt into it explicitly: add `matrix_appservice_slack_database_engine: nedb` to your vars.yml file and re-run this same command. + Alternatively, to migrate your existing neDB database to Postgres: + 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) + 2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_slack_data_path }} postgres_connection_string_variable_name=matrix_appservice_slack_database_connString'`) + 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) + when: "matrix_appservice_slack_nedb_stat_result.stat.exists" + when: "matrix_appservice_slack_database_engine == 'postgres'" diff --git a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 index 6c491134..0ae13738 100644 --- a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 +++ b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 @@ -9,9 +9,11 @@ homeserver: url: "{{ matrix_appservice_slack_homeserver_url }}" server_name: "{{ matrix_domain }}" +{% if matrix_appservice_slack_database_engine == 'nedb' %} dbdir: "/data" +{% endif %} db: - engine: "postgres" - connectionString: "postgresql://matrix_appservice_slack:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_appservice_slack') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_slack" + engine: "{{ matrix_appservice_slack_database_engine }}" + connectionString: {{ matrix_appservice_slack_database_connString | to_json }} matrix_admin_room: "{{ matrix_appservice_slack_control_room_id }}" From bc376c2fb2f2da5172dfe6ef130317921a23070f Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 15:55:35 +0200 Subject: [PATCH 151/217] Add database migration utility to matrix-postgres role --- .../tasks/util/migrate_db_to_postgres.yml | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml new file mode 100644 index 00000000..ec635f99 --- /dev/null +++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml @@ -0,0 +1,111 @@ +--- + +- name: Fail if Postgres not enabled + fail: + msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot migrate." + when: "not matrix_postgres_enabled|bool" + +- name: Fail if util called incorrectly (missing matrix_postgres_db_migration_request) + fail: + msg: "The `matrix_postgres_db_migration_request` variable needs to be provided to this util." + when: "matrix_postgres_db_migration_request is not defined" + +- name: Fail if util called incorrectly (invalid matrix_postgres_db_migration_request) + fail: + msg: "The `matrix_postgres_db_migration_request` variable needs to contain `{{ item }}`." + with_items: + - src + - dst + - caller + - engine_variable_name + - systemd_services_to_stop + when: "item not in matrix_postgres_db_migration_request" + +- name: Check if the provided source database file exists + stat: + path: "{{ matrix_postgres_db_migration_request.src }}" + register: matrix_postgres_db_migration_request_src_stat_result + +- name: Fail if provided source database file doesn't exist + fail: + msg: "File cannot be found on the server at {{ matrix_postgres_db_migration_request.src }}" + when: "not matrix_postgres_db_migration_request_src_stat_result.stat.exists" + +- name: Fail if we cannot migrate on the current architecture ({{ matrix_architecture }}) + fail: + msg: >- + {{ matrix_postgres_db_migration_request.engine_variable_name }} has been set to `postgres` (which is our new default now). + However, we've discovered an existing file-based database ({{ matrix_postgres_db_migration_request.engine_old_default }}) in {{ matrix_postgres_db_migration_request.src }}. + It appears that you've been using this bridge with a file-based database engine until now. + To continue using {{ matrix_postgres_db_migration_request.engine_old_default }}, opt into it explicitly: add `{{ matrix_postgres_db_migration_request.engine_variable_name }}: {{ matrix_postgres_db_migration_request.engine_old_default }}` to your vars.yml file and re-run this same command. + We'd normally auto-migrate you to Postgres, but we can't do it on the {{ matrix_architecture }} architecture. Our pgloader container image only supports amd64 (for now). + Learn more here: https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/740 + when: "matrix_architecture != 'amd64'" + + +# Defaults + +- name: Set postgres_start_wait_time, if not provided + set_fact: + postgres_start_wait_time: 15 + when: "postgres_start_wait_time|default('') == ''" + +# Actual import work + +# matrix-postgres is most likely started already +- name: Ensure matrix-postgres is started + service: + name: matrix-postgres + state: started + daemon_reload: yes + register: matrix_postgres_service_start_result + +- name: Wait a bit, so that Postgres can start + wait_for: + timeout: "{{ postgres_start_wait_time }}" + delegate_to: 127.0.0.1 + become: false + when: "matrix_postgres_service_start_result.changed|bool" + +# We only stop services here, leaving it to the caller to start them later. +# +# We can't start them, because they probably need to be reconfigured too (changing the configuration from using SQLite to Postgres, etc.), +# before starting. +# +# Since the caller will be starting them, it might make sense to leave stopping to it as well. +# However, we don't do it, because it's simpler having it here, and it also gets to happen only if we'll be doing an import. +# If we bailed out (somewhere above), nothing would have gotten stopped. It's nice to leave this running in such cases. +- name: Ensure systemd services stopped + service: + name: "{{ item }}" + state: stopped + with_items: "{{ matrix_postgres_db_migration_request.systemd_services_to_stop }}" + +- name: Import SQLite database from {{ matrix_postgres_db_migration_request.src }} into Postgres + command: + cmd: >- + {{ matrix_host_command_docker }} run + --rm + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --network={{ matrix_docker_network }} + --mount type=bind,src={{ matrix_postgres_db_migration_request.src }},dst=/in.db,ro + --entrypoint=/bin/sh + {{ matrix_postgres_pgloader_docker_image }} + -c + 'pgloader /in.db {{ matrix_postgres_db_migration_request.dst }}' + +- name: Archive SQLite database ({{ matrix_postgres_db_migration_request.src }} -> {{ matrix_postgres_db_migration_request.src }}.backup) + command: + cmd: "mv {{ matrix_postgres_db_migration_request.src }} {{ matrix_postgres_db_migration_request.src }}.backup" + +- name: Inject result + set_fact: + matrix_playbook_runtime_results: | + {{ + matrix_playbook_runtime_results|default([]) + + + [ + "NOTE: Your {{ matrix_postgres_db_migration_request.engine_variable_name }} database file has been imported into Postgres. The original file has been moved from `{{ matrix_postgres_db_migration_request.src }}` to `{{ matrix_postgres_db_migration_request.src }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file." + ] + }} From dc7850e83cc0d23f0f5b6085e8e974f6e89f0e0e Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 16:03:03 +0200 Subject: [PATCH 152/217] Fix wording and variable names a bit --- .../tasks/util/migrate_db_to_postgres.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml index ec635f99..60539a91 100644 --- a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml +++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml @@ -35,9 +35,9 @@ fail: msg: >- {{ matrix_postgres_db_migration_request.engine_variable_name }} has been set to `postgres` (which is our new default now). - However, we've discovered an existing file-based database ({{ matrix_postgres_db_migration_request.engine_old_default }}) in {{ matrix_postgres_db_migration_request.src }}. + However, we've discovered an existing file-based database ({{ matrix_postgres_db_migration_request.engine_old }}) in {{ matrix_postgres_db_migration_request.src }}. It appears that you've been using this bridge with a file-based database engine until now. - To continue using {{ matrix_postgres_db_migration_request.engine_old_default }}, opt into it explicitly: add `{{ matrix_postgres_db_migration_request.engine_variable_name }}: {{ matrix_postgres_db_migration_request.engine_old_default }}` to your vars.yml file and re-run this same command. + To continue using {{ matrix_postgres_db_migration_request.engine_old }}, opt into it explicitly: add `{{ matrix_postgres_db_migration_request.engine_variable_name }}: {{ matrix_postgres_db_migration_request.engine_old }}` to your vars.yml file and re-run this same command. We'd normally auto-migrate you to Postgres, but we can't do it on the {{ matrix_architecture }} architecture. Our pgloader container image only supports amd64 (for now). Learn more here: https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/740 when: "matrix_architecture != 'amd64'" @@ -75,13 +75,13 @@ # Since the caller will be starting them, it might make sense to leave stopping to it as well. # However, we don't do it, because it's simpler having it here, and it also gets to happen only if we'll be doing an import. # If we bailed out (somewhere above), nothing would have gotten stopped. It's nice to leave this running in such cases. -- name: Ensure systemd services stopped +- name: Ensure systemd services blocking the database import are stopped service: name: "{{ item }}" state: stopped with_items: "{{ matrix_postgres_db_migration_request.systemd_services_to_stop }}" -- name: Import SQLite database from {{ matrix_postgres_db_migration_request.src }} into Postgres +- name: Import {{ matrix_postgres_db_migration_request.engine_old }} database from {{ matrix_postgres_db_migration_request.src }} into Postgres command: cmd: >- {{ matrix_host_command_docker }} run @@ -95,7 +95,7 @@ -c 'pgloader /in.db {{ matrix_postgres_db_migration_request.dst }}' -- name: Archive SQLite database ({{ matrix_postgres_db_migration_request.src }} -> {{ matrix_postgres_db_migration_request.src }}.backup) +- name: Archive {{ matrix_postgres_db_migration_request.engine_old }} database ({{ matrix_postgres_db_migration_request.src }} -> {{ matrix_postgres_db_migration_request.src }}.backup) command: cmd: "mv {{ matrix_postgres_db_migration_request.src }} {{ matrix_postgres_db_migration_request.src }}.backup" @@ -106,6 +106,6 @@ matrix_playbook_runtime_results|default([]) + [ - "NOTE: Your {{ matrix_postgres_db_migration_request.engine_variable_name }} database file has been imported into Postgres. The original file has been moved from `{{ matrix_postgres_db_migration_request.src }}` to `{{ matrix_postgres_db_migration_request.src }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file." + "NOTE: Your {{ matrix_postgres_db_migration_request.engine_old }} database file has been imported into Postgres. The original database file has been moved from `{{ matrix_postgres_db_migration_request.src }}` to `{{ matrix_postgres_db_migration_request.src }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file." ] }} From ac37091d01c4b38f59c1a57c7b83305bb335ae6d Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 16:03:32 +0200 Subject: [PATCH 153/217] Enable automatic (SQLite -> Postgres) migration for matrix-reminder-bot --- .../tasks/setup_install.yml | 46 ++++++++++++------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml index 4fbafa3a..fea1e00d 100644 --- a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml +++ b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml @@ -1,23 +1,29 @@ --- -- block: - - name: Check if an SQLite database already exists - stat: - path: "{{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }}" - register: matrix_bot_matrix_reminder_bot_sqlite_database_path_local_stat_result +- set_fact: + matrix_bot_matrix_reminder_bot_requires_restart: false - - name: Fail if an SQLite database already exists when using Postgres - fail: - msg: >- - matrix_bot_matrix_reminder_bot_database_engine has been set to `postgres` (which is our new default now). - However, we've discovered an existing SQLite database in {{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }}. - It appears that you've been using this bridge with the SQLite engine until now. - To continue using SQLite, opt into it explicitly: add `matrix_bot_matrix_reminder_bot_database_engine: sqlite` to your vars.yml file and re-run this same command. - Alternatively, to migrate your existing SQLite database to Postgres: - 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_bot_matrix_reminder_bot_database_connection_string'`) - 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) - when: "matrix_bot_matrix_reminder_bot_sqlite_database_path_local_stat_result.stat.exists" +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }}" + register: matrix_bot_matrix_reminder_bot_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }}" + dst: "{{ matrix_bot_matrix_reminder_bot_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_bot_matrix_reminder_bot_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-bot-matrix-reminder-bot.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_bot_matrix_reminder_bot_requires_restart: true + when: "matrix_bot_matrix_reminder_bot_sqlite_database_path_local_stat_result.stat.exists|bool" when: "matrix_bot_matrix_reminder_bot_database_engine == 'postgres'" - name: Ensure matrix-reminder-bot paths exist @@ -59,3 +65,9 @@ service: daemon_reload: yes when: "matrix_bot_matrix_reminder_bot_systemd_service_result.changed|bool" + +- name: Ensure matrix-bot-matrix-reminder-bot.service restarted, if necessary + service: + name: "matrix-bot-matrix-reminder-bot.service" + state: restarted + when: "matrix_bot_matrix_reminder_bot_requires_restart|bool" From c1431b28f0384002843b5a059f4fe11caa3d8a57 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 16:13:57 +0200 Subject: [PATCH 154/217] Make use of matrix_postgres_db_migration_request.caller --- roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml index 60539a91..b77ce4b0 100644 --- a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml +++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml @@ -34,7 +34,7 @@ - name: Fail if we cannot migrate on the current architecture ({{ matrix_architecture }}) fail: msg: >- - {{ matrix_postgres_db_migration_request.engine_variable_name }} has been set to `postgres` (which is our new default now). + {{ matrix_postgres_db_migration_request.engine_variable_name }} (part of {{ matrix_postgres_db_migration_request.caller }}) has been set to `postgres` (which is our new default now). However, we've discovered an existing file-based database ({{ matrix_postgres_db_migration_request.engine_old }}) in {{ matrix_postgres_db_migration_request.src }}. It appears that you've been using this bridge with a file-based database engine until now. To continue using {{ matrix_postgres_db_migration_request.engine_old }}, opt into it explicitly: add `{{ matrix_postgres_db_migration_request.engine_variable_name }}: {{ matrix_postgres_db_migration_request.engine_old }}` to your vars.yml file and re-run this same command. From bbc08722c5ba4c02e39f747e0631696022316321 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 16:14:23 +0200 Subject: [PATCH 155/217] Enable automatic (SQLite -> Postgres) migration for matrix-mautrix-facebook --- .../tasks/setup_install.yml | 46 ++++++++++++------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml index 1a0ff70c..7f310446 100644 --- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml @@ -8,24 +8,30 @@ The matrix-bridge-mautrix-facebook role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" -- block: - - name: Check if an SQLite database already exists - stat: - path: "{{ matrix_mautrix_facebook_sqlite_database_path_local }}" - register: matrix_mautrix_facebook_sqlite_database_path_local_stat_result +- set_fact: + matrix_mautrix_facebook_requires_restart: false - - name: Fail if an SQLite database already exists when using Postgres - fail: - msg: >- - matrix_mautrix_facebook_database_engine has been set to `postgres` (which is our new default now). - However, we've discovered an existing SQLite database in {{ matrix_mautrix_facebook_sqlite_database_path_local }}. - It appears that you've been using this bridge with the SQLite engine until now. - To continue using SQLite, opt into it explicitly: add `matrix_mautrix_facebook_database_engine: sqlite` to your vars.yml file and re-run this same command. - Alternatively, to migrate your existing SQLite database to Postgres: - 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_facebook_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_facebook_database_connection_string'`) - 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) - when: "matrix_mautrix_facebook_sqlite_database_path_local_stat_result.stat.exists" +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mautrix_facebook_sqlite_database_path_local }}" + register: matrix_mautrix_facebook_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_mautrix_facebook_sqlite_database_path_local }}" + dst: "{{ matrix_mautrix_facebook_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_mautrix_facebook_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-mautrix-facebook.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_mautrix_facebook_requires_restart: true + when: "matrix_mautrix_facebook_sqlite_database_path_local_stat_result.stat.exists|bool" when: "matrix_mautrix_facebook_database_engine == 'postgres'" - name: Ensure Mautrix Facebook image is pulled @@ -114,3 +120,9 @@ service: daemon_reload: yes when: "matrix_mautrix_facebook_systemd_service_result.changed" + +- name: Ensure matrix-mautrix-facebook.service restarted, if necessary + service: + name: "matrix-mautrix-facebook.service" + state: restarted + when: "matrix_mautrix_facebook_requires_restart|bool" From 3ba852026663407c0db3707ee9a659e850186728 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 16:18:38 +0200 Subject: [PATCH 156/217] Enable automatic (SQLite -> Postgres) migration for matrix-mautrix-hangouts --- .../tasks/setup_install.yml | 46 ++++++++++++------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml index 02c34357..15af9626 100644 --- a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml @@ -8,24 +8,30 @@ The matrix-bridge-mautrix-hangouts role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" -- block: - - name: Check if an SQLite database already exists - stat: - path: "{{ matrix_mautrix_hangouts_sqlite_database_path_local }}" - register: matrix_mautrix_hangouts_sqlite_database_path_local_stat_result +- set_fact: + matrix_mautrix_hangouts_requires_restart: false - - name: Fail if an SQLite database already exists when using Postgres - fail: - msg: >- - matrix_mautrix_hangouts_database_engine has been set to `postgres` (which is our new default now). - However, we've discovered an existing SQLite database in {{ matrix_mautrix_hangouts_sqlite_database_path_local }}. - It appears that you've been using this bridge with the SQLite engine until now. - To continue using SQLite, opt into it explicitly: add `matrix_mautrix_hangouts_database_engine: sqlite` to your vars.yml file and re-run this same command. - Alternatively, to migrate your existing SQLite database to Postgres: - 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_hangouts_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_hangouts_database_connection_string'`) - 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) - when: "matrix_mautrix_hangouts_sqlite_database_path_local_stat_result.stat.exists" +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mautrix_hangouts_sqlite_database_path_local }}" + register: matrix_mautrix_hangouts_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_mautrix_hangouts_sqlite_database_path_local }}" + dst: "{{ matrix_mautrix_hangouts_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_mautrix_hangouts_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-mautrix-hangouts.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_mautrix_hangouts_requires_restart: true + when: "matrix_mautrix_hangouts_sqlite_database_path_local_stat_result.stat.exists|bool" when: "matrix_mautrix_hangouts_database_engine == 'postgres'" - name: Ensure Mautrix Hangouts image is pulled @@ -113,3 +119,9 @@ service: daemon_reload: yes when: "matrix_mautrix_hangouts_systemd_service_result.changed" + +- name: Ensure matrix-mautrix-hangouts.service restarted, if necessary + service: + name: "matrix-mautrix-hangouts.service" + state: restarted + when: "matrix_mautrix_hangouts_requires_restart|bool" From 86a8091768bbe8871212761c99de5aa734462039 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 16:19:54 +0200 Subject: [PATCH 157/217] Enable automatic (SQLite -> Postgres) migration for matrix-mautrix-telegram --- .../tasks/setup_install.yml | 46 ++++++++++++------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml index d898c590..6a37974a 100644 --- a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml @@ -8,24 +8,30 @@ The matrix-bridge-mautrix-telegram role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" -- block: - - name: Check if an SQLite database already exists - stat: - path: "{{ matrix_mautrix_telegram_sqlite_database_path_local }}" - register: matrix_mautrix_telegram_sqlite_database_path_local_stat_result +- set_fact: + matrix_mautrix_telegram_requires_restart: false - - name: Fail if an SQLite database already exists when using Postgres - fail: - msg: >- - matrix_mautrix_telegram_database_engine has been set to `postgres` (which is our new default now). - However, we've discovered an existing SQLite database in {{ matrix_mautrix_telegram_sqlite_database_path_local }}. - It appears that you've been using this bridge with the SQLite engine until now. - To continue using SQLite, opt into it explicitly: add `matrix_mautrix_telegram_database_engine: sqlite` to your vars.yml file and re-run this same command. - Alternatively, to migrate your existing SQLite database to Postgres: - 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_telegram_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_telegram_database_connection_string'`) - 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) - when: "matrix_mautrix_telegram_sqlite_database_path_local_stat_result.stat.exists" +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mautrix_telegram_sqlite_database_path_local }}" + register: matrix_mautrix_telegram_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_mautrix_telegram_sqlite_database_path_local }}" + dst: "{{ matrix_mautrix_telegram_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_mautrix_telegram_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-mautrix-telegram.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_mautrix_telegram_requires_restart: true + when: "matrix_mautrix_telegram_sqlite_database_path_local_stat_result.stat.exists|bool" when: "matrix_mautrix_telegram_database_engine == 'postgres'" - name: Ensure Mautrix Telegram image is pulled @@ -91,3 +97,9 @@ service: daemon_reload: yes when: "matrix_mautrix_telegram_systemd_service_result.changed" + +- name: Ensure matrix-mautrix-telegram.service restarted, if necessary + service: + name: "matrix-mautrix-telegram.service" + state: restarted + when: "matrix_mautrix_telegram_requires_restart|bool" From 13f84e2ad588668f26010e8f8de07f35c6527e97 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 16:21:01 +0200 Subject: [PATCH 158/217] Enable automatic (SQLite -> Postgres) migration for matrix-mautrix-whatsapp --- .../tasks/setup_install.yml | 46 ++++++++++++------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml index 9521e0a5..8d894a84 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml @@ -8,24 +8,30 @@ The matrix-bridge-mautrix-whatsapp role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" -- block: - - name: Check if an SQLite database already exists - stat: - path: "{{ matrix_mautrix_whatsapp_sqlite_database_path_local }}" - register: matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result +- set_fact: + matrix_mautrix_whatsapp_requires_restart: false - - name: Fail if an SQLite database already exists when using Postgres - fail: - msg: >- - matrix_mautrix_whatsapp_database_engine has been set to `postgres` (which is our new default now). - However, we've discovered an existing SQLite database in {{ matrix_mautrix_whatsapp_sqlite_database_path_local }}. - It appears that you've been using this bridge with the SQLite engine until now. - To continue using SQLite, opt into it explicitly: add `matrix_mautrix_whatsapp_database_engine: sqlite` to your vars.yml file and re-run this same command. - Alternatively, to migrate your existing SQLite database to Postgres: - 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_whatsapp_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_whatsapp_database_connection_string'`) - 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) - when: "matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result.stat.exists" +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mautrix_whatsapp_sqlite_database_path_local }}" + register: matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_mautrix_whatsapp_sqlite_database_path_local }}" + dst: "{{ matrix_mautrix_whatsapp_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_mautrix_whatsapp_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-mautrix-whatsapp.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_mautrix_whatsapp_requires_restart: true + when: "matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result.stat.exists|bool" when: "matrix_mautrix_whatsapp_database_engine == 'postgres'" - name: Ensure Mautrix Whatsapp image is pulled @@ -100,3 +106,9 @@ service: daemon_reload: yes when: "matrix_mautrix_whatsapp_systemd_service_result.changed" + +- name: Ensure matrix-mautrix-whatsapp.service restarted, if necessary + service: + name: "matrix-mautrix-whatsapp.service" + state: restarted + when: "matrix_mautrix_whatsapp_requires_restart|bool" From cba973d6b555b9288e8791eecb601c4e80aba4f6 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 16:24:56 +0200 Subject: [PATCH 159/217] Enable automatic (SQLite -> Postgres) migration for matrix-appservice-discord --- .../tasks/setup_install.yml | 46 ++++++++++++------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml index a9d8da4c..8bf50e28 100644 --- a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml @@ -1,23 +1,29 @@ --- -- block: - - name: Check if an SQLite database already exists - stat: - path: "{{ matrix_appservice_discord_sqlite_database_path_local }}" - register: matrix_appservice_discord_sqlite_database_path_local_stat_result +- set_fact: + matrix_appservice_discord_requires_restart: false - - name: Fail if an SQLite database already exists when using Postgres - fail: - msg: >- - matrix_appservice_discord_database_engine has been set to `postgres` (which is our new default now). - However, we've discovered an existing SQLite database in {{ matrix_appservice_discord_sqlite_database_path_local }}. - It appears that you've been using this bridge with the SQLite engine until now. - To continue using SQLite, opt into it explicitly: add `matrix_appservice_discord_database_engine: sqlite` to your vars.yml file and re-run this same command. - Alternatively, to migrate your existing SQLite database to Postgres: - 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_discord_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_appservice_discord_database_connString'`) - 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) - when: "matrix_appservice_discord_sqlite_database_path_local_stat_result.stat.exists" +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_appservice_discord_sqlite_database_path_local }}" + register: matrix_appservice_discord_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_appservice_discord_sqlite_database_path_local }}" + dst: "{{ matrix_appservice_discord_database_connString }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_appservice_discord_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-appservice-discord.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_appservice_discord_requires_restart: true + when: "matrix_appservice_discord_sqlite_database_path_local_stat_result.stat.exists|bool" when: "matrix_appservice_discord_database_engine == 'postgres'" - name: Ensure Appservice Discord image is pulled @@ -100,3 +106,9 @@ service: daemon_reload: yes when: "matrix_appservice_discord_systemd_service_result.changed" + +- name: Ensure matrix-appservice-discord.service restarted, if necessary + service: + name: "matrix-appservice-discord.service" + state: restarted + when: "matrix_appservice_discord_requires_restart|bool" From 7248eb3c11d77303dbc395f03c2aeb8ed85e4639 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 16:25:44 +0200 Subject: [PATCH 160/217] Fix syntax error in roles/matrix-bridge-appservice-irc/defaults/main.yml --- roles/matrix-bridge-appservice-irc/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-appservice-irc/defaults/main.yml b/roles/matrix-bridge-appservice-irc/defaults/main.yml index a1c76253..bf23b0ab 100644 --- a/roles/matrix-bridge-appservice-irc/defaults/main.yml +++ b/roles/matrix-bridge-appservice-irc/defaults/main.yml @@ -22,7 +22,7 @@ matrix_appservice_irc_database_password: ~ matrix_appservice_irc_database_hostname: 'matrix-postgres' matrix_appservice_irc_database_port: 5432 matrix_appservice_irc_database_name: matrix_appservice_irc -matrix_appservice_irc_databasa + matrix_appservice_irc_database_connString: >-2 {%- if matrix_appservice_irc_database_engine == 'postgres' -%} postgresql://{{ matrix_appservice_irc_database_username }}:{{ matrix_appservice_irc_database_password }}@{{ matrix_appservice_irc_database_hostname }}:{{ matrix_appservice_irc_database_port }}/{{ matrix_appservice_irc_database_name }}?sslmode=disable From 0ca48f3532d211df9e44c67d9661a95420701660 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 16:57:51 +0200 Subject: [PATCH 161/217] Separate matrix-ma1sd install/uninstall tasks --- roles/matrix-ma1sd/tasks/main.yml | 9 ++- .../{setup_ma1sd.yml => setup_install.yml} | 57 ++----------------- roles/matrix-ma1sd/tasks/setup_uninstall.yml | 35 ++++++++++++ 3 files changed, 48 insertions(+), 53 deletions(-) rename roles/matrix-ma1sd/tasks/{setup_ma1sd.yml => setup_install.yml} (69%) create mode 100644 roles/matrix-ma1sd/tasks/setup_uninstall.yml diff --git a/roles/matrix-ma1sd/tasks/main.yml b/roles/matrix-ma1sd/tasks/main.yml index f5ac34d6..0b8a114e 100644 --- a/roles/matrix-ma1sd/tasks/main.yml +++ b/roles/matrix-ma1sd/tasks/main.yml @@ -8,7 +8,14 @@ - setup-all - setup-ma1sd -- import_tasks: "{{ role_path }}/tasks/setup_ma1sd.yml" +- import_tasks: "{{ role_path }}/tasks/setup_install.yml" + when: "run_setup|bool and matrix_ma1sd_enabled|bool" + tags: + - setup-all + - setup-ma1sd + +- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml" + when: "run_setup|bool and not matrix_ma1sd_enabled|bool" tags: - setup-all - setup-ma1sd diff --git a/roles/matrix-ma1sd/tasks/setup_ma1sd.yml b/roles/matrix-ma1sd/tasks/setup_install.yml similarity index 69% rename from roles/matrix-ma1sd/tasks/setup_ma1sd.yml rename to roles/matrix-ma1sd/tasks/setup_install.yml index 46acb428..aa11ae7a 100644 --- a/roles/matrix-ma1sd/tasks/setup_ma1sd.yml +++ b/roles/matrix-ma1sd/tasks/setup_install.yml @@ -1,9 +1,5 @@ --- -# -# Tasks related to setting up ma1sd -# - - name: Ensure ma1sd paths exist file: path: "{{ item.path }}" @@ -15,10 +11,9 @@ - { path: "{{ matrix_ma1sd_config_path }}", when: true } - { path: "{{ matrix_ma1sd_data_path }}", when: true } - { path: "{{ matrix_ma1sd_docker_src_files_path }}", when: "{{ matrix_ma1sd_container_image_self_build }}"} - when: matrix_ma1sd_enabled|bool and item.when + when: "item.when|bool" - import_tasks: "{{ role_path }}/tasks/migrate_mxisd.yml" - when: matrix_ma1sd_enabled|bool - name: Ensure ma1sd image is pulled docker_image: @@ -26,7 +21,7 @@ source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_ma1sd_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_ma1sd_docker_image_force_pull }}" - when: matrix_ma1sd_enabled|bool and not matrix_ma1sd_container_image_self_build + when: "not matrix_ma1sd_container_image_self_build|bool" - block: - name: Ensure gradle is installed for self-building (Debian) @@ -72,7 +67,7 @@ repository: "{{ matrix_ma1sd_docker_image }}" force_tag: yes source: local - when: "matrix_ma1sd_enabled|bool and matrix_ma1sd_container_image_self_build" + when: "matrix_ma1sd_container_image_self_build|bool" - name: Ensure ma1sd config installed copy: @@ -81,7 +76,6 @@ mode: 0644 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" - when: matrix_ma1sd_enabled|bool - name: Ensure custom templates are installed if any copy: @@ -95,7 +89,7 @@ - {value: "{{ matrix_ma1sd_threepid_medium_email_custom_session_validation_template }}", location: 'validate-template.eml'} - {value: "{{ matrix_ma1sd_threepid_medium_email_custom_unbind_fraudulent_template }}", location: 'unbind-fraudulent.eml'} - {value: "{{ matrix_ma1sd_threepid_medium_email_custom_matrixid_template }}", location: 'mxid-template.eml'} - when: "matrix_ma1sd_enabled|bool and matrix_ma1sd_threepid_medium_email_custom_templates_enabled|bool and item.value" + when: "matrix_ma1sd_threepid_medium_email_custom_templates_enabled|bool and item.value" - name: Ensure matrix-ma1sd.service installed template: @@ -103,49 +97,8 @@ dest: "{{ matrix_systemd_path }}/matrix-ma1sd.service" mode: 0644 register: matrix_ma1sd_systemd_service_result - when: matrix_ma1sd_enabled|bool - name: Ensure systemd reloaded after matrix-ma1sd.service installation service: daemon_reload: yes - when: "matrix_ma1sd_enabled|bool and matrix_ma1sd_systemd_service_result.changed" - -# -# Tasks related to getting rid of ma1sd (if it was previously enabled) -# - -- name: Check existence of matrix-ma1sd service - stat: - path: "{{ matrix_systemd_path }}/matrix-ma1sd.service" - register: matrix_ma1sd_service_stat - -- name: Ensure matrix-ma1sd is stopped - service: - name: matrix-ma1sd - state: stopped - daemon_reload: yes - register: stopping_result - when: "not matrix_ma1sd_enabled|bool and matrix_ma1sd_service_stat.stat.exists" - -- name: Ensure matrix-ma1sd.service doesn't exist - file: - path: "{{ matrix_systemd_path }}/matrix-ma1sd.service" - state: absent - when: "not matrix_ma1sd_enabled|bool and matrix_ma1sd_service_stat.stat.exists" - -- name: Ensure systemd reloaded after matrix-ma1sd.service removal - service: - daemon_reload: yes - when: "not matrix_ma1sd_enabled|bool and matrix_ma1sd_service_stat.stat.exists" - -- name: Ensure Matrix ma1sd paths don't exist - file: - path: "{{ matrix_ma1sd_base_path }}" - state: absent - when: "not matrix_ma1sd_enabled|bool" - -- name: Ensure ma1sd Docker image doesn't exist - docker_image: - name: "{{ matrix_ma1sd_docker_image }}" - state: absent - when: "not matrix_ma1sd_enabled|bool" + when: "matrix_ma1sd_systemd_service_result.changed|bool" diff --git a/roles/matrix-ma1sd/tasks/setup_uninstall.yml b/roles/matrix-ma1sd/tasks/setup_uninstall.yml new file mode 100644 index 00000000..b36ab508 --- /dev/null +++ b/roles/matrix-ma1sd/tasks/setup_uninstall.yml @@ -0,0 +1,35 @@ +--- + +- name: Check existence of matrix-ma1sd service + stat: + path: "{{ matrix_systemd_path }}/matrix-ma1sd.service" + register: matrix_ma1sd_service_stat + +- name: Ensure matrix-ma1sd is stopped + service: + name: matrix-ma1sd + state: stopped + daemon_reload: yes + register: stopping_result + when: "matrix_ma1sd_service_stat.stat.exists|bool" + +- name: Ensure matrix-ma1sd.service doesn't exist + file: + path: "{{ matrix_systemd_path }}/matrix-ma1sd.service" + state: absent + when: "matrix_ma1sd_service_stat.stat.exists|bool" + +- name: Ensure systemd reloaded after matrix-ma1sd.service removal + service: + daemon_reload: yes + when: "matrix_ma1sd_service_stat.stat.exists|bool" + +- name: Ensure Matrix ma1sd paths don't exist + file: + path: "{{ matrix_ma1sd_base_path }}" + state: absent + +- name: Ensure ma1sd Docker image doesn't exist + docker_image: + name: "{{ matrix_ma1sd_docker_image }}" + state: absent From af3ea67bbaa699ad6f03db6633f1dd1902ab9290 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 17:16:25 +0200 Subject: [PATCH 162/217] Add (SQLite + Postgres) support and automatic migration to matrix-ma1sd --- group_vars/matrix_servers | 10 ++++++ roles/matrix-ma1sd/defaults/main.yml | 22 ++++++++++++ roles/matrix-ma1sd/tasks/setup_install.yml | 36 +++++++++++++++++++ roles/matrix-ma1sd/templates/ma1sd.yaml.j2 | 16 +++++++-- .../tasks/util/migrate_db_to_postgres.yml | 2 +- 5 files changed, 82 insertions(+), 4 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index d6d6cc4a..735d1324 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -835,6 +835,10 @@ matrix_ma1sd_systemd_wanted_services_list: | (['matrix-mailer.service'] if matrix_mailer_enabled else []) }} +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_ma1sd_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_ma1sd_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'ma1sd.db') | to_uuid }}" + ###################################################################### # # /matrix-ma1sd @@ -953,6 +957,12 @@ matrix_postgres_db_name: "homeserver" matrix_postgres_additional_databases: | {{ + ([{ + 'name': matrix_ma1sd_database_db_name, + 'username': matrix_ma1sd_database_username, + 'password': matrix_ma1sd_database_password, + }] if (matrix_ma1sd_enabled and matrix_ma1sd_database_engine == 'postgres' and matrix_ma1sd_database_hostname == 'matrix-postgres') else []) + + ([{ 'name': matrix_bot_matrix_reminder_bot_database_db_name, 'username': matrix_bot_matrix_reminder_bot_database_username, diff --git a/roles/matrix-ma1sd/defaults/main.yml b/roles/matrix-ma1sd/defaults/main.yml index f7a0782d..42e36e92 100644 --- a/roles/matrix-ma1sd/defaults/main.yml +++ b/roles/matrix-ma1sd/defaults/main.yml @@ -39,6 +39,28 @@ matrix_ma1sd_systemd_wanted_services_list: [] # Enabling this is discouraged. Learn more here: https://github.com/ma1uta/ma1sd/blob/master/docs/features/identity.md#lookups matrix_ma1sd_matrixorg_forwarding_enabled: false + +# Database-related configuration fields. +# +# To use SQLite, stick to these defaults. +# +# To use Postgres: +# - change the engine (`matrix_ma1sd_database_engine: 'postgres'`) +# - adjust your database credentials via the `matrix_ma1sd_postgres_*` variables +matrix_ma1sd_database_engine: 'sqlite' + +matrix_ma1sd_sqlite_database_path_local: "{{ matrix_ma1sd_data_path }}/ma1sd.db" +matrix_ma1sd_sqlite_database_path_in_container: "/var/ma1sd/ma1sd.db" + +matrix_ma1sd_database_username: 'matrix_ma1sd' +matrix_ma1sd_database_password: 'some-password' +matrix_ma1sd_database_hostname: 'matrix-postgres' +matrix_ma1sd_database_port: 5432 +matrix_ma1sd_database_db_name: 'matrix_ma1sd' + +matrix_ma1sd_database_connection_string: 'postgresql://{{ matrix_ma1sd_database_username }}:{{ matrix_ma1sd_database_password }}@{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_db_name }}' + + # ma1sd has serveral supported identity stores. # One of them is storing identities directly in Synapse's database. # Learn more here: https://github.com/ma1uta/ma1sd/blob/master/docs/stores/synapse.md diff --git a/roles/matrix-ma1sd/tasks/setup_install.yml b/roles/matrix-ma1sd/tasks/setup_install.yml index aa11ae7a..9ae5f077 100644 --- a/roles/matrix-ma1sd/tasks/setup_install.yml +++ b/roles/matrix-ma1sd/tasks/setup_install.yml @@ -15,6 +15,36 @@ - import_tasks: "{{ role_path }}/tasks/migrate_mxisd.yml" + +# These (SQLite -> Postgres) migration tasks are usually at the top, +# but we'd like to run them after `migrate_mxisd.yml`, which requires the ma1sd paths to exist. +- set_fact: + matrix_ma1sd_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_ma1sd_sqlite_database_path_local }}" + register: matrix_ma1sd_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_ma1sd_sqlite_database_path_local }}" + dst: "{{ matrix_ma1sd_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_ma1sd_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-ma1sd.service'] + pgloader_options: ['--with "quote identifiers"'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_ma1sd_requires_restart: true + when: "matrix_ma1sd_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_ma1sd_database_engine == 'postgres'" + - name: Ensure ma1sd image is pulled docker_image: name: "{{ matrix_ma1sd_docker_image }}" @@ -102,3 +132,9 @@ service: daemon_reload: yes when: "matrix_ma1sd_systemd_service_result.changed|bool" + +- name: Ensure matrix-ma1sd.service restarted, if necessary + service: + name: "matrix-ma1sd.service" + state: restarted + when: "matrix_ma1sd_requires_restart|bool" diff --git a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 index 84585707..43b4022d 100644 --- a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 +++ b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 @@ -11,9 +11,19 @@ key: path: /var/ma1sd/sign.key storage: - provider: - sqlite: - database: /var/ma1sd/ma1sd.db + {% if matrix_ma1sd_database_engine == 'sqlite' %} + backend: sqlite + provider: + sqlite: + database: {{ matrix_ma1sd_sqlite_database_path_in_container|to_json }} + {% elif matrix_ma1sd_database_engine == 'postgres' %} + backend: postgresql + provider: + postgresql: + database: //{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_db_name }} + username: {{ matrix_ma1sd_database_username|to_json }} + password: {{ matrix_ma1sd_database_password|to_json }} + {% endif %} {% if matrix_ma1sd_dns_overwrite_enabled %} dns: diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml index b77ce4b0..5d917538 100644 --- a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml +++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml @@ -93,7 +93,7 @@ --entrypoint=/bin/sh {{ matrix_postgres_pgloader_docker_image }} -c - 'pgloader /in.db {{ matrix_postgres_db_migration_request.dst }}' + 'pgloader {{ matrix_postgres_db_migration_request.pgloader_options|default([])|join(' ') }} /in.db {{ matrix_postgres_db_migration_request.dst }}' - name: Archive {{ matrix_postgres_db_migration_request.engine_old }} database ({{ matrix_postgres_db_migration_request.src }} -> {{ matrix_postgres_db_migration_request.src }}.backup) command: From 3ab7dd1abe07574085da2705b3f79ca6773c4f29 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 17:22:31 +0200 Subject: [PATCH 163/217] Potentially add matrix-postgres.service as required for bridges, etc. --- group_vars/matrix_servers | 41 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 735d1324..1223d8a6 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -61,6 +61,8 @@ matrix_appservice_discord_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} matrix_appservice_discord_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'discord.as.token') | to_uuid }}" @@ -207,6 +209,8 @@ matrix_mautrix_facebook_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} matrix_mautrix_facebook_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'fb.as.token') | to_uuid }}" @@ -244,6 +248,8 @@ matrix_mautrix_hangouts_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} matrix_mautrix_hangouts_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'ho.as.token') | to_uuid }}" @@ -282,6 +288,8 @@ matrix_mautrix_telegram_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} matrix_mautrix_telegram_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'telegr.as.token') | to_uuid }}" @@ -318,6 +326,8 @@ matrix_mautrix_whatsapp_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} matrix_mautrix_whatsapp_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'whats.as.token') | to_uuid }}" @@ -378,6 +388,8 @@ matrix_mx_puppet_skype_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} matrix_mx_puppet_skype_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'skype.as.tok') | to_uuid }}" @@ -413,6 +425,8 @@ matrix_mx_puppet_slack_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} matrix_mx_puppet_slack_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxslk.as.tok') | to_uuid }}" @@ -447,6 +461,8 @@ matrix_mx_puppet_twitter_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} matrix_mx_puppet_twitter_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxtwt.as.tok') | to_uuid }}" @@ -484,6 +500,8 @@ matrix_mx_puppet_instagram_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} matrix_mx_puppet_instagram_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxig.as.tok') | to_uuid }}" @@ -518,6 +536,8 @@ matrix_mx_puppet_discord_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} matrix_mx_puppet_discord_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxdsc.as.tok') | to_uuid }}" @@ -552,6 +572,8 @@ matrix_mx_puppet_steam_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} matrix_mx_puppet_steam_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxste.as.tok') | to_uuid }}" @@ -580,6 +602,13 @@ matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key # We don't enable bots by default. matrix_bot_matrix_reminder_bot_enabled: false +matrix_bot_matrix_reminder_bot_systemd_required_services_list: | + {{ + ['docker.service'] + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + }} + # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_bot_matrix_reminder_bot_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" matrix_bot_matrix_reminder_bot_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'reminder.bot.db') | to_uuid }}" @@ -678,6 +707,13 @@ matrix_dimension_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_ena matrix_integration_manager_rest_url: "{{ matrix_dimension_integrations_rest_url if matrix_dimension_enabled else None }}" matrix_integration_manager_ui_url: "{{ matrix_dimension_integrations_ui_url if matrix_dimension_enabled else None }}" +matrix_dimension_systemd_required_services_list: | + {{ + ['docker.service'] + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + }} + # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_dimension_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" matrix_dimension_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.dimension.db') | to_uuid }}" @@ -826,6 +862,11 @@ matrix_ma1sd_threepid_medium_email_connectors_smtp_tls: 0 matrix_ma1sd_self_check_validate_certificates: "{{ false if matrix_ssl_retrieval_method == 'self-signed' else true }}" +matrix_ma1sd_systemd_required_services_list: | + {{ + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + }} + matrix_ma1sd_systemd_wanted_services_list: | {{ (['matrix-corporal.service'] if matrix_corporal_enabled else ['matrix-synapse.service']) From 13d8a9b39c1a5c5a225ad2c82d1a876210e341c0 Mon Sep 17 00:00:00 2001 From: transcaffeine Date: Mon, 14 Dec 2020 16:27:06 +0100 Subject: [PATCH 164/217] hint supported automatic migration nedb->postgres --- .../tasks/validate_config.yml | 2 +- roles/matrix-postgres/tasks/import_nedb.yml | 117 ++++++++++++++++++ roles/matrix-postgres/tasks/main.yml | 16 +++ 3 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 roles/matrix-postgres/tasks/import_nedb.yml diff --git a/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml b/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml index e0e932bc..9b89a340 100644 --- a/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml +++ b/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml @@ -49,7 +49,7 @@ To continue using neDB, opt into it explicitly: add `matrix_appservice_irc_database_engine: nedb` to your vars.yml file and re-run this same command. Alternatively, to migrate your existing neDB database to Postgres: 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_irc_data_path }} postgres_connection_string_variable_name=matrix_appservice_irc_database_connString'`) + 2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-irc-nedb --extra-vars='nedb_database_path={{ matrix_appservice_irc_data_path }} postgres_connection_string_variable_name=matrix_appservice_irc_database_connString'`) 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) when: "matrix_appservice_irc_nedb_stat_result.stat.exists" when: "matrix_appservice_irc_database_engine == 'postgres'" diff --git a/roles/matrix-postgres/tasks/import_nedb.yml b/roles/matrix-postgres/tasks/import_nedb.yml new file mode 100644 index 00000000..cc1f9d78 --- /dev/null +++ b/roles/matrix-postgres/tasks/import_nedb.yml @@ -0,0 +1,117 @@ +--- + +# Pre-checks + +- name: Fail if Postgres not enabled + fail: + msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot import." + when: "not matrix_postgres_enabled|bool" + +- name: Fail if playbook called incorrectly + fail: + msg: "The `nedb_database_path` variable needs to be provided to this playbook, via --extra-vars" + when: "nedb_database_path is not defined or nedb_database_path.startswith('<')" + +- name: Check if the provided nedb database file exists + stat: + path: "{{ nedb_database_path }}" + register: nedb_database_path_stat_result + +- name: Fail if provided SQLite database file doesn't exist + fail: + msg: "File cannot be found on the server at {{ nedb_database_path }}" + when: "not nedb_database_path_stat_result.stat.exists" + +# We either expect `postgres_db_connection_string` specifying a full Postgres database connection string, +# or `postgres_connection_string_variable_name`, specifying a name of a variable, which contains a valid connection string. + +- block: + - name: Fail if postgres_connection_string_variable_name points to an undefined variable + fail: msg="postgres_connection_string_variable_name is defined, but there is no variable with the name `{{ postgres_connection_string_variable_name }}`" + when: "postgres_connection_string_variable_name not in vars" + + - name: Get Postgres connection string from variable + set_fact: + postgres_db_connection_string: "{{ lookup('vars', postgres_connection_string_variable_name) }}" + when: 'postgres_connection_string_variable_name is defined' + +- name: Fail if playbook called incorrectly + fail: + msg: >- + Either a `postgres_db_connection_string` variable or a `postgres_connection_string_variable_name` needs to be provided to this playbook, via `--extra-vars`. + Example: `--extra-vars="postgres_db_connection_string=postgresql://username:password@localhost:/database_name"` or `--extra-vars="postgres_connection_string_variable_name=matrix_appservice_discord_database_connString"` + when: "postgres_db_connection_string is not defined or not postgres_db_connection_string.startswith('postgresql://')" + + +# Defaults + +- name: Set postgres_start_wait_time, if not provided + set_fact: + postgres_start_wait_time: 15 + when: "postgres_start_wait_time|default('') == ''" + + +# Actual import work + +- name: Ensure matrix-postgres is started + service: + name: matrix-postgres + state: started + daemon_reload: yes + register: matrix_postgres_service_start_result + +- name: Wait a bit, so that Postgres can start + wait_for: + timeout: "{{ postgres_start_wait_time }}" + delegate_to: 127.0.0.1 + become: false + when: "matrix_postgres_service_start_result.changed|bool" + +# See https://github.com/matrix-org/matrix-appservice-irc/wiki/Migrating-from-NEdB-to-PostgreSQL +- name: Import appservice_irc NeDB database from {{ sqlite_database_path }} into Postgres + when: database == 'appservice_irc' + command: + cmd: >- + {{ matrix_host_command_docker }} run + --rm + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --network={{ matrix_docker_network }} + --mount type=bind,src={{ matrix_appservice_irc_data_path }}:/data:ro + --entrypoint=/bin/sh + {{ matrix_appservice_irc_docker_image }} + -c + './scripts/migrate-db-to-pgres.sh -d /data -p passkey.pem -c {{ postgres_db_connection_string }}' + +# No migration.sh available, but found this: +# https://github.com/matrix-org/matrix-appservice-slack/blob/develop/src/scripts/migrateToPostgres.ts +# Usage should be similar to appservice_irc +- name: Import appservice_slack NeDB database from {{ sqlite_database_path }} into Postgres + when: database == 'appservice_slack' + command: + cmd: >- + {{ matrix_host_command_docker }} run + --rm + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --network={{ matrix_docker_network }} + --mount type=bind,src={{ matrix_appservice_irc_data_path }}:/data:ro + --entrypoint=/bin/sh + {{ matrix_appservice_slack_docker_image }} + -c + 'node /lib/scripts/migrate-db-to-pgres.js -d /data -p passkey.pem -c {{ postgres_db_connection_string }}' + +- name: Archive NeDB database ({{ sqlite_database_path }} -> {{ sqlite_database_path }}.backup) + command: + cmd: "mv {{ sqlite_database_path }} {{ sqlite_database_path }}.backup" + +- name: Inject result + set_fact: + matrix_playbook_runtime_results: | + {{ + matrix_playbook_runtime_results|default([]) + + + [ + "NOTE: Your NeDB database file has been imported into Postgres. The original directory has been moved from `{{ nedb_database_path }}` to `{{ nedb_database_path }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file." + ] + }} diff --git a/roles/matrix-postgres/tasks/main.yml b/roles/matrix-postgres/tasks/main.yml index b9c2ae7c..51801314 100644 --- a/roles/matrix-postgres/tasks/main.yml +++ b/roles/matrix-postgres/tasks/main.yml @@ -32,6 +32,22 @@ tags: - import-generic-sqlite-db +# Imports appservice-irc NeDB into postgres +- import_tasks: "{{ role_path }}/tasks/import_nedb.yml" + vars: + database: appservice_irc + when: run_postgres_import_nedb|bool + tags: + - import-irc-nedb + +# Imports slacks neDB to postgres. +- import_tasks: "{{ role_path }}/tasks/import_nedb.yml" + vars: + database: appservice_slack + when: run_postgres_import_nedb|bool + tags: + - import-slack-nedb + - import_tasks: "{{ role_path }}/tasks/upgrade_postgres.yml" when: run_postgres_upgrade|bool tags: From 516ccb2b2b3eee4d110a6056302d80dcf902f03a Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 18:12:03 +0200 Subject: [PATCH 165/217] Separate matrix-registration install/uninstall tasks --- roles/matrix-registration/tasks/main.yml | 9 +++- .../tasks/{setup.yml => setup_install.yml} | 50 ++----------------- .../tasks/setup_uninstall.yml | 30 +++++++++++ 3 files changed, 43 insertions(+), 46 deletions(-) rename roles/matrix-registration/tasks/{setup.yml => setup_install.yml} (57%) create mode 100644 roles/matrix-registration/tasks/setup_uninstall.yml diff --git a/roles/matrix-registration/tasks/main.yml b/roles/matrix-registration/tasks/main.yml index 4a884ccd..4fef6abe 100644 --- a/roles/matrix-registration/tasks/main.yml +++ b/roles/matrix-registration/tasks/main.yml @@ -8,7 +8,14 @@ - setup-all - setup-matrix-registration -- import_tasks: "{{ role_path }}/tasks/setup.yml" +- import_tasks: "{{ role_path }}/tasks/setup_install.yml" + when: "run_setup|bool and matrix_registration_enabled|bool" + tags: + - setup-all + - setup-matrix-registration + +- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml" + when: "run_setup|bool and not matrix_registration_enabled|bool" tags: - setup-all - setup-matrix-registration diff --git a/roles/matrix-registration/tasks/setup.yml b/roles/matrix-registration/tasks/setup_install.yml similarity index 57% rename from roles/matrix-registration/tasks/setup.yml rename to roles/matrix-registration/tasks/setup_install.yml index dfe1ba8b..77c35581 100644 --- a/roles/matrix-registration/tasks/setup.yml +++ b/roles/matrix-registration/tasks/setup_install.yml @@ -1,9 +1,5 @@ --- -# -# Tasks related to setting up matrix-registration -# - - name: Ensure matrix-registration paths exist file: path: "{{ item.path }}" @@ -16,7 +12,7 @@ - { path: "{{ matrix_registration_config_path }}", when: true } - { path: "{{ matrix_registration_data_path }}", when: true } - { path: "{{ matrix_registration_docker_src_files_path }}", when: "{{ matrix_registration_container_image_self_build }}"} - when: matrix_registration_enabled|bool and item.when + when: "item.when|bool" - name: Ensure matrix-registration image is pulled docker_image: @@ -24,7 +20,7 @@ source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_registration_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_registration_docker_image_force_pull }}" - when: "matrix_registration_enabled|bool and not matrix_registration_container_image_self_build|bool" + when: "not matrix_registration_container_image_self_build|bool" - name: Ensure matrix-registration repository is present when self-building git: @@ -33,7 +29,7 @@ version: "{{ matrix_registration_container_image_self_build_branch }}" force: "yes" register: matrix_registration_git_pull_results - when: "matrix_registration_enabled|bool and matrix_registration_container_image_self_build|bool" + when: "matrix_registration_container_image_self_build|bool" - name: Ensure matrix-registration Docker image is built docker_image: @@ -44,7 +40,7 @@ dockerfile: Dockerfile path: "{{ matrix_registration_docker_src_files_path }}" pull: yes - when: "matrix_registration_enabled|bool and matrix_registration_container_image_self_build|bool" + when: "matrix_registration_container_image_self_build|bool" - name: Ensure matrix-registration config installed copy: @@ -53,7 +49,6 @@ mode: 0644 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" - when: matrix_registration_enabled|bool - name: Ensure matrix-registration.service installed template: @@ -61,43 +56,8 @@ dest: "{{ matrix_systemd_path }}/matrix-registration.service" mode: 0644 register: matrix_registration_systemd_service_result - when: matrix_registration_enabled|bool - name: Ensure systemd reloaded after matrix-registration.service installation service: daemon_reload: yes - when: "matrix_registration_enabled|bool and matrix_registration_systemd_service_result.changed" - -# -# Tasks related to getting rid of matrix-registration (if it was previously enabled) -# - -- name: Check existence of matrix-registration service - stat: - path: "{{ matrix_systemd_path }}/matrix-registration.service" - register: matrix_registration_service_stat - -- name: Ensure matrix-registration is stopped - service: - name: matrix-registration - state: stopped - daemon_reload: yes - register: stopping_result - when: "not matrix_registration_enabled|bool and matrix_registration_service_stat.stat.exists" - -- name: Ensure matrix-registration.service doesn't exist - file: - path: "{{ matrix_systemd_path }}/matrix-registration.service" - state: absent - when: "not matrix_registration_enabled|bool and matrix_registration_service_stat.stat.exists" - -- name: Ensure systemd reloaded after matrix-registration.service removal - service: - daemon_reload: yes - when: "not matrix_registration_enabled|bool and matrix_registration_service_stat.stat.exists" - -- name: Ensure matrix-registration Docker image doesn't exist - docker_image: - name: "{{ matrix_registration_docker_image }}" - state: absent - when: "not matrix_registration_enabled|bool" + when: "matrix_registration_systemd_service_result.changed|bool" diff --git a/roles/matrix-registration/tasks/setup_uninstall.yml b/roles/matrix-registration/tasks/setup_uninstall.yml new file mode 100644 index 00000000..573f8170 --- /dev/null +++ b/roles/matrix-registration/tasks/setup_uninstall.yml @@ -0,0 +1,30 @@ +--- + +- name: Check existence of matrix-registration service + stat: + path: "{{ matrix_systemd_path }}/matrix-registration.service" + register: matrix_registration_service_stat + +- name: Ensure matrix-registration is stopped + service: + name: matrix-registration + state: stopped + daemon_reload: yes + register: stopping_result + when: "matrix_registration_service_stat.stat.exists|bool" + +- name: Ensure matrix-registration.service doesn't exist + file: + path: "{{ matrix_systemd_path }}/matrix-registration.service" + state: absent + when: "matrix_registration_service_stat.stat.exists|bool" + +- name: Ensure systemd reloaded after matrix-registration.service removal + service: + daemon_reload: yes + when: "matrix_registration_service_stat.stat.exists|bool" + +- name: Ensure matrix-registration Docker image doesn't exist + docker_image: + name: "{{ matrix_registration_docker_image }}" + state: absent From 8d7459387895a6a3c14ef4c1a751353c740c5a9d Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 18:58:37 +0200 Subject: [PATCH 166/217] Prepare matrix-registation for (SQLite + Postgres) support Auto-migration and everything seems to work. It's just that matrix-registration cannot load the Python modules required for talking to a Postgres database. Tracked here: https://github.com/ZerataX/matrix-registration/issues/44 Until this gets fixed, we'll continue default to 'sqlite'. --- group_vars/matrix_servers | 18 +++++++++++ roles/matrix-registration/defaults/main.yml | 30 +++++++++++++++++ .../tasks/setup_install.yml | 32 +++++++++++++++++++ .../tasks/validate_config.yml | 7 ++++ .../templates/config.yaml.j2 | 2 +- 5 files changed, 88 insertions(+), 1 deletion(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 1223d8a6..ad62a86e 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1010,6 +1010,12 @@ matrix_postgres_additional_databases: | 'password': matrix_bot_matrix_reminder_bot_database_password, }] if (matrix_bot_matrix_reminder_bot_enabled and matrix_bot_matrix_reminder_bot_database_engine == 'postgres' and matrix_bot_matrix_reminder_bot_database_hostname == 'matrix-postgres') else []) + + ([{ + 'name': matrix_registration_database_db_name, + 'username': matrix_registration_database_username, + 'password': matrix_registration_database_password, + }] if (matrix_registration_enabled and matrix_registration_database_engine == 'postgres' and matrix_registration_database_hostname == 'matrix-postgres') else []) + + ([{ 'name': matrix_appservice_discord_database_db_name, 'username': matrix_appservice_discord_database_username, @@ -1291,6 +1297,18 @@ matrix_registration_api_validate_certs: "{{ false if matrix_ssl_retrieval_method matrix_registration_container_image_self_build: "{{ matrix_architecture != 'amd64' }}" +matrix_registration_systemd_required_services_list: | + {{ + ['docker.service'] + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + }} + +# We'd like to use 'postgres' if matrix_postgres_enabled, but the container image doesn't seem to support that. +# Learn more here: https://github.com/ZerataX/matrix-registration/issues/44 +matrix_registration_database_engine: 'sqlite' +matrix_registration_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx.registr.db') | to_uuid }}" + ###################################################################### # # /matrix-registration diff --git a/roles/matrix-registration/defaults/main.yml b/roles/matrix-registration/defaults/main.yml index 95147665..5afe4dee 100644 --- a/roles/matrix-registration/defaults/main.yml +++ b/roles/matrix-registration/defaults/main.yml @@ -32,6 +32,36 @@ matrix_registration_systemd_wanted_services_list: [] # Takes an ":" or "" value (e.g. "127.0.0.1:8767"), or empty string to not expose. matrix_registration_container_http_host_bind_port: '' +# Database-related configuration fields. +# +# To use SQLite, stick to these defaults. +# +# To use Postgres: +# - change the engine (`matrix_registration_database_engine: 'postgres'`) +# - adjust your database credentials via the `matrix_registration_postgres_*` variables +matrix_registration_database_engine: 'sqlite' + +matrix_registration_sqlite_database_path_local: "{{ matrix_registration_data_path }}/db.sqlite3" +matrix_registration_sqlite_database_path_in_container: "/data/db.sqlite3" + +matrix_registration_database_username: 'matrix_registration' +matrix_registration_database_password: 'some-password' +matrix_registration_database_hostname: 'matrix-postgres' +matrix_registration_database_port: 5432 +matrix_registration_database_db_name: 'matrix_registration' + +matrix_registration_database_connection_string: 'postgresql://{{ matrix_registration_database_username }}:{{ matrix_registration_database_password }}@{{ matrix_registration_database_hostname }}:{{ matrix_registration_database_port }}/{{ matrix_registration_database_db_name }}' + +# For some reason, matrix-registraiton expects the `db` field to be like this: `sqlite:////data/db.sqlite3`. +# (seems like one too many slashes, but..) +matrix_registration_db: "{{ + { + 'sqlite': ('sqlite:///' + matrix_registration_sqlite_database_path_in_container), + 'postgres': matrix_registration_database_connection_string, + }[matrix_registration_database_engine] +}}" + + # The path at which Matrix Registration will be exposed on `matrix.DOMAIN` # (only applies when matrix-nginx-proxy is used). matrix_registration_public_endpoint: /matrix-registration diff --git a/roles/matrix-registration/tasks/setup_install.yml b/roles/matrix-registration/tasks/setup_install.yml index 77c35581..489bab8b 100644 --- a/roles/matrix-registration/tasks/setup_install.yml +++ b/roles/matrix-registration/tasks/setup_install.yml @@ -1,5 +1,31 @@ --- +- set_fact: + matrix_registration_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_registration_sqlite_database_path_local }}" + register: matrix_registration_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_registration_sqlite_database_path_local }}" + dst: "{{ matrix_registration_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_registration_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-registration.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_registration_requires_restart: true + when: "matrix_registration_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_registration_database_engine == 'postgres'" + - name: Ensure matrix-registration paths exist file: path: "{{ item.path }}" @@ -61,3 +87,9 @@ service: daemon_reload: yes when: "matrix_registration_systemd_service_result.changed|bool" + +- name: Ensure matrix-registration.service restarted, if necessary + service: + name: "matrix-registration.service" + state: restarted + when: "matrix_registration_requires_restart|bool" diff --git a/roles/matrix-registration/tasks/validate_config.yml b/roles/matrix-registration/tasks/validate_config.yml index 90466b46..80293bcb 100644 --- a/roles/matrix-registration/tasks/validate_config.yml +++ b/roles/matrix-registration/tasks/validate_config.yml @@ -18,3 +18,10 @@ when: "item.old in vars" with_items: - {'old': 'matrix_registration_docker_repo', 'new': 'matrix_registration_container_image_self_build_repo'} + +- name: Fail if Postgres usage attempted + fail: + msg: > + matrix-registration doesn't support using Postgres just yet. + Learn more here: https://github.com/ZerataX/matrix-registration/issues/44 + when: "matrix_registration_database_engine == 'postgres'" diff --git a/roles/matrix-registration/templates/config.yaml.j2 b/roles/matrix-registration/templates/config.yaml.j2 index f3b1c57b..ae0ba5ff 100644 --- a/roles/matrix-registration/templates/config.yaml.j2 +++ b/roles/matrix-registration/templates/config.yaml.j2 @@ -3,7 +3,7 @@ server_name: {{ matrix_registration_server_name|to_json }} shared_secret: {{ matrix_registration_shared_secret|to_json }} admin_secret: {{ matrix_registration_admin_secret|to_json }} riot_instance: {{ matrix_registration_riot_instance|to_json }} -db: 'sqlite:////data/db.sqlite3' +db: {{ matrix_registration_db|to_json }} host: '0.0.0.0' port: 5000 rate_limit: ["100 per day", "10 per minute"] From 374f43735aa362bbbb2a9068ef2fd8e81b5ab561 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 20:05:31 +0200 Subject: [PATCH 167/217] Separate matrix-dimension install/uninstall tasks --- roles/matrix-dimension/tasks/main.yml | 10 ++- .../tasks/setup_dimension.yml | 85 ------------------- .../matrix-dimension/tasks/setup_install.yml | 36 ++++++++ .../tasks/setup_uninstall.yml | 35 ++++++++ 4 files changed, 79 insertions(+), 87 deletions(-) delete mode 100644 roles/matrix-dimension/tasks/setup_dimension.yml create mode 100644 roles/matrix-dimension/tasks/setup_install.yml create mode 100644 roles/matrix-dimension/tasks/setup_uninstall.yml diff --git a/roles/matrix-dimension/tasks/main.yml b/roles/matrix-dimension/tasks/main.yml index 1888f945..aad55286 100644 --- a/roles/matrix-dimension/tasks/main.yml +++ b/roles/matrix-dimension/tasks/main.yml @@ -8,8 +8,14 @@ - setup-all - setup-dimension -- import_tasks: "{{ role_path }}/tasks/setup_dimension.yml" - when: run_setup|bool +- import_tasks: "{{ role_path }}/tasks/setup_install.yml" + when: run_setup|bool and matrix_dimension_enabled|bool + tags: + - setup-all + - setup-dimension + +- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml" + when: run_setup|bool and not matrix_dimension_enabled|bool tags: - setup-all - setup-dimension diff --git a/roles/matrix-dimension/tasks/setup_dimension.yml b/roles/matrix-dimension/tasks/setup_dimension.yml deleted file mode 100644 index 2437a547..00000000 --- a/roles/matrix-dimension/tasks/setup_dimension.yml +++ /dev/null @@ -1,85 +0,0 @@ ---- - -# -# Tasks related to setting up the dimension -# - -- name: Ensure Dimension base path exists - file: - path: "{{ matrix_dimension_base_path }}" - state: directory - mode: 0770 - owner: "{{ matrix_user_username }}" - group: "{{ matrix_dimension_user_gid }}" - when: matrix_dimension_enabled|bool - -- name: Ensure Dimension config installed - copy: - content: "{{ matrix_dimension_configuration|to_nice_yaml }}" - dest: "{{ matrix_dimension_base_path }}/config.yaml" - mode: 0640 - owner: "{{ matrix_user_username }}" - group: "{{ matrix_dimension_user_gid }}" - when: matrix_dimension_enabled|bool - -- name: Ensure Dimension image is pulled - docker_image: - name: "{{ matrix_dimension_docker_image }}" - source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" - force_source: "{{ matrix_dimension_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_dimension_docker_image_force_pull }}" - when: matrix_dimension_enabled|bool - -- name: Ensure matrix-dimension.service installed - template: - src: "{{ role_path }}/templates/systemd/matrix-dimension.service.j2" - dest: "{{ matrix_systemd_path }}/matrix-dimension.service" - mode: 0644 - register: matrix_dimension_systemd_service_result - when: matrix_dimension_enabled|bool - -- name: Ensure systemd reloaded after matrix-dimension.service installation - service: - daemon_reload: yes - when: "matrix_dimension_enabled|bool and matrix_dimension_systemd_service_result.changed" - -# -# Tasks related to getting rid of the dimension (if it was previously enabled) -# - -- name: Check existence of matrix-dimension service - stat: - path: "{{ matrix_systemd_path }}/matrix-dimension.service" - register: matrix_dimension_service_stat - when: "not matrix_dimension_enabled|bool" - -- name: Ensure matrix-dimension is stopped - service: - name: matrix-dimension - state: stopped - daemon_reload: yes - register: stopping_result - when: "not matrix_dimension_enabled|bool and matrix_dimension_service_stat.stat.exists" - -- name: Ensure matrix-dimension.service doesn't exist - file: - path: "{{ matrix_systemd_path }}/matrix-dimension.service" - state: absent - when: "not matrix_dimension_enabled|bool and matrix_dimension_service_stat.stat.exists" - -- name: Ensure systemd reloaded after matrix-dimension.service removal - service: - daemon_reload: yes - when: "not matrix_dimension_enabled|bool and matrix_dimension_service_stat.stat.exists" - -- name: Ensure Dimension environment variables path doesn't exist - file: - path: "{{ matrix_dimension_base_path }}" - state: absent - when: "not matrix_dimension_enabled|bool" - -- name: Ensure Dimension Docker image doesn't exist - docker_image: - name: "{{ matrix_dimension_docker_image }}" - state: absent - when: "not matrix_dimension_enabled|bool" diff --git a/roles/matrix-dimension/tasks/setup_install.yml b/roles/matrix-dimension/tasks/setup_install.yml new file mode 100644 index 00000000..e118dd62 --- /dev/null +++ b/roles/matrix-dimension/tasks/setup_install.yml @@ -0,0 +1,36 @@ +--- + +- name: Ensure Dimension base path exists + file: + path: "{{ matrix_dimension_base_path }}" + state: directory + mode: 0770 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_dimension_user_gid }}" + +- name: Ensure Dimension config installed + copy: + content: "{{ matrix_dimension_configuration|to_nice_yaml }}" + dest: "{{ matrix_dimension_base_path }}/config.yaml" + mode: 0640 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_dimension_user_gid }}" + +- name: Ensure Dimension image is pulled + docker_image: + name: "{{ matrix_dimension_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_dimension_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_dimension_docker_image_force_pull }}" + +- name: Ensure matrix-dimension.service installed + template: + src: "{{ role_path }}/templates/systemd/matrix-dimension.service.j2" + dest: "{{ matrix_systemd_path }}/matrix-dimension.service" + mode: 0644 + register: matrix_dimension_systemd_service_result + +- name: Ensure systemd reloaded after matrix-dimension.service installation + service: + daemon_reload: yes + when: "matrix_dimension_systemd_service_result.changed|bool" diff --git a/roles/matrix-dimension/tasks/setup_uninstall.yml b/roles/matrix-dimension/tasks/setup_uninstall.yml new file mode 100644 index 00000000..9bc4ac8b --- /dev/null +++ b/roles/matrix-dimension/tasks/setup_uninstall.yml @@ -0,0 +1,35 @@ +--- + +- name: Check existence of matrix-dimension service + stat: + path: "{{ matrix_systemd_path }}/matrix-dimension.service" + register: matrix_dimension_service_stat + +- name: Ensure matrix-dimension is stopped + service: + name: matrix-dimension + state: stopped + daemon_reload: yes + register: stopping_result + when: "matrix_dimension_service_stat.stat.exists|bool" + +- name: Ensure matrix-dimension.service doesn't exist + file: + path: "{{ matrix_systemd_path }}/matrix-dimension.service" + state: absent + when: "matrix_dimension_service_stat.stat.exists|bool" + +- name: Ensure systemd reloaded after matrix-dimension.service removal + service: + daemon_reload: yes + when: "matrix_dimension_service_stat.stat.exists|bool" + +- name: Ensure Dimension base directory doesn't exist + file: + path: "{{ matrix_dimension_base_path }}" + state: absent + +- name: Ensure Dimension Docker image doesn't exist + docker_image: + name: "{{ matrix_dimension_docker_image }}" + state: absent From 0790a7b2a8bc21e34d12f4436b88e9a3ce75c526 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 20:31:07 +0200 Subject: [PATCH 168/217] Add support for matrix_dimension_systemd_{required,wanted}_services_list We were referencing them from `group_vars/matrix_servers` since recently, but there were no such variables and they weren't being put to use. --- roles/matrix-dimension/defaults/main.yml | 6 ++++++ .../templates/systemd/matrix-dimension.service.j2 | 9 +++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/roles/matrix-dimension/defaults/main.yml b/roles/matrix-dimension/defaults/main.yml index 3b69227a..7ead6e89 100644 --- a/roles/matrix-dimension/defaults/main.yml +++ b/roles/matrix-dimension/defaults/main.yml @@ -15,6 +15,12 @@ matrix_dimension_base_path: "{{ matrix_base_data_path }}/dimension" matrix_dimension_docker_image: "docker.io/turt2live/matrix-dimension:latest" matrix_dimension_docker_image_force_pull: "{{ matrix_dimension_docker_image.endswith(':latest') }}" +# List of systemd services that matrix-dimension.service depends on. +matrix_dimension_systemd_required_services_list: ['docker.service'] + +# List of systemd services that matrix-dimension.service wants +matrix_dimension_systemd_wanted_services_list: [] + # The user and group id correspond to the node user in the `turt2live/matrix-dimension` image. matrix_dimension_user_uid: '1000' matrix_dimension_user_gid: '1000' diff --git a/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 b/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 index 94c38491..1c900a16 100644 --- a/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 +++ b/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 @@ -1,8 +1,13 @@ #jinja2: lstrip_blocks: "True" [Unit] Description=Matrix Dimension -After=docker.service -Requires=docker.service +{% for service in matrix_dimension_systemd_required_services_list %} +Requires={{ service }} +After={{ service }} +{% endfor %} +{% for service in matrix_dimension_systemd_wanted_services_list %} +Wants={{ service }} +{% endfor %} DefaultDependencies=no [Service] From 2a502db2394af179bcb6def67ac89c7fd83d2d95 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 21:01:47 +0200 Subject: [PATCH 169/217] Add (SQLite + Postgres) support and automatic migration to matrix-dimension --- group_vars/matrix_servers | 9 +++-- roles/matrix-dimension/defaults/main.yml | 22 +++++++++++++ .../matrix-dimension/tasks/setup_install.yml | 33 +++++++++++++++++++ .../matrix-dimension/templates/config.yaml.j2 | 6 +++- .../systemd/matrix-dimension.service.j2 | 4 ++- 5 files changed, 67 insertions(+), 7 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index ad62a86e..d7163ab7 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -716,7 +716,7 @@ matrix_dimension_systemd_required_services_list: | # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_dimension_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_dimension_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.dimension.db') | to_uuid }}" +matrix_dimension_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db') | to_uuid }}" ###################################################################### # @@ -1092,13 +1092,12 @@ matrix_postgres_additional_databases: | }] if (matrix_mx_puppet_steam_enabled and matrix_mx_puppet_steam_database_engine == 'postgres' and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else []) - + ([{ + + + ([{ 'name': matrix_dimension_database_db_name, 'username': matrix_dimension_database_username, 'password': matrix_dimension_database_password, - }] if (matrix_dimension_enabled - and matrix_dimension_database_engine == 'postgres' - and matrix_dimension_database_hostname == 'matrix-postgres') else []) + }] if (matrix_dimension_enabled and matrix_dimension_database_engine == 'postgres' and matrix_dimension_database_hostname == 'matrix-postgres') else []) }} ###################################################################### diff --git a/roles/matrix-dimension/defaults/main.yml b/roles/matrix-dimension/defaults/main.yml index 7ead6e89..a6a3bf5d 100644 --- a/roles/matrix-dimension/defaults/main.yml +++ b/roles/matrix-dimension/defaults/main.yml @@ -40,6 +40,28 @@ matrix_dimension_integrations_jitsi_widget_url: "https://{{ matrix_server_fqn_di matrix_dimension_homeserver_federationUrl: "http://matrix-synapse:8048" + +# Database-related configuration fields. +# +# To use SQLite, stick to these defaults. +# +# To use Postgres: +# - change the engine (`matrix_dimension_database_engine: 'postgres'`) +# - adjust your database credentials via the `matrix_dimension_postgres_*` variables +matrix_dimension_database_engine: 'sqlite' + +matrix_dimension_sqlite_database_path_local: "{{ matrix_dimension_base_path }}/dimension.db" +matrix_dimension_sqlite_database_path_in_container: "dimension.db" + +matrix_dimension_database_username: 'matrix_dimension' +matrix_dimension_database_password: 'some-password' +matrix_dimension_database_hostname: 'matrix-postgres' +matrix_dimension_database_port: 5432 +matrix_dimension_database_db_name: 'matrix_dimension' + +matrix_dimension_database_connection_string: 'postgres://{{ matrix_dimension_database_username }}:{{ matrix_dimension_database_password }}@{{ matrix_dimension_database_hostname }}:{{ matrix_dimension_database_port }}/{{ matrix_dimension_database_db_name }}' + + # Default Dimension configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. # diff --git a/roles/matrix-dimension/tasks/setup_install.yml b/roles/matrix-dimension/tasks/setup_install.yml index e118dd62..9a264449 100644 --- a/roles/matrix-dimension/tasks/setup_install.yml +++ b/roles/matrix-dimension/tasks/setup_install.yml @@ -1,5 +1,32 @@ --- +- set_fact: + matrix_dimension_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_dimension_sqlite_database_path_local }}" + register: matrix_dimension_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_dimension_sqlite_database_path_local }}" + dst: "{{ matrix_dimension_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_dimension_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-dimension.service'] + pgloader_options: ['--with "quote identifiers"'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_dimension_requires_restart: true + when: "matrix_dimension_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_dimension_database_engine == 'postgres'" + - name: Ensure Dimension base path exists file: path: "{{ matrix_dimension_base_path }}" @@ -34,3 +61,9 @@ service: daemon_reload: yes when: "matrix_dimension_systemd_service_result.changed|bool" + +- name: Ensure matrix-dimension.service restarted, if necessary + service: + name: "matrix-dimension.service" + state: restarted + when: "matrix_dimension_requires_restart|bool" diff --git a/roles/matrix-dimension/templates/config.yaml.j2 b/roles/matrix-dimension/templates/config.yaml.j2 index b7ca1ee3..200871e7 100644 --- a/roles/matrix-dimension/templates/config.yaml.j2 +++ b/roles/matrix-dimension/templates/config.yaml.j2 @@ -44,7 +44,11 @@ widgetBlacklist: # Where the database for Dimension is database: - uri: "postgres://matrix_dimension:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_dimension') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_dimension" +{% if matrix_dimension_database_engine == 'sqlite' %} + file: {{ matrix_dimension_sqlite_database_path_in_container|to_json }} +{% elif matrix_dimension_database_engine == 'postgres' %} + uri: {{ matrix_dimension_database_connection_string|to_json }} +{% endif %} # Display settings that apply to self-hosted go-neb instances goneb: diff --git a/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 b/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 index 1c900a16..ff10224a 100644 --- a/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 +++ b/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 @@ -16,7 +16,9 @@ ExecStartPre=-{{ matrix_host_command_docker }} kill matrix-dimension ExecStartPre=-{{ matrix_host_command_docker }} rm matrix-dimension # Fixup database ownership if it got changed somehow (during a server migration, etc.) -ExecStartPre=-{{ matrix_host_command_chown }} {{ matrix_dimension_user_uid }}:{{ matrix_dimension_user_gid }} {{ matrix_dimension_base_path }}/dimension.db +{% if matrix_dimension_database_engine == 'sqlite' %} +ExecStartPre=-{{ matrix_host_command_chown }} {{ matrix_dimension_user_uid }}:{{ matrix_dimension_user_gid }} {{ matrix_dimension_sqlite_database_path_local }} +{% endif %} ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-dimension \ --log-driver=none \ From dd797ba6a76e18f8b2bfa676b6f3e78f18f7cf47 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 22:28:20 +0200 Subject: [PATCH 170/217] Fix Postgres database importing/upgrading conflicts We were running into conflicts, because having initialized the roles (users) and databases, trying to import leads to errors (role XXX already exists, etc.). We were previously ignoring the Synapse database (`homeserver`) when upgrading/importing, because that one gets created by default whenever the container starts. For our additional databases, it's a similar situation now. It's not created by default as soon as Postgres starts with an empty database, but rather we create it as part of running the playbook. So we either need to skip those role/database creation statements while upgrading/importing, or to avoid creating the additional database and rely on the import for that. I've gone for the former, because it's already similar to what we were doing and it's simpler (it lets `setup_postgres.yml` be the same in all scenarios). --- group_vars/matrix_servers | 14 ++++++++++++++ roles/matrix-postgres/defaults/main.yml | 16 ++++++++++++++++ roles/matrix-postgres/tasks/import_postgres.yml | 4 ++-- roles/matrix-postgres/tasks/upgrade_postgres.yml | 4 ++-- 4 files changed, 34 insertions(+), 4 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index d7163ab7..932b288a 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1100,6 +1100,20 @@ matrix_postgres_additional_databases: | }] if (matrix_dimension_enabled and matrix_dimension_database_engine == 'postgres' and matrix_dimension_database_hostname == 'matrix-postgres') else []) }} +matrix_postgres_import_roles_to_ignore: | + {{ + [matrix_postgres_connection_username] + + + matrix_postgres_additional_databases|map(attribute='username') + }} + +matrix_postgres_import_databases_to_ignore: | + {{ + [matrix_postgres_db_name] + + + matrix_postgres_additional_databases|map(attribute='name') + }} + ###################################################################### # # /matrix-postgres diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index 5755742f..0c516281 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -43,6 +43,22 @@ matrix_postgres_container_postgres_bind_port: "" # password: some_password matrix_postgres_additional_databases: [] +# A list of roles/users to avoid creating when importing (or upgrading) the database. +# If a dump file contains the roles and they've also been created beforehand (see `matrix_postgres_additional_databases`), +# importing would fail. +# We either need to not create them or to ignore the `CREATE ROLE` statements in the dump. +matrix_postgres_import_roles_to_ignore: [matrix_postgres_connection_username] + +matrix_postgres_import_roles_ignore_regex: "^CREATE ROLE ({{ matrix_postgres_import_roles_to_ignore|join('|') }});" + +# A list of databases to avoid creating when importing (or upgrading) the database. +# If a dump file contains the databases and they've also been created beforehand (see `matrix_postgres_additional_databases`), +# importing would fail. +# We either need to not create them or to ignore the `CREATE DATABASE` statements in the dump. +matrix_postgres_import_databases_to_ignore: [matrix_postgres_db_name] + +matrix_postgres_import_databases_ignore_regex: "^CREATE DATABASE ({{ matrix_postgres_import_databases_to_ignore|join('|') }})\\s" + # The number of seconds to wait after starting `matrix-postgres.service` # and before trying to run queries for creating additional databases/users against it. # diff --git a/roles/matrix-postgres/tasks/import_postgres.yml b/roles/matrix-postgres/tasks/import_postgres.yml index 33d98691..c26affbb 100644 --- a/roles/matrix-postgres/tasks/import_postgres.yml +++ b/roles/matrix-postgres/tasks/import_postgres.yml @@ -74,8 +74,8 @@ {{ matrix_postgres_docker_image_latest }} -c "cat /{{ server_path_postgres_dump|basename }} | {{ 'gunzip |' if server_path_postgres_dump.endswith('.gz') else '' }} - grep -vE '^CREATE ROLE {{ matrix_postgres_connection_username }}' | - grep -vE '^CREATE DATABASE {{ matrix_postgres_db_name }}' | + grep -vE '{{ matrix_postgres_import_roles_ignore_regex }}' | + grep -vE '{{ matrix_postgres_import_databases_ignore_regex }}' | psql -v ON_ERROR_STOP=1 -h matrix-postgres" # This is a hack. diff --git a/roles/matrix-postgres/tasks/upgrade_postgres.yml b/roles/matrix-postgres/tasks/upgrade_postgres.yml index 72f327b3..564265d8 100644 --- a/roles/matrix-postgres/tasks/upgrade_postgres.yml +++ b/roles/matrix-postgres/tasks/upgrade_postgres.yml @@ -135,8 +135,8 @@ {{ matrix_postgres_docker_image_latest }} -c "cat /in/{{ postgres_dump_name }} | {{ 'gunzip |' if postgres_dump_name.endswith('.gz') else '' }} - grep -vE '^CREATE ROLE {{ matrix_postgres_connection_username }}' | - grep -vE '^CREATE DATABASE {{ matrix_postgres_db_name }}' | + grep -vE '{{ matrix_postgres_import_roles_ignore_regex }}' | + grep -vE '{{ matrix_postgres_import_databases_ignore_regex }}' | psql -v ON_ERROR_STOP=1 -h matrix-postgres" # This is a hack. From e2ba46bf013785079eeca380a9926598b9fd1fe4 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 14 Dec 2020 22:40:37 +0200 Subject: [PATCH 171/217] Fix Jinja2 syntax error (else if -> elif) --- roles/matrix-bridge-appservice-irc/defaults/main.yml | 2 +- roles/matrix-bridge-appservice-slack/defaults/main.yml | 2 +- roles/matrix-bridge-mx-puppet-discord/defaults/main.yml | 2 +- roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml | 2 +- roles/matrix-bridge-mx-puppet-skype/defaults/main.yml | 2 +- roles/matrix-bridge-mx-puppet-slack/defaults/main.yml | 2 +- roles/matrix-bridge-mx-puppet-steam/defaults/main.yml | 2 +- roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/roles/matrix-bridge-appservice-irc/defaults/main.yml b/roles/matrix-bridge-appservice-irc/defaults/main.yml index bf23b0ab..5110fd85 100644 --- a/roles/matrix-bridge-appservice-irc/defaults/main.yml +++ b/roles/matrix-bridge-appservice-irc/defaults/main.yml @@ -26,7 +26,7 @@ matrix_appservice_irc_database_name: matrix_appservice_irc matrix_appservice_irc_database_connString: >-2 {%- if matrix_appservice_irc_database_engine == 'postgres' -%} postgresql://{{ matrix_appservice_irc_database_username }}:{{ matrix_appservice_irc_database_password }}@{{ matrix_appservice_irc_database_hostname }}:{{ matrix_appservice_irc_database_port }}/{{ matrix_appservice_irc_database_name }}?sslmode=disable - {%- else if matrix_appservice_irc_database_engine == 'nedb' -%} + {%- elif matrix_appservice_irc_database_engine == 'nedb' -%} {{ matrix_appservice_irc_database_engine }}://{{ matrix_appservice_irc_database_file }} {%- endif -%} diff --git a/roles/matrix-bridge-appservice-slack/defaults/main.yml b/roles/matrix-bridge-appservice-slack/defaults/main.yml index e2127c29..365493ff 100644 --- a/roles/matrix-bridge-appservice-slack/defaults/main.yml +++ b/roles/matrix-bridge-appservice-slack/defaults/main.yml @@ -55,7 +55,7 @@ matrix_appservice_slack_database_file: /data matrix_appservice_slack_database_connString: >-2 {%- if matrix_appservice_slack_database_engine == 'postgres' -%} postgresql://{{ matrix_appservice_slack_database_username }}:{{ matrix_appservice_slack_database_password }}@{{ matrix_appservice_slack_database_hostname }}:{{ matrix_appservice_slack_database_port }}/{{ matrix_appservice_slack_database_name }}?sslmode=disable - {%- else if matrix_appservice_slack_database_engine == 'nedb' -%} + {%- elif matrix_appservice_slack_database_engine == 'nedb' -%} {{ matrix_appservice_slack_database_engine }}://{{ matrix_appservice_slack_database_file }} {%- endif -%} diff --git a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml index 5a65f33e..a0ad1c0a 100644 --- a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml @@ -66,7 +66,7 @@ matrix_mx_puppet_discord_database_file: /data/database.db matrix_mx_puppet_discord_database_connString: >-2 {%- if matrix_mx_puppet_discord_database_engine == 'postgres' -%} postgresql://{{ matrix_mx_puppet_discord_database_username }}:{{ matrix_mx_puppet_discord_database_password }}@{{ matrix_mx_puppet_discord_database_hostname }}:{{ matrix_mx_puppet_discord_database_port }}/{{ matrix_mx_puppet_discord_database_name }}?sslmode=disable - {%- else if matrix_mx_puppet_discord_database_engine == 'sqlite' -%} + {%- elif matrix_mx_puppet_discord_database_engine == 'sqlite' -%} {{ matrix_mx_puppet_discord_database_engine }}://{{ matrix_mx_puppet_discord_database_file }} {%- endif -%} diff --git a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml index c30f9b8b..4c40bda4 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml @@ -56,7 +56,7 @@ matrix_mx_puppet_instagram_database_file: /data/database.db matrix_mx_puppet_instagram_database_connString: >-2 {%- if matrix_mx_puppet_instagram_database_engine == 'postgres' -%} postgresql://{{ matrix_mx_puppet_instagram_database_username }}:{{ matrix_mx_puppet_instagram_database_password }}@{{ matrix_mx_puppet_instagram_database_hostname }}:{{ matrix_mx_puppet_instagram_database_port }}/{{ matrix_mx_puppet_instagram_database_name }}?sslmode=disable - {%- else if matrix_mx_puppet_instagram_database_engine == 'sqlite' -%} + {%- elif matrix_mx_puppet_instagram_database_engine == 'sqlite' -%} {{ matrix_mx_puppet_instagram_database_engine }}://{{ matrix_mx_puppet_instagram_database_file }} {%- endif -%} diff --git a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml index 3b7448c6..db0f4ede 100644 --- a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml @@ -64,7 +64,7 @@ matrix_mx_puppet_skype_database_file: /data/database.db matrix_mx_puppet_skype_database_connString: >-2 {%- if matrix_mx_puppet_skype_database_engine == 'postgres' -%} postgresql://{{ matrix_mx_puppet_skype_database_username }}:{{ matrix_mx_puppet_skype_database_password }}@{{ matrix_mx_puppet_skype_database_hostname }}:{{ matrix_mx_puppet_skype_database_port }}/{{ matrix_mx_puppet_skype_database_name }}?sslmode=disable - {%- else if matrix_mx_puppet_skype_database_engine == 'sqlite' -%} + {%- elif matrix_mx_puppet_skype_database_engine == 'sqlite' -%} {{ matrix_mx_puppet_skype_database_engine }}://{{ matrix_mx_puppet_skype_database_file }} {%- endif -%} diff --git a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml index 5f91beb8..2ed72611 100644 --- a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml @@ -68,7 +68,7 @@ matrix_mx_puppet_slack_database_file: /data/database.db matrix_mx_puppet_slack_database_connString: >-2 {%- if matrix_mx_puppet_slack_database_engine == 'postgres' -%} postgresql://{{ matrix_mx_puppet_slack_database_username }}:{{ matrix_mx_puppet_slack_database_password }}@{{ matrix_mx_puppet_slack_database_hostname }}:{{ matrix_mx_puppet_slack_database_port }}/{{ matrix_mx_puppet_slack_database_name }}?sslmode=disable - {%- else if matrix_mx_puppet_slack_database_engine == 'sqlite' -%} + {%- elif matrix_mx_puppet_slack_database_engine == 'sqlite' -%} {{ matrix_mx_puppet_slack_database_engine }}://{{ matrix_mx_puppet_slack_database_file }} {%- endif -%} diff --git a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml index be343db4..77f7804b 100644 --- a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml @@ -65,7 +65,7 @@ matrix_mx_puppet_steam_database_file: /data/database.db matrix_mx_puppet_steam_database_connString: >-2 {%- if matrix_mx_puppet_steam_database_engine == 'postgres' -%} postgresql://{{ matrix_mx_puppet_steam_database_username }}:{{ matrix_mx_puppet_steam_database_password }}@{{ matrix_mx_puppet_steam_database_hostname }}:{{ matrix_mx_puppet_steam_database_port }}/{{ matrix_mx_puppet_steam_database_name }}?sslmode=disable - {%- else if matrix_mx_puppet_steam_database_engine == 'sqlite' -%} + {%- elif matrix_mx_puppet_steam_database_engine == 'sqlite' -%} {{ matrix_mx_puppet_steam_database_engine }}://{{ matrix_mx_puppet_steam_database_file }} {%- endif -%} diff --git a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml index f536029c..91aa264a 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml @@ -71,7 +71,7 @@ matrix_mx_puppet_twitter_database_file: /data/database.db matrix_mx_puppet_twitter_database_connString: >-2 {%- if matrix_mx_puppet_twitter_database_engine == 'postgres' -%} postgresql://{{ matrix_mx_puppet_twitter_database_username }}:{{ matrix_mx_puppet_twitter_database_password }}@{{ matrix_mx_puppet_twitter_database_hostname }}:{{ matrix_mx_puppet_twitter_database_port }}/{{ matrix_mx_puppet_twitter_database_name }}?sslmode=disable - {%- else if matrix_mx_puppet_twitter_database_engine == 'sqlite' -%} + {%- elif matrix_mx_puppet_twitter_database_engine == 'sqlite' -%} {{ matrix_mx_puppet_twitter_database_engine }}://{{ matrix_mx_puppet_twitter_database_file }} {%- endif -%} From 6a3f6fbcb56b8aa2da098d1d4cbd1820a347ee34 Mon Sep 17 00:00:00 2001 From: Tomas Strand Date: Tue, 15 Dec 2020 14:15:40 +0200 Subject: [PATCH 172/217] Missing ` in traefik domains instructions --- docs/configuring-playbook-own-webserver.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuring-playbook-own-webserver.md b/docs/configuring-playbook-own-webserver.md index 8c671df4..c930da40 100644 --- a/docs/configuring-playbook-own-webserver.md +++ b/docs/configuring-playbook-own-webserver.md @@ -144,7 +144,7 @@ matrix_nginx_proxy_container_extra_arguments: - '--label "traefik.enable=true"' # The Nginx proxy container will receive traffic from these subdomains - - '--label "traefik.http.routers.matrix-nginx-proxy.rule=Host(`{{ matrix_server_fqn_matrix }}`,`{{ matrix_server_fqn_element }}`,`{{ matrix_server_fqn_dimension }},`{{ matrix_server_fqn_jitsi }}`)"' + - '--label "traefik.http.routers.matrix-nginx-proxy.rule=Host(`{{ matrix_server_fqn_matrix }}`,`{{ matrix_server_fqn_element }}`,`{{ matrix_server_fqn_dimension }}`,`{{ matrix_server_fqn_jitsi }}`)"' # (The 'web-secure' entrypoint must bind to port 443 in Traefik config) - '--label "traefik.http.routers.matrix-nginx-proxy.entrypoints=web-secure"' From 69f71f48a660a104bd394309edae83026e75636c Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 15 Dec 2020 22:00:24 +0200 Subject: [PATCH 173/217] Upgrade matrix-registration (v0.7.1 -> 0.7.2) and use official image This switches us to a container image maintained by the matrix-registration developer. 0.7.2 also supports a `base_url` configuration option we can use to make it easier to reverse-proxy at a different base URL. We still keep some workarounds, because of this issue: https://github.com/ZerataX/matrix-registration/issues/47 --- roles/matrix-registration/defaults/main.yml | 6 ++++-- roles/matrix-registration/tasks/init.yml | 9 +++------ roles/matrix-registration/templates/config.yaml.j2 | 1 + 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/roles/matrix-registration/defaults/main.yml b/roles/matrix-registration/defaults/main.yml index 95147665..d89bf070 100644 --- a/roles/matrix-registration/defaults/main.yml +++ b/roles/matrix-registration/defaults/main.yml @@ -12,9 +12,9 @@ matrix_registration_config_path: "{{ matrix_registration_base_path }}/config" matrix_registration_data_path: "{{ matrix_registration_base_path }}/data" matrix_registration_docker_src_files_path: "{{ matrix_registration_base_path }}/docker-src" -matrix_registration_version: "v0.7.1" +matrix_registration_version: "v0.7.2" -matrix_registration_docker_image: "{{ matrix_registration_docker_image_name_prefix }}devture/zeratax-matrix-registration:{{ matrix_registration_version }}" +matrix_registration_docker_image: "{{ matrix_registration_docker_image_name_prefix }}zeratax/matrix-registration:{{ matrix_registration_version }}" matrix_registration_docker_image_name_prefix: "{{ 'localhost/' if matrix_registration_container_image_self_build else 'docker.io/' }}" matrix_registration_docker_image_force_pull: "{{ matrix_registration_docker_image.endswith(':latest') }}" @@ -36,6 +36,8 @@ matrix_registration_container_http_host_bind_port: '' # (only applies when matrix-nginx-proxy is used). matrix_registration_public_endpoint: /matrix-registration +matrix_registration_base_url: "{{ matrix_registration_public_endpoint }}" + matrix_registration_api_register_endpoint: "{{ matrix_homeserver_url }}{{ matrix_registration_public_endpoint }}/register" matrix_registration_api_token_endpoint: "{{ matrix_homeserver_url }}{{ matrix_registration_public_endpoint }}/token" diff --git a/roles/matrix-registration/tasks/init.yml b/roles/matrix-registration/tasks/init.yml index 8a7cdc75..bdb3928e 100644 --- a/roles/matrix-registration/tasks/init.yml +++ b/roles/matrix-registration/tasks/init.yml @@ -30,15 +30,12 @@ {% endif %} {# - Workaround matrix-registration serving static files at /static - (see https://github.com/ZerataX/matrix-registration/issues/29) - - Also fixing the form, which goes to /register. + Workaround matrix-registration serving the background image at /static + (see https://github.com/ZerataX/matrix-registration/issues/47) #} sub_filter_once off; - sub_filter_types text/html text/css; + sub_filter_types text/css; sub_filter "/static/" "{{ matrix_registration_public_endpoint }}/static/"; - sub_filter "/register" "{{ matrix_registration_public_endpoint }}/register"; } - name: Register matrix-registration proxying configuration with matrix-nginx-proxy diff --git a/roles/matrix-registration/templates/config.yaml.j2 b/roles/matrix-registration/templates/config.yaml.j2 index f3b1c57b..1b2464e5 100644 --- a/roles/matrix-registration/templates/config.yaml.j2 +++ b/roles/matrix-registration/templates/config.yaml.j2 @@ -28,3 +28,4 @@ logging: # password requirements password: min_length: 8 +base_url: {{ matrix_registration_base_url|to_json }} From 1bd5c240e582bfc4d4d1fcaaea5be326f5186e92 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 15 Dec 2020 23:18:29 +0200 Subject: [PATCH 174/217] Add support for executing additional DB migration statements In cases where pgloader is not enough and we need to do some additional migration work after it, we can now use `additional_psql_statements_list` and `additional_psql_statements_db_name`. This is to be used when migrating `matrix-registration`'s data at the very least. --- .../tasks/util/migrate_db_to_postgres.yml | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml index 5d917538..d01611ef 100644 --- a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml +++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml @@ -99,6 +99,29 @@ command: cmd: "mv {{ matrix_postgres_db_migration_request.src }} {{ matrix_postgres_db_migration_request.src }}.backup" +- block: + # We can't use `{{ role_path }}` here, neither with `import_tasks`, nor with `include_tasks`, + # because it refers to the role that included this util, and not to the role this file belongs to. + - import_tasks: "roles/matrix-postgres/tasks/util/detect_existing_postgres_version.yml" + + - set_fact: + matrix_postgres_docker_image_to_use: "{{ matrix_postgres_docker_image_latest if matrix_postgres_detected_version_corresponding_docker_image == '' else matrix_postgres_detected_version_corresponding_docker_image }}" + + - name: Execute additional Postgres SQL migration statements + command: + cmd: >- + {{ matrix_host_command_docker }} run + --rm + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --env-file={{ matrix_postgres_base_path }}/env-postgres-psql + --network={{ matrix_docker_network }} + {{ matrix_postgres_docker_image_to_use }} + psql --host=matrix-postgres --dbname={{ matrix_postgres_db_migration_request.additional_psql_statements_db_name }} --command='{{ item }}' + with_items: "{{ matrix_postgres_db_migration_request.additional_psql_statements_list }}" + + when: "matrix_postgres_db_migration_request.additional_psql_statements_list|default([])|length > 0" + - name: Inject result set_fact: matrix_playbook_runtime_results: | From a197968b7f95c6f7e6d5f3882cb25e1dc1892124 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 15 Dec 2020 23:19:56 +0200 Subject: [PATCH 175/217] Make matrix-registration use Postgres by default Now that 0.7.2 is out, the Docker image supports Postgres and we can do the (SQLite -> Postgres) migration. I've also found out that we needed to fix up the `tokens.ex_date` column data type a bit to prevent matrix-registration from raising exceptions when comparing `datetime.now()` with `ex_date` coming from the database. Example: > File "/usr/local/lib/python3.8/site-packages/matrix_registration/tokens.py", line 58, in valid > expired = self.ex_date < datetime.now() > TypeError: can't compare offset-naive and offset-aware datetimes --- group_vars/matrix_servers | 5 ++--- roles/matrix-registration/defaults/main.yml | 1 - roles/matrix-registration/tasks/setup_install.yml | 5 +++++ roles/matrix-registration/tasks/validate_config.yml | 7 ------- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 932b288a..fda40efd 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1317,9 +1317,8 @@ matrix_registration_systemd_required_services_list: | (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} -# We'd like to use 'postgres' if matrix_postgres_enabled, but the container image doesn't seem to support that. -# Learn more here: https://github.com/ZerataX/matrix-registration/issues/44 -matrix_registration_database_engine: 'sqlite' +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_registration_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" matrix_registration_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx.registr.db') | to_uuid }}" ###################################################################### diff --git a/roles/matrix-registration/defaults/main.yml b/roles/matrix-registration/defaults/main.yml index b39f02c4..d85faf89 100644 --- a/roles/matrix-registration/defaults/main.yml +++ b/roles/matrix-registration/defaults/main.yml @@ -88,7 +88,6 @@ matrix_registration_admin_secret: "" matrix_registration_riot_instance: "https://riot.im/app/" - # Default matrix-registration configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. # diff --git a/roles/matrix-registration/tasks/setup_install.yml b/roles/matrix-registration/tasks/setup_install.yml index 489bab8b..708cb1df 100644 --- a/roles/matrix-registration/tasks/setup_install.yml +++ b/roles/matrix-registration/tasks/setup_install.yml @@ -18,6 +18,11 @@ engine_variable_name: 'matrix_registration_database_engine' engine_old: 'sqlite' systemd_services_to_stop: ['matrix-registration.service'] + # pgloader makes `ex_date` of type `TIMESTAMP WITH TIMEZONE`, + # which makes matrix-registration choke on it later on when comparing dates. + additional_psql_statements_list: + - ALTER TABLE tokens ALTER COLUMN ex_date TYPE TIMESTAMP WITHOUT TIME ZONE; + additional_psql_statements_db_name: "{{ matrix_registration_database_db_name }}" - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" diff --git a/roles/matrix-registration/tasks/validate_config.yml b/roles/matrix-registration/tasks/validate_config.yml index 80293bcb..90466b46 100644 --- a/roles/matrix-registration/tasks/validate_config.yml +++ b/roles/matrix-registration/tasks/validate_config.yml @@ -18,10 +18,3 @@ when: "item.old in vars" with_items: - {'old': 'matrix_registration_docker_repo', 'new': 'matrix_registration_container_image_self_build_repo'} - -- name: Fail if Postgres usage attempted - fail: - msg: > - matrix-registration doesn't support using Postgres just yet. - Learn more here: https://github.com/ZerataX/matrix-registration/issues/44 - when: "matrix_registration_database_engine == 'postgres'" From a4b8baee494e670c15999be1c2c14b314e6d6f04 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 16 Dec 2020 03:32:31 +0200 Subject: [PATCH 176/217] Fix inability to send (Matrix -> Discord) messages via appservice-discord Revert "Correct inabillity for appservice-discord to connect" This reverts commit 673e19f8307bdfc76dc4d1e63dcab40c84ebb37e. While certain things do work even with such a local URL, sending messages leads to an error like this: > [DiscordBot] verbose: DiscordAPIError: Invalid Form Body > avatar_url: Not a well formed URL. Fixes https://github.com/Half-Shot/matrix-appservice-discord/issues/649 The sample configuration file for appservice-discord https://github.com/Half-Shot/matrix-appservice-discord/blob/c29cfc72f55f1f81e43e2d71f5c080d752d1a884/config/config.sample.yaml#L8 explicitly says that we need a public URL. --- roles/matrix-bridge-appservice-discord/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml index 357b93d6..9ca06b05 100644 --- a/roles/matrix-bridge-appservice-discord/defaults/main.yml +++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml @@ -37,7 +37,7 @@ matrix_appservice_discord_appservice_url: 'http://matrix-appservice-discord:9005 matrix_appservice_discord_bridge_domain: "{{ matrix_domain }}" # As of right now, the homeserver URL must be a public URL. See below. -matrix_appservice_discord_bridge_homeserverUrl: "http://matrix-synapse:8008" +matrix_appservice_discord_bridge_homeserverUrl: "{{ matrix_homeserver_url }}" matrix_appservice_discord_bridge_disablePresence: false matrix_appservice_discord_bridge_enableSelfServiceBridging: false From db69a51653271d8bf563cc2ef952064fa05bb825 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 16 Dec 2020 16:25:38 +0200 Subject: [PATCH 177/217] Do not recommend usePrivilegedIntents for appservice-discord We're getting reports of it being broken and I can confirm that the bridge works well without it, so I don't see why we'd be asking people to enable it. Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/751 --- ...configuring-playbook-bridge-appservice-discord.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/configuring-playbook-bridge-appservice-discord.md b/docs/configuring-playbook-bridge-appservice-discord.md index 11be678e..f3efc555 100644 --- a/docs/configuring-playbook-bridge-appservice-discord.md +++ b/docs/configuring-playbook-bridge-appservice-discord.md @@ -14,20 +14,18 @@ Instructions loosely based on [this](https://github.com/Half-Shot/matrix-appserv 1. Create a Discord Application [here](https://discordapp.com/developers/applications). 2. Retrieve Client ID. 3. Create a bot from the Bot tab and retrieve the Bot token. -4. From the Bot tab, enable all checkboxes related to Privileged Gateway Intents (you can skip this step if you're not using `matrix_appservice_discord_auth_usePrivilegedIntents: true` below) -5. Enable the bridge with the following configuration in your `vars.yml` file: +4. Enable the bridge with the following configuration in your `vars.yml` file: ```yaml matrix_appservice_discord_enabled: true matrix_appservice_discord_client_id: "YOUR DISCORD APP CLIENT ID" matrix_appservice_discord_bot_token: "YOUR DISCORD APP BOT TOKEN" -matrix_appservice_discord_auth_usePrivilegedIntents: true ``` -6. If you've already installed Matrix services using the playbook before, you'll need to re-run it (`--tags=setup-all,start`). If not, proceed with [configuring other playbook services](configuring-playbook.md) and then with [Installing](installing.md). Get back to this guide once ready. -7. Retrieve Discord invite link from the `{{ matrix_appservice_discord_config_path }}/invite_link` file on the server (this defaults to `/matrix/appservice-discord/config/invite_link`). You need to peek at the file on the server via SSH, etc., because it's not available via HTTP(S). -8. Invite the Bot to Discord servers you wish to bridge. Administrator permission is recommended. -9. Room addresses follow this syntax: `#_discord_guildid_channelid`. You can easily find the guild and channel ids by logging into Discord in a browser and opening the desired channel. The URL will have this format: `discordapp.com/channels/guild_id/channel_id`. Once you have figured out the appropriate room addrss, you can join by doing `/join #_discord_guildid_channelid` in your Matrix client. +5. If you've already installed Matrix services using the playbook before, you'll need to re-run it (`--tags=setup-all,start`). If not, proceed with [configuring other playbook services](configuring-playbook.md) and then with [Installing](installing.md). Get back to this guide once ready. +6. Retrieve Discord invite link from the `{{ matrix_appservice_discord_config_path }}/invite_link` file on the server (this defaults to `/matrix/appservice-discord/config/invite_link`). You need to peek at the file on the server via SSH, etc., because it's not available via HTTP(S). +7. Invite the Bot to Discord servers you wish to bridge. Administrator permission is recommended. +8. Room addresses follow this syntax: `#_discord_guildid_channelid`. You can easily find the guild and channel ids by logging into Discord in a browser and opening the desired channel. The URL will have this format: `discordapp.com/channels/guild_id/channel_id`. Once you have figured out the appropriate room addrss, you can join by doing `/join #_discord_guildid_channelid` in your Matrix client. Other configuration options are available via the `matrix_appservice_discord_configuration_extension_yaml` variable. From e2e6cfaa8b66a995aa4b4ef6b6409d071e648fe9 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 16 Dec 2020 18:14:52 +0200 Subject: [PATCH 178/217] Document Ansible's Python interpreter discovery some more Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/570 Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/641 --- docs/prerequisites.md | 2 +- examples/hosts | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/prerequisites.md b/docs/prerequisites.md index daf6ff80..488f6c6f 100644 --- a/docs/prerequisites.md +++ b/docs/prerequisites.md @@ -10,7 +10,7 @@ This playbook doesn't support running on ARM (see [this issue](https://github.co - `root` access to your server (or a user capable of elevating to `root` via `sudo`). -- [Python](https://www.python.org/) being installed on the server. Most distributions install Python by default, but some don't (e.g. Ubuntu 18.04) and require manual installation (something like `apt-get install python`). +- [Python](https://www.python.org/) being installed on the server. Most distributions install Python by default, but some don't (e.g. Ubuntu 18.04) and require manual installation (something like `apt-get install python3`). On some distros, Ansible may incorrectly [detect the Python version](https://docs.ansible.com/ansible/latest/reference_appendices/interpreter_discovery.html) (2 vs 3) and you may need to explicitly specify the interpreter path in `inventory/hosts` during installation (e.g. `ansible_python_interpreter=/usr/bin/python3`) - A `cron`-like tool installed on the server such as `cron` or `anacron` to automatically schedule the Let's Encrypt SSL certificates's renewal. *This can be ignored if you use your own SSL certificates.* diff --git a/examples/hosts b/examples/hosts index 3b14f09e..daf2cfc5 100644 --- a/examples/hosts +++ b/examples/hosts @@ -10,6 +10,9 @@ # # If you're running this Ansible playbook on the same server as the one you're installing to, # consider adding an additional `ansible_connection=local` argument below. +# +# Ansible may fail to discover which Python interpreter to use on the host for some distros (like Ubuntu 20.04). +# You may sometimes need to explicitly add `ansible_python_interpreter=/usr/bin/python3` to lines below. [matrix_servers] matrix. ansible_host= ansible_ssh_user=root From ed159cc7427c9e856b037864bb9e923f5b7fb2ca Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 17 Dec 2020 11:33:18 +0200 Subject: [PATCH 179/217] Move matrix_architecture to matrix-base We were only defining this in `group_vars/matrix_servers`, which is inconsistent with how we normally do things. --- group_vars/matrix_servers | 14 -------------- roles/matrix-base/defaults/main.yml | 6 ++++++ 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 78a03851..f5eb1cb8 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -24,20 +24,6 @@ matrix_identity_server_url: "{{ ('https://' + matrix_server_fqn_matrix) if matri # ###################################################################### -###################################################################### -# -# matrix-architecture -# -###################################################################### - -matrix_architecture: "amd64" - -###################################################################### -# -# /matrix-architecture -# -###################################################################### - ###################################################################### # diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index e0522ba8..aabbbed8 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -23,6 +23,12 @@ matrix_server_fqn_jitsi: "jitsi.{{ matrix_domain }}" matrix_federation_public_port: 8448 +# The architecture that your server runs. +# Recognized values by us are 'amd64', 'arm32' and 'arm64'. +# Not all architectures support all services, so your experience (on non-amd64) may vary. +# See docs/alternative-architectures.md +matrix_architecture: amd64 + matrix_user_username: "matrix" matrix_user_groupname: "matrix" From 55f252a6ed276d000e193158a4390b1315354cb2 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 17 Dec 2020 11:37:30 +0200 Subject: [PATCH 180/217] Do not hardcode amd64 in setup_debian.yml Until now, we've only supported non-amd64 on Raspbian. Seems like there are now people running Debian/Ubuntu on ARM, so we were forcing them into amd64 Docker packages. I've gotten a report that this change fixes support for Ubuntu Server 20.04 on RPi 4B. --- roles/matrix-base/defaults/main.yml | 5 +++++ roles/matrix-base/tasks/server_base/setup_debian.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index aabbbed8..2cc4b42f 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -29,6 +29,11 @@ matrix_federation_public_port: 8448 # See docs/alternative-architectures.md matrix_architecture: amd64 +# The architecture for Debian packages. +# See: https://wiki.debian.org/SupportedArchitectures +# We just remap from our `matrix_architecture` values to what Debian and possibly other distros call things. +matrix_debian_arch: "{{ 'armhf' if matrix_architecture == 'arm32' else matrix_architecture }}" + matrix_user_username: "matrix" matrix_user_groupname: "matrix" diff --git a/roles/matrix-base/tasks/server_base/setup_debian.yml b/roles/matrix-base/tasks/server_base/setup_debian.yml index 6d8d18fa..e30d3b93 100644 --- a/roles/matrix-base/tasks/server_base/setup_debian.yml +++ b/roles/matrix-base/tasks/server_base/setup_debian.yml @@ -20,7 +20,7 @@ - name: Ensure Docker repository is enabled apt_repository: - repo: "deb [arch=amd64] https://download.docker.com/linux/{{ ansible_distribution|lower }} {{ ansible_distribution_release }} stable" + repo: "deb [arch={{ matrix_debian_arch }}] https://download.docker.com/linux/{{ ansible_distribution|lower }} {{ ansible_distribution_release }} stable" state: present update_cache: yes when: matrix_docker_installation_enabled|bool and matrix_docker_package_name == 'docker-ce' From f545de53f742e4443bf26608e67da786b8d6dc16 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 17 Dec 2020 11:39:18 +0200 Subject: [PATCH 181/217] Do not hardcode "ubuntu" for the Docker APT key URL Well, `ubuntu` or `debian`, the same key is served right now, so it doesn't really matter. This seems cleaner and less prone to breakage though. --- roles/matrix-base/tasks/server_base/setup_debian.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-base/tasks/server_base/setup_debian.yml b/roles/matrix-base/tasks/server_base/setup_debian.yml index e30d3b93..37706d1f 100644 --- a/roles/matrix-base/tasks/server_base/setup_debian.yml +++ b/roles/matrix-base/tasks/server_base/setup_debian.yml @@ -11,7 +11,7 @@ - name: Ensure Docker's APT key is trusted apt_key: - url: https://download.docker.com/linux/ubuntu/gpg + url: "https://download.docker.com/linux/{{ ansible_distribution|lower }}/gpg" id: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 state: present register: add_repository_key From a09ed58892965241f0ee73b11139b5cfb5995cde Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 17 Dec 2020 11:45:32 +0200 Subject: [PATCH 182/217] Ensure gnupg installed on Raspbian It's likely installed by default, but it doesn't hurt to specify it. It also makes us more the same with `setup_debian.yml`. --- roles/matrix-base/tasks/server_base/setup_raspbian.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/matrix-base/tasks/server_base/setup_raspbian.yml b/roles/matrix-base/tasks/server_base/setup_raspbian.yml index a9a5d20a..a84228a5 100644 --- a/roles/matrix-base/tasks/server_base/setup_raspbian.yml +++ b/roles/matrix-base/tasks/server_base/setup_raspbian.yml @@ -5,6 +5,7 @@ name: - apt-transport-https - ca-certificates + - gnupg state: present update_cache: yes From 349fbb64343bc3a877db02948ef6e715fa8c0128 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 17 Dec 2020 11:47:34 +0200 Subject: [PATCH 183/217] Do not hardcode armhf for Raspbian Raspbian doesn't seem to support arm64, so this is somewhat pointless right now. However, they might in the future. Doing this should also unify us some more with `setup_debian.yml` with the ultimate goal of eliminating `setup_raspbian.yml`. --- roles/matrix-base/tasks/server_base/setup_raspbian.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-base/tasks/server_base/setup_raspbian.yml b/roles/matrix-base/tasks/server_base/setup_raspbian.yml index a84228a5..daf6c165 100644 --- a/roles/matrix-base/tasks/server_base/setup_raspbian.yml +++ b/roles/matrix-base/tasks/server_base/setup_raspbian.yml @@ -20,7 +20,7 @@ - name: Ensure Docker repository is enabled apt_repository: - repo: "deb [arch=armhf] https://download.docker.com/linux/raspbian {{ ansible_distribution_release }} stable" + repo: "deb [arch={{ matrix_debian_arch }}] https://download.docker.com/linux/raspbian {{ ansible_distribution_release }} stable" state: present update_cache: yes when: matrix_docker_installation_enabled|bool and matrix_docker_package_name == 'docker-ce' From 8748f3d443d7b7388f41d06f2efab17f291735eb Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 17 Dec 2020 11:49:56 +0200 Subject: [PATCH 184/217] Move python{,3}-docker installation to another task This also adds support for installing python3-docker (not python-docker) in systems that run Python 3. --- roles/matrix-base/tasks/server_base/setup_raspbian.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-base/tasks/server_base/setup_raspbian.yml b/roles/matrix-base/tasks/server_base/setup_raspbian.yml index daf6c165..421905a0 100644 --- a/roles/matrix-base/tasks/server_base/setup_raspbian.yml +++ b/roles/matrix-base/tasks/server_base/setup_raspbian.yml @@ -28,7 +28,6 @@ - name: Ensure APT packages are installed apt: name: - - python-docker - "{{ matrix_ntpd_package }}" - fuse state: latest @@ -38,5 +37,6 @@ apt: name: - "{{ matrix_docker_package_name }}" + - "python{{'3' if ansible_python.version.major == 3 else ''}}-docker" state: latest when: matrix_docker_installation_enabled|bool From d0ee86e0a54f5ae93c872550d8b66f5b660d0b1d Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 21 Dec 2020 15:44:14 +0200 Subject: [PATCH 185/217] Fix matrix_corporal_docker_image_name_prefix referencing matrix_synapse_ stuff --- roles/matrix-corporal/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-corporal/defaults/main.yml b/roles/matrix-corporal/defaults/main.yml index 1cab3119..cccaadd0 100644 --- a/roles/matrix-corporal/defaults/main.yml +++ b/roles/matrix-corporal/defaults/main.yml @@ -23,7 +23,7 @@ matrix_corporal_container_extra_arguments: [] matrix_corporal_systemd_required_services_list: ['docker.service'] matrix_corporal_docker_image: "{{ matrix_corporal_docker_image_name_prefix }}devture/matrix-corporal:{{ matrix_corporal_docker_image_tag }}" -matrix_corporal_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_container_image_self_build else 'docker.io/' }}" +matrix_corporal_docker_image_name_prefix: "{{ 'localhost/' if matrix_corporal_container_image_self_build else 'docker.io/' }}" matrix_corporal_docker_image_tag: "1.11.0" matrix_corporal_docker_image_force_pull: "{{ matrix_corporal_docker_image.endswith(':latest') }}" From dcd47166366ad5272525513f3aed409b752057a8 Mon Sep 17 00:00:00 2001 From: louis Date: Sun, 20 Dec 2020 17:30:28 +0100 Subject: [PATCH 186/217] add option to disable nginx access log --- docs/configuring-playbook-nginx.md | 9 ++++++++- roles/matrix-nginx-proxy/defaults/main.yml | 2 ++ roles/matrix-nginx-proxy/templates/nginx/nginx.conf.j2 | 4 ++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/configuring-playbook-nginx.md b/docs/configuring-playbook-nginx.md index 945864e9..5693c569 100644 --- a/docs/configuring-playbook-nginx.md +++ b/docs/configuring-playbook-nginx.md @@ -24,7 +24,6 @@ matrix_nginx_proxy_proxy_matrix_nginx_status_allowed_addresses: - 1.1.1.1 ``` - ## Synapse + OpenID Connect for Single-Sign-On If you want to use OpenID Connect as an SSO provider (as per the [Synapse OpenID docs](https://github.com/matrix-org/synapse/blob/develop/docs/openid.md)), you need to use the following configuration (in your `vars.yml` file) to instruct nginx to forward `/_synapse/oidc` to Synapse: @@ -32,3 +31,11 @@ If you want to use OpenID Connect as an SSO provider (as per the [Synapse OpenID ```yaml matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_oidc_api_enabled: true ``` + +## Disable Nginx access logs + +This will disable the access logging for nginx. + +```yaml +matrix_nginx_proxy_access_log_enabled: false +``` diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 7331c589..b18bedde 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -92,6 +92,8 @@ matrix_nginx_proxy_base_domain_homepage_template: |- +# Option to disable the access log +matrix_nginx_proxy_access_log_enabled: true # Controls whether proxying the riot domain should be done. matrix_nginx_proxy_proxy_riot_compat_redirect_enabled: false diff --git a/roles/matrix-nginx-proxy/templates/nginx/nginx.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/nginx.conf.j2 index 51aa8a00..975c8b4f 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/nginx.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/nginx.conf.j2 @@ -33,7 +33,11 @@ http { '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; + {% if matrix_nginx_proxy_access_log_enabled %} access_log /var/log/nginx/access.log main; + {% else %} + access_log off; + {% endif %} sendfile on; #tcp_nopush on; From 6488e11d692d0a44fc5c53e81d986ada35c92be4 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 22 Dec 2020 10:52:36 +0200 Subject: [PATCH 187/217] Relocate some tasks --- .../tasks/setup_install.yml | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml index c3f2b01f..9022f309 100644 --- a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml @@ -8,6 +8,23 @@ The matrix-bridge-mx-puppet-discord role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" +- name: Check if an old database file already exists + stat: + path: "{{ matrix_mx_puppet_discord_base_path }}/database.db" + register: matrix_mx_puppet_discord_stat_database + +- block: + - name: (Data relocation) Ensure matrix-mx-puppet-discord.service is stopped + service: + name: matrix-mx-puppet-discord + state: stopped + daemon_reload: yes + failed_when: False + + - name: (Data relocation) Move mx-puppet-discord database file to ./data directory + command: "mv {{ matrix_mx_puppet_discord_base_path }}/database.db {{ matrix_mx_puppet_discord_data_path }}/database.db" + when: "matrix_mx_puppet_discord_stat_database.stat.exists" + - name: Ensure MX Puppet Discord image is pulled docker_image: name: "{{ matrix_mx_puppet_discord_docker_image }}" @@ -49,23 +66,6 @@ pull: yes when: "matrix_mx_puppet_discord_enabled|bool and matrix_mx_puppet_discord_container_image_self_build|bool" -- name: Check if an old database file already exists - stat: - path: "{{ matrix_mx_puppet_discord_base_path }}/database.db" - register: matrix_mx_puppet_discord_stat_database - -- name: (Data relocation) Ensure matrix-mx-puppet-discord.service is stopped - service: - name: matrix-mx-puppet-discord - state: stopped - daemon_reload: yes - failed_when: false - when: "matrix_mx_puppet_discord_stat_database.stat.exists" - -- name: (Data relocation) Move mx-puppet-discord database file to ./data directory - command: "mv {{ matrix_mx_puppet_discord_base_path }}/database.db {{ matrix_mx_puppet_discord_data_path }}/database.db" - when: "matrix_mx_puppet_discord_stat_database.stat.exists" - - name: Ensure mx-puppet-discord config.yaml installed copy: content: "{{ matrix_mx_puppet_discord_configuration|to_nice_yaml }}" From 9b4bf7358784f4f7df163cbf922e643dca5f0977 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 22 Dec 2020 11:08:07 +0200 Subject: [PATCH 188/217] Fix undefined variable reference --- roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 index c24c7e54..27f0c567 100644 --- a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 @@ -100,7 +100,7 @@ namePatterns: group: :name database: -{% if matrix_mx_puppet_slack_discord_database_engine == 'sqlite' %} +{% if matrix_mx_puppet_discord_database_engine == 'sqlite' %} # Use SQLite3 as a database backend # The name of the database file filename: /data/database.db From 149872e00c59960494877bac032b9a4d0a5d40ea Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 22 Dec 2020 11:10:10 +0200 Subject: [PATCH 189/217] Add support for automatic (Postgres -> SQLite) migration to mx-puppet-discord --- .../defaults/main.yml | 15 +++++------ .../tasks/setup_install.yml | 26 +++++++++++++++++++ .../tasks/validate_config.yml | 20 -------------- .../templates/config.yaml.j2 | 4 +-- 4 files changed, 35 insertions(+), 30 deletions(-) diff --git a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml index a0ad1c0a..ad86a241 100644 --- a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml @@ -57,18 +57,17 @@ matrix_mx_puppet_discord_login_shared_secret: '' # Database configuration matrix_mx_puppet_discord_database_engine: 'sqlite' + +matrix_mx_puppet_discord_sqlite_database_path_local: "{{ matrix_mx_puppet_discord_data_path }}/database.db" +matrix_mx_puppet_discord_sqlite_database_path_in_container: "/data/database.db" + matrix_mx_puppet_discord_database_username: matrix_mx_puppet_discord matrix_mx_puppet_discord_database_password: ~ matrix_mx_puppet_discord_database_hostname: 'matrix-postgres' matrix_mx_puppet_discord_database_port: 5432 -matrix_mx_puppet_discord_database_name: matrix_mx_puppet_discord -matrix_mx_puppet_discord_database_file: /data/database.db -matrix_mx_puppet_discord_database_connString: >-2 - {%- if matrix_mx_puppet_discord_database_engine == 'postgres' -%} - postgresql://{{ matrix_mx_puppet_discord_database_username }}:{{ matrix_mx_puppet_discord_database_password }}@{{ matrix_mx_puppet_discord_database_hostname }}:{{ matrix_mx_puppet_discord_database_port }}/{{ matrix_mx_puppet_discord_database_name }}?sslmode=disable - {%- elif matrix_mx_puppet_discord_database_engine == 'sqlite' -%} - {{ matrix_mx_puppet_discord_database_engine }}://{{ matrix_mx_puppet_discord_database_file }} - {%- endif -%} +matrix_mx_puppet_discord_database_db_name: matrix_mx_puppet_discord + +matrix_mx_puppet_discord_database_connection_string: 'postgresql://{{ matrix_mx_puppet_discord_database_username }}:{{ matrix_mx_puppet_discord_database_password }}@{{ matrix_mx_puppet_discord_database_hostname }}:{{ matrix_mx_puppet_discord_database_port }}/{{ matrix_mx_puppet_discord_database_db_name }}?sslmode=disable' # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml index 9022f309..08a199c4 100644 --- a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml @@ -25,6 +25,32 @@ command: "mv {{ matrix_mx_puppet_discord_base_path }}/database.db {{ matrix_mx_puppet_discord_data_path }}/database.db" when: "matrix_mx_puppet_discord_stat_database.stat.exists" +- set_fact: + matrix_mx_puppet_discord_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mx_puppet_discord_sqlite_database_path_local }}" + register: matrix_mx_puppet_discord_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_mx_puppet_discord_sqlite_database_path_local }}" + dst: "{{ matrix_mx_puppet_discord_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_mx_puppet_discord_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-mx-puppet-discord.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_mx_puppet_discord_requires_restart: true + when: "matrix_mx_puppet_discord_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_mx_puppet_discord_database_engine == 'postgres'" + - name: Ensure MX Puppet Discord image is pulled docker_image: name: "{{ matrix_mx_puppet_discord_docker_image }}" diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml index 8db47ed1..c253eda2 100644 --- a/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml +++ b/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml @@ -8,23 +8,3 @@ with_items: - "matrix_mx_puppet_discord_appservice_token" - "matrix_mx_puppet_discord_homeserver_token" - -- block: - - name: Check if a SQLite database already exists - stat: - path: "{{ matrix_mx_puppet_discord_data_path }}/database.db" - register: matrix_mx_puppet_discord_sqlite_database_path_local_stat_result - - - name: Fail if an SQLite database already exists when using Postgres - fail: - msg: >-2 - matrix_mx_puppet_discord_database_engine has been set to `postgres` (which is our new default now). - However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_discord_data_path }}/database.db. - It appears that you've been using this bridge with the SQLite engine until now. - To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_discord_database_engine: sqlite` to your vars.yml file and re-run this same command. - Alternatively, to migrate your existing SQLite database to Postgres: - 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_discord_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_discord_database_connString'`) - 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) - when: "matrix_mx_puppet_discord_sqlite_database_path_local_stat_result.stat.exists" - when: "matrix_mx_puppet_discord_database_engine == 'postgres'" diff --git a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 index 27f0c567..1f4548d8 100644 --- a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 @@ -103,14 +103,14 @@ database: {% if matrix_mx_puppet_discord_database_engine == 'sqlite' %} # Use SQLite3 as a database backend # The name of the database file - filename: /data/database.db + filename: {{ matrix_mx_puppet_discord_sqlite_database_path_in_container|to_json }} {% else %} # Use Postgres as a database backend # If set, will be used instead of SQLite3 # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - connString: {{ matrix_mx_puppet_discord_database_connString|to_json }} + connString: {{ matrix_mx_puppet_discord_database_connection_string|to_json }} {% endif %} logging: From e64758c11951b7d2720431b139be0edc454085bc Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 22 Dec 2020 13:24:53 +0200 Subject: [PATCH 190/217] Add missing restart task Should have been part of 149872e00c5 --- .../matrix-bridge-mx-puppet-discord/tasks/setup_install.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml index 08a199c4..9801bf82 100644 --- a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml @@ -119,3 +119,9 @@ service: daemon_reload: yes when: "matrix_mx_puppet_discord_systemd_service_result.changed" + +- name: Ensure matrix-mx-puppet-discord.service restarted, if necessary + service: + name: "matrix-mx-puppet-discord.service" + state: restarted + when: "matrix_mx_puppet_discord_requires_restart|bool" From 44c9f4daca05fca27aa1fe8399349a34a9837015 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 22 Dec 2020 13:30:52 +0200 Subject: [PATCH 191/217] Add support for automatic (Postgres -> SQLite) migration to mx-puppet-instagram --- .../defaults/main.yml | 15 ++++----- .../tasks/setup_install.yml | 33 ++++++++++++++++++- .../tasks/validate_config.yml | 20 ----------- .../templates/config.yaml.j2 | 4 +-- 4 files changed, 41 insertions(+), 31 deletions(-) diff --git a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml index 4c40bda4..61bf3dcf 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml @@ -47,18 +47,17 @@ matrix_mx_puppet_instagram_homeserver_token: '' matrix_mx_puppet_instagram_login_shared_secret: '' matrix_mx_puppet_instagram_database_engine: sqlite + +matrix_mx_puppet_instagram_sqlite_database_path_local: "{{ matrix_mx_puppet_instagram_data_path }}/database.db" +matrix_mx_puppet_instagram_sqlite_database_path_in_container: "/data/database.db" + matrix_mx_puppet_instagram_database_username: matrix_mx_puppet_instagram matrix_mx_puppet_instagram_database_password: ~ matrix_mx_puppet_instagram_database_hostname: 'matrix-postgres' matrix_mx_puppet_instagram_database_port: 5432 -matrix_mx_puppet_instagram_database_name: matrix_mx_puppet_instagram -matrix_mx_puppet_instagram_database_file: /data/database.db -matrix_mx_puppet_instagram_database_connString: >-2 - {%- if matrix_mx_puppet_instagram_database_engine == 'postgres' -%} - postgresql://{{ matrix_mx_puppet_instagram_database_username }}:{{ matrix_mx_puppet_instagram_database_password }}@{{ matrix_mx_puppet_instagram_database_hostname }}:{{ matrix_mx_puppet_instagram_database_port }}/{{ matrix_mx_puppet_instagram_database_name }}?sslmode=disable - {%- elif matrix_mx_puppet_instagram_database_engine == 'sqlite' -%} - {{ matrix_mx_puppet_instagram_database_engine }}://{{ matrix_mx_puppet_instagram_database_file }} - {%- endif -%} +matrix_mx_puppet_instagram_database_db_name: matrix_mx_puppet_instagram + +matrix_mx_puppet_instagram_database_connection_string: 'postgresql://{{ matrix_mx_puppet_instagram_database_username }}:{{ matrix_mx_puppet_instagram_database_password }}@{{ matrix_mx_puppet_instagram_database_hostname }}:{{ matrix_mx_puppet_instagram_database_port }}/{{ matrix_mx_puppet_instagram_database_db_name }}?sslmode=disable' # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. diff --git a/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml index 78bd78c1..76bbd629 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml @@ -8,6 +8,33 @@ The matrix-bridge-mx-puppet-instagram role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" + +- set_fact: + matrix_mx_puppet_instagram_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mx_puppet_instagram_sqlite_database_path_local }}" + register: matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_mx_puppet_instagram_sqlite_database_path_local }}" + dst: "{{ matrix_mx_puppet_instagram_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_mx_puppet_instagram_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-mx-puppet-instagram.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_mx_puppet_instagram_requires_restart: true + when: "matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_mx_puppet_instagram_database_engine == 'postgres'" + - name: Ensure mx-puppet-instagram image is pulled docker_image: name: "{{ matrix_mx_puppet_instagram_docker_image }}" @@ -77,4 +104,8 @@ daemon_reload: yes when: "matrix_mx_puppet_instagram_systemd_service_result.changed" - +- name: Ensure matrix-mx-puppet-instagram.service restarted, if necessary + service: + name: "matrix-mx-puppet-instagram.service" + state: restarted + when: "matrix_mx_puppet_instagram_requires_restart|bool" diff --git a/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml index c2115cbb..b6d9d994 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml +++ b/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml @@ -8,23 +8,3 @@ with_items: - "matrix_mx_puppet_instagram_appservice_token" - "matrix_mx_puppet_instagram_homeserver_token" - -- block: - - name: Check if an SQLite database already exists - stat: - path: "{{ matrix_mx_puppet_instagram_data_path }}/database.db" - register: matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result - - - name: Fail if an SQLite database already exists when using Postgres - fail: - msg: >- - matrix_mx_puppet_instagram_database_engine has been set to `postgres` (which is our new default now). - However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_instagram_data_path }}/database.db. - It appears that you've been using this bridge with the SQLite engine until now. - To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_instagram_database_engine: sqlite` to your vars.yml file and re-run this same command. - Alternatively, to migrate your existing SQLite database to Postgres: - 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_instagram_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_instagram_database_connString'`) - 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) - when: "matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result.stat.exists" - when: "matrix_mx_puppet_instagram_database_engine == 'postgres'" diff --git a/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 index d89c5bdb..b830da2b 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 @@ -50,11 +50,11 @@ database: # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - connString: {{ matrix_mx_puppet_instagram_database_connString | to_json }} + connString: {{ matrix_mx_puppet_instagram_database_connection_string|to_json }} {% else %} # Use SQLite3 as a database backend # The name of the database file - filename: {{ matrix_mx_puppet_instagram_database_file }} + filename: {{ matrix_mx_puppet_instagram_sqlite_database_path_in_container|to_json }} {% endif %} logging: From d135cd9cd3117ca659459db626d8ed942d474325 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 22 Dec 2020 13:44:33 +0200 Subject: [PATCH 192/217] Ensure mx-puppet-discord directories are created before attempting migration Our old (base-path -> data-path) SQLite migration can't work otherwise. It's probably not necessary to keep it anymore, but since we still do, at least we should take care to ensure it works. --- .../tasks/setup_install.yml | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml index 9801bf82..3e3b2f94 100644 --- a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml @@ -8,6 +8,20 @@ The matrix-bridge-mx-puppet-discord role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" +- name: Ensure MX Puppet Discord paths exist + file: + path: "{{ item.path }}" + state: directory + mode: 0750 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + with_items: + - { path: "{{ matrix_mx_puppet_discord_base_path }}", when: true } + - { path: "{{ matrix_mx_puppet_discord_config_path }}", when: true } + - { path: "{{ matrix_mx_puppet_discord_data_path }}", when: true } + - { path: "{{ matrix_mx_puppet_discord_docker_src_files_path }}", when: "{{ matrix_mx_puppet_discord_container_image_self_build }}" } + when: matrix_mx_puppet_discord_enabled|bool and item.when|bool + - name: Check if an old database file already exists stat: path: "{{ matrix_mx_puppet_discord_base_path }}/database.db" @@ -59,20 +73,6 @@ force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_discord_docker_image_force_pull }}" when: matrix_mx_puppet_discord_enabled|bool and not matrix_mx_puppet_discord_container_image_self_build -- name: Ensure MX Puppet Discord paths exist - file: - path: "{{ item.path }}" - state: directory - mode: 0750 - owner: "{{ matrix_user_username }}" - group: "{{ matrix_user_groupname }}" - with_items: - - { path: "{{ matrix_mx_puppet_discord_base_path }}", when: true } - - { path: "{{ matrix_mx_puppet_discord_config_path }}", when: true } - - { path: "{{ matrix_mx_puppet_discord_data_path }}", when: true } - - { path: "{{ matrix_mx_puppet_discord_docker_src_files_path }}", when: "{{ matrix_mx_puppet_discord_container_image_self_build }}" } - when: matrix_mx_puppet_discord_enabled|bool and item.when|bool - - name: Ensure MX Puppet Discord repository is present on self build git: repo: "{{ matrix_mx_puppet_discord_container_image_self_build_repo }}" From 10e0fa17adffde2b79048d597828f8beed00d05c Mon Sep 17 00:00:00 2001 From: Dan Arnfield Date: Tue, 22 Dec 2020 08:23:37 -0600 Subject: [PATCH 193/217] Update nginx (1.19.5 -> 1.19.6) --- roles/matrix-nginx-proxy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index b18bedde..61653db4 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -3,7 +3,7 @@ matrix_nginx_proxy_enabled: true # We use an official nginx image, which we fix-up to run unprivileged. # An alternative would be an `nginxinc/nginx-unprivileged` image, but # that is frequently out of date. -matrix_nginx_proxy_docker_image: "docker.io/nginx:1.19.5-alpine" +matrix_nginx_proxy_docker_image: "docker.io/nginx:1.19.6-alpine" matrix_nginx_proxy_docker_image_force_pull: "{{ matrix_nginx_proxy_docker_image.endswith(':latest') }}" matrix_nginx_proxy_base_path: "{{ matrix_base_data_path }}/nginx-proxy" From c3b63c6c97e99c81af3fe5fda48a72979dc8b5af Mon Sep 17 00:00:00 2001 From: Dan Arnfield Date: Tue, 22 Dec 2020 08:29:37 -0600 Subject: [PATCH 194/217] Update element-web (1.7.15 -> 1.7.16) --- roles/matrix-client-element/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index 8b032ac2..d0297193 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -3,7 +3,7 @@ matrix_client_element_enabled: true matrix_client_element_container_image_self_build: false matrix_client_element_container_image_self_build_repo: "https://github.com/vector-im/riot-web.git" -matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:v1.7.15" +matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:v1.7.16" matrix_client_element_docker_image_name_prefix: "{{ 'localhost/' if matrix_client_element_container_image_self_build else 'docker.io/' }}" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" From e49eb078a25bab1e320ac6d5c56f58461f8b19ad Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 22 Dec 2020 16:29:47 +0200 Subject: [PATCH 195/217] Add support for automatic (Postgres -> SQLite) migration to mx-puppet-skype --- .../defaults/main.yml | 15 ++-- .../tasks/setup_install.yml | 82 +++++++++++++------ .../tasks/validate_config.yml | 20 ----- .../templates/config.yaml.j2 | 4 +- 4 files changed, 66 insertions(+), 55 deletions(-) diff --git a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml index db0f4ede..f3e17a59 100644 --- a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml @@ -55,18 +55,17 @@ matrix_mx_puppet_skype_login_shared_secret: '' # Database configuration, role default is `sqlite` but playbook default is `postgres` matrix_mx_puppet_skype_database_engine: sqlite + +matrix_mx_puppet_skype_sqlite_database_path_local: "{{ matrix_mx_puppet_skype_data_path }}/database.db" +matrix_mx_puppet_skype_sqlite_database_path_in_container: "/data/database.db" + matrix_mx_puppet_skype_database_username: matrix_mx_puppet_skype matrix_mx_puppet_skype_database_password: ~ matrix_mx_puppet_skype_database_hostname: 'matrix-postgres' matrix_mx_puppet_skype_database_port: 5432 -matrix_mx_puppet_skype_database_name: matrix_mx_puppet_skype -matrix_mx_puppet_skype_database_file: /data/database.db -matrix_mx_puppet_skype_database_connString: >-2 - {%- if matrix_mx_puppet_skype_database_engine == 'postgres' -%} - postgresql://{{ matrix_mx_puppet_skype_database_username }}:{{ matrix_mx_puppet_skype_database_password }}@{{ matrix_mx_puppet_skype_database_hostname }}:{{ matrix_mx_puppet_skype_database_port }}/{{ matrix_mx_puppet_skype_database_name }}?sslmode=disable - {%- elif matrix_mx_puppet_skype_database_engine == 'sqlite' -%} - {{ matrix_mx_puppet_skype_database_engine }}://{{ matrix_mx_puppet_skype_database_file }} - {%- endif -%} +matrix_mx_puppet_skype_database_db_name: matrix_mx_puppet_skype + +matrix_mx_puppet_skype_database_connection_string: 'postgresql://{{ matrix_mx_puppet_skype_database_username }}:{{ matrix_mx_puppet_skype_database_password }}@{{ matrix_mx_puppet_skype_database_hostname }}:{{ matrix_mx_puppet_skype_database_port }}/{{ matrix_mx_puppet_skype_database_db_name }}?sslmode=disable' # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. diff --git a/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml index fb5185ed..9289a793 100644 --- a/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml @@ -8,14 +8,6 @@ The matrix-bridge-mx-puppet-skype role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" -- name: Ensure MX Puppet Skype image is pulled - docker_image: - name: "{{ matrix_mx_puppet_skype_docker_image }}" - source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" - force_source: "{{ matrix_mx_puppet_skype_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_skype_docker_image_force_pull }}" - when: matrix_mx_puppet_skype_enabled|bool and not matrix_mx_puppet_skype_container_image_self_build - - name: Ensure MX Puppet Skype paths exist file: path: "{{ item.path }}" @@ -30,6 +22,57 @@ - { path: "{{ matrix_mx_puppet_skype_docker_src_files_path }}", when: "{{ matrix_mx_puppet_skype_container_image_self_build }}" } when: matrix_mx_puppet_skype_enabled|bool and item.when|bool +- name: Check if an old database file already exists + stat: + path: "{{ matrix_mx_puppet_skype_base_path }}/database.db" + register: matrix_mx_puppet_skype_stat_database + +- name: (Data relocation) Ensure matrix-mx-puppet-skype.service is stopped + service: + name: matrix-mx-puppet-skype + state: stopped + daemon_reload: yes + failed_when: false + when: "matrix_mx_puppet_skype_stat_database.stat.exists" + +- name: (Data relocation) Move mx-puppet-skype database file to ./data directory + command: "mv {{ matrix_mx_puppet_skype_base_path }}/database.db {{ matrix_mx_puppet_skype_data_path }}/database.db" + when: "matrix_mx_puppet_skype_stat_database.stat.exists" + +- set_fact: + matrix_mx_puppet_skype_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mx_puppet_skype_sqlite_database_path_local }}" + register: matrix_mx_puppet_skype_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_mx_puppet_skype_sqlite_database_path_local }}" + dst: "{{ matrix_mx_puppet_skype_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_mx_puppet_skype_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-mx-puppet-skype.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_mx_puppet_skype_requires_restart: true + when: "matrix_mx_puppet_skype_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_mx_puppet_skype_database_engine == 'postgres'" + +- name: Ensure MX Puppet Skype image is pulled + docker_image: + name: "{{ matrix_mx_puppet_skype_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_mx_puppet_skype_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_skype_docker_image_force_pull }}" + when: matrix_mx_puppet_skype_enabled|bool and not matrix_mx_puppet_skype_container_image_self_build + - name: Ensure MX Puppet Skype repository is present on self build git: repo: "{{ matrix_mx_puppet_skype_container_image_self_build_repo }}" @@ -49,23 +92,6 @@ pull: yes when: "matrix_mx_puppet_skype_enabled|bool and matrix_mx_puppet_skype_container_image_self_build|bool" -- name: Check if an old database file already exists - stat: - path: "{{ matrix_mx_puppet_skype_base_path }}/database.db" - register: matrix_mx_puppet_skype_stat_database - -- name: (Data relocation) Ensure matrix-mx-puppet-skype.service is stopped - service: - name: matrix-mx-puppet-skype - state: stopped - daemon_reload: yes - failed_when: false - when: "matrix_mx_puppet_skype_stat_database.stat.exists" - -- name: (Data relocation) Move mx-puppet-skype database file to ./data directory - command: "mv {{ matrix_mx_puppet_skype_base_path }}/database.db {{ matrix_mx_puppet_skype_data_path }}/database.db" - when: "matrix_mx_puppet_skype_stat_database.stat.exists" - - name: Ensure mx-puppet-skype config.yaml installed copy: content: "{{ matrix_mx_puppet_skype_configuration|to_nice_yaml }}" @@ -93,3 +119,9 @@ service: daemon_reload: yes when: "matrix_mx_puppet_skype_systemd_service_result.changed" + +- name: Ensure matrix-mx-puppet-skype.service restarted, if necessary + service: + name: "matrix-mx-puppet-skype.service" + state: restarted + when: "matrix_mx_puppet_skype_requires_restart|bool" diff --git a/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml index c7100e51..7ed433b1 100644 --- a/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml +++ b/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml @@ -8,23 +8,3 @@ with_items: - "matrix_mx_puppet_skype_appservice_token" - "matrix_mx_puppet_skype_homeserver_token" - -- block: - - name: Check if an SQLite database already exists - stat: - path: "{{ matrix_mx_puppet_skype_data_path }}/database.db" - register: matrix_mx_puppet_skype_sqlite_database_path_local_stat_result - - - name: Fail if an SQLite database already exists when using Postgres - fail: - msg: >- - matrix_mx_puppet_skype_database_engine has been set to `postgres` (which is our new default now). - However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_skype_data_path }}/database.db. - It appears that you've been using this bridge with the SQLite engine until now. - To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_skype_database_engine: sqlite` to your vars.yml file and re-run this same command. - Alternatively, to migrate your existing SQLite database to Postgres: - 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_skype_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_skype_database_connString'`) - 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) - when: "matrix_mx_puppet_skype_sqlite_database_path_local_stat_result.stat.exists" - when: "matrix_mx_puppet_skype_database_engine == 'postgres'" diff --git a/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 index a32523b1..d41d3a23 100644 --- a/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 @@ -74,11 +74,11 @@ database: # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - connString: {{ matrix_mx_puppet_skype_database_connString | to_json }} + connString: {{ matrix_mx_puppet_skype_database_connection_string|to_json }} {% else %} # Use SQLite3 as a database backend # The name of the database file - filename: {{ matrix_mx_puppet_skype_database_file }} + filename: {{ matrix_mx_puppet_skype_sqlite_database_path_in_container|to_json }} {% endif %} provisioning: From 262a25f997a032826b0348f3d966bcac55b9959e Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 22 Dec 2020 16:39:21 +0200 Subject: [PATCH 196/217] Add support for automatic (Postgres -> SQLite) migration to mx-puppet-slack --- .../defaults/main.yml | 15 ++-- .../tasks/setup_install.yml | 74 +++++++++++++------ .../tasks/validate_config.yml | 20 ----- .../templates/config.yaml.j2 | 4 +- 4 files changed, 62 insertions(+), 51 deletions(-) diff --git a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml index 2ed72611..9aad602a 100644 --- a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml @@ -59,18 +59,17 @@ matrix_mx_puppet_slack_login_shared_secret: '' # Database configuration, role uses 'sqlite' per default but playbook sets up postgres by default matrix_mx_puppet_slack_database_engine: sqlite + +matrix_mx_puppet_slack_sqlite_database_path_local: "{{ matrix_mx_puppet_slack_data_path }}/database.db" +matrix_mx_puppet_slack_sqlite_database_path_in_container: "/data/database.db" + matrix_mx_puppet_slack_database_username: matrix_mx_puppet_slack matrix_mx_puppet_slack_database_password: ~ matrix_mx_puppet_slack_database_hostname: 'matrix-postgres' matrix_mx_puppet_slack_database_port: 5432 -matrix_mx_puppet_slack_database_name: matrix_mx_puppet_slack -matrix_mx_puppet_slack_database_file: /data/database.db -matrix_mx_puppet_slack_database_connString: >-2 - {%- if matrix_mx_puppet_slack_database_engine == 'postgres' -%} - postgresql://{{ matrix_mx_puppet_slack_database_username }}:{{ matrix_mx_puppet_slack_database_password }}@{{ matrix_mx_puppet_slack_database_hostname }}:{{ matrix_mx_puppet_slack_database_port }}/{{ matrix_mx_puppet_slack_database_name }}?sslmode=disable - {%- elif matrix_mx_puppet_slack_database_engine == 'sqlite' -%} - {{ matrix_mx_puppet_slack_database_engine }}://{{ matrix_mx_puppet_slack_database_file }} - {%- endif -%} +matrix_mx_puppet_slack_database_db_name: matrix_mx_puppet_slack + +matrix_mx_puppet_slack_database_connection_string: 'postgresql://{{ matrix_mx_puppet_slack_database_username }}:{{ matrix_mx_puppet_slack_database_password }}@{{ matrix_mx_puppet_slack_database_hostname }}:{{ matrix_mx_puppet_slack_database_port }}/{{ matrix_mx_puppet_slack_database_db_name }}?sslmode=disable' # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. diff --git a/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml index 2e45ecf6..5d68435c 100644 --- a/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml @@ -8,14 +8,6 @@ The matrix-bridge-mx-puppet-slack role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" -- name: Ensure MX Puppet Slack image is pulled - docker_image: - name: "{{ matrix_mx_puppet_slack_docker_image }}" - source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" - force_source: "{{ matrix_mx_puppet_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_slack_docker_image_force_pull }}" - when: matrix_mx_puppet_slack_enabled|bool and not matrix_mx_puppet_slack_container_image_self_build - - name: Ensure MX Puppet Slack paths exist file: path: "{{ item.path }}" @@ -30,6 +22,53 @@ - { path: "{{ matrix_mx_puppet_slack_docker_src_files_path }}", when: "{{ matrix_mx_puppet_slack_container_image_self_build }}" } when: matrix_mx_puppet_slack_enabled|bool and item.when|bool +- name: Check if an old database file already exists + stat: + path: "{{ matrix_mx_puppet_slack_base_path }}/database.db" + register: matrix_mx_puppet_slack_stat_database + +- name: (Data relocation) Ensure matrix-mx-puppet-slack.service is stopped + service: + name: matrix-mx-puppet-slack + state: stopped + daemon_reload: yes + failed_when: false + when: "matrix_mx_puppet_slack_stat_database.stat.exists" + +- set_fact: + matrix_mx_puppet_slack_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mx_puppet_slack_sqlite_database_path_local }}" + register: matrix_mx_puppet_slack_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_mx_puppet_slack_sqlite_database_path_local }}" + dst: "{{ matrix_mx_puppet_slack_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_mx_puppet_slack_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-mx-puppet-slack.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_mx_puppet_slack_requires_restart: true + when: "matrix_mx_puppet_slack_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_mx_puppet_slack_database_engine == 'postgres'" + +- name: Ensure MX Puppet Slack image is pulled + docker_image: + name: "{{ matrix_mx_puppet_slack_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_mx_puppet_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_slack_docker_image_force_pull }}" + when: matrix_mx_puppet_slack_enabled|bool and not matrix_mx_puppet_slack_container_image_self_build + - name: Ensure MX Puppet Slack repository is present on self build git: repo: "{{ matrix_mx_puppet_slack_container_image_self_build_repo }}" @@ -49,19 +88,6 @@ pull: yes when: "matrix_mx_puppet_slack_enabled|bool and matrix_mx_puppet_slack_container_image_self_build" -- name: Check if an old database file already exists - stat: - path: "{{ matrix_mx_puppet_slack_base_path }}/database.db" - register: matrix_mx_puppet_slack_stat_database - -- name: (Data relocation) Ensure matrix-mx-puppet-slack.service is stopped - service: - name: matrix-mx-puppet-slack - state: stopped - daemon_reload: yes - failed_when: false - when: "matrix_mx_puppet_slack_stat_database.stat.exists" - - name: (Data relocation) Move mx-puppet-slack database file to ./data directory command: "mv {{ matrix_mx_puppet_slack_base_path }}/database.db {{ matrix_mx_puppet_slack_data_path }}/database.db" when: "matrix_mx_puppet_slack_stat_database.stat.exists" @@ -93,3 +119,9 @@ service: daemon_reload: yes when: "matrix_mx_puppet_slack_systemd_service_result.changed" + +- name: Ensure matrix-mx-puppet-slack.service restarted, if necessary + service: + name: "matrix-mx-puppet-slack.service" + state: restarted + when: "matrix_mx_puppet_slack_requires_restart|bool" diff --git a/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml index bc722ee6..3a0bca11 100644 --- a/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml +++ b/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml @@ -8,23 +8,3 @@ with_items: - "matrix_mx_puppet_slack_appservice_token" - "matrix_mx_puppet_slack_homeserver_token" - -- block: - - name: Check if sqlite database already exists - stat: - path: "{{ matrix_mx_puppet_slack_data_path }}/database.db" - register: matrix_mx_puppet_slack_sqlite_database_path_local_stat_result - - - name: Fail if an SQLite database already exists when using Postgres - fail: - msg: >- - matrix_mx_puppet_slack_database_engine has been set to `postgres` (which is our new default now). - However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_slack_data_path }}/database.db. - It appears that you've been using this bridge with the SQLite engine until now. - To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_slack_database_engine: sqlite` to your vars.yml file and re-run this same command. - Alternatively, to migrate your existing SQLite database to Postgres: - 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_slack_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_slack_database_connString'`) - 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) - when: "matrix_mx_puppet_slack_sqlite_database_path_local_stat_result.stat.exists" - when: "matrix_mx_puppet_slack_database_engine == 'postgres'" diff --git a/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 index 58c5c11a..af6b5cb8 100644 --- a/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 @@ -64,11 +64,11 @@ database: # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - connString: {{ matrix_mx_puppet_slack_database_connString | to_json }} + connString: {{ matrix_mx_puppet_slack_database_connection_string|to_json }} {% else %} # Use SQLite3 as a database backend # The name of the database file - filename: {{ matrix_mx_puppet_slack_database_file }} + filename: {{ matrix_mx_puppet_slack_sqlite_database_path_in_container|to_json }} {% endif %} logging: From 69cc2145d2c710d3874fe2ec12315266561e10a1 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 22 Dec 2020 16:51:59 +0200 Subject: [PATCH 197/217] Add support for automatic (Postgres -> SQLite) migration to mx-puppet-steam --- group_vars/matrix_servers | 2 +- .../defaults/main.yml | 15 ++-- .../tasks/setup_install.yml | 82 +++++++++++++------ .../tasks/validate_config.yml | 22 ----- .../templates/config.yaml.j2 | 4 +- 5 files changed, 67 insertions(+), 58 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index fda40efd..283d5b6e 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -583,7 +583,7 @@ matrix_mx_puppet_steam_homeserver_token: "{{ matrix_synapse_macaroon_secret_key matrix_mx_puppet_steam_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}" # Postgres is the default, except if not using `matrix_postgres` (internal postgres) -matrix_mx_puppet_stream_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_mx_puppet_steam_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_steam.db') | to_uuid }}" ###################################################################### diff --git a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml index 77f7804b..c1f6a385 100644 --- a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml @@ -56,18 +56,17 @@ matrix_mx_puppet_steam_homeserver_token: '' matrix_mx_puppet_steam_login_shared_secret: '' matrix_mx_puppet_steam_database_engine: sqlite + +matrix_mx_puppet_steam_sqlite_database_path_local: "{{ matrix_mx_puppet_steam_data_path }}/database.db" +matrix_mx_puppet_steam_sqlite_database_path_in_container: "/data/database.db" + matrix_mx_puppet_steam_database_username: matrix_mx_puppet_steam matrix_mx_puppet_steam_database_password: ~ matrix_mx_puppet_steam_database_hostname: 'matrix-postgres' matrix_mx_puppet_steam_database_port: 5432 -matrix_mx_puppet_steam_database_name: matrix_mx_puppet_steam -matrix_mx_puppet_steam_database_file: /data/database.db -matrix_mx_puppet_steam_database_connString: >-2 - {%- if matrix_mx_puppet_steam_database_engine == 'postgres' -%} - postgresql://{{ matrix_mx_puppet_steam_database_username }}:{{ matrix_mx_puppet_steam_database_password }}@{{ matrix_mx_puppet_steam_database_hostname }}:{{ matrix_mx_puppet_steam_database_port }}/{{ matrix_mx_puppet_steam_database_name }}?sslmode=disable - {%- elif matrix_mx_puppet_steam_database_engine == 'sqlite' -%} - {{ matrix_mx_puppet_steam_database_engine }}://{{ matrix_mx_puppet_steam_database_file }} - {%- endif -%} +matrix_mx_puppet_steam_database_db_name: matrix_mx_puppet_steam + +matrix_mx_puppet_steam_database_connection_string: 'postgresql://{{ matrix_mx_puppet_steam_database_username }}:{{ matrix_mx_puppet_steam_database_password }}@{{ matrix_mx_puppet_steam_database_hostname }}:{{ matrix_mx_puppet_steam_database_port }}/{{ matrix_mx_puppet_steam_database_db_name }}?sslmode=disable' # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. diff --git a/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml index 7b7f8211..71f6d889 100644 --- a/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml @@ -8,14 +8,6 @@ The matrix-bridge-mx-puppet-steam role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" -- name: Ensure MX Puppet Steam image is pulled - docker_image: - name: "{{ matrix_mx_puppet_steam_docker_image }}" - source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" - force_source: "{{ matrix_mx_puppet_steam_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_steam_docker_image_force_pull }}" - when: matrix_mx_puppet_steam_enabled|bool and not matrix_mx_puppet_steam_container_image_self_build - - name: Ensure MX Puppet Steam paths exist file: path: "{{ item.path }}" @@ -30,6 +22,57 @@ - { path: "{{ matrix_mx_puppet_steam_docker_src_files_path }}", when: "{{ matrix_mx_puppet_steam_container_image_self_build }}" } when: matrix_mx_puppet_steam_enabled|bool and item.when|bool +- name: Check if an old database file already exists + stat: + path: "{{ matrix_mx_puppet_steam_base_path }}/database.db" + register: matrix_mx_puppet_steam_stat_database + +- name: (Data relocation) Ensure matrix-mx-puppet-steam.service is stopped + service: + name: matrix-mx-puppet-steam + state: stopped + daemon_reload: yes + failed_when: false + when: "matrix_mx_puppet_steam_stat_database.stat.exists" + +- name: (Data relocation) Move mx-puppet-steam database file to ./data directory + command: "mv {{ matrix_mx_puppet_steam_base_path }}/database.db {{ matrix_mx_puppet_steam_data_path }}/database.db" + when: "matrix_mx_puppet_steam_stat_database.stat.exists" + +- set_fact: + matrix_mx_puppet_steam_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mx_puppet_steam_sqlite_database_path_local }}" + register: matrix_mx_puppet_steam_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_mx_puppet_steam_sqlite_database_path_local }}" + dst: "{{ matrix_mx_puppet_steam_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_mx_puppet_steam_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-mx-puppet-steam.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_mx_puppet_steam_requires_restart: true + when: "matrix_mx_puppet_steam_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_mx_puppet_steam_database_engine == 'postgres'" + +- name: Ensure MX Puppet Steam image is pulled + docker_image: + name: "{{ matrix_mx_puppet_steam_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_mx_puppet_steam_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_steam_docker_image_force_pull }}" + when: matrix_mx_puppet_steam_enabled|bool and not matrix_mx_puppet_steam_container_image_self_build + - name: Ensure MX Puppet Steam repository is present on self build git: repo: "{{ matrix_mx_puppet_steam_container_image_self_build_repo }}" @@ -49,23 +92,6 @@ pull: yes when: "matrix_mx_puppet_steam_enabled|bool and matrix_mx_puppet_steam_container_image_self_build" -- name: Check if an old database file already exists - stat: - path: "{{ matrix_mx_puppet_steam_base_path }}/database.db" - register: matrix_mx_puppet_steam_stat_database - -- name: (Data relocation) Ensure matrix-mx-puppet-steam.service is stopped - service: - name: matrix-mx-puppet-steam - state: stopped - daemon_reload: yes - failed_when: false - when: "matrix_mx_puppet_steam_stat_database.stat.exists" - -- name: (Data relocation) Move mx-puppet-steam database file to ./data directory - command: "mv {{ matrix_mx_puppet_steam_base_path }}/database.db {{ matrix_mx_puppet_steam_data_path }}/database.db" - when: "matrix_mx_puppet_steam_stat_database.stat.exists" - - name: Ensure mx-puppet-steam config.yaml installed copy: content: "{{ matrix_mx_puppet_steam_configuration|to_nice_yaml }}" @@ -93,3 +119,9 @@ service: daemon_reload: yes when: "matrix_mx_puppet_steam_systemd_service_result.changed" + +- name: Ensure matrix-mx-puppet-steam.service restarted, if necessary + service: + name: "matrix-mx-puppet-steam.service" + state: restarted + when: "matrix_mx_puppet_steam_requires_restart|bool" diff --git a/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml index 50ce15d9..a8bc6a42 100644 --- a/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml +++ b/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml @@ -8,25 +8,3 @@ with_items: - "matrix_mx_puppet_steam_appservice_token" - "matrix_mx_puppet_steam_homeserver_token" - -- block: - - name: Check if an SQLite database already exists - stat: - path: "{{ matrix_mx_puppet_steam_data_path }}" - register: matrix_mx_puppet_steam_sqlite_database_path_local_stat_result - - - name: Fail if an SQLite database already exists when using Postgres - fail: - msg: >- - matrix_mx_puppet_steam_database_engine has been set to `postgres` (which is our new default now). - However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_steam_data_path }}/database.db. - It appears that you've been using this bridge with the SQLite engine until now. - To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_steam_database_engine: sqlite` to your vars.yml file and re-run this same command. - Alternatively, to migrate your existing SQLite database to Postgres: - 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_steam_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_steam_database_connString'`) - 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) - when: "matrix_mx_puppet_steam_sqlite_database_path_local_stat_result.stat.exists" - when: "matrix_mx_puppet_steam_database_engine == 'postgres'" - - diff --git a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 index 8eda278d..149e08b6 100644 --- a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 @@ -67,11 +67,11 @@ database: # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - connString: {{ matrix_mx_puppet_steam_database_connString | to_json }} + connString: {{ matrix_mx_puppet_steam_database_connection_string|to_json }} {% else %} # Use SQLite3 as a database backend # The name of the database file - filename: {{ matrix_mx_puppet_steam_database_file }} + filename: {{ matrix_mx_puppet_steam_sqlite_database_path_in_container|to_json }} {% endif %} logging: From ab6563ce4e9273cc4e45fbf1b50339c10aff9452 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 22 Dec 2020 17:09:08 +0200 Subject: [PATCH 198/217] Add support for automatic (Postgres -> SQLite) migration to mx-puppet-twitter --- .../defaults/main.yml | 17 ++-- .../tasks/setup_install.yml | 82 +++++++++++++------ .../tasks/validate_config.yml | 22 ----- .../templates/config.yaml.j2 | 4 +- 4 files changed, 67 insertions(+), 58 deletions(-) diff --git a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml index 91aa264a..da7a9ae7 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml @@ -62,18 +62,17 @@ matrix_mx_puppet_twitter_login_shared_secret: '' # Database configuration matrix_mx_puppet_twitter_database_engine: sqlite -matrix_mx_puppet_twitter_database_username: matrix:mx_puppet_twitter + +matrix_mx_puppet_twitter_sqlite_database_path_local: "{{ matrix_mx_puppet_twitter_data_path }}/database.db" +matrix_mx_puppet_twitter_sqlite_database_path_in_container: "/data/database.db" + +matrix_mx_puppet_twitter_database_username: mx_puppet_twitter matrix_mx_puppet_twitter_database_password: ~ matrix_mx_puppet_twitter_database_hostname: 'matrix-postgres' matrix_mx_puppet_twitter_database_port: 5432 -matrix_mx_puppet_twitter_database_name: matrix_mx_puppet_twitter -matrix_mx_puppet_twitter_database_file: /data/database.db -matrix_mx_puppet_twitter_database_connString: >-2 - {%- if matrix_mx_puppet_twitter_database_engine == 'postgres' -%} - postgresql://{{ matrix_mx_puppet_twitter_database_username }}:{{ matrix_mx_puppet_twitter_database_password }}@{{ matrix_mx_puppet_twitter_database_hostname }}:{{ matrix_mx_puppet_twitter_database_port }}/{{ matrix_mx_puppet_twitter_database_name }}?sslmode=disable - {%- elif matrix_mx_puppet_twitter_database_engine == 'sqlite' -%} - {{ matrix_mx_puppet_twitter_database_engine }}://{{ matrix_mx_puppet_twitter_database_file }} - {%- endif -%} +matrix_mx_puppet_twitter_database_db_name: matrix_mx_puppet_twitter + +matrix_mx_puppet_twitter_database_connection_string: 'postgresql://{{ matrix_mx_puppet_twitter_database_username }}:{{ matrix_mx_puppet_twitter_database_password }}@{{ matrix_mx_puppet_twitter_database_hostname }}:{{ matrix_mx_puppet_twitter_database_port }}/{{ matrix_mx_puppet_twitter_database_db_name }}?sslmode=disable' # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. diff --git a/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml index 663f822c..3893981a 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml @@ -8,14 +8,6 @@ The matrix-bridge-mx-puppet-twitter role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" -- name: Ensure MX Puppet Twitter image is pulled - docker_image: - name: "{{ matrix_mx_puppet_twitter_docker_image }}" - source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" - force_source: "{{ matrix_mx_puppet_twitter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_twitter_docker_image_force_pull }}" - when: matrix_mx_puppet_twitter_enabled|bool and not matrix_mx_puppet_twitter_container_image_self_build - - name: Ensure MX Puppet Twitter paths exist file: path: "{{ item.path }}" @@ -30,6 +22,57 @@ - { path: "{{ matrix_mx_puppet_twitter_docker_src_files_path }}", when: "{{ matrix_mx_puppet_twitter_container_image_self_build }}" } when: matrix_mx_puppet_twitter_enabled|bool and item.when|bool +- name: Check if an old database file already exists + stat: + path: "{{ matrix_mx_puppet_twitter_base_path }}/database.db" + register: matrix_mx_puppet_twitter_stat_database + +- name: (Data relocation) Ensure matrix-mx-puppet-twitter.service is stopped + service: + name: matrix-mx-puppet-twitter + state: stopped + daemon_reload: yes + failed_when: false + when: "matrix_mx_puppet_twitter_stat_database.stat.exists" + +- name: (Data relocation) Move mx-puppet-twitter database file to ./data directory + command: "mv {{ matrix_mx_puppet_twitter_base_path }}/database.db {{ matrix_mx_puppet_twitter_data_path }}/database.db" + when: "matrix_mx_puppet_twitter_stat_database.stat.exists" + +- set_fact: + matrix_mx_puppet_twitter_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mx_puppet_twitter_sqlite_database_path_local }}" + register: matrix_mx_puppet_twitter_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_mx_puppet_twitter_sqlite_database_path_local }}" + dst: "{{ matrix_mx_puppet_twitter_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_mx_puppet_twitter_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-mx-puppet-twitter.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_mx_puppet_twitter_requires_restart: true + when: "matrix_mx_puppet_twitter_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_mx_puppet_twitter_database_engine == 'postgres'" + +- name: Ensure MX Puppet Twitter image is pulled + docker_image: + name: "{{ matrix_mx_puppet_twitter_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_mx_puppet_twitter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_twitter_docker_image_force_pull }}" + when: matrix_mx_puppet_twitter_enabled|bool and not matrix_mx_puppet_twitter_container_image_self_build + - name: Ensure MX Puppet Twitter repository is present on self build git: repo: "{{ matrix_mx_puppet_twitter_container_image_self_build_repo }}" @@ -49,23 +92,6 @@ pull: yes when: "matrix_mx_puppet_twitter_enabled|bool and matrix_mx_puppet_twitter_container_image_self_build" -- name: Check if an old database file already exists - stat: - path: "{{ matrix_mx_puppet_twitter_base_path }}/database.db" - register: matrix_mx_puppet_twitter_stat_database - -- name: (Data relocation) Ensure matrix-mx-puppet-twitter.service is stopped - service: - name: matrix-mx-puppet-twitter - state: stopped - daemon_reload: yes - failed_when: false - when: "matrix_mx_puppet_twitter_stat_database.stat.exists" - -- name: (Data relocation) Move mx-puppet-twitter database file to ./data directory - command: "mv {{ matrix_mx_puppet_twitter_base_path }}/database.db {{ matrix_mx_puppet_twitter_data_path }}/database.db" - when: "matrix_mx_puppet_twitter_stat_database.stat.exists" - - name: Ensure mx-puppet-twitter config.yaml installed copy: content: "{{ matrix_mx_puppet_twitter_configuration|to_nice_yaml }}" @@ -93,3 +119,9 @@ service: daemon_reload: yes when: "matrix_mx_puppet_twitter_systemd_service_result.changed" + +- name: Ensure matrix-mx-puppet-twitter.service restarted, if necessary + service: + name: "matrix-mx-puppet-twitter.service" + state: restarted + when: "matrix_mx_puppet_twitter_requires_restart|bool" diff --git a/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml index 0b3bd8f8..d13a39e1 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml +++ b/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml @@ -8,25 +8,3 @@ with_items: - "matrix_mx_puppet_twitter_appservice_token" - "matrix_mx_puppet_twitter_homeserver_token" - -- block: - - name: Check if an SQLite database already exists - stat: - path: "{{ matrix_mx_puppet_twitter_sqlite_data_path }}/database.db" - register: matrix_mx_puppet_twitter_sqlite_database_path_local_stat_result - - - name: Fail if an SQLite database already exists when using Postgres - fail: - msg: >- - matrix_mx_puppet_twitter_database_engine has been set to `postgres` (which is our new default now). - However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_twitter_data_path }}/database.db. - It appears that you've been using this bridge with the SQLite engine until now. - To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_twitter_database_engine: sqlite` to your vars.yml file and re-run this same command. - Alternatively, to migrate your existing SQLite database to Postgres: - 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_twitter_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_twitter_database_connString'`) - 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) - when: "matrix_mx_puppet_twitter_sqlite_database_path_local_stat_result.stat.exists" - when: "matrix_mx_puppet_twitter_database_engine == 'postgres'" - - diff --git a/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 index cecb171d..bdecf1dc 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 @@ -60,11 +60,11 @@ database: # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - connString: {{ matrix_mx_puppet_twitter_database_connString | to_json }} + connString: {{ matrix_mx_puppet_twitter_database_connection_string|to_json }} {% else %} # Use SQLite3 as a database backend # The name of the database file - filename: {{ matrix_mx_puppet_twitter_database_file }} + filename: {{ matrix_mx_puppet_twitter_sqlite_database_path_in_container|to_json }} {% endif %} logging: From 15f4cc924d30a62256cd30a0c73311223d74eac6 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 22 Dec 2020 17:10:02 +0200 Subject: [PATCH 199/217] Rename variables (_database_db_name -> _database_name) --- group_vars/matrix_servers | 34 +++++++++---------- .../defaults/main.yml | 4 +-- .../defaults/main.yml | 4 +-- .../defaults/main.yml | 4 +-- .../defaults/main.yml | 4 +-- .../defaults/main.yml | 4 +-- .../defaults/main.yml | 4 +-- .../defaults/main.yml | 4 +-- .../defaults/main.yml | 4 +-- .../defaults/main.yml | 4 +-- .../defaults/main.yml | 4 +-- .../defaults/main.yml | 4 +-- .../defaults/main.yml | 4 +-- roles/matrix-dimension/defaults/main.yml | 4 +-- roles/matrix-ma1sd/defaults/main.yml | 4 +-- roles/matrix-ma1sd/templates/ma1sd.yaml.j2 | 2 +- roles/matrix-registration/defaults/main.yml | 4 +-- .../tasks/setup_install.yml | 2 +- 18 files changed, 49 insertions(+), 49 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 283d5b6e..21846592 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -999,55 +999,55 @@ matrix_postgres_db_name: "homeserver" matrix_postgres_additional_databases: | {{ ([{ - 'name': matrix_ma1sd_database_db_name, + 'name': matrix_ma1sd_database_name, 'username': matrix_ma1sd_database_username, 'password': matrix_ma1sd_database_password, }] if (matrix_ma1sd_enabled and matrix_ma1sd_database_engine == 'postgres' and matrix_ma1sd_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_bot_matrix_reminder_bot_database_db_name, + 'name': matrix_bot_matrix_reminder_bot_database_name, 'username': matrix_bot_matrix_reminder_bot_database_username, 'password': matrix_bot_matrix_reminder_bot_database_password, }] if (matrix_bot_matrix_reminder_bot_enabled and matrix_bot_matrix_reminder_bot_database_engine == 'postgres' and matrix_bot_matrix_reminder_bot_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_registration_database_db_name, + 'name': matrix_registration_database_name, 'username': matrix_registration_database_username, 'password': matrix_registration_database_password, }] if (matrix_registration_enabled and matrix_registration_database_engine == 'postgres' and matrix_registration_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_appservice_discord_database_db_name, + 'name': matrix_appservice_discord_database_name, 'username': matrix_appservice_discord_database_username, 'password': matrix_appservice_discord_database_password, }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_appservice_slack_database_db_name, + 'name': matrix_appservice_slack_database_name, 'username': matrix_appservice_slack_database_username, 'password': matrix_appservice_slack_database_password, }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_appservice_irc_database_db_name, + 'name': matrix_appservice_irc_database_name, 'username': matrix_appservice_irc_database_username, 'password': matrix_appservice_irc_database_password, }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mautrix_facebook_database_db_name, + 'name': matrix_mautrix_facebook_database_name, 'username': matrix_mautrix_facebook_database_username, 'password': matrix_mautrix_facebook_database_password, }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mautrix_hangouts_database_db_name, + 'name': matrix_mautrix_hangouts_database_name, 'username': matrix_mautrix_hangouts_database_username, 'password': matrix_mautrix_hangouts_database_password, }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mautrix_telegram_database_db_name, + 'name': matrix_mautrix_telegram_database_name, 'username': matrix_mautrix_telegram_database_username, 'password': matrix_mautrix_telegram_database_password, }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_mautrix_telegram_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mautrix_whatsapp_database_db_name, + 'name': matrix_mautrix_whatsapp_database_name, 'username': matrix_mautrix_whatsapp_database_username, 'password': matrix_mautrix_whatsapp_database_password, }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres') else []) @@ -1057,36 +1057,36 @@ matrix_postgres_additional_databases: | 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid, }] if matrix_sms_bridge_enabled else []) + ([{ - 'name': matrix_mx_puppet_skype_database_db_name, + 'name': matrix_mx_puppet_skype_database_name, 'username': matrix_mx_puppet_skype_database_username, 'password': matrix_mx_puppet_skype_database_password, }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mx_puppet_slack_database_db_name, + 'name': matrix_mx_puppet_slack_database_name, 'username': matrix_mx_puppet_slack_database_username, 'password': matrix_mx_puppet_slack_database_password, }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mx_puppet_twitter_database_db_name, + 'name': matrix_mx_puppet_twitter_database_name, 'username': matrix_mx_puppet_twitter_database_username, 'password': matrix_mx_puppet_twitter_database_password, }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mx_puppet_instagram_database_db_name, + 'name': matrix_mx_puppet_instagram_database_name, 'username': matrix_mx_puppet_instagram_database_username, 'password': matrix_mx_puppet_instagram_database_password, }] if (matrix_mx_puppet_instagram_enabled and matrix_mx_puppet_instagram_database_engine == 'postgres' and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mx_puppet_discord_database_db_name, + 'name': matrix_mx_puppet_discord_database_name, 'username': matrix_mx_puppet_discord_database_username, 'password': matrix_mx_puppet_discord_database_password, }] if (matrix_mx_puppet_discord_enabled and matrix_mx_puppet_discord_database_engine == 'postgres' and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_mx_puppet_steam_database_db_name, + 'name': matrix_mx_puppet_steam_database_name, 'username': matrix_mx_puppet_steam_database_username, 'password': matrix_mx_puppet_steam_database_password, }] if (matrix_mx_puppet_steam_enabled @@ -1094,7 +1094,7 @@ matrix_postgres_additional_databases: | and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else []) + ([{ - 'name': matrix_dimension_database_db_name, + 'name': matrix_dimension_database_name, 'username': matrix_dimension_database_username, 'password': matrix_dimension_database_password, }] if (matrix_dimension_enabled and matrix_dimension_database_engine == 'postgres' and matrix_dimension_database_hostname == 'matrix-postgres') else []) diff --git a/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml b/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml index 70e7b2d2..29bc8307 100644 --- a/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml +++ b/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml @@ -37,9 +37,9 @@ matrix_bot_matrix_reminder_bot_database_username: 'matrix_reminder_bot' matrix_bot_matrix_reminder_bot_database_password: 'some-password' matrix_bot_matrix_reminder_bot_database_hostname: 'matrix-postgres' matrix_bot_matrix_reminder_bot_database_port: 5432 -matrix_bot_matrix_reminder_bot_database_db_name: 'matrix_reminder_bot' +matrix_bot_matrix_reminder_bot_database_name: 'matrix_reminder_bot' -matrix_bot_matrix_reminder_bot_database_connection_string: 'postgres://{{ matrix_bot_matrix_reminder_bot_database_username }}:{{ matrix_bot_matrix_reminder_bot_database_password }}@{{ matrix_bot_matrix_reminder_bot_database_hostname }}:{{ matrix_bot_matrix_reminder_bot_database_port }}/{{ matrix_bot_matrix_reminder_bot_database_db_name }}' +matrix_bot_matrix_reminder_bot_database_connection_string: 'postgres://{{ matrix_bot_matrix_reminder_bot_database_username }}:{{ matrix_bot_matrix_reminder_bot_database_password }}@{{ matrix_bot_matrix_reminder_bot_database_hostname }}:{{ matrix_bot_matrix_reminder_bot_database_port }}/{{ matrix_bot_matrix_reminder_bot_database_name }}' matrix_bot_matrix_reminder_bot_storage_database: "{{ { diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml index 8e453a15..c7cdddb6 100644 --- a/roles/matrix-bridge-appservice-discord/defaults/main.yml +++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml @@ -57,12 +57,12 @@ matrix_appservice_discord_database_username: 'matrix_appservice_discord' matrix_appservice_discord_database_password: 'some-password' matrix_appservice_discord_database_hostname: 'matrix-postgres' matrix_appservice_discord_database_port: 5432 -matrix_appservice_discord_database_db_name: 'matrix_appservice_discord' +matrix_appservice_discord_database_name: 'matrix_appservice_discord' # These 2 variables are what actually ends up in the bridge configuration. # It's best if you don't change them directly, but rather redefine the sub-variables that constitute them. matrix_appservice_discord_database_filename: "{{ matrix_appservice_discord_sqlite_database_path_in_container }}" -matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_username }}:{{ matrix_appservice_discord_database_password }}@{{ matrix_appservice_discord_database_hostname }}:{{ matrix_appservice_discord_database_port }}/{{ matrix_appservice_discord_database_db_name }}' +matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_username }}:{{ matrix_appservice_discord_database_password }}@{{ matrix_appservice_discord_database_hostname }}:{{ matrix_appservice_discord_database_port }}/{{ matrix_appservice_discord_database_name }}' # Tells whether the bot should make use of "Privileged Gateway Intents". diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml index c02197a5..580934db 100644 --- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml @@ -49,9 +49,9 @@ matrix_mautrix_facebook_database_username: 'matrix_mautrix_facebook' matrix_mautrix_facebook_database_password: 'some-password' matrix_mautrix_facebook_database_hostname: 'matrix-postgres' matrix_mautrix_facebook_database_port: 5432 -matrix_mautrix_facebook_database_db_name: 'matrix_mautrix_facebook' +matrix_mautrix_facebook_database_name: 'matrix_mautrix_facebook' -matrix_mautrix_facebook_database_connection_string: 'postgresql://{{ matrix_mautrix_facebook_database_username }}:{{ matrix_mautrix_facebook_database_password }}@{{ matrix_mautrix_facebook_database_hostname }}:{{ matrix_mautrix_facebook_database_port }}/{{ matrix_mautrix_facebook_database_db_name }}' +matrix_mautrix_facebook_database_connection_string: 'postgresql://{{ matrix_mautrix_facebook_database_username }}:{{ matrix_mautrix_facebook_database_password }}@{{ matrix_mautrix_facebook_database_hostname }}:{{ matrix_mautrix_facebook_database_port }}/{{ matrix_mautrix_facebook_database_name }}' matrix_mautrix_facebook_appservice_database: "{{ { diff --git a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml index 5d38289b..984bec48 100644 --- a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml @@ -56,9 +56,9 @@ matrix_mautrix_hangouts_database_username: 'matrix_mautrix_hangouts' matrix_mautrix_hangouts_database_password: 'some-password' matrix_mautrix_hangouts_database_hostname: 'matrix-postgres' matrix_mautrix_hangouts_database_port: 5432 -matrix_mautrix_hangouts_database_db_name: 'matrix_mautrix_hangouts' +matrix_mautrix_hangouts_database_name: 'matrix_mautrix_hangouts' -matrix_mautrix_hangouts_database_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_database_username }}:{{ matrix_mautrix_hangouts_database_password }}@{{ matrix_mautrix_hangouts_database_hostname }}:{{ matrix_mautrix_hangouts_database_port }}/{{ matrix_mautrix_hangouts_database_db_name }}' +matrix_mautrix_hangouts_database_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_database_username }}:{{ matrix_mautrix_hangouts_database_password }}@{{ matrix_mautrix_hangouts_database_hostname }}:{{ matrix_mautrix_hangouts_database_port }}/{{ matrix_mautrix_hangouts_database_name }}' matrix_mautrix_hangouts_appservice_database: "{{ { diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml index fcdf95db..ceebd3ec 100644 --- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml @@ -60,9 +60,9 @@ matrix_mautrix_telegram_database_username: 'matrix_mautrix_telegram' matrix_mautrix_telegram_database_password: 'some-password' matrix_mautrix_telegram_database_hostname: 'matrix-postgres' matrix_mautrix_telegram_database_port: 5432 -matrix_mautrix_telegram_database_db_name: 'matrix_mautrix_telegram' +matrix_mautrix_telegram_database_name: 'matrix_mautrix_telegram' -matrix_mautrix_telegram_database_connection_string: 'postgresql://{{ matrix_mautrix_telegram_database_username }}:{{ matrix_mautrix_telegram_database_password }}@{{ matrix_mautrix_telegram_database_hostname }}:{{ matrix_mautrix_telegram_database_port }}/{{ matrix_mautrix_telegram_database_db_name }}' +matrix_mautrix_telegram_database_connection_string: 'postgresql://{{ matrix_mautrix_telegram_database_username }}:{{ matrix_mautrix_telegram_database_password }}@{{ matrix_mautrix_telegram_database_hostname }}:{{ matrix_mautrix_telegram_database_port }}/{{ matrix_mautrix_telegram_database_name }}' matrix_mautrix_telegram_appservice_database: "{{ { diff --git a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml index 7e198b36..0d4f9852 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml @@ -44,9 +44,9 @@ matrix_mautrix_whatsapp_database_username: 'matrix_mautrix_whatsapp' matrix_mautrix_whatsapp_database_password: 'some-password' matrix_mautrix_whatsapp_database_hostname: 'matrix-postgres' matrix_mautrix_whatsapp_database_port: 5432 -matrix_mautrix_whatsapp_database_db_name: 'matrix_mautrix_whatsapp' +matrix_mautrix_whatsapp_database_name: 'matrix_mautrix_whatsapp' -matrix_mautrix_whatsapp_database_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_database_username }}:{{ matrix_mautrix_whatsapp_database_password }}@{{ matrix_mautrix_whatsapp_database_hostname }}:{{ matrix_mautrix_whatsapp_database_port }}/{{ matrix_mautrix_whatsapp_database_db_name }}' +matrix_mautrix_whatsapp_database_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_database_username }}:{{ matrix_mautrix_whatsapp_database_password }}@{{ matrix_mautrix_whatsapp_database_hostname }}:{{ matrix_mautrix_whatsapp_database_port }}/{{ matrix_mautrix_whatsapp_database_name }}' matrix_mautrix_whatsapp_appservice_database_type: "{{ { diff --git a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml index ad86a241..97b20313 100644 --- a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml @@ -65,9 +65,9 @@ matrix_mx_puppet_discord_database_username: matrix_mx_puppet_discord matrix_mx_puppet_discord_database_password: ~ matrix_mx_puppet_discord_database_hostname: 'matrix-postgres' matrix_mx_puppet_discord_database_port: 5432 -matrix_mx_puppet_discord_database_db_name: matrix_mx_puppet_discord +matrix_mx_puppet_discord_database_name: matrix_mx_puppet_discord -matrix_mx_puppet_discord_database_connection_string: 'postgresql://{{ matrix_mx_puppet_discord_database_username }}:{{ matrix_mx_puppet_discord_database_password }}@{{ matrix_mx_puppet_discord_database_hostname }}:{{ matrix_mx_puppet_discord_database_port }}/{{ matrix_mx_puppet_discord_database_db_name }}?sslmode=disable' +matrix_mx_puppet_discord_database_connection_string: 'postgresql://{{ matrix_mx_puppet_discord_database_username }}:{{ matrix_mx_puppet_discord_database_password }}@{{ matrix_mx_puppet_discord_database_hostname }}:{{ matrix_mx_puppet_discord_database_port }}/{{ matrix_mx_puppet_discord_database_name }}?sslmode=disable' # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. diff --git a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml index 61bf3dcf..cd08c010 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml @@ -55,9 +55,9 @@ matrix_mx_puppet_instagram_database_username: matrix_mx_puppet_instagram matrix_mx_puppet_instagram_database_password: ~ matrix_mx_puppet_instagram_database_hostname: 'matrix-postgres' matrix_mx_puppet_instagram_database_port: 5432 -matrix_mx_puppet_instagram_database_db_name: matrix_mx_puppet_instagram +matrix_mx_puppet_instagram_database_name: matrix_mx_puppet_instagram -matrix_mx_puppet_instagram_database_connection_string: 'postgresql://{{ matrix_mx_puppet_instagram_database_username }}:{{ matrix_mx_puppet_instagram_database_password }}@{{ matrix_mx_puppet_instagram_database_hostname }}:{{ matrix_mx_puppet_instagram_database_port }}/{{ matrix_mx_puppet_instagram_database_db_name }}?sslmode=disable' +matrix_mx_puppet_instagram_database_connection_string: 'postgresql://{{ matrix_mx_puppet_instagram_database_username }}:{{ matrix_mx_puppet_instagram_database_password }}@{{ matrix_mx_puppet_instagram_database_hostname }}:{{ matrix_mx_puppet_instagram_database_port }}/{{ matrix_mx_puppet_instagram_database_name }}?sslmode=disable' # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. diff --git a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml index f3e17a59..83cd3dc5 100644 --- a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml @@ -63,9 +63,9 @@ matrix_mx_puppet_skype_database_username: matrix_mx_puppet_skype matrix_mx_puppet_skype_database_password: ~ matrix_mx_puppet_skype_database_hostname: 'matrix-postgres' matrix_mx_puppet_skype_database_port: 5432 -matrix_mx_puppet_skype_database_db_name: matrix_mx_puppet_skype +matrix_mx_puppet_skype_database_name: matrix_mx_puppet_skype -matrix_mx_puppet_skype_database_connection_string: 'postgresql://{{ matrix_mx_puppet_skype_database_username }}:{{ matrix_mx_puppet_skype_database_password }}@{{ matrix_mx_puppet_skype_database_hostname }}:{{ matrix_mx_puppet_skype_database_port }}/{{ matrix_mx_puppet_skype_database_db_name }}?sslmode=disable' +matrix_mx_puppet_skype_database_connection_string: 'postgresql://{{ matrix_mx_puppet_skype_database_username }}:{{ matrix_mx_puppet_skype_database_password }}@{{ matrix_mx_puppet_skype_database_hostname }}:{{ matrix_mx_puppet_skype_database_port }}/{{ matrix_mx_puppet_skype_database_name }}?sslmode=disable' # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. diff --git a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml index 9aad602a..70b98ece 100644 --- a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml @@ -67,9 +67,9 @@ matrix_mx_puppet_slack_database_username: matrix_mx_puppet_slack matrix_mx_puppet_slack_database_password: ~ matrix_mx_puppet_slack_database_hostname: 'matrix-postgres' matrix_mx_puppet_slack_database_port: 5432 -matrix_mx_puppet_slack_database_db_name: matrix_mx_puppet_slack +matrix_mx_puppet_slack_database_name: matrix_mx_puppet_slack -matrix_mx_puppet_slack_database_connection_string: 'postgresql://{{ matrix_mx_puppet_slack_database_username }}:{{ matrix_mx_puppet_slack_database_password }}@{{ matrix_mx_puppet_slack_database_hostname }}:{{ matrix_mx_puppet_slack_database_port }}/{{ matrix_mx_puppet_slack_database_db_name }}?sslmode=disable' +matrix_mx_puppet_slack_database_connection_string: 'postgresql://{{ matrix_mx_puppet_slack_database_username }}:{{ matrix_mx_puppet_slack_database_password }}@{{ matrix_mx_puppet_slack_database_hostname }}:{{ matrix_mx_puppet_slack_database_port }}/{{ matrix_mx_puppet_slack_database_name }}?sslmode=disable' # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. diff --git a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml index c1f6a385..15fa889f 100644 --- a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml @@ -64,9 +64,9 @@ matrix_mx_puppet_steam_database_username: matrix_mx_puppet_steam matrix_mx_puppet_steam_database_password: ~ matrix_mx_puppet_steam_database_hostname: 'matrix-postgres' matrix_mx_puppet_steam_database_port: 5432 -matrix_mx_puppet_steam_database_db_name: matrix_mx_puppet_steam +matrix_mx_puppet_steam_database_name: matrix_mx_puppet_steam -matrix_mx_puppet_steam_database_connection_string: 'postgresql://{{ matrix_mx_puppet_steam_database_username }}:{{ matrix_mx_puppet_steam_database_password }}@{{ matrix_mx_puppet_steam_database_hostname }}:{{ matrix_mx_puppet_steam_database_port }}/{{ matrix_mx_puppet_steam_database_db_name }}?sslmode=disable' +matrix_mx_puppet_steam_database_connection_string: 'postgresql://{{ matrix_mx_puppet_steam_database_username }}:{{ matrix_mx_puppet_steam_database_password }}@{{ matrix_mx_puppet_steam_database_hostname }}:{{ matrix_mx_puppet_steam_database_port }}/{{ matrix_mx_puppet_steam_database_name }}?sslmode=disable' # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. diff --git a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml index da7a9ae7..28639fda 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml @@ -70,9 +70,9 @@ matrix_mx_puppet_twitter_database_username: mx_puppet_twitter matrix_mx_puppet_twitter_database_password: ~ matrix_mx_puppet_twitter_database_hostname: 'matrix-postgres' matrix_mx_puppet_twitter_database_port: 5432 -matrix_mx_puppet_twitter_database_db_name: matrix_mx_puppet_twitter +matrix_mx_puppet_twitter_database_name: matrix_mx_puppet_twitter -matrix_mx_puppet_twitter_database_connection_string: 'postgresql://{{ matrix_mx_puppet_twitter_database_username }}:{{ matrix_mx_puppet_twitter_database_password }}@{{ matrix_mx_puppet_twitter_database_hostname }}:{{ matrix_mx_puppet_twitter_database_port }}/{{ matrix_mx_puppet_twitter_database_db_name }}?sslmode=disable' +matrix_mx_puppet_twitter_database_connection_string: 'postgresql://{{ matrix_mx_puppet_twitter_database_username }}:{{ matrix_mx_puppet_twitter_database_password }}@{{ matrix_mx_puppet_twitter_database_hostname }}:{{ matrix_mx_puppet_twitter_database_port }}/{{ matrix_mx_puppet_twitter_database_name }}?sslmode=disable' # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. diff --git a/roles/matrix-dimension/defaults/main.yml b/roles/matrix-dimension/defaults/main.yml index a6a3bf5d..8a691946 100644 --- a/roles/matrix-dimension/defaults/main.yml +++ b/roles/matrix-dimension/defaults/main.yml @@ -57,9 +57,9 @@ matrix_dimension_database_username: 'matrix_dimension' matrix_dimension_database_password: 'some-password' matrix_dimension_database_hostname: 'matrix-postgres' matrix_dimension_database_port: 5432 -matrix_dimension_database_db_name: 'matrix_dimension' +matrix_dimension_database_name: 'matrix_dimension' -matrix_dimension_database_connection_string: 'postgres://{{ matrix_dimension_database_username }}:{{ matrix_dimension_database_password }}@{{ matrix_dimension_database_hostname }}:{{ matrix_dimension_database_port }}/{{ matrix_dimension_database_db_name }}' +matrix_dimension_database_connection_string: 'postgres://{{ matrix_dimension_database_username }}:{{ matrix_dimension_database_password }}@{{ matrix_dimension_database_hostname }}:{{ matrix_dimension_database_port }}/{{ matrix_dimension_database_name }}' # Default Dimension configuration template which covers the generic use case. diff --git a/roles/matrix-ma1sd/defaults/main.yml b/roles/matrix-ma1sd/defaults/main.yml index 42e36e92..2932f3ed 100644 --- a/roles/matrix-ma1sd/defaults/main.yml +++ b/roles/matrix-ma1sd/defaults/main.yml @@ -56,9 +56,9 @@ matrix_ma1sd_database_username: 'matrix_ma1sd' matrix_ma1sd_database_password: 'some-password' matrix_ma1sd_database_hostname: 'matrix-postgres' matrix_ma1sd_database_port: 5432 -matrix_ma1sd_database_db_name: 'matrix_ma1sd' +matrix_ma1sd_database_name: 'matrix_ma1sd' -matrix_ma1sd_database_connection_string: 'postgresql://{{ matrix_ma1sd_database_username }}:{{ matrix_ma1sd_database_password }}@{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_db_name }}' +matrix_ma1sd_database_connection_string: 'postgresql://{{ matrix_ma1sd_database_username }}:{{ matrix_ma1sd_database_password }}@{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_name }}' # ma1sd has serveral supported identity stores. diff --git a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 index 43b4022d..9a426c47 100644 --- a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 +++ b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 @@ -20,7 +20,7 @@ storage: backend: postgresql provider: postgresql: - database: //{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_db_name }} + database: //{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_name }} username: {{ matrix_ma1sd_database_username|to_json }} password: {{ matrix_ma1sd_database_password|to_json }} {% endif %} diff --git a/roles/matrix-registration/defaults/main.yml b/roles/matrix-registration/defaults/main.yml index d85faf89..065e0c48 100644 --- a/roles/matrix-registration/defaults/main.yml +++ b/roles/matrix-registration/defaults/main.yml @@ -48,9 +48,9 @@ matrix_registration_database_username: 'matrix_registration' matrix_registration_database_password: 'some-password' matrix_registration_database_hostname: 'matrix-postgres' matrix_registration_database_port: 5432 -matrix_registration_database_db_name: 'matrix_registration' +matrix_registration_database_name: 'matrix_registration' -matrix_registration_database_connection_string: 'postgresql://{{ matrix_registration_database_username }}:{{ matrix_registration_database_password }}@{{ matrix_registration_database_hostname }}:{{ matrix_registration_database_port }}/{{ matrix_registration_database_db_name }}' +matrix_registration_database_connection_string: 'postgresql://{{ matrix_registration_database_username }}:{{ matrix_registration_database_password }}@{{ matrix_registration_database_hostname }}:{{ matrix_registration_database_port }}/{{ matrix_registration_database_name }}' # For some reason, matrix-registraiton expects the `db` field to be like this: `sqlite:////data/db.sqlite3`. # (seems like one too many slashes, but..) diff --git a/roles/matrix-registration/tasks/setup_install.yml b/roles/matrix-registration/tasks/setup_install.yml index 708cb1df..9b6d1260 100644 --- a/roles/matrix-registration/tasks/setup_install.yml +++ b/roles/matrix-registration/tasks/setup_install.yml @@ -22,7 +22,7 @@ # which makes matrix-registration choke on it later on when comparing dates. additional_psql_statements_list: - ALTER TABLE tokens ALTER COLUMN ex_date TYPE TIMESTAMP WITHOUT TIME ZONE; - additional_psql_statements_db_name: "{{ matrix_registration_database_db_name }}" + additional_psql_statements_db_name: "{{ matrix_registration_database_name }}" - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" From 815a80c290d2527a74a26acb47e6f1fbbb3f000b Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 22 Dec 2020 17:16:56 +0200 Subject: [PATCH 200/217] Fix alignment issues --- group_vars/matrix_servers | 53 ++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 21846592..318ebac7 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1021,78 +1021,85 @@ matrix_postgres_additional_databases: | 'username': matrix_appservice_discord_database_username, 'password': matrix_appservice_discord_database_password, }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_database_hostname == 'matrix-postgres') else []) - + ([{ + + + ([{ 'name': matrix_appservice_slack_database_name, 'username': matrix_appservice_slack_database_username, 'password': matrix_appservice_slack_database_password, }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres') else []) - + ([{ + + + ([{ 'name': matrix_appservice_irc_database_name, 'username': matrix_appservice_irc_database_username, 'password': matrix_appservice_irc_database_password, }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres') else []) - + ([{ + + + ([{ 'name': matrix_mautrix_facebook_database_name, 'username': matrix_mautrix_facebook_database_username, 'password': matrix_mautrix_facebook_database_password, }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_database_hostname == 'matrix-postgres') else []) - + ([{ + + + ([{ 'name': matrix_mautrix_hangouts_database_name, 'username': matrix_mautrix_hangouts_database_username, 'password': matrix_mautrix_hangouts_database_password, }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres') else []) - + ([{ + + + ([{ 'name': matrix_mautrix_telegram_database_name, 'username': matrix_mautrix_telegram_database_username, 'password': matrix_mautrix_telegram_database_password, }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_mautrix_telegram_database_hostname == 'matrix-postgres') else []) - + ([{ + + + ([{ 'name': matrix_mautrix_whatsapp_database_name, 'username': matrix_mautrix_whatsapp_database_username, 'password': matrix_mautrix_whatsapp_database_password, }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres') else []) - + ([{ + + + ([{ 'name': 'matrix_bridge_sms', 'username': 'matrix_bridge_sms', 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid, }] if matrix_sms_bridge_enabled else []) - + ([{ + + + ([{ 'name': matrix_mx_puppet_skype_database_name, 'username': matrix_mx_puppet_skype_database_username, 'password': matrix_mx_puppet_skype_database_password, }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres') else []) - + ([{ + + + ([{ 'name': matrix_mx_puppet_slack_database_name, 'username': matrix_mx_puppet_slack_database_username, 'password': matrix_mx_puppet_slack_database_password, }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres') else []) - + ([{ + + + ([{ 'name': matrix_mx_puppet_twitter_database_name, 'username': matrix_mx_puppet_twitter_database_username, 'password': matrix_mx_puppet_twitter_database_password, }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres') else []) - + ([{ + + + ([{ 'name': matrix_mx_puppet_instagram_database_name, 'username': matrix_mx_puppet_instagram_database_username, 'password': matrix_mx_puppet_instagram_database_password, - }] if (matrix_mx_puppet_instagram_enabled - and matrix_mx_puppet_instagram_database_engine == 'postgres' - and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres') else []) - + ([{ + }] if (matrix_mx_puppet_instagram_enabled and matrix_mx_puppet_instagram_database_engine == 'postgres' and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres') else []) + + + ([{ 'name': matrix_mx_puppet_discord_database_name, 'username': matrix_mx_puppet_discord_database_username, 'password': matrix_mx_puppet_discord_database_password, - }] if (matrix_mx_puppet_discord_enabled - and matrix_mx_puppet_discord_database_engine == 'postgres' - and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres') else []) - + ([{ + }] if (matrix_mx_puppet_discord_enabled and matrix_mx_puppet_discord_database_engine == 'postgres' and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres') else []) + + + ([{ 'name': matrix_mx_puppet_steam_database_name, 'username': matrix_mx_puppet_steam_database_username, 'password': matrix_mx_puppet_steam_database_password, - }] if (matrix_mx_puppet_steam_enabled - and matrix_mx_puppet_steam_database_engine == 'postgres' - and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else []) - + + }] if (matrix_mx_puppet_steam_enabled and matrix_mx_puppet_steam_database_engine == 'postgres' and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else []) + + ([{ 'name': matrix_dimension_database_name, 'username': matrix_dimension_database_username, From 715bdf2c6487c56be5ad17a4e7b80d94efa46ce2 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 22 Dec 2020 19:32:43 +0200 Subject: [PATCH 201/217] Add support for automatic (nedb -> Postgres) migration to mx-appservice-irc --- group_vars/matrix_servers | 3 +- roles/matrix-base/defaults/main.yml | 1 + .../defaults/main.yml | 17 +++-- .../tasks/migrate_nedb_to_postgres.yml | 53 ++++++++++++++++ .../tasks/setup_install.yml | 63 ++++++++++++------- .../tasks/validate_config.yml | 20 ------ .../templates/config.yaml.j2 | 4 +- roles/matrix-postgres/tasks/import_nedb.yml | 21 +------ roles/matrix-postgres/tasks/main.yml | 8 --- 9 files changed, 109 insertions(+), 81 deletions(-) create mode 100644 roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 318ebac7..4ddadb43 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -181,8 +181,7 @@ matrix_appservice_irc_appservice_token: "{{ matrix_synapse_macaroon_secret_key | matrix_appservice_irc_homeserver_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'irc.hs.token') | to_uuid }}" -# Postgres is the default, except if not using `matrix_postgres` (internal postgres) -matrix_appservice_irc_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_appservice_irc_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'nedb' }}" matrix_appservice_irc_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.irc.db') | to_uuid }}" diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index e0522ba8..d0fdcdf8 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -106,6 +106,7 @@ matrix_docker_package_name: docker-ce run_postgres_import: true run_postgres_upgrade: true run_postgres_import_sqlite_db: true +run_postgres_import_nedb: true run_postgres_vacuum: true run_synapse_register_user: true run_synapse_update_user_password: true diff --git a/roles/matrix-bridge-appservice-irc/defaults/main.yml b/roles/matrix-bridge-appservice-irc/defaults/main.yml index 5110fd85..0b671e76 100644 --- a/roles/matrix-bridge-appservice-irc/defaults/main.yml +++ b/roles/matrix-bridge-appservice-irc/defaults/main.yml @@ -23,12 +23,17 @@ matrix_appservice_irc_database_hostname: 'matrix-postgres' matrix_appservice_irc_database_port: 5432 matrix_appservice_irc_database_name: matrix_appservice_irc -matrix_appservice_irc_database_connString: >-2 - {%- if matrix_appservice_irc_database_engine == 'postgres' -%} - postgresql://{{ matrix_appservice_irc_database_username }}:{{ matrix_appservice_irc_database_password }}@{{ matrix_appservice_irc_database_hostname }}:{{ matrix_appservice_irc_database_port }}/{{ matrix_appservice_irc_database_name }}?sslmode=disable - {%- elif matrix_appservice_irc_database_engine == 'nedb' -%} - {{ matrix_appservice_irc_database_engine }}://{{ matrix_appservice_irc_database_file }} - {%- endif -%} +# This is just the Postgres connection string, if Postgres is used. +# Naming clashes with `matrix_appservice_irc_database_connectionString` somewhat. +matrix_appservice_irc_database_connection_string: 'postgresql://{{ matrix_appservice_irc_database_username }}:{{ matrix_appservice_irc_database_password }}@{{ matrix_appservice_irc_database_hostname }}:{{ matrix_appservice_irc_database_port }}/{{ matrix_appservice_irc_database_name }}?sslmode=disable' + +# This is what actually goes into `database.connectionString` for the bridge. +matrix_appservice_irc_database_connectionString: "{{ + { + 'nedb': 'nedb:///data', + 'postgres': matrix_appservice_irc_database_connection_string, + }[matrix_appservice_irc_database_engine] +}}" matrix_appservice_irc_ircService_servers: [] diff --git a/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml b/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml new file mode 100644 index 00000000..bc6525ec --- /dev/null +++ b/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml @@ -0,0 +1,53 @@ +- name: Fail if Postgres not enabled + fail: + msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot migrate." + when: "not matrix_postgres_enabled|bool" + +# Defaults + +- name: Set postgres_start_wait_time, if not provided + set_fact: + postgres_start_wait_time: 15 + when: "postgres_start_wait_time|default('') == ''" + +# Actual import work + +- name: Ensure matrix-postgres is started + service: + name: matrix-postgres + state: started + daemon_reload: yes + register: matrix_postgres_service_start_result + +- name: Wait a bit, so that Postgres can start + wait_for: + timeout: "{{ postgres_start_wait_time }}" + delegate_to: 127.0.0.1 + become: false + when: "matrix_postgres_service_start_result.changed|bool" + +- name: Ensure matrix-appservice-irc is stopped + service: + name: matrix-appservice-irc + state: stopped + +- name: Import appservice-irc NeDB database into Postgres + command: + cmd: >- + {{ matrix_host_command_docker }} run + --rm + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --network={{ matrix_docker_network }} + --mount type=bind,src={{ matrix_appservice_irc_data_path }},dst=/data + --entrypoint=/bin/sh + {{ matrix_appservice_irc_docker_image }} + -c + '/usr/local/bin/node /app/lib/scripts/migrate-db-to-pgres.js --dbdir /data --privateKey /data/passkey.pem --connectionString {{ matrix_appservice_irc_database_connection_string }}' + +- name: Archive NeDB database files + command: + cmd: "mv {{ matrix_appservice_irc_data_path }}/{{ item }} {{ matrix_appservice_irc_data_path }}/{{ item }}.backup" + with_items: + - rooms.db + - users.db diff --git a/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml b/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml index 5e313347..a748df96 100644 --- a/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml @@ -1,12 +1,5 @@ --- -- name: Ensure Appservice IRC image is pulled - docker_image: - name: "{{ matrix_appservice_irc_docker_image }}" - source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" - force_source: "{{ matrix_appservice_irc_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_irc_docker_image_force_pull }}" - - name: Ensure Appservice IRC paths exist file: path: "{{ item }}" @@ -24,25 +17,49 @@ path: "{{ matrix_appservice_irc_base_path }}/passkey.pem" register: matrix_appservice_irc_stat_passkey -- name: (Data relocation) Ensure matrix-appservice-irc.service is stopped - service: - name: matrix-appservice-irc - state: stopped - daemon_reload: yes - failed_when: false +- block: + - name: (Data relocation) Ensure matrix-appservice-irc.service is stopped + service: + name: matrix-appservice-irc + state: stopped + daemon_reload: yes + failed_when: false + + - name: (Data relocation) Move AppService IRC passkey.pem file to ./data directory + command: "mv {{ matrix_appservice_irc_base_path }}/passkey.pem {{ matrix_appservice_irc_data_path }}/passkey.pem" + + - name: (Data relocation) Move AppService IRC database files to ./data directory + command: "mv {{ matrix_appservice_irc_base_path }}/{{ item }} {{ matrix_appservice_irc_data_path }}/{{ item }}" + with_items: + - rooms.db + - users.db + failed_when: false when: "matrix_appservice_irc_stat_passkey.stat.exists" -- name: (Data relocation) Move AppService IRC passkey.pem file to ./data directory - command: "mv {{ matrix_appservice_irc_base_path }}/passkey.pem {{ matrix_appservice_irc_data_path }}/passkey.pem" - when: "matrix_appservice_irc_stat_passkey.stat.exists" -- name: (Data relocation) Move AppService IRC database files to ./data directory - command: "mv {{ matrix_appservice_irc_base_path }}/{{ item }} {{ matrix_appservice_irc_data_path }}/{{ item }}" - with_items: - - rooms.db - - users.db - failed_when: false - when: "matrix_appservice_irc_stat_passkey.stat.exists" +- set_fact: + matrix_appservice_irc_requires_restart: false + +- block: + - name: Check if a nedb database already exists + stat: + path: "{{ matrix_appservice_irc_data_path }}/users.db" + register: matrix_appservice_irc_nedb_database_path_local_stat_result + + - block: + - import_tasks: "{{ role_path }}/tasks/migrate_nedb_to_postgres.yml" + + - set_fact: + matrix_appservice_irc_requires_restart: true + when: "matrix_appservice_irc_nedb_database_path_local_stat_result.stat.exists|bool" + when: "matrix_appservice_irc_database_engine == 'postgres'" + +- name: Ensure Appservice IRC image is pulled + docker_image: + name: "{{ matrix_appservice_irc_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_appservice_irc_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_irc_docker_image_force_pull }}" - name: Ensure Matrix Appservice IRC config installed copy: diff --git a/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml b/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml index 9b89a340..bd08427c 100644 --- a/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml +++ b/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml @@ -33,23 +33,3 @@ when: "item.old in vars" with_items: - {'old': 'matrix_appservice_irc_container_expose_client_server_api_port', 'new': ''} - -- block: - - name: Check if a neDB database already exists - stat: - path: "{{ matrix_appservice_irc_data_path }}/" - register: matrix_appservice_irc_nedb_stat_result - - - name: Fail if an neDB database already exists when using Postgres - fail: - msg: >-2 - matrix_appservice_irc_database_engine has been set to `postgres` (which is our new default now). - However, we've discovered an existing neDB database in {{ matrix_appservice_irc_data_path }}/. - It appears that you've been using this bridge with the neDB engine until now. - To continue using neDB, opt into it explicitly: add `matrix_appservice_irc_database_engine: nedb` to your vars.yml file and re-run this same command. - Alternatively, to migrate your existing neDB database to Postgres: - 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-irc-nedb --extra-vars='nedb_database_path={{ matrix_appservice_irc_data_path }} postgres_connection_string_variable_name=matrix_appservice_irc_database_connString'`) - 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) - when: "matrix_appservice_irc_nedb_stat_result.stat.exists" - when: "matrix_appservice_irc_database_engine == 'postgres'" diff --git a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 index 0da28403..94bbda7b 100644 --- a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 +++ b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 @@ -127,8 +127,8 @@ advanced: # Use an external database to store bridge state. database: # database engine (must be 'postgres' or 'nedb'). Default: nedb - engine: {{ matrix_appservice_irc_database_engine }} + engine: {{ matrix_appservice_irc_database_engine|to_json }} # Either a PostgreSQL connection string, or a path to the NeDB storage directory. # For postgres, it must start with postgres:// # For NeDB, it must start with nedb://. The path is relative to the project directory. - connectionString: {{ matrix_appservice_irc_database_connString | to_json }} + connectionString: {{ matrix_appservice_irc_database_connectionString|to_json }} diff --git a/roles/matrix-postgres/tasks/import_nedb.yml b/roles/matrix-postgres/tasks/import_nedb.yml index cc1f9d78..2a3dd587 100644 --- a/roles/matrix-postgres/tasks/import_nedb.yml +++ b/roles/matrix-postgres/tasks/import_nedb.yml @@ -67,22 +67,6 @@ become: false when: "matrix_postgres_service_start_result.changed|bool" -# See https://github.com/matrix-org/matrix-appservice-irc/wiki/Migrating-from-NEdB-to-PostgreSQL -- name: Import appservice_irc NeDB database from {{ sqlite_database_path }} into Postgres - when: database == 'appservice_irc' - command: - cmd: >- - {{ matrix_host_command_docker }} run - --rm - --user={{ matrix_user_uid }}:{{ matrix_user_gid }} - --cap-drop=ALL - --network={{ matrix_docker_network }} - --mount type=bind,src={{ matrix_appservice_irc_data_path }}:/data:ro - --entrypoint=/bin/sh - {{ matrix_appservice_irc_docker_image }} - -c - './scripts/migrate-db-to-pgres.sh -d /data -p passkey.pem -c {{ postgres_db_connection_string }}' - # No migration.sh available, but found this: # https://github.com/matrix-org/matrix-appservice-slack/blob/develop/src/scripts/migrateToPostgres.ts # Usage should be similar to appservice_irc @@ -95,15 +79,12 @@ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} --cap-drop=ALL --network={{ matrix_docker_network }} - --mount type=bind,src={{ matrix_appservice_irc_data_path }}:/data:ro + --mount type=bind,src={{ matrix_appservice_irc_data_path }},dst=/data,ro --entrypoint=/bin/sh {{ matrix_appservice_slack_docker_image }} -c 'node /lib/scripts/migrate-db-to-pgres.js -d /data -p passkey.pem -c {{ postgres_db_connection_string }}' -- name: Archive NeDB database ({{ sqlite_database_path }} -> {{ sqlite_database_path }}.backup) - command: - cmd: "mv {{ sqlite_database_path }} {{ sqlite_database_path }}.backup" - name: Inject result set_fact: diff --git a/roles/matrix-postgres/tasks/main.yml b/roles/matrix-postgres/tasks/main.yml index 51801314..86b8f872 100644 --- a/roles/matrix-postgres/tasks/main.yml +++ b/roles/matrix-postgres/tasks/main.yml @@ -32,14 +32,6 @@ tags: - import-generic-sqlite-db -# Imports appservice-irc NeDB into postgres -- import_tasks: "{{ role_path }}/tasks/import_nedb.yml" - vars: - database: appservice_irc - when: run_postgres_import_nedb|bool - tags: - - import-irc-nedb - # Imports slacks neDB to postgres. - import_tasks: "{{ role_path }}/tasks/import_nedb.yml" vars: From 9b95e1937c61a2b21d3d11b3d497e16c6176d593 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 22 Dec 2020 19:34:08 +0200 Subject: [PATCH 202/217] Auto-restart matrix-appservice-irc after (nedb -> Postgres) migration --- roles/matrix-bridge-appservice-irc/tasks/setup_install.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml b/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml index a748df96..00568c0d 100644 --- a/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml @@ -36,7 +36,6 @@ failed_when: false when: "matrix_appservice_irc_stat_passkey.stat.exists" - - set_fact: matrix_appservice_irc_requires_restart: false @@ -164,3 +163,9 @@ service: daemon_reload: yes when: "matrix_appservice_irc_systemd_service_result.changed" + +- name: Ensure matrix-appservice-irc.service restarted, if necessary + service: + name: "matrix-appservice-irc.service" + state: restarted + when: "matrix_appservice_irc_requires_restart|bool" From 8675dedbdb310425d309f6bf1725d7a6c415279e Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 22 Dec 2020 19:56:52 +0200 Subject: [PATCH 203/217] Add support for automatic (nedb -> Postgres) migration to matrix-appservice-slack --- roles/matrix-base/defaults/main.yml | 1 - .../tasks/migrate_nedb_to_postgres.yml | 11 +++ .../defaults/main.yml | 19 ++-- .../tasks/migrate_nedb_to_postgres.yml | 66 +++++++++++++ .../tasks/setup_install.yml | 37 +++++-- .../tasks/validate_config.yml | 20 ---- .../templates/config.yaml.j2 | 7 +- roles/matrix-postgres/tasks/import_nedb.yml | 98 ------------------- roles/matrix-postgres/tasks/main.yml | 8 -- 9 files changed, 123 insertions(+), 144 deletions(-) create mode 100644 roles/matrix-bridge-appservice-slack/tasks/migrate_nedb_to_postgres.yml delete mode 100644 roles/matrix-postgres/tasks/import_nedb.yml diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index d0fdcdf8..e0522ba8 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -106,7 +106,6 @@ matrix_docker_package_name: docker-ce run_postgres_import: true run_postgres_upgrade: true run_postgres_import_sqlite_db: true -run_postgres_import_nedb: true run_postgres_vacuum: true run_synapse_register_user: true run_synapse_update_user_password: true diff --git a/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml b/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml index bc6525ec..3fab195a 100644 --- a/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml +++ b/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml @@ -51,3 +51,14 @@ with_items: - rooms.db - users.db + +- name: Inject result + set_fact: + matrix_playbook_runtime_results: | + {{ + matrix_playbook_runtime_results|default([]) + + + [ + "NOTE: Your appservice-irc database files have been imported into Postgres. The original database files have been moved from `{{ matrix_appservice_irc_data_path }}/*.db` to `{{ matrix_appservice_irc_data_path }}/*.db.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete these files." + ] + }} diff --git a/roles/matrix-bridge-appservice-slack/defaults/main.yml b/roles/matrix-bridge-appservice-slack/defaults/main.yml index 365493ff..28646a3c 100644 --- a/roles/matrix-bridge-appservice-slack/defaults/main.yml +++ b/roles/matrix-bridge-appservice-slack/defaults/main.yml @@ -51,13 +51,18 @@ matrix_appservice_slack_database_password: ~ matrix_appservice_slack_database_hostname: 'matrix-postgres' matrix_appservice_slack_database_port: 5432 matrix_appservice_slack_database_name: matrix_appservice_slack -matrix_appservice_slack_database_file: /data -matrix_appservice_slack_database_connString: >-2 - {%- if matrix_appservice_slack_database_engine == 'postgres' -%} - postgresql://{{ matrix_appservice_slack_database_username }}:{{ matrix_appservice_slack_database_password }}@{{ matrix_appservice_slack_database_hostname }}:{{ matrix_appservice_slack_database_port }}/{{ matrix_appservice_slack_database_name }}?sslmode=disable - {%- elif matrix_appservice_slack_database_engine == 'nedb' -%} - {{ matrix_appservice_slack_database_engine }}://{{ matrix_appservice_slack_database_file }} - {%- endif -%} + +# This is just the Postgres connection string, if Postgres is used. +# Naming clashes with `matrix_appservice_slack_database_connectionString` somewhat. +matrix_appservice_slack_database_connection_string: 'postgresql://{{ matrix_appservice_slack_database_username }}:{{ matrix_appservice_slack_database_password }}@{{ matrix_appservice_slack_database_hostname }}:{{ matrix_appservice_slack_database_port }}/{{ matrix_appservice_slack_database_name }}?sslmode=disable' + +# This is what actually goes into `database.connectionString` for the bridge. +matrix_appservice_slack_database_connectionString: "{{ + { + 'nedb': 'nedb:///data', + 'postgres': matrix_appservice_slack_database_connection_string, + }[matrix_appservice_slack_database_engine] +}}" matrix_appservice_slack_configuration_yaml: "{{ lookup('template', 'templates/config.yaml.j2') }}" diff --git a/roles/matrix-bridge-appservice-slack/tasks/migrate_nedb_to_postgres.yml b/roles/matrix-bridge-appservice-slack/tasks/migrate_nedb_to_postgres.yml new file mode 100644 index 00000000..fedad977 --- /dev/null +++ b/roles/matrix-bridge-appservice-slack/tasks/migrate_nedb_to_postgres.yml @@ -0,0 +1,66 @@ +- name: Fail if Postgres not enabled + fail: + msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot migrate." + when: "not matrix_postgres_enabled|bool" + +# Defaults + +- name: Set postgres_start_wait_time, if not provided + set_fact: + postgres_start_wait_time: 15 + when: "postgres_start_wait_time|default('') == ''" + +# Actual import work + +- name: Ensure matrix-postgres is started + service: + name: matrix-postgres + state: started + daemon_reload: yes + register: matrix_postgres_service_start_result + +- name: Wait a bit, so that Postgres can start + wait_for: + timeout: "{{ postgres_start_wait_time }}" + delegate_to: 127.0.0.1 + become: false + when: "matrix_postgres_service_start_result.changed|bool" + +- name: Ensure matrix-appservice-slack is stopped + service: + name: matrix-appservice-slack + state: stopped + +- name: Import appservice-slack NeDB database into Postgres + command: + cmd: >- + {{ matrix_host_command_docker }} run + --rm + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --network={{ matrix_docker_network }} + --mount type=bind,src={{ matrix_appservice_slack_data_path }},dst=/data + --entrypoint=/bin/sh + {{ matrix_appservice_slack_docker_image }} + -c + '/usr/local/bin/node /usr/src/app/lib/scripts/migrateToPostgres.js --dbdir /data --connectionString {{ matrix_appservice_slack_database_connection_string }}' + +- name: Archive NeDB database files + command: + cmd: "mv {{ matrix_appservice_slack_data_path }}/{{ item }} {{ matrix_appservice_slack_data_path }}/{{ item }}.backup" + with_items: + - teams.db + - room-store.db + - user-store.db + - event-store.db + +- name: Inject result + set_fact: + matrix_playbook_runtime_results: | + {{ + matrix_playbook_runtime_results|default([]) + + + [ + "NOTE: Your appservice-slack database files have been imported into Postgres. The original database files have been moved from `{{ matrix_appservice_slack_data_path }}/*.db` to `{{ matrix_appservice_slack_data_path }}/*.db.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete these files." + ] + }} diff --git a/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml b/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml index 94e0fedf..29b0f39e 100644 --- a/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml @@ -1,12 +1,5 @@ --- -- name: Ensure Appservice Slack image is pulled - docker_image: - name: "{{ matrix_appservice_slack_docker_image }}" - source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" - force_source: "{{ matrix_appservice_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_slack_docker_image_force_pull }}" - - name: Ensure AppService Slack paths exist file: path: "{{ item }}" @@ -19,6 +12,30 @@ - "{{ matrix_appservice_slack_config_path }}" - "{{ matrix_appservice_slack_data_path }}" +- set_fact: + matrix_appservice_slack_requires_restart: false + +- block: + - name: Check if a nedb database already exists + stat: + path: "{{ matrix_appservice_slack_data_path }}/teams.db" + register: matrix_appservice_slack_nedb_database_path_local_stat_result + + - block: + - import_tasks: "{{ role_path }}/tasks/migrate_nedb_to_postgres.yml" + + - set_fact: + matrix_appservice_slack_requires_restart: true + when: "matrix_appservice_slack_nedb_database_path_local_stat_result.stat.exists|bool" + when: "matrix_appservice_slack_database_engine == 'postgres'" + +- name: Ensure Appservice Slack image is pulled + docker_image: + name: "{{ matrix_appservice_slack_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_appservice_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_slack_docker_image_force_pull }}" + - name: Ensure Matrix Appservice Slack config installed copy: content: "{{ matrix_appservice_slack_configuration|to_nice_yaml }}" @@ -46,3 +63,9 @@ service: daemon_reload: yes when: "matrix_appservice_slack_systemd_service_result.changed" + +- name: Ensure matrix-appservice-slack.service restarted, if necessary + service: + name: "matrix-appservice-slack.service" + state: restarted + when: "matrix_appservice_slack_requires_restart|bool" diff --git a/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml b/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml index 5421b112..8af10f2f 100644 --- a/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml +++ b/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml @@ -10,23 +10,3 @@ - "matrix_appservice_slack_appservice_token" - "matrix_appservice_slack_homeserver_token" - "matrix_appservice_slack_id_token" - -- block: - - name: Check if a neDB database already exists - stat: - path: "{{ matrix_appservice_slack_data_path }}/" - register: matrix_appservice_slack_nedb_stat_result - - - name: Fail if an neDB database already exists when using Postgres - fail: - msg: >-2 - matrix_appservice_slack_database_engine has been set to `postgres` (which is our new default now). - However, we've discovered an existing neDB database in {{ matrix_appservice_slack_data_path }}/. - It appears that you've been using this bridge with the neDB engine until now. - To continue using neDB, opt into it explicitly: add `matrix_appservice_slack_database_engine: nedb` to your vars.yml file and re-run this same command. - Alternatively, to migrate your existing neDB database to Postgres: - 1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - 2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_slack_data_path }} postgres_connection_string_variable_name=matrix_appservice_slack_database_connString'`) - 3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) - when: "matrix_appservice_slack_nedb_stat_result.stat.exists" - when: "matrix_appservice_slack_database_engine == 'postgres'" diff --git a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 index 0ae13738..bf8072c1 100644 --- a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 +++ b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 @@ -11,9 +11,10 @@ homeserver: {% if matrix_appservice_slack_database_engine == 'nedb' %} dbdir: "/data" -{% endif %} +{% else %} db: - engine: "{{ matrix_appservice_slack_database_engine }}" - connectionString: {{ matrix_appservice_slack_database_connString | to_json }} + engine: {{ matrix_appservice_slack_database_engine|to_json }} + connectionString: {{ matrix_appservice_slack_database_connectionString|to_json }} +{% endif %} matrix_admin_room: "{{ matrix_appservice_slack_control_room_id }}" diff --git a/roles/matrix-postgres/tasks/import_nedb.yml b/roles/matrix-postgres/tasks/import_nedb.yml deleted file mode 100644 index 2a3dd587..00000000 --- a/roles/matrix-postgres/tasks/import_nedb.yml +++ /dev/null @@ -1,98 +0,0 @@ ---- - -# Pre-checks - -- name: Fail if Postgres not enabled - fail: - msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot import." - when: "not matrix_postgres_enabled|bool" - -- name: Fail if playbook called incorrectly - fail: - msg: "The `nedb_database_path` variable needs to be provided to this playbook, via --extra-vars" - when: "nedb_database_path is not defined or nedb_database_path.startswith('<')" - -- name: Check if the provided nedb database file exists - stat: - path: "{{ nedb_database_path }}" - register: nedb_database_path_stat_result - -- name: Fail if provided SQLite database file doesn't exist - fail: - msg: "File cannot be found on the server at {{ nedb_database_path }}" - when: "not nedb_database_path_stat_result.stat.exists" - -# We either expect `postgres_db_connection_string` specifying a full Postgres database connection string, -# or `postgres_connection_string_variable_name`, specifying a name of a variable, which contains a valid connection string. - -- block: - - name: Fail if postgres_connection_string_variable_name points to an undefined variable - fail: msg="postgres_connection_string_variable_name is defined, but there is no variable with the name `{{ postgres_connection_string_variable_name }}`" - when: "postgres_connection_string_variable_name not in vars" - - - name: Get Postgres connection string from variable - set_fact: - postgres_db_connection_string: "{{ lookup('vars', postgres_connection_string_variable_name) }}" - when: 'postgres_connection_string_variable_name is defined' - -- name: Fail if playbook called incorrectly - fail: - msg: >- - Either a `postgres_db_connection_string` variable or a `postgres_connection_string_variable_name` needs to be provided to this playbook, via `--extra-vars`. - Example: `--extra-vars="postgres_db_connection_string=postgresql://username:password@localhost:/database_name"` or `--extra-vars="postgres_connection_string_variable_name=matrix_appservice_discord_database_connString"` - when: "postgres_db_connection_string is not defined or not postgres_db_connection_string.startswith('postgresql://')" - - -# Defaults - -- name: Set postgres_start_wait_time, if not provided - set_fact: - postgres_start_wait_time: 15 - when: "postgres_start_wait_time|default('') == ''" - - -# Actual import work - -- name: Ensure matrix-postgres is started - service: - name: matrix-postgres - state: started - daemon_reload: yes - register: matrix_postgres_service_start_result - -- name: Wait a bit, so that Postgres can start - wait_for: - timeout: "{{ postgres_start_wait_time }}" - delegate_to: 127.0.0.1 - become: false - when: "matrix_postgres_service_start_result.changed|bool" - -# No migration.sh available, but found this: -# https://github.com/matrix-org/matrix-appservice-slack/blob/develop/src/scripts/migrateToPostgres.ts -# Usage should be similar to appservice_irc -- name: Import appservice_slack NeDB database from {{ sqlite_database_path }} into Postgres - when: database == 'appservice_slack' - command: - cmd: >- - {{ matrix_host_command_docker }} run - --rm - --user={{ matrix_user_uid }}:{{ matrix_user_gid }} - --cap-drop=ALL - --network={{ matrix_docker_network }} - --mount type=bind,src={{ matrix_appservice_irc_data_path }},dst=/data,ro - --entrypoint=/bin/sh - {{ matrix_appservice_slack_docker_image }} - -c - 'node /lib/scripts/migrate-db-to-pgres.js -d /data -p passkey.pem -c {{ postgres_db_connection_string }}' - - -- name: Inject result - set_fact: - matrix_playbook_runtime_results: | - {{ - matrix_playbook_runtime_results|default([]) - + - [ - "NOTE: Your NeDB database file has been imported into Postgres. The original directory has been moved from `{{ nedb_database_path }}` to `{{ nedb_database_path }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file." - ] - }} diff --git a/roles/matrix-postgres/tasks/main.yml b/roles/matrix-postgres/tasks/main.yml index 86b8f872..b9c2ae7c 100644 --- a/roles/matrix-postgres/tasks/main.yml +++ b/roles/matrix-postgres/tasks/main.yml @@ -32,14 +32,6 @@ tags: - import-generic-sqlite-db -# Imports slacks neDB to postgres. -- import_tasks: "{{ role_path }}/tasks/import_nedb.yml" - vars: - database: appservice_slack - when: run_postgres_import_nedb|bool - tags: - - import-slack-nedb - - import_tasks: "{{ role_path }}/tasks/upgrade_postgres.yml" when: run_postgres_upgrade|bool tags: From ad1425eee4f8d9eb1f92f5f0acecef8da3447280 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 23 Dec 2020 09:08:54 +0200 Subject: [PATCH 204/217] Add pgloader self-building support (for ARM) --- group_vars/matrix_servers | 2 + roles/matrix-postgres/defaults/main.yml | 13 ++++- .../tasks/util/migrate_db_to_postgres.yml | 53 +++++++++++++++---- 3 files changed, 57 insertions(+), 11 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 4ddadb43..c8af3cf2 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -995,6 +995,8 @@ matrix_postgres_connection_username: "synapse" matrix_postgres_connection_password: "synapse-password" matrix_postgres_db_name: "homeserver" +matrix_postgres_pgloader_container_image_self_build: "{{ matrix_architecture != 'amd64' }}" + matrix_postgres_additional_databases: | {{ ([{ diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index 0c516281..8f1d0d78 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -65,4 +65,15 @@ matrix_postgres_import_databases_ignore_regex: "^CREATE DATABASE ({{ matrix_post # For most (subsequent) runs, Postgres would already be running, so no waiting will be happening at all. matrix_postgres_additional_databases_postgres_start_wait_timeout_seconds: 15 -matrix_postgres_pgloader_docker_image: "docker.io/illagrenan/pgloader:3.6.2" + +matrix_postgres_pgloader_container_image_self_build: false +matrix_postgres_pgloader_container_image_self_build_repo: "https://github.com/illagrenan/pgloader-docker.git" +matrix_postgres_pgloader_container_image_self_build_repo_branch: "v{{ matrix_postgres_pgloader_docker_image_tag }}" +matrix_postgres_pgloader_container_image_self_build_src_path: "{{ matrix_postgres_base_path }}/pgloader-container-src" + +# We use illagrenan/pgloader, instead of the more official dimitri/pgloader image, +# because the official one only provides a `latest` tag. +matrix_postgres_pgloader_docker_image: "{{ matrix_postgres_pgloader_docker_image_name_prefix }}illagrenan/pgloader:{{ matrix_postgres_pgloader_docker_image_tag }}" +matrix_postgres_pgloader_docker_image_name_prefix: "{{ 'localhost/' if matrix_postgres_pgloader_container_image_self_build else 'docker.io/' }}" +matrix_postgres_pgloader_docker_image_tag: "3.6.2" +matrix_postgres_pgloader_docker_image_force_pull: "{{ matrix_postgres_pgloader_docker_image.endswith(':latest') }}" diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml index d01611ef..0da48c64 100644 --- a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml +++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml @@ -31,17 +31,50 @@ msg: "File cannot be found on the server at {{ matrix_postgres_db_migration_request.src }}" when: "not matrix_postgres_db_migration_request_src_stat_result.stat.exists" -- name: Fail if we cannot migrate on the current architecture ({{ matrix_architecture }}) - fail: - msg: >- - {{ matrix_postgres_db_migration_request.engine_variable_name }} (part of {{ matrix_postgres_db_migration_request.caller }}) has been set to `postgres` (which is our new default now). - However, we've discovered an existing file-based database ({{ matrix_postgres_db_migration_request.engine_old }}) in {{ matrix_postgres_db_migration_request.src }}. - It appears that you've been using this bridge with a file-based database engine until now. - To continue using {{ matrix_postgres_db_migration_request.engine_old }}, opt into it explicitly: add `{{ matrix_postgres_db_migration_request.engine_variable_name }}: {{ matrix_postgres_db_migration_request.engine_old }}` to your vars.yml file and re-run this same command. - We'd normally auto-migrate you to Postgres, but we can't do it on the {{ matrix_architecture }} architecture. Our pgloader container image only supports amd64 (for now). - Learn more here: https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/740 - when: "matrix_architecture != 'amd64'" +- block: + - name: Ensure pgloader repository is present on self-build + git: + repo: "{{ matrix_postgres_pgloader_container_image_self_build_repo }}" + dest: "{{ matrix_postgres_pgloader_container_image_self_build_src_path }}" + version: "{{ matrix_postgres_pgloader_container_image_self_build_repo_branch }}" + force: "yes" + register: matrix_postgres_pgloader_git_pull_results + # If `stable` is used, we hit an error when processing /opt/src/pgloader/build/quicklisp/dists/quicklisp/software/uax-15-20201220-git/data/CompositionExclusions.txt: + # > the octet sequence #(194) cannot be decoded + # + # The issue is described here and is not getting fixed for months: https://github.com/dimitri/pgloader/pull/1179 + # + # Although we're not using the dimitri/pgloader image, the one we're using suffers from the same problem. + - name: Switch pgloader base image from Debian stable (likely 10.x/Buster) to Bullseye + lineinfile: + path: "{{ matrix_postgres_pgloader_container_image_self_build_src_path }}/Dockerfile" + regexp: "{{ item.match }}" + line: "{{ item.replace }}" + with_items: + - match: '^FROM debian:stable-slim as builder$' + replace: 'FROM debian:bullseye-slim as builder' + - match: '^FROM debian:stable-slim$' + replace: 'FROM debian:bullseye-slim' + + - name: Ensure pgloader Docker image is built + docker_image: + name: "{{ matrix_postgres_pgloader_docker_image }}" + source: build + force_source: "{{ matrix_postgres_pgloader_git_pull_results.changed }}" + build: + dockerfile: Dockerfile + path: "{{ matrix_postgres_pgloader_container_image_self_build_src_path }}" + pull: yes + when: "matrix_postgres_pgloader_container_image_self_build|bool" + +- name: Ensure pgloader Docker image is pulled + docker_image: + name: "{{ matrix_postgres_pgloader_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_postgres_pgloader_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_postgres_pgloader_docker_image_force_pull }}" + when: "not matrix_postgres_pgloader_container_image_self_build" # Defaults From 3475b98b76ff27af5250dceba1e90195e2825ed7 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 23 Dec 2020 11:02:02 +0200 Subject: [PATCH 205/217] Announce the big move to all-on-Postgres Related to - https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/740 - https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/686 --- CHANGELOG.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 946ec5b0..efc5c536 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,31 @@ +# 2020-12-23 + +## The big move to all-on-Postgres (potentially dangerous) + +**TLDR**: all your bridges (and other services) will likely be auto-migrated from SQLite/nedb to Postgres, hopefully without trouble. You can opt-out (see how below), if too worried about breakage. + +Until now, we've only used Postgres as a database for Synapse. All other services (bridges, bots, etc.) were kept simple and used a file-based database (SQLite or nedb). + +Since [this huge pull request](https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/740), **all of our services now use Postgres by default**. Thanks to [Johanna Dorothea Reichmann](https://github.com/jdreichmann) for starting the work on it and for providing great input! + +Moving all services to Postgres brings a few **benefits** to us: + +- **improved performance** +- **improved compatibility**. Most bridges are deprecating SQLite/nedb support or offer less features when not on Postgres. +- **easier backups**. It's still some effort to take a proper backup (Postgres dump + various files, keys), but a Postgres dump now takes you much further. +- we're now **more prepared to introduce other services** that need a Postgres database - [Dendrite](https://github.com/matrix-org/dendrite), the [mautrix-signal](https://github.com/tulir/mautrix-signal) bridge (existing [pull request](https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/686)), etc. + +### Key takeway + +- existing installations that use an [external Postgres](https://github.com/spantaleev/matrix-docker-ansible-deploy/blob/master/docs/configuring-playbook-external-postgres.md) server should be unaffected (they remain on SQLite/nedb for all services, except Synapse) + +- for existing installations which use our integrated Postgres database server (`matrix-postgres`, which is the default), **we automatically migrate data** from SQLite/nedb to Postgres and **archive the database files** (`something.db` -> `something.db.backup`), so you can restore them if you need to go back (see how below). + +- this is a **very large and somewhat untested change** (potentially dangerous), so **if you're not feeling confident/experimental, opt-out** of it for now (see below). Still, it's the new default and what we (and various bridges) will focus on going forward, so don't stick to old ways for too long. + +- you can remain on SQLite/nedb (at least for now) by adding a variable like this to your `vars.yml` file for each service you use: `matrix_COMPONENT_database_engine: sqlite` (e.g. `matrix_mautrix_facebook_database_engine: sqlite`). Some services (like `appservice-irc` and `appservice-slack`) don't use SQLite, so use `nedb`, instead of `sqlite` for them. If the playbook had already migrated you to Postgres, you will need to rename back the database files (`something.db.backup` -> `something.db`). + + # 2020-12-11 ## synapse-janitor support removed From 4fe1248d95b116177315edb986a2c86122423c86 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 23 Dec 2020 11:21:06 +0200 Subject: [PATCH 206/217] Update changelog entry to be more informative --- CHANGELOG.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index efc5c536..c4a586e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,9 +21,23 @@ Moving all services to Postgres brings a few **benefits** to us: - for existing installations which use our integrated Postgres database server (`matrix-postgres`, which is the default), **we automatically migrate data** from SQLite/nedb to Postgres and **archive the database files** (`something.db` -> `something.db.backup`), so you can restore them if you need to go back (see how below). -- this is a **very large and somewhat untested change** (potentially dangerous), so **if you're not feeling confident/experimental, opt-out** of it for now (see below). Still, it's the new default and what we (and various bridges) will focus on going forward, so don't stick to old ways for too long. +### Opting-out of the Postgres migration -- you can remain on SQLite/nedb (at least for now) by adding a variable like this to your `vars.yml` file for each service you use: `matrix_COMPONENT_database_engine: sqlite` (e.g. `matrix_mautrix_facebook_database_engine: sqlite`). Some services (like `appservice-irc` and `appservice-slack`) don't use SQLite, so use `nedb`, instead of `sqlite` for them. If the playbook had already migrated you to Postgres, you will need to rename back the database files (`something.db.backup` -> `something.db`). +This is a **very large and somewhat untested change** (potentially dangerous), so **if you're not feeling confident/experimental, opt-out** of it for now. Still, it's the new default and what we (and various bridges) will focus on going forward, so don't stick to old ways for too long. + +You can remain on SQLite/nedb (at least for now) by adding a variable like this to your `vars.yml` file for each service you use: `matrix_COMPONENT_database_engine: sqlite` (e.g. `matrix_mautrix_facebook_database_engine: sqlite`). + +Some services (like `appservice-irc` and `appservice-slack`) don't use SQLite, so use `nedb`, instead of `sqlite` for them. + +### Going back to SQLite/nedb if things went wrong + +If you went with the Postgres migration and it went badly for you (some bridge not working as expected or not working at all), do this: + +- stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) +- SSH into the server and rename the old database files (`something.db.backup` -> `something.db`). Example: `mv /matrix/mautrix-facebook/data/mautrix-facebook.db.backup /matrix/mautrix-facebook/data/mautrix-facebook.db` +- switch the affected service back to SQLite (e.g. `matrix_mautrix_facebook_database_engine: sqlite`). Some services (like `appservice-irc` and `appservice-slack`) don't use SQLite, so use `nedb`, instead of `sqlite` for them. +- re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`) +- [get in touch](README.md#support) with us # 2020-12-11 From c5f8b1f61bf544bdcae0cd916c3d48d17886d277 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 23 Dec 2020 11:40:22 +0200 Subject: [PATCH 207/217] Fix mautrix-whatsapp Postgres connection string to not use SSL by default --- roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml index 0d4f9852..beda6d7d 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml @@ -46,7 +46,7 @@ matrix_mautrix_whatsapp_database_hostname: 'matrix-postgres' matrix_mautrix_whatsapp_database_port: 5432 matrix_mautrix_whatsapp_database_name: 'matrix_mautrix_whatsapp' -matrix_mautrix_whatsapp_database_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_database_username }}:{{ matrix_mautrix_whatsapp_database_password }}@{{ matrix_mautrix_whatsapp_database_hostname }}:{{ matrix_mautrix_whatsapp_database_port }}/{{ matrix_mautrix_whatsapp_database_name }}' +matrix_mautrix_whatsapp_database_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_database_username }}:{{ matrix_mautrix_whatsapp_database_password }}@{{ matrix_mautrix_whatsapp_database_hostname }}:{{ matrix_mautrix_whatsapp_database_port }}/{{ matrix_mautrix_whatsapp_database_name }}?sslmode=disable' matrix_mautrix_whatsapp_appservice_database_type: "{{ { From 4675c8a715bae4da5288d14badf5d18f9e7a30e0 Mon Sep 17 00:00:00 2001 From: Jan Date: Wed, 23 Dec 2020 11:23:09 +0100 Subject: [PATCH 208/217] Update updating-users-passwords.md Fixed markdown error --- docs/updating-users-passwords.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/updating-users-passwords.md b/docs/updating-users-passwords.md index 785bc17c..7d2f2832 100644 --- a/docs/updating-users-passwords.md +++ b/docs/updating-users-passwords.md @@ -26,7 +26,7 @@ and then connecting to the postgres server and executing: ``` UPDATE users SET password_hash = '' WHERE name = '@someone:server.com' ``` -` + where `` is the hash returned by the docker command above. From 9f00970c90795c92dad9d1e83ea7fa4f8e2afa80 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 23 Dec 2020 12:31:52 +0200 Subject: [PATCH 209/217] Do not use salts longer than 16 characters We've hit this problem before as well. Certain Ansible installations choke on it. --- group_vars/matrix_servers | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 904b2291..44848ae5 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -208,7 +208,7 @@ matrix_mautrix_facebook_bridge_presence: "{{ matrix_synapse_use_presence if matr # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mautrix_facebook_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mautrix_facebook_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_facebook.db') | to_uuid }}" +matrix_mautrix_facebook_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau_fb.db') | to_uuid }}" ###################################################################### # @@ -247,7 +247,7 @@ matrix_mautrix_hangouts_login_shared_secret: "{{ matrix_synapse_ext_password_pro # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mautrix_hangouts_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mautrix_hangouts_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_hangouts.db') | to_uuid }}" +matrix_mautrix_hangouts_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau_hangouts.db') | to_uuid }}" ###################################################################### # @@ -289,7 +289,7 @@ matrix_mautrix_telegram_login_shared_secret: "{{ matrix_synapse_ext_password_pro # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mautrix_telegram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mautrix_telegram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_telegram.db') | to_uuid }}" +matrix_mautrix_telegram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau_telegram.db') | to_uuid }}" ###################################################################### # @@ -323,7 +323,7 @@ matrix_mautrix_whatsapp_login_shared_secret: "{{ matrix_synapse_ext_password_pro # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mautrix_whatsapp_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mautrix_whatsapp_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_whatsapp.db') | to_uuid }}" +matrix_mautrix_whatsapp_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau_whatsapp.db') | to_uuid }}" ###################################################################### # @@ -385,7 +385,7 @@ matrix_mx_puppet_skype_login_shared_secret: "{{ matrix_synapse_ext_password_prov # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_skype_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_skype_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_skype.db') | to_uuid }}" +matrix_mx_puppet_skype_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_skype.db') | to_uuid }}" ###################################################################### # @@ -422,7 +422,7 @@ matrix_mx_puppet_slack_login_shared_secret: "{{ matrix_synapse_ext_password_prov # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_slack.db') | to_uuid }}" +matrix_mx_puppet_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_slack.db') | to_uuid }}" ###################################################################### # @@ -460,7 +460,7 @@ matrix_mx_puppet_twitter_container_http_host_bind_port: "{{ '' if matrix_nginx_p # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_twitter_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_twitter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_twitter.db') | to_uuid }}" +matrix_mx_puppet_twitter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_twitter.db') | to_uuid }}" ###################################################################### # @@ -497,7 +497,7 @@ matrix_mx_puppet_instagram_login_shared_secret: "{{ matrix_synapse_ext_password_ # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_instagram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_instagram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_instagram.db') | to_uuid }}" +matrix_mx_puppet_instagram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_ig.db') | to_uuid }}" ###################################################################### # @@ -533,7 +533,7 @@ matrix_mx_puppet_discord_login_shared_secret: "{{ matrix_synapse_ext_password_pr # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_discord_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_discord.db') | to_uuid }}" +matrix_mx_puppet_discord_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_dsc.db') | to_uuid }}" ###################################################################### # @@ -569,7 +569,7 @@ matrix_mx_puppet_steam_login_shared_secret: "{{ matrix_synapse_ext_password_prov # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_steam_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_steam.db') | to_uuid }}" +matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_steam.db') | to_uuid }}" ###################################################################### # @@ -1048,7 +1048,7 @@ matrix_postgres_additional_databases: | ([{ 'name': 'matrix_bridge_sms', 'username': 'matrix_bridge_sms', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid, + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db') | to_uuid, }] if matrix_sms_bridge_enabled else []) + ([{ From 8b6174786b86f3ec09246228df7ad21e9a178f8f Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 23 Dec 2020 12:57:30 +0200 Subject: [PATCH 210/217] Fixup Dimension database schema a bit after pgloader import --- roles/matrix-dimension/tasks/setup_install.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/matrix-dimension/tasks/setup_install.yml b/roles/matrix-dimension/tasks/setup_install.yml index 9a264449..b7f9b8e0 100644 --- a/roles/matrix-dimension/tasks/setup_install.yml +++ b/roles/matrix-dimension/tasks/setup_install.yml @@ -19,6 +19,12 @@ engine_old: 'sqlite' systemd_services_to_stop: ['matrix-dimension.service'] pgloader_options: ['--with "quote identifiers"'] + # pgloader makes `isSelfBot` of type `smallint`, instead of `boolean`. + # We need to fix it up + additional_psql_statements_list: + - 'ALTER TABLE dimension_users ALTER COLUMN "isSelfBot" DROP default;' + - 'ALTER TABLE dimension_users ALTER COLUMN "isSelfBot" TYPE BOOLEAN USING("isSelfBot"::text::boolean);' + - 'ALTER TABLE dimension_users ALTER COLUMN "isSelfBot" SET default false;' - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" From ea804f2f9fc16cdfbd9d1e30ff22f10944b8f28a Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 23 Dec 2020 12:59:48 +0200 Subject: [PATCH 211/217] Do not use underscore in salts passed to sha512 Some Ansible installations choke on it, it seems. Similar to 9f00970c907 --- group_vars/matrix_servers | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 44848ae5..a54d8baa 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -208,7 +208,7 @@ matrix_mautrix_facebook_bridge_presence: "{{ matrix_synapse_use_presence if matr # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mautrix_facebook_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mautrix_facebook_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau_fb.db') | to_uuid }}" +matrix_mautrix_facebook_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau.fb.db') | to_uuid }}" ###################################################################### # @@ -247,7 +247,7 @@ matrix_mautrix_hangouts_login_shared_secret: "{{ matrix_synapse_ext_password_pro # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mautrix_hangouts_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mautrix_hangouts_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau_hangouts.db') | to_uuid }}" +matrix_mautrix_hangouts_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau.hangouts.db') | to_uuid }}" ###################################################################### # @@ -289,7 +289,7 @@ matrix_mautrix_telegram_login_shared_secret: "{{ matrix_synapse_ext_password_pro # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mautrix_telegram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mautrix_telegram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau_telegram.db') | to_uuid }}" +matrix_mautrix_telegram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau.telegram.db') | to_uuid }}" ###################################################################### # @@ -323,7 +323,7 @@ matrix_mautrix_whatsapp_login_shared_secret: "{{ matrix_synapse_ext_password_pro # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mautrix_whatsapp_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mautrix_whatsapp_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mau_whatsapp.db') | to_uuid }}" +matrix_mautrix_whatsapp_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mauwhatsapp.db') | to_uuid }}" ###################################################################### # @@ -385,7 +385,7 @@ matrix_mx_puppet_skype_login_shared_secret: "{{ matrix_synapse_ext_password_prov # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_skype_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_skype_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_skype.db') | to_uuid }}" +matrix_mx_puppet_skype_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup.skype.db') | to_uuid }}" ###################################################################### # @@ -422,7 +422,7 @@ matrix_mx_puppet_slack_login_shared_secret: "{{ matrix_synapse_ext_password_prov # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_slack.db') | to_uuid }}" +matrix_mx_puppet_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup.slack.db') | to_uuid }}" ###################################################################### # @@ -460,7 +460,7 @@ matrix_mx_puppet_twitter_container_http_host_bind_port: "{{ '' if matrix_nginx_p # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_twitter_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_twitter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_twitter.db') | to_uuid }}" +matrix_mx_puppet_twitter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup.twitter.db') | to_uuid }}" ###################################################################### # @@ -497,7 +497,7 @@ matrix_mx_puppet_instagram_login_shared_secret: "{{ matrix_synapse_ext_password_ # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_instagram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_instagram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_ig.db') | to_uuid }}" +matrix_mx_puppet_instagram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup.ig.db') | to_uuid }}" ###################################################################### # @@ -533,7 +533,7 @@ matrix_mx_puppet_discord_login_shared_secret: "{{ matrix_synapse_ext_password_pr # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_discord_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_dsc.db') | to_uuid }}" +matrix_mx_puppet_discord_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup.dsc.db') | to_uuid }}" ###################################################################### # @@ -569,7 +569,7 @@ matrix_mx_puppet_steam_login_shared_secret: "{{ matrix_synapse_ext_password_prov # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_mx_puppet_steam_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" -matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup_steam.db') | to_uuid }}" +matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxpup.steam.db') | to_uuid }}" ###################################################################### # @@ -1048,7 +1048,7 @@ matrix_postgres_additional_databases: | ([{ 'name': 'matrix_bridge_sms', 'username': 'matrix_bridge_sms', - 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db') | to_uuid, + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge.sms.db') | to_uuid, }] if matrix_sms_bridge_enabled else []) + ([{ From be0c59956525f6219aa510344d03d09e850fa7ca Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 23 Dec 2020 13:33:25 +0200 Subject: [PATCH 212/217] Feed more slashes to mautrix bridges when using SQLite This makes the `sqlite://` URI match what we were using before and what the config expects. --- roles/matrix-bridge-mautrix-facebook/defaults/main.yml | 2 +- roles/matrix-bridge-mautrix-hangouts/defaults/main.yml | 2 +- roles/matrix-bridge-mautrix-telegram/defaults/main.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml index 580934db..80e54e0e 100644 --- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml @@ -55,7 +55,7 @@ matrix_mautrix_facebook_database_connection_string: 'postgresql://{{ matrix_maut matrix_mautrix_facebook_appservice_database: "{{ { - 'sqlite': ('sqlite://' + matrix_mautrix_facebook_sqlite_database_path_in_container), + 'sqlite': ('sqlite:///' + matrix_mautrix_facebook_sqlite_database_path_in_container), 'postgres': matrix_mautrix_facebook_database_connection_string, }[matrix_mautrix_facebook_database_engine] }}" diff --git a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml index 984bec48..e266bcbb 100644 --- a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml @@ -62,7 +62,7 @@ matrix_mautrix_hangouts_database_connection_string: 'postgresql://{{ matrix_maut matrix_mautrix_hangouts_appservice_database: "{{ { - 'sqlite': ('sqlite://' + matrix_mautrix_hangouts_sqlite_database_path_in_container), + 'sqlite': ('sqlite:///' + matrix_mautrix_hangouts_sqlite_database_path_in_container), 'postgres': matrix_mautrix_hangouts_database_connection_string, }[matrix_mautrix_hangouts_database_engine] }}" diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml index ceebd3ec..539f0a9c 100644 --- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml @@ -66,7 +66,7 @@ matrix_mautrix_telegram_database_connection_string: 'postgresql://{{ matrix_maut matrix_mautrix_telegram_appservice_database: "{{ { - 'sqlite': ('sqlite://' + matrix_mautrix_telegram_sqlite_database_path_in_container), + 'sqlite': ('sqlite:///' + matrix_mautrix_telegram_sqlite_database_path_in_container), 'postgres': matrix_mautrix_telegram_database_connection_string, }[matrix_mautrix_telegram_database_engine] }}" From 019a4d7dcd774776aa9c7f6fef80cb9c58cfba97 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 23 Dec 2020 10:38:47 +0000 Subject: [PATCH 213/217] Use role relative paths for things --- roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml | 2 +- roles/matrix-bridge-appservice-discord/tasks/setup_install.yml | 2 +- roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml | 2 +- roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml | 2 +- roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml | 2 +- roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml | 2 +- roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml | 2 +- roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml | 2 +- roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml | 2 +- roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml | 2 +- roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml | 2 +- roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml | 2 +- roles/matrix-dimension/tasks/setup_install.yml | 2 +- roles/matrix-ma1sd/tasks/setup_install.yml | 2 +- roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml | 2 +- roles/matrix-registration/tasks/setup_install.yml | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml index fea1e00d..195485e4 100644 --- a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml +++ b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml @@ -19,7 +19,7 @@ engine_old: 'sqlite' systemd_services_to_stop: ['matrix-bot-matrix-reminder-bot.service'] - - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml" - set_fact: matrix_bot_matrix_reminder_bot_requires_restart: true diff --git a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml index 8bf50e28..6d3fdd0f 100644 --- a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml @@ -19,7 +19,7 @@ engine_old: 'sqlite' systemd_services_to_stop: ['matrix-appservice-discord.service'] - - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml" - set_fact: matrix_appservice_discord_requires_restart: true diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml index 7f310446..59998463 100644 --- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml @@ -27,7 +27,7 @@ engine_old: 'sqlite' systemd_services_to_stop: ['matrix-mautrix-facebook.service'] - - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml" - set_fact: matrix_mautrix_facebook_requires_restart: true diff --git a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml index 15af9626..2ce8a441 100644 --- a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml @@ -27,7 +27,7 @@ engine_old: 'sqlite' systemd_services_to_stop: ['matrix-mautrix-hangouts.service'] - - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml" - set_fact: matrix_mautrix_hangouts_requires_restart: true diff --git a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml index 6a37974a..e9a93c72 100644 --- a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml @@ -27,7 +27,7 @@ engine_old: 'sqlite' systemd_services_to_stop: ['matrix-mautrix-telegram.service'] - - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml" - set_fact: matrix_mautrix_telegram_requires_restart: true diff --git a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml index 8d894a84..1cfa60f8 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml @@ -27,7 +27,7 @@ engine_old: 'sqlite' systemd_services_to_stop: ['matrix-mautrix-whatsapp.service'] - - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml" - set_fact: matrix_mautrix_whatsapp_requires_restart: true diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml index 3e3b2f94..c7865e98 100644 --- a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml @@ -58,7 +58,7 @@ engine_old: 'sqlite' systemd_services_to_stop: ['matrix-mx-puppet-discord.service'] - - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml" - set_fact: matrix_mx_puppet_discord_requires_restart: true diff --git a/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml index 76bbd629..5701a916 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml @@ -28,7 +28,7 @@ engine_old: 'sqlite' systemd_services_to_stop: ['matrix-mx-puppet-instagram.service'] - - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml" - set_fact: matrix_mx_puppet_instagram_requires_restart: true diff --git a/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml index 9289a793..68a1d7f4 100644 --- a/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml @@ -58,7 +58,7 @@ engine_old: 'sqlite' systemd_services_to_stop: ['matrix-mx-puppet-skype.service'] - - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml" - set_fact: matrix_mx_puppet_skype_requires_restart: true diff --git a/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml index 5d68435c..04eab20a 100644 --- a/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml @@ -54,7 +54,7 @@ engine_old: 'sqlite' systemd_services_to_stop: ['matrix-mx-puppet-slack.service'] - - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml" - set_fact: matrix_mx_puppet_slack_requires_restart: true diff --git a/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml index 71f6d889..6b574656 100644 --- a/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml @@ -58,7 +58,7 @@ engine_old: 'sqlite' systemd_services_to_stop: ['matrix-mx-puppet-steam.service'] - - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml" - set_fact: matrix_mx_puppet_steam_requires_restart: true diff --git a/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml index 3893981a..1c48c030 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml @@ -58,7 +58,7 @@ engine_old: 'sqlite' systemd_services_to_stop: ['matrix-mx-puppet-twitter.service'] - - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml" - set_fact: matrix_mx_puppet_twitter_requires_restart: true diff --git a/roles/matrix-dimension/tasks/setup_install.yml b/roles/matrix-dimension/tasks/setup_install.yml index b7f9b8e0..ca12f367 100644 --- a/roles/matrix-dimension/tasks/setup_install.yml +++ b/roles/matrix-dimension/tasks/setup_install.yml @@ -26,7 +26,7 @@ - 'ALTER TABLE dimension_users ALTER COLUMN "isSelfBot" TYPE BOOLEAN USING("isSelfBot"::text::boolean);' - 'ALTER TABLE dimension_users ALTER COLUMN "isSelfBot" SET default false;' - - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml" - set_fact: matrix_dimension_requires_restart: true diff --git a/roles/matrix-ma1sd/tasks/setup_install.yml b/roles/matrix-ma1sd/tasks/setup_install.yml index 9ae5f077..a0a32728 100644 --- a/roles/matrix-ma1sd/tasks/setup_install.yml +++ b/roles/matrix-ma1sd/tasks/setup_install.yml @@ -38,7 +38,7 @@ systemd_services_to_stop: ['matrix-ma1sd.service'] pgloader_options: ['--with "quote identifiers"'] - - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml" - set_fact: matrix_ma1sd_requires_restart: true diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml index 0da48c64..de0327b7 100644 --- a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml +++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml @@ -135,7 +135,7 @@ - block: # We can't use `{{ role_path }}` here, neither with `import_tasks`, nor with `include_tasks`, # because it refers to the role that included this util, and not to the role this file belongs to. - - import_tasks: "roles/matrix-postgres/tasks/util/detect_existing_postgres_version.yml" + - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/detect_existing_postgres_version.yml" - set_fact: matrix_postgres_docker_image_to_use: "{{ matrix_postgres_docker_image_latest if matrix_postgres_detected_version_corresponding_docker_image == '' else matrix_postgres_detected_version_corresponding_docker_image }}" diff --git a/roles/matrix-registration/tasks/setup_install.yml b/roles/matrix-registration/tasks/setup_install.yml index 9b6d1260..2b806fe0 100644 --- a/roles/matrix-registration/tasks/setup_install.yml +++ b/roles/matrix-registration/tasks/setup_install.yml @@ -24,7 +24,7 @@ - ALTER TABLE tokens ALTER COLUMN ex_date TYPE TIMESTAMP WITHOUT TIME ZONE; additional_psql_statements_db_name: "{{ matrix_registration_database_name }}" - - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml" - set_fact: matrix_registration_requires_restart: true From 21662af3be5c13fe03ed304a29860935a7a72365 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 23 Dec 2020 14:11:21 +0200 Subject: [PATCH 214/217] Archive database only after additional_psql_statements_list had executed --- .../matrix-postgres/tasks/util/migrate_db_to_postgres.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml index de0327b7..af95815f 100644 --- a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml +++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml @@ -128,10 +128,6 @@ -c 'pgloader {{ matrix_postgres_db_migration_request.pgloader_options|default([])|join(' ') }} /in.db {{ matrix_postgres_db_migration_request.dst }}' -- name: Archive {{ matrix_postgres_db_migration_request.engine_old }} database ({{ matrix_postgres_db_migration_request.src }} -> {{ matrix_postgres_db_migration_request.src }}.backup) - command: - cmd: "mv {{ matrix_postgres_db_migration_request.src }} {{ matrix_postgres_db_migration_request.src }}.backup" - - block: # We can't use `{{ role_path }}` here, neither with `import_tasks`, nor with `include_tasks`, # because it refers to the role that included this util, and not to the role this file belongs to. @@ -155,6 +151,10 @@ when: "matrix_postgres_db_migration_request.additional_psql_statements_list|default([])|length > 0" +- name: Archive {{ matrix_postgres_db_migration_request.engine_old }} database ({{ matrix_postgres_db_migration_request.src }} -> {{ matrix_postgres_db_migration_request.src }}.backup) + command: + cmd: "mv {{ matrix_postgres_db_migration_request.src }} {{ matrix_postgres_db_migration_request.src }}.backup" + - name: Inject result set_fact: matrix_playbook_runtime_results: | From 80c72615c7bfdcf47644d55f033e88f4e610cf25 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 23 Dec 2020 14:11:47 +0200 Subject: [PATCH 215/217] Fixup all Dimension boolean fields after pgloader import This is 8b6174786b86f3 done right. There were many more fields that we had to account for. --- .../matrix-dimension/tasks/setup_install.yml | 46 ++++++++++++++++--- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/roles/matrix-dimension/tasks/setup_install.yml b/roles/matrix-dimension/tasks/setup_install.yml index ca12f367..26a75bcb 100644 --- a/roles/matrix-dimension/tasks/setup_install.yml +++ b/roles/matrix-dimension/tasks/setup_install.yml @@ -10,6 +10,44 @@ register: matrix_dimension_sqlite_database_path_local_stat_result - block: + # pgloader makes a few columns `smallint`, instead of `boolean`. + # We need to fix them up. + - set_fact: + matrix_dimension_pgloader_additional_psql_statements_list: [] + + - set_fact: + matrix_dimension_pgloader_additional_psql_statements_list: | + {{ + matrix_dimension_pgloader_additional_psql_statements_list + + + ([] if item.default == '' else ['ALTER TABLE ' + item.table + ' ALTER COLUMN "' + item.column + '" DROP default;']) + + + (['ALTER TABLE ' + item.table + ' ALTER COLUMN "' + item.column + '" TYPE BOOLEAN USING("' + item.column + '"::text::boolean);']) + + + ([] if item.default == '' else ['ALTER TABLE ' + item.table + ' ALTER COLUMN "' + item.column + '" SET default ' + item.default + ';']) + }} + with_items: + - {'table': 'dimension_widgets', 'column': 'isEnabled', 'default': ''} + - {'table': 'dimension_widgets', 'column': 'isPublic', 'default': ''} + - {'table': 'dimension_webhook_bridges', 'column': 'isEnabled', 'default': ''} + - {'table': 'dimension_user_sticker_packs', 'column': 'isSelected', 'default': ''} + - {'table': 'dimension_scalar_tokens', 'column': 'isDimensionToken', 'default': ''} + - {'table': 'dimension_users', 'column': 'isSelfBot', 'default': 'false'} + - {'table': 'dimension_telegram_bridges', 'column': 'allowTgPuppets', 'default': ''} + - {'table': 'dimension_telegram_bridges', 'column': 'allowMxPuppets', 'default': ''} + - {'table': 'dimension_telegram_bridges', 'column': 'isEnabled', 'default': ''} + - {'table': 'dimension_sticker_packs', 'column': 'isEnabled', 'default': ''} + - {'table': 'dimension_sticker_packs', 'column': 'isPublic', 'default': ''} + - {'table': 'dimension_slack_bridges', 'column': 'isEnabled', 'default': ''} + - {'table': 'dimension_neb_integrations', 'column': 'isPublic', 'default': ''} + - {'table': 'dimension_irc_bridges', 'column': 'isEnabled', 'default': ''} + - {'table': 'dimension_irc_bridge_networks', 'column': 'isEnabled', 'default': ''} + - {'table': 'dimension_gitter_bridges', 'column': 'isEnabled', 'default': ''} + - {'table': 'dimension_custom_simple_bots', 'column': 'isEnabled', 'default': ''} + - {'table': 'dimension_custom_simple_bots', 'column': 'isPublic', 'default': ''} + - {'table': 'dimension_bridges', 'column': 'isEnabled', 'default': ''} + - {'table': 'dimension_bridges', 'column': 'isPublic', 'default': ''} + - set_fact: matrix_postgres_db_migration_request: src: "{{ matrix_dimension_sqlite_database_path_local }}" @@ -19,12 +57,8 @@ engine_old: 'sqlite' systemd_services_to_stop: ['matrix-dimension.service'] pgloader_options: ['--with "quote identifiers"'] - # pgloader makes `isSelfBot` of type `smallint`, instead of `boolean`. - # We need to fix it up - additional_psql_statements_list: - - 'ALTER TABLE dimension_users ALTER COLUMN "isSelfBot" DROP default;' - - 'ALTER TABLE dimension_users ALTER COLUMN "isSelfBot" TYPE BOOLEAN USING("isSelfBot"::text::boolean);' - - 'ALTER TABLE dimension_users ALTER COLUMN "isSelfBot" SET default false;' + additional_psql_statements_list: "{{ matrix_dimension_pgloader_additional_psql_statements_list }}" + additional_psql_statements_db_name: "{{ matrix_dimension_database_name }}" - import_tasks: "{{ role_path }}/../matrix-postgres/tasks/util/migrate_db_to_postgres.yml" From a2a4218e9573ad99a8c87a9edece1c4453cc3bff Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 23 Dec 2020 15:39:12 +0200 Subject: [PATCH 216/217] Make mautrix-python-based bridges E2EE happier Fixes a problem like this: > File "/usr/lib/python3.8/site-packages/mautrix/bridge/e2ee.py", line 79, in __init__ > raise RuntimeError("Unsupported database scheme") mautrix-python's e2ee.py module expects to find `postgres://` instead of `postgresql://`. --- roles/matrix-bridge-mautrix-facebook/defaults/main.yml | 2 +- roles/matrix-bridge-mautrix-hangouts/defaults/main.yml | 2 +- roles/matrix-bridge-mautrix-telegram/defaults/main.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml index 80e54e0e..da9b1889 100644 --- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml @@ -51,7 +51,7 @@ matrix_mautrix_facebook_database_hostname: 'matrix-postgres' matrix_mautrix_facebook_database_port: 5432 matrix_mautrix_facebook_database_name: 'matrix_mautrix_facebook' -matrix_mautrix_facebook_database_connection_string: 'postgresql://{{ matrix_mautrix_facebook_database_username }}:{{ matrix_mautrix_facebook_database_password }}@{{ matrix_mautrix_facebook_database_hostname }}:{{ matrix_mautrix_facebook_database_port }}/{{ matrix_mautrix_facebook_database_name }}' +matrix_mautrix_facebook_database_connection_string: 'postgres://{{ matrix_mautrix_facebook_database_username }}:{{ matrix_mautrix_facebook_database_password }}@{{ matrix_mautrix_facebook_database_hostname }}:{{ matrix_mautrix_facebook_database_port }}/{{ matrix_mautrix_facebook_database_name }}' matrix_mautrix_facebook_appservice_database: "{{ { diff --git a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml index e266bcbb..8dfee030 100644 --- a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml @@ -58,7 +58,7 @@ matrix_mautrix_hangouts_database_hostname: 'matrix-postgres' matrix_mautrix_hangouts_database_port: 5432 matrix_mautrix_hangouts_database_name: 'matrix_mautrix_hangouts' -matrix_mautrix_hangouts_database_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_database_username }}:{{ matrix_mautrix_hangouts_database_password }}@{{ matrix_mautrix_hangouts_database_hostname }}:{{ matrix_mautrix_hangouts_database_port }}/{{ matrix_mautrix_hangouts_database_name }}' +matrix_mautrix_hangouts_database_connection_string: 'postgres://{{ matrix_mautrix_hangouts_database_username }}:{{ matrix_mautrix_hangouts_database_password }}@{{ matrix_mautrix_hangouts_database_hostname }}:{{ matrix_mautrix_hangouts_database_port }}/{{ matrix_mautrix_hangouts_database_name }}' matrix_mautrix_hangouts_appservice_database: "{{ { diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml index 539f0a9c..3f81617a 100644 --- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml @@ -62,7 +62,7 @@ matrix_mautrix_telegram_database_hostname: 'matrix-postgres' matrix_mautrix_telegram_database_port: 5432 matrix_mautrix_telegram_database_name: 'matrix_mautrix_telegram' -matrix_mautrix_telegram_database_connection_string: 'postgresql://{{ matrix_mautrix_telegram_database_username }}:{{ matrix_mautrix_telegram_database_password }}@{{ matrix_mautrix_telegram_database_hostname }}:{{ matrix_mautrix_telegram_database_port }}/{{ matrix_mautrix_telegram_database_name }}' +matrix_mautrix_telegram_database_connection_string: 'postgres://{{ matrix_mautrix_telegram_database_username }}:{{ matrix_mautrix_telegram_database_password }}@{{ matrix_mautrix_telegram_database_hostname }}:{{ matrix_mautrix_telegram_database_port }}/{{ matrix_mautrix_telegram_database_name }}' matrix_mautrix_telegram_appservice_database: "{{ { From befffa926bd0f71ab20c702fcbb6d0aeb58d730f Mon Sep 17 00:00:00 2001 From: David Gnedt Date: Wed, 23 Dec 2020 19:24:45 +0100 Subject: [PATCH 217/217] Fix concatenation of additional databases Otherwise the postgres upgrade fails with the following error: Unexpected templating type error occurred on ({{ [matrix_postgres_connection_username] + matrix_postgres_additional_databases|map(attribute='username') }} ): can only concatenate list (not "generator") to list --- group_vars/matrix_servers | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index a54d8baa..b8fc9eb4 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1098,14 +1098,14 @@ matrix_postgres_import_roles_to_ignore: | {{ [matrix_postgres_connection_username] + - matrix_postgres_additional_databases|map(attribute='username') + matrix_postgres_additional_databases|map(attribute='username')|list }} matrix_postgres_import_databases_to_ignore: | {{ [matrix_postgres_db_name] + - matrix_postgres_additional_databases|map(attribute='name') + matrix_postgres_additional_databases|map(attribute='name')|list }} ######################################################################