From 328d981b05d6c85743b806a2d34590055f5799ad Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 7 Jun 2019 11:46:35 +0300 Subject: [PATCH 01/22] Fix undefined variables in mxisd and Dimension configuration --- roles/matrix-bridge-appservice-discord/defaults/main.yml | 2 +- roles/matrix-bridge-appservice-irc/defaults/main.yml | 2 +- roles/matrix-dimension/defaults/main.yml | 1 + roles/matrix-mxisd/defaults/main.yml | 2 ++ 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml index 23c5df25..8faca889 100644 --- a/roles/matrix-bridge-appservice-discord/defaults/main.yml +++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml @@ -128,6 +128,6 @@ matrix_appservice_discord_configuration_extension_yaml: | # # fininished handling it, causing us to echo it back to the room) # discordSendDelay: 750 -matrix_appservice_discord_configuration_extension: "{{ matrix_appservice_discord_configuration_extension_yaml|from_yaml if matrix_appservice_discord_configuration_extension_yaml|from_yaml else {} }}" +matrix_appservice_discord_configuration_extension: "{{ matrix_appservice_discord_configuration_extension_yaml|from_yaml if matrix_appservice_discord_configuration_extension_yaml|from_yaml is mapping else {} }}" matrix_appservice_discord_configuration: "{{ matrix_appservice_discord_configuration_yaml|from_yaml|combine(matrix_appservice_discord_configuration_extension, recursive=True) }}" diff --git a/roles/matrix-bridge-appservice-irc/defaults/main.yml b/roles/matrix-bridge-appservice-irc/defaults/main.yml index a4b1ef22..91531f95 100644 --- a/roles/matrix-bridge-appservice-irc/defaults/main.yml +++ b/roles/matrix-bridge-appservice-irc/defaults/main.yml @@ -433,6 +433,6 @@ matrix_appservice_irc_configuration_extension_yaml: | # # the database. # # -matrix_appservice_irc_configuration_extension: "{{ matrix_appservice_irc_configuration_extension_yaml|from_yaml if matrix_appservice_irc_configuration_extension_yaml|from_yaml else {} }}" +matrix_appservice_irc_configuration_extension: "{{ matrix_appservice_irc_configuration_extension_yaml|from_yaml if matrix_appservice_irc_configuration_extension_yaml|from_yaml is mapping else {} }}" matrix_appservice_irc_configuration: "{{ matrix_appservice_irc_configuration_yaml|from_yaml|combine(matrix_appservice_irc_configuration_extension, recursive=True) }}" diff --git a/roles/matrix-dimension/defaults/main.yml b/roles/matrix-dimension/defaults/main.yml index d2c5a011..923b1269 100644 --- a/roles/matrix-dimension/defaults/main.yml +++ b/roles/matrix-dimension/defaults/main.yml @@ -128,6 +128,7 @@ matrix_dimension_configuration_extension_yaml: | # botToken: "YourTokenHere" # +matrix_dimension_configuration_extension: "{{ matrix_dimension_configuration_extension_yaml|from_yaml if matrix_dimension_configuration_extension_yaml|from_yaml is mapping else {} }}" # Holds the final Dimension configuration (a combination of the default and its extension). # You most likely don't need to touch this variable. Instead, see `matrix_dimension_configuration_yaml`. diff --git a/roles/matrix-mxisd/defaults/main.yml b/roles/matrix-mxisd/defaults/main.yml index b7fc5137..dd4b7afb 100644 --- a/roles/matrix-mxisd/defaults/main.yml +++ b/roles/matrix-mxisd/defaults/main.yml @@ -161,6 +161,8 @@ matrix_mxisd_configuration_extension_yaml: | # bindDn: CN=My Mxisd User,OU=Users,DC=example,DC=org # bindPassword: TheUserPassword +matrix_mxisd_configuration_extension: "{{ matrix_mxisd_configuration_extension_yaml|from_yaml if matrix_mxisd_configuration_extension_yaml|from_yaml is mapping else {} }}" + # Holds the final mxisd configuration (a combination of the default and its extension). # You most likely don't need to touch this variable. Instead, see `matrix_mxisd_configuration_yaml`. matrix_mxisd_configuration: "{{ matrix_mxisd_configuration_yaml|from_yaml|combine(matrix_mxisd_configuration_extension, recursive=True) }}" From d6d6c152a382513212f38058b30594c3214af2cd Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 7 Jun 2019 12:15:37 +0300 Subject: [PATCH 02/22] Delay bridge startup to ensure Synapse is up Bridges start matrix-synapse.service as a dependency, but Synapse is sometimes slow to start, while bridges are quick to hit it and die (if unavailable). They'll auto-restart later, but .. this still breaks `--tags=start`, which doesn't wait long enough for such a restart to happen. This attempts to slow down bridge startup enough to ensure Synapse is up and no failures happen at all. --- .../templates/systemd/matrix-appservice-discord.service.j2 | 3 +++ .../templates/systemd/matrix-appservice-irc.service.j2 | 3 +++ .../templates/systemd/matrix-mautrix-facebook.service.j2 | 3 +++ .../templates/systemd/matrix-mautrix-telegram.service.j2 | 3 +++ .../templates/systemd/matrix-mautrix-whatsapp.service.j2 | 3 +++ .../templates/synapse/systemd/matrix-synapse.service.j2 | 2 +- 6 files changed, 16 insertions(+), 1 deletion(-) diff --git a/roles/matrix-bridge-appservice-discord/templates/systemd/matrix-appservice-discord.service.j2 b/roles/matrix-bridge-appservice-discord/templates/systemd/matrix-appservice-discord.service.j2 index 4ae408c5..f82c1284 100644 --- a/roles/matrix-bridge-appservice-discord/templates/systemd/matrix-appservice-discord.service.j2 +++ b/roles/matrix-bridge-appservice-discord/templates/systemd/matrix-appservice-discord.service.j2 @@ -14,6 +14,9 @@ Type=simple ExecStartPre=-/usr/bin/docker kill matrix-appservice-discord ExecStartPre=-/usr/bin/docker rm matrix-appservice-discord +# Intentional delay, so that the homeserver (we likely depend on) can manage to start. +ExecStartPre=/bin/sleep 5 + ExecStart=/usr/bin/docker run --rm --name matrix-appservice-discord \ --log-driver=none \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ diff --git a/roles/matrix-bridge-appservice-irc/templates/systemd/matrix-appservice-irc.service.j2 b/roles/matrix-bridge-appservice-irc/templates/systemd/matrix-appservice-irc.service.j2 index b50f058e..14a54794 100644 --- a/roles/matrix-bridge-appservice-irc/templates/systemd/matrix-appservice-irc.service.j2 +++ b/roles/matrix-bridge-appservice-irc/templates/systemd/matrix-appservice-irc.service.j2 @@ -14,6 +14,9 @@ Type=simple ExecStartPre=-/usr/bin/docker kill matrix-appservice-irc ExecStartPre=-/usr/bin/docker rm matrix-appservice-irc +# Intentional delay, so that the homeserver (we likely depend on) can manage to start. +ExecStartPre=/bin/sleep 5 + ExecStart=/usr/bin/docker run --rm --name matrix-appservice-irc \ --log-driver=none \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ diff --git a/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 b/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 index aa06ccc4..424fb76a 100644 --- a/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 +++ b/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 @@ -21,6 +21,9 @@ ExecStartPre=/usr/bin/docker run --rm --name matrix-mautrix-facebook-db \ {{ matrix_mautrix_facebook_docker_image }} \ alembic -x config=/data/config.yaml upgrade head +# Intentional delay, so that the homeserver (we likely depend on) can manage to start. +ExecStartPre=/bin/sleep 5 + ExecStart=/usr/bin/docker run --rm --name matrix-mautrix-facebook \ --log-driver=none \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ diff --git a/roles/matrix-bridge-mautrix-telegram/templates/systemd/matrix-mautrix-telegram.service.j2 b/roles/matrix-bridge-mautrix-telegram/templates/systemd/matrix-mautrix-telegram.service.j2 index c47ad866..878e2585 100644 --- a/roles/matrix-bridge-mautrix-telegram/templates/systemd/matrix-mautrix-telegram.service.j2 +++ b/roles/matrix-bridge-mautrix-telegram/templates/systemd/matrix-mautrix-telegram.service.j2 @@ -21,6 +21,9 @@ ExecStartPre=/usr/bin/docker run --rm --name matrix-mautrix-telegram-db \ {{ matrix_mautrix_telegram_docker_image }} \ alembic -x config=/data/config.yaml upgrade head +# Intentional delay, so that the homeserver (we likely depend on) can manage to start. +ExecStartPre=/bin/sleep 5 + ExecStart=/usr/bin/docker run --rm --name matrix-mautrix-telegram \ --log-driver=none \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ diff --git a/roles/matrix-bridge-mautrix-whatsapp/templates/systemd/matrix-mautrix-whatsapp.service.j2 b/roles/matrix-bridge-mautrix-whatsapp/templates/systemd/matrix-mautrix-whatsapp.service.j2 index 83b59680..f0913864 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/templates/systemd/matrix-mautrix-whatsapp.service.j2 +++ b/roles/matrix-bridge-mautrix-whatsapp/templates/systemd/matrix-mautrix-whatsapp.service.j2 @@ -14,6 +14,9 @@ Type=simple ExecStartPre=-/usr/bin/docker kill matrix-mautrix-whatsapp ExecStartPre=-/usr/bin/docker rm matrix-mautrix-whatsapp +# Intentional delay, so that the homeserver (we likely depend on) can manage to start. +ExecStartPre=/bin/sleep 5 + ExecStart=/usr/bin/docker run --rm --name matrix-mautrix-whatsapp \ --log-driver=none \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 index 071a0eda..ef691cc4 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 @@ -17,7 +17,7 @@ ExecStartPre=-/usr/bin/docker rm matrix-synapse # Allow for some time before starting, so that media store can mount. # Mounting can happen later too, but if we start writing, # we'd write files to the local filesystem and fusermount will complain. -ExecStartPre=/bin/sleep 5 +ExecStartPre=/bin/sleep 3 {% endif %} ExecStart=/usr/bin/docker run --rm --name matrix-synapse \ From 04bc50a282cb181e389aa0d7c39bbd26e27dc2b3 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 7 Jun 2019 13:40:12 +0300 Subject: [PATCH 03/22] Make Facebook bridge docs more detailed --- ...guring-playbook-bridge-mautrix-facebook.md | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/docs/configuring-playbook-bridge-mautrix-facebook.md b/docs/configuring-playbook-bridge-mautrix-facebook.md index 20274cfa..6df9d991 100644 --- a/docs/configuring-playbook-bridge-mautrix-facebook.md +++ b/docs/configuring-playbook-bridge-mautrix-facebook.md @@ -8,4 +8,59 @@ See the project's [documentation](https://github.com/tulir/mautrix-facebook/wiki matrix_mautrix_facebook_enabled: true ``` +## Usage + You then need to start a chat with `@facebookbot:YOUR_DOMAIN` (where `YOUR_DOMAIN` is your base domain, not the `matrix.` domain). + + +### Set up bridging + +Send `login YOUR_FACEBOOK_EMAIL_ADDRESS YOUR_FACEBOOK_PASSWORD` to the bridge bot to enable bridging for your Facebook/Messenger account. + +You can learn more here about authentication from the bridge's [official documentation on Authentication](https://github.com/tulir/mautrix-facebook/wiki/Authentication). + +If you run into trouble, check the [Troubleshooting](#troubleshooting) section below. + +After successfully enabling bridging, you may wish to [set up Double Puppeting](#set-up-double-puppeting). + + +### Set up Double Puppeting + +**Note**: Double Puppeting can be configured only after you've already [set up bridging](#set-up-bridging). + +If you'd like to use [Double Puppeting](https://github.com/tulir/mautrix-facebook/wiki/Authentication#double-puppeting) (hint: you most likely do), you should: + +- retrieve a Matrix access token for yourself. You can use the following command: + +``` +curl \ +--data '{"identifier": {"type": "m.id.user", "user": "YOUR_MATRIX_USERNAME" }, "password": "YOUR_MATRIX_PASSWORD", "type": "m.login.password", "device_id": "Mautrix-Facebook", "initial_device_display_name": "Mautrix-Facebook"}' \ +https://matrix.DOMAIN/_matrix/client/r0/login +``` + +- send the access token to the bot. Example: `login-matrix MATRIX_ACCESS_TOKEN_HERE` + +- make sure you don't log out the `Mautrix-Facebook` device some time in the future, as that would break the Double Puppeting feature + + +## Troubleshooting + +### Facebook rejecting login attempts and forcing you to change password + +If your Matrix server is in a wildly different location than where you usually use your Facebook account from, the bridge's login attempts may be outright rejected by Facebook. Along with that, Facebook may even force you to change the account's password. + +If you happen to run into this problem while [setting up bridging](#set-up-bridging), try to first get a successful session up by logging in to Facebook through the Matrix server's IP address. + +The easiest way to do this may be to use [sshutle](https://sshuttle.readthedocs.io/) to proxy your traffic through the Matrix server. + +Example command for proxying your traffic through the Matrix server: + +``` +sshuttle -r root@matrix.DOMAIN:22 0/0 +``` + +Once connected, you should be able to verify that you're browsing the web through the Matrix server's IP by checking [icanhazip](https://icanhazip.com/). + +Then proceed to log in to [Facebook/Messenger](https://www.facebook.com/). + +Once logged in, proceed to [set up bridging](#set-up-bridging). From 330648a3e065ba892931603f0fd29c6b93c632ce Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 7 Jun 2019 14:04:59 +0300 Subject: [PATCH 04/22] Make Facebook bridge configuration playbook-managed Related to #193, but for the Facebook bridge. (other bridges can be changed to do the same later). This patch makes the bridge configuration entirely managed by the Ansible playbook. The bridge's `config.yaml` and `registration.yaml` configuration files are regenerated every time the playbook runs. This allows us to apply updates to those files and to avoid people having to manage the configuration files manually on the server. ------------------------------------------------------------- A deficiency of the current approach to dumping YAML configuration in `config.yaml` is that we strip all comments from it. Later on, when the bridge actually starts, it will load and redump (this time with comments), which will make the `config.yaml` file change. Subsequent playbook runs will report "changed" for the "Ensure mautrix-facebook config.yaml installed" task, which is a little strange. We might wish to improve this in the future, if possible. Still, it's better to have a (usually) somewhat meaningless "changed" task than to what we had -- never rebuilding the configuration. --- CHANGELOG.md | 13 ++ group_vars/matrix_servers | 4 + .../defaults/main.yml | 163 +++++++++++++++++- .../tasks/main.yml | 6 + .../tasks/setup_install.yml | 38 ++-- .../tasks/validate_config.yml | 10 ++ .../templates/config.yaml.j2 | 109 ------------ 7 files changed, 206 insertions(+), 137 deletions(-) create mode 100644 roles/matrix-bridge-mautrix-facebook/tasks/validate_config.yml delete mode 100644 roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 999280b5..53f3a782 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +# 2019-06-07 + +## (BC Break) Facebook bridge configuration is now entirely managed by the playbook + +Until now, configuration files for the [Facebook bridge](docs/configuring-playbook-bridge-mautrix-facebook.md) were created by the playbook initially, but never modified later on. + +From now on, the playbook will keep those configuration in sync for you. + +This means that if you were making manual changes to the `/matrix/mautrix-facebook/config.yaml` or `/matrix/mautrix-facebook/registration.yaml` configuration files, those would be lost the next time you run the playbook. + +You need to migrate your manual changes over to the new `matrix_mautrix_facebook_configuration_extension_yaml` variable, so that the playbook would apply them for you. + + # 2019-05-25 ## Support for exposing container ports publicly (not just to the host) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index dcc06d8d..cf8d4830 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -90,6 +90,10 @@ matrix_appservice_irc_systemd_required_services_list: | # We don't enable bridges by default. matrix_mautrix_facebook_enabled: false +matrix_mautrix_facebook_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'facebook-appservice-token') | to_uuid }}" + +matrix_mautrix_facebook_homeserver_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'facebook-homeserver-token') | to_uuid }}" + matrix_mautrix_facebook_systemd_required_services_list: | {{ ['docker.service'] diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml index 4ff467dd..05d37fcd 100644 --- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml @@ -7,7 +7,7 @@ matrix_mautrix_facebook_docker_image: "tulir/mautrix-facebook:latest" matrix_mautrix_facebook_base_path: "{{ matrix_base_data_path }}/mautrix-facebook" -matrix_mautrix_facebook_homeserver_address: 'https://{{ matrix_server_fqn_matrix }}' +matrix_mautrix_facebook_homeserver_address: 'http://matrix-synapse:8008' matrix_mautrix_facebook_homeserver_domain: '{{ matrix_domain }}' matrix_mautrix_facebook_appservice_address: 'http://matrix-mautrix-facebook:8080' @@ -19,3 +19,164 @@ matrix_mautrix_facebook_systemd_required_services_list: ['docker.service'] # List of systemd services that matrix-mautrix-facebook.service wants matrix_mautrix_facebook_systemd_wanted_services_list: [] + +matrix_mautrix_facebook_appservice_token: '' +matrix_mautrix_facebook_homeserver_token: '' + +# Default mxisd configuration template which covers the generic use case. +# You can customize it by controlling the various variables inside it. +# +# For a more advanced customization, you can extend the default (see `matrix_mautrix_facebook_configuration_extension_yaml`) +# or completely replace this variable with your own template. +matrix_mautrix_facebook_configuration_yaml: | + #jinja2: lstrip_blocks: "True" + # Homeserver details + homeserver: + # The address that this appservice can use to connect to the homeserver. + address: {{ matrix_mautrix_facebook_homeserver_address }} + # The domain of the homeserver (for MXIDs, etc). + domain: {{ matrix_mautrix_facebook_homeserver_domain }} + # Whether or not to verify the SSL certificate of the homeserver. + # Only applies if address starts with https:// + verify_ssl: true + + # Application service host/registration related details + # Changing these values requires regeneration of the registration. + appservice: + # The address that the homeserver can use to connect to this appservice. + address: {{ matrix_mautrix_facebook_appservice_address }} + + # The hostname and port where this appservice should listen. + hostname: 0.0.0.0 + port: 8080 + # The maximum body size of appservice API requests (from the homeserver) in mebibytes + # Usually 1 is enough, but on high-traffic bridges you might need to increase this to avoid 413s + max_body_size: 1 + + # The full URI to the database. SQLite and Postgres are fully supported. + # Other DBMSes supported by SQLAlchemy may or may not work. + # Format examples: + # SQLite: sqlite:///filename.db + # Postgres: postgres://username:password@hostname/dbname + database: sqlite:////data/mautrix-facebook.db + + # The unique ID of this appservice. + id: facebook + # Username of the appservice bot. + bot_username: facebookbot + # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty + # to leave display name/avatar as-is. + bot_displayname: Facebook bridge bot + bot_avatar: mxc://maunium.net/ddtNPZSKMNqaUzqrHuWvUADv + + # Authentication tokens for AS <-> HS communication. + as_token: "{{ matrix_mautrix_facebook_appservice_token }}" + hs_token: "{{ matrix_mautrix_facebook_homeserver_token }}" + + # Bridge config + bridge: + # Localpart template of MXIDs for Facebook users. + # {userid} is replaced with the user ID of the Facebook user. + username_template: "facebook_{userid}" + # Displayname template for Facebook users. + # {displayname} is replaced with the display name of the Facebook user + # as defined below in displayname_preference. + # Keys available for displayname_preference are also available here. + displayname_template: '{displayname} (FB)' + # Available keys: + # "name" (full name) + # "first_name" + # "last_name" + # "nickname" + # "own_nickname" (user-specific!) + displayname_preference: + - name + + # The prefix for commands. Only required in non-management rooms. + command_prefix: "!fb" + + # Number of chats to sync (and create portals for) on startup/login. + # Maximum 20, set 0 to disable automatic syncing. + initial_chat_sync: 10 + # Whether or not the Facebook users of logged in Matrix users should be + # invited to private chats when the user sends a message from another client. + invite_own_puppet_to_pm: false + # Whether or not to use /sync to get presence, read receipts and typing notifications when using + # your own Matrix account as the Matrix puppet for your Facebook account. + sync_with_custom_puppets: true + # Whether or not to bridge presence in both directions. Facebook allows users not to broadcast + # presence, but then it won't send other users' presence to the client. + presence: true + + # Permissions for using the bridge. + # Permitted values: + # user - Use the bridge with puppeting. + # admin - Use and administrate the bridge. + # Permitted keys: + # * - All Matrix users + # domain - All users on that homeserver + # mxid - Specific user + permissions: + '{{ matrix_mautrix_facebook_homeserver_domain }}': user + + # Python logging configuration. + # + # See section 16.7.2 of the Python documentation for more info: + # https://docs.python.org/3.6/library/logging.config.html#configuration-dictionary-schema + logging: + version: 1 + formatters: + colored: + (): mautrix_facebook.util.ColorFormatter + format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s" + normal: + format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s" + handlers: + file: + class: logging.handlers.RotatingFileHandler + formatter: normal + filename: /data/mautrix-facebook.log + maxBytes: 10485760 + backupCount: 10 + console: + class: logging.StreamHandler + formatter: colored + loggers: + mau: + level: DEBUG + fbchat: + level: DEBUG + aiohttp: + level: INFO + root: + level: DEBUG + handlers: [file, console] + +matrix_mautrix_facebook_configuration_extension_yaml: | + # Your custom YAML configuration goes here. + # This configuration extends the default starting configuration (`matrix_mautrix_facebook_configuration_yaml`). + # + # You can override individual variables from the default configuration, or introduce new ones. + # + # If you need something more special, you can take full control by + # completely redefining `matrix_mautrix_facebook_configuration_yaml`. + +matrix_mautrix_facebook_configuration_extension: "{{ matrix_mautrix_facebook_configuration_extension_yaml|from_yaml if matrix_mautrix_facebook_configuration_extension_yaml|from_yaml is mapping else {} }}" + +# Holds the final configuration (a combination of the default and its extension). +# You most likely don't need to touch this variable. Instead, see `matrix_mautrix_facebook_configuration_yaml`. +matrix_mautrix_facebook_configuration: "{{ matrix_mautrix_facebook_configuration_yaml|from_yaml|combine(matrix_mautrix_facebook_configuration_extension, recursive=True) }}" + +matrix_mautrix_facebook_registration_yaml: | + id: facebook + as_token: "{{ matrix_mautrix_facebook_appservice_token }}" + hs_token: "{{ matrix_mautrix_facebook_homeserver_token }}" + namespaces: + users: + - exclusive: true + regex: '@facebook_.+:{{ matrix_mautrix_facebook_homeserver_domain }}' + url: {{ matrix_mautrix_facebook_appservice_address }} + sender_localpart: facebookbot + rate_limited: false + +matrix_mautrix_facebook_registration: "{{ matrix_mautrix_facebook_registration_yaml|from_yaml }}" diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/main.yml b/roles/matrix-bridge-mautrix-facebook/tasks/main.yml index d3e3ce32..54fb6f9d 100644 --- a/roles/matrix-bridge-mautrix-facebook/tasks/main.yml +++ b/roles/matrix-bridge-mautrix-facebook/tasks/main.yml @@ -2,6 +2,12 @@ tags: - always +- import_tasks: "{{ role_path }}/tasks/validate_config.yml" + when: "run_setup|bool and matrix_mautrix_facebook_enabled|bool" + tags: + - setup-all + - setup-mautrix-facebook + - import_tasks: "{{ role_path }}/tasks/setup_install.yml" when: "run_setup|bool and matrix_mautrix_facebook_enabled|bool" tags: diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml index ae313e36..e9597c29 100644 --- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml @@ -21,19 +21,21 @@ owner: "{{ matrix_user_username }}" group: "{{ matrix_user_username }}" -- name: Check if a mautrix-facebook configuration file exists - stat: - path: "{{ matrix_mautrix_facebook_base_path }}/config.yaml" - register: mautrix_facebook_config_file_stat - -- name: Ensure Matrix Mautrix facebook config installed - template: - src: "{{ role_path }}/templates/config.yaml.j2" +- name: Ensure mautrix-facebook config.yaml installed + copy: + content: "{{ matrix_mautrix_facebook_configuration|to_nice_yaml }}" dest: "{{ matrix_mautrix_facebook_base_path }}/config.yaml" mode: 0644 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_username }}" - when: "not mautrix_facebook_config_file_stat.stat.exists" + +- name: Ensure mautrix-facebook registration.yaml installed + copy: + content: "{{ matrix_mautrix_facebook_registration|to_nice_yaml }}" + dest: "{{ matrix_mautrix_facebook_base_path }}/registration.yaml" + mode: 0644 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_username }}" - name: Ensure matrix-mautrix-facebook.service installed template: @@ -47,24 +49,6 @@ daemon_reload: yes when: "matrix_mautrix_facebook_systemd_service_result.changed" -- name: Check if a mautrix-facebook registration file exists - stat: - path: "{{ matrix_mautrix_facebook_base_path }}/registration.yaml" - register: mautrix_facebook_registration_file_stat - -- name: Generate matrix-mautrix-facebook registration.yaml if it doesn't exist - shell: - cmd: >- - /usr/bin/docker run - --rm - --user={{ matrix_user_uid }}:{{ matrix_user_gid }} - --cap-drop=ALL - --name matrix-mautrix-facebook-gen - -v {{ matrix_mautrix_facebook_base_path }}:/data:z - {{ matrix_mautrix_facebook_docker_image }} - python3 -m mautrix_facebook -g -c /data/config.yaml -r /data/registration.yaml - when: "not mautrix_facebook_registration_file_stat.stat.exists" - # If the matrix-synapse role is not used, these variables may not exist. - set_fact: matrix_synapse_container_extra_arguments: > diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/validate_config.yml b/roles/matrix-bridge-mautrix-facebook/tasks/validate_config.yml new file mode 100644 index 00000000..dfbe072b --- /dev/null +++ b/roles/matrix-bridge-mautrix-facebook/tasks/validate_config.yml @@ -0,0 +1,10 @@ +--- + +- name: Fail if required settings not defined + fail: + msg: >- + You need to define a required configuration setting (`{{ item }}`). + when: "vars[item] == ''" + with_items: + - "matrix_mautrix_facebook_appservice_token" + - "matrix_mautrix_facebook_homeserver_token" diff --git a/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 deleted file mode 100644 index 86033c3c..00000000 --- a/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 +++ /dev/null @@ -1,109 +0,0 @@ -#jinja2: lstrip_blocks: "True" -# Homeserver details -homeserver: - # The address that this appservice can use to connect to the homeserver. - address: {{ matrix_mautrix_facebook_homeserver_address }} - # The domain of the homeserver (for MXIDs, etc). - domain: {{ matrix_mautrix_facebook_homeserver_domain }} - # Whether or not to verify the SSL certificate of the homeserver. - # Only applies if address starts with https:// - verify_ssl: true - -# Application service host/registration related details -# Changing these values requires regeneration of the registration. -appservice: - # The address that the homeserver can use to connect to this appservice. - address: {{ matrix_mautrix_facebook_appservice_address }} - - # The hostname and port where this appservice should listen. - hostname: 0.0.0.0 - port: 8080 - # The maximum body size of appservice API requests (from the homeserver) in mebibytes - # Usually 1 is enough, but on high-traffic bridges you might need to increase this to avoid 413s - max_body_size: 1 - - # The full URI to the database. SQLite and Postgres are fully supported. - # Other DBMSes supported by SQLAlchemy may or may not work. - # Format examples: - # SQLite: sqlite:///filename.db - # Postgres: postgres://username:password@hostname/dbname - database: sqlite:////data/mautrix-facebook.db - - # The unique ID of this appservice. - id: facebook - # Username of the appservice bot. - bot_username: facebookbot - # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty - # to leave display name/avatar as-is. - bot_displayname: Facebook bridge bot - bot_avatar: mxc://maunium.net/ddtNPZSKMNqaUzqrHuWvUADv - - # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify. - as_token: "This value is generated when generating the registration" - hs_token: "This value is generated when generating the registration" - -# Bridge config -bridge: - # Localpart template of MXIDs for Facebook users. - # {userid} is replaced with the user ID of the Facebook user. - username_template: "facebook_{userid}" - - # The prefix for commands. Only required in non-management rooms. - command_prefix: "!fb" - - # Number of chats to sync (and create portals for) on startup/login. - # Maximum 20, set 0 to disable automatic syncing. - initial_chat_sync: 10 - # Whether or not the Facebook users of logged in Matrix users should be - # invited to private chats when the user sends a message from another client. - invite_own_puppet_to_pm: false - # Whether or not to use /sync to get presence, read receipts and typing notifications when using - # your own Matrix account as the Matrix puppet for your Facebook account. - sync_with_custom_puppets: true - # Whether or not to bridge presence in both directions. Facebook allows users not to broadcast - # presence, but then it won't send other users' presence to the client. - presence: true - - # Permissions for using the bridge. - # Permitted values: - # user - Use the bridge with puppeting. - # admin - Use and administrate the bridge. - # Permitted keys: - # * - All Matrix users - # domain - All users on that homeserver - # mxid - Specific user - permissions: - '{{ matrix_mautrix_facebook_homeserver_domain }}': user - -# Python logging configuration. -# -# See section 16.7.2 of the Python documentation for more info: -# https://docs.python.org/3.6/library/logging.config.html#configuration-dictionary-schema -logging: - version: 1 - formatters: - colored: - (): mautrix_facebook.util.ColorFormatter - format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s" - normal: - format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s" - handlers: - file: - class: logging.handlers.RotatingFileHandler - formatter: normal - filename: /data/mautrix-facebook.log - maxBytes: 10485760 - backupCount: 10 - console: - class: logging.StreamHandler - formatter: colored - loggers: - mau: - level: DEBUG - fbchat: - level: DEBUG - aiohttp: - level: INFO - root: - level: DEBUG - handlers: [file, console] From 4f0bcc624f8cb9ca67a691705871f3899ea0569a Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 7 Jun 2019 14:29:51 +0300 Subject: [PATCH 05/22] Fix typo --- docs/configuring-playbook-bridge-mautrix-facebook.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuring-playbook-bridge-mautrix-facebook.md b/docs/configuring-playbook-bridge-mautrix-facebook.md index 6df9d991..b48281f4 100644 --- a/docs/configuring-playbook-bridge-mautrix-facebook.md +++ b/docs/configuring-playbook-bridge-mautrix-facebook.md @@ -51,7 +51,7 @@ If your Matrix server is in a wildly different location than where you usually u If you happen to run into this problem while [setting up bridging](#set-up-bridging), try to first get a successful session up by logging in to Facebook through the Matrix server's IP address. -The easiest way to do this may be to use [sshutle](https://sshuttle.readthedocs.io/) to proxy your traffic through the Matrix server. +The easiest way to do this may be to use [sshuttle](https://sshuttle.readthedocs.io/) to proxy your traffic through the Matrix server. Example command for proxying your traffic through the Matrix server: From 172b0fa88c341ba4d102535e40d1959128820671 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 7 Jun 2019 14:50:55 +0300 Subject: [PATCH 06/22] Separate Facebook bridge configuration and data Using a separate directory allows easier backups (only need to back up the Ansible playbook configuration and the bridge's `./data` directory). The playbook takes care of migrating an existing database file from the base directory into the `./data` directory. In the future, we can also mount the configuration read-only, to ensure the bridge won't touch it. For now, mautrix-facebook is keen on rebuilding the `config.yaml` file on startup though, so this will have to wait. --- .../defaults/main.yml | 2 + .../tasks/setup_install.yml | 37 ++++++++++++++++--- .../matrix-mautrix-facebook.service.j2 | 10 +++-- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml index 05d37fcd..a323b03c 100644 --- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml @@ -6,6 +6,8 @@ matrix_mautrix_facebook_enabled: true matrix_mautrix_facebook_docker_image: "tulir/mautrix-facebook:latest" matrix_mautrix_facebook_base_path: "{{ matrix_base_data_path }}/mautrix-facebook" +matrix_mautrix_facebook_config_path: "{{ matrix_mautrix_facebook_base_path }}/config" +matrix_mautrix_facebook_data_path: "{{ matrix_mautrix_facebook_base_path }}/data" matrix_mautrix_facebook_homeserver_address: 'http://matrix-synapse:8008' matrix_mautrix_facebook_homeserver_domain: '{{ matrix_domain }}' diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml index e9597c29..a442498e 100644 --- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml @@ -13,18 +13,45 @@ name: "{{ matrix_mautrix_facebook_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" -- name: Ensure Mautrix Facebook base directory exists +- name: Ensure Mautrix Facebook paths exist file: - path: "{{ matrix_mautrix_facebook_base_path }}" + path: "{{ item }}" state: directory mode: 0750 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_username }}" + recurse: true + with_items: + - "{{ matrix_mautrix_facebook_base_path }}" + - "{{ matrix_mautrix_facebook_config_path }}" + - "{{ matrix_mautrix_facebook_data_path }}" + +- name: Check if an old database file already exists + stat: + path: "{{ matrix_mautrix_facebook_base_path }}/mautrix-facebook.db" + register: matrix_mautrix_facebook_stat_database + +- name: (Data relocation) Ensure matrix-mautrix-facebook.service is stopped + service: + name: matrix-mautrix-facebook + state: stopped + daemon_reload: yes + register: stopping_result + failed_when: false + when: "matrix_mautrix_facebook_stat_database.stat.exists" + +- name: (Data relocation) Move mautrix-facebook database file to ./data directory + command: "mv {{ matrix_mautrix_facebook_base_path }}/mautrix-facebook.db {{ matrix_mautrix_facebook_data_path }}/mautrix-facebook.db" + when: "matrix_mautrix_facebook_stat_database.stat.exists" + +- name: (Data relocation) Move mautrix-facebook database file to ./data directory + command: "mv {{ matrix_mautrix_facebook_base_path }}/mautrix-facebook.db {{ matrix_mautrix_facebook_data_path }}/mautrix-facebook.db" + when: "matrix_mautrix_facebook_stat_database.stat.exists" - name: Ensure mautrix-facebook config.yaml installed copy: content: "{{ matrix_mautrix_facebook_configuration|to_nice_yaml }}" - dest: "{{ matrix_mautrix_facebook_base_path }}/config.yaml" + dest: "{{ matrix_mautrix_facebook_config_path }}/config.yaml" mode: 0644 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_username }}" @@ -32,7 +59,7 @@ - name: Ensure mautrix-facebook registration.yaml installed copy: content: "{{ matrix_mautrix_facebook_registration|to_nice_yaml }}" - dest: "{{ matrix_mautrix_facebook_base_path }}/registration.yaml" + dest: "{{ matrix_mautrix_facebook_config_path }}/registration.yaml" mode: 0644 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_username }}" @@ -54,7 +81,7 @@ matrix_synapse_container_extra_arguments: > {{ matrix_synapse_container_extra_arguments|default([]) }} + - {{ ["--mount type=bind,src={{ matrix_mautrix_facebook_base_path }}/registration.yaml,dst=/matrix-mautrix-facebook-registration.yaml,ro"] }} + {{ ["--mount type=bind,src={{ matrix_mautrix_facebook_config_path }}/registration.yaml,dst=/matrix-mautrix-facebook-registration.yaml,ro"] }} matrix_synapse_app_service_config_files: > {{ matrix_synapse_app_service_config_files|default([]) }} diff --git a/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 b/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 index 424fb76a..7bc4f189 100644 --- a/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 +++ b/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 @@ -17,9 +17,10 @@ ExecStartPre=/usr/bin/docker run --rm --name matrix-mautrix-facebook-db \ --log-driver=none \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ --cap-drop=ALL \ - -v {{ matrix_mautrix_facebook_base_path }}:/data:z \ + -v {{ matrix_mautrix_facebook_data_path }}:/data:z \ + -v {{ matrix_mautrix_facebook_config_path }}:/config:z \ {{ matrix_mautrix_facebook_docker_image }} \ - alembic -x config=/data/config.yaml upgrade head + alembic -x config=/config/config.yaml upgrade head # Intentional delay, so that the homeserver (we likely depend on) can manage to start. ExecStartPre=/bin/sleep 5 @@ -29,12 +30,13 @@ ExecStart=/usr/bin/docker run --rm --name matrix-mautrix-facebook \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ --cap-drop=ALL \ --network={{ matrix_docker_network }} \ - -v {{ matrix_mautrix_facebook_base_path }}:/data:z \ + -v {{ matrix_mautrix_facebook_data_path }}:/data:z \ + -v {{ matrix_mautrix_facebook_config_path }}:/config:z \ {% for arg in matrix_mautrix_facebook_container_extra_arguments %} {{ arg }} \ {% endfor %} {{ matrix_mautrix_facebook_docker_image }} \ - python3 -m mautrix_facebook -c /data/config.yaml + python3 -m mautrix_facebook -c /config/config.yaml ExecStop=-/usr/bin/docker kill matrix-mautrix-facebook ExecStop=-/usr/bin/docker rm matrix-mautrix-facebook From bf446b6e15a87fd6af0340fbe277826aba1c3a79 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 7 Jun 2019 15:05:53 +0300 Subject: [PATCH 07/22] Fix double mv command --- roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml index a442498e..a7e23a6d 100644 --- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml @@ -36,7 +36,6 @@ name: matrix-mautrix-facebook state: stopped daemon_reload: yes - register: stopping_result failed_when: false when: "matrix_mautrix_facebook_stat_database.stat.exists" @@ -44,10 +43,6 @@ command: "mv {{ matrix_mautrix_facebook_base_path }}/mautrix-facebook.db {{ matrix_mautrix_facebook_data_path }}/mautrix-facebook.db" when: "matrix_mautrix_facebook_stat_database.stat.exists" -- name: (Data relocation) Move mautrix-facebook database file to ./data directory - command: "mv {{ matrix_mautrix_facebook_base_path }}/mautrix-facebook.db {{ matrix_mautrix_facebook_data_path }}/mautrix-facebook.db" - when: "matrix_mautrix_facebook_stat_database.stat.exists" - - name: Ensure mautrix-facebook config.yaml installed copy: content: "{{ matrix_mautrix_facebook_configuration|to_nice_yaml }}" From 67c13d0a77f4552c1febbf22da418bdefb3d634d Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 7 Jun 2019 15:11:25 +0300 Subject: [PATCH 08/22] Update changelog --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53f3a782..16b2d25f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,10 @@ From now on, the playbook will keep those configuration in sync for you. This means that if you were making manual changes to the `/matrix/mautrix-facebook/config.yaml` or `/matrix/mautrix-facebook/registration.yaml` configuration files, those would be lost the next time you run the playbook. -You need to migrate your manual changes over to the new `matrix_mautrix_facebook_configuration_extension_yaml` variable, so that the playbook would apply them for you. +The bridge now stores configuration in a subdirectory (`/matrix/mautrix-facebook/config`), so your old configuration remains in the base directory (`/matrix/mautrix-facebook`). +You need to migrate any manual changes over to the new `matrix_mautrix_facebook_configuration_extension_yaml` variable, so that the playbook would apply them for you. + +Likewise, data is now also stored in a subdirectory (`/matrix/mautrix-facebook/data`). When you run the playbook with an existing database file (`/matrix/mautrix-facebook/mautrix-facebook.db`), the playbook will stop the bridge and relocate the database file to the `./data` directory. There's no data-loss involved. You'll need to restart the bridge manually though (`--tags=start`). # 2019-05-25 From 18baeabdf25d8e752817aef1be55e987378eec40 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 7 Jun 2019 15:18:29 +0300 Subject: [PATCH 09/22] Do not create Facebook bridge directories with recurse: true I'm not sure what I had in mind when I added this earlier, but I think we'd better go without it. --- roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml index a7e23a6d..d797688e 100644 --- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml @@ -20,7 +20,6 @@ mode: 0750 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_username }}" - recurse: true with_items: - "{{ matrix_mautrix_facebook_base_path }}" - "{{ matrix_mautrix_facebook_config_path }}" From a9953dd641319905f1e83081aec1c84a781b7457 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 7 Jun 2019 15:48:13 +0300 Subject: [PATCH 10/22] Make Facebook/Telegram bridges not log to files We log to journald anyway. There's no need for double-logging. It should not that matrix-synapse logs to journald and to files, but that's likely to change in the future as well. Because Synapse's logs are insanely verbose right now (and may get dropped by journald), it's more reliable to have file-logging too. As Synapse matures and gets more stable, logging should hopefully get less, we should be able to only use journald and stop writing to files for it as well. --- roles/matrix-bridge-mautrix-facebook/defaults/main.yml | 8 +------- .../templates/config.yaml.j2 | 8 +------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml index a323b03c..b9039dc6 100644 --- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml @@ -134,12 +134,6 @@ matrix_mautrix_facebook_configuration_yaml: | normal: format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s" handlers: - file: - class: logging.handlers.RotatingFileHandler - formatter: normal - filename: /data/mautrix-facebook.log - maxBytes: 10485760 - backupCount: 10 console: class: logging.StreamHandler formatter: colored @@ -152,7 +146,7 @@ matrix_mautrix_facebook_configuration_yaml: | level: INFO root: level: DEBUG - handlers: [file, console] + handlers: [console] matrix_mautrix_facebook_configuration_extension_yaml: | # Your custom YAML configuration goes here. diff --git a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 index b2854646..0a8869a6 100644 --- a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 @@ -251,12 +251,6 @@ logging: precise: format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s" handlers: - file: - class: logging.handlers.RotatingFileHandler - formatter: precise - filename: /data/mautrix-telegram.log - maxBytes: 10485760 - backupCount: 10 console: class: logging.StreamHandler formatter: precise @@ -269,4 +263,4 @@ logging: level: INFO root: level: DEBUG - handlers: [file, console] + handlers: [console] From 3567d9adbab7d6f08046591c34fe551093581504 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 7 Jun 2019 16:07:01 +0300 Subject: [PATCH 11/22] Fix typo --- roles/matrix-postgres/tasks/validate_config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/matrix-postgres/tasks/validate_config.yml b/roles/matrix-postgres/tasks/validate_config.yml index 8c3e0fbe..4985a4c7 100644 --- a/roles/matrix-postgres/tasks/validate_config.yml +++ b/roles/matrix-postgres/tasks/validate_config.yml @@ -13,10 +13,10 @@ - name: Fail if required Postgres settings not defined fail: msg: > - You need to define a required configuration setting (`{{ item }}`) for using mxisd. + You need to define a required configuration setting (`{{ item }}`). when: "vars[item] == ''" with_items: - "matrix_postgres_connection_hostname" - "matrix_postgres_connection_username" - "matrix_postgres_connection_password" - - "matrix_postgres_db_name" \ No newline at end of file + - "matrix_postgres_db_name" From 44156fe659b31a7a650884855950a9f5f58e68ef Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 7 Jun 2019 17:44:32 +0300 Subject: [PATCH 12/22] Fix Ansible 2.8 deprecation in Dimension role --- roles/matrix-dimension/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/matrix-dimension/defaults/main.yml b/roles/matrix-dimension/defaults/main.yml index 923b1269..7f52ade6 100644 --- a/roles/matrix-dimension/defaults/main.yml +++ b/roles/matrix-dimension/defaults/main.yml @@ -15,8 +15,8 @@ matrix_dimension_base_path: "{{ matrix_base_data_path }}/dimension" matrix_dimension_docker_image: "turt2live/matrix-dimension:latest" # The user and group id correspond to the node user in the `turt2live/matrix-dimension` image. -matrix_dimension_user_uid: 1000 -matrix_dimension_user_gid: 1000 +matrix_dimension_user_uid: '1000' +matrix_dimension_user_gid: '1000' # Controls whether the matrix-dimension container exposes its HTTP port (tcp/8184 in the container). # From 79f4bcf5beccd040784078d223b9931406297522 Mon Sep 17 00:00:00 2001 From: Aaron Raimist Date: Fri, 7 Jun 2019 16:02:41 -0500 Subject: [PATCH 13/22] Enable sentry.io integration --- roles/matrix-synapse/defaults/main.yml | 6 ++++++ roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 | 6 ++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index f343d5ff..e69f8114 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -205,6 +205,12 @@ matrix_synapse_url_preview_enabled: true matrix_synapse_metrics_enabled: false matrix_synapse_metrics_port: 9100 +# Send ERROR logs to sentry.io for easier tracking +# To set this up: go to sentry.io, create a python project, and set +# matrix_synapse_sentry_dsn to the URL it gives you. +# See https://github.com/matrix-org/synapse/issues/4632 for important privacy concerns +matrix_synapse_sentry_dsn: "" + # Postgres database information matrix_synapse_database_host: "" matrix_synapse_database_user: "" diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index 9e7777fd..8b9cf4f3 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -872,8 +872,10 @@ enable_metrics: {{ matrix_synapse_metrics_enabled|to_json }} # information, and it in turn may then diseminate sensitive information # through insecure notification channels if so configured. # -#sentry: -# dsn: "..." +{% if matrix_synapse_sentry_dsn != "" %} +sentry: + dsn: {{ matrix_synapse_sentry_dsn|to_json }} +{% endif %} # Whether or not to report anonymized homeserver usage statistics. report_stats: {{ matrix_synapse_report_stats|to_json }} From 6fce809d1098300fdc6ef8f8c40210db625251b6 Mon Sep 17 00:00:00 2001 From: Aaron Raimist Date: Sun, 9 Jun 2019 20:35:35 -0500 Subject: [PATCH 14/22] Add config option to be able to access database outside of container --- roles/matrix-postgres/defaults/main.yml | 8 ++++++++ .../templates/systemd/matrix-postgres.service.j2 | 3 +++ 2 files changed, 11 insertions(+) diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index b3dc1e8f..f04452e2 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -15,3 +15,11 @@ matrix_postgres_docker_image_latest: "{{ matrix_postgres_docker_image_v11 }}" # A list of extra arguments to pass to the container matrix_postgres_container_extra_arguments: [] + +# Controls whether the matrix-postgres container exposes a port (tcp/5432 in the +# container) that can be used to access the database with psql. +# +# psql postgresql://username:password@localhost:/database_name +# +# Takes an ":" or "" value (e.g. "127.0.0.1:5432"), or empty string to not expose. +matrix_postgres_container_postgres_bind_port: "" diff --git a/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2 b/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2 index 52fefa8c..4a197003 100644 --- a/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2 +++ b/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2 @@ -17,6 +17,9 @@ ExecStart=/usr/bin/docker run --rm --name matrix-postgres \ --tmpfs=/tmp:rw,noexec,nosuid,size=100m \ --tmpfs=/run/postgresql:rw,noexec,nosuid,size=100m \ --network={{ matrix_docker_network }} \ + {% if matrix_postgres_container_postgres_bind_port %} + -p {{ matrix_postgres_container_postgres_bind_port }}:5432 \ + {% endif %} --env-file={{ matrix_postgres_base_path }}/env-postgres-server \ -v {{ matrix_postgres_data_path }}:/var/lib/postgresql/data:rw \ -v /etc/passwd:/etc/passwd:ro \ From 4f87f7e43e5389cb9bd835d55b652b261cad42b6 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 10 Jun 2019 08:24:37 +0300 Subject: [PATCH 15/22] Explain matrix_postgres_container_postgres_bind_port a little more Previously, it only mentioned exposing for psql-usage purposes. Realistically, it can be used for much more. Especially given that psql can be easily accessed via our matrix-postgres-cli script, without exposing the container port. --- roles/matrix-postgres/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index f04452e2..9e951fb4 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -17,7 +17,7 @@ matrix_postgres_docker_image_latest: "{{ matrix_postgres_docker_image_v11 }}" matrix_postgres_container_extra_arguments: [] # Controls whether the matrix-postgres container exposes a port (tcp/5432 in the -# container) that can be used to access the database with psql. +# container) that can be used to access the database from outside the container (e.g. with psql) # # psql postgresql://username:password@localhost:/database_name # From 7d3adc451220751d3b1928bd27216546142d4a2c Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 10 Jun 2019 14:23:51 +0300 Subject: [PATCH 16/22] Automatically force-pull :latest images We do use some `:latest` images by default for the following services: - matrix-dimension - Goofys (in the matrix-synapse role) - matrix-bridge-appservice-irc - matrix-bridge-appservice-discord - matrix-bridge-mautrix-facebook - matrix-bridge-mautrix-whatsapp It's terribly unfortunate that those software projects don't release anything other than `:latest`, but that's how it is for now. Updating that software requires that users manually do `docker pull` on the server. The playbook didn't force-repull images that it already had. With this patch, it starts doing so. Any image tagged `:latest` will be force re-pulled by the playbook every time it's executed. It should be noted that even though we ask the `docker_image` module to force-pull, it only reports "changed" when it actually pulls something new. This is nice, because it lets people know exactly when something gets updated, as opposed to giving the indication that it's always updating the images (even though it isn't). --- roles/matrix-bridge-appservice-discord/defaults/main.yml | 1 + .../matrix-bridge-appservice-discord/tasks/setup_install.yml | 2 ++ roles/matrix-bridge-appservice-irc/defaults/main.yml | 1 + roles/matrix-bridge-appservice-irc/tasks/setup_install.yml | 2 ++ roles/matrix-bridge-mautrix-facebook/defaults/main.yml | 1 + roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml | 2 ++ roles/matrix-bridge-mautrix-telegram/defaults/main.yml | 1 + roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml | 2 ++ roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml | 1 + roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml | 2 ++ roles/matrix-corporal/defaults/main.yml | 2 ++ roles/matrix-corporal/tasks/setup_corporal.yml | 2 ++ roles/matrix-coturn/defaults/main.yml | 1 + roles/matrix-coturn/tasks/setup_coturn.yml | 2 ++ roles/matrix-dimension/defaults/main.yml | 1 + roles/matrix-dimension/tasks/setup_dimension.yml | 2 ++ roles/matrix-mailer/defaults/main.yml | 1 + roles/matrix-mailer/tasks/setup_mailer.yml | 2 ++ roles/matrix-mxisd/defaults/main.yml | 2 ++ roles/matrix-mxisd/tasks/setup_mxisd.yml | 2 ++ roles/matrix-nginx-proxy/defaults/main.yml | 4 +++- roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml | 2 ++ .../matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml | 2 ++ roles/matrix-postgres/defaults/main.yml | 5 +++++ roles/matrix-postgres/tasks/setup_postgres.yml | 2 ++ roles/matrix-riot-web/defaults/main.yml | 1 + roles/matrix-riot-web/tasks/setup_riot_web.yml | 2 ++ roles/matrix-synapse/defaults/main.yml | 2 ++ roles/matrix-synapse/tasks/goofys/setup_install.yml | 2 ++ roles/matrix-synapse/tasks/synapse/setup_install.yml | 2 ++ 30 files changed, 55 insertions(+), 1 deletion(-) diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml index 8faca889..a415d255 100644 --- a/roles/matrix-bridge-appservice-discord/defaults/main.yml +++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml @@ -4,6 +4,7 @@ matrix_appservice_discord_enabled: true matrix_appservice_discord_docker_image: "halfshot/matrix-appservice-discord:latest" +matrix_appservice_discord_docker_image_force_pull: "{{ matrix_appservice_discord_docker_image.endswith(':latest') }}" matrix_appservice_discord_base_path: "{{ matrix_base_data_path }}/appservice-discord" diff --git a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml index 90d52ef5..0f683c84 100644 --- a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml @@ -12,6 +12,8 @@ docker_image: name: "{{ matrix_appservice_discord_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_appservice_discord_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_discord_docker_image_force_pull }}" - name: Ensure Appservice Discord base directory exists file: diff --git a/roles/matrix-bridge-appservice-irc/defaults/main.yml b/roles/matrix-bridge-appservice-irc/defaults/main.yml index 91531f95..9f05fc4e 100644 --- a/roles/matrix-bridge-appservice-irc/defaults/main.yml +++ b/roles/matrix-bridge-appservice-irc/defaults/main.yml @@ -4,6 +4,7 @@ matrix_appservice_irc_enabled: true matrix_appservice_irc_docker_image: "tedomum/matrix-appservice-irc:latest" +matrix_appservice_irc_docker_image_force_pull: "{{ matrix_appservice_irc_docker_image.endswith(':latest') }}" matrix_appservice_irc_base_path: "{{ matrix_base_data_path }}/appservice-irc" diff --git a/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml b/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml index 96849c65..62a81e37 100644 --- a/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml @@ -12,6 +12,8 @@ docker_image: name: "{{ matrix_appservice_irc_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_appservice_irc_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_irc_docker_image_force_pull }}" - name: Ensure Appservice IRC base directory exists file: diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml index b9039dc6..75752cda 100644 --- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml @@ -4,6 +4,7 @@ matrix_mautrix_facebook_enabled: true matrix_mautrix_facebook_docker_image: "tulir/mautrix-facebook:latest" +matrix_mautrix_facebook_docker_image_force_pull: "{{ matrix_mautrix_facebook_docker_image.endswith(':latest') }}" matrix_mautrix_facebook_base_path: "{{ matrix_base_data_path }}/mautrix-facebook" matrix_mautrix_facebook_config_path: "{{ matrix_mautrix_facebook_base_path }}/config" diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml index d797688e..a69a58c9 100644 --- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml @@ -12,6 +12,8 @@ docker_image: name: "{{ matrix_mautrix_facebook_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_mautrix_facebook_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mautrix_facebook_docker_image_force_pull }}" - name: Ensure Mautrix Facebook paths exist file: diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml index c0f3b0a0..f06f0a76 100644 --- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml @@ -4,6 +4,7 @@ matrix_mautrix_telegram_enabled: true matrix_mautrix_telegram_docker_image: "tulir/mautrix-telegram:v0.5.2" +matrix_mautrix_telegram_docker_image_force_pull: "{{ matrix_mautrix_telegram_docker_image.endswith(':latest') }}" matrix_mautrix_telegram_base_path: "{{ matrix_base_data_path }}/mautrix-telegram" diff --git a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml index 0655369b..2ad0694a 100644 --- a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml @@ -12,6 +12,8 @@ docker_image: name: "{{ matrix_mautrix_telegram_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_mautrix_telegram_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mautrix_telegram_docker_image_force_pull }}" - name: Ensure Mautrix Telegram base directory exists file: diff --git a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml index 32375b87..946e64af 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml @@ -4,6 +4,7 @@ matrix_mautrix_whatsapp_enabled: true matrix_mautrix_whatsapp_docker_image: "tulir/mautrix-whatsapp:latest" +matrix_mautrix_whatsapp_docker_image_force_pull: "{{ matrix_mautrix_whatsapp_docker_image.endswith(':latest') }}" matrix_mautrix_whatsapp_base_path: "{{ matrix_base_data_path }}/mautrix-whatsapp" diff --git a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml index ac7361ac..38e4e035 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml @@ -12,6 +12,8 @@ docker_image: name: "{{ matrix_mautrix_whatsapp_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_mautrix_whatsapp_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mautrix_whatsapp_docker_image_force_pull }}" - name: Ensure Mautrix Whatsapp base directory exists file: diff --git a/roles/matrix-corporal/defaults/main.yml b/roles/matrix-corporal/defaults/main.yml index 47e834a4..af9d1001 100644 --- a/roles/matrix-corporal/defaults/main.yml +++ b/roles/matrix-corporal/defaults/main.yml @@ -20,6 +20,8 @@ matrix_corporal_container_extra_arguments: [] matrix_corporal_systemd_required_services_list: ['docker.service'] matrix_corporal_docker_image: "devture/matrix-corporal:1.4.0" +matrix_corporal_docker_image_force_pull: "{{ matrix_corporal_docker_image.endswith(':latest') }}" + matrix_corporal_base_path: "{{ matrix_base_data_path }}/corporal" matrix_corporal_config_dir_path: "{{ matrix_corporal_base_path }}/config" matrix_corporal_cache_dir_path: "{{ matrix_corporal_base_path }}/cache" diff --git a/roles/matrix-corporal/tasks/setup_corporal.yml b/roles/matrix-corporal/tasks/setup_corporal.yml index 9c65b11c..1fe19732 100644 --- a/roles/matrix-corporal/tasks/setup_corporal.yml +++ b/roles/matrix-corporal/tasks/setup_corporal.yml @@ -21,6 +21,8 @@ docker_image: name: "{{ matrix_corporal_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_corporal_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_corporal_docker_image_force_pull }}" when: matrix_corporal_enabled|bool - name: Ensure Matrix Corporal config installed diff --git a/roles/matrix-coturn/defaults/main.yml b/roles/matrix-coturn/defaults/main.yml index b12cabe7..08cc3877 100644 --- a/roles/matrix-coturn/defaults/main.yml +++ b/roles/matrix-coturn/defaults/main.yml @@ -1,6 +1,7 @@ matrix_coturn_enabled: true matrix_coturn_docker_image: "instrumentisto/coturn:4.5.1.1" +matrix_coturn_docker_image_force_pull: "{{ matrix_coturn_docker_image.endswith(':latest') }}" # The Docker network that Coturn would be put into. # diff --git a/roles/matrix-coturn/tasks/setup_coturn.yml b/roles/matrix-coturn/tasks/setup_coturn.yml index 6a7a9f18..548d3c7a 100644 --- a/roles/matrix-coturn/tasks/setup_coturn.yml +++ b/roles/matrix-coturn/tasks/setup_coturn.yml @@ -8,6 +8,8 @@ docker_image: name: "{{ matrix_coturn_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_coturn_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_coturn_docker_image_force_pull }}" when: matrix_coturn_enabled|bool - name: Ensure Coturn configuration path exists diff --git a/roles/matrix-dimension/defaults/main.yml b/roles/matrix-dimension/defaults/main.yml index 7f52ade6..57c732c4 100644 --- a/roles/matrix-dimension/defaults/main.yml +++ b/roles/matrix-dimension/defaults/main.yml @@ -13,6 +13,7 @@ matrix_dimension_widgets_allow_self_signed_ssl_certificates: false matrix_dimension_base_path: "{{ matrix_base_data_path }}/dimension" matrix_dimension_docker_image: "turt2live/matrix-dimension:latest" +matrix_dimension_docker_image_force_pull: "{{ matrix_dimension_docker_image.endswith(':latest') }}" # The user and group id correspond to the node user in the `turt2live/matrix-dimension` image. matrix_dimension_user_uid: '1000' diff --git a/roles/matrix-dimension/tasks/setup_dimension.yml b/roles/matrix-dimension/tasks/setup_dimension.yml index 436bdd6d..0b52c5a2 100644 --- a/roles/matrix-dimension/tasks/setup_dimension.yml +++ b/roles/matrix-dimension/tasks/setup_dimension.yml @@ -26,6 +26,8 @@ docker_image: name: "{{ matrix_dimension_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_dimension_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_dimension_docker_image_force_pull }}" when: matrix_dimension_enabled|bool - name: Ensure matrix-dimension.service installed diff --git a/roles/matrix-mailer/defaults/main.yml b/roles/matrix-mailer/defaults/main.yml index 184fd3cc..6e5b3467 100644 --- a/roles/matrix-mailer/defaults/main.yml +++ b/roles/matrix-mailer/defaults/main.yml @@ -3,6 +3,7 @@ matrix_mailer_enabled: true matrix_mailer_base_path: "{{ matrix_base_data_path }}/mailer" matrix_mailer_docker_image: "devture/exim-relay:4.91-r3-0" +matrix_mailer_docker_image_force_pull: "{{ matrix_mailer_docker_image.endswith(':latest') }}" # The user/group that the container runs with. # These match the `exim` user/group within the container image. diff --git a/roles/matrix-mailer/tasks/setup_mailer.yml b/roles/matrix-mailer/tasks/setup_mailer.yml index b60d031d..c357ead3 100644 --- a/roles/matrix-mailer/tasks/setup_mailer.yml +++ b/roles/matrix-mailer/tasks/setup_mailer.yml @@ -24,6 +24,8 @@ docker_image: name: "{{ matrix_mailer_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_mailer_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mailer_docker_image_force_pull }}" when: matrix_mailer_enabled|bool - name: Ensure matrix-mailer.service installed diff --git a/roles/matrix-mxisd/defaults/main.yml b/roles/matrix-mxisd/defaults/main.yml index dd4b7afb..8aea3388 100644 --- a/roles/matrix-mxisd/defaults/main.yml +++ b/roles/matrix-mxisd/defaults/main.yml @@ -4,6 +4,8 @@ matrix_mxisd_enabled: true matrix_mxisd_docker_image: "kamax/mxisd:1.4.4" +matrix_mxisd_docker_image_force_pull: "{{ matrix_mxisd_docker_image.endswith(':latest') }}" + matrix_mxisd_base_path: "{{ matrix_base_data_path }}/mxisd" matrix_mxisd_config_path: "{{ matrix_mxisd_base_path }}/config" matrix_mxisd_data_path: "{{ matrix_mxisd_base_path }}/data" diff --git a/roles/matrix-mxisd/tasks/setup_mxisd.yml b/roles/matrix-mxisd/tasks/setup_mxisd.yml index 4f9fdca8..2bafa179 100644 --- a/roles/matrix-mxisd/tasks/setup_mxisd.yml +++ b/roles/matrix-mxisd/tasks/setup_mxisd.yml @@ -20,6 +20,8 @@ docker_image: name: "{{ matrix_mxisd_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_mxisd_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mxisd_docker_image_force_pull }}" when: matrix_mxisd_enabled|bool - name: Ensure mxisd config installed diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index d9983b99..47897489 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -2,8 +2,9 @@ matrix_nginx_proxy_enabled: true # We use an official nginx image, which we fix-up to run unprivileged. # An alternative would be an `nginxinc/nginx-unprivileged` image, but -# those as more frequently out of date. +# that is frequently out of date. matrix_nginx_proxy_docker_image: "nginx:1.15.12-alpine" +matrix_nginx_proxy_docker_image_force_pull: "{{ matrix_nginx_proxy_docker_image.endswith(':latest') }}" matrix_nginx_proxy_base_path: "{{ matrix_base_data_path }}/nginx-proxy" matrix_nginx_proxy_data_path: "{{ matrix_nginx_proxy_base_path }}/data" @@ -144,6 +145,7 @@ matrix_ssl_domains_to_obtain_certificates_for: [] # Controls whether to obtain production or staging certificates from Let's Encrypt. matrix_ssl_lets_encrypt_staging: false matrix_ssl_lets_encrypt_certbot_docker_image: "certbot/certbot:v0.33.1" +matrix_ssl_lets_encrypt_certbot_docker_image_force_pull: "{{ matrix_ssl_lets_encrypt_certbot_docker_image.endswith(':latest') }}" matrix_ssl_lets_encrypt_certbot_standalone_http_port: 2402 matrix_ssl_lets_encrypt_support_email: ~ diff --git a/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml b/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml index b775c333..693c1bd0 100644 --- a/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml +++ b/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml @@ -98,6 +98,8 @@ docker_image: name: "{{ matrix_nginx_proxy_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_nginx_proxy_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_nginx_proxy_docker_image_force_pull }}" when: matrix_nginx_proxy_enabled|bool - name: Ensure matrix-nginx-proxy.service installed diff --git a/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml b/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml index 970ecc94..9dafdb10 100644 --- a/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml +++ b/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml @@ -37,6 +37,8 @@ docker_image: name: "{{ matrix_ssl_lets_encrypt_certbot_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_ssl_lets_encrypt_certbot_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_ssl_lets_encrypt_certbot_docker_image_force_pull }}" when: "matrix_ssl_retrieval_method == 'lets-encrypt'" - name: Obtain Let's Encrypt certificates diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index 9e951fb4..3971eb92 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -13,6 +13,11 @@ matrix_postgres_docker_image_v10: "postgres:10.8-alpine" matrix_postgres_docker_image_v11: "postgres:11.3-alpine" matrix_postgres_docker_image_latest: "{{ matrix_postgres_docker_image_v11 }}" +# This variable is assigned at runtime. Overriding its value has no effect. +matrix_postgres_docker_image_to_use: '{{ matrix_postgres_docker_image_latest }}' + +matrix_postgres_docker_image_force_pull: "{{ matrix_postgres_docker_image_to_use.endswith(':latest') }}" + # A list of extra arguments to pass to the container matrix_postgres_container_extra_arguments: [] diff --git a/roles/matrix-postgres/tasks/setup_postgres.yml b/roles/matrix-postgres/tasks/setup_postgres.yml index 3df4c6e0..7cb10a14 100644 --- a/roles/matrix-postgres/tasks/setup_postgres.yml +++ b/roles/matrix-postgres/tasks/setup_postgres.yml @@ -28,6 +28,8 @@ docker_image: name: "{{ matrix_postgres_docker_image_to_use }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_postgres_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_postgres_docker_image_force_pull }}" when: matrix_postgres_enabled|bool # We always create these directories, even if an external Postgres is used, diff --git a/roles/matrix-riot-web/defaults/main.yml b/roles/matrix-riot-web/defaults/main.yml index 7f7cba9d..9ca25c33 100644 --- a/roles/matrix-riot-web/defaults/main.yml +++ b/roles/matrix-riot-web/defaults/main.yml @@ -1,6 +1,7 @@ matrix_riot_web_enabled: true matrix_riot_web_docker_image: "bubuntux/riot-web:v1.2.1" +matrix_riot_web_docker_image_force_pull: "{{ matrix_riot_web_docker_image.endswith(':latest') }}" matrix_riot_web_data_path: "{{ matrix_base_data_path }}/riot-web" diff --git a/roles/matrix-riot-web/tasks/setup_riot_web.yml b/roles/matrix-riot-web/tasks/setup_riot_web.yml index 1d420177..f72d4369 100644 --- a/roles/matrix-riot-web/tasks/setup_riot_web.yml +++ b/roles/matrix-riot-web/tasks/setup_riot_web.yml @@ -17,6 +17,8 @@ docker_image: name: "{{ matrix_riot_web_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_riot_web_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_riot_web_docker_image_force_pull }}" when: matrix_riot_web_enabled|bool - name: Ensure Matrix riot-web config files installed diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index e69f8114..1e59e111 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -4,6 +4,7 @@ matrix_synapse_enabled: true matrix_synapse_docker_image: "matrixdotorg/synapse:v0.99.5.2" +matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse" matrix_synapse_config_dir_path: "{{ matrix_synapse_base_path }}/config" @@ -259,6 +260,7 @@ matrix_synapse_ext_password_provider_ldap_filter: "" matrix_s3_media_store_enabled: false matrix_s3_media_store_custom_endpoint_enabled: false matrix_s3_goofys_docker_image: "ewoutp/goofys:latest" +matrix_s3_goofys_docker_image_force_pull: "{{ matrix_s3_goofys_docker_image.endswith(':latest') }}" matrix_s3_media_store_custom_endpoint: "your-custom-endpoint" matrix_s3_media_store_bucket_name: "your-bucket-name" matrix_s3_media_store_aws_access_key: "your-aws-access-key" diff --git a/roles/matrix-synapse/tasks/goofys/setup_install.yml b/roles/matrix-synapse/tasks/goofys/setup_install.yml index 0a73d45e..9a4b9a7e 100644 --- a/roles/matrix-synapse/tasks/goofys/setup_install.yml +++ b/roles/matrix-synapse/tasks/goofys/setup_install.yml @@ -2,6 +2,8 @@ docker_image: name: "{{ matrix_s3_goofys_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_s3_goofys_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_s3_goofys_docker_image_force_pull }}" # This will throw a Permission Denied error if already mounted - name: Check Matrix Goofys external storage mountpoint path diff --git a/roles/matrix-synapse/tasks/synapse/setup_install.yml b/roles/matrix-synapse/tasks/synapse/setup_install.yml index 56747c1d..0058de0b 100644 --- a/roles/matrix-synapse/tasks/synapse/setup_install.yml +++ b/roles/matrix-synapse/tasks/synapse/setup_install.yml @@ -22,6 +22,8 @@ docker_image: name: "{{ matrix_synapse_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_synapse_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_synapse_docker_image_force_pull }}" - name: Check if a Synapse signing key exists stat: From e4068e55eeae32023a1ce0f54ff32f9c68abc134 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 11 Jun 2019 20:30:18 +0300 Subject: [PATCH 17/22] Upgrade Synapse (0.99.5.2 -> 1.0.0) --- roles/matrix-synapse/defaults/main.yml | 2 +- .../templates/synapse/homeserver.yaml.j2 | 169 ++++++++++++++++-- 2 files changed, 152 insertions(+), 19 deletions(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 1e59e111..ef585e37 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -3,7 +3,7 @@ matrix_synapse_enabled: true -matrix_synapse_docker_image: "matrixdotorg/synapse:v0.99.5.2" +matrix_synapse_docker_image: "matrixdotorg/synapse:v1.0.0" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse" diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index 8b9cf4f3..6f3213be 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -74,6 +74,16 @@ use_presence: {{ matrix_synapse_use_presence|to_json }} # #restrict_public_rooms_to_local_users: true +# The default room version for newly created rooms. +# +# Known room versions are listed here: +# https://matrix.org/docs/spec/#complete-list-of-room-versions +# +# For example, for room version 1, default_room_version should be set +# to "1". +# +#default_room_version: "4" + # The GC threshold parameters to pass to `gc.set_threshold`, if defined # #gc_thresholds: [700, 10, 10] @@ -256,6 +266,22 @@ listeners: # Monthly Active User Blocking # +# Used in cases where the admin or server owner wants to limit to the +# number of monthly active users. +# +# 'limit_usage_by_mau' disables/enables monthly active user blocking. When +# anabled and a limit is reached the server returns a 'ResourceLimitError' +# with error type Codes.RESOURCE_LIMIT_EXCEEDED +# +# 'max_mau_value' is the hard limit of monthly active users above which +# the server will start blocking user actions. +# +# 'mau_trial_days' is a means to add a grace period for active users. It +# means that users must be active for this number of days before they +# can be considered active and guards against the case where lots of users +# sign up in a short space of time never to return after their initial +# session. +# #limit_usage_by_mau: False #max_mau_value: 50 #mau_trial_days: 2 @@ -305,12 +331,12 @@ tls_certificate_path: {{ matrix_synapse_tls_certificate_path|to_json }} # tls_private_key_path: {{ matrix_synapse_tls_private_key_path|to_json }} -# Whether to verify TLS certificates when sending federation traffic. +# Whether to verify TLS server certificates for outbound federation requests. # -# This currently defaults to `false`, however this will change in -# Synapse 1.0 when valid federation certificates will be required. +# Defaults to `true`. To disable certificate verification, uncomment the +# following line. # -#federation_verify_certificates: true +#federation_verify_certificates: false # Skip federation certificate verification on the following whitelist # of domains. @@ -764,7 +790,9 @@ enable_registration: {{ matrix_synapse_enable_registration|to_json }} # This means that, if a validity period is set, and Synapse is restarted (it will # then derive an expiration date from the current validity period), and some time # after that the validity period changes and Synapse is restarted, the users' -# expiration dates won't be updated unless their account is manually renewed. +# expiration dates won't be updated unless their account is manually renewed. This +# date will be randomly selected within a range [now + period - d ; now + period], +# where d is equal to 10% of the validity period. # #account_validity: # enabled: True @@ -944,12 +972,43 @@ signing_key_path: "/data/{{ matrix_server_fqn_matrix }}.signing.key" # The trusted servers to download signing keys from. # -#perspectives: -# servers: -# "matrix.org": -# verify_keys: -# "ed25519:auto": -# key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw" +# When we need to fetch a signing key, each server is tried in parallel. +# +# Normally, the connection to the key server is validated via TLS certificates. +# Additional security can be provided by configuring a `verify key`, which +# will make synapse check that the response is signed by that key. +# +# This setting supercedes an older setting named `perspectives`. The old format +# is still supported for backwards-compatibility, but it is deprecated. +# +# Options for each entry in the list include: +# +# server_name: the name of the server. required. +# +# verify_keys: an optional map from key id to base64-encoded public key. +# If specified, we will check that the response is signed by at least +# one of the given keys. +# +# accept_keys_insecurely: a boolean. Normally, if `verify_keys` is unset, +# and federation_verify_certificates is not `true`, synapse will refuse +# to start, because this would allow anyone who can spoof DNS responses +# to masquerade as the trusted key server. If you know what you are doing +# and are sure that your network environment provides a secure connection +# to the key server, you can set this to `true` to override this +# behaviour. +# +# An example configuration might look like: +# +#trusted_key_servers: +# - server_name: "my_trusted_server.example.com" +# verify_keys: +# "ed25519:auto": "abcdefghijklmnopqrstuvwxyzabcdefghijklmopqr" +# - server_name: "my_other_trusted_server.example.com" +# +# The default configuration is: +# +#trusted_key_servers: +# - server_name: "matrix.org" # Enable SAML2 for registration and login. Uses pysaml2. @@ -1026,14 +1085,73 @@ password_config: -# Enable sending emails for notification events or expiry notices -# Defining a custom URL for Riot is only needed if email notifications -# should contain links to a self-hosted installation of Riot; when set -# the "app_name" setting is ignored. +# Enable sending emails for password resets, notification events or +# account expiry notices # # If your SMTP server requires authentication, the optional smtp_user & # smtp_pass variables should be used # +#email: +# enable_notifs: false +# smtp_host: "localhost" +# smtp_port: 25 # SSL: 465, STARTTLS: 587 +# smtp_user: "exampleusername" +# smtp_pass: "examplepassword" +# require_transport_security: False +# notif_from: "Your Friendly %(app)s Home Server " +# app_name: Matrix +# +# # Enable email notifications by default +# notif_for_new_users: True +# +# # Defining a custom URL for Riot is only needed if email notifications +# # should contain links to a self-hosted installation of Riot; when set +# # the "app_name" setting is ignored +# riot_base_url: "http://localhost/riot" +# +# # Enable sending password reset emails via the configured, trusted +# # identity servers +# # +# # IMPORTANT! This will give a malicious or overtaken identity server +# # the ability to reset passwords for your users! Make absolutely sure +# # that you want to do this! It is strongly recommended that password +# # reset emails be sent by the homeserver instead +# # +# # If this option is set to false and SMTP options have not been +# # configured, resetting user passwords via email will be disabled +# #trust_identity_server_for_password_resets: false +# +# # Configure the time that a validation email or text message code +# # will expire after sending +# # +# # This is currently used for password resets +# #validation_token_lifetime: 1h +# +# # Template directory. All template files should be stored within this +# # directory +# # +# #template_dir: res/templates +# +# # Templates for email notifications +# # +# notif_template_html: notif_mail.html +# notif_template_text: notif_mail.txt +# +# # Templates for account expiry notices +# # +# expiry_template_html: notice_expiry.html +# expiry_template_text: notice_expiry.txt +# +# # Templates for password reset emails sent by the homeserver +# # +# #password_reset_template_html: password_reset.html +# #password_reset_template_text: password_reset.txt +# +# # Templates for password reset success and failure pages that a user +# # will see after attempting to reset their password +# # +# #password_reset_template_success_html: password_reset_success.html +# #password_reset_template_failure_html: password_reset_failure.html {% if matrix_synapse_email_enabled %} email: enable_notifs: true @@ -1147,9 +1265,9 @@ push: # # 'search_all_users' defines whether to search all users visible to your HS # when searching the user directory, rather than limiting to users visible -# in public rooms. Defaults to false. If you set it True, you'll have to run -# UPDATE user_directory_stream_pos SET stream_id = NULL; -# on your database to tell it to rebuild the user_directory search indexes. +# in public rooms. Defaults to false. If you set it True, you'll have to +# rebuild the user_directory search indexes, see +# https://github.com/matrix-org/synapse/blob/master/docs/user_directory.md # #user_directory: # enabled: true @@ -1207,6 +1325,21 @@ push: # +# Local statistics collection. Used in populating the room directory. +# +# 'bucket_size' controls how large each statistics timeslice is. It can +# be defined in a human readable short form -- e.g. "1d", "1y". +# +# 'retention' controls how long historical statistics will be kept for. +# It can be defined in a human readable short form -- e.g. "1d", "1y". +# +# +#stats: +# enabled: true +# bucket_size: 1d +# retention: 1y + + # Server Notices room configuration # # Uncomment this section to enable a room which can be used to send notices From a8dc0befa9f99d0a3cf11cee8b233fc30a3bf49e Mon Sep 17 00:00:00 2001 From: Michael Haak Date: Tue, 11 Jun 2019 22:06:25 +0200 Subject: [PATCH 18/22] Added a basic guide on migrating to another server --- docs/installing.md | 1 + docs/maintenance-migrating.md | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 docs/maintenance-migrating.md diff --git a/docs/installing.md b/docs/installing.md index 90209e57..78bd1be1 100644 --- a/docs/installing.md +++ b/docs/installing.md @@ -39,3 +39,4 @@ Now that the services are running, you might want to: - or [set up the Dimension Integrations Manager](configuring-playbook-dimension.md) - or [check if services work](maintenance-checking-services.md) - or learn how to [upgrade your services when new versions are released](maintenance-upgrading-services.md) +- or learn how to [migrate to another server](maintenance-migrating.md) diff --git a/docs/maintenance-migrating.md b/docs/maintenance-migrating.md new file mode 100644 index 00000000..2487b674 --- /dev/null +++ b/docs/maintenance-migrating.md @@ -0,0 +1,7 @@ +# Migrating to new server + +1. Backup directory `/matrix`. Make sure to preserve owner and permission (use `cp -p`)! +2. Remove old server from matrix-docker-ansible-deploy `hosts` file and add new server. +3. Run `ansible-playbook -i inventory/hosts setup.yml --tags=setup-all`. This will create the matrix user and group. +4. Copy backup from old server to new server. Make sure to preserve owner and permission (use `cp -p`)! +5. Run `ansible-playbook -i inventory/hosts setup.yml --tags=start` From d262028d825fea9b5e7176b62fad6e745e4d3edf Mon Sep 17 00:00:00 2001 From: Aaron Raimist Date: Tue, 11 Jun 2019 20:19:15 -0500 Subject: [PATCH 19/22] Upgrade mxisd (1.4.4 -> 1.4.5) --- roles/matrix-mxisd/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-mxisd/defaults/main.yml b/roles/matrix-mxisd/defaults/main.yml index 8aea3388..e0988c65 100644 --- a/roles/matrix-mxisd/defaults/main.yml +++ b/roles/matrix-mxisd/defaults/main.yml @@ -3,7 +3,7 @@ matrix_mxisd_enabled: true -matrix_mxisd_docker_image: "kamax/mxisd:1.4.4" +matrix_mxisd_docker_image: "kamax/mxisd:1.4.5" matrix_mxisd_docker_image_force_pull: "{{ matrix_mxisd_docker_image.endswith(':latest') }}" matrix_mxisd_base_path: "{{ matrix_base_data_path }}/mxisd" From 483bdd8c0170da3be3d81f16791742d421ab6eaf Mon Sep 17 00:00:00 2001 From: Aaron Raimist Date: Tue, 11 Jun 2019 21:18:06 -0500 Subject: [PATCH 20/22] Allow default room version to be configured --- roles/matrix-synapse/defaults/main.yml | 2 ++ roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index ef585e37..50301d5b 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -286,3 +286,5 @@ matrix_synapse_room_list_publication_rules: alias: "*" room_id: "*" action: allow + +matrix_synapse_default_room_version: "4" diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index 6f3213be..869f313f 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -82,7 +82,7 @@ use_presence: {{ matrix_synapse_use_presence|to_json }} # For example, for room version 1, default_room_version should be set # to "1". # -#default_room_version: "4" +default_room_version: "{{ matrix_synapse_default_room_version|to_json }}" # The GC threshold parameters to pass to `gc.set_threshold`, if defined # From f4574961c7cd800d525043585c78ffa51782c33d Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 12 Jun 2019 09:17:35 +0300 Subject: [PATCH 21/22] Prevent double-quotes around default room version Using `|to_json` on a string is expected to correctly wrap it in quotes (e.g. `"4"`). Wrapping it explicitly in double-quotes results in undesirable double-quoting (`""4""`). --- roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index 869f313f..43ae99b6 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -82,7 +82,7 @@ use_presence: {{ matrix_synapse_use_presence|to_json }} # For example, for room version 1, default_room_version should be set # to "1". # -default_room_version: "{{ matrix_synapse_default_room_version|to_json }}" +default_room_version: {{ matrix_synapse_default_room_version|to_json }} # The GC threshold parameters to pass to `gc.set_threshold`, if defined # From 8a7b3d5bd0ca7a349ddc1654756d6898ef65d597 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 12 Jun 2019 09:36:19 +0300 Subject: [PATCH 22/22] Make instructions simpler and safer Changes to the original are: - it tells people to stop and disable services, so that: - services won't be running while you are copying files - services won't accidentally start again later - it does the file-copying in 1 step - it does copying before running `--tags=setup-all`, so that existing files (SSL certificates, etc.) can be reused. Otherwise, the playbook starts from a blank slate, retrieves them anew, generates new signing keys anew, etc. Only to have those replaced by your own old backup later. - it mentions DNS changes - combines `--tags=setup-all,start` into a single step, thanks to the files being already copied --- docs/maintenance-migrating.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/maintenance-migrating.md b/docs/maintenance-migrating.md index 2487b674..fcfe4e16 100644 --- a/docs/maintenance-migrating.md +++ b/docs/maintenance-migrating.md @@ -1,7 +1,8 @@ # Migrating to new server -1. Backup directory `/matrix`. Make sure to preserve owner and permission (use `cp -p`)! -2. Remove old server from matrix-docker-ansible-deploy `hosts` file and add new server. -3. Run `ansible-playbook -i inventory/hosts setup.yml --tags=setup-all`. This will create the matrix user and group. -4. Copy backup from old server to new server. Make sure to preserve owner and permission (use `cp -p`)! -5. Run `ansible-playbook -i inventory/hosts setup.yml --tags=start` +1. Prepare by lowering DNS TTL for your domains (`matrix.DOMAIN`, etc.), so that DNS record changes (step 4 below) would happen faster, leading ot less downtime +2. Stop all services on the old server and make sure they won't be starting again. Execute this on the old server: `systemctl disable --now matrix*` +3. Copy directory `/matrix` from the old server to the new server. Make sure to preserve ownership and permissions (use `cp -p` or `rsync -ar`)! +4. Make sure your DNS records are adjusted to point to the new server's IP address +5. Remove old server from the `inventory/hosts` file and add new server. +6. Run `ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`. This will create the matrix user and group and start all services on the new server