From 413d9ec1433498aaafd6e671eb53642ee77d75a8 Mon Sep 17 00:00:00 2001 From: Aaron Raimist Date: Wed, 2 Oct 2019 21:35:44 -0500 Subject: [PATCH 1/4] WIP: Upgrade Synapse (1.3.1 -> 1.4.0rc2) --- roles/matrix-synapse/defaults/main.yml | 9 +- .../templates/synapse/homeserver.yaml.j2 | 282 +++++++++++++----- 2 files changed, 216 insertions(+), 75 deletions(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index f368c388..9a859468 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -3,7 +3,7 @@ matrix_synapse_enabled: true -matrix_synapse_docker_image: "matrixdotorg/synapse:v1.3.1" +matrix_synapse_docker_image: "matrixdotorg/synapse:v1.4.0rc2" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse" @@ -308,6 +308,13 @@ matrix_synapse_default_room_version: "4" # If not, you can also control its value manually. matrix_synapse_spam_checker: ~ +matrix_synapse_trusted_key_servers: + - server_name: "matrix.org" + +matrix_synapse_redaction_retention_period: 7d + +matrix_synapse_user_ips_max_age: 28d + # Default Synapse configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. # diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index 91f371a2..7b602c71 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -105,6 +105,9 @@ federation_domain_whitelist: {{ matrix_synapse_federation_domain_whitelist|to_js # blacklist IP address CIDR ranges. If this option is not specified, or # specified with an empty list, no ip range blacklist will be enforced. # +# As of Synapse v1.4.0 this option also affects any outbound requests to identity +# servers provided by user input. +# # (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly # listed here, since they correspond to unroutable addresses.) # @@ -131,8 +134,8 @@ federation_ip_range_blacklist: # # type: the type of listener. Normally 'http', but other valid options are: # 'manhole' (see docs/manhole.md), -# 'metrics' (see docs/metrics-howto.rst), -# 'replication' (see docs/workers.rst). +# 'metrics' (see docs/metrics-howto.md), +# 'replication' (see docs/workers.md). # # tls: set to true to enable TLS for this listener. Will use the TLS # key/cert specified in tls_private_key_path / tls_certificate_path. @@ -167,12 +170,12 @@ federation_ip_range_blacklist: # # media: the media API (/_matrix/media). # -# metrics: the metrics interface. See docs/metrics-howto.rst. +# metrics: the metrics interface. See docs/metrics-howto.md. # # openid: OpenID authentication. # # replication: the HTTP replication API (/_synapse/replication). See -# docs/workers.rst. +# docs/workers.md. # # static: static resources under synapse/static (/_matrix/static). (Mostly # useful for 'fallback authentication'.) @@ -311,6 +314,23 @@ listeners: # #allow_per_room_profiles: false +# How long to keep redacted events in unredacted form in the database. After +# this period redacted events get replaced with their redacted form in the DB. +# +# Defaults to `7d`. Set to `null` to disable. +# +#redaction_retention_period: 28d + +redaction_retention_period: {{ matrix_synapse_redaction_retention_period }} + +# How long to track users' last seen time and IPs in the database. +# +# Defaults to `28d`. Set to `null` to disable clearing out of old rows. +# +#user_ips_max_age: 14d + +user_ips_max_age: {{ matrix_synapse_user_ips_max_age }} + ## TLS ## @@ -442,7 +462,7 @@ acme: # # If unspecified, we will use CONFDIR/client.key. # - account_key_file: /data/acme_account.key + #account_key_file: /data/acme_account.key # List of allowed TLS fingerprints for this server to publish along # with the signing keys for this server. Other matrix servers that @@ -493,7 +513,8 @@ event_cache_size: "{{ matrix_synapse_event_cache_size }}" ## Logging ## -# A yaml python logging config file +# A yaml python logging config file as described by +# https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema # log_config: "/data/{{ matrix_server_fqn_matrix }}.log.config" @@ -518,13 +539,15 @@ log_config: "/data/{{ matrix_server_fqn_matrix }}.log.config" # - one for login that ratelimits login requests based on the account the # client is attempting to log into, based on the amount of failed login # attempts for this account. +# - one for ratelimiting redactions by room admins. If this is not explicitly +# set then it uses the same ratelimiting as per rc_message. This is useful +# to allow room admins to deal with abuse quickly. # # The defaults are as shown below. # #rc_message: # per_second: 0.2 # burst_count: 10 -# rc_message: {{ matrix_synapse_rc_message|to_json }} # #rc_registration: @@ -543,6 +566,10 @@ rc_registration: {{ matrix_synapse_rc_registration|to_json }} # per_second: 0.17 # burst_count: 3 rc_login: {{ matrix_synapse_rc_login|to_json }} +# +#rc_admin_redaction: +# per_second: 1 +# burst_count: 50 # Ratelimiting settings for incoming federation @@ -907,11 +934,45 @@ allow_guest_access: {{ matrix_synapse_allow_guest_access|to_json }} # Also defines the ID server which will be called when an account is # deactivated (one will be picked arbitrarily). # +# Note: This option is deprecated. Since v0.99.4, Synapse has tracked which identity +# server a 3PID has been bound to. For 3PIDs bound before then, Synapse runs a +# background migration script, informing itself that the identity server all of its +# 3PIDs have been bound to is likely one of the below. +# +# As of Synapse v1.4.0, all other functionality of this option has been deprecated, and +# it is now solely used for the purposes of the background migration script, and can be +# removed once it has run. {% if matrix_synapse_trusted_third_party_id_servers|length > 0 %} trusted_third_party_id_servers: {{ matrix_synapse_trusted_third_party_id_servers|to_nice_yaml }} {% endif %} +# Handle threepid (email/phone etc) registration and password resets through a set of +# *trusted* identity servers. Note that this allows the configured identity server to +# reset passwords for accounts! +# +# Be aware that if `email` is not set, and SMTP options have not been +# configured in the email config block, registration and user password resets via +# email will be globally disabled. +# +# Additionally, if `msisdn` is not set, registration and password resets via msisdn +# will be disabled regardless. This is due to Synapse currently not supporting any +# method of sending SMS messages on its own. +# +# To enable using an identity server for operations regarding a particular third-party +# identifier type, set the value to the URL of that identity server as shown in the +# examples below. +# +# Servers handling the these requests must answer the `/requestToken` endpoints defined +# by the Matrix Identity Service API specification: +# https://matrix.org/docs/spec/identity_service/latest +# +# If a delegate is specified, the config option public_baseurl must also be filled out. +# +account_threepid_delegates: + #email: https://example.com # Delegate email sending to example.org + #msisdn: http://localhost:8090 # Delegate SMS sending to this local process + # Users who register on this homeserver will automatically be joined # to these rooms # @@ -950,9 +1011,24 @@ sentry: dsn: {{ matrix_synapse_sentry_dsn|to_json }} {% endif %} +# Flags to enable Prometheus metrics which are not suitable to be +# enabled by default, either for performance reasons or limited use. +# +metrics_flags: + # Publish synapse_federation_known_servers, a g auge of the number of + # servers this homeserver knows about, including itself. May cause + # performance problems on large homeservers. + # + #known_servers: true + # Whether or not to report anonymized homeserver usage statistics. report_stats: {{ matrix_synapse_report_stats|to_json }} +# The endpoint to report the anonymized homeserver usage statistics to. +# Defaults to https://matrix.org/report-usage-stats/push +# +#report_stats_endpoint: https://example.com/report-usage-stats/push + ## API Configuration ## @@ -1022,6 +1098,10 @@ signing_key_path: "/data/{{ matrix_server_fqn_matrix }}.signing.key" # This setting supercedes an older setting named `perspectives`. The old format # is still supported for backwards-compatibility, but it is deprecated. # +# 'trusted_key_servers' defaults to matrix.org, but using it will generate a +# warning on start-up. To suppress this warning, set +# 'suppress_key_server_warning' to true. +# # Options for each entry in the list include: # # server_name: the name of the server. required. @@ -1046,20 +1126,31 @@ signing_key_path: "/data/{{ matrix_server_fqn_matrix }}.signing.key" # "ed25519:auto": "abcdefghijklmnopqrstuvwxyzabcdefghijklmopqr" # - server_name: "my_other_trusted_server.example.com" # -# The default configuration is: +trusted_key_servers: {{ matrix_synapse_trusted_key_servers|to_json }} + + +# Uncomment the following to disable the warning that is emitted when the +# trusted_key_servers include 'matrix.org'. See above. # -#trusted_key_servers: -# - server_name: "matrix.org" +#suppress_key_server_warning: true + +# The signing keys to use when acting as a trusted key server. If not specified +# defaults to the server signing key. +# +# Can contain multiple keys, one per line. +# +#key_server_signing_keys_path: "key_server_signing_keys.key" # Enable SAML2 for registration and login. Uses pysaml2. # -# `sp_config` is the configuration for the pysaml2 Service Provider. -# See pysaml2 docs for format of config. +# At least one of `sp_config` or `config_path` must be set in this section to +# enable SAML login. # -# Default values will be used for the 'entityid' and 'service' settings, -# so it is not normally necessary to specify them unless you need to -# override them. +# (You will probably also want to set the following options to `false` to +# disable the regular login/registration flows: +# * enable_registration +# * password_config.enabled # # Once SAML support is enabled, a metadata file will be exposed at # https://:/_matrix/saml2/metadata.xml, which you may be able to @@ -1067,52 +1158,85 @@ signing_key_path: "/data/{{ matrix_server_fqn_matrix }}.signing.key" # the IdP to use an ACS location of # https://:/_matrix/saml2/authn_response. # -#saml2_config: -# sp_config: -# # point this to the IdP's metadata. You can use either a local file or -# # (preferably) a URL. -# metadata: -# #local: ["saml2/idp.xml"] -# remote: -# - url: https://our_idp/metadata.xml -# -# # By default, the user has to go to our login page first. If you'd like to -# # allow IdP-initiated login, set 'allow_unsolicited: True' in a -# # 'service.sp' section: -# # -# #service: -# # sp: -# # allow_unsolicited: True -# -# # The examples below are just used to generate our metadata xml, and you -# # may well not need it, depending on your setup. Alternatively you -# # may need a whole lot more detail - see the pysaml2 docs! -# -# description: ["My awesome SP", "en"] -# name: ["Test SP", "en"] -# -# organization: -# name: Example com -# display_name: -# - ["Example co", "en"] -# url: "http://example.com" -# -# contact_person: -# - given_name: Bob -# sur_name: "the Sysadmin" -# email_address": ["admin@example.com"] -# contact_type": technical -# -# # Instead of putting the config inline as above, you can specify a -# # separate pysaml2 configuration file: -# # -# config_path: "/data/sp_conf.py" -# -# # the lifetime of a SAML session. This defines how long a user has to -# # complete the authentication process, if allow_unsolicited is unset. -# # The default is 5 minutes. -# # -# # saml_session_lifetime: 5m +saml2_config: + # `sp_config` is the configuration for the pysaml2 Service Provider. + # See pysaml2 docs for format of config. + # + # Default values will be used for the 'entityid' and 'service' settings, + # so it is not normally necessary to specify them unless you need to + # override them. + # + #sp_config: + # # point this to the IdP's metadata. You can use either a local file or + # # (preferably) a URL. + # metadata: + # #local: ["saml2/idp.xml"] + # remote: + # - url: https://our_idp/metadata.xml + # + # # By default, the user has to go to our login page first. If you'd like + # # to allow IdP-initiated login, set 'allow_unsolicited: True' in a + # # 'service.sp' section: + # # + # #service: + # # sp: + # # allow_unsolicited: true + # + # # The examples below are just used to generate our metadata xml, and you + # # may well not need them, depending on your setup. Alternatively you + # # may need a whole lot more detail - see the pysaml2 docs! + # + # description: ["My awesome SP", "en"] + # name: ["Test SP", "en"] + # + # organization: + # name: Example com + # display_name: + # - ["Example co", "en"] + # url: "http://example.com" + # + # contact_person: + # - given_name: Bob + # sur_name: "the Sysadmin" + # email_address": ["admin@example.com"] + # contact_type": technical + + # Instead of putting the config inline as above, you can specify a + # separate pysaml2 configuration file: + # + #config_path: "CONFDIR/sp_conf.py" + + # the lifetime of a SAML session. This defines how long a user has to + # complete the authentication process, if allow_unsolicited is unset. + # The default is 5 minutes. + # + #saml_session_lifetime: 5m + + # The SAML attribute (after mapping via the attribute maps) to use to derive + # the Matrix ID from. 'uid' by default. + # + #mxid_source_attribute: displayName + + # The mapping system to use for mapping the saml attribute onto a matrix ID. + # Options include: + # * 'hexencode' (which maps unpermitted characters to '=xx') + # * 'dotreplace' (which replaces unpermitted characters with '.'). + # The default is 'hexencode'. + # + #mxid_mapping: dotreplace + + # In previous versions of synapse, the mapping from SAML attribute to MXID was + # always calculated dynamically rather than stored in a table. For backwards- + # compatibility, we will look for user_ids matching such a pattern before + # creating a new account. + # + # This setting controls the SAML attribute which will be used for this + # backwards-compatibility lookup. Typically it should be 'uid', but if the + # attribute maps are changed, it may be necessary to change it. + # + # The default is 'uid'. + # + #grandfathered_mxid_source_attribute: upn @@ -1178,19 +1302,6 @@ password_config: # # # riot_base_url: "http://localhost/riot" # -# # Enable sending password reset emails via the configured, trusted -# # identity servers -# # -# # IMPORTANT! This will give a malicious or overtaken identity server -# # the ability to reset passwords for your users! Make absolutely sure -# # that you want to do this! It is strongly recommended that password -# # reset emails be sent by the homeserver instead -# # -# # If this option is set to false and SMTP options have not been -# # configured, resetting user passwords via email will be disabled -# # -# #trust_identity_server_for_password_resets: false -# # # Configure the time that a validation email or text message code # # will expire after sending # # @@ -1222,11 +1333,34 @@ password_config: # #password_reset_template_html: password_reset.html # #password_reset_template_text: password_reset.txt # +# # Templates for registration emails sent by the homeserver +# # +# #registration_template_html: registration.html +# #registration_template_text: registration.txt +# +# # Templates for validation emails sent by the homeserver when adding an email to +# # your user account +# # +# #add_threepid_template_html: add_threepid.html +# #add_threepid_template_text: add_threepid.txt +# # # Templates for password reset success and failure pages that a user # # will see after attempting to reset their password # # # #password_reset_template_success_html: password_reset_success.html # #password_reset_template_failure_html: password_reset_failure.html +# +# # Templates for registration success and failure pages that a user +# # will see after attempting to register using an email or phone +# # +# #registration_template_success_html: registration_success.html +# #registration_template_failure_html: registration_failure.html +# +# # Templates for success and failure pages that a user will see after attempting +# # to add an email or phone to their account +# # +# #add_threepid_success_html: add_threepid_success.html +# #add_threepid_failure_html: add_threepid_failure.html {% if matrix_synapse_email_enabled %} email: enable_notifs: true From 392f8202bd44630570ea4da6b24fcb931eff419c Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 3 Oct 2019 18:07:33 +0300 Subject: [PATCH 2/4] Make SAML2 configuration match sample config generated using generate command --- roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index 7b602c71..9d8da346 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -1204,7 +1204,7 @@ saml2_config: # Instead of putting the config inline as above, you can specify a # separate pysaml2 configuration file: # - #config_path: "CONFDIR/sp_conf.py" + #config_path: "/data/sp_conf.py" # the lifetime of a SAML session. This defines how long a user has to # complete the authentication process, if allow_unsolicited is unset. From a37b96d829918061b1a9dae48775c243c096604e Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 3 Oct 2019 19:20:59 +0300 Subject: [PATCH 3/4] Upgrade Synapse to 1.4.0 --- roles/matrix-synapse/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 9a859468..41a9862b 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -3,7 +3,7 @@ matrix_synapse_enabled: true -matrix_synapse_docker_image: "matrixdotorg/synapse:v1.4.0rc2" +matrix_synapse_docker_image: "matrixdotorg/synapse:v1.4.0" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse" From c3229209e8ba5a7144fd7f2c4c3a6058c6f9c699 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 3 Oct 2019 19:21:07 +0300 Subject: [PATCH 4/4] Update changelog --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a6f3b08..18ac6568 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,18 @@ +# 2019-10-03 + +## Synapse 1.4.0 + +Synapse 1.4.0 [is out](https://matrix.org/blog/2019/10/03/synapse-1-4-0-released) with lots of changes related to privacy. + +Its new defaults (which we adopt as well) mean that certain old data will automatically get purged after a certain number of days. 1.4.0 automatically garbage collects redacted messages (defaults to 7 days) and removes unused IP and user agent information stored in the user_ips table (defaults to 30 days). If you'd like to preserve this data, we encourage you to look at the `redaction_retention_period` and `user_ips_max_age` options (controllable by the `matrix_synapse_redaction_retention_period` and `matrix_synapse_user_ips_max_age` playbook variables, respectively) before doing the upgrade. If you'd like to keep data indefinitely, set these variables to `null` (e.g. `matrix_synapse_redaction_retention_period: ~`). + +From now on the `trusted_key_servers` setting for Synapse is configurable. It still defaults to `matrix.org` just like it always has, but in a more explicit way now. If you'd like to use another trusted key server, adjust the `matrix_synapse_trusted_key_servers` playbook variable. + +Synapse 1.4.0 also changes lots of things related to identity server integration. +Because Synapse will now by default be responsible for validating email addresses for user accounts, running without an identity server looks more feasible. +We still [have concerns](https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/275/files#r331104117) over disabling the identity server by default, so for now it remains enabled. + + # 2019-09-09 ## Synapse Simple Antispam support