Merge pull request #207 from tommes0815/whatsapp-config-playbook-managed

Whatsapp config playbook managed

CHANGELOG.md

@@ -1,3 +1,20 @@
+# 2019-06-24
+
+## (BC Break) WhatsApp bridge configuration is now entirely managed by the playbook
+
+Until now, configuration files for the [WhatsApp bridge](docs/configuring-playbook-bridge-mautrix-whatsapp.md) were created by the playbook initially, but never modified later on.
+
+From now on, the playbook will keep the configuration in sync for you.
+
+This means that if you were making manual changes to the `/matrix/mautrix-whatsapp/config.yaml` or `/matrix/mautrix-whatsapp/registration.yaml` configuration files, those would be lost the next time you run the playbook.
+
+The bridge now stores configuration in a subdirectory (`/matrix/mautrix-whatsapp/config`), so your old configuration remains in the base directory (`/matrix/mautrix-whatsapp`).
+You need to migrate any manual changes over to the new `matrix_mautrix_whatsapp_configuration_extension_yaml` variable, so that the playbook would apply them for you.
+
+Likewise, data is now also stored in a subdirectory (`/matrix/mautrix-whatsapp/data`). When you run the playbook with an existing database file (`/matrix/mautrix-whatsapp/mautrix-whatsapp.db`), the playbook will stop the bridge and relocate the database file to the `./data` directory. There's no data-loss involved. You'll need to restart the bridge manually though (`--tags=start`).
+
+We're now following the default configuration for the WhatsApp bridge.
+
 # 2019-06-20
 
 ## (BC Break) IRC bridge configuration is now entirely managed by the playbook

group_vars/matrix_servers

@@ -164,6 +164,10 @@ matrix_mautrix_whatsapp_systemd_required_services_list: |
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
   }}
 
+matrix_mautrix_whatsapp_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'whatsapp-appservice-token') | to_uuid }}"
+
+matrix_mautrix_whatsapp_homeserver_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'whatsapp-homeserver-token') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-mautrix-whatsapp

roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml

@@ -7,6 +7,8 @@ matrix_mautrix_whatsapp_docker_image: "tulir/mautrix-whatsapp:latest"
 matrix_mautrix_whatsapp_docker_image_force_pull: "{{ matrix_mautrix_whatsapp_docker_image.endswith(':latest') }}"
 
 matrix_mautrix_whatsapp_base_path: "{{ matrix_base_data_path }}/mautrix-whatsapp"
+matrix_mautrix_whatsapp_config_path: "{{ matrix_mautrix_whatsapp_base_path }}/config"
+matrix_mautrix_whatsapp_data_path: "{{ matrix_mautrix_whatsapp_base_path }}/data"
 
 matrix_mautrix_whatsapp_homeserver_address: "https://{{ matrix_server_fqn_matrix }}"
 matrix_mautrix_whatsapp_homeserver_domain: "{{ matrix_domain }}"
@@ -20,3 +22,170 @@ matrix_mautrix_whatsapp_systemd_required_services_list: ['docker.service']
 
 # List of systemd services that matrix-mautrix-whatsapp.service wants
 matrix_mautrix_whatsapp_systemd_wanted_services_list: []
+
+matrix_mautrix_whatsapp_appservice_token: ''
+matrix_mautrix_whatsapp_homeserver_token: ''
+
+# Default mautrix-whatsapp configuration template which covers the generic use case.
+# You can customize it by controlling the various variables inside it.
+#
+# For a more advanced customization, you can extend the default (see `matrix_mautrix_whatsapp_configuration_extension_yaml`)
+# or completely replace this variable with your own template.
+matrix_mautrix_whatsapp_configuration_yaml: |
+  # Homeserver details.
+  homeserver:
+      # The address that this appservice can use to connect to the homeserver.
+      address: {{ matrix_mautrix_whatsapp_homeserver_address }}
+      # The domain of the homeserver (for MXIDs, etc).
+      domain: {{ matrix_mautrix_whatsapp_homeserver_domain }}
+  # Application service host/registration related details.
+  # Changing these values requires regeneration of the registration.
+
+  appservice:
+      # The address that the homeserver can use to connect to this appservice.
+      address: {{ matrix_mautrix_whatsapp_appservice_address }}
+
+      # The hostname and port where this appservice should listen.
+      hostname: 0.0.0.0
+      port: 8080
+
+      # Database config.
+      database:
+          # The database type. "sqlite3" and "postgres" are supported.
+          type: sqlite3
+          # The database URI.
+          #   SQLite: File name is enough. https://github.com/mattn/go-sqlite3#connection-string
+          #   Postgres: Connection string. For example, postgres://user:password@host/database
+          uri: mautrix-whatsapp.db
+          # Maximum number of connections. Mostly relevant for Postgres.
+          max_open_conns: 20
+          max_idle_conns: 2
+
+      # Path to the Matrix room state store.
+      state_store_path: ./mx-state.json
+
+      # The unique ID of this appservice.
+      id: whatsapp
+      # Appservice bot details.
+      bot:
+          # Username of the appservice bot.
+          username: whatsappbot
+          # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty
+          # to leave display name/avatar as-is.
+          displayname: WhatsApp bridge bot
+          avatar: mxc://maunium.net/NeXNQarUbrlYBiPCpprYsRqr
+
+      # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
+      as_token: "{{ matrix_mautrix_whatsapp_appservice_token }}"
+      hs_token: "{{ matrix_mautrix_whatsapp_homeserver_token }}"
+
+  # Bridge config
+  bridge:
+      # Localpart template of MXIDs for WhatsApp users.
+      # {{ '{{.}}' }} is replaced with the phone number of the WhatsApp user.
+      username_template: "{{ 'whatsapp_{{.}}' }}"
+      # Displayname template for WhatsApp users.
+      # {{ '{{.Notify'}}' }} - nickname set by the WhatsApp user
+      # {{ '{{.Jid}}' }}    - phone number (international format)
+      # The following variables are also available, but will cause problems on multi-user instances:
+      # {{ '{{.Name}}' }}   - display name from contact list
+      # {{ '{{.Short}}' }}  - short display name from contact list
+      displayname_template: "{{ '{{if .Notify}}{{.Notify}}{{else}}{{.Jid}}{{end}} (WA)' }}"
+      # WhatsApp connection timeout in seconds.
+      connection_timeout: 20
+      # Maximum number of times to retry connecting on connection error.
+      max_connection_attempts: 3
+      # Number of seconds to wait between connection attempts.
+      # Negative numbers are exponential backoff: -connection_retry_delay + 1 + 2^attempts
+      connection_retry_delay: -1
+      # Whether or not the bridge should send a notice to the user's management room when it retries connecting.
+      # If false, it will only report when it stops retrying.
+      report_connection_retry: true
+      # Number of seconds to wait for contacts and chats to be sent at startup before syncing.
+      # If you have lots of chats, it might take more than a second.
+      contact_wait_delay: 1
+      # Number of chats to sync for new users.
+      initial_chat_sync_count: 10
+      # Number of old messages to fill when creating new portal rooms.
+      initial_history_fill_count: 20
+      # Maximum number of chats to sync when recovering from downtime.
+      # Set to -1 to sync all new chats during downtime.
+      recovery_chat_sync_limit: -1
+      # Whether or not to sync history when recovering from downtime.
+      recovery_history_backfill: true
+      # Maximum number of seconds since last message in chat to skip
+      # syncing the chat in any case. This setting will take priority
+      # over both recovery_chat_sync_limit and initial_chat_sync_count.
+      # Default is 3 days = 259200 seconds
+      sync_max_chat_age: 259200
+
+      # Whether or not to sync with custom puppets to receive EDUs that
+      # are not normally sent to appservices.
+      sync_with_custom_puppets: true
+      # Whether or not to invite own WhatsApp user's Matrix puppet into private
+      # chat portals when backfilling if needed.
+      # This always uses the default puppet instead of custom puppets due to
+      # rate limits and timestamp massaging.
+      invite_own_puppet_for_backfilling: true
+      # Whether or not to explicitly set the avatar and room name for private
+      # chat portal rooms. This can be useful if the previous field works fine,
+      # but causes room avatar/name bugs.
+      private_chat_portal_meta: false
+
+      # The prefix for commands. Only required in non-management rooms.
+      command_prefix: "!wa"
+
+      # Permissions for using the bridge.
+      # Permitted values:
+      #     user - Access to use the bridge to chat with a WhatsApp account.
+      #    admin - User level and some additional administration tools
+      # Permitted keys:
+      #        * - All Matrix users
+      #   domain - All users on that homeserver
+      #     mxid - Specific user
+      permissions:
+          "{{ matrix_mautrix_whatsapp_homeserver_domain }}": user
+  # Logging config.
+  logging:
+      # The directory for log files. Will be created if not found.
+      directory: ./logs
+      # Available variables: .Date for the file date and .Index for different log files on the same day.
+      file_name_format: "{{ '{{.Date}}-{{.Index}}.log' }}"
+      # Date format for file names in the Go time format: https://golang.org/pkg/time/#pkg-constants
+      file_date_format: "2006-01-02"
+      # Log file permissions.
+      file_mode: 0600
+      # Timestamp format for log entries in the Go time format.
+      timestamp_format: "Jan _2, 2006 15:04:05"
+      # Minimum severity for log messages.
+      # Options: debug, info, warn, error, fatal
+      print_level: debug
+
+matrix_mautrix_whatsapp_configuration_extension_yaml: |
+  # Your custom YAML configuration goes here.
+  # This configuration extends the default starting configuration (`matrix_mautrix_whatsapp_configuration_yaml`).
+  #
+  # You can override individual variables from the default configuration, or introduce new ones.
+  #
+  # If you need something more special, you can take full control by
+  # completely redefining `matrix_mautrix_whatsapp_configuration_yaml`.
+
+matrix_mautrix_whatsapp_configuration_extension: "{{ matrix_mautrix_whatsapp_configuration_extension_yaml|from_yaml if matrix_mautrix_whatsapp_configuration_extension_yaml|from_yaml is mapping else {} }}"
+
+# Holds the final configuration (a combination of the default and its extension).
+# You most likely don't need to touch this variable. Instead, see `matrix_mautrix_whatsapp_configuration_yaml`.
+matrix_mautrix_whatsapp_configuration: "{{ matrix_mautrix_whatsapp_configuration_yaml|from_yaml|combine(matrix_mautrix_whatsapp_configuration_extension, recursive=True) }}"
+
+matrix_mautrix_whatsapp_registration_yaml: |
+  id: whatsapp
+  url: {{ matrix_mautrix_whatsapp_appservice_address }}
+  as_token: "{{ matrix_mautrix_whatsapp_appservice_token }}"
+  hs_token: "{{ matrix_mautrix_whatsapp_homeserver_token }}"
+  sender_localpart: whatsappbot
+  rate_limited: false
+  namespaces:
+      users:
+      - regex: '^@whatsapp_[0-9]+:{{ matrix_mautrix_whatsapp_homeserver_domain }}$'
+        exclusive: true
+
+matrix_mautrix_whatsapp_registration: "{{ matrix_mautrix_whatsapp_registration_yaml|from_yaml }}"

roles/matrix-bridge-mautrix-whatsapp/tasks/init.yml

@@ -7,7 +7,7 @@
     matrix_synapse_container_extra_arguments: >
       {{ matrix_synapse_container_extra_arguments|default([]) }}
       +
-      {{ ["--mount type=bind,src={{ matrix_mautrix_whatsapp_base_path }}/registration.yaml,dst=/matrix-mautrix-whatsapp-registration.yaml,ro"] }}
+      {{ ["--mount type=bind,src={{ matrix_mautrix_whatsapp_config_path }}/registration.yaml,dst=/matrix-mautrix-whatsapp-registration.yaml,ro"] }}
 
     matrix_synapse_app_service_config_files: >
       {{ matrix_synapse_app_service_config_files|default([]) }}

roles/matrix-bridge-mautrix-whatsapp/tasks/main.yml

@@ -2,6 +2,12 @@
   tags:
     - always
 
+- import_tasks: "{{ role_path }}/tasks/validate_config.yml"
+  when: "run_setup|bool and matrix_mautrix_whatsapp_enabled|bool"
+  tags:
+    - setup-all
+    - setup-mautrix-whatsapp
+
 - import_tasks: "{{ role_path }}/tasks/setup_install.yml"
   when: "run_setup and matrix_mautrix_whatsapp_enabled"
   tags:

roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml

@@ -15,27 +15,59 @@
     force_source: "{{ matrix_mautrix_whatsapp_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
     force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mautrix_whatsapp_docker_image_force_pull }}"
 
-- name: Ensure Mautrix Whatsapp base directory exists
+- name: Ensure Mautrix Whatsapp paths exists
   file:
-    path: "{{ matrix_mautrix_whatsapp_base_path }}"
+    path: "{{ item }}"
     state: directory
     mode: 0750
     owner: "{{ matrix_user_username }}"
     group: "{{ matrix_user_username }}"
+  with_items:
+    - "{{ matrix_mautrix_whatsapp_base_path }}"
+    - "{{ matrix_mautrix_whatsapp_config_path }}"
+    - "{{ matrix_mautrix_whatsapp_data_path }}"
     
-- name: Check if a mautrix-whatsapp configuration file exists
+- name: Check if an old database file exists
   stat:
-    path: "{{ matrix_mautrix_whatsapp_base_path }}/config.yaml"
+    path: "{{ matrix_mautrix_whatsapp_base_path }}/mautrix-whatsapp.db"
-  register: mautrix_whatsapp_config_file_stat
+  register: matrix_mautrix_whatsapp_stat_database
   
-- name: Ensure Matrix Mautrix whatsapp config installed
+- name: Check if an old matrix state file exists
-  template:
+  stat:
-    src: "{{ role_path }}/templates//config.yaml.j2"
+    path: "{{ matrix_mautrix_whatsapp_base_path }}/mx-state.json"
-    dest: "{{ matrix_mautrix_whatsapp_base_path }}/config.yaml"
+  register: matrix_mautrix_whatsapp_stat_mx_state
+
+- name: (Data relocation) Ensure matrix-mautrix-whatsapp.service is stopped
+  service:
+    name: matrix-mautrix-whatsapp
+    state: stopped
+    daemon_reload: yes
+  failed_when: false
+  when: "matrix_mautrix_whatsapp_stat_database.stat.exists"
+
+- name: (Data relocation) Move mautrix-whatsapp database file to ./data directory
+  command: "mv {{ matrix_mautrix_whatsapp_base_path }}/mautrix-whatsapp.db {{ matrix_mautrix_whatsapp_data_path }}/mautrix-whatsapp.db"
+  when: "matrix_mautrix_whatsapp_stat_database.stat.exists"
+  
+- name: (Data relocation) Move mautrix-whatsapp mx-state file to ./data directory
+  command: "mv {{ matrix_mautrix_whatsapp_base_path }}/mx-state.json {{ matrix_mautrix_whatsapp_data_path }}/mx-state.json"
+  when: "matrix_mautrix_whatsapp_stat_mx_state.stat.exists"
+
+- name: Ensure mautrix-whatsapp config.yaml installed
+  copy:
+    content: "{{ matrix_mautrix_whatsapp_configuration|to_nice_yaml }}"
+    dest: "{{ matrix_mautrix_whatsapp_config_path }}/config.yaml"
+    mode: 0644
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_username }}"
+
+- name: Ensure mautrix-whatsapp registration.yaml installed
+  copy:
+    content: "{{ matrix_mautrix_whatsapp_registration|to_nice_yaml }}"
+    dest: "{{ matrix_mautrix_whatsapp_config_path }}/registration.yaml"
     mode: 0644
     owner: "{{ matrix_user_username }}"
     group: "{{ matrix_user_username }}"
-  when: "not mautrix_whatsapp_config_file_stat.stat.exists"
 
 - name: Ensure matrix-mautrix-whatsapp.service installed
   template:
@@ -48,21 +80,3 @@
   service:
     daemon_reload: yes
   when: "matrix_mautrix_whatsapp_systemd_service_result.changed"
-
-- name: Check if a mautrix-whatsapp registration file exists
-  stat:
-    path: "{{ matrix_mautrix_whatsapp_base_path }}/registration.yaml"
-  register: mautrix_whatsapp_registration_file_stat
-
-- name: Generate matrix-mautrix-whatsapp registration.yaml if it doesn't exist
-  shell:
-    cmd: >-
-      /usr/bin/docker run
-      --rm
-      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
-      --cap-drop=ALL
-      --name matrix-mautrix-whatsapp-gen
-      -v {{ matrix_mautrix_whatsapp_base_path }}:/data:z
-      {{ matrix_mautrix_whatsapp_docker_image }}
-      /usr/bin/mautrix-whatsapp -g -c /data/config.yaml -r /data/registration.yaml
-  when: "not mautrix_whatsapp_registration_file_stat.stat.exists"

roles/matrix-bridge-mautrix-whatsapp/tasks/validate_config.yml

@@ -0,0 +1,11 @@
+---
+
+- name: Fail if required settings not defined
+  fail:
+    msg: >-
+      You need to define a required configuration setting (`{{ item }}`).
+  when: "vars[item] == ''"
+  with_items:
+    - "matrix_mautrix_whatsapp_appservice_token"
+    - "matrix_mautrix_whatsapp_homeserver_token"
+

roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2

@@ -1,87 +0,0 @@
-#jinja2: lstrip_blocks: "True"
-# Homeserver details.
-homeserver:
-  # The address that this appservice can use to connect to the homeserver.
-  address: {{ matrix_mautrix_whatsapp_homeserver_address }}
-  # The domain of the homeserver (for MXIDs, etc).
-  domain: {{ matrix_mautrix_whatsapp_homeserver_domain }}
-
-# Application service host/registration related details.
-# Changing these values requires regeneration of the registration.
-appservice:
-  # The address that the homeserver can use to connect to this appservice.
-  address: {{ matrix_mautrix_whatsapp_appservice_address }}
-
-  # The hostname and port where this appservice should listen.
-  hostname: 0.0.0.0
-  port: 8080
-
-  # Database config.
-  database:
-    # The database type. Only "sqlite3" is supported.
-    type: sqlite3
-    # The database URI. Usually file name. https://github.com/mattn/go-sqlite3#connection-string
-    uri: mautrix-whatsapp.db
-  # Path to the Matrix room state store.
-  state_store_path: ./mx-state.json
-
-  # The unique ID of this appservice.
-  id: whatsapp
-  # Appservice bot details.
-  bot:
-    # Username of the appservice bot.
-    username: whatsappbot
-    # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty
-    # to leave display name/avatar as-is.
-    displayname: WhatsApp bridge bot
-    avatar: remove
-
-  # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
-  as_token: "This value is generated when generating the registration"
-  hs_token: "This value is generated when generating the registration"
-
-# Bridge config. Currently unused.
-bridge:
-  # {% raw %}
-  # protecting the go templates inside the raw section.
-  # Localpart template of MXIDs for WhatsApp users.
-  # {{.}} is replaced with the phone number of the WhatsApp user.
-  username_template: whatsapp_{{.}}
-  # Displayname template for WhatsApp users.
-  # {{.Notify}} - nickname set by the WhatsApp user
-  # {{.Jid}}    - phone number (international format)
-  # The following variables are also available, but will cause problems on multi-user instances:
-  # {{.Name}}   - display name from contact list
-  # {{.Short}}  - short display name from contact list
-  displayname_template: "{{if .Notify}}{{.Notify}}{{else}}{{.Jid}}{{end}} (WA)"
-  # {% endraw %}
-
-  # The prefix for commands. Only required in non-management rooms.
-  command_prefix: "!wa"
-
-  # Permissions for using the bridge.
-  # Permitted values:
-  #     user - Access to use the bridge to chat with a WhatsApp account.
-  #    admin - User level and some additional administration tools
-  # Permitted keys:
-  #        * - All Matrix users
-  #   domain - All users on that homeserver
-  #     mxid - Specific user
-  permissions:
-    '{{ matrix_mautrix_whatsapp_homeserver_domain }}': user
-
-# Logging config.
-logging:
-  # The directory for log files. Will be created if not found.
-  directory: ./logs
-  # Available variables: .Date for the file date and .Index for different log files on the same day.
-  file_name_format: {% raw %}"{{.Date}}-{{.Index}}.log"{% endraw %}
-  # Date format for file names in the Go time format: https://golang.org/pkg/time/#pkg-constants
-  file_date_format: 2006-01-02
-  # Log file permissions.
-  file_mode: 0600
-  # Timestamp format for log entries in the Go time format.
-  timestamp_format: Jan _2, 2006 15:04:05
-  # Minimum severity for log messages.
-  # Options: debug, info, warn, error, fatal
-  print_level: debug

roles/matrix-bridge-mautrix-whatsapp/templates/systemd/matrix-mautrix-whatsapp.service.j2

@@ -22,13 +22,14 @@ ExecStart=/usr/bin/docker run --rm --name matrix-mautrix-whatsapp \
 			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
 			--cap-drop=ALL \
 			--network={{ matrix_docker_network }} \
-			-v {{ matrix_mautrix_whatsapp_base_path }}:/data:z \
+			-v {{ matrix_mautrix_whatsapp_config_path }}:/config:z \
+			-v {{ matrix_mautrix_whatsapp_data_path }}:/data:z \
 			--workdir=/data \
 			{% for arg in matrix_mautrix_whatsapp_container_extra_arguments %}
 			{{ arg }} \
 			{% endfor %}
 			{{ matrix_mautrix_whatsapp_docker_image }} \
-			/usr/bin/mautrix-whatsapp
+			/usr/bin/mautrix-whatsapp -c /config/config.yaml -r /config/registration.yaml
 
 ExecStop=-/usr/bin/docker kill matrix-mautrix-whatsapp
 ExecStop=-/usr/bin/docker rm matrix-mautrix-whatsapp