updated whatsapp config, backfill doesn't work

roles/matrix-base/defaults/main.yml

@@ -98,6 +98,10 @@ matrix_identity_server_url: ~
 matrix_integration_manager_rest_url: ~
 matrix_integration_manager_ui_url: ~
 
+# Enable backfilling history sync payloads from bridges using batch sending?
+# This requires a server with MSC2716 support, which is currently an experimental feature in synapse.
+matrix_bridges_backfill_enabled: false
+
 # The domain name where a Jitsi server is self-hosted.
 # If set, `/.well-known/matrix/client` will suggest Element clients to use that Jitsi server.
 # See: https://github.com/vector-im/element-web/blob/develop/docs/jitsi.md#configuring-element-to-use-your-self-hosted-jitsi-server
@@ -121,6 +125,8 @@ matrix_client_element_e2ee_secure_backup_setup_methods: []
 # The Docker network that all services would be put into
 matrix_docker_network: "matrix"
 
+
+
 # Controls whether we'll preserve the vars.yml file on the Matrix server.
 # If you have a differently organized inventory, you may wish to disable this feature,
 # or to repoint `matrix_vars_yml_snapshotting_src` to the file you'd like to preserve.

roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2

@@ -7,6 +7,10 @@ homeserver:
     domain: {{ matrix_mautrix_whatsapp_homeserver_domain }}
 # Application service host/registration related details.
 # Changing these values requires regeneration of the registration.
+    # The URL to push real-time bridge status to.
+    # If set, the bridge will make POST requests to this URL whenever a user's whatsapp connection state changes.
+    # The bridge will use the appservice as_token to authorize requests.
+#    status_endpoint: "null"
 
 appservice:
     # The address that the homeserver can use to connect to this appservice.
@@ -28,9 +32,6 @@ appservice:
         max_open_conns: 20
         max_idle_conns: 2
 
-    # Path to the Matrix room state store.
-    state_store_path: ./mx-state.json
-
     # The unique ID of this appservice.
     id: whatsapp
     # Appservice bot details.
@@ -51,79 +52,142 @@ bridge:
     # Localpart template of MXIDs for WhatsApp users.
     # {{ '{{.}}' }} is replaced with the phone number of the WhatsApp user.
     username_template: "{{ 'whatsapp_{{.}}' }}"
-    # Displayname template for WhatsApp users.
-    # {{ '{{.Notify'}}' }} - nickname set by the WhatsApp user
-    # {{ '{{.Jid}}' }}    - phone number (international format)
-    # The following variables are also available, but will cause problems on multi-user instances:
-    # {{ '{{.Name}}' }}   - display name from contact list
-    # {{ '{{.Short}}' }}  - short display name from contact list
-    displayname_template: "{{ '{{if .Notify}}{{.Notify}}{{else}}{{.Jid}}{{end}} (WA)' }}"
-    # WhatsApp connection timeout in seconds.
-    connection_timeout: 20
-    # Maximum number of times to retry connecting on connection error.
-    max_connection_attempts: 3
-    # Number of seconds to wait between connection attempts.
-    # Negative numbers are exponential backoff: -connection_retry_delay + 1 + 2^attempts
-    connection_retry_delay: -1
-    # Whether or not the bridge should send a notice to the user's management room when it retries connecting.
-    # If false, it will only report when it stops retrying.
-    report_connection_retry: true
-    # Maximum number of seconds to wait for chats to be sent at startup.
-    # If this is too low and you have lots of chats, it could cause backfilling to fail.
-    chat_list_wait: 30
-    # Maximum number of seconds to wait to sync portals before force unlocking message processing.
-    # If this is too low and you have lots of chats, it could cause backfilling to fail.
-    portal_sync_wait: 600
+    displayname_template: "{{ '{{if .PushName}}{{.PushName}}{{else if .BusinessName}}{{.BusinessName}}{{else}}{{.JID}}{{end}} (WA)' }}"
 
-    # Whether or not to send call start/end notices to Matrix.
-    call_notices:
-        start: true
-        end: true
+    # Should the bridge send a read receipt from the bridge bot when a message has been sent to WhatsApp?
+    delivery_receipts: false
+    # Should incoming calls send a message to the Matrix room?
+    call_start_notices: true
+    # Should another user's cryptographic identity changing send a message to Matrix?
+    identity_change_notices: false
+    # Should a "reactions not yet supported" warning be sent to the Matrix room when a user reacts to a message?
+    reaction_notices: true
 
-    # Number of chats to sync for new users.
-    initial_chat_sync_count: 10
-    # Number of old messages to fill when creating new portal rooms.
-    initial_history_fill_count: 20
-    # Maximum number of chats to sync when recovering from downtime.
-    # Set to -1 to sync all new chats during downtime.
-    recovery_chat_sync_limit: -1
-    # Whether or not to sync history when recovering from downtime.
-    recovery_history_backfill: true
-    # Maximum number of seconds since last message in chat to skip
-    # syncing the chat in any case. This setting will take priority
-    # over both recovery_chat_sync_limit and initial_chat_sync_count.
-    # Default is 3 days = 259200 seconds
-    sync_max_chat_age: 259200
+    portal_message_buffer: 128
 
-    # Whether or not to sync with custom puppets to receive EDUs that
-    # are not normally sent to appservices.
+    # Settings for handling history sync payloads. These settings only apply right after login,
+    # because the phone only sends the history sync data once, and there's no way to re-request it
+    # (other than logging out and back in again).
+    history_sync:
+        # Should the bridge create portals for chats in the history sync payload?
+        create_portals: true
+        # Maximum age of chats in seconds to create portals for. Set to 0 to create portals for all chats in sync payload.
+        max_age: 604800
+        # Enable backfilling history sync payloads from WhatsApp using batch sending?
+        # This requires a server with MSC2716 support, which is currently an experimental feature in synapse.
+        # It can be enabled by setting experimental_features -> msc2716_enabled to true in homeserver.yaml.
+        # Note that as of Synapse 1.46, there are still some bugs with the implementation, especially if using event persistence workers.
+        backfill: {{ matrix_bridges_backfill_enabled }}
+        # Use double puppets for backfilling?
+        # In order to use this, the double puppets must be in the appservice's user ID namespace
+        # (because the bridge can't use the double puppet access token with batch sending).
+        # This only affects double puppets on the local server, double puppets on other servers will never be used.
+        double_puppet_backfill: {{ matrix_bridges_backfill_enabled }}
+        # Should the bridge request a full sync from the phone when logging in?
+        # This bumps the size of history syncs from 3 months to 1 year.
+        request_full_sync: false
+
+    user_avatar_sync: true
+    # Should Matrix users leaving groups be bridged to WhatsApp?
+    bridge_matrix_leave: true
+
+
+    # Should the bridge sync with double puppeting to receive EDUs that aren't normally sent to appservices.
     sync_with_custom_puppets: true
-    # Shared secret for https://github.com/devture/matrix-synapse-shared-secret-auth
+    # Should the bridge update the m.direct account data event when double puppeting is enabled.
+    # Note that updating the m.direct event is not atomic (except with mautrix-asmux)
+    # and is therefore prone to race conditions.
+    sync_direct_chat_list: false
+    # When double puppeting is enabled, users can use `!wa toggle` to change whether
+    # presence and read receipts are bridged. These settings set the default values.
+    # Existing users won't be affected when these are changed.
+    default_bridge_receipts: true
+    default_bridge_presence: true
+    # Servers to always allow double puppeting from
+    double_puppet_server_map: "{{ matrix_domain }}"
+
+    # Allow using double puppeting from any server with a valid client .well-known file.
+    double_puppet_allow_discovery: false
+    # Shared secrets for https://github.com/devture/matrix-synapse-shared-secret-auth
     #
-    # If set, custom puppets will be enabled automatically for local users
+    # If set, double puppeting will be enabled automatically for local users
     # instead of users having to find an access token and run `login-matrix`
     # manually.
-    login_shared_secret: {{ matrix_mautrix_whatsapp_login_shared_secret|to_json }}
+    login_shared_secret_map:
+        "{{ matrix_mautrix_whatsapp_homeserver_domain }}": {{ matrix_mautrix_whatsapp_login_shared_secret|to_json }}
 
-    # Whether or not to invite own WhatsApp user's Matrix puppet into private
-    # chat portals when backfilling if needed.
-    # This always uses the default puppet instead of custom puppets due to
-    # rate limits and timestamp massaging.
-    invite_own_puppet_for_backfilling: true
-    # Whether or not to explicitly set the avatar and room name for private
-    # chat portal rooms. This can be useful if the previous field works fine,
-    # but causes room avatar/name bugs.
+
+    # Should the bridge explicitly set the avatar and room name for private chat portal rooms?
     private_chat_portal_meta: false
-
+    # Should Matrix m.notice-type messages be bridged?
+    bridge_notices: true
+    # Set this to true to tell the bridge to re-send m.bridge events to all rooms on the next run.
+    # This field will automatically be changed back to false after it, except if the config file is not writable.
+    resend_bridge_info: false
+    # When using double puppeting, should muted chats be muted in Matrix?
+    mute_bridging: false
+    # When using double puppeting, should archived chats be moved to a specific tag in Matrix?
+    # Note that WhatsApp unarchives chats when a message is received, which will also be mirrored to Matrix.
+    # This can be set to a tag (e.g. m.lowpriority), or null to disable.
+    archive_tag: null
+    # Same as above, but for pinned chats. The favorite tag is called m.favourite
+    pinned_tag: null
+    # Should mute status and tags only be bridged when the portal room is created?
+    tag_only_on_create: true
+    # Should WhatsApp status messages be bridged into a Matrix room?
+    # Disabling this won't affect already created status broadcast rooms.
+    enable_status_broadcast: true
+    # Should the bridge use thumbnails from WhatsApp?
+    # They're disabled by default due to very low resolution.
+    whatsapp_thumbnail: false
     # Allow invite permission for user. User can invite any bots to room with whatsapp
     # users (private chat and groups)
     allow_user_invite: false
+    # Whether or not created rooms should have federation enabled.
+    # If false, created portal rooms will never be federated.
+    federate_rooms: true
 
     # The prefix for commands. Only required in non-management rooms.
     command_prefix: "!wa"
 
+    # Messages sent upon joining a management room.
+    # Markdown is supported. The defaults are listed below.
+    management_room_text:
+        # Sent when joining a room.
+        welcome: "Hello, I'm a WhatsApp bridge bot."
+        # Sent when joining a management room and the user is already logged in.
+        welcome_connected: "Use `help` for help."
+        # Sent when joining a management room and the user is not logged in.
+        welcome_unconnected: "Use `help` for help or `login` to log in."
+        # Optional extra text sent when joining a management room.
+        additional_help: ""
+
+    # End-to-bridge encryption support options.
+    #
+    # See https://docs.mau.fi/bridges/general/end-to-bridge-encryption.html for more info.
+    encryption:
+        # Allow encryption, work in group chat rooms with e2ee enabled
+        allow: false
+        # Default to encryption, force-enable encryption in all portals the bridge creates
+        # This will cause the bridge bot to be in private chats for the encryption to work properly.
+        # It is recommended to also set private_chat_portal_meta to true when using this.
+        default: false
+        # Options for automatic key sharing.
+        key_sharing:
+            # Enable key sharing? If enabled, key requests for rooms where users are in will be fulfilled.
+            # You must use a client that supports requesting keys from other users to use this feature.
+            allow: false
+            # Require the requesting device to have a valid cross-signing signature?
+            # This doesn't require that the bridge has verified the device, only that the user has verified it.
+            # Not yet implemented.
+            require_cross_signing: false
+            # Require devices to be verified by the bridge?
+            # Verification by the bridge is not yet implemented.
+            require_verification: true
+
     # Permissions for using the bridge.
     # Permitted values:
+    #    relay - Talk through the relaybot (if enabled), no access otherwise
     #     user - Access to use the bridge to chat with a WhatsApp account.
     #    admin - User level and some additional administration tools
     # Permitted keys:
@@ -133,15 +197,13 @@ bridge:
     permissions:
         "{{ matrix_mautrix_whatsapp_homeserver_domain }}": user
 
-    relaybot:
-        # Whether or not relaybot support is enabled.
+    # Settings for relay mode
+    relay:
+        # Whether relay mode should be allowed. If allowed, `!wa set-relay` can be used to turn any
+        # authenticated user into a relaybot for that chat.
         enabled: false
-        # The management room for the bot. This is where all status notifications are posted and
-        # in this room, you can use `!wa <command>` instead of `!wa relaybot <command>`. Omitting
-        # the command prefix completely like in user management rooms is not possible.
-        management: '!foo:example.com'
-        # List of users to invite to all created rooms that include the relaybot.
-        invites: []
+        # Should only admins be allowed to set themselves as relay users?
+        admin_only: true
         # The formats to use when sending messages to WhatsApp via the relaybot.
         message_formats:
             m.text: "<b>{{ '{{ .Sender.Displayname }}' }}</b>: {{ '{{ .Message }}' }}"
@@ -152,6 +214,7 @@ bridge:
             m.audio: "<b>{{ '{{ .Sender.Displayname }}' }}</b>: sent an audio file"
             m.video: "<b>{{ '{{ .Sender.Displayname }}' }}</b>: sent a video"
             m.location: "<b>{{ '{{ .Sender.Displayname }}' }}</b>: sent a location"
+
 # Logging config.
 logging:
     # The directory for log files. Will be created if not found.

roles/matrix-synapse/templates/synapse/homeserver.yaml.j2

@@ -2875,6 +2875,11 @@ opentracing:
 #  events: worker1
 #  typing: worker1
 
+{% if matrix_bridges_backfill_enabled %}
+Experimental:
+  msc2716_enabled: true
+{% endif %}
+
 # The worker that is used to run background tasks (e.g. cleaning up expired
 # data). If not provided this defaults to the main process.
 #