Add mautrix-telegram to bridge with telegram services

examples/host-vars.yml

@@ -25,3 +25,10 @@ matrix_coturn_turn_static_auth_secret: ""
 # A secret used to protect access keys issued by the server.
 # You can put any string here, but generating a strong one is preferred (e.g. `pwgen -s 64 1`).
 matrix_synapse_macaroon_secret_key: ""
+
+# Mautrix telegram
+# Enable telegram bridge
+matrix_mautrix_telegram_enabled: false
+# Get your own API keys at https://my.telegram.org/apps
+matrix_mautrix_telegram_api_id: YOUR_TELEGRAM_APP_ID
+matrix_mautrix_telegram_api_hash: YOUR_TELEGRAM_API_HASH

roles/matrix-server/defaults/main.yml

@@ -84,6 +84,11 @@ matrix_synapse_container_additional_volumes: []
 # Contains definition objects like this: `{"name": "..", "level": "DEBUG"}
 matrix_synapse_additional_loggers: []
 
+# A list of service config files
+# This list gets populated dynamically based on Synapse extensions that have been enabled.
+# Conatains fs paths
+matrix_synapse_app_service_config_files: []
+
 # This is set dynamically during execution depending on whether
 # any password providers have been enabled or not.
 matrix_synapse_password_providers_enabled: false
@@ -110,6 +115,7 @@ matrix_nginx_riot_web_data_path: "{{ matrix_base_data_path }}/riot-web"
 matrix_coturn_base_path: "{{ matrix_base_data_path }}/coturn"
 matrix_coturn_config_path: "{{ matrix_coturn_base_path }}/turnserver.conf"
 matrix_scratchpad_dir: "{{ matrix_base_data_path }}/scratchpad"
+matrix_mautrix_telegram_base_path: "{{ matrix_base_data_path }}/mautrix-telegram"
 
 matrix_docker_image_postgres_v9: "postgres:9.6.10-alpine"
 matrix_docker_image_postgres_v10: "postgres:10.5-alpine"
@@ -122,6 +128,7 @@ matrix_docker_image_goofys: "cloudproto/goofys:latest"
 matrix_docker_image_coturn: "instrumentisto/coturn:4.5.0.7"
 matrix_docker_image_mailer: "panubo/postfix:latest"
 matrix_docker_image_mxisd: "kamax/mxisd:1.1.1"
+matrix_docker_image_mautrix_telegram: "tulir/mautrix-telegram:v0.3.0"
 
 # The Docker network that all services would be put into
 matrix_docker_network: "matrix"
@@ -213,6 +220,10 @@ matrix_riot_web_enabled: true
 matrix_riot_web_default_identity_server_url: "https://{{ matrix_synapse_trusted_third_party_id_servers[0] }}"
 
 
+# Matrix mautrix is a Matrix <-> Telegram bridge
+matrix_mautrix_telegram_enabled: false
+
+
 # By default, this playbook sets up its own nginx proxy server on port 80/443.
 # This is fine if you're dedicating the whole server to Matrix.
 # But in case that's not the case, you may wish to prevent that

roles/matrix-server/tasks/setup_synapse_ext.yml

@@ -3,3 +3,5 @@
 - include: tasks/setup_synapse_ext_rest_auth.yml
 
 - include: tasks/setup_synapse_ext_shared_secret_auth.yml
+
+- include: tasks/setup_synapse_ext_mautrix_telegram.yml

roles/matrix-server/tasks/setup_synapse_ext_mautrix_telegram.yml

@@ -0,0 +1,65 @@
+---
+
+- name: Ensure Mautrix Telegram image is pulled
+  docker_image:
+    name: "{{ matrix_docker_image_mautrix_telegram }}"
+  when: "matrix_mautrix_telegram_enabled"
+
+- name: Ensure Mautrix Telegram configuration path exists
+  file:
+    path: "{{ matrix_mautrix_telegram_base_path }}"
+    state: directory
+    mode: 0750
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_username }}"
+  when: "matrix_mautrix_telegram_enabled"
+
+- name: Ensure Matrix Mautrix telegram config installed
+  template:
+    src: "{{ role_path }}/templates/mautrix-telegram/config.yaml.j2"
+    dest: "{{ matrix_mautrix_telegram_base_path }}/config.yaml"
+    mode: 0644
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_username }}"
+  when: "matrix_mautrix_telegram_enabled"
+
+- name: Ensure matrix-mautrix-telegram.service installed
+  template:
+    src: "{{ role_path }}/templates/systemd/matrix-mautrix-telegram.service.j2"
+    dest: "/etc/systemd/system/matrix-mautrix-telegram.service"
+    mode: 0644
+  when: "matrix_mautrix_telegram_enabled"
+
+- stat: "path={{ matrix_mautrix_telegram_base_path }}/registration.yaml"
+  register: mautrix_registration_file
+
+- name: Generate matrix-mautrix-telegram registration.yaml if it doesn't exist
+  shell: /usr/bin/docker run --rm --name matrix-mautrix-telegram-gen -v {{ matrix_mautrix_telegram_base_path }}:/data:z {{ matrix_docker_image_mautrix_telegram }} python3 -m mautrix_telegram -g -c /data/config.yaml -r /data/registration.yaml
+  when: "matrix_mautrix_telegram_enabled and mautrix_registration_file.stat.exists == False"
+
+- set_fact:
+    matrix_synapse_app_service_config_file_mautrix_telegram: '/app-registration/mautrix-telegram.yml'
+
+- set_fact:
+    matrix_synapse_container_additional_volumes: >
+      {{ matrix_synapse_container_additional_volumes }}
+      +
+      {{ [{'src': '{{ matrix_mautrix_telegram_base_path }}/registration.yaml', 'dst': '{{ matrix_synapse_app_service_config_file_mautrix_telegram }}', 'options': 'ro'}] }}
+  when: "matrix_mautrix_telegram_enabled"
+
+- set_fact:
+    matrix_synapse_app_service_config_files: >
+      {{ matrix_synapse_app_service_config_files }}
+      +
+      {{ ["{{ matrix_synapse_app_service_config_file_mautrix_telegram }}"] | to_nice_json  }}
+  when: "matrix_mautrix_telegram_enabled"
+
+#
+# Tasks related to getting rid of matrix-mautrix-telegram (if it was previously enabled)
+#
+
+- name: Ensure matrix-mautrix-telegram.service doesn't exist
+  file:
+    path: "{{ role_path }}/templates/systemd/matrix-mautrix-telegram.service.j2"
+    state: absent
+  when: "not matrix_mautrix_telegram_enabled"

roles/matrix-server/tasks/start.yml

@@ -33,3 +33,7 @@
 - name: Ensure matrix-corporal autoruns and is restarted
   service: name=matrix-corporal enabled=yes state=restarted daemon_reload=yes
   when: matrix_corporal_enabled
+
+- name: Ensure matrix-mautrix-telegram autoruns and is restarted
+  service: name=matrix-mautrix-telegram enabled=yes state=restarted daemon_reload=yes
+  when: matrix_mautrix_telegram_enabled

roles/matrix-server/templates/mautrix-telegram/config.yaml.j2

@@ -0,0 +1,273 @@
+# Homeserver details
+homeserver:
+    # The address that this appservice can use to connect to the homeserver.
+    address: https://{{ hostname_matrix }}
+    # The domain of the homeserver (for MXIDs, etc).
+    domain: {{ hostname_identity }}
+    # Whether or not to verify the SSL certificate of the homeserver.
+    # Only applies if address starts with https://
+    verify_ssl: true
+
+# Application service host/registration related details
+# Changing these values requires regeneration of the registration.
+appservice:
+    # The address that the homeserver can use to connect to this appservice.
+    address: http://matrix-mautrix-telegram:8080
+
+    # The hostname and port where this appservice should listen.
+    hostname: 0.0.0.0
+    port: 8080
+    # The maximum body size of appservice API requests (from the homeserver) in mebibytes
+    # Usually 1 is enough, but on high-traffic bridges you might need to increase this to avoid 413s
+    max_body_size: 1
+
+    # The full URI to the database. SQLite and Postgres are fully supported.
+    # Other DBMSes supported by SQLAlchemy may or may not work.
+    # Format examples:
+    #   SQLite:   sqlite:///filename.db
+    #   Postgres: postgres://username:password@hostname/dbname
+    database: sqlite:///mautrix-telegram.db
+
+    # Public part of web server for out-of-Matrix interaction with the bridge.
+    # Used for things like login if the user wants to make sure the 2FA password isn't stored in
+    # the HS database.
+    public:
+        # Whether or not the public-facing endpoints should be enabled.
+        enabled: true
+        # The prefix to use in the public-facing endpoints.
+        prefix: /public
+        # The base URL where the public-facing endpoints are available. The prefix is not added
+        # implicitly.
+        external: https://{{ hostname_matrix }}/public
+
+    # Provisioning API part of the web server for automated portal creation and fetching information.
+    # Used by things like Dimension (https://dimension.t2bot.io/).
+    provisioning:
+        # Whether or not the provisioning API should be enabled.
+        enabled: false
+        # The prefix to use in the provisioning API endpoints.
+        prefix: /_matrix/provision/v1
+        # The shared secret to authorize users of the API.
+        # Set to "generate" to generate and save a new token.
+        shared_secret: generate
+
+    # The unique ID of this appservice.
+    id: telegram
+    # Username of the appservice bot.
+    bot_username: telegrambot
+    # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty
+    # to leave display name/avatar as-is.
+    bot_displayname: Telegram bridge bot
+    bot_avatar: mxc://maunium.net/tJCRmUyJDsgRNgqhOgoiHWbX
+
+    # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
+    as_token: "This value is generated when generating the registration"
+    hs_token: "This value is generated when generating the registration"
+
+# Bridge config
+bridge:
+    # Localpart template of MXIDs for Telegram users.
+    # {userid} is replaced with the user ID of the Telegram user.
+    username_template: "telegram_{userid}"
+    # Localpart template of room aliases for Telegram portal rooms.
+    # {groupname} is replaced with the name part of the public channel/group invite link ( https://t.me/{} )
+    alias_template: "telegram_{groupname}"
+    # Displayname template for Telegram users.
+    # {displayname} is replaced with the display name of the Telegram user.
+    displayname_template: "{displayname} (Telegram)"
+
+    # Set the preferred order of user identifiers which to use in the Matrix puppet display name.
+    # In the (hopefully unlikely) scenario that none of the given keys are found, the numeric user
+    # ID is used.
+    #
+    # If the bridge is working properly, a phone number or an username should always be known, but
+    # the other one can very well be empty.
+    #
+    # Valid keys:
+    #   "full name"          (First and/or last name)
+    #   "full name reversed" (Last and/or first name)
+    #   "first name"
+    #   "last name"
+    #   "username"
+    #   "phone number"
+    displayname_preference:
+    - full name
+    - username
+    - phone number
+
+    # Show message editing as a reply to the original message.
+    # If this is false, message edits are not shown at all, as Matrix does not support editing yet.
+    edits_as_replies: false
+    # Highlight changed/added parts in edits. Requires lxml.
+    highlight_edits: false
+    # Whether or not Matrix bot messages (type m.notice) should be bridged.
+    bridge_notices: true
+    # Whether to bridge Telegram bot messages as m.notices or m.texts.
+    bot_messages_as_notices: true
+    # Maximum number of members to sync per portal when starting up. Other members will be
+    # synced when they send messages. The maximum is 10000, after which the Telegram server
+    # will not send any more members.
+    # Defaults to no local limit (-> limited to 10000 by server)
+    max_initial_member_sync: -1
+    # Whether or not to sync the member list in channels.
+    # If no channel admins have logged into the bridge, the bridge won't be able to sync the member
+    # list regardless of this setting.
+    sync_channel_members: true
+    # The maximum number of simultaneous Telegram deletions to handle.
+    # A large number of simultaneous redactions could put strain on your homeserver.
+    max_telegram_delete: 10
+    # Allow logging in within Matrix. If false, the only way to log in is using the out-of-Matrix
+    # login website (see appservice.public config section)
+    allow_matrix_login: true
+    # Use inline images instead of m.image to make rich captions possible.
+    # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
+    inline_images: true
+    # Whether or not to bridge plaintext highlights.
+    # Only enable this if your displayname_template has some static part that the bridge can use to
+    # reliably identify what is a plaintext highlight.
+    plaintext_highlights: false
+    # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
+    public_portals: true
+    # Whether to send stickers as the new native m.sticker type or normal m.images.
+    # Old versions of Riot don't support the new type at all.
+    # Remember that proper sticker support always requires Pillow to convert webp into png.
+    native_stickers: true
+    # Whether or not to fetch and handle Telegram updates at startup from the time the bridge was down.
+    # WARNING: Probably buggy, might get stuck in infinite loop.
+    catch_up: false
+    # Whether or not to use /sync to get presence, read receipts and typing notifications when using
+    # your own Matrix account as the Matrix puppet for your Telegram account.
+    sync_with_custom_puppets: true
+
+    # Some config options related to Telegram message deduplication.
+    # The default values are usually fine, but some debug messages/warnings might recommend you
+    # change these.
+    deduplication:
+        # Whether or not to check the database if the message about to be sent is a duplicate.
+        pre_db_check: false
+        # The number of latest events to keep when checking for duplicates.
+        # You might need to increase this on high-traffic bridge instances.
+        cache_queue_length: 20
+
+    # The formats to use when sending messages to Telegram via the relay bot.
+    #
+    # Telegram doesn't have built-in emotes, so the m.emote format is also used for non-relaybot users.
+    #
+    # Available variables:
+    #   $sender_displayname    - The display name of the sender (e.g. Example User)
+    #   $sender_username       - The username (Matrix ID localpart) of the sender (e.g. exampleuser)
+    #   $sender_mxid           - The Matrix ID of the sender (e.g. @exampleuser:example.com)
+    #   $message               - The message content as HTML
+    message_formats:
+        m.text: "<b>$sender_displayname</b>: $message"
+        m.emote: "* <b>$sender_displayname</b> $message"
+        m.file: "<b>$sender_displayname</b> sent a file: $message"
+        m.image: "<b>$sender_displayname</b> sent an image: $message"
+        m.audio: "<b>$sender_displayname</b> sent an audio file: $message"
+        m.video: "<b>$sender_displayname</b> sent a video: $message"
+        m.location: "<b>$sender_displayname</b> sent a location: $message"
+
+    # The formats to use when sending state events to Telegram via the relay bot.
+    #
+    # Variables from `message_formats` that have the `sender_` prefix are available without the prefix.
+    # In name_change events, `$prev_displayname` is the previous displayname.
+    #
+    # Set format to an empty string to disable the messages for that event.
+    state_event_formats:
+        join: "<b>$displayname</b> joined the room."
+        leave: "<b>$displayname</b> left the room."
+        name_change: "<b>$prev_displayname</b> changed their name to <b>$displayname</b>"
+
+    # Filter rooms that can/can't be bridged. Can also be managed using the `filter` and
+    # `filter-mode` management commands.
+    #
+    # Filters do not affect direct chats.
+    # An empty blacklist will essentially disable the filter.
+    filter:
+        # Filter mode to use. Either "blacklist" or "whitelist".
+        # If the mode is "blacklist", the listed chats will never be bridged.
+        # If the mode is "whitelist", only the listed chats can be bridged.
+        mode: blacklist
+        # The list of group/channel IDs to filter.
+        list: []
+
+    # The prefix for commands. Only required in non-management rooms.
+    command_prefix: "!tg"
+
+    # Permissions for using the bridge.
+    # Permitted values:
+    #   relaybot - Only use the bridge via the relaybot, no access to commands.
+    #       user - Relaybot level + access to commands to create bridges.
+    #  puppeting - User level + logging in with a Telegram account.
+    #       full - Full access to use the bridge, i.e. previous levels + Matrix login.
+    #      admin - Full access to use the bridge and some extra administration commands.
+    # Permitted keys:
+    #        * - All Matrix users
+    #   domain - All users on that homeserver
+    #     mxid - Specific user
+    permissions:
+        "*": "puppeting"
+
+    # Options related to the message relay Telegram bot.
+    relaybot:
+        # Whether or not to allow creating portals from Telegram.
+        authless_portals: false
+        # Whether or not to allow Telegram group admins to use the bot commands.
+        whitelist_group_admins: false
+        # Whether or not to ignore incoming events sent by the relay bot.
+        ignore_own_incoming_events: true
+        # List of usernames/user IDs who are also allowed to use the bot commands.
+        whitelist:
+        - myusername
+        - 12345678
+
+# Telegram config
+telegram:
+    # Get your own API keys at https://my.telegram.org/apps
+    api_id: {{ matrix_mautrix_telegram_api_id }}
+    api_hash: {{ matrix_mautrix_telegram_api_hash }}
+    # (Optional) Create your own bot at https://t.me/BotFather
+    bot_token: disabled
+    # Telethon proxy configuration.
+    # You must install PySocks from pip for proxies to work.
+    proxy:
+        # Allowed types: disabled, socks4, socks5, http
+        type: disabled
+        # Proxy IP address and port.
+        address: 127.0.0.1
+        port: 1080
+        # Whether or not to perform DNS resolving remotely.
+        rdns: true
+        # Proxy authentication (optional).
+        username: ""
+        password: ""
+
+# Python logging configuration.
+#
+# See section 16.7.2 of the Python documentation for more info:
+# https://docs.python.org/3.6/library/logging.config.html#configuration-dictionary-schema
+logging:
+    version: 1
+    formatters:
+        precise:
+            format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s"
+    handlers:
+        file:
+            class: logging.handlers.RotatingFileHandler
+            formatter: precise
+            filename: ./mautrix-telegram.log
+            maxBytes: 10485760
+            backupCount: 10
+        console:
+            class: logging.StreamHandler
+            formatter: precise
+    loggers:
+        mau:
+            level: DEBUG
+        telethon:
+            level: DEBUG
+        aiohttp:
+            level: INFO
+    root:
+        level: DEBUG
+        handlers: [file, console]

roles/matrix-server/templates/synapse/homeserver.yaml.j2

@@ -461,7 +461,7 @@ room_invite_state_types:
 
 
 # A list of application service config file to use
-app_service_config_files: []
+app_service_config_files: {{ matrix_synapse_app_service_config_files }}
 
 
 macaroon_secret_key: "{{ matrix_synapse_macaroon_secret_key }}"

roles/matrix-server/templates/systemd/matrix-mautrix-telegram.service.j2

@@ -0,0 +1,23 @@
+[Unit]
+Description=Matrix Mautrix Telegram server
+After=docker.service
+Requires=docker.service
+Requires=matrix-synapse.service
+After=matrix-synapse.service
+
+[Service]
+Type=simple
+ExecStartPre=-/usr/bin/docker kill matrix-mautrix-telegram
+ExecStartPre=-/usr/bin/docker rm matrix-mautrix-telegram
+ExecStart=/usr/bin/docker run --rm --name matrix-mautrix-telegram \
+			-e "UID={{ matrix_user_uid }}" -e "GID={{ matrix_user_gid }}" \
+			--network={{ matrix_docker_network }} \
+			-v {{ matrix_mautrix_telegram_base_path }}:/data:z \
+			{{ matrix_docker_image_mautrix_telegram }}
+ExecStop=-/usr/bin/docker kill matrix-mautrix-telegram
+ExecStop=-/usr/bin/docker rm matrix-mautrix-telegram
+Restart=always
+RestartSec=30
+
+[Install]
+WantedBy=multi-user.target