From 9c68b057b0616c65948257869accaa5c771258be Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 7 Sep 2017 18:26:41 +0300 Subject: [PATCH] Add support for storing Matrix Synapse's media_store to Amazon S3 --- README.md | 44 ++++++++++++++++- roles/matrix-server/defaults/main.yml | 12 +++-- .../tasks/import_media_store.yml | 26 +++++++++- roles/matrix-server/tasks/main.yml | 5 ++ roles/matrix-server/tasks/setup_s3fs.yml | 49 +++++++++++++++++++ roles/matrix-server/tasks/setup_synapse.yml | 19 ++++++- roles/matrix-server/tasks/start.yml | 4 ++ .../templates/s3fs-credentials.j2 | 1 + .../templates/systemd/matrix-s3fs.service.j2 | 35 +++++++++++++ .../systemd/matrix-synapse.service.j2 | 4 ++ 10 files changed, 192 insertions(+), 7 deletions(-) create mode 100644 roles/matrix-server/tasks/setup_s3fs.yml create mode 100644 roles/matrix-server/templates/s3fs-credentials.j2 create mode 100644 roles/matrix-server/templates/systemd/matrix-s3fs.service.j2 diff --git a/README.md b/README.md index 8fc23d42..42971461 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,13 @@ Using this playbook, you can get the following services configured on your serve - a [Matrix Synapse](https://github.com/matrix-org/synapse) homeserver - storing your data and managing your presence in the [Matrix](http://matrix.org/) network +- (optional) [Amazon S3](https://aws.amazon.com/s3/) storage for your Matrix Synapse's content repository (`media_store`) files using [s3fs-fuse](https://github.com/s3fs-fuse/s3fs-fuse) + - a [PostgreSQL](https://www.postgresql.org/) database for Matrix Synapse - providing better performance than the default [SQLite](https://sqlite.org/) database -- a [STUN server](https://github.com/coturn/coturn) for WebRTC audio/video calls +- a [STUN/TURN server](https://github.com/coturn/coturn) for WebRTC audio/video calls -- a [Riot](https://riot.im/) web UI +- a [Riot](https://riot.im/) web UI, which is configured to connect to your own Matrix Synapse server by default - free [Let's Encrypt](https://letsencrypt.org/) SSL certificate, which secures the connection to the Synapse server and the Riot web UI @@ -33,6 +35,8 @@ This is similar to the [EMnify/matrix-synapse-auto-deploy](https://github.com/EM - this one retrieves and automatically renews free [Let's Encrypt](https://letsencrypt.org/) **SSL certificates** for you +- this one optionally can store the `media_store` content repository files on [Amazon S3](https://aws.amazon.com/s3/) + Special thanks goes to: - [EMnify/matrix-synapse-auto-deploy](https://github.com/EMnify/matrix-synapse-auto-deploy) - for the inspiration @@ -91,6 +95,42 @@ You can follow these steps: - edit the inventory hosts file (`inventory/hosts`) to your liking +## Amazon S3 configuration (optional) + +If you'd like to store Matrix Synapse's content repository (`media_store`) files on Amazon S3, +you can let this playbook configure [s3fs-fuse](https://github.com/s3fs-fuse/s3fs-fuse) for you. + +You'll need an Amazon S3 bucket and some IAM user credentials (access key + secret key) with full write access to the bucket. Example security policy: + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "Stmt1400105486000", + "Effect": "Allow", + "Action": [ + "s3:*" + ], + "Resource": [ + "arn:aws:s3:::your-bucket-name", + "arn:aws:s3:::your-bucket-name/*" + ] + } + ] +} +``` + +You then need to enable S3 support in your configuration file (`inventory/matrix./vars.yml`). +It would be something like this: + +``` +matrix_s3_media_store_bucket_name: "your-bucket-name" +matrix_s3_media_store_aws_access_key: "access-key-goes-here" +matrix_s3_media_store_aws_secret_key: "secret-key-goes-here" +``` + + ## Installing Once you have your server and you have [configured your DNS records](#configuring-dns), you can proceed with installing. diff --git a/roles/matrix-server/defaults/main.yml b/roles/matrix-server/defaults/main.yml index 9f298373..47910501 100644 --- a/roles/matrix-server/defaults/main.yml +++ b/roles/matrix-server/defaults/main.yml @@ -35,9 +35,10 @@ matrix_nginx_riot_web_data_path: "{{ matrix_base_data_path }}/riot-web" matrix_scratchpad_dir: "{{ matrix_base_data_path }}/scratchpad" docker_postgres_image: "postgres:9.6.3-alpine" -docker_matrix_image: "silviof/docker-matrix" +docker_matrix_image: "silviof/docker-matrix:latest" docker_nginx_image: "nginx:1.13.3-alpine" -docker_riot_image: "silviof/matrix-riot-docker" +docker_riot_image: "silviof/matrix-riot-docker:latest" +docker_s3fs_image: "xueshanf/s3fs:latest" # Specifies when to restart the Matrix services so that # a new SSL certificate could go into effect (UTC time). @@ -51,4 +52,9 @@ matrix_coturn_turn_external_ip_address: "{{ ansible_host }}" matrix_max_upload_size_mb: 10 matrix_max_log_file_size_mb: 100 -matrix_max_log_files_count: 10 \ No newline at end of file +matrix_max_log_files_count: 10 + +matrix_s3_media_store_enabled: false +matrix_s3_media_store_bucket_name: "your-bucket-name" +matrix_s3_media_store_aws_access_key: "your-aws-access-key" +matrix_s3_media_store_aws_secret_key: "your-aws-secret-key" \ No newline at end of file diff --git a/roles/matrix-server/tasks/import_media_store.yml b/roles/matrix-server/tasks/import_media_store.yml index f6cd95f9..7b191be4 100644 --- a/roles/matrix-server/tasks/import_media_store.yml +++ b/roles/matrix-server/tasks/import_media_store.yml @@ -42,13 +42,37 @@ # It's wasteful to preserve owner/group now. We chown below anyway. owner: no group: no + # The default of times=yes does not work when s3fs is used. + times: "{{ False if matrix_s3_media_store_enabled else True }}" + perms: "{{ False if matrix_s3_media_store_enabled else True }}" -- name: Ensure media store permissions are correct +# This is for the generic case and fails for remote file systems, +# because the base path (matrix_synapse_media_store_path) is a mount point. +- name: Ensure media store permissions are correct (generic case) file: path: "{{ matrix_synapse_media_store_path }}" owner: "{{ matrix_user_username }}" group: "{{ matrix_user_username }}" recurse: yes + when: "not matrix_s3_media_store_enabled" + +- name: Determine media store subdirectories + find: paths="{{ local_path_media_store }}" file_type=directory + delegate_to: 127.0.0.1 + become: false + register: media_store_directories_result + when: "matrix_s3_media_store_enabled" + +# This is the s3fs special case. We chown the subdirectories one by one, +# without touching the base directory. +- name: Ensure media store permissions are correct (s3fs) + file: + path: "{{ matrix_synapse_media_store_path }}/{{ item.path|basename }}" + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_username }}" + recurse: yes + with_items: "{{ media_store_directories_result.files }}" + when: "matrix_s3_media_store_enabled" - name: Ensure Matrix Synapse is started (if it previously was) service: name="{{ item }}" state=started daemon_reload=yes diff --git a/roles/matrix-server/tasks/main.yml b/roles/matrix-server/tasks/main.yml index 25cad895..9d54d86c 100644 --- a/roles/matrix-server/tasks/main.yml +++ b/roles/matrix-server/tasks/main.yml @@ -1,5 +1,10 @@ --- +- include: tasks/setup_s3fs.yml + tags: + - setup-main + - setup-s3fs + - include: tasks/setup_base.yml tags: - setup-main diff --git a/roles/matrix-server/tasks/setup_s3fs.yml b/roles/matrix-server/tasks/setup_s3fs.yml new file mode 100644 index 00000000..f3312195 --- /dev/null +++ b/roles/matrix-server/tasks/setup_s3fs.yml @@ -0,0 +1,49 @@ +# +# Tasks related to setting up s3fs +# + +- name: Ensure S3fs Docker image is pulled + docker_image: + name: "{{ docker_s3fs_image }}" + when: matrix_s3_media_store_enabled + +- name: Ensure s3fs-credentials file created + template: + src: "{{ role_path }}/templates/s3fs-credentials.j2" + dest: "{{ matrix_base_data_path }}/s3fs-credentials" + owner: root + mode: 0600 + when: matrix_s3_media_store_enabled + +- name: Ensure matrix-s3fs.service installed + template: + src: "{{ role_path }}/templates/systemd/matrix-s3fs.service.j2" + dest: "/etc/systemd/system/matrix-s3fs.service" + mode: 0644 + when: matrix_s3_media_store_enabled + +# +# Tasks related to getting rid of s3fs (if it was previously enabled) +# +- name: Ensure matrix-s3fs is stopped + service: name=matrix-s3fs state=stopped daemon_reload=yes + register: stopping_result + when: "not matrix_s3_media_store_enabled" + +- name: Ensure matrix-s3fs.service doesn't exist + file: + path: "{{ matrix_base_data_path }}/s3fs-credentials" + state: absent + when: "not matrix_s3_media_store_enabled" + +- name: Ensure s3fs-credentials doesn't exist + file: + path: "{{ matrix_base_data_path }}/s3fs-credentials" + state: absent + when: "not matrix_s3_media_store_enabled" + +- name: Ensure S3fs Docker image doesn't exist + docker_image: + name: "{{ docker_s3fs_image }}" + state: absent + when: "not matrix_s3_media_store_enabled" \ No newline at end of file diff --git a/roles/matrix-server/tasks/setup_synapse.yml b/roles/matrix-server/tasks/setup_synapse.yml index 91329ba5..e53dbc50 100644 --- a/roles/matrix-server/tasks/setup_synapse.yml +++ b/roles/matrix-server/tasks/setup_synapse.yml @@ -11,7 +11,24 @@ - "{{ matrix_synapse_base_path }}" - "{{ matrix_synapse_config_dir_path }}" - "{{ matrix_synapse_run_path }}" - - "{{ matrix_synapse_media_store_path }}" + # We handle matrix_synapse_media_store_path below, not here, + # because if it's using S3fs and it's already mounted (from before), + # trying to chown/chmod it here will cause trouble. + +- name: Check Matrix Synapse media store path + stat: path="{{ matrix_synapse_media_store_path }}" + register: local_path_media_store_stat + +# This is separate and conditional, to ensure we don't execute it +# if the path already exists (and is likely used by an s3fs mount). +- name: Ensure Matrix media store path exists + file: + path: "{{ matrix_synapse_media_store_path }}" + state: directory + mode: 0750 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_username }}" + when: "not local_path_media_store_stat.stat.exists" - name: Ensure Matrix Docker image is pulled docker_image: diff --git a/roles/matrix-server/tasks/start.yml b/roles/matrix-server/tasks/start.yml index 31ec3abc..110244ee 100644 --- a/roles/matrix-server/tasks/start.yml +++ b/roles/matrix-server/tasks/start.yml @@ -3,6 +3,10 @@ - name: Ensure matrix-postgres autoruns and is restarted service: name=matrix-postgres enabled=yes state=restarted daemon_reload=yes +- name: Ensure matrix-s3fs autoruns and is restarted + service: name=matrix-s3fs enabled=yes state=restarted daemon_reload=yes + when: matrix_s3_media_store_enabled + - name: Ensure matrix-synapse autoruns and is restarted service: name=matrix-synapse enabled=yes state=restarted daemon_reload=yes diff --git a/roles/matrix-server/templates/s3fs-credentials.j2 b/roles/matrix-server/templates/s3fs-credentials.j2 new file mode 100644 index 00000000..51e9365f --- /dev/null +++ b/roles/matrix-server/templates/s3fs-credentials.j2 @@ -0,0 +1 @@ +{{ matrix_s3_media_store_aws_access_key }}:{{ matrix_s3_media_store_aws_secret_key }} \ No newline at end of file diff --git a/roles/matrix-server/templates/systemd/matrix-s3fs.service.j2 b/roles/matrix-server/templates/systemd/matrix-s3fs.service.j2 new file mode 100644 index 00000000..51212fb7 --- /dev/null +++ b/roles/matrix-server/templates/systemd/matrix-s3fs.service.j2 @@ -0,0 +1,35 @@ +[Unit] +Description=Matrix S3fs media store +After=docker.service +Requires=docker.service + +[Service] +Type=simple +ExecStartPre=-/usr/bin/docker kill %n +ExecStartPre=-/usr/bin/docker rm %n +ExecStartPre=-/usr/bin/mkdir /tmp/matrix-s3fs-cache +ExecStart=/usr/bin/docker run --rm --name %n \ + -v {{ matrix_base_data_path }}/s3fs-credentials:/s3fs-credentials \ + --security-opt apparmor:unconfined \ + --cap-add mknod \ + --cap-add sys_admin \ + --device=/dev/fuse \ + -v {{ matrix_synapse_media_store_path }}:/media-store:shared \ + -v /tmp/matrix-s3fs-cache:/s3fs-cache \ + {{ docker_s3fs_image }} \ + /usr/bin/s3fs -f \ + -o allow_other \ + -o use_cache=/s3fs-cache \ + -o storage_class=standard_ia \ + -o passwd_file=/s3fs-credentials \ + {{ matrix_s3_media_store_bucket_name }} /media-store +TimeoutStartSec=5min +ExecStop=-/usr/bin/docker stop %n +ExecStop=-/usr/bin/docker kill %n +ExecStop=-/usr/bin/docker rm %n +ExecStop=-/usr/bin/rm -rf /tmp/matrix-s3fs-cache +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/roles/matrix-server/templates/systemd/matrix-synapse.service.j2 b/roles/matrix-server/templates/systemd/matrix-synapse.service.j2 index c1f7b1f6..8752f1c3 100644 --- a/roles/matrix-server/templates/systemd/matrix-synapse.service.j2 +++ b/roles/matrix-server/templates/systemd/matrix-synapse.service.j2 @@ -4,6 +4,10 @@ After=docker.service Requires=docker.service Requires=matrix-postgres.service After=matrix-postgres.service +{% if matrix_s3_media_store_enabled %} +After=matrix-s3fs.service +Requires=matrix-s3fs.service +{% endif %} [Service] Type=simple