From efc78fb9d349b225a8ddb0523cc5b3a0e536011c Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 20 Feb 2018 21:36:08 +0200 Subject: [PATCH] Switch from s3fs to Goofys Improves performance of media store operations. --- README.md | 11 +-- roles/matrix-server/defaults/main.yml | 5 +- .../tasks/import_media_store.yml | 24 ++----- roles/matrix-server/tasks/main.yml | 5 ++ roles/matrix-server/tasks/setup_base.yml | 2 + roles/matrix-server/tasks/setup_goofys.yml | 70 +++++++++++++++++++ roles/matrix-server/tasks/setup_s3fs.yml | 31 +------- roles/matrix-server/tasks/setup_synapse.yml | 9 ++- roles/matrix-server/tasks/start.yml | 4 +- .../matrix-server/templates/env/env-goofys.j2 | 2 + .../systemd/matrix-goofys.service.j2 | 32 +++++++++ .../templates/systemd/matrix-s3fs.service.j2 | 35 ---------- .../systemd/matrix-synapse.service.j2 | 12 +++- 13 files changed, 144 insertions(+), 98 deletions(-) create mode 100644 roles/matrix-server/tasks/setup_goofys.yml create mode 100644 roles/matrix-server/templates/env/env-goofys.j2 create mode 100644 roles/matrix-server/templates/systemd/matrix-goofys.service.j2 delete mode 100644 roles/matrix-server/templates/systemd/matrix-s3fs.service.j2 diff --git a/README.md b/README.md index 27177dc8..c3c58929 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Using this playbook, you can get the following services configured on your serve - a [Matrix Synapse](https://github.com/matrix-org/synapse) homeserver - storing your data and managing your presence in the [Matrix](http://matrix.org/) network -- (optional) [Amazon S3](https://aws.amazon.com/s3/) storage for your Matrix Synapse's content repository (`media_store`) files using [s3fs-fuse](https://github.com/s3fs-fuse/s3fs-fuse) +- (optional) [Amazon S3](https://aws.amazon.com/s3/) storage for your Matrix Synapse's content repository (`media_store`) files using [Goofys](https://github.com/kahing/goofys) - (optional default) [PostgreSQL](https://www.postgresql.org/) database for Matrix Synapse - providing better performance than the default [SQLite](https://sqlite.org/) database. Using an external PostgreSQL server [is possible](#using-an-external-postgresql-server-optional) as well @@ -111,11 +111,11 @@ By default, this playbook configures your server to store Matrix Synapse's conte If that's alright, you can skip ahead. If you'd like to store Matrix Synapse's content repository (`media_store`) files on Amazon S3, -you can let this playbook configure [s3fs-fuse](https://github.com/s3fs-fuse/s3fs-fuse) for you. +you can let this playbook configure [Goofys](https://github.com/kahing/goofys) for you. You'll need an Amazon S3 bucket and some IAM user credentials (access key + secret key) with full write access to the bucket. Example security policy: -``` +```json { "Version": "2012-10-17", "Statement": [ @@ -137,11 +137,12 @@ You'll need an Amazon S3 bucket and some IAM user credentials (access key + secr You then need to enable S3 support in your configuration file (`inventory/matrix./vars.yml`). It would be something like this: -``` +```yaml matrix_s3_media_store_enabled: true matrix_s3_media_store_bucket_name: "your-bucket-name" matrix_s3_media_store_aws_access_key: "access-key-goes-here" matrix_s3_media_store_aws_secret_key: "secret-key-goes-here" +matrix_s3_media_store_region: "eu-central-1" ``` @@ -153,7 +154,7 @@ If that's alright, you can skip ahead. If you'd like to use an external PostgreSQL server that you manage, you can edit your configuration file (`inventory/matrix./vars.yml`). It should be something like this: -``` +```yaml matrix_postgres_use_external: true matrix_postgres_connection_hostname: "your-postgres-server-hostname" matrix_postgres_connection_username: "your-postgres-server-username" diff --git a/roles/matrix-server/defaults/main.yml b/roles/matrix-server/defaults/main.yml index 7b9ebf46..7008f110 100644 --- a/roles/matrix-server/defaults/main.yml +++ b/roles/matrix-server/defaults/main.yml @@ -30,7 +30,8 @@ matrix_environment_variables_data_path: "{{ matrix_base_data_path }}/environment matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse" matrix_synapse_config_dir_path: "{{ matrix_synapse_base_path }}/config" matrix_synapse_run_path: "{{ matrix_synapse_base_path }}/run" -matrix_synapse_media_store_path: "{{ matrix_synapse_base_path }}/media-store" +matrix_synapse_storage_path: "{{ matrix_synapse_base_path }}/storage" +matrix_synapse_media_store_path: "{{ matrix_synapse_storage_path }}/media-store" matrix_postgres_data_path: "{{ matrix_base_data_path }}/postgres" matrix_nginx_proxy_data_path: "{{ matrix_base_data_path }}/nginx-proxy" matrix_nginx_proxy_confd_path: "{{ matrix_nginx_proxy_data_path }}/conf.d" @@ -42,6 +43,7 @@ docker_matrix_image: "avhost/docker-matrix:v0.26.0" docker_nginx_image: "nginx:1.13.8-alpine" docker_riot_image: "avhost/docker-matrix-riot:latest" docker_s3fs_image: "xueshanf/s3fs:latest" +docker_goofys_image: "cloudproto/goofys:latest" # To avoid Synapse's macaroon secret key from changing every time # a new config is built from scratch, you can specify one here. @@ -61,6 +63,7 @@ matrix_s3_media_store_enabled: false matrix_s3_media_store_bucket_name: "your-bucket-name" matrix_s3_media_store_aws_access_key: "your-aws-access-key" matrix_s3_media_store_aws_secret_key: "your-aws-secret-key" +matrix_s3_media_store_region: "eu-central-1" # By default, this playbook installs the Riot.IM web UI on the `hostname_riot` domain. # If you wish to connect to your Matrix server by other means, diff --git a/roles/matrix-server/tasks/import_media_store.yml b/roles/matrix-server/tasks/import_media_store.yml index 7b191be4..13515b33 100644 --- a/roles/matrix-server/tasks/import_media_store.yml +++ b/roles/matrix-server/tasks/import_media_store.yml @@ -42,12 +42,11 @@ # It's wasteful to preserve owner/group now. We chown below anyway. owner: no group: no - # The default of times=yes does not work when s3fs is used. times: "{{ False if matrix_s3_media_store_enabled else True }}" perms: "{{ False if matrix_s3_media_store_enabled else True }}" -# This is for the generic case and fails for remote file systems, -# because the base path (matrix_synapse_media_store_path) is a mount point. +# This is for the generic case and fails in other cases (remote file systems), +# because in such cases the base path (matrix_synapse_media_store_path) is a mount point. - name: Ensure media store permissions are correct (generic case) file: path: "{{ matrix_synapse_media_store_path }}" @@ -56,23 +55,8 @@ recurse: yes when: "not matrix_s3_media_store_enabled" -- name: Determine media store subdirectories - find: paths="{{ local_path_media_store }}" file_type=directory - delegate_to: 127.0.0.1 - become: false - register: media_store_directories_result - when: "matrix_s3_media_store_enabled" - -# This is the s3fs special case. We chown the subdirectories one by one, -# without touching the base directory. -- name: Ensure media store permissions are correct (s3fs) - file: - path: "{{ matrix_synapse_media_store_path }}/{{ item.path|basename }}" - owner: "{{ matrix_user_username }}" - group: "{{ matrix_user_username }}" - recurse: yes - with_items: "{{ media_store_directories_result.files }}" - when: "matrix_s3_media_store_enabled" +# We don't chown for Goofys, because due to the way it's mounted, +# all files become owned by whoever needs to own them. - name: Ensure Matrix Synapse is started (if it previously was) service: name="{{ item }}" state=started daemon_reload=yes diff --git a/roles/matrix-server/tasks/main.yml b/roles/matrix-server/tasks/main.yml index 088bd495..8904cb53 100644 --- a/roles/matrix-server/tasks/main.yml +++ b/roles/matrix-server/tasks/main.yml @@ -23,6 +23,11 @@ - setup-main - setup-s3fs +- include: tasks/setup_goofys.yml + tags: + - setup-main + - setup-goofys + - include: tasks/setup_synapse.yml tags: - setup-main diff --git a/roles/matrix-server/tasks/setup_base.yml b/roles/matrix-server/tasks/setup_base.yml index 99065a37..04342536 100644 --- a/roles/matrix-server/tasks/setup_base.yml +++ b/roles/matrix-server/tasks/setup_base.yml @@ -25,6 +25,7 @@ - docker-python - firewalld - ntp + - fuse when: ansible_distribution == 'CentOS' - name: Ensure APT usage dependencies are installed (Debian) @@ -60,6 +61,7 @@ - docker-ce - python-docker - ntp + - fuse when: ansible_os_family == 'Debian' - name: Ensure firewalld is started and autoruns diff --git a/roles/matrix-server/tasks/setup_goofys.yml b/roles/matrix-server/tasks/setup_goofys.yml new file mode 100644 index 00000000..ab2a6365 --- /dev/null +++ b/roles/matrix-server/tasks/setup_goofys.yml @@ -0,0 +1,70 @@ +# +# Tasks related to setting up Goofys +# + +- name: Ensure Goofys Docker image is pulled + docker_image: + name: "{{ docker_goofys_image }}" + when: matrix_s3_media_store_enabled + +# This will throw a Permission Denied error if already mounted +- name: Check Matrix Goofys external storage mountpoint path + stat: path="{{ matrix_synapse_media_store_path }}" + register: local_path_matrix_synapse_media_store_path_stat + ignore_errors: yes + when: matrix_s3_media_store_enabled + +- name: Ensure Matrix Goofys external storage mountpoint exists + file: + path: "{{ matrix_synapse_media_store_path }}" + state: directory + mode: 0750 + owner: "{{ matrix_user_uid }}" + group: "{{ matrix_user_gid }}" + when: "matrix_s3_media_store_enabled and not local_path_matrix_synapse_media_store_path_stat.failed and not local_path_matrix_synapse_media_store_path_stat.stat.exists" + +- name: Ensure goofys environment variables file created + template: + src: "{{ role_path }}/templates/env/env-goofys.j2" + dest: "{{ matrix_environment_variables_data_path }}/goofys" + owner: root + mode: 0600 + when: matrix_s3_media_store_enabled + +- name: Ensure matrix-goofys.service installed + template: + src: "{{ role_path }}/templates/systemd/matrix-goofys.service.j2" + dest: "/etc/systemd/system/matrix-goofys.service" + mode: 0644 + when: matrix_s3_media_store_enabled + +# +# Tasks related to getting rid of goofys (if it was previously enabled) +# + +- name: Check existence of matrix-goofys service + stat: path="/etc/systemd/system/matrix-goofys.service" + register: matrix_goofys_service_stat + +- name: Ensure matrix-goofys is stopped + service: name=matrix-goofys state=stopped daemon_reload=yes + register: stopping_result + when: "not matrix_s3_media_store_enabled and matrix_goofys_service_stat.stat.exists" + +- name: Ensure matrix-goofys.service doesn't exist + file: + path: "/etc/systemd/system/matrix-goofys.service" + state: absent + when: "not matrix_s3_media_store_enabled and matrix_goofys_service_stat.stat.exists" + +- name: Ensure goofys environment variables file doesn't exist + file: + path: "{{ matrix_environment_variables_data_path }}/goofys" + state: absent + when: "not matrix_s3_media_store_enabled" + +- name: Ensure Goofys Docker image doesn't exist + docker_image: + name: "{{ docker_goofys_image }}" + state: absent + when: "not matrix_s3_media_store_enabled" diff --git a/roles/matrix-server/tasks/setup_s3fs.yml b/roles/matrix-server/tasks/setup_s3fs.yml index 15fe590b..49f64990 100644 --- a/roles/matrix-server/tasks/setup_s3fs.yml +++ b/roles/matrix-server/tasks/setup_s3fs.yml @@ -1,29 +1,5 @@ # -# Tasks related to setting up s3fs -# - -- name: Ensure S3fs Docker image is pulled - docker_image: - name: "{{ docker_s3fs_image }}" - when: matrix_s3_media_store_enabled - -- name: Ensure s3fs-credentials file created - template: - src: "{{ role_path }}/templates/s3fs-credentials.j2" - dest: "{{ matrix_base_data_path }}/s3fs-credentials" - owner: root - mode: 0600 - when: matrix_s3_media_store_enabled - -- name: Ensure matrix-s3fs.service installed - template: - src: "{{ role_path }}/templates/systemd/matrix-s3fs.service.j2" - dest: "/etc/systemd/system/matrix-s3fs.service" - mode: 0644 - when: matrix_s3_media_store_enabled - -# -# Tasks related to getting rid of s3fs (if it was previously enabled) +# Tasks related to getting rid of s3fs (if it was previously installed) # - name: Check existence of matrix-s3fs service @@ -33,22 +9,19 @@ - name: Ensure matrix-s3fs is stopped service: name=matrix-s3fs state=stopped daemon_reload=yes register: stopping_result - when: "not matrix_s3_media_store_enabled and matrix_s3fs_service_stat.stat.exists" + when: "matrix_s3fs_service_stat.stat.exists" - name: Ensure matrix-s3fs.service doesn't exist file: path: "/etc/systemd/system/matrix-s3fs.service" state: absent - when: "not matrix_s3_media_store_enabled and matrix_s3fs_service_stat.stat.exists" - name: Ensure s3fs-credentials doesn't exist file: path: "{{ matrix_base_data_path }}/s3fs-credentials" state: absent - when: "not matrix_s3_media_store_enabled" - name: Ensure S3fs Docker image doesn't exist docker_image: name: "{{ docker_s3fs_image }}" state: absent - when: "not matrix_s3_media_store_enabled" diff --git a/roles/matrix-server/tasks/setup_synapse.yml b/roles/matrix-server/tasks/setup_synapse.yml index 362dc047..2c67a676 100644 --- a/roles/matrix-server/tasks/setup_synapse.yml +++ b/roles/matrix-server/tasks/setup_synapse.yml @@ -10,16 +10,19 @@ with_items: - "{{ matrix_synapse_config_dir_path }}" - "{{ matrix_synapse_run_path }}" + - "{{ matrix_synapse_storage_path }}" # We handle matrix_synapse_media_store_path below, not here, # because if it's using S3fs and it's already mounted (from before), # trying to chown/chmod it here will cause trouble. +# This will throw a Permission Denied error if already mounted using fuse - name: Check Matrix Synapse media store path stat: path="{{ matrix_synapse_media_store_path }}" register: local_path_media_store_stat + ignore_errors: yes # This is separate and conditional, to ensure we don't execute it -# if the path already exists (and is likely used by an s3fs mount). +# if the path already exists or we failed to check, because it's mounted using fuse. - name: Ensure Matrix media store path exists file: path: "{{ matrix_synapse_media_store_path }}" @@ -27,7 +30,7 @@ mode: 0750 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_username }}" - when: "not local_path_media_store_stat.stat.exists" + when: "not local_path_media_store_stat.failed and not local_path_media_store_stat.stat.exists" - name: Ensure Matrix Docker image is pulled docker_image: @@ -74,7 +77,7 @@ - {"regexp": "^turn_allow_guests:", "line": 'turn_allow_guests: False'} - {"regexp": "^url_preview_enabled:", "line": 'url_preview_enabled: True'} - {"regexp": "^max_upload_size:", "line": 'max_upload_size: "{{ matrix_max_upload_size_mb }}M"'} - - {"regexp": "^media_store_path:", "line": 'media_store_path: "/matrix-media-store"'} + - {"regexp": "^media_store_path:", "line": 'media_store_path: "/matrix-storage/media-store"'} - name: Augment Matrix config (configure Macaroon secret) lineinfile: "dest={{ matrix_synapse_config_dir_path }}/homeserver.yaml" diff --git a/roles/matrix-server/tasks/start.yml b/roles/matrix-server/tasks/start.yml index 3b1cafb0..058f4c7a 100644 --- a/roles/matrix-server/tasks/start.yml +++ b/roles/matrix-server/tasks/start.yml @@ -4,8 +4,8 @@ service: name=matrix-postgres enabled=yes state=restarted daemon_reload=yes when: "not matrix_postgres_use_external" -- name: Ensure matrix-s3fs autoruns and is restarted - service: name=matrix-s3fs enabled=yes state=restarted daemon_reload=yes +- name: Ensure matrix-goofys autoruns and is restarted + service: name=matrix-goofys enabled=yes state=restarted daemon_reload=yes when: matrix_s3_media_store_enabled - name: Ensure matrix-synapse autoruns and is restarted diff --git a/roles/matrix-server/templates/env/env-goofys.j2 b/roles/matrix-server/templates/env/env-goofys.j2 new file mode 100644 index 00000000..a9e4b692 --- /dev/null +++ b/roles/matrix-server/templates/env/env-goofys.j2 @@ -0,0 +1,2 @@ +AWS_ACCESS_KEY={{ matrix_s3_media_store_aws_access_key }} +AWS_SECRET_KEY={{ matrix_s3_media_store_aws_secret_key }} \ No newline at end of file diff --git a/roles/matrix-server/templates/systemd/matrix-goofys.service.j2 b/roles/matrix-server/templates/systemd/matrix-goofys.service.j2 new file mode 100644 index 00000000..9107ca9a --- /dev/null +++ b/roles/matrix-server/templates/systemd/matrix-goofys.service.j2 @@ -0,0 +1,32 @@ +[Unit] +Description=Matrix Goofys media store +After=docker.service +Requires=docker.service + +[Service] +Type=simple +ExecStartPre=-/usr/bin/docker kill %n +ExecStartPre=-/usr/bin/docker rm %n +ExecStart=/usr/bin/docker run --rm --name %n \ + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + -v /etc/passwd:/etc/passwd:ro \ + -v /etc/group:/etc/group:ro \ + --security-opt apparmor:unconfined \ + --cap-add mknod \ + --cap-add sys_admin \ + --device=/dev/fuse \ + -v {{ matrix_synapse_media_store_path }}:/s3:shared \ + --env-file={{ matrix_environment_variables_data_path }}/goofys \ + --entrypoint /bin/sh \ + {{ docker_goofys_image }} \ + -c 'goofys -f --storage-class=STANDARD_IA --region {{ matrix_s3_media_store_region }} --stat-cache-ttl 60m0s --type-cache-ttl 60m0s --dir-mode 0700 --file-mode 0700 {{ matrix_s3_media_store_bucket_name }} /s3' +TimeoutStartSec=5min +ExecStop=-/usr/bin/docker stop %n +ExecStop=-/usr/bin/docker kill %n +ExecStop=-/usr/bin/docker rm %n +ExecStop=-/bin/fusermount -u {{ matrix_synapse_media_store_path }} +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/roles/matrix-server/templates/systemd/matrix-s3fs.service.j2 b/roles/matrix-server/templates/systemd/matrix-s3fs.service.j2 deleted file mode 100644 index 51212fb7..00000000 --- a/roles/matrix-server/templates/systemd/matrix-s3fs.service.j2 +++ /dev/null @@ -1,35 +0,0 @@ -[Unit] -Description=Matrix S3fs media store -After=docker.service -Requires=docker.service - -[Service] -Type=simple -ExecStartPre=-/usr/bin/docker kill %n -ExecStartPre=-/usr/bin/docker rm %n -ExecStartPre=-/usr/bin/mkdir /tmp/matrix-s3fs-cache -ExecStart=/usr/bin/docker run --rm --name %n \ - -v {{ matrix_base_data_path }}/s3fs-credentials:/s3fs-credentials \ - --security-opt apparmor:unconfined \ - --cap-add mknod \ - --cap-add sys_admin \ - --device=/dev/fuse \ - -v {{ matrix_synapse_media_store_path }}:/media-store:shared \ - -v /tmp/matrix-s3fs-cache:/s3fs-cache \ - {{ docker_s3fs_image }} \ - /usr/bin/s3fs -f \ - -o allow_other \ - -o use_cache=/s3fs-cache \ - -o storage_class=standard_ia \ - -o passwd_file=/s3fs-credentials \ - {{ matrix_s3_media_store_bucket_name }} /media-store -TimeoutStartSec=5min -ExecStop=-/usr/bin/docker stop %n -ExecStop=-/usr/bin/docker kill %n -ExecStop=-/usr/bin/docker rm %n -ExecStop=-/usr/bin/rm -rf /tmp/matrix-s3fs-cache -Restart=always -RestartSec=5 - -[Install] -WantedBy=multi-user.target diff --git a/roles/matrix-server/templates/systemd/matrix-synapse.service.j2 b/roles/matrix-server/templates/systemd/matrix-synapse.service.j2 index 717bd3a8..fd8ddb4a 100644 --- a/roles/matrix-server/templates/systemd/matrix-synapse.service.j2 +++ b/roles/matrix-server/templates/systemd/matrix-synapse.service.j2 @@ -7,14 +7,20 @@ Requires=matrix-postgres.service After=matrix-postgres.service {% endif %} {% if matrix_s3_media_store_enabled %} -After=matrix-s3fs.service -Requires=matrix-s3fs.service +After=matrix-goofys.service +Requires=matrix-goofys.service {% endif %} [Service] Type=simple ExecStartPre=-/usr/bin/docker kill matrix-synapse ExecStartPre=-/usr/bin/docker rm matrix-synapse +{% if matrix_s3_media_store_enabled %} +# Allow for some time before starting, so that media store can mount. +# Mounting can happen later too, but if we start writing, +# we'd write files to the local filesystem and fusermount will complain. +ExecStartPre=/bin/sleep 5 +{% endif %} ExecStart=/usr/bin/docker run --rm --name matrix-synapse \ {% if not matrix_postgres_use_external %} --link matrix-postgres:{{ matrix_postgres_connection_hostname }} \ @@ -28,7 +34,7 @@ ExecStart=/usr/bin/docker run --rm --name matrix-synapse \ -p {{ matrix_coturn_turn_udp_min_port }}-{{ matrix_coturn_turn_udp_max_port }}:{{ matrix_coturn_turn_udp_min_port }}-{{ matrix_coturn_turn_udp_max_port }}/udp \ -v {{ matrix_synapse_config_dir_path }}:/data \ -v {{ matrix_synapse_run_path }}:/matrix-run \ - -v {{ matrix_synapse_media_store_path }}:/matrix-media-store \ + -v {{ matrix_synapse_storage_path }}:/matrix-storage:slave \ {{ docker_matrix_image }} ExecStop=-/usr/bin/docker kill matrix-synapse ExecStop=-/usr/bin/docker rm matrix-synapse