Switch from s3fs to Goofys

Improves performance of media store operations.
This commit is contained in:
Slavi Pantaleev 2018-02-20 21:36:08 +02:00
parent db686c3f8e
commit efc78fb9d3
13 changed files with 144 additions and 98 deletions

View file

@ -10,7 +10,7 @@ Using this playbook, you can get the following services configured on your serve
- a [Matrix Synapse](https://github.com/matrix-org/synapse) homeserver - storing your data and managing your presence in the [Matrix](http://matrix.org/) network
- (optional) [Amazon S3](https://aws.amazon.com/s3/) storage for your Matrix Synapse's content repository (`media_store`) files using [s3fs-fuse](https://github.com/s3fs-fuse/s3fs-fuse)
- (optional) [Amazon S3](https://aws.amazon.com/s3/) storage for your Matrix Synapse's content repository (`media_store`) files using [Goofys](https://github.com/kahing/goofys)
- (optional default) [PostgreSQL](https://www.postgresql.org/) database for Matrix Synapse - providing better performance than the default [SQLite](https://sqlite.org/) database. Using an external PostgreSQL server [is possible](#using-an-external-postgresql-server-optional) as well
@ -111,11 +111,11 @@ By default, this playbook configures your server to store Matrix Synapse's conte
If that's alright, you can skip ahead.
If you'd like to store Matrix Synapse's content repository (`media_store`) files on Amazon S3,
you can let this playbook configure [s3fs-fuse](https://github.com/s3fs-fuse/s3fs-fuse) for you.
you can let this playbook configure [Goofys](https://github.com/kahing/goofys) for you.
You'll need an Amazon S3 bucket and some IAM user credentials (access key + secret key) with full write access to the bucket. Example security policy:
```
```json
{
"Version": "2012-10-17",
"Statement": [
@ -137,11 +137,12 @@ You'll need an Amazon S3 bucket and some IAM user credentials (access key + secr
You then need to enable S3 support in your configuration file (`inventory/matrix.<your-domain>/vars.yml`).
It would be something like this:
```
```yaml
matrix_s3_media_store_enabled: true
matrix_s3_media_store_bucket_name: "your-bucket-name"
matrix_s3_media_store_aws_access_key: "access-key-goes-here"
matrix_s3_media_store_aws_secret_key: "secret-key-goes-here"
matrix_s3_media_store_region: "eu-central-1"
```
@ -153,7 +154,7 @@ If that's alright, you can skip ahead.
If you'd like to use an external PostgreSQL server that you manage, you can edit your configuration file (`inventory/matrix.<your-domain>/vars.yml`).
It should be something like this:
```
```yaml
matrix_postgres_use_external: true
matrix_postgres_connection_hostname: "your-postgres-server-hostname"
matrix_postgres_connection_username: "your-postgres-server-username"

View file

@ -30,7 +30,8 @@ matrix_environment_variables_data_path: "{{ matrix_base_data_path }}/environment
matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse"
matrix_synapse_config_dir_path: "{{ matrix_synapse_base_path }}/config"
matrix_synapse_run_path: "{{ matrix_synapse_base_path }}/run"
matrix_synapse_media_store_path: "{{ matrix_synapse_base_path }}/media-store"
matrix_synapse_storage_path: "{{ matrix_synapse_base_path }}/storage"
matrix_synapse_media_store_path: "{{ matrix_synapse_storage_path }}/media-store"
matrix_postgres_data_path: "{{ matrix_base_data_path }}/postgres"
matrix_nginx_proxy_data_path: "{{ matrix_base_data_path }}/nginx-proxy"
matrix_nginx_proxy_confd_path: "{{ matrix_nginx_proxy_data_path }}/conf.d"
@ -42,6 +43,7 @@ docker_matrix_image: "avhost/docker-matrix:v0.26.0"
docker_nginx_image: "nginx:1.13.8-alpine"
docker_riot_image: "avhost/docker-matrix-riot:latest"
docker_s3fs_image: "xueshanf/s3fs:latest"
docker_goofys_image: "cloudproto/goofys:latest"
# To avoid Synapse's macaroon secret key from changing every time
# a new config is built from scratch, you can specify one here.
@ -61,6 +63,7 @@ matrix_s3_media_store_enabled: false
matrix_s3_media_store_bucket_name: "your-bucket-name"
matrix_s3_media_store_aws_access_key: "your-aws-access-key"
matrix_s3_media_store_aws_secret_key: "your-aws-secret-key"
matrix_s3_media_store_region: "eu-central-1"
# By default, this playbook installs the Riot.IM web UI on the `hostname_riot` domain.
# If you wish to connect to your Matrix server by other means,

View file

@ -42,12 +42,11 @@
# It's wasteful to preserve owner/group now. We chown below anyway.
owner: no
group: no
# The default of times=yes does not work when s3fs is used.
times: "{{ False if matrix_s3_media_store_enabled else True }}"
perms: "{{ False if matrix_s3_media_store_enabled else True }}"
# This is for the generic case and fails for remote file systems,
# because the base path (matrix_synapse_media_store_path) is a mount point.
# This is for the generic case and fails in other cases (remote file systems),
# because in such cases the base path (matrix_synapse_media_store_path) is a mount point.
- name: Ensure media store permissions are correct (generic case)
file:
path: "{{ matrix_synapse_media_store_path }}"
@ -56,23 +55,8 @@
recurse: yes
when: "not matrix_s3_media_store_enabled"
- name: Determine media store subdirectories
find: paths="{{ local_path_media_store }}" file_type=directory
delegate_to: 127.0.0.1
become: false
register: media_store_directories_result
when: "matrix_s3_media_store_enabled"
# This is the s3fs special case. We chown the subdirectories one by one,
# without touching the base directory.
- name: Ensure media store permissions are correct (s3fs)
file:
path: "{{ matrix_synapse_media_store_path }}/{{ item.path|basename }}"
owner: "{{ matrix_user_username }}"
group: "{{ matrix_user_username }}"
recurse: yes
with_items: "{{ media_store_directories_result.files }}"
when: "matrix_s3_media_store_enabled"
# We don't chown for Goofys, because due to the way it's mounted,
# all files become owned by whoever needs to own them.
- name: Ensure Matrix Synapse is started (if it previously was)
service: name="{{ item }}" state=started daemon_reload=yes

View file

@ -23,6 +23,11 @@
- setup-main
- setup-s3fs
- include: tasks/setup_goofys.yml
tags:
- setup-main
- setup-goofys
- include: tasks/setup_synapse.yml
tags:
- setup-main

View file

@ -25,6 +25,7 @@
- docker-python
- firewalld
- ntp
- fuse
when: ansible_distribution == 'CentOS'
- name: Ensure APT usage dependencies are installed (Debian)
@ -60,6 +61,7 @@
- docker-ce
- python-docker
- ntp
- fuse
when: ansible_os_family == 'Debian'
- name: Ensure firewalld is started and autoruns

View file

@ -0,0 +1,70 @@
#
# Tasks related to setting up Goofys
#
- name: Ensure Goofys Docker image is pulled
docker_image:
name: "{{ docker_goofys_image }}"
when: matrix_s3_media_store_enabled
# This will throw a Permission Denied error if already mounted
- name: Check Matrix Goofys external storage mountpoint path
stat: path="{{ matrix_synapse_media_store_path }}"
register: local_path_matrix_synapse_media_store_path_stat
ignore_errors: yes
when: matrix_s3_media_store_enabled
- name: Ensure Matrix Goofys external storage mountpoint exists
file:
path: "{{ matrix_synapse_media_store_path }}"
state: directory
mode: 0750
owner: "{{ matrix_user_uid }}"
group: "{{ matrix_user_gid }}"
when: "matrix_s3_media_store_enabled and not local_path_matrix_synapse_media_store_path_stat.failed and not local_path_matrix_synapse_media_store_path_stat.stat.exists"
- name: Ensure goofys environment variables file created
template:
src: "{{ role_path }}/templates/env/env-goofys.j2"
dest: "{{ matrix_environment_variables_data_path }}/goofys"
owner: root
mode: 0600
when: matrix_s3_media_store_enabled
- name: Ensure matrix-goofys.service installed
template:
src: "{{ role_path }}/templates/systemd/matrix-goofys.service.j2"
dest: "/etc/systemd/system/matrix-goofys.service"
mode: 0644
when: matrix_s3_media_store_enabled
#
# Tasks related to getting rid of goofys (if it was previously enabled)
#
- name: Check existence of matrix-goofys service
stat: path="/etc/systemd/system/matrix-goofys.service"
register: matrix_goofys_service_stat
- name: Ensure matrix-goofys is stopped
service: name=matrix-goofys state=stopped daemon_reload=yes
register: stopping_result
when: "not matrix_s3_media_store_enabled and matrix_goofys_service_stat.stat.exists"
- name: Ensure matrix-goofys.service doesn't exist
file:
path: "/etc/systemd/system/matrix-goofys.service"
state: absent
when: "not matrix_s3_media_store_enabled and matrix_goofys_service_stat.stat.exists"
- name: Ensure goofys environment variables file doesn't exist
file:
path: "{{ matrix_environment_variables_data_path }}/goofys"
state: absent
when: "not matrix_s3_media_store_enabled"
- name: Ensure Goofys Docker image doesn't exist
docker_image:
name: "{{ docker_goofys_image }}"
state: absent
when: "not matrix_s3_media_store_enabled"

View file

@ -1,29 +1,5 @@
#
# Tasks related to setting up s3fs
#
- name: Ensure S3fs Docker image is pulled
docker_image:
name: "{{ docker_s3fs_image }}"
when: matrix_s3_media_store_enabled
- name: Ensure s3fs-credentials file created
template:
src: "{{ role_path }}/templates/s3fs-credentials.j2"
dest: "{{ matrix_base_data_path }}/s3fs-credentials"
owner: root
mode: 0600
when: matrix_s3_media_store_enabled
- name: Ensure matrix-s3fs.service installed
template:
src: "{{ role_path }}/templates/systemd/matrix-s3fs.service.j2"
dest: "/etc/systemd/system/matrix-s3fs.service"
mode: 0644
when: matrix_s3_media_store_enabled
#
# Tasks related to getting rid of s3fs (if it was previously enabled)
# Tasks related to getting rid of s3fs (if it was previously installed)
#
- name: Check existence of matrix-s3fs service
@ -33,22 +9,19 @@
- name: Ensure matrix-s3fs is stopped
service: name=matrix-s3fs state=stopped daemon_reload=yes
register: stopping_result
when: "not matrix_s3_media_store_enabled and matrix_s3fs_service_stat.stat.exists"
when: "matrix_s3fs_service_stat.stat.exists"
- name: Ensure matrix-s3fs.service doesn't exist
file:
path: "/etc/systemd/system/matrix-s3fs.service"
state: absent
when: "not matrix_s3_media_store_enabled and matrix_s3fs_service_stat.stat.exists"
- name: Ensure s3fs-credentials doesn't exist
file:
path: "{{ matrix_base_data_path }}/s3fs-credentials"
state: absent
when: "not matrix_s3_media_store_enabled"
- name: Ensure S3fs Docker image doesn't exist
docker_image:
name: "{{ docker_s3fs_image }}"
state: absent
when: "not matrix_s3_media_store_enabled"

View file

@ -10,16 +10,19 @@
with_items:
- "{{ matrix_synapse_config_dir_path }}"
- "{{ matrix_synapse_run_path }}"
- "{{ matrix_synapse_storage_path }}"
# We handle matrix_synapse_media_store_path below, not here,
# because if it's using S3fs and it's already mounted (from before),
# trying to chown/chmod it here will cause trouble.
# This will throw a Permission Denied error if already mounted using fuse
- name: Check Matrix Synapse media store path
stat: path="{{ matrix_synapse_media_store_path }}"
register: local_path_media_store_stat
ignore_errors: yes
# This is separate and conditional, to ensure we don't execute it
# if the path already exists (and is likely used by an s3fs mount).
# if the path already exists or we failed to check, because it's mounted using fuse.
- name: Ensure Matrix media store path exists
file:
path: "{{ matrix_synapse_media_store_path }}"
@ -27,7 +30,7 @@
mode: 0750
owner: "{{ matrix_user_username }}"
group: "{{ matrix_user_username }}"
when: "not local_path_media_store_stat.stat.exists"
when: "not local_path_media_store_stat.failed and not local_path_media_store_stat.stat.exists"
- name: Ensure Matrix Docker image is pulled
docker_image:
@ -74,7 +77,7 @@
- {"regexp": "^turn_allow_guests:", "line": 'turn_allow_guests: False'}
- {"regexp": "^url_preview_enabled:", "line": 'url_preview_enabled: True'}
- {"regexp": "^max_upload_size:", "line": 'max_upload_size: "{{ matrix_max_upload_size_mb }}M"'}
- {"regexp": "^media_store_path:", "line": 'media_store_path: "/matrix-media-store"'}
- {"regexp": "^media_store_path:", "line": 'media_store_path: "/matrix-storage/media-store"'}
- name: Augment Matrix config (configure Macaroon secret)
lineinfile: "dest={{ matrix_synapse_config_dir_path }}/homeserver.yaml"

View file

@ -4,8 +4,8 @@
service: name=matrix-postgres enabled=yes state=restarted daemon_reload=yes
when: "not matrix_postgres_use_external"
- name: Ensure matrix-s3fs autoruns and is restarted
service: name=matrix-s3fs enabled=yes state=restarted daemon_reload=yes
- name: Ensure matrix-goofys autoruns and is restarted
service: name=matrix-goofys enabled=yes state=restarted daemon_reload=yes
when: matrix_s3_media_store_enabled
- name: Ensure matrix-synapse autoruns and is restarted

View file

@ -0,0 +1,2 @@
AWS_ACCESS_KEY={{ matrix_s3_media_store_aws_access_key }}
AWS_SECRET_KEY={{ matrix_s3_media_store_aws_secret_key }}

View file

@ -0,0 +1,32 @@
[Unit]
Description=Matrix Goofys media store
After=docker.service
Requires=docker.service
[Service]
Type=simple
ExecStartPre=-/usr/bin/docker kill %n
ExecStartPre=-/usr/bin/docker rm %n
ExecStart=/usr/bin/docker run --rm --name %n \
--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
-v /etc/passwd:/etc/passwd:ro \
-v /etc/group:/etc/group:ro \
--security-opt apparmor:unconfined \
--cap-add mknod \
--cap-add sys_admin \
--device=/dev/fuse \
-v {{ matrix_synapse_media_store_path }}:/s3:shared \
--env-file={{ matrix_environment_variables_data_path }}/goofys \
--entrypoint /bin/sh \
{{ docker_goofys_image }} \
-c 'goofys -f --storage-class=STANDARD_IA --region {{ matrix_s3_media_store_region }} --stat-cache-ttl 60m0s --type-cache-ttl 60m0s --dir-mode 0700 --file-mode 0700 {{ matrix_s3_media_store_bucket_name }} /s3'
TimeoutStartSec=5min
ExecStop=-/usr/bin/docker stop %n
ExecStop=-/usr/bin/docker kill %n
ExecStop=-/usr/bin/docker rm %n
ExecStop=-/bin/fusermount -u {{ matrix_synapse_media_store_path }}
Restart=always
RestartSec=5
[Install]
WantedBy=multi-user.target

View file

@ -1,35 +0,0 @@
[Unit]
Description=Matrix S3fs media store
After=docker.service
Requires=docker.service
[Service]
Type=simple
ExecStartPre=-/usr/bin/docker kill %n
ExecStartPre=-/usr/bin/docker rm %n
ExecStartPre=-/usr/bin/mkdir /tmp/matrix-s3fs-cache
ExecStart=/usr/bin/docker run --rm --name %n \
-v {{ matrix_base_data_path }}/s3fs-credentials:/s3fs-credentials \
--security-opt apparmor:unconfined \
--cap-add mknod \
--cap-add sys_admin \
--device=/dev/fuse \
-v {{ matrix_synapse_media_store_path }}:/media-store:shared \
-v /tmp/matrix-s3fs-cache:/s3fs-cache \
{{ docker_s3fs_image }} \
/usr/bin/s3fs -f \
-o allow_other \
-o use_cache=/s3fs-cache \
-o storage_class=standard_ia \
-o passwd_file=/s3fs-credentials \
{{ matrix_s3_media_store_bucket_name }} /media-store
TimeoutStartSec=5min
ExecStop=-/usr/bin/docker stop %n
ExecStop=-/usr/bin/docker kill %n
ExecStop=-/usr/bin/docker rm %n
ExecStop=-/usr/bin/rm -rf /tmp/matrix-s3fs-cache
Restart=always
RestartSec=5
[Install]
WantedBy=multi-user.target

View file

@ -7,14 +7,20 @@ Requires=matrix-postgres.service
After=matrix-postgres.service
{% endif %}
{% if matrix_s3_media_store_enabled %}
After=matrix-s3fs.service
Requires=matrix-s3fs.service
After=matrix-goofys.service
Requires=matrix-goofys.service
{% endif %}
[Service]
Type=simple
ExecStartPre=-/usr/bin/docker kill matrix-synapse
ExecStartPre=-/usr/bin/docker rm matrix-synapse
{% if matrix_s3_media_store_enabled %}
# Allow for some time before starting, so that media store can mount.
# Mounting can happen later too, but if we start writing,
# we'd write files to the local filesystem and fusermount will complain.
ExecStartPre=/bin/sleep 5
{% endif %}
ExecStart=/usr/bin/docker run --rm --name matrix-synapse \
{% if not matrix_postgres_use_external %}
--link matrix-postgres:{{ matrix_postgres_connection_hostname }} \
@ -28,7 +34,7 @@ ExecStart=/usr/bin/docker run --rm --name matrix-synapse \
-p {{ matrix_coturn_turn_udp_min_port }}-{{ matrix_coturn_turn_udp_max_port }}:{{ matrix_coturn_turn_udp_min_port }}-{{ matrix_coturn_turn_udp_max_port }}/udp \
-v {{ matrix_synapse_config_dir_path }}:/data \
-v {{ matrix_synapse_run_path }}:/matrix-run \
-v {{ matrix_synapse_media_store_path }}:/matrix-media-store \
-v {{ matrix_synapse_storage_path }}:/matrix-storage:slave \
{{ docker_matrix_image }}
ExecStop=-/usr/bin/docker kill matrix-synapse
ExecStop=-/usr/bin/docker rm matrix-synapse