Put s3-storage-provider scripts under /matrix/synapse/ext/s3-storage-provider/bin, not /usr/local/bin

This commit is contained in:
Slavi Pantaleev 2022-11-27 09:34:46 +02:00
parent 735e7355a5
commit 7c5c3aedc4
8 changed files with 25 additions and 27 deletions

View file

@ -62,7 +62,7 @@ Migrating your existing data can happen in multiple ways:
Instead of using `s3_media_upload` directly, which is very slow and painful for an initial data migration, we recommend [using another tool in combination with `s3_media_upload`](#using-another-tool-in-combination-with-s3_media_upload).
To copy your existing files, SSH into the server and run `/usr/local/bin/matrix-synapse-s3-storage-provider-shell`.
To copy your existing files, SSH into the server and run `/matrix/synapse/ext/s3-storage-provider/bin/shell`.
This launches a Synapse container, which has access to the local media store, Postgres database, S3 store and has some convenient environment variables configured for you to use (`MEDIA_PATH`, `BUCKET`, `ENDPOINT`, `UPDATE_DB_DAYS`, etc).
@ -76,12 +76,12 @@ Then use the following commands (`$` values come from environment variables - th
The `s3_media_upload upload` command may take a lot of time to complete.
Instead of running the above commands manually in the shell, you can also run the `/usr/local/bin/matrix-synapse-s3-storage-provider-migrate` script which will run the same commands automatically. We demonstrate how to do it manually, because:
Instead of running the above commands manually in the shell, you can also run the `/matrix/synapse/ext/s3-storage-provider/bin/migrate` script which will run the same commands automatically. We demonstrate how to do it manually, because:
- it's what the upstream project demonstrates and it teaches you how to use the `s3_media_upload` tool
- allows you to check and verify the output of each command, to catch mistakes
- includes progress bars and detailed output for each command
- allows you to easily interrupt slow-running commands, etc. (the `/usr/local/bin/matrix-synapse-s3-storage-provider-migrate` starts a container without interactive TTY support, so `Ctrl+C` may not work and you and require killing via `docker kill ..`)
- allows you to easily interrupt slow-running commands, etc. (the `/matrix/synapse/ext/s3-storage-provider/bin/migrate` starts a container without interactive TTY support, so `Ctrl+C` may not work and you and require killing via `docker kill ..`)
### Using another tool in combination with `s3_media_upload`
@ -119,7 +119,7 @@ As described in [How it works?](#how-it-works) above, when new media is uploaded
By default, we periodically ensure that all local files are uploaded to S3 and are then removed from the local filesystem. This is done automatically using:
- the `/usr/local/bin/matrix-synapse-s3-storage-provider-migrate` script
- the `/matrix/synapse/ext/s3-storage-provider/bin/migrate` script
- .. invoked via the `matrix-synapse-s3-storage-provider-migrate.service` service
- .. triggered by the `matrix-synapse-s3-storage-provider-migrate.timer` timer, every day at 05:00

View file

@ -60,6 +60,7 @@ matrix_synapse_bin_path: "{{ matrix_synapse_base_path }}/bin"
matrix_synapse_ext_path: "{{ matrix_synapse_base_path }}/ext"
matrix_synapse_ext_s3_storage_provider_base_path: "{{ matrix_synapse_base_path }}/ext/s3-storage-provider"
matrix_synapse_ext_s3_storage_provider_bin_path: "{{ matrix_synapse_ext_s3_storage_provider_base_path }}/bin"
matrix_synapse_container_client_api_port: 8008

View file

@ -7,20 +7,23 @@
#
# Below are additional tasks for setting up various helper scripts, etc.
- name: Ensure s3-storage-provider paths exists
ansible.builtin.file:
path: "{{ item }}"
state: directory
mode: 0750
owner: "{{ matrix_user_username }}"
group: "{{ matrix_user_groupname }}"
with_items:
- "{{ matrix_synapse_ext_s3_storage_provider_base_path }}"
- "{{ matrix_synapse_ext_s3_storage_provider_bin_path }}"
- name: Ensure s3-storage-provider env file installed
ansible.builtin.template:
src: "{{ role_path }}/templates/synapse/ext/s3-storage-provider/env.j2"
dest: "{{ matrix_synapse_ext_s3_storage_provider_base_path }}/env"
mode: 0640
- name: Ensure s3-storage-provider data path exists
ansible.builtin.file:
path: "{{ matrix_synapse_ext_s3_storage_provider_base_path }}/data"
state: directory
mode: 0750
owner: "{{ matrix_user_username }}"
group: "{{ matrix_user_groupname }}"
- name: Ensure s3-storage-provider database.yaml file installed
ansible.builtin.template:
src: "{{ role_path }}/templates/synapse/ext/s3-storage-provider/database.yaml.j2"
@ -29,12 +32,12 @@
- name: Ensure s3-storage-provider scripts installed
ansible.builtin.template:
src: "{{ role_path }}/templates/synapse/ext/s3-storage-provider/usr-local-bin/{{ item }}.j2"
dest: "{{ matrix_local_bin_path }}/{{ item }}"
src: "{{ role_path }}/templates/synapse/ext/s3-storage-provider/bin/{{ item }}.j2"
dest: "{{ matrix_synapse_ext_s3_storage_provider_bin_path }}/{{ item }}"
mode: 0750
with_items:
- matrix-synapse-s3-storage-provider-shell
- matrix-synapse-s3-storage-provider-migrate
- shell
- migrate
- name: Ensure matrix-synapse-s3-storage-provider-migrate.service and timer are installed
ansible.builtin.template:
@ -45,8 +48,3 @@
- matrix-synapse-s3-storage-provider-migrate.service
- matrix-synapse-s3-storage-provider-migrate.timer
register: matrix_synapse_s3_storage_provider_systemd_service_result
- name: Ensure systemd reloaded after matrix-synapse-s3-storage-provider-migrate.service installation
ansible.builtin.service:
daemon_reload: true
when: matrix_synapse_s3_storage_provider_systemd_service_result.changed | bool

View file

@ -11,9 +11,5 @@
- name: Ensure s3-storage-provider files don't exist
ansible.builtin.file:
path: "{{ item }}"
path: "{{ matrix_synapse_ext_s3_storage_provider_base_path }}"
state: absent
with_items:
- "{{ matrix_local_bin_path }}/matrix-synapse-s3-storage-provider-shell"
- "{{ matrix_local_bin_path }}/matrix-synapse-s3-storage-provider-migrate"
- "{{ matrix_synapse_ext_s3_storage_provider_base_path }}"

View file

@ -13,7 +13,10 @@
- {path: "{{ matrix_synapse_bin_path }}", when: true}
- {path: "{{ matrix_synapse_docker_src_files_path }}", when: "{{ matrix_synapse_container_image_self_build }}"}
- {path: "{{ matrix_synapse_customized_docker_src_files_path }}", when: "{{ matrix_synapse_container_image_customizations_enabled }}"}
- {path: "{{ matrix_synapse_ext_s3_storage_provider_base_path }}", when: "{{ matrix_synapse_ext_synapse_s3_storage_provider_enabled }}"}
- {path: "{{ matrix_synapse_ext_s3_storage_provider_bin_path }}", when: "{{ matrix_synapse_ext_synapse_s3_storage_provider_enabled }}"}
# We handle matrix_synapse_media_store_path elsewhere (in ./synapse/setup_install.yml),
# because if it's using Goofys and it's already mounted (from before),
# trying to chown/chmod it here will cause trouble.

View file

@ -4,4 +4,4 @@ Description=Migrates locally-stored Synapse media store files to S3
[Service]
Type=oneshot
Environment="HOME={{ devture_systemd_docker_base_systemd_unit_home_path }}"
ExecStart={{ matrix_local_bin_path }}/matrix-synapse-s3-storage-provider-migrate
ExecStart={{ matrix_synapse_ext_s3_storage_provider_bin_path }}/migrate