Merge pull request #868 from foxcris/postgres-backup

- Added a postgres-backup role
This commit is contained in:
Slavi Pantaleev 2021-04-05 10:53:04 +03:00 committed by GitHub
commit 0a107dc0ce
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 329 additions and 0 deletions

View file

@ -0,0 +1,22 @@
# Setting up postgres backup (optional)
The playbook can install and configure [docker-postgres-backup-local](https://github.com/prodrigestivill/docker-postgres-backup-local) for you.
## Adjusting the playbook configuration
| Name | Default value | Description |
| :-------------------------------- | :--------------------------- | :--------------------------------------------------------------- |
|matrix_postgres_backup_enabled|false|Set to true to use [docker-postgres-backup-local](https://github.com/prodrigestivill/docker-postgres-backup-local) to create automatic database backups|
|matrix_postgres_backup_schedule| '@daily' |Cron-schedule specifying the interval between postgres backups.|
|matrix_postgres_backup_keep_days|"7"|Number of daily backups to keep|
|matrix_postgres_backup_keep_weeks|"4"|Number of weekly backups to keep|
|matrix_postgres_backup_keep_months|"12"|Number of monthly backups to keep|
|matrix_postgres_backup_path | "{{ matrix_base_data_path }}/postgres-backup" | Storagepath for the database backups|
## Installing
After configuring the playbook, run the [installation](installing.md) command again:
```
ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start
```

View file

@ -1774,3 +1774,30 @@ matrix_registration_database_password: "{{ matrix_synapse_macaroon_secret_key |
# /matrix-registration
#
######################################################################
######################################################################
#
# matrix-postgres-backup
#
######################################################################
matrix_postgres_backup_connection_hostname: "{{ matrix_postgres_connection_hostname }}"
matrix_postgres_backup_connection_port: "{{ matrix_postgres_connection_port }}"
matrix_postgres_backup_connection_username: "{{ matrix_postgres_connection_username }}"
matrix_postgres_backup_connection_password: "{{ matrix_postgres_connection_password }}"
# the default matrix synapse databse is not always part of the matrix_postgres_additional_databases variable thus we have to add it if the default database is used
matrix_postgres_backup_databases: |
{{
(([{
'name': matrix_synapse_database_database
}] if (matrix_synapse_enabled and matrix_synapse_database_database == matrix_postgres_db_name and matrix_synapse_database_host == 'matrix-postgres') else [])
+
matrix_postgres_additional_databases)|map(attribute='name')|list
}}
######################################################################
#
# /matrix-postgres-backup
#
######################################################################

View file

@ -0,0 +1,38 @@
matrix_postgres_backup_enabled: false
matrix_postgres_backup_connection_hostname: "matrix-postgres"
matrix_postgres_backup_connection_port: 5432
matrix_postgres_backup_connection_username: "matrix"
matrix_postgres_backup_connection_password: ""
matrix_postgres_backup_extra_opts: "-Z9 --schema=public --blobs"
matrix_postgres_backup_schedule: "@daily"
matrix_postgres_backup_keep_days: "7"
matrix_postgres_backup_keep_weeks: "4"
matrix_postgres_backup_keep_months: "12"
matrix_postgres_backup_healthcheck_port: "8080"
matrix_postgres_backup_databases: []
matrix_postgres_backup_path: "{{ matrix_base_data_path }}/postgres-backup"
matrix_postgres_base_path: "{{ matrix_base_data_path }}/postgres"
matrix_postgres_data_path: "{{ matrix_postgres_base_path }}/data"
matrix_postgres_backup_architecture: amd64
# matrix_postgres_docker_image_suffix controls whether we use Alpine-based images (`-alpine`) or the normal Debian-based images.
# Alpine-based Postgres images are smaller and we usually prefer them, but they don't work on ARM32 (tested on a Raspberry Pi 3 running Raspbian 10.7).
# On ARM32, `-alpine` images fail with the following error:
# > LOG: startup process (PID 37) was terminated by signal 11: Segmentation fault
matrix_postgres_backup_docker_image_suffix: "{{ '-alpine' if matrix_postgres_backup_architecture in ['amd64', 'arm64'] else '' }}"
matrix_postgres_backup_docker_image_v9: "docker.io/prodrigestivill/postgres-backup-local:9.6{{ matrix_postgres_backup_docker_image_suffix }}"
matrix_postgres_backup_docker_image_v10: "docker.io/prodrigestivill/postgres-backup-local:10{{ matrix_postgres_backup_docker_image_suffix }}"
matrix_postgres_backup_docker_image_v11: "docker.io/prodrigestivill/postgres-backup-local:11{{ matrix_postgres_backup_docker_image_suffix }}"
matrix_postgres_backup_docker_image_v12: "docker.io/prodrigestivill/postgres-backup-local:12{{ matrix_postgres_backup_docker_image_suffix }}"
matrix_postgres_backup_docker_image_v13: "docker.io/prodrigestivill/postgres-backup-local:13{{ matrix_postgres_backup_docker_image_suffix }}"
matrix_postgres_backup_docker_image_latest: "{{ matrix_postgres_backup_docker_image_v13 }}"
# This variable is assigned at runtime. Overriding its value has no effect.
matrix_postgres_backup_docker_image_to_use: '{{ matrix_postgres_backup_docker_image_latest }}'
matrix_postgres_backup_docker_image_force_pull: "{{ matrix_postgres_backup_docker_image_to_use.endswith(':latest') }}"

View file

@ -0,0 +1,3 @@
- set_fact:
matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-postgres-backup.service'] }}"
when: matrix_postgres_backup_enabled|bool

View file

@ -0,0 +1,17 @@
---
- import_tasks: "{{ role_path }}/tasks/init.yml"
tags:
- always
- import_tasks: "{{ role_path }}/tasks/validate_config.yml"
when: "run_setup|bool and matrix_postgres_enabled|bool"
tags:
- setup-all
- setup-postgres-backup
- import_tasks: "{{ role_path }}/tasks/setup_postgres_backup.yml"
when: run_setup|bool
tags:
- setup-all
- setup-postgres-backup

View file

@ -0,0 +1,103 @@
---
#
# Tasks related to setting up an internal postgres server
#
- import_tasks: "{{ role_path }}/tasks/util/detect_existing_postgres_version.yml"
when: matrix_postgres_enabled|bool
# If we have found an existing version (installed from before), we use its corresponding Docker image.
# If not, we install using the latest Postgres.
#
# Upgrading is supposed to be performed separately and explicitly (see `upgrade_postgres.yml`).
- set_fact:
matrix_postgres_backup_docker_image_to_use: "{{ matrix_postgres_backup_docker_image_latest if matrix_postgres_backup_detected_version_corresponding_docker_image == '' else matrix_postgres_backup_detected_version_corresponding_docker_image }}"
when: matrix_postgres_backup_enabled|bool
- name: Ensure postgres backup Docker image is pulled
docker_image:
name: "{{ matrix_postgres_backup_docker_image_to_use }}"
source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
force_source: "{{ matrix_postgres_backup_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_postgres_backup_docker_image_force_pull }}"
when: matrix_postgres_backup_enabled|bool
- name: Ensure Postgres backup paths exist
file:
path: "{{ item }}"
state: directory
mode: 0700
owner: "{{ matrix_user_username }}"
group: "{{ matrix_user_groupname }}"
with_items:
- "{{ matrix_postgres_backup_path }}"
when: matrix_postgres_backup_enabled|bool
- name: Ensure Postgres environment variables file created
template:
src: "{{ role_path }}/templates/{{ item }}.j2"
dest: "{{ matrix_postgres_backup_path }}/{{ item }}"
mode: 0640
with_items:
- "env-postgres-backup"
when: matrix_postgres_backup_enabled|bool
- name: Ensure matrix-postgres-backup.service installed
template:
src: "{{ role_path }}/templates/systemd/matrix-postgres-backup.service.j2"
dest: "{{ matrix_systemd_path }}/matrix-postgres-backup.service"
mode: 0644
register: matrix_postgres_backup_systemd_service_result
when: matrix_postgres_backup_enabled|bool
- name: Ensure systemd reloaded after matrix-postgres-backup.service installation
service:
daemon_reload: yes
when: "matrix_postgres_backup_enabled|bool and matrix_postgres_backup_systemd_service_result.changed"
#
# Tasks related to getting rid of the internal postgres backup server (if it was previously enabled)
#
- name: Check existence of matrix-postgres-backup service
stat:
path: "{{ matrix_systemd_path }}/matrix-postgres-backup.service"
register: matrix_postgres_backup_service_stat
when: "not matrix_postgres_backup_enabled|bool"
- name: Ensure matrix-postgres-backup is stopped
service:
name: matrix-postgres-backup
state: stopped
daemon_reload: yes
when: "not matrix_postgres_backup_enabled|bool and matrix_postgres_backup_service_stat.stat.exists"
- name: Ensure matrix-postgres-backup.service doesn't exist
file:
path: "{{ matrix_systemd_path }}/matrix-postgres-backup.service"
state: absent
when: "not matrix_postgres_backup_enabled|bool and matrix_postgres_backup_service_stat.stat.exists"
- name: Ensure systemd reloaded after matrix-postgres-backup.service removal
service:
daemon_reload: yes
when: "not matrix_postgres_backup_enabled|bool and matrix_postgres_backup_service_stat.stat.exists"
- name: Check existence of matrix-postgres-backup backup path
stat:
path: "{{ matrix_postgres_backup_path }}"
register: matrix_postgres_backup_path_stat
when: "not matrix_postgres_backup_enabled|bool"
# We just want to notify the user. Deleting data is too destructive.
- name: Inject warning if matrix-postgres backup data remains
set_fact:
matrix_playbook_runtime_results: |
{{
matrix_playbook_runtime_results|default([])
+
[
"NOTE: You are not using the local backup service to backup the PostgreSQL database, but some old data remains from before in `{{ matrix_postgres_backup_path }}`. Feel free to delete it."
]
}}
when: "not matrix_postgres_backup_enabled|bool and matrix_postgres_backup_path_stat.stat.exists"

View file

@ -0,0 +1,56 @@
---
# This utility aims to determine if there is some existing Postgres version in use or not.
# If there is, it also tries to detect the Docker image that corresponds to that version.
- name: Initialize Postgres version determination variables (default to empty)
set_fact:
matrix_postgres_detection_pg_version_path: "{{ matrix_postgres_data_path }}/PG_VERSION"
matrix_postgres_detected_existing: false
matrix_postgres_detected_version: ""
matrix_postgres_detected_version_corresponding_docker_image: ""
- name: Determine existing Postgres version (check PG_VERSION file)
stat:
path: "{{ matrix_postgres_detection_pg_version_path }}"
register: result_pg_version_stat
- set_fact:
matrix_postgres_detected_existing: true
when: "result_pg_version_stat.stat.exists"
- name: Determine existing Postgres version (read PG_VERSION file)
slurp:
src: "{{ matrix_postgres_detection_pg_version_path }}"
register: result_pg_version
when: matrix_postgres_detected_existing|bool
- name: Determine existing Postgres version (make sense of PG_VERSION file)
set_fact:
matrix_postgres_detected_version: "{{ result_pg_version['content']|b64decode|replace('\n', '') }}"
when: matrix_postgres_detected_existing|bool
- name: Determine corresponding Docker image to detected version (assume default of latest)
set_fact:
matrix_postgres_backup_detected_version_corresponding_docker_image: "{{ matrix_postgres_backup_docker_image_latest }}"
when: "matrix_postgres_detected_version != ''"
- name: Determine corresponding Docker image to detected version (use 9.x, if detected)
set_fact:
matrix_postgres_backup_detected_version_corresponding_docker_image: "{{ matrix_postgres_backup_docker_image_v9 }}"
when: "matrix_postgres_detected_version.startswith('9.')"
- name: Determine corresponding Docker image to detected version (use 10.x, if detected)
set_fact:
matrix_postgres_backup_detected_version_corresponding_docker_image: "{{ matrix_postgres_backup_docker_image_v10 }}"
when: "matrix_postgres_detected_version == '10' or matrix_postgres_detected_version.startswith('10.')"
- name: Determine corresponding Docker image to detected version (use 11.x, if detected)
set_fact:
matrix_postgres_backup_detected_version_corresponding_docker_image: "{{ matrix_postgres_backup_docker_image_v11 }}"
when: "matrix_postgres_detected_version == '11' or matrix_postgres_detected_version.startswith('11.')"
- name: Determine corresponding Docker image to detected version (use 12.x, if detected)
set_fact:
matrix_postgres_backup_detected_version_corresponding_docker_image: "{{ matrix_postgres_backup_docker_image_v12 }}"
when: "matrix_postgres_detected_version == '12' or matrix_postgres_detected_version.startswith('12.')"

View file

@ -0,0 +1,18 @@
---
- name: Fail if required Postgres settings not defined
fail:
msg: >-
You need to define a required configuration setting (`{{ item }}`).
when: "vars[item] == ''"
with_items:
- "matrix_postgres_backup_connection_hostname"
- "matrix_postgres_backup_connection_username"
- "matrix_postgres_backup_connection_password"
- "matrix_postgres_backup_connection_port"
- "matrix_postgres_backup_schedule"
- "matrix_postgres_backup_keep_days"
- "matrix_postgres_backup_keep_weeks"
- "matrix_postgres_backup_keep_months"
- "matrix_postgres_backup_path"
- "matrix_postgres_backup_databases"

View file

@ -0,0 +1,12 @@
#jinja2: lstrip_blocks: "True"
POSTGRES_USER={{ matrix_postgres_backup_connection_username }}
POSTGRES_PASSWORD={{ matrix_postgres_backup_connection_password }}
POSTGRES_HOST={{ matrix_postgres_backup_connection_hostname }}
POSTGRES_DB={{ matrix_postgres_backup_databases|join(', ') }}
POSTGRES_EXTRA_OPTS={{ matrix_postgres_backup_extra_opts }}
SCHEDULE={{ matrix_postgres_backup_schedule }}
BACKUP_KEEP_DAYS={{ matrix_postgres_backup_keep_days }}
BACKUP_KEEP_WEEKS={{ matrix_postgres_backup_keep_weeks }}
BACKUP_KEEP_MONTHS={{ matrix_postgres_backup_keep_months }}
HEALTHCHECK_PORT={{ matrix_postgres_backup_healthcheck_port }}
POSTGRES_PORT={{ matrix_postgres_backup_connection_port }}

View file

@ -0,0 +1,31 @@
#jinja2: lstrip_blocks: "True"
[Unit]
Description=Automatic Backup of Matrix Postgres server
After=docker.service
Requires=docker.service
DefaultDependencies=no
[Service]
Type=simple
Environment="HOME={{ matrix_systemd_unit_home_path }}"
ExecStartPre=-{{ matrix_host_command_docker }} stop matrix-postgres-backup
ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-postgres-backup 2>/dev/null'
ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-postgres-backup \
--log-driver=none \
--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
--cap-drop=ALL \
--read-only \
--network={{ matrix_docker_network }} \
--env-file={{ matrix_postgres_backup_path }}/env-postgres-backup \
--mount type=bind,src={{ matrix_postgres_backup_path }},dst=/backups \
{{ matrix_postgres_backup_docker_image_to_use }}
ExecStop=-{{ matrix_host_command_docker }} stop matrix-postgres-backup
ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-postgres-backup 2>/dev/null'
Restart=always
RestartSec=30
SyslogIdentifier=matrix-postgres-backup
[Install]
WantedBy=multi-user.target

View file

@ -51,4 +51,6 @@
- matrix-nginx-proxy
- matrix-coturn
- matrix-aux
- matrix-postgres-backup
- matrix-common-after