WIP: postgres: create databases for all services

If a service is enabled, a database for it is created in postgres with a uniqque password. The service can then use this database for data storage instead of relying on sqlite.
This commit is contained in:
transcaffeine 2020-11-29 15:20:22 +01:00
parent d08b27784f
commit d9f4914e0d
No known key found for this signature in database
GPG key ID: 03624C433676E465
16 changed files with 118 additions and 21 deletions

View file

@ -889,6 +889,70 @@ matrix_postgres_connection_username: "synapse"
matrix_postgres_connection_password: "synapse-password"
matrix_postgres_db_name: "homeserver"
matrix_postgres_additional_databases: |
{{
([{
name: 'matrix_appservice_discord',
pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string
}] if matrix_appservice_discord_enabled else [])
+ ([{
name: 'matrix_appservice_slack'
pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string
}] if matrix_appservice_slack_enabled else [])
+ ([{
name: 'matrix_appservice_irc'
pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string
}] if matrix_appservice_irc_enabled else [])
+ ([{
name: 'mautrix-bridge-facebook'
pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string
}] if matrix_mautrix_facebook_enabled else [])
+ ([{
name: 'mautrix_bridge_hangouts'
pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string
}] if matrix_mautrix_hangouts_enabled else [])
+ ([{
name: 'mautrix_bridge_telegram'
pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string
}] if matrix_mautrix_telegram_enabled else [])
+ ([{
name: 'mautrix_bridge_whatsapp'
pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string
}] if matrix_mautrix_whatsapp_enabled else [])
+ ([{
name: 'matrix_bridge_sms'
pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string
}] if matrix_sms_bridge_enabled else [])
+ ([{
name: 'matrix_puppet_skype'
pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string
}] if matrix_mx_puppet_skype_enabled else [])
+ ([{
name: 'matrix_puppet_slack'
pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string
}] if matrix_mx_puppet_slack_enabled else [])
+ ([{
name: 'matrix_puppet_twitter'
pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string
}] if matrix_mx_puppet_twitter_enabled else [])
+ ([{
name: 'matrix_puppet_instagram'
pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string
] if matrix_mx_puppet_instagram_enabled else [])
+ ([{
name: 'matrix_puppet_discord'
pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string
}] if matrix_mx_puppet_discord_enabled else [])
+ ([{
name: 'matrix_puppet_steam'
pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string
}] if matrix_mx_puppet_steam_enabled else [])
+ ([{
name: 'matrix_dimension'
pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string
}] if matrix_dimension_enabled else [])
}}
######################################################################
#
# /matrix-postgres

View file

@ -58,8 +58,8 @@ database:
# If you are migrating, see https://github.com/Half-Shot/matrix-appservice-discord/blob/master/docs/howto.md#migrate-to-postgres-from-sqlite
# WARNING: You will almost certainly be fine with sqlite unless your bridge
# is in heavy demand and you suffer from IO slowness.
filename: "/data/discord.db"
# connString: "postgresql://user:password@localhost/database_name"
#filename: "/data/discord.db"
connString: "postgresql://matrix_appservice_discord:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_appservice_discord') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_discord"
room:
# Set the default visibility of alias rooms, defaults to "public".
# One of: "public", "private"

View file

@ -127,8 +127,8 @@ advanced:
# Use an external database to store bridge state.
database:
# database engine (must be 'postgres' or 'nedb'). Default: nedb
engine: "nedb"
engine: "postgres"
# Either a PostgreSQL connection string, or a path to the NeDB storage directory.
# For postgres, it must start with postgres://
# For NeDB, it must start with nedb://. The path is relative to the project directory.
connectionString: "nedb:///data"
connectionString: "postgres://matrix_appservice_irc:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_appservice_irc') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_irc"

View file

@ -10,5 +10,8 @@ homeserver:
server_name: "{{ matrix_domain }}"
dbdir: "/data"
db:
engine: "postgres"
connectionString: "postgresql://matrix_appservice_slack:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_appservice_slack') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_slack"
matrix_admin_room: "{{ matrix_appservice_slack_control_room_id }}"

View file

@ -27,7 +27,7 @@ appservice:
# Format examples:
# SQLite: sqlite:///filename.db
# Postgres: postgres://username:password@hostname/dbname
database: sqlite:////data/mautrix-facebook.db
database: sqlite://matrix_bridge_facebook:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_bridge_facebook') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_bridge_facebook
# Public part of web server for out-of-Matrix interaction with the bridge.
public:

View file

@ -27,7 +27,7 @@ appservice:
# Format examples:
# SQLite: sqlite:///filename.db
# Postgres: postgres://username:password@hostname/dbname
database: sqlite:////data/mautrix-hangouts.db
database: postgres://mautrix_bridge_hangouts:{{ matrix_additional_databases | selectattr('name', 'equalto', 'mautrix_bridge_hangouts') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/mautrix_bridge_hangouts
# The unique ID of this appservice.
id: hangouts

View file

@ -27,7 +27,7 @@ appservice:
# Format examples:
# SQLite: sqlite:///filename.db
# Postgres: postgres://username:password@hostname/dbname
database: sqlite:////data/mautrix-telegram.db
database: postgres://mautrix_bridge_telegram:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_bridge_telegram') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/mautrix_bridge_telegram
# Public part of web server for out-of-Matrix interaction with the bridge.
# Used for things like login if the user wants to make sure the 2FA password isn't stored in

View file

@ -23,7 +23,7 @@ appservice:
# The database URI.
# SQLite: File name is enough. https://github.com/mattn/go-sqlite3#connection-string
# Postgres: Connection string. For example, postgres://user:password@host/database
uri: mautrix-whatsapp.db
uri: postgres://matrix_bridge_whatsapp@{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_bridge_whatsapp') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_bridge_whatsapp
# Maximum number of connections. Mostly relevant for Postgres.
max_open_conns: 20
max_idle_conns: 2

View file

@ -105,10 +105,10 @@ database:
# Connection string to connect to the Postgres instance
# with username "user", password "pass", host "localhost" and database name "dbname".
# Modify each value as necessary
#connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
connString: "postgres://matrix_puppet_discord:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_discord') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_discord?sslmode=disable"
# Use SQLite3 as a database backend
# The name of the database file
filename: /data/database.db
#filename: /data/database.db
logging:
# Log level of console output

View file

@ -49,10 +49,10 @@ database:
# Connection string to connect to the Postgres instance
# with username "user", password "pass", host "localhost" and database name "dbname".
# Modify each value as necessary
#connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
connString: "postgres://matrix_puppet_instagram:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_instagram') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_instagram?sslmode=disable"
# Use SQLite3 as a database backend
# The name of the database file
filename: /data/database.db
#filename: /data/database.db
logging:
# Log level of console output

View file

@ -73,10 +73,10 @@ database:
# Connection string to connect to the Postgres instance
# with username "user", password "pass", host "localhost" and database name "dbname".
# Modify each value as necessary
#connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
connString: "postgres://matrix_puppet_skype:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_skype') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_skype?sslmode=disable"
# Use SQLite3 as a database backend
# The name of the database file
filename: /data/database.db
#filename: /data/database.db
provisioning:
# Regex of Matrix IDs allowed to use the puppet bridge

View file

@ -63,10 +63,10 @@ database:
# Connection string to connect to the Postgres instance
# with username "user", password "pass", host "localhost" and database name "dbname".
# Modify each value as necessary
#connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
connString: "postgres://matrix_puppet_slack:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_slack') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_slack?sslmode=disable"
# Use SQLite3 as a database backend
# The name of the database file
filename: /data/database.db
#filename: /data/database.db
logging:
# Log level of console output

View file

@ -66,10 +66,10 @@ database:
# Connection string to connect to the Postgres instance
# with username "user", password "pass", host "localhost" and database name "dbname".
# Modify each value as necessary
#connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
connString: "postgres://matrix_puppet_steam:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_steam') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_steam?sslmode=disable"
# Use SQLite3 as a database backend
# The name of the database file
filename: /data/database.db
#filename: /data/database.db
logging:
# Log level of console output

View file

@ -59,10 +59,10 @@ database:
# Connection string to connect to the Postgres instance
# with username "user", password "pass", host "localhost" and database name "dbname".
# Modify each value as necessary
#connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
connString: "postgres://matrix_puppet_twitter:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_twitter') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_twitter?sslmode=disable"
# Use SQLite3 as a database backend
# The name of the database file
filename: /data/database.db
#filename: /data/database.db
logging:
# Log level of console output

View file

@ -44,7 +44,7 @@ widgetBlacklist:
# Where the database for Dimension is
database:
file: "dimension.db"
uri: "postgres://matrix_dimension:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_dimension') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_dimension"
# Display settings that apply to self-hosted go-neb instances
goneb:

View file

@ -162,3 +162,33 @@
- matrix-change-user-admin-status
- matrix-postgres-update-user-password-hash
when: "not matrix_postgres_enabled|bool"
# Create additional databases
- name: Retrieve IP of postgres container
shell: "docker inspect matrix-postgres | jq -r '.[0].NetworkSettings.Networks.{{ matrix_docker_network }}.IPAddress'"
register: matirx_postgres_container_ip
- name: Create additional users in postgres
postgresql_user:
name: "{{ item.name }}"
password: "{{ item.pass }}"
login_host: "{{ matrx_postgres_container_ip.stdout }}"
login_port: 5432
login_user: "{{ matrix_postgres_connection_username }}"
login_password: "{{ matrix_postgres_connection_password }}"
login_db: "{{ matrix_postgres_db_name }}"
loop: matrix_postgres_additional_databases
when: matrix_postgres_enabed|bool
- name: Create additional users in postgres
postgresql_db:
name: "{{ item.name }}"
owner: "{{ item.name }}"
lc_ctype: 'C'
lc_collate: 'C'
login_host: "{{ matrx_postgres_container_ip.stdout }}"
login_port: 5432
login_user: "{{ matrix_postgres_connection_username }}"
login_password: "{{ matrix_postgres_connection_password }}"
loop: matrix_postgres_additional_databases
when: matrix_postgres_enabled|bool