From 2f75ae7e6256707cc0558e85524b0a2487c10ba6 Mon Sep 17 00:00:00 2001 From: teutat3s Date: Thu, 25 Jan 2024 20:21:50 +0100 Subject: [PATCH 1/4] feat(matrix-synapse): enable metrics Following: https://github.com/matrix-org/synapse/blob/develop/docs/metrics-howto.md --- hosts/flora-6/apps/prometheus.nix | 19 ++++++++++-- hosts/nachtigall/apps/matrix/synapse.nix | 30 +++++++++++++++++++ hosts/nachtigall/apps/nginx-matrix.nix | 4 --- .../apps/nginx-prometheus-exporters.nix | 3 ++ 4 files changed, 50 insertions(+), 6 deletions(-) diff --git a/hosts/flora-6/apps/prometheus.nix b/hosts/flora-6/apps/prometheus.nix index 686045a..11bc0bf 100644 --- a/hosts/flora-6/apps/prometheus.nix +++ b/hosts/flora-6/apps/prometheus.nix @@ -26,7 +26,7 @@ }; scrapeConfigs = [ { - job_name = "http-targets"; + job_name = "node-exporter-http"; static_configs = [{ targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ]; labels = { @@ -35,7 +35,7 @@ }]; } { - job_name = "https-targets"; + job_name = "node-exporter-https"; scheme = "https"; metrics_path = "/metrics"; basic_auth = { @@ -49,6 +49,21 @@ }; }]; } + { + job_name = "matrix-synapse"; + scheme = "https"; + metrics_path = "/_synapse/metrics"; + basic_auth = { + username = "hakkonaut"; + password_file = "${config.age.secrets.nachtigall-metrics-prometheus-basic-auth-password.path}"; + }; + static_configs = [{ + targets = [ "nachtigall.pub.solar" ]; + labels = { + instance = "nachtigall"; + }; + }]; + } ]; }; } diff --git a/hosts/nachtigall/apps/matrix/synapse.nix b/hosts/nachtigall/apps/matrix/synapse.nix index e35c914..240897a 100644 --- a/hosts/nachtigall/apps/matrix/synapse.nix +++ b/hosts/nachtigall/apps/matrix/synapse.nix @@ -38,6 +38,32 @@ in allow_unsafe_locale = false; txn_limit = 0; }; + listeners = [ + { + bind_addresses = [ + "127.0.0.1" + ]; + port = 8008; + resources = [ + { + compress = true; + names = [ + "client" + ]; + } + { + compress = false; + names = [ + "federation" + "metrics" + ]; + } + ]; + tls = false; + type = "http"; + x_forwarded = true; + } + ]; account_threepid_delegates.msisdn = ""; alias_creation_rules = [{ @@ -68,6 +94,7 @@ in encryption_enabled_by_default_for_room_type = "off"; event_cache_size = "100K"; federation_rr_transactions_per_room_per_second = 50; + federation_client_minimum_tls_version = "1.2"; forget_rooms_on_leave = true; include_profile_data_on_invite = true; instance_map = { }; @@ -162,6 +189,7 @@ in stream_writers = { }; trusted_key_servers = [{ server_name = "matrix.org"; }]; + suppress_key_server_warning = true; turn_allow_guests = false; turn_uris = [ @@ -213,6 +241,8 @@ in ]; }; + withJemalloc = true; + extraConfigFiles = [ "/run/agenix/matrix-synapse-secret-config.yaml" diff --git a/hosts/nachtigall/apps/nginx-matrix.nix b/hosts/nachtigall/apps/nginx-matrix.nix index c943146..a65a3dc 100644 --- a/hosts/nachtigall/apps/nginx-matrix.nix +++ b/hosts/nachtigall/apps/nginx-matrix.nix @@ -88,10 +88,6 @@ in gzip_types text/plain application/json; ''; locations = { - # TODO: Configure metrics - # "/metrics" = { - # }; - # For telegram "/c3c3f34b-29fb-5feb-86e5-98c75ec8214b" = { proxyPass = "http://127.0.0.1:8009"; diff --git a/hosts/nachtigall/apps/nginx-prometheus-exporters.nix b/hosts/nachtigall/apps/nginx-prometheus-exporters.nix index f46e651..358f9fe 100644 --- a/hosts/nachtigall/apps/nginx-prometheus-exporters.nix +++ b/hosts/nachtigall/apps/nginx-prometheus-exporters.nix @@ -14,6 +14,9 @@ locations."/metrics" = { proxyPass = "http://127.0.0.1:${toString(config.services.prometheus.exporters.node.port)}"; }; + locations."/_synapse/metrics" = { + proxyPass = "http://127.0.0.1:${toString (builtins.map (listener: listener.port) config.services.matrix-synapse.settings.listeners)}"; + }; }; }; } From 3cfdd9d20aa554893b27d8173b1a50e0d2a10b48 Mon Sep 17 00:00:00 2001 From: teutat3s Date: Sat, 27 Jan 2024 11:19:30 +0100 Subject: [PATCH 2/4] refactor(matrix-synapse): get first listener port --- hosts/nachtigall/apps/matrix/irc.nix | 6 +++++- hosts/nachtigall/apps/nginx-prometheus-exporters.nix | 9 ++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/hosts/nachtigall/apps/matrix/irc.nix b/hosts/nachtigall/apps/matrix/irc.nix index 50e566c..3b40664 100644 --- a/hosts/nachtigall/apps/matrix/irc.nix +++ b/hosts/nachtigall/apps/matrix/irc.nix @@ -1,4 +1,8 @@ { config, lib, pkgs, ... }: +let + # Get port from first element in list of matrix-synapse listeners + synapsePort = "${toString (lib.findFirst (listener: listener.port != null) "" config.services.matrix-synapse.settings.listeners).port}"; +in { systemd.services.matrix-appservice-irc.serviceConfig.SystemCallFilter = lib.mkForce [ "@system-service @pkey" @@ -13,7 +17,7 @@ settings = { homeserver = { domain = "pub.solar"; - url = "http://127.0.0.1:${toString (builtins.map (listener: listener.port) config.services.matrix-synapse.settings.listeners)}"; + url = "http://127.0.0.1:${synapsePort}"; media_url = "https://matrix.pub.solar"; enablePresence = false; }; diff --git a/hosts/nachtigall/apps/nginx-prometheus-exporters.nix b/hosts/nachtigall/apps/nginx-prometheus-exporters.nix index 358f9fe..e1f8e30 100644 --- a/hosts/nachtigall/apps/nginx-prometheus-exporters.nix +++ b/hosts/nachtigall/apps/nginx-prometheus-exporters.nix @@ -1,5 +1,8 @@ -{ config, flake, ... }: - +{ config, flake, lib, ... }: +let + # Get port from first element in list of matrix-synapse listeners + synapsePort = "${toString (lib.findFirst (listener: listener.port != null) "" config.services.matrix-synapse.settings.listeners).port}"; +in { age.secrets.nachtigall-metrics-nginx-basic-auth = { file = "${flake.self}/secrets/nachtigall-metrics-nginx-basic-auth.age"; @@ -15,7 +18,7 @@ proxyPass = "http://127.0.0.1:${toString(config.services.prometheus.exporters.node.port)}"; }; locations."/_synapse/metrics" = { - proxyPass = "http://127.0.0.1:${toString (builtins.map (listener: listener.port) config.services.matrix-synapse.settings.listeners)}"; + proxyPass = "http://127.0.0.1:${synapsePort}"; }; }; }; From 62429bca082542920705fd1fc542672b8507219e Mon Sep 17 00:00:00 2001 From: teutat3s Date: Sat, 27 Jan 2024 20:23:03 +0100 Subject: [PATCH 3/4] fix(matrix-synapse): make sure to find element in list of config.services.matrix-synapse.settings.listeners.*.resources that sets names = "client" instead of just using the first element in the list of listeners --- hosts/nachtigall/apps/matrix/irc.nix | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/hosts/nachtigall/apps/matrix/irc.nix b/hosts/nachtigall/apps/matrix/irc.nix index 3b40664..fb2c1e0 100644 --- a/hosts/nachtigall/apps/matrix/irc.nix +++ b/hosts/nachtigall/apps/matrix/irc.nix @@ -1,7 +1,16 @@ { config, lib, pkgs, ... }: let - # Get port from first element in list of matrix-synapse listeners - synapsePort = "${toString (lib.findFirst (listener: listener.port != null) "" config.services.matrix-synapse.settings.listeners).port}"; + # Find element in list config.services.matrix-synapse.settings.listeners.*.resources + # that sets names = "client" + nameHasClient = name: name == "client"; + resourceHasClient = resource: builtins.any nameHasClient resource.names; + listenerWithClient = lib.findFirst + (listener: + builtins.any resourceHasClient listener.resources) + (throw "Found no matrix-synapse.settings.listeners.*.resources.*.names containing string client") + config.services.matrix-synapse.settings.listeners + ; + synapseClientPort = "${toString listenerWithClient.port}"; in { systemd.services.matrix-appservice-irc.serviceConfig.SystemCallFilter = lib.mkForce [ @@ -17,7 +26,7 @@ in settings = { homeserver = { domain = "pub.solar"; - url = "http://127.0.0.1:${synapsePort}"; + url = "http://127.0.0.1:${synapseClientPort}"; media_url = "https://matrix.pub.solar"; enablePresence = false; }; From 69b976607fe2f3d00fa978098fb22c025bcc598c Mon Sep 17 00:00:00 2001 From: teutat3s Date: Sat, 27 Jan 2024 20:24:11 +0100 Subject: [PATCH 4/4] fix(matrix-synapse): make sure to find element in list of config.services.matrix-synapse.settings.listeners that sets type = "metrics" instead of just using the first element in the list --- hosts/nachtigall/apps/matrix/synapse.nix | 15 ++++++++++++++- .../apps/nginx-prometheus-exporters.nix | 13 ++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/hosts/nachtigall/apps/matrix/synapse.nix b/hosts/nachtigall/apps/matrix/synapse.nix index 240897a..93cff67 100644 --- a/hosts/nachtigall/apps/matrix/synapse.nix +++ b/hosts/nachtigall/apps/matrix/synapse.nix @@ -55,7 +55,6 @@ in compress = false; names = [ "federation" - "metrics" ]; } ]; @@ -63,6 +62,20 @@ in type = "http"; x_forwarded = true; } + { + bind_addresses = [ + "127.0.0.1" + ]; + port = 8012; + resources = [ + { + names = [ + "metrics" + ]; + } + ]; + type = "metrics"; + } ]; account_threepid_delegates.msisdn = ""; diff --git a/hosts/nachtigall/apps/nginx-prometheus-exporters.nix b/hosts/nachtigall/apps/nginx-prometheus-exporters.nix index e1f8e30..af5678c 100644 --- a/hosts/nachtigall/apps/nginx-prometheus-exporters.nix +++ b/hosts/nachtigall/apps/nginx-prometheus-exporters.nix @@ -1,7 +1,14 @@ { config, flake, lib, ... }: let - # Get port from first element in list of matrix-synapse listeners - synapsePort = "${toString (lib.findFirst (listener: listener.port != null) "" config.services.matrix-synapse.settings.listeners).port}"; + # Find element in list config.services.matrix-synapse.settings.listeners + # that sets type = "metrics" + listenerWithMetrics = lib.findFirst + (listener: + listener.type == "metrics") + (throw "Found no matrix-synapse.settings.listeners.*.type containing string metrics") + config.services.matrix-synapse.settings.listeners + ; + synapseMetricsPort = "${toString listenerWithMetrics.port}"; in { age.secrets.nachtigall-metrics-nginx-basic-auth = { @@ -18,7 +25,7 @@ in proxyPass = "http://127.0.0.1:${toString(config.services.prometheus.exporters.node.port)}"; }; locations."/_synapse/metrics" = { - proxyPass = "http://127.0.0.1:${synapsePort}"; + proxyPass = "http://127.0.0.1:${synapseMetricsPort}"; }; }; };