metrics: refactor to use firewall rules

add additional prometheus-exporters for nextcloud, php-fpm, nginx,
postgres, zfs

hosts/default.nix

@@ -50,7 +50,6 @@
           self.nixosModules.nginx-mastodon-files
           self.nixosModules.mediawiki
           self.nixosModules.nextcloud
-          self.nixosModules.nginx-prometheus-exporters
           self.nixosModules.nginx-website
           self.nixosModules.nginx-website-miom
           self.nixosModules.opensearch

hosts/nachtigall/default.nix

@@ -7,6 +7,7 @@
     ./configuration.nix
 
     ./networking.nix
+    ./prometheus-exporters.nix
     ./wireguard.nix
     ./backups.nix
     "${flake.inputs.fork}/nixos/modules/services/matrix/matrix-authentication-service.nix"

hosts/nachtigall/prometheus-exporters.nix

@@ -0,0 +1,76 @@
+{
+  config,
+  lib,
+  flake,
+  ...
+}:
+let
+  # Find element in list config.services.matrix-synapse.settings.listeners
+  # that sets type = "metrics"
+  listenerWithMetrics =
+    lib.findFirst (listener: listener.type == "metrics")
+      (throw "Found no matrix-synapse.settings.listeners.*.type containing string metrics")
+      config.services.matrix-synapse.settings.listeners;
+  synapseMetricsPort = "${toString listenerWithMetrics.port}";
+
+in
+{
+  age.secrets."nextcloud-serverinfo-token" = {
+    file = "${flake.self}/secrets/nextcloud-serverinfo-token.age";
+    mode = "400";
+    owner = "nextcloud-exporter";
+  };
+
+  services.prometheus = {
+    exporters = {
+      # https://github.com/xperimental/nextcloud-exporter
+      nextcloud = {
+        enable = true;
+        openFirewall = true;
+        firewallRules = [
+          ''iifname "wg-ssh" tcp dport ${config.services.prometheus.exporters.nextcloud.port} accept''
+        ];
+        url = "https://cloud.pub.solar";
+        tokenFile = config.age.secrets."nextcloud-serverinfo-token".path;
+        port = 9205;
+      };
+      # https://github.com/nginx/nginx-prometheus-exporter
+      nginx = {
+        enable = true;
+        openFirewall = true;
+        firewallRules = [
+          ''iifname "wg-ssh" tcp dport ${config.services.prometheus.exporters.nginx.port} accept''
+        ];
+        port = 9113;
+      };
+      # https://github.com/hipages/php-fpm_exporter
+      php-fpm = {
+        enable = true;
+        openFirewall = true;
+        firewallRules = [
+          ''iifname "wg-ssh" tcp dport ${config.services.prometheus.exporters.php-fpm.port} accept''
+        ];
+        port = 9253;
+      };
+      # https://github.com/prometheus-community/postgres_exporter
+      postgres = {
+        enable = true;
+        dataSourceName = "postgres_exporter@:5432/postgres?host=/run/postgresql";
+        openFirewall = true;
+        firewallRules = [
+          ''iifname "wg-ssh" tcp dport ${config.services.prometheus.exporters.postgres.port} accept''
+        ];
+        port = 9187;
+      };
+      # https://github.com/pdf/zfs_exporter
+      zfs = {
+        enable = true;
+        openFirewall = true;
+        firewallRules = [
+          ''iifname "wg-ssh" tcp dport ${config.services.prometheus.exporters.zfs.port} accept''
+        ];
+        port = 9134;
+      };
+    };
+  };
+}

modules/matrix/default.nix

@@ -8,6 +8,13 @@
 let
   publicDomain = "matrix.${config.pub-solar-os.networking.domain}";
   serverDomain = "${config.pub-solar-os.networking.domain}";
+  # Find element in list config.services.matrix-synapse.settings.listeners
+  # that sets type = "metrics"
+  listenerWithMetrics =
+    lib.findFirst (listener: listener.type == "metrics")
+      (throw "Found no matrix-synapse.settings.listeners.*.type containing string metrics")
+      config.services.matrix-synapse.settings.listeners;
+  synapseMetricsPort = "${toString listenerWithMetrics.port}";
 in
 {
   options.pub-solar-os = {
@@ -45,6 +52,9 @@ in
   };
 
   config = lib.mkIf config.pub-solar-os.matrix.enable {
+    # Only expose matrix-synapse metrics port via wireguard interface
+    networking.firewall.interfaces.wg-ssh.allowedTCPPorts = [ synapseMetricsPort ];
+
     services.matrix-synapse = {
       enable = true;
       settings = {
@@ -80,7 +90,7 @@ in
             x_forwarded = true;
           }
           {
-            bind_addresses = [ "127.0.0.1" ];
+            bind_addresses = [ "0.0.0.0" ];
             port = 8012;
             resources = [ { names = [ "metrics" ]; } ];
             tls = false;

modules/nginx-prometheus-exporters/default.nix

@@ -1,31 +0,0 @@
-{
-  config,
-  flake,
-  lib,
-  ...
-}:
-let
-  # Find element in list config.services.matrix-synapse.settings.listeners
-  # that sets type = "metrics"
-  listenerWithMetrics =
-    lib.findFirst (listener: listener.type == "metrics")
-      (throw "Found no matrix-synapse.settings.listeners.*.type containing string metrics")
-      config.services.matrix-synapse.settings.listeners;
-  synapseMetricsPort = "${toString listenerWithMetrics.port}";
-in
-{
-  services.nginx.virtualHosts = {
-    "nachtigall.wg.${config.pub-solar-os.networking.domain}" = {
-      listenAddresses = [
-        "10.7.6.1"
-        "[fd00:fae:fae:fae:fae:1::]"
-      ];
-      locations."/metrics" = {
-        proxyPass = "http://127.0.0.1:${toString (config.services.prometheus.exporters.node.port)}";
-      };
-      locations."/_synapse/metrics" = {
-        proxyPass = "http://127.0.0.1:${synapseMetricsPort}";
-      };
-    };
-  };
-}

modules/nginx/default.nix

@@ -15,6 +15,7 @@ in
   services.nginx = {
     enable = true;
     enableReload = true;
+    statusPage = true;
     proxyCachePath.cache = {
       enable = true;
     };

modules/prometheus-exporters/default.nix

@@ -1,12 +1,13 @@
 { config, ... }:
 {
-  # Only expose prometheus exporter port via wireguard interface
-  networking.firewall.interfaces.wg-ssh.allowedTCPPorts = [ 9002 ];
-
   services.prometheus = {
     exporters = {
       node = {
         enable = true;
+        openFirewall = true;
+        firewallRules = [
+          ''iifname "wg-ssh" tcp dport ${config.services.prometheus.exporters.node.port} accept''
+        ];
         enabledCollectors = [ "systemd" ];
         port = 9002;
       };

modules/prometheus/default.nix

@@ -5,6 +5,15 @@
   flake,
   ...
 }:
+let
+  # Find element in list config.services.matrix-synapse.settings.listeners
+  # that sets type = "metrics"
+  listenerWithMetrics =
+    lib.findFirst (listener: listener.type == "metrics")
+      (throw "Found no matrix-synapse.settings.listeners.*.type containing string metrics")
+      flake.self.nixosConfigurations.nachtigall.config.services.matrix-synapse.settings.listeners;
+  synapseMetricsPort = "${toString listenerWithMetrics.port}";
+in
 {
   age.secrets.alertmanager-envfile = {
     file = "${flake.self}/secrets/alertmanager-envfile.age";
@@ -38,13 +47,6 @@
   services.prometheus = {
     enable = true;
     port = 9001;
-    exporters = {
-      node = {
-        enable = true;
-        enabledCollectors = [ "systemd" ];
-        port = 9002;
-      };
-    };
     globalConfig = {
       scrape_interval = "10s";
       scrape_timeout = "9s";
@@ -54,7 +56,9 @@
         job_name = "node-exporter";
         static_configs = [
           {
-            targets = [ "nachtigall.wg.${config.pub-solar-os.networking.domain}" ];
+            targets = [
+              "nachtigall.wg.${config.pub-solar-os.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
+            ];
             labels = {
               instance = "nachtigall";
             };
@@ -106,7 +110,7 @@
         metrics_path = "/_synapse/metrics";
         static_configs = [
           {
-            targets = [ "nachtigall.wg.${config.pub-solar-os.networking.domain}" ];
+            targets = [ "nachtigall.wg.${config.pub-solar-os.networking.domain}:${synapseMetricsPort}" ];
             labels = {
               instance = "nachtigall";
             };
@@ -149,6 +153,87 @@
           }
         ];
       }
+      {
+        job_name = "nextcloud";
+        scrape_interval = "5m";
+        static_configs = [
+          {
+            targets = [
+              "nachtigall.wg.${config.pub-solar-os.networking.domain}:${toString config.services.prometheus.exporters.nextcloud.port}"
+            ];
+            labels = {
+              instance = "nachtigall";
+            };
+          }
+        ];
+      }
+      {
+        job_name = "nginx";
+        static_configs = [
+          {
+            targets = [
+              "nachtigall.wg.${config.pub-solar-os.networking.domain}:${toString config.services.prometheus.exporters.nginx.port}"
+            ];
+            labels = {
+              instance = "nachtigall";
+            };
+          }
+        ];
+      }
+      {
+        job_name = "php-fpm";
+        static_configs = [
+          {
+            targets = [
+              "nachtigall.wg.${config.pub-solar-os.networking.domain}:${toString config.services.prometheus.exporters.php-fpm.port}"
+            ];
+            labels = {
+              instance = "nachtigall";
+            };
+          }
+        ];
+      }
+      {
+        job_name = "postgres";
+        relabel_configs = [
+          {
+            source_labels = [
+              "__address__"
+            ];
+            target_label = "__param_target";
+          }
+          {
+            source_labels = [
+              "__param_target"
+            ];
+            target_label = "instance";
+          }
+          {
+            target_label = "__address__";
+            replacement = "127.0.0.1:9116";
+          }
+        ];
+        static_configs = [
+          {
+            targets = [
+              "nachtigall.wg.${config.pub-solar-os.networking.domain}:${toString config.services.prometheus.exporters.postgres.port}"
+            ];
+          }
+        ];
+      }
+      {
+        job_name = "zfs";
+        static_configs = [
+          {
+            targets = [
+              "nachtigall.wg.${config.pub-solar-os.networking.domain}:${toString config.services.prometheus.exporters.zfs.port}"
+            ];
+            labels = {
+              instance = "nachtigall";
+            };
+          }
+        ];
+      }
     ];
 
     ruleFiles = [

secrets/secrets.nix

@@ -81,6 +81,7 @@ in
 
   "nextcloud-secrets.age".publicKeys = nachtigallKeys ++ adminKeys;
   "nextcloud-admin-pass.age".publicKeys = nachtigallKeys ++ adminKeys;
+  "nextcloud-serverinfo-token.age".publicKeys = nachtigallKeys ++ adminKeys;
 
   "searx-environment.age".publicKeys = nachtigallKeys ++ adminKeys;