blackbox-exporter: scrape every 60m, add URLs

style: fix formatting
monitoring: scrape blackbox every 5m, alert 21 days
2024-11-06 21:54:16 +01:00 · 2024-11-06 21:31:41 +01:00 · 2024-11-06 21:31:41 +01:00 · 2024-11-06 21:31:41 +01:00 · 2024-11-06 21:31:40 +01:00 · 2024-11-06 21:31:40 +01:00
19 changed files with 1318 additions and 69 deletions
--- a/flake.lock
+++ b/flake.lock
@ -94,11 +94,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1731895210,
-        "narHash": "sha256-z76Q/OXLxO/RxMII3fIt/TG665DANiE2lVvnolK2lXk=",
+        "lastModified": 1729712798,
+        "narHash": "sha256-a+Aakkb+amHw4biOZ0iMo8xYl37uUL48YEXIC5PYJ/8=",
        "owner": "nix-community",
        "repo": "disko",
-        "rev": "639d1520df9417ca2761536c3072688569e83c80",
+        "rev": "09a776702b004fdf9c41a024e1299d575ee18a7d",
        "type": "github"
      },
      "original": {
@ -185,11 +185,11 @@
        "nixpkgs-lib": "nixpkgs-lib"
      },
      "locked": {
-        "lastModified": 1730504689,
-        "narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=",
+        "lastModified": 1727826117,
+        "narHash": "sha256-K5ZLCyfO/Zj9mPFldf3iwS6oZStJcU4tSpiXTMYaaL0=",
        "owner": "hercules-ci",
        "repo": "flake-parts",
-        "rev": "506278e768c2a08bec68eb62932193e341f55c90",
+        "rev": "3d04084d54bedc3d6b8b736c70ef449225c361b1",
        "type": "github"
      },
      "original": {
@ -280,11 +280,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1731518114,
-        "narHash": "sha256-h9Wb3VjmXBZwTO3prRweUKwp2H9hZHCQKrkbU+2WPQs=",
+        "lastModified": 1730041422,
+        "narHash": "sha256-aEz5/yUJN/PSEXwPBuKMs2FbAmz68fDIQ9B0tVRVmTo=",
        "ref": "main",
-        "rev": "060ecccc5f8c92a0705ab91ff047811efd559468",
-        "revCount": 36,
+        "rev": "09f7b1ed16c99f5fb5c5f9a2a73ccc9ff0645b35",
+        "revCount": 32,
        "type": "git",
        "url": "https://git.pub.solar/pub-solar/keycloak-theme"
      },
@ -320,11 +320,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1732016537,
-        "narHash": "sha256-XwXUK+meYnlhdQz2TVE4Wv+tsx1CkdGbDPt1tRzCNH4=",
+        "lastModified": 1729757100,
+        "narHash": "sha256-x+8uGaX66V5+fUBHY23Q/OQyibQ38nISzxgj7A7Jqds=",
        "owner": "lnl7",
        "repo": "nix-darwin",
-        "rev": "61cee20168a3ebb71a9efd70a55adebaadfbe4d4",
+        "rev": "04193f188e4144d7047f83ad1de81d6034d175cd",
        "type": "github"
      },
      "original": {
@ -336,11 +336,11 @@
    },
    "nixpkgs": {
      "locked": {
-        "lastModified": 1731797254,
-        "narHash": "sha256-df3dJApLPhd11AlueuoN0Q4fHo/hagP75LlM5K1sz9g=",
+        "lastModified": 1729449015,
+        "narHash": "sha256-Gf04dXB0n4q0A9G5nTGH3zuMGr6jtJppqdeljxua1fo=",
        "owner": "nixos",
        "repo": "nixpkgs",
-        "rev": "e8c38b73aeb218e27163376a2d617e61a2ad9b59",
+        "rev": "89172919243df199fe237ba0f776c3e3e3d72367",
        "type": "github"
      },
      "original": {
@ -352,14 +352,14 @@
    },
    "nixpkgs-lib": {
      "locked": {
-        "lastModified": 1730504152,
-        "narHash": "sha256-lXvH/vOfb4aGYyvFmZK/HlsNsr/0CVWlwYvo2rxJk3s=",
+        "lastModified": 1727825735,
+        "narHash": "sha256-0xHYkMkeLVQAMa7gvkddbPqpxph+hDzdu1XdGPJR+Os=",
        "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/cc2f28000298e1269cea6612cd06ec9979dd5d7f.tar.gz"
+        "url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz"
      },
      "original": {
        "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/cc2f28000298e1269cea6612cd06ec9979dd5d7f.tar.gz"
+        "url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz"
      }
    },
    "root": {
@ -484,11 +484,11 @@
    },
    "unstable": {
      "locked": {
-        "lastModified": 1731676054,
-        "narHash": "sha256-OZiZ3m8SCMfh3B6bfGC/Bm4x3qc1m2SVEAlkV6iY7Yg=",
+        "lastModified": 1729665710,
+        "narHash": "sha256-AlcmCXJZPIlO5dmFzV3V2XF6x/OpNWUV8Y/FMPGd8Z4=",
        "owner": "nixos",
        "repo": "nixpkgs",
-        "rev": "5e4fbfb6b3de1aa2872b76d49fafc942626e2add",
+        "rev": "2768c7d042a37de65bb1b5b3268fc987e534c49d",
        "type": "github"
      },
      "original": {
--- a/modules/core/nix.nix
+++ b/modules/core/nix.nix
@ -11,17 +11,6 @@
    permittedInsecurePackages = [ "olm-3.2.16" ];
  };

-  system.activationScripts.diff-closures = {
-    text = ''
-      if [[ -e /run/current-system ]]; then
-        ${config.nix.package}/bin/nix store diff-closures \
-          /run/current-system "$systemConfig" \
-          --extra-experimental-features nix-command
-      fi
-    '';
-    supportsDryActivation = true;
-  };
-
  nix = {
    # Use default version alias for nix package
    package = pkgs.nix;
--- a/modules/coturn/default.nix
+++ b/modules/coturn/default.nix
@ -5,8 +5,9 @@
  ...
 }:
 {
-  age.secrets."coturn-static-auth-secret" = {
-    file = "${flake.self}/secrets/coturn-static-auth-secret.age";
+  age.secrets."nachtigall-coturn-static-auth-secret" = {
+    file = "${flake.self}/secrets/nachtigall-coturn-static-auth-secret.age";
+    path = "/run/agenix/coturn-static-auth-secret";
    mode = "400";
    owner = "turnserver";
  };
@ -18,7 +19,7 @@
    min-port = 49000;
    max-port = 50000;
    use-auth-secret = true;
-    static-auth-secret-file = config.age.secrets."coturn-static-auth-secret".path;
+    static-auth-secret-file = config.age.secrets."nachtigall-coturn-static-auth-secret".path;
    realm = "turn.${config.pub-solar-os.networking.domain}";
    cert = "${config.security.acme.certs.${realm}.directory}/full.pem";
    pkey = "${config.security.acme.certs.${realm}.directory}/key.pem";
--- a/modules/forgejo/default.nix
+++ b/modules/forgejo/default.nix
@ -141,12 +141,6 @@
        LOGIN_REMEMBER_DAYS = 365;
      };

-      # See https://docs.gitea.com/administration/config-cheat-sheet#migrations-migrations
-      migrations = {
-        # This allows migrations from the same forgejo instance
-        ALLOW_LOCALNETWORKS = true;
-      };
-
      # https://forgejo.org/docs/next/admin/config-cheat-sheet/#indexer-indexer
      indexer = {
        REPO_INDEXER_ENABLED = true;
--- a/modules/grafana/default.nix
+++ b/modules/grafana/default.nix
@ -28,6 +28,11 @@
      group = "grafana";
      user = "grafana";
    };
+    "grafana-dashboards/blackbox-exporter_rev3.json" = {
+      source = ./grafana-dashboards/blackbox-exporter_rev3.json;
+      group = "grafana";
+      user = "grafana";
+    };
    "grafana-dashboards/synapse.json" = {
      source = ./grafana-dashboards/synapse.json;
      group = "grafana";
--- a/modules/grafana/grafana-dashboards/blackbox-exporter_rev3.json
+++ b/modules/grafana/grafana-dashboards/blackbox-exporter_rev3.json
--- a/modules/matrix-irc/default.nix
+++ b/modules/matrix-irc/default.nix
@ -35,8 +35,7 @@ in
          port = 1113;
        };
        logging = {
-          # set to debug for debugging
-          level = "warn";
+          level = "debug";
          maxFiles = 5;
          toCosole = true;
        };
--- a/modules/matrix/default.nix
+++ b/modules/matrix/default.nix
@ -124,17 +124,6 @@ in
        enable_room_list_search = true;
        encryption_enabled_by_default_for_room_type = "off";
        event_cache_size = "100K";
-
-        # https://github.com/element-hq/synapse/issues/11203
-        # No YAML deep-merge, so this needs to be in secret extraConfigFiles
-        # together with msc3861
-        #experimental_features = {
-        #  # Room summary API
-        #  msc3266_enabled = true;
-        #  # Rendezvous server for QR Code generation
-        #  msc4108_enabled = true;
-        #};
-
        federation_rr_transactions_per_room_per_second = 50;
        federation_client_minimum_tls_version = "1.2";
        forget_rooms_on_leave = true;
@ -294,9 +283,7 @@ in
      createDatabase = true;
      extraConfigFiles = config.pub-solar-os.matrix.matrix-authentication-service.extra-config-files;

-      # https://element-hq.github.io/matrix-authentication-service/reference/configuration.html
      settings = {
-        account.email_change_allowed = false;
        http.public_base = "https://mas.${config.pub-solar-os.networking.domain}";
        http.issuer = "https://mas.${config.pub-solar-os.networking.domain}";
        http.listeners = [
--- a/modules/nginx-matrix/default.nix
+++ b/modules/nginx-matrix/default.nix
@ -24,6 +24,14 @@ let
      secure_backup_required = false;
      secure_backup_setup_methods = [ ];
    };
+    "m.integrations" = {
+      managers = [
+        {
+          api_url = "https://dimension.${domain}/api/v1/scalar";
+          ui_url = "https://dimension.${domain}/element";
+        }
+      ];
+    };
  };
  wellKnownServer = domain: { "m.server" = "matrix.${domain}:8448"; };
  wellKnownSupport = {
--- a/modules/prometheus/alert-rules.nix
+++ b/modules/prometheus/alert-rules.nix
@ -142,8 +142,8 @@ lib.mapAttrsToList

    cpu_using_90percent = {
      condition = ''100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) >= 90'';
-      time = "20m";
-      description = "{{$labels.instance}} is running with cpu usage > 90% for at least 20 minutes: {{$value}}";
+      time = "10m";
+      description = "{{$labels.instance}} is running with cpu usage > 90% for at least 10 minutes: {{$value}}";
    };

    reboot = {
@ -198,10 +198,10 @@ lib.mapAttrsToList
      description = "{{$labels.instance}}: healtcheck {{$labels.job}} fails!";
      };
    */
-    #cert_expiry = {
-    #  condition = "(probe_ssl_earliest_cert_expiry - time())/(3600*24) < 30";
-    #  description = "{{$labels.instance}}: The TLS certificate will expire in less than 30 days: {{$value}}s";
-    #};
+    cert_expiry = {
+      condition = "(probe_ssl_earliest_cert_expiry - time())/(3600*24) < 21";
+      description = "{{$labels.instance}}: The TLS certificate will expire in less than 21 days: {{$value}}s";
+    };

    # ignore devices that disabled S.M.A.R.T (example if attached via USB)

@ -234,10 +234,10 @@ lib.mapAttrsToList
      };
    */

-    #host_memory_under_memory_pressure = {
-    #  condition = "rate(node_vmstat_pgmajfault[1m]) > 1000";
-    #  description = "{{$labels.instance}}: The node is under heavy memory pressure. High rate of major page faults: {{$value}}";
-    #};
+    host_memory_under_memory_pressure = {
+      condition = "rate(node_vmstat_pgmajfault[1m]) > 1000";
+      description = "{{$labels.instance}}: The node is under heavy memory pressure. High rate of major page faults: {{$value}}";
+    };

    # ext4_errors = {
    #   condition = "ext4_errors_value > 0";
--- a/modules/prometheus/default.nix
+++ b/modules/prometheus/default.nix
@ -5,6 +5,22 @@
  flake,
  ...
 }:
+let
+  blackboxTargets = [
+    "https://pub.solar"
+    "https://chat.pub.solar"
+    "https://cloud.pub.solar"
+    "https://collabora.pub.solar"
+    "https://git.pub.solar"
+    "https://grafana.pub.solar"
+    "https://list.pub.solar"
+    "https://mastodon.pub.solar"
+    "https://obs-portal.pub.solar"
+    "https://stream.pub.solar"
+    "https://wiki.pub.solar"
+    "https://www.pub.solar"
+  ];
+in
 {
  age.secrets.alertmanager-envfile = {
    file = "${flake.self}/secrets/alertmanager-envfile.age";
@ -39,6 +55,32 @@
    enable = true;
    port = 9001;
    exporters = {
+      blackbox = {
+        enable = true;
+        # Default port is 9115
+        openFirewall = false;
+
+        configFile = pkgs.writeTextFile {
+          name = "blackbox-exporter-config";
+          text = ''
+            modules:
+              http_2xx:
+                prober: http
+                timeout: 5s
+                http:
+                  valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
+                  valid_status_codes: []  # Defaults to 2xx
+                  method: GET
+                  no_follow_redirects: false
+                  fail_if_ssl: false
+                  fail_if_not_ssl: false
+                  tls_config:
+                    insecure_skip_verify: false
+                  preferred_ip_protocol: "ip4" # defaults to "ip6"
+                  ip_protocol_fallback: true  # fallback to "ip6"
+          '';
+        };
+      };
      node = {
        enable = true;
        enabledCollectors = [ "systemd" ];
@ -50,6 +92,30 @@
      scrape_timeout = "9s";
    };
    scrapeConfigs = [
+      {
+        job_name = "blackbox";
+        scrape_interval = "60m";
+        metrics_path = "/probe";
+        params = {
+          module = [ "http_2xx" ];
+        };
+        static_configs = [ { targets = blackboxTargets; } ];
+
+        relabel_configs = [
+          {
+            source_labels = [ "__address__" ];
+            target_label = "__param_target";
+          }
+          {
+            source_labels = [ "__param_target" ];
+            target_label = "instance";
+          }
+          {
+            target_label = "__address__";
+            replacement = "127.0.0.1:9115"; # The blackbox exporter's real hostname:port.
+          }
+        ];
+      }
      {
        job_name = "node-exporter";
        static_configs = [
--- a/secrets/drone-db-secrets.age
+++ b/secrets/drone-db-secrets.age
@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 Y0ZZaw 5slOxDM4xGALMpYxFm1WBX4Sds7itgPBMIiY97d7Egk
+mZNzn4I6obUHAdox6eVR4H82EZagZ1IrCcq0CDtK44Y
+-> ssh-ed25519 uYcDNw w5lzhmA8wIMXihKF25d5jx4/Cc5BFE3Lw6ad60b0wBg
+v9z03cpts6oVlcTQ48hMw8rjWHp1JUOov2qCUjFN4bs
+-> ssh-rsa f5THog
+A93Usdjf6yJFLFqDiy6UUIJ4faBgQXIvk0pZlABlj9M5n7fSf9uzR6sSih4HNCvm
+sMkZ4wKyQHJnUB4Uc2jGrdcWqpmP5MLYHhj74Rxsi6heZuCRf94KH7sE/03A958w
+jAV4v9z4EqmkvWLNQi/hxMVMs5A61Vs63WIX/TA6vhL8Yrn0FeIKlRZYUVIeMu42
+pbEuLWeIzbUioAuEA1ZKV5VDx+6ack8TS/Dj5bTNEnzFWpjnHjO0/GeQU+aaQZTe
+Zy325TcRosT0V7PIh0tDQZKKRpOH/e9LnDkd8NIjyfEsGdDYaP1EVOYVxPCqUDAh
+A0kV1kkTiBzaXDkuakc+HDCIxtYXLWthsmbD+vI3D7FlTl0CY4fOP0wwO/0rS5Yp
+KDuxjz89II1H4+ZvlcPUihyW7OEj4d+NwFQy+7Qq0Y9Ii0NONXNsnx17FKXJwOMo
+NKyLo097FvHV7k8F9wv9mmZboRulDAoRyDngeO0+SJA90uJass04DuiZvK+g3Hry
+xVzbkk59j9EQqUogopW/oSeSbUP0pvcKOahGcSIW8vmadDTgnN7zzqf3fq+dJ2TM
+QD2IXAwvoTFBE+9DnPOtptk1X1D2umZuAWTzGAseXOImrPFZ+bEr5MV3qLGlg7sL
+yA7Mvbp4diVdH5aePzeBefhxrYphz+yfCbELFTYam9g
+-> ssh-rsa kFDS0A
+Ng0FhTDjASWJkrlNh+UZxU/dU/wfmoV1/fwTv6Xg69k/2qU9lk0oR6e5xAimvX6u
+h7rKAYt3zSRIFveGczPCflC1nycG9wLSpaoJghav+q+muoDQ/fbSKSgHFXITC7Me
+f/wblyWvJsUQbjxSW3g6/8EGz6FvpTnycPtD2vbRj+Ctq72GPA2ZWg/OC4jAUlDs
+r6X0Ql2jwWzy3Y12v0mPknlBezN8cIfjBmoNOWokUeGJIBjujlS7loA1yif09BLg
+PTSLCY1YH3QYcm6lCXK0HaNcMjSSk/ZK9D0wROriF9PBbkpWgg5NlIrqGaeqPN9z
+QwRR2DvhuCa1br57F36Y2LKGphYjmhWAtzCyQ0h9YQ+AzEy9uFCbK0IFyyeVl/fN
+HBGgxacJBcEGsNV3mbJvh6dn1348eex0GgaQEf1B/lu/y66WHbmSqVyUDfWkqEz
+IytAC43VT2rKgg+B5u0d/JhLDLwXTp7iVDy52ul1n7keJHk8t1GDaufAXbWqalQ4
+vuyxs6ghSIXUi27IZrYblg/OEPFTBfcoMXkmCgyx5a+eK+DhnBazWjy5j+vgp2so
+ZQRQurbG02qpZasTwBM3iy4ZklX/uFjsKnk0c/YcmK4YcMviHcQQjdjKruEE93u+
+Za1KE+qZGLkhFCd9O3ZPMtEjRjpN10XIs5ylKQ9MKU4
+-> piv-p256 vRzPNw AiNjNIR0OGHBu5Qn+bvn+Lk5VnpI2BQ3eJ3+2/FTJfZC
+elT3acRVdmtBl0qC5YbvfntxkJrsZwEJqlF6aN5hhWw
+-> piv-p256 zqq/iw AjIzSibkqG+YcP894QekM61Wsty6MaKBghlWapHfU0Jn
+HyXBp8DxtnNsfuzZq13bwgma5CzLTf3UB5Eht6XUwe8
+-> ssh-ed25519 YFSOsg WRBQZZYM+X26hfoH4zvNWQulZvVWP/Ha5OgkUmGK/Q4
+5Hw4ZDNawn5YRC673Op/sbpexOKeL3gez2B7oZxUKhA
+-> ssh-ed25519 iHV63A wyr8R4DlqLAu0XypddVoFimK2ZMncWaa+KWV7vMEQm8
+puV3g1t5AbnEgC0S1U4ft1evB7KuNppEi1g/AtxHgWE
+-> ssh-ed25519 BVsyTA 0N3iyyGqTCRAHHcK7QfN5xRttorc2E2GL0RDTIVIBU4
+Bph0OujqmXzi9IswduX9Mbh+yRdPKOwCf3fBv2zUzqI
+-> ssh-ed25519 +3V2lQ 0p90VtsxWyGFaeeoTISIxQRyeKVk0HoGGq71tjpIPjg
+sRf73Tp3BJ0DsTnJO2xVGyCKjaX7C7oydXj+39dKMUg
+--- +/HCG0s/x+c03NG5qrgliJ+5EXXI6UnuJz5XDv2aphY
+ÞšÂ<>™Ý@»=£L¬“7*®„ÐFq<46>UÒ*ûUê¿‰»È$e=þLgJ|*1Ï½BÚE ZG—_Ü5ê²ð—²ŽíÂ,òöÛi<C39B>_'¸d7
Ý3Ú“Nä3ãç¡*»ðªê<C2AA>£ŽáŽòqýŸ‰Oy#¶([l³†pÄf¼õ¾¥ö
--- a/secrets/drone-secrets.age
+++ b/secrets/drone-secrets.age
--- a/secrets/forgejo-actions-runner-token.age
+++ b/secrets/forgejo-actions-runner-token.age
@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 Y0ZZaw tm4AmC8yXPgR82lgsQR4VZn4xfGiK8o6fIn8pKPY6XQ
+IDnsYVD8noh2HdPNvjY/M5G+meR5rwvVI5SVN/cHEVM
+-> ssh-ed25519 uYcDNw ZB7GyOvD8S8XLqE1AeMXWTPcJnvEntWbZ7TGg8CJVxc
+Bfb/+P2DEiKI9ZWH105rLAYQXTUwWftWtudUGnVtjSU
+-> ssh-rsa f5THog
+Q3Hqks6BMGmP8TXnUkbblO8btrVdls7AUdxDW8e/w5biis/4awZVBHuZCLpiasM7
+7RWXcUep2VVyfCMb+8tedaf5a1MpGPDkZvdbxhfDVWZakh7vsEnth+gK2QsV0h8e
+eIgfDMA9J6DHXNCr4EYSf22PxY12KPqGqsMpVBhOZQuXoJwJy0ob3jbJEOYfPlu6
+V+TLYQNGQ2UQ+A7zjpUrGz1L+u7rUejY4Rv1BmCakg1bLEs8oSDmIVmsuVmFPqOI
+wqQJvnYlRAdioVQZwYCiqJech2QJ0ZhtC/ZeVp4c6TgFwB1ottxiAI8l7Bz1nFzW
+/E85qU3Jkh1tcNcLsVHj8+tnxwn1SSQ3xQxcOT9l1Po98sNapK7mwd/xx3pJ1hH8
+5YUQAtG80e5YmOBxkabVI4s612wACfK9JrIdL+uyIIzGeNHgoimjx7GuOCQ2ut6L
+gj/+Rcv3e0ERkNaXTXGkcdrsMTt45lGmyxUgxz6lbHgtqq+r+BHogiQ9cdPKwXuK
+wom4AvaMOBKCxtB4qVsuNHRd0I8OaA9Ab9SUvHCRvzCkJRHP4qc0zRJif7Rk4qRl
+rAGYwVxq3DRk2HHCQCYC26VqLU6B5LuAAqOipVhOeTfbgaSGD6Wfrt+XBBPmWB7o
+i9zDrk5GKehsPeDKgjh9uVd1y+IBHcWoYBxR5WPTYnA
+-> ssh-rsa kFDS0A
+HS3y8/5wAej0jv0sQqhdGwWd79vUwkrLkoKKPmqo9HlJaO70Cr1bnAIdyA8PBphs
+NIRjIcdClbUBuelZudzuhHuEzH8/JMAVwgHoIiUxviEIr6JpJVBagtvSHp1nfDdn
+x/hkpt2isSYb6fVzYgewqkdD3tv9XEm/WR2JmzlfaNTV9N9x/HNJYy5iYoTWRxKr
+e8R7txdmgRaDDxpbkJdWBcoV9HVgytTMtvBkqGViWzaFDopb9nDlfN/C7/BkCp6H
+9b65JqznpIChoJV7+sK5SEw8VcFj7ikIHzREWscEn8XBb7Kth2iVukaEPM+BgGZz
+Irk0IdkSb/XmQFwsOLnQViwUjkFXGXwHyMdHIcU2qEzZ4PN0PgEengILt9vqWJs0
+qHxrA7sKiC1D8S0i1+Dn+DiI//1s5Bbmp1jk626tH6fNKqSOlpwM47IGArTxCAFd
+NMinIBnR47DUCXWheirsWF6yP7kwX7vOW1dR4UVJJnVPKkgjklwCZvJiNAo0Soo+
+95zuugaeobsJ+qz2Pv+l8BGYriOFpRIAu7YTy9yY7mqHwC5MoeY0G8eNg5UmEzFz
+JsEbKPsZcsMg4WdywzLU2aufK4M/Cd7lVPGZzuZ8hJHBF/EvTFov7L/HK5VnhZar
+CYtILdyiVvmMZ7dhEARG1GG988W9wMendikmKpM4dTA
+-> piv-p256 vRzPNw AuCJRxHGmvv25VTHpnbfMLyLIj8K+daFD97wwHvFAqHl
+m4lPR+5h3+xmdL0OBfmNoxSM/O5Ca+2lVRLwITUtVmQ
+-> piv-p256 zqq/iw ApNbp/6seWw6gCj/QWKLYlmuHaqdaSKVI+Hup1fKAO2O
+xpNXgDXMVFe15eS+L9lGaI0Ip2F13SuhjCTQLDtBIr0
+-> ssh-ed25519 YFSOsg v9BtvUZh5HIvN7nsnErVrHbWTwRhWpj/SlHoiiJSIR0
+ol9z9juHfOCuZsSpuRl/zGFuJg5RzpKK1YnX+VDLDTA
+-> ssh-ed25519 iHV63A yfa9P22C7+wCMqtRRSyiOhcFnLWPI50jMWxWpLarMT8
+VpU+Uv/20JJGkTWTATiF5JImMsDKwyHMj+Wp0mMC/MQ
+-> ssh-ed25519 BVsyTA dUj/mHSyOm61h2ETa4tSX7Cyf+KRN8AMXCVKwMbJTE0
+E1EEPqVQSqlxSAi7DaMlaS1Az1D5XsasrPrIdwylSAE
+-> ssh-ed25519 +3V2lQ AWBlXeuJq76UgQR0xQVPMN0NUq/mqa2iDlIDBBp/Wjo
+XiliN4FB5YQ7qmTyV3AIbvoCY8UoGS6Vi5LpVWrH9kg
+--- V4Tp84/WJUM+/l+eEjlypE4Lx47BtkGFpEnNIdIB35E
+ô¡qzîL¡çzYNŽv×0[¿AöKN<ÛÀ<C39B>ö¦÷:,D<>¢S¹:!$`Õw©^‡àŽhi†O'(ž7!=ÿC(ÿï>´ûDÐ¾ÙFÐ=J
--- a/secrets/matrix-synapse-secret-config.yaml.age
+++ b/secrets/matrix-synapse-secret-config.yaml.age
--- a/secrets/nachtigall-coturn-static-auth-secret.age
+++ b/secrets/nachtigall-coturn-static-auth-secret.age
--- a/secrets/restic-password-droppie.age
+++ b/secrets/restic-password-droppie.age
@ -0,0 +1,29 @@
+age-encryption.org/v1
+-> ssh-ed25519 iDKjwg tSrbeVPpTxjeuCg6JiOdeIcaGWq/PUOyXrt+ZNIWmzI
+fz4B2nMu36G1DYPyNiu30Txx4cOLGyUreuBLqktORhY
+-> ssh-ed25519 uYcDNw zySKp8qUnCKOCbIP17bWYvJ/TsKxLl/v/AEsfVQpHTc
+SYqaUEXJ7mSA2wY6I3i/hirEiqvXVnWH1NYtXbaMzbw
+-> ssh-rsa kFDS0A
+MiyJjq7nn9B+PF9Lf6vD496FZUnUy5zI9H2kunJLhmYnr+7ZJaPGI+MW0cuPjsmM
+1XhmUC72zA9WTiuReGijl9GhIyU0D6vPx+ezQrx18dUGRJEqnvk5C75SCUuPGtUi
+DamPxJc781mnDSyW91MgRrqlBVwhfJ9Zmws3+/j8LODWb0T9Jvy0ywmFX4ilapr9
+6dqbNmwBZkfh2CKB9LtMG/DuSNicgHscpt4Mt7EogJRdgBrGIcA1u8BXUOwf+R8G
+Ya8VzoQV0iVhGtbff3g/6/0+UX9KEZBQc/aiT2ItrgUmgMyjAZo13Dj3pD2wPdfa
+t346symp7bcYxRea/U0hfmPdnixIKgBLTUZkzrMFza26QOvnSwiMxWY+//Vmbp3z
+0YCsyOQTQJVB+Q0pYE0+9EVI9bInP0LxHhM7mIdJroscRtY1YphjZUlFmcUXk0iC
+5g3/wdLr8KdT4aZ93m4WVgkBUzg8chhfMHb9uFvpHT9xeZHrV3u30TuJpmmuqMXK
+JD3+9xRzNVHwvfZuhD8B3vkK2tYO7pTxJytkJtdolx/uSju8SCm9F4lwt5y8rQd4
+7676QS0IWS1WItcpLrsgk9nWZUNzy78gBc39jtJUOsFDqJSdAWACq/jgYpnx6Bja
+5aZonRAp3fyCEavUpS4AQUJZGVx9EFp9LqXeUiIkCJU
+-> ssh-ed25519 YFSOsg vJ7xw4zDSKBFuv8FT6ZnB83uGwHDnDsEwsvQ6urZkEk
+gOkuwjOX8U/qmPuGrPPb+FYy6PuJjrgCNFYrukkaIPA
+-> ssh-ed25519 iHV63A FzZRHQB39iDTLm4y1QLWtfDi3jZn6i59nuSlf0d5mww
+5kiyKN7KFTv8pJgRAZTUMgR2+TInByFotDhAw307P6g
+-> ssh-ed25519 BVsyTA rTjBaEI6AwwmpK9Q8elVJaSN60deXOMEmZPrsH9O1nY
+HikNbHzx3BEJ3hn4YPFr2p/cnSu3qr9cvgdKCxN6xY8
+-> h|2u%UF\-grease Gcp2NG
+y6OjgzrOYEVgrY2+fv8a1xrVuoHQIPncGuGeLI0zHh4xf1qbzD2vKYp+W1fRBE8d
+/yPEGUe/T/ZjO+F4oF266HUR23wancFeFoGpfJgWQVS2oc0Z4aMuNebf/+Kw087W
+
+--- GJBLNxBoH7vI1mkcCmfbm9UiG4xuwIOWS7IaJKuO6cc
+C<EFBFBD>3=¼,¦{-TçÄéÈµ	™-ˆ]äÌ·»æq=‡“û
Òú–=~}Î1®˜ lß„¿fúì_[žU_å›A€G.ÒdŽêÖrëa`×åUyÉPt+Ì½òö‡¯!<
--- a/secrets/secrets.nix
+++ b/secrets/secrets.nix
@ -92,7 +92,7 @@ in
  "mediawiki-oidc-client-secret.age".publicKeys = nachtigallKeys ++ adminKeys;
  "mediawiki-secret-key.age".publicKeys = nachtigallKeys ++ adminKeys;

-  "coturn-static-auth-secret.age".publicKeys = nachtigallKeys ++ adminKeys;
+  "nachtigall-coturn-static-auth-secret.age".publicKeys = nachtigallKeys ++ adminKeys;

  "grafana-admin-password.age".publicKeys = trinkgenossinKeys ++ adminKeys;
  "grafana-keycloak-client-secret.age".publicKeys = trinkgenossinKeys ++ adminKeys;
--- a/secrets/staging-matrix-synapse-secret-config.yaml.age
+++ b/secrets/staging-matrix-synapse-secret-config.yaml.age
Author	SHA1	Message	Date
teutat3s	10cc5e914b	blackbox-exporter: scrape every 60m, add URLs All checks were successful Flake checks / Check (pull_request) Successful in 24m13s Details	2024-11-06 21:54:16 +01:00
teutat3s	a9c54dc496	style: fix formatting	2024-11-06 21:31:41 +01:00
teutat3s	84e2b22771	monitoring: scrape blackbox every 5m, alert 21 days before certificate expiry	2024-11-06 21:31:41 +01:00
teutat3s	d2806156c9	grafana: add blackbox exporter dashboard	2024-11-06 21:31:41 +01:00
Pablo Ovelleiro Corral	15783ba0e8	fix formatting	2024-11-06 21:31:40 +01:00
Pablo Ovelleiro Corral	6548edee9a	Add blackbox-exporter config	2024-11-06 21:31:40 +01:00