Add reverseproxy for alerts.pub.solar

Reviewed-on: pub-solar/infra#153
Reviewed-by: Hendrik Sokolowski <hensoko@noreply.git.pub.solar>

flake.lock

@@ -180,11 +180,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1712386041,
-        "narHash": "sha256-dA82pOMQNnCJMAsPG7AXG35VmCSMZsJHTFlTHizpKWQ=",
+        "lastModified": 1714043624,
+        "narHash": "sha256-Xn2r0Jv95TswvPlvamCC46wwNo8ALjRCMBJbGykdhcM=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "d6bb9f934f2870e5cbc5b94c79e9db22246141ff",
+        "rev": "86853e31dc1b62c6eeed11c667e8cdd0285d4411",
         "type": "github"
       },
       "original": {
@@ -224,11 +224,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1713543876,
-        "narHash": "sha256-olEWxacm1xZhAtpq+ZkEyQgR4zgfE7ddpNtZNvubi3g=",
+        "lastModified": 1713946171,
+        "narHash": "sha256-lc75rgRQLdp4Dzogv5cfqOg6qYc5Rp83oedF2t0kDp8=",
         "owner": "lnl7",
         "repo": "nix-darwin",
-        "rev": "9e7c20ffd056e406ddd0276ee9d89f09c5e5f4ed",
+        "rev": "230a197063de9287128e2c68a7a4b0cd7d0b50a7",
         "type": "github"
       },
       "original": {
@@ -255,11 +255,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1713564160,
-        "narHash": "sha256-YguPZpiejgzLEcO36/SZULjJQ55iWcjAmf3lYiyV1Fo=",
+        "lastModified": 1713995372,
+        "narHash": "sha256-fFE3M0vCoiSwCX02z8VF58jXFRj9enYUSTqjyHAjrds=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "bc194f70731cc5d2b046a6c1b3b15f170f05999c",
+        "rev": "dd37924974b9202f8226ed5d74a252a9785aedf8",
         "type": "github"
       },
       "original": {
@@ -405,11 +405,11 @@
     },
     "unstable": {
       "locked": {
-        "lastModified": 1713537308,
-        "narHash": "sha256-XtTSSIB2DA6tOv+l0FhvfDMiyCmhoRbNB+0SeInZkbk=",
+        "lastModified": 1713895582,
+        "narHash": "sha256-cfh1hi+6muQMbi9acOlju3V1gl8BEaZBXBR9jQfQi4U=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "5c24cf2f0a12ad855f444c30b2421d044120c66f",
+        "rev": "572af610f6151fd41c212f897c71f7056e3fb518",
         "type": "github"
       },
       "original": {

hosts/flora-6/apps/alert-rules.nix

@@ -0,0 +1,260 @@
+{ lib }:
+
+let
+  # docker's filesystems disappear quickly, leading to false positives
+  deviceFilter = ''path!~"^(/var/lib/docker|/nix/store).*"'';
+in
+lib.mapAttrsToList
+  (name: opts: {
+    alert = name;
+    expr = opts.condition;
+    for = opts.time or "2m";
+    labels = { };
+    annotations.description = opts.description;
+  })
+  ({
+
+    # prometheus_too_many_restarts = {
+    #   condition = ''changes(process_start_time_seconds{job=~"prometheus|alertmanager"}[15m]) > 2'';
+    #   description = "Prometheus has restarted more than twice in the last 15 minutes. It might be crashlooping.";
+    # };
+
+    # alert_manager_config_not_synced = {
+    #   condition = ''count(count_values("config_hash", alertmanager_config_hash)) > 1'';
+    #   description = "Configurations of AlertManager cluster instances are out of sync.";
+    # };
+
+    #alert_manager_e2e_dead_man_switch = {
+    #  condition = "vector(1)";
+    #  description = "Prometheus DeadManSwitch is an always-firing alert. It's used as an end-to-end test of Prometheus through the Alertmanager.";
+    #};
+
+    # prometheus_not_connected_to_alertmanager = {
+    #   condition = "prometheus_notifications_alertmanagers_discovered < 1";
+    #   description = "Prometheus cannot connect the alertmanager\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}";
+    # };
+
+    # prometheus_rule_evaluation_failures = {
+    #   condition = "increase(prometheus_rule_evaluation_failures_total[3m]) > 0";
+    #   description = "Prometheus encountered {{ $value }} rule evaluation failures, leading to potentially ignored alerts.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}";
+    # };
+
+    # prometheus_template_expansion_failures = {
+    #   condition = "increase(prometheus_template_text_expansion_failures_total[3m]) > 0";
+    #   time = "0m";
+    #   description = "Prometheus encountered {{ $value }} template text expansion failures\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}";
+    # };
+
+    # promtail_file_lagging = {
+    #   condition = ''abs(promtail_file_bytes_total - promtail_read_bytes_total) > 1e6'';
+    #   time = "15m";
+    #   description = ''{{ $labels.instance }} {{ $labels.job }} {{ $labels.path }} has been lagging by more than 1MB for more than 15m.'';
+    # };
+
+    filesystem_full_80percent = {
+      condition = ''
+        100 - ((node_filesystem_avail_bytes{fstype!="rootfs",mountpoint="/"} * 100) / node_filesystem_size_bytes{fstype!="rootfs",mountpoint="/"}) > 80'';
+      time = "10m";
+      description =
+        "{{$labels.instance}} device {{$labels.device}} on {{$labels.mountpoint}} got less than 20% space left on its filesystem.";
+    };
+
+    # filesystem_inodes_full = {
+    #   condition = ''disk_inodes_free / disk_inodes_total < 0.10'';
+    #   time = "10m";
+    #   description = "{{$labels.instance}} device {{$labels.device}} on {{$labels.mountpoint}} got less than 10% inodes left on its filesystem.";
+    # };
+
+    # daily_task_not_run = {
+    #   # give 6 hours grace period
+    #   condition = ''time() - task_last_run{state="ok",frequency="daily"} > (24 + 6) * 60 * 60'';
+    #   description = "{{$labels.instance}}: {{$labels.name}} was not run in the last 24h";
+    # };
+
+    # daily_task_failed = {
+    #   condition = ''task_last_run{state="fail"}'';
+    #   description = "{{$labels.instance}}: {{$labels.name}} failed to run";
+    # };
+    # } // (lib.genAttrs [
+    #   "borgbackup-turingmachine"
+    #   "borgbackup-eve"
+    #   "borgbackup-datastore"
+    # ]
+    #   (name: {
+    #     condition = ''absent_over_time(task_last_run{name="${name}"}[1d])'';
+    #     description = "status of ${name} is unknown: no data for a day";
+    #   }))
+    # // {
+
+    # borgbackup_matchbox_not_run = {
+    #   # give 6 hours grace period
+    #   condition = ''time() - task_last_run{state="ok",frequency="daily",name="borgbackup-matchbox"} > 7 * 24 * 60 * 60'';
+    #   description = "{{$labels.instance}}: {{$labels.name}} was not run in the last week";
+    # };
+
+    # borgbackup_matchbox = {
+    #   condition = ''absent_over_time(task_last_run{name="borgbackup-matchbox"}[7d])'';
+    #   description = "status of borgbackup-matchbox is unknown: no data for a week";
+    # };
+
+    # homeassistant = {
+    #   condition = ''
+    #     homeassistant_entity_available{domain="persistent_notification", entity!="persistent_notification.http_login"} >= 0'';
+    #   description =
+    #     "homeassistant notification {{$labels.entity}} ({{$labels.friendly_name}}): {{$value}}";
+    # };
+
+    swap_using_20percent = {
+      condition =
+        "node_memory_SwapTotal_bytes - (node_memory_SwapCached_bytes + node_memory_SwapFree_bytes) > node_memory_SwapTotal_bytes * 0.2";
+      time = "30m";
+      description =
+        "{{$labels.instance}} is using 20% of its swap space for at least 30 minutes.";
+    };
+
+    systemd_service_failed = {
+      condition = ''node_systemd_unit_state{state="failed"} == 1'';
+      description =
+        "{{$labels.instance}} failed to (re)start service {{$labels.name}}.";
+    };
+
+    restic_backup_too_old = {
+      condition = ''(time() - restic_snapshots_latest_time)/(60*60) > 24'';
+      description = "{{$labels.instance}} not backed up for more than 24 hours. ({{$value}})";
+    };
+
+    host_down = {
+      condition = ''up{job="node-stats", instance!~"ahorn.wireguard:9100|kartoffel.wireguard:9100|mega.wireguard:9100"} == 0'';
+      description = "{{$labels.instance}} is down!";
+    };
+
+    # service_not_running = {
+    #   condition = ''systemd_units_active_code{name=~"teamspeak3-server.service|tt-rss.service", sub!="running"}'';
+    #   description = "{{$labels.instance}} should have a running {{$labels.name}}.";
+    # };
+
+    ram_using_90percent = {
+      condition =
+        "node_memory_Buffers_bytes + node_memory_MemFree_bytes + node_memory_Cached_bytes < node_memory_MemTotal_bytes * 0.1";
+      time = "1h";
+      description =
+        "{{$labels.instance}} is using at least 90% of its RAM for at least 1 hour.";
+    };
+
+    cpu_using_90percent = {
+      condition = ''
+        100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) >= 90'';
+      time = "10m";
+      description =
+        "{{$labels.instance}} is running with cpu usage > 90% for at least 10 minutes: {{$value}}";
+    };
+
+    reboot = {
+      condition = "node_boot_time_seconds < 300";
+      description = "{{$labels.instance}} just rebooted.";
+    };
+
+    uptime = {
+      condition = "(time() - node_boot_time_seconds ) / (60*60*24) > 30";
+      description =
+        "Uptime monster: {{$labels.instance}} has been up for more than 30 days.";
+    };
+
+    flake_nixpkgs_outdated = {
+      condition = ''
+        (time() - flake_input_last_modified{input="nixpkgs"}) / (60*60*24) > 30'';
+      description =
+        "Nixpkgs outdated: Nixpkgs on {{$labels.instance}} has not been updated in 30 days";
+    };
+
+    /* ping = {
+      condition = "ping_result_code{type!='mobile'} != 0";
+      description = "{{$labels.url}}: ping from {{$labels.instance}} has failed!";
+      };
+
+      ping_high_latency = {
+      condition = "ping_average_response_ms{type!='mobile'} > 5000";
+      description = "{{$labels.instance}}: ping probe from {{$labels.source}} is encountering high latency!";
+      };
+    */
+    http_status = {
+      condition = ''
+        probe_http_status_code{instance!~"https://megaclan3000.de"} != 200'';
+      description =
+        "http request failed from {{$labels.instance}}: {{$labels.result}}!";
+    };
+    /* http_match_failed = {
+      condition = "http_response_response_string_match == 0";
+      description = "{{$labels.server}} : http body not as expected; status code: {{$labels.status_code}}!";
+      };
+      dns_query = {
+      condition = "dns_query_result_code != 0";
+      description = "{{$labels.domain}} : could retrieve A record {{$labels.instance}} from server {{$labels.server}}: {{$labels.result}}!";
+      };
+      secure_dns_query = {
+      condition = "secure_dns_state != 0";
+      description = "{{$labels.domain}} : could retrieve A record {{$labels.instance}} from server {{$labels.server}}: {{$labels.result}} for protocol {{$labels.protocol}}!";
+      };
+      connection_failed = {
+      condition = "net_response_result_code != 0";
+      description = "{{$labels.server}}: connection to {{$labels.port}}({{$labels.protocol}}) failed from {{$labels.instance}}";
+      };
+      healthchecks = {
+      condition = "hc_check_up == 0";
+      description = "{{$labels.instance}}: healtcheck {{$labels.job}} fails!";
+      };
+    */
+    cert_expiry = {
+      condition = "(probe_ssl_earliest_cert_expiry - time())/(3600*24) < 30";
+      description =
+        "{{$labels.instance}}: The TLS certificate will expire in less than 30 days: {{$value}}s";
+    };
+
+    # ignore devices that disabled S.M.A.R.T (example if attached via USB)
+
+    # smart_errors = {
+    #   condition = ''smart_device_health_ok{enabled!="Disabled"} != 1'';
+    #   description =
+    #     "{{$labels.instance}}: S.M.A.R.T reports: {{$labels.device}} ({{$labels.model}}) has errors.";
+    # };
+
+    oom_kills = {
+      condition = "increase(node_vmstat_oom_kill[5m]) > 0";
+      description = "{{$labels.instance}}: OOM kill detected";
+    };
+
+    /* unusual_disk_read_latency = {
+      condition =
+      "rate(diskio_read_time[1m]) / rate(diskio_reads[1m]) > 0.1 and rate(diskio_reads[1m]) > 0";
+      description = ''
+      {{$labels.instance}}: Disk latency is growing (read operations > 100ms)
+      '';
+      };
+
+      unusual_disk_write_latency = {
+      condition =
+      "rate(diskio_write_time[1m]) / rate(diskio_write[1m]) > 0.1 and rate(diskio_write[1m]) > 0";
+      description = ''
+      {{$labels.instance}}: Disk latency is growing (write operations > 100ms)
+      '';
+      };
+    */
+
+    host_memory_under_memory_pressure = {
+      condition = "rate(node_vmstat_pgmajfault[1m]) > 1000";
+      description =
+        "{{$labels.instance}}: The node is under heavy memory pressure. High rate of major page faults: {{$value}}";
+    };
+
+    # ext4_errors = {
+    #   condition = "ext4_errors_value > 0";
+    #   description =
+    #     "{{$labels.instance}}: ext4 has reported {{$value}} I/O errors: check /sys/fs/ext4/*/errors_count";
+    # };
+
+    # alerts_silences_changed = {
+    #   condition = ''abs(delta(alertmanager_silences{state="active"}[1h])) >= 1'';
+    #   description =
+    #     "alertmanager: number of active silences has changed: {{$value}}";
+    # };
+  })

hosts/flora-6/apps/caddy.nix

@@ -37,6 +37,14 @@
                     reverse_proxy :${toString config.services.loki.configuration.server.http_listen_port}
         '';
       };
+      "alerts.pub.solar" = {
+        logFormat = lib.mkForce ''
+          output discard
+        '';
+        extraConfig = ''
+        reverse_proxy 10.7.6.2:${toString config.services.prometheus.alertmanager.port}
+        '';
+      };
       "grafana.pub.solar" = {
         logFormat = lib.mkForce ''
           output discard

hosts/flora-6/apps/prometheus.nix

@@ -65,5 +65,50 @@
         }];
       }
     ];
+
+    ruleFiles = [
+      (pkgs.writeText "prometheus-rules.yml" (builtins.toJSON {
+        groups = [{
+          name = "alerting-rules";
+          rules = import ./alert-rules.nix { inherit lib; };
+        }];
+      }))
+    ];
+
+    alertmanagers = [{ static_configs = [{ targets = [ "localhost:9093" ]; }]; }];
+
+    alertmanager = {
+      enable = true;
+      # port = 9093; # Default
+      webExternalUrl = "https://alerts.pub.solar"; # TODO use a proper url?
+      # environmentFile = "${config.age.secrets.nachtigall-alertmanager-envfile.path}";
+      configuration = {
+
+        route = {
+          receiver = "all";
+          group_by = [ "instance" ];
+          group_wait = "30s";
+          group_interval = "2m";
+          repeat_interval = "24h";
+        };
+
+        receivers = [{
+          name = "all";
+          # Email config documentation: https://prometheus.io/docs/alerting/latest/configuration/#email_config
+          email_configs = [{
+            send_resolved = true;
+            to = "TODO";
+            from = "alerts@pub.solar";
+            smarthost = "TODO";
+            auth_username = "TODO";
+            auth_password_file = "${config.age.secrets.nachtigall-alertmanager-smtp-password.path}";
+            require_tls = true;
+          }];
+          # TODO:
+          # For matrix notifications, look into: https://github.com/pinpox/matrix-hook and add a webhook
+          #   webhook_configs = [ { url = "http://127.0.0.1:11000/alert"; } ];
+        }];
+      };
+    };
   };
 }

hosts/nachtigall/apps/matrix/irc.nix

@@ -13,11 +13,6 @@ let
   synapseClientPort = "${toString listenerWithClient.port}";
 in
 {
-  systemd.services.matrix-appservice-irc.serviceConfig.SystemCallFilter = lib.mkForce [
-    "@system-service @pkey"
-    "~@privileged @resources"
-    "@chown"
-  ];
   services.matrix-appservice-irc = {
     enable = true;
     localpart = "irc_bot";

hosts/nachtigall/apps/obs-portal.nix

@@ -1,140 +0,0 @@
-{ config
-, lib
-, pkgs
-, self
-, flake
-, ...
-}: let 
-  configPy = pkgs.writeText "obs-portal-config.py" ''
-DEBUG = False
-VERBOSE = DEBUG
-AUTO_RESTART = DEBUG
-LEAN_MODE = False
-FRONTEND_URL = None
-FRONTEND_HTTPS = True
-FRONTEND_DIR = "../frontend/build/"
-FRONTEND_CONFIG = {
-    "imprintUrl": "https://pub.solar/about",
-    "privacyPolicyUrl": "https://pub.solar/privacy",
-    "mapHome": {"zoom": 12, "latitude": 50.93, "longitude": 6.97},
-    "banner": {
-        "text": "This is an installation serving the Cologne/Bonn region run for Team OBSKöln by pub.solar n.e.V.",
-        "style": "info"
-    },
-}
-TILES_FILE = None
-ADDITIONAL_CORS_ORIGINS = None
-  '';
-
-  env = {
-    OBS_KEYCLOAK_URI = "auth.pub.solar";
-    OBS_PORTAL_URI = "obs-portal.pub.solar";
-
-    OBS_POSTGRES_MAX_OVERFLOW = "20";
-    OBS_POSTGRES_POOL_SIZE = "40";
-
-    OBS_HOST = "0.0.0.0";
-    OBS_PORT = "3000";
-    OBS_KEYCLOAK_URL = "https://auth.pub.solar/realms/pub.solar/";
-    OBS_KEYCLOAK_CLIENT_ID = "openbikesensor-portal";
-    OBS_DEDICATED_WORKER = "True";
-    OBS_DATA_DIR = "/data";
-    OBS_PROXIES_COUNT = "1";
-  };
-in {
-  age.secrets.obs-portal-env = {
-    file = "${flake.self}/secrets/obs-portal-env.age";
-    mode = "600";
-  };
-
-  age.secrets.obs-portal-database-env = {
-    file = "${flake.self}/secrets/obs-portal-database-env.age";
-    mode = "600";
-  };
-
-  systemd.services."docker-network-obs-portal" =
-    let
-      docker = config.virtualisation.oci-containers.backend;
-      dockerBin = "${pkgs.${docker}}/bin/${docker}";
-    in
-    {
-      serviceConfig.Type = "oneshot";
-      before = [ "docker-obs-portal.service" ];
-      script = ''
-        ${dockerBin} network inspect obs-portal-net >/dev/null 2>&1 || ${dockerBin} network create obs-portal-net --subnet 172.20.0.0/24
-      '';
-    };
-
-  services.nginx.virtualHosts."obs-portal.pub.solar" = {
-    enableACME = true;
-    forceSSL = true;
-
-    locations."/" = {
-      proxyWebsockets = true;
-      extraConfig = ''
-        proxy_pass http://127.0.0.1:3001;
-        proxy_set_header Host $host;
-      '';
-    };
-  };
-
-  virtualisation = {
-    oci-containers = {
-      backend = "docker";
-
-      containers."obs-portal" = {
-        image = "git.pub.solar/pub-solar/obs-portal:latest";
-        autoStart = true;
-        ports = [ "localhost:3001:${env.OBS_PORT}" ];
-
-        environment = env;
-        environmentFiles = [ config.age.secrets.obs-portal-env.path ];
-
-        volumes = [
-          "${configPy}:/opt/obs/api/config.py"
-          "/var/lib/obs-portal${env.OBS_DATA_DIR}:${env.OBS_DATA_DIR}"
-          "/var/lib/obs-portal/tiles/:/tiles"
-          "/var/lib/obs-portal/pbf/:/pbf"
-        ];
-
-        extraOptions = [
-          "--network=obs-portal-net"
-        ];
-      };
-
-      containers."obs-portal-worker" = {
-        image = "git.pub.solar/pub-solar/obs-portal:latest";
-        autoStart = true;
-
-        cmd = [ "python" "tools/process_track.py" ];
-
-        environment = env;
-        environmentFiles = [ config.age.secrets.obs-portal-env.path ];
-
-        volumes = [
-          "${configPy}:/opt/obs/api/config.py"
-          "/var/lib/obs-portal${env.OBS_DATA_DIR}:${env.OBS_DATA_DIR}"
-        ];
-
-        extraOptions = [
-          "--network=obs-portal-net"
-        ];
-      };
-
-      containers."obs-portal-db" = {
-        image = "openmaptiles/postgis:7.0";
-        autoStart = true;
-
-        environmentFiles = [ config.age.secrets.obs-portal-database-env.path ];
-
-        volumes = [
-          "/var/lib/postgres-obs-portal/data:/var/lib/postgresql/data"
-        ];
-
-        extraOptions = [
-          "--network=obs-portal-net"
-        ];
-      };
-    };
-  };
-}

hosts/nachtigall/default.nix

@@ -32,7 +32,6 @@
       ./apps/promtail.nix
       ./apps/searx.nix
       ./apps/tmate.nix
-      ./apps/obs-portal.nix
 
       ./apps/matrix/irc.nix
       ./apps/matrix/mautrix-telegram.nix

secrets/obs-portal-database-env.age

@@ -1,27 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 iDKjwg hAoEiOaK1U0HImALePEYHiE6xebOOqtVujaBWgNBZF8
-ecf/ykqYPihRJxI/Y7Oh6QhWSyncwevlzEZoRqm3aGM
--> ssh-ed25519 uYcDNw NcIttsTn6wPCmoOYGtZ66IYhthjLDI3sYFe4pbW6cB4
-9hv4dEYoXXWSZ2pG1hy68vmTf++v+g3q7wVhT6cAog0
--> ssh-rsa kFDS0A
-KoW3J2Tw90chM6Oy17umOQN0WFI4je7CBk3IgdImsd4Mz5q17/nXlhVlFFhx4ZEk
-Or9LaqytVk1NA6J4+suMRlx4Pd6oberXu1KBkFQMr1B3LKhNOaOZ+W1mrbQLGG9U
-YUTyOpkHxVkw0IOsvxB/0reMCHtjKHo661zFjim1YFmEk0WRt4hU1XqsMNiE4wbc
-GF0t9EWMN2pU2p7DpX/DzVTqu8yk8SQhCZc9kfzWcuawwf0rcjwUJ/Rk1MH5tMpK
-odRXXl1slPPwQinE+KJqeyrfuRDHqwqmxnOfOWG6KQwWkVSE1btiHEvfuuLOjSjl
-3wO+veRC9hW5sSCPANoFbuSQ1dprmoyaZnOyeRTbgw91ks/ogLBezF/KSkaMQeHx
-XRnfcceBmeeqHl9L3Z+3EmBjwIqu2Og0pvhDU8G/ZeA0cHS/22QYGzeD/gOqaEW7
-d1VyA6LZd8PxIjoBamdipIpY0TqZ8+cA/yaUKNnYXXRSlKQ5ggPxh7ZXfvRbGg+m
-WbNiHxBPcTK7/Bpzes4LJVcx0Ar4XeDxVQe1MITLpFWh+FDEQZEA3630JngZ153J
-vBvw+VFedPSr6Ov+/33/J3LKC0XRatGnc++AWfo4rWPLCE6qovEDyY+wmct8gv0j
-rMEK7OaNfyy+Z21mjrkwcEUbyoGt9ksEplaRblE0Lsk
--> ssh-ed25519 YFSOsg LmLRtBYMSzjid3VkUgAQvDOS9r0imWSKE7fm0t/x41Y
-0mae0vsNmaS5aVOKezXit7KV44JKLpU+GWpuA++dCVo
--> ssh-ed25519 iHV63A Tc2z2JciftAikoj4Hv9IBgkcYWAcyGuPJTNA3Yw2K1w
-cO5o/pbaZAtTvXUskOah9vWP/Tuvyi3QDM7g4AQ+b8s
--> ssh-ed25519 BVsyTA mk6n6ytaI4V9JVoUZFtwfFOgaLYc6gvVOcSZXQj/FVI
-etqbUCqe0eY81qaVco7pMJjhfM+sA/bXLMW0bEsCLxI
---- CmNq6ZPxFoFTsySVfr7BTHV0tm9cbRYGG6IR7DNgbEY
-!è烈í}
-ùSê<>ŸSl®Ds;!ÁjršZçR"—ë#ž­¿»ÙÅ~!›Ÿ¤6AùwEn ? kËAcx~—ŽÜGVæ&M¯ý¾ä,
-a›U

secrets/secrets.nix

@@ -1,5 +1,4 @@
 let
-<<<<<<< HEAD
   admins = import ../logins/admins.nix;
 
   nachtigall-host = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP7G0ufi+MNvaAZLDgpieHrABPGN7e/kD5kMFwSk4ABj root@nachtigall";
@@ -65,7 +64,4 @@ in
 
   "nachtigall-metrics-nginx-basic-auth.age".publicKeys = nachtigallKeys ++ adminKeys;
   "nachtigall-metrics-prometheus-basic-auth-password.age".publicKeys = flora6Keys ++ nachtigallKeys ++ adminKeys;
-
-  "obs-portal-env.age".publicKeys = nachtigallKeys ++ adminKeys;
-  "obs-portal-database-env.age".publicKeys = nachtigallKeys ++ adminKeys;
 }

terraform/dns.tf

@@ -1,181 +1,186 @@
 # https://registry.terraform.io/providers/namecheap/namecheap/latest/docs
 resource "namecheap_domain_records" "pub-solar" {
-  domain = "pub.solar"
-  mode = "OVERWRITE"
+  domain     = "pub.solar"
+  mode       = "OVERWRITE"
   email_type = "MX"
 
   record {
     hostname = "flora-6"
-    type = "A"
-    address = "80.71.153.210"
+    type     = "A"
+    address  = "80.71.153.210"
   }
   record {
     hostname = "auth"
-    type = "CNAME"
-    address = "nachtigall.pub.solar."
+    type     = "CNAME"
+    address  = "nachtigall.pub.solar."
   }
   record {
     hostname = "ci"
-    type = "A"
-    address = "80.71.153.210"
+    type     = "A"
+    address  = "80.71.153.210"
+  }
+  record {
+    hostname = "alerts"
+    type     = "CNAME"
+    address  = "flora-6.pub.solar."
   }
   record {
     hostname = "git"
-    type = "CNAME"
-    address = "nachtigall.pub.solar."
+    type     = "CNAME"
+    address  = "nachtigall.pub.solar."
   }
   record {
     hostname = "stream"
-    type = "CNAME"
-    address = "nachtigall.pub.solar."
+    type     = "CNAME"
+    address  = "nachtigall.pub.solar."
   }
   record {
     hostname = "list"
-    type = "CNAME"
-    address = "nachtigall.pub.solar."
+    type     = "CNAME"
+    address  = "nachtigall.pub.solar."
   }
   record {
     hostname = "obs-portal"
-    type = "A"
-    address = "80.71.153.210"
+    type     = "A"
+    address  = "80.71.153.210"
   }
   record {
     hostname = "vpn"
-    type = "A"
-    address = "80.71.153.210"
+    type     = "A"
+    address  = "80.71.153.210"
   }
   record {
     hostname = "cache"
-    type = "A"
-    address = "95.217.225.160"
+    type     = "A"
+    address  = "95.217.225.160"
   }
   record {
     hostname = "factorio"
-    type = "A"
-    address = "80.244.242.2"
+    type     = "A"
+    address  = "80.244.242.2"
   }
   record {
     hostname = "collabora"
-    type = "CNAME"
-    address = "nachtigall.pub.solar."
+    type     = "CNAME"
+    address  = "nachtigall.pub.solar."
   }
   record {
     hostname = "@"
-    type = "ALIAS"
-    address = "nachtigall.pub.solar."
-    ttl = 300
+    type     = "ALIAS"
+    address  = "nachtigall.pub.solar."
+    ttl      = 300
   }
   record {
     hostname = "chat"
-    type = "CNAME"
-    address = "nachtigall.pub.solar."
+    type     = "CNAME"
+    address  = "nachtigall.pub.solar."
   }
   record {
     hostname = "cloud"
-    type = "CNAME"
-    address = "nachtigall.pub.solar."
+    type     = "CNAME"
+    address  = "nachtigall.pub.solar."
   }
   record {
     hostname = "turn"
-    type = "CNAME"
-    address = "nachtigall.pub.solar."
+    type     = "CNAME"
+    address  = "nachtigall.pub.solar."
   }
   record {
     hostname = "grafana"
-    type = "A"
-    address = "80.71.153.210"
+    type     = "A"
+    address  = "80.71.153.210"
   }
   record {
     hostname = "hpb"
-    type = "A"
-    address = "80.71.153.239"
+    type     = "A"
+    address  = "80.71.153.239"
   }
   record {
     hostname = "files"
-    type = "CNAME"
-    address = "nachtigall.pub.solar."
+    type     = "CNAME"
+    address  = "nachtigall.pub.solar."
   }
   record {
     hostname = "search"
-    type = "CNAME"
-    address = "nachtigall.pub.solar."
+    type     = "CNAME"
+    address  = "nachtigall.pub.solar."
   }
   record {
     hostname = "wiki"
-    type = "CNAME"
-    address = "nachtigall.pub.solar."
+    type     = "CNAME"
+    address  = "nachtigall.pub.solar."
   }
   record {
     hostname = "mastodon"
-    type = "CNAME"
-    address = "nachtigall.pub.solar."
+    type     = "CNAME"
+    address  = "nachtigall.pub.solar."
   }
   record {
     hostname = "matrix"
-    type = "CNAME"
-    address = "nachtigall.pub.solar."
+    type     = "CNAME"
+    address  = "nachtigall.pub.solar."
   }
   record {
     hostname = "tmate"
-    type = "CNAME"
-    address = "nachtigall.pub.solar."
+    type     = "CNAME"
+    address  = "nachtigall.pub.solar."
   }
   record {
     hostname = "www"
-    type = "CNAME"
-    address = "nachtigall.pub.solar."
+    type     = "CNAME"
+    address  = "nachtigall.pub.solar."
   }
   record {
     hostname = "@"
-    type = "TXT"
-    address = "v=spf1 include:spf.greenbaum.zone a:list.pub.solar ~all"
+    type     = "TXT"
+    address  = "v=spf1 include:spf.greenbaum.zone a:list.pub.solar ~all"
   }
   record {
     hostname = "list"
-    type = "TXT"
-    address = "v=spf1 a:list.pub.solar ?all"
+    type     = "TXT"
+    address  = "v=spf1 a:list.pub.solar ?all"
   }
   record {
     hostname = "_dmarc"
-    type = "TXT"
-    address = "v=DMARC1; p=reject;"
+    type     = "TXT"
+    address  = "v=DMARC1; p=reject;"
   }
   record {
     hostname = "_dmarc.list"
-    type = "TXT"
-    address = "v=DMARC1; p=reject;"
+    type     = "TXT"
+    address  = "v=DMARC1; p=reject;"
   }
   record {
     hostname = "modoboa._domainkey"
-    type = "TXT"
-    address = "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx/EqLMpk0MyL1aQ0JVG44ypTRbZBVA13MFjEntxAvowaWtq1smRbnEwTTKgqUOrUyaM4dVmli1dedne4mk/ncqRAm02KuhtTY+5wXfhTKK53EhqehbKwH+Qvzb12983Qwdau/QTHiFHwXHufMaSsCvd9CRWCp9q68Q7noQqndJeLHT6L0eECd2Zk3ZxJuh+Fxdb7+Kw68Tf6z13Rs+MU01qLM7x0jmSQHa4cv2pk+7NTGMBRp6fVskfbqev5nFkZWJ7rhXEbP9Eukd/L3ro/ubs1quWJotG02gPRKE8fgkm1Ytlws1/pnqpuvKXQS1HzBEP1X2ExezJMzQ1SnZCigQIDAQAB"
+    type     = "TXT"
+    address  = "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx/EqLMpk0MyL1aQ0JVG44ypTRbZBVA13MFjEntxAvowaWtq1smRbnEwTTKgqUOrUyaM4dVmli1dedne4mk/ncqRAm02KuhtTY+5wXfhTKK53EhqehbKwH+Qvzb12983Qwdau/QTHiFHwXHufMaSsCvd9CRWCp9q68Q7noQqndJeLHT6L0eECd2Zk3ZxJuh+Fxdb7+Kw68Tf6z13Rs+MU01qLM7x0jmSQHa4cv2pk+7NTGMBRp6fVskfbqev5nFkZWJ7rhXEbP9Eukd/L3ro/ubs1quWJotG02gPRKE8fgkm1Ytlws1/pnqpuvKXQS1HzBEP1X2ExezJMzQ1SnZCigQIDAQAB"
   }
   record {
     hostname = "@"
-    type = "MX"
-    address = "mail.greenbaum.zone."
-    mx_pref = "0"
+    type     = "MX"
+    address  = "mail.greenbaum.zone."
+    mx_pref  = "0"
   }
   record {
     hostname = "list"
-    type = "MX"
-    address = "list.pub.solar."
-    mx_pref = "0"
+    type     = "MX"
+    address  = "list.pub.solar."
+    mx_pref  = "0"
   }
   record {
     hostname = "nachtigall"
-    type = "A"
-    address = "138.201.80.102"
+    type     = "A"
+    address  = "138.201.80.102"
   }
   record {
     hostname = "nachtigall"
-    type = "AAAA"
-    address = "2a01:4f8:172:1c25::1"
+    type     = "AAAA"
+    address  = "2a01:4f8:172:1c25::1"
   }
   record {
     hostname = "matrix.test"
-    type = "CNAME"
-    address = "nachtigall.pub.solar."
+    type     = "CNAME"
+    address  = "nachtigall.pub.solar."
   }
   # SRV records can only be changed via NameCheap Web UI
   # add comment