obs-portal: init obs-portal on nachtigall

This follows the official installation instructions at https://github.com/openbikesensor/portal/blob/main/docs/production-deployment.md Unfortunately, the postgres database needs to have postgis enabled, so we'll have to start a second instance. To stay close to the official deployment instructions, this is running in docker. The secrets were taken from the old installation instance. During initial installation, we'll need to import data from the old instance into this one, which might take a while.
2024-04-23 23:47:30 +02:00
11 changed files with 262 additions and 403 deletions
--- a/flake.lock
+++ b/flake.lock
@ -180,11 +180,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1714043624,
-        "narHash": "sha256-Xn2r0Jv95TswvPlvamCC46wwNo8ALjRCMBJbGykdhcM=",
+        "lastModified": 1712386041,
+        "narHash": "sha256-dA82pOMQNnCJMAsPG7AXG35VmCSMZsJHTFlTHizpKWQ=",
        "owner": "nix-community",
        "repo": "home-manager",
-        "rev": "86853e31dc1b62c6eeed11c667e8cdd0285d4411",
+        "rev": "d6bb9f934f2870e5cbc5b94c79e9db22246141ff",
        "type": "github"
      },
      "original": {
@ -224,11 +224,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1713946171,
-        "narHash": "sha256-lc75rgRQLdp4Dzogv5cfqOg6qYc5Rp83oedF2t0kDp8=",
+        "lastModified": 1713543876,
+        "narHash": "sha256-olEWxacm1xZhAtpq+ZkEyQgR4zgfE7ddpNtZNvubi3g=",
        "owner": "lnl7",
        "repo": "nix-darwin",
-        "rev": "230a197063de9287128e2c68a7a4b0cd7d0b50a7",
+        "rev": "9e7c20ffd056e406ddd0276ee9d89f09c5e5f4ed",
        "type": "github"
      },
      "original": {
@ -255,11 +255,11 @@
    },
    "nixpkgs": {
      "locked": {
-        "lastModified": 1713995372,
-        "narHash": "sha256-fFE3M0vCoiSwCX02z8VF58jXFRj9enYUSTqjyHAjrds=",
+        "lastModified": 1713564160,
+        "narHash": "sha256-YguPZpiejgzLEcO36/SZULjJQ55iWcjAmf3lYiyV1Fo=",
        "owner": "nixos",
        "repo": "nixpkgs",
-        "rev": "dd37924974b9202f8226ed5d74a252a9785aedf8",
+        "rev": "bc194f70731cc5d2b046a6c1b3b15f170f05999c",
        "type": "github"
      },
      "original": {
@ -405,11 +405,11 @@
    },
    "unstable": {
      "locked": {
-        "lastModified": 1713895582,
-        "narHash": "sha256-cfh1hi+6muQMbi9acOlju3V1gl8BEaZBXBR9jQfQi4U=",
+        "lastModified": 1713537308,
+        "narHash": "sha256-XtTSSIB2DA6tOv+l0FhvfDMiyCmhoRbNB+0SeInZkbk=",
        "owner": "nixos",
        "repo": "nixpkgs",
-        "rev": "572af610f6151fd41c212f897c71f7056e3fb518",
+        "rev": "5c24cf2f0a12ad855f444c30b2421d044120c66f",
        "type": "github"
      },
      "original": {
--- a/hosts/flora-6/apps/alert-rules.nix
+++ b/hosts/flora-6/apps/alert-rules.nix
@ -1,260 +0,0 @@
-{ lib }:
-
-let
-  # docker's filesystems disappear quickly, leading to false positives
-  deviceFilter = ''path!~"^(/var/lib/docker|/nix/store).*"'';
-in
-lib.mapAttrsToList
-  (name: opts: {
-    alert = name;
-    expr = opts.condition;
-    for = opts.time or "2m";
-    labels = { };
-    annotations.description = opts.description;
-  })
-  ({
-
-    # prometheus_too_many_restarts = {
-    #   condition = ''changes(process_start_time_seconds{job=~"prometheus|alertmanager"}[15m]) > 2'';
-    #   description = "Prometheus has restarted more than twice in the last 15 minutes. It might be crashlooping.";
-    # };
-
-    # alert_manager_config_not_synced = {
-    #   condition = ''count(count_values("config_hash", alertmanager_config_hash)) > 1'';
-    #   description = "Configurations of AlertManager cluster instances are out of sync.";
-    # };
-
-    #alert_manager_e2e_dead_man_switch = {
-    #  condition = "vector(1)";
-    #  description = "Prometheus DeadManSwitch is an always-firing alert. It's used as an end-to-end test of Prometheus through the Alertmanager.";
-    #};
-
-    # prometheus_not_connected_to_alertmanager = {
-    #   condition = "prometheus_notifications_alertmanagers_discovered < 1";
-    #   description = "Prometheus cannot connect the alertmanager\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}";
-    # };
-
-    # prometheus_rule_evaluation_failures = {
-    #   condition = "increase(prometheus_rule_evaluation_failures_total[3m]) > 0";
-    #   description = "Prometheus encountered {{ $value }} rule evaluation failures, leading to potentially ignored alerts.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}";
-    # };
-
-    # prometheus_template_expansion_failures = {
-    #   condition = "increase(prometheus_template_text_expansion_failures_total[3m]) > 0";
-    #   time = "0m";
-    #   description = "Prometheus encountered {{ $value }} template text expansion failures\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}";
-    # };
-
-    # promtail_file_lagging = {
-    #   condition = ''abs(promtail_file_bytes_total - promtail_read_bytes_total) > 1e6'';
-    #   time = "15m";
-    #   description = ''{{ $labels.instance }} {{ $labels.job }} {{ $labels.path }} has been lagging by more than 1MB for more than 15m.'';
-    # };
-
-    filesystem_full_80percent = {
-      condition = ''
-        100 - ((node_filesystem_avail_bytes{fstype!="rootfs",mountpoint="/"} * 100) / node_filesystem_size_bytes{fstype!="rootfs",mountpoint="/"}) > 80'';
-      time = "10m";
-      description =
-        "{{$labels.instance}} device {{$labels.device}} on {{$labels.mountpoint}} got less than 20% space left on its filesystem.";
-    };
-
-    # filesystem_inodes_full = {
-    #   condition = ''disk_inodes_free / disk_inodes_total < 0.10'';
-    #   time = "10m";
-    #   description = "{{$labels.instance}} device {{$labels.device}} on {{$labels.mountpoint}} got less than 10% inodes left on its filesystem.";
-    # };
-
-    # daily_task_not_run = {
-    #   # give 6 hours grace period
-    #   condition = ''time() - task_last_run{state="ok",frequency="daily"} > (24 + 6) * 60 * 60'';
-    #   description = "{{$labels.instance}}: {{$labels.name}} was not run in the last 24h";
-    # };
-
-    # daily_task_failed = {
-    #   condition = ''task_last_run{state="fail"}'';
-    #   description = "{{$labels.instance}}: {{$labels.name}} failed to run";
-    # };
-    # } // (lib.genAttrs [
-    #   "borgbackup-turingmachine"
-    #   "borgbackup-eve"
-    #   "borgbackup-datastore"
-    # ]
-    #   (name: {
-    #     condition = ''absent_over_time(task_last_run{name="${name}"}[1d])'';
-    #     description = "status of ${name} is unknown: no data for a day";
-    #   }))
-    # // {
-
-    # borgbackup_matchbox_not_run = {
-    #   # give 6 hours grace period
-    #   condition = ''time() - task_last_run{state="ok",frequency="daily",name="borgbackup-matchbox"} > 7 * 24 * 60 * 60'';
-    #   description = "{{$labels.instance}}: {{$labels.name}} was not run in the last week";
-    # };
-
-    # borgbackup_matchbox = {
-    #   condition = ''absent_over_time(task_last_run{name="borgbackup-matchbox"}[7d])'';
-    #   description = "status of borgbackup-matchbox is unknown: no data for a week";
-    # };
-
-    # homeassistant = {
-    #   condition = ''
-    #     homeassistant_entity_available{domain="persistent_notification", entity!="persistent_notification.http_login"} >= 0'';
-    #   description =
-    #     "homeassistant notification {{$labels.entity}} ({{$labels.friendly_name}}): {{$value}}";
-    # };
-
-    swap_using_20percent = {
-      condition =
-        "node_memory_SwapTotal_bytes - (node_memory_SwapCached_bytes + node_memory_SwapFree_bytes) > node_memory_SwapTotal_bytes * 0.2";
-      time = "30m";
-      description =
-        "{{$labels.instance}} is using 20% of its swap space for at least 30 minutes.";
-    };
-
-    systemd_service_failed = {
-      condition = ''node_systemd_unit_state{state="failed"} == 1'';
-      description =
-        "{{$labels.instance}} failed to (re)start service {{$labels.name}}.";
-    };
-
-    restic_backup_too_old = {
-      condition = ''(time() - restic_snapshots_latest_time)/(60*60) > 24'';
-      description = "{{$labels.instance}} not backed up for more than 24 hours. ({{$value}})";
-    };
-
-    host_down = {
-      condition = ''up{job="node-stats", instance!~"ahorn.wireguard:9100|kartoffel.wireguard:9100|mega.wireguard:9100"} == 0'';
-      description = "{{$labels.instance}} is down!";
-    };
-
-    # service_not_running = {
-    #   condition = ''systemd_units_active_code{name=~"teamspeak3-server.service|tt-rss.service", sub!="running"}'';
-    #   description = "{{$labels.instance}} should have a running {{$labels.name}}.";
-    # };
-
-    ram_using_90percent = {
-      condition =
-        "node_memory_Buffers_bytes + node_memory_MemFree_bytes + node_memory_Cached_bytes < node_memory_MemTotal_bytes * 0.1";
-      time = "1h";
-      description =
-        "{{$labels.instance}} is using at least 90% of its RAM for at least 1 hour.";
-    };
-
-    cpu_using_90percent = {
-      condition = ''
-        100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) >= 90'';
-      time = "10m";
-      description =
-        "{{$labels.instance}} is running with cpu usage > 90% for at least 10 minutes: {{$value}}";
-    };
-
-    reboot = {
-      condition = "node_boot_time_seconds < 300";
-      description = "{{$labels.instance}} just rebooted.";
-    };
-
-    uptime = {
-      condition = "(time() - node_boot_time_seconds ) / (60*60*24) > 30";
-      description =
-        "Uptime monster: {{$labels.instance}} has been up for more than 30 days.";
-    };
-
-    flake_nixpkgs_outdated = {
-      condition = ''
-        (time() - flake_input_last_modified{input="nixpkgs"}) / (60*60*24) > 30'';
-      description =
-        "Nixpkgs outdated: Nixpkgs on {{$labels.instance}} has not been updated in 30 days";
-    };
-
-    /* ping = {
-      condition = "ping_result_code{type!='mobile'} != 0";
-      description = "{{$labels.url}}: ping from {{$labels.instance}} has failed!";
-      };
-
-      ping_high_latency = {
-      condition = "ping_average_response_ms{type!='mobile'} > 5000";
-      description = "{{$labels.instance}}: ping probe from {{$labels.source}} is encountering high latency!";
-      };
-    */
-    http_status = {
-      condition = ''
-        probe_http_status_code{instance!~"https://megaclan3000.de"} != 200'';
-      description =
-        "http request failed from {{$labels.instance}}: {{$labels.result}}!";
-    };
-    /* http_match_failed = {
-      condition = "http_response_response_string_match == 0";
-      description = "{{$labels.server}} : http body not as expected; status code: {{$labels.status_code}}!";
-      };
-      dns_query = {
-      condition = "dns_query_result_code != 0";
-      description = "{{$labels.domain}} : could retrieve A record {{$labels.instance}} from server {{$labels.server}}: {{$labels.result}}!";
-      };
-      secure_dns_query = {
-      condition = "secure_dns_state != 0";
-      description = "{{$labels.domain}} : could retrieve A record {{$labels.instance}} from server {{$labels.server}}: {{$labels.result}} for protocol {{$labels.protocol}}!";
-      };
-      connection_failed = {
-      condition = "net_response_result_code != 0";
-      description = "{{$labels.server}}: connection to {{$labels.port}}({{$labels.protocol}}) failed from {{$labels.instance}}";
-      };
-      healthchecks = {
-      condition = "hc_check_up == 0";
-      description = "{{$labels.instance}}: healtcheck {{$labels.job}} fails!";
-      };
-    */
-    cert_expiry = {
-      condition = "(probe_ssl_earliest_cert_expiry - time())/(3600*24) < 30";
-      description =
-        "{{$labels.instance}}: The TLS certificate will expire in less than 30 days: {{$value}}s";
-    };
-
-    # ignore devices that disabled S.M.A.R.T (example if attached via USB)
-
-    # smart_errors = {
-    #   condition = ''smart_device_health_ok{enabled!="Disabled"} != 1'';
-    #   description =
-    #     "{{$labels.instance}}: S.M.A.R.T reports: {{$labels.device}} ({{$labels.model}}) has errors.";
-    # };
-
-    oom_kills = {
-      condition = "increase(node_vmstat_oom_kill[5m]) > 0";
-      description = "{{$labels.instance}}: OOM kill detected";
-    };
-
-    /* unusual_disk_read_latency = {
-      condition =
-      "rate(diskio_read_time[1m]) / rate(diskio_reads[1m]) > 0.1 and rate(diskio_reads[1m]) > 0";
-      description = ''
-      {{$labels.instance}}: Disk latency is growing (read operations > 100ms)
-      '';
-      };
-
-      unusual_disk_write_latency = {
-      condition =
-      "rate(diskio_write_time[1m]) / rate(diskio_write[1m]) > 0.1 and rate(diskio_write[1m]) > 0";
-      description = ''
-      {{$labels.instance}}: Disk latency is growing (write operations > 100ms)
-      '';
-      };
-    */
-
-    host_memory_under_memory_pressure = {
-      condition = "rate(node_vmstat_pgmajfault[1m]) > 1000";
-      description =
-        "{{$labels.instance}}: The node is under heavy memory pressure. High rate of major page faults: {{$value}}";
-    };
-
-    # ext4_errors = {
-    #   condition = "ext4_errors_value > 0";
-    #   description =
-    #     "{{$labels.instance}}: ext4 has reported {{$value}} I/O errors: check /sys/fs/ext4/*/errors_count";
-    # };
-
-    # alerts_silences_changed = {
-    #   condition = ''abs(delta(alertmanager_silences{state="active"}[1h])) >= 1'';
-    #   description =
-    #     "alertmanager: number of active silences has changed: {{$value}}";
-    # };
-  })
--- a/hosts/flora-6/apps/caddy.nix
+++ b/hosts/flora-6/apps/caddy.nix
@ -37,14 +37,6 @@
                    reverse_proxy :${toString config.services.loki.configuration.server.http_listen_port}
        '';
      };
-      "alerts.pub.solar" = {
-        logFormat = lib.mkForce ''
-          output discard
-        '';
-        extraConfig = ''
-        reverse_proxy 10.7.6.2:${toString config.services.prometheus.alertmanager.port}
-        '';
-      };
      "grafana.pub.solar" = {
        logFormat = lib.mkForce ''
          output discard
--- a/hosts/flora-6/apps/prometheus.nix
+++ b/hosts/flora-6/apps/prometheus.nix
@ -65,50 +65,5 @@
        }];
      }
    ];
-
-    ruleFiles = [
-      (pkgs.writeText "prometheus-rules.yml" (builtins.toJSON {
-        groups = [{
-          name = "alerting-rules";
-          rules = import ./alert-rules.nix { inherit lib; };
-        }];
-      }))
-    ];
-
-    alertmanagers = [{ static_configs = [{ targets = [ "localhost:9093" ]; }]; }];
-
-    alertmanager = {
-      enable = true;
-      # port = 9093; # Default
-      webExternalUrl = "https://alerts.pub.solar"; # TODO use a proper url?
-      # environmentFile = "${config.age.secrets.nachtigall-alertmanager-envfile.path}";
-      configuration = {
-
-        route = {
-          receiver = "all";
-          group_by = [ "instance" ];
-          group_wait = "30s";
-          group_interval = "2m";
-          repeat_interval = "24h";
-        };
-
-        receivers = [{
-          name = "all";
-          # Email config documentation: https://prometheus.io/docs/alerting/latest/configuration/#email_config
-          email_configs = [{
-            send_resolved = true;
-            to = "TODO";
-            from = "alerts@pub.solar";
-            smarthost = "TODO";
-            auth_username = "TODO";
-            auth_password_file = "${config.age.secrets.nachtigall-alertmanager-smtp-password.path}";
-            require_tls = true;
-          }];
-          # TODO:
-          # For matrix notifications, look into: https://github.com/pinpox/matrix-hook and add a webhook
-          #   webhook_configs = [ { url = "http://127.0.0.1:11000/alert"; } ];
-        }];
-      };
-    };
  };
 }
--- a/hosts/nachtigall/apps/matrix/irc.nix
+++ b/hosts/nachtigall/apps/matrix/irc.nix
@ -13,6 +13,11 @@ let
  synapseClientPort = "${toString listenerWithClient.port}";
 in
 {
+  systemd.services.matrix-appservice-irc.serviceConfig.SystemCallFilter = lib.mkForce [
+    "@system-service @pkey"
+    "~@privileged @resources"
+    "@chown"
+  ];
  services.matrix-appservice-irc = {
    enable = true;
    localpart = "irc_bot";
--- a/hosts/nachtigall/apps/obs-portal.nix
+++ b/hosts/nachtigall/apps/obs-portal.nix
@ -0,0 +1,140 @@
+{ config
+, lib
+, pkgs
+, self
+, flake
+, ...
+}: let 
+  configPy = pkgs.writeText "obs-portal-config.py" ''
+DEBUG = False
+VERBOSE = DEBUG
+AUTO_RESTART = DEBUG
+LEAN_MODE = False
+FRONTEND_URL = None
+FRONTEND_HTTPS = True
+FRONTEND_DIR = "../frontend/build/"
+FRONTEND_CONFIG = {
+    "imprintUrl": "https://pub.solar/about",
+    "privacyPolicyUrl": "https://pub.solar/privacy",
+    "mapHome": {"zoom": 12, "latitude": 50.93, "longitude": 6.97},
+    "banner": {
+        "text": "This is an installation serving the Cologne/Bonn region run for Team OBSKöln by pub.solar n.e.V.",
+        "style": "info"
+    },
+}
+TILES_FILE = None
+ADDITIONAL_CORS_ORIGINS = None
+  '';
+
+  env = {
+    OBS_KEYCLOAK_URI = "auth.pub.solar";
+    OBS_PORTAL_URI = "obs-portal.pub.solar";
+
+    OBS_POSTGRES_MAX_OVERFLOW = "20";
+    OBS_POSTGRES_POOL_SIZE = "40";
+
+    OBS_HOST = "0.0.0.0";
+    OBS_PORT = "3000";
+    OBS_KEYCLOAK_URL = "https://auth.pub.solar/realms/pub.solar/";
+    OBS_KEYCLOAK_CLIENT_ID = "openbikesensor-portal";
+    OBS_DEDICATED_WORKER = "True";
+    OBS_DATA_DIR = "/data";
+    OBS_PROXIES_COUNT = "1";
+  };
+in {
+  age.secrets.obs-portal-env = {
+    file = "${flake.self}/secrets/obs-portal-env.age";
+    mode = "600";
+  };
+
+  age.secrets.obs-portal-database-env = {
+    file = "${flake.self}/secrets/obs-portal-database-env.age";
+    mode = "600";
+  };
+
+  systemd.services."docker-network-obs-portal" =
+    let
+      docker = config.virtualisation.oci-containers.backend;
+      dockerBin = "${pkgs.${docker}}/bin/${docker}";
+    in
+    {
+      serviceConfig.Type = "oneshot";
+      before = [ "docker-obs-portal.service" ];
+      script = ''
+        ${dockerBin} network inspect obs-portal-net >/dev/null 2>&1 || ${dockerBin} network create obs-portal-net --subnet 172.20.0.0/24
+      '';
+    };
+
+  services.nginx.virtualHosts."obs-portal.pub.solar" = {
+    enableACME = true;
+    forceSSL = true;
+
+    locations."/" = {
+      proxyWebsockets = true;
+      extraConfig = ''
+        proxy_pass http://127.0.0.1:3001;
+        proxy_set_header Host $host;
+      '';
+    };
+  };
+
+  virtualisation = {
+    oci-containers = {
+      backend = "docker";
+
+      containers."obs-portal" = {
+        image = "git.pub.solar/pub-solar/obs-portal:latest";
+        autoStart = true;
+        ports = [ "localhost:3001:${env.OBS_PORT}" ];
+
+        environment = env;
+        environmentFiles = [ config.age.secrets.obs-portal-env.path ];
+
+        volumes = [
+          "${configPy}:/opt/obs/api/config.py"
+          "/var/lib/obs-portal${env.OBS_DATA_DIR}:${env.OBS_DATA_DIR}"
+          "/var/lib/obs-portal/tiles/:/tiles"
+          "/var/lib/obs-portal/pbf/:/pbf"
+        ];
+
+        extraOptions = [
+          "--network=obs-portal-net"
+        ];
+      };
+
+      containers."obs-portal-worker" = {
+        image = "git.pub.solar/pub-solar/obs-portal:latest";
+        autoStart = true;
+
+        cmd = [ "python" "tools/process_track.py" ];
+
+        environment = env;
+        environmentFiles = [ config.age.secrets.obs-portal-env.path ];
+
+        volumes = [
+          "${configPy}:/opt/obs/api/config.py"
+          "/var/lib/obs-portal${env.OBS_DATA_DIR}:${env.OBS_DATA_DIR}"
+        ];
+
+        extraOptions = [
+          "--network=obs-portal-net"
+        ];
+      };
+
+      containers."obs-portal-db" = {
+        image = "openmaptiles/postgis:7.0";
+        autoStart = true;
+
+        environmentFiles = [ config.age.secrets.obs-portal-database-env.path ];
+
+        volumes = [
+          "/var/lib/postgres-obs-portal/data:/var/lib/postgresql/data"
+        ];
+
+        extraOptions = [
+          "--network=obs-portal-net"
+        ];
+      };
+    };
+  };
+}
--- a/hosts/nachtigall/default.nix
+++ b/hosts/nachtigall/default.nix
@ -32,6 +32,7 @@
      ./apps/promtail.nix
      ./apps/searx.nix
      ./apps/tmate.nix
+      ./apps/obs-portal.nix

      ./apps/matrix/irc.nix
      ./apps/matrix/mautrix-telegram.nix
--- a/secrets/obs-portal-database-env.age
+++ b/secrets/obs-portal-database-env.age
@ -0,0 +1,27 @@
+age-encryption.org/v1
+-> ssh-ed25519 iDKjwg hAoEiOaK1U0HImALePEYHiE6xebOOqtVujaBWgNBZF8
+ecf/ykqYPihRJxI/Y7Oh6QhWSyncwevlzEZoRqm3aGM
+-> ssh-ed25519 uYcDNw NcIttsTn6wPCmoOYGtZ66IYhthjLDI3sYFe4pbW6cB4
+9hv4dEYoXXWSZ2pG1hy68vmTf++v+g3q7wVhT6cAog0
+-> ssh-rsa kFDS0A
+KoW3J2Tw90chM6Oy17umOQN0WFI4je7CBk3IgdImsd4Mz5q17/nXlhVlFFhx4ZEk
+Or9LaqytVk1NA6J4+suMRlx4Pd6oberXu1KBkFQMr1B3LKhNOaOZ+W1mrbQLGG9U
+YUTyOpkHxVkw0IOsvxB/0reMCHtjKHo661zFjim1YFmEk0WRt4hU1XqsMNiE4wbc
+GF0t9EWMN2pU2p7DpX/DzVTqu8yk8SQhCZc9kfzWcuawwf0rcjwUJ/Rk1MH5tMpK
+odRXXl1slPPwQinE+KJqeyrfuRDHqwqmxnOfOWG6KQwWkVSE1btiHEvfuuLOjSjl
+3wO+veRC9hW5sSCPANoFbuSQ1dprmoyaZnOyeRTbgw91ks/ogLBezF/KSkaMQeHx
+XRnfcceBmeeqHl9L3Z+3EmBjwIqu2Og0pvhDU8G/ZeA0cHS/22QYGzeD/gOqaEW7
+d1VyA6LZd8PxIjoBamdipIpY0TqZ8+cA/yaUKNnYXXRSlKQ5ggPxh7ZXfvRbGg+m
+WbNiHxBPcTK7/Bpzes4LJVcx0Ar4XeDxVQe1MITLpFWh+FDEQZEA3630JngZ153J
+vBvw+VFedPSr6Ov+/33/J3LKC0XRatGnc++AWfo4rWPLCE6qovEDyY+wmct8gv0j
+rMEK7OaNfyy+Z21mjrkwcEUbyoGt9ksEplaRblE0Lsk
+-> ssh-ed25519 YFSOsg LmLRtBYMSzjid3VkUgAQvDOS9r0imWSKE7fm0t/x41Y
+0mae0vsNmaS5aVOKezXit7KV44JKLpU+GWpuA++dCVo
+-> ssh-ed25519 iHV63A Tc2z2JciftAikoj4Hv9IBgkcYWAcyGuPJTNA3Yw2K1w
+cO5o/pbaZAtTvXUskOah9vWP/Tuvyi3QDM7g4AQ+b8s
+-> ssh-ed25519 BVsyTA mk6n6ytaI4V9JVoUZFtwfFOgaLYc6gvVOcSZXQj/FVI
+etqbUCqe0eY81qaVco7pMJjhfM+sA/bXLMW0bEsCLxI
+--- CmNq6ZPxFoFTsySVfr7BTHV0tm9cbRYGG6IR7DNgbEY
+!èçƒˆí}
+ùSê<>ŸSl®Ds;!ÁjršZçR"—ë#ž¿»ÙÅ~!›Ÿ¤6AùwEn ? kËAcx~—ŽÜGVæ&M¯ý¾ä,
+a›U
--- a/secrets/obs-portal-env.age
+++ b/secrets/obs-portal-env.age
--- a/secrets/secrets.nix
+++ b/secrets/secrets.nix
@ -1,4 +1,5 @@
 let
+<<<<<<< HEAD
  admins = import ../logins/admins.nix;

  nachtigall-host = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP7G0ufi+MNvaAZLDgpieHrABPGN7e/kD5kMFwSk4ABj root@nachtigall";
@ -64,4 +65,7 @@ in

  "nachtigall-metrics-nginx-basic-auth.age".publicKeys = nachtigallKeys ++ adminKeys;
  "nachtigall-metrics-prometheus-basic-auth-password.age".publicKeys = flora6Keys ++ nachtigallKeys ++ adminKeys;
+
+  "obs-portal-env.age".publicKeys = nachtigallKeys ++ adminKeys;
+  "obs-portal-database-env.age".publicKeys = nachtigallKeys ++ adminKeys;
 }
--- a/terraform/dns.tf
+++ b/terraform/dns.tf
@ -1,186 +1,181 @@
 # https://registry.terraform.io/providers/namecheap/namecheap/latest/docs
 resource "namecheap_domain_records" "pub-solar" {
-  domain     = "pub.solar"
-  mode       = "OVERWRITE"
+  domain = "pub.solar"
+  mode = "OVERWRITE"
  email_type = "MX"

  record {
    hostname = "flora-6"
-    type     = "A"
-    address  = "80.71.153.210"
+    type = "A"
+    address = "80.71.153.210"
  }
  record {
    hostname = "auth"
-    type     = "CNAME"
-    address  = "nachtigall.pub.solar."
+    type = "CNAME"
+    address = "nachtigall.pub.solar."
  }
  record {
    hostname = "ci"
-    type     = "A"
-    address  = "80.71.153.210"
-  }
-  record {
-    hostname = "alerts"
-    type     = "CNAME"
-    address  = "flora-6.pub.solar."
+    type = "A"
+    address = "80.71.153.210"
  }
  record {
    hostname = "git"
-    type     = "CNAME"
-    address  = "nachtigall.pub.solar."
+    type = "CNAME"
+    address = "nachtigall.pub.solar."
  }
  record {
    hostname = "stream"
-    type     = "CNAME"
-    address  = "nachtigall.pub.solar."
+    type = "CNAME"
+    address = "nachtigall.pub.solar."
  }
  record {
    hostname = "list"
-    type     = "CNAME"
-    address  = "nachtigall.pub.solar."
+    type = "CNAME"
+    address = "nachtigall.pub.solar."
  }
  record {
    hostname = "obs-portal"
-    type     = "A"
-    address  = "80.71.153.210"
+    type = "A"
+    address = "80.71.153.210"
  }
  record {
    hostname = "vpn"
-    type     = "A"
-    address  = "80.71.153.210"
+    type = "A"
+    address = "80.71.153.210"
  }
  record {
    hostname = "cache"
-    type     = "A"
-    address  = "95.217.225.160"
+    type = "A"
+    address = "95.217.225.160"
  }
  record {
    hostname = "factorio"
-    type     = "A"
-    address  = "80.244.242.2"
+    type = "A"
+    address = "80.244.242.2"
  }
  record {
    hostname = "collabora"
-    type     = "CNAME"
-    address  = "nachtigall.pub.solar."
+    type = "CNAME"
+    address = "nachtigall.pub.solar."
  }
  record {
    hostname = "@"
-    type     = "ALIAS"
-    address  = "nachtigall.pub.solar."
-    ttl      = 300
+    type = "ALIAS"
+    address = "nachtigall.pub.solar."
+    ttl = 300
  }
  record {
    hostname = "chat"
-    type     = "CNAME"
-    address  = "nachtigall.pub.solar."
+    type = "CNAME"
+    address = "nachtigall.pub.solar."
  }
  record {
    hostname = "cloud"
-    type     = "CNAME"
-    address  = "nachtigall.pub.solar."
+    type = "CNAME"
+    address = "nachtigall.pub.solar."
  }
  record {
    hostname = "turn"
-    type     = "CNAME"
-    address  = "nachtigall.pub.solar."
+    type = "CNAME"
+    address = "nachtigall.pub.solar."
  }
  record {
    hostname = "grafana"
-    type     = "A"
-    address  = "80.71.153.210"
+    type = "A"
+    address = "80.71.153.210"
  }
  record {
    hostname = "hpb"
-    type     = "A"
-    address  = "80.71.153.239"
+    type = "A"
+    address = "80.71.153.239"
  }
  record {
    hostname = "files"
-    type     = "CNAME"
-    address  = "nachtigall.pub.solar."
+    type = "CNAME"
+    address = "nachtigall.pub.solar."
  }
  record {
    hostname = "search"
-    type     = "CNAME"
-    address  = "nachtigall.pub.solar."
+    type = "CNAME"
+    address = "nachtigall.pub.solar."
  }
  record {
    hostname = "wiki"
-    type     = "CNAME"
-    address  = "nachtigall.pub.solar."
+    type = "CNAME"
+    address = "nachtigall.pub.solar."
  }
  record {
    hostname = "mastodon"
-    type     = "CNAME"
-    address  = "nachtigall.pub.solar."
+    type = "CNAME"
+    address = "nachtigall.pub.solar."
  }
  record {
    hostname = "matrix"
-    type     = "CNAME"
-    address  = "nachtigall.pub.solar."
+    type = "CNAME"
+    address = "nachtigall.pub.solar."
  }
  record {
    hostname = "tmate"
-    type     = "CNAME"
-    address  = "nachtigall.pub.solar."
+    type = "CNAME"
+    address = "nachtigall.pub.solar."
  }
  record {
    hostname = "www"
-    type     = "CNAME"
-    address  = "nachtigall.pub.solar."
+    type = "CNAME"
+    address = "nachtigall.pub.solar."
  }
  record {
    hostname = "@"
-    type     = "TXT"
-    address  = "v=spf1 include:spf.greenbaum.zone a:list.pub.solar ~all"
+    type = "TXT"
+    address = "v=spf1 include:spf.greenbaum.zone a:list.pub.solar ~all"
  }
  record {
    hostname = "list"
-    type     = "TXT"
-    address  = "v=spf1 a:list.pub.solar ?all"
+    type = "TXT"
+    address = "v=spf1 a:list.pub.solar ?all"
  }
  record {
    hostname = "_dmarc"
-    type     = "TXT"
-    address  = "v=DMARC1; p=reject;"
+    type = "TXT"
+    address = "v=DMARC1; p=reject;"
  }
  record {
    hostname = "_dmarc.list"
-    type     = "TXT"
-    address  = "v=DMARC1; p=reject;"
+    type = "TXT"
+    address = "v=DMARC1; p=reject;"
  }
  record {
    hostname = "modoboa._domainkey"
-    type     = "TXT"
-    address  = "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx/EqLMpk0MyL1aQ0JVG44ypTRbZBVA13MFjEntxAvowaWtq1smRbnEwTTKgqUOrUyaM4dVmli1dedne4mk/ncqRAm02KuhtTY+5wXfhTKK53EhqehbKwH+Qvzb12983Qwdau/QTHiFHwXHufMaSsCvd9CRWCp9q68Q7noQqndJeLHT6L0eECd2Zk3ZxJuh+Fxdb7+Kw68Tf6z13Rs+MU01qLM7x0jmSQHa4cv2pk+7NTGMBRp6fVskfbqev5nFkZWJ7rhXEbP9Eukd/L3ro/ubs1quWJotG02gPRKE8fgkm1Ytlws1/pnqpuvKXQS1HzBEP1X2ExezJMzQ1SnZCigQIDAQAB"
+    type = "TXT"
+    address = "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx/EqLMpk0MyL1aQ0JVG44ypTRbZBVA13MFjEntxAvowaWtq1smRbnEwTTKgqUOrUyaM4dVmli1dedne4mk/ncqRAm02KuhtTY+5wXfhTKK53EhqehbKwH+Qvzb12983Qwdau/QTHiFHwXHufMaSsCvd9CRWCp9q68Q7noQqndJeLHT6L0eECd2Zk3ZxJuh+Fxdb7+Kw68Tf6z13Rs+MU01qLM7x0jmSQHa4cv2pk+7NTGMBRp6fVskfbqev5nFkZWJ7rhXEbP9Eukd/L3ro/ubs1quWJotG02gPRKE8fgkm1Ytlws1/pnqpuvKXQS1HzBEP1X2ExezJMzQ1SnZCigQIDAQAB"
  }
  record {
    hostname = "@"
-    type     = "MX"
-    address  = "mail.greenbaum.zone."
-    mx_pref  = "0"
+    type = "MX"
+    address = "mail.greenbaum.zone."
+    mx_pref = "0"
  }
  record {
    hostname = "list"
-    type     = "MX"
-    address  = "list.pub.solar."
-    mx_pref  = "0"
+    type = "MX"
+    address = "list.pub.solar."
+    mx_pref = "0"
  }
  record {
    hostname = "nachtigall"
-    type     = "A"
-    address  = "138.201.80.102"
+    type = "A"
+    address = "138.201.80.102"
  }
  record {
    hostname = "nachtigall"
-    type     = "AAAA"
-    address  = "2a01:4f8:172:1c25::1"
+    type = "AAAA"
+    address = "2a01:4f8:172:1c25::1"
  }
  record {
    hostname = "matrix.test"
-    type     = "CNAME"
-    address  = "nachtigall.pub.solar."
+    type = "CNAME"
+    address = "nachtigall.pub.solar."
  }
  # SRV records can only be changed via NameCheap Web UI
  # add comment