alerts.pub.solar: use DNS challenge for cert

delite, blue-shell

add instance labels to garage scrape jobs

hosts/default.nix

@@ -82,8 +82,6 @@
           self.nixosModules.overlays
           self.nixosModules.unlock-luks-on-boot
           self.nixosModules.core
-          self.nixosModules.prometheus-exporters
-          self.nixosModules.promtail
 
           self.nixosModules.garage
           self.nixosModules.nginx
@@ -105,8 +103,8 @@
           self.nixosModules.overlays
           self.nixosModules.unlock-luks-on-boot
           self.nixosModules.core
-          #self.nixosModules.prometheus-exporters
-          #self.nixosModules.promtail
+          self.nixosModules.prometheus-exporters
+          self.nixosModules.promtail
 
           self.nixosModules.garage
           self.nixosModules.nginx
@@ -122,8 +120,8 @@
           self.nixosModules.overlays
           self.nixosModules.unlock-luks-on-boot
           self.nixosModules.core
-          #self.nixosModules.prometheus-exporters
-          #self.nixosModules.promtail
+          self.nixosModules.prometheus-exporters
+          self.nixosModules.promtail
 
           self.nixosModules.garage
           self.nixosModules.nginx

hosts/delite/configuration.nix

@@ -9,7 +9,7 @@
 
   boot.kernelParams = [
     "boot.shell_on_fail=1"
-    "ip=dhcp"
+    "ip=5.255.119.132::5.255.119.1:255.255.255.0:delite::off"
   ];
 
   # This option defines the first version of NixOS you have installed on this particular machine,

modules/loki/default.nix

@@ -25,7 +25,7 @@
           };
         };
         replication_factor = 1;
-        path_prefix = "/data/loki";
+        path_prefix = "/var/lib/loki";
         storage = {
           filesystem = {
             chunks_directory = "chunks/";

modules/prometheus/alert-rules.nix

@@ -250,4 +250,10 @@ lib.mapAttrsToList
     #   description =
     #     "alertmanager: number of active silences has changed: {{$value}}";
     # };
+
+    garage_cluster_healthy = {
+      condition = "cluster_healthy = 0";
+      time = "15m";
+      description = "garage cluster on {{$labels.instance}} is not healthy: {{$labels.result}}!";
+    };
   })

modules/prometheus/default.nix

@@ -12,6 +12,15 @@
     owner = "alertmanager";
   };
 
+  security.acme.certs = {
+    "alerts.${config.pub-solar-os.networking.domain}" = {
+      # disable http challenge
+      webroot = null;
+      # enable dns challenge
+      dnsProvider = "namecheap";
+    };
+  };
+
   services.nginx.virtualHosts."alerts.${config.pub-solar-os.networking.domain}" = {
     enableACME = true;
     forceSSL = true;
@@ -74,6 +83,22 @@
               instance = "trinkgenossin";
             };
           }
+          {
+            targets = [
+              "delite.wg.${config.pub-solar-os.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
+            ];
+            labels = {
+              instance = "delite";
+            };
+          }
+          {
+            targets = [
+              "blue-shell.wg.${config.pub-solar-os.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
+            ];
+            labels = {
+              instance = "blue-shell";
+            };
+          }
         ];
       }
       {
@@ -94,9 +119,26 @@
           {
             targets = [
               "trinkgenossin.wg.${config.pub-solar-os.networking.domain}:3903"
+            ];
+            labels = {
+              instance = "trinkgenossin";
+            };
+          }
+          {
+            targets = [
               "delite.wg.${config.pub-solar-os.networking.domain}:3903"
+            ];
+            labels = {
+              instance = "delite";
+            };
+          }
+          {
+            targets = [
               "blue-shell.wg.${config.pub-solar-os.networking.domain}:3903"
             ];
+            labels = {
+              instance = "blue-shell";
+            };
           }
         ];
       }