Merge pull request 'loki: set log retention to 1w' (#349) from loki-logs-retention into main

Reviewed-on: #349

docs/nextcloud.md

@@ -6,10 +6,10 @@ Set loglevel to `0` for debug logs:
 services.nextcloud.settings.loglevel = 0;
 ```
 
-Then, logs appear in the `phpfpm-nextcloud.service` logs:
+Then, logs appear in the `phpfpm-nextcloud.service` and `nextcloud-*.service` logs:
 
 ```bash
-sudo journalctl -fu phpfpm-nextcloud
+sudo journalctl --follow --unit phpfpm-nextcloud --unit nextcloud-*.service
 ```
 
 Make sure to set the loglevel back to the default `2` warning after debugging:

hosts/default.nix

@@ -133,6 +133,7 @@
           self.nixosModules.prometheus
           self.nixosModules.prometheus-exporters
           self.nixosModules.loki
+          self.nixosModules.promtail
           self.nixosModules.forgejo-actions-runner
         ];
       };

modules/core/default.nix

@@ -7,6 +7,7 @@
 }:
 {
   imports = [
+    ./logging.nix
     ./nix.nix
     ./networking.nix
     ./terminal-tooling.nix

modules/core/logging.nix

@@ -0,0 +1,12 @@
+{
+  config,
+  pkgs,
+  lib,
+  flake,
+  ...
+}:
+{
+  services.journald.extraConfig = ''
+    MaxRetentionSec=3day
+  '';
+}

modules/grafana/default.nix

@@ -94,6 +94,9 @@
 
   services.grafana = {
     enable = true;
+    declarativePlugins = with pkgs.grafanaPlugins; [
+      grafana-lokiexplore-app
+    ];
     settings = {
       server = {
         # Listening Address

modules/loki/default.nix

@@ -35,9 +35,12 @@
       };
       ingester = {
         chunk_encoding = "snappy";
-        chunk_idle_period = "1h";
+        chunk_idle_period = "8h";
+        max_chunk_age = "8h";
       };
       pattern_ingester.enabled = true;
+      # 2x CPU cores
+      querier.max_concurrent = 16;
       query_range = {
         results_cache = {
           cache = {
@@ -57,14 +60,15 @@
           };
         };
       };
-      # Keep logs for 4 weeks
+      # Keep logs for 1 week
       # https://grafana.com/docs/loki/latest/operations/storage/retention/
       limits_config = {
         allow_structured_metadata = true;
         ingestion_rate_mb = 8;
         ingestion_burst_size_mb = 12;
-        retention_period = "4w";
-        split_queries_by_interval = "0";
+        retention_period = "1w";
+        split_queries_by_interval = "1h";
+        tsdb_max_query_parallelism = 32;
         volume_enabled = true;
       };
       compactor = {
@@ -100,40 +104,4 @@
       };
     };
   };
-
-  services.promtail = {
-    enable = true;
-    configuration = {
-      server = {
-        http_listen_port = 9080;
-        grpc_listen_port = 0;
-      };
-      positions = {
-        filename = "/tmp/positions.yaml";
-      };
-      clients = [
-        {
-          url = "http://trinkgenossin.wg.pub.solar:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
-        }
-      ];
-      scrape_configs = [
-        {
-          job_name = "journal";
-          journal = {
-            max_age = "24h";
-            labels = {
-              job = "systemd-journal";
-              host = "trinkgenossin";
-            };
-          };
-          relabel_configs = [
-            {
-              source_labels = [ "__journal__systemd_unit" ];
-              target_label = "unit";
-            }
-          ];
-        }
-      ];
-    };
-  };
 }

modules/nextcloud/default.nix

@@ -180,7 +180,7 @@
         activity_expire_days = "14";
         integrity.check.disabled = false;
         updater.release.channel = "stable";
-        loglevel = 0;
+        loglevel = 2;
         debug = false;
         maintenance_window_start = "1";
         # maintenance = false;

modules/promtail/default.nix

@@ -33,16 +33,11 @@
           };
           relabel_configs = [
             {
-              source_labels = [ "__journal__systemd_unit" ];
-              target_label = "unit";
-            }
-            {
-              source_labels = [ "__journal__systemd_user_unit" ];
-              target_label = "user_unit";
-            }
-            {
-              source_labels = [ "__journal_syslog_identifier" ];
-              target_label = "identifier";
+              source_labels = [
+                "__journal__systemd_unit"
+                "__journal__systemd_user_unit"
+              ];
+              target_label = "service";
             }
             {
               source_labels = [ "__journal_priority_keyword" ];