Refactor test-vm, add queue + scheduler workers

Replace nginx with caddy

Prepare for nixos module implementation

test-vm/configuration.nix

@@ -6,7 +6,56 @@
     "${modulesPath}/virtualisation/qemu-vm.nix"
   ];
 
-  config = {
+  config =
+  let
+    penv = pkgs.python3.buildEnv.override {
+      extraLibs = [
+        pkgs.python3.pkgs.frappe
+        pkgs.python3.pkgs.erpnext
+        pkgs.python3.pkgs.bench
+      ];
+    };
+    appsFile = pkgs.writeText "erpnext-apps.txt" ''
+      frappe
+      erpnext
+    '';
+    # In a module, this could be provided by a use as a file as it could
+    # contain secrets and we don't want this in the nix-store. But here it
+    # is OK.
+    commonSiteConfig = pkgs.writeText "erpnext-common_site_config.json" ''
+    {
+      "db_host": "localhost",
+      "db_port": 3306,
+      "db_name": "erpnext" ,
+      "db_password": "erpnext" ,
+      "redis_cache": "redis://localhost:6379",
+      "redis_queue": "redis://localhost:6379",
+      "redis_socketio": "redis://localhost:12311",
+      "socketio_port": 3000
+    }
+    '';
+    defaultServiceConfig = {
+      User = "erpnext";
+      NoNewPrivileges = true;
+      Type = "simple";
+      BindReadOnlyPaths = [
+        "/etc/hosts:/etc/hosts"
+        "${pkgs.frappe-app}:${pkgs.frappe-app}"
+        "${pkgs.frappe-app}/share/apps/frappe:/var/lib/erpnext/bench/apps/frappe"
+        "${pkgs.erpnext-app}:${pkgs.erpnext-app}"
+        "${pkgs.erpnext-app}/share/apps/erpnext:/var/lib/erpnext/bench/apps/erpnext"
+        "${pkgs.frappe-erpnext-assets}/share/sites/assets:/var/lib/erpnext/bench/sites/assets"
+        "${appsFile}:/var/lib/erpnext/bench/sites/apps.txt"
+        "${commonSiteConfig}:/var/lib/erpnext/bench/sites/common_site_config.json"
+        "${penv}:/var/lib/erpnext/bench/env"
+      ];
+      BindPaths = [
+        "/var/lib/erpnext:/var/lib/erpnext"
+      ];
+      WorkingDirectory = "/var/lib/erpnext/bench";
+    };
+  in
+  {
     services.qemuGuest.enable = true;
     system.stateVersion = "23.05";
 
@@ -88,13 +137,14 @@
       home = "/var/lib/erpnext";
       createHome = true;
     };
+    users.groups.erpnext = {};
 
-    systemd.services.setup-mysql = {
+    systemd.services.erpnext-setup-mysql = {
       enable = true;
-      before = [ "erpnext.service" ];
+      before = [ "erpnext-web.service" ];
       after = [ "mysql.service" ];
-      wantedBy = [ "erpnext.service" ];
+      wantedBy = [ "erpnext-web.service" ];
-      partOf = [ "erpnext.service" ];
+      partOf = [ "erpnext-web.service" ];
       script = ''
         ${pkgs.mariadb-client}/bin/mysql -e "SET PASSWORD FOR 'root'@'localhost' = PASSWORD('password')";
       '';
@@ -104,11 +154,11 @@
       };
     };
 
-    systemd.services.ensure-bench-dir = {
+    systemd.services.erpnext-ensure-bench-dir = {
       enable = true;
-      before = [ "erpnext.service" ];
+      before = [ "erpnext-web.service" ];
-      wantedBy = [ "erpnext.service" ];
+      wantedBy = [ "erpnext-web.service" ];
-      partOf = [ "erpnext.service" ];
+      partOf = [ "erpnext-web.service" ];
       script = ''
         for subdir in apps sites config/pids logs; do
           mkdir -p /var/lib/erpnext/bench/$subdir
@@ -121,82 +171,137 @@
       };
     };
 
-    services.nginx = {
+    systemd.services.erpnext-nodejs-socketio = {
       enable = true;
-      appendHttpConfig = builtins.readFile "${pkgs.erpnext-nginx-conf}";
+      before = [ "erpnext-web.service" ];
+      wantedBy = [ "erpnext-web.service" ];
+      partOf = [ "erpnext-web.service" ];
+      description = "ERPNext Node.js HTTP server for socket.io ";
+      confinement = {
+        enable = true;
+        packages = [ pkgs.nodejs ];
       };
-
+      serviceConfig = defaultServiceConfig // {
-    systemd.services.erpnext =
+        ExecStart = ''
-      let
+          ${pkgs.nodejs}/bin/node /var/lib/erpnext/bench/apps/frappe/socketio.js
-        penv = pkgs.python3.buildEnv.override {
-          extraLibs = [
-            pkgs.python3.pkgs.frappe
-            pkgs.python3.pkgs.erpnext
-            pkgs.python3.pkgs.bench
-          ];
-        };
-        appsFile = pkgs.writeText "erpnext-apps.txt" ''
-          frappe
-          erpnext
         '';
+      };
+    };
 
-        # In a module, this could be provided by a use as a file as it could
+    services.caddy = {
-        # contain secrets and we don't want this in the nix-store. But here it
+      enable = true;
-        # is OK.
+      email = "admins@pub.solar";
-        commonSiteConfig = pkgs.writeText "erpnext-common_site_config.json" ''
+      globalConfig = ''
-          {
+        local_certs
-            "db_host": "localhost",
+      '';
-            "db_port": 3306,
+      virtualHosts = {
-            "db_name": "erpnext" ,
+        "localhost:8081" = {
-            "db_password": "erpnext" ,
+          extraConfig = ''
-            "redis_cache": "redis://localhost:6379?db=0",
+            handle /assets/* {
-            "redis_queue": "redis://localhost:6379?db=1",
+              root * ${pkgs.frappe-erpnext-assets}/share/sites
-            "redis_socketio": "redis://localhost:6379?db=2",
+              file_server
-            "socketio_port": 3000
             }
+            handle /socket.io/* {
+              reverse_proxy :3000
+            }
+
+            reverse_proxy :9090
           '';
-      in
+        };
-        {
+      };
+    };
+
+    systemd.services.erpnext-web = {
       enable = true;
       wantedBy = [ "multi-user.target" ];
       after = [ "mysql.service" "redis.service" "redis-socketio.service" ];
-          description = "ERPNext";
+      description = "ERPNext web server";
       confinement = {
         enable = true;
-            packages = [ pkgs.mariadb-client pkgs.nodejs penv ];
+        packages = [ pkgs.mariadb-client penv ];
       };
-          script = ''
+      serviceConfig = defaultServiceConfig // {
+        TimeoutStartSec = "300s";
+        ExecStartPre = pkgs.writeScript "erpnext-server.worker-init" ''
+          #!/bin/sh
           export PYTHON_PATH=${penv}/${pkgs.python3.sitePackages}
           export PATH="${pkgs.mariadb-client}/bin:${pkgs.nodejs}/bin:${penv}/bin:$PATH"
 
           # Upstream initializes the DB with this command
-            # TODO: Make this idempotent
+          bench new-site localhost --mariadb-root-password password --admin-password admin --install-app erpnext || true
-            cd /var/lib/erpnext/bench/sites
+        '';
-            bench new-site localhost --mariadb-root-password password --admin-password admin || true
+        ExecStart = ''
-            bench --site localhost install-app erpnext
+          ${penv}/bin/gunicorn \
-
+            --chdir="/var/lib/erpnext/bench/sites" \
-            # TODO: Run these as systemd units
+            --bind=0.0.0.0:9090 \
-            node /var/lib/erpnext/bench/apps/frappe/socketio.js &
+            --threads=4 \
-            gunicorn --chdir="/var/lib/erpnext/bench/sites" --bind=0.0.0.0:9090 --threads=4 --workers=2 --worker-class=gthread --worker-tmp-dir=/dev/shm --timeout=120 --preload frappe.app:application
+            --workers=2 \
+            --worker-class=gthread \
+            --worker-tmp-dir=/dev/shm \
+            --timeout=120 \
+            --preload \
+            frappe.app:application
+        '';
+      };
+    };
+    systemd.services.erpnext-queue-short = {
+      enable = true;
+      wantedBy = [ "multi-user.target" ];
+      after = [ "mysql.service" "redis.service" "redis-socketio.service" ];
+      description = "ERPNext short queue server";
+      confinement = {
+        enable = true;
+        packages = [ penv ];
+      };
+      serviceConfig = defaultServiceConfig // {
+        ExecStart = ''
+          ${penv}/bin/bench worker --queue short
+        '';
+      };
+    };
+    systemd.services.erpnext-queue-default = {
+      enable = true;
+      wantedBy = [ "multi-user.target" ];
+      after = [ "mysql.service" "redis.service" "redis-socketio.service" ];
+      description = "ERPNext default queue server";
+      confinement = {
+        enable = true;
+        packages = [ penv ];
+      };
+      serviceConfig = defaultServiceConfig // {
+        ExecStart = ''
+          ${penv}/bin/bench worker --queue default
+        '';
+      };
+    };
+    systemd.services.erpnext-queue-long = {
+      enable = true;
+      wantedBy = [ "multi-user.target" ];
+      after = [ "mysql.service" "redis.service" "redis-socketio.service" ];
+      description = "ERPNext long queue server";
+      confinement = {
+        enable = true;
+        packages = [ penv ];
+      };
+      serviceConfig = defaultServiceConfig // {
+        ExecStart = ''
+          ${penv}/bin/bench worker --queue long
+        '';
+      };
+    };
+    systemd.services.erpnext-scheduler = {
+      enable = true;
+      wantedBy = [ "multi-user.target" ];
+      after = [ "mysql.service" "redis.service" "redis-socketio.service" ];
+      description = "ERPNext scheduler server";
+      confinement = {
+        enable = true;
+        packages = [ penv ];
+      };
+      serviceConfig = defaultServiceConfig // {
+        ExecStart = ''
+          ${penv}/bin/bench schedule
         '';
-          serviceConfig = {
-            User = "erpnext";
-            NoNewPrivileges = true;
-            Type = "simple";
-            BindReadOnlyPaths = [
-              "/etc/hosts:/etc/hosts"
-              "${pkgs.frappe-app}:${pkgs.frappe-app}"
-              "${pkgs.frappe-app}/share/apps/frappe:/var/lib/erpnext/bench/apps/frappe"
-              "${pkgs.erpnext-app}:${pkgs.erpnext-app}"
-              "${pkgs.erpnext-app}/share/apps/erpnext:/var/lib/erpnext/bench/apps/erpnext"
-              "${pkgs.frappe-erpnext-assets}/share/sites/assets:/var/lib/erpnext/bench/sites/assets"
-              "${appsFile}:/var/lib/erpnext/bench/sites/apps.txt"
-              "${commonSiteConfig}:/var/lib/erpnext/bench/sites/common_site_config.json"
-              "${penv}:/var/lib/erpnext/bench/env"
-            ];
-            BindPaths = [
-              "/var/lib/erpnext:/var/lib/erpnext"
-            ];
       };
     };
   };