Add module to setup wireguard backed zfs enabled k3s cluster

modules/home-controller/default.nix

@@ -0,0 +1,131 @@
+{ lib, config, pkgs, ... }:
+with lib;
+let
+  psCfg = config.pub-solar;
+  cfg = config.pub-solar.home-controller;
+in
+{
+  imports = [
+    ./k3s.nix
+    ./wireguard.nix
+  ];
+
+  options.pub-solar.home-controller = {
+    enable = mkEnableOption "Control your home";
+
+    role = mkOption {
+      description = ''
+        Whether the node should run as a server or agent.
+        Note that the server, by default, also runs as an agent.
+      '';
+      default = "server";
+      type = types.enum [ "server" "agent" ];
+    };
+
+    ownIp = mkOption {
+      description = ''
+        Internal ip in wireguard used for cluster control-plane communication.
+      '';
+      type = types.str;
+    };
+
+    k3s = {
+      enableLocalStorage = mkOption {
+        description = ''
+          Enable local storage provisioner.
+        '';
+        default = false;
+        type = types.bool;
+      };
+
+      defaultLocalStoragePath = mkOption {
+        description = ''
+          Default path to use for local storage provisioner.
+        '';
+        default = "/var/lib/rancher/k3s/storage";
+        type = types.path;
+      };
+
+      flannelBackend = mkOption {
+        description = ''
+          Flannel backend to use.
+        '';
+        default = "wireguard-native";
+        type = types.str;
+      };
+
+      serverAddr = mkOption {
+        description = ''
+          Set server address of master
+        '';
+        default = "";
+        type = types.str;
+        example = "https://api.kube:6443";
+      };
+
+      tokenFile = mkOption {
+        description = ''
+          Location of token file used to join cluster.
+        '';
+        default = "";
+        type = types.str;
+      };
+
+      enableZfs = mkOption {
+        description = ''
+          Enable when k3s should use a ZFS compatible runtime.
+        '';
+        default = false;
+        type = types.bool;
+      };
+
+      zfsPool = mkOption {
+        description = ''
+          The ZFS pool to use and create a containerd volume in.
+        '';
+        default = "zroot";
+        type = types.str;
+      };
+    };
+
+    wireguard = {
+      privateKeyFile = mkOption {
+        description = ''
+          Location of private key file
+        '';
+        type = types.path;
+      };
+
+      listenPort = mkOption {
+        description = ''
+          Port for wireguard.
+        '';
+        default = 51899;
+        type = types.int;
+      };
+
+      peers = mkOption {
+        description = ''
+          Wireguard peers.
+        '';
+        type = types.listOf types.attrs;
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    boot.kernelModules = [ "rbd" ];
+
+    networking.extraHosts =
+      ''
+        192.168.42.231 ringo.local
+        192.168.42.232 giggles.local
+        192.168.42.234 cox.local
+        192.168.42.236 companion.local
+        10.0.1.11      api.kube giggles.kube
+        10.0.1.12      cox.kube
+        10.0.1.13      companion.kube
+        10.0.1.21      ringo.kube
+      '';
+  };
+}

modules/home-controller/k3s.nix

@@ -0,0 +1,77 @@
+{ lib, config, pkgs, ... }:
+with lib;
+let
+  psCfg = config.pub-solar;
+  cfg = config.pub-solar.home-controller;
+in
+{
+  config = mkIf cfg.enable {
+    environment.systemPackages = with pkgs; [
+      kubernetes-helm
+    ];
+
+    environment.sessionVariables = lib.mkIf (cfg.role == "server") rec {
+      KUBECONFIG = "/etc/rancher/k3s/k3s.yaml";
+    };
+
+    networking.firewall.enable = lib.mkForce false;
+
+    services.k3s = {
+      enable = true;
+      role = cfg.role;
+      serverAddr = lib.mkIf (cfg.k3s.serverAddr != "") cfg.k3s.serverAddr;
+      tokenFile = lib.mkIf (cfg.k3s.tokenFile != "") cfg.k3s.tokenFile;
+      extraFlags = concatStringsSep " " (
+        [
+          "--node-ip ${cfg.ownIp}"
+          "--container-runtime-endpoint unix:///run/containerd/containerd.sock"
+
+          "${optionalString (cfg.role == "server") "--disable servicelb"}"
+          "${optionalString (cfg.role == "server") "--disable traefik"}"
+
+          "${optionalString (cfg.role == "server") "--bind-address ${cfg.ownIp}"}"
+
+          "${optionalString (cfg.role == "server" && cfg.k3s.flannelBackend != "") "--flannel-backend=${cfg.k3s.flannelBackend}"}"
+
+          "${optionalString (cfg.role == "server" && !cfg.k3s.enableLocalStorage) "--disable local-storage"}"
+          "${optionalString (cfg.role == "server" && cfg.k3s.enableLocalStorage) "--default-local-storage-path ${cfg.k3s.defaultLocalStoragePath}"}"
+
+          "${optionalString cfg.k3s.enableZfs "--snapshotter=zfs"}"
+        ]
+      );
+    };
+
+    systemd.services.containerd = mkIf cfg.k3s.enableZfs {
+      serviceConfig = {
+        ExecStartPre = [
+          "-${pkgs.zfs}/bin/zfs create -o mountpoint=/var/lib/containerd/io.containerd.snapshotter.v1.zfs ${cfg.k3s.zfsPool}/containerd"
+        ];
+      };
+    };
+
+    systemd.services.k3s = {
+      after = [ "containerd.service" ];
+      requisite = [ "containerd.service" ];
+    };
+
+    virtualisation.containerd = {
+      enable = true;
+      settings =
+        let
+          fullCNIPlugins = pkgs.buildEnv {
+            name = "full-cni";
+            paths = with pkgs; [
+              cni-plugins
+              cni-plugin-flannel
+            ];
+          };
+        in
+        {
+          plugins."io.containerd.grpc.v1.cri".cni = {
+            bin_dir = "${fullCNIPlugins}/bin";
+            conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d/";
+          };
+        };
+    };
+  };
+}

modules/home-controller/wireguard.nix

@@ -0,0 +1,23 @@
+{ lib, config, pkgs, ... }:
+with lib;
+let
+  psCfg = config.pub-solar;
+  cfg = config.pub-solar.home-controller;
+in
+{
+  config = mkIf cfg.enable {
+    systemd.services.wireguard-wghome.serviceConfig.Restart = "on-failure";
+    systemd.services.wireguard-wghome.serviceConfig.RestartSec = "5s";
+
+    networking.firewall.allowedUDPPorts = [ cfg.wireguard.listenPort ];
+
+    networking.wireguard.interfaces = {
+      wghome = {
+        ips = [ cfg.ownIp ];
+        listenPort = cfg.wireguard.listenPort;
+        privateKeyFile = cfg.wireguard.privateKeyFile;
+        peers = cfg.wireguard.peers;
+      };
+    };
+  };
+}