wireguard: fix routing w/ tunneled wireguard

hosts/droppie/networking.nix

@@ -25,18 +25,16 @@
       "fd00:b12f:acab:1312:acab:3::/96"
     ];
     privateKeyFile = config.age.secrets.wg-private-key.path;
-    useDNS = false;
   };
 
-  age.secrets.wg-public-key.file = "${flake.self}/secrets/wg-public-droppie.age";
+  age.secrets.wg-tunnel-key.file = "${flake.self}/secrets/wg-tunnel-droppie.age";
 
-  pub-solar.wireguard.public = {
+  pub-solar.wireguard.tunnel = {
     ownIPs = [
-      "10.70.16.244/32"
+      "10.69.139.214/32"
-      "fc00:bbbb:bbbb:bb01::7:10f3/128"
+      "fc00:bbbb:bbbb:bb01::6:8bd5/128"
     ];
-    privateKeyFile = config.age.secrets.wg-public-key.path;
+    privateKeyFile = config.age.secrets.wg-tunnel-key.path;
-    useDNS = true;
     peer = {
       publicKey = "m9w2Fr0rcN6R1a9HYrGnUTU176rTZIq2pcsovPd9sms=";
       endpoint = "[2a02:6ea0:d406:1::a18f]:3019";

hosts/frikandel/wireguard.nix

@@ -20,6 +20,7 @@
   };
 
   networking.firewall.allowedUDPPorts = [ 51899 ];
+
   networking.firewall.extraForwardRules = [
     "iifname { != wg-private } reject"
     "iifname wg-private accept"
@@ -62,7 +63,8 @@
             "10.13.12.2/32"
             "fd00:b12f:acab:1312:acab:2::/96"
           ];
-          persistentKeepalive = 25;
+          persistentKeepalive = 30;
+          dynamicEndpointRefreshSeconds = 30;
         }
         { # droppie
           publicKey = "qsnBMoj9Z16D8PJ5ummRtIfT5AiMpoF3SoOCo4sbyiw=";
@@ -70,8 +72,8 @@
             "10.13.12.3/32"
             "fd00:b12f:acab:1312:acab:3::/96"
           ];
-
+          persistentKeepalive = 30;
-          persistentKeepalive = 25;
+          dynamicEndpointRefreshSeconds = 30;
         }
         { # chocolatebar
           publicKey = "nk8EtGE/QsnSEm1lhLS3/w83nOBD2OGYhODIf92G91A=";
@@ -79,8 +81,8 @@
             "10.13.12.5/32"
             "fd00:b12f:acab:1312:acab:5::/96"
           ];
-
+          persistentKeepalive = 30;
-          persistentKeepalive = 25;
+          dynamicEndpointRefreshSeconds = 30;
         }
         { # biolimo
           publicKey = "4ymN7wwBuhF+h+5fFN0TqXmVyOe1AsWiTqRL0jJ3CDc=";
@@ -88,8 +90,8 @@
             "10.13.12.6/32"
             "fd00:b12f:acab:1312:acab:6::/96"
           ];
-
+          persistentKeepalive = 30;
-          persistentKeepalive = 25;
+          dynamicEndpointRefreshSeconds = 30;
         }
         { # stroopwafel
           publicKey = "5iNRg13utOJ30pX2Z8SjwPNUFwfH2zonlbeYW2mKFkU=";
@@ -97,8 +99,8 @@
             "10.13.12.8/32"
             "fd00:b12f:acab:1312:acab:8::/96"
           ];
-
+          persistentKeepalive = 30;
-          persistentKeepalive = 25;
+          dynamicEndpointRefreshSeconds = 30;
         }
       ];
     };

hosts/stroopwafel/networking.nix

@@ -18,14 +18,14 @@
     privateKeyFile = config.age.secrets.wg-private-key.path;
   };
 
-  age.secrets.wg-public-key.file = "${flake.self}/secrets/wg-public-stroopwafel.age";
+  age.secrets.wg-tunnel-key.file = "${flake.self}/secrets/wg-tunnel-stroopwafel.age";
 
-  pub-solar.wireguard.public = {
+  pub-solar.wireguard.tunnel = {
     ownIPs = [
-      "10.68.3.133/32"
+      "10.65.141.174/32"
-      "fc00:bbbb:bbbb:bb01::5:384/128"
+      "fc00:bbbb:bbbb:bb01::2:8dad/128"
     ];
-    privateKeyFile = config.age.secrets.wg-public-key.path;
+    privateKeyFile = config.age.secrets.wg-tunnel-key.path;
     peer = {
       publicKey = "5FZW+fNA2iVBSY99HFl+KjGc9AFVNE+UFAedLNhu8lc=";
       endpoint = "146.70.134.2:3565";

modules/wireguard/default.nix

@@ -6,6 +6,6 @@
 }: {
   imports = [
     ./private.nix
-    ./public.nix
+    ./tunnel.nix
   ];
 }

modules/wireguard/private.nix

@@ -75,9 +75,13 @@ in {
         + (if cfg.fullTunnel then ''
           defaultRoute=$(${pkgs.iproute2}/bin/ip r | ${pkgs.gnugrep}/bin/grep "default via" | head -n 1 | ${pkgs.gawk}/bin/awk '{ print $3 " " $4 " " $5 }')
           ipv4=$(${pkgs.dnsutils}/bin/dig +short A vpn.b12f.io)
-          ${pkgs.iproute2}/bin/ip route add $ipv4 via $defaultRoute
+          ${pkgs.iproute2}/bin/ip route add $ipv4 metric 256 via $defaultRoute
           ipv6=$(${pkgs.dnsutils}/bin/dig +short AAAA vpn.b12f.io)
-          ${pkgs.iproute2}/bin/ip route add $ipv6 via $defaultRoute
+          ${pkgs.iproute2}/bin/ip route add $ipv6 metric 256 via $defaultRoute
+          ip -4 route delete default dev wg-private || true
+          ip -4 route replace default dev wg-private metric 512
+          ip -6 route delete default dev wg-private || true
+          ip -6 route replace default dev wg-private metric 512
         '' else "");
         postShutdown = lib.mkIf cfg.useDNS ''
           resolvconf -d wg-private -f
@@ -94,7 +98,8 @@ in {
               "::/0"
             ] else []);
             endpoint = "vpn.b12f.io:51899";
-            persistentKeepalive = 25;
+            persistentKeepalive = 30;
+            dynamicEndpointRefreshSeconds = 30;
           }
         ];
       };

modules/wireguard/public.nix

@@ -1,83 +0,0 @@
-{
-  lib,
-  config,
-  pkgs,
-  ...
-}:
-with lib; let
-  psCfg = config.pub-solar;
-  cfg = config.pub-solar.wireguard.public;
-in {
-  options.pub-solar.wireguard.public = {
-    ownIPs = mkOption {
-      description = "Internal ips in wireguard used for cluster control-plane communication.";
-      type = types.listOf types.str;
-      default = [];
-    };
-
-    privateKeyFile = mkOption {
-      description = "Location of private key file";
-      type = types.path;
-    };
-
-    peer = {
-      publicKey = mkOption {
-        description = "Public key of the peer";
-        type = types.str;
-      };
-      endpoint = mkOption {
-        description = "Peer endpoint address";
-        type = types.str;
-      };
-    };
-
-    useDNS = mkOption {
-      description = "Whether to use the DNS of the interface as default";
-      default = true;
-      type = types.bool;
-    };
-  };
-
-  config = mkIf (length cfg.ownIPs != 0){
-    networking.firewall.allowedUDPPorts = [51820];
-
-    systemd.services.wireguard-wg-public = mkIf (length config.pub-solar.wireguard.private.ownIPs != 0) {
-      after = [ "wireguard-wg-private.service" ];
-    };
-
-    networking.wireguard.interfaces = {
-      wg-public = {
-        listenPort = 51820;
-        ips = cfg.ownIPs;
-        privateKeyFile = cfg.privateKeyFile;
-        postSetup = let
-          splitEndpoint = (strings.splitString ":" cfg.peer.endpoint);
-          joinIPV6 = p: ip: p + (if (stringLength ip > 0) then ":" else "") + ip;
-          isIPV4 = length splitEndpoint < 3;
-          endpointIP = (if isIPV4
-            then elemAt splitEndpoint 0
-            else lists.fold joinIPV6 "" ((lists.take ((length splitEndpoint) - 1)) splitEndpoint)
-          );
-        in mkIf cfg.useDNS ''
-          printf "nameserver 10.64.0.1" | resolvconf -a wg-public -m 0 -x
-          defaultRoute=$(${pkgs.iproute2}/bin/ip r | ${pkgs.gnugrep}/bin/grep "default via" | head -n 1 | ${pkgs.gawk}/bin/awk '{ print $3 " " $4 " " $5 }')
-          ${pkgs.iproute2}/bin/ip ${if isIPV4 then "" else "-6"} route add ${endpointIP}${if isIPV4 then "" else "/128"} via $defaultRoute
-        '';
-        postShutdown = mkIf cfg.useDNS ''
-          resolvconf -d wg-public -f
-        '';
-        peers = [
-          {
-            publicKey = cfg.peer.publicKey;
-            allowedIPs = [
-              "0.0.0.0/0"
-              "::/0"
-            ];
-            endpoint = cfg.peer.endpoint;
-            persistentKeepalive = 25;
-          }
-        ];
-      };
-    };
-  };
-}

modules/wireguard/tunnel.nix

@@ -0,0 +1,96 @@
+{
+  lib,
+  config,
+  pkgs,
+  ...
+}:
+with lib; let
+  psCfg = config.pub-solar;
+  cfg = config.pub-solar.wireguard.tunnel;
+in {
+  options.pub-solar.wireguard.tunnel = {
+    ownIPs = mkOption {
+      description = "Internal ips in wireguard used for cluster control-plane communication.";
+      type = types.listOf types.str;
+      default = [];
+    };
+
+    privateKeyFile = mkOption {
+      description = "Location of private key file";
+      type = types.path;
+    };
+
+    peer = {
+      publicKey = mkOption {
+        description = "Public key of the peer";
+        type = types.str;
+      };
+      endpoint = mkOption {
+        description = "Peer endpoint address";
+        type = types.str;
+      };
+    };
+
+    useDNS = mkOption {
+      description = "Whether to use the DNS of the interface as default";
+      default = false;
+      type = types.bool;
+    };
+  };
+
+  config = mkIf (length cfg.ownIPs != 0){
+    networking.firewall.allowedUDPPorts = [51820];
+
+    systemd.services.wireguard-wg-tunnel = mkIf (length config.pub-solar.wireguard.private.ownIPs != 0) {
+      after = [ "wireguard-wg-private.service" ];
+    };
+
+    networking.wireguard.interfaces = let
+      splitEndpoint = (strings.splitString ":" cfg.peer.endpoint);
+      joinIPV6 = p: ip: p + (if (stringLength ip > 0) then ":" else "") + ip;
+      isIPV4 = length splitEndpoint < 3;
+      ipFlag = if isIPV4 then "-4" else "-6";
+      endpointIP = (if isIPV4
+        then elemAt splitEndpoint 0
+        else lists.fold joinIPV6 "" ((lists.take ((length splitEndpoint) - 1)) splitEndpoint)
+      );
+      endpointIPStripped = strings.removePrefix "[" (strings.removeSuffix "]" endpointIP);
+    in {
+      wg-tunnel = {
+        listenPort = 51820;
+        ips = cfg.ownIPs;
+        privateKeyFile = cfg.privateKeyFile;
+        postSetup = ''
+          defaultRoute=$(${pkgs.iproute2}/bin/ip ${ipFlag} r | ${pkgs.gnugrep}/bin/grep "default via" | head -n 1 | ${pkgs.gawk}/bin/awk '{ print $3 " " $4 " " $5 }')
+          ${pkgs.iproute2}/bin/ip ${ipFlag} route add "${endpointIPStripped}${if isIPV4 then "/32" else "/128"}" metric 256 via $defaultRoute
+          ip -4 route delete default dev wg-tunnel || true
+          ip -4 route add default dev wg-tunnel metric 512
+          ip -6 route delete default dev wg-tunnel || true
+          ip -6 route add default dev wg-tunnel metric 512
+        '' + (if cfg.useDNS
+          then ''printf "nameserver 10.64.0.1" | resolvconf -a wg-tunnel -m 0 -x''
+          else "");
+        postShutdown = ''
+          addedRoute=$(${pkgs.iproute2}/bin/ip ${ipFlag} r | ${pkgs.gnugrep}/bin/grep "${endpointIPStripped}" | head -n 1 | ${pkgs.gawk}/bin/awk '{ print $1 " " $2 " " $3 " " $4 " " $5 }')
+          if [ -n "$addedRoute" ]; then
+            ${pkgs.iproute2}/bin/ip ${ipFlag} route delete $addedRoute
+          fi
+        '' + (if cfg.useDNS
+          then ''resolvconf -d wg-tunnel -f''
+          else "");
+        peers = [
+          {
+            publicKey = cfg.peer.publicKey;
+            allowedIPs = [
+              "0.0.0.0/0"
+              "::/0"
+            ];
+            endpoint = cfg.peer.endpoint;
+            persistentKeepalive = 30;
+            dynamicEndpointRefreshSeconds = 30;
+          }
+        ];
+      };
+    };
+  };
+}

secrets/secrets.nix

@@ -80,8 +80,8 @@ in {
   "wg-private-droppie.age".publicKeys = droppieKeys ++ baseKeys;
   "wg-private-frikandel-server.age".publicKeys = frikandelKeys ++ baseKeys;
 
-  "wg-public-stroopwafel.age".publicKeys = stroopwafelKeys ++ baseKeys;
+  "wg-tunnel-stroopwafel.age".publicKeys = stroopwafelKeys ++ baseKeys;
-  "wg-public-droppie.age".publicKeys = droppieKeys ++ baseKeys;
+  "wg-tunnel-droppie.age".publicKeys = droppieKeys ++ baseKeys;
 
   "invoiceplane-db-password.age".publicKeys = pieKeys ++ baseKeys;
   "invoiceplane-db-secrets.env.age".publicKeys = pieKeys ++ baseKeys;

secrets/wg-tunnel-stroopwafel.age

@@ -0,0 +1,24 @@
+age-encryption.org/v1
+-> ssh-ed25519 b0WFDg SR5wr78Fr0kObzfzfr1UwFukdELVPZ5hht71Fa5fFxs
+3QndmGk9sbYth4QskAV1L8aVqK3JvIQo7wmo6tMsvMg
+-> ssh-rsa kFDS0A
+T6+WM2QvqQVErNTl8Fww+zCU6jA8gtZwFjZzeTlKxUMkAizae5X7ahsii9yRB6Sq
+d1eHjzSqfO87lTDyX/QTL88Gf07wpGYKMuphQheA+mFUQwUlGauhCXFcPH+hs1t/
+NRQLwqPnDp0tIL3toBbYMMyUSR170IEM76c0Q5aU6JCYN+lhFAJS7z3jN/uAOzlG
+KUnDtf/KV35jpIIW9rkkReOLHvFAkgtw2il6FVUq1OPN8Kkyl/HWOIAslC+ct/Ui
+459Cv0m3Keaa1lBOTa9UnrJrIevNZE7V3N0XZMvoYLPgNZisUvblyrJABQRAKA8g
+ivGn3l8DifN9UJwpW1SP/fej4ZqPZrrBUld4eerxoKyxLBqlrGiCBA3aYXsH6szS
+ke7K0G2/q+wUsGd7zaXuwSoozbcRb9hbz2BlVVeyPJwlkt9HmLDMCFLbN+l4/72j
+9tUrkmloKliyRPEOWmL+cVzzMZR7qZMPUmhW+N6Pb74KsfItfwoauJff2oNefC6u
+ngJbErZLBplpBYDYMlPXPXWj0D1ZJXQj99fu1kPuKY0O7ji2COewR2yCcuox7Ecd
+Zg8NyID97l6Gqy2MXOJU+QT7o9b6d0wm43X0nuSfbMnkjBsyYs+s4vzeRwC63J3K
+9xG+0qk9DPTaIvsxwjNL1022/SPXJWFtTI1PMVrnJP4
+-> piv-p256 zqq/iw AhBisE2Fe0vaKERI8QXX4PkYrl15sVVB4Gvhb+ztzYd3
+fyrDkCPy3uDC0V3QIxZ+pcEgYCaBTaRnQpyMykHOMrY
+-> piv-p256 vRzPNw A3cCq7Gh+pCEKMb4fuAAeYGKOCroXeoIaz0uV9PVpAY9
+sLd2F+4tMlRcslGQIXMkTDkfc6IQZZoywI3Cf/nFuEk
+-> \b*z-grease
+GMVOh1rMrdQqWp9U263dPc3h1ngcEsrE/ZUmtgD7urq6KsqnQaS0WJqtsZ5VzbWS
+
+--- A+ftgvNAqu9R+3wZSILocxfuTN782u1YzEvjIE1Lp9I
+àNç„i«ÌáBÚOýYµúh±PŸÑ:;¥{‹“¦XJÐÔŽÀ7Ñ¡íѬÎéH6¼’!<21>_ˆM”ƒ³SÖ¹òÞGЋIÖ?€ßº /Þ