Merge #236

236: init pkgs-lib and add tests and shell r=nrdxp a=Pacman99

Towards the goal of removing top level folders and putting more things in lib. This shifts shell and tests to lib under the `pkgs-lib` namespace. This namespace is separated by architecture, because those lib functions rely on nixpkgs architecture specific derivations. I think this is the cleanest way to do it, you don't have to instantiate pkgs-lib for each arch, and it exposes some of these functions for others to use.

This PR also fixes multi-arch checks.

Co-authored-by: Pacman99 <pachum99@gmail.com>

flake.nix

@@ -30,7 +30,7 @@
 
   outputs = inputs@{ deploy, nixos, nur, self, utils, ... }:
     let
-      lib = import ./lib { inherit self nixos inputs; };
+      lib = import ./lib { inherit self nixos utils inputs; };
     in
     lib.mkFlake
       {

lib/default.nix

@@ -19,6 +19,8 @@ lib.makeExtensible (final:
 
     mkFlake = callLibs ./mkFlake;
 
+    pkgs-lib = callLibs ./pkgs-lib;
+
     inherit (attrs) mapFilterAttrs genAttrs' safeReadDir
       pathsToImportedAttrs concatAttrs filterPackages;
     inherit (lists) pathsIn;

lib/mkFlake/default.nix

@@ -34,27 +34,21 @@ let
   systemOutputs = utils.lib.eachDefaultSystem (system:
     let
       pkgs = multiPkgs.${system};
+      pkgs-lib = dev.pkgs-lib.${system};
       # all packages that are defined in ./pkgs
       legacyPackages = os.mkPackages { inherit pkgs; };
     in
     {
-      checks =
-        let
-          tests = nixos.lib.optionalAttrs (system == "x86_64-linux")
-            (import "${self}/tests" { inherit self pkgs; });
-          deployHosts = nixos.lib.filterAttrs
-            (n: _: self.nixosConfigurations.${n}.config.nixpkgs.system == system)
-            self.deploy.nodes;
-          deployChecks = deploy.lib.${system}.deployChecks { nodes = deployHosts; };
-        in
-        nixos.lib.recursiveUpdate tests deployChecks;
+      checks = pkgs-lib.tests.mkChecks {
+        inherit (self.deploy) nodes;
+        hosts = self.nixosConfigurations;
+        homes = self.homeConfigurations;
+      };
 
       inherit legacyPackages;
       packages = dev.filterPackages system legacyPackages;
 
-      devShell = import "${self}/shell" {
-        inherit self system;
-      };
+      devShell = pkgs-lib.shell;
     });
 in
 outputs // systemOutputs

lib/pkgs-lib/default.nix

@@ -0,0 +1,20 @@
+args@{ lib, dev, utils, nixos, ... }:
+lib.genAttrs utils.lib.defaultSystems (system:
+  lib.makeExtensible (final:
+    let
+      pkgs = import nixos { inherit system; };
+      callLibs = file: import file
+        (args // {
+          inherit pkgs system;
+          pkgs-lib = final;
+        });
+    in
+    with final;
+    {
+      inherit callLibs;
+
+      tests = callLibs ./tests;
+      shell = callLibs ./shell;
+    }
+  )
+)

lib/pkgs-lib/shell/default.nix

@@ -1,25 +1,23 @@
-{ self ? (import ../compat).defaultNix
-, system ? builtins.currentSystem
-, extern ? import ../extern { inherit (self) inputs; }
-, overrides ? import ../overrides
-}:
+{ lib, dev, inputs, system, nixos, ... }:
 let
-  pkgs = (self.lib.os.mkPkgs {
-    inherit overrides extern;
-  }).${system};
+  overlays = [
+    inputs.devshell.overlay
+    (final: prev: {
+      deploy-rs =
+        inputs.deploy.packages.${prev.system}.deploy-rs;
+    })
+  ];
 
-  inherit (pkgs) lib;
+  pkgs = dev.os.pkgImport nixos overlays system;
+
+  flk = pkgs.callPackage ./flk.nix { };
 
   installPkgs = (lib.nixosSystem {
     inherit system;
     modules = [ ];
   }).config.system.build;
-
-  flk = pkgs.callPackage ./flk.nix { };
-
 in
-pkgs.devshell.mkShell
-{
+pkgs.devshell.mkShell {
   imports = [ (pkgs.devshell.importTOML ./devshell.toml) ];
 
   packages = with installPkgs; [

lib/pkgs-lib/tests/default.nix

@@ -1,17 +1,29 @@
-{ self, pkgs }:
+{ pkgs-lib, pkgs, system, inputs, nixos, lib, ... }:
 let
-  inherit (self.inputs) nixos;
-  inherit (self.nixosConfigurations.NixOS.config.lib) testModule specialArgs;
+  mkChecks = { hosts, nodes, homes ? { } }:
+    let
+      deployHosts = lib.filterAttrs
+        (n: _: hosts.${n}.config.nixpkgs.system == system)
+        nodes;
+      deployChecks = inputs.deploy.lib.${system}.deployChecks { nodes = deployHosts; };
+      tests =
+        { libTests = libTests; }
+        // lib.optionalAttrs (deployHosts != { }) {
+          profilesTest = profilesTest (hosts.${(builtins.head (builtins.attrNames deployHosts))});
+        } // lib.mapAttrs (n: v: v.home.activationPackage) homes;
 
-  mkTest =
+    in
+    lib.recursiveUpdate tests deployChecks;
+
+  mkTest = host:
     let
       nixosTesting =
         (import "${nixos}/nixos/lib/testing-python.nix" {
-          inherit (pkgs.stdenv.hostPlatform) system;
-          inherit specialArgs;
+          inherit system;
+          inherit (host.config.lib) specialArgs;
           inherit pkgs;
           extraConfigurations = [
-            testModule
+            host.config.lib.testModule
           ];
         });
     in
@@ -27,9 +39,8 @@ let
         else loadedTest;
     in
     nixosTesting.makeTest calledTest;
-in
-{
-  profilesTest = mkTest {
+
+  profilesTest = host: mkTest host {
     name = "profiles";
 
     machine = { suites, ... }: {
@@ -41,14 +52,12 @@ in
     '';
   };
 
-  homeTest = self.homeConfigurations."nixos@NixOS".home.activationPackage;
-
   libTests = pkgs.runCommandNoCC "devos-lib-tests"
     {
       buildInputs = [
         pkgs.nix
         (
-          let tests = import ./lib.nix { inherit self pkgs; };
+          let tests = pkgs-lib.callLibs ./lib.nix;
           in
           if tests == [ ]
           then null
@@ -70,5 +79,5 @@ in
 
     touch $out
   '';
-}
-
+in
+{ inherit mkTest libTests profilesTest mkChecks; }

lib/pkgs-lib/tests/lib.nix

@@ -1,6 +1,5 @@
-{ self, pkgs }:
-let inherit (self.inputs.nixos) lib; in
-with self.lib;
+{ pkgs, lib, dev, ... }:
+with dev;
 lib.runTests {
   testConcatAttrs = {
     expr = concatAttrs [{ foo = 1; } { bar = 2; } { baz = 3; }];
@@ -28,29 +27,24 @@ lib.runTests {
     expected = { foobar = 1; };
   };
 
-  testPathsIn =
-    let testPaths = pkgs.runCommandNoCC "test-paths-in" { } ''
-      mkdir -p $out/{foo,bar,baz}
-    '';
-    in
-    {
-      expr = pathsIn testPaths;
+  testPathsIn = {
+    expr = pathsIn (toString ./testPathsIn);
 
-      expected = [
-        "${testPaths}/bar"
-        "${testPaths}/baz"
-        "${testPaths}/foo"
+    expected = map toString [
+      ./testPathsIn/bar
+      ./testPathsIn/baz
+      ./testPathsIn/foo
     ];
   };
 
   testPathsToImportedAttrs = {
     expr =
       pathsToImportedAttrs [
-        "${self}/tests/testPathsToImportedAttrs/dir"
-        "${self}/tests/testPathsToImportedAttrs/foo.nix"
-        "${self}/tests/testPathsToImportedAttrs/bar.nix"
-        "${self}/tests/testPathsToImportedAttrs/t.nix"
-        "${self}/tests/testPathsToImportedAttrs/f.nix"
+        (toString ./testPathsToImportedAttrs/dir)
+        ./testPathsToImportedAttrs/foo.nix
+        ./testPathsToImportedAttrs/bar.nix
+        ./testPathsToImportedAttrs/t.nix
+        ./testPathsToImportedAttrs/f.nix
       ];
 
     expected = {
@@ -70,7 +64,7 @@ lib.runTests {
   ];
 
   testSafeReadDir = {
-    expr = safeReadDir "${self}/tests/profiles" // safeReadDir "${self}/nonexistentdir";
+    expr = safeReadDir ./profiles // safeReadDir ./nonexistentdir;
     expected = {
       foo = "directory";
       t = "directory";

tests/README.md

@@ -1,36 +0,0 @@
-# Testing
-
-Testing is always an important aspect of any software development project, and
-NixOS offers some incredibly powerful tools to write tests for your
-configuration, and, optionally, run them in
-[CI](../doc/integrations/hercules.md).
-
-## Lib Tests
-You can easily write tests for your own library functions in the
-___tests/lib.nix___ file and they will be run on every `nix flake check` or
-during a CI run.
-
-## Unit Tests
-Unit tests are can be created from regular derivations, and they can do
-almost anything you can imagine. By convention, it is best to test your
-packages during their [check phase][check]. All packages and their tests will
-be built during CI.
-
-## Integration Tests
-You can write integration tests for one or more NixOS VMs that can,
-optionally, be networked together, and yes, it's as awesome as it sounds!
-
-Be sure to use the `mkTest` function, in the [___tests/default.nix___][default]
-which wraps the official [testing-python][testing-python] function to ensure
-that the system is setup exactly as it is for a bare DevOS system. There are
-already great resources for learning how to use these tests effectively,
-including the official [docs][test-doc], a fantastic [blog post][test-blog],
-and the examples in [nixpkgs][nixos-tests].
-
-[test-doc]: https://nixos.org/manual/nixos/stable/index.html#sec-nixos-tests
-[test-blog]: https://www.haskellforall.com/2020/11/how-to-use-nixos-for-lightweight.html
-[default]: https://github.com/divnix/devos/tree/core/tests/default.nix
-[run-test]: https://github.com/NixOS/nixpkgs/blob/6571462647d7316aff8b8597ecdf5922547bf365/lib/debug.nix#L154-L166
-[nixos-tests]: https://github.com/NixOS/nixpkgs/tree/master/nixos/tests
-[testing-python]: https://github.com/NixOS/nixpkgs/tree/master/nixos/lib/testing-python.nix
-[check]: https://nixos.org/manual/nixpkgs/stable/#ssec-check-phase