treewide cleanups and refactoring for initial tests (#157)

- [x] refactor lib into separate files, similar to NixOS/nixpkgs/lib.
- [x] refactor ci to automatically generate derivations from flake outputs
- [x] remove cluttered indirection statements throughout the codebase
- [x] refactor hosts to allow for upcoming integration tests
- [x] improve ambiguity in the existing docs 
- [x] add [BORS](https://bors.tech) support
- [x] add initial integration test
- [x] write tests documentation
- [x] test lib
- [x] improve version string generation, and do so automatically for pkgs/flake.nix sources

Clean up the codebase as best we can in preparation for #152 and add tests. From now on, all PRs will be merged with BORS.
This commit is contained in:
Timothy DeHerrera 2021-03-14 07:10:51 +00:00
parent b06adb81ba
commit c012f2f4ed
33 changed files with 756 additions and 445 deletions

2
.envrc
View file

@ -1,2 +1,2 @@
watch_file **/*.nix
watch_file shell/* flake.nix
use flake || use nix

View file

@ -1,29 +1,31 @@
[![Build](https://img.shields.io/github/checks-status/divnix/devos/core)](https://hercules-ci.com/github/divnix/devos/jobs)
[![Bors enabled](https://bors.tech/images/badge_small.svg)](https://app.bors.tech/repositories/32678)
[![MIT License](https://img.shields.io/github/license/divnix/devos)][mit]
[![NixOS 20.09](https://img.shields.io/badge/NixOS-v20.09-blue.svg?style=flat&logo=NixOS&logoColor=white)](https://nixos.org)
> #### ⚠ Advisory ⚠
> DevOS leverages the [flakes][flakes] feature available via an _experimental_
> branch of [nix][nix]. Until nix 3.0 is released, this project should be
> considered unstable, though quite usable as flakes have been maturing
> _well_
> [for a while](https://github.com/divnix/devos/tree/17713c22d07c54525c728c62060a0428b76dee3b).
> DevOS requires the [flakes][flakes] feature available via an _experimental_
> branch of [nix][nix]. Until nix 3.0 is released, this project
> should be considered unstable, though quite usable as flakes have been
> maturing _well_ [for a while](https://github.com/divnix/devos/tree/17713c22d07c54525c728c62060a0428b76dee3b).
# Introduction
DevOS grants a simple way to use, deploy and manage [NixOS][nixos] systems for
personal and productive use. It does this by providing a convenient repository
structure, integrating several popular projects like
[home-manager][home-manager], and [devshell][devshell], and offering useful
conveniences like
personal and productive use. A sane repository structure is provided,
integrating several popular projects like [home-manager][home-manager],
[devshell][devshell], and [more](./doc/integrations).
Stiving for ___nix first™___ solutions with unobstrusive implementations,
a [flake centric][flake-doc] approach is taken for useful conveniences such as
[automatic source updates](./pkgs#automatic-source-updates).
Skip the indeterminate nature of other systems, _and_ the perceived difficulty
of Nix. It's easier than you think!
Skip the indeterminate nature of other systems, _and_ the perceived
tedium of bootstrapping Nix. It's easier than you think!
### Status
Alpha. A lot of the implementation is less than perfect, and huge redesigns
_will_ happen. There are unstable versions (0._x_._x_) to help users keep
track of changes and progress.
### Status: Alpha
A lot of the implementation is less than perfect, and huge
[redesigns](https://github.com/divnix/devos/issues/152) _will_ happen. There
are unstable versions (0._x_._x_) to help users keep track of changes and
progress.
## Getting Started
Check out the [guide](https://devos.divnix.com/doc/start) to get up and running.
@ -36,8 +38,8 @@ make critical comments about the [code][please]. 😜
NixOS provides an amazing abstraction to manage our environment, but that new
power can sometimes bring feelings of overwhelm and confusion. Having a turing
complete system can easily lead to unlimited complexity if we do it wrong.
Instead, we should have a community consensus on how to manage a NixOS system.
Help us reach that goal!
Instead, we should have a community consensus on how to manage a NixOS system
and its satellite projects, from which best practices can evolve.
___The future is declarative! 🎉___
@ -73,6 +75,7 @@ DevOS is licensed under the [MIT License][mit].
[nixos]: https://nixos.org/manual/nixos/stable
[home-manager]: https://nix-community.github.io/home-manager
[flakes]: https://nixos.wiki/wiki/Flakes
[flake-doc]: https://github.com/NixOS/nix/blob/master/src/nix/flake.md
[core]: https://github.com/divnix/devos
[community]: https://github.com/divnix/devos/tree/community
[dotfiles]: https://github.com/hlissner/dotfiles

View file

@ -16,6 +16,7 @@
- [Profiles](./profiles/README.md)
- [Secrets](./secrets/README.md)
- [Suites](./suites/README.md)
- [Tests](./tests/README.md)
- [Users](./users/README.md)
- [flk](./doc/flk/index.md)
- [up](./doc/flk/up.md)
@ -25,6 +26,6 @@
- [install](./doc/flk/install.md)
- [home](./doc/flk/home.md)
- [Integrations](doc/integrations/index.md)
- [deploy-rs](./doc/integrations/deploy.md)
- [hercules-ci](./doc/integrations/hercules.md)
- [Deploy RS](./doc/integrations/deploy.md)
- [Hercules CI](./doc/integrations/hercules.md)
- [Contributing](./doc/README.md)

12
bors.toml Normal file
View file

@ -0,0 +1,12 @@
status = [
"ci/hercules/evaluation",
"ci/hercules/derivations"
]
required_approvals = 1
up_to_date_approvals = true
delete_merged_branches = true
use_squash_merge = true

View file

@ -1,8 +1,8 @@
let
inherit (default.inputs.nixos.lib) recurseIntoAttrs;
inherit (default.inputs.nixos) lib;
default = (import ./compat).defaultNix;
in
builtins.mapAttrs (_: v: recurseIntoAttrs v) default.packages // {
builtins.mapAttrs (_: v: lib.recurseIntoAttrs v) default.packages // {
shell = import ./shell.nix;
}

View file

@ -1 +0,0 @@
# Contributing

View file

@ -4,6 +4,8 @@ relevant docs. Each directory contains its own README.md, which will
automatically be pulled into the [mdbook](https://devos.divnix.com). The book is
rendered on every change, so the docs should always be up to date.
We also use [BORS](https://bors.tech) to ensure that all pull requests pass the
test suite once at least one review is completed.
## Community PRs
While much of your work in this template may be idiosyncratic in nature. Anything

View file

@ -27,28 +27,14 @@
srcs.url = "path:./pkgs";
};
outputs =
inputs@{ ci-agent
, deploy
, devshell
, home
, nixos
, nixos-hardware
, nur
, override
, self
, utils
, ...
}:
outputs = inputs@{ deploy, nixos, nur, self, utils, ... }:
let
inherit (utils.lib) eachDefaultSystem flattenTreeSystem;
inherit (nixos.lib) recursiveUpdate;
inherit (self.lib) overlays nixosModules genPackages genPkgs
genHomeActivationPackages mkNodes;
inherit (self) lib;
inherit (lib) os;
extern = import ./extern { inherit inputs; };
pkgs' = genPkgs { inherit self; };
pkgs' = os.mkPkgs { inherit self; };
outputs =
let
@ -56,36 +42,42 @@
pkgs = pkgs'.${system};
in
{
inherit nixosModules overlays;
nixosConfigurations =
import ./hosts (recursiveUpdate inputs {
import ./hosts (nixos.lib.recursiveUpdate inputs {
inherit pkgs system extern;
inherit (pkgs) lib;
});
nixosModules =
let moduleList = import ./modules/module-list.nix;
in lib.pathsToImportedAttrs moduleList;
overlay = import ./pkgs;
overlays = lib.pathsToImportedAttrs (lib.pathsIn ./overlays);
lib = import ./lib { inherit nixos pkgs; };
templates.flk.path = ./.;
templates.flk.description = "flk template";
defaultTemplate = self.templates.flk;
deploy.nodes = mkNodes deploy self.nixosConfigurations;
deploy.nodes = os.mkNodes deploy self.nixosConfigurations;
checks = builtins.mapAttrs
(system: deployLib: deployLib.deployChecks self.deploy)
deploy.lib;
checks =
let
tests = import ./tests { inherit self pkgs; };
deployChecks = builtins.mapAttrs
(system: deployLib: deployLib.deployChecks self.deploy)
deploy.lib;
in
nixos.lib.recursiveUpdate tests deployChecks;
};
systemOutputs = eachDefaultSystem (system:
systemOutputs = utils.lib.eachDefaultSystem (system:
let pkgs = pkgs'.${system}; in
{
packages = flattenTreeSystem system
(genPackages {
packages = utils.lib.flattenTreeSystem system
(os.mkPackages {
inherit self pkgs;
});
@ -94,9 +86,9 @@
};
legacyPackages.hmActivationPackages =
genHomeActivationPackages { inherit self; };
os.mkHomeActivation { inherit self; };
}
);
in
recursiveUpdate outputs systemOutputs;
nixos.lib.recursiveUpdate outputs systemOutputs;
}

View file

@ -2,7 +2,6 @@
, home
, lib
, nixos
, nixos-hardware
, override
, pkgs
, self
@ -10,81 +9,89 @@
, ...
}:
let
inherit (lib.flk) recImport nixosSystemExtended defaultImports;
inherit (builtins) attrValues removeAttrs;
inherit (lib) dev;
suites = import ../suites { inherit lib; };
config = hostName:
nixosSystemExtended {
inherit system;
specialArgs = extern.specialArgs // { inherit suites; };
modules =
modules =
let
core = ../profiles/core;
modOverrides = { config, overrideModulesPath, ... }:
let
core = ../profiles/core;
modOverrides = { config, overrideModulesPath, ... }:
let
overrides = import ../overrides;
inherit (overrides) modules disabledModules;
in
{
disabledModules = modules ++ disabledModules;
imports = map
(path: "${overrideModulesPath}/${path}")
modules;
};
global = {
home-manager.useGlobalPkgs = true;
home-manager.useUserPackages = true;
hardware.enableRedistributableFirmware = lib.mkDefault true;
networking.hostName = hostName;
nix.nixPath = [
"nixpkgs=${nixos}"
"nixos-config=${self}/compat/nixos"
"home-manager=${home}"
];
nixpkgs = { inherit pkgs; };
nix.registry = {
devos.flake = self;
nixos.flake = nixos;
override.flake = override;
};
system.configurationRevision = lib.mkIf (self ? rev) self.rev;
};
local = {
require = [
"${toString ./.}/${hostName}.nix"
];
};
# Everything in `./modules/list.nix`.
flakeModules =
attrValues self.nixosModules;
overrides = import ../overrides;
inherit (overrides) modules disabledModules;
in
flakeModules ++ [
core
global
local
modOverrides
] ++ extern.modules;
{
disabledModules = modules ++ disabledModules;
imports = map
(path: "${overrideModulesPath}/${path}")
modules;
};
global = {
home-manager.useGlobalPkgs = true;
home-manager.useUserPackages = true;
hardware.enableRedistributableFirmware = lib.mkDefault true;
nix.nixPath = [
"nixpkgs=${nixos}"
"nixos-config=${self}/compat/nixos"
"home-manager=${home}"
];
nixpkgs = { inherit pkgs; };
nix.registry = {
devos.flake = self;
nixos.flake = nixos;
override.flake = override;
};
system.configurationRevision = lib.mkIf (self ? rev) self.rev;
};
# Everything in `./modules/list.nix`.
flakeModules =
builtins.attrValues self.nixosModules;
in
flakeModules ++ [
core
global
modOverrides
] ++ extern.modules;
specialArgs = extern.specialArgs // { inherit suites; };
mkHostConfig = hostName:
let
local = {
require = [
"${toString ./.}/${hostName}.nix"
];
networking = { inherit hostName; };
};
in
dev.os.devosSystem {
inherit system specialArgs;
modules = modules ++ [
local
{
lib = { inherit specialArgs; };
lib.testModule = {
imports = modules;
};
}
];
};
hosts = recImport {
dir = ./.;
_import = config;
};
hosts = dev.os.recImport
{
dir = ./.;
_import = mkHostConfig;
};
in
hosts

29
lib/attrs.nix Normal file
View file

@ -0,0 +1,29 @@
{ lib, ... }:
rec {
# mapFilterAttrs ::
# (name -> value -> bool )
# (name -> value -> { name = any; value = any; })
# attrs
mapFilterAttrs = seive: f: attrs:
lib.filterAttrs
seive
(lib.mapAttrs' f attrs);
# Generate an attribute set by mapping a function over a list of values.
genAttrs' = values: f: lib.listToAttrs (map f values);
# Convert a list of file paths to attribute set
# that has the filenames stripped of nix extension as keys
# and imported content of the file as value.
#
pathsToImportedAttrs = paths:
let
paths' = lib.filter (lib.hasSuffix ".nix") paths;
in
genAttrs' paths' (path: {
name = lib.removeSuffix ".nix" (baseNameOf path);
value = import path;
});
concatAttrs = lib.fold (attr: sum: lib.recursiveUpdate sum attr) { };
}

View file

@ -1,232 +1,21 @@
{ nixos, pkgs, ... }:
let
inherit (builtins) attrNames attrValues isAttrs readDir listToAttrs mapAttrs
pathExists filter;
args@{ nixos, pkgs, ... }:
let inherit (nixos) lib; in
lib.makeExtensible (self:
let callLibs = file: import file
({
inherit lib;
inherit (nixos.lib) fold filterAttrs hasSuffix mapAttrs' nameValuePair removeSuffix
recursiveUpdate genAttrs nixosSystem mkForce substring optionalAttrs;
dev = self;
} // args);
in
with self;
{
inherit callLibs;
# mapFilterAttrs ::
# (name -> value -> bool )
# (name -> value -> { name = any; value = any; })
# attrs
mapFilterAttrs = seive: f: attrs: filterAttrs seive (mapAttrs' f attrs);
attrs = callLibs ./attrs.nix;
os = callLibs ./devos;
lists = callLibs ./lists.nix;
# Generate an attribute set by mapping a function over a list of values.
genAttrs' = values: f: listToAttrs (map f values);
# pkgImport :: Nixpkgs -> Overlays -> System -> Pkgs
pkgImport = nixpkgs: overlays: system:
import nixpkgs {
inherit system overlays;
config = { allowUnfree = true; };
};
# Convert a list to file paths to attribute set
# that has the filenames stripped of nix extension as keys
# and imported content of the file as value.
#
pathsToImportedAttrs = paths:
let
paths' = filter (hasSuffix ".nix") paths;
in
genAttrs' paths' (path: {
name = removeSuffix ".nix" (baseNameOf path);
value = import path;
});
overlayPaths =
let
overlayDir = ../overlays;
fullPath = name: overlayDir + "/${name}";
in
map fullPath (attrNames (readDir overlayDir));
/**
Synopsis: mkNodes _nixosConfigurations_
Generate the `nodes` attribute expected by deploy-rs
where _nixosConfigurations_ are `nodes`.
**/
mkNodes = deploy: mapAttrs (_: config: {
hostname = config.config.networking.hostName;
profiles.system = {
user = "root";
path = deploy.lib.x86_64-linux.activate.nixos config;
};
});
/**
Synopsis: mkProfileAttrs _path_
Recursively import the subdirs of _path_ containing a default.nix.
Example:
let profiles = mkProfileAttrs ./profiles; in
assert profiles ? core.default; 0
**/
mkProfileAttrs = dir:
let
imports =
let
files = readDir dir;
p = n: v:
v == "directory"
&& n != "profiles";
in
filterAttrs p files;
f = n: _:
optionalAttrs
(pathExists "${dir}/${n}/default.nix")
{ default = "${dir}/${n}"; }
// mkProfileAttrs "${dir}/${n}";
in
mapAttrs f imports;
in
{
inherit mkProfileAttrs mapFilterAttrs genAttrs' pkgImport
pathsToImportedAttrs mkNodes;
overlays = pathsToImportedAttrs overlayPaths;
mkVersion = src: "${substring 0 8 src.lastModifiedDate}_${src.shortRev}";
genPkgs = { self }:
let inherit (self) inputs;
in
(inputs.utils.lib.eachDefaultSystem
(system:
let
extern = import ../extern { inherit inputs; };
overridePkgs = pkgImport inputs.override [ ] system;
overridesOverlay = (import ../overrides).packages;
overlays = [
(overridesOverlay overridePkgs)
self.overlay
(final: prev: {
srcs = self.inputs.srcs.inputs;
lib = (prev.lib or { }) // {
inherit (nixos.lib) nixosSystem;
flk = self.lib;
utils = inputs.utils.lib;
};
})
]
++ extern.overlays
++ (attrValues self.overlays);
in
{ pkgs = pkgImport nixos overlays system; }
)
).pkgs;
profileMap = map (profile: profile.default);
recImport = { dir, _import ? base: import "${dir}/${base}.nix" }:
mapFilterAttrs
(_: v: v != null)
(n: v:
if n != "default.nix" && hasSuffix ".nix" n && v == "regular"
then
let name = removeSuffix ".nix" n; in nameValuePair (name) (_import name)
else
nameValuePair ("") (null))
(readDir dir);
nixosSystemExtended = { modules, ... } @ args:
nixosSystem (args // {
modules =
let
modpath = "nixos/modules";
cd = "installer/cd-dvd/installation-cd-minimal-new-kernel.nix";
ciConfig =
(nixosSystem (args // {
modules =
let
# remove host module
modules' = filter (x: ! x ? require) modules;
in
modules' ++ [
({ suites, ... }: {
imports = with suites;
allProfiles ++ allUsers;
boot.loader.systemd-boot.enable = true;
boot.loader.efi.canTouchEfiVariables = true;
fileSystems."/" = { device = "/dev/disk/by-label/nixos"; };
})
];
})).config;
isoConfig = (nixosSystem
(args // {
modules = modules ++ [
"${nixos}/${modpath}/${cd}"
({ config, ... }: {
isoImage.isoBaseName = "nixos-" + config.networking.hostName;
# confilcts with networking.wireless which might be slightly
# more useful on a stick
networking.networkmanager.enable = mkForce false;
# confilcts with networking.wireless
networking.wireless.iwd.enable = mkForce false;
})
];
})).config;
in
modules ++ [{
system.build = {
iso = isoConfig.system.build.isoImage;
ci = ciConfig.system.build.toplevel;
};
}];
});
nixosModules =
let
# binary cache
cachix = import ../cachix.nix;
cachixAttrs = { inherit cachix; };
# modules
moduleList = import ../modules/module-list.nix;
modulesAttrs = pathsToImportedAttrs moduleList;
in
recursiveUpdate cachixAttrs modulesAttrs;
genHomeActivationPackages = { self }:
let hmConfigs =
builtins.mapAttrs
(_: config: config.config.home-manager.users)
self.nixosConfigurations;
in
mapAttrs
(_: x: mapAttrs
(_: cfg: cfg.home.activationPackage)
x)
hmConfigs;
genPackages = { self, pkgs }:
let
inherit (self) overlay overlays;
packagesNames = attrNames (overlay null null)
++ attrNames (fold
(attr: sum: recursiveUpdate sum attr)
{ }
(attrValues
(mapAttrs (_: v: v null null) overlays)
)
);
in
fold
(key: sum: recursiveUpdate sum {
${key} = pkgs.${key};
})
{ }
packagesNames;
}
inherit (attrs) mapFilterAttrs genAttrs' pathsToImportedAttrs concatAttrs;
inherit (lists) pathsIn;
})

26
lib/devos/default.nix Normal file
View file

@ -0,0 +1,26 @@
{ lib, nixos, dev, ... }:
{
# pkgImport :: Nixpkgs -> Overlays -> System -> Pkgs
pkgImport = nixpkgs: overlays: system:
import nixpkgs {
inherit system overlays;
config = { allowUnfree = true; };
};
profileMap = map (profile: profile.default);
mkNodes = dev.callLibs ./mkNodes.nix;
mkProfileAttrs = dev.callLibs ./mkProfileAttrs.nix;
mkPkgs = dev.callLibs ./mkPkgs.nix;
recImport = dev.callLibs ./recImport.nix;
devosSystem = dev.callLibs ./devosSystem.nix;
mkHomeActivation = dev.callLibs ./mkHomeActivation.nix;
mkPackages = dev.callLibs ./mkPackages.nix;
}

30
lib/devos/devosSystem.nix Normal file
View file

@ -0,0 +1,30 @@
{ lib, nixos, ... }:
{ modules, ... } @ args:
lib.nixosSystem (args // {
modules =
let
modpath = "nixos/modules";
cd = "installer/cd-dvd/installation-cd-minimal-new-kernel.nix";
isoConfig = (lib.nixosSystem
(args // {
modules = modules ++ [
"${nixos}/${modpath}/${cd}"
({ config, ... }: {
isoImage.isoBaseName = "nixos-" + config.networking.hostName;
# confilcts with networking.wireless which might be slightly
# more useful on a stick
networking.networkmanager.enable = lib.mkForce false;
# confilcts with networking.wireless
networking.wireless.iwd.enable = lib.mkForce false;
})
];
})).config;
in
modules ++ [{
system.build = {
iso = isoConfig.system.build.isoImage;
};
}];
})

View file

@ -0,0 +1,13 @@
{ lib, ... }:
{ self }:
let hmConfigs =
lib.mapAttrs
(_: config: config.config.home-manager.users)
self.nixosConfigurations;
in
lib.mapAttrs
(_: x: lib.mapAttrs
(_: cfg: cfg.home.activationPackage)
x)
hmConfigs

16
lib/devos/mkNodes.nix Normal file
View file

@ -0,0 +1,16 @@
{ lib, ... }:
/**
Synopsis: mkNodes _nixosConfigurations_
Generate the `nodes` attribute expected by deploy-rs
where _nixosConfigurations_ are `nodes`.
**/
deploy: lib.mapAttrs (_: config: {
hostname = config.config.networking.hostName;
profiles.system = {
user = "root";
path = deploy.lib.x86_64-linux.activate.nixos config;
};
})

18
lib/devos/mkPackages.nix Normal file
View file

@ -0,0 +1,18 @@
{ lib, dev, ... }:
{ self, pkgs }:
let
inherit (self) overlay overlays;
packagesNames = lib.attrNames (overlay null null)
++ lib.attrNames (dev.concatAttrs
(lib.attrValues
(lib.mapAttrs (_: v: v null null) overlays)
)
);
in
lib.fold
(key: sum: lib.recursiveUpdate sum {
${key} = pkgs.${key};
})
{ }
packagesNames

66
lib/devos/mkPkgs.nix Normal file
View file

@ -0,0 +1,66 @@
{ lib, dev, nixos, ... }:
{ self }:
let inherit (self) inputs;
in
(inputs.utils.lib.eachDefaultSystem
(system:
let
extern = import ../../extern { inherit inputs; };
overridePkgs = dev.os.pkgImport inputs.override [ ] system;
overridesOverlay = (import ../../overrides).packages;
overlays = [
(overridesOverlay overridePkgs)
self.overlay
(final: prev: {
srcs =
let
mkVersion = name: input:
let
inputs = (builtins.fromJSON
(builtins.readFile ../../flake.lock)).nodes;
ref =
if lib.hasAttrByPath [ name "original" "ref" ] inputs
then inputs.${name}.original.ref
else "";
version =
let version' = builtins.match
"[[:alpha:]]*[-._]?([0-9]+(\.[0-9]+)*)+"
ref;
in
if lib.isList version'
then lib.head version'
else if input ? lastModifiedDate && input ? shortRev
then "${lib.substring 0 8 input.lastModifiedDate}_${input.shortRev}"
else null;
in
version;
in
lib.mapAttrs
(name: input:
let
version = mkVersion name input;
in
input // lib.optionalAttrs (! isNull version)
{
inherit version;
}
)
self.inputs.srcs.inputs;
lib = prev.lib.extend (lfinal: lprev: {
inherit dev;
inherit (lib) nixosSystem;
utils = inputs.utils.lib;
});
})
]
++ extern.overlays
++ (lib.attrValues self.overlays);
in
{ pkgs = dev.os.pkgImport nixos overlays system; }
)
).pkgs

View file

@ -0,0 +1,35 @@
{ lib, ... }:
let mkProfileAttrs =
/**
Synopsis: mkProfileAttrs _path_
Recursively collect the subdirs of _path_ containing a default.nix into attrs.
This sets a contract, eliminating ambiguity for _default.nix_ living under the
profile directory.
Example:
let profiles = mkProfileAttrs ./profiles; in
assert profiles ? core.default; 0
**/
dir:
let
imports =
let
files = builtins.readDir dir;
p = n: v:
v == "directory"
&& n != "profiles";
in
lib.filterAttrs p files;
f = n: _:
lib.optionalAttrs
(lib.pathExists "${dir}/${n}/default.nix")
{ default = "${dir}/${n}"; }
// mkProfileAttrs "${dir}/${n}";
in
lib.mapAttrs f imports;
in mkProfileAttrs

12
lib/devos/recImport.nix Normal file
View file

@ -0,0 +1,12 @@
{ lib, dev, ... }:
{ dir, _import ? base: import "${dir}/${base}.nix" }:
dev.mapFilterAttrs
(_: v: v != null)
(n: v:
if n != "default.nix" && lib.hasSuffix ".nix" n && v == "regular"
then
let name = lib.removeSuffix ".nix" n; in lib.nameValuePair (name) (_import name)
else
lib.nameValuePair ("") (null))
(builtins.readDir dir)

8
lib/lists.nix Normal file
View file

@ -0,0 +1,8 @@
{ lib, ... }:
{
pathsIn = dir:
let
fullPath = name: "${toString dir}/${name}";
in
map fullPath (lib.attrNames (builtins.readDir dir));
}

View file

@ -1,32 +1,31 @@
let
inherit (default.inputs.nixos.lib) mapAttrs recurseIntoAttrs;
inherit (default.inputs.nixos) lib;
default = (import "${../.}/compat").defaultNix;
packages = import ../default.nix;
ciSystems = [
"aarch64-linux"
"i686-linux"
"x86_64-linux"
];
filterSystems = lib.filterAttrs
(system: _: lib.elem system ciSystems);
recurseIntoAttrsRecursive = lib.mapAttrs (_: v:
if lib.isAttrs v
then recurseIntoAttrsRecursive (lib.recurseIntoAttrs v)
else v
);
systemOutputs = lib.filterAttrs
(_: set: lib.isAttrs set
&& lib.any
(system: set ? ${system})
ciSystems
)
default.outputs;
ciDrvs = lib.mapAttrs (_: system: filterSystems system) systemOutputs;
in
{
checks = recurseIntoAttrs (mapAttrs (_: v: recurseIntoAttrs v) {
inherit (default.checks)
aarch64-linux
i686-linux
x86_64-linux
;
});
# platforms supported by our hercules-ci agent
inherit (packages)
aarch64-linux
i686-linux
x86_64-linux
;
devShell = recurseIntoAttrs {
inherit (default.devShell)
aarch64-linux
i686-linux
x86_64-linux
;
};
nixos = default.nixosConfigurations.NixOS.config.system.build.ci;
}
recurseIntoAttrsRecursive ciDrvs

View file

@ -16,38 +16,31 @@ And, as usual, every package in the overlay is also available to any NixOS
## Automatic Source Updates
There is the added, but optional, convenience of declaring your sources in
_pkgs/flake.nix_ as an input. This allows updates to be managed automatically
by simply [updating](../doc/flk/update.md#updating-package-sources) the lock
file. No more manually entering sha256 hashes!
_pkgs/flake.nix_ as an input. You can then access them from the `srcs` package.
This allows updates to be managed automatically by simply
[updating](../doc/flk/update.md#updating-package-sources) the lock file. No
more manually entering sha256 hashes!
As an added bonus, version strings are also generated automatically from either
the flake ref, or the date and git revision of the source. For examples,
definitely checkout the [community branch](../#community-profiles).
## Example
pkgs/development/libraries/libinih/default.nix:
```nix
{ stdenv, meson, ninja, lib, srcs, ... }:
let version = "r53";
in
let inherit (srcs) libinih; in
stdenv.mkDerivation {
pname = "libinih";
inherit version;
src = srcs.libinih;
# version will resolve to 53, as specified in the final example below
inherit (libinih) version;
src = libinih;
buildInputs = [ meson ninja ];
mesonFlags = ''
-Ddefault_library=shared
-Ddistro_install=true
'';
meta = with lib; {
description = "Simple .INI file parser in C";
homepage = "https://github.com/benhoyt/inih";
maintainers = [ maintainers.divnix ];
license = licenses.bsd3;
platforms = platforms.all;
inherit version;
};
# ...
}
```

View file

@ -1,43 +1,34 @@
# Profiles
Profiles are simply NixOS modules which contain generic expressions suitable
for any host. A good example is the configuration for a text editor, or
window manager. If you need some concrete examples, just checkout the
community [branch](https://github.com/divnix/devos/tree/community/profiles).
Profiles are a convenient shorthand for the [_definition_][definition] of
[options][options] in contrast to their [_declaration_][declaration]. They're
built into the NixOS module system for a reason: to elegantly provide a clear
separation of concerns.
If you need guidance, a community [branch](https://github.com/divnix/devos/tree/community/profiles)
is maintained to help get up to speed on their usage.
## Constraints
For the sake of consistency, a profile should always be defined in a
_default.nix_ containing a valid [nixos module](https://nixos.wiki/wiki/Module)
which ___does not___ declare any new
[module options](https://nixos.org/manual/nixos/stable/index.html#sec-option-declarations).
If you need to do that, use the [modules directory](../modules).
___default.nix___ containing a [nixos module config][config].
A profile's directory is used for quick modularization of
[interelated bits](./#subprofiles).
> ##### _Note:_
> [hercules-ci](../doc/integrations/hercules.md) expects all profiles to be
> defined in a _default.nix_. Similarly, [suites](../suites) expect a
> _default.nix_ as well.
### Example
#### Correct ✔
profiles/develop/default.nix:
```nix
{ ... }:
{
programs.zsh.enable = true;
}
```
#### Incorrect ❌
profiles/develop.nix:
```nix
{
options = {};
}
```
> ##### _Notes:_
> * For _declaring_ module options, there's the [modules](../modules) directory.
> * This directory takes inspiration from
> [upstream](https://github.com/NixOS/nixpkgs/tree/master/nixos/modules/profiles)
> .
> * Sticking to a simple [spec][spec] has refreshing advantages.
> [hercules-ci](../doc/integrations/hercules.md) expects all profiles to be
> defined in a ___default.nix___, allowing them to be built automatically when
> added. Congruently, [suites](../suites) expect ___default.nix___ to avoid
> having to manage their paths manually.
## Subprofiles
Profiles can also define subprofiles. They follow the same constraints outlined
above. A good top level profile should be a high level concern, such a your
personal development environment, and the subprofiles should be more concrete
above. A good top level profile should be a high level concern, such as your
personal development environment while the subprofiles should be more focused
program configurations such as your text editor, and shell configs. This way,
you can either pull in the whole development profile, or pick and choose
individual programs.
@ -62,8 +53,13 @@ profiles/develop/zsh/default.nix:
```
## Conclusion
Profiles are the most important concept in devos. They allow us to keep our
nix expressions self contained and modular. This way we can maximize reuse
while minimizing boilerplate. Always strive to keep your profiles as generic
and modular as possible. Anything machine specific belongs in your
[host](../hosts) files.
Profiles are the most important concept in DevOS. They allow us to keep our
Nix expressions self contained and modular. This way we can maximize reuse
across hosts while minimizing boilerplate. Remember, anything machine
specific belongs in your [host](../hosts) files instead.
[definition]: https://nixos.org/manual/nixos/stable/index.html#sec-option-definitions
[declaration]: https://nixos.org/manual/nixos/stable/index.html#sec-option-declarations
[options]: https://nixos.org/manual/nixos/stable/index.html#sec-writing-modules
[spec]: https://github.com/divnix/devos/tree/core/lib/devos/mkProfileAttrs.nix
[config]: https://nixos.wiki/wiki/Module#structure

View file

@ -2,7 +2,7 @@
, system ? builtins.currentSystem
}:
let
pkgs = (self.lib.genPkgs { inherit self; }).${system};
pkgs = (self.lib.os.mkPkgs { inherit self; }).${system};
inherit (pkgs) lib;

View file

@ -1,10 +1,9 @@
{ lib }:
let
inherit (builtins) mapAttrs isFunction;
inherit (lib.flk) mkProfileAttrs profileMap;
inherit (lib) dev;
profiles = mkProfileAttrs (toString ../profiles);
users = mkProfileAttrs (toString ../users);
profiles = dev.os.mkProfileAttrs (toString ../profiles);
users = dev.os.mkProfileAttrs (toString ../users);
allProfiles =
let defaults = lib.collect (x: x ? default) profiles;
@ -19,6 +18,6 @@ let
base = [ users.nixos users.root ];
};
in
mapAttrs (_: v: profileMap v) suites // {
lib.mapAttrs (_: v: dev.os.profileMap v) suites // {
inherit allProfiles allUsers;
}

View file

@ -0,0 +1,61 @@
From 9f33ab62d99c98e3f5bddd64532f15f482cf01b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Janne=20He=C3=9F?= <janne@hess.ooo>
Date: Tue, 2 Jun 2020 16:27:07 +0200
Subject: [PATCH 04/22] nixos/testing: Add support for specialArgs
Since using flakes disallows the usage of <unstable> (which I use in
some tests), this adds an alternative. By setting specialArgs, all VMs
can get the `unstable` flake input as an arg. This is not possible with
extraConfigurations, as that would lead to infinite recursions.
---
nixos/lib/build-vms.nix | 8 +++++---
nixos/lib/testing-python.nix | 4 +++-
2 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/nixos/lib/build-vms.nix b/nixos/lib/build-vms.nix
index 1bad63b9194..b1575fc13bb 100644
--- a/nixos/lib/build-vms.nix
+++ b/nixos/lib/build-vms.nix
@@ -3,8 +3,10 @@
minimal ? false
, # Ignored
config ? null
- # Nixpkgs, for qemu, lib and more
-, pkgs
+, # Nixpkgs, for qemu, lib and more
+ pkgs
+, # !!! See comment about args in lib/modules.nix
+ specialArgs ? {}
, # NixOS configuration to add to the VMs
extraConfigurations ? []
}:
@@ -31,7 +33,7 @@ rec {
nodes: configurations:
import ./eval-config.nix {
- inherit system;
+ inherit system specialArgs;
modules = configurations ++ extraConfigurations;
baseModules = (import ../modules/module-list.nix) ++
[ ../modules/virtualisation/qemu-vm.nix
diff --git a/nixos/lib/testing-python.nix b/nixos/lib/testing-python.nix
index 76a2022082c..498f97336c0 100644
--- a/nixos/lib/testing-python.nix
+++ b/nixos/lib/testing-python.nix
@@ -4,10 +4,12 @@
, minimal ? false
# Ignored
, config ? {}
+ # !!! See comment about args in lib/modules.nix
+, specialArgs ? {}
# Modules to add to each VM
, extraConfigurations ? [] }:
-with import ./build-vms.nix { inherit system pkgs minimal extraConfigurations; };
+with import ./build-vms.nix { inherit system pkgs minimal specialArgs extraConfigurations; };
with pkgs;
rec {
--
2.29.2

36
tests/README.md Normal file
View file

@ -0,0 +1,36 @@
# Testing
Testing is always an important aspect of any software development project, and
NixOS offers some incredibly powerful tools to write tests for your
configuration, and, optionally, run them in
[CI](../doc/integrations/hercules.md).
## Lib Tests
You can easily write tests for your own library functions in the
___tests/lib.nix___ file and they will be run on every `nix flake check` or
during a CI run.
## Unit Tests
Unit tests are can be created from regular derivations, and they can do
almost anything you can imagine. By convention, it is best to test your
packages during their [check phase][check]. All packages and their tests will
be built during CI.
## Integration Tests
You can write integration tests for one or more NixOS VMs that can,
optionally, be networked together, and yes, it's as awesome as it sounds!
Be sure to use the `mkTest` function, in the [___tests/default.nix___][default]
which wraps the official [testing-python][testing-python] function to ensure
that the system is setup exactly as it is for a bare DevOS system. There are
already great resources for learning how to use these tests effectively,
including the official [docs][test-doc], a fantastic [blog post][test-blog],
and the examples in [nixpkgs][nixos-tests].
[test-doc]: https://nixos.org/manual/nixos/stable/index.html#sec-nixos-tests
[test-blog]: https://www.haskellforall.com/2020/11/how-to-use-nixos-for-lightweight.html
[default]: https://github.com/divnix/devos/tree/core/tests/default.nix
[run-test]: https://github.com/NixOS/nixpkgs/blob/6571462647d7316aff8b8597ecdf5922547bf365/lib/debug.nix#L154-L166
[nixos-tests]: https://github.com/NixOS/nixpkgs/tree/master/nixos/tests
[testing-python]: https://github.com/NixOS/nixpkgs/tree/master/nixos/lib/testing-python.nix
[check]: https://nixos.org/manual/nixpkgs/stable/#ssec-check-phase

103
tests/default.nix Normal file
View file

@ -0,0 +1,103 @@
{ self, pkgs }:
let
inherit (self.inputs) nixos;
inherit (self.nixosConfigurations.NixOS.config.lib) testModule specialArgs;
# current release 20.09 does not support the specialArgs required for us
# to use tests as we would normally use hosts. Using the "testing-python.nix"
# from the override flake would build the test-vm from an unstable os
# different than the one our systems are running. Instead simply patch nixpkgs
# to include the updated version. This can be removed in the next release
patchedNixpkgs =
pkgs.stdenv.mkDerivation {
name = "nixpkgs-patched";
src = nixos;
patches = [ ./0004-nixos-testing-Add-support-for-specialArgs.patch ];
dontBuild = true;
dontFixup = true;
versionSuffix = "pre${
if nixos ? lastModified
then builtins.substring 0 8 (nixos.lastModifiedDate or nixos.lastModified)
else toString nixos.revCount}.${nixos.shortRev or "dirty"}";
configurePhase = ''
echo -n $VERSION_SUFFIX > .version-suffix
echo -n ${nixos.rev or nixos.shortRev or "dirty"} > .git-revision
'';
installPhase = ''
cp -r $PWD $out
'';
};
mkTest =
let
nixosTesting =
(import "${patchedNixpkgs}/nixos/lib/testing-python.nix" {
inherit (pkgs.stdenv.hostPlatform) system;
inherit specialArgs;
inherit pkgs;
extraConfigurations = [
testModule
];
});
in
test:
let
loadedTest =
if builtins.typeOf test == "path"
then import test
else test;
calledTest =
if pkgs.lib.isFunction loadedTest
then pkgs.callPackage loadedTest { }
else loadedTest;
in
nixosTesting.makeTest calledTest;
in
{
x86_64-linux = {
profilesTest = mkTest {
name = "profiles";
machine = { suites, ... }: {
imports = suites.allProfiles ++ suites.allUsers;
};
testScript = ''
machine.systemctl("is-system-running --wait")
'';
};
libTests = pkgs.runCommandNoCC "devos-lib-tests"
{
buildInputs = [
pkgs.nix
(
let tests = import ./lib.nix { inherit self pkgs; };
in
if tests == [ ]
then null
else throw (builtins.toJSON tests)
)
];
} ''
datadir="${pkgs.nix}/share"
export TEST_ROOT=$(pwd)/test-tmp
export NIX_BUILD_HOOK=
export NIX_CONF_DIR=$TEST_ROOT/etc
export NIX_LOCALSTATE_DIR=$TEST_ROOT/var
export NIX_LOG_DIR=$TEST_ROOT/var/log/nix
export NIX_STATE_DIR=$TEST_ROOT/var/nix
export NIX_STORE_DIR=$TEST_ROOT/store
export PAGER=cat
cacheDir=$TEST_ROOT/binary-cache
nix-store --init
touch $out
'';
};
}

62
tests/lib.nix Normal file
View file

@ -0,0 +1,62 @@
{ self, pkgs }:
let inherit (self.inputs.nixos) lib; in
with self.lib;
lib.runTests {
testConcatAttrs = {
expr = concatAttrs [{ foo = 1; } { bar = 2; } { baz = 3; }];
expected = { foo = 1; bar = 2; baz = 3; };
};
testGenAttrs' = {
expr = genAttrs'
[ "/foo/bar" "/baz/buzz" ]
(path: {
name = baseNameOf path;
value = "${path}/fizz";
});
expected = { bar = "/foo/bar/fizz"; buzz = "/baz/buzz/fizz"; };
};
testMapFilterAttrs = {
expr = mapFilterAttrs
(n: v: n == "foobar" && v == 1)
(n: v: lib.nameValuePair ("${n}bar") (v + 1))
{ foo = 0; bar = 2; };
expected = { foobar = 1; };
};
testPathsIn =
let testPaths = pkgs.runCommandNoCC "test-paths-in" { } ''
mkdir -p $out/{foo,bar,baz}
'';
in
{
expr = pathsIn testPaths;
expected = [
"${testPaths}/bar"
"${testPaths}/baz"
"${testPaths}/foo"
];
};
testPathsToImportedAttrs = {
expr =
pathsToImportedAttrs [
./testPathsToImportedAttrs/foo.nix
./testPathsToImportedAttrs/bar.nix
./testPathsToImportedAttrs/t.nix
./testPathsToImportedAttrs/f.nix
];
expected = {
foo = { bar = 1; };
bar = { foo = 2; };
t = true;
f = false;
};
};
}

View file

@ -0,0 +1 @@
{ foo = 2; }

View file

@ -0,0 +1 @@
true && false

View file

@ -0,0 +1 @@
{ bar = 1; }

View file

@ -0,0 +1 @@
true || false