Merge #325

325: feature: Add pkgs fetch manager nvfetcher r=blaggacao a=GTrunSec

Regarding motivation in https://github.com/divnix/devos/issues/299

Any question here?

Example: 
-  overlay 
```
inherit (prev.sources.<PackageName>) pname version src;
```

- pkgs 
```
stdenv.mkDerivation rec {
 inherit (sources.<PackageName>) pname version src;
```

Resolves #299 

Co-authored-by: GTrunSec <gtrunsec@hardenedlinux.org>
Co-authored-by: David Arnold <dgx.arnold@gmail.com>

.gitignore

@@ -8,3 +8,5 @@ doc/index.html
 vm
 iso
 doi
+
+pkgs/_sources/.shake*

README.md

@@ -69,6 +69,7 @@ following giants][giants]:
 ### :family: — like family
 - [`numtide/devshell`][devshell]
 - [`serokell/deploy-rs`][deploy]
+- [`berberman/nvfetcher`][nvfetcher]
 - [`NixOS/nixpkgs`][nixpkgs]
 
 :heart:
@@ -94,22 +95,23 @@ goals are sufficiently upstreamed into "the Nix", dissolved.
 # License
 DevOS is licensed under the [MIT License][mit].
 
-[mk-flake]: https://github.com/divnix/digga/tree/master/src/mkFlake
-[nixpkgs]: https://github.com/NixOS/nixpkgs
-[deploy]: https://github.com/serokell/deploy-rs
-[toc]: https://github.com/divnix/devos/blob/core/flake.nix
-[giants]: https://en.wikipedia.org/wiki/Standing_on_the_shoulders_of_giants
-[digga]: https://github.com/divnix/digga
-[fup]: https://github.com/gytis-ivaskevicius/flake-utils-plus
-[fu]: https://github.com/numtide/flake-utils
-[devshell]: https://github.com/numtide/devshell
-[nix]: https://nixos.org/manual/nix/stable
-[mit]: https://mit-license.org
-[nixos]: https://nixos.org/manual/nixos/stable
-[home-manager]: https://nix-community.github.io/home-manager
-[flakes]: https://nixos.wiki/wiki/Flakes
-[flake-doc]: https://github.com/NixOS/nix/blob/master/src/nix/flake.md
-[core]: https://github.com/divnix/devos
 [community]: https://github.com/divnix/devos/tree/community
+[core]: https://github.com/divnix/devos
+[deploy]: https://github.com/serokell/deploy-rs
+[devshell]: https://github.com/numtide/devshell
+[digga]: https://github.com/divnix/digga
 [dotfiles]: https://github.com/hlissner/dotfiles
+[flake-doc]: https://github.com/NixOS/nix/blob/master/src/nix/flake.md
+[flakes]: https://nixos.wiki/wiki/Flakes
+[fu]: https://github.com/numtide/flake-utils
+[fup]: https://github.com/gytis-ivaskevicius/flake-utils-plus
+[giants]: https://en.wikipedia.org/wiki/Standing_on_the_shoulders_of_giants
+[home-manager]: https://nix-community.github.io/home-manager
+[mit]: https://mit-license.org
+[mk-flake]: https://github.com/divnix/digga/tree/master/src/mkFlake
+[nix]: https://nixos.org/manual/nix/stable
+[nixos]: https://nixos.org/manual/nixos/stable
+[nixpkgs]: https://github.com/NixOS/nixpkgs
+[nvfetcher]: https://github.com/berberman/nvfetcher
 [please]: https://github.com/nrdxp/devos/tree/nrd
+[toc]: https://github.com/divnix/devos/blob/core/flake.nix

doc/SUMMARY.md

@@ -31,5 +31,6 @@
 - [Integrations](./integrations/index.md)
   - [Cachix](./integrations/cachix.md)
   - [Deploy RS](./integrations/deploy.md)
+  - [NvFetcher](./integrations/nvfetcher.md)
   - [Hercules CI](./integrations/hercules.md)
 - [Contributing](./CONTRIBUTING.md)

doc/integrations/nvfetcher.md

@@ -0,0 +1,43 @@
+# nvfetcher
+[NvFetcher][nvf] is a workflow companion for updating nix sources.
+
+You can specify an origin source and an update configuration, and
+nvfetcher can for example track updates to a specific branch and
+automatically update your nix sources configuration on each run
+to the tip of that branch.
+
+All package source declaration is done in [sources.toml][sources.toml].
+
+From within the devshell of this repo, run `nvfetcher`, a wrapped
+version of `nvfetcher` that knows where to find and place its files
+and commit the results.
+
+## Usage
+
+Statically fetching (not tracking) a particular tag from a github repo:
+```toml
+[manix]
+src.manual = "v0.6.3"
+fetch.github = "mlvzk/manix"
+```
+
+Tracking the latest github _release_ from a github repo:
+```toml
+[manix]
+src.github = "mlvzk/manix" # responsible for tracking
+fetch.github = "mlvzk/manix" # responsible for fetching
+```
+
+Tracking the latest commit of a git repository and fetch from a git repo:
+```toml
+[manix]
+src.git = "https://github.com/mlvzk/manix.git" # responsible for tracking
+fetch.git = "https://github.com/mlvzk/manix.git" # responsible for fetching
+```
+
+> ##### _Note:_
+> Please refer to the [NvFetcher Readme][nvf-readme] for more options.
+
+[nvf: https://github.com/berberman/nvfetcher
+[nvf-readme]: https://github.com/berberman/nvfetcher#readme
+[sources.toml]: https://github.com/divnix/devos/tree/core/pkgs/sources.toml

flake.lock

@@ -166,6 +166,22 @@
         "type": "github"
       }
     },
+    "flake-compat_3": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1606424373,
+        "narHash": "sha256-oq8d4//CJOrVj+EcOaSXvMebvuTkmBJuT5tzlfewUnQ=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "99f1c2157fba4bfe6211a321fd0ee43199025dbf",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
     "flake-utils": {
       "locked": {
         "lastModified": 1623660459,
@@ -181,6 +197,21 @@
         "type": "github"
       }
     },
+    "flake-utils_2": {
+      "locked": {
+        "lastModified": 1620759905,
+        "narHash": "sha256-WiyWawrgmyN0EdmiHyG2V+fqReiVi8bM9cRdMaKQOFg=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "b543720b25df6ffdfcf9227afafc5b8c1fabfae8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
     "home": {
       "inputs": {
         "nixpkgs": [
@@ -332,20 +363,26 @@
         "type": "indirect"
       }
     },
-    "pkgs": {
+    "nvfetcher": {
       "inputs": {
+        "flake-compat": "flake-compat_3",
+        "flake-utils": "flake-utils_2",
         "nixpkgs": [
-          "nixos"
+          "latest"
         ]
       },
       "locked": {
-        "narHash": "sha256-Zs7dc0dNNa0Z3//+Gckxj7SKrMqVovY0xZZ1z8xWnEg=",
+        "lastModified": 1624534673,
-        "path": "./pkgs",
+        "narHash": "sha256-7HWt8Xh4aIFfGKAFQus5euhYxcWLe6kXz1DsGuV0WbU=",
-        "type": "path"
+        "owner": "berberman",
+        "repo": "nvfetcher",
+        "rev": "a8514f53c7999d23b48d2f42de63660bc3d7850f",
+        "type": "github"
       },
       "original": {
-        "path": "./pkgs",
+        "owner": "berberman",
-        "type": "path"
+        "repo": "nvfetcher",
+        "type": "github"
       }
     },
     "pre-commit-hooks-nix": {
@@ -375,7 +412,7 @@
         "nixos": "nixos",
         "nixos-hardware": "nixos-hardware",
         "nur": "nur",
-        "pkgs": "pkgs"
+        "nvfetcher": "nvfetcher"
       }
     },
     "utils": {

flake.nix

@@ -24,13 +24,12 @@
       agenix.inputs.nixpkgs.follows = "latest";
       nixos-hardware.url = "github:nixos/nixos-hardware";
 
-      pkgs.url = "path:./pkgs";
+      nvfetcher.url = "github:berberman/nvfetcher";
-      pkgs.inputs.nixpkgs.follows = "nixos";
+      nvfetcher.inputs.nixpkgs.follows = "latest";
     };
 
   outputs =
     { self
-    , pkgs
     , digga
     , nixos
     , ci-agent
@@ -38,6 +37,7 @@
     , nixos-hardware
     , nur
     , agenix
+    , nvfetcher
     , ...
     } @ inputs:
     digga.lib.mkFlake {
@@ -49,10 +49,11 @@
         nixos = {
           imports = [ (digga.lib.importers.overlays ./overlays) ];
           overlays = [
-            ./pkgs/default.nix
-            pkgs.overlay # for `srcs`
             nur.overlay
             agenix.overlay
+            nvfetcher.overlay
+            (final: prev: { nvfetcher-bin = nvfetcher.defaultPackage.${final.system}; })
+            ./pkgs/default.nix
           ];
         };
         latest = { };
@@ -109,7 +110,14 @@
       };
 
       devshell.externalModules = { pkgs, ... }: {
-        packages = [ pkgs.agenix ];
+        commands = [
+          { package = pkgs.agenix; category = "secrets"; }
+          {
+            name = pkgs.nvfetcher-bin.pname;
+            help = pkgs.nvfetcher-bin.meta.description;
+            command = "cd $DEVSHELL_ROOT/pkgs; ${pkgs.nvfetcher-bin}/bin/nvfetcher -c ./sources.toml --no-output $@; nixpkgs-fmt _sources/";
+          }
+        ];
       };
 
       homeConfigurations = digga.lib.mkHomeConfigurations self.nixosConfigurations;

overlays/manix.nix

@@ -0,0 +1,5 @@
+final: prev: {
+  manix = prev.manix.overrideAttrs (o: rec{
+    inherit (prev.sources.manix) pname version src;
+  });
+}

overlays/overrides.nix

@@ -7,7 +7,6 @@ channels: final: prev: {
     dhall
     discord
     element-desktop
-    manix
     rage
     nixpkgs-fmt
     qutebrowser

pkgs/_sources/generated.nix

@@ -0,0 +1,17 @@
+# This file was generated by nvfetcher, please do not modify it manually.
+{ fetchgit, fetchurl }:
+{
+  manix = {
+    pname = "manix";
+    version = "d08e7ca185445b929f097f8bfb1243a8ef3e10e4";
+    src = fetchgit {
+      url = "https://github.com/mlvzk/manix";
+      rev = "d08e7ca185445b929f097f8bfb1243a8ef3e10e4";
+      fetchSubmodules = false;
+      deepClone = false;
+      leaveDotGit = false;
+      sha256 = "1b7xi8c2drbwzfz70czddc4j33s7g1alirv12dwl91hbqxifx8qs";
+    };
+
+  };
+}

pkgs/default.nix

@@ -1 +1,5 @@
-final: prev: { }
+final: prev: {
+  # keep sources this first
+  sources = prev.callPackage (import ./_sources/generated.nix) { };
+  # then, call packages with `final.callPackage`
+}

pkgs/flake.lock

@@ -1,25 +0,0 @@
-{
-  "nodes": {
-    "nixpkgs": {
-      "locked": {
-        "lastModified": 1615926763,
-        "narHash": "sha256-yeq8A3EPNuQVlsxlEQrIRsklfJwJK0Us6jtcG/u8wNs=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "b702a56d417647de4090ac56c0f18bdc7e646610",
-        "type": "github"
-      },
-      "original": {
-        "id": "nixpkgs",
-        "type": "indirect"
-      }
-    },
-    "root": {
-      "inputs": {
-        "nixpkgs": "nixpkgs"
-      }
-    }
-  },
-  "root": "root",
-  "version": 7
-}

pkgs/flake.nix

@@ -1,52 +0,0 @@
-{
-  description = "Package Sources";
-
-  inputs = { };
-
-  outputs = { self, nixpkgs, ... }: {
-    overlay = final: prev: {
-      inherit (self) srcs;
-    };
-
-    srcs =
-      let
-        inherit (nixpkgs) lib;
-
-        mkVersion = name: input:
-          let
-            inputs = (builtins.fromJSON
-              (builtins.readFile ./flake.lock)).nodes;
-
-            ref =
-              if lib.hasAttrByPath [ name "original" "ref" ] inputs
-              then inputs.${name}.original.ref
-              else "";
-
-            version =
-              let version' = builtins.match
-                "[[:alpha:]]*[-._]?([0-9]+(\.[0-9]+)*)+"
-                ref;
-              in
-              if lib.isList version'
-              then lib.head version'
-              else if input ? lastModifiedDate && input ? shortRev
-              then "${lib.substring 0 8 input.lastModifiedDate}_${input.shortRev}"
-              else null;
-          in
-          version;
-      in
-      lib.mapAttrs
-        (pname: input:
-          let
-            version = mkVersion pname input;
-          in
-          input // { inherit pname; }
-          // lib.optionalAttrs (! isNull version)
-            {
-              inherit version;
-            }
-        )
-        (lib.filterAttrs (n: _: n != "nixpkgs")
-          self.inputs);
-  };
-}

pkgs/sources.toml

@@ -0,0 +1,4 @@
+# nvfetcher.toml
+[manix]
+src.git = "https://github.com/mlvzk/manix"
+fetch.github = "mlvzk/manix"