From c701a4dd299aaa82589e4e4d4ce49ae330729a25 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 5 Apr 2023 16:10:09 +0200 Subject: [PATCH 1/8] lib.sources.pathType and co.: Move to lib.filesystem These functions only work with the filesystem, they don't import anything as sources --- lib/default.nix | 5 +++-- lib/filesystem.nix | 26 ++++++++++++++++++++++++++ lib/sources.nix | 37 ++++++++++++++++++------------------- 3 files changed, 47 insertions(+), 21 deletions(-) diff --git a/lib/default.nix b/lib/default.nix index 0424db36b2e..8fea4b8ad63 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -117,10 +117,11 @@ let inherit (self.meta) addMetaAttrs dontDistribute setName updateName appendToName mapDerivationAttrset setPrio lowPrio lowPrioSet hiPrio hiPrioSet getLicenseFromSpdxId getExe; - inherit (self.sources) pathType pathIsDirectory cleanSourceFilter + inherit (self.filesystem) pathType pathIsDirectory pathIsRegularFile; + inherit (self.sources) cleanSourceFilter cleanSource sourceByRegex sourceFilesBySuffices commitIdFromGitRepo cleanSourceWith pathHasContext - canCleanSource pathIsRegularFile pathIsGitRepo; + canCleanSource pathIsGitRepo; inherit (self.modules) evalModules setDefaultModuleLocation unifyModuleSyntax applyModuleArgsIfFunction mergeModules mergeModules' mergeOptionDecls evalOptionValue mergeDefinitions diff --git a/lib/filesystem.nix b/lib/filesystem.nix index 94819605520..2d8a62639ec 100644 --- a/lib/filesystem.nix +++ b/lib/filesystem.nix @@ -2,12 +2,38 @@ { lib }: let + inherit (builtins) + getAttr + readDir + pathExists + ; + inherit (lib.strings) hasPrefix ; + + inherit (lib.filesystem) + pathType + ; in { + + /* + Returns the type of a path: regular (for file), symlink, or directory. + */ + pathType = path: getAttr (baseNameOf path) (readDir (dirOf path)); + + /* + Returns true if the path exists and is a directory, false otherwise. + */ + pathIsDirectory = path: if pathExists path then (pathType path) == "directory" else false; + + /* + Returns true if the path exists and is a regular file, false otherwise. + */ + pathIsRegularFile = path: if pathExists path then (pathType path) == "regular" else false; + /* A map of all haskell packages defined in the given path, identified by having a cabal file with the same name as the diff --git a/lib/sources.nix b/lib/sources.nix index 3ad7dc63355..d990777c6fc 100644 --- a/lib/sources.nix +++ b/lib/sources.nix @@ -18,21 +18,11 @@ let pathExists readFile ; - - /* - Returns the type of a path: regular (for file), symlink, or directory. - */ - pathType = path: getAttr (baseNameOf path) (readDir (dirOf path)); - - /* - Returns true if the path exists and is a directory, false otherwise. - */ - pathIsDirectory = path: if pathExists path then (pathType path) == "directory" else false; - - /* - Returns true if the path exists and is a regular file, false otherwise. - */ - pathIsRegularFile = path: if pathExists path then (pathType path) == "regular" else false; + inherit (lib.filesystem) + pathType + pathIsDirectory + pathIsRegularFile + ; /* A basic filter for `cleanSourceWith` that removes @@ -271,11 +261,20 @@ let }; in { - inherit - pathType - pathIsDirectory - pathIsRegularFile + pathType = lib.warnIf (lib.isInOldestRelease 2305) + "lib.sources.pathType has been moved to lib.filesystem.pathType." + lib.filesystem.pathType; + + pathIsDirectory = lib.warnIf (lib.isInOldestRelease 2305) + "lib.sources.pathIsDirectory has been moved to lib.filesystem.pathIsDirectory." + lib.filesystem.pathIsDirectory; + + pathIsRegularFile = lib.warnIf (lib.isInOldestRelease 2305) + "lib.sources.pathIsRegularFile has been moved to lib.filesystem.pathIsRegularFile." + lib.filesystem.pathIsRegularFile; + + inherit pathIsGitRepo commitIdFromGitRepo From a1dedc908d1d9f7e2b9490dc7b3b3020e8fe7d0b Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 5 Apr 2023 16:44:58 +0200 Subject: [PATCH 2/8] lib.filesystem.pathType and co.: Add tests Co-Authored-By: Robert Hensing --- lib/tests/filesystem.sh | 88 +++++++++++++++++++++++++++++++++++++++++ lib/tests/release.nix | 3 ++ 2 files changed, 91 insertions(+) create mode 100755 lib/tests/filesystem.sh diff --git a/lib/tests/filesystem.sh b/lib/tests/filesystem.sh new file mode 100755 index 00000000000..ab1504abb49 --- /dev/null +++ b/lib/tests/filesystem.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +# Tests lib/filesystem.nix +# Run: +# [nixpkgs]$ lib/tests/filesystem.sh +# or: +# [nixpkgs]$ nix-build lib/tests/release.nix + +set -euo pipefail +shopt -s inherit_errexit + +# Use +# || die +die() { + echo >&2 "test case failed: " "$@" + exit 1 +} + +if test -n "${TEST_LIB:-}"; then + NIX_PATH=nixpkgs="$(dirname "$TEST_LIB")" +else + NIX_PATH=nixpkgs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.."; pwd)" +fi +export NIX_PATH + +work="$(mktemp -d)" +clean_up() { + rm -rf "$work" +} +trap clean_up EXIT +cd "$work" + +mkdir directory +touch regular +ln -s target symlink +mkfifo fifo + +checkPathType() { + local path=$1 + local expectedPathType=$2 + local actualPathType=$(nix-instantiate --eval --strict --json 2>&1 \ + -E '{ path }: let lib = import ; in lib.filesystem.pathType path' \ + --argstr path "$path") + if [[ "$actualPathType" != "$expectedPathType" ]]; then + die "lib.filesystem.pathType \"$path\" == $actualPathType, but $expectedPathType was expected" + fi +} + +checkPathType "$PWD/directory" '"directory"' +checkPathType "$PWD/regular" '"regular"' +checkPathType "$PWD/symlink" '"symlink"' +checkPathType "$PWD/fifo" '"unknown"' + +checkPathIsDirectory() { + local path=$1 + local expectedIsDirectory=$2 + local actualIsDirectory=$(nix-instantiate --eval --strict --json 2>&1 \ + -E '{ path }: let lib = import ; in lib.filesystem.pathIsDirectory path' \ + --argstr path "$path") + if [[ "$actualIsDirectory" != "$expectedIsDirectory" ]]; then + die "lib.filesystem.pathIsDirectory \"$path\" == $actualIsDirectory, but $expectedIsDirectory was expected" + fi +} + +checkPathIsDirectory "$PWD/directory" "true" +checkPathIsDirectory "$PWD/regular" "false" +checkPathIsDirectory "$PWD/symlink" "false" +checkPathIsDirectory "$PWD/fifo" "false" +checkPathIsDirectory "$PWD/non-existent" "false" + +checkPathIsRegularFile() { + local path=$1 + local expectedIsRegularFile=$2 + local actualIsRegularFile=$(nix-instantiate --eval --strict --json 2>&1 \ + -E '{ path }: let lib = import ; in lib.filesystem.pathIsRegularFile path' \ + --argstr path "$path") + if [[ "$actualIsRegularFile" != "$expectedIsRegularFile" ]]; then + die "lib.filesystem.pathIsRegularFile \"$path\" == $actualIsRegularFile, but $expectedIsRegularFile was expected" + fi +} + +checkPathIsRegularFile "$PWD/directory" "false" +checkPathIsRegularFile "$PWD/regular" "true" +checkPathIsRegularFile "$PWD/symlink" "false" +checkPathIsRegularFile "$PWD/fifo" "false" +checkPathIsRegularFile "$PWD/non-existent" "false" + +echo >&2 tests ok diff --git a/lib/tests/release.nix b/lib/tests/release.nix index dbf6683d49a..f5c6e81030c 100644 --- a/lib/tests/release.nix +++ b/lib/tests/release.nix @@ -44,6 +44,9 @@ pkgs.runCommand "nixpkgs-lib-tests" { echo "Running lib/tests/modules.sh" bash lib/tests/modules.sh + echo "Running lib/tests/filesystem.sh" + TEST_LIB=$PWD/lib bash lib/tests/filesystem.sh + echo "Running lib/tests/sources.sh" TEST_LIB=$PWD/lib bash lib/tests/sources.sh From 5346636c20ff57410b25a6a8897d4e95f75e040c Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 5 Apr 2023 16:13:03 +0200 Subject: [PATCH 3/8] lib.filesystem: Minor refactor Co-Authored-By: Robert Hensing --- lib/filesystem.nix | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/filesystem.nix b/lib/filesystem.nix index 2d8a62639ec..f604eb90434 100644 --- a/lib/filesystem.nix +++ b/lib/filesystem.nix @@ -3,7 +3,6 @@ let inherit (builtins) - getAttr readDir pathExists ; @@ -22,17 +21,20 @@ in /* Returns the type of a path: regular (for file), symlink, or directory. */ - pathType = path: getAttr (baseNameOf path) (readDir (dirOf path)); + pathType = path: + (readDir (dirOf path)).${baseNameOf path}; /* Returns true if the path exists and is a directory, false otherwise. */ - pathIsDirectory = path: if pathExists path then (pathType path) == "directory" else false; + pathIsDirectory = path: + pathExists path && pathType path == "directory"; /* Returns true if the path exists and is a regular file, false otherwise. */ - pathIsRegularFile = path: if pathExists path then (pathType path) == "regular" else false; + pathIsRegularFile = path: + pathExists path && pathType path == "regular"; /* A map of all haskell packages defined in the given path, From bb6eab0bdbe060cb578fc9be5925a76a1635424f Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 5 Apr 2023 16:47:30 +0200 Subject: [PATCH 4/8] lib.filesystem.pathType: Fix for filesystem root argument Previously this function couldn't handle / being passed, it would throw an error: error: attribute '' missing at nixpkgs/lib/filesystem.nix:24:20: 23| */ 24| pathType = path: (readDir (dirOf path)).${baseNameOf path}; | ^ 25| Consequently this also fixes the lib.filesystem.{pathIsDirectory,pathIsRegularFile} functions. --- lib/filesystem.nix | 7 ++++++- lib/tests/filesystem.sh | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/filesystem.nix b/lib/filesystem.nix index f604eb90434..13f4ebb2640 100644 --- a/lib/filesystem.nix +++ b/lib/filesystem.nix @@ -22,7 +22,12 @@ in Returns the type of a path: regular (for file), symlink, or directory. */ pathType = path: - (readDir (dirOf path)).${baseNameOf path}; + # The filesystem root is the only path where `dirOf / == /` and + # `baseNameOf /` is not valid. We can detect this and directly return + # "directory", since we know the filesystem root can't be anything else. + if dirOf path == path + then "directory" + else (readDir (dirOf path)).${baseNameOf path}; /* Returns true if the path exists and is a directory, false otherwise. diff --git a/lib/tests/filesystem.sh b/lib/tests/filesystem.sh index ab1504abb49..61710da92ba 100755 --- a/lib/tests/filesystem.sh +++ b/lib/tests/filesystem.sh @@ -46,6 +46,7 @@ checkPathType() { fi } +checkPathType "/" '"directory"' checkPathType "$PWD/directory" '"directory"' checkPathType "$PWD/regular" '"regular"' checkPathType "$PWD/symlink" '"symlink"' @@ -62,6 +63,7 @@ checkPathIsDirectory() { fi } +checkPathIsDirectory "/" "true" checkPathIsDirectory "$PWD/directory" "true" checkPathIsDirectory "$PWD/regular" "false" checkPathIsDirectory "$PWD/symlink" "false" @@ -79,6 +81,7 @@ checkPathIsRegularFile() { fi } +checkPathIsRegularFile "/" "false" checkPathIsRegularFile "$PWD/directory" "false" checkPathIsRegularFile "$PWD/regular" "true" checkPathIsRegularFile "$PWD/symlink" "false" From d064d972f07b908db3efb909d6663bee3adf8d40 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 5 Apr 2023 16:58:29 +0200 Subject: [PATCH 5/8] lib.filesystem.pathType: Improve error for non-existent paths Previously it would fail with error: attribute 'nonexistent' missing at nixpkgs/lib/filesystem.nix:29:10: 28| if dirOf path == path then "directory" 29| else (readDir (dirOf path)).${baseNameOf path}; | ^ 30| --- lib/filesystem.nix | 6 +++++- lib/tests/filesystem.sh | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/filesystem.nix b/lib/filesystem.nix index 13f4ebb2640..db120a48c64 100644 --- a/lib/filesystem.nix +++ b/lib/filesystem.nix @@ -22,10 +22,14 @@ in Returns the type of a path: regular (for file), symlink, or directory. */ pathType = path: + if ! pathExists path + # Fail irrecoverably to mimic the historic behavior of this function and + # the new builtins.readFileType + then abort "lib.filesystem.pathType: Path ${toString path} does not exist." # The filesystem root is the only path where `dirOf / == /` and # `baseNameOf /` is not valid. We can detect this and directly return # "directory", since we know the filesystem root can't be anything else. - if dirOf path == path + else if dirOf path == path then "directory" else (readDir (dirOf path)).${baseNameOf path}; diff --git a/lib/tests/filesystem.sh b/lib/tests/filesystem.sh index 61710da92ba..4a5ffeb1243 100755 --- a/lib/tests/filesystem.sh +++ b/lib/tests/filesystem.sh @@ -51,6 +51,7 @@ checkPathType "$PWD/directory" '"directory"' checkPathType "$PWD/regular" '"regular"' checkPathType "$PWD/symlink" '"symlink"' checkPathType "$PWD/fifo" '"unknown"' +checkPathType "$PWD/non-existent" "error: evaluation aborted with the following error message: 'lib.filesystem.pathType: Path $PWD/non-existent does not exist.'" checkPathIsDirectory() { local path=$1 From 84a3d633d6f3653675d794ef5e8e90bf0cb7c502 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 5 Apr 2023 17:16:10 +0200 Subject: [PATCH 6/8] lib.filesystem.pathType and co.: Improve documentation --- lib/filesystem.nix | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/lib/filesystem.nix b/lib/filesystem.nix index db120a48c64..6dfd273b0a5 100644 --- a/lib/filesystem.nix +++ b/lib/filesystem.nix @@ -19,7 +19,18 @@ in { /* - Returns the type of a path: regular (for file), symlink, or directory. + The type of a path. The path needs to exist and be accessible. + The result is either "directory" for a directory, "regular" for a regular file, "symlink" for a symlink, or "unknown" for anything else. + + Type: + pathType :: Path -> String + + Example: + pathType /. + => "directory" + + pathType /some/file.nix + => "regular" */ pathType = path: if ! pathExists path @@ -34,13 +45,39 @@ in else (readDir (dirOf path)).${baseNameOf path}; /* - Returns true if the path exists and is a directory, false otherwise. + Whether a path exists and is a directory. + + Type: + pathIsDirectory :: Path -> Bool + + Example: + pathIsDirectory /. + => true + + pathIsDirectory /this/does/not/exist + => false + + pathIsDirectory /some/file.nix + => false */ pathIsDirectory = path: pathExists path && pathType path == "directory"; /* - Returns true if the path exists and is a regular file, false otherwise. + Whether a path exists and is a regular file, meaning not a symlink or any other special file type. + + Type: + pathIsRegularFile :: Path -> Bool + + Example: + pathIsRegularFile /. + => false + + pathIsRegularFile /this/does/not/exist + => false + + pathIsRegularFile /some/file.nix + => true */ pathIsRegularFile = path: pathExists path && pathType path == "regular"; From fcaa2b1097e46c286b4651f5f2d7653f52ae860b Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 5 Apr 2023 18:42:22 +0200 Subject: [PATCH 7/8] lib.filesystem.pathType: Use new builtins.readFileType if available Co-Authored-By: Robert Hensing --- lib/filesystem.nix | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/lib/filesystem.nix b/lib/filesystem.nix index 6dfd273b0a5..37b60e4fe11 100644 --- a/lib/filesystem.nix +++ b/lib/filesystem.nix @@ -32,17 +32,21 @@ in pathType /some/file.nix => "regular" */ - pathType = path: - if ! pathExists path - # Fail irrecoverably to mimic the historic behavior of this function and - # the new builtins.readFileType - then abort "lib.filesystem.pathType: Path ${toString path} does not exist." - # The filesystem root is the only path where `dirOf / == /` and - # `baseNameOf /` is not valid. We can detect this and directly return - # "directory", since we know the filesystem root can't be anything else. - else if dirOf path == path - then "directory" - else (readDir (dirOf path)).${baseNameOf path}; + pathType = + builtins.readFileType or + # Nix <2.14 compatibility shim + (path: + if ! pathExists path + # Fail irrecoverably to mimic the historic behavior of this function and + # the new builtins.readFileType + then abort "lib.filesystem.pathType: Path ${toString path} does not exist." + # The filesystem root is the only path where `dirOf / == /` and + # `baseNameOf /` is not valid. We can detect this and directly return + # "directory", since we know the filesystem root can't be anything else. + else if dirOf path == path + then "directory" + else (readDir (dirOf path)).${baseNameOf path} + ); /* Whether a path exists and is a directory. From 378bf1a6192a5d46ceb7e07af7be1d37b93b47c7 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Mon, 22 May 2023 14:17:42 +0200 Subject: [PATCH 8/8] lib/filesystem.nix: Update top comment Co-Authored-By: Robert Hensing --- lib/filesystem.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/filesystem.nix b/lib/filesystem.nix index 37b60e4fe11..4860d4d02a7 100644 --- a/lib/filesystem.nix +++ b/lib/filesystem.nix @@ -1,6 +1,8 @@ -# Functions for copying sources to the Nix store. +# Functions for querying information about the filesystem +# without copying any files to the Nix store. { lib }: +# Tested in lib/tests/filesystem.sh let inherit (builtins) readDir