b12f/nixpkgs
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity
nixpkgs/lib/fileset/internal.nix

537 lines
20 KiB
Nix
Raw Blame History

{ lib ? import ../. }:
let
inherit (builtins)
isAttrs
isPath
isString
pathExists
readDir
seq
split
trace
typeOf
;
inherit (lib.attrsets)
attrNames
attrValues
mapAttrs
setAttrByPath
zipAttrsWith
;
inherit (lib.filesystem)
pathType
;
inherit (lib.lists)
all
commonPrefix
drop
elemAt
filter
findFirst
findFirstIndex
foldl'
head
length
sublist
tail
;
inherit (lib.path)
append
splitRoot
;
inherit (lib.path.subpath)
components
join
;
inherit (lib.strings)
isStringLike
concatStringsSep
substring
stringLength
;
in
# Rare case of justified usage of rec:
# - This file is internal, so the return value doesn't matter, no need to make things overridable
# - The functions depend on each other
# - We want to expose all of these functions for easy testing
rec {
# If you change the internal representation, make sure to:
# - Increment this version
# - Add an additional migration function below
# - Update the description of the internal representation in ./README.md
_currentVersion = 3;
# Migrations between versions. The 0th element converts from v0 to v1, and so on
migrations = [
# Convert v0 into v1: Add the _internalBase{Root,Components} attributes
(
filesetV0:
let
parts = splitRoot filesetV0._internalBase;
in
filesetV0 // {
_internalVersion = 1;
_internalBaseRoot = parts.root;
_internalBaseComponents = components parts.subpath;
}
)
# Convert v1 into v2: filesetTree's can now also omit attributes to signal paths not being included
(
filesetV1:
# This change is backwards compatible (but not forwards compatible, so we still need a new version)
filesetV1 // {
_internalVersion = 2;
}
)
# Convert v2 into v3: filesetTree's now have a representation for an empty file set without a base path
(
filesetV2:
filesetV2 // {
# All v1 file sets are not the new empty file set
_internalIsEmptyWithoutBase = false;
_internalVersion = 3;
}
)
];
_noEvalMessage = ''
lib.fileset: Directly evaluating a file set is not supported.
To turn it into a usable source, use `lib.fileset.toSource`.
To pretty-print the contents, use `lib.fileset.trace` or `lib.fileset.traceVal`.'';
# The empty file set without a base path
_emptyWithoutBase = {
_type = "fileset";
_internalVersion = _currentVersion;
# The one and only!
_internalIsEmptyWithoutBase = true;
# Due to alphabetical ordering, this is evaluated last,
# which makes the nix repl output nicer than if it would be ordered first.
# It also allows evaluating it strictly up to this error, which could be useful
_noEval = throw _noEvalMessage;
};
# Create a fileset, see ./README.md#fileset
# Type: path -> filesetTree -> fileset
_create = base: tree:
let
# Decompose the base into its components
# See ../path/README.md for why we're not just using `toString`
parts = splitRoot base;
in
{
_type = "fileset";
_internalVersion = _currentVersion;
_internalIsEmptyWithoutBase = false;
_internalBase = base;
_internalBaseRoot = parts.root;
_internalBaseComponents = components parts.subpath;
_internalTree = tree;
# Due to alphabetical ordering, this is evaluated last,
# which makes the nix repl output nicer than if it would be ordered first.
# It also allows evaluating it strictly up to this error, which could be useful
_noEval = throw _noEvalMessage;
};
# Coerce a value to a fileset, erroring when the value cannot be coerced.
# The string gives the context for error messages.
# Type: String -> (fileset | Path) -> fileset
_coerce = context: value:
if value._type or "" == "fileset" then
if value._internalVersion > _currentVersion then
throw ''
${context} is a file set created from a future version of the file set library with a different internal representation:
- Internal version of the file set: ${toString value._internalVersion}
- Internal version of the library: ${toString _currentVersion}
Make sure to update your Nixpkgs to have a newer version of `lib.fileset`.''
else if value._internalVersion < _currentVersion then
let
# Get all the migration functions necessary to convert from the old to the current version
migrationsToApply = sublist value._internalVersion (_currentVersion - value._internalVersion) migrations;
in
foldl' (value: migration: migration value) value migrationsToApply
else
value
else if ! isPath value then
if isStringLike value then
throw ''
${context} ("${toString value}") is a string-like value, but it should be a path instead.
Paths represented as strings are not supported by `lib.fileset`, use `lib.sources` or derivations instead.''
else
throw ''
${context} is of type ${typeOf value}, but it should be a path instead.''
else if ! pathExists value then
throw ''
${context} (${toString value}) does not exist.''
else
_singleton value;
# Coerce many values to filesets, erroring when any value cannot be coerced,
# or if the filesystem root of the values doesn't match.
# Type: String -> [ { context :: String, value :: fileset | Path } ] -> [ fileset ]
_coerceMany = functionContext: list:
let
filesets = map ({ context, value }:
_coerce "${functionContext}: ${context}" value
) list;
# Find the first value with a base, there may be none!
firstWithBase = findFirst (fileset: ! fileset._internalIsEmptyWithoutBase) null filesets;
# This value is only accessed if first != null
firstBaseRoot = firstWithBase._internalBaseRoot;
# Finds the first element with a filesystem root different than the first element, if any
differentIndex = findFirstIndex (fileset:
# The empty value without a base doesn't have a base path
! fileset._internalIsEmptyWithoutBase
&& firstBaseRoot != fileset._internalBaseRoot
) null filesets;
in
# Only evaluates `differentIndex` if there are any elements with a base
if firstWithBase != null && differentIndex != null then
throw ''
${functionContext}: Filesystem roots are not the same:
${(head list).context}: root "${toString firstBaseRoot}"
${(elemAt list differentIndex).context}: root "${toString (elemAt filesets differentIndex)._internalBaseRoot}"
Different roots are not supported.''
else
filesets;
# Create a file set from a path.
# Type: Path -> fileset
_singleton = path:
let
type = pathType path;
in
if type == "directory" then
_create path type
else
# This turns a file path ./default.nix into a fileset with
# - _internalBase: ./.
# - _internalTree: {
# "default.nix" = <type>;
# }
# See ./README.md#single-files
_create (dirOf path)
{
${baseNameOf path} = type;
};
# Expand a directory representation to an equivalent one in attribute set form.
# All directory entries are included in the result.
# Type: Path -> filesetTree -> { <name> = filesetTree; }
_directoryEntries = path: value:
if value == "directory" then
readDir path
else
# Set all entries not present to null
mapAttrs (name: value: null) (readDir path)
// value;
/*
A normalisation of a filesetTree suitable filtering with `builtins.path`:
- Replace all directories that have no files with `null`.
This removes directories that would be empty
- Replace all directories with all files with `"directory"`.
This speeds up the source filter function
Note that this function is strict, it evaluates the entire tree
Type: Path -> filesetTree -> filesetTree
*/
_normaliseTreeFilter = path: tree:
if tree == "directory" || isAttrs tree then
let
entries = _directoryEntries path tree;
normalisedSubtrees = mapAttrs (name: _normaliseTreeFilter (path + "/${name}")) entries;
subtreeValues = attrValues normalisedSubtrees;
in
# This triggers either when all files in a directory are filtered out
# Or when the directory doesn't contain any files at all
if all isNull subtreeValues then
null
# Triggers when we have the same as a `readDir path`, so we can turn it back into an equivalent "directory".
else if all isString subtreeValues then
"directory"
else
normalisedSubtrees
else
tree;
/*
A minimal normalisation of a filesetTree, intended for pretty-printing:
- If all children of a path are recursively included or empty directories, the path itself is also recursively included
- If all children of a path are fully excluded or empty directories, the path itself is an empty directory
- Other empty directories are represented with the special "emptyDir" string
While these could be replaced with `null`, that would take another mapAttrs
Note that this function is partially lazy.
Type: Path -> filesetTree -> filesetTree (with "emptyDir"'s)
*/
_normaliseTreeMinimal = path: tree:
if tree == "directory" || isAttrs tree then
let
entries = _directoryEntries path tree;
normalisedSubtrees = mapAttrs (name: _normaliseTreeMinimal (path + "/${name}")) entries;
subtreeValues = attrValues normalisedSubtrees;
in
# If there are no entries, or all entries are empty directories, return "emptyDir".
# After this branch we know that there's at least one file
if all (value: value == "emptyDir") subtreeValues then
"emptyDir"
# If all subtrees are fully included or empty directories
# (both of which are coincidentally represented as strings), return "directory".
# This takes advantage of the fact that empty directories can be represented as included directories.
# Note that the tree == "directory" check allows avoiding recursion
else if tree == "directory" || all (value: isString value) subtreeValues then
"directory"
# If all subtrees are fully excluded or empty directories, return null.
# This takes advantage of the fact that empty directories can be represented as excluded directories
else if all (value: isNull value || value == "emptyDir") subtreeValues then
null
# Mix of included and excluded entries
else
normalisedSubtrees
else
tree;
# Trace a filesetTree in a pretty way when the resulting value is evaluated.
# This can handle both normal filesetTree's, and ones returned from _normaliseTreeMinimal
# Type: Path -> filesetTree (with "emptyDir"'s) -> Null
_printMinimalTree = base: tree:
let
treeSuffix = tree:
if isAttrs tree then
""
else if tree == "directory" then
" (all files in directory)"
else
# This does "leak" the file type strings of the internal representation,
# but this is the main reason these file type strings even are in the representation!
# TODO: Consider removing that information from the internal representation for performance.
# The file types can still be printed by querying them only during tracing
" (${tree})";
# Only for attribute set trees
traceTreeAttrs = prevLine: indent: tree:
foldl' (prevLine: name:
let
subtree = tree.${name};
# Evaluating this prints the line for this subtree
thisLine =
trace "${indent}- ${name}${treeSuffix subtree}" prevLine;
in
if subtree == null || subtree == "emptyDir" then
# Don't print anything at all if this subtree is empty
prevLine
else if isAttrs subtree then
# A directory with explicit entries
# Do print this node, but also recurse
traceTreeAttrs thisLine "${indent} " subtree
else
# Either a file, or a recursively included directory
# Do print this node but no further recursion needed
thisLine
) prevLine (attrNames tree);
# Evaluating this will print the first line
firstLine =
if tree == null || tree == "emptyDir" then
trace "(empty)" null
else
trace "${toString base}${treeSuffix tree}" null;
in
if isAttrs tree then
traceTreeAttrs firstLine "" tree
else
firstLine;
# Pretty-print a file set in a pretty way when the resulting value is evaluated
# Type: fileset -> Null
_printFileset = fileset:
if fileset._internalIsEmptyWithoutBase then
trace "(empty)" null
else
_printMinimalTree fileset._internalBase
(_normaliseTreeMinimal fileset._internalBase fileset._internalTree);
# Turn a fileset into a source filter function suitable for `builtins.path`
# Only directories recursively containing at least one files are recursed into
# Type: Path -> fileset -> (String -> String -> Bool)
_toSourceFilter = fileset:
let
# Simplify the tree, necessary to make sure all empty directories are null
# which has the effect that they aren't included in the result
tree = _normaliseTreeFilter fileset._internalBase fileset._internalTree;
# The base path as a string with a single trailing slash
baseString =
if fileset._internalBaseComponents == [] then
# Need to handle the filesystem root specially
"/"
else
"/" + concatStringsSep "/" fileset._internalBaseComponents + "/";
baseLength = stringLength baseString;
# Check whether a list of path components under the base path exists in the tree.
# This function is called often, so it should be fast.
# Type: [ String ] -> Bool
inTree = components:
let
recurse = index: localTree:
if isAttrs localTree then
# We have an attribute set, meaning this is a directory with at least one file
if index >= length components then
# The path may have no more components though, meaning the filter is running on the directory itself,
# so we always include it, again because there's at least one file in it.
true
else
# If we do have more components, the filter runs on some entry inside this directory, so we need to recurse
# We do +2 because builtins.split is an interleaved list of the inbetweens and the matches
recurse (index + 2) localTree.${elemAt components index}
else
# If it's not an attribute set it can only be either null (in which case it's not included)
# or a string ("directory" or "regular", etc.) in which case it's included
localTree != null;
in recurse 0 tree;
# Filter suited when there's no files
empty = _: _: false;
# Filter suited when there's some files
# This can't be used for when there's no files, because the base directory is always included
nonEmpty =
path: _:
let
# Add a slash to the path string, turning "/foo" to "/foo/",
# making sure to not have any false prefix matches below.
# Note that this would produce "//" for "/",
# but builtins.path doesn't call the filter function on the `path` argument itself,
# meaning this function can never receive "/" as an argument
pathSlash = path + "/";
in
# Same as `hasPrefix pathSlash baseString`, but more efficient.
# With base /foo/bar we need to include /foo:
# hasPrefix "/foo/" "/foo/bar/"
if substring 0 (stringLength pathSlash) baseString == pathSlash then
true
# Same as `! hasPrefix baseString pathSlash`, but more efficient.
# With base /foo/bar we need to exclude /baz
# ! hasPrefix "/baz/" "/foo/bar/"
else if substring 0 baseLength pathSlash != baseString then
false
else
# Same as `removePrefix baseString path`, but more efficient.
# From the above code we know that hasPrefix baseString pathSlash holds, so this is safe.
# We don't use pathSlash here because we only needed the trailing slash for the prefix matching.
# With base /foo and path /foo/bar/baz this gives
# inTree (split "/" (removePrefix "/foo/" "/foo/bar/baz"))
# == inTree (split "/" "bar/baz")
# == inTree [ "bar" "baz" ]
inTree (split "/" (substring baseLength (-1) path));
in
# Special case because the code below assumes that the _internalBase is always included in the result
# which shouldn't be done when we have no files at all in the base
# This also forces the tree before returning the filter, leads to earlier error messages
if fileset._internalIsEmptyWithoutBase || tree == null then
empty
else
nonEmpty;
# Computes the union of a list of filesets.
# The filesets must already be coerced and validated to be in the same filesystem root
# Type: [ Fileset ] -> Fileset
_unionMany = filesets:
let
# All filesets that have a base, aka not the ones that are the empty value without a base
filesetsWithBase = filter (fileset: ! fileset._internalIsEmptyWithoutBase) filesets;
# The first fileset that has a base.
# This value is only accessed if there are at all.
firstWithBase = head filesetsWithBase;
# To be able to union filesetTree's together, they need to have the same base path.
# Base paths can be unioned by taking their common prefix,
# e.g. such that `union /foo/bar /foo/baz` has the base path `/foo`
# A list of path components common to all base paths.
# Note that commonPrefix can only be fully evaluated,
# so this cannot cause a stack overflow due to a build-up of unevaluated thunks.
commonBaseComponents = foldl'
(components: el: commonPrefix components el._internalBaseComponents)
firstWithBase._internalBaseComponents
# We could also not do the `tail` here to avoid a list allocation,
# but then we'd have to pay for a potentially expensive
# but unnecessary `commonPrefix` call
(tail filesetsWithBase);
# The common base path assembled from a filesystem root and the common components
commonBase = append firstWithBase._internalBaseRoot (join commonBaseComponents);
# A list of filesetTree's that all have the same base path
# This is achieved by nesting the trees into the components they have over the common base path
# E.g. `union /foo/bar /foo/baz` has the base path /foo
# So the tree under `/foo/bar` gets nested under `{ bar = ...; ... }`,
# while the tree under `/foo/baz` gets nested under `{ baz = ...; ... }`
# Therefore allowing combined operations over them.
trees = map (fileset:
setAttrByPath
(drop (length commonBaseComponents) fileset._internalBaseComponents)
fileset._internalTree
) filesetsWithBase;
# Folds all trees together into a single one using _unionTree
# We do not use a fold here because it would cause a thunk build-up
# which could cause a stack overflow for a large number of trees
resultTree = _unionTrees trees;
in
# If there's no values with a base, we have no files
if filesetsWithBase == [ ] then
_emptyWithoutBase
else
_create commonBase resultTree;
# The union of multiple filesetTree's with the same base path.
# Later elements are only evaluated if necessary.
# Type: [ filesetTree ] -> filesetTree
_unionTrees = trees:
let
stringIndex = findFirstIndex isString null trees;
withoutNull = filter (tree: tree != null) trees;
in
if stringIndex != null then
# If there's a string, it's always a fully included tree (dir or file),
# no need to look at other elements
elemAt trees stringIndex
else if withoutNull == [ ] then
# If all trees are null, then the resulting tree is also null
null
else
# The non-null elements have to be attribute sets representing partial trees
# We need to recurse into those
zipAttrsWith (name: _unionTrees) withoutNull;
}
Reference in a new issue View git blame Copy permalink