From 4c52e34ca60c58fe50e73875550068099dcdebdb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 30 Jan 2018 22:10:34 -0500 Subject: [PATCH 1/3] stdenv: Clean up check meta args --- pkgs/stdenv/generic/check-meta.nix | 22 +++++++++++----------- pkgs/stdenv/generic/make-derivation.nix | 5 ++--- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/pkgs/stdenv/generic/check-meta.nix b/pkgs/stdenv/generic/check-meta.nix index 93d0f4cc980..ee1e657bdab 100644 --- a/pkgs/stdenv/generic/check-meta.nix +++ b/pkgs/stdenv/generic/check-meta.nix @@ -1,11 +1,9 @@ # Checks derivation meta and attrs for problems (like brokenness, # licenses, etc). -{ lib, config, system, meta, derivationArg, mkDerivationArg }: +{ lib, config, system, meta }: let - attrs = mkDerivationArg; # TODO: probably get rid of passing this one - # See discussion at https://github.com/NixOS/nixpkgs/pull/25304#issuecomment-298385426 # for why this defaults to false, but I (@copumpkin) want to default it to true soon. shouldCheckMeta = config.checkMeta or false; @@ -123,7 +121,7 @@ let ''; - handleEvalIssue = { reason , errormsg ? "" }: + handleEvalIssue = attrs: { reason , errormsg ? "" }: let msg = '' Package ‘${attrs.name or "«name-missing»"}’ in ${pos_str} ${errormsg}, refusing to evaluate. @@ -196,11 +194,13 @@ let { valid = false; reason = "unknown-meta"; errormsg = "has an invalid meta attrset:${lib.concatMapStrings (x: "\n\t - " + x) res}"; } else { valid = true; }; - validity = checkValidity attrs; + assertValidity = attrs: let + validity = checkValidity attrs; + in validity // { + # Throw an error if trying to evaluate an non-valid derivation + handled = if !validity.valid + then handleEvalIssue attrs (removeAttrs validity ["valid"]) + else true; + }; -in validity // { - # Throw an error if trying to evaluate an non-valid derivation - handled = if !validity.valid - then handleEvalIssue (removeAttrs validity ["valid"]) - else true; -} +in assertValidity diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index e8f78d7401f..18c4beb965b 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -204,12 +204,11 @@ rec { }); validity = import ./check-meta.nix { - inherit lib config meta derivationArg; - mkDerivationArg = attrs; + inherit lib config meta; # Nix itself uses the `system` field of a derivation to decide where # to build it. This is a bit confusing for cross compilation. inherit (stdenv) system; - }; + } attrs; # The meta attribute is passed in the resulting attribute set, # but it's not part of the actual derivation, i.e., it's not From c26252af3e8a102e8ac7ab67ae3ceb2c19845cac Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 31 Jan 2018 00:11:03 -0500 Subject: [PATCH 2/3] lib, stdenv: Check `meta.platforms` against host platform and be open world First, we need check against the host platform, not the build platform. That's simple enough. Second, we move away from exahustive finite case analysis (i.e. exhaustively listing all platforms the package builds on). That only work in a closed-world setting, where we know all platforms we might build one. But with cross compilation, we may be building for arbitrary platforms, So we need fancier filters. This is the closed world to open world change. The solution is instead of having a list of systems (strings in the form "foo-bar"), we have a list of of systems or "patterns", i.e. attributes that partially match the output of the parsers in `lib.systems.parse`. The "check meta" logic treats the systems strings as an exact whitelist just as before, but treats the patterns as a fuzzy whitelist, intersecting the actual `hostPlatform` with the pattern and then checking for equality. (This is done using `matchAttrs`). The default convenience lists for `meta.platforms` are now changed to be lists of patterns (usually a single pattern) in `lib/systems/for-meta.nix` for maximum flexibility under this new system. Fixes #30902 --- lib/default.nix | 2 +- lib/systems/default.nix | 1 + lib/systems/doubles.nix | 4 ++-- lib/systems/for-meta.nix | 27 +++++++++++++++++++++++++ pkgs/stdenv/generic/check-meta.nix | 13 ++++++++---- pkgs/stdenv/generic/make-derivation.nix | 2 +- 6 files changed, 41 insertions(+), 8 deletions(-) create mode 100644 lib/systems/for-meta.nix diff --git a/lib/default.nix b/lib/default.nix index 4c36f3b0d79..b5ccc3b5144 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -47,7 +47,7 @@ let filesystem = callLibs ./filesystem.nix; # back-compat aliases - platforms = systems.doubles; + platforms = systems.forMeta; inherit (builtins) add addErrorContext attrNames concatLists deepSeq elem elemAt filter genericClosure genList diff --git a/lib/systems/default.nix b/lib/systems/default.nix index d5a206e620c..bd408e00bb1 100644 --- a/lib/systems/default.nix +++ b/lib/systems/default.nix @@ -3,6 +3,7 @@ rec { doubles = import ./doubles.nix { inherit lib; }; + forMeta = import ./for-meta.nix { inherit lib; }; parse = import ./parse.nix { inherit lib; }; inspect = import ./inspect.nix { inherit lib; }; platforms = import ./platforms.nix { inherit lib; }; diff --git a/lib/systems/doubles.nix b/lib/systems/doubles.nix index 0e1ee62ac95..012a1786a3c 100644 --- a/lib/systems/doubles.nix +++ b/lib/systems/doubles.nix @@ -30,14 +30,14 @@ in rec { aarch64 = filterDoubles predicates.isAarch64; x86 = filterDoubles predicates.isx86; i686 = filterDoubles predicates.isi686; - mips = filterDoubles predicates.isMips; x86_64 = filterDoubles predicates.isx86_64; + mips = filterDoubles predicates.isMips; cygwin = filterDoubles predicates.isCygwin; darwin = filterDoubles predicates.isDarwin; freebsd = filterDoubles predicates.isFreeBSD; # Should be better, but MinGW is unclear, and HURD is bit-rotted. - gnu = filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnu; }); + gnu = filterDoubles (matchAttrs { kernel = parse.kernels.linux; abi = parse.abis.gnu; }); illumos = filterDoubles predicates.isSunOS; linux = filterDoubles predicates.isLinux; netbsd = filterDoubles predicates.isNetBSD; diff --git a/lib/systems/for-meta.nix b/lib/systems/for-meta.nix new file mode 100644 index 00000000000..0a7a4bf5b0c --- /dev/null +++ b/lib/systems/for-meta.nix @@ -0,0 +1,27 @@ +{ lib }: +let + inherit (lib.systems) parse; + inherit (lib.systems.inspect) patterns; + +in rec { + inherit (lib.systems.doubles) all mesaPlatforms; + none = []; + + arm = [ patterns.Arm ]; + aarch64 = [ patterns.Aarch64 ]; + x86 = [ patterns.x86 ]; + i686 = [ patterns.i686 ]; + x86_64 = [ patterns.x86_64 ]; + mips = [ patterns.Mips ]; + + cygwin = [ patterns.Cygwin ]; + darwin = [ patterns.Darwin ]; + freebsd = [ patterns.FreeBSD ]; + # Should be better, but MinGW is unclear, and HURD is bit-rotted. + gnu = [ { kernel = parse.kernels.linux; abi = parse.abis.gnu; } ]; + illumos = [ patterns.SunOS ]; + linux = [ patterns.Linux ]; + netbsd = [ patterns.NetBSD ]; + openbsd = [ patterns.OpenBSD ]; + unix = patterns.Unix; # Actually a list +} diff --git a/pkgs/stdenv/generic/check-meta.nix b/pkgs/stdenv/generic/check-meta.nix index ee1e657bdab..444bad84d8d 100644 --- a/pkgs/stdenv/generic/check-meta.nix +++ b/pkgs/stdenv/generic/check-meta.nix @@ -1,7 +1,7 @@ # Checks derivation meta and attrs for problems (like brokenness, # licenses, etc). -{ lib, config, system, meta }: +{ lib, config, hostPlatform, meta }: let # See discussion at https://github.com/NixOS/nixpkgs/pull/25304#issuecomment-298385426 @@ -144,7 +144,7 @@ let license = either (listOf lib.types.attrs) (either lib.types.attrs str); maintainers = listOf str; priority = int; - platforms = listOf str; + platforms = listOf (either str lib.systems.parsed.types.system); hydraPlatforms = listOf str; broken = bool; @@ -173,6 +173,11 @@ let else "key '${k}' is unrecognized; expected one of: \n\t [${lib.concatMapStringsSep ", " (x: "'${x}'") (lib.attrNames metaTypes)}]"; checkMeta = meta: if shouldCheckMeta then lib.remove null (lib.mapAttrsToList checkMetaAttr meta) else []; + checkPlatform = attrs: let + raw = attrs.meta.platforms; + uniform = map (x: if builtins.isString x then { system = x; } else { parsed = x; }) raw; + in lib.any (pat: lib.matchAttrs pat hostPlatform) uniform; + # Check if a derivation is valid, that is whether it passes checks for # e.g brokenness or license. # @@ -186,8 +191,8 @@ let { valid = false; reason = "blacklisted"; errormsg = "has a blacklisted license (‘${showLicense attrs.meta.license}’)"; } else if !allowBroken && attrs.meta.broken or false then { valid = false; reason = "broken"; errormsg = "is marked as broken"; } - else if !allowUnsupportedSystem && !allowBroken && attrs.meta.platforms or null != null && !lib.lists.elem system attrs.meta.platforms then - { valid = false; reason = "broken"; errormsg = "is not supported on ‘${system}’"; } + else if !allowUnsupportedSystem && !allowBroken && attrs.meta.platforms or null != null && !(checkPlatform attrs) then + { valid = false; reason = "broken"; errormsg = "is not supported on ‘${hostPlatform.config}’"; } else if !(hasAllowedInsecure attrs) then { valid = false; reason = "insecure"; errormsg = "is marked as insecure"; } else let res = checkMeta (attrs.meta or {}); in if res != [] then diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index 18c4beb965b..46df958b839 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -207,7 +207,7 @@ rec { inherit lib config meta; # Nix itself uses the `system` field of a derivation to decide where # to build it. This is a bit confusing for cross compilation. - inherit (stdenv) system; + inherit (stdenv) hostPlatform; } attrs; # The meta attribute is passed in the resulting attribute set, From eae19f3c28503a8623b0fee10bfb0b3322122637 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 31 Jan 2018 16:36:52 -0500 Subject: [PATCH 3/3] release-lib: Adapt to work with new meta.platforms `packagePlatforms` now filters `supportedSystems` with the new-style `meta.platforms`, rather than just plopping it in as is. --- pkgs/top-level/release-lib.nix | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pkgs/top-level/release-lib.nix b/pkgs/top-level/release-lib.nix index 3e842f8b556..e377ca30f3a 100644 --- a/pkgs/top-level/release-lib.nix +++ b/pkgs/top-level/release-lib.nix @@ -98,7 +98,18 @@ rec { packagePlatforms = mapAttrs (name: value: let res = builtins.tryEval ( if isDerivation value then - value.meta.hydraPlatforms or (value.meta.platforms or [ "x86_64-linux" ]) + # TODO(@Ericson2314) deduplicate with `checkPlatform` in + # `pkgs/stdenv/generic/check-meta.nix`. + value.meta.hydraPlatforms or (let + raw = value.meta.platforms or [ "x86_64-linux" ]; + toPattern = x: if builtins.isString x + then { system = x; } + else { parsed = x; }; + uniform = map toPattern raw; + pred = hostPlatform: + lib.any (pat: lib.matchAttrs pat hostPlatform) uniform; + pred' = system: pred (lib.systems.elaborate { inherit system; }); + in lib.filter pred' supportedSystems) else if value.recurseForDerivations or false || value.recurseForRelease or false then packagePlatforms value else