nodePackages: patch node2nix for npm v7+ and switch to building package set with current nodejs

This commit is contained in:
Lily Foster 2022-09-28 09:57:51 -04:00 committed by Yt
parent 7030b3d11c
commit 07b207c5e9
9 changed files with 12296 additions and 3405 deletions

View file

@ -772,6 +772,18 @@
which no longer has a downgrade path to releases 1.2 or older. which no longer has a downgrade path to releases 1.2 or older.
</para> </para>
</listitem> </listitem>
<listitem>
<para>
The <literal>nodePackages</literal> package set now defaults
to the LTS release in the <literal>nodejs</literal> package
again, instead of being pinned to
<literal>nodejs-14_x</literal>. Several updates to node2nix
have been made for compatibility with newer Node.js and npm
versions and a new <literal>postRebuild</literal> hook has
been added for packages to perform extra build steps before
the npm install step prunes dev dependencies.
</para>
</listitem>
</itemizedlist> </itemizedlist>
</section> </section>
</section> </section>

View file

@ -254,4 +254,6 @@ Available as [services.patroni](options.html#opt-services.patroni.enable).
- The `nomad` package now defaults to 1.3, which no longer has a downgrade path to releases 1.2 or older. - The `nomad` package now defaults to 1.3, which no longer has a downgrade path to releases 1.2 or older.
- The `nodePackages` package set now defaults to the LTS release in the `nodejs` package again, instead of being pinned to `nodejs-14_x`. Several updates to node2nix have been made for compatibility with newer Node.js and npm versions and a new `postRebuild` hook has been added for packages to perform extra build steps before the npm install step prunes dev dependencies.
<!-- To avoid merge conflicts, consider adding your item at an arbitrary place in the list instead. --> <!-- To avoid merge conflicts, consider adding your item at an arbitrary place in the list instead. -->

View file

@ -48,7 +48,7 @@ let
]; ];
# Follows https://github.com/rust-lang/rust-analyzer/blob/41949748a6123fd6061eb984a47f4fe780525e63/xtask/src/dist.rs#L39-L65 # Follows https://github.com/rust-lang/rust-analyzer/blob/41949748a6123fd6061eb984a47f4fe780525e63/xtask/src/dist.rs#L39-L65
postInstall = '' postRebuild = ''
jq ' jq '
.version = $ENV.version | .version = $ENV.version |
.releaseTag = $ENV.releaseTag | .releaseTag = $ENV.releaseTag |

View file

@ -2,7 +2,7 @@
{pkgs ? import <nixpkgs> { {pkgs ? import <nixpkgs> {
inherit system; inherit system;
}, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-14_x"}: }, system ? builtins.currentSystem, nodejs ? pkgs."nodejs-18_x"}:
let let
nodeEnv = import ./node-env.nix { nodeEnv = import ./node-env.nix {

View file

@ -13,7 +13,7 @@ rm -f ./node-env.nix
-i node-packages.json \ -i node-packages.json \
-o node-packages.nix \ -o node-packages.nix \
-c composition.nix \ -c composition.nix \
--pkg-name nodejs-14_x --pkg-name nodejs-18_x
# using --no-out-link in nix-build argument would cause the # using --no-out-link in nix-build argument would cause the
# gc to run before the script finishes # gc to run before the script finishes

View file

@ -165,7 +165,11 @@ let
if(process.argv[2] == "development") { if(process.argv[2] == "development") {
replaceDependencies(packageObj.devDependencies); replaceDependencies(packageObj.devDependencies);
} }
else {
packageObj.devDependencies = {};
}
replaceDependencies(packageObj.optionalDependencies); replaceDependencies(packageObj.optionalDependencies);
replaceDependencies(packageObj.peerDependencies);
/* Write the fixed package.json file */ /* Write the fixed package.json file */
fs.writeFileSync("package.json", JSON.stringify(packageObj, null, 2)); fs.writeFileSync("package.json", JSON.stringify(packageObj, null, 2));
@ -270,7 +274,7 @@ let
# Reconstructs a package-lock file from the node_modules/ folder structure and package.json files with dummy sha1 hashes # Reconstructs a package-lock file from the node_modules/ folder structure and package.json files with dummy sha1 hashes
reconstructPackageLock = writeTextFile { reconstructPackageLock = writeTextFile {
name = "addintegrityfields.js"; name = "reconstructpackagelock.js";
text = '' text = ''
var fs = require('fs'); var fs = require('fs');
var path = require('path'); var path = require('path');
@ -280,25 +284,43 @@ let
var lockObj = { var lockObj = {
name: packageObj.name, name: packageObj.name,
version: packageObj.version, version: packageObj.version,
lockfileVersion: 1, lockfileVersion: 2,
requires: true, requires: true,
packages: {
"": {
name: packageObj.name,
version: packageObj.version,
license: packageObj.license,
bin: packageObj.bin,
dependencies: packageObj.dependencies,
engines: packageObj.engines,
optionalDependencies: packageObj.optionalDependencies
}
},
dependencies: {} dependencies: {}
}; };
function augmentPackageJSON(filePath, dependencies) { function augmentPackageJSON(filePath, packages, dependencies) {
var packageJSON = path.join(filePath, "package.json"); var packageJSON = path.join(filePath, "package.json");
if(fs.existsSync(packageJSON)) { if(fs.existsSync(packageJSON)) {
var packageObj = JSON.parse(fs.readFileSync(packageJSON)); var packageObj = JSON.parse(fs.readFileSync(packageJSON));
packages[filePath] = {
version: packageObj.version,
integrity: "sha1-000000000000000000000000000=",
dependencies: packageObj.dependencies,
engines: packageObj.engines,
optionalDependencies: packageObj.optionalDependencies
};
dependencies[packageObj.name] = { dependencies[packageObj.name] = {
version: packageObj.version, version: packageObj.version,
integrity: "sha1-000000000000000000000000000=", integrity: "sha1-000000000000000000000000000=",
dependencies: {} dependencies: {}
}; };
processDependencies(path.join(filePath, "node_modules"), dependencies[packageObj.name].dependencies); processDependencies(path.join(filePath, "node_modules"), packages, dependencies[packageObj.name].dependencies);
} }
} }
function processDependencies(dir, dependencies) { function processDependencies(dir, packages, dependencies) {
if(fs.existsSync(dir)) { if(fs.existsSync(dir)) {
var files = fs.readdirSync(dir); var files = fs.readdirSync(dir);
@ -314,23 +336,84 @@ let
pkgFiles.forEach(function(entry) { pkgFiles.forEach(function(entry) {
if(stats.isDirectory()) { if(stats.isDirectory()) {
var pkgFilePath = path.join(filePath, entry); var pkgFilePath = path.join(filePath, entry);
augmentPackageJSON(pkgFilePath, dependencies); augmentPackageJSON(pkgFilePath, packages, dependencies);
} }
}); });
} else { } else {
augmentPackageJSON(filePath, dependencies); augmentPackageJSON(filePath, packages, dependencies);
} }
} }
}); });
} }
} }
processDependencies("node_modules", lockObj.dependencies); processDependencies("node_modules", lockObj.packages, lockObj.dependencies);
fs.writeFileSync("package-lock.json", JSON.stringify(lockObj, null, 2)); fs.writeFileSync("package-lock.json", JSON.stringify(lockObj, null, 2));
''; '';
}; };
# Script that links bins defined in package.json to the node_modules bin directory
# NPM does not do this for top-level packages itself anymore as of v7
linkBinsScript = writeTextFile {
name = "linkbins.js";
text = ''
var fs = require('fs');
var path = require('path');
var packageObj = JSON.parse(fs.readFileSync("package.json"));
var nodeModules = Array(packageObj.name.split("/").length).fill("..").join(path.sep);
if(packageObj.bin !== undefined) {
fs.mkdirSync(path.join(nodeModules, ".bin"))
if(typeof packageObj.bin == "object") {
Object.keys(packageObj.bin).forEach(function(exe) {
if(fs.existsSync(packageObj.bin[exe])) {
console.log("linking bin '" + exe + "'");
fs.symlinkSync(
path.join("..", packageObj.name, packageObj.bin[exe]),
path.join(nodeModules, ".bin", exe)
);
}
else {
console.log("skipping non-existent bin '" + exe + "'");
}
})
}
else {
if(fs.existsSync(packageObj.bin)) {
console.log("linking bin '" + packageObj.bin + "'");
fs.symlinkSync(
path.join("..", packageObj.name, packageObj.bin),
path.join(nodeModules, ".bin", packageObj.name.split("/").pop())
);
}
else {
console.log("skipping non-existent bin '" + packageObj.bin + "'");
}
}
}
else if(packageObj.directories !== undefined && packageObj.directories.bin !== undefined) {
fs.mkdirSync(path.join(nodeModules, ".bin"))
fs.readdirSync(packageObj.directories.bin).forEach(function(exe) {
if(fs.existsSync(path.join(packageObj.directories.bin, exe))) {
console.log("linking bin '" + exe + "'");
fs.symlinkSync(
path.join("..", packageObj.name, packageObj.directories.bin, exe),
path.join(nodeModules, ".bin", exe)
);
}
else {
console.log("skipping non-existent bin '" + exe + "'");
}
})
}
'';
};
prepareAndInvokeNPM = {packageName, bypassCache, reconstructLock, npmFlags, production}: prepareAndInvokeNPM = {packageName, bypassCache, reconstructLock, npmFlags, production}:
let let
forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com"; forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com";
@ -377,13 +460,18 @@ let
npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} rebuild npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} rebuild
runHook postRebuild
if [ "''${dontNpmInstall-}" != "1" ] if [ "''${dontNpmInstall-}" != "1" ]
then then
# NPM tries to download packages even when they already exist if npm-shrinkwrap is used. # NPM tries to download packages even when they already exist if npm-shrinkwrap is used.
rm -f npm-shrinkwrap.json rm -f npm-shrinkwrap.json
npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} install npm ${forceOfflineFlag} --nodedir=${nodeSources} --no-bin-links --ignore-scripts ${npmFlags} ${lib.optionalString production "--production"} install
fi fi
# Link executables defined in package.json
node ${linkBinsScript}
''; '';
# Builds and composes an NPM package including all its dependencies # Builds and composes an NPM package including all its dependencies

File diff suppressed because it is too large Load diff

View file

@ -9,6 +9,7 @@ let
callPackage callPackage
fetchFromGitHub fetchFromGitHub
fetchurl fetchurl
fetchpatch
nixosTests; nixosTests;
since = version: lib.versionAtLeast nodejs.version version; since = version: lib.versionAtLeast nodejs.version version;
@ -92,6 +93,8 @@ final: prev: {
bitwarden-cli = prev."@bitwarden/cli".override { bitwarden-cli = prev."@bitwarden/cli".override {
name = "bitwarden-cli"; name = "bitwarden-cli";
nativeBuildInputs = [ pkgs.pkg-config ];
buildInputs = with pkgs; [ pixman cairo pango ];
}; };
bower2nix = prev.bower2nix.override { bower2nix = prev.bower2nix.override {
@ -218,6 +221,10 @@ final: prev: {
libsecret libsecret
final.node-gyp-build final.node-gyp-build
final.node-pre-gyp final.node-pre-gyp
pixman
cairo
pango
] ++ lib.optionals stdenv.isDarwin [ ] ++ lib.optionals stdenv.isDarwin [
darwin.apple_sdk.frameworks.AppKit darwin.apple_sdk.frameworks.AppKit
darwin.apple_sdk.frameworks.Security darwin.apple_sdk.frameworks.Security
@ -314,16 +321,23 @@ final: prev: {
}; };
node2nix = prev.node2nix.override { node2nix = prev.node2nix.override {
nativeBuildInputs = [ pkgs.buildPackages.makeWrapper ]; # Get latest commit for misc fixes
# We need to use master because of a fix that replaces git:// url to https://.
src = fetchFromGitHub { src = fetchFromGitHub {
owner = "svanderburg"; owner = "svanderburg";
repo = "node2nix"; repo = "node2nix";
rev = "68f5735f9a56737e3fedceb182705985e3ab8799"; rev = "026360084db8a27095aafdac7125d7f1a93046c8";
sha256 = "sha256-NK6gDTkGx0GG7yPTwgtFC4ttQZPfcLaLp8W8OOMO6bg="; sha256 = "sha256-zO/xGG10v7HGv58RLX5SFd7QOXAL2vRxCRM8IfRZ8JA=";
}; };
nativeBuildInputs = [ pkgs.buildPackages.makeWrapper ];
postInstall = '' postInstall = let
# Needed to fix Node.js 16+ - PR svanderburg/node2nix#302
npmPatch = fetchpatch {
name = "emit-lockfile-v2-and-fix-bin-links-with-npmv7.patch";
url = "https://github.com/svanderburg/node2nix/commit/375a055041b5ee49ca5fb3f74a58ca197c90c7d5.patch";
hash = "sha256-uVYrXptJILojeur9s2O+J/f2vyPNCaZMn1GM/NoC5n8=";
};
in ''
patch -d $out/lib/node_modules/node2nix -p1 < ${npmPatch}
wrapProgram "$out/bin/node2nix" --prefix PATH : ${lib.makeBinPath [ pkgs.nix ]} wrapProgram "$out/bin/node2nix" --prefix PATH : ${lib.makeBinPath [ pkgs.nix ]}
''; '';
}; };
@ -486,10 +500,18 @@ final: prev: {
thelounge-theme-flat-blue = prev.thelounge-theme-flat-blue.override { thelounge-theme-flat-blue = prev.thelounge-theme-flat-blue.override {
nativeBuildInputs = [ final.node-pre-gyp ]; nativeBuildInputs = [ final.node-pre-gyp ];
# TODO: needed until upstream pins thelounge version 4.3.1+ (which fixes dependency on old sqlite3 and transitively very old node-gyp 3.x)
preRebuild = ''
rm -r node_modules/node-gyp
'';
}; };
thelounge-theme-flat-dark = prev.thelounge-theme-flat-dark.override { thelounge-theme-flat-dark = prev.thelounge-theme-flat-dark.override {
nativeBuildInputs = [ final.node-pre-gyp ]; nativeBuildInputs = [ final.node-pre-gyp ];
# TODO: needed until upstream pins thelounge version 4.3.1+ (which fixes dependency on old sqlite3 and transitively very old node-gyp 3.x)
preRebuild = ''
rm -r node_modules/node-gyp
'';
}; };
triton = prev.triton.override { triton = prev.triton.override {
@ -574,12 +596,6 @@ final: prev: {
}; };
wrangler = prev.wrangler.override (oldAttrs: { wrangler = prev.wrangler.override (oldAttrs: {
dontNpmInstall = true;
nativeBuildInputs = [ pkgs.buildPackages.makeWrapper ];
postInstall = ''
makeWrapper "$out/lib/node_modules/wrangler/bin/wrangler.js" "$out/bin/wrangler" \
--inherit-argv0
'';
meta = oldAttrs.meta // { broken = before "16.13"; }; meta = oldAttrs.meta // { broken = before "16.13"; };
}); });

View file

@ -6067,7 +6067,7 @@ with pkgs;
inherit (darwin.apple_sdk.frameworks) Security; inherit (darwin.apple_sdk.frameworks) Security;
}; };
inherit (nodePackages_latest) wrangler; inherit (nodePackages) wrangler;
wrangler_1 = callPackage ../development/tools/wrangler_1 { wrangler_1 = callPackage ../development/tools/wrangler_1 {
inherit (darwin.apple_sdk.frameworks) CoreFoundation CoreServices Security; inherit (darwin.apple_sdk.frameworks) CoreFoundation CoreServices Security;
@ -8521,10 +8521,7 @@ with pkgs;
nodePackages_latest = dontRecurseIntoAttrs nodejs_latest.pkgs; nodePackages_latest = dontRecurseIntoAttrs nodejs_latest.pkgs;
nodePackages = (dontRecurseIntoAttrs nodejs.pkgs).override { nodePackages = dontRecurseIntoAttrs nodejs.pkgs;
# It does not work on 16.x: https://github.com/NixOS/nixpkgs/issues/132456
nodejs = nodejs-14_x;
};
node2nix = nodePackages.node2nix; node2nix = nodePackages.node2nix;