From 4ba34c316cef6b6bf9c2a61c98840908e026e870 Mon Sep 17 00:00:00 2001 From: Lily Foster Date: Wed, 21 Sep 2022 20:43:50 -0400 Subject: [PATCH] Emit lockfile v2 and fix bin links with NPM v7+ Lockfile v2 mostly just has a bit of extra metadata and all dependencies are hoisted to the top-level with path-specific keys in a new lock value called "packages". This update emits enough of the format that NPM v7+ seem to be happy enough with it and does not try to rewrite it and cause ENOTCACHED errors with the sandbox. As of NPM v7+, it no longer links bins for the top-level project automatically unless a global install is selected[1][2]. Given a global install would cause more problems than it would solve, I added a simple script to perform the linking ourselves and instructed `npm install` to never link them for consistency. Closes #236, #293, #294 [1]: https://github.com/npm/cli/commit/e46400c9484f5c66a0ba405eeb8c1340594dbf05#diff-24c01909dabbe2fc000fb5b43d14b511fb335b2f0c2e8e7a671f7d567a33d577R17-R18 [2]: https://github.com/npm/cli/issues/4308 --- bin/node2nix.js | 18 +++++++--- lib/Package.js | 12 ++++++- nix/node-env.nix | 86 +++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 103 insertions(+), 13 deletions(-) diff --git a/bin/node2nix.js b/bin/node2nix.js index e4744b5..a04b6d5 100755 --- a/bin/node2nix.js +++ b/bin/node2nix.js @@ -28,7 +28,7 @@ var switches = [ ['-18', '--nodejs-18', 'Provides all settings to generate expression for usage with Node.js 18.x (default is: nodejs-14_x)'], ['--supplement-input FILE', 'A supplement package JSON file that are passed as build inputs to all packages defined in the input JSON file'], ['--supplement-output FILE', 'Path to a Nix expression representing a supplementing set of Nix packages provided as inputs to a project (defaults to: supplement.nix)'], - ['--include-peer-dependencies', 'Specifies whether to include peer dependencies. In npm 2.x, this is the default. (false by default)'], + ['--include-peer-dependencies', 'Specifies whether to include peer dependencies. In npm 2.x, this is the default. (true by default for Node.js 16+)'], ['--no-flatten', 'Simulate pre-npm 3.x isolated dependency structure. (false by default)'], ['--pkg-name NAME', 'Specifies the name of the Node.js package to use from Nixpkgs (defaults to: nodejs)'], ['--registry URL', 'URL referring to the NPM packages registry. It defaults to the official NPM one, but can be overridden to support private registries'], @@ -47,7 +47,7 @@ var parser = new optparse.OptionParser(switches); var help = false; var version = false; var production = true; -var includePeerDependencies = false; +var includePeerDependencies = true; var flatten = true; var inputJSON = "package.json"; var outputNix = "node-packages.nix"; @@ -118,61 +118,71 @@ parser.on('development', function(arg, value) { parser.on('nodejs-4', function(arg, value) { flatten = false; nodePackage = "nodejs-4_x"; - byPassCache = false; + bypassCache = false; + includePeerDependencies = false; }); parser.on('nodejs-6', function(arg, value) { flatten = true; nodePackage = "nodejs-6_x"; - byPassCache = false; + bypassCache = false; + includePeerDependencies = false; }); parser.on('nodejs-8', function(arg, value) { flatten = true; nodePackage = "nodejs-8_x"; bypassCache = true; + includePeerDependencies = false; }); parser.on('nodejs-10', function(arg, value) { flatten = true; nodePackage = "nodejs-10_x"; bypassCache = true; + includePeerDependencies = false; }); parser.on('nodejs-12', function(arg, value) { flatten = true; nodePackage = "nodejs-12_x"; bypassCache = true; + includePeerDependencies = false; }); parser.on('nodejs-13', function(arg, value) { flatten = true; nodePackage = "nodejs-13_x"; bypassCache = true; + includePeerDependencies = false; }); parser.on('nodejs-14', function(arg, value) { flatten = true; nodePackage = "nodejs-14_x"; bypassCache = true; + includePeerDependencies = false; }); parser.on('nodejs-16', function(arg, value) { flatten = true; nodePackage = "nodejs-16_x"; bypassCache = true; + includePeerDependencies = true; }); parser.on('nodejs-17', function(arg, value) { flatten = true; nodePackage = "nodejs-17_x"; bypassCache = true; + includePeerDependencies = true; }); parser.on('nodejs-18', function(arg, value) { flatten = true; nodePackage = "nodejs-18_x"; bypassCache = true; + includePeerDependencies = true; }); parser.on('include-peer-dependencies', function(arg, value) { diff --git a/lib/Package.js b/lib/Package.js index 9a6f0b7..df19507 100644 --- a/lib/Package.js +++ b/lib/Package.js @@ -196,7 +196,17 @@ Package.prototype.resolveDependenciesAndSources = function(callback) { function(callback) { if(self.deploymentConfig.includePeerDependencies) { - /* Bundle the peer dependencies, if applicable */ + /* Unconditionally remove optional peer dependencies */ + Object.keys(self.source.config.peerDependenciesMeta || {}).forEach(function(dependencyName) { + if(self.source.config.peerDependenciesMeta[dependencyName].optional) { + if(self.source.config.dependencies) + delete self.source.config.dependencies[dependencyName]; + if(self.source.config.peerDependencies) + delete self.source.config.peerDependencies[dependencyName]; + } + }); + + /* Bundle the required peer dependencies, if applicable */ self.bundleDependencies(resolvedDependencies, self.source.config.peerDependencies, callback); } else { callback(); diff --git a/nix/node-env.nix b/nix/node-env.nix index 2590dd2..8df94cf 100644 --- a/nix/node-env.nix +++ b/nix/node-env.nix @@ -165,7 +165,11 @@ let if(process.argv[2] == "development") { replaceDependencies(packageObj.devDependencies); } + else { + delete packageObj.devDependencies; + } replaceDependencies(packageObj.optionalDependencies); + replaceDependencies(packageObj.peerDependencies); /* Write the fixed package.json file */ fs.writeFileSync("package.json", JSON.stringify(packageObj, null, 2)); @@ -280,25 +284,43 @@ let var lockObj = { name: packageObj.name, version: packageObj.version, - lockfileVersion: 1, + lockfileVersion: 2, requires: true, + packages: { + "": { + name: packageObj.name, + version: packageObj.version, + license: packageObj.license, + bin: packageObj.bin, + dependencies: packageObj.dependencies, + engines: packageObj.engines, + optionalDependencies: packageObj.optionalDependencies + } + }, dependencies: {} }; - function augmentPackageJSON(filePath, dependencies) { + function augmentPackageJSON(filePath, packages, dependencies) { var packageJSON = path.join(filePath, "package.json"); if(fs.existsSync(packageJSON)) { var packageObj = JSON.parse(fs.readFileSync(packageJSON)); + packages[filePath] = { + version: packageObj.version, + integrity: "sha1-000000000000000000000000000=", + dependencies: packageObj.dependencies, + engines: packageObj.engines, + optionalDependencies: packageObj.optionalDependencies + }; dependencies[packageObj.name] = { version: packageObj.version, integrity: "sha1-000000000000000000000000000=", dependencies: {} }; - processDependencies(path.join(filePath, "node_modules"), dependencies[packageObj.name].dependencies); + processDependencies(path.join(filePath, "node_modules"), packages, dependencies[packageObj.name].dependencies); } } - function processDependencies(dir, dependencies) { + function processDependencies(dir, packages, dependencies) { if(fs.existsSync(dir)) { var files = fs.readdirSync(dir); @@ -314,23 +336,66 @@ let pkgFiles.forEach(function(entry) { if(stats.isDirectory()) { var pkgFilePath = path.join(filePath, entry); - augmentPackageJSON(pkgFilePath, dependencies); + augmentPackageJSON(pkgFilePath, packages, dependencies); } }); } else { - augmentPackageJSON(filePath, dependencies); + augmentPackageJSON(filePath, packages, dependencies); } } }); } } - processDependencies("node_modules", lockObj.dependencies); + processDependencies("node_modules", lockObj.packages, lockObj.dependencies); fs.writeFileSync("package-lock.json", JSON.stringify(lockObj, null, 2)); ''; }; + # Script that links bins defined in package.json to the node_modules bin directory + # NPM does not do this for top-level packages itself anymore as of v7 + linkBinsScript = writeTextFile { + name = "linkbins.js"; + text = '' + var fs = require('fs'); + var path = require('path'); + + var packageObj = JSON.parse(fs.readFileSync("package.json")); + + var nodeModules = Array(packageObj.name.split("/").length).fill("..").join(path.sep); + + if(packageObj.bin !== undefined) { + fs.mkdirSync(path.join(nodeModules, ".bin")) + + if(typeof packageObj.bin == "object") { + Object.keys(packageObj.bin).forEach(function(exe) { + fs.symlinkSync( + path.join("..", packageObj.name, packageObj.bin[exe]), + path.join(nodeModules, ".bin", exe) + ); + }) + } + else { + fs.symlinkSync( + path.join("..", packageObj.name, packageObj.bin), + path.join(nodeModules, ".bin", packageObj.name) + ); + } + } + else if(packageObj.directories !== undefined && packageObj.directories.bin !== undefined) { + fs.mkdirSync(path.join(nodeModules, ".bin")) + + fs.readdirSync(packageObj.directories.bin).forEach(function(exe) { + fs.symlinkSync( + path.join("..", packageObj.name, packageObj.bin[exe]), + path.join(nodeModules, ".bin", exe) + ); + }) + } + ''; + }; + prepareAndInvokeNPM = {packageName, bypassCache, reconstructLock, npmFlags, production}: let forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com"; @@ -377,13 +442,18 @@ let npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} rebuild + runHook postRebuild + if [ "''${dontNpmInstall-}" != "1" ] then # NPM tries to download packages even when they already exist if npm-shrinkwrap is used. rm -f npm-shrinkwrap.json - npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} install + npm ${forceOfflineFlag} --nodedir=${nodeSources} --no-bin-links --ignore-scripts ${npmFlags} ${lib.optionalString production "--production"} install fi + + # Link executables defined in package.json + node ${linkBinsScript} ''; # Builds and composes an NPM package including all its dependencies