From df31978ba594f365ee21901f43a293954806907c Mon Sep 17 00:00:00 2001 From: Maarten Hoogendoorn Date: Sat, 17 Dec 2016 16:19:50 +0100 Subject: [PATCH] Started to add yarn.lock file support --- bin/node2nix.js | 8 +++++++- lib/generator/collection.js | 6 ++++-- lib/generator/package.js | 5 +++-- lib/node2nix.js | 25 +++++++++++++++++++++---- lib/packagefetcher/yarn.js | 34 ++++++++++++++++++++++++++++++++++ lib/packageset.js | 25 ++++++++++++++----------- package.json | 3 ++- 7 files changed, 85 insertions(+), 21 deletions(-) create mode 100644 lib/packagefetcher/yarn.js diff --git a/bin/node2nix.js b/bin/node2nix.js index c8f1218..3c94478 100644 --- a/bin/node2nix.js +++ b/bin/node2nix.js @@ -11,6 +11,7 @@ var switches = [ ['-h', '--help', 'Shows help sections'], ['-v', '--version', 'Shows version'], ['-i', '--input FILE', 'Specifies a path to a JSON file containing an object with package settings or an array of dependencies (defaults to: package.json)'], + ['-y', '--yarn-lock-file-dir FILE', 'Specifies a path to the directory containing a yarn lock file which contains the versions to be used for all resolved dependencies'], ['-o', '--output FILE', 'Path to a Nix expression representing a registry of Node.js packages (defaults to: node-packages.nix)'], ['-c', '--composition FILE', 'Path to a Nix composition expression allowing someone to deploy the generated Nix packages from the command-line (defaults to: default.nix)'], ['-e', '--node-env FILE', 'Path to the Nix expression implementing functions that build NPM packages (defaults to: node-env.nix)'], @@ -35,6 +36,7 @@ var production = true; var includePeerDependencies = false; var flatten = false; var inputJSON = "package.json"; +var yarnDir; var outputNix = "node-packages.nix"; var compositionNix = "default.nix"; var supplementJSON; @@ -58,6 +60,10 @@ parser.on('input', function(arg, value) { inputJSON = value; }); +parser.on('yarn-lock-file-dir', function(arg, value) { + yarnDir = value; +}); + parser.on('output', function(arg, value) { outputNix = value; }); @@ -170,7 +176,7 @@ if(version) { } /* Perform the NPM to Nix conversion */ -node2nix.npmToNix(inputJSON, outputNix, compositionNix, nodeEnvNix, supplementJSON, supplementNix, production, includePeerDependencies, flatten, nodePackage, registryURL, function(err) { +node2nix.npmToNix(inputJSON, yarnDir, outputNix, compositionNix, nodeEnvNix, supplementJSON, supplementNix, production, includePeerDependencies, flatten, nodePackage, registryURL, function(err) { if(err) { process.stderr.write(err + "\n"); process.exit(1); diff --git a/lib/generator/collection.js b/lib/generator/collection.js index 4396d9e..983d5c7 100644 --- a/lib/generator/collection.js +++ b/lib/generator/collection.js @@ -7,6 +7,7 @@ var nijs = require('nijs'); * * @param {PackageSet} packageSet Maintains a set of packages and their dependencies * @param {Object} dependencies An array of objects representing package specifications + * @param {Object} yarnInfo Dependency information from yarn (can be undefined) * @param {String} baseDir Directory in which the referrer's package.json configuration resides * @param {Boolean} production Indicates whether to deploy the package in production mode * @param {Boolean} includePeerDependencies Indicates whether to include peer dependencies with the package @@ -15,7 +16,8 @@ var nijs = require('nijs'); * If an error occurs, the error parameter is set to contain the error * If the operation succeeds, it returns an object containing the abstract syntax of a Nix expression */ -function generateCollectionExpr(packageSet, dependencies, baseDir, production, includePeerDependencies, flatten, callback) { +function generateCollectionExpr(packageSet, dependencies, yarnInfo, baseDir, production, includePeerDependencies, flatten, callback) { + throw JSON.stringify({ msg: "not working with yarn yet", packageObj: packageObj}, null, " "); var body = {}; var i; @@ -46,7 +48,7 @@ function generateCollectionExpr(packageSet, dependencies, baseDir, production, i slasp.sequence([ function(callback) { /* Resolve the dependencies of each package */ - packageSet.resolveDependencies(resolvedDependencies, null, dependency, flatten, callback); + packageSet.resolveDependencies(resolvedDependencies, null, yarnInfo, dependency, flatten, callback); }, function(callback) { diff --git a/lib/generator/package.js b/lib/generator/package.js index 038dc6f..4332c2e 100644 --- a/lib/generator/package.js +++ b/lib/generator/package.js @@ -8,6 +8,7 @@ var nijs = require('nijs'); * * @param {PackageSet} packageSet Maintains a set of packages and their dependencies * @param {Object} packageObj A package.json configuration file of a package + * @param {Object} yarnInfo Dependency information from yarn (can be undefined) * @param {String} baseDir Directory in which the referrer's package.json configuration resides * @param {Object} src Directory in which the package.json file resides * @param {Boolean} production Indicates whether to deploy the package in production mode @@ -17,7 +18,7 @@ var nijs = require('nijs'); * If an error occurs, the error parameter is set to contain the error * If the operation succeeds, it returns an object containing the abstract syntax of a Nix expression */ -function generatePackageExpr(packageSet, packageObj, baseDir, src, production, includePeerDependencies, flatten, callback) { +function generatePackageExpr(packageSet, packageObj, yarnInfo, baseDir, src, production, includePeerDependencies, flatten, callback) { /* Construct metadata object for the package that we want to deploy */ var metadata = { packageObj: packageObj, @@ -29,7 +30,7 @@ function generatePackageExpr(packageSet, packageObj, baseDir, src, production, i /* Construct a Nix expression providing jobsets to deploy it */ slasp.sequence([ function(callback) { - packageSet.generatePackageArgsExpr(metadata, production, includePeerDependencies, flatten, callback); + packageSet.generatePackageArgsExpr(metadata, production, yarnInfo, includePeerDependencies, flatten, callback); }, function(callback, args) { diff --git a/lib/node2nix.js b/lib/node2nix.js index c84b69f..8ad1203 100644 --- a/lib/node2nix.js +++ b/lib/node2nix.js @@ -2,6 +2,7 @@ var fs = require('fs'); var path = require('path'); var slasp = require('slasp'); var nijs = require('nijs'); +const yarn_wrapper = require("../node_modules/yarn/lib/lockfile/wrapper"); var PackageSet = require('./packageset.js').PackageSet; var packageGenerator = require('./generator/package.js'); var collectionGenerator = require('./generator/collection.js'); @@ -45,6 +46,7 @@ exports.copyNodeEnvExpr = copyNodeEnvExpr; * configuration or an array of NPM dependencies. * * @param {String} inputJSON Path to a package.json or arbitrary JSON file + * @param {String} yarnDir Path to directory containing the yarn.lock file. May be undefined. * @param {String} outputNix Path to which the generated registry expression is written * @param {String} compositionNix Path to which the generated composition expression is written * @param {String} nodeEnvNix Path to which the NPM package build expression is written @@ -59,7 +61,7 @@ exports.copyNodeEnvExpr = copyNodeEnvExpr; * If an error occurs, the error parameter is set to contain the error * If the operation succeeds, it returns a string containing the registry expression containing the packages and all its dependencies */ -function npmToNix(inputJSON, outputNix, compositionNix, nodeEnvNix, supplementJSON, supplementNix, production, includePeerDependencies, flatten, nodePackage, registryURL, callback) { +function npmToNix(inputJSON, yarnDir, outputNix, compositionNix, nodeEnvNix, supplementJSON, supplementNix, production, includePeerDependencies, flatten, nodePackage, registryURL, callback) { var obj = JSON.parse(fs.readFileSync(inputJSON)); var version = JSON.parse(fs.readFileSync(path.join(__dirname, "..", "package.json"))).version; var disclaimer = "# This file has been generated by node2nix " + version + ". Do not edit!\n\n"; @@ -68,13 +70,28 @@ function npmToNix(inputJSON, outputNix, compositionNix, nodeEnvNix, supplementJS var packageSet = new PackageSet(registryURL, outputDir); slasp.sequence([ + /* Compute yarn info from lockfile */ + function (callback) { + if (typeof yarnDir == "undefined") { + callback(null, undefined); + } else { + yarn_wrapper.default.fromDirectory(yarnDir).then((lockfile) => { + if (!lockfile.cache) { + throw "yarn.lock could not be found or loaded"; + } + callback(null, lockfile.cache); + }).catch((err) => { + callback(err); + }); + } + }, /* Generate a Nix expression */ - function(callback) { + function(callback, yarnInfo) { if(typeof obj == "object" && obj !== null) { if(Array.isArray(obj)) { - collectionGenerator.generateCollectionExpr(packageSet, obj, baseDir, production, includePeerDependencies, flatten, callback); + collectionGenerator.generateCollectionExpr(packageSet, obj, yarnInfo, baseDir, production, includePeerDependencies, flatten, callback); } else { - packageGenerator.generatePackageExpr(packageSet, obj, baseDir, new nijs.NixFile({ value: "./." }), production, includePeerDependencies, flatten, callback); + packageGenerator.generatePackageExpr(packageSet, obj, yarnInfo, baseDir, new nijs.NixFile({ value: "./." }), production, includePeerDependencies, flatten, callback); } } else { callback("The provided JSON file must consist of an object or an array"); diff --git a/lib/packagefetcher/yarn.js b/lib/packagefetcher/yarn.js new file mode 100644 index 0000000..de0ce97 --- /dev/null +++ b/lib/packagefetcher/yarn.js @@ -0,0 +1,34 @@ +var path = require('path'); +var nijs = require('nijs'); + +function fetchMetaDataFromYarn(baseDir, dependencyName, versionSpec, yarnInfo, callback) { + var yarnKey = dependencyName + "@" + versionSpec; + var yarnDepInfo = yarnInfo[yarnKey]; + + var resolvedParts = yarnDepInfo.resolved.split("#"); + var url = resolvedParts[0]; + var shasum = resolvedParts[1]; + + var packageObj = { + name: dependencyName, + version: yarnDepInfo["version"], + dependencies: yarnDepInfo["dependencies"] + }; + + var yarnPkg = { + baseDir: path.join(baseDir, dependencyName), + packageObj: packageObj, + identifier: dependencyName + "-" + packageObj.version, + src: new nijs.NixFunInvocation({ + funExpr: new nijs.NixExpression("fetchurl"), + paramExpr: { + url: url, + sha1: shasum + } + }), + }; + + callback(null, yarnPkg); +} + +exports.fetchMetaDataFromYarn = fetchMetaDataFromYarn; diff --git a/lib/packageset.js b/lib/packageset.js index 4c8e9bc..a63ffc4 100644 --- a/lib/packageset.js +++ b/lib/packageset.js @@ -14,6 +14,7 @@ var fetchMetaDataFromNPMRegistry = require('./packagefetcher/npmregistry.js').fe var fetchMetaDataFromGit = require('./packagefetcher/git.js').fetchMetaDataFromGit; var fetchMetaDataFromHTTP = require('./packagefetcher/http.js').fetchMetaDataFromHTTP; var fetchMetaDataFromLocalDirectory = require('./packagefetcher/local.js').fetchMetaDataFromLocalDirectory; +var fetchMetaDataFromYarn = require('./packagefetcher/yarn.js').fetchMetaDataFromYarn; /** * @constructor @@ -58,12 +59,14 @@ function composeGitURL(baseURL, parsedUrl) { * corresponding to a Nix function invocation that fetches the package from * an external source */ -PackageSet.prototype.fetchMetaData = function(baseDir, dependencyName, versionSpec, callback) { +PackageSet.prototype.fetchMetaData = function(baseDir, dependencyName, yarnInfo, versionSpec, callback) { var parsedVersionSpec = semver.validRange(versionSpec, true); var parsedUrl = url.parse(versionSpec); - if(parsedVersionSpec !== null) { // If the version is valid semver range, fetch the package from the NPM registry + if (yarnInfo) { + fetchMetaDataFromYarn(baseDir, dependencyName, versionSpec, yarnInfo, callback); + } else if(parsedVersionSpec !== null) { // If the version is valid semver range, fetch the package from the NPM registry fetchMetaDataFromNPMRegistry(baseDir, dependencyName, parsedVersionSpec, this.registryURL, callback); } else if(parsedUrl.protocol == "github:") { // If the version is a GitHub repository, compose the corresponding Git URL and do a Git checkout fetchMetaDataFromGit(baseDir, dependencyName, composeGitURL("git://github.com", parsedUrl), callback); @@ -189,7 +192,7 @@ PackageSet.prototype.bindDependency = function(metadata, dependencyMetadata, fla * @param {function(String, Object)} callback Callback that gets invoked when the work is done. * If an error occured, the error parameter is set to contain an error message */ -PackageSet.prototype.resolveDependencies = function(resolvedDependencies, metadata, dependencies, flatten, callback) { +PackageSet.prototype.resolveDependencies = function(resolvedDependencies, metadata, yarnInfo, dependencies, flatten, callback) { var self = this; if(typeof dependencies == "object" && dependencies !== null) { // If the dependency set is empty then we don't have to generate anything @@ -214,7 +217,7 @@ PackageSet.prototype.resolveDependencies = function(resolvedDependencies, metada slasp.sequence([ function(callback) { // Fetch package meta data from an external source - self.fetchMetaData(baseDir, dependencyName, versionSpec, callback); + self.fetchMetaData(baseDir, dependencyName, yarnInfo, versionSpec, callback); }, function(callback, dependencyMetadata) { @@ -255,13 +258,13 @@ PackageSet.prototype.resolveDependencies = function(resolvedDependencies, metada * @param {function(String)} callback allback Callback that gets invoked when the work is done. * If an error occured, the error parameter is set to contain an error message. */ -PackageSet.prototype.resolveAllDependencies = function(metadata, production, includePeerDependencies, flatten, callback) { +PackageSet.prototype.resolveAllDependencies = function(metadata, production, yarnInfo, includePeerDependencies, flatten, callback) { var self = this; var resolvedDependencies = {}; slasp.sequence([ function(callback) { - self.resolveDependencies(resolvedDependencies, metadata, metadata.packageObj.dependencies, flatten, callback); + self.resolveDependencies(resolvedDependencies, metadata, yarnInfo, metadata.packageObj.dependencies, flatten, callback); }, function(callback) { @@ -269,14 +272,14 @@ PackageSet.prototype.resolveAllDependencies = function(metadata, production, inc if(production) { callback(); } else { - self.resolveDependencies(resolvedDependencies, metadata, metadata.packageObj.devDependencies, flatten, callback); + self.resolveDependencies(resolvedDependencies, metadata, yarnInfo, metadata.packageObj.devDependencies, flatten, callback); } }, function(callback) { /* Resolve the peer dependencies, if applicable */ if(includePeerDependencies) { - self.resolveDependencies(resolvedDependencies, metadata, metadata.packageObj.peerDependencies, flatten, callback); + self.resolveDependencies(resolvedDependencies, metadata, yarnInfo, metadata.packageObj.peerDependencies, flatten, callback); } else { callback(); } @@ -290,7 +293,7 @@ PackageSet.prototype.resolveAllDependencies = function(metadata, production, inc }, function(dependencyName, callback) { var resolvedDependency = resolvedDependencies[dependencyName]; - self.resolveAllDependencies(resolvedDependency, true /* Never include development dependencies of transitive dependencies */, includePeerDependencies, flatten, callback); + self.resolveAllDependencies(resolvedDependency, true /* Never include development dependencies of transitive dependencies */, yarnInfo, includePeerDependencies, flatten, callback); }, callback); } ], callback); @@ -375,13 +378,13 @@ PackageSet.prototype.generateDependencyListExpr = function(metadata) { * @param {function(String,Object)} callback Callback function that gets invoked when the work is done. The first parameter is set to an error message if some error occured. * The second parameter is set to an object containing the parameters. */ -PackageSet.prototype.generatePackageArgsExpr = function(metadata, production, includePeerDependencies, flatten, callback) { +PackageSet.prototype.generatePackageArgsExpr = function(metadata, production, yarnInfo, includePeerDependencies, flatten, callback) { var self = this; slasp.sequence([ function(callback) { - self.resolveAllDependencies(metadata, production, includePeerDependencies, flatten, callback); + self.resolveAllDependencies(metadata, production, yarnInfo, includePeerDependencies, flatten, callback); }, function(callback) { diff --git a/package.json b/package.json index 3cf4722..aa3ccd4 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,8 @@ "fs.extra": "1.2.x", "findit": "2.0.x", "slasp": "0.0.4", - "nijs": "0.0.23" + "nijs": "0.0.23", + "yarn": "0.18.x" }, "repository": { "type": "git",