From d54ce3154dfe5283fcfeffc13d4e003bbade6370 Mon Sep 17 00:00:00 2001 From: Dave Pacheco Date: Tue, 21 Feb 2012 15:32:16 -0800 Subject: [PATCH] add "npm shrinkwrap" --- doc/api/shrinkwrap.md | 20 +++++ doc/cli/install.md | 5 +- doc/cli/shrinkwrap.md | 154 +++++++++++++++++++++++++++++++++ lib/install.js | 192 ++++++++++++++++++++++++++++++++---------- lib/npm.js | 1 + lib/shrinkwrap.js | 84 ++++++++++++++++++ 6 files changed, 411 insertions(+), 45 deletions(-) create mode 100644 doc/api/shrinkwrap.md create mode 100644 doc/cli/shrinkwrap.md create mode 100644 lib/shrinkwrap.js diff --git a/doc/api/shrinkwrap.md b/doc/api/shrinkwrap.md new file mode 100644 index 00000000..6584d6a0 --- /dev/null +++ b/doc/api/shrinkwrap.md @@ -0,0 +1,20 @@ +npm-shrinkwrap(3) -- programmatically generate package shrinkwrap file +==================================================== + +## SYNOPSIS + + npm.commands.shrinkwrap(args, [silent,] callback) + +## DESCRIPTION + +This acts much the same ways as shrinkwrapping on the command-line. + +This command does not take any arguments, but 'args' must be defined. +Beyond that, if any arguments are passed in, npm will politely warn that it +does not take positional arguments. + +If the 'silent' parameter is set to true, nothing will be output to the screen, +but the shrinkwrap file will still be written. + +Finally, 'callback' is a function that will be called when the shrinkwrap has +been saved. diff --git a/doc/cli/install.md b/doc/cli/install.md index 22eb8234..903844a4 100644 --- a/doc/cli/install.md +++ b/doc/cli/install.md @@ -14,7 +14,9 @@ npm-install(1) -- Install a package ## DESCRIPTION -This command installs a package, and any packages that it depends on. +This command installs a package, and any packages that it depends on. If the +package has a shrinkwrap file, the installation of dependencies will be driven +by that. See npm-shrinkwrap(1). A `package` is: @@ -199,3 +201,4 @@ affects a real use-case, it will be investigated. * npm-folders(1) * npm-tag(1) * npm-rm(1) +* npm-shrinkwrap(1) diff --git a/doc/cli/shrinkwrap.md b/doc/cli/shrinkwrap.md new file mode 100644 index 00000000..9ed7750c --- /dev/null +++ b/doc/cli/shrinkwrap.md @@ -0,0 +1,154 @@ +npm-shrinkwrap(1) -- Lock down dependency versions +===================================================== + +## SYNOPSIS + + npm shrinkwrap + +## DESCRIPTION + +This command locks down the versions of a package's dependencies so that you can +control exactly which versions of each dependency will be used when your package +is installed. + +By default, "npm install" recursively installs the target's dependencies (as +specified in package.json), choosing the latest available version that satisfies +the dependency's semver pattern. In some situations, particularly when shipping +software where each change is tightly managed, it's desirable to fully specify +each version of each dependency recursively so that subsequent builds and +deploys do not inadvertently pick up newer versions of a dependency that satisfy +the semver pattern. Specifying specific semver patterns in each dependency's +package.json would facilitate this, but that's not always possible or desirable, +as when another author owns the npm package. It's also possible to check +dependencies directly into source control, but that may be undesirable for other +reasons. + +As an example, consider package A: + + { + "name": "A", + "version": "0.1.0" + "dependencies": { + "B": "<0.1.0" + } + } + +package B: + + { + "name": "B", + "version": "0.0.1" + "dependencies": { + "C": "<0.1.0" + } + } + +and package C: + + { + "name": "C + "version": "0.0.1" + } + +If these are the only versions of A, B, and C available in the registry, then +a normal "npm install A" will install: + + A@0.1.0 + B@0.0.1 + C@0.0.1 + +However, if B@0.0.2 is published, then a fresh "npm install A" will install: + + A@0.1.0 + B@0.0.2 + C@0.0.1 + +assuming the new version did not modify B's dependencies. Of course, the new +version of B could include a new version of C and any number of new +dependencies. If such changes are undesirable, the author of A could specify a +dependency on B@0.0.1. However, if A's author and B's author are not the same +person, there's no way for A's author to say that he or she does not want to +pull in newly published versions of C when B hasn't changed at all. + +In this case, A's author can use + + # npm shrinkwrap + +This generates npm-shrinkwrap.json, which will look something like this: + + { + "name": "A" + "version": "0.1.0" + "dependencies": { + "B": { + "version": "0.0.1" + "dependencies": { + "C": { + "version": "0.1.0" + } + } + } + } + } + +The shrinkwrap command has locked down the dependencies based on what's +currently installed in node_modules. When "npm install" installs a package with +a npm-shrinkwrap.json file in the package root, the shrinkwrap file (rather than +package.json files) completely drives the installation of that package and all +of its dependencies (recursively). So now the author publishes A@0.1.0, and +subsequent installs of this package will use B@0.0.1 and C@0.1.0, regardless the +dependencies and versions listed in A's, B's, and C's package.json files. + + +### Using shrinkwrapped packages + +Using a shrinkwrapped package is no different than using any other package: you +can "npm install" it by hand, or add a dependency to your package.json file and +"npm install" it. + +### Building shrinkwrapped packages + +To shrinkwrap an existing package: + +1. Run "npm install" in the package root to install the current versions of all + dependencies. +2. Validate that the package works as expected with these versions. +3. Run "npm shrinkwrap", add npm-shrinkwrap.json to git, and publish your + package. + +To add or update a dependency in a shrinkwrapped package: + +1. Run "npm install" in the package root to install the current versions of all + dependencies. +2. Add or update dependencies. "npm install" each new or updated package + individually and then update package.json. +3. Validate that the package works as expected with the new dependencies. +4. Run "npm shrinkwrap", commit the new npm-shrinkwrap.json, and publish your + package. + +You can use npm-outdated(1) to view dependencies with newer versions available. + +### Other notes + +Since "npm shrinkwrap" uses the locally installed packages to construct the +shrinkwrap file, devDependencies will be included if and only if you've +installed them already when you make the shrinkwrap. + +A shrinkwrap file must be consistent with the package's package.json file. "npm +shrinkwrap" will fail if required dependencies are not already installed, since +that would result in a shrinkwrap that wouldn't actually work. Similarly, the +command will fail if there are extraneous packages (not referenced by +package.json), since that would indicate that package.json is not correct. + +If shrinkwrapped package A depends on shrinkwrapped package B, B's shrinkwrap +will not be used as part of the installation of A. However, because A's +shrinkwrap is constructed from a valid installation of B and recursively +specifies all dependencies, the contents of B's shrinkwrap will implicitly be +included in A's shrinkwrap. + + +## SEE ALSO + +* npm-install(1) +* npm-json(1) +* npm-list(1) diff --git a/lib/install.js b/lib/install.js index 211a6612..e8d21449 100644 --- a/lib/install.js +++ b/lib/install.js @@ -20,7 +20,9 @@ install.usage = "npm install " + "\nnpm install @" + "\nnpm install @" + "\n\nCan specify one or more: npm install ./foo.tgz bar@stable /some/folder" - + "\nInstalls dependencies in ./package.json if no argument supplied" + + "\nIf no argument is supplied and ./npm-shrinkwrap.json is " + + "\npresent, installs dependencies specified in the shrinkwrap." + + "\nOtherwise, installs dependencies from ./package.json." install.completion = function (opts, cb) { // install can complete to a folder with a package.json, or any package. @@ -109,7 +111,8 @@ function install (args, cb_) { // or install current folder globally if (!args.length) { if (npm.config.get("global")) args = ["."] - else return readJson( path.resolve(where, "package.json") + else return readDependencies( null + , where , { dev: !npm.config.get("production") } , function (er, data) { if (er) return log.er(cb, "Couldn't read dependencies.")(er) @@ -123,7 +126,7 @@ function install (args, cb_) { , parsed = url.parse(target.replace(/^git\+/, "git")) target = dep + "@" + target return target - }), where, family, ancestors, false, data, cb) + }), where, family, ancestors, null, false, data, cb) }) } @@ -135,7 +138,68 @@ function install (args, cb_) { , ancestors = {} if (data) family[data.name] = ancestors[data.name] = data.version var fn = npm.config.get("global") ? installMany : installManyTop - fn(args, where, family, ancestors, true, data, cb) + fn(args, where, family, ancestors, null, true, data, cb) + }) + }) +} + +// reads dependencies for the package at "where". There are several cases, +// depending on our current state and the package's configuration: +// +// 1. If "wrap" is specified, then it's assumed we're processing a package +// underneath a shrinkwrap, so dependencies are read directly from the +// shrinkwrap. +// 2. Otherwise, if an npm-shrinkwrap.json file is present, dependencies are +// read from there. +// 3. Otherwise, dependencies come from package.json. +// +// Regardless of which case we fall into, "cb" is invoked with a first argument +// describing the full package (as though readJson had been used) but with +// "dependencies" read as described above. The second argument to "cb" is the +// shrinkwrap to use in processing this package's dependencies, which may be +// "wrap" (in case 1) or a new shrinkwrap (in case 2). +function readDependencies (wrap, where, opts, cb) +{ + readJson( path.resolve(where, "package.json") + , opts + , function (er, data) { + if (er) return cb(er) + + if (wrap) { + log.verbose([where, wrap], "readDependencies: using existing wrap") + var rv = {} + for (var key in data) + rv[key] = data[key] + rv["dependencies"] = {} + for (key in wrap) + rv["dependencies"][key] = wrap[key]["version"] + log.verbose([rv["dependencies"]], "readDependencies: returned deps") + return cb(null, rv, wrap) + } + + var wrapfile = path.resolve(where, "npm-shrinkwrap.json") + + fs.readFile(wrapfile, function (er, wrapjson) { + if (er) { + log.verbose("readDependencies: using package.json deps") + return cb(null, data, null) + } + + try { + var newwrap = JSON.parse(wrapjson) + } catch (ex) { + return cb(ex) + } + + log.info("using shrinkwrap file "+wrapfile) + var rv = {} + for (var key in data) + rv[key] = data[key] + rv["dependencies"] = {} + for (key in newwrap["dependencies"]) + rv["dependencies"][key] = newwrap["dependencies"][key]["version"] + log.verbose([rv["dependencies"]], "readDependencies: returned deps") + return cb(null, rv, newwrap["dependencies"]) }) }) } @@ -255,7 +319,8 @@ function treeify (installed) { // just like installMany, but also add the existing packages in // where/node_modules to the family object. -function installManyTop (what, where, family, ancestors, explicit, parent, cb_) { +function installManyTop (what, where, family, ancestors, unused, explicit, parent, + cb_) { function cb (er, d) { if (explicit || er) return cb_(er, d) @@ -286,7 +351,8 @@ function installManyTop_ (what, where, family, ancestors, explicit, parent, cb) : [] fs.readdir(nm, function (er, pkgs) { - if (er) return installMany(what, where, family, ancestors, explicit, parent, cb) + if (er) return installMany(what, where, family, ancestors, null, explicit, + parent, cb) pkgs = pkgs.filter(function (p) { return !p.match(/^[\._-]/) && (!explicit || names.indexOf(p) === -1) @@ -304,26 +370,38 @@ function installManyTop_ (what, where, family, ancestors, explicit, parent, cb) packages.forEach(function (p) { family[p[0]] = p[1] }) - return installMany(what, where, family, ancestors, explicit, parent, cb) + return installMany(what, where, family, ancestors, null, explicit, parent, + cb) }) }) } -function installMany (what, where, family, ancestors, explicit, parent, cb) { - // 'npm install foo' should install the version of foo - // that satisfies the dep in the current folder. - // This will typically return immediately, since we already read - // this file family, and it'll be cached. - readJson(path.resolve(where, "package.json"), function (er, data) { +function installMany (what, where, family, ancestors, oldwrap, explicit, parent, + cb) { + // readDependencies takes care of figuring out whether the list of + // dependencies we'll iterate below comes from an existing shrinkwrap from a + // parent level, a new shrinkwrap at this level, or package.json at this + // level, as well as which shrinkwrap (if any) our dependencies should use. + readDependencies(oldwrap, where, {}, function (er, data, wrap) { if (er) data = {} - d = data.dependencies || {} var parent = data + var d = data["dependencies"] || {} + + // if we're explicitly installing "what" into "where", then the shrinkwrap + // for "where" doesn't apply. This would be the case if someone were adding + // a new package to a shrinkwrapped package. (data.dependencies will not be + // used here except to indicate what packages are already present, so + // there's no harm in using that.) + if (explicit) + wrap = null + // what is a list of things. // resolve each one. asyncMap( what - , targetResolver(where, family, ancestors, explicit, d, parent) + , targetResolver(where, family, ancestors, wrap, explicit, d, + parent) , function (er, targets) { if (er) return cb(er) @@ -343,13 +421,15 @@ function installMany (what, where, family, ancestors, explicit, parent, cb) { }) asyncMap(targets, function (target, cb) { log(target._id, "installOne") - installOne(target, where, newPrev, newAnc, parent, cb) + var newWrap = wrap ? wrap[target.name]["dependencies"] || {} : null + installOne(target, where, newPrev, newAnc, newWrap, parent, cb) }, cb) }) }) } -function targetResolver (where, family, ancestors, explicit, deps, parent) { +function targetResolver (where, family, ancestors, wrap, explicit, deps, + parent) { var alreadyInstalledManually = explicit ? [] : null , nm = path.resolve(where, "node_modules") @@ -381,11 +461,29 @@ function targetResolver (where, family, ancestors, explicit, deps, parent) { return cb(null, []) } - if (family[what] && semver.satisfies(family[what], deps[what] || "")) { - return cb(null, []) + // check for a version installed higher in the tree. + // If installing from a shrinkwrap, it must match exactly. + if (family[what]) { + if (wrap && wrap[what]["version"] == family[what]) { + log.verbose("using existing "+what+" (matches shrinkwrap)") + return cb(null, []) + } + + if (!wrap && semver.satisfies(family[what], deps[what] || "")) { + log.verbose("using existing "+what+" (no shrinkwrap)") + return cb(null, []) + } } - if (deps[what]) { + if (wrap) { + name = what.split(/@/).shift() + if (wrap[name]) { + log.verbose("shrinkwrap: resolving "+what+" to "+wrap[name]["version"]) + what = name + "@" + wrap[name]["version"] + } else { + log.verbose("shrinkwrap: skipping "+what+" (not in shrinkwrap)") + } + } else if (deps[what]) { what = what + "@" + deps[what] } @@ -405,14 +503,14 @@ function targetResolver (where, family, ancestors, explicit, deps, parent) { // we've already decided to install this. if anything's in the way, // then uninstall it first. -function installOne (target, where, family, ancestors, parent, cb) { +function installOne (target, where, family, ancestors, wrap, parent, cb) { // the --link flag makes this a "link" command if it's at the // the top level. if (where === npm.prefix && npm.config.get("link") && !npm.config.get("global")) { - return localLink(target, where, family, ancestors, parent, cb) + return localLink(target, where, family, ancestors, wrap, parent, cb) } - installOne_(target, where, family, ancestors, parent, cb) + installOne_(target, where, family, ancestors, wrap, parent, cb) } function localLink (target, where, family, ancestors, parent, cb) { @@ -440,7 +538,7 @@ function localLink (target, where, family, ancestors, parent, cb) { } else { log.verbose(target._id, "install locally (no link)") - installOne_(target, where, family, ancestors, parent, cb) + installOne_(target, where, family, ancestors, wrap, parent, cb) } }) } @@ -464,7 +562,7 @@ function resultList (target, where, parentId) { , parentId && prettyWhere ] } -function installOne_ (target, where, family, ancestors, parent, cb) { +function installOne_ (target, where, family, ancestors, wrap, parent, cb) { var nm = path.resolve(where, "node_modules") , targetFolder = path.resolve(nm, target.name) , prettyWhere = relativize(where, process.cwd() + "/x") @@ -475,7 +573,7 @@ function installOne_ (target, where, family, ancestors, parent, cb) { ( [ [checkEngine, target] , [checkCycle, target, ancestors] , [checkGit, targetFolder] - , [write, target, targetFolder, family, ancestors] ] + , [write, target, targetFolder, family, ancestors, wrap] ] , function (er, d) { log.verbose(target._id, "installOne cb") if (er) return cb(er) @@ -559,7 +657,7 @@ function checkGit_ (folder, cb) { }) } -function write (target, targetFolder, family, ancestors, cb_) { +function write (target, targetFolder, family, ancestors, wrap, cb_) { var up = npm.config.get("unsafe-perm") , user = up ? null : npm.config.get("user") , group = up ? null : npm.config.get("group") @@ -586,23 +684,29 @@ function write (target, targetFolder, family, ancestors, cb_) { // up until this point, since we really don't care about it. , function (er) { if (er) return cb(er) - var deps = Object.keys(target.dependencies || {}) - installMany(deps.filter(function (d) { - // prefer to not install things that are satisfied by - // something in the "family" list. - return !semver.satisfies(family[d], target.dependencies[d]) - }).map(function (d) { - var t = target.dependencies[d] - , parsed = url.parse(t.replace(/^git\+/, "git")) - t = d + "@" + t - return t - }), targetFolder, family, ancestors, false, target, function (er, d) { - log.verbose(targetFolder, "about to build") - if (er) return cb(er) - npm.commands.build( [targetFolder] - , npm.config.get("global") - , true - , function (er) { return cb(er, d) }) + + // before continuing to installing dependencies, check for a shrinkwrap. + readDependencies(wrap, targetFolder, {}, function (er, data, wrap) { + var deps = Object.keys(data.dependencies || {}) + installMany(deps.filter(function (d) { + // prefer to not install things that are satisfied by + // something in the "family" list, unless we're installing + // from a shrinkwrap. + return wrap || !semver.satisfies(family[d], data.dependencies[d]) + }).map(function (d) { + var t = data.dependencies[d] + , parsed = url.parse(t.replace(/^git\+/, "git")) + t = d + "@" + t + return t + }), targetFolder, family, ancestors, wrap, false, target, + function (er, d) { + log.verbose(targetFolder, "about to build") + if (er) return cb(er) + npm.commands.build( [targetFolder] + , npm.config.get("global") + , true + , function (er) { return cb(er, d) }) + }) }) } ) } diff --git a/lib/npm.js b/lib/npm.js index de68393d..53197082 100644 --- a/lib/npm.js +++ b/lib/npm.js @@ -138,6 +138,7 @@ var commandCache = {} , "unpublish" , "owner" , "deprecate" + , "shrinkwrap" , "help" , "help-search" diff --git a/lib/shrinkwrap.js b/lib/shrinkwrap.js new file mode 100644 index 00000000..8a54dd5d --- /dev/null +++ b/lib/shrinkwrap.js @@ -0,0 +1,84 @@ +// emit JSON describing versions of all packages currently installed (for later +// use with shrinkwrap install) + +module.exports = exports = shrinkwrap + +var npm = require("./npm.js") + , output = require("./utils/output.js") + , log = require("./utils/log.js") + , fs = require('fs') + , path = require('path') + +shrinkwrap.usage = "npm shrinkwrap" + +function shrinkwrap (args, silent, cb) { + if (typeof cb !== "function") cb = silent, silent = false + + if (args.length) { + log.warn("shrinkwrap doesn't take positional args.") + } + + npm.commands.ls([], true, function (er, pkginfo) { + if (er) return cb(er) + + var wrapped = {} + var nerr + + if (pkginfo['name']) + wrapped['name'] = pkginfo['name'] + + nerr = shrinkwrapPkg(log, pkginfo['name'], pkginfo, wrapped) + if (nerr > 0) + return cb(new Error('failed with ' + nerr + ' errors')) + + // leave the version field out of the top-level, since it's not used and + // could only be confusing if it gets out of date. + delete wrapped['version'] + + fs.writeFile( path.join(process.cwd(), "npm-shrinkwrap.json") + , new Buffer(JSON.stringify(wrapped, null, 2) + "\n") + , function (er) { + if (er) return cb(er) + output.write("wrote npm-shrinkwrap.json", function (er) { + cb(er, wrapped) + }) + }) + }) +} + +function shrinkwrapPkg (log, pkgname, pkginfo, rv) { + var pkg, dep, nerr + + if (typeof (pkginfo) == 'string') { + log.error('required dependency not installed: ' + pkgname + '@' + pkginfo) + return (1) + } + + if ('version' in pkginfo) + rv['version'] = pkginfo['version'] + + if (Object.keys(pkginfo['dependencies']).length === 0) + return (0) + + rv['dependencies'] = {} + nerr = 0 + + for (pkg in pkginfo['dependencies']) { + dep = pkginfo['dependencies'][pkg] + rv['dependencies'][pkg] = {} + nerr += shrinkwrapPkg(log, pkg, dep, rv['dependencies'][pkg]) + + // package.json must be consistent with the shrinkwrap bundle + if (dep['extraneous']) { + log.error('package is extraneous: ' + pkg + '@' + dep['version']) + nerr++ + } + + if (dep['invalid']) { + log.error('package is invalid: ' + pkg) + nerr++ + } + } + + return (nerr) +}