add "npm shrinkwrap"

doc/api/shrinkwrap.md

@@ -0,0 +1,20 @@
+npm-shrinkwrap(3) -- programmatically generate package shrinkwrap file
+====================================================
+
+## SYNOPSIS
+
+    npm.commands.shrinkwrap(args, [silent,] callback)
+
+## DESCRIPTION
+
+This acts much the same ways as shrinkwrapping on the command-line.
+
+This command does not take any arguments, but 'args' must be defined.
+Beyond that, if any arguments are passed in, npm will politely warn that it
+does not take positional arguments.
+
+If the 'silent' parameter is set to true, nothing will be output to the screen,
+but the shrinkwrap file will still be written.
+
+Finally, 'callback' is a function that will be called when the shrinkwrap has
+been saved.

doc/cli/install.md

@@ -14,7 +14,9 @@ npm-install(1) -- Install a package
 
 ## DESCRIPTION
 
-This command installs a package, and any packages that it depends on.
+This command installs a package, and any packages that it depends on. If the
+package has a shrinkwrap file, the installation of dependencies will be driven
+by that. See npm-shrinkwrap(1).
 
 A `package` is:
 
@@ -199,3 +201,4 @@ affects a real use-case, it will be investigated.
 * npm-folders(1)
 * npm-tag(1)
 * npm-rm(1)
+* npm-shrinkwrap(1)

doc/cli/shrinkwrap.md

@@ -0,0 +1,154 @@
+npm-shrinkwrap(1) -- Lock down dependency versions
+=====================================================
+
+## SYNOPSIS
+
+    npm shrinkwrap
+
+## DESCRIPTION
+
+This command locks down the versions of a package's dependencies so that you can
+control exactly which versions of each dependency will be used when your package
+is installed.
+
+By default, "npm install" recursively installs the target's dependencies (as
+specified in package.json), choosing the latest available version that satisfies
+the dependency's semver pattern. In some situations, particularly when shipping
+software where each change is tightly managed, it's desirable to fully specify
+each version of each dependency recursively so that subsequent builds and
+deploys do not inadvertently pick up newer versions of a dependency that satisfy
+the semver pattern. Specifying specific semver patterns in each dependency's
+package.json would facilitate this, but that's not always possible or desirable,
+as when another author owns the npm package. It's also possible to check
+dependencies directly into source control, but that may be undesirable for other
+reasons.
+
+As an example, consider package A:
+
+    {
+        "name": "A",
+        "version": "0.1.0"
+        "dependencies": {
+            "B": "<0.1.0"
+        }
+    }
+
+package B:
+
+    {
+        "name": "B",
+        "version": "0.0.1"
+        "dependencies": {
+            "C": "<0.1.0"
+        }
+    }
+
+and package C:
+
+    {
+        "name": "C
+        "version": "0.0.1"
+    }
+
+If these are the only versions of A, B, and C available in the registry, then
+a normal "npm install A" will install:
+
+    A@0.1.0
+        B@0.0.1
+            C@0.0.1
+
+However, if B@0.0.2 is published, then a fresh "npm install A" will install:
+
+    A@0.1.0
+        B@0.0.2
+           C@0.0.1
+
+assuming the new version did not modify B's dependencies. Of course, the new
+version of B could include a new version of C and any number of new
+dependencies. If such changes are undesirable, the author of A could specify a
+dependency on B@0.0.1. However, if A's author and B's author are not the same
+person, there's no way for A's author to say that he or she does not want to
+pull in newly published versions of C when B hasn't changed at all.
+
+In this case, A's author can use
+
+    # npm shrinkwrap
+
+This generates npm-shrinkwrap.json, which will look something like this:
+
+    {
+      "name": "A"
+      "version": "0.1.0"
+      "dependencies": {
+        "B": {
+          "version": "0.0.1"
+          "dependencies": {
+            "C": {
+              "version": "0.1.0"
+            }
+          }
+        }
+      }
+    }
+
+The shrinkwrap command has locked down the dependencies based on what's
+currently installed in node_modules.  When "npm install" installs a package with
+a npm-shrinkwrap.json file in the package root, the shrinkwrap file (rather than
+package.json files) completely drives the installation of that package and all
+of its dependencies (recursively).  So now the author publishes A@0.1.0, and
+subsequent installs of this package will use B@0.0.1 and C@0.1.0, regardless the
+dependencies and versions listed in A's, B's, and C's package.json files.
+
+
+### Using shrinkwrapped packages
+
+Using a shrinkwrapped package is no different than using any other package: you
+can "npm install" it by hand, or add a dependency to your package.json file and
+"npm install" it.
+
+### Building shrinkwrapped packages
+
+To shrinkwrap an existing package:
+
+1. Run "npm install" in the package root to install the current versions of all
+   dependencies.
+2. Validate that the package works as expected with these versions.
+3. Run "npm shrinkwrap", add npm-shrinkwrap.json to git, and publish your
+   package.
+
+To add or update a dependency in a shrinkwrapped package:
+
+1. Run "npm install" in the package root to install the current versions of all
+   dependencies.
+2. Add or update dependencies. "npm install" each new or updated package
+   individually and then update package.json.
+3. Validate that the package works as expected with the new dependencies.
+4. Run "npm shrinkwrap", commit the new npm-shrinkwrap.json, and publish your
+   package.
+
+You can use npm-outdated(1) to view dependencies with newer versions available.
+
+### Other notes
+
+Since "npm shrinkwrap" uses the locally installed packages to construct the
+shrinkwrap file, devDependencies will be included if and only if you've
+installed them already when you make the shrinkwrap.
+
+A shrinkwrap file must be consistent with the package's package.json file. "npm
+shrinkwrap" will fail if required dependencies are not already installed, since
+that would result in a shrinkwrap that wouldn't actually work. Similarly, the
+command will fail if there are extraneous packages (not referenced by
+package.json), since that would indicate that package.json is not correct.
+
+If shrinkwrapped package A depends on shrinkwrapped package B, B's shrinkwrap
+will not be used as part of the installation of A. However, because A's
+shrinkwrap is constructed from a valid installation of B and recursively
+specifies all dependencies, the contents of B's shrinkwrap will implicitly be
+included in A's shrinkwrap.
+
+
+## SEE ALSO
+
+* npm-install(1)
+* npm-json(1)
+* npm-list(1)

lib/install.js

@@ -20,7 +20,9 @@ install.usage = "npm install <tarball file>"
               + "\nnpm install <pkg>@<version>"
               + "\nnpm install <pkg>@<version range>"
               + "\n\nCan specify one or more: npm install ./foo.tgz bar@stable /some/folder"
-              + "\nInstalls dependencies in ./package.json if no argument supplied"
+              + "\nIf no argument is supplied and ./npm-shrinkwrap.json is "
+              + "\npresent, installs dependencies specified in the shrinkwrap."
+              + "\nOtherwise, installs dependencies from ./package.json."
 
 install.completion = function (opts, cb) {
   // install can complete to a folder with a package.json, or any package.
@@ -109,7 +111,8 @@ function install (args, cb_) {
     // or install current folder globally
     if (!args.length) {
       if (npm.config.get("global")) args = ["."]
-      else return readJson( path.resolve(where, "package.json")
+      else return readDependencies( null
+                          , where
                           , { dev: !npm.config.get("production") }
                           , function (er, data) {
         if (er) return log.er(cb, "Couldn't read dependencies.")(er)
@@ -123,7 +126,7 @@ function install (args, cb_) {
             , parsed = url.parse(target.replace(/^git\+/, "git"))
           target = dep + "@" + target
           return target
-        }), where, family, ancestors, false, data, cb)
+        }), where, family, ancestors, null, false, data, cb)
       })
     }
 
@@ -135,7 +138,68 @@ function install (args, cb_) {
         , ancestors = {}
       if (data) family[data.name] = ancestors[data.name] = data.version
       var fn = npm.config.get("global") ? installMany : installManyTop
-      fn(args, where, family, ancestors, true, data, cb)
+      fn(args, where, family, ancestors, null, true, data, cb)
+    })
+  })
+}
+
+// reads dependencies for the package at "where". There are several cases,
+// depending on our current state and the package's configuration:
+//
+// 1. If "wrap" is specified, then it's assumed we're processing a package
+//    underneath a shrinkwrap, so dependencies are read directly from the
+//    shrinkwrap.
+// 2. Otherwise, if an npm-shrinkwrap.json file is present, dependencies are
+//    read from there.
+// 3. Otherwise, dependencies come from package.json.
+//
+// Regardless of which case we fall into, "cb" is invoked with a first argument
+// describing the full package (as though readJson had been used) but with
+// "dependencies" read as described above. The second argument to "cb" is the
+// shrinkwrap to use in processing this package's dependencies, which may be
+// "wrap" (in case 1) or a new shrinkwrap (in case 2).
+function readDependencies (wrap, where, opts, cb)
+{
+  readJson( path.resolve(where, "package.json")
+             , opts
+             , function (er, data) {
+    if (er)  return cb(er)
+
+    if (wrap) {
+      log.verbose([where, wrap], "readDependencies: using existing wrap")
+      var rv = {}
+      for (var key in data)
+        rv[key] = data[key]
+      rv["dependencies"] = {}
+      for (key in wrap)
+        rv["dependencies"][key] = wrap[key]["version"]
+      log.verbose([rv["dependencies"]], "readDependencies: returned deps")
+      return cb(null, rv, wrap)
+    }
+
+    var wrapfile = path.resolve(where, "npm-shrinkwrap.json")
+
+    fs.readFile(wrapfile, function (er, wrapjson) {
+      if (er) {
+        log.verbose("readDependencies: using package.json deps")
+        return cb(null, data, null)
+      }
+
+      try {
+        var newwrap = JSON.parse(wrapjson)
+      } catch (ex) {
+        return cb(ex)
+      }
+
+      log.info("using shrinkwrap file "+wrapfile)
+      var rv = {}
+      for (var key in data)
+        rv[key] = data[key]
+      rv["dependencies"] = {}
+      for (key in newwrap["dependencies"])
+        rv["dependencies"][key] = newwrap["dependencies"][key]["version"]
+      log.verbose([rv["dependencies"]], "readDependencies: returned deps")
+      return cb(null, rv, newwrap["dependencies"])
     })
   })
 }
@@ -255,7 +319,8 @@ function treeify (installed) {
 
 // just like installMany, but also add the existing packages in
 // where/node_modules to the family object.
-function installManyTop (what, where, family, ancestors, explicit, parent, cb_) {
+function installManyTop (what, where, family, ancestors, unused, explicit, parent,
+  cb_) {
 
   function cb (er, d) {
     if (explicit || er) return cb_(er, d)
@@ -286,7 +351,8 @@ function installManyTop_ (what, where, family, ancestors, explicit, parent, cb)
             : []
 
   fs.readdir(nm, function (er, pkgs) {
-    if (er) return installMany(what, where, family, ancestors, explicit, parent, cb)
+    if (er) return installMany(what, where, family, ancestors, null, explicit,
+      parent, cb)
     pkgs = pkgs.filter(function (p) {
       return !p.match(/^[\._-]/)
           && (!explicit || names.indexOf(p) === -1)
@@ -304,26 +370,38 @@ function installManyTop_ (what, where, family, ancestors, explicit, parent, cb)
       packages.forEach(function (p) {
         family[p[0]] = p[1]
       })
-      return installMany(what, where, family, ancestors, explicit, parent, cb)
+      return installMany(what, where, family, ancestors, null, explicit, parent,
+        cb)
     })
   })
 }
 
-function installMany (what, where, family, ancestors, explicit, parent, cb) {
-  // 'npm install foo' should install the version of foo
-  // that satisfies the dep in the current folder.
-  // This will typically return immediately, since we already read
-  // this file family, and it'll be cached.
-  readJson(path.resolve(where, "package.json"), function (er, data) {
+function installMany (what, where, family, ancestors, oldwrap, explicit, parent,
+  cb) {
+  // readDependencies takes care of figuring out whether the list of
+  // dependencies we'll iterate below comes from an existing shrinkwrap from a
+  // parent level, a new shrinkwrap at this level, or package.json at this
+  // level, as well as which shrinkwrap (if any) our dependencies should use.
+  readDependencies(oldwrap, where, {}, function (er, data, wrap) {
     if (er) data = {}
 
-    d = data.dependencies || {}
     var parent = data
 
+    var d = data["dependencies"] || {}
+
+    // if we're explicitly installing "what" into "where", then the shrinkwrap
+    // for "where" doesn't apply. This would be the case if someone were adding
+    // a new package to a shrinkwrapped package. (data.dependencies will not be
+    // used here except to indicate what packages are already present, so
+    // there's no harm in using that.)
+    if (explicit)
+      wrap = null
+
     // what is a list of things.
     // resolve each one.
     asyncMap( what
-            , targetResolver(where, family, ancestors, explicit, d, parent)
+            , targetResolver(where, family, ancestors, wrap, explicit, d,
+              parent)
             , function (er, targets) {
 
       if (er) return cb(er)
@@ -343,13 +421,15 @@ function installMany (what, where, family, ancestors, explicit, parent, cb) {
       })
       asyncMap(targets, function (target, cb) {
         log(target._id, "installOne")
-        installOne(target, where, newPrev, newAnc, parent, cb)
+        var newWrap = wrap ? wrap[target.name]["dependencies"] || {} : null
+        installOne(target, where, newPrev, newAnc, newWrap, parent, cb)
       }, cb)
     })
   })
 }
 
-function targetResolver (where, family, ancestors, explicit, deps, parent) {
+function targetResolver (where, family, ancestors, wrap, explicit, deps,
+  parent) {
   var alreadyInstalledManually = explicit ? [] : null
     , nm = path.resolve(where, "node_modules")
 
@@ -381,11 +461,29 @@ function targetResolver (where, family, ancestors, explicit, deps, parent) {
       return cb(null, [])
     }
 
-    if (family[what] && semver.satisfies(family[what], deps[what] || "")) {
-      return cb(null, [])
+    // check for a version installed higher in the tree.
+    // If installing from a shrinkwrap, it must match exactly.
+    if (family[what]) {
+      if (wrap && wrap[what]["version"] == family[what]) {
+        log.verbose("using existing "+what+" (matches shrinkwrap)")
+        return cb(null, [])
+      }
+
+      if (!wrap && semver.satisfies(family[what], deps[what] || "")) {
+        log.verbose("using existing "+what+" (no shrinkwrap)")
+        return cb(null, [])
+      }
     }
 
-    if (deps[what]) {
+    if (wrap) {
+      name = what.split(/@/).shift()
+      if (wrap[name]) {
+        log.verbose("shrinkwrap: resolving "+what+" to "+wrap[name]["version"])
+        what = name + "@" + wrap[name]["version"]
+      } else {
+        log.verbose("shrinkwrap: skipping "+what+" (not in shrinkwrap)")
+      }
+    } else if (deps[what]) {
       what = what + "@" + deps[what]
     }
 
@@ -405,14 +503,14 @@ function targetResolver (where, family, ancestors, explicit, deps, parent) {
 
 // we've already decided to install this.  if anything's in the way,
 // then uninstall it first.
-function installOne (target, where, family, ancestors, parent, cb) {
+function installOne (target, where, family, ancestors, wrap, parent, cb) {
   // the --link flag makes this a "link" command if it's at the
   // the top level.
   if (where === npm.prefix && npm.config.get("link")
       && !npm.config.get("global")) {
-    return localLink(target, where, family, ancestors, parent, cb)
+    return localLink(target, where, family, ancestors, wrap, parent, cb)
   }
-  installOne_(target, where, family, ancestors, parent, cb)
+  installOne_(target, where, family, ancestors, wrap, parent, cb)
 }
 
 function localLink (target, where, family, ancestors, parent, cb) {
@@ -440,7 +538,7 @@ function localLink (target, where, family, ancestors, parent, cb) {
 
     } else {
       log.verbose(target._id, "install locally (no link)")
-      installOne_(target, where, family, ancestors, parent, cb)
+      installOne_(target, where, family, ancestors, wrap, parent, cb)
     }
   })
 }
@@ -464,7 +562,7 @@ function resultList (target, where, parentId) {
          , parentId && prettyWhere ]
 }
 
-function installOne_ (target, where, family, ancestors, parent, cb) {
+function installOne_ (target, where, family, ancestors, wrap, parent, cb) {
   var nm = path.resolve(where, "node_modules")
     , targetFolder = path.resolve(nm, target.name)
     , prettyWhere = relativize(where, process.cwd() + "/x")
@@ -475,7 +573,7 @@ function installOne_ (target, where, family, ancestors, parent, cb) {
     ( [ [checkEngine, target]
       , [checkCycle, target, ancestors]
       , [checkGit, targetFolder]
-      , [write, target, targetFolder, family, ancestors] ]
+      , [write, target, targetFolder, family, ancestors, wrap] ]
     , function (er, d) {
         log.verbose(target._id, "installOne cb")
         if (er) return cb(er)
@@ -559,7 +657,7 @@ function checkGit_ (folder, cb) {
   })
 }
 
-function write (target, targetFolder, family, ancestors, cb_) {
+function write (target, targetFolder, family, ancestors, wrap, cb_) {
   var up = npm.config.get("unsafe-perm")
     , user = up ? null : npm.config.get("user")
     , group = up ? null : npm.config.get("group")
@@ -586,23 +684,29 @@ function write (target, targetFolder, family, ancestors, cb_) {
     // up until this point, since we really don't care about it.
     , function (er) {
       if (er) return cb(er)
-      var deps = Object.keys(target.dependencies || {})
-      installMany(deps.filter(function (d) {
-        // prefer to not install things that are satisfied by
-        // something in the "family" list.
-        return !semver.satisfies(family[d], target.dependencies[d])
-      }).map(function (d) {
-        var t = target.dependencies[d]
-          , parsed = url.parse(t.replace(/^git\+/, "git"))
-        t = d + "@" + t
-        return t
-      }), targetFolder, family, ancestors, false, target, function (er, d) {
-        log.verbose(targetFolder, "about to build")
-        if (er) return cb(er)
-        npm.commands.build( [targetFolder]
-                          , npm.config.get("global")
-                          , true
-                          , function (er) { return cb(er, d) })
+
+      // before continuing to installing dependencies, check for a shrinkwrap.
+      readDependencies(wrap, targetFolder, {}, function (er, data, wrap) {
+        var deps = Object.keys(data.dependencies || {})
+        installMany(deps.filter(function (d) {
+          // prefer to not install things that are satisfied by
+          // something in the "family" list, unless we're installing
+          // from a shrinkwrap.
+          return wrap || !semver.satisfies(family[d], data.dependencies[d])
+        }).map(function (d) {
+          var t = data.dependencies[d]
+            , parsed = url.parse(t.replace(/^git\+/, "git"))
+          t = d + "@" + t
+          return t
+        }), targetFolder, family, ancestors, wrap, false, target,
+          function (er, d) {
+            log.verbose(targetFolder, "about to build")
+            if (er) return cb(er)
+            npm.commands.build( [targetFolder]
+                              , npm.config.get("global")
+                              , true
+                              , function (er) { return cb(er, d) })
+        })
       })
     } )
 }

lib/npm.js

@@ -138,6 +138,7 @@ var commandCache = {}
               , "unpublish"
               , "owner"
               , "deprecate"
+              , "shrinkwrap"
 
               , "help"
               , "help-search"

lib/shrinkwrap.js

@@ -0,0 +1,84 @@
+// emit JSON describing versions of all packages currently installed (for later
+// use with shrinkwrap install)
+
+module.exports = exports = shrinkwrap
+
+var npm = require("./npm.js")
+  , output = require("./utils/output.js")
+  , log = require("./utils/log.js")
+  , fs = require('fs')
+  , path = require('path')
+
+shrinkwrap.usage = "npm shrinkwrap"
+
+function shrinkwrap (args, silent, cb) {
+  if (typeof cb !== "function") cb = silent, silent = false
+
+  if (args.length) {
+    log.warn("shrinkwrap doesn't take positional args.")
+  }
+
+  npm.commands.ls([], true, function (er, pkginfo) {
+    if (er) return cb(er)
+
+    var wrapped = {}
+    var nerr
+
+    if (pkginfo['name'])
+      wrapped['name'] = pkginfo['name']
+
+    nerr = shrinkwrapPkg(log, pkginfo['name'], pkginfo, wrapped)
+    if (nerr > 0)
+      return cb(new Error('failed with ' + nerr + ' errors'))
+
+    // leave the version field out of the top-level, since it's not used and
+    // could only be confusing if it gets out of date.
+    delete wrapped['version']
+
+    fs.writeFile( path.join(process.cwd(), "npm-shrinkwrap.json")
+                , new Buffer(JSON.stringify(wrapped, null, 2) + "\n")
+                , function (er) {
+      if (er) return cb(er)
+      output.write("wrote npm-shrinkwrap.json", function (er) {
+        cb(er, wrapped)
+      })
+    })
+  })
+}
+
+function shrinkwrapPkg (log, pkgname, pkginfo, rv) {
+  var pkg, dep, nerr
+
+  if (typeof (pkginfo) == 'string') {
+    log.error('required dependency not installed: ' + pkgname + '@' + pkginfo)
+    return (1)
+  }
+
+  if ('version' in pkginfo)
+    rv['version'] = pkginfo['version']
+
+  if (Object.keys(pkginfo['dependencies']).length === 0)
+    return (0)
+
+  rv['dependencies'] = {}
+  nerr = 0
+
+  for (pkg in pkginfo['dependencies']) {
+    dep = pkginfo['dependencies'][pkg]
+    rv['dependencies'][pkg] = {}
+    nerr += shrinkwrapPkg(log, pkg, dep, rv['dependencies'][pkg])
+
+    // package.json must be consistent with the shrinkwrap bundle
+    if (dep['extraneous']) {
+      log.error('package is extraneous: ' + pkg + '@' + dep['version'])
+      nerr++
+    }
+
+    if (dep['invalid']) {
+      log.error('package is invalid: ' + pkg)
+      nerr++
+    }
+  }
+
+  return (nerr)
+}