diff options
author | isaacs <i@izs.me> | 2012-06-28 19:08:32 -0700 |
---|---|---|
committer | isaacs <i@izs.me> | 2012-06-28 19:08:32 -0700 |
commit | c721604d254f57eefef715d4bd83329fac0750c7 (patch) | |
tree | 906a4b7bd1c3d6991293ca720cf8a7858efb43fd /deps/npm/lib | |
parent | f2a9ed487369ab7222522e1097708550adbe165c (diff) | |
download | node-new-c721604d254f57eefef715d4bd83329fac0750c7.tar.gz |
npm: Upgrade to 1.1.33
Support for parallel use of the cache folder
Retry on registry timeouts or network failures
Reduce 'engines' failures to a warning
Use new zsh completion if aviailable
Diffstat (limited to 'deps/npm/lib')
-rw-r--r-- | deps/npm/lib/cache.js | 274 | ||||
-rw-r--r-- | deps/npm/lib/install.js | 16 | ||||
-rw-r--r-- | deps/npm/lib/npm.js | 21 | ||||
-rw-r--r-- | deps/npm/lib/utils/cmd-shim.js | 2 | ||||
-rwxr-xr-x | deps/npm/lib/utils/completion.sh | 15 | ||||
-rw-r--r-- | deps/npm/lib/utils/config-defs.js | 19 | ||||
-rw-r--r-- | deps/npm/lib/utils/fetch.js | 43 | ||||
-rw-r--r-- | deps/npm/lib/utils/link.js | 12 |
8 files changed, 291 insertions, 111 deletions
diff --git a/deps/npm/lib/cache.js b/deps/npm/lib/cache.js index 399e903c57..c6ca8b477a 100644 --- a/deps/npm/lib/cache.js +++ b/deps/npm/lib/cache.js @@ -1,6 +1,30 @@ // XXX lib/utils/tar.js and this file need to be rewritten. +// URL-to-cache folder mapping: +// : -> ! +// @ -> _ +// http://registry.npmjs.org/foo/version -> cache/http!/... +// + /* +fetching a url: +1. Check for url in inFlightUrls. If present, add cb, and return. +2. create inFlightURL list +3. Acquire lock at {cache}/{sha(url)}.lock + retries = {cache-lock-retries, def=3} + stale = {cache-lock-stale, def=30000} + wait = {cache-lock-wait, def=100} +4. if lock can't be acquired, then fail +5. fetch url, clear lock, call cbs + +cache folders: +1. urls: http!/server.com/path/to/thing +2. c:\path\to\thing: file!/c!/path/to/thing +3. /path/to/thing: file!/path/to/thing +4. git@ private: git_github.com!isaacs/npm +5. git://public: git!/github.com/isaacs/npm +6. git+blah:// git-blah!/server.com/foo/bar + adding a folder: 1. tar into tmp/random/package.tgz 2. untar into tmp/random/contents/package, stripping one dir piece @@ -49,6 +73,9 @@ var mkdir = require("mkdirp") , fileCompletion = require("./utils/completion/file-completion.js") , url = require("url") , chownr = require("chownr") + , lockFile = require("lockfile") + , crypto = require("crypto") + , retry = require("retry") cache.usage = "npm cache add <tarball file>" + "\nnpm cache add <folder>" @@ -238,10 +265,26 @@ function add (args, cb) { default: // if we have a name and a spec, then try name@spec // if not, then try just spec (which may try name@"" if not found) - return name ? addNamed(name, spec, cb) : addLocal(spec, cb) + if (name) { + addNamed(name, spec, cb) + } else { + addLocal(spec, cb) + } } } +function fetchAndShaCheck (u, tmp, shasum, cb) { + fetch(u, tmp, function (er, response) { + if (er) { + log.error("fetch failed", u) + return cb(er, response) + } + if (!shasum) return cb() + // validate that the url we just downloaded matches the expected shasum. + sha.check(tmp, shasum, cb) + }) +} + // Only have a single download action at once for a given url // additional calls stack the callbacks. var inFlightURLs = {} @@ -255,29 +298,48 @@ function addRemoteTarball (u, shasum, name, cb_) { if (iF.length > 1) return function cb (er, data) { - var c - while (c = iF.shift()) c(er, data) - delete inFlightURLs[u] + unlock(u, function () { + var c + while (c = iF.shift()) c(er, data) + delete inFlightURLs[u] + }) } - log.verbose("addRemoteTarball", [u, shasum]) - var tmp = path.join(npm.tmp, Date.now()+"-"+Math.random(), "tmp.tgz") - mkdir(path.dirname(tmp), function (er) { + lock(u, function (er) { if (er) return cb(er) - fetch(u, tmp, function (er) { - if (er) { - log.error("fetch failed", u) - return cb(er) - } - if (!shasum) return done() - // validate that the url we just downloaded matches the expected shasum. - sha.check(tmp, shasum, done) + + log.verbose("addRemoteTarball", [u, shasum]) + var tmp = path.join(npm.tmp, Date.now()+"-"+Math.random(), "tmp.tgz") + mkdir(path.dirname(tmp), function (er) { + if (er) return cb(er) + // Tuned to spread 3 attempts over about a minute. + // See formula at <https://github.com/tim-kos/node-retry>. + var operation = retry.operation + ( { retries: npm.config.get("fetch-retries") + , factor: npm.config.get("fetch-retry-factor") + , minTimeout: npm.config.get("fetch-retry-mintimeout") + , maxTimeout: npm.config.get("fetch-retry-maxtimeout") }) + + operation.attempt(function (currentAttempt) { + log.info("retry", "fetch attempt " + currentAttempt + + " at " + (new Date()).toLocaleTimeString()) + fetchAndShaCheck(u, tmp, shasum, function (er, response) { + // Only retry on 408, 5xx or no `response`. + var statusCode = response && response.statusCode + var statusRetry = !statusCode || (statusCode === 408 || statusCode >= 500) + if (er && statusRetry && operation.retry(er)) { + log.info("retry", "will retry, error on last attempt: " + er) + return + } + done(er) + }) + }) }) + function done (er) { + if (er) return cb(er) + addLocalTarball(tmp, name, cb) + } }) - function done (er) { - if (er) return cb(er) - addLocalTarball(tmp, name, cb) - } } // For now, this is kind of dumb. Just basically treat git as @@ -292,48 +354,54 @@ function addRemoteGit (u, parsed, name, cb_) { if (iF.length > 1) return function cb (er, data) { - var c - while (c = iF.shift()) c(er, data) - delete inFlightURLs[u] + unlock(u, function () { + var c + while (c = iF.shift()) c(er, data) + delete inFlightURLs[u] + }) } - // figure out what we should check out. - var co = parsed.hash && parsed.hash.substr(1) || "master" - // git is so tricky! - // if the path is like ssh://foo:22/some/path then it works, but - // it needs the ssh:// - // If the path is like ssh://foo:some/path then it works, but - // only if you remove the ssh:// - u = u.replace(/^git\+/, "") - .replace(/#.*$/, "") - - // ssh paths that are scp-style urls don't need the ssh:// - if (parsed.pathname.match(/^\/?:/)) { - u = u.replace(/^ssh:\/\//, "") - } + lock(u, function (er) { + if (er) return cb(er) - log.verbose("addRemoteGit", [u, co]) + // figure out what we should check out. + var co = parsed.hash && parsed.hash.substr(1) || "master" + // git is so tricky! + // if the path is like ssh://foo:22/some/path then it works, but + // it needs the ssh:// + // If the path is like ssh://foo:some/path then it works, but + // only if you remove the ssh:// + u = u.replace(/^git\+/, "") + .replace(/#.*$/, "") + + // ssh paths that are scp-style urls don't need the ssh:// + if (parsed.pathname.match(/^\/?:/)) { + u = u.replace(/^ssh:\/\//, "") + } - var tmp = path.join(npm.tmp, Date.now()+"-"+Math.random()) - mkdir(path.dirname(tmp), function (er) { - if (er) return cb(er) - exec( npm.config.get("git"), ["clone", u, tmp], null, false - , function (er, code, stdout, stderr) { - stdout = (stdout + "\n" + stderr).trim() - if (er) { - log.error("git clone " + u, stdout) - return cb(er) - } - log.verbose("git clone "+u, stdout) - exec( npm.config.get("git"), ["checkout", co], null, false, tmp + log.verbose("addRemoteGit", [u, co]) + + var tmp = path.join(npm.tmp, Date.now()+"-"+Math.random()) + mkdir(path.dirname(tmp), function (er) { + if (er) return cb(er) + exec( npm.config.get("git"), ["clone", u, tmp], null, false , function (er, code, stdout, stderr) { stdout = (stdout + "\n" + stderr).trim() if (er) { - log.error("git checkout " + co, stdout) + log.error("git clone " + u, stdout) return cb(er) } - log.verbose("git checkout " + co, stdout) - addLocalDirectory(tmp, cb) + log.verbose("git clone "+u, stdout) + exec( npm.config.get("git"), ["checkout", co], null, false, tmp + , function (er, code, stdout, stderr) { + stdout = (stdout + "\n" + stderr).trim() + if (er) { + log.error("git checkout " + co, stdout) + return cb(er) + } + log.verbose("git checkout " + co, stdout) + addLocalDirectory(tmp, cb) + }) }) }) }) @@ -343,8 +411,10 @@ function addRemoteGit (u, parsed, name, cb_) { // only have one request in flight for a given // name@blah thing. var inFlightNames = {} -function addNamed (name, x, cb_) { +function addNamed (name, x, data, cb_) { + if (typeof cb_ !== "function") cb_ = data, data = null log.verbose("addNamed", [name, x]) + var k = name + "@" + x if (!inFlightNames[k]) inFlightNames[k] = [] var iF = inFlightNames[k] @@ -352,19 +422,27 @@ function addNamed (name, x, cb_) { if (iF.length > 1) return function cb (er, data) { - var c - while (c = iF.shift()) c(er, data) - delete inFlightNames[k] + unlock(k, function () { + var c + while (c = iF.shift()) c(er, data) + delete inFlightNames[k] + }) } log.verbose("addNamed", [semver.valid(x), semver.validRange(x)]) - return ( null !== semver.valid(x) ? addNameVersion - : null !== semver.validRange(x) ? addNameRange - : addNameTag - )(name, x, cb) + lock(k, function (er, fd) { + if (er) return cb(er) + + var fn = ( null !== semver.valid(x) ? addNameVersion + : null !== semver.validRange(x) ? addNameRange + : addNameTag + ) + fn(name, x, data, cb) + }) } -function addNameTag (name, tag, cb) { +function addNameTag (name, tag, data, cb) { + if (typeof cb !== "function") cb = data, data = null log.info("addNameTag", [name, tag]) var explicit = true if (!tag) { @@ -378,10 +456,10 @@ function addNameTag (name, tag, cb) { if (data["dist-tags"] && data["dist-tags"][tag] && data.versions[data["dist-tags"][tag]]) { var ver = data["dist-tags"][tag] - return addNameVersion(name, ver, data.versions[ver], cb) + return addNamed(name, ver, data.versions[ver], cb) } if (!explicit && Object.keys(data.versions).length) { - return addNameRange(name, "*", data, cb) + return addNamed(name, "*", data, cb) } return cb(installTargetsError(tag, data)) }) @@ -390,12 +468,14 @@ function addNameTag (name, tag, cb) { function engineFilter (data) { var npmv = npm.version , nodev = npm.config.get("node-version") + , strict = npm.config.get("engine-strict") if (!nodev || npm.config.get("force")) return data Object.keys(data.versions || {}).forEach(function (v) { var eng = data.versions[v].engines if (!eng) return + if (!strict && !data.versions[v].engineStrict) return if (eng.node && !semver.satisfies(nodev, eng.node) || eng.npm && !semver.satisfies(npmv, eng.npm)) { delete data.versions[v] @@ -438,12 +518,12 @@ function addNameRange (name, range, data, cb) { function next_ () { log.silly("addNameRange", "versions" - , [data.name, Object.keys(data.versions)]) + , [data.name, Object.keys(data.versions || {})]) // if the tagged version satisfies, then use that. var tagged = data["dist-tags"][npm.config.get("tag")] if (tagged && data.versions[tagged] && semver.satisfies(tagged, range)) { - return addNameVersion(name, tagged, data.versions[tagged], cb) + return addNamed(name, tagged, data.versions[tagged], cb) } // find the max satisfying version. @@ -454,7 +534,7 @@ function addNameRange (name, range, data, cb) { // if we don't have a registry connection, try to see if // there's a cached copy that will be ok. - addNameVersion(name, ms, data.versions[ms], cb) + addNamed(name, ms, data.versions[ms], cb) } } @@ -573,24 +653,29 @@ function addLocal (p, name, cb_) { if (typeof cb_ !== "function") cb_ = name, name = "" function cb (er, data) { - if (er) { - // if it doesn't have a / in it, it might be a - // remote thing. - if (p.indexOf("/") === -1 && p.charAt(0) !== "." - && (process.platform !== "win32" || p.indexOf("\\") === -1)) { - return addNamed(p, "", cb_) + unlock(p, function () { + if (er) { + // if it doesn't have a / in it, it might be a + // remote thing. + if (p.indexOf("/") === -1 && p.charAt(0) !== "." + && (process.platform !== "win32" || p.indexOf("\\") === -1)) { + return addNamed(p, "", cb_) + } + log.error("addLocal", "Could not install %s", p) + return cb_(er) } - log.error("addLocal", "Could not install %s", p) - return cb_(er) - } - return cb_(er, data) + return cb_(er, data) + }) } - // figure out if this is a folder or file. - fs.stat(p, function (er, s) { + lock(p, function (er) { if (er) return cb(er) - if (s.isDirectory()) addLocalDirectory(p, name, cb) - else addLocalTarball(p, name, cb) + // figure out if this is a folder or file. + fs.stat(p, function (er, s) { + if (er) return cb(er) + if (s.isDirectory()) addLocalDirectory(p, name, cb) + else addLocalTarball(p, name, cb) + }) }) } @@ -847,3 +932,32 @@ function deprCheck (data) { log.warn("deprecated", "%s: %s", data._id, data.deprecated) } } + +function lockFileName (u) { + var c = u.replace(/[^a-zA-Z0-9]+/g, '-') + , h = crypto.createHash("sha1").update(u).digest("hex") + return path.resolve(npm.config.get("cache"), h + "-" + c + ".lock") +} + +var madeCache = false +function lock (u, cb) { + // the cache dir needs to exist already for this. + if (madeCache) then() + else mkdir(npm.config.get("cache"), function (er) { + if (er) return cb(er) + madeCache = true + then() + }) + function then () { + var opts = { stale: npm.config.get("cache-lock-stale") + , retries: npm.config.get("cache-lock-retries") + , wait: npm.config.get("cache-lock-wait") } + var lf = lockFileName(u) + log.verbose("lock", u, lf) + lockFile.lock(lf, opts, cb) + } +} + +function unlock (u, cb) { + lockFile.unlock(lockFileName(u), cb) +} diff --git a/deps/npm/lib/install.js b/deps/npm/lib/install.js index fc1c26aaf0..1aabb46019 100644 --- a/deps/npm/lib/install.js +++ b/deps/npm/lib/install.js @@ -710,15 +710,21 @@ function checkEngine (target, cb) { var npmv = npm.version , force = npm.config.get("force") , nodev = force ? null : npm.config.get("node-version") + , strict = npm.config.get("engine-strict") || target.engineStrict , eng = target.engines if (!eng) return cb() if (nodev && eng.node && !semver.satisfies(nodev, eng.node) || eng.npm && !semver.satisfies(npmv, eng.npm)) { - var er = new Error("Unsupported") - er.code = "ENOTSUP" - er.required = eng - er.pkgid = target._id - return cb(er) + if (strict) { + var er = new Error("Unsupported") + er.code = "ENOTSUP" + er.required = eng + er.pkgid = target._id + return cb(er) + } else { + log.warn( "engine", "%s: wanted: %j (current: %j)" + , target._id, eng, {node: nodev, npm: npm.version} ) + } } return cb() } diff --git a/deps/npm/lib/npm.js b/deps/npm/lib/npm.js index 5b257d6738..589eb17fca 100644 --- a/deps/npm/lib/npm.js +++ b/deps/npm/lib/npm.js @@ -57,13 +57,13 @@ try { npm.version = j.version npm.nodeVersionRequired = j.engines.node if (!semver.satisfies(process.version, j.engines.node)) { - log.error("unsupported version", ["" - ,"npm requires node version: "+j.engines.node - ,"And you have: "+process.version - ,"which is not satisfactory." - ,"" - ,"Bad things will likely happen. You have been warned." - ,""].join("\n")) + log.warn("unsupported version", ["" + ,"npm requires node version: "+j.engines.node + ,"And you have: "+process.version + ,"which is not satisfactory." + ,"" + ,"Bad things will likely happen. You have been warned." + ,""].join("\n")) } } catch (ex) { try { @@ -98,6 +98,7 @@ var commandCache = {} , "apihelp" : "help" , "login": "adduser" , "add-user": "adduser" + , "tst": "test" } , aliasNames = Object.keys(aliases) @@ -287,6 +288,10 @@ function load (npm, conf, cb) { , E404: npm.E404 , EPUBLISHCONFLICT: npm.EPUBLISHCONFLICT , log: log + , retries: npm.config.get("fetch-retries") + , retryFactor: npm.config.get("fetch-retry-factor") + , retryMinTimeout: npm.config.get("fetch-retry-mintimeout") + , retryMaxTimeout: npm.config.get("fetch-retry-maxtimeout") }) var umask = parseInt(conf.umask, 8) @@ -443,7 +448,7 @@ Object.defineProperty(npm, "cache", var tmpFolder Object.defineProperty(npm, "tmp", { get : function () { - if (!tmpFolder) tmpFolder = "npm-"+Date.now() + if (!tmpFolder) tmpFolder = "npm-" + process.pid return path.resolve(npm.config.get("tmp"), tmpFolder) } , enumerable : true diff --git a/deps/npm/lib/utils/cmd-shim.js b/deps/npm/lib/utils/cmd-shim.js index 828b633632..6fce0e03a4 100644 --- a/deps/npm/lib/utils/cmd-shim.js +++ b/deps/npm/lib/utils/cmd-shim.js @@ -62,7 +62,7 @@ function writeShim_ (from, to, prog, args, cb) { var shTarget = path.relative(path.dirname(to), from) , target = shTarget.split("/").join("\\") , longProg - , shProg = prog.split("\\").join("/") + , shProg = prog && prog.split("\\").join("/") , shLongProg shTarget = shTarget.split("\\").join("/") args = args || "" diff --git a/deps/npm/lib/utils/completion.sh b/deps/npm/lib/utils/completion.sh index f19046b198..d027590597 100755 --- a/deps/npm/lib/utils/completion.sh +++ b/deps/npm/lib/utils/completion.sh @@ -11,7 +11,7 @@ COMP_WORDBREAKS=${COMP_WORDBREAKS/=/} COMP_WORDBREAKS=${COMP_WORDBREAKS/@/} export COMP_WORDBREAKS -if complete &>/dev/null; then +if type complete &>/dev/null; then _npm_completion () { local si="$IFS" IFS=$'\n' COMPREPLY=($(COMP_CWORD="$COMP_CWORD" \ @@ -22,7 +22,18 @@ if complete &>/dev/null; then IFS="$si" } complete -F _npm_completion npm -elif compctl &>/dev/null; then +elif type compdef &>/dev/null; then + _npm_completion() { + si=$IFS + compadd -- $(COMP_CWORD=$((CURRENT-1)) \ + COMP_LINE=$BUFFER \ + COMP_POINT=0 \ + npm completion -- "${words[@]}" \ + 2>/dev/null) + IFS=$si + } + compdef _npm_completion npm +elif type compctl &>/dev/null; then _npm_completion () { local cword line point words si read -Ac words diff --git a/deps/npm/lib/utils/config-defs.js b/deps/npm/lib/utils/config-defs.js index bb277efdea..ee2c22f101 100644 --- a/deps/npm/lib/utils/config-defs.js +++ b/deps/npm/lib/utils/config-defs.js @@ -123,6 +123,11 @@ Object.defineProperty(exports, "defaults", {get: function () { , cache : process.platform === "win32" ? path.resolve(process.env.APPDATA || home || temp, "npm-cache") : path.resolve( home || temp, ".npm") + + , "cache-lock-stale": 60000 + , "cache-lock-retries": 10 + , "cache-lock-wait": 10000 + , "cache-max": Infinity , "cache-min": 0 @@ -132,8 +137,14 @@ Object.defineProperty(exports, "defaults", {get: function () { , description : true , dev : false , editor : osenv.editor() + , "engine-strict": false , force : false + , "fetch-retries": 2 + , "fetch-retry-factor": 10 + , "fetch-retry-mintimeout": 10000 + , "fetch-retry-maxtimeout": 60000 + , git: "git" , global : false @@ -207,6 +218,9 @@ exports.types = , browser : String , ca: [null, String] , cache : path + , "cache-lock-stale": Number + , "cache-lock-retries": Number + , "cache-lock-wait": Number , "cache-max": Number , "cache-min": Number , color : ["always", Boolean] @@ -215,7 +229,12 @@ exports.types = , description : Boolean , dev : Boolean , editor : String + , "engine-strict": Boolean , force : Boolean + , "fetch-retries": Number + , "fetch-retry-factor": Number + , "fetch-retry-mintimeout": Number + , "fetch-retry-maxtimeout": Number , git: String , global : Boolean , globalconfig : path diff --git a/deps/npm/lib/utils/fetch.js b/deps/npm/lib/utils/fetch.js index f69975ba60..6042abc528 100644 --- a/deps/npm/lib/utils/fetch.js +++ b/deps/npm/lib/utils/fetch.js @@ -25,17 +25,31 @@ function fetch (remote, local, headers, cb) { function fetch_ (remote, local, headers, cb) { var fstr = fs.createWriteStream(local, { mode : npm.modes.file }) + var response = null + var calledback = false fstr.on("error", function (er) { fs.close(fstr.fd, function () {}) - if (fstr._ERROR) return + if (calledback) return + calledback = true cb(fstr._ERROR = er) }) fstr.on("open", function () { - makeRequest(remote, fstr, headers) + var req = makeRequest(remote, fstr, headers) + req.on("response", function (res) { + log.http(res.statusCode, remote) + response = res + }) }) fstr.on("close", function () { - if (fstr._ERROR) return - cb() + if (calledback) return + calledback = true + if (response && response.statusCode && response.statusCode >= 400) { + var er = new Error(response.statusCode + " " + + require("http").STATUS_CODES[response.statusCode]) + cb(fstr._ERROR = er, response) + } else { + cb(null, response) + } }) } @@ -55,14 +69,15 @@ function makeRequest (remote, fstr, headers) { ? "https-proxy" : "proxy") - request({ url: remote - , proxy: proxy - , strictSSL: npm.config.get("strict-ssl") - , ca: remote.host === regHost ? npm.config.get("ca") : undefined - , headers: { "user-agent": npm.config.get("user-agent") } - , onResponse: onResponse }).pipe(fstr) - function onResponse (er, res) { - if (er) return fstr.emit("error", er) - log.http(res.statusCode, remote.href) - } + var opts = { url: remote + , proxy: proxy + , strictSSL: npm.config.get("strict-ssl") + , ca: remote.host === regHost ? npm.config.get("ca") : undefined + , headers: { "user-agent": npm.config.get("user-agent") }} + var req = request(opts) + req.on("error", function (er) { + fstr.emit("error", er) + }) + req.pipe(fstr) + return req; } diff --git a/deps/npm/lib/utils/link.js b/deps/npm/lib/utils/link.js index e8cae155e5..9e01d82e61 100644 --- a/deps/npm/lib/utils/link.js +++ b/deps/npm/lib/utils/link.js @@ -20,10 +20,20 @@ function link (from, to, gently, cb) { if (typeof cb !== "function") cb = gently, gently = null if (npm.config.get("force")) gently = false + to = path.resolve(to) + var target = from = path.resolve(from) + if (process.platform !== "win32") { + // junctions on windows must be absolute + target = path.relative(path.dirname(to), from) + // if there is no folder in common, then it will be much + // longer, and using a relative link is dumb. + if (target.length >= from.length) target = from + } + chain ( [ [fs, "stat", from] , [rm, to, gently] , [mkdir, path.dirname(to)] - , [fs, "symlink", from, to, "junction"] ] + , [fs, "symlink", target, to, "junction"] ] , cb) } |