diff options
Diffstat (limited to 'deps/npm/lib/utils/tar.js')
| -rw-r--r-- | deps/npm/lib/utils/tar.js | 628 |
1 files changed, 114 insertions, 514 deletions
diff --git a/deps/npm/lib/utils/tar.js b/deps/npm/lib/utils/tar.js index f315bbf961..415eb7f9e2 100644 --- a/deps/npm/lib/utils/tar.js +++ b/deps/npm/lib/utils/tar.js @@ -1,115 +1,39 @@ -// XXX lib/cache.js and this file need to be rewritten. - // commands for packing and unpacking tarballs // this file is used by lib/cache.js var npm = require("../npm.js") , fs = require("graceful-fs") - , exec = require("./exec.js") - , find = require("./find.js") - , mkdir = require("./mkdir-p.js") - , asyncMap = require("slide").asyncMap , path = require("path") , log = require("./log.js") - , uidNumber = require("./uid-number.js") + , uidNumber = require("uid-number") , rm = require("rimraf") , readJson = require("./read-json.js") , relativize = require("./relativize.js") , cache = require("../cache.js") - , excludes = require("./excludes.js") , myUid = process.getuid && process.getuid() , myGid = process.getgid && process.getgid() , tar = require("tar") , zlib = require("zlib") , fstream = require("fstream") + , Packer = require("fstream-npm") + +if (process.env.SUDO_UID && myUid === 0) { + if (!isNaN(process.env.SUDO_UID)) myUid = +process.env.SUDO_UID + if (!isNaN(process.env.SUDO_GID)) myGid = +process.env.SUDO_GID +} exports.pack = pack exports.unpack = unpack -exports.makeList = makeList function pack (targetTarball, folder, pkg, dfc, cb) { + log.verbose([targetTarball, folder], "tar.pack") if (typeof cb !== "function") cb = dfc, dfc = true - folder = path.resolve(folder) - - log.verbose(folder, "pack") - - if (typeof pkg === "function") { - cb = pkg, pkg = null - return readJson(path.resolve(folder, "package.json"), function (er, pkg) { - if (er) return log.er(cb, "Couldn't find package.json in "+folder)(er) - pack(targetTarball, folder, pkg, dfc, cb) - }) - } - log.verbose(folder+" "+targetTarball, "pack") - var parent = path.dirname(folder) - , addFolder = path.basename(folder) - - var confEx = npm.config.get("ignore") - log.silly(folder, "makeList") - makeList(folder, pkg, dfc, function (er, files, cleanup) { - if (er) return cb(er) - // log.silly(files, "files") - return packFiles(targetTarball, parent, files, pkg, function (er) { - if (!cleanup || !cleanup.length) return cb(er) - // try to be a good citizen, even/especially in the event of failure. - cleanupResolveLinkDep(cleanup, function (er2) { - if (er || er2) { - if (er) log(er, "packing tarball") - if (er2) log(er2, "while cleaning up resolved deps") - } - return cb(er || er2) - }) - }) - }) -} - -function packFiles (targetTarball, parent, files, pkg, cb_) { - - var p - - files = files.map(function (f) { - p = f.split(/\/|\\/)[0] - return path.resolve(parent, f) - }) - - parent = path.resolve(parent, p) - - var called = false - function cb (er) { - if (called) return - called = true - cb_(er) - } log.verbose(targetTarball, "tarball") - log.verbose(parent, "parent") - fstream.Reader({ type: "Directory" - , path: parent - , filter: function () { - // files should *always* get into tarballs - // in a user-writable state, even if they're - // being installed from some wackey vm-mounted - // read-only filesystem. - this.props.mode = this.props.mode | 0200 - var inc = -1 !== files.indexOf(this.path) + log.verbose(folder, "folder") + new Packer({ path: folder, type: "Directory", isDirectory: true }) + .on("error", log.er(cb, "error reading "+folder)) - // WARNING! Hackety hack! - // XXX Fix this in a better way. - // Rename .gitignore to .npmignore if there is not a - // .npmignore file there already, the better to lock - // down installed packages with git for deployment. - if (this.basename === ".gitignore") { - if (this.parent._entries.indexOf(".npmignore") !== -1) { - return false - } - var d = path.dirname(this.path) - this.basename = ".npmignore" - this.path = path.join(d, ".npmignore") - } - return inc - } - }) - .on("error", log.er(cb, "error reading "+parent)) // By default, npm includes some proprietary attributes in the // package tarball. This is sane, and allowed by the spec. // However, npm *itself* excludes these from its own package, @@ -121,11 +45,14 @@ function packFiles (targetTarball, parent, files, pkg, cb_) { .on("error", log.er(cb, "gzip error "+targetTarball)) .pipe(fstream.Writer({ type: "File", path: targetTarball })) .on("error", log.er(cb, "Could not write "+targetTarball)) - .on("close", cb) + .on("close", function () { + cb() + }) } function unpack (tarball, unpackTarget, dMode, fMode, uid, gid, cb) { + log.verbose(tarball, "unpack") if (typeof cb !== "function") cb = gid, gid = null if (typeof cb !== "function") cb = uid, uid = null if (typeof cb !== "function") cb = fMode, fMode = npm.modes.file @@ -138,466 +65,139 @@ function unpack (tarball, unpackTarget, dMode, fMode, uid, gid, cb) { } function unpack_ ( tarball, unpackTarget, dMode, fMode, uid, gid, cb ) { - // If the desired target is /path/to/foo, - // then unpack into /path/to/.foo.npm/{something} - // rename that to /path/to/foo, and delete /path/to/.foo.npm var parent = path.dirname(unpackTarget) , base = path.basename(unpackTarget) - , tmp = path.resolve(parent, "___" + base + ".npm") - mkdir(tmp, dMode || npm.modes.exec, uid, gid, function (er) { - log.verbose([uid, gid], "unpack_ uid, gid") - log.verbose(unpackTarget, "unpackTarget") - if (er) return log.er(cb, "Could not create "+tmp)(er) - // cp the gzip of the tarball, pipe the stdout into tar's stdin + rm(unpackTarget, function (er) { + if (er) return cb(er) + // gzip {tarball} --decompress --stdout \ // | tar -mvxpf - --strip-components=1 -C {unpackTarget} - gunzTarPerm( tarball, tmp + gunzTarPerm( tarball, unpackTarget , dMode, fMode , uid, gid , function (er, folder) { if (er) return cb(er) - log.verbose(folder, "gunzed") - - rm(unpackTarget, function (er) { - if (er) return cb(er) - log.verbose(unpackTarget, "rm'ed") - - moveIntoPlace(folder, unpackTarget, function (er) { - if (er) return cb(er) - log.verbose([folder, unpackTarget], "renamed") - // curse you, nfs! It will lie and tell you that the - // mv is done, when in fact, it isn't. In theory, - // reading the file should cause it to wait until it's done. - readJson( path.resolve(unpackTarget, "package.json") - , function (er, data) { - // now we read the json, so we know it's there. - rm(tmp, function (er2) { cb(er || er2, data) }) - }) - }) - }) + readJson(path.resolve(folder, "package.json"), cb) }) }) } -// on Windows, A/V software can lock the directory, causing this -// to fail with an EACCES. Try again on failure, for up to 1 second. -// XXX Fix this by not unpacking into a temp directory, instead just -// renaming things on the way out of the tarball. -function moveIntoPlace (folder, unpackTarget, cb) { - var start = Date.now() - fs.rename(folder, unpackTarget, function CB (er) { - if (er - && process.platform === "win32" - && er.code === "EACCES" - && Date.now() - start < 1000) { - return fs.rename(folder, unpackTarget, CB) - } - cb(er) - }) -} - -function gunzTarPerm (tarball, tmp, dMode, fMode, uid, gid, cb) { +function gunzTarPerm (tarball, target, dMode, fMode, uid, gid, cb_) { if (!dMode) dMode = npm.modes.exec if (!fMode) fMode = npm.modes.file log.silly([dMode.toString(8), fMode.toString(8)], "gunzTarPerm modes") - fs.createReadStream(tarball) - .on("error", log.er(cb, "error reading "+tarball)) - .pipe(zlib.Unzip()) - .on("error", log.er(cb, "unzip error "+tarball)) - .pipe(tar.Extract({ type: "Directory", path: tmp })) - .on("error", log.er(cb, "Failed unpacking "+tarball)) - .on("close", afterUntar) - - // - // XXX Do all this in an Extract filter. - // - function afterUntar (er) { - log.silly(er, "afterUntar") - // if we're not doing ownership management, - // then we're done now. - if (er) return log.er(cb, "Failed unpacking "+tarball)(er) - - // HACK skip on windows - if (npm.config.get("unsafe-perm") && process.platform !== "win32") { - uid = process.getuid() - gid = process.getgid() - if (uid === 0) { - if (process.env.SUDO_UID) uid = +process.env.SUDO_UID - if (process.env.SUDO_GID) gid = +process.env.SUDO_GID - } - } - - if (process.platform === "win32") { - return fs.readdir(tmp, function (er, files) { - files = files.filter(function (f) { - return f && f.indexOf("\0") === -1 - }) - cb(er, files && path.resolve(tmp, files[0])) - }) - } - - find(tmp, function (f) { - return f !== tmp - }, function (er, files) { - if (er) return cb(er) - asyncMap(files, function (f, cb) { - f = path.resolve(f) - log.silly(f, "asyncMap in gTP") - fs.lstat(f, function (er, stat) { - - if (er || stat.isSymbolicLink()) return cb(er) - if (typeof uid === "number" && typeof gid === "number") { - fs.chown(f, uid, gid, chown) - } else chown() - - function chown (er) { - if (er) return cb(er) - var mode = stat.isDirectory() ? dMode : fMode - , oldMode = stat.mode & 0777 - , newMode = (oldMode | mode) & (~npm.modes.umask) - if (mode && newMode !== oldMode) { - log.silly(newMode.toString(8), "chmod "+path.basename(f)) - fs.chmod(f, newMode, cb) - } else cb() - } - }) - }, function (er) { - - if (er) return cb(er) - if (typeof myUid === "number" && typeof myGid === "number") { - fs.chown(tmp, myUid, myGid, chown) - } else chown() - - function chown (er) { - if (er) return cb(er) - fs.readdir(tmp, function (er, folder) { - folder = folder && folder.filter(function (f) { - return f && !f.match(/^\._/) - }) - cb(er, folder && path.resolve(tmp, folder[0])) - }) - } - }) - }) + var cbCalled = false + function cb (er) { + if (cbCalled) return + cbCalled = true + cb_(er, target) } -} - -function makeList (dir, pkg, dfc, cb) { - if (typeof cb !== "function") cb = dfc, dfc = true - if (typeof cb !== "function") cb = pkg, pkg = null - dir = path.resolve(dir) - - if (!pkg.path) pkg.path = dir - - var name = path.basename(dir) - - // since this is a top-level traversal, get the user and global - // exclude files, as well as the "ignore" config setting. - var confIgnore = npm.config.get("ignore").trim() - .split(/[\n\r\s\t]+/) - .filter(function (i) { return i.trim() }) - , userIgnore = npm.config.get("userignorefile") - , globalIgnore = npm.config.get("globalignorefile") - , userExclude - , globalExclude - - confIgnore.dir = dir - confIgnore.name = "confIgnore" - - var defIgnore = ["build/"] - defIgnore.dir = dir - - // TODO: only look these up once, and cache outside this function - excludes.parseIgnoreFile( userIgnore, null, dir - , function (er, uex) { - if (er) return cb(er) - userExclude = uex - next() - }) - excludes.parseIgnoreFile( globalIgnore, null, dir - , function (er, gex) { - if (er) return cb(er) - globalExclude = gex - next() - }) + var fst = fs.createReadStream(tarball) - function next () { - if (!globalExclude || !userExclude) return - var exList = [ defIgnore, confIgnore, globalExclude, userExclude ] - - makeList_(dir, pkg, exList, dfc, function (er, files, cleanup) { - if (er) return cb(er) - var dirLen = dir.replace(/(\/|\\)$/, "").length + 1 - log.silly([dir, dirLen], "dir, dirLen") - files = files.map(function (file) { - return path.join(name, file.substr(dirLen)) - }) - return cb(null, files, cleanup) - }) + // figure out who we're supposed to be, if we're not pretending + // to be a specific user. + if (npm.config.get("unsafe-perm") && process.platform !== "win32") { + uid = myUid + gid = myGid } -} - -// Patterns ending in slashes will only match targets -// ending in slashes. To implement this, add a / to -// the filename iff it lstats isDirectory() -function readDir (dir, pkg, dfc, cb) { - fs.readdir(dir, function (er, files) { - if (er) return cb(er) - files = files.filter(function (f) { - return f && f.charAt(0) !== "/" && f.indexOf("\0") === -1 - }) - asyncMap(files, function (file, cb) { - fs.lstat(path.resolve(dir, file), function (er, st) { - if (er) return cb(null, []) - // if it's a directory, then tack "/" onto the name - // so that it can match dir-only patterns in the - // include/exclude logic later. - if (st.isDirectory()) return cb(null, file + "/") - - // if it's a symlink, then we need to do some more - // complex stuff for GH-691 - if (st.isSymbolicLink()) return readSymlink(dir, file, pkg, dfc, cb) - - // otherwise, just let it on through. - return cb(null, file) - }) - }, cb) - }) -} - -// just see where this link is pointing, and resolve relative paths. -function shallowReal (link, cb) { - link = path.resolve(link) - fs.readlink(link, function (er, t) { - if (er) return cb(er) - return cb(null, path.resolve(path.dirname(link), t), t) - }) -} - -function readSymlink (dir, file, pkg, dfc, cb) { - var isNM = dfc - && path.basename(dir) === "node_modules" - && path.dirname(dir) === pkg.path - // see if this thing is pointing outside of the package. - // external symlinks are resolved for deps, ignored for other things. - // internal symlinks are allowed through. - var df = path.resolve(dir, file) - shallowReal(df, function (er, r, target) { - if (er) return cb(null, []) // wtf? exclude file. - if (r.indexOf(dir) === 0) return cb(null, file) // internal - if (!isNM) return cb(null, []) // external non-dep - // now the fun stuff! - fs.realpath(df, function (er, resolved) { - if (er) return cb(null, []) // can't add it. - readJson(path.resolve(resolved, "package.json"), function (er) { - if (er) return cb(null, []) // not a package - resolveLinkDep(dir, file, resolved, target, pkg, function (er, f, c) { - cb(er, f, c) - }) - }) - }) - }) -} - -// put the link back the way it was. -function cleanupResolveLinkDep (cleanup, cb) { - // cut it out of the list, so that cycles will be broken. - if (!cleanup) return cb() - asyncMap(cleanup, function (d, cb) { - rm(d[1], function (er) { - if (er) return cb(er) - fs.symlink(d[0], d[1], cb) - }) - }, cb) -} - -function resolveLinkDep (dir, file, resolved, target, pkg, cb) { - // we've already decided that this is a dep that will be bundled. - // make sure the data reflects this. - var bd = pkg.bundleDependencies || pkg.bundledDependencies || [] - delete pkg.bundledDependencies - pkg.bundleDependencies = bd - var f = path.resolve(dir, file) - , cleanup = [[target, f, resolved]] + function extractEntry (entry) { + log.silly(entry.path, "extracting entry") + // never create things that are user-unreadable, + // or dirs that are user-un-listable. Only leads to headaches. + var originalMode = entry.mode = entry.mode || entry.props.mode + entry.mode = entry.mode | (entry.type === "Directory" ? dMode : fMode) + entry.mode = entry.mode & (~npm.modes.umask) + entry.props.mode = entry.mode + if (originalMode !== entry.mode) { + log.silly([entry.path, originalMode, entry.mode], "modified mode") + } - if (bd.indexOf(file) === -1) { - // then we don't do this one. - // just move the symlink out of the way. - return rm(f, function (er) { - cb(er, file, cleanup) - }) + // if there's a specific owner uid/gid that we want, then set that + if (process.platform !== "win32" && + typeof uid === "number" && + typeof gid === "number") { + entry.props.uid = entry.uid = uid + entry.props.gid = entry.gid = gid + } } - rm(f, function (er) { - if (er) return cb(er) - cache.add(resolved, function (er, data) { - if (er) return cb(er) - cache.unpack(data.name, data.version, f, function (er, data) { - if (er) return cb(er) - // now clear out the cache entry, since it's weird, probably. - // pass the cleanup object along so that the thing getting the - // list of files knows what to clean up afterwards. - cache.clean([data._id], function (er) { cb(er, file, cleanup) }) - }) - }) - }) -} - -// exList is a list of ignore lists. -// Each exList item is an array of patterns of files to ignore -// -function makeList_ (dir, pkg, exList, dfc, cb) { - var files = null - , cleanup = null + var extractOpts = { type: "Directory", path: target, strip: 1 } - readDir(dir, pkg, dfc, function (er, f, c) { - if (er) return cb(er) - cleanup = c - files = f.map(function (f) { - // no nulls in paths! - return f.split(/\0/)[0] - }).filter(function (f) { - // always remove all source control folders and - // waf/vim/OSX garbage. this is a firm requirement. - return !( f === ".git/" - || f === ".lock-wscript" - || f === "CVS/" - || f === ".svn/" - || f === ".hg/" - || f.match(/^\..*\.swp/) - || f === ".DS_Store" - || f.match(/^\._/) - || f === "npm-debug.log" - || f === "" - || f.charAt(0) === "/" - ) - }) - - // if (files.length > 0) files.push(".") - - if (files.indexOf("package.json") !== -1 && dir !== pkg.path) { - // a package.json file starts the whole exclude/include - // logic all over. Otherwise, a parent could break its - // deps with its files list or .npmignore file. - readJson(path.resolve(dir, "package.json"), function (er, data) { - if (!er && typeof data === "object") { - data.path = dir - return makeList(dir, data, dfc, function (er, files) { - // these need to be mounted onto the directory now. - cb(er, files && files.map(function (f) { - return path.resolve(path.dirname(dir), f) - })) - }) - } - next() - }) - //next() - } else next() + if (process.platform !== "win32" && + typeof uid === "number" && + typeof gid === "number") { + extractOpts.uid = uid + extractOpts.gid = gid + } - // add a local ignore file, if found. - if (files.indexOf(".npmignore") === -1 - && files.indexOf(".gitignore") === -1) next() - else { - excludes.addIgnoreFile( path.resolve(dir, ".npmignore") - , ".gitignore" - , exList - , dir - , function (er, list) { - if (!er) exList = list - next(er) - }) + extractOpts.filter = function () { + // symbolic links are not allowed in packages. + if (this.type.match(/^.*Link$/)) { + log.warn( this.path.substr(target.length + 1) + + ' -> ' + this.linkpath + , "excluding symbolic link") + return false } - }) + return true + } - var n = 2 - , errState = null - function next (er) { - if (errState) return - if (er) return cb(errState = er, [], cleanup) - if (-- n > 0) return - if (!pkg) return cb(new Error("No package.json file in "+dir)) - if (pkg.path === dir && pkg.files) { - pkg.files = pkg.files.filter(function (f) { - f = f.trim() - return f && f.charAt(0) !== "#" - }) - if (!pkg.files.length) pkg.files = null - } - if (pkg.path === dir && pkg.files) { - // stuff on the files list MUST be there. - // ignore everything, then include the stuff on the files list. - var pkgFiles = ["*"].concat(pkg.files.map(function (f) { - return "!" + f - })) - pkgFiles.dir = dir - pkgFiles.packageFiles = true - exList.push(pkgFiles) - } - - if (path.basename(dir) === "node_modules" - && pkg.path === path.dirname(dir) - && dfc) { // do fancy crap - files = filterNodeModules(files, pkg) + fst.on("error", log.er(cb, "error reading "+tarball)) + fst.on("data", function OD (c) { + // detect what it is. + // Then, depending on that, we'll figure out whether it's + // a single-file module, gzipped tarball, or naked tarball. + // gzipped files all start with 1f8b08 + if (c[0] === 0x1F && + c[1] === 0x8B && + c[2] === 0x08) { + fst + .pipe(zlib.Unzip()) + .on("error", log.er(cb, "unzip error "+tarball)) + .pipe(tar.Extract(extractOpts)) + .on("entry", extractEntry) + .on("error", log.er(cb, "untar error "+tarball)) + .on("close", cb) + } else if (c.toString().match(/^package\//)) { + // naked tar + fst + .pipe(tar.Extract(extractOpts)) + .on("entry", extractEntry) + .on("error", log.er(cb, "untar error "+tarball)) + .on("close", cb) } else { - // If a directory is excluded, we still need to be - // able to *include* a file within it, and have that override - // the prior exclusion. - // - // This whole makeList thing probably needs to be rewritten - files = files.filter(function (f) { - return excludes.filter(dir, exList)(f) || f.slice(-1) === "/" - }) - } - - - asyncMap(files, function (file, cb) { - // if this is a dir, then dive into it. - // otherwise, don't. - file = path.resolve(dir, file) - - // in 0.6.0, fs.readdir can produce some really odd results. - // XXX: remove this and make the engines hash exclude 0.6.0 - if (file.indexOf(dir) !== 0) { - return cb(null, []) + // naked js file + var jsOpts = { path: path.resolve(target, "index.js") } + + if (process.platform !== "win32" && + typeof uid === "number" && + typeof gid === "number") { + jsOpts.uid = uid + jsOpts.gid = gid } - fs.lstat(file, function (er, st) { - if (er) return cb(er) - if (st.isDirectory()) { - return makeList_(file, pkg, exList, dfc, cb) - } - return cb(null, file) - }) - }, function (er, files, c) { - if (c) cleanup = (cleanup || []).concat(c) - if (files.length > 0) files.push(dir) - return cb(er, files, cleanup) - }) - } -} - -// only include node_modules folder that are: -// 1. not on the dependencies list or -// 2. on the "bundleDependencies" list. -function filterNodeModules (files, pkg) { - var bd = pkg.bundleDependencies || pkg.bundledDependencies || [] - , deps = Object.keys(pkg.dependencies || {}) - .filter(function (key) { return !pkg.dependencies[key].extraneous }) - .concat(Object.keys(pkg.devDependencies || {})) - - delete pkg.bundledDependencies - pkg.bundleDependencies = bd + fst + .pipe(fstream.Writer(jsOpts)) + .on("error", log.er(cb, "copy error "+tarball)) + .on("close", function () { + var j = path.resolve(target, "package.json") + readJson(j, function (er, d) { + if (er) { + log.error(tarball, "Not a package") + return cb(er) + } + fs.writeFile(j, JSON.stringify(d) + "\n", cb) + }) + }) + } - return files.filter(function (f) { - f = f.replace(/\/$/, "") - return f.charAt(0) !== "." - && f.charAt(0) !== "_" - && bd.indexOf(f) !== -1 + // now un-hook, and re-emit the chunk + fst.removeListener("data", OD) + fst.emit("data", c) }) } |