SERVER-13287 delegate to shell when compressing tar archives

Signed-off-by: Matt Kangas <matt.kangas@mongodb.com>
(cherry picked from commit b34245c1ff87833a307b2eac76bb710ec5033d28)

1 files changed, 86 insertions, 23 deletions

diff --git a/buildscripts/make_archive.py b/buildscripts/make_archive.py
index 4c12e901a64..840b134e717 100755
--- a/buildscripts/make_archive.py
+++ b/buildscripts/make_archive.py
@@ -29,17 +29,89 @@ For a detailed usage example, see src/SConscript.client or src/mongo/SConscript.
 import optparse
 import os
 import sys
+import shutil
+import zipfile
+from subprocess import (Popen, PIPE, STDOUT)
 
 def main(argv):
     opts = parse_options(argv[1:])
-    archive = open_archive_for_write(opts.output_filename, opts.archive_format)
+    if opts.archive_format in ('tar', 'tgz'):
+        make_tar_archive(opts)
+    elif opts.archive_format in ('zip'):
+        make_zip_archive(opts)
+    else:
+        raise ValueError('Unsupported archive format "%s"' % opts.archive_format)
+
+def delete_directory(dir):
+    '''Recursively deletes a directory and its contents.
+    '''
+    try:
+        shutil.rmtree(dir)
+    except Exception:
+        pass
+
+def make_tar_archive(opts):
+    '''Given the parsed options, generates the 'opt.output_filename'
+    tarball containing all the files in 'opt.input_filename' renamed
+    according to the mappings in 'opts.transformations'.
+
+    e.g. for an input file named "a/mongo/build/DISTSRC", and an
+    existing transformation {"a/mongo/build": "release"}, the input
+    file will be written to the tarball as "release/DISTSRC"
+
+    All files to be compressed are copied into new directories as
+    required by 'opts.transformations'. Once the tarball has been
+    created, all temporary directory structures created for the
+    purposes of compressing, are removed.
+    '''
+    tar_options = "cvf"
+    if opts.archive_format is 'tgz':
+        tar_options += "z"
+
+    # clean and create a temp directory to copy files to
+    enclosing_archive_directory = os.path.join("build", "archive")
+    delete_directory(enclosing_archive_directory)
+    os.makedirs(enclosing_archive_directory)
+    output_tarfile = os.path.join(os.getcwd(), opts.output_filename)
+
+    tar_command = ["tar", tar_options, output_tarfile]
+
+    for input_filename in opts.input_filenames:
+        preferred_filename = get_preferred_filename(input_filename, opts.transformations)
+        temp_file_location = os.path.join(enclosing_archive_directory, preferred_filename)
+        enclosing_file_directory = os.path.dirname(temp_file_location)
+        if not os.path.exists(enclosing_file_directory):
+            os.makedirs(enclosing_file_directory)
+        print "copying %s => %s" % (input_filename, temp_file_location)
+        shutil.copy2(input_filename, temp_file_location)
+        tar_command.append(preferred_filename)
+
+    print " ".join(tar_command)
+    # execute the full tar command
+    run_directory = os.path.join(os.getcwd(), enclosing_archive_directory)
+    proc = Popen(tar_command, stdout=PIPE, stderr=STDOUT, bufsize=0, cwd=run_directory)
+    proc.wait()
+
+    # delete temp directory
+    delete_directory(enclosing_archive_directory)
+
+def make_zip_archive(opts):
+    '''Given the parsed options, generates the 'opt.output_filename'
+    zipfile containing all the files in 'opt.input_filename' renamed
+    according to the mappings in 'opts.transformations'.
+
+    All files in 'opt.output_filename' are renamed before being
+    written into the zipfile.
+    '''
+    archive = open_zip_archive_for_write(opts.output_filename)
     try:
         for input_filename in opts.input_filenames:
             archive.add(input_filename, arcname=get_preferred_filename(input_filename,
-                                                                       opts.transformations))
+            opts.transformations))
     finally:
         archive.close()
 
+
 def parse_options(args):
     parser = optparse.OptionParser()
     parser.add_option('-o', dest='output_filename', default=None,
@@ -82,30 +154,21 @@ def parse_options(args):
 
     return opts
 
-def open_archive_for_write(filename, archive_format):
-    '''Open a tar or zip archive for write, with the given format, and return it.
-
-    The type of archive is determined by the "archive_format" parameter, which should be
-    "tar", "tgz" (for gzipped tar) or "zip".
+def open_zip_archive_for_write(filename):
+    '''Open a zip archive for writing and return it.
     '''
-
-    if archive_format in ('tar', 'tgz'):
-        import tarfile
-        mode = 'w'
-        if archive_format is 'tgz':
-            mode += '|gz'
-        return tarfile.open(filename, mode)
-    if archive_format is 'zip':
-        import zipfile
-        # Infuriatingly, Zipfile calls the "add" method "write", but they're otherwise identical,
-        # for our purposes.  WrappedZipFile is a minimal adapter class.
-        class WrappedZipFile(zipfile.ZipFile):
-            def add(self, filename, arcname):
-                return self.write(filename, arcname)
-        return WrappedZipFile(filename, 'w', zipfile.ZIP_DEFLATED)
-    raise ValueError('Unsupported archive format "%s"' % archive_format)
+    # Infuriatingly, Zipfile calls the "add" method "write", but they're otherwise identical,
+    # for our purposes.  WrappedZipFile is a minimal adapter class.
+    class WrappedZipFile(zipfile.ZipFile):
+        def add(self, filename, arcname):
+            return self.write(filename, arcname)
+    return WrappedZipFile(filename, 'w', zipfile.ZIP_DEFLATED)
 
 def get_preferred_filename(input_filename, transformations):
+    '''Does a prefix subsitution on 'input_filename' for the
+    first matching transformation in 'transformations' and
+    returns the substituted string
+    '''
     for match, replace in transformations:
         if input_filename.startswith(match):
             return replace + input_filename[len(match):]