SERVER-22150 Multiversion download script now uses the feed https://downloads.mongodb.org/full.json

2 files changed, 172 insertions, 77 deletions

diff --git a/buildscripts/setup_multiversion_mongodb.py b/buildscripts/setup_multiversion_mongodb.py
index 740bed8b138..b9294debc43 100644
--- a/buildscripts/setup_multiversion_mongodb.py
+++ b/buildscripts/setup_multiversion_mongodb.py
@@ -1,22 +1,41 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 import re
 import sys
 import os
 import tempfile
-import urllib2
 import subprocess
+import json
+import urlparse
 import tarfile
+import signal
+import threading
+import traceback
 import shutil
 import errno
+from contextlib import closing
 # To ensure it exists on the system
-import gzip
+import zipfile
 
 #
 # Useful script for installing multiple versions of MongoDB on a machine
 # Only really tested/works on Linux.
 #
 
+def dump_stacks(signal, frame):
+    print "======================================"
+    print "DUMPING STACKS due to SIGUSR1 signal"
+    print "======================================"
+    threads = threading.enumerate();
+
+    print "Total Threads: " + str(len(threads))
+
+    for id, stack in sys._current_frames().items():
+        print "Thread %d" % (id)
+        print "".join(traceback.format_stack(stack))
+    print "======================================"
+
+
 def version_tuple(version):
     """Returns a version tuple that can be used for numeric sorting
     of version strings such as '2.6.0-rc1' and '2.4.0'"""
@@ -24,6 +43,11 @@ def version_tuple(version):
     RC_OFFSET = -100
     version_parts = re.split(r'\.|-', version[0])
 
+    if version_parts[-1] == "pre":
+        # Prior to improvements for how the version string is managed within the server
+        # (SERVER-17782), the binary archives would contain a trailing "-pre".
+        version_parts.pop()
+
     if version_parts[-1].startswith("rc"):
         rc_part = version_parts.pop()
         rc_part = rc_part.split('rc')[1]
@@ -32,21 +56,50 @@ def version_tuple(version):
         # releases to be sorted in ascending order (e.g., 2.6.0-rc1,
         # 2.6.0-rc2, 2.6.0).
         version_parts.append(int(rc_part) + RC_OFFSET)
+    elif version_parts[0].startswith("v") and version_parts[-1] == "latest":
+        version_parts[0] = version_parts[0][1:]
+        # The "<branchname>-latest" versions are weighted the highest when a particular major
+        # release is requested.
+        version_parts[-1] = float("inf")
     else:
         # Non-RC releases have an extra 0 appended so version tuples like
         # (2, 6, 0, -100) and (2, 6, 0, 0) sort in ascending order.
         version_parts.append(0)
 
-    return tuple(map(int, version_parts))
+    return tuple(map(float, version_parts))
+
+
+def download_file(url, file_name):
+    """Returns True if download was successful. Raises error if download fails."""
+    proc = subprocess.Popen(["curl",
+                             "-L", "--silent",
+                             "--retry", "5",
+                             "--retry-max-time", "600",
+                             "--max-time", "120",
+                             "-o", file_name,
+                             url],
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    proc.communicate()
+    error_code = proc.returncode
+    if not error_code:
+        error_code = proc.wait()
+    if not error_code:
+        return True
+
+    raise Exception("Failed to download %s with error %d" % (url, error_code))
+
 
 class MultiVersionDownloader :
 
-    def __init__(self, install_dir, link_dir, platform):
+    def __init__(self, install_dir, link_dir, edition, platform_arch):
         self.install_dir = install_dir
         self.link_dir = link_dir
-        match = re.compile("(.*)\/(.*)").match(platform)
-        self.platform = match.group(1)
-        self.arch = match.group(2)
+        self.edition = edition
+        match = re.compile("(.*)\/(.*)").match(platform_arch)
+        if match:
+            self.platform_arch = match.group(1).lower() + "_" + match.group(2).lower()
+        else:
+            self.platform_arch = platform_arch.lower()
         self._links = None
 
     @property
@@ -56,31 +109,22 @@ class MultiVersionDownloader :
         return self._links
 
     def download_links(self):
-        href = "http://dl.mongodb.org/dl/%s/%s" \
-               % (self.platform.lower(), self.arch)
-
-        attempts_remaining = 5
-        timeout_seconds = 10
-        while True:
-            try:
-                html = urllib2.urlopen(href, timeout = timeout_seconds).read()
-                break
-            except Exception as e:
-                print "fetching links failed (%s), retrying..." % e
-                attempts_remaining -= 1
-                if attempts_remaining == 0 :
-                    raise Exception("Failed to get links after multiple retries")
+        temp_file = tempfile.mktemp()
+        download_file("https://downloads.mongodb.org/full.json", temp_file)
+        with open(temp_file) as f:
+            full_json = json.load(f)
+        os.remove(temp_file)
+        if 'versions' not in full_json:
+            raise Exception("No versions field in JSON: \n" + str(full_json))
 
         links = {}
-        for line in html.split():
-            match = re.compile("http:\/\/downloads\.mongodb\.org\/%s/mongodb-%s-%s-([^\"]*)\.tgz" \
-                % (self.platform.lower(), self.platform.lower(), self.arch)).search(line)
-
-            if match == None: continue
-
-            link = match.group(0)
-            version = match.group(1)
-            links[version] = link
+        for json_version in full_json['versions']:
+            if 'version' in json_version and 'downloads' in json_version:
+                for download in json_version['downloads']:
+                    if 'target' in download and 'edition' in download and \
+                        download['target'] == self.platform_arch and \
+                        download['edition'] == self.edition:
+                            links[json_version['version']] = download['archive']['url']
 
         return links
 
@@ -95,13 +139,16 @@ class MultiVersionDownloader :
 
         urls = []
         for link_version, link_url in self.links.iteritems():
-            if link_version.startswith(version):
-                # If we have a "-" in our version, exact match only
-                if version.find("-") >= 0:
-                    if link_version != version: continue
-                elif link_version.find("-") >= 0:
-                    continue
-
+            if link_version.startswith(version) or link_version == "v%s-latest" % (version):
+                # The 'link_version' is a candidate for the requested 'version' if
+                #   (a) it is a prefix of the requested version, or if
+                #   (b) it is the "<branchname>-latest" version and the requested version is for a
+                #       particular major release.
+                if "-" in version:
+                    # The requested 'version' contains a hyphen, so we only consider exact matches
+                    # to that version.
+                    if link_version != version:
+                        continue
                 urls.append((link_version, link_url))
 
         if len(urls) == 0:
@@ -112,6 +159,7 @@ class MultiVersionDownloader :
         full_version = urls[-1][0]
         url = urls[-1][1]
         extract_dir = url.split("/")[-1][:-4]
+        file_suffix = os.path.splitext(urlparse.urlparse(url).path)[1]
 
         # only download if we don't already have the directory
         already_downloaded = os.path.isdir(os.path.join( self.install_dir, extract_dir))
@@ -119,32 +167,50 @@ class MultiVersionDownloader :
             print "Skipping download for version %s (%s) since the dest already exists '%s'" \
                 % (version, full_version, extract_dir)
         else:
-            temp_dir = tempfile.mkdtemp()
-            temp_file = tempfile.mktemp(suffix=".tgz")
-    
-            data = urllib2.urlopen(url)
-    
             print "Downloading data for version %s (%s)..." % (version, full_version)
-    
-            with open(temp_file, 'wb') as f:
-                f.write(data.read())
-                print "Uncompressing data for version %s (%s)..." % (version, full_version)
-    
-            # Can't use cool with syntax b/c of python 2.6
-            tf = tarfile.open(temp_file, 'r:gz')
-    
-            try:
-                tf.extractall(path=temp_dir)
-            except:
-                tf.close()
-                raise
-    
-            tf.close()
-    
+            print "Download url is %s" % url
+
+            temp_dir = tempfile.mkdtemp()
+            temp_file = tempfile.mktemp(suffix=file_suffix)
+            download_file(url, temp_file)
+
+            print "Uncompressing data for version %s (%s)..." % (version, full_version)
+            first_file = ''
+            if file_suffix == ".zip":
+                # Support .zip downloads, used for Windows binaries.
+                with zipfile.ZipFile(temp_file) as zf:
+                    # Use the name of the root directory in the archive as the name of the directory
+                    # to extract the binaries into inside 'self.install_dir'. The name of the root
+                    # directory nearly always matches the parsed URL text, with the exception of
+                    # versions such as "v3.2-latest" that instead contain the githash.
+                    first_file = zf.namelist()[0]
+                    zf.extractall(temp_dir)
+            elif file_suffix == ".tgz":
+                # Support .tgz downloads, used for Linux binaries.
+                with closing(tarfile.open(temp_file, 'r:gz')) as tf:
+                    # Use the name of the root directory in the archive as the name of the directory
+                    # to extract the binaries into inside 'self.install_dir'. The name of the root
+                    # directory nearly always matches the parsed URL text, with the exception of
+                    # versions such as "v3.2-latest" that instead contain the githash.
+                    first_file = tf.getnames()[0]
+                    tf.extractall(path=temp_dir)
+            else:
+                raise Exception("Unsupported file extension %s" % file_suffix)
+
+            # Sometimes the zip will contain the root directory as the first file and
+            # os.path.dirname() will return ''.
+            extract_dir = os.path.dirname(first_file)
+            if not extract_dir:
+                extract_dir = first_file
             temp_install_dir = os.path.join(temp_dir, extract_dir)
-    
-            shutil.move(temp_install_dir, self.install_dir)
-    
+
+            # We may not have been able to determine whether we already downloaded the requested
+            # version due to the ambiguity in the parsed URL text, so we check for it again using
+            # the adjusted 'extract_dir' value.
+            already_downloaded = os.path.isdir(os.path.join(self.install_dir, extract_dir))
+            if not already_downloaded:
+                shutil.move(temp_install_dir, self.install_dir)
+
             shutil.rmtree(temp_dir)
             os.remove(temp_file)
 
@@ -162,11 +228,24 @@ class MultiVersionDownloader :
 
         for executable in os.listdir(os.path.join(installed_dir, "bin")):
 
-            link_name = "%s-%s" % (executable, version)
+            executable_name, executable_extension = os.path.splitext(executable)
+            link_name = "%s-%s%s" % (executable_name, version, executable_extension)
 
             try:
-                os.symlink(os.path.join(installed_dir, "bin", executable),\
-                           os.path.join(self.link_dir, link_name))
+                executable = os.path.join(installed_dir, "bin", executable)
+                executable_link = os.path.join(self.link_dir, link_name)
+                if os.name == "nt":
+                    # os.symlink is not supported on Windows, use a direct method instead.
+                    def symlink_ms(source, link_name):
+                        import ctypes
+                        csl = ctypes.windll.kernel32.CreateSymbolicLinkW
+                        csl.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32)
+                        csl.restype = ctypes.c_ubyte
+                        flags = 1 if os.path.isdir(source) else 0
+                        if csl(link_name, source.replace('/', '\\'), flags) == 0:
+                            raise ctypes.WinError()
+                    os.symlink = symlink_ms
+                os.symlink(executable, executable_link)
             except OSError as exc:
                 if exc.errno == errno.EEXIST:
                     pass
@@ -175,13 +254,20 @@ class MultiVersionDownloader :
 
 CL_HELP_MESSAGE = \
 """
-Downloads and installs particular mongodb versions (each binary is renamed to include its version) 
-into an install directory and symlinks the binaries with versions to another directory.
+Downloads and installs particular mongodb versions (each binary is renamed to include its version)
+into an install directory and symlinks the binaries with versions to another directory. This script
+supports community and enterprise builds.
 
-Usage: setup_multiversion_mongodb.py INSTALL_DIR LINK_DIR PLATFORM_AND_ARCH VERSION1 [VERSION2 VERSION3 ...]
+Usage: setup_multiversion_mongodb.py INSTALL_DIR LINK_DIR EDITION PLATFORM_AND_ARCH VERSION1 [VERSION2 VERSION3 ...]
 
-Ex: setup_multiversion_mongodb.py ./install ./link "Linux/x86_64" "2.0.6" "2.0.3-rc0" "2.0" "2.2" "2.3"
-Ex: setup_multiversion_mongodb.py ./install ./link "OSX/x86_64" "2.4" "2.2"
+EDITION is one of the following:
+    base (generic community builds)
+    enterprise
+    targeted (platform specific community builds, includes SSL)
+PLATFORM_AND_ARCH can be specified with just a platform, i.e., OSX, if it is supported.
+
+Ex: setup_multiversion_mongodb.py ./install ./link base "Linux/x86_64" "2.0.6" "2.0.3-rc0" "2.0" "2.2" "2.3"
+Ex: setup_multiversion_mongodb.py ./install ./link enterprise "OSX" "2.4" "2.2"
 
 After running the script you will have a directory structure like this:
 ./install/[mongodb-osx-x86_64-2.4.9, mongodb-osx-x86_64-2.2.7]
@@ -209,22 +295,33 @@ def parse_cl_args(args):
     link_dir = args[0]
 
     args = args[1:]
+    if len(args) == 0: raise_exception("Missing EDITION")
+
+    edition = args[0]
+    if edition not in ['base', 'enterprise', 'targeted']:
+        raise Exception("Unsupported edition %s" % edition)
+
+    args = args[1:]
     if len(args) == 0: raise_exception("Missing PLATFORM_AND_ARCH")
 
-    platform = args[0]
+    platform_arch = args[0]
 
     args = args[1:]
-    if re.compile(".*\/.*").match(platform) == None:
-        raise_exception("PLATFORM_AND_ARCH isn't of the correct format")
 
     if len(args) == 0: raise_exception("Missing VERSION1")
 
     versions = args
 
-    return (MultiVersionDownloader(install_dir, link_dir, platform), versions)
+    return (MultiVersionDownloader(install_dir, link_dir, edition, platform_arch), versions)
 
 def main():
 
+    # Listen for SIGUSR1 and dump stack if received.
+    try:
+        signal.signal(signal.SIGUSR1, dump_stacks)
+    except AttributeError:
+        print "Cannot catch signals on Windows"
+
     downloader, versions = parse_cl_args(sys.argv[1:])
 
     for version in versions:
@@ -234,4 +331,3 @@ def main():
 
 if __name__ == '__main__':
   main()
-
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index e563e0562ec..774c28a6005 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -548,7 +548,7 @@ tasks:
           ${decompress|unzip} mongodb*.${ext|tgz}
           cp mongodb*/bin/* .
           rm -rf /data/install /data/multiversion
-          ${python|python} buildscripts/setup_multiversion_mongodb.py /data/install /data/multiversion "Linux/x86_64" "1.8" "2.0" "2.2" "2.4"
+          ${python|python} buildscripts/setup_multiversion_mongodb.py /data/install /data/multiversion "base" "Linux/x86_64" "1.8" "2.0" "2.2" "2.4"
           PATH=$PATH:/data/multiversion ${python|python} buildscripts/smoke.py --nopreallocj --with-cleanbb --mongod ./mongod --mongo ./mongo --report-file report.json ${test_flags|} multiVersion
 
 - name: noPassthrough
@@ -1354,7 +1354,6 @@ buildvariants:
   - name: jsCore_compatibility
   - name: jsCore_small_oplog
   - name: mongosTest
-  - name: multiversion
   - name: noPassthrough
   - name: noPassthroughWithMongod
   - name: parallel