diff options
author | Jonathan Abrahams <jonathan@mongodb.com> | 2016-08-25 16:08:57 -0400 |
---|---|---|
committer | Jonathan Abrahams <jonathan@mongodb.com> | 2016-08-25 16:08:57 -0400 |
commit | 5fe2f5b2261bf9cfb82a39dd96edab9454e4ff11 (patch) | |
tree | 2f151b8e3b0f88cbd14de6616402d089a712edd8 | |
parent | 9216812bf0ad01f1df7e91c274f3069d8d37e29b (diff) | |
download | mongo-5fe2f5b2261bf9cfb82a39dd96edab9454e4ff11.tar.gz |
SERVER-22150 Multiversion download script now uses the feed https://downloads.mongodb.org/full.json
-rw-r--r-- | buildscripts/setup_multiversion_mongodb.py | 246 | ||||
-rw-r--r-- | etc/evergreen.yml | 4 |
2 files changed, 172 insertions, 78 deletions
diff --git a/buildscripts/setup_multiversion_mongodb.py b/buildscripts/setup_multiversion_mongodb.py index 740bed8b138..b9294debc43 100644 --- a/buildscripts/setup_multiversion_mongodb.py +++ b/buildscripts/setup_multiversion_mongodb.py @@ -1,22 +1,41 @@ -#!/usr/bin/python +#!/usr/bin/env python import re import sys import os import tempfile -import urllib2 import subprocess +import json +import urlparse import tarfile +import signal +import threading +import traceback import shutil import errno +from contextlib import closing # To ensure it exists on the system -import gzip +import zipfile # # Useful script for installing multiple versions of MongoDB on a machine # Only really tested/works on Linux. # +def dump_stacks(signal, frame): + print "======================================" + print "DUMPING STACKS due to SIGUSR1 signal" + print "======================================" + threads = threading.enumerate(); + + print "Total Threads: " + str(len(threads)) + + for id, stack in sys._current_frames().items(): + print "Thread %d" % (id) + print "".join(traceback.format_stack(stack)) + print "======================================" + + def version_tuple(version): """Returns a version tuple that can be used for numeric sorting of version strings such as '2.6.0-rc1' and '2.4.0'""" @@ -24,6 +43,11 @@ def version_tuple(version): RC_OFFSET = -100 version_parts = re.split(r'\.|-', version[0]) + if version_parts[-1] == "pre": + # Prior to improvements for how the version string is managed within the server + # (SERVER-17782), the binary archives would contain a trailing "-pre". + version_parts.pop() + if version_parts[-1].startswith("rc"): rc_part = version_parts.pop() rc_part = rc_part.split('rc')[1] @@ -32,21 +56,50 @@ def version_tuple(version): # releases to be sorted in ascending order (e.g., 2.6.0-rc1, # 2.6.0-rc2, 2.6.0). version_parts.append(int(rc_part) + RC_OFFSET) + elif version_parts[0].startswith("v") and version_parts[-1] == "latest": + version_parts[0] = version_parts[0][1:] + # The "<branchname>-latest" versions are weighted the highest when a particular major + # release is requested. + version_parts[-1] = float("inf") else: # Non-RC releases have an extra 0 appended so version tuples like # (2, 6, 0, -100) and (2, 6, 0, 0) sort in ascending order. version_parts.append(0) - return tuple(map(int, version_parts)) + return tuple(map(float, version_parts)) + + +def download_file(url, file_name): + """Returns True if download was successful. Raises error if download fails.""" + proc = subprocess.Popen(["curl", + "-L", "--silent", + "--retry", "5", + "--retry-max-time", "600", + "--max-time", "120", + "-o", file_name, + url], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + proc.communicate() + error_code = proc.returncode + if not error_code: + error_code = proc.wait() + if not error_code: + return True + + raise Exception("Failed to download %s with error %d" % (url, error_code)) + class MultiVersionDownloader : - def __init__(self, install_dir, link_dir, platform): + def __init__(self, install_dir, link_dir, edition, platform_arch): self.install_dir = install_dir self.link_dir = link_dir - match = re.compile("(.*)\/(.*)").match(platform) - self.platform = match.group(1) - self.arch = match.group(2) + self.edition = edition + match = re.compile("(.*)\/(.*)").match(platform_arch) + if match: + self.platform_arch = match.group(1).lower() + "_" + match.group(2).lower() + else: + self.platform_arch = platform_arch.lower() self._links = None @property @@ -56,31 +109,22 @@ class MultiVersionDownloader : return self._links def download_links(self): - href = "http://dl.mongodb.org/dl/%s/%s" \ - % (self.platform.lower(), self.arch) - - attempts_remaining = 5 - timeout_seconds = 10 - while True: - try: - html = urllib2.urlopen(href, timeout = timeout_seconds).read() - break - except Exception as e: - print "fetching links failed (%s), retrying..." % e - attempts_remaining -= 1 - if attempts_remaining == 0 : - raise Exception("Failed to get links after multiple retries") + temp_file = tempfile.mktemp() + download_file("https://downloads.mongodb.org/full.json", temp_file) + with open(temp_file) as f: + full_json = json.load(f) + os.remove(temp_file) + if 'versions' not in full_json: + raise Exception("No versions field in JSON: \n" + str(full_json)) links = {} - for line in html.split(): - match = re.compile("http:\/\/downloads\.mongodb\.org\/%s/mongodb-%s-%s-([^\"]*)\.tgz" \ - % (self.platform.lower(), self.platform.lower(), self.arch)).search(line) - - if match == None: continue - - link = match.group(0) - version = match.group(1) - links[version] = link + for json_version in full_json['versions']: + if 'version' in json_version and 'downloads' in json_version: + for download in json_version['downloads']: + if 'target' in download and 'edition' in download and \ + download['target'] == self.platform_arch and \ + download['edition'] == self.edition: + links[json_version['version']] = download['archive']['url'] return links @@ -95,13 +139,16 @@ class MultiVersionDownloader : urls = [] for link_version, link_url in self.links.iteritems(): - if link_version.startswith(version): - # If we have a "-" in our version, exact match only - if version.find("-") >= 0: - if link_version != version: continue - elif link_version.find("-") >= 0: - continue - + if link_version.startswith(version) or link_version == "v%s-latest" % (version): + # The 'link_version' is a candidate for the requested 'version' if + # (a) it is a prefix of the requested version, or if + # (b) it is the "<branchname>-latest" version and the requested version is for a + # particular major release. + if "-" in version: + # The requested 'version' contains a hyphen, so we only consider exact matches + # to that version. + if link_version != version: + continue urls.append((link_version, link_url)) if len(urls) == 0: @@ -112,6 +159,7 @@ class MultiVersionDownloader : full_version = urls[-1][0] url = urls[-1][1] extract_dir = url.split("/")[-1][:-4] + file_suffix = os.path.splitext(urlparse.urlparse(url).path)[1] # only download if we don't already have the directory already_downloaded = os.path.isdir(os.path.join( self.install_dir, extract_dir)) @@ -119,32 +167,50 @@ class MultiVersionDownloader : print "Skipping download for version %s (%s) since the dest already exists '%s'" \ % (version, full_version, extract_dir) else: - temp_dir = tempfile.mkdtemp() - temp_file = tempfile.mktemp(suffix=".tgz") - - data = urllib2.urlopen(url) - print "Downloading data for version %s (%s)..." % (version, full_version) - - with open(temp_file, 'wb') as f: - f.write(data.read()) - print "Uncompressing data for version %s (%s)..." % (version, full_version) - - # Can't use cool with syntax b/c of python 2.6 - tf = tarfile.open(temp_file, 'r:gz') - - try: - tf.extractall(path=temp_dir) - except: - tf.close() - raise - - tf.close() - + print "Download url is %s" % url + + temp_dir = tempfile.mkdtemp() + temp_file = tempfile.mktemp(suffix=file_suffix) + download_file(url, temp_file) + + print "Uncompressing data for version %s (%s)..." % (version, full_version) + first_file = '' + if file_suffix == ".zip": + # Support .zip downloads, used for Windows binaries. + with zipfile.ZipFile(temp_file) as zf: + # Use the name of the root directory in the archive as the name of the directory + # to extract the binaries into inside 'self.install_dir'. The name of the root + # directory nearly always matches the parsed URL text, with the exception of + # versions such as "v3.2-latest" that instead contain the githash. + first_file = zf.namelist()[0] + zf.extractall(temp_dir) + elif file_suffix == ".tgz": + # Support .tgz downloads, used for Linux binaries. + with closing(tarfile.open(temp_file, 'r:gz')) as tf: + # Use the name of the root directory in the archive as the name of the directory + # to extract the binaries into inside 'self.install_dir'. The name of the root + # directory nearly always matches the parsed URL text, with the exception of + # versions such as "v3.2-latest" that instead contain the githash. + first_file = tf.getnames()[0] + tf.extractall(path=temp_dir) + else: + raise Exception("Unsupported file extension %s" % file_suffix) + + # Sometimes the zip will contain the root directory as the first file and + # os.path.dirname() will return ''. + extract_dir = os.path.dirname(first_file) + if not extract_dir: + extract_dir = first_file temp_install_dir = os.path.join(temp_dir, extract_dir) - - shutil.move(temp_install_dir, self.install_dir) - + + # We may not have been able to determine whether we already downloaded the requested + # version due to the ambiguity in the parsed URL text, so we check for it again using + # the adjusted 'extract_dir' value. + already_downloaded = os.path.isdir(os.path.join(self.install_dir, extract_dir)) + if not already_downloaded: + shutil.move(temp_install_dir, self.install_dir) + shutil.rmtree(temp_dir) os.remove(temp_file) @@ -162,11 +228,24 @@ class MultiVersionDownloader : for executable in os.listdir(os.path.join(installed_dir, "bin")): - link_name = "%s-%s" % (executable, version) + executable_name, executable_extension = os.path.splitext(executable) + link_name = "%s-%s%s" % (executable_name, version, executable_extension) try: - os.symlink(os.path.join(installed_dir, "bin", executable),\ - os.path.join(self.link_dir, link_name)) + executable = os.path.join(installed_dir, "bin", executable) + executable_link = os.path.join(self.link_dir, link_name) + if os.name == "nt": + # os.symlink is not supported on Windows, use a direct method instead. + def symlink_ms(source, link_name): + import ctypes + csl = ctypes.windll.kernel32.CreateSymbolicLinkW + csl.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32) + csl.restype = ctypes.c_ubyte + flags = 1 if os.path.isdir(source) else 0 + if csl(link_name, source.replace('/', '\\'), flags) == 0: + raise ctypes.WinError() + os.symlink = symlink_ms + os.symlink(executable, executable_link) except OSError as exc: if exc.errno == errno.EEXIST: pass @@ -175,13 +254,20 @@ class MultiVersionDownloader : CL_HELP_MESSAGE = \ """ -Downloads and installs particular mongodb versions (each binary is renamed to include its version) -into an install directory and symlinks the binaries with versions to another directory. +Downloads and installs particular mongodb versions (each binary is renamed to include its version) +into an install directory and symlinks the binaries with versions to another directory. This script +supports community and enterprise builds. -Usage: setup_multiversion_mongodb.py INSTALL_DIR LINK_DIR PLATFORM_AND_ARCH VERSION1 [VERSION2 VERSION3 ...] +Usage: setup_multiversion_mongodb.py INSTALL_DIR LINK_DIR EDITION PLATFORM_AND_ARCH VERSION1 [VERSION2 VERSION3 ...] -Ex: setup_multiversion_mongodb.py ./install ./link "Linux/x86_64" "2.0.6" "2.0.3-rc0" "2.0" "2.2" "2.3" -Ex: setup_multiversion_mongodb.py ./install ./link "OSX/x86_64" "2.4" "2.2" +EDITION is one of the following: + base (generic community builds) + enterprise + targeted (platform specific community builds, includes SSL) +PLATFORM_AND_ARCH can be specified with just a platform, i.e., OSX, if it is supported. + +Ex: setup_multiversion_mongodb.py ./install ./link base "Linux/x86_64" "2.0.6" "2.0.3-rc0" "2.0" "2.2" "2.3" +Ex: setup_multiversion_mongodb.py ./install ./link enterprise "OSX" "2.4" "2.2" After running the script you will have a directory structure like this: ./install/[mongodb-osx-x86_64-2.4.9, mongodb-osx-x86_64-2.2.7] @@ -209,22 +295,33 @@ def parse_cl_args(args): link_dir = args[0] args = args[1:] + if len(args) == 0: raise_exception("Missing EDITION") + + edition = args[0] + if edition not in ['base', 'enterprise', 'targeted']: + raise Exception("Unsupported edition %s" % edition) + + args = args[1:] if len(args) == 0: raise_exception("Missing PLATFORM_AND_ARCH") - platform = args[0] + platform_arch = args[0] args = args[1:] - if re.compile(".*\/.*").match(platform) == None: - raise_exception("PLATFORM_AND_ARCH isn't of the correct format") if len(args) == 0: raise_exception("Missing VERSION1") versions = args - return (MultiVersionDownloader(install_dir, link_dir, platform), versions) + return (MultiVersionDownloader(install_dir, link_dir, edition, platform_arch), versions) def main(): + # Listen for SIGUSR1 and dump stack if received. + try: + signal.signal(signal.SIGUSR1, dump_stacks) + except AttributeError: + print "Cannot catch signals on Windows" + downloader, versions = parse_cl_args(sys.argv[1:]) for version in versions: @@ -234,4 +331,3 @@ def main(): if __name__ == '__main__': main() - diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 4f2a4d6a289..298737bd0ac 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -1104,7 +1104,7 @@ tasks: cp mongodb*/bin/* . rm -rf /data/install /data/multiversion - ${python|python} buildscripts/setup_multiversion_mongodb.py /data/install /data/multiversion "Linux/x86_64" "1.8" "2.0" "2.2" "2.4" "2.6" + ${python|python} buildscripts/setup_multiversion_mongodb.py /data/install /data/multiversion "base" "Linux/x86_64" "1.8" "2.0" "2.2" "2.4" "2.6" PATH=$PATH:/data/multiversion ${asan_symbolizer} ${python|python} buildscripts/smoke.py --nopreallocj --with-cleanbb --mongod ./mongod --mongo ./mongo --report-file report.json ${test_flags|} multiVersion - name: noPassthrough @@ -2387,7 +2387,6 @@ buildvariants: - name: jsCore_small_oplog_WT - name: mmap - name: mongosTest - - name: multiversion - name: noPassthrough - name: noPassthrough_WT - name: noPassthroughWithMongod @@ -3600,7 +3599,6 @@ buildvariants: - name: jsCore_small_oplog_WT - name: mmap - name: mongosTest - - name: multiversion - name: noPassthrough - name: noPassthrough_WT - name: noPassthroughWithMongod |