diff options
Diffstat (limited to 'buildscripts/setup_multiversion_mongodb.py')
-rwxr-xr-x | buildscripts/setup_multiversion_mongodb.py | 484 |
1 files changed, 6 insertions, 478 deletions
diff --git a/buildscripts/setup_multiversion_mongodb.py b/buildscripts/setup_multiversion_mongodb.py index 94e5d056816..3f089504d1d 100755 --- a/buildscripts/setup_multiversion_mongodb.py +++ b/buildscripts/setup_multiversion_mongodb.py @@ -1,481 +1,9 @@ #!/usr/bin/env python3 -"""Install multiple versions of MongoDB on a machine.""" - -import contextlib -import errno -import json -import optparse -import os -import re -import shutil -import signal -import sys -import tarfile -import tempfile -import threading -import traceback -import urllib.parse -import zipfile - -import requests -import requests.exceptions - - -def dump_stacks(_signal_num, _frame): # pylint: disable=unused-argument - """Dump stacks when SIGUSR1 is received.""" - print("======================================") - print("DUMPING STACKS due to SIGUSR1 signal") - print("======================================") - threads = threading.enumerate() - - print("Total Threads: {:d}".format(len(threads))) - - for tid, stack in list(sys._current_frames().items()): # pylint: disable=protected-access - print("Thread {:d}".format(tid)) - print("".join(traceback.format_stack(stack))) - print("======================================") - - -def get_version_parts(version, for_sorting=False): - """Return a list containing the components of the version string as numeric values. - - This function can be used for numeric sorting - of version strings such as '2.6.0-rc1' and '2.4.0' when the - 'for_sorting' parameter is specified as true. - """ - - rc_offset = -100 - version_parts = re.split(r"\.|-", version) - - if version_parts[-1] == "pre": - # Prior to improvements for how the version string is managed within the server - # (SERVER-17782), the binary archives would contain a trailing "-pre". - version_parts.pop() - - if version_parts[-1].startswith("rc"): - # RC versions are weighted down to allow future RCs and general - # releases to be sorted in ascending order (e.g., 2.6.0-rc1, - # 2.6.0-rc2, 2.6.0). - version_parts[-1] = int(version_parts[-1][2:]) + rc_offset - elif version_parts[0].startswith("v") and version_parts[-1] == "latest": - version_parts[0] = version_parts[0][1:] - # The "<branchname>-latest" versions are weighted the highest when a particular major - # release is requested. - version_parts[-1] = float("inf") - elif for_sorting: - # We want to have the number of components in the resulting version parts match the number - # of components in the 'version' string if we aren't going to be using them for sorting. - # Otherwise, we append an additional 0 to non-RC releases so that version lists like - # [2, 6, 0, -100] and [2, 6, 0, 0] sort in ascending order. - version_parts.append(0) - - return [float(part) for part in version_parts] - - -def download_file(url, file_name, download_retries=5): - """Return True if download was successful. Raises error if download fails.""" - - while download_retries > 0: - - with requests.Session() as session: - adapter = requests.adapters.HTTPAdapter(max_retries=download_retries) - session.mount(url, adapter) - response = session.get(url, stream=True) - response.raise_for_status() - - with open(file_name, "wb") as file_handle: - try: - for block in response.iter_content(1024 * 1000): - file_handle.write(block) - except requests.exceptions.ChunkedEncodingError as err: - download_retries -= 1 - if download_retries == 0: - raise Exception("Incomplete download for URL {}: {}".format(url, err)) - continue - - # Check if file download was completed. - if "Content-length" in response.headers: - url_content_length = int(response.headers["Content-length"]) - file_size = os.path.getsize(file_name) - # Retry download if file_size has an unexpected size. - if url_content_length != file_size: - download_retries -= 1 - if download_retries == 0: - raise Exception("Downloaded file size ({} bytes) doesn't match content length" - "({} bytes) for URL {}".format(file_size, url_content_length, - url)) - continue - - return True - - raise Exception("Unknown download problem for {} to file {}".format(url, file_name)) - - -class MultiVersionDownloader(object): # pylint: disable=too-many-instance-attributes - """Class to support multiversion downloads.""" - - def __init__( # pylint: disable=too-many-arguments - self, install_dir, link_dir, edition, platform, architecture, use_latest=False): - """Initialize MultiVersionDownloader.""" - self.install_dir = install_dir - self.link_dir = link_dir - self.edition = edition.lower() - self.platform = platform.lower() - self.architecture = architecture.lower() - self.generic_platform = "linux" - self.generic_architecture = "x86_64" - self.use_latest = use_latest - self._links = None - self._generic_links = None - - @property - def generic_links(self): - """Get a list of generic links.""" - if self._generic_links is None: - self._links, self._generic_links = self.download_links() - return self._generic_links - - @property - def links(self): - """Get a list of links.""" - if self._links is None: - self._links, self._generic_links = self.download_links() - return self._links - - @staticmethod - def is_major_minor_version(version): - """Return True if the version is specified as M.m.""" - if re.match(r"^\d+?\.\d+?$", version) is None: - return False - return True - - def download_links(self): - """Return the download and generic download links.""" - temp_file = tempfile.mktemp() - download_file("https://downloads.mongodb.org/full.json", temp_file) - with open(temp_file) as file_handle: - full_json = json.load(file_handle) - os.remove(temp_file) - if "versions" not in full_json: - raise Exception("No versions field in JSON: \n" + str(full_json)) - - links = {} - generic_links = {} - # The generic target contains a platform and architecture. - generic_target = "{}_{}".format(self.generic_platform, self.generic_architecture) - for json_version in full_json["versions"]: - if "version" not in json_version or 'downloads' not in json_version: - continue - version = json_version["version"] - for download in json_version["downloads"]: - if "target" not in download or "edition" not in download: - continue - if (download["target"].lower() == self.platform - and download["arch"].lower() == self.architecture - and download["edition"].lower() == self.edition): - links[version] = download["archive"]["url"] - elif (download["target"].lower() == generic_target - and download["edition"].lower() == "base"): - generic_links[version] = download["archive"]["url"] - - return links, generic_links - - def download_install(self, version): - """Download and install the version specified.""" - dl_file = self.download_version(version) - if dl_file: - installed_dir = self.uncompress_download(dl_file) - self.symlink_version(version, installed_dir) - - def download_version(self, version): # pylint: disable=too-many-branches,too-many-locals,too-many-statements - """Download the version specified and return file location. - - If no download occurs, file location is None. - """ - - try: - os.makedirs(self.install_dir) - except OSError as exc: - if exc.errno == errno.EEXIST and os.path.isdir(self.install_dir): - pass - else: - raise - - urls = [] - requested_version_parts = get_version_parts(version) - for link_version, link_url in self.links.items(): - link_version_parts = get_version_parts(link_version) - if link_version_parts[:len(requested_version_parts)] == requested_version_parts: - # The 'link_version' is a candidate for the requested 'version' if - # (a) it is a prefix of the requested version, or if - # (b) it is the "<branchname>-latest" version and the requested version is for a - # particular major release. - # This is equivalent to the 'link_version' having components equal to all of the - # version parts that make up 'version'. - if "-" in version: - # The requested 'version' contains a hyphen, so we only consider exact matches - # to that version. - if link_version != version: - continue - urls.append((link_version, link_url)) - - if not urls: - print( - "Cannot find a link for version {}, versions {} found.".format(version, self.links), - file=sys.stderr) - for ver, generic_url in self.generic_links.items(): - parts = get_version_parts(ver) - if parts[:len(requested_version_parts)] == requested_version_parts: - if "-" in version and ver != version: - continue - urls.append((ver, generic_url)) - if not urls: - if version not in ["4.7", "4.8"]: - raise Exception( - "No fall-back generic link available or version {}.".format(version)) - - print( - "manually constructing URL for {} releases; please note that only the latest release is available" - .format(version)) - - url_template = "https://downloads.mongodb.com/{bucket}/mongodb-{os_family}-{arch}-enterprise-{platform}-{version}.{suffix}" - - suffix = "tgz" - - if self.platform == "windows": - os_family = self.platform - bucket = self.platform - suffix = "zip" - elif self.platform == "osx": - os_family = "macos" - bucket = self.platform - else: - os_family = "linux" - bucket = "linux" - - if self.platform == "osx" or self.platform == "windows": - # macOS and Windows URLs don't have the platform portion. - platform = "" - elif self.platform == "amazon": - # The community version and the enterprise version have different platform names. - platform = "amzn64" - else: - platform = self.platform - - url = url_template.format(bucket=bucket, os_family=os_family, - arch=self.architecture, platform=platform, - version=version, suffix=suffix) - - # URLs with missing sections lead to double dashes. - url = url.replace("--", "-") - urls.append((version, url)) - else: - print("Falling back to generic architecture.") - - urls.sort(key=lambda link: get_version_parts(link[0], for_sorting=True)) - full_version = urls[-1][0] - url = urls[-1][1] - extract_dir = url.split("/")[-1][:-4] - file_suffix = os.path.splitext(urllib.parse.urlparse(url).path)[1] - - # Only download if we don't already have the directory. - # Note, we cannot detect if 'latest' has already been downloaded, as the name - # of the 'extract_dir' cannot be derived from the URL, since it contains the githash. - already_downloaded = os.path.isdir(os.path.join(self.install_dir, extract_dir)) - if already_downloaded: - print("Skipping download for version {} ({}) since the dest already exists '{}'".format( - version, full_version, extract_dir)) - return None - else: - temp_file = tempfile.mktemp(suffix=file_suffix) - - latest_downloaded = False - # We try to download 'v<version>-latest' if the 'version' is specified - # as Major.minor. If that fails, we then try to download the version that - # was specified. - if self.use_latest and self.is_major_minor_version(version): - latest_version = "v{}-latest".format(version) - latest_url = url.replace(full_version, latest_version) - print("Trying to download {}...".format(latest_version)) - print("Download url is {}".format(latest_url)) - try: - download_file(latest_url, temp_file) - full_version = latest_version - latest_downloaded = True - except requests.exceptions.HTTPError: - print("Failed to download {}".format(latest_url)) - - if not latest_downloaded: - print("Downloading data for version {} ({})...".format(version, full_version)) - print("Download url is {}".format(url)) - download_file(url, temp_file) - return temp_file - - def uncompress_download(self, dl_file): - """Download the version specified and return root of extracted directory.""" - - print("Uncompressing data to {}...".format(self.install_dir)) - first_file = "" - temp_dir = tempfile.mkdtemp() - _, file_suffix = os.path.splitext(dl_file) - if file_suffix == ".zip": - # Support .zip downloads, used for Windows binaries. - with zipfile.ZipFile(dl_file) as zip_handle: - # Use the name of the root directory in the archive as the name of the directory - # to extract the binaries into inside 'self.install_dir'. The name of the root - # directory nearly always matches the parsed URL text, with the exception of - # versions such as "v3.2-latest" that instead contain the githash. - first_file = zip_handle.namelist()[0] - zip_handle.extractall(temp_dir) - elif file_suffix == ".tgz": - # Support .tgz downloads, used for Linux binaries. - with contextlib.closing(tarfile.open(dl_file, "r:gz")) as tar_handle: - # Use the name of the root directory in the archive as the name of the directory - # to extract the binaries into inside 'self.install_dir'. The name of the root - # directory nearly always matches the parsed URL text, with the exception of - # versions such as "v3.2-latest" that instead contain the githash. - first_file = tar_handle.getnames()[0] - tar_handle.extractall(path=temp_dir) - else: - raise Exception("Unsupported file extension {}".format(file_suffix)) - - # Sometimes the zip will contain the root directory as the first file and - # os.path.dirname() will return ''. - extract_dir = os.path.dirname(first_file) - if not extract_dir: - extract_dir = first_file - temp_install_dir = os.path.join(temp_dir, extract_dir) - - # We may not have been able to determine whether we already downloaded the requested - # version due to the ambiguity in the parsed URL text, so we check for it again using - # the adjusted 'extract_dir' value. - already_downloaded = os.path.isdir(os.path.join(self.install_dir, extract_dir)) - if not already_downloaded: - shutil.move(temp_install_dir, self.install_dir) - - shutil.rmtree(temp_dir) - os.remove(dl_file) - - return os.path.abspath(os.path.join(self.install_dir, extract_dir)) - - def symlink_version(self, version, installed_dir): - """Symlink the binaries in the 'installed_dir' to the 'link_dir'.""" - try: - os.makedirs(self.link_dir) - except OSError as exc: - if exc.errno == errno.EEXIST and os.path.isdir(self.link_dir): - pass - else: - raise - - for executable in os.listdir(os.path.join(installed_dir, "bin")): - - executable_name, executable_extension = os.path.splitext(executable) - link_name = "{}-{}{}".format(executable_name, version, executable_extension) - - try: - executable = os.path.join(installed_dir, "bin", executable) - executable_link = os.path.join(self.link_dir, link_name) - if os.name == "nt": - # os.symlink is not supported on Windows, use a direct method instead. - def symlink_ms(source, link_name): - """Provide symlink for Windows.""" - import ctypes - csl = ctypes.windll.kernel32.CreateSymbolicLinkW - csl.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32) - csl.restype = ctypes.c_ubyte - flags = 1 if os.path.isdir(source) else 0 - if csl(link_name, source.replace("/", "\\"), flags) == 0: - raise ctypes.WinError() - - os.symlink = symlink_ms - os.symlink(executable, executable_link) - except OSError as exc: - if exc.errno == errno.EEXIST: - pass - else: - raise - - -def main(): - """Execute Main program.""" - - # Listen for SIGUSR1 and dump stack if received. - try: - signal.signal(signal.SIGUSR1, dump_stacks) - except AttributeError: - print("Cannot catch signals on Windows") - - parser = optparse.OptionParser(usage=""" -Downloads and installs particular mongodb versions (each binary is renamed -to include its version) into an install directory and symlinks the binaries -with versions to another directory. This script supports community and -enterprise builds. - -Usage: setup_multiversion_mongodb.py [options] ver1 [vers2 ...] - -Ex: setup_multiversion_mongodb.py --installDir ./install - --linkDir ./link - --edition base - --platform Linux 2.0.6 2.0.3-rc0 - --architecture x86_64 - 2.0 2.2 2.3 -Ex: setup_multiversion_mongodb.py --installDir ./install - --linkDir ./link - --edition enterprise - --platform osx - 2.4 2.2 - -After running the script you will have a directory structure like this: - ./install/[mongodb-osx-x86_64-2.4.9, mongodb-osx-x86_64-2.2.7] - ./link/[mongod-2.4.9, mongod-2.2.7, mongo-2.4.9...] - -You should then add ./link/ to your path so multi-version tests will work. - -Note: If "rc" is included in the version name, we'll use the exact rc, otherwise -we'll pull the highest non-rc version compatible with the version specified. -""") - - parser.add_option("-i", "--installDir", dest="install_dir", - help="Directory to install the download archive. [REQUIRED]", default=None) - parser.add_option( - "-l", "--linkDir", dest="link_dir", - help=("Directory to contain links to all binaries for each version in" - " the install directory. [REQUIRED]"), default=None) - editions = ["base", "enterprise", "targeted"] - parser.add_option( - "-e", "--edition", dest="edition", choices=editions, - help=("Edition of the build to download, choose from {}, [default:" - " '%default'].".format(editions)), default="base") - parser.add_option( - "-p", "--platform", dest="platform", - help=("Platform to download [REQUIRED]. Examples include: 'linux'," - " 'osx', 'rhel62', 'windows'."), default=None) - parser.add_option( - "-a", "--architecture", dest="architecture", - help=("Architecture to download, [default: '%default']. Examples include:" - " 'arm64', 'ppc64le', 's390x' and 'x86_64'."), default="x86_64") - parser.add_option( - "-u", "--useLatest", dest="use_latest", action="store_true", - help=("If specified, the latest (nightly) version will be downloaded," - " if it exists, for the version specified. For example, if specifying" - " version 3.2 for download, the nightly version for 3.2 will be" - " downloaded if it exists, otherwise the 'highest' version will be" - " downloaded, i.e., '3.2.17'"), default=False) - - options, versions = parser.parse_args() - - # Check for required options. - if not versions or not options.install_dir or not options.link_dir or not options.platform: - parser.print_help() - parser.exit(1) - - downloader = MultiVersionDownloader(options.install_dir, options.link_dir, options.edition, - options.platform, options.architecture, options.use_latest) - - for version in versions: - downloader.download_install(version) - +"""Stub file pointing users to the new invocation.""" if __name__ == "__main__": - main() + print( + "Hello! It seems you've executed 'buildscripts/setup_multiversion_mongodb.py'. We have\n" + "repackaged the setup multiversion as a subcommand of resmoke. It can now be invoked as\n" + "'./buildscripts/resmoke.py setup-multiversion' with all of the same arguments as before.\n" + "Please use './buildscripts/resmoke.py setup-multiversion --help' for more details.") |