summaryrefslogtreecommitdiff
path: root/buildscripts/setup_multiversion_mongodb.py
blob: bad860184bd53f5c430edf880da8cc35278296cd (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
#!/usr/bin/env python3
"""Install multiple versions of MongoDB on a machine."""

import contextlib
import errno
import json
import optparse
import os
import re
import shutil
import signal
import sys
import tarfile
import tempfile
import threading
import traceback
import urllib.parse
import zipfile

import requests
import requests.exceptions


def dump_stacks(_signal_num, _frame):  # pylint: disable=unused-argument
    """Dump stacks when SIGUSR1 is received."""
    print("======================================")
    print("DUMPING STACKS due to SIGUSR1 signal")
    print("======================================")
    threads = threading.enumerate()

    print("Total Threads: {:d}".format(len(threads)))

    for tid, stack in list(sys._current_frames().items()):  # pylint: disable=protected-access
        print("Thread {:d}".format(tid))
        print("".join(traceback.format_stack(stack)))
    print("======================================")


def get_version_parts(version, for_sorting=False):
    """Return a list containing the components of the version string as numeric values.

    This function can be used for numeric sorting
    of version strings such as '2.6.0-rc1' and '2.4.0' when the
    'for_sorting' parameter is specified as true.
    """

    rc_offset = -100
    version_parts = re.split(r"\.|-", version)

    if version_parts[-1] == "pre":
        # Prior to improvements for how the version string is managed within the server
        # (SERVER-17782), the binary archives would contain a trailing "-pre".
        version_parts.pop()

    if version_parts[-1].startswith("rc"):
        # RC versions are weighted down to allow future RCs and general
        # releases to be sorted in ascending order (e.g., 2.6.0-rc1,
        # 2.6.0-rc2, 2.6.0).
        version_parts[-1] = int(version_parts[-1][2:]) + rc_offset
    elif version_parts[0].startswith("v") and version_parts[-1] == "latest":
        version_parts[0] = version_parts[0][1:]
        # The "<branchname>-latest" versions are weighted the highest when a particular major
        # release is requested.
        version_parts[-1] = float("inf")
    elif for_sorting:
        # We want to have the number of components in the resulting version parts match the number
        # of components in the 'version' string if we aren't going to be using them for sorting.
        # Otherwise, we append an additional 0 to non-RC releases so that version lists like
        # [2, 6, 0, -100] and [2, 6, 0, 0] sort in ascending order.
        version_parts.append(0)

    return [float(part) for part in version_parts]


def download_file(url, file_name, download_retries=5):
    """Return True if download was successful. Raises error if download fails."""

    while download_retries > 0:

        with requests.Session() as session:
            adapter = requests.adapters.HTTPAdapter(max_retries=download_retries)
            session.mount(url, adapter)
            response = session.get(url, stream=True)
            response.raise_for_status()

            with open(file_name, "wb") as file_handle:
                try:
                    for block in response.iter_content(1024 * 1000):
                        file_handle.write(block)
                except requests.exceptions.ChunkedEncodingError as err:
                    download_retries -= 1
                    if download_retries == 0:
                        raise Exception("Incomplete download for URL {}: {}".format(url, err))
                    continue

        # Check if file download was completed.
        if "Content-length" in response.headers:
            url_content_length = int(response.headers["Content-length"])
            file_size = os.path.getsize(file_name)
            # Retry download if file_size has an unexpected size.
            if url_content_length != file_size:
                download_retries -= 1
                if download_retries == 0:
                    raise Exception("Downloaded file size ({} bytes) doesn't match content length"
                                    "({} bytes) for URL {}".format(file_size, url_content_length,
                                                                   url))
                continue

        return True

    raise Exception("Unknown download problem for {} to file {}".format(url, file_name))


class MultiVersionDownloader(object):  # pylint: disable=too-many-instance-attributes
    """Class to support multiversion downloads."""

    def __init__(  # pylint: disable=too-many-arguments
            self, install_dir, link_dir, edition, platform, architecture, use_latest=False):
        """Initialize MultiVersionDownloader."""
        self.install_dir = install_dir
        self.link_dir = link_dir
        self.edition = edition.lower()
        self.platform = platform.lower()
        self.architecture = architecture.lower()
        self.generic_platform = "linux"
        self.generic_architecture = "x86_64"
        self.use_latest = use_latest
        self._links = None
        self._generic_links = None

    @property
    def generic_links(self):
        """Get a list of generic links."""
        if self._generic_links is None:
            self._links, self._generic_links = self.download_links()
        return self._generic_links

    @property
    def links(self):
        """Get a list of links."""
        if self._links is None:
            self._links, self._generic_links = self.download_links()
        return self._links

    @staticmethod
    def is_major_minor_version(version):
        """Return True if the version is specified as M.m."""
        if re.match(r"^\d+?\.\d+?$", version) is None:
            return False
        return True

    def download_links(self):
        """Return the download and generic download links."""
        temp_file = tempfile.mktemp()
        download_file("https://downloads.mongodb.org/full.json", temp_file)
        with open(temp_file) as file_handle:
            full_json = json.load(file_handle)
        os.remove(temp_file)
        if "versions" not in full_json:
            raise Exception("No versions field in JSON: \n" + str(full_json))

        links = {}
        generic_links = {}
        # The generic target contains a platform and architecture.
        generic_target = "{}_{}".format(self.generic_platform, self.generic_architecture)
        for json_version in full_json["versions"]:
            if "version" not in json_version or 'downloads' not in json_version:
                continue
            version = json_version["version"]
            for download in json_version["downloads"]:
                if "target" not in download or "edition" not in download:
                    continue
                if (download["target"].lower() == self.platform
                        and download["arch"].lower() == self.architecture
                        and download["edition"].lower() == self.edition):
                    links[version] = download["archive"]["url"]
                elif (download["target"].lower() == generic_target
                      and download["edition"].lower() == "base"):
                    generic_links[version] = download["archive"]["url"]

        return links, generic_links

    def download_install(self, version):
        """Download and install the version specified."""
        dl_file = self.download_version(version)
        if dl_file:
            installed_dir = self.uncompress_download(dl_file)
            self.symlink_version(version, installed_dir)

    def download_version(self, version):  # pylint: disable=too-many-branches,too-many-locals,too-many-statements
        """Download the version specified and return file location.

        If no download occurs, file location is None.
        """

        try:
            os.makedirs(self.install_dir)
        except OSError as exc:
            if exc.errno == errno.EEXIST and os.path.isdir(self.install_dir):
                pass
            else:
                raise

        urls = []
        requested_version_parts = get_version_parts(version)
        for link_version, link_url in self.links.items():
            link_version_parts = get_version_parts(link_version)
            if link_version_parts[:len(requested_version_parts)] == requested_version_parts:
                # The 'link_version' is a candidate for the requested 'version' if
                #   (a) it is a prefix of the requested version, or if
                #   (b) it is the "<branchname>-latest" version and the requested version is for a
                #       particular major release.
                # This is equivalent to the 'link_version' having components equal to all of the
                # version parts that make up 'version'.
                if "-" in version:
                    # The requested 'version' contains a hyphen, so we only consider exact matches
                    # to that version.
                    if link_version != version:
                        continue
                urls.append((link_version, link_url))

        if not urls:
            print(
                "Cannot find a link for version {}, versions {} found.".format(version, self.links),
                file=sys.stderr)
            for ver, generic_url in self.generic_links.items():
                parts = get_version_parts(ver)
                if parts[:len(requested_version_parts)] == requested_version_parts:
                    if "-" in version and ver != version:
                        continue
                    urls.append((ver, generic_url))
            if not urls:
                raise Exception(
                    "No fall-back generic link available or version {}.".format(version))
            else:
                print("Falling back to generic architecture.")

        urls.sort(key=lambda link: get_version_parts(link[0], for_sorting=True))
        full_version = urls[-1][0]
        url = urls[-1][1]
        extract_dir = url.split("/")[-1][:-4]
        file_suffix = os.path.splitext(urllib.parse.urlparse(url).path)[1]

        # Only download if we don't already have the directory.
        # Note, we cannot detect if 'latest' has already been downloaded, as the name
        # of the 'extract_dir' cannot be derived from the URL, since it contains the githash.
        already_downloaded = os.path.isdir(os.path.join(self.install_dir, extract_dir))
        if already_downloaded:
            print("Skipping download for version {} ({}) since the dest already exists '{}'".format(
                version, full_version, extract_dir))
            return None
        else:
            temp_file = tempfile.mktemp(suffix=file_suffix)

            latest_downloaded = False
            # We try to download 'v<version>-latest' if the 'version' is specified
            # as Major.minor. If that fails, we then try to download the version that
            # was specified.
            if self.use_latest and self.is_major_minor_version(version):
                latest_version = "v{}-latest".format(version)
                latest_url = url.replace(full_version, latest_version)
                print("Trying to download {}...".format(latest_version))
                print("Download url is {}".format(latest_url))
                try:
                    download_file(latest_url, temp_file)
                    full_version = latest_version
                    latest_downloaded = True
                except requests.exceptions.HTTPError:
                    print("Failed to download {}".format(latest_url))

            if not latest_downloaded:
                print("Downloading data for version {} ({})...".format(version, full_version))
                print("Download url is {}".format(url))
                download_file(url, temp_file)
        return temp_file

    def uncompress_download(self, dl_file):
        """Download the version specified and return root of extracted directory."""

        print("Uncompressing data to {}...".format(self.install_dir))
        first_file = ""
        temp_dir = tempfile.mkdtemp()
        _, file_suffix = os.path.splitext(dl_file)
        if file_suffix == ".zip":
            # Support .zip downloads, used for Windows binaries.
            with zipfile.ZipFile(dl_file) as zip_handle:
                # Use the name of the root directory in the archive as the name of the directory
                # to extract the binaries into inside 'self.install_dir'. The name of the root
                # directory nearly always matches the parsed URL text, with the exception of
                # versions such as "v3.2-latest" that instead contain the githash.
                first_file = zip_handle.namelist()[0]
                zip_handle.extractall(temp_dir)
        elif file_suffix == ".tgz":
            # Support .tgz downloads, used for Linux binaries.
            with contextlib.closing(tarfile.open(dl_file, "r:gz")) as tar_handle:
                # Use the name of the root directory in the archive as the name of the directory
                # to extract the binaries into inside 'self.install_dir'. The name of the root
                # directory nearly always matches the parsed URL text, with the exception of
                # versions such as "v3.2-latest" that instead contain the githash.
                first_file = tar_handle.getnames()[0]
                tar_handle.extractall(path=temp_dir)
        else:
            raise Exception("Unsupported file extension {}".format(file_suffix))

        # Sometimes the zip will contain the root directory as the first file and
        # os.path.dirname() will return ''.
        extract_dir = os.path.dirname(first_file)
        if not extract_dir:
            extract_dir = first_file
        temp_install_dir = os.path.join(temp_dir, extract_dir)

        # We may not have been able to determine whether we already downloaded the requested
        # version due to the ambiguity in the parsed URL text, so we check for it again using
        # the adjusted 'extract_dir' value.
        already_downloaded = os.path.isdir(os.path.join(self.install_dir, extract_dir))
        if not already_downloaded:
            shutil.move(temp_install_dir, self.install_dir)

        shutil.rmtree(temp_dir)
        os.remove(dl_file)

        return os.path.abspath(os.path.join(self.install_dir, extract_dir))

    def symlink_version(self, version, installed_dir):
        """Symlink the binaries in the 'installed_dir' to the 'link_dir'."""
        try:
            os.makedirs(self.link_dir)
        except OSError as exc:
            if exc.errno == errno.EEXIST and os.path.isdir(self.link_dir):
                pass
            else:
                raise

        for executable in os.listdir(os.path.join(installed_dir, "bin")):

            executable_name, executable_extension = os.path.splitext(executable)
            link_name = "{}-{}{}".format(executable_name, version, executable_extension)

            try:
                executable = os.path.join(installed_dir, "bin", executable)
                executable_link = os.path.join(self.link_dir, link_name)
                if os.name == "nt":
                    # os.symlink is not supported on Windows, use a direct method instead.
                    def symlink_ms(source, link_name):
                        """Provide symlink for Windows."""
                        import ctypes
                        csl = ctypes.windll.kernel32.CreateSymbolicLinkW
                        csl.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32)
                        csl.restype = ctypes.c_ubyte
                        flags = 1 if os.path.isdir(source) else 0
                        if csl(link_name, source.replace("/", "\\"), flags) == 0:
                            raise ctypes.WinError()

                    os.symlink = symlink_ms
                os.symlink(executable, executable_link)
            except OSError as exc:
                if exc.errno == errno.EEXIST:
                    pass
                else:
                    raise


def main():
    """Execute Main program."""

    # Listen for SIGUSR1 and dump stack if received.
    try:
        signal.signal(signal.SIGUSR1, dump_stacks)
    except AttributeError:
        print("Cannot catch signals on Windows")

    parser = optparse.OptionParser(usage="""
Downloads and installs particular mongodb versions (each binary is renamed
to include its version) into an install directory and symlinks the binaries
with versions to another directory. This script supports community and
enterprise builds.

Usage: setup_multiversion_mongodb.py [options] ver1 [vers2 ...]

Ex: setup_multiversion_mongodb.py --installDir ./install
                                  --linkDir ./link
                                  --edition base
                                  --platform Linux 2.0.6 2.0.3-rc0
                                  --architecture x86_64
                                  2.0 2.2 2.3
Ex: setup_multiversion_mongodb.py --installDir ./install
                                  --linkDir ./link
                                  --edition enterprise
                                  --platform osx
                                  2.4 2.2

After running the script you will have a directory structure like this:
    ./install/[mongodb-osx-x86_64-2.4.9, mongodb-osx-x86_64-2.2.7]
    ./link/[mongod-2.4.9, mongod-2.2.7, mongo-2.4.9...]

You should then add ./link/ to your path so multi-version tests will work.

Note: If "rc" is included in the version name, we'll use the exact rc, otherwise
we'll pull the highest non-rc version compatible with the version specified.
""")

    parser.add_option("-i", "--installDir", dest="install_dir",
                      help="Directory to install the download archive. [REQUIRED]", default=None)
    parser.add_option(
        "-l", "--linkDir", dest="link_dir",
        help=("Directory to contain links to all binaries for each version in"
              " the install directory. [REQUIRED]"), default=None)
    editions = ["base", "enterprise", "targeted"]
    parser.add_option(
        "-e", "--edition", dest="edition", choices=editions,
        help=("Edition of the build to download, choose from {}, [default:"
              " '%default'].".format(editions)), default="base")
    parser.add_option(
        "-p", "--platform", dest="platform",
        help=("Platform to download [REQUIRED]. Examples include: 'linux',"
              " 'osx', 'rhel62', 'windows'."), default=None)
    parser.add_option(
        "-a", "--architecture", dest="architecture",
        help=("Architecture to download, [default: '%default']. Examples include:"
              " 'arm64', 'ppc64le', 's390x' and 'x86_64'."), default="x86_64")
    parser.add_option(
        "-u", "--useLatest", dest="use_latest", action="store_true",
        help=("If specified, the latest (nightly) version will be downloaded,"
              " if it exists, for the version specified. For example, if specifying"
              " version 3.2 for download, the nightly version for 3.2 will be"
              " downloaded if it exists, otherwise the 'highest' version will be"
              " downloaded, i.e., '3.2.17'"), default=False)

    options, versions = parser.parse_args()

    # Check for required options.
    if not versions or not options.install_dir or not options.link_dir or not options.platform:
        parser.print_help()
        parser.exit(1)

    downloader = MultiVersionDownloader(options.install_dir, options.link_dir, options.edition,
                                        options.platform, options.architecture, options.use_latest)

    for version in versions:
        downloader.download_install(version)


if __name__ == "__main__":
    main()