summaryrefslogtreecommitdiff
path: root/buildscripts/generate_compile_expansions_shared_cache.py
blob: 337e9669c0f71c94a849cf1d362a1547b9c05679 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
#!/usr/bin/env python3
"""
Generate the compile expansions file used by Evergreen as part of the push/release process.

Invoke by specifying an output file.
$ python generate_compile_expansions.py --out compile_expansions.yml
"""

import argparse
import json
import os
import re
import sys
import shlex
import yaml

VERSION_JSON = "version.json"


def generate_expansions():
    """Entry point for the script.

    This calls functions to generate version and scons cache expansions and
    writes them to a file.
    """
    args = parse_args()
    expansions = {}
    expansions.update(generate_version_expansions())
    expansions.update(generate_scons_cache_expansions())

    with open(args.out, "w") as out:
        print("saving compile expansions to {0}: ({1})".format(args.out, expansions))
        yaml.safe_dump(expansions, out, default_flow_style=False)


def parse_args():
    """Parse program arguments."""
    parser = argparse.ArgumentParser()
    parser.add_argument("--out", required=True)
    return parser.parse_args()


def generate_version_expansions():
    """Generate expansions from a version.json file if given, or $MONGO_VERSION."""
    expansions = {}

    if os.path.exists(VERSION_JSON):
        with open(VERSION_JSON, "r") as fh:
            data = fh.read()
            version_data = json.loads(data)
        version_line = version_data['version']
        version_parts = match_verstr(version_line)
        if not version_parts:
            raise ValueError("Unable to parse version.json")
    else:
        if not os.getenv("MONGO_VERSION"):
            raise Exception("$MONGO_VERSION not set and no version.json provided")
        version_line = os.getenv("MONGO_VERSION").lstrip("r")
        version_parts = match_verstr(version_line)
        if not version_parts:
            raise ValueError("Unable to parse version from stdin and no version.json provided")

    if version_parts[0]:
        expansions["suffix"] = "latest"
        expansions["src_suffix"] = "latest"
        expansions["is_release"] = "false"
    else:
        expansions["suffix"] = version_line
        expansions["src_suffix"] = "r{0}".format(version_line)
        expansions["is_release"] = "true"
    expansions["version"] = version_line

    return expansions


def generate_scons_cache_expansions():
    """Generate scons cache expansions from some files and environment variables."""
    expansions = {}

    # Get the scons cache mode
    scons_cache_mode = os.getenv("SCONS_CACHE_MODE", "nolinked")

    # Get the host uuid
    if sys.platform.startswith("win"):
        system_id_path = r"c:\mongodb-build-system-id"
    else:
        system_id_path = "/etc/mongodb-build-system-id"

    if os.path.isfile(system_id_path):
        with open(system_id_path, "r") as fh:
            system_uuid = fh.readline().strip()

    # Set the scons shared cache setting

    # Global shared cache using EFS
    if os.getenv("SCONS_CACHE_SCOPE") == "shared":
        if sys.platform.startswith("win"):
            shared_mount_root = 'X:\\'
        else:
            shared_mount_root = '/efs'
        default_cache_path = os.path.join(shared_mount_root, system_uuid, "scons-cache")
        expansions["scons_cache_path"] = default_cache_path
        expansions[
            "scons_cache_args"] = "--cache={0} --cache-signature-mode=validate --cache-dir={1}".format(
                scons_cache_mode, shlex.quote(default_cache_path))

    # Local shared cache - host-based
    elif os.getenv("SCONS_CACHE_SCOPE") == "local":

        if sys.platform.startswith("win"):
            default_cache_path_base = r"z:\data\scons-cache"
        else:
            default_cache_path_base = "/data/scons-cache"

        default_cache_path = os.path.join(default_cache_path_base, system_uuid)
        expansions["scons_cache_path"] = default_cache_path
        expansions[
            "scons_cache_args"] = "--cache={0} --cache-signature-mode=validate --cache-dir={1}".format(
                scons_cache_mode, shlex.quote(default_cache_path))
    # No cache
    else:
        # Anything else is 'none'
        print("No cache used")

    return expansions


def match_verstr(verstr):
    """Match a version string and capture the "extra" part.

    If the version is a release like "2.3.4" or "2.3.4-rc0", this will return
    None. If the version is a pre-release like "2.3.4-325-githash" or
    "2.3.4-pre-", this will return "-pre-" or "-325-githash" If the version
    begins with the letter 'r', it will also match, e.g. r2.3.4, r2.3.4-rc0,
    r2.3.4-git234, r2.3.4-rc0-234-githash If the version is invalid (i.e.
    doesn't start with "2.3.4" or "2.3.4-rc0", this will return False.
    """
    res = re.match(r'^r?(?:\d+\.\d+\.\d+(?:-rc\d+|-alpha\d+)?)(-.*)?', verstr)
    if not res:
        return False
    return res.groups()


if __name__ == "__main__":
    generate_expansions()