1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
|
#!/usr/bin/env python
"""Script to retrieve the etc/test_lifecycle.yml tag file from the metadata repository that
corresponds to the current repository.
Usage:
python buildscsripts/fetch_test_lifecycle.py evergreen-project revision
"""
from __future__ import absolute_import
from __future__ import print_function
import logging
import optparse
import os
import posixpath
import shutil
import sys
import textwrap
import yaml
# Get relative imports to work when the package is not installed on the PYTHONPATH.
if __name__ == "__main__" and __package__ is None:
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from buildscripts import git
LOGGER = logging.getLogger(__name__)
class MetadataRepository(object):
"""Represent the metadata repository containing the test lifecycle tags file."""
def __init__(self, repository, references_file, lifecycle_file):
"""Initlialize the MetadataRepository.
Args:
repository: the git.Repository object for the repository.
references_file: the relative path from the root of the repository to the references
yaml file.
lifecycle_file: the relative path from the root of the repository to the test lifecycle
tags yaml file.
"""
self._repository = repository
self._references_file = references_file
self._lifecycle_file = lifecycle_file
# The path to the lifecycle file, absolute or relative to the current working directory.
self.lifecycle_path = os.path.join(repository.directory, lifecycle_file)
def list_revisions(self):
"""List the revisions from the HEAD of this repository.
Returns:
A list of str containing the git hashes for all the revisions from the newest (HEAD)
to the oldest.
"""
return self._repository.git_rev_list(["HEAD", "--", self._lifecycle_file]).splitlines()
def _get_references_content(self, revision):
references_content = self._repository.git_cat_file(
["blob", "%s:%s" % (revision, self._references_file)])
return references_content
def get_reference(self, metadata_revision, project):
"""Retrieve the reference revision (a revision of the project 'project') associated with
the test lifecycle file present in the metadata repository at revision 'metadata_revision'.
Args:
metadata_revision: a revision (git hash) of this repository.
project: an Evergreen project name (e.g. mongodb-mongo-master).
"""
references_content = self._get_references_content(metadata_revision)
references = yaml.safe_load(references_content)
return references.get("test-lifecycle", {}).get(project)
def get_lifecycle_file_content(self, metadata_revision):
"""Return the content of the test lifecycle file as it was at the given revision."""
return self._repository.git_cat_file(
["blob", "%s:%s" % (metadata_revision, self._lifecycle_file)])
def _clone_repository(url, branch):
"""Clone the repository present at the URL 'url' and use the branch 'branch'."""
target_directory = posixpath.splitext(posixpath.basename(url))[0]
LOGGER.info("Cloning the repository %s into the directory %s", url, target_directory)
return git.Repository.clone(url, target_directory, branch)
def _get_metadata_revision(metadata_repo, mongo_repo, project, revision):
"""Get the metadata revision that corresponds to a given repository, project, revision."""
for metadata_revision in metadata_repo.list_revisions():
reference = metadata_repo.get_reference(metadata_revision, project)
if not reference:
# No reference for this revision. This should not happen but we keep trying in
# case we can find an older revision with a reference.
continue
if mongo_repo.is_ancestor(reference, revision):
# We found a reference that is a parent of the current revision.
return metadata_revision
return None
def fetch_test_lifecycle(metadata_repo_url, references_file, lifecycle_file, project, revision):
"""Fetch the test lifecycle file that corresponds to the given revision of the repository this
script is called from.
Args:
metadata_repo_url: the git repository URL for the metadata repository containing the test
lifecycle file.
references_file: the relative path from the root of the metadata repository to the
references file.
lifecycle_file: the relative path from the root of the metadata repository to the test
lifecycle file.
project: the Evergreen project name.
revision: the current repository revision.
"""
metadata_repo = MetadataRepository(
_clone_repository(metadata_repo_url, project), references_file, lifecycle_file)
mongo_repo = git.Repository(os.getcwd())
metadata_revision = _get_metadata_revision(metadata_repo, mongo_repo, project, revision)
if metadata_revision:
LOGGER.info("Using metadata repository revision '%s'", metadata_revision)
result = metadata_repo.get_lifecycle_file_content(metadata_revision)
else:
result = None
return result
def main():
"""
Utility to fetch the etc/test_lifecycle.yml file corresponding to a given revision from
the mongo-test-metadata repository.
"""
parser = optparse.OptionParser(
description=textwrap.dedent(main.__doc__), usage="Usage: %prog [options] evergreen-project")
parser.add_option("--revision", dest="revision", metavar="<revision>", default="HEAD",
help=("The project revision for which to retrieve the test lifecycle tags"
" file."))
parser.add_option("--metadataRepo", dest="metadata_repo_url", metavar="<metadata-repo-url>",
default="git@github.com:mongodb/mongo-test-metadata.git",
help=("The URL to the metadata repository that contains the test lifecycle"
" tags file."))
parser.add_option("--lifecycleFile", dest="lifecycle_file", metavar="<lifecycle-file>",
default="etc/test_lifecycle.yml",
help=("The path to the test lifecycle tags file, relative to the root of the"
" metadata repository. Defaults to '%default'."))
parser.add_option("--referencesFile", dest="references_file", metavar="<references-file>",
default="references.yml",
help=("The path to the metadata references file, relative to the root of the"
" metadata repository. Defaults to '%default'."))
parser.add_option("--destinationFile", dest="destination_file", metavar="<destination-file>",
default="etc/test_lifecycle.yml",
help=("The path where the lifecycle file should be available when this script"
" completes successfully. This path is absolute or relative to the"
" current working directory. Defaults to '%default'."))
parser.add_option("--logLevel", dest="log_level", metavar="<log-level>",
choices=["DEBUG", "INFO", "WARNING", "ERROR"], default="INFO",
help="The log level: DEBUG, INFO, WARNING or ERROR. Defaults to '%default'.")
parser.add_option("--logFile", dest="log_file", metavar="<log-file>", default=None,
help=("The destination file for the logs. If not set the script will log to"
" the standard output"))
options, args = parser.parse_args()
if len(args) != 1:
parser.print_help(file=sys.stderr)
print(file=sys.stderr)
parser.error("Must specify an Evergreen project")
evergreen_project = args[0]
logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s", level=options.log_level,
filename=options.log_file)
lifecycle_file_content = fetch_test_lifecycle(options.metadata_repo_url,
options.references_file, options.lifecycle_file,
evergreen_project, options.revision)
if not lifecycle_file_content:
LOGGER.error("Failed to fetch the test lifecycle tag file.")
sys.exit(1)
else:
LOGGER.info("Writing the test lifecycle file to '%s'.", options.destination_file)
with open(options.destination_file, "wb") as destf:
destf.write(lifecycle_file_content)
LOGGER.info("Done.")
if __name__ == "__main__":
main()
|
