1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
|
"""
Utility functions to create MongoDB processes.
Handles all the nitty-gritty parameter conversion.
"""
from __future__ import absolute_import
import json
import os
import os.path
import stat
from . import process as _process
from .. import utils
from .. import config
def mongod_program(logger, executable=None, process_kwargs=None, **kwargs):
"""
Returns a Process instance that starts a mongod executable with
arguments constructed from 'kwargs'.
"""
executable = utils.default_if_none(executable, config.DEFAULT_MONGOD_EXECUTABLE)
args = [executable]
# Apply the --setParameter command line argument. Command line options to resmoke.py override
# the YAML configuration.
suite_set_parameters = kwargs.pop("set_parameters", {})
if config.MONGOD_SET_PARAMETERS is not None:
suite_set_parameters.update(utils.load_yaml(config.MONGOD_SET_PARAMETERS))
_apply_set_parameters(args, suite_set_parameters)
shortcut_opts = {
"nojournal": config.NO_JOURNAL,
"nopreallocj": config.NO_PREALLOC_JOURNAL,
"storageEngine": config.STORAGE_ENGINE,
"wiredTigerCollectionConfigString": config.WT_COLL_CONFIG,
"wiredTigerEngineConfigString": config.WT_ENGINE_CONFIG,
"wiredTigerIndexConfigString": config.WT_INDEX_CONFIG,
}
# These options are just flags, so they should not take a value.
opts_without_vals = ("nojournal", "nopreallocj")
# Have the --nojournal command line argument to resmoke.py unset the journal option.
if shortcut_opts["nojournal"] is not None and "journal" in kwargs:
del kwargs["journal"]
for opt_name in shortcut_opts:
if shortcut_opts[opt_name] is not None:
# Command line options override the YAML configuration.
if opt_name in opts_without_vals:
kwargs[opt_name] = ""
else:
kwargs[opt_name] = shortcut_opts[opt_name]
# Apply the rest of the command line arguments.
_apply_kwargs(args, kwargs)
if "keyFile" in kwargs:
_set_keyfile_permissions(kwargs["keyFile"])
process_kwargs = utils.default_if_none(process_kwargs, {})
return _process.Process(logger, args, **process_kwargs)
def mongos_program(logger, executable=None, process_kwargs=None, **kwargs):
"""
Returns a Process instance that starts a mongos executable with
arguments constructed from 'kwargs'.
"""
executable = utils.default_if_none(executable, config.DEFAULT_MONGOS_EXECUTABLE)
args = [executable]
# Apply the --setParameter command line argument. Command line options to resmoke.py override
# the YAML configuration.
suite_set_parameters = kwargs.pop("set_parameters", {})
if config.MONGOS_SET_PARAMETERS is not None:
suite_set_parameters.update(utils.load_yaml(config.MONGOS_SET_PARAMETERS))
_apply_set_parameters(args, suite_set_parameters)
# Apply the rest of the command line arguments.
_apply_kwargs(args, kwargs)
if "keyFile" in kwargs:
_set_keyfile_permissions(kwargs["keyFile"])
process_kwargs = utils.default_if_none(process_kwargs, {})
return _process.Process(logger, args, **process_kwargs)
def mongo_shell_program(logger, executable=None, filename=None, process_kwargs=None, **kwargs):
"""
Returns a Process instance that starts a mongo shell with arguments
constructed from 'kwargs'.
"""
executable = utils.default_if_none(executable, config.DEFAULT_MONGO_EXECUTABLE)
args = [executable]
eval_sb = [] # String builder.
global_vars = kwargs.pop("global_vars", {})
shortcut_opts = {
"noJournal": (config.NO_JOURNAL, False),
"noJournalPrealloc": (config.NO_PREALLOC_JOURNAL, False),
"storageEngine": (config.STORAGE_ENGINE, ""),
"testName": (os.path.splitext(os.path.basename(filename))[0], ""),
"wiredTigerCollectionConfigString": (config.WT_COLL_CONFIG, ""),
"wiredTigerEngineConfigString": (config.WT_ENGINE_CONFIG, ""),
"wiredTigerIndexConfigString": (config.WT_INDEX_CONFIG, ""),
}
test_data = global_vars.get("TestData", {}).copy()
for opt_name in shortcut_opts:
(opt_value, opt_default) = shortcut_opts[opt_name]
if opt_value is not None:
test_data[opt_name] = opt_value
elif opt_name not in test_data:
# Only use 'opt_default' if the property wasn't set in the YAML configuration.
test_data[opt_name] = opt_default
global_vars["TestData"] = test_data
for var_name in global_vars:
_format_shell_vars(eval_sb, var_name, global_vars[var_name])
if "eval" in kwargs:
eval_sb.append(kwargs.pop("eval"))
eval_str = "; ".join(eval_sb)
args.append("--eval")
args.append(eval_str)
if config.SHELL_WRITE_MODE is not None:
kwargs["writeMode"] = config.SHELL_WRITE_MODE
# Apply the rest of the command line arguments.
_apply_kwargs(args, kwargs)
# Have the mongos shell run the specified file.
args.append(filename)
if "keyFile" in global_vars["TestData"]:
_set_keyfile_permissions(global_vars["TestData"]["keyFile"])
process_kwargs = utils.default_if_none(process_kwargs, {})
return _process.Process(logger, args, **process_kwargs)
def _format_shell_vars(sb, path, value):
"""
Formats 'value' in a way that can be passed to --eval.
If 'value' is a dictionary, then it is unrolled into the creation of
a new JSON object with properties assigned for each key of the
dictionary.
"""
# Only need to do special handling for JSON objects.
if not isinstance(value, dict):
sb.append("%s = %s" % (path, json.dumps(value)))
return
# Avoid including curly braces and colons in output so that the command invocation can be
# copied and run through bash.
sb.append("%s = new Object()" % (path))
for subkey in value:
_format_shell_vars(sb, ".".join((path, subkey)), value[subkey])
def dbtest_program(logger, executable=None, suites=None, process_kwargs=None, **kwargs):
"""
Returns a Process instance that starts a dbtest executable with
arguments constructed from 'kwargs'.
"""
executable = utils.default_if_none(executable, config.DEFAULT_DBTEST_EXECUTABLE)
args = [executable]
if suites is not None:
args.extend(suites)
if config.STORAGE_ENGINE is not None:
kwargs["storageEngine"] = config.STORAGE_ENGINE
for arg_name in kwargs:
arg_value = str(kwargs[arg_name])
args.append("--%s" % (arg_name))
if arg_value:
args.append(arg_value)
process_kwargs = utils.default_if_none(process_kwargs, {})
return _process.Process(logger, args, **process_kwargs)
def _apply_set_parameters(args, set_parameter):
"""
Converts key-value pairs from 'kwargs' into --setParameter key=value
arguments to an executable and appends them to 'args'.
"""
for param_name in set_parameter:
param_value = set_parameter[param_name]
# --setParameter takes boolean values as lowercase strings.
if isinstance(param_value, bool):
param_value = "true" if param_value else "false"
args.append("--setParameter")
args.append("%s=%s" % (param_name, param_value))
def _apply_kwargs(args, kwargs):
"""
Converts key-value pairs from 'kwargs' into --key value arguments
to an executable and appends them to 'args'.
A --flag without a value is represented with the empty string.
"""
for arg_name in kwargs:
arg_value = str(kwargs[arg_name])
args.append("--%s" % (arg_name))
if arg_value:
args.append(arg_value)
def _set_keyfile_permissions(keyfile_path):
"""
Change the permissions on 'keyfile_path' to 600, i.e. only the user
can read and write the file.
This necessary to avoid having the mongod/mongos fail to start up
because "permissions on 'keyfile_path' are too open".
"""
os.chmod(keyfile_path, stat.S_IRUSR | stat.S_IWUSR)
|
