From 5171d72f3a20559f7433d4a6a7283de189cb81a2 Mon Sep 17 00:00:00 2001 From: Trevor Guidry Date: Wed, 16 Nov 2022 17:24:50 +0000 Subject: SERVER-63104 add resmoke argument for generating all_feature_flags.txt locally (cherry picked from commit 22f38cf147eeedca45943ababe818213184eb754) --- buildscripts/idl/gen_all_feature_flag_list.py | 26 +++++++++---------- buildscripts/idl/idl/parser.py | 6 ++--- buildscripts/resmokelib/config.py | 10 +++++--- buildscripts/resmokelib/configure_resmoke.py | 36 +++++++++++++++------------ buildscripts/resmokelib/run/__init__.py | 21 ++++++---------- etc/evergreen.yml | 14 +++++------ etc/perf.yml | 2 +- etc/system_perf.yml | 2 +- evergreen/feature_flag_tags_check.sh | 4 +-- evergreen/gen_feature_flags.sh | 2 +- 10 files changed, 63 insertions(+), 60 deletions(-) diff --git a/buildscripts/idl/gen_all_feature_flag_list.py b/buildscripts/idl/gen_all_feature_flag_list.py index 518583898cb..c7496803b3b 100644 --- a/buildscripts/idl/gen_all_feature_flag_list.py +++ b/buildscripts/idl/gen_all_feature_flag_list.py @@ -30,7 +30,6 @@ Generate a file containing a list of disabled feature flags. Used by resmoke.py to run only feature flag tests. """ -import argparse import os import sys @@ -43,6 +42,7 @@ sys.path.append(os.path.normpath(os.path.join(os.path.abspath(__file__), '../../ # pylint: disable=wrong-import-position import buildscripts.idl.lib as lib +from buildscripts.idl.idl import parser def is_third_party_idl(idl_path: str) -> bool: @@ -56,13 +56,14 @@ def is_third_party_idl(idl_path: str) -> bool: return False -def gen_all_feature_flags(idl_dir: str, import_dirs: List[str]): +def gen_all_feature_flags(idl_dir: str = os.getcwd()): """Generate a list of all feature flags.""" all_flags = [] for idl_path in sorted(lib.list_idls(idl_dir)): if is_third_party_idl(idl_path): continue - for feature_flag in lib.parse_idl(idl_path, import_dirs).spec.feature_flags: + doc = parser.parse_file(open(idl_path), idl_path) + for feature_flag in doc.spec.feature_flags: if feature_flag.default.literal != "true": all_flags.append(feature_flag.name) @@ -72,18 +73,17 @@ def gen_all_feature_flags(idl_dir: str, import_dirs: List[str]): return list(set(all_flags) - set(force_disabled_flags)) -def main(): - """Run the main function.""" - arg_parser = argparse.ArgumentParser(description=__doc__) - arg_parser.add_argument("--import-dir", dest="import_dirs", type=str, action="append", - help="Directory to search for IDL import files") +def gen_all_feature_flags_file(filename: str = lib.ALL_FEATURE_FLAG_FILE): + """Output generated list of feature flags to specified file.""" + flags = gen_all_feature_flags() + with open(filename, "w") as output_file: + output_file.write("\n".join(flags)) + print("Generated: ", os.path.realpath(output_file.name)) - args = arg_parser.parse_args() - flags = gen_all_feature_flags(os.getcwd(), args.import_dirs) - with open(lib.ALL_FEATURE_FLAG_FILE, "w") as output_file: - for flag in flags: - output_file.write("%s\n" % flag) +def main(): + """Run the main function.""" + gen_all_feature_flags_file() if __name__ == '__main__': diff --git a/buildscripts/idl/idl/parser.py b/buildscripts/idl/idl/parser.py index b4d47c2c0c1..7df90b8e2e1 100644 --- a/buildscripts/idl/idl/parser.py +++ b/buildscripts/idl/idl/parser.py @@ -1026,7 +1026,7 @@ def _propagate_globals(spec): idltype.cpp_type = _prefix_with_namespace(cpp_namespace, idltype.cpp_type) -def _parse(stream, error_file_name): +def parse_file(stream, error_file_name): # type: (Any, str) -> syntax.IDLParsedSpec """ Parse a YAML document into an idl.syntax tree. @@ -1130,7 +1130,7 @@ def parse(stream, input_file_name, resolver): """ # pylint: disable=too-many-locals - root_doc = _parse(stream, input_file_name) + root_doc = parse_file(stream, input_file_name) if root_doc.errors: return root_doc @@ -1167,7 +1167,7 @@ def parse(stream, input_file_name, resolver): # Parse imported file with resolver.open(resolved_file_name) as file_stream: - parsed_doc = _parse(file_stream, resolved_file_name) + parsed_doc = parse_file(file_stream, resolved_file_name) # Check for errors if parsed_doc.errors: diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index 63e0858a63a..2566f367703 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -92,8 +92,9 @@ DEFAULTS = { "report_failure_status": "fail", "report_file": None, "run_all_feature_flag_tests": False, - "run_all_feature_flags_no_tests": False, + "run_no_feature_flag_tests": False, "additional_feature_flags": None, + "additional_feature_flags_file": None, "seed": int(time.time() * 256), # Taken from random.py code in Python 2.7. "service_executor": None, "shell_conn_string": None, @@ -379,8 +380,11 @@ INSTALL_DIR = None # Whether to run tests for feature flags. RUN_ALL_FEATURE_FLAG_TESTS = None -# Whether to run the server with feature flags. Defaults to true if `RUN_ALL_FEATURE_FLAG_TESTS` is true. -RUN_ALL_FEATURE_FLAGS = None +# Whether to run the tests with enabled feature flags +RUN_NO_FEATURE_FLAG_TESTS = None + +# the path to a file containing feature flags +ADDITIONAL_FEATURE_FLAGS_FILE = None # List of enabled feature flags. ENABLED_FEATURE_FLAGS = [] diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py index af8a0fb09fe..aa5a384f17d 100644 --- a/buildscripts/resmokelib/configure_resmoke.py +++ b/buildscripts/resmokelib/configure_resmoke.py @@ -15,6 +15,7 @@ import shlex import pymongo.uri_parser +from buildscripts.idl import gen_all_feature_flag_list from buildscripts.idl.lib import ALL_FEATURE_FLAG_FILE from buildscripts.resmokelib import config as _config @@ -52,14 +53,9 @@ def _validate_options(parser, args): "Cannot use --replayFile with additional test files listed on the command line invocation." ) - if args.run_all_feature_flag_tests or args.run_all_feature_flags_no_tests: - if not os.path.isfile(ALL_FEATURE_FLAG_FILE): - parser.error( - "To run tests with all feature flags, the %s file must exist and be placed in" - " your working directory. The file can be downloaded from the artifacts tarball" - " in Evergreen. Alternatively, if you know which feature flags you want to enable," - " you can use the --additionalFeatureFlags command line argument" % - ALL_FEATURE_FLAG_FILE) + if args.additional_feature_flags_file and not os.path.isfile( + args.additional_feature_flags_file): + parser.error("The specified additional feature flags file does not exist.") def get_set_param_errors(process_params): agg_set_params = collections.defaultdict(list) @@ -185,28 +181,36 @@ be invoked as either: - buildscripts/resmoke.py --installDir {shlex.quote(user_config['install_dir'])}""") raise RuntimeError(err) + def process_feature_flag_file(path): + with open(path) as fd: + return fd.read().split() + def setup_feature_flags(): _config.RUN_ALL_FEATURE_FLAG_TESTS = config.pop("run_all_feature_flag_tests") - _config.RUN_ALL_FEATURE_FLAGS = config.pop("run_all_feature_flags_no_tests") + _config.RUN_NO_FEATURE_FLAG_TESTS = config.pop("run_no_feature_flag_tests") + _config.ADDITIONAL_FEATURE_FLAGS_FILE = config.pop("additional_feature_flags_file") - # Running all feature flag tests implies running the fixtures with feature flags. if _config.RUN_ALL_FEATURE_FLAG_TESTS: - _config.RUN_ALL_FEATURE_FLAGS = True + print("Generating: ", ALL_FEATURE_FLAG_FILE) + gen_all_feature_flag_list.gen_all_feature_flags_file() all_ff = [] enabled_feature_flags = [] try: - with open(ALL_FEATURE_FLAG_FILE) as fd: - all_ff = fd.read().split() + all_ff = process_feature_flag_file(ALL_FEATURE_FLAG_FILE) except FileNotFoundError: # If we ask resmoke to run with all feature flags, the feature flags file # needs to exist. - if _config.RUN_ALL_FEATURE_FLAGS: + if _config.RUN_ALL_FEATURE_FLAG_TESTS or _config.RUN_NO_FEATURE_FLAG_TESTS: raise - if _config.RUN_ALL_FEATURE_FLAGS: + if _config.RUN_ALL_FEATURE_FLAG_TESTS: enabled_feature_flags = all_ff[:] + if _config.ADDITIONAL_FEATURE_FLAGS_FILE: + enabled_feature_flags.extend( + process_feature_flag_file(_config.ADDITIONAL_FEATURE_FLAGS_FILE)) + # Specify additional feature flags from the command line. # Set running all feature flag tests to True if this options is specified. additional_feature_flags = _tags_from_list(config.pop("additional_feature_flags")) @@ -231,7 +235,7 @@ be invoked as either: _config.EXCLUDE_WITH_ANY_TAGS.extend( utils.default_if_none(_tags_from_list(config.pop("exclude_with_any_tags")), [])) - if _config.RUN_ALL_FEATURE_FLAGS and not _config.RUN_ALL_FEATURE_FLAG_TESTS: + if _config.RUN_NO_FEATURE_FLAG_TESTS: # Don't run any feature flag tests. _config.EXCLUDE_WITH_ANY_TAGS.extend(all_feature_flags) else: diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py index 64652279fa6..7c54607f50d 100644 --- a/buildscripts/resmokelib/run/__init__.py +++ b/buildscripts/resmokelib/run/__init__.py @@ -835,15 +835,19 @@ class RunPlugin(PluginInterface): ) parser.add_argument( - "--runAllFeatureFlagsNoTests", dest="run_all_feature_flags_no_tests", - action="store_true", help= - "Run MongoDB servers with all feature flags enabled but don't run any tests tagged with these feature flags; used for multiversion suites" - ) + "--runNoFeatureFlagTests", dest="run_no_feature_flag_tests", action="store_true", + help=("Do not run any tests tagged with enabled feature flags." + " This argument has precedence over --runAllFeatureFlagTests" + "; used for multiversion suites")) parser.add_argument("--additionalFeatureFlags", dest="additional_feature_flags", action="append", metavar="featureFlag1, featureFlag2, ...", help="Additional feature flags") + parser.add_argument("--additionalFeatureFlagsFile", dest="additional_feature_flags_file", + action="store", metavar="FILE", + help="The path to a file with feature flags, delimited by newlines.") + parser.add_argument("--maxTestQueueSize", type=int, dest="max_test_queue_size", help=argparse.SUPPRESS) @@ -1191,15 +1195,6 @@ def to_local_args(input_args=None): # pylint: disable=too-many-branches,too-man if origin_suite is not None: setattr(parsed_args, "suite_files", origin_suite) - # Replace --runAllFeatureFlagTests with an explicit list of feature flags. The former relies on - # all_feature_flags.txt which may not exist in the local dev environment. - run_all_feature_flag_tests = getattr(parsed_args, "run_all_feature_flag_tests", None) - if run_all_feature_flag_tests is not None: - setattr(parsed_args, "additional_feature_flags", config.ENABLED_FEATURE_FLAGS) - del parsed_args.run_all_feature_flag_tests - - del parsed_args.run_all_feature_flags_no_tests - # The top-level parser has one subparser that contains all subcommand parsers. command_subparser = [ action for action in parser._actions # pylint: disable=protected-access diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 2e178dc18b8..6b9b4f6dd7c 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -861,7 +861,7 @@ buildvariants: # To force disable feature flags even on the all feature flags variant, please use this file: # buildscripts/resmokeconfig/fully_disabled_feature_flags.yml test_flags: >- - --runAllFeatureFlagTests + --additionalFeatureFlagsFile all_feature_flags.txt --excludeWithAnyTags=incompatible_with_windows_tls --excludeWithAnyTags=incompatible_with_shard_merge external_auth_jobs_max: 1 @@ -1362,7 +1362,7 @@ buildvariants: # To force disable feature flags even on the all feature flags variant, please use this file: # buildscripts/resmokeconfig/fully_disabled_feature_flags.yml test_flags: >- - --runAllFeatureFlagTests + --additionalFeatureFlagsFile all_feature_flags.txt --excludeWithAnyTags=incompatible_with_shard_merge tasks: &enterprise-rhel-80-64-bit-dynamic-all-feature-flags-tasks - name: cqf @@ -1725,7 +1725,7 @@ buildvariants: <<: *enterprise-rhel-80-64-bit-multiversion-expansions-template # No feature flag tests since they aren't compatible with the older binaries. test_flags: >- - --runAllFeatureFlagsNoTests + --runNoFeatureFlagTests --excludeWithAnyTags=incompatible_with_shard_merge tasks: - name: compile_test_and_package_parallel_core_stream_TG @@ -2166,7 +2166,7 @@ buildvariants: # buildscripts/resmokeconfig/fully_disabled_feature_flags.yml large_distro_name: rhel80-build test_flags: >- - --runAllFeatureFlagTests + --additionalFeatureFlagsFile all_feature_flags.txt --excludeWithAnyTags=incompatible_with_shard_merge separate_debug: off tasks: @@ -2324,7 +2324,7 @@ buildvariants: test_flags: >- --excludeWithAnyTags=requires_ocsp_stapling --excludeWithAnyTags=incompatible_with_shard_merge - --runAllFeatureFlagTests + --additionalFeatureFlagsFile all_feature_flags.txt multiversion_platform: rhel80 multiversion_edition: enterprise resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under UBSAN build. @@ -2516,7 +2516,7 @@ buildvariants: test_flags: >- --excludeWithAnyTags=requires_ocsp_stapling --excludeWithAnyTags=incompatible_with_shard_merge - --runAllFeatureFlagTests + --additionalFeatureFlagsFile all_feature_flags.txt resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under {A,UB}SAN build. hang_analyzer_dump_core: false scons_cache_scope: shared @@ -2980,7 +2980,7 @@ buildvariants: target_resmoke_time: 30 max_sub_suites: 3 test_flags: >- - --runAllFeatureFlagTests + --additionalFeatureFlagsFile all_feature_flags.txt --excludeWithAnyTags=resource_intensive --excludeWithAnyTags=incompatible_with_shard_merge diff --git a/etc/perf.yml b/etc/perf.yml index 959379ad806..ec082554750 100644 --- a/etc/perf.yml +++ b/etc/perf.yml @@ -320,7 +320,7 @@ functions: set -o errexit set -o verbose source "${workdir}/compile_venv/bin/activate" - python ./buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir src/mongo/db/modules/enterprise/src + python ./buildscripts/idl/gen_all_feature_flag_list.py mkdir -p mongodb/feature_flags cp ./all_feature_flags.txt mongodb/feature_flags - command: shell.exec diff --git a/etc/system_perf.yml b/etc/system_perf.yml index b940c0dc77e..75f680deef7 100755 --- a/etc/system_perf.yml +++ b/etc/system_perf.yml @@ -349,7 +349,7 @@ functions: set -o errexit set -o verbose source "${workdir}/compile_venv/bin/activate" - python ./buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir src/mongo/db/modules/enterprise/src + python ./buildscripts/idl/gen_all_feature_flag_list.py mkdir -p mongodb/feature_flags cp ./all_feature_flags.txt mongodb/feature_flags - command: shell.exec diff --git a/evergreen/feature_flag_tags_check.sh b/evergreen/feature_flag_tags_check.sh index 5b046843a76..825591d5b41 100755 --- a/evergreen/feature_flag_tags_check.sh +++ b/evergreen/feature_flag_tags_check.sh @@ -12,7 +12,7 @@ enterprise_path="src/mongo/db/modules/enterprise" diff_file_name="with_base_upstream.diff" # get the list of feature flags from the patched version -$python buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir "$enterprise_path"/src +$python buildscripts/idl/gen_all_feature_flag_list.py mv all_feature_flags.txt patch_all_feature_flags.txt # get the list of feature flags from the base commit @@ -29,7 +29,7 @@ if [ -s "$diff_file_name" ]; then fi popd -$python buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir "$enterprise_path"/src +$python buildscripts/idl/gen_all_feature_flag_list.py mv all_feature_flags.txt base_all_feature_flags.txt # print out the list of tests that previously had feature flag tag, that was diff --git a/evergreen/gen_feature_flags.sh b/evergreen/gen_feature_flags.sh index 4e6dc0e9b4c..b06f614c243 100755 --- a/evergreen/gen_feature_flags.sh +++ b/evergreen/gen_feature_flags.sh @@ -6,4 +6,4 @@ cd src set -o errexit set -o verbose activate_venv -$python buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir src/mongo/db/modules/enterprise/src +$python buildscripts/idl/gen_all_feature_flag_list.py -- cgit v1.2.1