SERVER-51883 Add all feature flags by default to the all feature flags

builders

3 files changed, 137 insertions, 22 deletions

diff --git a/buildscripts/idl/check_versioned_api_commands_have_idl_definitions.py b/buildscripts/idl/check_versioned_api_commands_have_idl_definitions.py
index 7e507751c93..f3c5f045e83 100644
--- a/buildscripts/idl/check_versioned_api_commands_have_idl_definitions.py
+++ b/buildscripts/idl/check_versioned_api_commands_have_idl_definitions.py
@@ -43,39 +43,19 @@ from pymongo import MongoClient
 sys.path.append(os.path.normpath(os.path.join(os.path.abspath(__file__), '../../..')))
 
 # pylint: disable=wrong-import-position,wrong-import-order
+from buildscripts.idl.lib import list_idls, parse_idl
 from buildscripts.resmokelib import configure_resmoke
 from buildscripts.resmokelib.logging import loggers
 from buildscripts.resmokelib.testing.fixtures import interface
 from buildscripts.resmokelib.testing.fixtures.fixturelib import FixtureLib
 from buildscripts.resmokelib.testing.fixtures.shardedcluster import ShardedClusterFixture
 from buildscripts.resmokelib.testing.fixtures.standalone import MongoDFixture
-from idl import parser, syntax
-from idl.compiler import CompilerImportResolver
+from idl import syntax
 
 LOGGER_NAME = 'check-idl-definitions'
 LOGGER = logging.getLogger(LOGGER_NAME)
 
 
-def list_idls(directory: str) -> Set[str]:
-    """Find all IDL files in the current directory."""
-    return {
-        os.path.join(dirpath, filename)
-        for dirpath, dirnames, filenames in os.walk(directory) for filename in filenames
-        if filename.endswith('.idl')
-    }
-
-
-def parse_idl(idl_path: str, import_directories: List[str]) -> syntax.IDLParsedSpec:
-    """Parse an IDL file or throw an error."""
-    parsed_doc = parser.parse(open(idl_path), idl_path, CompilerImportResolver(import_directories))
-
-    if parsed_doc.errors:
-        parsed_doc.errors.dump_errors()
-        raise ValueError(f"Cannot parse {idl_path}")
-
-    return parsed_doc
-
-
 def is_test_idl(idl_path: str) -> bool:
     """Check if an IDL file is a test file."""
     test_idls_subpaths = ["/idl/tests/", "unittest.idl"]
diff --git a/buildscripts/idl/gen_all_feature_flag_list.py b/buildscripts/idl/gen_all_feature_flag_list.py
new file mode 100644
index 00000000000..7b431fd2dab
--- /dev/null
+++ b/buildscripts/idl/gen_all_feature_flag_list.py
@@ -0,0 +1,77 @@
+# Copyright (C) 2020-present MongoDB, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the Server Side Public License, version 1,
+# as published by MongoDB, Inc.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# Server Side Public License for more details.
+#
+# You should have received a copy of the Server Side Public License
+# along with this program. If not, see
+# <http://www.mongodb.com/licensing/server-side-public-license>.
+#
+# As a special exception, the copyright holders give permission to link the
+# code of portions of this program with the OpenSSL library under certain
+# conditions as described in each individual source file and distribute
+# linked combinations including the program with the OpenSSL library. You
+# must comply with the Server Side Public License in all respects for
+# all of the code used other than as permitted herein. If you modify file(s)
+# with this exception, you may extend this exception to your version of the
+# file(s), but you are not obligated to do so. If you do not wish to do so,
+# delete this exception statement from your version. If you delete this
+# exception statement from all source files in the program, then also delete
+# it in the license file.
+"""
+Generate a file containing a list of disabled feature flags.
+
+Used by resmoke.py to run only feature flag tests.
+"""
+
+import argparse
+import os
+import sys
+
+from typing import List
+
+import yaml
+
+# Permit imports from "buildscripts".
+sys.path.append(os.path.normpath(os.path.join(os.path.abspath(__file__), '../../..')))
+
+# pylint: disable=wrong-import-position
+import buildscripts.idl.lib as lib
+
+
+def gen_all_feature_flags(idl_dir: str, import_dirs: List[str]):
+    """Generate a list of all feature flags."""
+    all_flags = []
+    for idl_path in sorted(lib.list_idls(idl_dir)):
+        for feature_flag in lib.parse_idl(idl_path, import_dirs).spec.feature_flags:
+            if feature_flag.default.literal != "true":
+                all_flags.append(feature_flag.name)
+
+    force_disabled_flags = yaml.safe_load(
+        open("buildscripts/resmokeconfig/fully_disabled_feature_flags.yml"))
+
+    return list(set(all_flags) - set(force_disabled_flags))
+
+
+def main():
+    """Run the main function."""
+    arg_parser = argparse.ArgumentParser(description=__doc__)
+    arg_parser.add_argument("--import-dir", dest="import_dirs", type=str, action="append",
+                            help="Directory to search for IDL import files")
+
+    args = arg_parser.parse_args()
+
+    flags = gen_all_feature_flags(os.getcwd(), args.import_dirs)
+    with open(lib.ALL_FEATURE_FLAG_FILE, "w") as output_file:
+        for flag in flags:
+            output_file.write("%s\n" % flag)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/buildscripts/idl/lib.py b/buildscripts/idl/lib.py
new file mode 100644
index 00000000000..c4ae1c08fea
--- /dev/null
+++ b/buildscripts/idl/lib.py
@@ -0,0 +1,58 @@
+# Copyright (C) 2020-present MongoDB, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the Server Side Public License, version 1,
+# as published by MongoDB, Inc.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# Server Side Public License for more details.
+#
+# You should have received a copy of the Server Side Public License
+# along with this program. If not, see
+# <http://www.mongodb.com/licensing/server-side-public-license>.
+#
+# As a special exception, the copyright holders give permission to link the
+# code of portions of this program with the OpenSSL library under certain
+# conditions as described in each individual source file and distribute
+# linked combinations including the program with the OpenSSL library. You
+# must comply with the Server Side Public License in all respects for
+# all of the code used other than as permitted herein. If you modify file(s)
+# with this exception, you may extend this exception to your version of the
+# file(s), but you are not obligated to do so. If you do not wish to do so,
+# delete this exception statement from your version. If you delete this
+# exception statement from all source files in the program, then also delete
+# it in the license file.
+"""Library functions and utility methods used across user-facing IDL scripts."""
+
+import os
+
+from typing import Set, List
+
+from buildscripts.idl.idl import syntax, parser
+from buildscripts.idl.idl.compiler import CompilerImportResolver
+
+# List of feature flags that are disabled by default. The file name is repeated in
+# evergreen.yml
+ALL_FEATURE_FLAG_FILE = "all_feature_flags.txt"
+
+
+def list_idls(directory: str) -> Set[str]:
+    """Find all IDL files in the current directory."""
+    return {
+        os.path.join(dirpath, filename)
+        for dirpath, dirnames, filenames in os.walk(directory) for filename in filenames
+        if filename.endswith('.idl')
+    }
+
+
+def parse_idl(idl_path: str, import_directories: List[str]) -> syntax.IDLParsedSpec:
+    """Parse an IDL file or throw an error."""
+    parsed_doc = parser.parse(open(idl_path), idl_path, CompilerImportResolver(import_directories))
+
+    if parsed_doc.errors:
+        parsed_doc.errors.dump_errors()
+        raise ValueError(f"Cannot parse {idl_path}")
+
+    return parsed_doc