SERVER-63947 Add column store jscore passthrough - plus fixes

Also fixes:
- $where reports depending on a full object, disqualifying it from using
  the column store index.
- projection analysis should not remove a projection if we have some
  match fields that are not needed in the output. This doesn't work yet.

4 files changed, 43 insertions, 25 deletions

diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index cb712de28f3..3e3207e5f43 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -908,6 +908,7 @@ buildvariants:
     - windows-2016-dc
   - name: .jscore .common !.sharding
   - name: jsCore_auth
+  - name: jsCore_column_store_indexes
   - name: jsCore_ese
   - name: jsCore_txns_large_txns_format
   - name: .jstestfuzz .common
@@ -1475,6 +1476,7 @@ buildvariants:
     distros:
     - rhel80-medium
   - name: .jscore .common
+  - name: jsCore_column_store_indexes
   - name: jsCore_minimum_batch_size
   - name: jsCore_repeat_queries
   - name: jsCore_txns_large_txns_format
@@ -1577,7 +1579,7 @@ buildvariants:
   - name: .integration
     distros:
     - rhel80-medium
-  - name: .jscore .common !jsCore
+  - name: .jscore .common !jsCore !.sbe_only
   - name: .jstestfuzz !.flow_control
   - name: .lint
   - name: .misc_js
@@ -2034,7 +2036,7 @@ buildvariants:
     distros:
     - ubuntu1804-xlarge
   - name: jsCore
-  - name: .aggregation !.encrypt !.feature_flag_guarded
+  - name: .aggregation !.encrypt
   - name: replica_sets_jscore_passthrough
   - name: sharded_collections_jscore_passthrough
   - name: sharding_gen
@@ -2247,6 +2249,7 @@ buildvariants:
   - name: initial_sync_fuzzer_gen
   - name: .integration !.standalone !.audit
   - name: .jscore .common
+  - name: jsCore_column_store_indexes
   - name: jsCore_minimum_batch_size
   - name: jsCore_txns_large_txns_format
   - name: json_schema
@@ -2326,7 +2329,7 @@ buildvariants:
   - name: external_auth_aws
   - name: initial_sync_fuzzer_gen
   - name: .integration !.standalone !.audit
-  - name: .jscore .common
+  - name: .jscore .common !.sbe_only
   - name: jsCore_minimum_batch_size
   - name: jsCore_txns_large_txns_format
   - name: json_schema
@@ -2405,6 +2408,7 @@ buildvariants:
   - name: initial_sync_fuzzer_gen
   - name: .integration !.audit
   - name: .jscore .common
+  - name: jsCore_column_store_indexes
   - name: jsCore_minimum_batch_size
   - name: jsCore_txns_large_txns_format
   - name: json_schema
@@ -2478,7 +2482,7 @@ buildvariants:
   - name: free_monitoring
   - name: initial_sync_fuzzer_gen
   - name: .integration !.audit
-  - name: .jscore .common
+  - name: .jscore .common !.sbe_only
   - name: jsCore_minimum_batch_size
   - name: jsCore_txns_large_txns_format
   - name: json_schema
@@ -2580,6 +2584,7 @@ buildvariants:
     - rhel80-xlarge
   - name: jsCore
   - name: jsCore_txns
+  - name: jsCore_column_store_indexes
 # Disabling these tests as they are not aware of feature flags.
 #  - name: compile_test_and_package_parallel_unittest_stream_TG
 #    distros:
diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml
index 641a3e3112d..6a3f161f7e2 100644
--- a/etc/evergreen_yml_components/definitions.yml
+++ b/etc/evergreen_yml_components/definitions.yml
@@ -5070,6 +5070,19 @@ tasks:
   - func: "run tests"
 
 - <<: *task_template
+  name: jsCore_column_store_indexes
+  # TODO SERVER-61644 Remove the 'feature_flag_guarded' tag and ensure this suite is run on all
+  # variants.
+  depends_on:
+  - name: jsCore
+  tags: ["jscore", "sbe_only", "feature_flag_guarded"]
+  commands:
+  - func: "do setup"
+  - func: "run tests"
+    vars:
+      suite: core_column_store_indexes
+
+- <<: *task_template
   name: mongosTest
   tags: ["misc_js", "non_read_maj", "non_live_record"]
   commands:
diff --git a/etc/evergreen_yml_components/variants/atlas.yml b/etc/evergreen_yml_components/variants/atlas.yml
index 070ee1a9ebb..91c20c51bb1 100644
--- a/etc/evergreen_yml_components/variants/atlas.yml
+++ b/etc/evergreen_yml_components/variants/atlas.yml
@@ -50,7 +50,7 @@ buildvariants:
   - name: .encrypt !.sharding !.replica_sets !.aggregation !.jscore
   - name: external_auth
   - name: external_auth_aws
-  - name: .jscore .common !.decimal !.sharding
+  - name: .jscore .common !.decimal !.sharding !.feature_flag_guarded
   - name: jsCore_txns_large_txns_format
   - name: .jstestfuzz .common
   - name: libunwind_tests
diff --git a/etc/evergreen_yml_components/variants/misc_release.yml b/etc/evergreen_yml_components/variants/misc_release.yml
index 6b3087b4057..f513eecd7f4 100644
--- a/etc/evergreen_yml_components/variants/misc_release.yml
+++ b/etc/evergreen_yml_components/variants/misc_release.yml
@@ -340,7 +340,7 @@ buildvariants:
   - name: jsCore
     distros:
     - amazon2-arm64-large
-  - name: .jscore .common !jsCore
+  - name: .jscore .common !jsCore !.feature_flag_guarded
   - name: jsCore_minimum_batch_size
   - name: jsCore_txns_large_txns_format
   - name: json_schema
@@ -434,7 +434,7 @@ buildvariants:
   - name: concurrency_replication_causal_consistency_gen
   - name: disk_wiredtiger
   - name: free_monitoring
-  - name: .jscore .common !.decimal
+  - name: .jscore .common !.decimal !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: multiversion_gen
@@ -489,7 +489,7 @@ buildvariants:
   - name: .jepsen_docker
     distros:
     - debian10-large
-  - name: .jscore .common !.decimal !.sharding
+  - name: .jscore .common !.decimal !.sharding !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: replica_sets_auth_gen
@@ -544,7 +544,7 @@ buildvariants:
   - name: concurrency_replication_causal_consistency_gen
   - name: disk_wiredtiger
   - name: free_monitoring
-  - name: .jscore .common !.decimal
+  - name: .jscore .common !.decimal !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: multiversion_gen
@@ -599,7 +599,7 @@ buildvariants:
   - name: .jepsen_docker
     distros:
     - debian11-large
-  - name: .jscore .common !.decimal !.sharding
+  - name: .jscore .common !.decimal !.sharding !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: replica_sets_auth_gen
@@ -958,7 +958,7 @@ buildvariants:
   - name: .encrypt !.sharding !.replica_sets !.aggregation !.jscore
   - name: external_auth
   - name: external_auth_aws
-  - name: .jscore .common !.decimal !.sharding
+  - name: .jscore .common !.decimal !.sharding !.feature_flag_guarded
   - name: jsCore_txns_large_txns_format
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
@@ -1021,7 +1021,7 @@ buildvariants:
   - name: concurrency_replication_causal_consistency_gen
   - name: disk_wiredtiger
   - name: free_monitoring
-  - name: .jscore .common !.decimal
+  - name: .jscore .common !.decimal !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: multiversion_gen
   - name: .logical_session_cache .one_sec
@@ -1067,7 +1067,7 @@ buildvariants:
   - name: audit
   - name: causally_consistent_jscore_txns_passthrough
   - name: .encrypt !.replica_sets !.aggregation !.sharding !.jscore
-  - name: .jscore .common !.decimal !.sharding
+  - name: .jscore .common !.decimal !.sharding !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: replica_sets_auth_gen
@@ -1115,7 +1115,7 @@ buildvariants:
   - name: audit
   - name: causally_consistent_jscore_txns_passthrough
   - name: .encrypt !.replica_sets !.aggregation !.sharding !.jscore
-  - name: .jscore .common !.decimal !.sharding
+  - name: .jscore .common !.decimal !.sharding !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: replica_sets_auth_gen
@@ -1166,7 +1166,7 @@ buildvariants:
   - name: concurrency_replication_causal_consistency_gen
   - name: disk_wiredtiger
   - name: free_monitoring
-  - name: .jscore .common !.decimal
+  - name: .jscore .common !.decimal !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: multiversion_gen
@@ -1289,7 +1289,7 @@ buildvariants:
   - name: .jepsen
     distros:
     - ubuntu1804-build
-  - name: .jscore .common !.decimal !.sharding
+  - name: .jscore .common !.decimal !.sharding !.feature_flag_guarded
   - name: jsCore_auth
   - name: .jstestfuzz .common
   - name: libunwind_tests
@@ -1349,7 +1349,7 @@ buildvariants:
   - name: .concurrency .common
   - name: concurrency_replication_causal_consistency_gen
   - name: fle
-  - name: .jscore .common !.auth
+  - name: .jscore .common !.auth !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: replica_sets_gen
@@ -1498,7 +1498,7 @@ buildvariants:
   - name: .encrypt !.aggregation !.replica_sets !.sharding !.jscore
   - name: external_auth
   - name: external_auth_aws
-  - name: .jscore .common !.decimal !.sharding
+  - name: .jscore .common !.decimal !.sharding !.feature_flag_guarded
   - name: jsCore_auth
   - name: .jstestfuzz .common
   - name: libunwind_tests
@@ -1553,7 +1553,7 @@ buildvariants:
   - name: .concurrency .common
   - name: concurrency_replication_causal_consistency_gen
   - name: fle
-  - name: .jscore .common !.auth
+  - name: .jscore .common !.auth !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: replica_sets_gen
@@ -1649,7 +1649,7 @@ buildvariants:
   - name: .concurrency .common
   - name: disk_wiredtiger
   - name: free_monitoring
-  - name: .jscore .common !.auth
+  - name: .jscore .common !.auth !.feature_flag_guarded
   - name: json_schema
   - name: .jstestfuzz !.initsync !.flow_control !.stepdowns
   - name: multiversion_gen
@@ -1718,7 +1718,7 @@ buildvariants:
     distros:
     - rhel70-small
   - name: disk_wiredtiger
-  - name: .jscore .common !.auth
+  - name: .jscore .common !.auth !.feature_flag_guarded
   - name: json_schema
   - name: .jstestfuzz !.initsync !.flow_control !.stepdowns
   - name: multiversion_gen
@@ -1767,7 +1767,7 @@ buildvariants:
   - name: disk_wiredtiger
   - name: free_monitoring
   - name: initial_sync_fuzzer_gen
-  - name: .jscore .common !.auth
+  - name: .jscore .common !.auth !.feature_flag_guarded
   - name: jsCore_txns_large_txns_format
   - name: .jstestfuzz .causal
   - name: .jstestfuzz .interrupt
@@ -1818,7 +1818,7 @@ buildvariants:
   - name: disk_wiredtiger
   - name: free_monitoring
   - name: initial_sync_fuzzer_gen
-  - name: .jscore .common !.auth
+  - name: .jscore .common !.auth !.feature_flag_guarded
   - name: jsCore_txns_large_txns_format
   - name: .jstestfuzz .causal
   - name: .jstestfuzz .interrupt
@@ -1868,7 +1868,7 @@ buildvariants:
   - name: causally_consistent_jscore_txns_passthrough
   # TODO: SERVER-66945 Re-enable ESE on enterprise macos
   # - name: .encrypt !.replica_sets !.sharding !.aggregation !.jscore
-  - name: .jscore .common !.decimal !.sharding
+  - name: .jscore .common !.decimal !.sharding !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: mqlrun
@@ -1907,7 +1907,7 @@ buildvariants:
   - name: auth_audit_gen
   - name: causally_consistent_jscore_txns_passthrough
   - name: .encrypt !.replica_sets !.sharding !.aggregation !.jscore
-  - name: .jscore .common !.decimal !.sharding
+  - name: .jscore .common !.decimal !.sharding !.feature_flag_guarded
   - name: .jstestfuzz .common
   - name: .logical_session_cache .one_sec
   - name: mqlrun