SERVER-52929 Correctly handle compound indexes with 32 keys

(cherry picked from commit 957ddd678dbd49a96318a13e8f669567e4eba3ab)
(cherry picked from commit 2faadd670bd54e7112c4fec6f6abc011d38b912c)

3 files changed, 85 insertions, 6 deletions

diff --git a/jstests/core/compound_index_max_fields.js b/jstests/core/compound_index_max_fields.js
new file mode 100644
index 00000000000..4213dfbb47e
--- /dev/null
+++ b/jstests/core/compound_index_max_fields.js
@@ -0,0 +1,58 @@
+/**
+ * Tests operations on indexes with the maximum number of compound index fields, 32.
+ *
+ *  @tags: [
+ *   assumes_unsharded_collection,
+ *   multiversion_incompatible,
+ *   requires_non_retryable_writes,
+ * ]
+ */
+(function() {
+
+const collName = jsTestName();
+const coll = db[collName];
+coll.drop();
+
+// Create a spec with alternating ascending and descending fields to keep things interesting.
+let spec = {};
+for (let i = 0; i < 32; i++) {
+    spec["f" + i] = (i % 2 == 0) ? 1 : -1;
+}
+
+const indexName = "big_index";
+assert.commandWorked(coll.createIndex(spec, {name: indexName}));
+assert.commandWorked(coll.insert({_id: 0}));
+for (let i = 0; i < 32; i++) {
+    assert.commandWorked(coll.update({_id: 0}, {
+        $set: {['f' + i]: 1},
+    }));
+}
+
+for (let i = 0; i < 32; i++) {
+    assert.eq(1, coll.find({['f' + i]: 1}).hint(indexName).itcount());
+}
+
+// Create an index that has one too many fields.
+let bigSpec = Object.extend({'f32': -1}, spec);
+assert.commandFailedWithCode(coll.createIndex(bigSpec), 13103);
+
+coll.drop();
+
+// Create a unique version of the same index from before.
+assert.commandWorked(coll.createIndex(spec, {unique: true, name: indexName}));
+
+let doc = {};
+let doc2 = {};
+for (let i = 0; i < 32; i++) {
+    doc['f' + i] = 1;
+    doc2['f' + i] = 2;
+}
+
+assert.commandWorked(coll.insert(doc));
+assert.commandWorked(coll.insert(doc2));
+
+for (let i = 0; i < 32; i++) {
+    let query = {['f' + i]: 1};
+    assert.eq(2, coll.find().hint(indexName).itcount(), "failed on query: " + tojson(query));
+}
+})();
diff --git a/src/mongo/bson/ordering.h b/src/mongo/bson/ordering.h
index c1052e36a90..db4fc18d06c 100644
--- a/src/mongo/bson/ordering.h
+++ b/src/mongo/bson/ordering.h
@@ -59,7 +59,13 @@ public:
     int get(int i) const {
         uassert(ErrorCodes::Overflow,
                 str::stream() << "Ordering offset is out of bounds: " << i,
-                i >= 0 && static_cast<size_t>(i) < kMaxCompoundIndexKeys);
+                i >= 0);
+        // Ordering only allows the first 32 fields to be inverted; any fields after the 32nd must
+        // be ascending. Return 1 to avoid a left shift a 32-bit integer by more than 31 bits, which
+        // is undefined behavior in C++.
+        if (static_cast<size_t>(i) >= kMaxCompoundIndexKeys) {
+            return 1;
+        }
         return ((1 << i) & bits) ? -1 : 1;
     }
 
diff --git a/src/mongo/db/storage/key_string_test.cpp b/src/mongo/db/storage/key_string_test.cpp
index 32d2919904c..465a43fe1a9 100644
--- a/src/mongo/db/storage/key_string_test.cpp
+++ b/src/mongo/db/storage/key_string_test.cpp
@@ -205,17 +205,32 @@ TEST(TypeBitsTest, AppendLotsOfZeroTypeBits) {
 }
 
 TEST_F(KeyStringTest, TooManyElementsInCompoundKey) {
-    // Construct an illegal KeyString with more than the limit of 32 elements in a compound index
-    // key. Encode 33 kBoolTrue ('o') values.
+    // Construct a KeyString with more than the limit of 32 elements in a compound index key. Encode
+    // 33 kBoolTrue ('o') values.
+    // Note that this KeyString encoding is legal, but it may not be legally stored in an index.
     const char* data = "ooooooooooooooooooooooooooooooooo";
     const size_t size = 33;
 
     KeyString ks(KeyString::Version::V1);
     ks.resetFromBuffer(data, size);
 
-    ASSERT_THROWS_CODE(KeyString::toBsonSafe(data, size, ALL_ASCENDING, ks.getTypeBits()),
-                       AssertionException,
-                       ErrorCodes::Overflow);
+    // No exceptions should be thrown.
+    KeyString::toBsonSafe(data, size, ALL_ASCENDING, ks.getTypeBits());
+    KeyString::getKeySize(ks.getBuffer(), ks.getSize(), ALL_ASCENDING, ks.getTypeBits());
+}
+
+TEST_F(KeyStringTest, MaxElementsInCompoundKey) {
+    // Construct a KeyString with 32 elements in a compound index key followed by an end byte.
+    // Encode 32 kBoolTrue ('o') values and an end byte, 0x4.
+    const char* data = "oooooooooooooooooooooooooooooooo\x4";
+    const size_t size = 33;
+
+    KeyString ks(KeyString::Version::V1);
+    ks.resetFromBuffer(data, size);
+
+    // No exceptions should be thrown.
+    KeyString::toBsonSafe(data, size, ALL_ASCENDING, ks.getTypeBits());
+    KeyString::getKeySize(ks.getBuffer(), ks.getSize(), ALL_ASCENDING, ks.getTypeBits());
 }
 
 TEST_F(KeyStringTest, ExceededBSONDepth) {