SERVER-38027 SERVER-37268 Partially enable hybrid index builds for background, non-unique indexes. Change background index builds to use the bulk builder and external sorter

1 files changed, 51 insertions, 10 deletions

diff --git a/src/mongo/db/index/index_build_interceptor.h b/src/mongo/db/index/index_build_interceptor.h
index 13ae79d10d2..b25fbf53c71 100644
--- a/src/mongo/db/index/index_build_interceptor.h
+++ b/src/mongo/db/index/index_build_interceptor.h
@@ -29,9 +29,10 @@
 
 #pragma once
 
+#include "mongo/db/index/index_access_method.h"
 #include "mongo/db/index/multikey_paths.h"
 #include "mongo/db/namespace_string.h"
-#include "mongo/db/record_id.h"
+#include "mongo/db/storage/temporary_record_store.h"
 #include "mongo/platform/atomic_word.h"
 
 namespace mongo {
@@ -44,13 +45,12 @@ class IndexBuildInterceptor {
 public:
     enum class Op { kInsert, kDelete };
 
-    IndexBuildInterceptor() : _sideWritesNs(makeTempSideWritesNs()) {}
-    IndexBuildInterceptor(NamespaceString sideWritesNs) : _sideWritesNs(sideWritesNs) {}
-
-    static NamespaceString makeTempSideWritesNs();
-
-    void ensureSideWritesCollectionExists(OperationContext* opCtx);
-    void removeSideWritesCollection(OperationContext* opCtx);
+    /**
+     * The OperationContext is used to construct a temporary table in the storage engine to
+     * intercept side writes. This interceptor must not exist longer than the operation context used
+     * to construct it, as the underlying TemporaryRecordStore needs it to destroy itself.
+     */
+    IndexBuildInterceptor(OperationContext* opCtx);
 
     /**
      * Client writes that are concurrent with an index build will have their index updates written
@@ -64,10 +64,51 @@ public:
                      const BSONObj* obj,
                      RecordId loc,
                      Op op,
-                     int64_t* numKeysOut);
+                     int64_t* const numKeysOut);
+
+    /**
+     * Performs a resumable scan on the side writes table, and either inserts or removes each key
+     * from the underlying IndexAccessMethod. This will only insert as many records as are visible
+     * in the current snapshot.
+     *
+     * This is resumable, so subsequent calls will start the scan at the record immediately
+     * following the last inserted record from a previous call to drainWritesIntoIndex.
+     */
+    Status drainWritesIntoIndex(OperationContext* opCtx,
+                                IndexAccessMethod* indexAccessMethod,
+                                const IndexDescriptor* indexDescriptor,
+                                const InsertDeleteOptions& options);
+
+    /**
+     * Returns 'true' if there are no visible records remaining to be applied from the side writes
+     * table. Ensure that this returns 'true' when an index build is completed.
+     */
+    bool areAllWritesApplied(OperationContext* opCtx) const;
+
+    /**
+      * When an index builder wants to commit, use this to retrieve any recorded multikey paths
+      * that were tracked during the build.
+      */
+    boost::optional<MultikeyPaths> getMultikeyPaths() const;
 
 private:
-    NamespaceString _sideWritesNs;
+    using SideWriteRecord = std::pair<RecordId, BSONObj>;
+
+    Status _applyWrite(OperationContext* opCtx,
+                       IndexAccessMethod* indexAccessMethod,
+                       const BSONObj& doc,
+                       const InsertDeleteOptions& options,
+                       int64_t* const keysInserted,
+                       int64_t* const keysDeleted);
+
+    // This temporary record store is owned by the interceptor and dropped along with it.
+    std::unique_ptr<TemporaryRecordStore> _sideWritesTable;
+
+    int64_t _numApplied{0};
+
+    AtomicInt64 _sideWritesCounter{0};
+
+    mutable stdx::mutex _multikeyPathMutex;
     boost::optional<MultikeyPaths> _multikeyPaths;
 };