Import wiredtiger: c9a4f73f9be0fcf5cc5f4818d4e5c598ab6a0c31 from branch mongodb-5.2

ref: 5ddf723b53..c9a4f73f9b
for: 5.2.0-rc2

WT-8421       Add a config to allow import for tables with a checkpoint timestamp smaller than or equal to the stable timestamp

6 files changed, 79 insertions, 40 deletions

diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py
index 48b5a6f5acc..8b3b11e05da 100644
--- a/src/third_party/wiredtiger/dist/api_data.py
+++ b/src/third_party/wiredtiger/dist/api_data.py
@@ -1409,6 +1409,12 @@ methods = {
     Config('import', '', r'''
         configure import of an existing object into the currently running database''',
         type='category', subconfig=[
+        Config('compare_timestamp', 'oldest', r'''
+            Allow importing files with timestamps smaller or equal to the configured
+            global timestamps. Note that the history of the files are not imported
+            together and thus snapshot read of historical data will not work with the
+            option "stable"''',
+            choices=['oldest', 'stable']),
         Config('enabled', 'false', r'''
             whether to import the input URI from disk''',
             type='boolean'),
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index f2f760c5b2c..166f354aaec 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-5.2",
-    "commit": "5ddf723b53f23b34e5c3ccddc9586a6f9a75e84a"
+    "commit": "c9a4f73f9be0fcf5cc5f4818d4e5c598ab6a0c31"
 }
diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c
index edcbca78e8e..23d6bfad626 100644
--- a/src/third_party/wiredtiger/src/config/config_def.c
+++ b/src/third_party/wiredtiger/src/config/config_def.c
@@ -260,6 +260,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create_encryption_subconfigs[] =
   {NULL, NULL, NULL, NULL, NULL, 0}};
 
 static const WT_CONFIG_CHECK confchk_WT_SESSION_create_import_subconfigs[] = {
+  {"compare_timestamp", "string", NULL, "choices=[\"oldest\",\"stable\"]", NULL, 0},
   {"enabled", "boolean", NULL, NULL, NULL, 0}, {"file_metadata", "string", NULL, NULL, NULL, 0},
   {"repair", "boolean", NULL, NULL, NULL, 0}, {NULL, NULL, NULL, NULL, NULL, 0}};
 
@@ -310,7 +311,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create[] = {
   {"huffman_key", "string", NULL, NULL, NULL, 0}, {"huffman_value", "string", NULL, NULL, NULL, 0},
   {"ignore_in_memory_cache_size", "boolean", NULL, NULL, NULL, 0},
   {"immutable", "boolean", NULL, NULL, NULL, 0},
-  {"import", "category", NULL, NULL, confchk_WT_SESSION_create_import_subconfigs, 3},
+  {"import", "category", NULL, NULL, confchk_WT_SESSION_create_import_subconfigs, 4},
   {"internal_item_max", "int", NULL, "min=0", NULL, 0},
   {"internal_key_max", "int", NULL, "min=0", NULL, 0},
   {"internal_key_truncate", "boolean", NULL, NULL, NULL, 0},
@@ -1245,8 +1246,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator",
     "collator=,columns=,dictionary=0,encryption=(keyid=,name=),"
     "exclusive=false,extractor=,format=btree,huffman_key=,"
     "huffman_value=,ignore_in_memory_cache_size=false,immutable=false"
-    ",import=(enabled=false,file_metadata=,repair=false),"
-    "internal_item_max=0,internal_key_max=0,"
+    ",import=(compare_timestamp=oldest,enabled=false,file_metadata=,"
+    "repair=false),internal_item_max=0,internal_key_max=0,"
     "internal_key_truncate=true,internal_page_max=4KB,key_format=u,"
     "key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB,"
     "leaf_value_max=0,log=(enabled=true),lsm=(auto_throttle=true,"
diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in
index 31b852d5fc0..aa33185db64 100644
--- a/src/third_party/wiredtiger/src/include/wiredtiger.in
+++ b/src/third_party/wiredtiger/src/include/wiredtiger.in
@@ -1169,13 +1169,17 @@ struct __wt_session {
 	 * by any update to a record in the table., a boolean flag; default \c false.}
 	 * @config{import = (, configure import of an existing object into the currently running
 	 * database., a set of related configuration options defined below.}
-	 * @config{    enabled, whether to import the input URI from disk., a
-	 * boolean flag; default \c false.}
-	 * @config{    file_metadata, the file
-	 * configuration extracted from the metadata of the export database., a string; default
-	 * empty.}
-	 * @config{    repair, whether to reconstruct the metadata from
-	 * the raw file content., a boolean flag; default \c false.}
+	 * @config{    compare_timestamp, Allow importing files with timestamps
+	 * smaller or equal to the configured global timestamps.  Note that the history of the files
+	 * are not imported together and thus snapshot read of historical data will not work with
+	 * the option "stable"., a string\, chosen from the following options: \c "oldest"\, \c
+	 * "stable"; default \c oldest.}
+	 * @config{    enabled, whether to import
+	 * the input URI from disk., a boolean flag; default \c false.}
+	 * @config{    file_metadata, the file configuration extracted from the
+	 * metadata of the export database., a string; default empty.}
+	 * @config{    repair, whether to reconstruct the metadata from the raw
+	 * file content., a boolean flag; default \c false.}
 	 * @config{ ),,}
 	 * @config{internal_key_max, This option is no longer supported\, retained for backward
 	 * compatibility., an integer greater than or equal to 0; default \c 0.}
diff --git a/src/third_party/wiredtiger/src/schema/schema_create.c b/src/third_party/wiredtiger/src/schema/schema_create.c
index 643529c27fd..16c868ba390 100644
--- a/src/third_party/wiredtiger/src/schema/schema_create.c
+++ b/src/third_party/wiredtiger/src/schema/schema_create.c
@@ -46,21 +46,28 @@ __wt_direct_io_size_check(
 
 /*
  * __check_imported_ts --
- *     Check the aggregated timestamps for each checkpoint in a file that we've imported. We're not
- *     allowed to import files with timestamps ahead of our oldest timestamp since a subsequent
- *     rollback to stable could result in data loss and historical reads could yield unexpected
- *     values. Therefore, this function should return non-zero to callers to signify that this is
- *     the case.
+ *     Check the aggregated timestamps for each checkpoint in a file that we've imported. By
+ *     default, we're not allowed to import files with timestamps ahead of the oldest timestamp
+ *     since a subsequent rollback to stable could result in data loss and historical reads could
+ *     yield unexpected values. Therefore, this function should return non-zero to callers to
+ *     signify that this is the case. If configured, it is possible to import files with timestamps
+ *     smaller than or equal to the stable timestamp. However, there is no history migrated with the
+ *     files and thus reading historical versions will not work.
  */
 static int
-__check_imported_ts(WT_SESSION_IMPL *session, const char *uri, const char *config)
+__check_imported_ts(
+  WT_SESSION_IMPL *session, const char *uri, const char *config, bool against_stable)
 {
     WT_CKPT *ckptbase, *ckpt;
     WT_DECL_RET;
     WT_TXN_GLOBAL *txn_global;
+    wt_timestamp_t ts;
+    const char *ts_name;
 
     ckptbase = NULL;
     txn_global = &S2C(session)->txn_global;
+    ts = against_stable ? txn_global->stable_timestamp : txn_global->oldest_timestamp;
+    ts_name = against_stable ? "stable" : "oldest";
 
     WT_ERR_NOTFOUND_OK(
       __wt_meta_ckptlist_get_from_config(session, false, &ckptbase, NULL, config), true);
@@ -70,11 +77,11 @@ __check_imported_ts(WT_SESSION_IMPL *session, const char *uri, const char *confi
 
     /* Now iterate over each checkpoint and compare the aggregate timestamps with our oldest. */
     WT_CKPT_FOREACH (ckptbase, ckpt) {
-        if (ckpt->ta.newest_start_durable_ts > txn_global->oldest_timestamp)
-            WT_ERR_MSG(session, EINVAL,
+        if (ckpt->ta.newest_start_durable_ts > ts)
+            WT_ERR_MSG(session, WT_ROLLBACK,
               "%s: import found aggregated newest start durable timestamp newer than the current "
-              "oldest timestamp, newest_start_durable_ts=%" PRIu64 ", oldest_ts=%" PRIu64,
-              uri, ckpt->ta.newest_start_durable_ts, txn_global->oldest_timestamp);
+              "%s timestamp, newest_start_durable_ts=%" PRIu64 ", %s_ts=%" PRIu64,
+              uri, ts_name, ckpt->ta.newest_start_durable_ts, ts_name, ts);
 
         /*
          * No need to check "newest stop" here as "newest stop durable" serves that purpose. When a
@@ -82,12 +89,12 @@ __check_imported_ts(WT_SESSION_IMPL *session, const char *uri, const char *confi
          * whereas "newest stop durable" refers to the newest non-max timestamp which is more useful
          * to us in terms of comparing with oldest.
          */
-        if (ckpt->ta.newest_stop_durable_ts > txn_global->oldest_timestamp) {
+        if (ckpt->ta.newest_stop_durable_ts > ts) {
             WT_ASSERT(session, ckpt->ta.newest_stop_durable_ts != WT_TS_MAX);
-            WT_ERR_MSG(session, EINVAL,
+            WT_ERR_MSG(session, WT_ROLLBACK,
               "%s: import found aggregated newest stop durable timestamp newer than the current "
-              "oldest timestamp, newest_stop_durable_ts=%" PRIu64 ", oldest_ts=%" PRIu64,
-              uri, ckpt->ta.newest_stop_durable_ts, txn_global->oldest_timestamp);
+              "%s timestamp, newest_stop_durable_ts=%" PRIu64 ", %s_ts=%" PRIu64,
+              uri, ts_name, ckpt->ta.newest_stop_durable_ts, ts_name, ts);
         }
     }
 
@@ -136,7 +143,7 @@ __create_file(
       *filecfg[] = {WT_CONFIG_BASE(session, file_meta), config, NULL, NULL, NULL, NULL};
     char *fileconf, *filemeta;
     uint32_t allocsize;
-    bool exists, import_repair, is_metadata;
+    bool against_stable, exists, import_repair, is_metadata;
 
     fileconf = filemeta = NULL;
 
@@ -249,10 +256,14 @@ __create_file(
 
         /*
          * Ensure that the timestamps in the imported data file are not in the future relative to
-         * our oldest timestamp.
+         * the configured global timestamp.
          */
-        if (import)
-            WT_ERR(__check_imported_ts(session, uri, fileconf));
+        if (import) {
+            against_stable =
+              __wt_config_getones(session, config, "import.compare_timestamp", &cval) == 0 &&
+              WT_STRING_MATCH("stable", cval.str, cval.len);
+            WT_ERR(__check_imported_ts(session, uri, fileconf, against_stable));
+        }
     }
 
     /*
diff --git a/src/third_party/wiredtiger/test/suite/test_import05.py b/src/third_party/wiredtiger/test/suite/test_import05.py
index 0307c6586cb..2a2ea7f6979 100644
--- a/src/third_party/wiredtiger/test/suite/test_import05.py
+++ b/src/third_party/wiredtiger/test/suite/test_import05.py
@@ -27,7 +27,7 @@
 # OTHER DEALINGS IN THE SOFTWARE.
 #
 # test_import05.py
-# Error conditions when trying to import files with timestamps past oldest.
+# Error conditions when trying to import files with timestamps past the configured global timestamp.
 
 import os, shutil
 import wiredtiger
@@ -52,9 +52,13 @@ class test_import05(test_import_base):
         ('file_metadata', dict(repair=False)),
         ('repair', dict(repair=True)),
     ]
-    scenarios = make_scenarios(optypes, import_types)
+    compare_timestamps = [
+        ('oldest', dict(global_ts='oldest')),
+        ('stable', dict(global_ts='stable')),
+    ]
+    scenarios = make_scenarios(optypes, import_types, compare_timestamps)
 
-    def test_file_import_ts_past_oldest(self):
+    def test_file_import_ts_past_global_ts(self):
         original_db_file = 'original_db_file'
         uri = 'file:' + original_db_file
         create_config = 'allocation_size=512,key_format=u,log=(enabled=true),value_format=u'
@@ -102,15 +106,22 @@ class test_import05(test_import_base):
 
         # Contruct the config string.
         if self.repair:
-            import_config = 'import=(enabled,repair=true)'
+            if self.global_ts == 'stable':
+                import_config = 'import=(enabled,repair=true,compare_timestamp=stable)'
+            else:
+                import_config = 'import=(enabled,repair=true)'
         else:
-            import_config = 'import=(enabled,repair=false,file_metadata=(' + \
-                original_db_file_config + '))'
+            if self.global_ts == 'stable':
+                import_config = 'import=(enabled,repair=false,compare_timestamp=stable,file_metadata=(' + \
+                    original_db_file_config + '))'
+            else:
+                import_config = 'import=(enabled,repair=false,file_metadata=(' + \
+                    original_db_file_config + '))'
 
         # Create error pattern. Depending on the situation, we substitute a different timestamp into
         # error message to check against.
         error_pattern = \
-            'import found aggregated {} timestamp newer than the current oldest timestamp'
+            'import found aggregated {} timestamp newer than the current'
 
         # Now begin trying to import the file.
         #
@@ -128,7 +139,10 @@ class test_import05(test_import_base):
         #
         # The table we're importing had an operation past this point so we're still expecting an
         # error.
-        self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(self.ts[-1] - 1))
+        if self.global_ts == 'stable':
+            self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(self.ts[-1] - 1))   
+        else: 
+            self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(self.ts[-1] - 1))
 
         # If our latest operation was an insert, we're expecting it to complain about the aggregated
         # start timestamp whereas if we did a delete, we should expect it to complain about stop.
@@ -139,7 +153,10 @@ class test_import05(test_import_base):
             self.assertRaisesException(wiredtiger.WiredTigerError,
                 lambda: self.session.create(uri, import_config))
 
-        # Now place oldest equal to the last insert/delete we made. This should succeed since all
-        # of our aggregated timestamps are now equal to or behind oldest.
-        self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(self.ts[-1]))
+        # Now place global timestamp equal to the last insert/delete we made. This should succeed
+        # since all of our aggregated timestamps are now equal to or behind the global timestamp.
+        if self.global_ts == 'stable':
+            self.conn.set_timestamp('stable_timestamp=' + self.timestamp_str(self.ts[-1]))   
+        else: 
+            self.conn.set_timestamp('oldest_timestamp=' + self.timestamp_str(self.ts[-1]))
         self.session.create(uri, import_config)