Import wiredtiger: aadac222429faa9b20d9344e3648a19be97811b9 from branch mongodb-4.4

ref: 03c93998a7..aadac22242
for: 4.4.7

WT-6204       Possible race between backup and checkpoint at file close
WT-6362       Ensure that history store operations in user session context are correct
WT-6387       Remove unused WT_CURSTD_UPDATE_LOCAL flag
WT-6538       Fix onpage prepare visibility check if the start and stop are from the same prepared transaction
WT-6893       Disable huffman config in compatibility test
WT-6956       Cut WiredTiger 10.0.0 release
WT-7076       Data placement for tiered storage in WiredTiger
WT-7092       Reduce calls to hash URI when opening/closing cached cursors by one
WT-7105       Add recovery error messages to include the URI
WT-7133       Fix bug in stat collection when target pages are reduced due to high HS cache pressure
WT-7173       Devise object naming scheme for tiered storage
WT-7176       Adding Ubuntu 18.04 ASAN variant to wiredtiger build
WT-7185       Avoid aborting a transaction if it is force evicting and oldest
WT-7186       Correct expected in-memory abort updates in prepare scenario
WT-7190       Limit eviction of non-history store pages when checkpoint is operating on history store
WT-7191       Replace FNV hash with City hash
WT-7229       Align out of order and mixed mode handling
WT-7230       CMake build system support for x86 POSIX targets
WT-7241       Add asserts to verify if prepared transaction abort mechanism is working as expected
WT-7253       Add import functionality to test/format
WT-7264       Creating a new configuration for search near that allows it to exit quickly when searching for prefixes
WT-7266       Test to validate re-reading files that were closed with active history
WT-7281       Add metric to record total sessions scanned
WT-7282       Make backup debugging messages into verbose messages
WT-7297       Fix search_near assertion
WT-7312       Keys/Values updated to String type and save the created keys
WT-7315       Implementation of the update thread operation in the test framework
WT-7316       Adding operation throttles, and modifying component functionality to separate core loop
WT-7329       Add hook capability to Python testing
WT-7332       Add ability to cycle through create and drop tables in workgen
WT-7345       Update incorrect copyright notices format
WT-7346       Connect new API changes to local storage extension
WT-7355       Create python hooks to validate tiered cursor implementation
WT-7356       Implement bulk load for tiered tables
WT-7365       Change the configuration file format
WT-7367       Do not remove unstable updates of an in-memory database btree page
WT-7368       Add WT_STORAGE_SOURCE.customize_file_system in place of locations
WT-7374       Add missing branch checking logic for doc-update task
WT-7376       Initialize tiered cursor name
WT-7380       Fix wiredtiger connection string to clear statistics
WT-7384       Fix an assert fire when inserting to the history store
WT-7385       Remove 'auth_token' from being reconfigurable
WT-7387       Replace cluster/member with hostid
WT-7388       Add parens to assignment in conditional
WT-7389       Remove on positioned tiered cursor should leave cursor positioned
WT-7390       Add --noremove flag to Python test runner
WT-7394       Coverity analysis defect 118020: Uninitialized scalar variable
WT-7395       Coverity analysis defect 118042: Dereference after null check
WT-7400       Set WT_HS_READ_ALL flag for the search before fixing the out of order timestamps
WT-7403       Random cursor on empty tiered table loops forever
WT-7409       Remove dead code
WT-7410       Split session flags into two to accommodate more session flags in future
WT-7413       Add an option to wtperf to run a backup operation
WT-7414       Create a python test to ensure that all tables that are dropped during backup are exists in the backup
WT-7415       Add new configuration files with backup option
WT-7419       Tiered local storage changes to use WT file system
WT-7420       Tiered local storage changed to flush files to bucket directory
WT-7423       Clear checkpoint LSN and backup metadata on import
WT-7425       Fix for the -C command line option
WT-7428       Move bucket storage to tiered structure
WT-7429       Set readonly metadata when switching tiered objects
WT-7447       Fix the assert fire because onpage out of order update is not popped from the stack
WT-7453       Coverity analysis defect 119968: Continue has no effect
WT-7454       Coverity analysis defect 119967: Continue has no effect
WT-7455       Coverity analysis defect 119966: Redundant test
WT-7456       Coverity analysis defect 119965: Uninitialized pointer read
WT-7457       Coverity: fix error in local store rename
WT-7458       Coverity analysis defect 119949: Redundant test
WT-7459       Coverity analysis defect 119947: Redundant test
WT-7463       Use wt_off_t to avoid incompatible pointer types
WT-7468       Fix tiered file and object metadata configuration
WT-7474       Reset cursor-order checks whenever a search is performed
WT-7475       Update format to use the new history-store cursor type
WT-7476       Update configuration handling to allow for optional configuration settings
WT-7477       Fix coverity bug: possible NULL dereference
WT-7478       Fix coverity printf arg type to match format
WT-7481       Fix the wrong assert of disk image write gen comparison with btree base write gen
WT-7484       Coverity analysis defect 120014: Uninitialized scalar variable
WT-7485       Coverity analysis defect 120018: Resource leak
WT-7488       Coverity analysis defect 120015: Dereference after null check
WT-7489       Avoid running RTS concurrently with checkpoint
WT-7496       Add operations each tier can support to data structure
WT-7500       Refactor tiered thread start code

1 files changed, 27 insertions, 115 deletions

diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c
index 17e40ef84e6..47a28e016f2 100644
--- a/src/third_party/wiredtiger/src/conn/conn_api.c
+++ b/src/third_party/wiredtiger/src/conn/conn_api.c
@@ -664,116 +664,6 @@ __wt_conn_remove_extractor(WT_SESSION_IMPL *session)
 }
 
 /*
- * __tiered_confchk --
- *     Check for a valid tiered storage source.
- */
-static int
-__tiered_confchk(
-  WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cname, WT_NAMED_STORAGE_SOURCE **nstoragep)
-{
-    WT_CONNECTION_IMPL *conn;
-    WT_NAMED_STORAGE_SOURCE *nstorage;
-
-    *nstoragep = NULL;
-
-    if (cname->len == 0 || WT_STRING_MATCH("none", cname->str, cname->len))
-        return (0);
-
-    conn = S2C(session);
-    TAILQ_FOREACH (nstorage, &conn->storagesrcqh, q)
-        if (WT_STRING_MATCH(nstorage->name, cname->str, cname->len)) {
-            *nstoragep = nstorage;
-            return (0);
-        }
-    WT_RET_MSG(session, EINVAL, "unknown storage source '%.*s'", (int)cname->len, cname->str);
-}
-
-/*
- * __wt_tiered_bucket_config --
- *     Given a configuration, configure the bucket storage.
- */
-int
-__wt_tiered_bucket_config(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cval, WT_CONFIG_ITEM *bucket,
-  WT_BUCKET_STORAGE **bstoragep)
-{
-    WT_BUCKET_STORAGE *bstorage, *new;
-    WT_CONNECTION_IMPL *conn;
-    WT_DECL_RET;
-    WT_NAMED_STORAGE_SOURCE *nstorage;
-#if 0
-    WT_STORAGE_SOURCE *custom, *storage;
-#else
-    WT_STORAGE_SOURCE *storage;
-#endif
-    uint64_t hash_bucket, hash;
-
-    *bstoragep = NULL;
-
-    bstorage = new = NULL;
-    conn = S2C(session);
-
-    __wt_spin_lock(session, &conn->storage_lock);
-
-    WT_ERR(__tiered_confchk(session, cval, &nstorage));
-    if (nstorage == NULL) {
-        if (bucket->len != 0)
-            WT_ERR_MSG(
-              session, EINVAL, "tiered_storage.bucket requires tiered_storage.name to be set");
-        goto out;
-    }
-
-    /*
-     * Check if tiered storage is set on the connection. If someone wants tiered storage on a table,
-     * it needs to be configured on the database as well.
-     */
-    if (conn->bstorage == NULL && bstoragep != &conn->bstorage)
-        WT_ERR_MSG(
-          session, EINVAL, "table tiered storage requires connection tiered storage to be set");
-    hash = __wt_hash_city64(bucket->str, bucket->len);
-    hash_bucket = hash & (conn->hash_size - 1);
-    TAILQ_FOREACH (bstorage, &nstorage->buckethashqh[hash_bucket], q)
-        if (WT_STRING_MATCH(bstorage->bucket, bucket->str, bucket->len))
-            goto out;
-
-    WT_ERR(__wt_calloc_one(session, &new));
-    WT_ERR(__wt_strndup(session, bucket->str, bucket->len, &new->bucket));
-    storage = nstorage->storage_source;
-#if 0
-    if (storage->customize != NULL) {
-        custom = NULL;
-        WT_ERR(storage->customize(storage, &session->iface, cfg_arg, &custom));
-        if (custom != NULL) {
-            bstorage->owned = 1;
-            storage = custom;
-        }
-    }
-#endif
-    new->storage_source = storage;
-    if (bstorage != NULL) {
-        new->object_size = bstorage->object_size;
-        new->retain_secs = bstorage->retain_secs;
-        WT_ERR(__wt_strdup(session, bstorage->auth_token, &new->auth_token));
-    }
-    TAILQ_INSERT_HEAD(&nstorage->bucketqh, new, q);
-    TAILQ_INSERT_HEAD(&nstorage->buckethashqh[hash_bucket], new, hashq);
-    F_SET(new, WT_BUCKET_FREE);
-
-out:
-    __wt_spin_unlock(session, &conn->storage_lock);
-    *bstoragep = new;
-    return (0);
-
-err:
-    if (bstorage != NULL) {
-        __wt_free(session, new->auth_token);
-        __wt_free(session, new->bucket);
-        __wt_free(session, new);
-    }
-    __wt_spin_unlock(session, &conn->storage_lock);
-    return (ret);
-}
-
-/*
  * __conn_add_storage_source --
  *     WT_CONNECTION->add_storage_source method.
  */
@@ -864,10 +754,6 @@ __wt_conn_remove_storage_source(WT_SESSION_IMPL *session)
         while ((bstorage = TAILQ_FIRST(&nstorage->bucketqh)) != NULL) {
             /* Remove from the connection's list, free memory. */
             TAILQ_REMOVE(&nstorage->bucketqh, bstorage, q);
-            storage = bstorage->storage_source;
-            WT_ASSERT(session, storage != NULL);
-            if (bstorage->owned && storage->terminate != NULL)
-                WT_TRET(storage->terminate(storage, (WT_SESSION *)session));
             __wt_free(session, bstorage->auth_token);
             __wt_free(session, bstorage->bucket);
             __wt_free(session, bstorage);
@@ -888,6 +774,25 @@ __wt_conn_remove_storage_source(WT_SESSION_IMPL *session)
 }
 
 /*
+ * __conn_ext_file_system_get --
+ *     WT_EXTENSION.file_system_get method. Get file system in use.
+ */
+static int
+__conn_ext_file_system_get(
+  WT_EXTENSION_API *wt_api, WT_SESSION *session, WT_FILE_SYSTEM **file_system)
+{
+    WT_FILE_SYSTEM *fs;
+
+    WT_UNUSED(session);
+
+    fs = ((WT_CONNECTION_IMPL *)wt_api->conn)->file_system;
+    if (fs == NULL)
+        return (WT_NOTFOUND);
+    *file_system = fs;
+    return (0);
+}
+
+/*
  * __conn_get_extension_api --
  *     WT_CONNECTION.get_extension_api method.
  */
@@ -911,6 +816,7 @@ __conn_get_extension_api(WT_CONNECTION *wt_conn)
     conn->extension_api.config_get_string = __wt_ext_config_get_string;
     conn->extension_api.config_parser_open = __wt_ext_config_parser_open;
     conn->extension_api.config_parser_open_arg = __wt_ext_config_parser_open_arg;
+    conn->extension_api.file_system_get = __conn_ext_file_system_get;
     conn->extension_api.metadata_insert = __wt_ext_metadata_insert;
     conn->extension_api.metadata_remove = __wt_ext_metadata_remove;
     conn->extension_api.metadata_search = __wt_ext_metadata_search;
@@ -2898,6 +2804,12 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, const char *c
     WT_ERR(__conn_load_extensions(session, cfg, false));
 
     /*
+     * Do some early initialization for tiered storage, as this may affect our choice of file system
+     * for some operations.
+     */
+    WT_ERR(__wt_tiered_conn_config(session, cfg, false));
+
+    /*
      * The metadata/log encryptor is configured after extensions, since
      * extensions may load encryptors.  We have to do this before creating
      * the metadata file.
@@ -2982,7 +2894,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, const char *c
      * FIXME-WT-6682: temporarily disable history store verification.
      */
     if (verify_meta) {
-        WT_ERR(__wt_open_internal_session(conn, "verify hs", false, 0, &verify_session));
+        WT_ERR(__wt_open_internal_session(conn, "verify hs", false, 0, 0, &verify_session));
         ret = __wt_hs_verify(verify_session);
         WT_TRET(__wt_session_close_internal(verify_session));
         WT_ERR(ret);