Import wiredtiger: 32fada0e21acc38c1a4a065da7307ddb6384a87d from branch mongodb-master

ref: a647d07c2f..32fada0e21
for: 6.3.0-rc0

WT-10514 Add a new option for verify command to process all tables

5 files changed, 92 insertions, 23 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 436ad746466..11f8bfb61b1 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "a647d07c2f87e2c3b922af30e011a081b471cf48"
+    "commit": "32fada0e21acc38c1a4a065da7307ddb6384a87d"
 }
diff --git a/src/third_party/wiredtiger/src/docs/command-line.dox b/src/third_party/wiredtiger/src/docs/command-line.dox
index 703642e8466..a551b27e460 100644
--- a/src/third_party/wiredtiger/src/docs/command-line.dox
+++ b/src/third_party/wiredtiger/src/docs/command-line.dox
@@ -469,8 +469,11 @@ The \c upgrade command has no command-specific options.
 @section util_verify wt verify
 Check the structural integrity of a table.
 
-The \c verify command verifies the specified table, exiting success if
-the data source is correct, and failure if the data source is corrupted.
+The \c verify command verifies the specified table if a URI argument is provided,
+exiting success if the data source is correct, and failure if the data source is corrupted.
+If no URI argument is provided, it verifies each table in the database, reporting on any
+that are damaged. Currently, the verify operation is not supported with tiered tables
+and the history store file.
 
 @subsection util_verify_synopsis Synopsis
 `wt [-BLmRrSVv] [-C config] [-E secretkey ] [-h directory] verify [-cstu] [-d dump_address | dump_blocks | dump_layout | dump_offsets=#,# | dump_pages ] [uri]`
diff --git a/src/third_party/wiredtiger/src/schema/schema_worker.c b/src/third_party/wiredtiger/src/schema/schema_worker.c
index 6c3329ee0f1..d9d5db77864 100644
--- a/src/third_party/wiredtiger/src/schema/schema_worker.c
+++ b/src/third_party/wiredtiger/src/schema/schema_worker.c
@@ -83,11 +83,11 @@ __wt_schema_worker(WT_SESSION_IMPL *session, const char *uri,
     WT_SESSION *wt_session;
     WT_TABLE *table;
     u_int i;
-    bool skip;
+    bool is_tiered, skip;
 
     table = NULL;
 
-    skip = false;
+    is_tiered = skip = false;
     if (name_func != NULL)
         WT_ERR(name_func(session, uri, &skip));
 
@@ -95,6 +95,12 @@ __wt_schema_worker(WT_SESSION_IMPL *session, const char *uri,
     if (skip)
         return (0);
 
+    /* FIXME-WT-10520 - Let verify process tiered storage related entries once it is supported. */
+    is_tiered = WT_PREFIX_MATCH(uri, "object:") || WT_PREFIX_MATCH(uri, "tier:") ||
+      WT_PREFIX_MATCH(uri, "tiered:");
+    if (file_func == __wt_verify && is_tiered)
+        WT_ERR(ENOTSUP);
+
     /* Get the btree handle(s) and call the underlying function. */
     if (WT_PREFIX_MATCH(uri, "file:")) {
         if (file_func != NULL)
@@ -124,6 +130,11 @@ __wt_schema_worker(WT_SESSION_IMPL *session, const char *uri,
          */
         for (i = 0; i < WT_COLGROUPS(table); i++) {
             colgroup = table->cgroups[i];
+
+            /* FIXME-WT-10520 - Let verify process tiered tables once it is supported. */
+            if (file_func == __wt_verify && WT_PREFIX_MATCH(colgroup->source, "tiered:"))
+                WT_ERR(ENOTSUP);
+
             skip = false;
             if (name_func != NULL)
                 WT_ERR(name_func(session, colgroup->name, &skip));
diff --git a/src/third_party/wiredtiger/src/utilities/util_verify.c b/src/third_party/wiredtiger/src/utilities/util_verify.c
index 5b1f63dfd10..e82ea5fa007 100644
--- a/src/third_party/wiredtiger/src/utilities/util_verify.c
+++ b/src/third_party/wiredtiger/src/utilities/util_verify.c
@@ -32,16 +32,35 @@ usage(void)
 }
 
 /*
+ * verify_one --
+ *     Verify the file specified by the URI.
+ */
+static int
+verify_one(WT_SESSION *session, char *config, char *uri)
+{
+    WT_DECL_RET;
+
+    if ((ret = session->verify(session, uri, config)) != 0)
+        ret = util_err(session, ret, "session.verify: %s", uri);
+    else if (verbose) {
+        /* Verbose configures a progress counter, move to the next line. */
+        printf("\n");
+    }
+    return (ret);
+}
+
+/*
  * util_verify --
  *     The verify command.
  */
 int
 util_verify(WT_SESSION *session, int argc, char *argv[])
 {
+    WT_CURSOR *cursor;
     WT_DECL_RET;
     size_t size;
     int ch;
-    char *config, *dump_offsets, *uri;
+    char *config, *dump_offsets, *key, *uri;
     bool do_not_clear_txn_id, dump_address, dump_app_data, dump_blocks, dump_layout, dump_pages,
       read_corrupt, stable_timestamp;
 
@@ -90,15 +109,6 @@ util_verify(WT_SESSION *session, int argc, char *argv[])
     argc -= __wt_optind;
     argv += __wt_optind;
 
-    /*
-     * The remaining argument is the table name. If we are verifying the history store we do not
-     * accept a URI. Otherwise, we need a URI to operate on.
-     */
-    if (argc != 1)
-        return (usage());
-    if ((uri = util_uri(session, *argv, "table")) == NULL)
-        return (1);
-
     if (do_not_clear_txn_id || dump_address || dump_app_data || dump_blocks || dump_layout ||
       dump_offsets != NULL || dump_pages || read_corrupt || stable_timestamp) {
         size = strlen("do_not_clear_txn_id,") + strlen("dump_address,") +
@@ -122,14 +132,45 @@ util_verify(WT_SESSION *session, int argc, char *argv[])
             goto err;
         }
     }
-    if ((ret = session->verify(session, uri, config)) != 0)
-        (void)util_err(session, ret, "session.verify: %s", uri);
-    else {
-        /*
-         * Verbose configures a progress counter, move to the next line.
-         */
-        if (verbose)
-            printf("\n");
+
+    /* Verify all the tables if no particular URI is specified. */
+    if (argc < 1) {
+        /* Open the metadata file and iterate through its entries, verifying each one. */
+        if ((ret = session->open_cursor(session, WT_METADATA_URI, NULL, NULL, &cursor)) != 0) {
+            /*
+             * If there is no metadata (yet), this will return ENOENT. Treat that the same as an
+             * empty metadata.
+             */
+            if (ret == ENOENT)
+                ret = 0;
+
+            WT_ERR(util_err(session, ret, "%s: WT_SESSION.open_cursor", WT_METADATA_URI));
+        }
+
+        while ((ret = cursor->next(cursor)) == 0) {
+            if ((ret = cursor->get_key(cursor, &key)) != 0)
+                WT_ERR(util_cerr(cursor, "get_key", ret));
+
+            /*
+             * Typically each WT file will have multiple entries, and so only run verify on table
+             * and lsm entries to prevent unnecessary work. Skip over the double up entries and also
+             * any entries that are not supported with verify.
+             *
+             * FIXME-WT-6682 - Let verify process the history store file once history store
+             * verification is implemented.
+             */
+            if ((WT_PREFIX_MATCH(key, "table:") || WT_PREFIX_MATCH(key, "lsm:")) &&
+              strcmp(key, WT_HS_URI) != 0 && strcmp(key, WT_METADATA_URI) != 0 &&
+              !WT_PREFIX_MATCH(key, WT_SYSTEM_PREFIX))
+                WT_TRET(verify_one(session, config, key));
+        }
+        if (ret == WT_NOTFOUND)
+            ret = 0;
+    } else {
+        if ((uri = util_uri(session, *argv, "table")) == NULL)
+            goto err;
+
+        ret = verify_one(session, config, uri);
     }
 
 err:
diff --git a/src/third_party/wiredtiger/test/suite/test_verify.py b/src/third_party/wiredtiger/test/suite/test_verify.py
index a252cc7724f..e1b479c684e 100755
--- a/src/third_party/wiredtiger/test/suite/test_verify.py
+++ b/src/third_party/wiredtiger/test/suite/test_verify.py
@@ -288,5 +288,19 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess):
         # does not exist. Ignore that.
         self.ignoreStderrPatternIfExists('No such file or directory')
 
+    def test_verify_all(self):
+        """
+        Test verify in a 'wt' process without a specific table URI argument.
+        """
+        params = 'key_format=S,value_format=S'
+        ntables = 3
+
+        for i in range(ntables):
+            self.session.create('table:' + self.tablename + str(i), params)
+            self.populate(self.tablename + str(i))
+        self.session.checkpoint()
+
+        self.runWt(["verify"])
+
 if __name__ == '__main__':
     wttest.run()