Import wiredtiger: 798793ffa1a8fb9960edfc782ad4a54b529e311d from branch mongodb-master

ref: a5581e7590..798793ffa1
for: 5.2.0

WT-8316       format backward compatibility mode fixes

6 files changed, 149 insertions, 78 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index c679fd263e1..41e7dfdec45 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "a5581e759002efc42ac455c3e0a3b8cc58cf569a"
+    "commit": "798793ffa1a8fb9960edfc782ad4a54b529e311d"
 }
diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c
index 4eb81bb7c19..2b6c6756224 100644
--- a/src/third_party/wiredtiger/test/format/config.c
+++ b/src/third_party/wiredtiger/test/format/config.c
@@ -115,12 +115,6 @@ config_promote(TABLE *table, CONFIG *cp, CONFIGV *v)
 }
 
 /*
- * We currently disable random LSM testing, that is, it can be specified explicitly but we won't
- * randomly choose LSM as a data_source configuration.
- */
-#define DISABLE_RANDOM_LSM_TESTING 1
-
-/*
  * config_table_am --
  *     Configure the table's access methods (type and source).
  */
@@ -172,8 +166,11 @@ config_table_am(TABLE *table)
             config_single(table, "runs.source=file", false);
             break;
         case 2: /* 20% */
-#if !defined(DISABLE_RANDOM_LSM_TESTING)
+#if 0
             /*
+             * We currently disable random LSM testing, that is, it can be specified explicitly but
+             * we won't randomly choose LSM as a data_source configuration.
+             *
              * LSM requires a row-store and backing disk. Don't configure LSM if in-memory,
              * timestamps or truncation are configured, they result in cache problems.
              *
@@ -181,9 +178,9 @@ config_table_am(TABLE *table)
              */
             if (table->type != ROW || GV(RUNS_IN_MEMORY))
                 break;
-            if (config_explicit(table, "transaction.timestamps") && TV(TRANSACTION_TIMESTAMPS))
+            if (config_explicit(NULL, "transaction.timestamps") && GV(TRANSACTION_TIMESTAMPS))
                 break;
-            if (GV(BACKUP) && config_explicit(table, "backup.incremental") &&
+            if (GV(BACKUP) && config_explicit(NULL, "backup.incremental") &&
               g.backup_incr_flag == INCREMENTAL_BLOCK)
                 break;
             if (config_explicit(table, "ops.truncate") && TV(OPS_TRUNCATE))
@@ -368,12 +365,8 @@ config_run(void)
 static void
 config_backup_incr(void)
 {
-    /* Incremental backup requires backup. */
     if (GV(BACKUP) == 0) {
-        if (!config_explicit(NULL, "backup.incremental"))
-            config_single(NULL, "backup.incremental=off", false);
-        if (g.backup_incr_flag != INCREMENTAL_OFF)
-            testutil_die(EINVAL, "backup.incremental requires backups be configured");
+        config_single(NULL, "backup.incremental=off", false);
         return;
     }
 
@@ -647,16 +640,24 @@ config_compression(TABLE *table, const char *conf_name)
     char confbuf[128];
     const char *cstr;
 
-    /* Return if already specified. */
-    if (config_explicit(table, conf_name))
-        return;
-
     /* Ignore logging compression if we're not doing logging. */
     if (strcmp(conf_name, "logging.compression") == 0 && GV(LOGGING) == 0) {
         config_single(NULL, "logging.compression=none", false);
         return;
     }
 
+    /* Return if already specified and it's a current compression engine. */
+    if (config_explicit(table, conf_name)) {
+        cstr = "none";
+        if (strcmp(conf_name, "logging.compression") == 0)
+            cstr = GVS(LOGGING_COMPRESSION);
+        if (strcmp(conf_name, "btree.compression") == 0)
+            cstr = TVS(BTREE_COMPRESSION);
+        if (cstr == NULL || memcmp(cstr, "bzip", strlen("bzip")) != 0)
+            return;
+        WARN("%s: bzip compression no longer supported", conf_name);
+    }
+
     /*
      * Select a compression type from the list of built-in engines. Listed percentages are only
      * correct if all of the possible engines are compiled in.
@@ -715,39 +716,40 @@ static void
 config_directio(void)
 {
     /*
-     * We don't roll the dice and set direct I/O, it has to be set explicitly. For that reason, any
-     * incompatible "permanent" option set with direct I/O is a configuration error.
+     * We don't roll the dice and set direct I/O, it has to be set explicitly. If there are any
+     * incompatible configurations set explicitly, turn off direct I/O, otherwise turn off the
+     * incompatible configurations.
      */
     if (!GV(DISK_DIRECT_IO))
         return;
+    testutil_assert(config_explicit(NULL, "disk.direct_io") == true);
+
+#undef DIO_CHECK
+#define DIO_CHECK(name, flag)                                                       \
+    if (GV(flag)) {                                                                 \
+        if (config_explicit(NULL, name)) {                                          \
+            WARN("%s not supported with direct I/O, turning off direct I/O", name); \
+            config_single(NULL, "disk.direct_io=off", false);                       \
+            return;                                                                 \
+        }                                                                           \
+        config_single(NULL, #name "=off", false);                                   \
+    }
 
     /*
      * Direct I/O may not work with backups, doing copies through the buffer cache after configuring
      * direct I/O in Linux won't work. If direct I/O is configured, turn off backups.
      */
-    if (GV(BACKUP)) {
-        if (config_explicit(NULL, "backup"))
-            testutil_die(EINVAL, "direct I/O is incompatible with backup configurations");
-        config_single(NULL, "backup=off", false);
-    }
+    DIO_CHECK("backup", BACKUP);
 
     /* Direct I/O may not work with imports for the same reason as for backups. */
-    if (GV(IMPORT)) {
-        if (config_explicit(NULL, "import"))
-            testutil_die(EINVAL, "direct I/O is incompatible with import configurations");
-        config_single(NULL, "import=0", false);
-    }
+    DIO_CHECK("import", IMPORT);
 
     /*
      * Direct I/O may not work with mmap. Theoretically, Linux ignores direct I/O configurations in
      * the presence of shared cache configurations (including mmap), but we've seen file corruption
      * and it doesn't make much sense (the library disallows the combination).
      */
-    if (GV(DISK_MMAP_ALL) != 0) {
-        if (config_explicit(NULL, "disk.mmap_all"))
-            testutil_die(EINVAL, "direct I/O is incompatible with mmap_all configurations");
-        config_single(NULL, "disk.mmap_all=off", false);
-    }
+    DIO_CHECK("disk.mmap_all", DISK_MMAP_ALL);
 
     /*
      * Turn off all external programs. Direct I/O is really, really slow on some machines and it can
@@ -755,11 +757,7 @@ config_directio(void)
      * format just hung, and the 15-minute timeout isn't effective. We could play games to handle
      * child process termination, but it's not worth the effort.
      */
-    if (GV(OPS_SALVAGE)) {
-        if (config_explicit(NULL, "ops.salvage"))
-            testutil_die(EINVAL, "direct I/O is incompatible with salvage configurations");
-        config_single(NULL, "ops.salvage=off", false);
-    }
+    DIO_CHECK("ops.salvage", OPS_SALVAGE);
 }
 
 /*
@@ -892,7 +890,9 @@ config_backup_incr_log_compatibility_check(void)
      * archival if log incremental backup is set.
      */
     if (GV(LOGGING_ARCHIVE) && config_explicit(NULL, "logging.archive"))
-        testutil_die(EINVAL, "backup.incremental=log is incompatible with logging.archive");
+        WARN("%s",
+          "backup.incremental=log is incompatible with logging.archive, turning off "
+          "logging.archive");
     if (GV(LOGGING_ARCHIVE))
         config_single(NULL, "logging.archive=0", false);
 }
@@ -919,19 +919,19 @@ config_lsm_reset(TABLE *table)
      *
      * FIXME: WT-4162.
      */
-    if (config_explicit(table, "ops.prepare"))
+    if (config_explicit(NULL, "ops.prepare"))
         testutil_die(EINVAL, "LSM (currently) incompatible with prepare configurations");
-    config_single(table, "ops.prepare=off", false);
-    if (config_explicit(table, "transaction.timestamps"))
+    config_single(NULL, "ops.prepare=off", false);
+    if (config_explicit(NULL, "transaction.timestamps"))
         testutil_die(EINVAL, "LSM (currently) incompatible with timestamp configurations");
-    config_single(table, "transaction.timestamps=off", false);
+    config_single(NULL, "transaction.timestamps=off", false);
 
     /*
      * LSM does not work with block-based incremental backup, change the incremental backup
      * mechanism if configured to be block based.
      */
     if (GV(BACKUP)) {
-        if (config_explicit(table, "backup.incremental"))
+        if (config_explicit(NULL, "backup.incremental"))
             testutil_die(
               EINVAL, "LSM (currently) incompatible with incremental backup configurations");
         config_single(NULL, "backup.incremental=log", false);
@@ -951,6 +951,7 @@ config_pct(TABLE *table)
         u_int order;      /* Order of assignment */
     } list[5];
     u_int i, max_order, max_slot, n, pct;
+    bool slot_available;
 
 #define CONFIG_MODIFY_ENTRY 2
     list[0].name = "ops.pct.delete";
@@ -974,13 +975,26 @@ config_pct(TABLE *table)
      * order in the list.
      */
     pct = 0;
+    slot_available = false;
     for (i = 0; i < WT_ELEMENTS(list); ++i)
         if (config_explicit(table, list[i].name))
             pct += *list[i].vp;
-        else
+        else {
             list[i].order = mmrand(NULL, 1, 1000);
-    if (pct > 100)
-        testutil_die(EINVAL, "operation percentages do not total to 100%%");
+            slot_available = true;
+        }
+
+    /*
+     * Some older configurations had broken percentages. If summing the explicitly specified
+     * percentages maxes us out, warn and keep running, leaving unspecified operations at 0.
+     */
+    if (pct > 100 || (pct < 100 && !slot_available)) {
+        WARN("operation percentages %s than 100, resetting to random values",
+          pct > 100 ? "greater" : "less");
+        for (i = 0; i < WT_ELEMENTS(list); ++i)
+            list[i].order = mmrand(NULL, 1, 1000);
+        pct = 0;
+    }
 
     /* Cursor modify isn't possible for fixed-length column store. */
     if (table->type == FIX) {
@@ -1042,10 +1056,6 @@ config_transaction(void)
         if (GV(TRANSACTION_IMPLICIT) && config_explicit(NULL, "transaction.implicit"))
             testutil_die(
               EINVAL, "transaction.timestamps is incompatible with implicit transactions");
-
-        /* FIXME-WT-6431: temporarily disable salvage with timestamps. */
-        if (GV(OPS_SALVAGE) && config_explicit(NULL, "ops.salvage"))
-            testutil_die(EINVAL, "transaction.timestamps is incompatible with salvage");
     }
 
     /*
@@ -1192,20 +1202,25 @@ config_print(bool error_display)
 
     /* Display global configuration values. */
     for (cp = configuration_list; cp->name != NULL; ++cp) {
+        /* Skip table count if tables not configured (implying an old-style CONFIG file). */
+        if (ntables == 0 && cp->off == V_GLOBAL_RUNS_TABLES)
+            continue;
+
         /* Skip mismatched objects and configurations. */
         if (!g.lsm_config && F_ISSET(cp, C_TYPE_LSM))
             continue;
-        /* Skip table count if tables not configured (implying an old-style CONFIG file). */
-        if (ntables == 0 && cp->off == V_GLOBAL_RUNS_TABLES)
+
+        /* Skip mismatched table items if the global table is the only table. */
+        if (ntables == 0 && F_ISSET(cp, C_TABLE) && !C_TYPE_MATCH(cp, tables[0]->type))
             continue;
 
-        /*
-         * Otherwise, print if we never configured any tables, if the global item was explicitly
-         * configured, or this isn't a table option.
-         */
+        /* Skip table items if not explicitly set and the global table isn't the only table. */
         gv = &tables[0]->v[cp->off];
-        if (ntables == 0 || gv->set || !F_ISSET(cp, C_TABLE))
-            config_print_one(fp, cp, gv, "");
+        if (ntables > 0 && F_ISSET(cp, C_TABLE) && !gv->set)
+            continue;
+
+        /* Print everything else. */
+        config_print_one(fp, cp, gv, "");
     }
 
     /* Display per-table configuration values. */
@@ -1308,7 +1323,7 @@ config_find(const char *s, size_t len, bool fatal)
     if (fatal)
         testutil_die(EINVAL, "%s: %s: unknown required configuration keyword", progname, s);
 
-    fprintf(stderr, "%s: %s: WARNING, ignoring unknown configuration keyword\n", progname, s);
+    WARN("%s: ignoring unknown configuration keyword", s);
     return (NULL);
 }
 
@@ -1340,6 +1355,9 @@ config_table_extend(u_int ntable)
 {
     u_int i;
 
+    if (g.backward_compatible)
+        testutil_die(0, "multiple tables not supported in backward compatibility mode");
+
     if (ntable <= ntables)
         return;
 
@@ -1373,9 +1391,21 @@ config_single(TABLE *table, const char *s, bool explicit)
     uint32_t steps, v1, v2;
     u_long ntable;
     u_int i;
+    const u_char *t;
     const char *equalp, *vp1, *vp2;
     char *endptr;
 
+    /*
+     * Check for corrupted input. Format has a syntax checking mode and this simplifies that work by
+     * checking for any unexpected characters. It's complicated by wiredtiger.config, as that
+     * configuration option includes JSON characters.
+     */
+    for (t = (const u_char *)s; *t != '\0'; ++t)
+        if (!__wt_isalnum(*t) && !__wt_isspace(*t) && strchr("()-.:=[]_,", *t) == NULL)
+            testutil_die(
+              EINVAL, "%s: configuration contains unexpected character %#x", progname, (u_int)*t);
+
+    /* Skip leading white space. */
     while (__wt_isspace((u_char)*s))
         ++s;
 
@@ -1408,6 +1438,12 @@ config_single(TABLE *table, const char *s, bool explicit)
         return;
     testutil_assert(F_ISSET(cp, C_TABLE) || table == tables[0]);
 
+    /* Ignore tables settings in backward compatible runs. */
+    if (g.backward_compatible && cp->off == V_GLOBAL_RUNS_TABLES) {
+        WARN("backward compatible run, ignoring %s setting", s);
+        return;
+    }
+
     ++equalp;
     v = &table->v[cp->off];
 
@@ -1474,9 +1510,22 @@ config_single(TABLE *table, const char *s, bool explicit)
      * variable" value.
      */
     v1 = config_value(s, vp1, range == RANGE_NONE ? '\0' : (range == RANGE_FIXED ? '-' : ':'));
-    if (!(v1 == 0 && F_ISSET(cp, C_ZERO_NOTSET)) && (v1 < cp->min || v1 > cp->maxset))
-        testutil_die(EINVAL, "%s: %s: value outside min/max values of %" PRIu32 "-%" PRIu32,
-          progname, s, cp->min, cp->maxset);
+    if (!(v1 == 0 && F_ISSET(cp, C_ZERO_NOTSET)) && (v1 < cp->min || v1 > cp->maxset)) {
+        /*
+         * Historically, btree.split_pct support ranges < 50; correct the value.
+         *
+         * Historically, btree.key_min allows ranges under the minimum; correct the value
+         */
+        if (cp->off == V_TABLE_BTREE_SPLIT_PCT && v1 < 50) {
+            v1 = 50;
+            WARN("correcting btree.split_pct value to %" PRIu32, v1);
+        } else if (cp->off == V_TABLE_BTREE_KEY_MIN && v1 < KEY_LEN_CONFIG_MIN) {
+            v1 = KEY_LEN_CONFIG_MIN;
+            WARN("correcting btree.key_min value to %" PRIu32, v1);
+        } else
+            testutil_die(EINVAL, "%s: %s: value outside min/max values of %" PRIu32 "-%" PRIu32,
+              progname, s, cp->min, cp->maxset);
+    }
 
     if (range != RANGE_NONE) {
         v2 = config_value(s, vp2, '\0');
diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h
index a0224d35d2d..79cacfc988f 100644
--- a/src/third_party/wiredtiger/test/format/format.h
+++ b/src/third_party/wiredtiger/test/format/format.h
@@ -427,4 +427,6 @@ void wts_verify(TABLE *, void *);
 #define CUR2S(c) ((WT_SESSION_IMPL *)((WT_CURSOR *)c)->session)
 #endif
 
+#define WARN(fmt, ...) fprintf(stderr, "%s: WARNING: " fmt "\n", progname, __VA_ARGS__);
+
 #include "format.i"
diff --git a/src/third_party/wiredtiger/test/format/kv.c b/src/third_party/wiredtiger/test/format/kv.c
index 54ed9732fb4..7760b74c3ea 100644
--- a/src/third_party/wiredtiger/test/format/kv.c
+++ b/src/third_party/wiredtiger/test/format/kv.c
@@ -42,7 +42,10 @@ key_init(TABLE *table, void *arg)
 
     (void)arg; /* unused argument */
 
-    testutil_check(__wt_snprintf(buf, sizeof(buf), "%s.%u", g.home_key, table->id));
+    if (ntables == 0)
+        testutil_check(__wt_snprintf(buf, sizeof(buf), "%s", g.home_key));
+    else
+        testutil_check(__wt_snprintf(buf, sizeof(buf), "%s.%u", g.home_key, table->id));
 
     /*
      * The key is a variable length item with a leading 10-digit value. Since we have to be able
diff --git a/src/third_party/wiredtiger/test/format/t.c b/src/third_party/wiredtiger/test/format/t.c
index b8a926670fd..aa7baad4c7c 100644
--- a/src/third_party/wiredtiger/test/format/t.c
+++ b/src/third_party/wiredtiger/test/format/t.c
@@ -145,6 +145,8 @@ format_process_env(void)
             set_alarm(0);                             \
     } while (0)
 
+static bool syntax_check; /* Only checking configuration syntax. */
+
 int
 main(int argc, char *argv[])
 {
@@ -160,10 +162,6 @@ main(int argc, char *argv[])
 
     (void)testutil_set_progname(argv);
 
-    /* The monitoring program looks for this line in the log file, push it out quickly. */
-    printf("%s: process %" PRIdMAX " running\n", progname, (intmax_t)getpid());
-    fflush(stdout);
-
     format_process_env();
 
     /*
@@ -176,8 +174,8 @@ main(int argc, char *argv[])
 
     /* Set values from the command line. */
     home = NULL;
-    quiet_flag = false;
-    while ((ch = __wt_getopt(progname, argc, argv, "1BC:c:h:qRrT:t")) != EOF)
+    quiet_flag = syntax_check = false;
+    while ((ch = __wt_getopt(progname, argc, argv, "1BC:c:h:qRSrT:t")) != EOF)
         switch (ch) {
         case '1':
             /* Ignored for backward compatibility. */
@@ -200,6 +198,9 @@ main(int argc, char *argv[])
         case 'R': /* Reopen (start running on an existing database) */
             g.reopen = true;
             break;
+        case 'S': /* Configuration syntax check */
+            syntax_check = true;
+            break;
         case 'T': /* Trace specifics. */
             trace_config(__wt_optarg);
             /* FALLTHROUGH */
@@ -211,6 +212,12 @@ main(int argc, char *argv[])
         }
     argv += __wt_optind;
 
+    /* format.sh looks for this line in the log file, push it out quickly. */
+    if (!syntax_check) {
+        printf("%s: process %" PRIdMAX " running\n", progname, (intmax_t)getpid());
+        fflush(stdout);
+    }
+
     __wt_random_init_seed(NULL, &g.rnd); /* Initialize the RNG. */
 
     /* Printable thread ID. */
@@ -265,6 +272,10 @@ main(int argc, char *argv[])
     config_run();
     g.configured = true;
 
+    /* If checking a CONFIG file syntax, we're done. */
+    if (syntax_check)
+        exit(0);
+
     /* Initialize locks to single-thread backups and timestamps. */
     lock_init(g.wts_session, &g.backup_lock);
     lock_init(g.wts_session, &g.ts_lock);
@@ -358,6 +369,10 @@ main(int argc, char *argv[])
 static void
 format_die(void)
 {
+    /* If only checking configuration syntax, no need to message or drop core. */
+    if (syntax_check)
+        exit(1);
+
     /*
      * Turn off progress reports and tracing so we don't obscure the error message or drop core when
      * using a session that's being closed. The lock we're about to acquire will act as a barrier to
diff --git a/src/third_party/wiredtiger/test/utility/misc.c b/src/third_party/wiredtiger/test/utility/misc.c
index 4d2d989a997..64de4cd0f01 100644
--- a/src/third_party/wiredtiger/test/utility/misc.c
+++ b/src/third_party/wiredtiger/test/utility/misc.c
@@ -53,10 +53,6 @@ testutil_die(int e, const char *fmt, ...)
     (void)fflush(stdout);
     (void)fflush(stderr);
 
-    /* Allow test programs to cleanup on fatal error. */
-    if (custom_die != NULL)
-        (*custom_die)();
-
     fprintf(stderr, "%s: FAILED", progname);
     if (fmt != NULL) {
         fprintf(stderr, ": ");
@@ -67,8 +63,14 @@ testutil_die(int e, const char *fmt, ...)
     if (e != 0)
         fprintf(stderr, ": %s", wiredtiger_strerror(e));
     fprintf(stderr, "\n");
-    fprintf(stderr, "%s: process aborting\n", progname);
+    (void)fflush(stderr);
 
+    /* Allow test programs to cleanup on fatal error. */
+    if (custom_die != NULL)
+        (*custom_die)();
+
+    /* Drop core. */
+    fprintf(stderr, "%s: process aborting\n", progname);
     __wt_abort(NULL);
 }