diff options
| author | Alex Gorrod <alexg@wiredtiger.com> | 2016-03-22 14:55:00 +1100 |
|---|---|---|
| committer | Alex Gorrod <alexg@wiredtiger.com> | 2016-03-22 14:55:05 +1100 |
| commit | 945022433fa60895a6bf412414a89da5a8c14e8c (patch) | |
| tree | d4038459477a402d62a5608e69db992be9fc2011 /src/third_party/wiredtiger/test | |
| parent | 8a5eb4206775a9497a44f01e9108743cc3cd25c4 (diff) | |
| download | mongo-945022433fa60895a6bf412414a89da5a8c14e8c.tar.gz | |
Import wiredtiger-wiredtiger-2.7.0-1122-g9cf8eb2.tar.gz from wiredtiger branch mongodb-3.4
ref: 444981a..9cf8eb2
SERVER-23040 Coverity analysis defect 98151: Dereference after null check
WT-2123 Don't clear allocated memory if not required
WT-2318 Configurable thread wake up time
WT-2322 Join cursor with isolation read-uncommitted may give different results with Bloom filters
WT-2345 Evicting tiny pages creates small pages on disk
WT-2375 Add tests for collators
WT-2381 Dump utility discards table config
WT-2384 lt, le conditions for ordering cursor in join cursor
WT-2391 De-prioritize eviction from indexes
WT-2404 Add streaming pack/unpack methods to the extension API
WT-2414 Avoid extractor calls for ordering cursor in join cursor
WT-2418 Rebalance operation failing with EBUSY
WT-2426 Deadlock caused by recent changes to checkpoint handle locking
WT-2431 Join statistics documentation needed
WT-2435 __wt_evict_file_exclusive_on/off cleanups
WT-2436 lt, le conditions for ref cursor with "strategy=bloom" in join cursor
WT-2443 Getting statistic for all indexes used in join cursor
WT-2444 Broken flag test in wtperf, whitespace
WT-2447 Join cursor reads main table
WT-2448 Add no_scale flag to relevant statistics
WT-2449 Configure should check for a 64-bit build
WT-2451 Allow eviction of metadata
WT-2454 checkpoint_sync=false does *not* prevent flushes/sync to disk.
WT-2456 Update Power8 CRC32 Code
WT-2457 Dropping an LSM table can fail with EBUSY when no user ops are active
WT-2459 Allow Configure scripts to provide the --tag option for libtool when compiling on PPC
WT-2460 Checkpoint failing with WT_ROLLBACK
WT-2461 Python sweep01 test failing
WT-2463 Test that measures idle CPU usage fails under valgrind
WT-2464 Valgrind errors.
WT-2465 Coverity 1352899: Dereference before null check
WT-2466 Coverity 1352893 Buffer not null terminated
WT-2467 Coverity 1352894: Logically dead code
WT-2468 Coverity 1352896: Explicit null dereferenced
WT-2469 Coverity 1352897: Integer overflowed argument
WT-2470 Coverity 1352898: Resource leak
WT-2471 Review WiredTiger "int" printf formats
WT-2473 MSVC doesn't support PRId64
WT-2475 Have reconf script remove cached configure results
WT-2476 btree->evict_lock is being accessed after being destroyed
WT-2477 Missing define in Windows wiredtiger_config.h
WT-2478 Valgrind test failures
WT-2481 Recent changes affect LSM performance
WT-2482 Coverity 1353015, 1353016, out-of-bounds access
WT-2483 readonly02 periodically fails
WT-2484 Coverity 1345809: unchecked return value
WT-2485 Test/format failure with Floating point exception
WT-2487 Release memory in manydbs test
WT-2489 Fix compiler warnings from /test/manydbs
WT-2490 search_near() returns wrong key for column-store
WT-2492 Windows test_config04.test_config04.test_invalid_config crashes
WT-2493 Verbose lsm_manager unsupported
WT-2494 Review calls to __wt_free, plus minor bug in an error path.
WT-2495 Missing memory initialization leads to crash on Windows
WT-2496 Testing revealed error unable to read root page
WT-2497 Enhance test/format to save a copy of backup
WT-2498 LSM tree drop hangs when a user cursor is open
WT-2499 LSM shutdown race causes segfault
WT-2501 Dropping a just opened LSM tree isn't safe
WT-2502 Memory leak in locking handles for checkpoint
WT-2503 Build warning in lsm_tree.c
WT-2506 Using an uninitialised value
Diffstat (limited to 'src/third_party/wiredtiger/test')
32 files changed, 1412 insertions, 163 deletions
diff --git a/src/third_party/wiredtiger/test/bloom/test_bloom.c b/src/third_party/wiredtiger/test/bloom/test_bloom.c index 183dc3d2d42..f95bc7faaf9 100644 --- a/src/third_party/wiredtiger/test/bloom/test_bloom.c +++ b/src/third_party/wiredtiger/test/bloom/test_bloom.c @@ -160,7 +160,7 @@ run(void) for (i = 0; i < g.c_ops; i++) { item.data = g.entries[i]; if ((ret = __wt_bloom_insert(bloomp, &item)) != 0) - testutil_die(ret, "__wt_bloom_insert: %d", i); + testutil_die(ret, "__wt_bloom_insert: %" PRIu32, i); } testutil_check(__wt_bloom_finalize(bloomp)); @@ -168,7 +168,8 @@ run(void) for (i = 0; i < g.c_ops; i++) { item.data = g.entries[i]; if ((ret = __wt_bloom_get(bloomp, &item)) != 0) { - fprintf(stderr, "get failed at record: %d\n", i); + fprintf(stderr, + "get failed at record: %" PRIu32 "\n", i); testutil_die(ret, "__wt_bloom_get"); } } @@ -201,7 +202,8 @@ run(void) testutil_die(ret, "__wt_bloom_get"); } free((void *)item.data); - printf("Out of %d ops, got %d false positives, %.4f%%\n", + printf( + "Out of %" PRIu32 " ops, got %" PRIu32 " false positives, %.4f%%\n", g.c_ops, fp, 100.0 * fp/g.c_ops); testutil_check(__wt_bloom_drop(bloomp, NULL)); } diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c index 0f28a86b675..c5524b3c63e 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c @@ -136,7 +136,7 @@ main(int argc, char *argv[]) printf("%s: process %" PRIu64 "\n", g.progname, (uint64_t)getpid()); for (cnt = 1; (runs == 0 || cnt <= runs) && g.status == 0; ++cnt) { - printf(" %d: %u workers, %u tables\n", + printf(" %d: %d workers, %d tables\n", cnt, g.nworkers, g.ntables); (void)cleanup(); /* Clean up previous runs */ diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order.c b/src/third_party/wiredtiger/test/cursor_order/cursor_order.c index 68d2f092c60..d8cfc0c1421 100644 --- a/src/third_party/wiredtiger/test/cursor_order/cursor_order.c +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order.c @@ -154,8 +154,10 @@ main(int argc, char *argv[]) printf("%s: process %" PRIu64 "\n", progname, (uint64_t)getpid()); for (cnt = 1; runs == 0 || cnt <= runs; ++cnt) { - printf(" %d: %u reverse scanners, %u writers\n", cnt, - (int)cfg->reverse_scanners, (int)cfg->append_inserters); + printf( + " %d: %" PRIu64 + " reverse scanners, %" PRIu64 " writers\n", + cnt, cfg->reverse_scanners, cfg->append_inserters); shutdown(); /* Clean up previous runs */ diff --git a/src/third_party/wiredtiger/test/fops/file.c b/src/third_party/wiredtiger/test/fops/file.c index 4cd92e7b590..ea15f1ee80d 100644 --- a/src/third_party/wiredtiger/test/fops/file.c +++ b/src/third_party/wiredtiger/test/fops/file.c @@ -147,7 +147,7 @@ obj_create_unique(int force) /* Generate a unique object name. */ if ((ret = pthread_rwlock_wrlock(&single)) != 0) testutil_die(ret, "pthread_rwlock_wrlock single"); - (void)snprintf(new_uri, sizeof(new_uri), "%s.%d", uri, ++uid); + (void)snprintf(new_uri, sizeof(new_uri), "%s.%u", uri, ++uid); if ((ret = pthread_rwlock_unlock(&single)) != 0) testutil_die(ret, "pthread_rwlock_unlock single"); diff --git a/src/third_party/wiredtiger/test/fops/fops.c b/src/third_party/wiredtiger/test/fops/fops.c index fbc9d9c6048..3333ff16858 100644 --- a/src/third_party/wiredtiger/test/fops/fops.c +++ b/src/third_party/wiredtiger/test/fops/fops.c @@ -109,7 +109,7 @@ fop(void *arg) __wt_random_init(&rnd); for (i = 0; i < nops; ++i, __wt_yield()) - switch (__wt_random(&rnd) % 9) { + switch (__wt_random(&rnd) % 10) { case 0: ++s->bulk; obj_bulk(); diff --git a/src/third_party/wiredtiger/test/format/backup.c b/src/third_party/wiredtiger/test/format/backup.c index 56657940514..2b1463bd0e3 100644 --- a/src/third_party/wiredtiger/test/format/backup.c +++ b/src/third_party/wiredtiger/test/format/backup.c @@ -67,6 +67,13 @@ copy_file(const char *name) "cp %s/%s %s/%s", g.home, name, g.home_backup, name); testutil_checkfmt(system(cmd), "backup copy: %s", cmd); free(cmd); + + len = strlen(g.home) + strlen(g.home_backup2) + strlen(name) * 2 + 20; + cmd = dmalloc(len); + (void)snprintf(cmd, len, + "cp %s/%s %s/%s", g.home, name, g.home_backup2, name); + testutil_checkfmt(system(cmd), "backup copy: %s", cmd); + free(cmd); } /* diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index c54fd061736..a129c5395fd 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -109,6 +109,7 @@ typedef struct { char *home; /* Home directory */ char *home_backup; /* Hot-backup directory */ + char *home_backup2; /* Saved Hot-backup directory */ char *home_backup_init; /* Initialize backup command */ char *home_bdb; /* BDB directory */ char *home_config; /* Run CONFIG file path */ diff --git a/src/third_party/wiredtiger/test/format/util.c b/src/third_party/wiredtiger/test/format/util.c index 347b2ea1db3..2e4c869366c 100644 --- a/src/third_party/wiredtiger/test/format/util.c +++ b/src/third_party/wiredtiger/test/format/util.c @@ -310,6 +310,10 @@ path_setup(const char *home) g.home_backup = dmalloc(len); snprintf(g.home_backup, len, "%s/%s", g.home, "BACKUP"); + len = strlen(g.home) + strlen("BACKUP2") + 2; + g.home_backup2 = dmalloc(len); + snprintf(g.home_backup2, len, "%s/%s", g.home, "BACKUP2"); + /* BDB directory. */ len = strlen(g.home) + strlen("bdb") + 2; g.home_bdb = dmalloc(len); @@ -340,13 +344,15 @@ path_setup(const char *home) /* Backup directory initialize command, remove and re-create it. */ #undef CMD #ifdef _WIN32 -#define CMD "del /s /q >:nul && mkdir %s" +#define CMD "del /s /q >:nul && mkdir %s %s" #else -#define CMD "rm -rf %s && mkdir %s" +#define CMD "rm -rf %s %s && mkdir %s %s" #endif - len = strlen(g.home_backup) * 2 + strlen(CMD) + 1; + len = strlen(g.home_backup) * 2 + + strlen(g.home_backup2) * 2 + strlen(CMD) + 1; g.home_backup_init = dmalloc(len); - snprintf(g.home_backup_init, len, CMD, g.home_backup, g.home_backup); + snprintf(g.home_backup_init, len, CMD, g.home_backup, g.home_backup2, + g.home_backup, g.home_backup2); /* * Salvage command, save the interesting files so we can replay the diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index a0e57dc2bee..81e484296e2 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -53,7 +53,8 @@ compressor(uint32_t compress_flag) default: break; } - testutil_die(EINVAL, "illegal compression flag: 0x%x", compress_flag); + testutil_die(EINVAL, + "illegal compression flag: %#" PRIx32, compress_flag); } /* @@ -71,7 +72,8 @@ encryptor(uint32_t encrypt_flag) default: break; } - testutil_die(EINVAL, "illegal encryption flag: 0x%x", encrypt_flag); + testutil_die(EINVAL, + "illegal encryption flag: %#" PRIx32, encrypt_flag); } static int @@ -313,7 +315,7 @@ wts_create(void) p += snprintf(p, REMAIN(p, end), "key_format=%s," "allocation_size=512,%s" - "internal_page_max=%d,leaf_page_max=%d", + "internal_page_max=%" PRIu32 ",leaf_page_max=%" PRIu32, (g.type == ROW) ? "u" : "r", g.c_firstfit ? "block_allocation=first," : "", maxintlpage, maxleafpage); @@ -325,15 +327,15 @@ wts_create(void) maxintlkey = mmrand(NULL, maxintlpage / 50, maxintlpage / 40); if (maxintlkey > 20) p += snprintf(p, REMAIN(p, end), - ",internal_key_max=%d", maxintlkey); + ",internal_key_max=%" PRIu32, maxintlkey); maxleafkey = mmrand(NULL, maxleafpage / 50, maxleafpage / 40); if (maxleafkey > 20) p += snprintf(p, REMAIN(p, end), - ",leaf_key_max=%d", maxleafkey); + ",leaf_key_max=%" PRIu32, maxleafkey); maxleafvalue = mmrand(NULL, maxleafpage * 10, maxleafpage / 40); if (maxleafvalue > 40 && maxleafvalue < 100 * 1024) p += snprintf(p, REMAIN(p, end), - ",leaf_value_max=%d", maxleafvalue); + ",leaf_value_max=%" PRIu32, maxleafvalue); switch (g.type) { case FIX: @@ -361,7 +363,7 @@ wts_create(void) ",huffman_value=english"); if (g.c_dictionary) p += snprintf(p, REMAIN(p, end), - ",dictionary=%d", mmrand(NULL, 123, 517)); + ",dictionary=%" PRIu32, mmrand(NULL, 123, 517)); break; } diff --git a/src/third_party/wiredtiger/test/manydbs/Makefile.am b/src/third_party/wiredtiger/test/manydbs/Makefile.am new file mode 100644 index 00000000000..53559b25243 --- /dev/null +++ b/src/third_party/wiredtiger/test/manydbs/Makefile.am @@ -0,0 +1,13 @@ +AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \ + -I$(top_srcdir)/test/utility + +noinst_PROGRAMS = t +t_SOURCES = manydbs.c +t_LDADD = $(top_builddir)/libwiredtiger.la +t_LDFLAGS = -static + +# Run this during a "make check" smoke test. +TESTS = smoke.sh + +clean-local: + rm -rf WiredTiger* *.core __* diff --git a/src/third_party/wiredtiger/test/manydbs/manydbs.c b/src/third_party/wiredtiger/test/manydbs/manydbs.c new file mode 100644 index 00000000000..1d3412a7b06 --- /dev/null +++ b/src/third_party/wiredtiger/test/manydbs/manydbs.c @@ -0,0 +1,264 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include <sys/wait.h> +#include <errno.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#ifndef _WIN32 +#include <unistd.h> +#endif + +#include <wiredtiger.h> + +#include "test_util.i" + +#define HOME_SIZE 512 +#define HOME_BASE "WT_HOME" +static char home[HOME_SIZE]; /* Base home directory */ +static char hometmp[HOME_SIZE]; /* Each conn home directory */ +static const char *progname; /* Program name */ +static const char * const uri = "table:main"; + +#define WTOPEN_CFG_COMMON \ + "create,log=(file_max=10M,archive=false,enabled)," \ + "statistics=(fast),statistics_log=(wait=5)," +#define WT_CONFIG0 \ + WTOPEN_CFG_COMMON \ + "transaction_sync=(enabled=false)" +#define WT_CONFIG1 \ + WTOPEN_CFG_COMMON \ + "transaction_sync=(enabled,method=none)" +#define WT_CONFIG2 \ + WTOPEN_CFG_COMMON \ + "transaction_sync=(enabled,method=fsync)" + +#define MAX_DBS 10 +#define MAX_IDLE_TIME 30 +#define IDLE_INCR 5 + +#define MAX_KV 100 +#define MAX_VAL 128 + +static void +usage(void) +{ + fprintf(stderr, + "usage: %s [-I] [-D maxdbs] [-h dir]\n", progname); + exit(EXIT_FAILURE); +} + +extern int __wt_optind; +extern char *__wt_optarg; + +void (*custom_die)(void) = NULL; + +WT_CONNECTION **connections = NULL; +WT_CURSOR **cursors = NULL; +WT_RAND_STATE rnd; +WT_SESSION **sessions = NULL; + +static int +get_stat(WT_SESSION *stat_session, int stat_field, uint64_t *valuep) +{ + WT_CURSOR *statc; + const char *desc, *pvalue; + int ret; + + testutil_check(stat_session->open_cursor(stat_session, + "statistics:", NULL, NULL, &statc)); + statc->set_key(statc, stat_field); + if ((ret = statc->search(statc)) != 0) + return (ret); + + ret = statc->get_value(statc, &desc, &pvalue, valuep); + testutil_check(statc->close(statc)); + return (ret); +} + +static int +run_ops(int dbs) +{ + WT_ITEM data; + int db_set, i, key; + uint32_t db; + uint8_t buf[MAX_VAL]; + + memset(buf, 0, sizeof(buf)); + for (i = 0; i < MAX_VAL; ++i) + buf[i] = (uint8_t)__wt_random(&rnd); + data.data = buf; + /* + * Write a small amount of data into a random subset of the databases. + */ + db_set = dbs / 4; + for (i = 0; i < db_set; ++i) { + db = __wt_random(&rnd) % (uint32_t)dbs; + printf("Write to database %" PRIu32 "\n", db); + for (key = 0; key < MAX_KV; ++key) { + data.size = __wt_random(&rnd) % MAX_VAL; + cursors[db]->set_key(cursors[db], key); + cursors[db]->set_value(cursors[db], &data); + testutil_check(cursors[db]->insert(cursors[db])); + } + } + return (0); +} + +int +main(int argc, char *argv[]) +{ + uint64_t cond_reset, cond_wait; + uint64_t *cond_reset_orig; + int cfg, ch, dbs, i; + bool idle; + const char *working_dir, *wt_cfg; + char cmd[128]; + + if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) + progname = argv[0]; + else + ++progname; + dbs = MAX_DBS; + working_dir = HOME_BASE; + idle = false; + while ((ch = __wt_getopt(progname, argc, argv, "D:h:I")) != EOF) + switch (ch) { + case 'D': + dbs = atoi(__wt_optarg); + break; + case 'h': + working_dir = __wt_optarg; + break; + case 'I': + idle = true; + break; + default: + usage(); + } + argc -= __wt_optind; + argv += __wt_optind; + if (argc != 0) + usage(); + + /* + * Allocate arrays for connection handles, sessions, statistics + * cursors and, if needed, data cursors. + */ + if ((connections = calloc( + (size_t)dbs, sizeof(WT_CONNECTION *))) == NULL) + testutil_die(ENOMEM, "connection array malloc"); + if ((sessions = calloc( + (size_t)dbs, sizeof(WT_SESSION *))) == NULL) + testutil_die(ENOMEM, "session array malloc"); + if ((cond_reset_orig = calloc((size_t)dbs, sizeof(uint64_t))) == NULL) + testutil_die(ENOMEM, "orig stat malloc"); + if (!idle && ((cursors = calloc( + (size_t)dbs, sizeof(WT_CURSOR *))) == NULL)) + testutil_die(ENOMEM, "cursor array malloc"); + memset(cmd, 0, sizeof(cmd)); + /* + * Set up all the directory names. + */ + testutil_work_dir_from_path(home, HOME_SIZE, working_dir); + testutil_make_work_dir(home); + __wt_random_init(&rnd); + for (i = 0; i < dbs; ++i) { + snprintf(hometmp, HOME_SIZE, "%s/%s.%d", home, HOME_BASE, i); + testutil_make_work_dir(hometmp); + /* + * Open each database. Rotate different configurations + * among them. Open a session and statistics cursor. + * If writing data, create the table and open a data cursor. + */ + cfg = i % 3; + if (cfg == 0) + wt_cfg = WT_CONFIG0; + else if (cfg == 1) + wt_cfg = WT_CONFIG1; + else + wt_cfg = WT_CONFIG2; + testutil_check(wiredtiger_open( + hometmp, NULL, wt_cfg, &connections[i])); + testutil_check(connections[i]->open_session(connections[i], + NULL, NULL, &sessions[i])); + if (!idle) { + testutil_check(sessions[i]->create(sessions[i], + uri, "key_format=Q,value_format=u")); + testutil_check(sessions[i]->open_cursor(sessions[i], + uri, NULL, NULL, &cursors[i])); + } + } + + sleep(10); + + /* + * Record original reset setting. There could have been some + * activity during the creation period. + */ + for (i = 0; i < dbs; ++i) + testutil_check(get_stat(sessions[i], + WT_STAT_CONN_COND_AUTO_WAIT_RESET, &cond_reset_orig[i])); + for (i = 0; i < MAX_IDLE_TIME; i += IDLE_INCR) { + if (!idle) + testutil_check(run_ops(dbs)); + printf("Sleep %d (%d of %d)\n", IDLE_INCR, i, MAX_IDLE_TIME); + sleep(IDLE_INCR); + } + for (i = 0; i < dbs; ++i) { + testutil_check(get_stat(sessions[i], + WT_STAT_CONN_COND_AUTO_WAIT_RESET, &cond_reset)); + testutil_check(get_stat(sessions[i], + WT_STAT_CONN_COND_AUTO_WAIT, &cond_wait)); + /* + * On an idle workload there should be no resets of condition + * variables during the idle period. Even with a light + * workload, resets should not be very common. We look for 5%. + */ + if (idle && cond_reset != cond_reset_orig[i]) + testutil_die(ERANGE, + "condition reset on idle connection %d of %" PRIu64, + i, cond_reset); + if (!idle && cond_reset > cond_wait / 20) + testutil_die(ERANGE, "connection %d condition reset %" + PRIu64 " exceeds 5%% of %" PRIu64, + i, cond_reset, cond_wait); + testutil_check(connections[i]->close(connections[i], NULL)); + } + + /* Cleanup allocated memory. */ + free(connections); + free(sessions); + free(cond_reset_orig); + if (!idle) + free(cursors); + + return (EXIT_SUCCESS); +} diff --git a/src/third_party/wiredtiger/test/manydbs/smoke.sh b/src/third_party/wiredtiger/test/manydbs/smoke.sh new file mode 100755 index 00000000000..c0e2976f154 --- /dev/null +++ b/src/third_party/wiredtiger/test/manydbs/smoke.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +set -e + +# Smoke-test format as part of running "make check". +# Run with: +# 1. The defaults +# 2. Set idle flag to turn off operations. +# 3. More dbs. +# +echo "manydbs: default with operations turned on" +$TEST_WRAPPER ./t +echo "manydbs: totally idle databases" +$TEST_WRAPPER ./t -I +echo "manydbs: 40 databases with operations" +$TEST_WRAPPER ./t -D 40 +echo "manydbs: 40 idle databases" +$TEST_WRAPPER ./t -I -D 40 diff --git a/src/third_party/wiredtiger/test/readonly/Makefile.am b/src/third_party/wiredtiger/test/readonly/Makefile.am index 384e197a1f8..3abcd2386a1 100644 --- a/src/third_party/wiredtiger/test/readonly/Makefile.am +++ b/src/third_party/wiredtiger/test/readonly/Makefile.am @@ -10,4 +10,4 @@ t_LDFLAGS = -static TESTS = smoke.sh clean-local: - rm -rf WiredTiger* *.core __* + rm -rf WT_RD* WiredTiger* *.core __* diff --git a/src/third_party/wiredtiger/test/readonly/readonly.c b/src/third_party/wiredtiger/test/readonly/readonly.c index 100ccbf81b7..41400da2605 100644 --- a/src/third_party/wiredtiger/test/readonly/readonly.c +++ b/src/third_party/wiredtiger/test/readonly/readonly.c @@ -42,9 +42,13 @@ #define HOME_SIZE 512 static char home[HOME_SIZE]; /* Program working dir lock file */ -static char home_wr[HOME_SIZE]; /* Writable dir copy no lock file */ -static char home_rd[HOME_SIZE]; /* Read-only dir */ -static char home_rd2[HOME_SIZE]; /* Read-only dir no lock file */ +#define HOME_WR_SUFFIX ".WRNOLOCK" /* Writable dir copy no lock file */ +static char home_wr[HOME_SIZE + sizeof(HOME_WR_SUFFIX)]; +#define HOME_RD_SUFFIX ".RD" /* Read-only dir */ +static char home_rd[HOME_SIZE + sizeof(HOME_RD_SUFFIX)]; +#define HOME_RD2_SUFFIX ".RDNOLOCK" /* Read-only dir no lock file */ +static char home_rd2[HOME_SIZE + sizeof(HOME_RD2_SUFFIX)]; + static const char *progname; /* Program name */ static const char *saved_argv0; /* Program command */ static const char * const uri = "table:main"; @@ -87,13 +91,14 @@ run_child(const char *homedir, int op, int expect) cfg = ENV_CONFIG_RD; else cfg = ENV_CONFIG_WR; - ret = wiredtiger_open(homedir, NULL, cfg, &conn); - if (expect == EXPECT_SUCCESS && ret != 0) - testutil_die(ret, "wiredtiger_open success err"); - if (expect == EXPECT_ERR) { - if (ret == 0) + if ((ret = wiredtiger_open(homedir, NULL, cfg, &conn)) == 0) { + if (expect == EXPECT_ERR) + testutil_die( + ret, "wiredtiger_open expected error, succeeded"); + } else { + if (expect == EXPECT_SUCCESS) testutil_die( - ret, "wiredtiger_open expected err succeeded"); + ret, "wiredtiger_open expected success, error"); /* * If we expect an error and got one, we're done. */ @@ -207,17 +212,14 @@ main(int argc, char *argv[]) if (argc != 0) usage(); - memset(buf, 0, sizeof(buf)); /* * Set up all the directory names. */ - testutil_work_dir_from_path(home, 512, working_dir); - strncpy(home_wr, home, HOME_SIZE); - strcat(home_wr, ".WRNOLOCK"); - strncpy(home_rd, home, HOME_SIZE); - strcat(home_rd, ".RD"); - strncpy(home_rd2, home, HOME_SIZE); - strcat(home_rd2, ".RDNOLOCK"); + testutil_work_dir_from_path(home, sizeof(home), working_dir); + (void)snprintf(home_wr, sizeof(home_wr), "%s%s", home, HOME_WR_SUFFIX); + (void)snprintf(home_rd, sizeof(home_rd), "%s%s", home, HOME_RD_SUFFIX); + (void)snprintf( + home_rd2, sizeof(home_rd2), "%s%s", home, HOME_RD2_SUFFIX); if (!child) { testutil_make_work_dir(home); testutil_make_work_dir(home_wr); @@ -260,6 +262,7 @@ main(int argc, char *argv[]) /* * Write data into the table and then cleanly shut down connection. */ + memset(buf, 0, sizeof(buf)); data.data = buf; data.size = MAX_VAL; for (i = 0; i < MAX_KV; ++i) { @@ -329,7 +332,8 @@ main(int argc, char *argv[]) * the child even though it should not be. So use 'system' to spawn * an entirely new process. */ - (void)snprintf(cmd, sizeof(cmd), "%s -R", saved_argv0); + (void)snprintf( + cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir); if ((status = system(cmd)) < 0) testutil_die(status, "system"); /* @@ -341,7 +345,8 @@ main(int argc, char *argv[]) /* * Scenario 2. Run child with writable config. */ - (void)snprintf(cmd, sizeof(cmd), "%s -W", saved_argv0); + (void)snprintf( + cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir); if ((status = system(cmd)) < 0) testutil_die(status, "system"); @@ -362,7 +367,8 @@ main(int argc, char *argv[]) /* * Scenario 3. Child read-only. */ - (void)snprintf(cmd, sizeof(cmd), "%s -R", saved_argv0); + (void)snprintf( + cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir); if ((status = system(cmd)) < 0) testutil_die(status, "system"); if (WEXITSTATUS(status) != 0) @@ -371,7 +377,8 @@ main(int argc, char *argv[]) /* * Scenario 4. Run child with writable config. */ - (void)snprintf(cmd, sizeof(cmd), "%s -W", saved_argv0); + (void)snprintf( + cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir); if ((status = system(cmd)) < 0) testutil_die(status, "system"); if (WEXITSTATUS(status) != 0) diff --git a/src/third_party/wiredtiger/test/recovery/random-abort.c b/src/third_party/wiredtiger/test/recovery/random-abort.c index c9cc10d2db3..f9c3ed28814 100644 --- a/src/third_party/wiredtiger/test/recovery/random-abort.c +++ b/src/third_party/wiredtiger/test/recovery/random-abort.c @@ -249,9 +249,10 @@ main(int argc, char *argv[]) if ((ret = conn->close(conn, NULL)) != 0) testutil_die(ret, "WT_CONNECTION:close"); if (absent) { - printf("%u record(s) absent from %u\n", absent, count); + printf("%" PRIu32 " record(s) absent from %" PRIu32 "\n", + absent, count); return (EXIT_FAILURE); } - printf("%u records verified\n", count); + printf("%" PRIu32 " records verified\n", count); return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/recovery/truncated-log.c b/src/third_party/wiredtiger/test/recovery/truncated-log.c index 23269e99d35..67fdb932c27 100644 --- a/src/third_party/wiredtiger/test/recovery/truncated-log.c +++ b/src/third_party/wiredtiger/test/recovery/truncated-log.c @@ -156,14 +156,16 @@ fill_db(void) "%" PRIu32 " %" PRIu32 "\n", save_lsn.l.offset, i - 1) == -1) testutil_die(errno, "fprintf"); - if (fclose(fp) != 0) - testutil_die(errno, "fclose"); - abort(); + break; } } first = false; } } + if (fclose(fp) != 0) + testutil_die(errno, "fclose"); + abort(); + /* NOTREACHED */ } extern int __wt_optind; @@ -243,8 +245,10 @@ main(int argc, char *argv[]) * The offset is the beginning of the last record. Truncate to * the middle of that last record (i.e. ahead of that offset). */ + if (offset > UINT64_MAX - V_SIZE) + testutil_die(ERANGE, "offset"); new_offset = offset + V_SIZE; - printf("Parent: Truncate to %u\n", (uint32_t)new_offset); + printf("Parent: Truncate to %" PRIu64 "\n", new_offset); if ((ret = truncate(LOG_FILE_1, (wt_off_t)new_offset)) != 0) testutil_die(errno, "truncate"); @@ -267,9 +271,10 @@ main(int argc, char *argv[]) if ((ret = conn->close(conn, NULL)) != 0) testutil_die(ret, "WT_CONNECTION:close"); if (count > max_key) { - printf("expected %u records found %u\n", max_key, count); + printf("expected %" PRIu32 " records found %" PRIu32 "\n", + max_key, count); return (EXIT_FAILURE); } - printf("%u records verified\n", count); + printf("%" PRIu32 " records verified\n", count); return (EXIT_SUCCESS); } diff --git a/src/third_party/wiredtiger/test/suite/helper.py b/src/third_party/wiredtiger/test/suite/helper.py index 3c460e23d08..f85d708880f 100644 --- a/src/third_party/wiredtiger/test/suite/helper.py +++ b/src/third_party/wiredtiger/test/suite/helper.py @@ -107,7 +107,10 @@ def copy_wiredtiger_home(olddir, newdir, aligned=True): for fname in os.listdir(olddir): fullname = os.path.join(olddir, fname) # Skip lock file, on Windows it is locked. - if os.path.isfile(fullname) and "WiredTiger.lock" not in fullname: + # Skip temporary log files. + if os.path.isfile(fullname) and "WiredTiger.lock" not in fullname and \ + "WiredTigerTmplog" not in fullname and \ + "WiredTigerPreplog" not in fullname: # Use a dd command that does not align on a block boundary. if aligned: shutil.copy(fullname, newdir) @@ -196,31 +199,36 @@ def complex_populate_index_count(): # config: prefix of the session.create configuration string # rows: entries to insert def complex_populate(self, uri, config, rows): - complex_populate_type(self, uri, config, rows, '') + complex_populate_type(self, uri, config, '', rows, '') +def complex_populate_cgconfig(self, uri, config, rows): + complex_populate_type(self, uri, config, config, rows, '') def complex_populate_lsm(self, uri, config, rows): - complex_populate_type(self, uri, config, rows, 'type=lsm') -def complex_populate_type(self, uri, config, rows, type): + complex_populate_type(self, uri, config, '', rows, 'type=lsm') +def complex_populate_cgconfig_lsm(self, uri, config, rows): + complex_populate_type(self, uri, config, config, rows, 'type=lsm') +def complex_populate_type(self, uri, config, cgconfig, rows, type): self.session.create(uri, config + ',value_format=SiSS,' + 'columns=(record,column2,column3,column4,column5),' + 'colgroups=(cgroup1,cgroup2,cgroup3,cgroup4,cgroup5,cgroup6)') cgname = 'colgroup:' + uri.split(":")[1] - self.session.create(cgname + ':cgroup1', 'columns=(column2)' + ',' + type) - self.session.create(cgname + ':cgroup2', 'columns=(column3)' + ',' + type) - self.session.create(cgname + ':cgroup3', 'columns=(column4)' + ',' + type) + cgcfg = ',' + cgconfig + ',' + type + self.session.create(cgname + ':cgroup1', 'columns=(column2)' + ',' + cgcfg) + self.session.create(cgname + ':cgroup2', 'columns=(column3)' + ',' + cgcfg) + self.session.create(cgname + ':cgroup3', 'columns=(column4)' + ',' + cgcfg) self.session.create( - cgname + ':cgroup4', 'columns=(column2,column3)' + ',' + type) + cgname + ':cgroup4', 'columns=(column2,column3)' + ',' + cgcfg) self.session.create( - cgname + ':cgroup5', 'columns=(column3,column4)' + ',' + type) + cgname + ':cgroup5', 'columns=(column3,column4)' + ',' + cgcfg) self.session.create( - cgname + ':cgroup6', 'columns=(column2,column4,column5)' + ',' + type) + cgname + ':cgroup6', 'columns=(column2,column4,column5)' + ',' + cgcfg) indxname = 'index:' + uri.split(":")[1] - self.session.create(indxname + ':indx1', 'columns=(column2)' + ',' + type) - self.session.create(indxname + ':indx2', 'columns=(column3)' + ',' + type) - self.session.create(indxname + ':indx3', 'columns=(column4)' + ',' + type) + self.session.create(indxname + ':indx1', 'columns=(column2)' + ',' + cgcfg) + self.session.create(indxname + ':indx2', 'columns=(column3)' + ',' + cgcfg) + self.session.create(indxname + ':indx3', 'columns=(column4)' + ',' + cgcfg) self.session.create( - indxname + ':indx4', 'columns=(column2,column4)' + ',' + type) + indxname + ':indx4', 'columns=(column2,column4)' + ',' + cgcfg) cursor = self.session.open_cursor(uri, None) for i in range(1, rows + 1): cursor[key_populate(cursor, i)] = \ @@ -228,9 +236,9 @@ def complex_populate_type(self, uri, config, rows, type): cursor.close() # add some indices after populating self.session.create( - indxname + ':indx5', 'columns=(column3,column5)' + ',' + type) + indxname + ':indx5', 'columns=(column3,column5)' + ',' + cgcfg) self.session.create( - indxname + ':indx6', 'columns=(column3,column5,column4)' + ',' + type) + indxname + ':indx6', 'columns=(column3,column5,column4)' + ',' + cgcfg) def complex_populate_colgroup_name(self, uri, i): return 'colgroup:' + uri.split(":")[1] + ':cgroup' + str(i + 1) diff --git a/src/third_party/wiredtiger/test/suite/test_bug008.py b/src/third_party/wiredtiger/test/suite/test_bug008.py index 8f0526d9cef..0243887e258 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug008.py +++ b/src/third_party/wiredtiger/test/suite/test_bug008.py @@ -33,65 +33,208 @@ import wiredtiger, wttest from helper import simple_populate, key_populate, value_populate from wtscenario import check_scenarios -# Tests for invisible updates. +# Test search/search-near operations, including invisible values and keys +# past the end of the table. class test_bug008(wttest.WiredTigerTestCase): + uri = 'file:test_bug008' # This is a btree layer test. scenarios = check_scenarios([ - ('fix', dict(fmt='key_format=r,value_format=8t', empty=1)), - ('row', dict(fmt='key_format=S', empty=0)), - ('var', dict(fmt='key_format=r', empty=0)) + ('fix', dict(fmt='key_format=r,value_format=8t', empty=1, colvar=0)), + ('row', dict(fmt='key_format=S', empty=0, colvar=0)), + ('var', dict(fmt='key_format=r', empty=0, colvar=1)) ]) + # Verify cursor search and search-near operations in an empty table. + def test_search_empty(self): + # Create the object and open a cursor. + self.session.create(self.uri, self.fmt) + cursor = self.session.open_cursor(self.uri, None) + + # Search for a record past the end of the table, which should fail. + cursor.set_key(key_populate(cursor, 100)) + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + + # Search-near for a record past the end of the table, which should fail. + cursor.set_key(key_populate(cursor, 100)) + self.assertEqual(cursor.search_near(), wiredtiger.WT_NOTFOUND) + + # Verify cursor search and search-near operations at and past the end of + # a file, with a set of on-page visible records. + def test_search_eot(self): + # Populate the tree and reopen the connection, forcing it to disk + # and moving the records to an on-page format. + simple_populate(self, self.uri, self.fmt, 100) + self.reopen_conn() + + # Open a cursor. + cursor = self.session.open_cursor(self.uri, None) + + # Search for a record at the end of the table, which should succeed. + cursor.set_key(key_populate(cursor, 100)) + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) + self.assertEqual(cursor.get_value(), value_populate(cursor, 100)) + + # Search-near for a record at the end of the table, which should + # succeed, returning the last record. + cursor.set_key(key_populate(cursor, 100)) + self.assertEqual(cursor.search_near(), 0) + self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) + self.assertEqual(cursor.get_value(), value_populate(cursor, 100)) + + # Search for a record past the end of the table, which should fail. + cursor.set_key(key_populate(cursor, 200)) + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + + # Search-near for a record past the end of the table, which should + # succeed, returning the last record. + cursor.set_key(key_populate(cursor, 200)) + self.assertEqual(cursor.search_near(), -1) + self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) + self.assertEqual(cursor.get_value(), value_populate(cursor, 100)) + + # Verify cursor search-near operations before and after a set of + # column-store duplicates. + def test_search_duplicate(self): + if self.colvar == 0: + return + + # Populate the tree. + simple_populate(self, self.uri, self.fmt, 105) + + # Set up deleted records before and after a set of duplicate records, + # and make sure search/search-near returns the correct record. + cursor = self.session.open_cursor(self.uri, None) + for i in range(20, 100): + cursor[key_populate(cursor, i)] = '=== IDENTICAL VALUE ===' + for i in range(15, 25): + cursor.set_key(key_populate(cursor, i)) + self.assertEqual(cursor.remove(), 0) + for i in range(95, 106): + cursor.set_key(key_populate(cursor, i)) + self.assertEqual(cursor.remove(), 0) + cursor.close() + + # Reopen the connection, forcing it to disk and moving the records to + # an on-page format. + self.reopen_conn() + + # Open a cursor. + cursor = self.session.open_cursor(self.uri, None) + + # Search-near for a record in the deleted set before the duplicate set, + # which should succeed, returning the first record in the duplicate set. + cursor.set_key(key_populate(cursor, 18)) + self.assertEqual(cursor.search_near(), 1) + self.assertEqual(cursor.get_key(), key_populate(cursor, 25)) + + # Search-near for a record in the deleted set after the duplicate set, + # which should succeed, returning the last record in the duplicate set. + cursor.set_key(key_populate(cursor, 98)) + self.assertEqual(cursor.search_near(), -1) + self.assertEqual(cursor.get_key(), key_populate(cursor, 94)) + # Verify cursor search and search-near operations on a file with a set of # on-page visible records, and a set of insert-list invisible records. def test_search_invisible_one(self): - uri = 'file:test_bug008' # This is a btree layer test. + # Populate the tree. + simple_populate(self, self.uri, self.fmt, 100) - # Populate the tree and reopen the connection, forcing it to disk - # and moving the records to an on-page format. - simple_populate(self, uri, self.fmt, 100) + # Delete a range of records. + for i in range(5, 10): + cursor = self.session.open_cursor(self.uri, None) + cursor.set_key(key_populate(cursor, i)) + self.assertEqual(cursor.remove(), 0) + + # Reopen the connection, forcing it to disk and moving the records to + # an on-page format. self.reopen_conn() - # Begin a transaction, and add some additional records. + # Add updates to the existing records (in both the deleted an undeleted + # range), as well as some new records after the end. Put the updates in + # a separate transaction so they're invisible to another cursor. self.session.begin_transaction() - cursor = self.session.open_cursor(uri, None) + cursor = self.session.open_cursor(self.uri, None) + for i in range(5, 10): + cursor[key_populate(cursor, i)] = value_populate(cursor, i + 1000) + for i in range(30, 40): + cursor[key_populate(cursor, i)] = value_populate(cursor, i + 1000) for i in range(100, 140): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[key_populate(cursor, i)] = value_populate(cursor, i + 1000) # Open a separate session and cursor. s = self.conn.open_session() - cursor = s.open_cursor(uri, None) + cursor = s.open_cursor(self.uri, None) - # Search for an invisible record. - cursor.set_key(key_populate(cursor, 130)) - if self.empty: - # Invisible updates to fixed-length column-store objects are - # invisible to the reader, but the fact that they exist past - # the end of the initial records causes the instantiation of - # empty records: confirm successful return of an empty row. - cursor.search() - self.assertEqual(cursor.get_key(), 130) - self.assertEqual(cursor.get_value(), 0) - else: - # Otherwise, we should not find any matching records. - self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + # Search for an existing record in the deleted range, should not find + # it. + for i in range(5, 10): + cursor.set_key(key_populate(cursor, i)) + if self.empty: + # Fixed-length column-store rows always exist. + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_key(), i) + self.assertEqual(cursor.get_value(), 0) + else: + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) - # Search-near for an invisible record, which should succeed, returning - # the last visible record. - cursor.set_key(key_populate(cursor, 130)) - cursor.search_near() - if self.empty: - # Invisible updates to fixed-length column-store objects are - # invisible to the reader, but the fact that they exist past - # the end of the initial records causes the instantiation of - # empty records: confirm successful return of an empty row. - cursor.search() - self.assertEqual(cursor.get_key(), 130) - self.assertEqual(cursor.get_value(), 0) - else: - # Otherwise, we should find the closest record for which we can see - # the value. - self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) - self.assertEqual(cursor.get_value(), value_populate(cursor, 100)) + # Search for an existing record in the updated range, should see the + # original value. + for i in range(30, 40): + cursor.set_key(key_populate(cursor, i)) + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_key(), key_populate(cursor, i)) + + # Search for a added record, should not find it. + for i in range(120, 130): + cursor.set_key(key_populate(cursor, i)) + if self.empty: + # Invisible updates to fixed-length column-store objects are + # invisible to the reader, but the fact that they exist past + # the end of the initial records causes the instantiation of + # empty records: confirm successful return of an empty row. + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_key(), i) + self.assertEqual(cursor.get_value(), 0) + else: + # Otherwise, we should not find any matching records. + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + + # Search-near for an existing record in the deleted range, should find + # the next largest record. (This depends on the implementation behavior + # which currently includes a bias to prefix search.) + for i in range(5, 10): + cursor.set_key(key_populate(cursor, i)) + if self.empty: + # Fixed-length column-store rows always exist. + self.assertEqual(cursor.search_near(), 0) + self.assertEqual(cursor.get_key(), i) + self.assertEqual(cursor.get_value(), 0) + else: + self.assertEqual(cursor.search_near(), 1) + self.assertEqual(cursor.get_key(), key_populate(cursor, 10)) + + # Search-near for an existing record in the updated range, should see + # the original value. + for i in range(30, 40): + cursor.set_key(key_populate(cursor, i)) + self.assertEqual(cursor.search_near(), 0) + self.assertEqual(cursor.get_key(), key_populate(cursor, i)) + + # Search-near for an added record, should find the previous largest + # record. + for i in range(120, 130): + cursor.set_key(key_populate(cursor, i)) + if self.empty: + # Invisible updates to fixed-length column-store objects are + # invisible to the reader, but the fact that they exist past + # the end of the initial records causes the instantiation of + # empty records: confirm successful return of an empty row. + self.assertEqual(cursor.search_near(), 0) + self.assertEqual(cursor.get_key(), i) + self.assertEqual(cursor.get_value(), 0) + else: + self.assertEqual(cursor.search_near(), -1) + self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) # Verify cursor search and search-near operations on a file with a set of # on-page visible records, a set of insert-list visible records, and a set @@ -101,28 +244,26 @@ class test_bug008(wttest.WiredTigerTestCase): # fallback happens, whether the correct position is in the page slots or # the insert list.) def test_search_invisible_two(self): - uri = 'file:test_bug008' # This is a btree layer test. - # Populate the tree and reopen the connection, forcing it to disk # and moving the records to an on-page format. - simple_populate(self, uri, self.fmt, 100) + simple_populate(self, self.uri, self.fmt, 100) self.reopen_conn() # Add some additional visible records. - cursor = self.session.open_cursor(uri, None) + cursor = self.session.open_cursor(self.uri, None) for i in range(100, 120): cursor[key_populate(cursor, i)] = value_populate(cursor, i) cursor.close() # Begin a transaction, and add some additional records. self.session.begin_transaction() - cursor = self.session.open_cursor(uri, None) + cursor = self.session.open_cursor(self.uri, None) for i in range(120, 140): cursor[key_populate(cursor, i)] = value_populate(cursor, i) # Open a separate session and cursor. s = self.conn.open_session() - cursor = s.open_cursor(uri, None) + cursor = s.open_cursor(self.uri, None) # Search for an invisible record. cursor.set_key(key_populate(cursor, 130)) diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint01.py b/src/third_party/wiredtiger/test/suite/test_checkpoint01.py index 9955944f73d..6e1ad7814ed 100644 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint01.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint01.py @@ -185,7 +185,7 @@ class test_checkpoint_cursor(wttest.WiredTigerTestCase): # Check dropping all checkpoints fails. msg = '/checkpoints cannot be dropped/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.checkpoint("name=checkpoint-2"), msg) + lambda: self.session.checkpoint("force,name=checkpoint-2"), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.checkpoint("drop=(checkpoint-2)"), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, diff --git a/src/third_party/wiredtiger/test/suite/test_collator.py b/src/third_party/wiredtiger/test/suite/test_collator.py new file mode 100644 index 00000000000..34b5c20247f --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_collator.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os +import wiredtiger, wttest, run +from wtscenario import check_scenarios, number_scenarios + +# test_collator.py +# Test indices using a custom extractor and collator. +class test_collator(wttest.WiredTigerTestCase): + """ + Test indices with a custom extractor to create an index, + with our own collator. + Our set of rows looks like a multiplication table: + row '0': '0,0,0,0' + row '1': '0,1,2,3' + row '2': '0,2,4,6' + with the twist that entries are mod 100. So, looking further: + row '40': '0,40,80,20' + + Each column is placed into its own index. Our collator reverses + the values. + """ + nentries = 100 + nindices = 4 + + # Return the wiredtiger_open extension argument for a shared library. + def extensionArg(self, exts): + extfiles = [] + for ext in exts: + (dirname, name, libname) = ext + if name != None and name != 'none': + testdir = os.path.dirname(__file__) + extdir = os.path.join(run.wt_builddir, 'ext', dirname) + extfile = os.path.join( + extdir, name, '.libs', 'libwiredtiger_' + libname + '.so') + if not os.path.exists(extfile): + self.skipTest('extension "' + extfile + '" not built') + if not extfile in extfiles: + extfiles.append(extfile) + if len(extfiles) == 0: + return '' + else: + return ',extensions=["' + '","'.join(extfiles) + '"]' + + # Override WiredTigerTestCase, we have extensions. + def setUpConnectionOpen(self, dir): + extarg = self.extensionArg([('extractors', 'csv', 'csv_extractor'), + ('collators', 'revint', 'revint_collator')]) + connarg = 'create,error_prefix="{0}: ",{1}'.format( + self.shortid(), extarg) + conn = self.wiredtiger_open(dir, connarg) + self.pr(`conn`) + return conn + + def create_indices(self): + # Create self.nindices index files, each with a column from the CSV + for i in range(0, self.nindices): + si = str(i) + self.session.create('index:collator:x' + si, + 'key_format=i,columns=(key),' + + 'collator=revint,' + + 'extractor=csv,app_metadata={"format" : "i",' + + '"field" : "' + si + '"}') + + def drop_indices(self): + for i in range(0, self.nindices): + self.session.drop("index:collator:x" + str(i)) + + def csv(self, s, i): + return s.split(',')[i] + + def expected_main_value(self, i): + return ','.join([str((i*j)%100) for j in range(0, self.nindices)]) + + # We split the population into two phases + # (in anticipation of future tests that create + # indices between the two population steps). + def populate(self): + cursor = self.session.open_cursor('table:collator', None, None) + for i in range(0, self.nentries): + cursor[i] = self.expected_main_value(i) + cursor.close() + + def check_entries(self): + cursor = self.session.open_cursor('table:collator', None, None) + icursor = [] + for i in range(0, self.nindices): + icursor.append(self.session.open_cursor('index:collator:x' + str(i), + None, None)) + i = 0 + for primkey, value in cursor: + # Check main table + expect = self.expected_main_value(i) + self.assertEqual(i, primkey) + self.assertEqual(value, expect) + for idx in range(0, self.nindices): + c = icursor[idx] + indexkey = (i*idx)%100 + c.set_key(indexkey) + self.assertEqual(c.search(), 0) + value = c.get_value() + key = c.get_key() + while value != expect and key == indexkey and \ + self.csv(value, idx) == self.csv(expect, idx): + self.assertEqual(0, c.next()) + value = c.get_value() + key = c.get_key() + self.assertEqual(value, expect) + i += 1 + self.assertEqual(self.nentries, i) + for i in range(0, self.nindices): + c = icursor[i] + c.reset() + expected = set(range(0, self.nentries)) + for key, val in c: + primkey = int(val.split(',')[1]) + expected.remove(primkey) + self.assertEquals(0, len(expected)) + c.close() + + def test_index(self): + self.session.create("table:collator", "key_format=i,value_format=S," + "columns=(primarykey,value)") + self.create_indices() + self.populate() + self.check_entries() + + # Drop and recreate all indices, everything should be there. + self.drop_indices() + self.create_indices() + self.check_entries() + + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_drop.py b/src/third_party/wiredtiger/test/suite/test_drop.py index 5663b85d661..52ea7251ab5 100644 --- a/src/third_party/wiredtiger/test/suite/test_drop.py +++ b/src/third_party/wiredtiger/test/suite/test_drop.py @@ -41,12 +41,11 @@ class test_drop(wttest.WiredTigerTestCase): scenarios = check_scenarios([ ('file', dict(uri='file:')), ('table', dict(uri='table:')), - #Not yet: drop failing with an open cursor needs handle locking - #('table-lsm', dict(uri='table:', extra_config=',type=lsm')), + ('table-lsm', dict(uri='table:', extra_config=',type=lsm')), ]) # Populate an object, remove it and confirm it no longer exists. - def drop(self, populate, with_cursor, close_session, drop_index): + def drop(self, populate, with_cursor, reopen, drop_index): uri = self.uri + self.name populate(self, uri, 'key_format=S' + self.extra_config, 10) @@ -57,7 +56,7 @@ class test_drop(wttest.WiredTigerTestCase): lambda: self.session.drop(uri, None)) cursor.close() - if close_session: + if reopen: self.reopen_conn() if drop_index: @@ -73,17 +72,17 @@ class test_drop(wttest.WiredTigerTestCase): # Try all combinations except dropping the index, the simple # case has no indices. for with_cursor in [False, True]: - for close_session in [False, True]: - self.drop(simple_populate, with_cursor, close_session, False) + for reopen in [False, True]: + self.drop(simple_populate, with_cursor, reopen, False) # A complex, multi-file table object. # Try all test combinations. if self.uri == "table:": for with_cursor in [False, True]: - for close_session in [False, True]: + for reopen in [False, True]: for drop_index in [False, True]: self.drop(complex_populate, with_cursor, - close_session, drop_index) + reopen, drop_index) # Test drop of a non-existent object: force succeeds, without force fails. def test_drop_dne(self): diff --git a/src/third_party/wiredtiger/test/suite/test_drop02.py b/src/third_party/wiredtiger/test/suite/test_drop02.py new file mode 100644 index 00000000000..677ba3866b2 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_drop02.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from helper import simple_populate + +# test_drop02.py +# Test dropping an LSM tree on first open. There was a bug where this +# would cause an assertion failure: WT-2501 +class test_drop02(wttest.WiredTigerTestCase): + name = 'test_drop02' + + # Populate an object, remove it and confirm it no longer exists. + def test_drop(self): + uri = 'lsm:' + self.name + simple_populate(self, uri, 'key_format=S', 100000) + self.reopen_conn() + + self.session.drop(uri, None) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_dump.py b/src/third_party/wiredtiger/test/suite/test_dump.py index c850d1b5d3f..fc1422155e2 100644 --- a/src/third_party/wiredtiger/test/suite/test_dump.py +++ b/src/third_party/wiredtiger/test/suite/test_dump.py @@ -29,8 +29,8 @@ import os import wiredtiger, wttest from helper import \ - complex_populate, complex_populate_check_cursor,\ - simple_populate, simple_populate_check_cursor + complex_populate, complex_populate_check, \ + simple_populate, simple_populate_check from suite_subprocess import suite_subprocess from wtscenario import multiply_scenarios, number_scenarios @@ -54,15 +54,24 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): ('string', dict(keyfmt='S')) ] types = [ - ('file', dict(type='file:', + ('file', dict(uri='file:', config='', lsm=False, populate=simple_populate, - populate_check=simple_populate_check_cursor)), - ('table-simple', dict(type='table:', + populate_check=simple_populate_check)), + ('lsm', dict(uri='lsm:', config='', lsm=True, populate=simple_populate, - populate_check=simple_populate_check_cursor)), - ('table-complex', dict(type='table:', + populate_check=simple_populate_check)), + ('table-simple', dict(uri='table:', config='', lsm=False, + populate=simple_populate, + populate_check=simple_populate_check)), + ('table-simple-lsm', dict(uri='table:', config='type=lsm', lsm=True, + populate=simple_populate, + populate_check=simple_populate_check)), + ('table-complex', dict(uri='table:', config='', lsm=False, + populate=complex_populate, + populate_check=complex_populate_check)), + ('table-complex-lsm', dict(uri='table:', config='type=lsm', lsm=True, populate=complex_populate, - populate_check=complex_populate_check_cursor)) + populate_check=complex_populate_check)) ] scenarios = number_scenarios( multiply_scenarios('.', types, keyfmt, dumpfmt)) @@ -94,9 +103,14 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): # Dump, re-load and do a content comparison. def test_dump(self): + # LSM and column-store isn't a valid combination. + if self.lsm and self.keyfmt == 'r': + return + # Create the object. - uri = self.type + self.name - self.populate(self, uri, 'key_format=' + self.keyfmt, self.nentries) + uri = self.uri + self.name + self.populate(self, uri, + self.config + ',key_format=' + self.keyfmt, self.nentries) # Dump the object. os.mkdir(self.dir) @@ -108,11 +122,17 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): # Re-load the object. self.runWt(['-h', self.dir, 'load', '-f', 'dump.out']) - # Check the contents + # Check the database contents + self.runWt(['list'], outfilename='list.out') + self.runWt(['-h', self.dir, 'list'], outfilename='list.out.new') + s1 = set(open('list.out').read().split()) + s2 = set(open('list.out.new').read().split()) + self.assertEqual(not s1.symmetric_difference(s2), True) + + # Check the object's contents conn = self.wiredtiger_open(self.dir) session = conn.open_session() - cursor = session.open_cursor(uri, None, None) - self.populate_check(self, cursor, self.nentries) + self.populate_check(self, uri, self.nentries) conn.close() # Re-load the object again. @@ -121,8 +141,7 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): # Check the contents, they shouldn't have changed. conn = self.wiredtiger_open(self.dir) session = conn.open_session() - cursor = session.open_cursor(uri, None, None) - self.populate_check(self, cursor, self.nentries) + self.populate_check(self, uri, self.nentries) conn.close() # Re-load the object again, but confirm -n (no overwrite) fails. @@ -130,7 +149,7 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): 'load', '-n', '-f', 'dump.out'], errfilename='errfile.out') self.check_non_empty_file('errfile.out') - # If there is are indices, dump one of them and check the output. + # If there are indices, dump one of them and check the output. if self.populate == complex_populate: indexuri = 'index:' + self.name + ':indx1' hexopt = ['-x'] if self.hex == 1 else [] diff --git a/src/third_party/wiredtiger/test/suite/test_join01.py b/src/third_party/wiredtiger/test/suite/test_join01.py index 539a3a3ae57..4aa2bc6e269 100644 --- a/src/third_party/wiredtiger/test/suite/test_join01.py +++ b/src/third_party/wiredtiger/test/suite/test_join01.py @@ -74,8 +74,18 @@ class test_join01(wttest.WiredTigerTestCase): # the join cursor and iterating again. def stats(self, jc, which): statcur = self.session.open_cursor('statistics:join', jc, None) - self.check_stats(statcur, 0, 'join: index:join01:index1: ' + - 'bloom filter false positives') + # pick a stat we always expect to see + statdesc = 'bloom filter false positives' + expectstats = [ + 'join: index:join01:index1: ' + statdesc, + 'join: index:join01:index2: ' + statdesc ] + if self.ref == 'index': + expectstats.append('join: index:join01:index0: ' + statdesc) + else: + expectstats.append('join: table:join01: ' + statdesc) + self.check_stats(statcur, expectstats) + statcur.reset() + self.check_stats(statcur, expectstats) statcur.close() def statstr_to_int(self, str): @@ -86,16 +96,14 @@ class test_join01(wttest.WiredTigerTestCase): parts = str.rpartition('(') return int(parts[2].rstrip(')')) - # string should appear with a minimum value of least "min". - def check_stats(self, statcursor, min, lookfor): + # All of the expect strings should appear + def check_stats(self, statcursor, expectstats): stringclass = ''.__class__ intclass = (0).__class__ # Reset the cursor, we're called multiple times. statcursor.reset() - found = False - foundval = 0 self.printVerbose(3, 'statistics:') for id, desc, valstr, val in statcursor: self.assertEqual(type(desc), stringclass) @@ -104,12 +112,11 @@ class test_join01(wttest.WiredTigerTestCase): self.assertEqual(val, self.statstr_to_int(valstr)) self.printVerbose(3, ' stat: \'' + desc + '\', \'' + valstr + '\', ' + str(val)) - if desc == lookfor: - found = True - foundval = val + if desc in expectstats: + expectstats.remove(desc) - self.assertTrue(found, 'in stats, did not see: ' + lookfor) - self.assertTrue(foundval >= min) + self.assertTrue(len(expectstats) == 0, + 'missing expected values in stats: ' + str(expectstats)) # Common function for testing the most basic functionality # of joins @@ -141,7 +148,8 @@ class test_join01(wttest.WiredTigerTestCase): # and examine primary keys 2,5,8,...,95,98,1,4,7,...,94,97. jc = self.session.open_cursor('join:table:join01' + proj_suffix, None, None) - c2 = self.session.open_cursor('index:join01:index2', None, None) + # Adding a projection to a reference cursor should be allowed. + c2 = self.session.open_cursor('index:join01:index2(v1)', None, None) c2.set_key(99) # skips all entries w/ primary key divisible by three self.assertEquals(0, c2.search()) self.session.join(jc, c2, 'compare=gt') @@ -159,12 +167,12 @@ class test_join01(wttest.WiredTigerTestCase): # Then select all numbers whose reverse string representation # is in '20' < x < '40'. - c1a = self.session.open_cursor('index:join01:index1', None, None) + c1a = self.session.open_cursor('index:join01:index1(v1)', None, None) c1a.set_key('21') self.assertEquals(0, c1a.search()) self.session.join(jc, c1a, 'compare=gt' + joincfg1) - c1b = self.session.open_cursor('index:join01:index1', None, None) + c1b = self.session.open_cursor('index:join01:index1(v1)', None, None) c1b.set_key('41') self.assertEquals(0, c1b.search()) self.session.join(jc, c1b, 'compare=lt' + joincfg1) diff --git a/src/third_party/wiredtiger/test/suite/test_join02.py b/src/third_party/wiredtiger/test/suite/test_join02.py index d122de8a0eb..a691c499cf6 100644 --- a/src/third_party/wiredtiger/test/suite/test_join02.py +++ b/src/third_party/wiredtiger/test/suite/test_join02.py @@ -179,15 +179,16 @@ class test_join02(wttest.WiredTigerTestCase): c.close() # Use the primary table in one of the joins. + # Use various projections, which should not matter for ref cursors c0a = self.session.open_cursor('table:join02', None, None) - c0b = self.session.open_cursor('table:join02', None, None) - c1a = self.session.open_cursor('index:join02:index1', None, None) + c0b = self.session.open_cursor('table:join02(v4)', None, None) + c1a = self.session.open_cursor('index:join02:index1(v0)', None, None) c1b = self.session.open_cursor('index:join02:index1', None, None) c2a = self.session.open_cursor('index:join02:index2', None, None) c2b = self.session.open_cursor('index:join02:index2', None, None) - c3a = self.session.open_cursor('index:join02:index3', None, None) - c3b = self.session.open_cursor('index:join02:index3', None, None) - c4a = self.session.open_cursor('index:join02:index4', None, None) + c3a = self.session.open_cursor('index:join02:index3(v4)', None, None) + c3b = self.session.open_cursor('index:join02:index3(v0)', None, None) + c4a = self.session.open_cursor('index:join02:index4(v1)', None, None) # Attach extra properties to each cursor. For cursors that # may appear on the 'left' side of a range CA < x < CB, diff --git a/src/third_party/wiredtiger/test/suite/test_join05.py b/src/third_party/wiredtiger/test/suite/test_join05.py new file mode 100644 index 00000000000..ef2be4c6460 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_join05.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import check_scenarios, multiply_scenarios, number_scenarios + +# test_join05.py +# Tests based on JIRA reports +class test_join05(wttest.WiredTigerTestCase): + + # test join having the first index just be lt/le + def test_wt_2384(self): + self.session.create("table:test_2384", + "key_format=i,value_format=i,columns=(k,v)") + self.session.create("index:test_2384:index", "columns=(v)") + cursor = self.session.open_cursor("table:test_2384", None, None) + cursor[1] = 11 + cursor[2] = 12 + cursor[3] = 13 + cursor.close() + + cursor = self.session.open_cursor("index:test_2384:index", None, None) + cursor.set_key(13) + self.assertEquals(cursor.search(), 0) + + jcursor = self.session.open_cursor("join:table:test_2384", None, None) + self.session.join(jcursor, cursor, "compare=lt") + + nr_found = 0 + while jcursor.next() == 0: + [k] = jcursor.get_keys() + [v] = jcursor.get_values() + #self.tty("jcursor: k=" + str(k) + ", v=" + str(v)) + nr_found += 1 + + self.assertEquals(nr_found, 2) + jcursor.close() + cursor.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_join06.py b/src/third_party/wiredtiger/test/suite/test_join06.py new file mode 100644 index 00000000000..9af6f93792f --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_join06.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os +import wiredtiger, wttest, run +from wtscenario import check_scenarios, multiply_scenarios, number_scenarios + +# test_join06.py +# Join operations +# Joins with a read-uncommitted +class test_join06(wttest.WiredTigerTestCase): + nentries = 1000 + + isoscen = [ + ('isolation_read_uncommitted', dict(uncommitted=True)), + ('isolation_default', dict(uncommitted=False)) + ] + + bloomscen = [ + ('bloom', dict(bloom=True)), + ('nobloom', dict(bloom=False)) + ] + + scenarios = number_scenarios(multiply_scenarios('.', isoscen, bloomscen)) + + def gen_values(self, i): + s = str(i) # 345 => "345" + f = s[0:1] + s[0:1] + s[0:1] # 345 => "333" + return [s, f] + + def gen_values2(self, i): + s = str(i) # 345 => "345" + l = s[-1:] + s[-1:] + s[-1:] # 345 => "555" + return [s, l] + + def populate(self, s, gen_values): + c = s.open_cursor('table:join06', None, None) + for i in range(0, self.nentries): + c.set_key(i) + c.set_value(*gen_values(i)) + c.insert() + c.close() + + # Common function for testing the most basic functionality + # of joins + def test_join(self): + self.session.create('table:join06', + 'columns=(k,v0,v1),key_format=i,value_format=SS') + self.session.create('index:join06:index0','columns=(v0)') + self.session.create('index:join06:index1','columns=(v1)') + + self.populate(self.session, self.gen_values) + + # TODO: needed? + #self.reopen_conn() + + if self.uncommitted: + self.session.begin_transaction('isolation=read-uncommitted') + + jc = self.session.open_cursor('join:table:join06', None, None) + c0 = self.session.open_cursor('index:join06:index0', None, None) + c0.set_key('520') + self.assertEquals(0, c0.search()) + self.session.join(jc, c0, 'compare=ge') + + joinconfig = 'compare=eq' + if self.bloom: + joinconfig += ',strategy=bloom,count=1000' + c1 = self.session.open_cursor('index:join06:index1', None, None) + c1.set_key('555') + self.assertEquals(0, c1.search()) + self.session.join(jc, c1, joinconfig) + + if self.uncommitted and self.bloom: + # Make sure that read-uncommitted with Bloom is not allowed. + # This is detected on the first next() operation. + msg = '/cannot be used with read-uncommitted/' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: jc.next(), msg) + return + + # Changes made in another session may or may not be visible to us, + # depending on the isolation level. + if self.uncommitted: + # isolation level is read-uncommitted, so we will see + # additions deletions made in our other session. + mbr = set(range(525,1000,10)) | set(range(55,100,10)) | set([520]) + else: + # default isolation level, so we should see a consistent + # set at the time we begin iteration. + mbr = set(range(520,600)) | set(range(53,60)) + + altered = False + + while jc.next() == 0: + [k] = jc.get_keys() + [v0,v1] = jc.get_values() + #self.tty('GOT: ' + str(k) + ': ' + str(jc.get_values())) + if altered and self.uncommitted: + self.assertEquals(self.gen_values2(k), [v0, v1]) + else: + self.assertEquals(self.gen_values(k), [v0, v1]) + if not k in mbr: + self.tty('**** ERROR: result ' + str(k) + ' is not in: ' + + str(mbr)) + self.assertTrue(k in mbr) + mbr.remove(k) + + # In another session, we remove entries for keys ending in 6, + # and add entries for keys ending in 5. Depending on the + # isolation level for the transaction, these changes may or + # may not be visible for the original session. + if not altered: + s = self.conn.open_session(None) + s.begin_transaction(None) + self.populate(s, self.gen_values2) + s.commit_transaction() + s.close() + altered = True + + if len(mbr) != 0: + self.tty('**** ERROR: did not see these: ' + str(mbr)) + self.assertEquals(0, len(mbr)) + + jc.close() + c1.close() + c0.close() + if self.uncommitted: + self.session.commit_transaction() + self.session.drop('table:join06') + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_lsm03.py b/src/third_party/wiredtiger/test/suite/test_lsm03.py new file mode 100644 index 00000000000..448d864c646 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_lsm03.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wtscenario, wttest +from helper import simple_populate + +# test_lsm03.py +# Check to make sure that LSM schema operations don't get EBUSY when +# there are no user operations active. +class test_lsm03(wttest.WiredTigerTestCase): + name = 'test_lsm03' + + # Use small pages so we generate some internal layout + # Setup LSM so multiple chunks are present + config = 'key_format=S,allocation_size=512,internal_page_max=512' + \ + ',leaf_page_max=1k,lsm=(chunk_size=512k,merge_min=10)' + + # Populate an object then drop it. + def test_lsm_drop_active(self): + uri = 'lsm:' + self.name + simple_populate(self, uri, self.config, 10000) + + # Force to disk + self.reopen_conn() + + # An open cursors should cause failure. + cursor = self.session.open_cursor(uri, None, None) + self.assertRaises(wiredtiger.WiredTigerError, + lambda: self.session.drop(uri, None)) + cursor.close() + + # Add enough records that a merge should be running + simple_populate(self, uri, self.config, 50000) + # The drop should succeed even when LSM work units are active + self.session.drop(uri) diff --git a/src/third_party/wiredtiger/test/suite/test_rebalance.py b/src/third_party/wiredtiger/test/suite/test_rebalance.py index 80cce6ed514..f2167e864c9 100644 --- a/src/third_party/wiredtiger/test/suite/test_rebalance.py +++ b/src/third_party/wiredtiger/test/suite/test_rebalance.py @@ -59,7 +59,7 @@ class test_rebalance(wttest.WiredTigerTestCase): if with_cursor: cursor = self.session.open_cursor(uri, None, None) self.assertRaises(wiredtiger.WiredTigerError, - lambda: self.session.drop(uri, None)) + lambda: self.session.rebalance(uri, None)) cursor.close() self.session.rebalance(uri, None) diff --git a/src/third_party/wiredtiger/test/suite/test_schema07.py b/src/third_party/wiredtiger/test/suite/test_schema07.py new file mode 100644 index 00000000000..ac397c6e1a1 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_schema07.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest + +# test_schema07.py +# Test that long-running tests don't fill the cache with metadata +class test_schema07(wttest.WiredTigerTestCase): + tablename = 'table:test_schema07' + + def conn_config(self, dir): + return 'cache_size=10MB' + + @wttest.longtest("Creating many tables shouldn't fill the cache") + def test_many_tables(self): + s = self.session + # We have a 10MB cache, metadata is (well) over 512B per table, + # if we can create 20K tables, something must be cleaning up. + for i in xrange(20000): + uri = '%s-%06d' % (self.tablename, i) + s.create(uri) + c = s.open_cursor(uri) + # This will block if the metadata fills the cache + c["key"] = "value" + c.close() + self.session.drop(uri) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_sweep01.py b/src/third_party/wiredtiger/test/suite/test_sweep01.py index f996dbfa06d..bccd2bce012 100644 --- a/src/third_party/wiredtiger/test/suite/test_sweep01.py +++ b/src/third_party/wiredtiger/test/suite/test_sweep01.py @@ -40,7 +40,7 @@ import wttest class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): tablebase = 'test_sweep01' uri = 'table:' + tablebase - numfiles = 50 + numfiles = 30 numkv = 1000 conn_config = 'file_manager=(close_handle_minimum=0,' + \ 'close_idle_time=6,close_scan_interval=2),' + \ @@ -87,7 +87,7 @@ class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): # # We've configured checkpoints to run every 5 seconds, sweep server to # run every 2 seconds and idle time to be 6 seconds. It should take - # about 8 seconds for a handle to be closed. Sleep for 12 seconds to be + # about 8 seconds for a handle to be closed. Sleep for double to be # safe. # uri = '%s.test' % self.uri @@ -105,13 +105,24 @@ class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri, None) k = 0 sleep = 0 - while sleep < 12: + max = 60 + final_nfile = 4 + while sleep < max: self.session.checkpoint() k = k+1 c[k] = 1 sleep += 2 time.sleep(2) + # Give slow machines time to process files. + stat_cursor = self.session.open_cursor('statistics:', None, None) + this_nfile = stat_cursor[stat.conn.file_open][2] + stat_cursor.close() + self.pr("==== loop " + str(sleep)) + self.pr("this_nfile " + str(this_nfile)) + if this_nfile == final_nfile: + break c.close() + self.pr("Sweep loop took " + str(sleep)) stat_cursor = self.session.open_cursor('statistics:', None, None) close2 = stat_cursor[stat.conn.dh_sweep_close][2] @@ -177,7 +188,7 @@ class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): self.assertEqual(nfile2 < nfile1, True) # The only files that should be left are the metadata, the lookaside # file, the lock file, and the active file. - if (nfile2 != 4): + if (nfile2 != final_nfile): print "close1: " + str(close1) + " close2: " + str(close2) print "remove1: " + str(remove1) + " remove2: " + str(remove2) print "sweep1: " + str(sweep1) + " sweep2: " + str(sweep2) @@ -186,7 +197,7 @@ class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): print "tod1: " + str(tod1) + " tod2: " + str(tod2) print "ref1: " + str(ref1) + " ref2: " + str(ref2) print "XX2: nfile1: " + str(nfile1) + " nfile2: " + str(nfile2) - self.assertEqual(nfile2 == 4, True) + self.assertEqual(nfile2 == final_nfile, True) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_util13.py b/src/third_party/wiredtiger/test/suite/test_util13.py new file mode 100644 index 00000000000..222f42cd7f1 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_util13.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os, re, string +from suite_subprocess import suite_subprocess +import itertools, wiredtiger, wttest + +from helper import complex_populate_cgconfig, complex_populate_cgconfig_lsm +from helper import simple_populate +from helper import complex_populate_check, simple_populate_check +from wtscenario import multiply_scenarios, number_scenarios + +# test_util13.py +# Utilities: wt dump, as well as the dump cursor +# Test that dump and load retain table configuration information. +# +class test_util13(wttest.WiredTigerTestCase, suite_subprocess): + """ + Test wt dump. We check for specific output and preservation of + non-default table create parameters. + """ + + pfx = 'test_util13' + nentries = 100 + dir = "dump_dir" + # + # Select table configuration settings that are not the default. + # + types = [ + ('file-simple', dict(uri='file:' + pfx, pop=simple_populate, + populate_check=simple_populate_check, + table_config='prefix_compression_min=3', cfg='')), + ('lsm-simple', dict(uri='lsm:' + pfx, pop=simple_populate, + populate_check=simple_populate_check, + table_config='lsm=(bloom_bit_count=29)', + cfg='bloom_bit_count=29')), + ('table-simple', dict(uri='table:' + pfx, pop=simple_populate, + populate_check=simple_populate_check, + table_config='split_pct=50', cfg='')), + ('table-complex', + dict(uri='table:' + pfx, pop=complex_populate_cgconfig, + populate_check=complex_populate_check, + table_config='allocation_size=512B', cfg='')), + ('table-complex-lsm', + dict(uri='table:' + pfx, pop=complex_populate_cgconfig_lsm, + populate_check=complex_populate_check, + table_config='lsm=(merge_max=5)', + cfg='merge_max=5')), + ] + + scenarios = number_scenarios(multiply_scenarios('.', types)) + + def compare_config(self, expected_cfg, actual_cfg): + # Replace '(' characters so configuration groups don't break parsing. + # If we ever want to look for config groups this will need to change. + #print "compare_config Actual config " + #print actual_cfg + #print "compare_config Expected config " + #print expected_cfg + cfg_orig = actual_cfg + if self.pop != simple_populate: + # + # If we have a complex config, strip out the colgroups and + # columns from the config. Doing so allows us to keep the + # split commands below usable because those two items don't + # have assignments in them. + # + nocolgrp = re.sub("colgroups=\((.+?)\),", '', actual_cfg) + cfg_orig = re.sub("columns=\((.+?)\),", '', nocolgrp) + + #print "Using original config " + #print cfg_orig + da = dict(kv.split('=') for kv in + cfg_orig.strip().replace('(',',').split(',')) + dx = dict(kv.split('=') for kv in + expected_cfg.strip().replace('(',',').split(',')) + + # Check that all items in our expected config subset are in + # the actual configuration and they match. + match = all(item in da.items() for item in dx.items()) + if match == False: + print "MISMATCH:" + print "Original dict: " + print da + print "Expected config: " + print dx + return match + + def compare_files(self, expect_subset, dump_out): + inheader = isconfig = False + for l1, l2 in zip(open(expect_subset, "rb"), open(dump_out, "rb")): + if isconfig: + if not self.compare_config(l1, l2): + return False + if inheader: + # This works because the expected subset has a format + # of URI and config lines alternating. + isconfig = not isconfig + if l1.strip() == 'Header': + inheader = True + if l1.strip() == 'Data': + break + return True + + def load_recheck(self, expect_subset, dump_out): + newdump = "newdump.out" + os.mkdir(self.dir) + self.runWt(['-h', self.dir, 'load', '-f', dump_out]) + # Check the contents + conn = self.wiredtiger_open(self.dir) + session = conn.open_session() + cursor = session.open_cursor(self.uri, None, None) + self.populate_check + conn.close() + dumpargs = ["-h"] + dumpargs.append(self.dir) + dumpargs.append("dump") + dumpargs.append(self.uri) + self.runWt(dumpargs, outfilename=newdump) + + self.assertTrue(self.compare_files(expect_subset, newdump)) + return True + + def test_dump_config(self): + # The number of btree_entries reported is influenced by the + # number of column groups and indices. Each insert will have + # a multiplied effect. + self.pop(self, self.uri, + 'key_format=S,value_format=S,' + self.table_config, self.nentries) + + ver = wiredtiger.wiredtiger_version() + verstring = str(ver[1]) + '.' + str(ver[2]) + '.' + str(ver[3]) + expectfile="expect.out" + with open(expectfile, "w") as expectout: + # Note: this output is sensitive to the precise output format + # generated by wt dump. If this is likely to change, we should + # make this test more accommodating. + expectout.write( + 'WiredTiger Dump (WiredTiger Version ' + verstring + ')\n') + expectout.write('Format=print\n') + expectout.write('Header\n') + expectout.write(self.uri + '\n') + # Check the config on the colgroup itself for complex tables. + if self.pop != simple_populate: + expectout.write('key_format=S\n') + expectout.write('colgroup:' + self.pfx + ':cgroup1\n') + if self.cfg == '': + expectout.write(self.table_config + '\n') + else: + expectout.write(self.cfg + '\n') + expectout.write('Data\n') + + self.pr('calling dump') + outfile="dump.out" + dumpargs = ["dump"] + dumpargs.append(self.uri) + self.runWt(dumpargs, outfilename=outfile) + + self.assertTrue(self.compare_files(expectfile, outfile)) + self.assertTrue(self.load_recheck(expectfile, outfile)) + +if __name__ == '__main__': + wttest.run() |