diff options
Diffstat (limited to 'src/third_party/wiredtiger/test')
21 files changed, 1727 insertions, 69 deletions
diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index 9bfba3cd0df..e4f7af2e1b2 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -65,6 +65,10 @@ static CONFIG c[] = { "if timed run should drop core", /* 0% */ C_BOOL, 0, 0, 0, &g.c_abort, NULL }, + { "alter", + "if altering the table is enabled", /* 10% */ + C_BOOL, 10, 0, 0, &g.c_alter, NULL }, + { "auto_throttle", "if LSM inserts are throttled", /* 90% */ C_BOOL, 90, 0, 0, &g.c_auto_throttle, NULL }, diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 820bc020c9b..c1f4875dbb2 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -140,6 +140,7 @@ typedef struct { char *config_open; /* Command-line configuration */ uint32_t c_abort; /* Config values */ + uint32_t c_alter; uint32_t c_auto_throttle; uint32_t c_backups; uint32_t c_bitcnt; @@ -242,7 +243,7 @@ typedef struct { } GLOBAL; extern GLOBAL g; -typedef struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) { +typedef struct { WT_RAND_STATE rnd; /* thread RNG state */ uint64_t search; /* operations */ @@ -276,6 +277,7 @@ void bdb_remove(uint64_t, int *); void bdb_update(const void *, size_t, const void *, size_t); #endif +void *alter(void *); void *backup(void *); void *compact(void *); void config_clear(void); diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 283e2912daa..940318c87a9 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -50,10 +50,10 @@ static void print_item(const char *, WT_ITEM *); void wts_ops(int lastrun) { - TINFO *tinfo, total; + TINFO **tinfo_list, *tinfo, total; WT_CONNECTION *conn; WT_SESSION *session; - pthread_t backup_tid, compact_tid, lrt_tid; + pthread_t alter_tid, backup_tid, compact_tid, lrt_tid; int64_t fourths, thread_ops; uint32_t i; int running; @@ -61,6 +61,7 @@ wts_ops(int lastrun) conn = g.wts_conn; session = NULL; /* -Wconditional-uninitialized */ + memset(&alter_tid, 0, sizeof(alter_tid)); memset(&backup_tid, 0, sizeof(backup_tid)); memset(&compact_tid, 0, sizeof(compact_tid)); memset(&lrt_tid, 0, sizeof(lrt_tid)); @@ -102,19 +103,24 @@ wts_ops(int lastrun) "=============== thread ops start ==============="); } - /* Create thread structure; start the worker threads. */ - tinfo = dcalloc((size_t)g.c_threads, sizeof(*tinfo)); + /* + * Create the per-thread structures and start the worker threads. + * Allocate the thread structures separately to minimize false sharing. + */ + tinfo_list = dcalloc((size_t)g.c_threads, sizeof(TINFO *)); for (i = 0; i < g.c_threads; ++i) { - tinfo[i].id = (int)i + 1; - tinfo[i].state = TINFO_RUNNING; - testutil_check( - pthread_create(&tinfo[i].tid, NULL, ops, &tinfo[i])); + tinfo_list[i] = tinfo = dcalloc(1, sizeof(TINFO)); + tinfo->id = (int)i + 1; + tinfo->state = TINFO_RUNNING; + testutil_check(pthread_create(&tinfo->tid, NULL, ops, tinfo)); } /* * If a multi-threaded run, start optional backup, compaction and * long-running reader threads. */ + if (g.c_alter) + testutil_check(pthread_create(&alter_tid, NULL, alter, NULL)); if (g.c_backups) testutil_check(pthread_create(&backup_tid, NULL, backup, NULL)); if (g.c_compact) @@ -128,21 +134,22 @@ wts_ops(int lastrun) /* Clear out the totals each pass. */ memset(&total, 0, sizeof(total)); for (i = 0, running = 0; i < g.c_threads; ++i) { - total.commit += tinfo[i].commit; - total.deadlock += tinfo[i].deadlock; - total.insert += tinfo[i].insert; - total.remove += tinfo[i].remove; - total.rollback += tinfo[i].rollback; - total.search += tinfo[i].search; - total.update += tinfo[i].update; - - switch (tinfo[i].state) { + tinfo = tinfo_list[i]; + total.commit += tinfo->commit; + total.deadlock += tinfo->deadlock; + total.insert += tinfo->insert; + total.remove += tinfo->remove; + total.rollback += tinfo->rollback; + total.search += tinfo->search; + total.update += tinfo->update; + + switch (tinfo->state) { case TINFO_RUNNING: running = 1; break; case TINFO_COMPLETE: - tinfo[i].state = TINFO_JOINED; - (void)pthread_join(tinfo[i].tid, NULL); + tinfo->state = TINFO_JOINED; + (void)pthread_join(tinfo->tid, NULL); break; case TINFO_JOINED: break; @@ -154,7 +161,7 @@ wts_ops(int lastrun) */ if (fourths == 0 || (thread_ops != -1 && - tinfo[i].ops >= (uint64_t)thread_ops)) { + tinfo->ops >= (uint64_t)thread_ops)) { /* * On the last execution, optionally drop core * for recovery testing. @@ -163,7 +170,7 @@ wts_ops(int lastrun) static char *core = NULL; *core = 0; } - tinfo[i].quit = 1; + tinfo->quit = 1; } } track("ops", 0ULL, &total); @@ -173,10 +180,14 @@ wts_ops(int lastrun) if (fourths != -1) --fourths; } - free(tinfo); + for (i = 0; i < g.c_threads; ++i) + free(tinfo_list[i]); + free(tinfo_list); /* Wait for the backup, compaction, long-running reader threads. */ g.workers_finished = 1; + if (g.c_alter) + (void)pthread_join(alter_tid, NULL); if (g.c_backups) (void)pthread_join(backup_tid, NULL); if (g.c_compact) @@ -455,7 +466,7 @@ ops(void *arg) * 10% of the time, perform some read-only operations * from a checkpoint. * - * Skip that if we single-threaded and doing checks + * Skip that if we are single-threaded and doing checks * against a Berkeley DB database, because that won't * work because the Berkeley DB database records won't * match the checkpoint. Also skip if we are using diff --git a/src/third_party/wiredtiger/test/format/util.c b/src/third_party/wiredtiger/test/format/util.c index a709aa93a2e..b9788f1ac75 100644 --- a/src/third_party/wiredtiger/test/format/util.c +++ b/src/third_party/wiredtiger/test/format/util.c @@ -459,3 +459,47 @@ fclose_and_clear(FILE **fpp) testutil_die(errno, "fclose"); return; } + +/* + * alter -- + * Periodically alter a table's metadata. + */ +void * +alter(void *arg) +{ + WT_CONNECTION *conn; + WT_SESSION *session; + u_int period; + bool access_value; + char buf[32]; + + (void)(arg); + conn = g.wts_conn; + + /* + * Only alter the access pattern hint. If we alter the cache resident + * setting we may end up with a setting that fills cache and doesn't + * allow it to be evicted. + */ + access_value = false; + + /* Open a session */ + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + while (!g.workers_finished) { + period = mmrand(NULL, 1, 10); + + snprintf(buf, sizeof(buf), + "access_pattern_hint=%s", access_value ? "random" : "none"); + access_value = !access_value; + if (session->alter(session, g.uri, buf) != 0) + break; + while (period > 0 && !g.workers_finished) { + --period; + sleep(1); + } + } + + testutil_check(session->close(session, NULL)); + return (NULL); +} diff --git a/src/third_party/wiredtiger/test/suite/test_alter01.py b/src/third_party/wiredtiger/test/suite/test_alter01.py new file mode 100644 index 00000000000..dfdf6b7a17e --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_alter01.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios + +# test_alter01.py +# Smoke-test the session alter operations. +class test_alter01(wttest.WiredTigerTestCase): + name = "alter01" + entries = 100 + # Settings for access_pattern_hint + types = [ + ('file', dict(uri='file:', use_cg=False, use_index=False)), + ('lsm', dict(uri='lsm:', use_cg=False, use_index=False)), + ('table-cg', dict(uri='table:', use_cg=True, use_index=False)), + ('table-index', dict(uri='table:', use_cg=False, use_index=True)), + ('table-simple', dict(uri='table:', use_cg=False, use_index=False)), + ] + hints = [ + ('default', dict(acreate='')), + ('none', dict(acreate='none')), + ('random', dict(acreate='random')), + ('sequential', dict(acreate='sequential')), + ] + access_alter=('', 'none', 'random', 'sequential') + # Settings for cache_resident + resid = [ + ('default', dict(ccreate='')), + ('false', dict(ccreate='false')), + ('true', dict(ccreate='true')), + ] + reopen = [ + ('no-reopen', dict(reopen=False)), + ('reopen', dict(reopen=True)), + ] + cache_alter=('', 'false', 'true') + scenarios = make_scenarios(types, hints, resid, reopen) + + def verify_metadata(self, metastr): + if metastr == '': + return + cursor = self.session.open_cursor('metadata:', None, None) + # + # Walk through all the metadata looking for the entries that are + # the file URIs for components of the table. + # + found = False + while True: + ret = cursor.next() + if ret != 0: + break + key = cursor.get_key() + check_meta = ((key.find("lsm:") != -1 or key.find("file:") != -1) \ + and key.find(self.name) != -1) + if check_meta: + value = cursor[key] + found = True + self.assertTrue(value.find(metastr) != -1) + cursor.close() + self.assertTrue(found == True) + + # Alter: Change the access pattern hint after creation + def test_alter01_access(self): + uri = self.uri + self.name + create_params = 'key_format=i,value_format=i,' + complex_params = '' + # + # If we're not explicitly setting the parameter, then don't + # modify create_params to test using the default. + # + if self.acreate != '': + access_param = 'access_pattern_hint=%s' % self.acreate + create_params += '%s,' % access_param + complex_params += '%s,' % access_param + else: + # NOTE: This is hard-coding the default value. If the default + # changes then this will fail and need to be fixed. + access_param = 'access_pattern_hint=none' + if self.ccreate != '': + cache_param = 'cache_resident=%s' % self.ccreate + create_params += '%s,' % cache_param + complex_params += '%s,' % cache_param + else: + # NOTE: This is hard-coding the default value. If the default + # changes then this will fail and need to be fixed. + cache_param = 'cache_resident=false' + + cgparam = '' + if self.use_cg or self.use_index: + cgparam = 'columns=(k,v),' + if self.use_cg: + cgparam += 'colgroups=(g0),' + + self.session.create(uri, create_params + cgparam) + # Add in column group or index settings. + if self.use_cg: + cgparam = 'columns=(v),' + suburi = 'colgroup:' + self.name + ':g0' + self.session.create(suburi, complex_params + cgparam) + if self.use_index: + suburi = 'index:' + self.name + ':i0' + self.session.create(suburi, complex_params + cgparam) + + # Put some data in table. + c = self.session.open_cursor(uri, None) + for k in range(self.entries): + c[k+1] = 1 + c.close() + + # Verify the string in the metadata + self.verify_metadata(access_param) + self.verify_metadata(cache_param) + + # Run through all combinations of the alter commands + # for all allowed settings. This tests having only one or + # the other set as well as having both set. It will also + # cover trying to change the setting to its current value. + for a in self.access_alter: + alter_param = '' + access_str = '' + if a != '': + access_str = 'access_pattern_hint=%s' % a + for c in self.cache_alter: + alter_param = '%s' % access_str + cache_str = '' + if c != '': + cache_str = 'cache_resident=%s' % c + alter_param += ',%s' % cache_str + if alter_param != '': + self.session.alter(uri, alter_param) + if self.reopen: + self.reopen_conn() + special = self.use_cg or self.use_index + if not special: + self.verify_metadata(access_str) + self.verify_metadata(cache_str) + else: + self.session.alter(suburi, alter_param) + self.verify_metadata(access_str) + self.verify_metadata(cache_str) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_bug011.py b/src/third_party/wiredtiger/test/suite/test_bug011.py index 29bb08ec2e5..969aaeb5b39 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug011.py +++ b/src/third_party/wiredtiger/test/suite/test_bug011.py @@ -30,25 +30,28 @@ import random, wiredtiger, wttest from wtdataset import SimpleDataSet # test_bug011.py -# Eviction working on more files than there are hazard pointers. +# Eviction working on more trees than the eviction server can walk +# simultaneously. There is a builtin limit of 1000 trees, we open double +# that, which makes this a long-running test. class test_bug011(wttest.WiredTigerTestCase): """ Test having eviction working on more files than the number of allocated hazard pointers. """ table_name = 'test_bug011' - ntables = 50 + ntables = 2000 nrows = 10000 nops = 10000 # Add connection configuration for this test. def conn_config(self, dir): - return 'cache_size=10MB,eviction_dirty_target=99,eviction_dirty_trigger=99,hazard_max=' + str(self.ntables / 2) + return 'cache_size=1GB' + @wttest.longtest("Eviction copes with lots of files") def test_eviction(self): cursors = [] datasets = [] for i in range(0, self.ntables): - this_uri = 'table:%s-%03d' % (self.table_name, i) + this_uri = 'table:%s-%05d' % (self.table_name, i) ds = SimpleDataSet(self, this_uri, self.nrows, config='allocation_size=1KB,leaf_page_max=1KB') ds.populate() @@ -57,9 +60,9 @@ class test_bug011(wttest.WiredTigerTestCase): # Switch over to on-disk trees with multiple leaf pages self.reopen_conn() - # Make sure we have a cursor for the table so it stays in cache. + # Make sure we have a cursor for every table so it stays in cache. for i in range(0, self.ntables): - this_uri = 'table:%s-%03d' % (self.table_name, i) + this_uri = 'table:%s-%05d' % (self.table_name, i) cursors.append(self.session.open_cursor(this_uri, None)) # Make use of the cache. diff --git a/src/third_party/wiredtiger/test/suite/test_compact02.py b/src/third_party/wiredtiger/test/suite/test_compact02.py index 3a4ca6cbc7e..7af76b5fd58 100644 --- a/src/third_party/wiredtiger/test/suite/test_compact02.py +++ b/src/third_party/wiredtiger/test/suite/test_compact02.py @@ -38,7 +38,7 @@ from wtscenario import make_scenarios class test_compact02(wttest.WiredTigerTestCase): types = [ - ('file', dict(uri='file:test_compact02')), + ('table', dict(uri='table:test_compact02')), ] cacheSize = [ ('default', dict(cacheSize='')), @@ -69,8 +69,9 @@ class test_compact02(wttest.WiredTigerTestCase): # 1. Create a table with the data, alternating record size. # 2. Checkpoint and get stats on the table to confirm the size. # 3. Delete the half of the records with the larger record size. - # 4. Call compact. - # 5. Get stats on compacted table. + # 4. Checkpoint so compact finds something to work with. + # 5. Call compact. + # 6. Get stats on compacted table. # nrecords = 22000 bigvalue = "abcdefghi" * 1074 # 9*1074 == 9666 @@ -99,7 +100,8 @@ class test_compact02(wttest.WiredTigerTestCase): self.home = '.' conn_params = 'create,' + \ cacheSize + ',error_prefix="%s: ",' % self.shortid() + \ - 'statistics=(all),eviction_dirty_target=99,eviction_dirty_trigger=99' + 'statistics=(all),' + \ + 'eviction_dirty_target=99,eviction_dirty_trigger=99' try: self.conn = wiredtiger.wiredtiger_open(self.home, conn_params) except wiredtiger.WiredTigerError as e: @@ -140,10 +142,13 @@ class test_compact02(wttest.WiredTigerTestCase): c.close() self.pr('Removed total ' + str((count * 9666) / mb) + 'MB') - # 4. Call compact. + # 4. Checkpoint + self.session.checkpoint() + + # 5. Call compact. self.session.compact(self.uri, None) - # 5. Get stats on compacted table. + # 6. Get stats on compacted table. sz = self.getSize() self.pr('After compact ' + str(sz / mb) + 'MB') diff --git a/src/third_party/wiredtiger/test/suite/test_config03.py b/src/third_party/wiredtiger/test/suite/test_config03.py index 88ca6ae3f39..6699f7d2650 100644 --- a/src/third_party/wiredtiger/test/suite/test_config03.py +++ b/src/third_party/wiredtiger/test/suite/test_config03.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!usr/bin/env python # # Public Domain 2014-2016 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. @@ -48,8 +48,6 @@ class test_config03(test_base03.test_base03): eviction_trigger_scenarios = wtscenario.quick_scenarios( 's_eviction_trigger', [50, 90, 95, 99], None) - hazard_max_scenarios = wtscenario.quick_scenarios('s_hazard_max', - [15, 50, 500], [0.4, 0.8, 0.8]) multiprocess_scenarios = wtscenario.quick_scenarios('s_multiprocess', [True,False], [1.0,1.0]) session_max_scenarios = wtscenario.quick_scenarios('s_session_max', @@ -66,13 +64,13 @@ class test_config03(test_base03.test_base03): verbose_scenarios = wtscenario.quick_scenarios('s_verbose', [None], None) config_vars = [ 'cache_size', 'create', 'error_prefix', 'eviction_target', - 'eviction_trigger', 'hazard_max', 'multiprocess', - 'session_max', 'verbose' ] + 'eviction_trigger', 'multiprocess', 'session_max', + 'verbose' ] scenarios = wtscenario.make_scenarios( cache_size_scenarios, create_scenarios, error_prefix_scenarios, eviction_target_scenarios, eviction_trigger_scenarios, - hazard_max_scenarios, multiprocess_scenarios, session_max_scenarios, + multiprocess_scenarios, session_max_scenarios, transactional_scenarios, verbose_scenarios, prune=1000) #wttest.WiredTigerTestCase.printVerbose(2, 'test_config03: running ' + \ diff --git a/src/third_party/wiredtiger/test/suite/test_config04.py b/src/third_party/wiredtiger/test/suite/test_config04.py index 204aa7e27d5..db8a5f4a16a 100644 --- a/src/third_party/wiredtiger/test/suite/test_config04.py +++ b/src/third_party/wiredtiger/test/suite/test_config04.py @@ -154,10 +154,6 @@ class test_config04(wttest.WiredTigerTestCase): 'eviction_trigger=86'), "/eviction target must be lower than the eviction trigger/") - def test_hazard_max(self): - # Note: There isn't any direct way to know that this was set. - self.common_test('hazard_max=50') - def test_invalid_config(self): msg = '/Unbalanced brackets/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, diff --git a/src/third_party/wiredtiger/test/suite/test_cursor10.py b/src/third_party/wiredtiger/test/suite/test_cursor10.py new file mode 100644 index 00000000000..b3cffeab4e9 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_cursor10.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios + +# test_cursor10.py +# Cursors with projections. +class test_cursor04(wttest.WiredTigerTestCase): + """ + Test cursor search and search_near + """ + table_name1 = 'test_cursor04' + nentries = 20 + + scenarios = make_scenarios([ + ('row', dict(key_format='S', uri='table')), + ('col', dict(key_format='r', uri='table')) + ]) + + def genkey(self, i): + if self.key_format == 'S': + return 'key' + str(i).zfill(5) # return key00001, key00002, etc. + else: + return long(i+1) + + def genvalue(self, i): + return [ 'v0:' + str(i), i+1, 'v2' + str(i+2), i+3 ] + + def extractkey(self, k): + if self.key_format == 'S': + return int(k[3:]) + else: + return long(k-1) + + def test_projection(self): + """ + Create entries, and read back in a regular and projected cursor + """ + tablearg = self.uri + ":" + self.table_name1 + create_args = 'columns=(k,v0,v1,v2,v3),value_format=SiSi,key_format=' \ + + self.key_format + self.session.create(tablearg, create_args) + + cursor = self.session.open_cursor(tablearg, None, None) + for i in range(0, self.nentries): + cursor.set_key(self.genkey(i)) + values = self.genvalue(i) + cursor.set_value(*values) + cursor.insert() + cursor.close() + cursor = self.session.open_cursor(tablearg, None, None) + count = 0 + for k,v0,v1,v2,v3 in cursor: + i = self.extractkey(k) + self.assertEqual(self.genkey(i), k) + self.assertEqual(self.genvalue(i), [v0,v1,v2,v3]) + count += 1 + self.assertEqual(count, self.nentries) + cursor.close() + cursor = self.session.open_cursor(tablearg + '(v3,v2,v1,v0,k)',\ + None, None) + count = 0 + for k1,v3,v2,v1,v0,k2 in cursor: + self.assertEqual(k1, k2) + i = self.extractkey(k1) + self.assertEqual(self.genkey(i), k1) + self.assertEqual(self.genvalue(i), [v0,v1,v2,v3]) + count += 1 + self.assertEqual(count, self.nentries) + cursor.close() + + def test_index_projection(self): + """ + Create entries, and read back in an index cursor with a projection + """ + tablearg = self.uri + ":" + self.table_name1 + indexarg = 'index:' + self.table_name1 + ':index1' + create_args = 'columns=(k,v0,v1,v2,v3),value_format=SiSi,key_format=' \ + + self.key_format + self.session.create(tablearg, create_args) + self.session.create(indexarg, 'columns=(v0,v2,v1,v3)') + cursor = self.session.open_cursor(tablearg, None, None) + for i in range(0, self.nentries): + cursor.set_key(self.genkey(i)) + values = self.genvalue(i) + cursor.set_value(*values) + cursor.insert() + cursor.close() + cursor = self.session.open_cursor(tablearg + '(v3,v2,v1,v0,k)',\ + None, None) + count = 0 + for k1,v3,v2,v1,v0,k2 in cursor: + self.assertEqual(k1, k2) + i = self.extractkey(k1) + self.assertEqual(self.genkey(i), k1) + self.assertEqual(self.genvalue(i), [v0,v1,v2,v3]) + count += 1 + self.assertEqual(count, self.nentries) + cursor.close() +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_hazard.py b/src/third_party/wiredtiger/test/suite/test_hazard.py new file mode 100644 index 00000000000..f2891fce526 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_hazard.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_hazard.py +# Hazard pointer tests. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet + +# Regression tests. +class test_hazard(wttest.WiredTigerTestCase): + + # Allocate a large number of hazard pointers in a session, forcing the + # hazard pointer array to repeatedly grow. + def test_hazard(self): + uri = "table:hazard" + ds = SimpleDataSet(self, uri, 1000) + ds.populate() + + # Open 10,000 cursors and pin a page to set a hazard pointer. + cursors = [] + for i in range(0, 10000): + c = self.session.open_cursor(uri, None) + c.set_key(ds.key(10)) + c.search() + cursors.append(c) + + # Close the cursors, clearing the hazard pointer. + for c in cursors: + c.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_index02.py b/src/third_party/wiredtiger/test/suite/test_index02.py index 9f39df003b1..4f424e5d3d2 100644 --- a/src/third_party/wiredtiger/test/suite/test_index02.py +++ b/src/third_party/wiredtiger/test/suite/test_index02.py @@ -27,20 +27,26 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest +from wtscenario import make_scenarios # test_index02.py # test search_near in indices class test_index02(wttest.WiredTigerTestCase): '''Test search_near in indices''' + scenarios = make_scenarios([ + ('index', dict(indexconfig='columns=(v)', ncol=1)), + ('index-with-key', dict(indexconfig='columns=(v,k)', ncol=2)), + ]) + basename = 'test_index02' tablename = 'table:' + basename indexname = 'index:' + basename + ":inverse" - def test_search_near(self): - '''Create a table, look for a nonexistent key''' + def test_search_near_exists(self): + '''Create a table, look for an existing key''' self.session.create(self.tablename, 'key_format=r,value_format=Q,columns=(k,v)') - self.session.create(self.indexname, 'columns=(v)') + self.session.create(self.indexname, self.indexconfig) cur = self.session.open_cursor(self.tablename, None, "append") cur.set_value(1) cur.insert() @@ -52,17 +58,60 @@ class test_index02(wttest.WiredTigerTestCase): cur.insert() cur.set_value(10) cur.insert() + cur.close() + + # Retry after reopening + for runs in xrange(2): + # search near should find a match + cur = self.session.open_cursor(self.indexname, None, None) + if self.ncol == 1: + cur.set_key(5) + else: + cur.set_key(5, 3) + self.assertEqual(cur.search_near(), 0) + + # Retry after reopening + self.reopen_conn() + + def test_search_near_between(self): + '''Create a table, look for a non-existing key''' + self.session.create(self.tablename, 'key_format=i,value_format=i,columns=(k,v)') + self.session.create(self.indexname, self.indexconfig) + cur = self.session.open_cursor(self.tablename) + for k in xrange(3): + cur[k] = 5 * k + 10 + cur.close() + + search_keys = [ 1, 11, 15, 19, 21 ] # search near should find a match - cur2 = self.session.open_cursor(self.indexname, None, None) - cur2.set_key(5) - self.assertEqual(cur2.search_near(), 0) + for runs in xrange(2): + cur = self.session.open_cursor(self.indexname, None, None) + for k in search_keys: + if self.ncol == 1: + cur.set_key(k) + else: + cur.set_key(k, 1) # [15,1] will completely match + exact = cur.search_near() + if self.ncol == 1: + found_key = cur.get_key() + else: + [ found_key, index ] = cur.get_key() + self.pr("search_near for " + str(k) + " found " + str(found_key) + " with exact " + str(exact)) + self.assertEqual(exact, cmp(found_key, k), "for key " + str(k)) + self.reopen_conn() - # Retry after reopening - self.reopen_conn() - cur3 = self.session.open_cursor(self.indexname, None, None) - cur3.set_key(5) - self.assertEqual(cur3.search_near(), 0) + def test_search_near_empty(self): + '''Create an empty table, look for a key''' + self.session.create(self.tablename, 'key_format=i,value_format=i,columns=(k,v)') + self.session.create(self.indexname, self.indexconfig) + + cur = self.session.open_cursor(self.indexname, None, None) + if self.ncol == 1: + cur.set_key(3) + else: + cur.set_key(3, 1) + self.assertEqual(cur.search_near(), wiredtiger.WT_NOTFOUND) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_inmem01.py b/src/third_party/wiredtiger/test/suite/test_inmem01.py index 1af43bbd9d9..388485db29b 100644 --- a/src/third_party/wiredtiger/test/suite/test_inmem01.py +++ b/src/third_party/wiredtiger/test/suite/test_inmem01.py @@ -59,7 +59,7 @@ class test_inmem01(wttest.WiredTigerTestCase): ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt, value_format=self.valuefmt, config=self.table_config) self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, - lambda:ds.populate(), msg) + ds.populate, msg) # Figure out the last key we successfully inserted, and check all # previous inserts are still there. @@ -77,7 +77,7 @@ class test_inmem01(wttest.WiredTigerTestCase): ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt, value_format=self.valuefmt, config=self.table_config) self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, - lambda:ds.populate(), msg) + ds.populate, msg) # Now that the database contains as much data as will fit into # the configured cache, verify removes succeed. @@ -93,7 +93,7 @@ class test_inmem01(wttest.WiredTigerTestCase): ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt, value_format=self.valuefmt, config=self.table_config) self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, - lambda:ds.populate(), msg) + ds.populate, msg) cursor = self.session.open_cursor(self.uri, None) cursor.prev() @@ -120,5 +120,50 @@ class test_inmem01(wttest.WiredTigerTestCase): break self.assertTrue(inserted) + # Custom "keep filling" helper + def fill(self, cursor, ds, start, end): + for i in xrange(start + 1, end + 1): + cursor[ds.key(i)] = ds.value(i) + + # Keep adding data to the cache until it becomes really full, make sure + # that reads aren't blocked. + @wttest.longtest("Try to wedge an in-memory cache") + def test_wedge(self): + # Try to really wedge the cache full + ds = SimpleDataSet(self, self.uri, 0, key_format=self.keyfmt, + value_format=self.valuefmt, config=self.table_config) + ds.populate() + cursor = self.session.open_cursor(self.uri, None) + + run = 0 + start, last_key = -1000, 0 + while last_key - start > 100: + msg = '/WT_CACHE_FULL.*/' + start = last_key + self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, + lambda: self.fill(cursor, ds, start, 10000000), msg) + cursor.reset() + sleep(1) + + # Figure out the last key we successfully inserted, and check all + # previous inserts are still there. + cursor.prev() + last_key = int(cursor.get_key()) + run += 1 + self.pr('Finished iteration ' + str(run) + ', last_key = ' + str(last_key)) + + self.pr('Checking ' + str(last_key) + ' keys') + ds = SimpleDataSet(self, self.uri, last_key, key_format=self.keyfmt, + value_format=self.valuefmt, config=self.table_config) + + # This test is *much* slower for fixed-length column stores: we fit + # many more records into the cache, so don't do as many passes through + # the data. + checks = 10 if self.valuefmt.endswith('t') else 100 + for run in xrange(checks): + ds.check() + self.pr('Finished check ' + str(run)) + sleep(1) + if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_join01.py b/src/third_party/wiredtiger/test/suite/test_join01.py index f3b13026896..2c4328dc7d3 100644 --- a/src/third_party/wiredtiger/test/suite/test_join01.py +++ b/src/third_party/wiredtiger/test/suite/test_join01.py @@ -110,8 +110,9 @@ class test_join01(wttest.WiredTigerTestCase): while jc.next() == 0: [k] = jc.get_keys() i = k - 1 - if do_proj: # our projection test simply reverses the values - [v2,v1,v0] = jc.get_values() + if do_proj: # our projection reverses the values and adds the key + [v2,v1,v0,kproj] = jc.get_values() + self.assertEquals(k, kproj) else: [v0,v1,v2] = jc.get_values() self.assertEquals(self.gen_values(i), [v0,v1,v2]) @@ -136,7 +137,7 @@ class test_join01(wttest.WiredTigerTestCase): if self.ref == 'index': expectstats.append('join: index:join01:index0: ' + statdesc) elif self.do_proj: - expectstats.append('join: table:join01(v2,v1,v0): ' + statdesc) + expectstats.append('join: table:join01(v2,v1,v0,k): ' + statdesc) else: expectstats.append('join: table:join01: ' + statdesc) self.check_stats(statcur, expectstats) @@ -228,7 +229,7 @@ class test_join01(wttest.WiredTigerTestCase): c.close() if do_proj: - proj_suffix = '(v2,v1,v0)' # Reversed values + proj_suffix = '(v2,v1,v0,k)' # Reversed values plus key else: proj_suffix = '' # Default projection (v0,v1,v2) diff --git a/src/third_party/wiredtiger/test/suite/test_nsnap04.py b/src/third_party/wiredtiger/test/suite/test_nsnap04.py index 60901dd2ee3..8d491540d74 100644 --- a/src/third_party/wiredtiger/test/suite/test_nsnap04.py +++ b/src/third_party/wiredtiger/test/suite/test_nsnap04.py @@ -38,14 +38,18 @@ class test_nsnap04(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:' + tablename nrows_per_itr = 10 - def check_named_snapshot(self, snapshot, expected): + def check_named_snapshot(self, snapshot, expected, skip_snapshot=False): new_session = self.conn.open_session() c = new_session.open_cursor(self.uri) - new_session.begin_transaction("snapshot=" + str(snapshot)) + if skip_snapshot: + new_session.begin_transaction() + else: + new_session.begin_transaction("snapshot=" + str(snapshot)) count = 0 for row in c: count += 1 new_session.commit_transaction() + new_session.close() # print "Checking snapshot %d, expect %d, found %d" % (snapshot, expected, count) self.assertEqual(count, expected) @@ -80,5 +84,34 @@ class test_nsnap04(wttest.WiredTigerTestCase, suite_subprocess): self.session.snapshot("name=0") self.check_named_snapshot(0, 2 * self.nrows_per_itr) + def test_include_updates(self): + # Populate a table + end = start = 0 + SimpleDataSet(self, self.uri, 0, key_format='i').populate() + + snapshots = [] + c = self.session.open_cursor(self.uri) + for i in xrange(self.nrows_per_itr): + c[i] = "some value" + + self.session.begin_transaction("isolation=snapshot") + count = 0 + for row in c: + count += 1 + self.session.snapshot("name=0,include_updates=true") + + self.check_named_snapshot(0, self.nrows_per_itr) + + # Insert some more content using the active session. + for i in xrange(self.nrows_per_itr): + c[self.nrows_per_itr + i] = "some value" + + self.check_named_snapshot(0, 2 * self.nrows_per_itr) + # Ensure transactions not tracking the snapshot don't see the updates + self.check_named_snapshot(0, self.nrows_per_itr, skip_snapshot=True) + self.session.commit_transaction() + # Ensure content is visible to non-snapshot transactions after commit + self.check_named_snapshot(0, 2 * self.nrows_per_itr, skip_snapshot=True) + if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/wtdataset.py b/src/third_party/wiredtiger/test/suite/wtdataset.py index cb77735510c..74e07e24e93 100644 --- a/src/third_party/wiredtiger/test/suite/wtdataset.py +++ b/src/third_party/wiredtiger/test/suite/wtdataset.py @@ -49,7 +49,7 @@ class BaseDataSet(object): def fill(self): c = self.testcase.session.open_cursor(self.uri, None) - for i in range(1, self.rows + 1): + for i in xrange(1, self.rows + 1): c[self.key(i)] = self.value(i) c.close() @@ -166,7 +166,7 @@ class SimpleIndexDataSet(SimpleDataSet): # Check values in the index. idxcursor = self.testcase.session.open_cursor(self.indexname) - for i in range(1, self.rows + 1): + for i in xrange(1, self.rows + 1): k = self.key(i) v = self.value(i) ik = (v, k) # The index key is columns=(v,k). diff --git a/src/third_party/wiredtiger/test/syscall/Makefile.am b/src/third_party/wiredtiger/test/syscall/Makefile.am new file mode 100644 index 00000000000..1d55b06c388 --- /dev/null +++ b/src/third_party/wiredtiger/test/syscall/Makefile.am @@ -0,0 +1,7 @@ +AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include +LDADD = $(top_builddir)/libwiredtiger.la +AM_LDFLAGS = -static +noinst_PROGRAMS = + +test_wt2336_base_SOURCES = wt2336_base/main.c +noinst_PROGRAMS += test_wt2336_base diff --git a/src/third_party/wiredtiger/test/syscall/syscall.py b/src/third_party/wiredtiger/test/syscall/syscall.py new file mode 100644 index 00000000000..59c2f347146 --- /dev/null +++ b/src/third_party/wiredtiger/test/syscall/syscall.py @@ -0,0 +1,857 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# syscall.py +# Command line syscall test runner +# +# For each .run file, run the corresponding program and collect strace +# output, comparing it to the contents of the .run file. +# +# run files are first preprocessed, which means the use of #ifdefs, #defines +# and #includes are allowed, as well as /**/ and // comments. +# Expressions are evaluated using the Python parser, so that: +# hex and octal numbers are accepted, constant values can be or-ed. +# +from __future__ import print_function +import argparse, distutils.spawn, fnmatch, os, platform, re, shutil, \ + subprocess, sys + +# A class that represents a context in which predefined constants can be +# set, and new variables can be assigned. +class VariableContext(object): + def __getitem__(self, key): + if key not in dir(self) or key.startswith('__'): + raise KeyError(key) + return getattr(self, key) + + def __setitem__(self, key, value): + setattr(self, key, value) + +################################################################ +# Changable parameters +# We expect these values to evolve as tests are added or modified. + +# Generally, system calls must be wrapped in an ASSERT_XX() "macro". +# Exceptions are calls in this list that return 0 on success, or +# those that are hardcoded in Runner.call_compare() +calls_returning_zero = [ 'close', 'ftruncate', 'fdatasync', 'rename' ] + +# Encapsulate all the defines we can use in our scripts. +# When this program is run, we'll find out their actual values on +# the host system. +defines_used = [ + 'HAVE_FTRUNCATE', 'O_ACCMODE', 'O_APPEND', 'O_ASYNC', + 'O_CLOEXEC', 'O_CREAT', 'O_EXCL', 'O_EXLOCK', 'O_NOATIME', + 'O_NOFOLLOW', 'O_NONBLOCK', 'O_RDONLY', 'O_RDWR', 'O_SHLOCK', + 'O_TRUNC', 'O_WRONLY' ] + +################################################################ + +# Patterns that are used to match the .run file and/or the output. +ident = r'([a-zA-Z_][a-zA-Z0-9_]*)' +outputpat = re.compile(r'OUTPUT\("([^"]*)"\)') +argpat = re.compile(r'''((?:[^,"']|"[^"]*"|'[^']*')+)''') +discardpat = re.compile(r';') + +# e.g. fd = open("blah", 0, 0); +assignpat = re.compile(ident + r'\s*=\s*' + ident + r'(\([^;]*\));') + +# e.g. ASSERT_EQ(close(fd), 0); +assertpat = re.compile(r'ASSERT_([ENLG][QET])\s*\(\s*' + ident + r'\s*(\(.*\))\s*,\s*([a-zA-Z0-9_]+)\);') + +# e.g. close(fd); must return 0 +callpat = re.compile(ident + r'(\(.*\));') + +# e.g. open("blah", 0x0, 0x0) = 6 0 +# We capture the errno (e.g. "0" or "Err#60"), but don't do anything with it. +# We don't currently test anything that is errno dependent. +dtruss_pat = re.compile(ident + r'(\(.*\))\s*=\s*(-*[0-9xA-F]+)\s+([-A-Za-z#0-9]*)') +# At the top of the dtruss output is a fixed string. +dtruss_init_pat = re.compile(r'\s*SYSCALL\(args\)\s*=\s*return\s*') + +strace_pat = re.compile(ident + r'(\(.*\))\s*=\s(-*[0-9]+)()') + +tracepat = re.compile(r'TRACE\("([^"]*)"\)') +runpat = re.compile(r'RUN\(([^\)]*)\)') +systempat = re.compile(r'SYSTEM\("([^"]*)"\)') +# If tracepat matches, set map['trace_syscalls'] to the 0'th group, etc. +headpatterns = [ [ tracepat, 'trace_syscalls', 0], + [ systempat, 'required_system', 0], + [ runpat, 'run_args', 0] ] + +pwrite_in = r'pwrite64' +pwrite_out = r'pwrite' + +# To create breakpoints while debugging this script +def bp(): + import pdb + pdb.set_trace() + +def msg(s): + print("syscall.py: " + s, file=sys.stderr) + +def die(s): + msg(s) + sys.exit(1) + +# If wttop appears as a prefix of pathname, strip it off. +def simplify_path(wttop, pathname): + wttop = os.path.join(wttop, "") + if pathname.startswith(wttop): + pathname = pathname[len(wttop):] + return pathname + +def printfile(pathname, abbrev): + print("================================================================") + print(abbrev + " (" + pathname + "):") + with open(pathname, 'r') as f: + shutil.copyfileobj(f, sys.stdout) + print("================================================================") + +# A line from a file: a modified string with the file name and line number +# associated with it. +class FileLine(str): + filename = None + linenum = 0 + def __new__(cls, value, *args, **kwargs): + return super(FileLine, cls).__new__(cls, value) + + def prefix(self): + return self.filename + ':' + str(self.linenum) + ': ' + + def range_prefix(self, otherline): + if self == otherline: + othernum = '' + elif otherline == None: + othernum = '-EOF' + else: + othernum = '-' + str(otherline.linenum) + return self.filename + ':' + str(self.linenum) + othernum + ': ' + +# Manage reading from a file, tracking line numbers. +class Reader(object): + # 'raw' means we don't ignore any lines + # 'is_cpp' means input lines beginning with '#' indicate file/linenumber + def __init__(self, wttop, filename, f, raw = True, is_cpp = False): + self.wttop = wttop + self.orig_filename = filename + self.filename = filename + self.f = f + self.linenum = 1 + self.raw = raw + self.is_cpp = is_cpp + if not self.f: + die(self.filename + ': cannot open') + + def __enter__(self): + if not self.f: + return False + return self + + def __exit__(self, typ, value, traceback): + if self.f: + self.f.close() + self.f = None + + # Return True if the line is to be ignored. + def ignore(self, line): + if self.raw: + return False + return line == '' + + # strip a line of comments + def strip_line(self, line): + if not line: + return None + line = line.strip() + if self.is_cpp and line.startswith('#'): + parts = line.split() + if len(parts) < 3 or not parts[1].isdigit(): + msg('bad cpp input: ' + line) + line = '' + self.linenum = int(parts[1]) - 1 + self.filename = parts[2].strip('"') + if self.filename == '<stdin>': + self.filename = self.orig_filename + if '//' in line: + if line.startswith('//'): + line = '' + else: + # This isn't exactly right, it would see "; //" + # within a string or comment. + m = re.match(r'^(.*;|.*\.\.\.)\s*//', line) + if m: + line = m.groups()[0].strip() + return line + + def readline(self): + line = self.strip_line(self.f.readline()) + while line != None and self.ignore(line): + self.linenum += 1 + line = self.strip_line(self.f.readline()) + if line: + line = FileLine(line) + line.filename = self.filename + line.linenum = self.linenum + self.linenum += 1 + else: + line = '' # make this somewhat compatible with file.readline + return line + + def close(self): + self.f.close() + +# Read from a regular file. +class FileReader(Reader): + def __init__(self, wttop, filename, raw = True): + return super(FileReader, self).__init__(wttop, filename, + open(filename), raw, False) + +# Read from the C preprocessor run on a file. +class PreprocessedReader(Reader): + def __init__(self, wttop, filename, predefines, raw = True): + sourcedir = os.path.dirname(filename) + cmd = ['cc', '-E', '-I' + sourcedir] + for name in dir(predefines): + if not name.startswith('__'): + cmd.append('-D' + name + '=' + str(predefines[name])) + cmd.append('-') + proc = subprocess.Popen(cmd, stdin=open(filename), + stdout=subprocess.PIPE) + super(PreprocessedReader, self).__init__(wttop, filename, + proc.stdout, raw, True) + +# Track options discovered in the 'head' section of the .run file. +class HeadOpts: + def __init__(self): + self.run_args = None + self.required_system = None + self.trace_syscalls = None + +# Manage a run of the target program characterized by a .run file, +# comparing output from the run and reporting differences. +class Runner: + def __init__(self, wttopdir, runfilename, exedir, testexe, + strace, args, variables, defines): + self.variables = variables + self.defines = defines + self.wttopdir = wttopdir + self.runfilename = runfilename + self.testexe = testexe + self.exedir = exedir + self.strace = strace + self.args = args + self.headopts = HeadOpts() + self.dircreated = False + self.strip_syscalls = None + outfilename = args.outfilename + errfilename = args.errfilename + if outfilename == None: + self.outfilename = os.path.join(exedir, 'stdout.txt') + else: + self.outfilename = outfilename + if errfilename == None: + self.errfilename = os.path.join(exedir, 'stderr.txt') + else: + self.errfilename = errfilename + + self.runfile = PreprocessedReader(self.wttopdir, runfilename, + self.defines, False) + + def init(self, systemtype): + # Read up until 'RUN()', setting attributes of self.headopts + runline = '?' + m = None + while runline: + runline = self.runfile.readline() + m = None + for pat,attr,group in headpatterns: + m = re.match(pat, runline) + if m: + setattr(self.headopts, attr, m.groups()[group]) + break + if not m: + self.fail(runline, "unknown header option: " + runline) + return [ False, False ] + if self.headopts.run_args != None: # found RUN()? + break + if not self.headopts.trace_syscalls: + msg("'" + self.runfile.filename + "': needs TRACE(...)") + return [ False, False ] + runargs = self.headopts.run_args.strip() + if len(runargs) > 0: + if len(runargs) < 2 or runargs[0] != '"' or runargs[-1] != '"': + msg("'" + self.runfile.filename + + "': Missing double quotes in RUN arguments") + return [ False, False ] + runargs = runargs[1:-1] + self.runargs = runargs.split() + #print("SYSCALLS: " + self.headopts.trace_syscalls + if self.headopts.required_system != None and \ + self.headopts.required_system != systemtype: + msg("skipping '" + self.runfile.filename + "': for '" + + self.headopts.required_system + "', this system is '" + + systemtype + "'") + return [ False, True ] + return [ True, False ] + + def close(self, forcePreserve): + self.runfile.close() + if self.exedir and self.dircreated and \ + not self.args.preserve and not forcePreserve: + os.chdir('..') + shutil.rmtree(self.exedir) + + def fail(self, line, s): + # make it work if line is None or is a plain string. + try: + prefix = simplify_path(self.wttopdir, line.prefix()) + except: + prefix = 'syscall.py: ' + print(prefix + s, file=sys.stderr) + + def failrange(self, line, lineto, s): + # make it work if line is None or is a plain string. + try: + prefix = simplify_path(self.wttopdir, line.range_prefix(lineto)) + except: + prefix = 'syscall.py: ' + print(prefix + s, file=sys.stderr) + + def str_match(self, s1, s2): + fuzzyRight = False + if len(s1) < 2 or len(s2) < 2: + return False + if s1[-3:] == '...': + fuzzyRight = True + s1 = s1[:-3] + if s2[-3:] == '...': + s2 = s2[:-3] + if s1[0] != '"' or s1[-1] != '"' or s2[0] != '"' or s2[-1] != '"': + return False + s1 = s1[1:-1] + s2 = s2[1:-1] + # We allow a trailing \0 + if s1[-2:] == '\\0': + s1 = s1[:-2] + if s2[-2:] == '\\0': + s2 = s2[:-2] + if fuzzyRight: + return s2.startswith(s2) + else: + return s1 == s2 + + def expr_eval(self, s): + return eval(s, {}, self.variables) + + def arg_match(self, a1, a2): + a1 = a1.strip() + a2 = a2.strip() + if a1 == a2: + return True + if len(a1) == 0 or len(a2) == 0: + return False + if a1[0] == '"': + return self.str_match(a1, a2) + #print(' arg_match: <' + a1 + '> <' + a2 + '>') + try: + a1value = self.expr_eval(a1) + except Exception: + self.fail(a1, 'unknown expression: ' + a1) + return False + try: + a2value = self.expr_eval(a2) + except Exception: + self.fail(a2, 'unknown expression: ' + a2) + return False + return a1value == a2value or int(a1value) == int(a2value) + + def split_args(self, s): + if s[0] == '(': + s = s[1:] + if s[-1] == ')': + s = s[:-1] + return argpat.split(s)[1::2] + + def args_match(self, args1, args2): + #print('args_match: ' + str(s1) + ', ' + str(s2)) + pos = 0 + for a1 in args1: + a1 = a1.strip() + if a1 == '...': # match anything? + return True + if pos >= len(args2): + return False + if not self.arg_match(a1, args2[pos]): + return False + pos += 1 + if pos < len(args2): + return False + return True + + # func(args); is shorthand for for ASSERT_EQ(func(args), xxx); + # where xxx may be 0 or may be derived from one of the args. + def call_compare(self, callname, result, eargs, errline): + if callname in calls_returning_zero: + return self.compare("EQ", result, "0", errline) + elif callname == 'pwrite' or callname == 'pwrite64': + return self.compare("EQ", + re.sub(pwrite_in, pwrite_out, result), + re.sub(pwrite_in, pwrite_out, eargs[2]), + errline) + else: + self.fail(errline, 'call ' + callname + + ': not known, use ASSERT_EQ()') + + def compare(self, compareop, left, right, errline): + l = self.expr_eval(left) + r = self.expr_eval(right) + if (compareop == "EQ" and l == r) or \ + (compareop == "NE" and l != r) or \ + (compareop == "LT" and l < r) or \ + (compareop == "LE" and l <= r) or \ + (compareop == "GT" and l > r) or \ + (compareop == "GE" and l >= r): + return True + else: + self.fail(errline, + 'call returned value: ' + left + ', comparison: (' + + left + ' ' + compareop + ' ' + right + + ') at line: ' + errline) + return False + + def match_report(self, runline, errline, verbose, skiplines, result, desc): + if result: + if verbose: + print('MATCH:') + print(' ' + runline.prefix() + runline) + print(' ' + errline.prefix() + errline) + else: + if verbose: + if not skiplines: + msg('Expecting ' + desc) + print(' ' + runline.prefix() + runline + + ' does not match:') + print(' ' + errline.prefix() + errline) + else: + print(' (... match) ' + errline.prefix() + errline) + return result + + def match(self, runline, errline, verbose, skiplines): + m = re.match(outputpat, runline) + if m: + outwant = m.groups()[0] + return self.match_report(runline, errline, verbose, skiplines, + errline == outwant, 'output line') + if self.args.systype == 'Linux': + em = re.match(strace_pat, errline) + elif self.args.systype == 'Darwin': + em = re.match(dtruss_pat, errline) + if not em: + self.fail(errline, 'Unknown strace/dtruss output: ' + errline) + return False + gotcall = re.sub(pwrite_in, pwrite_out, em.groups()[0]) + # filtering syscalls here if needed. If it's not a match, + # mark the errline so it is retried. + if self.strip_syscalls != None and gotcall not in self.strip_syscalls: + errline.skip = True + return False + m = re.match(assignpat, runline) + if m: + if m.groups()[1] != gotcall: + return self.match_report(runline, errline, verbose, skiplines, + False, 'syscall to match assignment') + + rargs = self.split_args(m.groups()[2]) + eargs = self.split_args(em.groups()[1]) + result = self.args_match(rargs, eargs) + if result: + self.variables[m.groups()[0]] = em.groups()[2] + return self.match_report(runline, errline, verbose, skiplines, + result, 'syscall to match assignment') + + # pattern groups using example ASSERT_EQ(close(fd), 0); + # 0 : comparison op ("EQ") + # 1 : function call name "close" + # 2 : function call args "(fd)" + # 3 : comparitor "0" + m = re.match(assertpat, runline) + if m: + if m.groups()[1] != gotcall: + return self.match_report(runline, errline, verbose, skiplines, + False, 'syscall to match ASSERT') + + rargs = self.split_args(m.groups()[2]) + eargs = self.split_args(em.groups()[1]) + result = self.args_match(rargs, eargs) + if not result: + return self.match_report(runline, errline, verbose, skiplines, + result, 'syscall to match ASSERT') + result = self.compare(m.groups()[0], em.groups()[2], + m.groups()[3], errline) + return self.match_report(runline, errline, verbose, skiplines, + result, 'ASSERT') + + # A call without an enclosing ASSERT is reduced to an ASSERT, + # depending on the particular system call. + m = re.match(callpat, runline) + if m: + if m.groups()[0] != gotcall: + return self.match_report(runline, errline, verbose, skiplines, + False, 'syscall') + + rargs = self.split_args(m.groups()[1]) + eargs = self.split_args(em.groups()[1]) + result = self.args_match(rargs, eargs) + if not result: + return self.match_report(runline, errline, verbose, skiplines, + result, 'syscall') + result = self.call_compare(m.groups()[0], em.groups()[2], + eargs, errline) + return self.match_report(runline, errline, verbose, skiplines, + result, 'syscall') + + self.fail(runline, 'unrecognized pattern in runfile:' + runline) + return False + + def match_lines(self): + outfile = FileReader(self.wttopdir, self.outfilename, True) + errfile = FileReader(self.wttopdir, self.errfilename, True) + + if outfile.readline(): + self.fail(None, 'output file has content, expected to be empty') + return False + + with outfile, errfile: + runlines = self.order_runfile(self.runfile) + errline = errfile.readline() + errline = re.sub(pwrite_in, pwrite_out, errline) + if re.match(dtruss_init_pat, errline): + errline = errfile.readline() + skiplines = False + for runline in runlines: + runline = re.sub(pwrite_in, pwrite_out, runline) + if runline == '...': + skiplines = True + if self.args.verbose: + print('Fuzzy matching:') + print(' ' + runline.prefix() + runline) + continue + first_errline = errline + while errline and not self.match(runline, errline, + self.args.verbose, skiplines): + if skiplines or hasattr(errline, 'skip'): + errline = errfile.readline() + else: + self.fail(runline, "expecting " + runline) + self.failrange(first_errline, errline, "does not match") + return False + if not errline: + self.fail(runline, "failed to match line: " + runline) + self.failrange(first_errline, errline, "does not match") + return False + errline = errfile.readline() + if re.match(dtruss_init_pat, errline): + errline = errfile.readline() + skiplines = False + if errline and not skiplines: + self.fail(errline, "extra lines seen starting at " + errline) + return False + return True + + def order_runfile(self, f): + # In OS X, dtruss is implemented using dtrace's apparently buffered + # printf writes to stdout, but that is all redirected to stderr. + # Because of that, the test program's writes to stderr do not + # interleave with dtruss output as it does with Linux's strace + # (which writes directly to stderr). On OS X, we get the program's + # output first, we compensate for this by moving all the + # OUTPUT statements in the runfile to match first. This simple + # approach will break if there is more data generated by OUTPUT + # statements than a stdio buffer's size. + matchout = (self.args.systype == 'Darwin') + out = [] + nonout = [] + s = f.readline() + while s: + if matchout and re.match(outputpat, s): + out.append(s) + elif not re.match(discardpat, s): + nonout.append(s) + s = f.readline() + out.extend(nonout) + return out + + def run(self): + if not self.exedir: + self.fail(None, "Execution directory not set") + return False + if not os.path.isfile(self.testexe): + msg("'" + self.testexe + "': no such file") + return False + + shutil.rmtree(self.exedir, ignore_errors=True) + os.mkdir(self.exedir) + self.dircreated = True + os.chdir(self.exedir) + + callargs = list(self.strace) + trace_syscalls = self.headopts.trace_syscalls + if self.args.systype == 'Linux': + callargs.extend(['-e', 'trace=' + trace_syscalls ]) + elif self.args.systype == 'Darwin': + # dtrace has no option to limit the syscalls to be traced, + # so we'll filter the output. + self.strip_syscalls = re.sub(pwrite_in, pwrite_out, + self.headopts.trace_syscalls).split(',') + callargs.append(self.testexe) + callargs.extend(self.runargs) + + outfile = open(self.outfilename, 'w') + errfile = open(self.errfilename, 'w') + if self.args.verbose: + print('RUNNING: ' + str(callargs)) + subret = subprocess.call(callargs, stdout=outfile, stderr=errfile) + outfile.close() + errfile.close() + if subret != 0: + msg("'" + self.testexe + "': exit value " + str(subret)) + printfile(self.outfilename, "output") + printfile(self.errfilename, "error") + return False + return True + +# Run the syscall program. +class SyscallCommand: + def __init__(self, disttop, builddir): + self.disttop = disttop + self.builddir = builddir + + def parse_args(self, argv): + srcdir = os.path.join(self.disttop, 'test', 'syscall') + self.exetopdir = os.path.join(self.builddir, 'test', 'syscall') + self.incdir1 = os.path.join(self.disttop, 'src', 'include') + self.incdir2 = self.builddir + + ap = argparse.ArgumentParser('Syscall test runner') + ap.add_argument('--systype', + help='override system type (Linux/Windows/Darwin)') + ap.add_argument('--errfile', dest='errfilename', + help='do not run the program, use this file as stderr') + ap.add_argument('--outfile', dest='outfilename', + help='do not run the program, use this file as stdout') + ap.add_argument('--preserve', action="store_true", + help='keep the WT_TEST.* directories') + ap.add_argument('--verbose', action="store_true", + help='add some verbose information') + ap.add_argument('tests', nargs='*', + help='the tests to run (defaults to all)') + args = ap.parse_args() + + if not args.systype: + args.systype = platform.system() # Linux, Windows, Darwin + + self.dorun = True + if args.errfilename or args.outfilename: + if len(args.tests) != 1: + msg("one test is required when --errfile or --outfile" + + " is specified") + return False + if not args.outfilename: + args.outfilename = os.devnull + if not args.errfilename: + args.errfilename = os.devnull + self.dorun = False + + # for now, we permit Linux and Darwin + straceexe = None + if args.systype == 'Linux': + strace = [ 'strace' ] + straceexe = 'strace' + elif args.systype == 'Darwin': + strace = [ 'sudo', 'dtruss' ] + straceexe = 'dtruss' + else: + msg("systype '" + args.systype + "' unsupported") + return False + if not distutils.spawn.find_executable(straceexe): + msg("strace: does not exist") + return False + self.args = args + self.strace = strace + return True + + def runone(self, runfilename, exedir, testexe, args): + result = True + runner = Runner(self.disttop, runfilename, exedir, testexe, + self.strace, args, self.variables, self.defines) + okay, skip = runner.init(args.systype) + if not okay: + if not skip: + result = False + else: + if testexe: + print('running ' + testexe) + if not runner.run(): + result = False + if result: + print('comparing:') + print(' ' + simplify_path(self.disttop, runfilename)) + print(' ' + simplify_path(self.disttop, runner.errfilename)) + result = runner.match_lines() + if not result and args.verbose: + printfile(runfilename, "runfile") + printfile(runner.errfilename, "trace output") + runner.close(not result) + if not result: + print('************************ FAILED ************************') + print(' see results in ' + exedir) + print('') + return result + + # Create a C program to get values for all defines we need. + # The output of the program is Python code that we'll execute + # directly to set the values. + def build_system_defines(self): + # variables is a symbol table that is used to + # evaluate expressions both in the .run file and + # in the output file. This is needed for strace, + # which shows system call flags in symbolic form. + self.variables = VariableContext() + # defines is a symbol table that is used to + # create preprocessor defines, effectively evaluating + # all flag defines in the .run file. + self.defines = VariableContext() + program = \ + '#include <stdio.h>\n' + \ + '#include <fcntl.h>\n' + \ + '#include <wt_internal.h>\n' + \ + 'int main() {\n' + for define in defines_used: + program += '#ifdef ' + define + '\n' + # output is Python that sets attributes of 'o'. + program += ' printf("o.' + define + '=%d\\n", ' + \ + define + ');\n' + program += '#endif\n' + program += \ + ' return(0);\n' + \ + '}\n' + probe_c = os.path.join(self.exetopdir, "syscall_probe.c") + probe_exe = os.path.join(self.exetopdir, "syscall_probe") + with open(probe_c, "w") as f: + f.write(program) + ccargs = ['cc', '-o', probe_exe] + ccargs.append('-I' + self.incdir1) + ccargs.append('-I' + self.incdir2) + if self.args.systype == 'Linux': + ccargs.append('-D_GNU_SOURCE') + ccargs.append(probe_c) + subret = subprocess.call(ccargs) + if subret != 0: + msg("probe compilation returned " + str(subret)) + return False + proc = subprocess.Popen([probe_exe], stdout=subprocess.PIPE) + out, err = proc.communicate() + subret = proc.returncode + if subret != 0 or err: + msg("probe run returned " + str(subret) + ", error=" + str(err)) + return False + if self.args.verbose: + print('probe output:\n' + out) + o = self.defines # The 'o' object will be modified. + exec(out) # Run the produced Python. + o = self.variables # Set these in variables too, so strace + exec(out) # symbolic output is evaluated. + if not self.args.preserve: + os.remove(probe_c) + os.remove(probe_exe) + return True + + def execute(self): + args = self.args + result = True + if not self.build_system_defines(): + die('cannot build system defines') + if not self.dorun: + for testname in args.tests: + result = self.runone(testname, None, None, args) and result + else: + if len(args.tests) > 0: + tests = [] + for arg in args.tests: + abspath = os.path.abspath(arg) + tests.append([os.path.dirname(abspath), [], + [os.path.basename(abspath)]]) + else: + tests = os.walk(syscalldir) + os.chdir(self.exetopdir) + for path, subdirs, files in tests: + testnum = -1 if len(files) <= 1 else 0 + for name in files: + if fnmatch.fnmatch(name, '*.run'): + testname = os.path.basename(os.path.normpath(path)) + runfilename = os.path.join(path, name) + testexe = os.path.join(self.exetopdir, + 'test_' + testname) + exedir = os.path.join(self.exetopdir, + 'WT_TEST.' + testname) + # If there are multiple tests in this directory, + # give each one its own execution dir. + if testnum >= 0: + exedir += '.' + str(testnum) + testnum += 1 + result = self.runone(runfilename, exedir, + testexe, args) and result + return result + +# Set paths, determining the top of the build. +syscalldir = sys.path[0] +wt_disttop = os.path.dirname(os.path.dirname(syscalldir)) + +# Note: this code is borrowed from test/suite/run.py +# Check for a local build that contains the wt utility. First check in +# current working directory, then in build_posix and finally in the disttop +# directory. This isn't ideal - if a user has multiple builds in a tree we +# could pick the wrong one. +if os.path.isfile(os.path.join(os.getcwd(), 'wt')): + wt_builddir = os.getcwd() +elif os.path.isfile(os.path.join(wt_disttop, 'wt')): + wt_builddir = wt_disttop +elif os.path.isfile(os.path.join(wt_disttop, 'build_posix', 'wt')): + wt_builddir = os.path.join(wt_disttop, 'build_posix') +elif os.path.isfile(os.path.join(wt_disttop, 'wt.exe')): + wt_builddir = wt_disttop +else: + die('unable to find useable WiredTiger build') + +cmd = SyscallCommand(wt_disttop, wt_builddir) +if not cmd.parse_args(sys.argv): + die('bad usage') +if not cmd.execute(): + sys.exit(1) +sys.exit(0) diff --git a/src/third_party/wiredtiger/test/syscall/wt2336_base/base.run b/src/third_party/wiredtiger/test/syscall/wt2336_base/base.run new file mode 100644 index 00000000000..7d2c42ce64e --- /dev/null +++ b/src/third_party/wiredtiger/test/syscall/wt2336_base/base.run @@ -0,0 +1,180 @@ +// Public Domain 2014-2016 MongoDB, Inc. +// Public Domain 2008-2014 WiredTiger, Inc. +// +// This is free and unencumbered software released into the public domain. +// +// Anyone is free to copy, modify, publish, use, compile, sell, or +// distribute this software, either in source code form or as a compiled +// binary, for any purpose, commercial or non-commercial, and by any +// means. +// +// In jurisdictions that recognize copyright laws, the author or authors +// of this software dedicate any and all copyright interest in the +// software to the public domain. We make this dedication for the benefit +// of the public at large and to the detriment of our heirs and +// successors. We intend this dedication to be an overt act of +// relinquishment in perpetuity of all present and future rights to this +// software under copyright law. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +// IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// +// base.run +// Command line syscall test runner +// +#ifdef __linux__ +SYSTEM("Linux"); +#define OPEN_EXISTING(name, flags) open(name, flags) +#else /* __linux__ */ +SYSTEM("Darwin"); +#define O_NOATIME 0 +#define OPEN_EXISTING(name, flags) open(name, flags, 0) +#endif /* __linux__ */ + +#ifdef HAVE_FTRUNCATE +#ifdef __linux__ +#define FTRUNCATE(fd, len) ftruncate(fd, len) +#else +#define FTRUNCATE(fd, len) ftruncate(fd, len, len) +#endif +#else +#define FTRUNCATE(fd, len) /* do nothing */ +#endif + +TRACE("close,fdatasync,fsync,ftruncate,open,pwrite64,rename"); +RUN(""); +... +OUTPUT("--------------wiredtiger_open"); +// lock == 3 +lock = open("./WiredTiger.lock", O_RDWR|O_CREAT|O_CLOEXEC, 0666); +pwrite64(lock, "WiredTiger lock file\n", 0x15, 0x0); +fd = open("./WiredTiger", O_RDWR|O_CREAT|O_CLOEXEC, 0666); +pwrite64(fd, "WiredTiger\nWiredTiger"..., ...); +#ifdef __linux__ +fdatasync(fd); +#endif /* __linux__ */ +close(fd); + +... // On Linux, there are calls to open and read "/proc/meminfo" here. + +fd = open("./WiredTiger.basecfg.set", O_RDWR|O_CREAT|O_EXCL|O_CLOEXEC, 0666); +pwrite64(fd, "# Do not modify this file."..., ...); +#ifdef __linux__ +fdatasync(fd); +#endif /* __linux__ */ +close(fd); +rename("./WiredTiger.basecfg.set", "./WiredTiger.basecfg"); + +#ifdef __linux__ +dir = open("./", O_RDONLY); +fdatasync(dir); +close(dir); +#endif + +fd = open("./WiredTiger.wt", O_RDWR|O_CREAT|O_EXCL|O_NOATIME|O_CLOEXEC, 0666); + +#ifdef __linux__ +dir = open("./", O_RDONLY); +fdatasync(dir); +close(dir); +#endif /* __linux__ */ + +pwrite64(fd, ""..., 0x1000, 0x0); +#ifdef __linux__ +fdatasync(fd); +#endif /* __linux__ */ +close(fd); + +wt = OPEN_EXISTING("./WiredTiger.wt\0", O_RDWR|O_NOATIME|O_CLOEXEC); +FTRUNCATE(wt, 0x1000); + +fd = open("./WiredTiger.turtle.set\0", O_RDWR|O_CREAT|O_EXCL|O_CLOEXEC, 0666); +pwrite64(fd, "WiredTiger version string\nWiredTiger"..., ...); +#ifdef __linux__ +fdatasync(fd); +#endif /* __linux__ */ +close(fd); +rename("./WiredTiger.turtle.set", "./WiredTiger.turtle"); + +... // There is a second open of turtle here, is it important? + +fd = open("./WiredTigerLAS.wt", O_RDWR|O_CREAT|O_EXCL|O_NOATIME|O_CLOEXEC, 0666); + +#ifdef __linux__ +dir = open("./", O_RDONLY); +fdatasync(dir); +close(dir); +#endif /* __linux__ */ + +pwrite64(fd, ""..., 0x1000, 0x0); + +#ifdef __linux__ +fdatasync(fd); +#endif /* __linux__ */ + +close(fd); +fd = OPEN_EXISTING("./WiredTigerLAS.wt", O_RDWR|O_NOATIME|O_CLOEXEC); +FTRUNCATE(fd, 0x1000); +fd = OPEN_EXISTING("./WiredTiger.turtle", O_RDWR|O_CLOEXEC); +close(fd); +pwrite64(wt, ""..., 0x1000, 0x1000); +pwrite64(wt, ""..., 0x1000, 0x2000); +pwrite64(wt, ""..., 0x1000, 0x3000); +#ifdef __linux__ +fdatasync(wt); +#endif /* __linux__ */ +fd = OPEN_EXISTING("./WiredTiger.turtle", O_RDWR|O_CLOEXEC); + +close(fd); +fd = open("./WiredTiger.turtle.set", O_RDWR|O_CREAT|O_EXCL|O_CLOEXEC, 0666); +pwrite64(fd, "WiredTiger version string\nWiredTiger"..., ...); +#ifdef __linux__ +fdatasync(fd); +#endif /* __linux__ */ +close(fd); +rename("./WiredTiger.turtle.set", "./WiredTiger.turtle"); +#ifdef __linux__ +dir = open("./", O_RDONLY); +fdatasync(dir); +close(dir); +fdatasync(wt); +#endif /* __linux__ */ + +OUTPUT("--------------open_session"); +OUTPUT("--------------create"); + +hello = open("./hello.wt", O_RDWR|O_CREAT|O_EXCL|O_NOATIME|O_CLOEXEC, 0666); +#ifdef __linux__ +dir = open("./", O_RDONLY); +fdatasync(dir); +close(dir); +#endif /* __linux__ */ +pwrite64(hello, "A\330\001"..., 0x1000, 0x0); +#ifdef __linux__ +fdatasync(hello); +#endif /* __linux__ */ +close(hello); +hello = OPEN_EXISTING("./hello.wt", O_RDWR|O_NOATIME|O_CLOEXEC); +FTRUNCATE(hello, 0x1000); +turtle = OPEN_EXISTING("./WiredTiger.turtle", O_RDWR|O_CLOEXEC); +close(turtle); +... // more changes to turtle +OUTPUT("--------------rename"); +#ifdef __linux__ +FTRUNCATE(hello, 0x1000); +#endif /* __linux__ */ +close(hello); +rename("./hello.wt", "./world.wt"); +// Changes to turtle, more writes to wt. +turtle = OPEN_EXISTING("./WiredTiger.turtle", O_RDWR|O_CLOEXEC); +close(turtle); +... +OUTPUT("--------------drop"); +... +OUTPUT("--------------WT_CONNECTION::close"); +... diff --git a/src/third_party/wiredtiger/test/syscall/wt2336_base/main.c b/src/third_party/wiredtiger/test/syscall/wt2336_base/main.c new file mode 100644 index 00000000000..22420371dd0 --- /dev/null +++ b/src/third_party/wiredtiger/test/syscall/wt2336_base/main.c @@ -0,0 +1,69 @@ +#include <stdlib.h> + +#include <unistd.h> // TODO +#include <fcntl.h> // TODO +#include <wt_internal.h> + +static void fail(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); + +static void +fail(int ret) { + fprintf(stderr, + "%s: %d (%s)\n", + "wt2336_fileop_basic", ret, wiredtiger_strerror(ret)); + exit(ret); +} + +#define SEPARATOR "--------------" + +int +main(int argc, char *argv[]) +{ + int ret; + WT_CONNECTION *conn; + WT_SESSION *session; + + (void)argc; + (void)argv; + fprintf(stderr, SEPARATOR "wiredtiger_open\n"); + if ((ret = wiredtiger_open(".", NULL, "create", &conn)) != 0) + fail(ret); + + usleep(100); + fflush(stderr); + fprintf(stderr, SEPARATOR "open_session\n"); + fflush(stderr); + + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + fail(ret); + + usleep(100); + fflush(stderr); + fprintf(stderr, SEPARATOR "create\n"); + fflush(stderr); + + if ((ret = session->create( + session, "table:hello", "key_format=S,value_format=S")) != 0) + fail(ret); + + usleep(100); + fprintf(stderr, SEPARATOR "rename\n"); + + if ((ret = session->rename( + session, "table:hello", "table:world", NULL)) != 0) + fail(ret); + + fflush(stdout); + fprintf(stderr, SEPARATOR "drop\n"); + fflush(stdout); + + if ((ret = session->drop(session, "table:world", NULL)) != 0) + fail(ret); + + fprintf(stderr, SEPARATOR "WT_CONNECTION::close\n"); + + if ((ret = conn->close(conn, NULL)) != 0) + fail(ret); + + return (0); +} diff --git a/src/third_party/wiredtiger/test/windows/windows_shim.h b/src/third_party/wiredtiger/test/windows/windows_shim.h index a6050e92bca..648b991b1a2 100644 --- a/src/third_party/wiredtiger/test/windows/windows_shim.h +++ b/src/third_party/wiredtiger/test/windows/windows_shim.h @@ -87,7 +87,7 @@ int usleep(useconds_t useconds); /* - * Emulate the <pthread.h> support we need for the tests + * Emulate the <pthread.h> support we need for tests and example code. */ typedef CRITICAL_SECTION pthread_mutex_t; typedef CONDITION_VARIABLE pthread_cond_t; |