diff options
Diffstat (limited to 'src/third_party/wiredtiger/test')
112 files changed, 1036 insertions, 734 deletions
diff --git a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c index d08c7f69695..493cdaf5114 100644 --- a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c +++ b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c @@ -33,7 +33,7 @@ static int compare_cursors( WT_CURSOR *, const char *, WT_CURSOR *, const char *); static int diagnose_key_error(WT_CURSOR *, int, WT_CURSOR *, int); static int real_checkpointer(void); -static int verify_checkpoint(WT_SESSION *); +static int verify_consistency(WT_SESSION *, bool); /* * start_checkpoints -- @@ -105,6 +105,11 @@ real_checkpointer(void) } while (g.running) { + /* Check for consistency of online data */ + if ((ret = verify_consistency(session, false)) != 0) + return (log_print_err( + "verify_consistency (online)", ret, 1)); + /* Execute a checkpoint */ if ((ret = session->checkpoint( session, checkpoint_config)) != 0) @@ -115,8 +120,9 @@ real_checkpointer(void) goto done; /* Verify the content of the checkpoint. */ - if ((ret = verify_checkpoint(session)) != 0) - return (log_print_err("verify_checkpoint", ret, 1)); + if ((ret = verify_consistency(session, true)) != 0) + return (log_print_err( + "verify_consistency (offline)", ret, 1)); } done: if ((ret = session->close(session, NULL)) != 0) @@ -126,40 +132,48 @@ done: if ((ret = session->close(session, NULL)) != 0) } /* - * verify_checkpoint -- + * verify_consistency -- * Open a cursor on each table at the last checkpoint and walk through * the tables in parallel. The key/values should match across all tables. */ static int -verify_checkpoint(WT_SESSION *session) +verify_consistency(WT_SESSION *session, bool use_checkpoint) { WT_CURSOR **cursors; uint64_t key_count; int i, ret, t_ret; - char ckpt[128], next_uri[128]; - const char *type0, *typei; + const char *ckpt, *type0, *typei; + char ckpt_buf[128], next_uri[128]; ret = t_ret = 0; key_count = 0; - testutil_check(__wt_snprintf( - ckpt, sizeof(ckpt), "checkpoint=%s", g.checkpoint_name)); cursors = calloc((size_t)g.ntables, sizeof(*cursors)); if (cursors == NULL) - return (log_print_err("verify_checkpoint", ENOMEM, 1)); + return (log_print_err("verify_consistency", ENOMEM, 1)); + + if (use_checkpoint) { + testutil_check(__wt_snprintf(ckpt_buf, sizeof(ckpt_buf), + "checkpoint=%s", g.checkpoint_name)); + ckpt = ckpt_buf; + } else { + ckpt = NULL; + testutil_check(session->begin_transaction( + session, "isolation=snapshot")); + } for (i = 0; i < g.ntables; i++) { /* * TODO: LSM doesn't currently support reading from * checkpoints. */ - if (g.cookies[i].type == LSM) + if (use_checkpoint && g.cookies[i].type == LSM) continue; testutil_check(__wt_snprintf( next_uri, sizeof(next_uri), "table:__wt%04d", i)); if ((ret = session->open_cursor( session, next_uri, NULL, ckpt, &cursors[i])) != 0) { (void)log_print_err( - "verify_checkpoint:session.open_cursor", ret, 1); + "verify_consistency:session.open_cursor", ret, 1); goto err; } } @@ -199,7 +213,7 @@ verify_checkpoint(WT_SESSION *session) continue; else if (ret == WT_NOTFOUND || t_ret == WT_NOTFOUND) { (void)log_print_err( - "verify_checkpoint tables with different" + "verify_consistency tables with different" " amount of data", EFAULT, 1); goto err; } @@ -211,21 +225,24 @@ verify_checkpoint(WT_SESSION *session) (void)diagnose_key_error( cursors[0], 0, cursors[i], i); (void)log_print_err( - "verify_checkpoint - mismatching data", + "verify_consistency - mismatching data", EFAULT, 1); goto err; } } } - printf("Finished verifying a checkpoint with %d tables and %" PRIu64 - " keys\n", g.ntables, key_count); + printf("Finished verifying a %s with %d tables and %" PRIu64 + " keys\n", use_checkpoint ? "checkpoint" : "snapshot", + g.ntables, key_count); err: for (i = 0; i < g.ntables; i++) { if (cursors[i] != NULL && (ret = cursors[i]->close(cursors[i])) != 0) (void)log_print_err( - "verify_checkpoint:cursor close", ret, 1); + "verify_consistency:cursor close", ret, 1); } + if (!use_checkpoint) + testutil_check(session->commit_transaction(session, NULL)); free(cursors); return (ret); } diff --git a/src/third_party/wiredtiger/test/checkpoint/workers.c b/src/third_party/wiredtiger/test/checkpoint/workers.c index ca1bca04528..33836c67110 100644 --- a/src/third_party/wiredtiger/test/checkpoint/workers.c +++ b/src/third_party/wiredtiger/test/checkpoint/workers.c @@ -126,13 +126,22 @@ worker_op(WT_CURSOR *cursor, uint64_t keyno, u_int new_val) char valuebuf[64]; cursor->set_key(cursor, keyno); - testutil_check(__wt_snprintf( - valuebuf, sizeof(valuebuf), "%037u", new_val)); - cursor->set_value(cursor, valuebuf); - if ((ret = cursor->insert(cursor)) != 0) { - if (ret == WT_ROLLBACK) - return (WT_ROLLBACK); - return (log_print_err("cursor.insert", ret, 1)); + /* Roughly 5% removes. */ + if (new_val % 19 == 0) { + if ((ret = cursor->remove(cursor)) != 0) { + if (ret == WT_ROLLBACK) + return (WT_ROLLBACK); + return (log_print_err("cursor.remove", ret, 1)); + } + } else { + testutil_check(__wt_snprintf( + valuebuf, sizeof(valuebuf), "%037u", new_val)); + cursor->set_value(cursor, valuebuf); + if ((ret = cursor->insert(cursor)) != 0) { + if (ret == WT_ROLLBACK) + return (WT_ROLLBACK); + return (log_print_err("cursor.insert", ret, 1)); + } } return (0); } diff --git a/src/third_party/wiredtiger/test/csuite/Makefile.am b/src/third_party/wiredtiger/test/csuite/Makefile.am index 362d0775a88..53175472c1e 100644 --- a/src/third_party/wiredtiger/test/csuite/Makefile.am +++ b/src/third_party/wiredtiger/test/csuite/Makefile.am @@ -127,6 +127,10 @@ test_wt4333_handle_locks_SOURCES = wt4333_handle_locks/main.c noinst_PROGRAMS += test_wt4333_handle_locks all_TESTS += test_wt4333_handle_locks +test_wt4699_json_SOURCES = wt4699_json/main.c +noinst_PROGRAMS += test_wt4699_json +all_TESTS += test_wt4699_json + # Run this during a "make check" smoke test. TESTS = $(all_TESTS) LOG_COMPILER = $(TEST_WRAPPER) diff --git a/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c b/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c index fd734b1a4a2..32b9f8f42a8 100644 --- a/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c @@ -30,16 +30,11 @@ #include <sys/wait.h> #include <signal.h> -#define CKPT_DISTANCE 1 #define CORRUPT "file:zzz-corrupt.SS" #define KEY "key" #define VALUE "value,value,value" -#define DB0 "CKPT0" -#define DB1 "CKPT1" -#define DB2 "CKPT2" #define SAVE "SAVE" -#define TEST "TEST" /* * NOTE: This assumes the default page size of 4096. If that changes these @@ -47,7 +42,7 @@ */ #define APP_MD_SIZE 4096 #define APP_BUF_SIZE (3 * 1024) -#define APP_STR "long app metadata. " +#define APP_STR "Long app metadata intended to force a page per entry. " static uint64_t data_val; static const char *home; @@ -363,59 +358,6 @@ copy_database(const char *sfx) } /* - * move_data_ahead -- - * Update the tables with new data and take a checkpoint twice. - * WiredTiger keeps the previous checkpoint so we do it twice so that - * the old checkpoint address no longer exists. - */ -static void -move_data_ahead(TABLE_INFO *table_data) -{ - TABLE_INFO *t; - uint64_t i; - - i = 0; - while (i < CKPT_DISTANCE) { - ++data_val; - for (t = table_data; t->name != NULL; t++) - cursor_insert(t->name, data_val); - ++i; - fprintf(stderr, "MOVE DATA: inserted %" PRIu64 ". CKPT.\n", - data_val); - testutil_check(wt_session->checkpoint(wt_session, NULL)); - } -} - -/* - * make_database_copies -- - * Make copies of the database so that we can test various mix and match - * of turtle files and metadata files. We take some checkpoints and - * update the data too. - */ -static void -make_database_copies(TABLE_INFO *table_data) -{ - /* - * If we're running an out-of-sync test, then we want to make copies - * of the turtle and metadata file, then checkpoint and again save a - * copy of the turtle file and the metadata file. Then we add more data - * and checkpoint again at least twice. Using the original and current - * files we can test various out of sync scenarios. - */ - /* - * Take a checkpoint and make a copy. - */ - testutil_check(wt_session->checkpoint(wt_session, NULL)); - copy_database(DB0); - - move_data_ahead(table_data); - copy_database(DB1); - - move_data_ahead(table_data); - copy_database(DB2); -} - -/* * wt_open_corrupt -- * Call wiredtiger_open and expect a corruption error. */ @@ -534,125 +476,6 @@ run_all_verification(const char *sfx, TABLE_INFO *t) open_normal(sfx, t); } -static void -setup_database(const char *src, const char *turtle_dir, const char *meta_dir) -{ - WT_DECL_RET; - char buf[1024]; - - /* - * Remove the test home directory and copy the source to it. - * Then copy the saved turtle and/or metadata file from the - * given args. - */ - testutil_check(__wt_snprintf(buf, sizeof(buf), - "rm -rf ./%s.%s; mkdir ./%s.%s; " - "cp -p %s.%s/* ./%s.%s", - home, TEST, home, TEST, home, src, home, TEST)); - printf("copy: %s\n", buf); - if ((ret = system(buf)) < 0) - testutil_die(ret, "system: %s", buf); - - /* Copy turtle if given. */ - if (turtle_dir != NULL) { - testutil_check(__wt_snprintf(buf, sizeof(buf), - "cp -p %s.%s/%s.%s %s.%s/%s", - home, turtle_dir, WT_METADATA_TURTLE, SAVE, - home, TEST, WT_METADATA_TURTLE)); - printf("copy: %s\n", buf); - if ((ret = system(buf)) < 0) - testutil_die(ret, "system: %s", buf); - } - /* Copy metadata if given. */ - if (meta_dir != NULL) { - testutil_check(__wt_snprintf(buf, sizeof(buf), - "cp -p %s.%s/%s.%s %s.%s/%s", - home, meta_dir, WT_METAFILE, SAVE, - home, TEST, WT_METAFILE)); - printf("copy: %s\n", buf); - if ((ret = system(buf)) < 0) - testutil_die(ret, "system: %s", buf); - } -} - -static void -out_of_sync(TABLE_INFO *table_data) -{ - /* - * We have five directories: - * - The main database directory that we just corrupted/salvaged. - * - A .SAVE copy of the main directory that is coherent prior to - * corrupting. Essentially a copy of the second checkpoint dir. - * - A copy of the main directory before the first checkpoint. DB0 - * - A copy of the main directory after the first checkpoint. DB1 - * - A copy of the main directory after the second checkpoint. DB2 - * - * We want to make a copy of a source directory and then copy a - * turtle or metadata file from another directory. Then detect the - * error, run with salvage and confirm. - */ - /* - * Run in DB0, bring in future metadata from DB1. - */ - test_out_of_sync = true; - printf( - "#\n# OUT OF SYNC: %s with future metadata from %s\n#\n", DB0, DB1); - setup_database(DB0, NULL, DB1); - run_all_verification(TEST, table_data); - - /* - * Run in DB0, bring in future turtle file from DB1. - */ - printf( - "#\n# OUT OF SYNC: %s with future turtle from %s\n#\n", DB0, DB1); - setup_database(DB0, DB1, NULL); - run_all_verification(TEST, table_data); - - /* - * Run in DB1, bring in old metadata file from DB0. - */ - printf("#\n# OUT OF SYNC: %s with old metadata from %s\n#\n", DB1, DB0); - setup_database(DB1, NULL, DB0); - run_all_verification(TEST, table_data); - - /* - * Run in DB1, bring in old turtle file from DB0. - */ - printf("#\n# OUT OF SYNC: %s with old turtle from %s\n#\n", DB1, DB0); - setup_database(DB1, DB0, NULL); - run_all_verification(TEST, table_data); - - /* - * Run in DB1, bring in future metadata file from DB2. - */ - printf( - "#\n# OUT OF SYNC: %s with future metadata from %s\n#\n", DB1, DB2); - setup_database(DB1, NULL, DB2); - run_all_verification(TEST, table_data); - - /* - * Run in DB1, bring in future turtle file from DB2. - */ - printf( - "#\n# OUT OF SYNC: %s with future turtle from %s\n#\n", DB1, DB2); - setup_database(DB1, DB2, NULL); - run_all_verification(TEST, table_data); - - /* - * Run in DB2, bring in old metadata file from DB1. - */ - printf("#\n# OUT OF SYNC: %s with old metadata from %s\n#\n", DB2, DB1); - setup_database(DB2, NULL, DB1); - run_all_verification(TEST, table_data); - - /* - * Run in DB2, bring in old turtle file from DB1. - */ - printf("#\n# OUT OF SYNC: %s with old turtle from %s\n#\n", DB2, DB1); - setup_database(DB2, DB1, NULL); - run_all_verification(TEST, table_data); -} - int main(int argc, char *argv[]) { @@ -700,10 +523,6 @@ main(int argc, char *argv[]) for (t = table_data; t->name != NULL; t++) create_data(t); - /* - * Take some checkpoints and add more data for out of sync testing. - */ - make_database_copies(table_data); testutil_check(opts->conn->close(opts->conn, NULL)); opts->conn = NULL; @@ -737,8 +556,6 @@ main(int argc, char *argv[]) testutil_die(ret, "system: %s", buf); run_all_verification(NULL, &table_data[0]); - out_of_sync(&table_data[0]); - /* * We need to set up the string before we clean up * the structure. Then after the clean up we will diff --git a/src/third_party/wiredtiger/test/csuite/wt4699_json/main.c b/src/third_party/wiredtiger/test/csuite/wt4699_json/main.c new file mode 100644 index 00000000000..636798c0ffd --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/wt4699_json/main.c @@ -0,0 +1,97 @@ +/*- + * Public Domain 2014-2019 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +/* + * JIRA ticket reference: WT-4699 + * Test case description: Use a JSON dump cursor on a projection, + * and overwrite the projection string. + * Failure mode: On the first retrieval of a JSON key/value, a configure + * parse error is returned. + */ + +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + WT_CURSOR *c; + WT_SESSION *session; + char *jsonkey, *jsonvalue; + char projection[1000]; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check(wiredtiger_open(opts->home, NULL, + "create", &opts->conn)); + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + /* Create a single record in a table with two fields in its value. */ + testutil_check(session->create(session, opts->uri, + "key_format=i,value_format=ii,columns=(k,v0,v1)")); + testutil_check( + session->open_cursor(session, opts->uri, NULL, NULL, &c)); + c->set_key(c, 1); + c->set_value(c, 1, 1); + testutil_check(c->insert(c)); + testutil_check(c->close(c)); + + /* + * Open a dump JSON cursor on a projection of the table. + * The fields will be listed in a different order. + */ + strcpy(projection, opts->uri); + strcat(projection, "(v1,v0,k)"); + testutil_check( + session->open_cursor(session, projection, NULL, "dump=json", &c)); + testutil_check(c->next(c)); + /* Overwrite the projection, with not enough columns */ + strcpy(projection, opts->uri); + strcat(projection, "(aaa,bbb)"); + testutil_check(c->get_key(c, &jsonkey)); + + /* + * Here's where we would get the parse error. + * When a JSON dump is performed on a projection, we need to format + * all the field names and values in the order listed. + * The implementation uses the projection string from the + * open_cursor call to determine the field names. + */ + testutil_check(c->get_value(c, &jsonvalue)); + testutil_assert(strstr(jsonvalue, "aaa") == NULL); + printf("KEY: %s\n", jsonkey); + printf("VALUE: %s\n", jsonvalue); + testutil_assert(c->next(c) == WT_NOTFOUND); + testutil_check(c->close(c)); + testutil_check(session->close(session, NULL)); + testutil_cleanup(opts); + return (EXIT_SUCCESS); +} diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml index 00869b45294..56010762762 100644 --- a/src/third_party/wiredtiger/test/evergreen.yml +++ b/src/third_party/wiredtiger/test/evergreen.yml @@ -658,6 +658,20 @@ tasks: ${test_env_vars|} $(pwd)/test_wt4156_metadata_salvage 2>&1 + - name: csuite-wt4699-json-test + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - command: shell.exec + params: + working_dir: "wiredtiger/build_posix/test/csuite" + script: | + set -o errexit + set -o verbose + + ${test_env_vars|} $(pwd)/test_wt4699_json 2>&1 + - name: csuite-rwlock-test depends_on: - name: compile @@ -1171,6 +1185,7 @@ buildvariants: - name: csuite-wt4105-large-doc-small-upd-test - name: csuite-wt4117-checksum-test - name: csuite-wt4156-metadata-salvage-test + - name: csuite-wt4699-json-test - name: csuite-rwlock-test - name: csuite-wt2246-col-append-test - name: csuite-wt2323-join-visibility-test diff --git a/src/third_party/wiredtiger/test/suite/helper.py b/src/third_party/wiredtiger/test/suite/helper.py index 0b5db24f0d9..72801cc18a4 100644..100755 --- a/src/third_party/wiredtiger/test/suite/helper.py +++ b/src/third_party/wiredtiger/test/suite/helper.py @@ -39,9 +39,9 @@ def compare_files(self, filename1, filename2): self.pr('compare_files: ' + filename1 + ', ' + filename2) bufsize = 4096 if os.path.getsize(filename1) != os.path.getsize(filename2): - print 'file comparison failed: ' + filename1 + ' size ' +\ + print('file comparison failed: ' + filename1 + ' size ' +\ str(os.path.getsize(filename1)) + ' != ' + filename2 +\ - ' size ' + str(os.path.getsize(filename2)) + ' size ' + str(os.path.getsize(filename2))) return False with open(filename1, "rb") as fp1: with open(filename2, "rb") as fp2: diff --git a/src/third_party/wiredtiger/test/suite/run.py b/src/third_party/wiredtiger/test/suite/run.py index b82a77fe97c..5895325a3d7 100755 --- a/src/third_party/wiredtiger/test/suite/run.py +++ b/src/third_party/wiredtiger/test/suite/run.py @@ -30,8 +30,14 @@ # Command line test runner # +from __future__ import print_function import glob, json, os, re, sys +try: + xrange +except NameError: #python3 + xrange = range + # Set paths suitedir = sys.path[0] wt_disttop = os.path.dirname(os.path.dirname(suitedir)) @@ -56,14 +62,14 @@ elif os.path.isfile(os.path.join(wt_disttop, 'build_posix', 'wt')): elif os.path.isfile(os.path.join(wt_disttop, 'wt.exe')): wt_builddir = wt_disttop else: - print 'Unable to find useable WiredTiger build' + print('Unable to find useable WiredTiger build') sys.exit(1) # Cannot import wiredtiger and supporting utils until we set up paths # We want our local tree in front of any installed versions of WiredTiger. # Don't change sys.path[0], it's the dir containing the invoked python script. + sys.path.insert(1, os.path.join(wt_builddir, 'lang', 'python')) -sys.path.insert(1, os.path.join(wt_disttop, 'lang', 'python')) # Append to a colon separated path in the environment def append_env_path(name, value): @@ -95,7 +101,7 @@ unittest = wttest.unittest from testscenarios.scenarios import generate_scenarios def usage(): - print 'Usage:\n\ + print('Usage:\n\ $ cd build_posix\n\ $ python ../test/suite/run.py [ options ] [ tests ]\n\ \n\ @@ -122,7 +128,7 @@ Tests:\n\ \n\ When -C or -c are present, there may not be any tests named.\n\ When -s is present, there must be a test named.\n\ -' +') # capture the category (AKA 'subsuite') part of a test name, # e.g. test_util03 -> util @@ -342,7 +348,7 @@ if __name__ == '__main__': configfile = args.pop(0) configwrite = True continue - print 'unknown arg: ' + arg + print('unknown arg: ' + arg) usage() sys.exit(2) testargs.append(arg) @@ -379,7 +385,7 @@ if __name__ == '__main__': for test in tests: dryOutput.add(test.shortDesc()) for line in dryOutput: - print line + print(line) else: result = wttest.runsuite(tests, parallel) sys.exit(0 if result.wasSuccessful() else 1) diff --git a/src/third_party/wiredtiger/test/suite/suite_subprocess.py b/src/third_party/wiredtiger/test/suite/suite_subprocess.py index 853a5cde091..d04a281807a 100755 --- a/src/third_party/wiredtiger/test/suite/suite_subprocess.py +++ b/src/third_party/wiredtiger/test/suite/suite_subprocess.py @@ -26,6 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. +from __future__ import print_function import os, re, subprocess, sys from run import wt_builddir from wttest import WiredTigerTestCase @@ -69,14 +70,14 @@ class suite_subprocess: lines.pop(0) hasPrevious = True if hasError: - print '**************** ' + match + ' in output file: ' + filename + ' ****************' + print('**************** ' + match + ' in output file: ' + filename + ' ****************') if hasPrevious: - print '...' + print('...') for line in lines: - print line, + print(line, end=' ') if hasNext: - print '...' - print '********************************' + print('...') + print('********************************') self.fail('ERROR found in output file: ' + filename) # If the string is of the form '/.../', then return just the embedded @@ -133,8 +134,8 @@ class suite_subprocess: if filesize > 0: with open(filename, 'r') as f: contents = f.read(1000) - print 'ERROR: ' + filename + ' expected to be empty, but contains:\n' - print contents + '...\n' + print('ERROR: ' + filename + ' expected to be empty, but contains:\n') + print(contents + '...\n') self.assertEqual(filesize, 0, filename + ': expected to be empty') def check_non_empty_file(self, filename): @@ -143,7 +144,7 @@ class suite_subprocess: """ filesize = os.path.getsize(filename) if filesize == 0: - print 'ERROR: ' + filename + ' should not be empty (this command expected error output)' + print('ERROR: ' + filename + ' should not be empty (this command expected error output)') self.assertNotEqual(filesize, 0, filename + ': expected to not be empty') def verbose_env(self, envvar): @@ -202,23 +203,23 @@ class suite_subprocess: infilepart = "" if infilename != None: infilepart = "<" + infilename + " " - print str(procargs) - print "*********************************************" - print "**** Run 'wt' via: run " + \ + print(str(procargs)) + print("*********************************************") + print("**** Run 'wt' via: run " + \ " ".join(procargs[3:]) + infilepart + \ - ">" + wtoutname + " 2>" + wterrname - print "*********************************************" + ">" + wtoutname + " 2>" + wterrname) + print("*********************************************") returncode = subprocess.call(procargs) elif self._lldbSubprocess: infilepart = "" if infilename != None: infilepart = "<" + infilename + " " - print str(procargs) - print "*********************************************" - print "**** Run 'wt' via: run " + \ + print(str(procargs)) + print("*********************************************") + print("**** Run 'wt' via: run " + \ " ".join(procargs[3:]) + infilepart + \ - ">" + wtoutname + " 2>" + wterrname - print "*********************************************" + ">" + wtoutname + " 2>" + wterrname) + print("*********************************************") returncode = subprocess.call(procargs) elif infilename: with open(infilename, "r") as wtin: diff --git a/src/third_party/wiredtiger/test/suite/test_alter02.py b/src/third_party/wiredtiger/test/suite/test_alter02.py index 449247b6988..7a135d99b74 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_alter02.py +++ b/src/third_party/wiredtiger/test/suite/test_alter02.py @@ -78,7 +78,7 @@ class test_alter02(wttest.WiredTigerTestCase): try: self.conn = wiredtiger.wiredtiger_open(self.home, conn_params) except wiredtiger.WiredTigerError as e: - print "Failed conn at '%s' with config '%s'" % (dir, conn_params) + print("Failed conn at '%s' with config '%s'" % (dir, conn_params)) self.session = self.conn.open_session() # Verify the metadata string for this URI and that its setting in the @@ -116,12 +116,12 @@ class test_alter02(wttest.WiredTigerTestCase): keys = c.get_key() # txnid, rectype, optype, fileid, logrec_key, logrec_value values = c.get_value() - try: - if self.value in str(values[5]): # logrec_value + # We are only looking for log records that that have a key/value + # pair. + if values[4] != b'': + if self.value.encode() in values[5]: # logrec_value count += 1 - self.assertFalse(value2 in str(values[5])) - except: - pass + self.assertFalse(self.value2.encode() in values[5]) c.close() self.assertEqual(count, expected_keys) diff --git a/src/third_party/wiredtiger/test/suite/test_async01.py b/src/third_party/wiredtiger/test/suite/test_async01.py index 5b13bf5ba5c..fc50327aab1 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_async01.py +++ b/src/third_party/wiredtiger/test/suite/test_async01.py @@ -117,7 +117,7 @@ class test_async01(wttest.WiredTigerTestCase, suite_subprocess): """ table_name1 = 'test_async01' nentries = 100 - async_ops = nentries / 2 + async_ops = nentries // 2 async_threads = 3 current = {} @@ -140,7 +140,7 @@ class test_async01(wttest.WiredTigerTestCase, suite_subprocess): if self.tablekind == 'row': return 'key' + str(i) else: - return long(i+1) + return self.recno(i+1) def genvalue(self, i): if self.tablekind == 'fix': diff --git a/src/third_party/wiredtiger/test/suite/test_async02.py b/src/third_party/wiredtiger/test/suite/test_async02.py index f5319648543..739b27f69bd 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_async02.py +++ b/src/third_party/wiredtiger/test/suite/test_async02.py @@ -114,7 +114,7 @@ class test_async02(wttest.WiredTigerTestCase, suite_subprocess): """ table_name1 = 'test_async02' nentries = 100 - async_ops = nentries / 2 + async_ops = nentries // 2 async_threads = 3 current = {} @@ -137,7 +137,7 @@ class test_async02(wttest.WiredTigerTestCase, suite_subprocess): if self.tablekind == 'row': return 'key' + str(i) else: - return long(i+1) + return self.recno(i+1) def genvalue(self, i): if self.tablekind == 'fix': @@ -187,8 +187,8 @@ class test_async02(wttest.WiredTigerTestCase, suite_subprocess): self.conn.async_flush() self.pr('flushed') - k = self.genkey(self.nentries / 2) - v = self.genvalue(self.nentries / 2) + k = self.genkey(self.nentries // 2) + v = self.genvalue(self.nentries // 2) k1 = self.genkey(self.nentries + 1) v1 = self.genvalue(self.nentries + 1) self.current[k] = wiredtiger.WT_DUPLICATE_KEY diff --git a/src/third_party/wiredtiger/test/suite/test_autoclose.py b/src/third_party/wiredtiger/test/suite/test_autoclose.py index e1ddc4c3359..efe29b3ca83 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_autoclose.py +++ b/src/third_party/wiredtiger/test/suite/test_autoclose.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import wiredtiger, wttest, exceptions +import wiredtiger, wttest # test_autoclose class test_autoclose(wttest.WiredTigerTestCase): @@ -57,7 +57,7 @@ class test_autoclose(wttest.WiredTigerTestCase): inscursor = self.open_cursor() inscursor['key1'] = 'value1' inscursor.close() - self.assertRaisesHavingMessage(exceptions.RuntimeError, + self.assertRaisesHavingMessage(RuntimeError, lambda: inscursor.next(), '/wt_cursor.* is None/') self.drop_table() @@ -72,7 +72,7 @@ class test_autoclose(wttest.WiredTigerTestCase): inscursor = self.open_cursor() inscursor['key1'] = 'value1' self.session.close() - self.assertRaisesHavingMessage(exceptions.RuntimeError, + self.assertRaisesHavingMessage(RuntimeError, lambda: inscursor.next(), '/wt_cursor.* is None/') self.close_conn() @@ -86,7 +86,7 @@ class test_autoclose(wttest.WiredTigerTestCase): inscursor = self.open_cursor() inscursor['key1'] = 'value1' self.close_conn() - self.assertRaisesHavingMessage(exceptions.RuntimeError, + self.assertRaisesHavingMessage(RuntimeError, lambda: inscursor.next(), '/wt_cursor.* is None/') @@ -120,11 +120,11 @@ class test_autoclose(wttest.WiredTigerTestCase): # Note: SWIG generates a TypeError in this case. # A RuntimeError to match the other cases would be better. inscursor2.close() - self.assertRaises(exceptions.TypeError, + self.assertRaises(TypeError, lambda: inscursor.compare(inscursor2)) inscursor2 = None - self.assertRaisesHavingMessage(exceptions.RuntimeError, + self.assertRaisesHavingMessage(RuntimeError, lambda: inscursor.compare(inscursor2), '/wt_cursor.* is None/') @@ -133,7 +133,7 @@ class test_autoclose(wttest.WiredTigerTestCase): Use a session handle after it is explicitly closed. """ self.session.close() - self.assertRaisesHavingMessage(exceptions.RuntimeError, + self.assertRaisesHavingMessage(RuntimeError, lambda: self.create_table(), '/wt_session.* is None/') self.close_conn() @@ -143,7 +143,7 @@ class test_autoclose(wttest.WiredTigerTestCase): Use a session handle after the connection is closed. """ self.close_conn() - self.assertRaisesHavingMessage(exceptions.RuntimeError, + self.assertRaisesHavingMessage(RuntimeError, lambda: self.create_table(), '/wt_session.* is None/') @@ -153,7 +153,7 @@ class test_autoclose(wttest.WiredTigerTestCase): """ conn = self.conn self.close_conn() - self.assertRaisesHavingMessage(exceptions.RuntimeError, + self.assertRaisesHavingMessage(RuntimeError, lambda: conn.open_session(None), '/wt_connection.* is None/') diff --git a/src/third_party/wiredtiger/test/suite/test_backup02.py b/src/third_party/wiredtiger/test/suite/test_backup02.py index 749ca3ff5f6..3f468a2b7bf 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_backup02.py +++ b/src/third_party/wiredtiger/test/suite/test_backup02.py @@ -26,7 +26,10 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import Queue +try: + import Queue as queue # python2 +except ImportError: + import queue import threading, time, wiredtiger, wttest from wtthread import backup_thread, checkpoint_thread, op_thread @@ -57,14 +60,14 @@ class test_backup02(wttest.WiredTigerTestCase): bkp = backup_thread(self.conn, 'backup.dir', done) bkp.start() - queue = Queue.Queue() + work_queue = queue.Queue() my_data = 'a' * self.dsize - for i in xrange(self.nops): - queue.put_nowait(('gi', i, my_data)) + for i in range(self.nops): + work_queue.put_nowait(('gi', i, my_data)) opthreads = [] - for i in xrange(self.nthreads): - t = op_thread(self.conn, uris, self.fmt, queue, done) + for i in range(self.nthreads): + t = op_thread(self.conn, uris, self.fmt, work_queue, done) opthreads.append(t) t.start() @@ -74,10 +77,10 @@ class test_backup02(wttest.WiredTigerTestCase): time.sleep(0.1) my_data = str(more_time) + 'a' * (self.dsize - len(str(more_time))) more_time = more_time - 0.1 - for i in xrange(self.nops): - queue.put_nowait(('gu', i, my_data)) + for i in range(self.nops): + work_queue.put_nowait(('gu', i, my_data)) - queue.join() + work_queue.join() done.set() # # Wait for checkpoint thread to notice status change. # ckpt.join() diff --git a/src/third_party/wiredtiger/test/suite/test_backup04.py b/src/third_party/wiredtiger/test/suite/test_backup04.py index d63c3f87152..1234652f9be 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_backup04.py +++ b/src/third_party/wiredtiger/test/suite/test_backup04.py @@ -26,7 +26,6 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import Queue import threading, time, wiredtiger, wttest import glob, os, shutil from helper import compare_files @@ -34,6 +33,10 @@ from suite_subprocess import suite_subprocess from wtdataset import SimpleDataSet, simple_key from wtscenario import make_scenarios from wtthread import op_thread +try: + xrange +except NameError: #python3 + xrange = range # test_backup04.py # Utilities: wt backup @@ -67,14 +70,14 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): def populate(self, uri, dsize, rows): self.pr('populate: ' + uri + ' with ' + str(rows) + ' rows') cursor = self.session.open_cursor(uri, None) - for i in range(1, rows + 1): + for i in xrange(1, rows + 1): cursor[simple_key(cursor, i)] = str(i) + ':' + 'a' * dsize cursor.close() def update(self, uri, dsize, upd, rows): self.pr('update: ' + uri + ' with ' + str(rows) + ' rows') cursor = self.session.open_cursor(uri, None) - for i in range(1, rows + 1): + for i in xrange(1, rows + 1): cursor[simple_key(cursor, i)] = str(i) + ':' + upd * dsize cursor.close() diff --git a/src/third_party/wiredtiger/test/suite/test_backup06.py b/src/third_party/wiredtiger/test/suite/test_backup06.py index cb4d76e14fb..d4efba4c6f0 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup06.py +++ b/src/third_party/wiredtiger/test/suite/test_backup06.py @@ -95,8 +95,8 @@ class test_backup06(wttest.WiredTigerTestCase, suite_subprocess): dh_after = stat_cursor[stat.conn.dh_conn_handle_count][2] stat_cursor.close() if (dh_before != dh_after): - print "Dhandles open before backup open: " + str(dh_before) - print "Dhandles open after backup open: " + str(dh_after) + print("Dhandles open before backup open: " + str(dh_before)) + print("Dhandles open after backup open: " + str(dh_after)) self.assertEqual(dh_after == dh_before, True) cursor.close() diff --git a/src/third_party/wiredtiger/test/suite/test_backup09.py b/src/third_party/wiredtiger/test/suite/test_backup09.py index e80f7f048bf..2141e4f8768 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_backup09.py +++ b/src/third_party/wiredtiger/test/suite/test_backup09.py @@ -73,11 +73,9 @@ class test_backup09(wttest.WiredTigerTestCase): cursor[doc_id] = doc_id last_doc_in_backup = doc_id - self.assertEqual(1, len(filter(lambda x: - x.startswith('WiredTigerLog.'), os.listdir('.')))) + self.assertEqual(1, len([x for x in os.listdir('.') if x.startswith('WiredTigerLog.')])) backup_cursor = self.session.open_cursor('backup:') - self.assertEqual(2, len(filter(lambda x: - x.startswith('WiredTigerLog.'), os.listdir('.')))) + self.assertEqual(2, len([x for x in os.listdir('.') if x.startswith('WiredTigerLog.')])) for i in range(10): doc_id += 1 @@ -91,7 +89,7 @@ class test_backup09(wttest.WiredTigerTestCase): os.mkdir(self.backup_dir) if self.all_log_files: helper.copy_wiredtiger_home('.', self.backup_dir) - log_files_copied = filter(lambda x: x.startswith('WiredTigerLog.'), os.listdir(self.backup_dir)) + log_files_copied = [x for x in os.listdir(self.backup_dir) if x.startswith('WiredTigerLog.')] self.assertEqual(len(log_files_copied), 2) else: while True: diff --git a/src/third_party/wiredtiger/test/suite/test_base05.py b/src/third_party/wiredtiger/test/suite/test_base05.py index 60f52bf39ac..688075b4392 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_base05.py +++ b/src/third_party/wiredtiger/test/suite/test_base05.py @@ -28,6 +28,10 @@ import wiredtiger, wttest from wtscenario import make_scenarios +try: + xrange +except NameError: #python3 + xrange = range # test_base05.py # Cursor operations @@ -127,7 +131,7 @@ class test_base05(wttest.WiredTigerTestCase): non_english_strings = [ # This notation creates 'string' objects that have embedded unicode. '\u20320\u22909', - '\u1571\u1604\u1587\u1617\u1604\u1575\u1605\u32\u1593\u1604\u1610\u1603\u1605', + '\u1571\u1604\u1587\u1617\u1604\u1575\u1605\u0032\u1593\u1604\u1610\u1603\u1605', '\u1513\u1500\u1493\u1501', '\u20170\u26085\u12399', '\u50504\u45397\u54616\u49464\u50836', @@ -146,7 +150,7 @@ class test_base05(wttest.WiredTigerTestCase): """ nstrings = 2 << (n % 10) result = '' - for i in range(nstrings): + for i in xrange(nstrings): if (n + i) % 20 == 0: reflist = self.non_english_strings else: @@ -164,7 +168,7 @@ class test_base05(wttest.WiredTigerTestCase): self.pr('creating cursor') cursor = self.session.open_cursor('table:' + self.table_name1) numbers = {} - for i in range(0, self.nentries): + for i in xrange(0, self.nentries): numbers[i] = i key = self.mixed_string(i) value = self.mixed_string(i+1) @@ -172,7 +176,7 @@ class test_base05(wttest.WiredTigerTestCase): # quick spot check to make sure searches work for divisor in [3, 5, 7]: - i = self.nentries / divisor + i = self.nentries // divisor key = self.mixed_string(i) value = self.mixed_string(i+1) cursor.set_key(key) @@ -205,16 +209,16 @@ class test_base05(wttest.WiredTigerTestCase): self.pr('creating cursor') cursor = self.session.open_cursor('table:' + self.table_name1) strlist = self.non_english_strings - for i in range(0, len(strlist)): + for i in xrange(0, len(strlist)): if convert: - key = val = unicode(strlist[i]) + key = val = str(strlist[i]) else: key = val = strlist[i] cursor[key] = val - for i in range(0, len(strlist)): + for i in xrange(0, len(strlist)): if convert: - key = val = unicode(strlist[i]) + key = val = str(strlist[i]) else: key = val = strlist[i] cursor.set_key(key) diff --git a/src/third_party/wiredtiger/test/suite/test_bug021.py b/src/third_party/wiredtiger/test/suite/test_bug021.py index 3c359e46eea..17e63b6a06d 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug021.py +++ b/src/third_party/wiredtiger/test/suite/test_bug021.py @@ -62,8 +62,8 @@ class test_bug021(wttest.WiredTigerTestCase): c.close() if actual != expected: - print 'expected: ', expected - print ' actual: ', actual + print('expected: ', expected) + print(' actual: ', actual) self.assertEqual(expected, actual) def test_implicit_record_cursor_insert_next(self): @@ -72,21 +72,21 @@ class test_bug021(wttest.WiredTigerTestCase): # Check cursor next/operation inside trailing implicit keys. cursor.set_key(62) - self.assertEquals(cursor.search(), 0) - self.assertEquals(cursor.next(), 0) - self.assertEquals(cursor.next(), 0) + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.next(), 0) + self.assertEqual(cursor.next(), 0) cursor.set_value(3) - self.assertEquals(cursor.insert(), 0) + self.assertEqual(cursor.insert(), 0) current[62 + 2] = 3 self.check(current) # Check cursor prev/operation inside trailing implicit keys. cursor.set_key(68) - self.assertEquals(cursor.search(), 0) - self.assertEquals(cursor.prev(), 0) - self.assertEquals(cursor.prev(), 0) + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.prev(), 0) + self.assertEqual(cursor.prev(), 0) cursor.set_value(7) - self.assertEquals(cursor.insert(), 0) + self.assertEqual(cursor.insert(), 0) current[68 - 2] = 7 def test_implicit_record_cursor_insert_prev(self): @@ -95,21 +95,21 @@ class test_bug021(wttest.WiredTigerTestCase): # Check cursor next/operation inside leading implicit keys. cursor.set_key(2) - self.assertEquals(cursor.search(), 0) - self.assertEquals(cursor.next(), 0) - self.assertEquals(cursor.next(), 0) + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.next(), 0) + self.assertEqual(cursor.next(), 0) cursor.set_value(3) - self.assertEquals(cursor.insert(), 0) + self.assertEqual(cursor.insert(), 0) current[2 + 2] = 3 self.check(current) # Check cursor prev/operation inside leading implicit keys. cursor.set_key(18) - self.assertEquals(cursor.search(), 0) - self.assertEquals(cursor.prev(), 0) - self.assertEquals(cursor.prev(), 0) + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.prev(), 0) + self.assertEqual(cursor.prev(), 0) cursor.set_value(7) - self.assertEquals(cursor.insert(), 0) + self.assertEqual(cursor.insert(), 0) current[18 - 2] = 7 self.check(current) @@ -119,19 +119,19 @@ class test_bug021(wttest.WiredTigerTestCase): # Check cursor next/operation inside trailing implicit keys. cursor.set_key(62) - self.assertEquals(cursor.search(), 0) + self.assertEqual(cursor.search(), 0) for i in range(1, 5): - self.assertEquals(cursor.next(), 0) - self.assertEquals(cursor.remove(), 0) + self.assertEqual(cursor.next(), 0) + self.assertEqual(cursor.remove(), 0) current[62 + i] = 0 self.check(current) # Check cursor prev/operation inside trailing implicit keys. cursor.set_key(68) - self.assertEquals(cursor.search(), 0) + self.assertEqual(cursor.search(), 0) for i in range(1, 5): - self.assertEquals(cursor.prev(), 0) - self.assertEquals(cursor.remove(), 0) + self.assertEqual(cursor.prev(), 0) + self.assertEqual(cursor.remove(), 0) current[68 - i] = 0 self.check(current) @@ -141,20 +141,20 @@ class test_bug021(wttest.WiredTigerTestCase): # Check cursor next/operation inside leading implicit keys. cursor.set_key(2) - self.assertEquals(cursor.search(), 0) + self.assertEqual(cursor.search(), 0) for i in range(1, 5): - self.assertEquals(cursor.next(), 0) - self.assertEquals(cursor.remove(), 0) + self.assertEqual(cursor.next(), 0) + self.assertEqual(cursor.remove(), 0) current[2 + i] = 0 self.check(current) # Check cursor prev/operation inside leading implicit keys. cursor.set_key(18) - self.assertEquals(cursor.search(), 0) + self.assertEqual(cursor.search(), 0) for i in range(1, 5): current[18 - i] = 0 - self.assertEquals(cursor.prev(), 0) - self.assertEquals(cursor.remove(), 0) + self.assertEqual(cursor.prev(), 0) + self.assertEqual(cursor.remove(), 0) current[18 - i] = 0 self.check(current) @@ -164,22 +164,22 @@ class test_bug021(wttest.WiredTigerTestCase): # Check cursor next/operation inside trailing implicit keys. cursor.set_key(62) - self.assertEquals(cursor.search(), 0) + self.assertEqual(cursor.search(), 0) for i in range(1, 5): - self.assertEquals(cursor.next(), 0) + self.assertEqual(cursor.next(), 0) cursor.set_value(i) self.session.breakpoint() - self.assertEquals(cursor.update(), 0) + self.assertEqual(cursor.update(), 0) current[62 + i] = i self.check(current) # Check cursor prev/operation inside trailing implicit keys. cursor.set_key(68) - self.assertEquals(cursor.search(), 0) + self.assertEqual(cursor.search(), 0) for i in range(1, 5): - self.assertEquals(cursor.prev(), 0) + self.assertEqual(cursor.prev(), 0) cursor.set_value(i) - self.assertEquals(cursor.update(), 0) + self.assertEqual(cursor.update(), 0) current[68 - i] = i self.check(current) @@ -189,22 +189,22 @@ class test_bug021(wttest.WiredTigerTestCase): # Check cursor next/operation inside leading implicit keys. cursor.set_key(2) - self.assertEquals(cursor.search(), 0) + self.assertEqual(cursor.search(), 0) for i in range(1, 5): - self.assertEquals(cursor.next(), 0) + self.assertEqual(cursor.next(), 0) cursor.set_value(i) - self.assertEquals(cursor.update(), 0) + self.assertEqual(cursor.update(), 0) current[2 + i] = i self.check(current) # Check cursor prev/operation inside leading implicit keys. cursor.set_key(18) - self.assertEquals(cursor.search(), 0) + self.assertEqual(cursor.search(), 0) for i in range(1, 5): current[18 - i] = 0 - self.assertEquals(cursor.prev(), 0) + self.assertEqual(cursor.prev(), 0) cursor.set_value(i) - self.assertEquals(cursor.update(), 0) + self.assertEqual(cursor.update(), 0) current[18 - i] = i self.check(current) diff --git a/src/third_party/wiredtiger/test/suite/test_bulk01.py b/src/third_party/wiredtiger/test/suite/test_bulk01.py index cda17a67b06..79459abc796 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_bulk01.py +++ b/src/third_party/wiredtiger/test/suite/test_bulk01.py @@ -33,6 +33,10 @@ import wiredtiger, wttest from wtdataset import SimpleDataSet, simple_key, simple_value from wtscenario import make_scenarios +try: + xrange +except NameError: #python3 + xrange = range # Smoke test bulk-load. class test_bulk_load(wttest.WiredTigerTestCase): @@ -60,7 +64,7 @@ class test_bulk_load(wttest.WiredTigerTestCase): self.session.create(uri, 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk") - for i in range(1, 1000): + for i in xrange(1, 1000): cursor[simple_key(cursor, i)] = simple_value(cursor, i) # Test a bulk-load triggers variable-length column-store RLE correctly. @@ -75,8 +79,8 @@ class test_bulk_load(wttest.WiredTigerTestCase): self.session.create(uri, 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk") - for i in range(1, 1000): - cursor[simple_key(cursor, i)] = simple_value(cursor, i/7) + for i in xrange(1, 1000): + cursor[simple_key(cursor, i)] = simple_value(cursor, i//7) # Test a bulk-load variable-length column-store append ignores any key. def test_bulk_load_var_append(self): @@ -87,11 +91,11 @@ class test_bulk_load(wttest.WiredTigerTestCase): self.session.create(uri, 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk,append") - for i in range(1, 1000): + for i in xrange(1, 1000): cursor[simple_key(cursor, 37)] = simple_value(cursor, i) cursor.close() cursor = self.session.open_cursor(uri, None, None) - for i in range(1, 1000): + for i in xrange(1, 1000): cursor.set_key(simple_key(cursor, i)) cursor.search() self.assertEqual(cursor.get_value(), simple_value(cursor, i)) @@ -105,7 +109,7 @@ class test_bulk_load(wttest.WiredTigerTestCase): self.session.create(uri, 'key_format=' + self.keyfmt + ',value_format=' + self.valfmt) cursor = self.session.open_cursor(uri, None, "bulk") - for i in range(1, 1000): + for i in xrange(1, 1000): if i % 7 == 0: cursor[simple_key(cursor, i)] = simple_value(cursor, i) @@ -117,7 +121,7 @@ class test_bulk_load(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri, None, None) # Verify all the records are there, in their proper state. - for i in range(1, 1000): + for i in xrange(1, 1000): cursor.set_key(simple_key(cursor, i)) if i % 7 == 0: cursor.search() diff --git a/src/third_party/wiredtiger/test/suite/test_calc_modify.py b/src/third_party/wiredtiger/test/suite/test_calc_modify.py index e3326717945..71ec7a45054 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_calc_modify.py +++ b/src/third_party/wiredtiger/test/suite/test_calc_modify.py @@ -28,6 +28,7 @@ import random, string import wiredtiger, wttest +from wtscenario import make_scenarios r = random.Random(42) # Make things repeatable @@ -52,17 +53,27 @@ class test_calc_modify(wttest.WiredTigerTestCase): REMOVE = 2 REPLACE = 3 + valuefmt = [ + ('item', dict(valuefmt='u')), + ('string', dict(valuefmt='S')), + ] + scenarios = make_scenarios(valuefmt) + def mkstring(self, size, repeat_size=1): - pattern = ''.join(r.choice(string.ascii_letters + string.digits) for _ in xrange(repeat_size)) - return (pattern * ((size + repeat_size - 1) / repeat_size))[:size] + choices = string.ascii_letters + string.digits + if self.valuefmt == 'S': + pattern = ''.join(r.choice(choices) for _ in range(repeat_size)) + else: + pattern = b''.join(bytes([r.choice(choices.encode())]) for _ in range(repeat_size)) + return (pattern * ((size + repeat_size - 1) // repeat_size))[:size] def one_test(self, c, k, oldsz, repeatsz, nmod, maxdiff): oldv = self.mkstring(oldsz, repeatsz) - offsets = sorted(r.sample(xrange(oldsz), nmod)) - modsizes = sorted(r.sample(xrange(maxdiff), nmod + 1)) - lengths = [modsizes[i+1] - modsizes[i] for i in xrange(nmod)] - modtypes = [r.choice((self.ADD, self.REMOVE, self.REPLACE)) for _ in xrange(nmod)] + offsets = sorted(r.sample(range(oldsz), nmod)) + modsizes = sorted(r.sample(range(maxdiff), nmod + 1)) + lengths = [modsizes[i+1] - modsizes[i] for i in range(nmod)] + modtypes = [r.choice((self.ADD, self.REMOVE, self.REPLACE)) for _ in range(nmod)] self.pr("offsets: %s" % offsets) self.pr("modsizes: %s" % modsizes) @@ -70,8 +81,8 @@ class test_calc_modify(wttest.WiredTigerTestCase): self.pr("modtypes: %s" % modtypes) orig = oldv - newv = '' - for i in xrange(nmod): + newv = '' if self.valuefmt == 'S' else b'' + for i in range(nmod): if i > 0 and offsets[i] - offsets[i - 1] < maxdiff: continue newv += orig[:offsets[i]] @@ -104,13 +115,13 @@ class test_calc_modify(wttest.WiredTigerTestCase): self.assertEqual(c[k], newv) def test_calc_modify(self): - self.session.create(self.uri, 'key_format=i,value_format=u') + self.session.create(self.uri, 'key_format=i,value_format=' + self.valuefmt) c = self.session.open_cursor(self.uri) - for k in xrange(1000): + for k in range(1000): size = r.randint(1000, 10000) repeats = r.randint(1, size) nmods = r.randint(1, 10) - maxdiff = r.randint(64, size / 10) + maxdiff = r.randint(64, size // 10) self.pr("size %s, repeats %s, nmods %s, maxdiff %s" % (size, repeats, nmods, maxdiff)) self.one_test(c, k, size, repeats, nmods, maxdiff) diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint01.py b/src/third_party/wiredtiger/test/suite/test_checkpoint01.py index 8c758943f68..02849bc773f 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint01.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint01.py @@ -67,14 +67,14 @@ class test_checkpoint(wttest.WiredTigerTestCase): # checkpoint the object, and verify it (which verifies all underlying # checkpoints individually). def build_file_with_checkpoints(self): - for checkpoint_name, entry in self.checkpoints.iteritems(): + for checkpoint_name, entry in self.checkpoints.items(): self.add_records(checkpoint_name) self.session.checkpoint("name=" + checkpoint_name) # Create a dictionary of sorted records a checkpoint should include. def list_expected(self, name): records = {} - for checkpoint_name, entry in self.checkpoints.iteritems(): + for checkpoint_name, entry in self.checkpoints.items(): start, stop = entry[0] for i in range(start, stop+1): records['%010d KEY------' % i] =\ @@ -98,7 +98,7 @@ class test_checkpoint(wttest.WiredTigerTestCase): # Physically verify the file, including the individual checkpoints. self.session.verify(self.uri, None) - for checkpoint_name, entry in self.checkpoints.iteritems(): + for checkpoint_name, entry in self.checkpoints.items(): if entry[1] == 0: self.assertRaises(wiredtiger.WiredTigerError, lambda: self.session.open_cursor( @@ -131,7 +131,7 @@ class test_checkpoint(wttest.WiredTigerTestCase): # Drop remaining checkpoints, all subsequent checkpoint opens should # fail. self.session.checkpoint("drop=(from=all)") - for checkpoint_name, entry in self.checkpoints.iteritems(): + for checkpoint_name, entry in self.checkpoints.items(): self.checkpoints[checkpoint_name] =\ (self.checkpoints[checkpoint_name][0], 0) self.check() diff --git a/src/third_party/wiredtiger/test/suite/test_checkpoint02.py b/src/third_party/wiredtiger/test/suite/test_checkpoint02.py index a3f37474cd9..3a738b52103 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_checkpoint02.py +++ b/src/third_party/wiredtiger/test/suite/test_checkpoint02.py @@ -26,10 +26,17 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import Queue +try: + import Queue as queue # python2 +except ImportError: + import queue import threading, time, wiredtiger, wttest from wtthread import checkpoint_thread, op_thread from wtscenario import make_scenarios +try: + xrange +except NameError: #python3 + xrange = range # test_checkpoint02.py # Run background checkpoints repeatedly while doing inserts and other @@ -49,20 +56,20 @@ class test_checkpoint02(wttest.WiredTigerTestCase): uris = list() uris.append(self.uri) - queue = Queue.Queue() + work_queue = queue.Queue() my_data = 'a' * self.dsize for i in xrange(self.nops): if i % 191 == 0 and i != 0: - queue.put_nowait(('b', i, my_data)) - queue.put_nowait(('i', i, my_data)) + work_queue.put_nowait(('b', i, my_data)) + work_queue.put_nowait(('i', i, my_data)) opthreads = [] - for i in xrange(self.nthreads): - t = op_thread(self.conn, uris, self.fmt, queue, done) + for i in range(self.nthreads): + t = op_thread(self.conn, uris, self.fmt, work_queue, done) opthreads.append(t) t.start() - queue.join() + work_queue.join() done.set() for t in opthreads: t.join() diff --git a/src/third_party/wiredtiger/test/suite/test_compact02.py b/src/third_party/wiredtiger/test/suite/test_compact02.py index b4fa534c354..5bfdd2945fa 100644 --- a/src/third_party/wiredtiger/test/suite/test_compact02.py +++ b/src/third_party/wiredtiger/test/suite/test_compact02.py @@ -77,7 +77,7 @@ class test_compact02(wttest.WiredTigerTestCase): bigvalue = "abcdefghi" * 1074 # 9*1074 == 9666 smallvalue = "ihgfedcba" * 303 # 9*303 == 2727 - fullsize = nrecords / 2 * len(bigvalue) + nrecords / 2 * len(smallvalue) + fullsize = nrecords // 2 * len(bigvalue) + nrecords // 2 * len(smallvalue) # Return the size of the file def getSize(self): @@ -105,7 +105,7 @@ class test_compact02(wttest.WiredTigerTestCase): try: self.conn = wiredtiger.wiredtiger_open(self.home, conn_params) except wiredtiger.WiredTigerError as e: - print "Failed conn at '%s' with config '%s'" % (dir, conn_params) + print("Failed conn at '%s' with config '%s'" % (dir, conn_params)) self.session = self.conn.open_session(None) # Create a table, add keys with both big and small values. @@ -128,7 +128,7 @@ class test_compact02(wttest.WiredTigerTestCase): # 2. Checkpoint and get stats on the table to confirm the size. self.session.checkpoint() sz = self.getSize() - self.pr('After populate ' + str(sz / mb) + 'MB') + self.pr('After populate ' + str(sz // mb) + 'MB') self.assertGreater(sz, self.fullsize) # 3. Delete the half of the records with the larger record size. @@ -140,7 +140,7 @@ class test_compact02(wttest.WiredTigerTestCase): c.set_key(i) c.remove() c.close() - self.pr('Removed total ' + str((count * 9666) / mb) + 'MB') + self.pr('Removed total ' + str((count * 9666) // mb) + 'MB') # 4. Checkpoint self.session.checkpoint() @@ -157,10 +157,10 @@ class test_compact02(wttest.WiredTigerTestCase): # 6. Get stats on compacted table. sz = self.getSize() - self.pr('After compact ' + str(sz / mb) + 'MB') + self.pr('After compact ' + str(sz // mb) + 'MB') # After compact, the file size should be less than half the full size. - self.assertLess(sz, self.fullsize / 2) + self.assertLess(sz, self.fullsize // 2) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_compress01.py b/src/third_party/wiredtiger/test/suite/test_compress01.py index 7406e971d19..b344e194ace 100644 --- a/src/third_party/wiredtiger/test/suite/test_compress01.py +++ b/src/third_party/wiredtiger/test/suite/test_compress01.py @@ -66,12 +66,12 @@ class test_compress01(wttest.WiredTigerTestCase): params = 'key_format=S,value_format=S,leaf_page_max=4096' self.session.create(self.uri, params) cursor = self.session.open_cursor(self.uri, None) - for idx in xrange(1,self.nrecords): - cursor.set_key(`idx`) - if idx / 12 == 0: - cursor.set_value(`idx` + self.bigvalue) + for idx in range(1,self.nrecords): + cursor.set_key(repr(idx)) + if idx // 12 == 0: + cursor.set_value(repr(idx) + self.bigvalue) else: - cursor.set_value(`idx` + "abcdefg") + cursor.set_value(repr(idx) + "abcdefg") cursor.insert() cursor.close() @@ -79,13 +79,13 @@ class test_compress01(wttest.WiredTigerTestCase): self.reopen_conn() cursor = self.session.open_cursor(self.uri, None) - for idx in xrange(1,self.nrecords): - cursor.set_key(`idx`) + for idx in range(1,self.nrecords): + cursor.set_key(repr(idx)) self.assertEqual(cursor.search(), 0) - if idx / 12 == 0: - self.assertEquals(cursor.get_value(), `idx` + self.bigvalue) + if idx // 12 == 0: + self.assertEqual(cursor.get_value(), repr(idx) + self.bigvalue) else: - self.assertEquals(cursor.get_value(), `idx` + "abcdefg") + self.assertEqual(cursor.get_value(), repr(idx) + "abcdefg") cursor.close() if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_config01.py b/src/third_party/wiredtiger/test/suite/test_config01.py index e77e80a1bce..5a866613c05 100644 --- a/src/third_party/wiredtiger/test/suite/test_config01.py +++ b/src/third_party/wiredtiger/test/suite/test_config01.py @@ -42,7 +42,7 @@ class test_config01(test_base03.test_base03): if hasattr(self, 'cache_size'): wtopen_args += ',cache_size=' + str(self.cache_size) conn = self.wiredtiger_open(dir, wtopen_args) - self.pr(`conn`) + self.pr(repr(conn)) return conn if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_config02.py b/src/third_party/wiredtiger/test/suite/test_config02.py index 82a56a0a245..9fbd12f23ab 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_config02.py +++ b/src/third_party/wiredtiger/test/suite/test_config02.py @@ -152,7 +152,7 @@ class test_config02(wttest.WiredTigerTestCase): self.skipTest('Unix specific test skipped on Windows') dir = 'subdir' os.mkdir(dir) - os.chmod(dir, 0555) + os.chmod(dir, 0o555) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.wiredtiger_open(dir, 'create'), '/Permission denied/') diff --git a/src/third_party/wiredtiger/test/suite/test_config03.py b/src/third_party/wiredtiger/test/suite/test_config03.py index c5f98f85898..a5d9ee199a0 100644 --- a/src/third_party/wiredtiger/test/suite/test_config03.py +++ b/src/third_party/wiredtiger/test/suite/test_config03.py @@ -123,7 +123,7 @@ class test_config03(test_base03.test_base03): self.verbose(3, 'wiredtiger_open with args: ' + args) conn = self.wiredtiger_open(dir, args) - self.pr(`conn`) + self.pr(repr(conn)) return conn if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_config04.py b/src/third_party/wiredtiger/test/suite/test_config04.py index b4045814c13..b4045814c13 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_config04.py +++ b/src/third_party/wiredtiger/test/suite/test_config04.py diff --git a/src/third_party/wiredtiger/test/suite/test_cursor01.py b/src/third_party/wiredtiger/test/suite/test_cursor01.py index 6da208e2f45..770b433cf14 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor01.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor01.py @@ -55,7 +55,7 @@ class test_cursor01(wttest.WiredTigerTestCase): if self.tablekind == 'row': return 'key' + str(i) else: - return long(i+1) + return self.recno(i+1) def genvalue(self, i): if self.tablekind == 'fix': diff --git a/src/third_party/wiredtiger/test/suite/test_cursor04.py b/src/third_party/wiredtiger/test/suite/test_cursor04.py index 0ff539a252f..b57d24da847 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor04.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor04.py @@ -82,7 +82,7 @@ class test_cursor04(wttest.WiredTigerTestCase): if self.tablekind == 'row': return 'key' + str(i).zfill(5) # return key00001, key00002, etc. else: - return long(i+1) + return self.recno(i+1) def genvalue(self, i): if self.tablekind == 'fix': diff --git a/src/third_party/wiredtiger/test/suite/test_cursor07.py b/src/third_party/wiredtiger/test/suite/test_cursor07.py index c26720f13e1..82a606ed1bc 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor07.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor07.py @@ -59,8 +59,8 @@ class test_cursor07(wttest.WiredTigerTestCase, suite_subprocess): def test_log_cursor(self): # print "Creating %s with config '%s'" % (self.uri, self.create_params) - create_params = 'key_format=i,value_format=S' - create_nolog_params = 'key_format=i,value_format=S,log=(enabled=false)' + create_params = 'key_format=i,value_format=u' + create_nolog_params = 'key_format=i,value_format=u,log=(enabled=false)' self.session.create(self.uri1, create_params) c1 = self.session.open_cursor(self.uri1, None) self.session.create(self.uri2, create_nolog_params) @@ -69,8 +69,8 @@ class test_cursor07(wttest.WiredTigerTestCase, suite_subprocess): c3 = self.session.open_cursor(self.uri3, None) # A binary value. - value = u'\u0001\u0002abcd\u0003\u0004' - value_nolog = u'\u0001\u0002dcba\u0003\u0004' + value = b'\x01\x02abcd\x03\x04' + value_nolog = b'\x01\x02dcba\x03\x04' # We want to test both adding data to a table that is not logged # that is part of the same transaction as a table that is logged @@ -100,12 +100,11 @@ class test_cursor07(wttest.WiredTigerTestCase, suite_subprocess): keys = c.get_key() # txnid, rectype, optype, fileid, logrec_key, logrec_value values = c.get_value() - try: - if value in str(values[5]): # logrec_value + # We are only looking for log records that that have a key/value + # pair. + if values[4] != b'': + if value in values[5]: # logrec_value count += 1 - self.assertFalse(value2 in str(values[5])) - except: - pass c.close() self.assertEqual(count, self.nkeys) diff --git a/src/third_party/wiredtiger/test/suite/test_cursor08.py b/src/third_party/wiredtiger/test/suite/test_cursor08.py index f1e52607cf2..a21c259c879 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor08.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor08.py @@ -70,7 +70,7 @@ class test_cursor08(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(self.uri, None) # A binary value. - value = u'\u0001\u0002abcd\u0003\u0004' + value = '\u0001\u0002abcd\u0003\u0004' self.session.begin_transaction() for k in range(self.nkeys): @@ -89,11 +89,11 @@ class test_cursor08(wttest.WiredTigerTestCase, suite_subprocess): keys = c.get_key() # txnid, rectype, optype, fileid, logrec_key, logrec_value values = c.get_value() - try: - if value in str(values[5]): # logrec_value + # We are only looking for log records that that have a key/value + # pair. + if values[4] != b'': + if value.encode() in values[5]: # logrec_value count += 1 - except: - pass c.close() self.assertEqual(count, self.nkeys) diff --git a/src/third_party/wiredtiger/test/suite/test_cursor10.py b/src/third_party/wiredtiger/test/suite/test_cursor10.py index 7cb12248512..f6b100ffd21 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor10.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor10.py @@ -47,7 +47,7 @@ class test_cursor10(wttest.WiredTigerTestCase): if self.key_format == 'S': return 'key' + str(i).zfill(5) # return key00001, key00002, etc. else: - return long(i+1) + return self.recno(i+1) def genvalue(self, i): return [ 'v0:' + str(i), i+1, 'v2' + str(i+2), i+3 ] @@ -56,7 +56,7 @@ class test_cursor10(wttest.WiredTigerTestCase): if self.key_format == 'S': return int(k[3:]) else: - return long(k-1) + return self.recno(k-1) def test_projection(self): """ diff --git a/src/third_party/wiredtiger/test/suite/test_cursor12.py b/src/third_party/wiredtiger/test/suite/test_cursor12.py index 84fdad8f242..121180430f7 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor12.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor12.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import random, string +import random, string, sys import wiredtiger, wttest from helper import copy_wiredtiger_home from wtdataset import SimpleDataSet @@ -180,6 +180,39 @@ class test_cursor12(wttest.WiredTigerTestCase): } ] + def setUp(self): + if sys.version_info[0] >= 3 and self.valuefmt == 'u': + # Python3 distinguishes bytes from strings + self.nullbyte = b'\x00' + self.spacebyte = b' ' + else: + self.nullbyte = '\x00' + self.spacebyte = ' ' + super(test_cursor12, self).setUp() + + # Convert a string to the correct type for the value. + def make_value(self, s): + if self.valuefmt == 'u': + return bytes(s.encode()) + else: + return s + + def fix_mods(self, mods): + if bytes != str and self.valuefmt == 'u': + # In Python3, bytes and strings are independent types, and + # the WiredTiger API needs bytes when the format calls for bytes. + newmods = [] + for mod in mods: + # We need to check because we may converted some of the Modify + # records already. + if type(mod.data) == str: + newmods.append(wiredtiger.Modify( + self.make_value(mod.data), mod.offset, mod.size)) + else: + newmods.append(mod) + mods = newmods + return mods + # Create a set of modified records and verify in-memory reads. def modify_load(self, ds, single): # For each test in the list: @@ -190,7 +223,7 @@ class test_cursor12(wttest.WiredTigerTestCase): c = self.session.open_cursor(self.uri, None) for i in self.list: c.set_key(ds.key(row)) - c.set_value(i['o']) + c.set_value(self.make_value(i['o'])) self.assertEquals(c.update(), 0) c.reset() @@ -200,6 +233,7 @@ class test_cursor12(wttest.WiredTigerTestCase): for j in i['mods']: mod = wiredtiger.Modify(j[0], j[1], j[2]) mods.append(mod) + mods = self.fix_mods(mods) self.assertEquals(c.modify(mods), 0) self.session.commit_transaction() c.reset() @@ -207,7 +241,8 @@ class test_cursor12(wttest.WiredTigerTestCase): c.set_key(ds.key(row)) self.assertEquals(c.search(), 0) v = c.get_value() - self.assertEquals(v.replace("\x00", " "), i['f']) + expect = self.make_value(i['f']) + self.assertEquals(v.replace(self.nullbyte, self.spacebyte), expect) if not single: row = row + 1 @@ -223,7 +258,8 @@ class test_cursor12(wttest.WiredTigerTestCase): c.set_key(ds.key(row)) self.assertEquals(c.search(), 0) v = c.get_value() - self.assertEquals(v.replace("\x00", " "), i['f']) + expect = self.make_value(i['f']) + self.assertEquals(v.replace(self.nullbyte, self.spacebyte), expect) if not single: row = row + 1 @@ -292,15 +328,17 @@ class test_cursor12(wttest.WiredTigerTestCase): c = self.session.open_cursor(self.uri, None) self.session.begin_transaction() c.set_key(ds.key(10)) - orig = 'abcdefghijklmnopqrstuvwxyz' + orig = self.make_value('abcdefghijklmnopqrstuvwxyz') c.set_value(orig) self.assertEquals(c.update(), 0) for i in range(0, 50000): - new = "".join([random.choice(string.digits) for i in xrange(5)]) + new = self.make_value("".join([random.choice(string.digits) \ + for i in range(5)])) orig = orig[:10] + new + orig[15:] mods = [] mod = wiredtiger.Modify(new, 10, 5) mods.append(mod) + mods = self.fix_mods(mods) self.assertEquals(c.modify(mods), 0) self.session.commit_transaction() @@ -322,6 +360,7 @@ class test_cursor12(wttest.WiredTigerTestCase): mods = [] mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) + mods = self.fix_mods(mods) c.set_key(ds.key(10)) self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND) @@ -348,6 +387,7 @@ class test_cursor12(wttest.WiredTigerTestCase): mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) c.set_key(ds.key(30)) + mods = self.fix_mods(mods) self.assertEqual(c.modify(mods), 0) # Test that another transaction cannot modify our uncommitted record. @@ -359,6 +399,7 @@ class test_cursor12(wttest.WiredTigerTestCase): mods = [] mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) + mods = self.fix_mods(mods) xc.set_key(ds.key(30)) self.assertEqual(xc.modify(mods), wiredtiger.WT_NOTFOUND) xs.rollback_transaction() @@ -371,6 +412,7 @@ class test_cursor12(wttest.WiredTigerTestCase): mods = [] mod = wiredtiger.Modify('ABCD', 3, 3) mods.append(mod) + mods = self.fix_mods(mods) c.set_key(ds.key(30)) self.assertEqual(c.modify(mods), wiredtiger.WT_NOTFOUND) self.session.rollback_transaction() diff --git a/src/third_party/wiredtiger/test/suite/test_cursor13.py b/src/third_party/wiredtiger/test/suite/test_cursor13.py index 3391f1e03ac..6f9126cce6a 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor13.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor13.py @@ -375,11 +375,11 @@ class test_cursor13_big_base(test_cursor13_base): # create some number (self.deep) of cached cursors. def create_uri_map(self, baseuri): uri_map = {} - for i in xrange(0, self.nuris): + for i in range(0, self.nuris): uri = self.uriname(i) cursors = [] self.session.create(uri, None) - for j in xrange(0, self.deep): + for j in range(0, self.deep): cursors.append(self.session.open_cursor(uri, None)) for c in cursors: c.close() @@ -478,8 +478,8 @@ class test_cursor13_sweep(test_cursor13_big_base): # Close cursors in half of the range, and don't # use them during this round, so they will be # closed by sweep. - half = self.nuris / 2 - potential_dead += self.close_uris(uri_map, xrange(0, half)) + half = self.nuris // 2 + potential_dead += self.close_uris(uri_map, list(range(0, half))) bottom_range = half # Let the dhandle sweep run and find the closed cursors. time.sleep(3.0) @@ -489,7 +489,7 @@ class test_cursor13_sweep(test_cursor13_big_base): # The session cursor sweep runs at most once a second and # traverses a fraction of the cached cursors. We'll run for # ten seconds with pauses to make sure we see sweep activity. - pause_point = self.opens_per_round / 100 + pause_point = self.opens_per_round // 100 if pause_point == 0: pause_point = 1 pause_duration = 0.1 @@ -522,7 +522,7 @@ class test_cursor13_sweep(test_cursor13_big_base): # We'll pass the test if we see at least 20% of the 'potentially # dead' cursors swept. There may be more, since the 1% per second # is a minimum. - min_swept = 2 * potential_dead / 10 + min_swept = 2 * potential_dead // 10 self.assertGreaterEqual(swept, min_swept) # No strict equality test for the reopen stats. When we've swept diff --git a/src/third_party/wiredtiger/test/suite/test_cursor14.py b/src/third_party/wiredtiger/test/suite/test_cursor14.py index f650f922a09..4d37069b221 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor14.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor14.py @@ -53,7 +53,7 @@ class test_cursor14(wttest.WiredTigerTestCase): ds = self.dataset(self, uri, 100, key_format=self.keyfmt) ds.populate() - for i in xrange(66000): + for i in range(66000): cursor = self.session.open_cursor(uri, None, None) if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_compare.py b/src/third_party/wiredtiger/test/suite/test_cursor_compare.py index 2597a8592a3..49682e48215 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor_compare.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_compare.py @@ -26,7 +26,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import wiredtiger, wttest, exceptions +import wiredtiger, wttest from wtdataset import SimpleDataSet, ComplexDataSet, ComplexLSMDataSet from wtscenario import filter_scenarios, make_scenarios @@ -99,7 +99,7 @@ class test_cursor_comparison(wttest.WiredTigerTestCase): wiredtiger.WiredTigerError, lambda: cX.compare(c1), msg) msg = '/wt_cursor.* is None/' self.assertRaisesHavingMessage( - exceptions.RuntimeError, lambda: cX.compare(None), msg) + RuntimeError, lambda: cX.compare(None), msg) if ix0_0 != None: self.assertEqual(ix0_0.compare(ix0_1), 0) ix0_1.reset() @@ -188,7 +188,7 @@ class test_cursor_comparison(wttest.WiredTigerTestCase): wiredtiger.WiredTigerError, lambda: cX.equals(c1), msg) msg = '/wt_cursor.* is None/' self.assertRaisesHavingMessage( - exceptions.RuntimeError, lambda: cX.equals(None), msg) + RuntimeError, lambda: cX.equals(None), msg) if ix0_0 != None: self.assertTrue(ix0_0.equals(ix0_1)) ix0_1.reset() diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_pin.py b/src/third_party/wiredtiger/test/suite/test_cursor_pin.py index 8b9784b10d0..692f6e09c26 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor_pin.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_pin.py @@ -111,11 +111,11 @@ class test_cursor_pin(wttest.WiredTigerTestCase): for i in range(self.nentries + 1000, self.nentries + 2001): c[ds.key(i)] = ds.value(i) self.forward(c, ds, self.nentries + 5000, - list(range(self.nentries + 1, self.nentries + 1000) +\ - range(self.nentries + 2001, self.nentries + 3000))) + list(list(range(self.nentries + 1, self.nentries + 1000)) +\ + list(range(self.nentries + 2001, self.nentries + 3000)))) self.backward(c, ds, self.nentries + 5000, - list(range(self.nentries + 1, self.nentries + 1000) +\ - range(self.nentries + 2001, self.nentries + 3000))) + list(list(range(self.nentries + 1, self.nentries + 1000)) +\ + list(range(self.nentries + 2001, self.nentries + 3000)))) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_random02.py b/src/third_party/wiredtiger/test/suite/test_cursor_random02.py index 46faa0ce3fc..6532247e271 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor_random02.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_random02.py @@ -87,13 +87,13 @@ class test_cursor_random02(wttest.WiredTigerTestCase): ''' self.tty('differentKeys: ' + str(differentKeys) + ' of ' + \ str(num_entries) + ', ' + \ - str((int)((differentKeys * 100) / num_entries)) + '%') + str((int)((differentKeys * 100) // num_entries)) + '%') ''' # Can't test for non-sequential data when there is 1 item in the table if num_entries > 1: self.assertGreater(num_entries - 1, sequentialKeys, 'cursor is returning sequential data') - self.assertGreater(differentKeys, num_entries / 4, + self.assertGreater(differentKeys, num_entries // 4, 'next_random random distribution not adequate') if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_tracker.py b/src/third_party/wiredtiger/test/suite/test_cursor_tracker.py index 6c96b85dd1e..967bf81138f 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_cursor_tracker.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_tracker.py @@ -179,7 +179,7 @@ class TestCursorTracker(wttest.WiredTigerTestCase): def stretch_content(self, s, sizes): result = s if sizes != None: - sha224 = hashlib.sha224(s) + sha224 = hashlib.sha224(s.encode()) md5 = sha224.digest() low = sizes[0] - len(s) if low < 0: @@ -188,7 +188,7 @@ class TestCursorTracker(wttest.WiredTigerTestCase): if high < 0: high = 0 diff = high - low - nextra = (ord(md5[4]) % (diff + 1)) + low + nextra = (self.ord_byte(md5[4]) % (diff + 1)) + low extra = sha224.hexdigest() while len(extra) < nextra: extra = extra + extra @@ -237,7 +237,7 @@ class TestCursorTracker(wttest.WiredTigerTestCase): # 64 bit key maj = ((bits >> 32) & 0xffffffff) + 1 min = (bits >> 16) & 0xffff - return long((maj << 16) | min) + return self.recno((maj << 16) | min) def decode_key_col_or_fix(self, bits): maj = ((bits << 16) & 0xffffffff) - 1 @@ -296,7 +296,7 @@ class TestCursorTracker(wttest.WiredTigerTestCase): def bitspos(self, bits): list = self.bitlist - return next(i for i in xrange(len(list)) if list[i] == bits) + return next(i for i in range(len(list)) if list[i] == bits) def cur_insert(self, cursor, major, minor): bits = self.triple_to_bits(major, minor, 0) @@ -333,7 +333,7 @@ class TestCursorTracker(wttest.WiredTigerTestCase): cursor.remove() def cur_recno_search(self, cursor, recno): - wtkey = long(recno) + wtkey = self.recno(recno) self.traceapi('cursor.set_key(' + str(wtkey) + ')') cursor.set_key(wtkey) if recno > 0 and recno <= len(self.bitlist): diff --git a/src/third_party/wiredtiger/test/suite/test_durable_ts03.py b/src/third_party/wiredtiger/test/suite/test_durable_ts03.py index e606a223953..9546c8f267c 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_durable_ts03.py +++ b/src/third_party/wiredtiger/test/suite/test_durable_ts03.py @@ -43,9 +43,9 @@ class test_durable_ts03(wttest.WiredTigerTestCase): uri = 'table:test_durable_ts03' nrows = 3000 self.session.create(uri, 'key_format=i,value_format=u') - valueA = "aaaaa" * 100 - valueB = "bbbbb" * 100 - valueC = "ccccc" * 100 + valueA = b"aaaaa" * 100 + valueB = b"bbbbb" * 100 + valueC = b"ccccc" * 100 # Start with setting a stable and oldest timestamp. self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1) + \ diff --git a/src/third_party/wiredtiger/test/suite/test_empty.py b/src/third_party/wiredtiger/test/suite/test_empty.py index 9b4c53b9774..f7949871a11 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_empty.py +++ b/src/third_party/wiredtiger/test/suite/test_empty.py @@ -46,7 +46,7 @@ class test_empty(wttest.WiredTigerTestCase): # Creating an object and then closing it shouldn't write any blocks. def test_empty_create(self): uri = self.type + self.name - self.session.create(uri, 'key_format=' + self.fmt) + self.session.create(uri, 'key_format=' + self.fmt + ',value_format=S') self.session.close() name = self.name if self.type == "table:": @@ -59,7 +59,7 @@ class test_empty(wttest.WiredTigerTestCase): def empty(self): uri = self.type + self.name self.session = self.conn.open_session() - self.session.create(uri, 'key_format=' + self.fmt) + self.session.create(uri, 'key_format=' + self.fmt + ',value_format=S') # Add a few records to the object and remove them. cursor = self.session.open_cursor(uri, None, None) diff --git a/src/third_party/wiredtiger/test/suite/test_empty_value.py b/src/third_party/wiredtiger/test/suite/test_empty_value.py index 6374f4c2a77..27ff6f2391c 100644 --- a/src/third_party/wiredtiger/test/suite/test_empty_value.py +++ b/src/third_party/wiredtiger/test/suite/test_empty_value.py @@ -47,7 +47,7 @@ class test_row_store_empty_values(wttest.WiredTigerTestCase): # Create the object, open the cursor, insert some records with zero-length values. self.session.create(uri, 'value_format=u,key_format=S') cursor = self.session.open_cursor(uri, None) - for i in xrange(1, nentries + 1): + for i in range(1, nentries + 1): cursor[simple_key(cursor, i)] = "" cursor.close() diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt01.py b/src/third_party/wiredtiger/test/suite/test_encrypt01.py index 323a6c7add2..21a507b7aed 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt01.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt01.py @@ -94,7 +94,7 @@ class test_encrypt01(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(self.uri, None) r = random.Random() r.seed(0) - for idx in xrange(1,self.nrecords): + for idx in range(1,self.nrecords): start = r.randint(0,9) key = self.bigvalue[start:r.randint(0,100)] + str(idx) val = self.bigvalue[start:r.randint(0,10000)] + str(idx) @@ -109,7 +109,7 @@ class test_encrypt01(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(self.uri, None) r.seed(0) - for idx in xrange(1,self.nrecords): + for idx in range(1,self.nrecords): start = r.randint(0,9) key = self.bigvalue[start:r.randint(0,100)] + str(idx) val = self.bigvalue[start:r.randint(0,10000)] + str(idx) diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt02.py b/src/third_party/wiredtiger/test/suite/test_encrypt02.py index 38ff2c04202..5dba7918105 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt02.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt02.py @@ -70,7 +70,7 @@ class test_encrypt02(wttest.WiredTigerTestCase, suite_subprocess): cursor = self.session.open_cursor(self.uri, None) r = random.Random() r.seed(0) - for idx in xrange(1,self.nrecords): + for idx in range(1,self.nrecords): start = r.randint(0,9) key = self.bigvalue[start:r.randint(0,100)] + str(idx) val = self.bigvalue[start:r.randint(0,10000)] + str(idx) @@ -85,7 +85,7 @@ class test_encrypt02(wttest.WiredTigerTestCase, suite_subprocess): cursor = self.session.open_cursor(self.uri, None) r.seed(0) - for idx in xrange(1,self.nrecords): + for idx in range(1,self.nrecords): start = r.randint(0,9) key = self.bigvalue[start:r.randint(0,100)] + str(idx) val = self.bigvalue[start:r.randint(0,10000)] + str(idx) diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt04.py b/src/third_party/wiredtiger/test/suite/test_encrypt04.py index 46be3d4b77a..e58c97b5bac 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt04.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt04.py @@ -120,11 +120,11 @@ class test_encrypt04(wttest.WiredTigerTestCase, suite_subprocess): if str(-1000) in str(err): self.got_forceerror = True raise - self.pr(`conn`) + self.pr(repr(conn)) return conn def create_records(self, cursor, r, low, high): - for idx in xrange(low, high): + for idx in range(low, high): start = r.randint(0,9) key = self.bigvalue[start:r.randint(0,100)] + str(idx) val = self.bigvalue[start:r.randint(0,10000)] + str(idx) @@ -133,13 +133,13 @@ class test_encrypt04(wttest.WiredTigerTestCase, suite_subprocess): cursor.insert() def check_records(self, cursor, r, low, high): - for idx in xrange(low, high): + for idx in range(low, high): start = r.randint(0,9) key = self.bigvalue[start:r.randint(0,100)] + str(idx) val = self.bigvalue[start:r.randint(0,10000)] + str(idx) cursor.set_key(key) self.assertEqual(cursor.search(), 0) - self.assertEquals(cursor.get_value(), val) + self.assertEqual(cursor.get_value(), val) # Evaluate expression, which either must succeed (if expect_okay) # or must fail (if !expect_okay). diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt06.py b/src/third_party/wiredtiger/test/suite/test_encrypt06.py index 1bec3b04570..7663a8d6d7e 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_encrypt06.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt06.py @@ -110,8 +110,9 @@ class test_encrypt06(wttest.WiredTigerTestCase): return (f.read().find(match) != -1) def match_string_in_rundir(self, match): + byte_match = match.encode() for fname in os.listdir('.'): - if self.match_string_in_file(fname, match): + if self.match_string_in_file(fname, byte_match): return True return False @@ -177,7 +178,7 @@ class test_encrypt06(wttest.WiredTigerTestCase): c0 = s.open_cursor(pfx + name0, None) c1 = s.open_cursor(pfx + name1, None) - for idx in xrange(1,self.nrecords): + for idx in range(1,self.nrecords): c0.set_key(str(idx) + txt0) c1.set_key(str(idx) + txt1) c0.set_value(txt0 * (idx % 97), txt0 * 3, txt0 * 5, txt0 * 7) diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt07.py b/src/third_party/wiredtiger/test/suite/test_encrypt07.py index d7ece82ded8..6577e62fa20 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_encrypt07.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt07.py @@ -61,7 +61,7 @@ class test_encrypt07(test_salvage.test_salvage): # (to find a physical spot to damage) we'll need to search for # the rot13 encrypted string. def damage(self, tablename): - self.damage_inner(tablename, self.rot13(self.unique)) + self.damage_inner(tablename, self.rot13(self.unique).encode()) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_env01.py b/src/third_party/wiredtiger/test/suite/test_env01.py index 79ea7b1a594..b4c8591a024 100644 --- a/src/third_party/wiredtiger/test/suite/test_env01.py +++ b/src/third_party/wiredtiger/test/suite/test_env01.py @@ -126,7 +126,7 @@ class test_priv01(wttest.WiredTigerTestCase): edir = 'envdir' os.mkdir(edir) if os.getuid() != os.geteuid(): - print 'Running ' + str(self) + ' as privileged user' + print('Running ' + str(self) + ' as privileged user') self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.common_test(None, edir, None), '/WIREDTIGER_HOME environment variable set but\ @@ -137,7 +137,7 @@ class test_priv01(wttest.WiredTigerTestCase): os.mkdir(edir) privarg = 'use_environment_priv=true' if os.getuid() != os.geteuid(): - print 'Running ' + str(self) + ' as privileged user' + print('Running ' + str(self) + ' as privileged user') self.common_test(None, edir, privarg) self.checkfiles(edir) self.checknofiles(".") diff --git a/src/third_party/wiredtiger/test/suite/test_index01.py b/src/third_party/wiredtiger/test/suite/test_index01.py index cdd0650bca6..dbf47b22b7e 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_index01.py +++ b/src/third_party/wiredtiger/test/suite/test_index01.py @@ -37,9 +37,10 @@ class test_index01(wttest.WiredTigerTestCase): tablename = 'table:' + basename indexbase = 'index:' + basename NUM_INDICES = 6 - index = ['%s:index%d' % (indexbase, i) for i in xrange(NUM_INDICES)] def create_table(self): + self.index = ['%s:index%d' % (self.indexbase, i) \ + for i in range(self.NUM_INDICES)] self.pr('create table') self.session.create(self.tablename, 'key_format=Si,value_format=SSii,columns=(name,ID,dept,job,salary,year)') self.session.create(self.index[0], 'columns=(dept)') @@ -130,7 +131,7 @@ class test_index01(wttest.WiredTigerTestCase): '''Create a table, look for a nonexistent key''' self.create_table() self.check_exists('jones', 10, wiredtiger.WT_NOTFOUND) - for i in xrange(self.NUM_INDICES): + for i in range(self.NUM_INDICES): self.assertEqual(list(self.index_iter(i)), []) self.drop_table() @@ -140,7 +141,7 @@ class test_index01(wttest.WiredTigerTestCase): self.insert('smith', 1, 'HR', 'manager', 100000, 1970) self.check_exists('smith', 1, 0) result = '' - for i in xrange(self.NUM_INDICES): + for i in range(self.NUM_INDICES): result += '\n'.join(repr(cols) for cols in self.index_iter(i)) result += '\n\n' @@ -162,7 +163,7 @@ class test_index01(wttest.WiredTigerTestCase): self.update_nonexistent('Smith', 1, 'HR', 'janitor', 1000, 1970) self.check_exists('smith', 1, 0) result = '' - for i in xrange(self.NUM_INDICES): + for i in range(self.NUM_INDICES): result += '\n'.join(repr(cols) for cols in self.index_iter(i)) result += '\n\n' @@ -186,7 +187,7 @@ class test_index01(wttest.WiredTigerTestCase): self.check_exists('jones', 2, 0) self.insert_duplicate('smith', 1, 'HR', 'manager', 100000, 1970) result = '' - for i in xrange(self.NUM_INDICES): + for i in range(self.NUM_INDICES): result += '\n'.join(repr(cols) for cols in self.index_iter(i)) result += '\n\n' @@ -213,7 +214,7 @@ class test_index01(wttest.WiredTigerTestCase): self.check_exists('smith', 1, 0) self.remove('smith', 1) self.check_exists('smith', 1, wiredtiger.WT_NOTFOUND) - for i in xrange(self.NUM_INDICES): + for i in range(self.NUM_INDICES): self.assertEqual(list(self.index_iter(i)), []) self.drop_table() diff --git a/src/third_party/wiredtiger/test/suite/test_index02.py b/src/third_party/wiredtiger/test/suite/test_index02.py index d92d4d5d59d..86369d9d083 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_index02.py +++ b/src/third_party/wiredtiger/test/suite/test_index02.py @@ -29,6 +29,14 @@ import wiredtiger, wttest from wtscenario import make_scenarios +def cmp(a, b): + if a > b: + return 1 + elif b > a: + return -1 + else: + return 0 + # test_index02.py # test search_near in indices class test_index02(wttest.WiredTigerTestCase): @@ -61,7 +69,7 @@ class test_index02(wttest.WiredTigerTestCase): cur.close() # Retry after reopening - for runs in xrange(2): + for runs in range(2): # search near should find a match cur = self.session.open_cursor(self.indexname, None, None) if self.ncol == 1: @@ -78,14 +86,14 @@ class test_index02(wttest.WiredTigerTestCase): self.session.create(self.tablename, 'key_format=i,value_format=i,columns=(k,v)') self.session.create(self.indexname, self.indexconfig) cur = self.session.open_cursor(self.tablename) - for k in xrange(3): + for k in range(3): cur[k] = 5 * k + 10 cur.close() search_keys = [ 1, 11, 15, 19, 21 ] # search near should find a match - for runs in xrange(2): + for runs in range(2): cur = self.session.open_cursor(self.indexname, None, None) for k in search_keys: if self.ncol == 1: diff --git a/src/third_party/wiredtiger/test/suite/test_index03.py b/src/third_party/wiredtiger/test/suite/test_index03.py index 71312a3db4f..dd6adb75c7e 100644 --- a/src/third_party/wiredtiger/test/suite/test_index03.py +++ b/src/third_party/wiredtiger/test/suite/test_index03.py @@ -59,7 +59,7 @@ class test_index03(wttest.WiredTigerTestCase): c1 = session.open_cursor(uri, None) # Having cursors open across drops is not currently allowed. # On the drop side, we need to begin using the cursor - for i in xrange(100, 200): + for i in range(100, 200): c1[self.key(i)] = self.value(i) self.assertRaises(wiredtiger.WiredTigerError, diff --git a/src/third_party/wiredtiger/test/suite/test_inmem01.py b/src/third_party/wiredtiger/test/suite/test_inmem01.py index 91b3383efce..b86d124cb69 100644 --- a/src/third_party/wiredtiger/test/suite/test_inmem01.py +++ b/src/third_party/wiredtiger/test/suite/test_inmem01.py @@ -102,7 +102,7 @@ class test_inmem01(wttest.WiredTigerTestCase): # Now that the database contains as much data as will fit into # the configured cache, verify removes succeed. cursor = self.session.open_cursor(self.uri, None) - for i in range(1, last_key / 4, 1): + for i in range(1, last_key // 4, 1): cursor.set_key(ds.key(i)) cursor.remove() @@ -125,7 +125,7 @@ class test_inmem01(wttest.WiredTigerTestCase): # Custom "keep filling" helper def fill(self, cursor, ds, start, end): - for i in xrange(start + 1, end + 1): + for i in range(start + 1, end + 1): cursor[ds.key(i)] = ds.value(i) # Keep adding data to the cache until it becomes really full, make sure @@ -163,7 +163,7 @@ class test_inmem01(wttest.WiredTigerTestCase): # many more records into the cache, so don't do as many passes through # the data. checks = 10 if self.valuefmt.endswith('t') else 100 - for run in xrange(checks): + for run in range(checks): ds.check() self.pr('Finished check ' + str(run)) sleep(1) diff --git a/src/third_party/wiredtiger/test/suite/test_inmem02.py b/src/third_party/wiredtiger/test/suite/test_inmem02.py index f3f69a0e7ac..113fe88299b 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_inmem02.py +++ b/src/third_party/wiredtiger/test/suite/test_inmem02.py @@ -44,7 +44,8 @@ class test_inmem02(wttest.WiredTigerTestCase): # Create a new table that is allowed to exceed the cache size, do this # before filling the cache so that the create succeeds self.session.create( - self.uri + '_over', 'ignore_in_memory_cache_size=true') + self.uri + '_over', + 'key_format=S,value_format=S,ignore_in_memory_cache_size=true') # Populate a table with enough data to fill the cache. msg = '/WT_CACHE_FULL.*/' diff --git a/src/third_party/wiredtiger/test/suite/test_join02.py b/src/third_party/wiredtiger/test/suite/test_join02.py index 7fdb0668a64..026f926f5e8 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_join02.py +++ b/src/third_party/wiredtiger/test/suite/test_join02.py @@ -65,7 +65,7 @@ class test_join02(wttest.WiredTigerTestCase): def gen_values(self, i): s = str(i) - x = 'x' * i + x = b'x' * i rs = s[::-1] f = int(s[0:1]) return [i, s, x, rs, f] @@ -225,11 +225,11 @@ class test_join02(wttest.WiredTigerTestCase): c1b.eqmembers = self.mkmbr(lambda x: str(x) == '733') c1b.name = 'c1b' - c2a.low = [ 'x' * 321 ] + c2a.low = [ b'x' * 321 ] c2a.gtmembers = self.mkmbr(lambda x: x > 321) c2a.eqmembers = self.mkmbr(lambda x: x == 321) c2a.name = 'c2a' - c2b.high = [ 'x' * 765 ] + c2b.high = [ b'x' * 765 ] c2b.ltmembers = self.mkmbr(lambda x: x < 765) c2b.eqmembers = self.mkmbr(lambda x: x == 765) c2b.name = 'c2b' diff --git a/src/third_party/wiredtiger/test/suite/test_jsondump02.py b/src/third_party/wiredtiger/test/suite/test_jsondump02.py index 6eea26df6af..b22212a651f 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_jsondump02.py +++ b/src/third_party/wiredtiger/test/suite/test_jsondump02.py @@ -26,10 +26,21 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import os +import os, sys import wiredtiger, wttest from suite_subprocess import suite_subprocess +# In Python2, Unicode and strings are different types, +# and need to be converted. In Python3, there is no separate +# unicode type, unicode characters are just embedded as UTF-8 +# in strings. +_python3 = (sys.version_info >= (3, 0, 0)) +def encode(s): + if _python3: + return s + else: + return s.encode('utf-8') + # test_jsondump.py # Test dump output from json cursors. class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): @@ -117,13 +128,12 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): self.set_kv(self.table_uri1, 'KEY001', '\'\"({[]})\"\'\\, etc. allowed') # \u03c0 is pi in Unicode, converted by Python to UTF-8: 0xcf 0x80. # Here's how UTF-8 might be used. - self.set_kv(self.table_uri1, 'KEY002', u'\u03c0'.encode('utf-8')) - # 0xf5-0xff are illegal in Unicode, but may occur legally in C strings. - self.set_kv(self.table_uri1, 'KEY003', '\xff\xfe') + self.set_kv(self.table_uri1, 'KEY002', encode(u'\u03c0')) + self.set_kv(self.table_uri1, 'KEY003', encode(u'\u0abc')) self.set_kv2(self.table_uri2, 'KEY000', 123, 'str0') self.set_kv2(self.table_uri2, 'KEY001', 234, 'str1') - self.set_kv(self.table_uri3, 1, '\x01\x02\x03') - self.set_kv(self.table_uri3, 2, '\x77\x88\x99\x00\xff\xfe') + self.set_kv(self.table_uri3, 1, b'\x01\x02\x03') + self.set_kv(self.table_uri3, 2, b'\x77\x88\x99\x00\x66\x55') self.populate_squarecube(self.table_uri4) table1_json = ( @@ -131,7 +141,7 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): ('"key0" : "KEY001"', '"value0" : ' + '"\'\\\"({[]})\\\"\'\\\\, etc. allowed"'), ('"key0" : "KEY002"', '"value0" : "\\u00cf\\u0080"'), - ('"key0" : "KEY003"', '"value0" : "\\u00ff\\u00fe"')) + ('"key0" : "KEY003"', '"value0" : "\\u00e0\\u00aa\\u00bc"')) self.check_json(self.table_uri1, table1_json) self.session.truncate(self.table_uri1, None, None, None) @@ -156,7 +166,7 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): # bad tokens self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.load_json(self.table_uri2, - (('"abc\u"', ''),)), + (('"abc\\u"', ''),)), '/invalid Unicode/') # bad tokens @@ -222,7 +232,7 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): table3_json = ( ('"key0" : 1', '"value0" : "\\u0001\\u0002\\u0003"'), ('"key0" : 2', - '"value0" : "\\u0077\\u0088\\u0099\\u0000\\u00ff\\u00fe"')) + '"value0" : "\\u0077\\u0088\\u0099\\u0000\\u0066\\u0055"')) self.check_json(self.table_uri3, table3_json) table4_json = ( ('"ikey" : 1,\n"Skey" : "key1"', @@ -323,14 +333,25 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): # i==0 : v:[0x00, 0x01, 0x02] # i==1 : v:[0x01, 0x02, 0x03] # etc. - # A null byte is disallowed in a string value, it is replaced by 'X' + # A null byte or any byte >= 0x7f is disallowed in a string value, + # it is replaced by 'X' def generate_value(self, i, v, isstring): for j in range(0, 3): - val = (i + j) % 256 - if isstring and val == 0: - val = 88 # 'X' + val = i + j + if val >= 256 or (isstring and (val == 0 or val >= 128)): + val = ord('X') v[j] = val + # In Python3, we cannot simply shove random bytes with values >= 0x80 + # into a string, as strings are unicode aware, so we test only up to 0x80. + # Real Unicode strings are tested elsewhere. + def bytes_to_str(self, barray): + mask = 0x7f + result = '' + for b in barray: + result += chr(b & mask) + return result + def test_json_all_bytes(self): """ Test the generated JSON for all byte values in byte array and @@ -343,12 +364,15 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): c6 = self.session.open_cursor(self.table_uri6, None, None) k = bytearray(b'\x00\x00') v = bytearray(b'\x00\x00\x00') - for i in range(0, 512): + for i in range(0, 256): self.generate_key(i, k) self.generate_value(i, v, False) - c5[str(k)] = str(v) + # A 'u' format requires a bytes type with Python3 + c5[bytes(k)] = bytes(v) self.generate_value(i, v, True) # no embedded nuls - c6[str(k)] = str(v) + kstr = self.bytes_to_str(k) + vstr = self.bytes_to_str(v) + c6[kstr] = vstr c5.close() c6.close() @@ -381,10 +405,10 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): table5_json = [] table6_json = [] - for i in range(0, 512): + for i in range(0, 256): self.generate_key(i, k) self.generate_value(i, v, False) - j = i if (i > 0 and i < 254) or (i > 256 and i < 510) else 88 + j = i if (i > 0 and i < 126) else 88 table5_json.append(('"key0" : "' + bin_unicode[k[0]] + bin_unicode[k[1]] + '"', '"value0" : "' + bin_unicode[v[0]] + diff --git a/src/third_party/wiredtiger/test/suite/test_las01.py b/src/third_party/wiredtiger/test/suite/test_las01.py index 91dbab64450..25f34c01952 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_las01.py +++ b/src/third_party/wiredtiger/test/suite/test_las01.py @@ -49,7 +49,7 @@ class test_las01(wttest.WiredTigerTestCase): session.begin_transaction() cursor.set_key(ds.key(nrows + i)) cursor.set_value(value) - self.assertEquals(cursor.update(), 0) + self.assertEqual(cursor.update(), 0) if timestamp == True: session.commit_transaction('commit_timestamp=' + timestamp_str(i + 1)) cursor.close() @@ -82,10 +82,10 @@ class test_las01(wttest.WiredTigerTestCase): cursor = session.open_cursor(uri, None) # Skip the initial rows, which were not updated for i in range(0, nrows+1): - self.assertEquals(cursor.next(), 0) + self.assertEqual(cursor.next(), 0) if (check_value != cursor.get_value()): - print "Check value : " + str(check_value) - print "value : " + str(cursor.get_value()) + print("Check value : " + str(check_value)) + print("value : " + str(cursor.get_value())) self.assertTrue(check_value == cursor.get_value()) cursor.close() session.close() @@ -97,20 +97,20 @@ class test_las01(wttest.WiredTigerTestCase): nrows = 100 ds = SimpleDataSet(self, uri, nrows, key_format="S", value_format='u') ds.populate() - bigvalue = "aaaaa" * 100 + bigvalue = b"aaaaa" * 100 # Initially load huge data cursor = self.session.open_cursor(uri) for i in range(1, 10000): cursor.set_key(ds.key(nrows + i)) cursor.set_value(bigvalue) - self.assertEquals(cursor.insert(), 0) + self.assertEqual(cursor.insert(), 0) cursor.close() self.session.checkpoint() # Scenario: 1 # Check to see LAS working with old snapshot - bigvalue1 = "bbbbb" * 100 + bigvalue1 = b"bbbbb" * 100 self.session.snapshot("name=xxx") # Update the values in different session after snapshot self.large_updates(self.session, uri, bigvalue1, ds, nrows) @@ -120,7 +120,7 @@ class test_las01(wttest.WiredTigerTestCase): # Scenario: 2 # Check to see LAS working with old reader - bigvalue2 = "ccccc" * 100 + bigvalue2 = b"ccccc" * 100 session2 = self.conn.open_session() session2.begin_transaction('isolation=snapshot') self.large_updates(self.session, uri, bigvalue2, ds, nrows) @@ -131,8 +131,8 @@ class test_las01(wttest.WiredTigerTestCase): # Scenario: 3 # Check to see LAS working with modify operations - bigvalue3 = "ccccc" * 100 - bigvalue3 = 'AA' + bigvalue3[2:] + bigvalue3 = b"ccccc" * 100 + bigvalue3 = b'AA' + bigvalue3[2:] session2 = self.conn.open_session() session2.begin_transaction('isolation=snapshot') # Apply two modify operations - replacing the first two items with 'A' @@ -147,7 +147,7 @@ class test_las01(wttest.WiredTigerTestCase): # Scenario: 4 # Check to see LAS working with old timestamp - bigvalue4 = "ddddd" * 100 + bigvalue4 = b"ddddd" * 100 self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) self.large_updates(self.session, uri, bigvalue4, ds, nrows, timestamp=True) # Check to see data can be see only till the stable_timestamp diff --git a/src/third_party/wiredtiger/test/suite/test_las02.py b/src/third_party/wiredtiger/test/suite/test_las02.py index d7d54659d51..2ab4d4ec918 100644 --- a/src/third_party/wiredtiger/test/suite/test_las02.py +++ b/src/third_party/wiredtiger/test/suite/test_las02.py @@ -79,17 +79,17 @@ class test_las02(wttest.WiredTigerTestCase): ',stable_timestamp=' + timestamp_str(1)) bigvalue = "aaaaa" * 100 - self.large_updates(uri, bigvalue, ds, nrows / 3, 1) + self.large_updates(uri, bigvalue, ds, nrows // 3, 1) # Check that all updates are seen - self.check(bigvalue, uri, nrows / 3, 1) + self.check(bigvalue, uri, nrows // 3, 1) # Check to see lookaside working with old timestamp bigvalue2 = "ddddd" * 100 self.large_updates(uri, bigvalue2, ds, nrows, 100) # Check that the new updates are only seen after the update timestamp - self.check(bigvalue, uri, nrows / 3, 1) + self.check(bigvalue, uri, nrows // 3, 1) self.check(bigvalue2, uri, nrows, 100) # Force out most of the pages by updating a different tree @@ -98,16 +98,16 @@ class test_las02(wttest.WiredTigerTestCase): # Now truncate half of the records self.session.begin_transaction() end = self.session.open_cursor(uri) - end.set_key(ds.key(nrows / 2)) + end.set_key(ds.key(nrows // 2)) self.session.truncate(None, None, end) end.close() self.session.commit_transaction('commit_timestamp=' + timestamp_str(200)) # Check that the truncate is visible after commit - self.check(bigvalue2, uri, nrows / 2, 200) + self.check(bigvalue2, uri, nrows // 2, 200) # Repeat earlier checks - self.check(bigvalue, uri, nrows / 3, 1) + self.check(bigvalue, uri, nrows // 3, 1) self.check(bigvalue2, uri, nrows, 100) if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_las03.py b/src/third_party/wiredtiger/test/suite/test_las03.py index 3c87c3503b8..60ad25cf6d7 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_las03.py +++ b/src/third_party/wiredtiger/test/suite/test_las03.py @@ -63,7 +63,7 @@ class test_las03(wttest.WiredTigerTestCase): nrows = 100 ds = SimpleDataSet(self, uri, nrows, key_format="S", value_format='u') ds.populate() - bigvalue = "aaaaa" * 100 + bigvalue = b"aaaaa" * 100 # Initially load huge data cursor = self.session.open_cursor(uri) @@ -73,7 +73,7 @@ class test_las03(wttest.WiredTigerTestCase): self.session.checkpoint() # Check to see LAS working with old timestamp - bigvalue2 = "ddddd" * 100 + bigvalue2 = b"ddddd" * 100 self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) las_writes_start = self.get_stat(stat.conn.cache_write_lookaside) self.large_updates(self.session, uri, bigvalue2, ds, nrows, 10000) diff --git a/src/third_party/wiredtiger/test/suite/test_lsm02.py b/src/third_party/wiredtiger/test/suite/test_lsm02.py index 344a9c9229e..278f9fa82f0 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_lsm02.py +++ b/src/third_party/wiredtiger/test/suite/test_lsm02.py @@ -43,19 +43,19 @@ class test_lsm02(wttest.WiredTigerTestCase): cursor.set_key(key) cursor.search() if value != cursor.get_value(): - print 'Unexpected value from LSM tree' + print('Unexpected value from LSM tree') cursor.close() # Put some special values that start with the LSM tombstone def test_lsm_tombstone(self): self.session.create(self.uri, 'key_format=S,value_format=u') - v = '\x14\x14' + v = b'\x14\x14' self.add_key(self.uri, 'k1', v) self.verify_key_exists(self.uri, 'k1', v) - v = '\x14\x14\0\0\0\0\0\0' + v = b'\x14\x14\0\0\0\0\0\0' self.add_key(self.uri, 'k2', v) self.verify_key_exists(self.uri, 'k2', v) - v += 'a' * 1000 + v += b'a' * 1000 self.add_key(self.uri, 'k3', v) self.verify_key_exists(self.uri, 'k3', v) diff --git a/src/third_party/wiredtiger/test/suite/test_metadata_cursor01.py b/src/third_party/wiredtiger/test/suite/test_metadata_cursor01.py index e5a6efe3a0b..40e38d884dc 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_metadata_cursor01.py +++ b/src/third_party/wiredtiger/test/suite/test_metadata_cursor01.py @@ -48,7 +48,7 @@ class test_metadata_cursor01(wttest.WiredTigerTestCase): if self.tablekind == 'row': return 'key' + str(i) else: - return long(i+1) + return self.recno(i+1) def genvalue(self, i): if self.tablekind == 'fix': diff --git a/src/third_party/wiredtiger/test/suite/test_nsnap01.py b/src/third_party/wiredtiger/test/suite/test_nsnap01.py index 50f7b8c745b..995b1d2d874 100644 --- a/src/third_party/wiredtiger/test/suite/test_nsnap01.py +++ b/src/third_party/wiredtiger/test/suite/test_nsnap01.py @@ -63,9 +63,9 @@ class test_nsnap01(wttest.WiredTigerTestCase, suite_subprocess): # if there are more than 10 snapshots active, drop the first half snapshots = [] c = self.session.open_cursor(self.uri) - for n in xrange(self.nrows / self.nrows_per_snap): + for n in range(self.nrows // self.nrows_per_snap): if len(snapshots) > self.nsnapshots: - middle = len(snapshots) / 2 + middle = len(snapshots) // 2 dropcfg = ",drop=(to=%d)" % snapshots[middle][0] snapshots = snapshots[middle + 1:] else: @@ -73,10 +73,10 @@ class test_nsnap01(wttest.WiredTigerTestCase, suite_subprocess): self.session.snapshot("name=%d%s" % (n, dropcfg)) snapshots.append((n, end - start)) - for i in xrange(2 * self.nrows_per_snap): + for i in range(2 * self.nrows_per_snap): c[end + i] = "some value" end += 2 * self.nrows_per_snap - for i in xrange(self.nrows_per_snap): + for i in range(self.nrows_per_snap): del c[start + i] start += self.nrows_per_snap diff --git a/src/third_party/wiredtiger/test/suite/test_nsnap02.py b/src/third_party/wiredtiger/test/suite/test_nsnap02.py index e57c26633ca..be979bd9206 100644 --- a/src/third_party/wiredtiger/test/suite/test_nsnap02.py +++ b/src/third_party/wiredtiger/test/suite/test_nsnap02.py @@ -70,13 +70,13 @@ class test_nsnap02(wttest.WiredTigerTestCase, suite_subprocess): # Each snapshot removes a (smaller) bunch of old data snapshots = [] c = self.session.open_cursor(self.uri) - for n in xrange(self.nsnapshots): + for n in range(self.nsnapshots): self.session.snapshot("name=%d" % (n)) snapshots.append((n, end - start, 0)) - for i in xrange(2 * self.nrows_per_snap): + for i in range(2 * self.nrows_per_snap): c[end + i] = "some value" end += 2 * self.nrows_per_snap - for i in xrange(self.nrows_per_snap): + for i in range(self.nrows_per_snap): del c[start + i] start += self.nrows_per_snap return snapshots diff --git a/src/third_party/wiredtiger/test/suite/test_nsnap03.py b/src/third_party/wiredtiger/test/suite/test_nsnap03.py index e69a58f1925..44eac820138 100644 --- a/src/third_party/wiredtiger/test/suite/test_nsnap03.py +++ b/src/third_party/wiredtiger/test/suite/test_nsnap03.py @@ -65,9 +65,9 @@ class test_nsnap03(wttest.WiredTigerTestCase, suite_subprocess): # if there are more than 10 snapshots active, drop the first half snapshots = [] c = self.session.open_cursor(self.uri) - for n in xrange(self.nrows / self.nrows_per_snap): + for n in range(self.nrows // self.nrows_per_snap): if len(snapshots) > self.nsnapshots: - middle = len(snapshots) / 2 + middle = len(snapshots) // 2 dropcfg = ",drop=(to=%d)" % snapshots[middle][0] snapshots = snapshots[middle + 1:] else: @@ -81,10 +81,10 @@ class test_nsnap03(wttest.WiredTigerTestCase, suite_subprocess): self.session.snapshot("name=%d%s" % (n, dropcfg)) snapshots.append((n, end - start)) - for i in xrange(2 * self.nrows_per_snap): + for i in range(2 * self.nrows_per_snap): c[end + i] = "some value" end += 2 * self.nrows_per_snap - for i in xrange(self.nrows_per_snap): + for i in range(self.nrows_per_snap): del c[start + i] start += self.nrows_per_snap diff --git a/src/third_party/wiredtiger/test/suite/test_nsnap04.py b/src/third_party/wiredtiger/test/suite/test_nsnap04.py index eecf0540106..df56c4cffec 100644 --- a/src/third_party/wiredtiger/test/suite/test_nsnap04.py +++ b/src/third_party/wiredtiger/test/suite/test_nsnap04.py @@ -60,7 +60,7 @@ class test_nsnap04(wttest.WiredTigerTestCase, suite_subprocess): snapshots = [] c = self.session.open_cursor(self.uri) - for i in xrange(self.nrows_per_itr): + for i in range(self.nrows_per_itr): c[i] = "some value" # Start a new transaction in a different session @@ -75,7 +75,7 @@ class test_nsnap04(wttest.WiredTigerTestCase, suite_subprocess): self.check_named_snapshot(0, self.nrows_per_itr) # Insert some more content using the original session. - for i in xrange(self.nrows_per_itr): + for i in range(self.nrows_per_itr): c[2 * self.nrows_per_itr + i] = "some value" self.check_named_snapshot(0, self.nrows_per_itr) @@ -91,7 +91,7 @@ class test_nsnap04(wttest.WiredTigerTestCase, suite_subprocess): snapshots = [] c = self.session.open_cursor(self.uri) - for i in xrange(self.nrows_per_itr): + for i in range(self.nrows_per_itr): c[i] = "some value" self.session.begin_transaction("isolation=snapshot") @@ -103,7 +103,7 @@ class test_nsnap04(wttest.WiredTigerTestCase, suite_subprocess): self.check_named_snapshot(0, self.nrows_per_itr) # Insert some more content using the active session. - for i in xrange(self.nrows_per_itr): + for i in range(self.nrows_per_itr): c[self.nrows_per_itr + i] = "some value" self.check_named_snapshot(0, 2 * self.nrows_per_itr) diff --git a/src/third_party/wiredtiger/test/suite/test_pack.py b/src/third_party/wiredtiger/test/suite/test_pack.py index fd0231c4a74..83893ca97af 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_pack.py +++ b/src/third_party/wiredtiger/test/suite/test_pack.py @@ -50,7 +50,7 @@ class test_pack(wttest.WiredTigerTestCase): uri = 'table:' + test_pack.name + '-' + fmtname idx_uri = 'index:' + test_pack.name + '-' + fmtname + ':inverse' nargs = len(v) - colnames = ",".join("v" + str(x) for x in xrange(nargs)) + colnames = ",".join("v" + str(x) for x in range(nargs)) self.session.create(uri, "columns=(k," + colnames + ")," + "key_format=i,value_format=" + fmt) self.session.create(idx_uri, "columns=(" + colnames + ")") @@ -92,16 +92,16 @@ class test_pack(wttest.WiredTigerTestCase): self.check('10SS', 'aaaaa\x00\x00\x00\x00\x00', 'something') self.check('S10S', 'something', 'aaaaa\x00\x00\x00\x00\x00') - self.check('u', r"\x42" * 20) - self.check('uu', r"\x42" * 10, r"\x42" * 10) - self.check('3u', r"\x4") - self.check('3uu', r"\x4", r"\x42" * 10) - self.check('u3u', r"\x42" * 10, r"\x4") - self.check('u', '\x00') - self.check('u', '') - self.check('uu', '', '\x00') - self.check('uu', '\x00', '') - self.check('uu', '', '') + self.check('u', b"\x42" * 20) + self.check('uu', b"\x42" * 10, b"\x42" * 10) + self.check('3u', b"\x04\x03\x02") + self.check('3uu', b"\x04\x03\x02", b"\x42" * 10) + self.check('u3u', b"\x42" * 10, b"\x04\x03\x02") + self.check('u', b'\x00') + self.check('u', b'') + self.check('uu', b'', b'\x00') + self.check('uu', b'\x00', b'') + self.check('uu', b'', b'') self.check('s', "4") self.check("1s", "4") diff --git a/src/third_party/wiredtiger/test/suite/test_prepare01.py b/src/third_party/wiredtiger/test/suite/test_prepare01.py index c23d0a869b2..596ceb6fdf8 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare01.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare01.py @@ -112,8 +112,8 @@ class test_prepare01(wttest.WiredTigerTestCase): self.check(cursor, 0, 0) self.session.begin_transaction("ignore_prepare=false") - for i in xrange(self.nentries): - if i > 0 and i % (self.nentries / 37) == 0: + for i in range(self.nentries): + if i > 0 and i % (self.nentries // 37) == 0: self.check(cursor, committed, i) self.session.prepare_transaction("prepare_timestamp=2a") self.session.timestamp_transaction("commit_timestamp=3a") diff --git a/src/third_party/wiredtiger/test/suite/test_prepare03.py b/src/third_party/wiredtiger/test/suite/test_prepare03.py index 512efe0c008..9abfa7b2957 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_prepare03.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare03.py @@ -56,7 +56,7 @@ class test_prepare03(wttest.WiredTigerTestCase): if self.tablekind == 'row': return 'key' + str(i) else: - return long(i+1) + return self.recno(i+1) def genvalue(self, i): if self.tablekind == 'fix': @@ -161,7 +161,7 @@ class test_prepare03(wttest.WiredTigerTestCase): # Search for a specific key. # Verify we get the expected error and then later we can update and # remove it. - cursor.set_key(self.genkey(self.nentries/2)) + cursor.set_key(self.genkey(self.nentries//2)) self.session.begin_transaction() self.session.prepare_transaction("prepare_timestamp=2a") self.assertRaisesWithMessage(wiredtiger.WiredTigerError, @@ -178,7 +178,7 @@ class test_prepare03(wttest.WiredTigerTestCase): self.session.timestamp_transaction("durable_timestamp=2b") self.session.commit_transaction() cursor.search() - cursor.set_value(self.genvalue(self.nentries + self.nentries/2)) + cursor.set_value(self.genvalue(self.nentries + self.nentries//2)) cursor.update() cursor.remove() diff --git a/src/third_party/wiredtiger/test/suite/test_prepare04.py b/src/third_party/wiredtiger/test/suite/test_prepare04.py index 4193c3299b7..9d3d78804c0 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare04.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare04.py @@ -72,7 +72,7 @@ class test_prepare04(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(self.uri) # Insert keys 1..100 each with timestamp=key, in some order - orig_keys = range(1, 101) + orig_keys = list(range(1, 101)) keys = orig_keys[:] random.shuffle(keys) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_lookaside01.py b/src/third_party/wiredtiger/test/suite/test_prepare_lookaside01.py index 9b17aa8ac9b..acfbefb7312 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_prepare_lookaside01.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_lookaside01.py @@ -58,7 +58,7 @@ class test_prepare_lookaside01(wttest.WiredTigerTestCase): self.conn.set_timestamp('stable_timestamp=' + timestamp_str(1)) # Commit some updates to get eviction and lookaside fired up - bigvalue1 = "bbbbb" * 100 + bigvalue1 = b"bbbbb" * 100 cursor = self.session.open_cursor(uri) for i in range(1, nsessions * nkeys): self.session.begin_transaction() @@ -71,7 +71,7 @@ class test_prepare_lookaside01(wttest.WiredTigerTestCase): # prepared updates to the lookaside sessions = [0] * nsessions cursors = [0] * nsessions - bigvalue2 = "ccccc" * 100 + bigvalue2 = b"ccccc" * 100 for j in range (0, nsessions): sessions[j] = self.conn.open_session() sessions[j].begin_transaction("isolation=snapshot") @@ -88,7 +88,7 @@ class test_prepare_lookaside01(wttest.WiredTigerTestCase): # Commit more regular updates. To do this, the pages that were just # evicted need to be read back. This ensures reading prepared updates # from the lookaside - bigvalue3 = "ddddd" * 100 + bigvalue3 = b"ddddd" * 100 cursor = self.session.open_cursor(uri) for i in range(1, nsessions * nkeys): self.session.begin_transaction() @@ -110,7 +110,7 @@ class test_prepare_lookaside01(wttest.WiredTigerTestCase): nrows = 100 ds = SimpleDataSet(self, uri, nrows, key_format="S", value_format='u') ds.populate() - bigvalue = "aaaaa" * 100 + bigvalue = b"aaaaa" * 100 # Initially load huge data cursor = self.session.open_cursor(uri) diff --git a/src/third_party/wiredtiger/test/suite/test_prepare_lookaside02.py b/src/third_party/wiredtiger/test/suite/test_prepare_lookaside02.py index 01c59885874..daeeff28fef 100644 --- a/src/third_party/wiredtiger/test/suite/test_prepare_lookaside02.py +++ b/src/third_party/wiredtiger/test/suite/test_prepare_lookaside02.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. # # test_prepare_lookaside02.py -# Prepare updates can be resolved for both commit / rollback operations. +# Prepare updates can be resolved for both commit // rollback operations. # from helper import copy_wiredtiger_home @@ -63,7 +63,7 @@ class test_prepare_lookaside02(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(self.uri) # Insert keys 1..100 each with timestamp=key, in some order - orig_keys = range(1, 101) + orig_keys = list(range(1, 101)) keys = orig_keys[:] random.shuffle(keys) diff --git a/src/third_party/wiredtiger/test/suite/test_readonly01.py b/src/third_party/wiredtiger/test/suite/test_readonly01.py index 21e4c735374..062b0698048 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_readonly01.py +++ b/src/third_party/wiredtiger/test/suite/test_readonly01.py @@ -100,8 +100,8 @@ class test_readonly01(wttest.WiredTigerTestCase, suite_subprocess): if self.dirchmod and os.name == 'posix': for f in os.listdir(self.home): if os.path.isfile(f): - os.chmod(f, 0444) - os.chmod(self.home, 0555) + os.chmod(f, 0o444) + os.chmod(self.home, 0o555) self.conn = self.setUpConnectionOpen(self.home) self.session = self.setUpSessionOpen(self.conn) diff --git a/src/third_party/wiredtiger/test/suite/test_reconfig02.py b/src/third_party/wiredtiger/test/suite/test_reconfig02.py index 7d5a88c0d41..68e9704b879 100644 --- a/src/third_party/wiredtiger/test/suite/test_reconfig02.py +++ b/src/third_party/wiredtiger/test/suite/test_reconfig02.py @@ -81,7 +81,7 @@ class test_reconfig02(wttest.WiredTigerTestCase): # # Potentially loop a few times in case it is a very slow system. self.conn.reconfigure("log=(prealloc=true)") - for x in xrange(0, 100): + for x in range(0, 100): time.sleep(1) prep_logs = fnmatch.filter(os.listdir('.'), "*Prep*") if len(prep_logs) != 0: diff --git a/src/third_party/wiredtiger/test/suite/test_salvage.py b/src/third_party/wiredtiger/test/suite/test_salvage.py index ee9a3cde687..0ccff73373b 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_salvage.py +++ b/src/third_party/wiredtiger/test/suite/test_salvage.py @@ -46,7 +46,7 @@ class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): key = '' for i in range(0, self.nentries): key += str(i) - if i == self.nentries / 2: + if i == self.nentries // 2: val = self.unique + '0' else: val = key + key @@ -62,7 +62,7 @@ class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): i = 0 for gotkey, gotval in cursor: wantkey += str(i) - if i == self.nentries / 2: + if i == self.nentries // 2: wantval = self.unique + '0' else: wantval = wantkey + wantkey @@ -87,7 +87,7 @@ class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): wantkey += str(i) if gotkey != wantkey: continue - if i == self.nentries / 2: + if i == self.nentries // 2: wantval = self.unique + '0' else: wantval = wantkey + wantkey @@ -106,7 +106,16 @@ class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): cursor.close() def damage(self, tablename): - self.damage_inner(tablename, self.unique) + self.damage_inner(tablename, self.unique.encode()) + + def read_byte(self, fp): + """ + Return a single byte from a file opened in binary mode. + """ + c = fp.read(1) + if self.is_python3(): + c = c[0] # In python3, the read returns bytes (an array). + return c def damage_inner(self, tablename, unique): """ @@ -114,6 +123,7 @@ class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): and modify it. """ self.close_conn() + self.assertTrue(type(unique) == bytes) # we close the connection to guarantee everything is # flushed and closed from the WT point of view. filename = tablename + ".wt" @@ -123,19 +133,19 @@ class test_salvage(wttest.WiredTigerTestCase, suite_subprocess): match = unique matchlen = len(match) flen = os.fstat(fp.fileno()).st_size - c = fp.read(1) + c = self.read_byte(fp) while fp.tell() != flen: if match[matchpos] == c: matchpos += 1 if matchpos == matchlen: # We're already positioned, so alter it fp.seek(-1, 1) - fp.write('G') + fp.write(b'G') matchpos = 0 found = 1 else: matchpos = 0 - c = fp.read(1) + c = self.read_byte(fp) # Make sure we found the embedded string self.assertTrue(found) fp.close() diff --git a/src/third_party/wiredtiger/test/suite/test_schema02.py b/src/third_party/wiredtiger/test/suite/test_schema02.py index 85091913a6e..87f64336fe0 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_schema02.py +++ b/src/third_party/wiredtiger/test/suite/test_schema02.py @@ -182,7 +182,7 @@ class test_schema02(wttest.WiredTigerTestCase): cursor = self.session.open_cursor('table:main', None, None) # spot check via search n = self.nentries - for i in (n / 5, 0, n - 1, n - 2, 1): + for i in (n // 5, 0, n - 1, n - 2, 1): cursor.set_key(i, 'key' + str(i)) square = i * i cube = square * i diff --git a/src/third_party/wiredtiger/test/suite/test_schema03.py b/src/third_party/wiredtiger/test/suite/test_schema03.py index 82b106ea5ae..76d709f2eae 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_schema03.py +++ b/src/third_party/wiredtiger/test/suite/test_schema03.py @@ -79,7 +79,7 @@ class tabconfig: elif format == 'i': keys.append(rev) elif format == 'r': - keys.append(long(i+1)) + keys.append(self.recno(i+1)) return keys def gen_values(self, i): @@ -315,7 +315,7 @@ class test_schema03(wttest.WiredTigerTestCase): if self.SHOW_PYTHON: if self.SHOW_PYTHON_ONLY_TABLE == None or self.current_table in self.SHOW_PYTHON_ONLY_TABLE: if self.SHOW_PYTHON_ONLY_SCEN == None or self.scenario_number in self.SHOW_PYTHON_ONLY_SCEN: - print ' ' + s + print(' ' + s) def join_names(self, sep, prefix, list): return sep.join([prefix + str(val) for val in list]) @@ -330,14 +330,14 @@ class test_schema03(wttest.WiredTigerTestCase): def finished_step(self, name): if self.s_restart == name: - print " # Reopening connection at step: " + name + print(" # Reopening connection at step: " + name) self.reopen_conn() def test_schema(self): rand = suite_random.suite_random() if self.SHOW_PYTHON: - print ' ################################################' - print ' # Running scenario ' + str(self.scenario_number) + print(' ################################################') + print(' # Running scenario ' + str(self.scenario_number)) ntables = self.s_ntable @@ -431,8 +431,8 @@ class test_schema03(wttest.WiredTigerTestCase): self.show_python("self.session.create('table:" + tc.tablename + "', '" + config + "')") self.session.create("table:" + tc.tablename, config) - tc.columns_for_groups(range(tc.nkeys, tc.nkeys + tc.nvalues)) - tc.columns_for_indices(range(0, tc.nkeys + tc.nvalues)) + tc.columns_for_groups(list(range(tc.nkeys, tc.nkeys + tc.nvalues))) + tc.columns_for_indices(list(range(0, tc.nkeys + tc.nvalues))) self.finished_step('table') @@ -461,7 +461,7 @@ class test_schema03(wttest.WiredTigerTestCase): for tc in tabconfigs: self.current_table = tc.tableidx max = rand.rand_range(0, self.nentries) - self.populate(tc, xrange(0, max)) + self.populate(tc, list(range(0, max))) self.finished_step('populate0') @@ -476,7 +476,7 @@ class test_schema03(wttest.WiredTigerTestCase): # populate second batch for tc in tabconfigs: self.current_table = tc.tableidx - self.populate(tc, xrange(tc.nentries, self.nentries)) + self.populate(tc, list(range(tc.nentries, self.nentries))) self.finished_step('populate1') diff --git a/src/third_party/wiredtiger/test/suite/test_schema04.py b/src/third_party/wiredtiger/test/suite/test_schema04.py index a69846f7558..157c89802e1 100644 --- a/src/third_party/wiredtiger/test/suite/test_schema04.py +++ b/src/third_party/wiredtiger/test/suite/test_schema04.py @@ -66,9 +66,9 @@ class test_schema04(wttest.WiredTigerTestCase): cursor = self.session.open_cursor('table:schema04', None, None) if phase == 0: range_from = 0 - range_to = self.nentries / 2 + range_to = self.nentries // 2 else: - range_from = self.nentries / 2 + range_from = self.nentries // 2 range_to = self.nentries for i in range(range_from, range_to): diff --git a/src/third_party/wiredtiger/test/suite/test_schema05.py b/src/third_party/wiredtiger/test/suite/test_schema05.py index 5a9be65e315..9a5e5d49c31 100644 --- a/src/third_party/wiredtiger/test/suite/test_schema05.py +++ b/src/third_party/wiredtiger/test/suite/test_schema05.py @@ -84,9 +84,9 @@ class test_schema05(wttest.WiredTigerTestCase): cursor = self.session.open_cursor('table:schema05', None, None) if phase == 0: range_from = 0 - range_to = self.nentries / 2 + range_to = self.nentries // 2 elif phase == 1: - range_from = self.nentries / 2 + range_from = self.nentries // 2 range_to = self.nentries - 5 else: range_from = self.nentries - 5 diff --git a/src/third_party/wiredtiger/test/suite/test_schema06.py b/src/third_party/wiredtiger/test/suite/test_schema06.py index 818525fafd4..f516175bffc 100644 --- a/src/third_party/wiredtiger/test/suite/test_schema06.py +++ b/src/third_party/wiredtiger/test/suite/test_schema06.py @@ -91,7 +91,7 @@ class test_schema06(wttest.WiredTigerTestCase): cursor = self.session.open_cursor('table:main', None, None) # spot check via search n = self.nentries - for i in (n / 5, 0, n - 1, n - 2, 1): + for i in (n // 5, 0, n - 1, n - 2, 1): cursor.set_key(i, 'key' + str(i)) square = i * i cube = square * i @@ -124,7 +124,7 @@ class test_schema06(wttest.WiredTigerTestCase): cursor = self.session.open_cursor('index:main:S1i4', None, None) count = 0 for s1key, i4key, s1, i2, s3, i4 in cursor: - i = int(i4key ** (1 / 3.0) + 0.0001) # cuberoot + i = int(i4key ** (1 // 3.0) + 0.0001) # cuberoot self.assertEqual(s1key, s1) self.assertEqual(i4key, i4) ikey = i @@ -145,7 +145,7 @@ class test_schema06(wttest.WiredTigerTestCase): cursor = self.session.open_cursor('index:main:i2S1i4', None, None) count = 0 for i2key, s1key, i4key, s1, i2, s3, i4 in cursor: - i = int(i4key ** (1 / 3.0) + 0.0001) # cuberoot + i = int(i4key ** (1 // 3.0) + 0.0001) # cuberoot self.assertEqual(i2key, i2) self.assertEqual(s1key, s1) self.assertEqual(i4key, i4) diff --git a/src/third_party/wiredtiger/test/suite/test_schema07.py b/src/third_party/wiredtiger/test/suite/test_schema07.py index 1c1755439f3..b35cca1611a 100644 --- a/src/third_party/wiredtiger/test/suite/test_schema07.py +++ b/src/third_party/wiredtiger/test/suite/test_schema07.py @@ -40,7 +40,7 @@ class test_schema07(wttest.WiredTigerTestCase): s = self.session # We have a 10MB cache, metadata is (well) over 512B per table, # if we can create 20K tables, something must be cleaning up. - for i in xrange(20000): + for i in range(20000): uri = '%s-%06d' % (self.tablename, i) s.create(uri) c = s.open_cursor(uri) diff --git a/src/third_party/wiredtiger/test/suite/test_split.py b/src/third_party/wiredtiger/test/suite/test_split.py index cde840fb055..233ae2133bc 100644 --- a/src/third_party/wiredtiger/test/suite/test_split.py +++ b/src/third_party/wiredtiger/test/suite/test_split.py @@ -49,7 +49,7 @@ class test_split(wttest.WiredTigerTestCase): # IF IT FAILS, IT MAY BE RECONCILIATION ISN'T CREATING THE SAME SIZE # PAGES AS BEFORE. - # Create a 4KB page (more than 3KB): 40 records w / 10 byte keys + # Create a 4KB page (more than 3KB): 40 records w // 10 byte keys # and 81 byte values. for i in range(35): cursor['%09d' % i] = 8 * ('%010d' % i) diff --git a/src/third_party/wiredtiger/test/suite/test_stat01.py b/src/third_party/wiredtiger/test/suite/test_stat01.py index c815b99d6bd..ea4e0b04f48 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_stat01.py +++ b/src/third_party/wiredtiger/test/suite/test_stat01.py @@ -107,7 +107,8 @@ class test_stat01(wttest.WiredTigerTestCase): # Test simple object statistics. def test_basic_data_source_stats(self): # Build an object. - config = self.config + ',key_format=' + self.keyfmt + config = self.config + ',key_format=' + self.keyfmt + \ + ',value_format=S' self.session.create(self.uri, config) cursor = self.session.open_cursor(self.uri, None, None) value = "" diff --git a/src/third_party/wiredtiger/test/suite/test_stat04.py b/src/third_party/wiredtiger/test/suite/test_stat04.py index 153ed983308..1717f70ecc7 100644 --- a/src/third_party/wiredtiger/test/suite/test_stat04.py +++ b/src/third_party/wiredtiger/test/suite/test_stat04.py @@ -93,7 +93,7 @@ class test_stat04(wttest.WiredTigerTestCase, suite_subprocess): count += 1 # Remove a number of entries, at each step checking that stats match. - for i in range(0, self.nentries / 37): + for i in range(0, self.nentries // 37): cursor.set_key(self.genkey(i*11 % self.nentries)) if cursor.remove() == 0: count -= 1 diff --git a/src/third_party/wiredtiger/test/suite/test_stat09.py b/src/third_party/wiredtiger/test/suite/test_stat09.py new file mode 100644 index 00000000000..8198ebd780d --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_stat09.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2019 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import random +import wiredtiger, wttest + +# test_stat09.py +# Check oldest active read timestamp statistic + +def timestamp_str(t): + return '%x' % t + +class test_stat09(wttest.WiredTigerTestCase): + tablename = 'test_stat09' + uri = 'table:' + tablename + conn_config = 'statistics=(all)' + + # Check the oldest active read statistic to be at the expected values + def check_stat_oldest_read(self, statcursor, expected_oldest, all_committed): + self.check_stats(statcursor, expected_oldest, + 'transaction: transaction read timestamp of the oldest active reader') + + # If the active oldest timestamp is 0, it implies there are no active readers, + # the pinned range because of them is expected to be 0 in that case + if expected_oldest == 0: + expected_pinned = 0 + else: + expected_pinned = all_committed - expected_oldest + self.check_stats(statcursor, expected_pinned, + 'transaction: transaction range of timestamps pinned by the oldest ' + 'active read timestamp') + + # Do a quick check of the entries in the stats cursor, the "lookfor" + # string should appear with the exact val of "expected_val". + def check_stats(self, statcursor, expected_val, lookfor): + # Reset the cursor, we're called multiple times. + statcursor.reset() + + found = False + foundval = 0 + for id, desc, valstr, val in statcursor: + if desc == lookfor: + found = True + foundval = val + self.printVerbose(2, ' stat: \'' + desc + '\', \'' + + valstr + '\', ' + str(val)) + break + + self.assertTrue(found, 'in stats, did not see: ' + lookfor) + self.assertTrue(foundval == expected_val) + + def test_oldest_active_read(self): + self.session.create(self.uri, 'key_format=i,value_format=i') + c = self.session.open_cursor(self.uri) + + # Insert some data: keys 1..100 each with timestamp=key, in some order + commit_range = 100 + orig_keys = list(range(1, commit_range + 1)) + keys = orig_keys[:] + random.shuffle(keys) + + for k in keys: + self.session.begin_transaction() + c[k] = 1 + self.session.commit_transaction('commit_timestamp=' + timestamp_str(k)) + + # Create a cursor on statistics that we can use repeatedly + allstat_cursor = self.session.open_cursor('statistics:', None, None) + + # There being no active reader, the corresponding statistic should be 0 + self.check_stat_oldest_read(allstat_cursor, 0, commit_range) + + # Introduce multiple transactions with varying read_timestamp + s1 = self.conn.open_session() + s1.begin_transaction('read_timestamp=' + timestamp_str(10)) + s2 = self.conn.open_session() + s2.begin_transaction('read_timestamp=' + timestamp_str(20)) + s3 = self.conn.open_session() + s3.begin_transaction('read_timestamp=' + timestamp_str(30)) + s4 = self.conn.open_session() + s4.begin_transaction('read_timestamp=' + timestamp_str(40)) + s5 = self.conn.open_session() + s5.begin_transaction('read_timestamp=' + timestamp_str(50)) + + # Check oldest reader + self.check_stat_oldest_read(allstat_cursor, 10, commit_range) + + # Close the oldest reader and check again + s1.commit_transaction() + self.check_stat_oldest_read(allstat_cursor, 20, commit_range) + + # Set and advance the oldest timestamp, it should be ignored for + # determining the oldest active read. + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(5)) + self.check_stat_oldest_read(allstat_cursor, 20, commit_range) + + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(30)) + self.check_stat_oldest_read(allstat_cursor, 20, commit_range) + + self.conn.set_timestamp('oldest_timestamp=' + timestamp_str(150)) + self.check_stat_oldest_read(allstat_cursor, 20, commit_range) + + # Move the commit timestamp and check again + commit_range = 200 + s2.commit_transaction('commit_timestamp=' + timestamp_str(commit_range)) + self.check_stat_oldest_read(allstat_cursor, 30, commit_range) + + # Close all the readers and check the oldest reader, it should be back to 0 + s3.commit_transaction() + s4.commit_transaction() + s5.commit_transaction() + self.check_stat_oldest_read(allstat_cursor, 0, commit_range) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_sweep01.py b/src/third_party/wiredtiger/test/suite/test_sweep01.py index 976a671081d..026c89b4dc9 100644 --- a/src/third_party/wiredtiger/test/suite/test_sweep01.py +++ b/src/third_party/wiredtiger/test/suite/test_sweep01.py @@ -153,55 +153,55 @@ class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): # in the presence of recent checkpoints. # if (close1 >= close2): - print "XX: close1: " + str(close1) + " close2: " + str(close2) - print "remove1: " + str(remove1) + " remove2: " + str(remove2) - print "sweep1: " + str(sweep1) + " sweep2: " + str(sweep2) - print "sclose1: " + str(sclose1) + " sclose2: " + str(sclose2) - print "ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2) - print "tod1: " + str(tod1) + " tod2: " + str(tod2) - print "ref1: " + str(ref1) + " ref2: " + str(ref2) - print "nfile1: " + str(nfile1) + " nfile2: " + str(nfile2) + print("XX: close1: " + str(close1) + " close2: " + str(close2)) + print("remove1: " + str(remove1) + " remove2: " + str(remove2)) + print("sweep1: " + str(sweep1) + " sweep2: " + str(sweep2)) + print("sclose1: " + str(sclose1) + " sclose2: " + str(sclose2)) + print("ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2)) + print("tod1: " + str(tod1) + " tod2: " + str(tod2)) + print("ref1: " + str(ref1) + " ref2: " + str(ref2)) + print("nfile1: " + str(nfile1) + " nfile2: " + str(nfile2)) self.assertEqual(close1 < close2, True) if (remove1 >= remove2): - print "close1: " + str(close1) + " close2: " + str(close2) - print "XX: remove1: " + str(remove1) + " remove2: " + str(remove2) - print "sweep1: " + str(sweep1) + " sweep2: " + str(sweep2) - print "sclose1: " + str(sclose1) + " sclose2: " + str(sclose2) - print "ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2) - print "tod1: " + str(tod1) + " tod2: " + str(tod2) - print "ref1: " + str(ref1) + " ref2: " + str(ref2) - print "nfile1: " + str(nfile1) + " nfile2: " + str(nfile2) + print("close1: " + str(close1) + " close2: " + str(close2)) + print("XX: remove1: " + str(remove1) + " remove2: " + str(remove2)) + print("sweep1: " + str(sweep1) + " sweep2: " + str(sweep2)) + print("sclose1: " + str(sclose1) + " sclose2: " + str(sclose2)) + print("ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2)) + print("tod1: " + str(tod1) + " tod2: " + str(tod2)) + print("ref1: " + str(ref1) + " ref2: " + str(ref2)) + print("nfile1: " + str(nfile1) + " nfile2: " + str(nfile2)) self.assertEqual(remove1 < remove2, True) if (sweep1 >= sweep2): - print "close1: " + str(close1) + " close2: " + str(close2) - print "remove1: " + str(remove1) + " remove2: " + str(remove2) - print "XX: sweep1: " + str(sweep1) + " sweep2: " + str(sweep2) - print "sclose1: " + str(sclose1) + " sclose2: " + str(sclose2) - print "ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2) - print "tod1: " + str(tod1) + " tod2: " + str(tod2) - print "ref1: " + str(ref1) + " ref2: " + str(ref2) + print("close1: " + str(close1) + " close2: " + str(close2)) + print("remove1: " + str(remove1) + " remove2: " + str(remove2)) + print("XX: sweep1: " + str(sweep1) + " sweep2: " + str(sweep2)) + print("sclose1: " + str(sclose1) + " sclose2: " + str(sclose2)) + print("ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2)) + print("tod1: " + str(tod1) + " tod2: " + str(tod2)) + print("ref1: " + str(ref1) + " ref2: " + str(ref2)) self.assertEqual(sweep1 < sweep2, True) if (nfile2 >= nfile1): - print "close1: " + str(close1) + " close2: " + str(close2) - print "remove1: " + str(remove1) + " remove2: " + str(remove2) - print "sweep1: " + str(sweep1) + " sweep2: " + str(sweep2) - print "sclose1: " + str(sclose1) + " sclose2: " + str(sclose2) - print "ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2) - print "tod1: " + str(tod1) + " tod2: " + str(tod2) - print "ref1: " + str(ref1) + " ref2: " + str(ref2) - print "XX: nfile1: " + str(nfile1) + " nfile2: " + str(nfile2) + print("close1: " + str(close1) + " close2: " + str(close2)) + print("remove1: " + str(remove1) + " remove2: " + str(remove2)) + print("sweep1: " + str(sweep1) + " sweep2: " + str(sweep2)) + print("sclose1: " + str(sclose1) + " sclose2: " + str(sclose2)) + print("ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2)) + print("tod1: " + str(tod1) + " tod2: " + str(tod2)) + print("ref1: " + str(ref1) + " ref2: " + str(ref2)) + print("XX: nfile1: " + str(nfile1) + " nfile2: " + str(nfile2)) self.assertEqual(nfile2 < nfile1, True) # The only files that should be left are the metadata, the lookaside # file, the lock file, and the active file. if (nfile2 != final_nfile): - print "close1: " + str(close1) + " close2: " + str(close2) - print "remove1: " + str(remove1) + " remove2: " + str(remove2) - print "sweep1: " + str(sweep1) + " sweep2: " + str(sweep2) - print "sclose1: " + str(sclose1) + " sclose2: " + str(sclose2) - print "ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2) - print "tod1: " + str(tod1) + " tod2: " + str(tod2) - print "ref1: " + str(ref1) + " ref2: " + str(ref2) - print "XX2: nfile1: " + str(nfile1) + " nfile2: " + str(nfile2) + print("close1: " + str(close1) + " close2: " + str(close2)) + print("remove1: " + str(remove1) + " remove2: " + str(remove2)) + print("sweep1: " + str(sweep1) + " sweep2: " + str(sweep2)) + print("sclose1: " + str(sclose1) + " sclose2: " + str(sclose2)) + print("ssweep1: " + str(ssweep1) + " ssweep2: " + str(ssweep2)) + print("tod1: " + str(tod1) + " tod2: " + str(tod2)) + print("ref1: " + str(ref1) + " ref2: " + str(ref2)) + print("XX2: nfile1: " + str(nfile1) + " nfile2: " + str(nfile2)) self.assertEqual(nfile2 == final_nfile, True) if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp02.py b/src/third_party/wiredtiger/test/suite/test_timestamp02.py index 3918f8cdeeb..c8a60b6f11f 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp02.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp02.py @@ -59,7 +59,7 @@ class test_timestamp02(wttest.WiredTigerTestCase, suite_subprocess): actual = dict((k, v) for k, v in c if v != 0) self.assertTrue(actual == expected) # Search for the expected items as well as iterating - for k, v in expected.iteritems(): + for k, v in expected.items(): self.assertEqual(c[k], v, "for key " + str(k)) c.close() if txn_config: @@ -71,7 +71,7 @@ class test_timestamp02(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(self.uri) # Insert keys 1..100 each with timestamp=key, in some order - orig_keys = range(1, 101) + orig_keys = list(range(1, 101)) keys = orig_keys[:] random.shuffle(keys) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp03.py b/src/third_party/wiredtiger/test/suite/test_timestamp03.py index 029e3292aac..4dc1117bb2e 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp03.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp03.py @@ -83,7 +83,7 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): actual = dict((k, v) for k, v in cur if v != 0) self.assertTrue(actual == expected) # Search for the expected items as well as iterating - for k, v in expected.iteritems(): + for k, v in expected.items(): self.assertEqual(cur[k], v, "for key " + str(k)) cur.close() if txn_config: @@ -176,7 +176,7 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): # Insert keys 1..100 each with timestamp=key, in some order nkeys = 100 - orig_keys = range(1, nkeys+1) + orig_keys = list(range(1, nkeys+1)) keys = orig_keys[:] random.shuffle(keys) @@ -267,20 +267,21 @@ class test_timestamp03(wttest.WiredTigerTestCase, suite_subprocess): # Scenario: 4a # This scenario is same as earlier one with read_timestamp earlier than - # oldest_timestamp and using the option of round_to_oldest + # oldest_timestamp and using the option of rounding read_timestamp to + # the oldest_timestamp earlier_ts = timestamp_str(90) self.check(self.session, - 'read_timestamp=' + earlier_ts +',round_to_oldest=true', + 'read_timestamp=' + earlier_ts +',roundup_timestamps=(read=true)', self.table_ts_log, dict((k, self.value) for k in orig_keys)) self.check(self.session, - 'read_timestamp=' + earlier_ts +',round_to_oldest=true', + 'read_timestamp=' + earlier_ts +',roundup_timestamps=(read=true)', self.table_ts_nolog, dict((k, self.value) for k in orig_keys)) # Tables not using the timestamps should see updated values (i.e. value2). self.check(self.session, - 'read_timestamp=' + earlier_ts +',round_to_oldest=true', + 'read_timestamp=' + earlier_ts +',roundup_timestamps=(read=true)', self.table_nots_log, dict((k, self.value2) for k in orig_keys)) self.check(self.session, - 'read_timestamp=' + earlier_ts +',round_to_oldest=true', + 'read_timestamp=' + earlier_ts +',roundup_timestamps=(read=true)', self.table_nots_nolog, dict((k, self.value2) for k in orig_keys)) # Scenario: 5 diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp04.py b/src/third_party/wiredtiger/test/suite/test_timestamp04.py index 9251bfab06e..cf692d43edd 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp04.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp04.py @@ -70,12 +70,12 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): if missing == False: actual = dict((k, v) for k, v in cur if v != 0) if actual != expected: - print "missing: ", sorted(set(expected.items()) - set(actual.items())) - print "extras: ", sorted(set(actual.items()) - set(expected.items())) + print("missing: ", sorted(set(expected.items()) - set(actual.items()))) + print("extras: ", sorted(set(actual.items()) - set(expected.items()))) self.assertTrue(actual == expected) # Search for the expected items as well as iterating. - for k, v in expected.iteritems(): + for k, v in expected.items(): if missing == False: self.assertEqual(cur[k], v, "for key " + str(k)) else: @@ -104,7 +104,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): try: self.conn = wiredtiger.wiredtiger_open(self.home, conn_params) except wiredtiger.WiredTigerError as e: - print "Failed conn at '%s' with config '%s'" % (dir, conn_params) + print("Failed conn at '%s' with config '%s'" % (dir, conn_params)) self.session = self.conn.open_session(None) def test_rollback_to_stable(self): @@ -131,7 +131,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Insert keys each with timestamp=key, in some order. key_range = 10000 - keys = range(1, key_range + 1) + keys = list(range(1, key_range + 1)) # Set keys 1-key_range to value 1. for k in keys: @@ -159,7 +159,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Scenario: 2 # Roll back half timestamps. - stable_ts = timestamp_str(key_range / 2) + stable_ts = timestamp_str(key_range // 2) self.conn.set_timestamp('stable_timestamp=' + stable_ts) self.conn.rollback_to_stable() stat_cursor = self.session.open_cursor('statistics:', None, None) @@ -182,9 +182,9 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Check that we see the inserted value (i.e. 1) for the keys in a # timestamped table until the stable_timestamp only. self.check(self.session, 'read_timestamp=' + latest_ts, - self.table_ts_nolog, dict((k, 1) for k in keys[:(key_range / 2)])) + self.table_ts_nolog, dict((k, 1) for k in keys[:(key_range // 2)])) self.check(self.session, 'read_timestamp=' + latest_ts, - self.table_ts_nolog, dict((k, 1) for k in keys[(key_range / 2 + 1):]), missing=True) + self.table_ts_nolog, dict((k, 1) for k in keys[(key_range // 2 + 1):]), missing=True) # For logged tables, the behavior of rollback_to_stable changes based on # whether connection level logging is enabled or not. @@ -198,9 +198,9 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Check that we see the insertions are rolled back in timestamped tables # until the stable_timestamp. self.check(self.session, 'read_timestamp=' + latest_ts, - self.table_ts_log, dict((k, 1) for k in keys[:(key_range / 2)])) + self.table_ts_log, dict((k, 1) for k in keys[:(key_range // 2)])) self.check(self.session, 'read_timestamp=' + latest_ts, - self.table_ts_log, dict((k, 1) for k in keys[(key_range / 2 + 1):]), missing=True) + self.table_ts_log, dict((k, 1) for k in keys[(key_range // 2 + 1):]), missing=True) # Bump the oldest timestamp, we're not going back. self.conn.set_timestamp('oldest_timestamp=' + stable_ts) @@ -229,7 +229,7 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # Scenario: 4 # Advance the stable_timestamp by a quarter range and rollback. # Three-fourths of the later timestamps will be rolled back. - rolled_range = key_range + key_range / 4 + rolled_range = key_range + key_range // 4 stable_ts = timestamp_str(rolled_range) self.conn.set_timestamp('stable_timestamp=' + stable_ts) self.conn.rollback_to_stable() @@ -257,10 +257,10 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # the updated value (i.e. 2) for the first quarter keys and old values # (i.e. 1) for the second quarter keys. self.check(self.session, 'read_timestamp=' + latest_ts, - self.table_ts_nolog, dict((k, 2 if k <= key_range / 4 else 1) - for k in keys[:(key_range / 2)])) + self.table_ts_nolog, dict((k, 2 if k <= key_range // 4 else 1) + for k in keys[:(key_range // 2)])) self.check(self.session, 'read_timestamp=' + latest_ts, - self.table_ts_nolog, dict((k, 1) for k in keys[(1 + key_range / 2):]), missing=True) + self.table_ts_nolog, dict((k, 1) for k in keys[(1 + key_range // 2):]), missing=True) # For logged tables behavior changes for rollback_to_stable based on # whether connection level logging is enabled or not. @@ -275,10 +275,10 @@ class test_timestamp04(wttest.WiredTigerTestCase, suite_subprocess): # the updated value (i.e. 2) for the first quarter keys and old values # (i.e. 1) for the second quarter keys. self.check(self.session, 'read_timestamp=' + latest_ts, - self.table_ts_log, dict((k, (2 if k <= key_range / 4 else 1)) - for k in keys[:(key_range / 2)])) + self.table_ts_log, dict((k, (2 if k <= key_range // 4 else 1)) + for k in keys[:(key_range // 2)])) self.check(self.session, 'read_timestamp=' + latest_ts, - self.table_ts_log, dict((k, 1) for k in keys[(1 + key_range / 2):]), missing=True) + self.table_ts_log, dict((k, 1) for k in keys[(1 + key_range // 2):]), missing=True) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp05.py b/src/third_party/wiredtiger/test/suite/test_timestamp05.py index 5b63b54597b..6ef6a92851e 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp05.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp05.py @@ -69,9 +69,7 @@ class test_timestamp05(wttest.WiredTigerTestCase, suite_subprocess): # Insert keys 1..100 each with timestamp=key, in some order nkeys = 100 - keys = range(1, nkeys+1) - - for k in keys: + for k in range(1, nkeys+1): c[k] = 'some value' # Start timestamps at 50 diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp06.py b/src/third_party/wiredtiger/test/suite/test_timestamp06.py index 95e3dbce427..a085f0f0d27 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp06.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp06.py @@ -72,11 +72,11 @@ class test_timestamp06(wttest.WiredTigerTestCase, suite_subprocess): cur = session.open_cursor(tablename, None) actual = dict((k, v) for k, v in cur if v != 0) if actual != expected: - print "missing: ", sorted(set(expected.items()) - set(actual.items())) - print "extras: ", sorted(set(actual.items()) - set(expected.items())) + print("missing: ", sorted(set(expected.items()) - set(actual.items()))) + print("extras: ", sorted(set(actual.items()) - set(expected.items()))) self.assertTrue(actual == expected) # Search for the expected items as well as iterating - for k, v in expected.iteritems(): + for k, v in expected.items(): self.assertEqual(cur[k], v, "for key " + str(k)) cur.close() if txn_config: @@ -131,7 +131,7 @@ class test_timestamp06(wttest.WiredTigerTestCase, suite_subprocess): # Insert keys 1..100 nkeys = 100 - orig_keys = range(1, nkeys+1) + orig_keys = list(range(1, nkeys+1)) keys = orig_keys[:] random.shuffle(keys) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp07.py b/src/third_party/wiredtiger/test/suite/test_timestamp07.py index ba29fa24bd7..ab76ff7ea10 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_timestamp07.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp07.py @@ -191,7 +191,7 @@ class test_timestamp07(wttest.WiredTigerTestCase, suite_subprocess): # print "tables created" # Insert keys 1..nkeys each with timestamp=key, in some order. - orig_keys = range(1, self.nkeys+1) + orig_keys = list(range(1, self.nkeys+1)) keys = orig_keys[:] random.shuffle(keys) diff --git a/src/third_party/wiredtiger/test/suite/test_timestamp13.py b/src/third_party/wiredtiger/test/suite/test_timestamp13.py index 65e0cb7df6f..1585a4cb194 100644 --- a/src/third_party/wiredtiger/test/suite/test_timestamp13.py +++ b/src/third_party/wiredtiger/test/suite/test_timestamp13.py @@ -107,11 +107,11 @@ class test_timestamp13(wttest.WiredTigerTestCase, suite_subprocess): 'key_format=i,value_format=i' + self.extra_config) self.conn.set_timestamp('oldest_timestamp=10') - self.session.begin_transaction('isolation=snapshot') # Rounding to the oldest timestamp will allow the stale read_timestamp # to succeed. The follow-up call to get the read timestamp returns the # chosen read timestamp. - self.session.timestamp_transaction('read_timestamp=5,round_to_oldest=true') + self.session.begin_transaction('isolation=snapshot,roundup_timestamps=(read=true)') + self.session.timestamp_transaction('read_timestamp=5') self.assertTimestampsEqual( self.session.query_timestamp('get=read'), '10') diff --git a/src/third_party/wiredtiger/test/suite/test_truncate01.py b/src/third_party/wiredtiger/test/suite/test_truncate01.py index a136b8220f6..6d941fb8043 100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate01.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate01.py @@ -159,8 +159,8 @@ class test_truncate_cursor_end(wttest.WiredTigerTestCase): c2 = self.session.open_cursor(uri, None) c2.set_key(ds.key(2000)) self.session.truncate(None, c1, c2, None) - self.assertEquals(c1.close(), 0) - self.assertEquals(c2.close(), 0) + self.assertEqual(c1.close(), 0) + self.assertEqual(c2.close(), 0) self.session.drop(uri) if self.type == "table:": @@ -171,8 +171,8 @@ class test_truncate_cursor_end(wttest.WiredTigerTestCase): c2 = self.session.open_cursor(uri, None) c2.set_key(ds.key(2000)) self.session.truncate(None, c1, c2, None) - self.assertEquals(c1.close(), 0) - self.assertEquals(c2.close(), 0) + self.assertEqual(c1.close(), 0) + self.assertEqual(c2.close(), 0) self.session.drop(uri) # Test truncation of empty objects. @@ -199,14 +199,14 @@ class test_truncate_empty(wttest.WiredTigerTestCase): c1.set_key(simple_key(c1, 1000)) c2 = self.session.open_cursor(uri, None) c2.set_key(simple_key(c2, 2000)) - self.assertEquals(self.session.truncate(None, c1, c2, None), 0) + self.assertEqual(self.session.truncate(None, c1, c2, None), 0) # Test truncation of empty objects using a URI def test_truncate_empty_uri(self): uri = self.type + self.name self.session.create(uri, ',key_format=' + self.keyfmt + ',value_format=S') - self.assertEquals(self.session.truncate(uri, None, None, None), 0) + self.assertEqual(self.session.truncate(uri, None, None, None), 0) # Test session.truncate. class test_truncate_cursor(wttest.WiredTigerTestCase): @@ -272,7 +272,7 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri, None) for i in range(begin, end + 1): expected[ds.key(i)] = [0] - for k, v in expected.iteritems(): + for k, v in expected.items(): cursor.set_key(k) if v == [0] and \ cursor.key_format == 'r' and cursor.value_format == '8t': diff --git a/src/third_party/wiredtiger/test/suite/test_txn01.py b/src/third_party/wiredtiger/test/suite/test_txn01.py index 2de136d9231..daf9b5193c6 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn01.py +++ b/src/third_party/wiredtiger/test/suite/test_txn01.py @@ -110,8 +110,8 @@ class test_txn01(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(self.uri, None) self.check(cursor, 0, 0) self.session.begin_transaction() - for i in xrange(self.nentries): - if i > 0 and i % (self.nentries / 37) == 0: + for i in range(self.nentries): + if i > 0 and i % (self.nentries // 37) == 0: self.check(cursor, committed, i) self.session.commit_transaction() committed = i diff --git a/src/third_party/wiredtiger/test/suite/test_txn02.py b/src/third_party/wiredtiger/test/suite/test_txn02.py index b91cf56f84c..9a090aaab04 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn02.py +++ b/src/third_party/wiredtiger/test/suite/test_txn02.py @@ -91,7 +91,7 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): # Each check_log() call takes a second, so we don't call it for # every scenario, we'll limit it to the value of checklog_calls. checklog_calls = 100 if wttest.islongtest() else 2 - checklog_mod = (len(scenarios) / checklog_calls + 1) + checklog_mod = (len(scenarios) // checklog_calls + 1) _debug = False def debug(self, msg): @@ -124,7 +124,7 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): c = session.open_cursor(self.uri, None) actual = dict((k, v) for k, v in c if v != 0) # Search for the expected items as well as iterating - for k, v in expected.iteritems(): + for k, v in expected.items(): self.assertEqual(c[k], v) c.close() if txn_config: diff --git a/src/third_party/wiredtiger/test/suite/test_txn04.py b/src/third_party/wiredtiger/test/suite/test_txn04.py index 62f9630139a..42ed7baab41 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn04.py +++ b/src/third_party/wiredtiger/test/suite/test_txn04.py @@ -81,7 +81,7 @@ class test_txn04(wttest.WiredTigerTestCase, suite_subprocess): c = session.open_cursor(self.uri, None) actual = dict((k, v) for k, v in c if v != 0) # Search for the expected items as well as iterating - for k, v in expected.iteritems(): + for k, v in expected.items(): self.assertEqual(c[k], v) c.close() if txn_config: diff --git a/src/third_party/wiredtiger/test/suite/test_txn05.py b/src/third_party/wiredtiger/test/suite/test_txn05.py index 4ce67a9df61..6fd72827618 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn05.py +++ b/src/third_party/wiredtiger/test/suite/test_txn05.py @@ -82,7 +82,7 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): c = session.open_cursor(self.uri, None) actual = dict((k, v) for k, v in c if v != 0) # Search for the expected items as well as iterating - for k, v in expected.iteritems(): + for k, v in expected.items(): self.assertEqual(c[k], v) c.close() if txn_config: @@ -237,7 +237,7 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): # Check the log state after the entire op completes # and run recovery. - if self.scenario_number % (len(test_txn05.scenarios) / 100 + 1) == 0: + if self.scenario_number % (len(test_txn05.scenarios) // 100 + 1) == 0: self.check_log(committed) if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_txn06.py b/src/third_party/wiredtiger/test/suite/test_txn06.py index bad62f55292..c0bbe332c79 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_txn06.py +++ b/src/third_party/wiredtiger/test/suite/test_txn06.py @@ -45,7 +45,7 @@ class test_txn06(wttest.WiredTigerTestCase, suite_subprocess): SimpleDataSet(self, self.source_uri, self.nrows).populate() # Now scan the table and copy the rows into a new table - c_src = self.session.create(self.uri, "key_format=S") + c_src = self.session.create(self.uri, "key_format=S,value_format=S") c_src = self.session.open_cursor(self.source_uri) c = self.session.open_cursor(self.uri) for k, v in c_src: diff --git a/src/third_party/wiredtiger/test/suite/test_txn07.py b/src/third_party/wiredtiger/test/suite/test_txn07.py index ceb86344cdc..3bc708fb9d1 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn07.py +++ b/src/third_party/wiredtiger/test/suite/test_txn07.py @@ -93,7 +93,7 @@ class test_txn07(wttest.WiredTigerTestCase, suite_subprocess): c = session.open_cursor(self.uri, None) actual = dict((k, v) for k, v in c if v != 0) # Search for the expected items as well as iterating - for k, v in expected.iteritems(): + for k, v in expected.items(): self.assertEqual(c[k], v) c.close() if txn_config: diff --git a/src/third_party/wiredtiger/test/suite/test_txn09.py b/src/third_party/wiredtiger/test/suite/test_txn09.py index c6a695a50ad..f216cbc04ac 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn09.py +++ b/src/third_party/wiredtiger/test/suite/test_txn09.py @@ -92,7 +92,7 @@ class test_txn09(wttest.WiredTigerTestCase, suite_subprocess): c = session.open_cursor(self.uri, None) actual = dict((k, v) for k, v in c if v != 0) # Search for the expected items as well as iterating - for k, v in expected.iteritems(): + for k, v in expected.items(): self.assertEqual(c[k], v) c.close() if txn_config: diff --git a/src/third_party/wiredtiger/test/suite/test_txn19.py b/src/third_party/wiredtiger/test/suite/test_txn19.py index 98053a5c7a8..604d8bed8bb 100755 --- a/src/third_party/wiredtiger/test/suite/test_txn19.py +++ b/src/third_party/wiredtiger/test/suite/test_txn19.py @@ -124,7 +124,7 @@ class test_txn19(wttest.WiredTigerTestCase, suite_subprocess): # (having no records initially). The last log file is this # (nrecords/2 + 1), given that we start with log 1. def record_to_logfile(self, recordnum): - return recordnum / 2 + 1 + return recordnum // 2 + 1 # Returns the first record number in a log file. def logfile_to_record(self, logfile): diff --git a/src/third_party/wiredtiger/test/suite/test_unicode01.py b/src/third_party/wiredtiger/test/suite/test_unicode01.py index c46c8529224..baa85e1daf8 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_unicode01.py +++ b/src/third_party/wiredtiger/test/suite/test_unicode01.py @@ -26,21 +26,27 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import wiredtiger, wtscenario, wttest +import sys, wiredtiger, wtscenario, wttest +_python3 = (sys.version_info >= (3, 0, 0)) # test_unicode01.py -# Make sure UTF8 config can be passed to WT_SESSION::create -# -# There are a couple of tricks here: -# 1) we don't want to treat the whole file as UTF-8, because this would -# be the only one in our tree and that causes problems for some -# scripts; and -# 2) we can't pass in a Unicode object directly because the -# SWIG-generated code expects a simple Python string. +# Make sure UTF8 config can be passed to WT_SESSION::create. +# Python turns Unicode strings into UTF-8. class test_unicode01(wttest.WiredTigerTestCase): def test_unicode(self): - self.session.create('table:t', - u'app_metadata={"name" : "Employ\xe9s"}'.encode('utf-8')) + # We use valid Unicode characters that are examples in + # the Unicode standard. + metadata_unicode = u'app_metadata={"name" : "Employ\u222b\u67d2\ud4db"}' + + # In Python2, Unicode and strings are different types, + # and need to be converted. In Python3, there is no separate + # unicode type, unicode characters are just embedded as UTF-8 + # in strings. + if _python3: + metadata_string = metadata_unicode + else: + metadata_string = metadata_unicode.encode('utf-8') + self.session.create('table:t', metadata_string) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_util01.py b/src/third_party/wiredtiger/test/suite/test_util01.py index 9287983ce91..d4c74ec3a01 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_util01.py +++ b/src/third_party/wiredtiger/test/suite/test_util01.py @@ -26,10 +26,12 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import string, os +import string, os, sys from suite_subprocess import suite_subprocess import wiredtiger, wttest +_python3 = (sys.version_info >= (3, 0, 0)) + # test_util01.py # Utilities: wt dump, as well as the dump cursor class test_util01(wttest.WiredTigerTestCase, suite_subprocess): @@ -57,9 +59,10 @@ class test_util01(wttest.WiredTigerTestCase, suite_subprocess): def compare_files(self, filename1, filename2): inheader = isconfig = False - for l1, l2 in zip(open(filename1, "rb"), open(filename2, "rb")): + for l1, l2 in zip(open(filename1, "r"), open(filename2, "r")): if isconfig: if not self.compare_config(l1, l2): + self.tty('Failed comparing: ' + l1 + '<<<>>>' + l2) return False elif l1 != l2: return False @@ -71,31 +74,41 @@ class test_util01(wttest.WiredTigerTestCase, suite_subprocess): inheader = isconfig = False return True - def get_string(self, i, len): + def get_bytes(self, i, len): """ Return a pseudo-random, but predictable string that uses all characters. As a special case, key 0 returns all characters 1-255 repeated """ - ret = '' + ret = b'' if i == 0: for j in range (0, len): - # we ensure that there are no internal nulls, that would - # truncate the string when we're using the 'S' encoding - # The last char in a string is null anyway, so that's tested. - ret += chr(j%255 + 1) + ret += bytes([j%255 + 1]) else: - for j in range(0, len / 3): + for j in range(0, len // 3): k = i + j - # no internal nulls... - ret += chr(k%255 + 1) + chr((k*3)%255 + 1) + chr((k*7)%255 + 1) - return ret + ret += bytes([k%255 + 1, (k*3)%255 + 1, (k*7)%255 + 1]) + return ret + bytes([0]) # Add a final null byte def get_key(self, i): - return ("%0.6d" % i) + ':' + self.get_string(i, 20) + return (b"%0.6d" % i) + b':' + self.get_bytes(i, 20) def get_value(self, i): - return self.get_string(i, 1000) + return self.get_bytes(i, 1000) + + if _python3: + def _ord(self, byte): + return byte + + def _byte_to_str(self, byte): + return chr(byte) + + else: + def _ord(self, byte): + return ord(byte) + + def _byte_to_str(self, byte): + return byte def dumpstr(self, s, hexoutput): """ @@ -105,6 +118,7 @@ class test_util01(wttest.WiredTigerTestCase, suite_subprocess): """ result = '' for c in s: + c = self._byte_to_str(c) if hexoutput: result += "%0.2x" % ord(c) elif c == '\\': @@ -114,13 +128,18 @@ class test_util01(wttest.WiredTigerTestCase, suite_subprocess): else: result += '\\' + "%0.2x" % ord(c) if hexoutput: - result += '00\n' + result += '\n' else: - result += '\\00\n' + result += '\n' return result def table_config(self): - return 'key_format=S,value_format=S' + # Using u configuration lets us store and print all the byte values. + return 'key_format=u,value_format=u' + + def dump_kv_to_line(self, b): + # The output from dump is a 'u' format. + return b.strip(b'\x00').decode() + '\n' def dump(self, usingapi, hexoutput): params = self.table_config() @@ -160,7 +179,8 @@ class test_util01(wttest.WiredTigerTestCase, suite_subprocess): dumpcurs = self.session.open_cursor('table:' + self.tablename, None, dumpopt) for key, val in dumpcurs: - dumpout.write(str(key) + "\n" + str(val) + "\n") + dumpout.write(self.dump_kv_to_line(key) + \ + self.dump_kv_to_line(val)) dumpcurs.close() else: dumpargs = ["dump"] diff --git a/src/third_party/wiredtiger/test/suite/test_util02.py b/src/third_party/wiredtiger/test/suite/test_util02.py index 277b799cfd7..4af6f5747f7 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_util02.py +++ b/src/third_party/wiredtiger/test/suite/test_util02.py @@ -65,7 +65,7 @@ class test_util02(wttest.WiredTigerTestCase, suite_subprocess): # The last char in a string is null anyway, so that's tested. ret += chr(j%255 + 1) else: - for j in range(0, len / 3): + for j in range(0, len // 3): k = i + j # no internal nulls... ret += chr(k%255 + 1) + chr((k*3)%255 + 1) + chr((k*7)%255 + 1) @@ -75,7 +75,7 @@ class test_util02(wttest.WiredTigerTestCase, suite_subprocess): if self.key_format == 'S': return ("%0.6d" % i) + ':' + self.get_string(i, 20) elif self.key_format == 'r': - return long(i + 1) + return self.recno(i + 1) else: return i + 1 diff --git a/src/third_party/wiredtiger/test/suite/test_util09.py b/src/third_party/wiredtiger/test/suite/test_util09.py index 37a2ad8f165..98a96d0c7a9 100644 --- a/src/third_party/wiredtiger/test/suite/test_util09.py +++ b/src/third_party/wiredtiger/test/suite/test_util09.py @@ -39,7 +39,7 @@ class test_util09(wttest.WiredTigerTestCase, suite_subprocess): def populate_file(self, filename, low, high): """ - Insert some simple key / value lines into the file + Insert some simple key // value lines into the file """ keys = {} with open("loadtext.in", "w") as f: diff --git a/src/third_party/wiredtiger/test/suite/test_util13.py b/src/third_party/wiredtiger/test/suite/test_util13.py index ea13093a230..5a92fb6a8ad 100644 --- a/src/third_party/wiredtiger/test/suite/test_util13.py +++ b/src/third_party/wiredtiger/test/suite/test_util13.py @@ -100,11 +100,11 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): # the actual configuration and they match. match = all(item in da.items() for item in dx.items()) if match == False: - print "MISMATCH:" - print "Original dict: " - print da - print "Expected config: " - print dx + print("MISMATCH:") + print("Original dict: ") + print(da) + print("Expected config: ") + print(dx) return match def compare_files(self, expect_subset, dump_out): diff --git a/src/third_party/wiredtiger/test/suite/test_verify.py b/src/third_party/wiredtiger/test/suite/test_verify.py index 39fb5efa672..dda68f43cd2 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_verify.py +++ b/src/third_party/wiredtiger/test/suite/test_verify.py @@ -29,6 +29,10 @@ import os, struct from suite_subprocess import suite_subprocess import wiredtiger, wttest +try: + xrange +except NameError: #python3 + xrange = range # test_verify.py # Utilities: wt verify @@ -42,7 +46,7 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): """ cursor = self.session.open_cursor('table:' + tablename, None, None) key = '' - for i in range(0, self.nentries): + for i in xrange(0, self.nentries): key += str(i) cursor[key] = key + key cursor.close() @@ -77,7 +81,7 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): filename = tablename + ".wt" filesize = os.path.getsize(filename) - position = (filesize * pct) / 100 + position = (filesize * pct) // 100 self.pr('damaging file at: ' + str(position)) fp = open(filename, "r+b") @@ -130,7 +134,7 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): self.session.create('table:' + self.tablename, params) self.populate(self.tablename) with self.open_and_position(self.tablename, 75) as f: - for i in range(0, 4096): + for i in xrange(0, 4096): f.write(struct.pack('B', 0)) # open_and_position closed the session/connection, reopen them now. @@ -149,7 +153,7 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): self.session.create('table:' + self.tablename, params) self.populate(self.tablename) with self.open_and_position(self.tablename, 75) as f: - for i in range(0, 4096): + for i in xrange(0, 4096): f.write(struct.pack('B', 0)) self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out", failure=True) @@ -164,8 +168,8 @@ class test_verify(wttest.WiredTigerTestCase, suite_subprocess): self.session.create('table:' + self.tablename, params) self.populate(self.tablename) with self.open_and_position(self.tablename, 25) as f: - for i in range(0, 100): - f.write('\x01\xff\x80') + for i in xrange(0, 100): + f.write(b'\x01\xff\x80') self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out", failure=True) self.check_non_empty_file("verifyerr.out") diff --git a/src/third_party/wiredtiger/test/suite/wtdataset.py b/src/third_party/wiredtiger/test/suite/wtdataset.py index b0c9d370517..9620c2127b0 100644..100755 --- a/src/third_party/wiredtiger/test/suite/wtdataset.py +++ b/src/third_party/wiredtiger/test/suite/wtdataset.py @@ -26,6 +26,10 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # +import sys +_python3 = (sys.version_info >= (3, 0, 0)) +if _python3: + xrange = range class BaseDataSet(object): """ @@ -73,6 +77,8 @@ class BaseDataSet(object): def key_by_format(i, key_format): if key_format == 'i' or key_format == 'r': return i + elif _python3 and key_format == 'u': + return bytes(('%015d' % i).encode()) elif key_format == 'S' or key_format == 'u': return str('%015d' % i) else: @@ -84,6 +90,8 @@ class BaseDataSet(object): def value_by_format(i, value_format): if value_format == 'i' or value_format == 'r': return i + elif _python3 and value_format == 'u': + return bytes((str(i) + ': abcdefghijklmnopqrstuvwxyz').encode()) elif value_format == 'S' or value_format == 'u': return str(i) + ': abcdefghijklmnopqrstuvwxyz' elif value_format == '8t': diff --git a/src/third_party/wiredtiger/test/suite/wtscenario.py b/src/third_party/wiredtiger/test/suite/wtscenario.py index 8b18ee91ca9..610b0de1723 100644 --- a/src/third_party/wiredtiger/test/suite/wtscenario.py +++ b/src/third_party/wiredtiger/test/suite/wtscenario.py @@ -33,8 +33,8 @@ import suite_random # Support scenarios based testing def powerrange(start, stop, mult): """ - Like xrange, generates a range from start to stop. - Unlike xrange, the range is inclusive of stop, + Like range, generates a range from start to stop. + Unlike range, the range is inclusive of stop, each step is multiplicative, and as a special case, the stop value is returned as the last item. """ @@ -269,9 +269,9 @@ class wtscenario: for lmax in powerrange(lmin, 512*megabyte, 1024): for cache in [megabyte, 32*megabyte, 1000*megabyte]: scen = wtscenario() - scen.ioverflow = max(imin / 40, 40) + scen.ioverflow = max(imin // 40, 40) scen.imax = imax - scen.loverflow = max(lmin / 40, 40) + scen.loverflow = max(lmin // 40, 40) scen.lmax = lmax scen.cache_size = cache s.append((scen.shortName(), dict(session_create_scenario=scen))) diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index ca4a8295373..d3d75d24c42 100755 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -29,6 +29,7 @@ # WiredTigerTestCase # parent class for all test cases # +from __future__ import print_function # If unittest2 is available, use it in preference to (the old) unittest try: @@ -45,6 +46,8 @@ def shortenWithEllipsis(s, maxlen): s = s[0:maxlen-3] + '...' return s +_python3 = (sys.version_info >= (3, 0, 0)) + class CapturedFd(object): """ CapturedFd encapsulates a file descriptor (e.g. 1 or 2) that is diverted @@ -192,7 +195,7 @@ class WiredTigerTestCase(unittest.TestCase): WiredTigerTestCase._parentTestdir = d WiredTigerTestCase._builddir = builddir WiredTigerTestCase._origcwd = os.getcwd() - WiredTigerTestCase._resultfile = open(os.path.join(d, 'results.txt'), "w", 0) # unbuffered + WiredTigerTestCase._resultfile = open(os.path.join(d, 'results.txt'), "w", 1) # line buffered WiredTigerTestCase._gdbSubprocess = gdbSub WiredTigerTestCase._lldbSubprocess = lldbSub WiredTigerTestCase._longtest = longtest @@ -293,7 +296,7 @@ class WiredTigerTestCase(unittest.TestCase): else: extfiles[ext] = complete if len(extfiles) != 0: - result = ',extensions=[' + ','.join(extfiles.values()) + ']' + result = ',extensions=[' + ','.join(list(extfiles.values())) + ']' return result # Can be overridden, but first consider setting self.conn_config @@ -311,10 +314,10 @@ class WiredTigerTestCase(unittest.TestCase): try: conn = self.wiredtiger_open(home, conn_param) except wiredtiger.WiredTigerError as e: - print "Failed wiredtiger_open: dir '%s', config '%s'" % \ - (home, conn_param) + print("Failed wiredtiger_open: dir '%s', config '%s'" % \ + (home, conn_param)) raise e - self.pr(`conn`) + self.pr(repr(conn)) return conn # Replacement for wiredtiger.wiredtiger_open that returns @@ -392,13 +395,25 @@ class WiredTigerTestCase(unittest.TestCase): self.tearDown() raise + # Used as part of tearDown determining if there is an error. + def list2reason(self, result, fieldname): + exc_list = getattr(result, fieldname, None) + if exc_list and exc_list[-1][0] is self: + return exc_list[-1][1] + def tearDown(self): - excinfo = sys.exc_info() - passed = (excinfo == (None, None, None)) - if passed: - skipped = False - else: - skipped = (excinfo[0] == unittest.SkipTest) + # This approach works for all our support Python versions and + # is suggested by one of the answers in: + # https://stackoverflow.com/questions/4414234/getting-pythons-unittest-results-in-a-teardown-method + if hasattr(self, '_outcome'): # Python 3.4+ + result = self.defaultTestResult() # these 2 methods have no side effects + self._feedErrorsToResult(result, self._outcome.errors) + else: # Python 3.2 - 3.3 or 3.0 - 3.1 and 2.7 + result = getattr(self, '_outcomeForDoCleanups', self._resultForDoCleanups) + error = self.list2reason(result, 'errors') + failure = self.list2reason(result, 'failures') + passed = not error and not failure + self.pr('finishing') # Close all connections that weren't explicitly closed. @@ -425,26 +440,26 @@ class WiredTigerTestCase(unittest.TestCase): os.chdir(self.origcwd) # Make sure no read-only files or directories were left behind - os.chmod(self.testdir, 0777) + os.chmod(self.testdir, 0o777) for root, dirs, files in os.walk(self.testdir): for d in dirs: - os.chmod(os.path.join(root, d), 0777) + os.chmod(os.path.join(root, d), 0o777) for f in files: - os.chmod(os.path.join(root, f), 0666) + os.chmod(os.path.join(root, f), 0o666) + self.pr('passed=' + str(passed)) # Clean up unless there's a failure - if (passed or skipped) and not WiredTigerTestCase._preserveFiles: + if passed and not WiredTigerTestCase._preserveFiles: shutil.rmtree(self.testdir, ignore_errors=True) else: self.pr('preserving directory ' + self.testdir) elapsed = time.time() - self.starttime if elapsed > 0.001 and WiredTigerTestCase._verbose >= 2: - print "%s: %.2f seconds" % (str(self), elapsed) - if not passed and not skipped: - print "ERROR in " + str(self) + print("%s: %.2f seconds" % (str(self), elapsed)) + if not passed: + print("ERROR in " + str(self)) self.pr('FAIL') - self.prexception(excinfo) self.pr('preserving directory ' + self.testdir) if WiredTigerTestCase._verbose > 2: self.prhead('TEST COMPLETED') @@ -515,7 +530,7 @@ class WiredTigerTestCase(unittest.TestCase): raised = False try: expr() - except BaseException, err: + except BaseException as err: if not isinstance(err, exceptionType): self.fail('Exception of incorrect type raised, got type: ' + \ str(type(err))) @@ -550,7 +565,7 @@ class WiredTigerTestCase(unittest.TestCase): """ try: expr() - except BaseException, err: + except BaseException as err: sys.stderr.write('Exception: ' + str(err)) raise @@ -607,7 +622,7 @@ class WiredTigerTestCase(unittest.TestCase): @staticmethod def prout(s): - os.write(WiredTigerTestCase._dupout, s + '\n') + os.write(WiredTigerTestCase._dupout, str.encode(s + '\n')) def pr(self, s): """ @@ -632,6 +647,30 @@ class WiredTigerTestCase(unittest.TestCase): traceback.print_exception(excinfo[0], excinfo[1], excinfo[2], None, WiredTigerTestCase._resultfile) WiredTigerTestCase._resultfile.write('\n') + def recno(self, i): + """ + return a recno key + """ + if _python3: + return i + else: + return long(i) + + def ord_byte(self, b): + """ + return the 'ord' of a single byte. + In Python2 a set of bytes is represented as a string, and a single + byte is a string of length one. In Python3, bytes are an array of + ints, so no explicit ord() call is needed. + """ + if _python3: + return b + else: + return ord(b) + + def is_python3(self): + return _python3 + # print directly to tty, useful for debugging def tty(self, message): WiredTigerTestCase.tty(message) diff --git a/src/third_party/wiredtiger/test/suite/wtthread.py b/src/third_party/wiredtiger/test/suite/wtthread.py index 5eab6527fe8..482bdd73c4b 100644..100755 --- a/src/third_party/wiredtiger/test/suite/wtthread.py +++ b/src/third_party/wiredtiger/test/suite/wtthread.py @@ -26,7 +26,10 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. -import Queue +try: + import Queue as queue # python2 +except ImportError: + import queue import os, shutil, sys, threading, time, wiredtiger, wttest from helper import compare_tables @@ -86,13 +89,13 @@ class backup_thread(threading.Thread): # wttest to do that.. if not compare_tables( self, sess, uris, "checkpoint=WiredTigerCheckpoint"): - print "Error: checkpoint tables differ." + print("Error: checkpoint tables differ.") else: wttest.WiredTigerTestCase.printVerbose( 3, "Checkpoint tables match") if not compare_tables(self, bkp_session, uris): - print "Error: backup tables differ." + print("Error: backup tables differ.") else: wttest.WiredTigerTestCase.printVerbose( 3, "Backup tables match") @@ -109,11 +112,11 @@ class backup_thread(threading.Thread): # 'd' for drop a table # 't' for create a table and insert a single item into it class op_thread(threading.Thread): - def __init__(self, conn, uris, key_fmt, queue, done): + def __init__(self, conn, uris, key_fmt, work_queue, done): self.conn = conn self.uris = uris self.key_fmt = key_fmt - self.queue = queue + self.work_queue = work_queue self.done = done threading.Thread.__init__(self) @@ -127,7 +130,7 @@ class op_thread(threading.Thread): cursors.append(sess.open_cursor(next_uri, None, None)) while not self.done.isSet(): try: - op, key, value = self.queue.get_nowait() + op, key, value = self.work_queue.get_nowait() if op == 'gi': # Group insert a number of tables. sess.begin_transaction() for next_cur in cursors: @@ -166,8 +169,8 @@ class op_thread(threading.Thread): # These operations can fail, if the drop in another # thread happened pass - self.queue.task_done() - except Queue.Empty: + self.work_queue.task_done() + except queue.Empty: # Wait on the queue until done is flagged time.sleep(0.01) if (len(self.uris) == 1): |