diff options
| author | Michael Cahill <michael.cahill@wiredtiger.com> | 2012-02-03 13:25:00 +1100 |
|---|---|---|
| committer | Michael Cahill <michael.cahill@wiredtiger.com> | 2012-02-03 13:25:00 +1100 |
| commit | 3438d9140b9937ad16350087c7cfed379c7ebf9b (patch) | |
| tree | 966fc0e033bf0fd29d188f03dae93fe5e4687080 | |
| parent | c01aa57f3619417152d3b27d91856b9bf03d3bb0 (diff) | |
| parent | 8347925459db58c6c74dc80f437d90ed6ffeac54 (diff) | |
| download | mongo-3438d9140b9937ad16350087c7cfed379c7ebf9b.tar.gz | |
merge tip
465 files changed, 91147 insertions, 0 deletions
diff --git a/.hgignore b/.hgignore new file mode 100644 index 00000000000..a523a281339 --- /dev/null +++ b/.hgignore @@ -0,0 +1,24 @@ +~$ +\.l?o$ +\.swp$ +\.pyc$ +\.class$ +\.dSYM$ +/Makefile.in$ +/tags$ +^configure.*$ +^build.*/(\.deps|COPYING|ChangeLog|INSTALL|Makefile.*|NEWS|README) +^build.*/(aclocal\.m4|config\..*|configure|stamp-h1|prototype.chk|w.*\.h) +^build.*/(ex.*|lib.*|test.*|wt_.*|_wiredtiger.so) +^build_posix/gnu-support/(compile|depcomp|install-sh|ltmain.sh|missing) +^build_posix/autom4te.cache +^docs/(doxygen.log|installdox|search) +^docs/[^/]*\.(css|html|js|png)$ +^docs/java +^docs/latex +^docs/python +^docs/swig +^lang/python/(wiredtiger.py|wiredtiger_wrap.c) +^releases +^src/server +^test/bt/(CONFIG|__rand|__wt.bdb|__wt.run|__wt.wt|db|t|vgout\..*) diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 00000000000..5a67734db28 --- /dev/null +++ b/AUTHORS @@ -0,0 +1 @@ +WiredTiger, Inc. diff --git a/bench/tcbench/Makefile.am b/bench/tcbench/Makefile.am new file mode 100644 index 00000000000..a48dafbcf7d --- /dev/null +++ b/bench/tcbench/Makefile.am @@ -0,0 +1,5 @@ +INCLUDES = -I$(top_builddir) +LDADD = $(top_builddir)/libwiredtiger.la + +noinst_PROGRAMS = wttest +wttest_SOURCES = wttest.c diff --git a/bench/tcbench/README b/bench/tcbench/README new file mode 100644 index 00000000000..5e4b2307f3a --- /dev/null +++ b/bench/tcbench/README @@ -0,0 +1,35 @@ +================================================================ + Comparison of key/value stores +================================================================ + +In this directory is a simple test of WiredTiger that inserts, then +reads, some simple records in a single file. + +This code is designed to match the functionality of the Tokyo Cabinet +"bros" tests, so that the results can be compared across various +key/value stores. + +To run the test standalone, first build WiredTiger in the top-level +"build_posix" directory, then do the following in this directory: + + $ make + $ ./wttest write file:casket.wt 1000000 + $ ./wttest read file:casket.wt 1000000 + +To compare the results from WiredTiger with various other stores: + + 1. Configure and build Tokyo Cabinet + + 2. Apply the tokyocabinet-test.patch in the bros subdirectory; + this patch updates the Makefile to build wttest and to build a + local version of Berkeley DB, updates the "reporter" script to + run wttest, as well as fixing bugs in the reporter script. + + 3. Build the test programs, using a command something like: + $ env \ + WT_HOME=path-to-WiredTiger \ + BDB_HOME=path-to-BerkeleyDB \ + make bdbtest tctest wttest + + 4. Run "reporter": + $ ./reporter diff --git a/bench/tcbench/tokyocabinet-test.patch b/bench/tcbench/tokyocabinet-test.patch new file mode 100644 index 00000000000..cd293c6bbcb --- /dev/null +++ b/bench/tcbench/tokyocabinet-test.patch @@ -0,0 +1,193 @@ +*** Makefile.orig Mon Jan 26 02:44:10 2009 +--- Makefile Tue Sep 13 15:17:04 2011 +*************** +*** 12,18 **** + + # Targets + MYBINS = tctest qdbmtest ndbmtest sdbmtest gdbmtest tdbtest cdbtest bdbtest \ +! maptest sqltest cmpsqltctest + + + +--- 12,18 ---- + + # Targets + MYBINS = tctest qdbmtest ndbmtest sdbmtest gdbmtest tdbtest cdbtest bdbtest \ +! maptest sqltest cmpsqltctest wttest + + + +*************** +*** 98,106 **** + + + bdbtest : bdbtest.c +! gcc -I/usr/local/bdb/include -D_GNU_SOURCE=1 \ + -Wall -O3 -o $@ bdbtest.c \ +! -static -L/usr/local/bdb/lib -ldb -lpthread -lc + + + maptest : maptest.cc +--- 98,112 ---- + + + bdbtest : bdbtest.c +! gcc -I$(BDB_HOME)/build_unix -D_GNU_SOURCE=1 \ + -Wall -O3 -o $@ bdbtest.c \ +! -static $(BDB_HOME)/build_unix/libdb.a -lpthread -lc +! +! +! wttest : $(WT_HOME)/bench/tcbench/wttest.c +! gcc -I$(WT_HOME)/build_posix -D_GNU_SOURCE=1 \ +! -Wall -O3 -o $@ $(WT_HOME)/bench/tcbench/wttest.c \ +! -static $(WT_HOME)/build_posix/.libs/libwiredtiger.a -lpthread -lc + + + maptest : maptest.cc +--- reporter.orig 2008-07-27 22:30:39.000000000 +1000 ++++ reporter 2011-08-05 17:20:26.500784436 +1000 +@@ -11,9 +11,9 @@ + + use constant { + RECNUM => 1000000, +- TESTCOUNT => 20, ++ TESTCOUNT => 7, + REMOVETOP => 2, +- REMOVEBOTTOM => 8, ++ REMOVEBOTTOM => 2, + }; + + my @commands = ( +@@ -47,6 +47,27 @@ + './bdbtest btread -rnd casket.bdbb_r ' . RECNUM, + './tctest flwrite casket.tcf ' . RECNUM, + './tctest flread casket.tcf ' . RECNUM, ++ ++ './wttest write -bulk file:casket.wt_b ' . RECNUM, ++ './wttest read file:casket.wt_b ' . RECNUM, ++ './wttest write file:casket.wt_a ' . RECNUM, ++ './wttest read file:casket.wt_a ' . RECNUM, ++ './wttest write -rnd file:casket.wt_r ' . RECNUM, ++ './wttest read -rnd file:casket.wt_r ' . RECNUM, ++ ++ './wttest vlcswrite -bulk file:casket.vlwt_b ' . RECNUM, ++ './wttest vlcsread file:casket.vlwt_b ' . RECNUM, ++ './wttest vlcswrite file:casket.vlwt_a ' . RECNUM, ++ './wttest vlcsread file:casket.vlwt_a ' . RECNUM, ++ './wttest vlcswrite -rnd file:casket.vlwt_r ' . RECNUM, ++ './wttest vlcsread -rnd file:casket.vlwt_r ' . RECNUM, ++ ++ './wttest flcswrite file:casket.flwt_b ' . RECNUM, ++ './wttest flcsread file:casket.flwt_b ' . RECNUM, ++ './wttest flcswrite file:casket.flwt_a ' . RECNUM, ++ './wttest flcsread file:casket.flwt_a ' . RECNUM, ++ './wttest flcswrite -rnd file:casket.flwt_r ' . RECNUM, ++ './wttest flcsread -rnd file:casket.flwt_r ' . RECNUM, + ); + + my @names = ( +@@ -65,6 +86,15 @@ + 'casket.bdbb', + 'casket.bdbb_r', + 'casket.tcf', ++ 'casket.wt_b', ++ 'casket.wt_a', ++ 'casket.wt_r', ++ 'casket.vlwt_b', ++ 'casket.vlwt_a', ++ 'casket.vlwt_r', ++ 'casket.flwt_b', ++ 'casket.flwt_a', ++ 'casket.flwt_r', + ); + + foreach my $name (@names){ +@@ -120,22 +150,36 @@ + } + printf("\n"); + +-printf("%s,%.5f,%.5f,%d\n", "TC", $table[0][1], $table[1][1], $sizes[0]); +-printf("%s,%.5f,%.5f,%d\n", "QDBM", $table[2][1], $table[3][1], $sizes[1]); +-printf("%s,%.5f,%.5f,%d\n", "NDBM", $table[4][1], $table[5][1], $sizes[2]); +-printf("%s,%.5f,%.5f,%d\n", "SDBM", $table[6][1], $table[7][1], $sizes[3]); +-printf("%s,%.5f,%.5f,%d\n", "GDBM", $table[8][1], $table[9][1], $sizes[4]); +-printf("%s,%.5f,%.5f,%d\n", "TDB", $table[10][1], $table[11][1], $sizes[5]); +-printf("%s,%.5f,%.5f,%d\n", "CDB", $table[12][1], $table[13][1], $sizes[6]); +-printf("%s,%.5f,%.5f,%d\n", "BDB", $table[14][1], $table[15][1], $sizes[7]); ++printf("%s,%.5f,%.5f,%d\n", "TC-HASH", $table[0][1], $table[1][1], $sizes[0]); ++printf("%s,%.5f,%.5f,%d\n", "QDBM-HASH", $table[2][1], $table[3][1], $sizes[1]); ++#printf("%s,%.5f,%.5f,%d\n", "NDBM", $table[4][1], $table[5][1], $sizes[2]); ++#printf("%s,%.5f,%.5f,%d\n", "SDBM", $table[6][1], $table[7][1], $sizes[3]); ++#printf("%s,%.5f,%.5f,%d\n", "GDBM", $table[8][1], $table[9][1], $sizes[4]); ++#printf("%s,%.5f,%.5f,%d\n", "TDB", $table[10][1], $table[11][1], $sizes[5]); ++#printf("%s,%.5f,%.5f,%d\n", "CDB", $table[12][1], $table[13][1], $sizes[6]); ++printf("%s,%.5f,%.5f,%d\n", "BDB-HASH", $table[14][1], $table[15][1], $sizes[7]); ++ ++# Ordered trees, in-order writes + printf("%s,%.5f,%.5f,%d\n", "TC-BT-ASC", $table[16][1], $table[17][1], $sizes[8]); +-printf("%s,%.5f,%.5f,%d\n", "TC-BT-RND", $table[18][1], $table[19][1], $sizes[9]); + printf("%s,%.5f,%.5f,%d\n", "QDBM-BT-ASC", $table[20][1], $table[21][1], $sizes[10]); +-printf("%s,%.5f,%.5f,%d\n", "QDBM-BT-RND", $table[22][1], $table[23][1], $sizes[11]); + printf("%s,%.5f,%.5f,%d\n", "BDB-BT-ASC", $table[24][1], $table[25][1], $sizes[12]); +-printf("%s,%.5f,%.5f,%d\n", "BDB-BT-RND", $table[26][1], $table[27][1], $sizes[13]); +-printf("%s,%.5f,%.5f,%d\n", "TC-FIXED", $table[28][1], $table[29][1], $sizes[14]); ++printf("%s,%.5f,%.5f,%d\n", "WT-BT-BULK", $table[30][1], $table[31][1], $sizes[15]); ++printf("%s,%.5f,%.5f,%d\n", "WT-BT-ASC", $table[32][1], $table[33][1], $sizes[16]); + ++# Ordered trees, random writes ++printf("%s,%.5f,%.5f,%d\n", "TC-BT-RND", $table[18][1], $table[19][1], $sizes[9]); ++printf("%s,%.5f,%.5f,%d\n", "QDBM-BT-RND", $table[22][1], $table[23][1], $sizes[11]); ++printf("%s,%.5f,%.5f,%d\n", "BDB-BT-RND", $table[26][1], $table[27][1], $sizes[13]); ++printf("%s,%.5f,%.5f,%d\n", "WT-BT-RND", $table[34][1], $table[35][1], $sizes[17]); + ++# Fixed-size / column stores ++printf("%s,%.5f,%.5f,%d\n", "TC-FIXED", $table[28][1], $table[29][1], $sizes[14]); ++printf("%s,%.5f,%.5f,%d\n", "WT-VAR-BULK", $table[36][1], $table[37][1], $sizes[18]); ++printf("%s,%.5f,%.5f,%d\n", "WT-VAR", $table[38][1], $table[39][1], $sizes[19]); ++printf("%s,%.5f,%.5f,%d\n", "WT-VAR-RND", $table[40][1], $table[41][1], $sizes[20]); ++ ++printf("%s,%.5f,%.5f,%d\n", "WT-FIX-BULK", $table[42][1], $table[43][1], $sizes[21]); ++printf("%s,%.5f,%.5f,%d\n", "WT-FIX", $table[44][1], $table[45][1], $sizes[22]); ++printf("%s,%.5f,%.5f,%d\n", "WT-FIX-RND", $table[46][1], $table[47][1], $sizes[23]); + + # END OF FILE +--- reporter.orig 2011-08-05 17:20:26.500784436 +1000 ++++ reporter 2011-09-14 06:19:44.702015434 +1000 +@@ -33,16 +33,16 @@ + './cdbtest read casket.cdbh ' . RECNUM, + './bdbtest write casket.bdbh ' . RECNUM, + './bdbtest read casket.bdbh ' . RECNUM, +- './tctest btwrite casket.tcb ' . RECNUM, +- './tctest btread casket.tcb ' . RECNUM, ++ './tctest btwrite casket.tcb_a ' . RECNUM, ++ './tctest btread casket.tcb_a ' . RECNUM, + './tctest btwrite -rnd casket.tcb_r ' . RECNUM, + './tctest btread -rnd casket.tcb_r ' . RECNUM, +- './qdbmtest btwrite casket.qdbb ' . RECNUM, +- './qdbmtest btread casket.qdbb ' . RECNUM, ++ './qdbmtest btwrite casket.qdbb_a ' . RECNUM, ++ './qdbmtest btread casket.qdbb_a ' . RECNUM, + './qdbmtest btwrite -rnd casket.qdbb_r ' . RECNUM, + './qdbmtest btread -rnd casket.qdbb_r ' . RECNUM, +- './bdbtest btwrite casket.bdbb ' . RECNUM, +- './bdbtest btread casket.bdbb ' . RECNUM, ++ './bdbtest btwrite casket.bdbb_a ' . RECNUM, ++ './bdbtest btread casket.bdbb_a ' . RECNUM, + './bdbtest btwrite -rnd casket.bdbb_r ' . RECNUM, + './bdbtest btread -rnd casket.bdbb_r ' . RECNUM, + './tctest flwrite casket.tcf ' . RECNUM, +@@ -79,11 +79,11 @@ + 'casket.tdbh', + 'casket.cdbh', + 'casket.bdbh', +- 'casket.tcb', ++ 'casket.tcb_a', + 'casket.tcb_r', +- 'casket.qdbb', ++ 'casket.qdbb_a', + 'casket.qdbb_r', +- 'casket.bdbb', ++ 'casket.bdbb_a', + 'casket.bdbb_r', + 'casket.tcf', + 'casket.wt_b', diff --git a/bench/tcbench/wttest.c b/bench/tcbench/wttest.c new file mode 100644 index 00000000000..9cb0ce8d804 --- /dev/null +++ b/bench/tcbench/wttest.c @@ -0,0 +1,608 @@ +/************************************************************************************************* + * Microbenchmark of WiredTiger + * Designed to be comparable with the TokyoCabinet "bros" tests. + *************************************************************************************************/ + + +#include <stdbool.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <wiredtiger.h> + +#undef TRUE +#define TRUE 1 /* boolean true */ +#undef FALSE +#define FALSE 0 /* boolean false */ + +#define RECBUFSIZ 32 /* buffer for records */ + + +/* global variables */ +const char *progname; /* program name */ +int showprgr; /* whether to show progression */ + + +/* function prototypes */ +int main(int argc, char **argv); +void usage(void); +int setup(char *name, const char *kf, const char *vf, const char *cconfig, WT_CURSOR **cursor); +int teardown(void); +int runwrite(int argc, char **argv); +int runread(int argc, char **argv); +int runvlcswrite(int argc, char **argv); +int runvlcsread(int argc, char **argv); +int runflcswrite(int argc, char **argv); +int runflcsread(int argc, char **argv); +int myrand(void); +int dowrite(char *name, int rnum, int bulk, int rnd); +int doread(char *name, int rnum, int rnd); +int dovlcswrite(char *name, int rnum, int bulk, int rnd); +int dovlcsread(char *name, int rnum, int rnd); +int doflcswrite(char *name, int rnum, int bulk, int rnd); +int doflcsread(char *name, int rnum, int rnd); + + +/* main routine */ +int main(int argc, char **argv){ + int rv; + progname = argv[0]; + showprgr = TRUE; + if(getenv("HIDEPRGR")) + showprgr = FALSE; + srand48(1978); + if(argc < 2) + usage(); + rv = 0; + if(!strcmp(argv[1], "write")){ + rv = runwrite(argc, argv); + } else if(!strcmp(argv[1], "read")){ + rv = runread(argc, argv); + } else if(!strcmp(argv[1], "vlcswrite")){ + rv = runvlcswrite(argc, argv); + } else if(!strcmp(argv[1], "vlcsread")){ + rv = runvlcsread(argc, argv); + } else if(!strcmp(argv[1], "flcswrite")){ + rv = runflcswrite(argc, argv); + } else if(!strcmp(argv[1], "flcsread")){ + rv = runflcsread(argc, argv); + } else { + usage(); + } + return rv; +} + + +/* print the usage and exit */ +void usage(void){ + fprintf(stderr, "%s: test cases for WiredTiger\n", progname); + fprintf(stderr, "\n"); + fprintf(stderr, "usage:\n"); + fprintf(stderr, " %s write [-bulk|-rnd] name rnum\n", progname); + fprintf(stderr, " %s read [-rnd] name rnum\n", progname); + fprintf(stderr, " %s vlcswrite [-bulk|-rnd] name rnum\n", progname); + fprintf(stderr, " %s vlcsread [-rnd] name rnum\n", progname); + fprintf(stderr, " %s flcswrite [-bulk|-rnd] name rnum\n", progname); + fprintf(stderr, " %s flcsread [-rnd] name rnum\n", progname); + fprintf(stderr, "\n"); + exit(1); +} + + +/* parse arguments of write command */ +int runwrite(int argc, char **argv){ + char *name, *rstr; + int bulk, i, rnd, rnum, rv; + name = NULL; + rstr = NULL; + bulk = rnd = FALSE; + rnum = 0; + for(i = 2; i < argc; i++){ + if(!name && argv[i][0] == '-'){ + if(!name && !strcmp(argv[i], "-bulk")) + bulk = TRUE; + else if(!name && !strcmp(argv[i], "-rnd")) + rnd = TRUE; + else + usage(); + } else if(!name){ + name = argv[i]; + } else if(!rstr){ + rstr = argv[i]; + } else { + usage(); + } + } + if(!name || !rstr) + usage(); + rnum = atoi(rstr); + if(rnum < 1) + usage(); + rv = dowrite(name, rnum, bulk, rnd); + return rv; +} + + +/* parse arguments of read command */ +int runread(int argc, char **argv){ + char *name, *rstr; + int i, rnd, rnum, rv; + name = NULL; + rstr = NULL; + rnd = FALSE; + rnum = 0; + for(i = 2; i < argc; i++){ + if(!name && argv[i][0] == '-'){ + if(!name && !strcmp(argv[i], "-rnd")) + rnd = TRUE; + else + usage(); + } else if(!name){ + name = argv[i]; + } else if(!rstr){ + rstr = argv[i]; + } else { + usage(); + } + } + if(!name || !rstr) + usage(); + rnum = atoi(rstr); + if(rnum < 1) + usage(); + rv = doread(name, rnum, rnd); + return rv; +} + + +/* parse arguments of write command */ +int runvlcswrite(int argc, char **argv){ + char *name, *rstr; + int bulk, i, rnd, rnum, rv; + name = NULL; + rstr = NULL; + rnum = 0; + bulk = rnd = FALSE; + for(i = 2; i < argc; i++){ + if(!name && argv[i][0] == '-'){ + if(!name && !strcmp(argv[i], "-bulk")) + bulk = TRUE; + else if(!name && !strcmp(argv[i], "-rnd")) + rnd = TRUE; + else + usage(); + } else if(!name){ + name = argv[i]; + } else if(!rstr){ + rstr = argv[i]; + } else { + usage(); + } + } + if(!name || !rstr) + usage(); + rnum = atoi(rstr); + if(rnum < 1) + usage(); + rv = dovlcswrite(name, rnum, bulk, rnd); + return rv; +} + + +/* parse arguments of read command */ +int runvlcsread(int argc, char **argv){ + char *name, *rstr; + int i, rnd, rnum, rv; + name = NULL; + rstr = NULL; + rnum = 0; + rnd = FALSE; + for(i = 2; i < argc; i++){ + if(!name && argv[i][0] == '-'){ + if(!name && !strcmp(argv[i], "-rnd")) + rnd = TRUE; + else + usage(); + } else if(!name){ + name = argv[i]; + } else if(!rstr){ + rstr = argv[i]; + } else { + usage(); + } + } + if(!name || !rstr) + usage(); + rnum = atoi(rstr); + if(rnum < 1) + usage(); + rv = dovlcsread(name, rnum, rnd); + return rv; +} + + +/* parse arguments of write command */ +int runflcswrite(int argc, char **argv){ + char *name, *rstr; + int bulk, i, rnd, rnum, rv; + name = NULL; + rstr = NULL; + rnum = 0; + bulk = rnd = FALSE; + for(i = 2; i < argc; i++){ + if(!name && argv[i][0] == '-'){ + if(!name && !strcmp(argv[i], "-bulk")) + bulk = TRUE; + else if(!name && !strcmp(argv[i], "-rnd")) + rnd = TRUE; + else + usage(); + } else if(!name){ + name = argv[i]; + } else if(!rstr){ + rstr = argv[i]; + } else { + usage(); + } + } + if(!name || !rstr) + usage(); + rnum = atoi(rstr); + if(rnum < 1) + usage(); + rv = doflcswrite(name, rnum, bulk, rnd); + return rv; +} + + +/* parse arguments of read command */ +int runflcsread(int argc, char **argv){ + char *name, *rstr; + int i, rnd, rnum, rv; + name = NULL; + rstr = NULL; + rnum = 0; + rnd = FALSE; + for(i = 2; i < argc; i++){ + if(!name && argv[i][0] == '-'){ + if(!name && !strcmp(argv[i], "-rnd")) + rnd = TRUE; + else + usage(); + } else if(!name){ + name = argv[i]; + } else if(!rstr){ + rstr = argv[i]; + } else { + usage(); + } + } + if(!name || !rstr) + usage(); + rnum = atoi(rstr); + if(rnum < 1) + usage(); + rv = doflcsread(name, rnum, rnd); + return rv; +} + + +/* pseudo random number generator */ +int myrand(void){ + static int cnt = 0; + return (int)((lrand48() + cnt++) & 0x7FFFFFFF); +} + +WT_CONNECTION *conn; + +int setup(char *name, const char *kf, const char *vf, const char *cconfig, WT_CURSOR **cursor){ + WT_SESSION *session; + int creating, ret; + char tconfig[64]; + + creating = (kf != NULL); + + if((ret = wiredtiger_open(NULL, NULL, "create", &conn) != 0) || + (ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + return ret; + + /* If we get a configuration, create the table. */ + if(creating) { + (void)session->drop(session, name, "force"); + snprintf(tconfig, sizeof(tconfig), "key_format=%s,value_format=%s", kf, vf); + if ((ret = session->create(session, name, tconfig)) != 0) + return ret; + } + + return session->open_cursor(session, name, NULL, cconfig, cursor); +} + +int teardown(void){ + int ret = 0; + if (conn != NULL) { + ret = conn->close(conn, NULL); + conn = NULL; + } + return ret; +} + +/* perform write command */ +int dowrite(char *name, int rnum, int bulk, int rnd){ + WT_CURSOR *c; + WT_ITEM key, value; + int i, err; + char buf[RECBUFSIZ]; + if(showprgr) + printf("<Write Test of Row Store>\n name=%s rnum=%d\n\n", name, rnum); + /* open a database */ + if(setup(name, "u", "u", bulk ? "bulk" : NULL, &c) != 0) { + fprintf(stderr, "create failed\n"); + (void)teardown(); + return 1; + } + err = FALSE; + key.data = value.data = buf; + key.size = value.size = 8; + /* loop for each record */ + for(i = 1; i <= rnum; i++){ + /* store a record */ + sprintf(buf, "%08d", rnd ? myrand() % rnum + 1 : i); + c->set_key(c, &key); + c->set_value(c, &value); + if((err = c->insert(c)) != 0 && err != WT_DUPLICATE_KEY) { + fprintf(stderr, "insert failed\n"); + break; + } + /* print progression */ + if(showprgr && rnum > 250 && i % (rnum / 250) == 0){ + putchar('.'); + fflush(stdout); + if(i == rnum || i % (rnum / 10) == 0){ + printf(" (%08d)\n", i); + fflush(stdout); + } + } + } + /* close the database */ + if(teardown() != 0) { + fprintf(stderr, "close failed\n"); + return 1; + } + if(showprgr && !err) + printf("ok\n\n"); + return err ? 1 : 0; +} + + +/* perform read command */ +int doread(char *name, int rnum, int rnd){ + WT_CURSOR *c; + WT_ITEM key, value; + int i, err; + char buf[RECBUFSIZ]; + if(showprgr) + printf("<Read Test of Row Store>\n name=%s rnum=%d\n\n", name, rnum); + /* open a database */ + if(setup(name, NULL, NULL, NULL, &c) != 0){ + fprintf(stderr, "open failed\n"); + return 1; + } + err = FALSE; + key.data = value.data = buf; + key.size = value.size = 8; + /* loop for each record */ + for(i = 1; i <= rnum; i++){ + /* store a record */ + sprintf(buf, "%08d", rnd ? myrand() % rnum + 1 : i); + c->set_key(c, &key); + if(c->search(c) != 0){ + fprintf(stderr, "search failed\n"); + err = TRUE; + break; + } + /* Include the cost of getting the value. */ + c->get_value(c, &value); + /* print progression */ + if(showprgr && rnum > 250 && i % (rnum / 250) == 0){ + putchar('.'); + fflush(stdout); + if(i == rnum || i % (rnum / 10) == 0){ + printf(" (%08d)\n", i); + fflush(stdout); + } + } + } + /* close the database */ + if(teardown() != 0) { + fprintf(stderr, "close failed\n"); + return 1; + } + if(showprgr && !err) + printf("ok\n\n"); + return err ? 1 : 0; +} + +/* perform write command */ +int dovlcswrite(char *name, int rnum, int bulk, int rnd){ + WT_CURSOR *c; + WT_ITEM value; + int i, err; + char buf[RECBUFSIZ]; + if(showprgr) + printf("<Write Test of var-length Column Store>\n name=%s rnum=%d\n\n", name, rnum); + /* open a database */ + if(setup(name, "r", "u", bulk ? "bulk" : NULL, &c) != 0) { + fprintf(stderr, "create failed\n"); + (void)teardown(); + return 1; + } + err = FALSE; + value.data = buf; + value.size = 8; + /* loop for each record */ + for(i = 1; i <= rnum; i++){ + /* store a record */ + sprintf(buf, "%08d", i); + c->set_key(c, (uint64_t)(rnd ? myrand() % rnum + 1 : i)); + c->set_value(c, &value); + if((err = c->insert(c)) != 0 && err != WT_DUPLICATE_KEY) { + fprintf(stderr, "insert failed\n"); + break; + } + /* print progression */ + if(showprgr && rnum > 250 && i % (rnum / 250) == 0){ + putchar('.'); + fflush(stdout); + if(i == rnum || i % (rnum / 10) == 0){ + printf(" (%08d)\n", i); + fflush(stdout); + } + } + } + /* close the database */ + if(teardown() != 0) { + fprintf(stderr, "close failed\n"); + return 1; + } + if(showprgr && !err) + printf("ok\n\n"); + return err ? 1 : 0; +} + + +/* perform read command */ +int dovlcsread(char *name, int rnum, int rnd){ + WT_CURSOR *c; + int i, err; + WT_ITEM value; + if(showprgr) + printf("<Read Test of var-length Column Store>\n name=%s rnum=%d\n\n", name, rnum); + /* open a database */ + if(setup(name, NULL, NULL, NULL, &c) != 0){ + fprintf(stderr, "open failed\n"); + return 1; + } + err = FALSE; + /* loop for each record */ + for(i = 1; i <= rnum; i++){ + c->set_key(c, (uint64_t)(rnd ? myrand() % rnum + 1 : i)); + if(c->search(c) != 0){ + fprintf(stderr, "search failed\n"); + err = TRUE; + break; + } + /* Include the cost of getting the value. */ + c->get_value(c, &value); + /* print progression */ + if(showprgr && rnum > 250 && i % (rnum / 250) == 0){ + putchar('.'); + fflush(stdout); + if(i == rnum || i % (rnum / 10) == 0){ + printf(" (%08d)\n", i); + fflush(stdout); + } + } + } + /* close the database */ + if(teardown() != 0) { + fprintf(stderr, "close failed\n"); + return 1; + } + if(showprgr && !err) + printf("ok\n\n"); + return err ? 1 : 0; +} + + +/* perform write command */ +int doflcswrite(char *name, int rnum, int bulk, int rnd){ + WT_CURSOR *c; + uint8_t value; + int i, err; + char buf[RECBUFSIZ]; + if(showprgr) + printf("<Write Test of var-length Column Store>\n name=%s rnum=%d\n\n", name, rnum); + /* open a database */ + if(setup(name, "r", "8t", bulk ? "bulk" : NULL, &c) != 0) { + fprintf(stderr, "create failed\n"); + (void)teardown(); + return 1; + } + err = FALSE; + value = 42; + /* loop for each record */ + for(i = 1; i <= rnum; i++){ + /* store a record */ + sprintf(buf, "%08d", i); + c->set_key(c, (uint64_t)(rnd ? myrand() % rnum + 1 : i)); + c->set_value(c, value); + if((err = c->insert(c)) != 0 && err != WT_DUPLICATE_KEY) { + fprintf(stderr, "insert failed\n"); + break; + } + /* print progression */ + if(showprgr && rnum > 250 && i % (rnum / 250) == 0){ + putchar('.'); + fflush(stdout); + if(i == rnum || i % (rnum / 10) == 0){ + printf(" (%08d)\n", i); + fflush(stdout); + } + } + } + /* close the database */ + if(teardown() != 0) { + fprintf(stderr, "close failed\n"); + return 1; + } + if(showprgr && !err) + printf("ok\n\n"); + return err ? 1 : 0; +} + + +/* perform read command */ +int doflcsread(char *name, int rnum, int rnd){ + WT_CURSOR *c; + uint8_t value; + int i, err; + if(showprgr) + printf("<Read Test of var-length Column Store>\n name=%s rnum=%d\n\n", name, rnum); + /* open a database */ + if(setup(name, NULL, NULL, NULL, &c) != 0){ + fprintf(stderr, "open failed\n"); + return 1; + } + err = FALSE; + /* loop for each record */ + for(i = 1; i <= rnum; i++){ + c->set_key(c, (uint64_t)(rnd ? myrand() % rnum + 1 : i)); + if(c->search(c) != 0){ + fprintf(stderr, "search failed\n"); + err = TRUE; + break; + } + /* Include the cost of getting the value. */ + c->get_value(c, &value); + /* print progression */ + if(showprgr && rnum > 250 && i % (rnum / 250) == 0){ + putchar('.'); + fflush(stdout); + if(i == rnum || i % (rnum / 10) == 0){ + printf(" (%08d)\n", i); + fflush(stdout); + } + } + } + /* close the database */ + if(teardown() != 0) { + fprintf(stderr, "close failed\n"); + return 1; + } + if(showprgr && !err) + printf("ok\n\n"); + return err ? 1 : 0; +} + +/* END OF FILE */ diff --git a/build_posix/Make.base b/build_posix/Make.base new file mode 100644 index 00000000000..4202f0fc19c --- /dev/null +++ b/build_posix/Make.base @@ -0,0 +1,48 @@ +ACLOCAL_AMFLAGS = -I build_posix/aclocal + +# BEGIN SUBDIRS, maintained by makemake and subdirs.list +# END SUBDIRS + +lib_LTLIBRARIES = libwiredtiger.la +LDADD = $(lib_LTLIBRARIES) + +bin_PROGRAMS = wt +wt_SOURCES =\ + src/utilities/util_cpyright.c \ + src/utilities/util_create.c \ + src/utilities/util_drop.c \ + src/utilities/util_dump.c \ + src/utilities/util_dumpfile.c \ + src/utilities/util_getopt.c \ + src/utilities/util_list.c \ + src/utilities/util_load.c \ + src/utilities/util_loadtext.c \ + src/utilities/util_main.c \ + src/utilities/util_misc.c \ + src/utilities/util_printlog.c \ + src/utilities/util_read.c \ + src/utilities/util_rename.c \ + src/utilities/util_salvage.c \ + src/utilities/util_stat.c \ + src/utilities/util_upgrade.c \ + src/utilities/util_verbose.c \ + src/utilities/util_verify.c \ + src/utilities/util_write.c + +include_HEADERS= wiredtiger.h +INCLUDES = -I$(srcdir)/src/include + +pkgconfigdir = $(libdir)/pkgconfig +pkgconfig_DATA = wiredtiger.pc + +$(srcdir)/Makefile.am: $(srcdir)/build_posix/Make.base $(srcdir)/build_posix/makemake $(srcdir)/dist/filelist + @cd $(srcdir)/build_posix && sh makemake + +libtool: $(LIBTOOL_DEPS) + $(SHELL) ./config.status libtool + +$(top_srcdir)/src/include/extern.h: auto-includes.chk +$(top_srcdir)/src/include/wt_internal.in: auto-includes.chk + +auto-includes.chk: $(libwiredtiger_la_SOURCES) + @(cd $(srcdir)/dist && sh s_prototypes && sh s_typedef -b) && touch $@ diff --git a/build_posix/Make.subdirs b/build_posix/Make.subdirs new file mode 100644 index 00000000000..409b10ed5d1 --- /dev/null +++ b/build_posix/Make.subdirs @@ -0,0 +1,17 @@ +# List of sub-directories, used by makemake to create Makefile.am +# +# The format is: +# <dir>[ <condition>] +# +# If the directory exists, it is added to AUTO_SUBDIRS. +# If a condition is included, the subdir is made conditional via AM_CONDITIONAL +bench/tcbench +examples/c +ext/collators/reverse +ext/compressors/bzip2_compress BZIP2 +ext/compressors/snappy_compress SNAPPY +ext/compressors/nop_compress +lang/python PYTHON +test/format HAVE_BDB +test/salvage +test/thread diff --git a/build_posix/aclocal/cond-if.m4 b/build_posix/aclocal/cond-if.m4 new file mode 100644 index 00000000000..df4b2c4b8ac --- /dev/null +++ b/build_posix/aclocal/cond-if.m4 @@ -0,0 +1,14 @@ +dnl AC_CONFIG_FILES conditionalization requires using AM_COND_IF, however +dnl AM_COND_IF is new to Automake 1.11. To use it on new Automake without +dnl requiring same, a fallback implementation for older Autoconf is provided. +dnl Note that disabling of AC_CONFIG_FILES requires Automake 1.11, this code +dnl is correct only in terms of m4sh generated script. +m4_ifndef([AM_COND_IF], [AC_DEFUN([AM_COND_IF], [ +if test -z "$$1_TRUE"; then : + m4_n([$2])[]dnl +m4_ifval([$3], +[else + $3 +])dnl +fi[]dnl +])]) diff --git a/build_posix/aclocal/options.m4 b/build_posix/aclocal/options.m4 new file mode 100644 index 00000000000..16b250cae07 --- /dev/null +++ b/build_posix/aclocal/options.m4 @@ -0,0 +1,100 @@ +# Optional configuration. +AC_DEFUN([AM_OPTIONS], [ + +AH_TEMPLATE(HAVE_ATTACH, [Define to 1 to pause for debugger attach on failure.]) +AC_MSG_CHECKING(if --enable-attach option specified) +AC_ARG_ENABLE(attach, + [AC_HELP_STRING([--enable-attach], + [Configure for debugger attach on failure.])], r=$enableval, r=no) +case "$r" in +no) wt_cv_enable_attach=no;; +*) AC_DEFINE(HAVE_ATTACH) + wt_cv_enable_attach=yes;; +esac +AC_MSG_RESULT($wt_cv_enable_attach) + +AC_MSG_CHECKING(if --enable-bzip2 option specified) +AC_ARG_ENABLE(bzip2, + [AC_HELP_STRING([--enable-bzip2], + [Build the bzip2 compressor extension.])], r=$enableval, r=no) +case "$r" in +no) wt_cv_enable_bzip2=no;; +*) wt_cv_enable_bzip2=yes;; +esac +AC_MSG_RESULT($wt_cv_enable_bzip2) +AM_CONDITIONAL([BZIP2], [test x$wt_cv_enable_bzip2 = xyes]) + +AC_MSG_CHECKING(if --enable-debug option specified) +AC_ARG_ENABLE(debug, + [AC_HELP_STRING([--enable-debug], + [Configure for debug symbols.])], r=$enableval, r=no) +case "$r" in +no) wt_cv_enable_debug=no;; +*) wt_cv_enable_debug=yes;; +esac +AC_MSG_RESULT($wt_cv_enable_debug) + +AH_TEMPLATE(HAVE_DIAGNOSTIC, [Define to 1 for diagnostic tests.]) +AC_MSG_CHECKING(if --enable-diagnostic option specified) +AC_ARG_ENABLE(diagnostic, + [AC_HELP_STRING([--enable-diagnostic], + [Configure for diagnostic tests.])], r=$enableval, r=no) +case "$r" in +no) wt_cv_enable_diagnostic=no;; +*) AC_DEFINE(HAVE_DIAGNOSTIC) + wt_cv_enable_diagnostic=yes;; +esac +AC_MSG_RESULT($wt_cv_enable_diagnostic) + +AC_MSG_CHECKING(if --enable-python option specified) +AC_ARG_ENABLE(python, + [AC_HELP_STRING([--enable-python], + [Configure for python symbols.])], r=$enableval, r=no) +case "$r" in +no) wt_cv_enable_python=no;; +*) wt_cv_enable_python=yes;; +esac +AC_MSG_RESULT($wt_cv_enable_python) +AM_CONDITIONAL(PYTHON, test x$wt_cv_enable_python = xyes) +if test "$enable_shared" = "no"; then + AC_MSG_ERROR([--enable-python requires shared libraries]) +fi + +AC_MSG_CHECKING(if --enable-snappy option specified) +AC_ARG_ENABLE(snappy, + [AC_HELP_STRING([--enable-snappy], + [Build the snappy compressor extension.])], r=$enableval, r=no) +case "$r" in +no) wt_cv_enable_snappy=no;; +*) wt_cv_enable_snappy=yes;; +esac +AC_MSG_RESULT($wt_cv_enable_snappy) +AM_CONDITIONAL([SNAPPY], [test x$wt_cv_enable_snappy = xyes]) + +AH_TEMPLATE(HAVE_VERBOSE, [Define to 1 to support the Env.verbose_set method.]) +AC_MSG_CHECKING(if --enable-verbose option specified) +AC_ARG_ENABLE(verbose, + [AC_HELP_STRING([--enable-verbose], + [Configure for Env.verbose_set method.])], r=$enableval, r=yes) +case "$r" in +no) wt_cv_enable_verbose=no;; +*) AC_DEFINE(HAVE_VERBOSE) + wt_cv_enable_verbose=yes;; +esac +AC_MSG_RESULT($wt_cv_enable_verbose) + +AC_MSG_CHECKING(if --with-spinlock option specified) +AH_TEMPLATE(SPINLOCK_TYPE, [Spinlock type from mutex.h.]) +AC_ARG_WITH(spinlock, + [AC_HELP_STRING([--with-spinlock], + [Spinlock type (pthread_mutex or gcc).])], + [], + [with_spinlock=pthread]) +case "$with_spinlock" in +pthread) AC_DEFINE(SPINLOCK_TYPE, SPINLOCK_PTHREAD_MUTEX);; +gcc) AC_DEFINE(SPINLOCK_TYPE, SPINLOCK_GCC);; +*) AC_MSG_ERROR([Unknown spinlock type "$with_spinlock"]);; +esac +AC_MSG_RESULT($with_spinlock) + +]) diff --git a/build_posix/aclocal/types.m4 b/build_posix/aclocal/types.m4 new file mode 100644 index 00000000000..844c8eadc73 --- /dev/null +++ b/build_posix/aclocal/types.m4 @@ -0,0 +1,140 @@ +# AM_SIGNED_TYPES, AM_UNSIGNED_TYPES -- +# Search standard type names for something of the same size and +# signed-ness as the type we want to declare. +# +# $1 AC_SUBST variable +# $2 typedef name +# $3 number of bytes +AC_DEFUN([AM_SIGNED_TYPES], [ + case "$3" in + "$ac_cv_sizeof_int") + $1="typedef int $2;";; + "$ac_cv_sizeof_char") + $1="typedef char $2;";; + "$ac_cv_sizeof_short") + $1="typedef short $2;";; + "$ac_cv_sizeof_long") + $1="typedef long $2;";; + "$ac_cv_sizeof_long_long") + $1="typedef long long $2;";; + *) + AC_MSG_ERROR([No signed $3-byte type found]);; + esac]) +]) +AC_DEFUN([AM_UNSIGNED_TYPES], [ + case "$3" in + "$ac_cv_sizeof_unsigned_int") + $1="typedef unsigned int $2;";; + "$ac_cv_sizeof_unsigned_char") + $1="typedef unsigned char $2;";; + "$ac_cv_sizeof_unsigned_short") + $1="typedef unsigned short $2;";; + "$ac_cv_sizeof_unsigned_long") + $1="typedef unsigned long $2;";; + "$ac_cv_sizeof_unsigned_long_long") + $1="typedef unsigned long long $2;";; + *) + AC_MSG_ERROR([No unsigned $3-byte type found]);; + esac]) +]) +# AM_TYPES -- +# Create any missing types. +AC_DEFUN([AM_TYPES], [ + # Basic list of include files that might have types. We also use + # as the list of includes directly included by wiredtiger.h. + std_includes=" +#include <sys/types.h> +#include <stdarg.h> +#include <stdint.h> +#include <stdio.h>" + AC_SUBST(wiredtiger_includes_decl) + wiredtiger_includes_decl="$std_includes" + + # Look for variable-sized type names, and if we don't find them, + # create our own. + AC_SUBST(u_char_decl) + AC_CHECK_TYPE(u_char,, + [u_char_decl="typedef unsigned char u_char;"], $std_includes) + AC_SUBST(u_short_decl) + AC_CHECK_TYPE(u_short,, + [u_short_decl="typedef unsigned short u_short;"], $std_includes) + AC_SUBST(u_int_decl) + AC_CHECK_TYPE(u_int,, + [u_int_decl="typedef unsigned int u_int;"], $std_includes) + AC_SUBST(u_long_decl) + AC_CHECK_TYPE(u_long,, + [u_long_decl="typedef unsigned long u_long;"], $std_includes) + AC_SUBST(u_quad_decl) + AC_CHECK_TYPE(u_quad,, + [u_quad_decl="typedef unsigned long long u_quad;"], $std_includes) + + # Look for fixed-size type names, and if we don't find them, create + # our own. + # + # First, figure out the sizes of the standard types. + AC_CHECK_SIZEOF(char,, $std_includes) + AC_CHECK_SIZEOF(unsigned char,, $std_includes) + AC_CHECK_SIZEOF(short,, $std_includes) + AC_CHECK_SIZEOF(unsigned short,, $std_includes) + AC_CHECK_SIZEOF(int,, $std_includes) + AC_CHECK_SIZEOF(unsigned int,, $std_includes) + AC_CHECK_SIZEOF(long,, $std_includes) + AC_CHECK_SIZEOF(unsigned long,, $std_includes) + AC_CHECK_SIZEOF(long long,, $std_includes) + AC_CHECK_SIZEOF(unsigned long long,, $std_includes) + AC_CHECK_SIZEOF(char *,, $std_includes) + + # Second, check for the types we really want, and if we don't find + # them, search for something of the same size and signed-ness. + AC_SUBST(u_int8_decl) + AC_CHECK_TYPE(u_int8_t,, [ + AM_UNSIGNED_TYPES(u_int8_decl, u_int8_t, 1)], $std_includes) + AC_SUBST(int8_decl) + AC_CHECK_TYPE(int8_t,, [ + AM_SIGNED_TYPES(int8_decl, int8_t, 1)], $std_includes) + AC_SUBST(u_int16_decl) + AC_CHECK_TYPE(u_int16_t,, [ + AM_UNSIGNED_TYPES(u_int16_decl, u_int16_t, 2)], $std_includes) + AC_SUBST(int16_decl) + AC_CHECK_TYPE(int16_t,, [ + AM_SIGNED_TYPES(int16_decl, int16_t, 2)], $std_includes) + AC_SUBST(u_int32_decl) + AC_CHECK_TYPE(u_int32_t,, [ + AM_UNSIGNED_TYPES(u_int32_decl, u_int32_t, 4)], $std_includes) + AC_SUBST(int32_decl) + AC_CHECK_TYPE(int32_t,, [ + AM_SIGNED_TYPES(int32_decl, int32_t, 4)], $std_includes) + AC_SUBST(u_int64_decl) + AC_CHECK_TYPE(u_int64_t,, [ + AM_UNSIGNED_TYPES(u_int64_decl, u_int64_t, 8)], $std_includes) + AC_SUBST(int64_decl) + AC_CHECK_TYPE(int64_t,, [ + AM_SIGNED_TYPES(int64_decl, int64_t, 8)], $std_includes) + + # We additionally require FILE, off_t, pid_t, size_t, ssize_t, + # time_t, uintmax_t and uintptr_t. + AC_SUBST(FILE_t_decl) + AC_CHECK_TYPE(FILE *,, AC_MSG_ERROR([No FILE type.]), $std_includes) + AC_SUBST(off_t_decl) + AC_CHECK_TYPE(off_t,, AC_MSG_ERROR([No off_t type.]), $std_includes) + AC_SUBST(pid_t_decl) + AC_CHECK_TYPE(pid_t,, AC_MSG_ERROR([No pid_t type.]), $std_includes) + AC_SUBST(size_t_decl) + AC_CHECK_TYPE(size_t,, AC_MSG_ERROR([No size_t type.]), $std_includes) + AC_SUBST(ssize_t_decl) + AC_CHECK_TYPE(ssize_t,, AC_MSG_ERROR([No size_t type.]), $std_includes) + AC_SUBST(time_t_decl) + AC_CHECK_TYPE(time_t,, AC_MSG_ERROR([No time_t type.]), $std_includes) + + # Some systems don't have a uintmax_t type (for example, FreeBSD 6.2. + # In this case, use an unsigned long long. + AC_SUBST(uintmax_t_decl) + AC_CHECK_TYPE(uintmax_t,, [AC_CHECK_TYPE(unsigned long long, + [uintmax_t_decl="typedef unsigned long long uintmax_t;"], + [uintmax_t_decl="typedef unsigned long uintmax_t;"], + $std_includes)]) + + AC_SUBST(uintptr_t_decl) + AC_CHECK_TYPE(uintptr_t,, + AC_MSG_ERROR([No uintptr_t type.]), $std_includes) +]) diff --git a/build_posix/aclocal/version-set.m4 b/build_posix/aclocal/version-set.m4 new file mode 100644 index 00000000000..f808f217b61 --- /dev/null +++ b/build_posix/aclocal/version-set.m4 @@ -0,0 +1,14 @@ +dnl build by dist/s_version + +VERSION_MAJOR=0 +VERSION_MINOR=10 +VERSION_PATCH=0 +VERSION_STRING='"WiredTiger 0.10.0: (January 13, 2012)"' + +AC_SUBST(VERSION_MAJOR) +AC_SUBST(VERSION_MINOR) +AC_SUBST(VERSION_PATCH) +AC_SUBST(VERSION_STRING) + +VERSION_NOPATCH=0.10 +AC_SUBST(VERSION_NOPATCH) diff --git a/build_posix/aclocal/version.m4 b/build_posix/aclocal/version.m4 new file mode 100644 index 00000000000..d134cf15fad --- /dev/null +++ b/build_posix/aclocal/version.m4 @@ -0,0 +1,2 @@ +dnl WiredTiger product version for AC_INIT. Maintained by dist/s_version +0.10.0 diff --git a/build_posix/configure.ac.in b/build_posix/configure.ac.in new file mode 100644 index 00000000000..8cc5419a690 --- /dev/null +++ b/build_posix/configure.ac.in @@ -0,0 +1,89 @@ +PACKAGE=wiredtiger +AC_INIT(WiredTiger, m4_normalize(m4_include([build_posix/aclocal/version.m4])), + [support@wiredtiger.com]) + +m4_include([build_posix/aclocal/version-set.m4]) + +AC_CONFIG_AUX_DIR([build_posix/gnu-support]) +AC_CONFIG_MACRO_DIR([build_posix/aclocal]) +AC_CONFIG_SRCDIR([dist/RELEASE]) + +AM_INIT_AUTOMAKE +AM_OPTIONS + +# If CFLAGS was not set on entry and we are not debugging, default to -O3 +wt_test_CFLAGS=${CFLAGS+set} +if test "$wt_test_CFLAGS" != set && test "$wt_cv_enable_debug" != "yes"; then + CFLAGS="-O3" +fi + +define([AC_LIBTOOL_LANG_CXX_CONFIG], [:])dnl +define([AC_LIBTOOL_LANG_F77_CONFIG], [:])dnl +LT_INIT([pic-only]) +AC_SUBST([LIBTOOL_DEPS]) + +AC_PROG_CC(cc gcc) +# AC_PROG_CXX(c++ g++) + +# Add debugging flags if requested and not already in CFLAGS +if test "$wt_cv_enable_debug" = "yes"; then + case "$CFLAGS" in + *-g*) ;; + *) AM_CFLAGS="$AM_CFLAGS -g" ;; + esac +fi + +if test "$GCC" = "yes"; then + # The Solaris gcc compiler gets the additional -pthreads flag. + if test "`uname -s`" = "SunOS"; then + AM_CFLAGS="$AM_CFLAGS -pthreads" + fi +else + # The Solaris native compiler gets the additional -mt flag. + if test "`uname -s`" = "SunOS"; then + AM_CFLAGS="$AM_CFLAGS -mt" + fi +fi +AC_SUBST(AM_CFLAGS) + +AM_CONDITIONAL([DEBUG], [test "$wt_cv_enable_debug" = "yes"]) + +# Python API +if test "$wt_cv_enable_python" = "yes"; then + AM_PATH_PYTHON([2.6]) + AC_PATH_PROG([SWIG], [swig], [swig]) +fi + +AM_TYPES + +AC_PROG_INSTALL + +AC_CHECK_LIB(pthread, pthread_create) +AC_CHECK_LIB(dl, dlopen) +AC_CHECK_LIB(rt, sched_yield) +AC_CHECK_FUNCS([fcntl]) +AC_SYS_LARGEFILE + +AC_C_BIGENDIAN + +# Is there a link to a BDB tree that we can use for test/format? +AM_CONDITIONAL([HAVE_BDB], [test -d ../test/format && test -e db]) + +# Warn that diagnostic builds should not be used in production +if test "$wt_cv_enable_diagnostic" = "yes"; then + AC_MSG_WARN([DIAGNOSTIC BUILDS ARE NOT RECOMMENDED FOR PRODUCTION DEPLOYMENT.]) +fi + +# Output files +AC_CONFIG_HEADERS([wiredtiger_config.h:build_posix/config.hin]) + +# BEGIN check existence -- maintained by reconf and Make.subdirs +# END check existence + +AC_CONFIG_FILES([ + Makefile + wiredtiger.h:src/include/wiredtiger.in + wt_internal.h:src/include/wt_internal.in + wiredtiger.pc:build_posix/wiredtiger.pc.in +]) +AC_OUTPUT diff --git a/build_posix/makemake b/build_posix/makemake new file mode 100755 index 00000000000..5db0e484174 --- /dev/null +++ b/build_posix/makemake @@ -0,0 +1,38 @@ +#! /bin/sh +# +# Build Makefile.am + +# Process Make.base, insert subdirs that exist from Make.subdirs +# (in release trees, some of the subdirs might be excluded). +(sed -n '1,/BEGIN SUBDIRS/p' Make.base + +SUBDIRS=. +sed -e 's/#.*$//' -e '/^$/d' Make.subdirs | (while read dir cond ; do + test -d ../$dir || continue + if test -n "$cond" ; then + cat <<EOF +if ${cond} + ${cond}_BUILD = $dir +endif +EOF + SUBDIRS="$SUBDIRS \\ + \$(${cond}_BUILD)" + else + SUBDIRS="$SUBDIRS \\ + $dir" + fi +done + +echo +echo "SUBDIRS = $SUBDIRS") + +# Write the rest of Make.base +sed -n '/END SUBDIRS/,$p' Make.base + +echo +echo "libwiredtiger_la_LDFLAGS = -release @VERSION@" +echo "libwiredtiger_la_SOURCES=\\" +sed -e '/^[a-z]/! d' \ + -e 's/.*/ & \\/' \ + -e '$s/ \\$//' < ../dist/filelist +) > ../Makefile.am diff --git a/build_posix/reconf b/build_posix/reconf new file mode 100755 index 00000000000..506625f6864 --- /dev/null +++ b/build_posix/reconf @@ -0,0 +1,64 @@ +#! /bin/sh + +t=/tmp/__configure.$$ +trap 'rm -f $t; exit 0' 0 1 2 3 13 15 + +# There's a cleanup function so we can easily clean out the directory. +clean() +{ + test -f Makefile && make distclean > /dev/null + rm -rf ../Makefile.am ../Makefile.in \ + ../aclocal.m4 ../autom4te.cache ../config.hin ../config.hin~ \ + ../configure flexlint.out mklog +} + +while : + do case "$1" in + -c) + clean # Clean and leave empty + exit 0;; + *) + clean # Clean and then re-create + break;; + esac +done + +# Build configure.ac +( +echo "# DO NOT EDIT" +echo "# This file is built automatically from build_posix/configure.ac.in." + +sed -n '1,/BEGIN check existence/p' configure.ac.in + +sed -e 's/#.*$//' -e '/^$/d' Make.subdirs | while read dir cond ; do + test -d ../$dir || continue + echo 'AC_CONFIG_FILES(['$dir/Makefile'])' +done + +sed -n '/END check existence/,$p' configure.ac.in +) > ../configure.ac + +# Build Makefile.am +sh ./makemake + +# From here on, work in the top of the tree +cd .. + +# Initialize standard automake files. +cp LICENSE COPYING +cat << END > NEWS +To view release and installation documentation, load the distribution +file docs/index.html into your web browser. +END +cp NEWS ChangeLog +cp NEWS INSTALL + +autoreconf --install + +# Make sure any missing files are writable +chmod 755 build_posix/gnu-support/* + +# Cleanup +rm -rf autom4te.cache + +exit 0 diff --git a/build_posix/wiredtiger.pc.in b/build_posix/wiredtiger.pc.in new file mode 100644 index 00000000000..be257efcef3 --- /dev/null +++ b/build_posix/wiredtiger.pc.in @@ -0,0 +1,11 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: WiredTiger +Description: The WiredTiger Data Engine +Requires: +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lwiredtiger-@VERSION_NOPATCH@ +Cflags: -I${includedir}/wiredtiger-@VERSION_NOPATCH@ diff --git a/dist/RELEASE b/dist/RELEASE new file mode 100644 index 00000000000..78e09485d0a --- /dev/null +++ b/dist/RELEASE @@ -0,0 +1,10 @@ +WIREDTIGER_VERSION_MAJOR=0 +WIREDTIGER_VERSION_MINOR=10 +WIREDTIGER_VERSION_PATCH=0 +WIREDTIGER_VERSION="$WIREDTIGER_VERSION_MAJOR.$WIREDTIGER_VERSION_MINOR.$WIREDTIGER_VERSION_PATCH" + +WIREDTIGER_RELEASE_DATE=`date "+%B %e, %Y"` + +WIREDTIGER_VERSION_STRING="WiredTiger $WIREDTIGER_VERSION: ($WIREDTIGER_RELEASE_DATE)" + +WIREDTIGER_VERSION_UNIQUE_NAME=`printf "_%d%03d" $WIREDTIGER_VERSION_MAJOR $WIREDTIGER_VERSION_MINOR` diff --git a/dist/api_data.py b/dist/api_data.py new file mode 100644 index 00000000000..7629ba0f4e4 --- /dev/null +++ b/dist/api_data.py @@ -0,0 +1,411 @@ +# This file is a python script that describes the WiredTiger API. + +class Error: + def __init__(self, name, desc, long_desc=None, **flags): + self.name = name + self.desc = desc + self.long_desc = long_desc + self.flags = flags + +errors = [ + Error('WT_DEADLOCK', 'conflict between concurrent operations', ''' + This error is generated when an operation cannot be completed + due to a conflict with concurrent operations. The operation + should be retried. If a transaction is in progress, it should + be rolled back and the operation retried in a new + transaction.'''), + Error('WT_DUPLICATE_KEY', 'attempt to insert an existing key', ''' + This error is generated when the application attempts to insert + a record with the same key as an existing record without the + 'overwrite' configuration to WT_SESSION::open_cursor.'''), + Error('WT_ERROR', 'non-specific WiredTiger error', ''' + This error is generated for cases that are not covered by + specific error returns.'''), + Error('WT_NOTFOUND', 'item not found', ''' + This return value indicates that a search operation did not + find a record matching the application's search key. This + includes implicit search operations in WT_CURSOR::update or + WT_CURSOR::remove operations.'''), + Error('WT_RESTART', 'restart the operation (internal)', undoc=True), +] + +class Method: + def __init__(self, config, **flags): + self.config = config + self.flags = flags + +class Config: + def __init__(self, name, default, desc, **flags): + self.name = name + self.default = default + self.desc = desc + self.flags = flags + + def __cmp__(self, other): + return cmp(self.name, other.name) + +# All schema objects can have column names (optional for simple tables). +column_meta = [ + Config('columns', '', r''' + list of the column names. Comma-separated list of the form + <code>(column[,...])</code>. For tables, the number of entries + must match the total number of values in \c key_format and \c + value_format. For colgroups and indices, all column names must + appear in the list of columns for the table''', + type='list'), +] + +filename_meta = [ + Config('filename', '', r''' + override the default filename derived from the object name'''), +] + +format_meta = column_meta + [ + Config('key_format', 'u', r''' + the format of the data packed into key items. See @ref + schema_format_types for details. By default, the key_format is + \c 'u' and applications use WT_ITEM structures to manipulate + raw byte arrays. By default, records are stored in row-store + files: keys of type \c 'r' are record numbers and records + referenced by record number are stored in column-store files''', + type='format'), + Config('value_format', 'u', r''' + the format of the data packed into value items. See @ref + schema_format_types for details. By default, the value_format + is \c 'u' and applications use a WT_ITEM structure to + manipulate raw byte arrays. Value items of type 't' are + bitfields, and when configured with record number type keys, + will be stored using a fixed-length store''', + type='format'), +] + +# Per-file configuration +file_meta = format_meta + [ + Config('allocation_size', '512B', r''' + the file unit allocation size, in bytes, must a power-of-two; + smaller values decrease the file space required by overflow + items, and the default value of 512B is a good choice absent + requirements from the operating system or storage device''', + min='512B', max='128MB'), + Config('block_compressor', '', r''' + configure a compressor for file blocks. Permitted values are + empty (off) or \c "bzip2", \c "snappy" or custom compression + engine \c "name" created with WT_CONNECTION::add_compressor. + See @ref compression for more information'''), + Config('checksum', 'true', r''' + configure file block checksums; if false, the block + manager is free to not write or check block checksums. + This can increase performance in applications where + compression provides checksum functionality or read-only + applications where blocks require no verification''', + type='boolean'), + Config('collator', '', r''' + configure custom collation for keys. Value must be a collator + name created with WT_CONNECTION::add_collator'''), + Config('huffman_key', '', r''' + configure Huffman encoding for keys. Permitted values are + empty (off), \c "english" or \c "<filename>". See @ref + huffman for more information'''), + Config('huffman_value', '', r''' + configure Huffman encoding for values. Permitted values are + empty (off), \c "english" or \c "<filename>". See @ref + huffman for more information'''), + Config('internal_key_truncate', 'true', r''' + configure internal key truncation, discarding unnecessary + trailing bytes on internal keys''', + type='boolean'), + Config('internal_page_max', '2KB', r''' + the maximum page size for internal nodes, in bytes; the size + must be a multiple of the allocation size and is significant + for applications wanting to avoid excessive L2 cache misses + while searching the tree''', + min='512B', max='512MB'), + Config('internal_item_max', '0', r''' + the maximum key size stored on internal nodes, in bytes. If + zero, a maximum is calculated to permit at least 8 keys per + internal page''', + min=0), + Config('key_gap', '10', r''' + the maximum gap between instantiated keys in a Btree leaf page, + constraining the number of keys processed to instantiate a + random Btree leaf page key''', + min='0'), + Config('leaf_page_max', '1MB', r''' + the maximum page size for leaf nodes, in bytes; the size must + be a multiple of the allocation size, and is significant for + applications wanting to maximize sequential data transfer from + a storage device''', + min='512B', max='512MB'), + Config('leaf_item_max', '0', r''' + the maximum key or value size stored on leaf nodes, in bytes. + If zero, a size is calculated to permit at least 8 items + (values or row store keys) per leaf page''', + min=0), + Config('prefix_compression', 'true', r''' + configure row-store format key prefix compression''', + type='boolean'), + Config('split_pct', '75', r''' + the Btree page split size as a percentage of the maximum Btree + page size, that is, when a Btree page is split, it will be + split into smaller pages, where each page is the specified + percentage of the maximum Btree page size''', + min='25', max='100'), + Config('type', 'btree', r''' + the file type''', + choices=['btree']), +] + +table_only_meta = [ + Config('colgroups', '', r''' + comma-separated list of names of column groups. Each column + group is stored separately, keyed by the primary key of the + table. If no column groups are specified, all columns are + stored together in a single file. All value columns in the + table must appear in at least one column group. Each column + group must be created with a separate call to + WT_SESSION::create''', type='list'), +] + +colgroup_meta = column_meta + filename_meta + +index_meta = column_meta + filename_meta + +table_meta = format_meta + table_only_meta + +methods = { +'file.meta' : Method(file_meta), + +'colgroup.meta' : Method(colgroup_meta), + +'index.meta' : Method(index_meta), + +'table.meta' : Method(table_meta), + +'cursor.close' : Method([ +]), + +'session.close' : Method([]), + +'session.create' : Method(table_meta + file_meta + filename_meta + [ + Config('exclusive', 'false', r''' + fail if the object exists. When false (the default), if the + object exists, check that its settings match the specified + configuration''', + type='boolean'), +]), + +'session.drop' : Method([ + Config('force', 'false', r''' + return success if the object does not exist''', + type='boolean'), + ]), + +'session.log_printf' : Method([]), + +'session.open_cursor' : Method([ + Config('append', 'false', r''' + only supported by cursors with record number keys: append the + value as a new record, creating a new record number key''', + type='boolean'), + Config('bulk', 'false', r''' + configure the cursor for bulk loads; bulk-load is a fast + load path for empty objects, only empty objects may be + bulk-loaded''', + type='boolean'), + Config('clear_on_close', 'false', r''' + for statistics cursors, reset statistics counters when the cursor is + closed''', + type='boolean'), + Config('dump', '', r''' + configure the cursor for dump format inputs and outputs: + "hex" selects a simple hexadecimal format, "print" + selects a format where only non-printing characters are + hexadecimal encoded. The cursor dump format is compatible + with the @ref utility_dump and @ref utility_load commands''', + choices=['hex', 'print']), + Config('isolation', 'read-committed', r''' + the isolation level for this cursor, ignored for transactional + cursors''', + choices=['snapshot', 'read-committed', 'read-uncommitted']), + Config('overwrite', 'false', r''' + change the behavior of the cursor's insert method to + overwrite previously existing values''', + type='boolean'), + Config('raw', 'false', r''' + ignore the encodings for the key and value, manage data as if + the formats were \c "u". See @ref cursor_raw for more details''', + type='boolean'), + Config('statistics', 'false', r''' + configure the cursor for statistics''', + type='boolean'), +]), + +'session.rename' : Method([]), +'session.salvage' : Method([ + Config('force', 'false', r''' + force salvage even of files that do not appear to be WiredTiger + files''', + type='boolean'), +]), +'session.sync' : Method([]), +'session.truncate' : Method([]), +'session.upgrade' : Method([]), +'session.verify' : Method([]), +'session.dumpfile' : Method([]), + +'session.begin_transaction' : Method([ + Config('isolation', 'read-committed', r''' + the isolation level for this transaction''', + choices=['serializable', 'snapshot', 'read-committed', + 'read-uncommitted']), + Config('name', '', r''' + name of the transaction for tracing and debugging'''), + Config('sync', 'full', r''' + how to sync log records when the transaction commits''', + choices=['full', 'flush', 'write', 'none']), + Config('priority', 0, r''' + priority of the transaction for resolving conflicts. + Transactions with higher values are less likely to abort''', + min='-100', max='100'), +]), + +'session.commit_transaction' : Method([]), +'session.rollback_transaction' : Method([]), + +'session.checkpoint' : Method([ + Config('archive', 'false', r''' + remove log files no longer required for transactional + durability''', + type='boolean'), + Config('flush_cache', 'true', r''' + flush the cache''', + type='boolean'), + Config('flush_log', 'true', r''' + flush the log to disk''', + type='boolean'), + Config('log_size', '0', r''' + only proceed if more than the specified number of bytes of log + records have been written since the last checkpoint''', + min='0'), + Config('force', 'false', r''' + write a new checkpoint even if nothing has changed since the + last one''', + type='boolean'), + Config('timeout', '0', r''' + only proceed if more than the specified number of milliseconds + have elapsed since the last checkpoint''', + min='0'), +]), + +'connection.add_cursor_type' : Method([]), +'connection.add_collator' : Method([]), +'connection.add_compressor' : Method([]), +'connection.add_extractor' : Method([]), +'connection.close' : Method([]), + +'connection.load_extension' : Method([ + Config('entry', 'wiredtiger_extension_init', r''' + the entry point of the extension'''), + Config('prefix', '', r''' + a prefix for all names registered by this extension (e.g., to + make namespaces distinct or during upgrades'''), +]), + +'connection.open_session' : Method([]), + +'wiredtiger_open' : Method([ + Config('cache_size', '100MB', r''' + maximum heap memory to allocate for the cache''', + min='1MB', max='10TB'), + Config('create', 'false', r''' + create the database if it does not exist''', + type='boolean'), + Config('home_environment', 'false', r''' + use the \c WIREDTIGER_HOME environment variable for naming + unless the process is running with special privileges. + See @ref home for more information''', + type='boolean'), + Config('home_environment_priv', 'false', r''' + use the \c WIREDTIGER_HOME environment variable for naming + regardless of whether or not the process is running with + special privileges. See @ref home for more information''', + type='boolean'), + Config('extensions', '', r''' + list of extensions to load. Optional values are passed as the + \c config parameter to WT_CONNECTION::load_extension. Complex + paths may need quoting, for example, + <code>extensions=("/path/to/ext.so"="entry=my_entry")</code>''', + type='list'), + Config('error_prefix', '', r''' + prefix string for error messages'''), + Config('eviction_target', '80', r''' + continue evicting until the cache becomes less full than this (as a + percentage). Must be less than \c eviction_trigger''', + min=10, max=99), + Config('eviction_trigger', '95', r''' + trigger eviction when the cache becomes this full (as a percentage)''', + min=10, max=99), + Config('hazard_max', '30', r''' + number of simultaneous hazard references per session handle''', + min='15'), + Config('logging', 'false', r''' + enable logging''', + type='boolean'), + Config('multiprocess', 'false', r''' + permit sharing between processes (will automatically start an + RPC server for primary processes and use RPC for secondary + processes)''', + type='boolean'), + Config('session_max', '50', r''' + maximum expected number of sessions (including server + threads)''', + min='1'), + Config('transactional', 'false', r''' + support transactional semantics''', + type='boolean'), + Config('verbose', '', r''' + enable messages for various events. Options are given as a + list, such as <code>"verbose=[evictserver,read]"</code>''', + type='list', choices=[ + 'block', + 'evict', + 'evictserver', + 'fileops', + 'hazard', + 'mutex', + 'read', + 'readserver', + 'reconcile', + 'salvage', + 'verify', + 'write']), +]), +} + +flags = { +################################################### +# Internal routine flag declarations +################################################### + 'page_free' : [ 'PAGE_FREE_IGNORE_DISK' ], + 'rec_evict' : [ 'REC_SINGLE' ], + 'verbose' : [ + 'VERB_block', + 'VERB_evict', + 'VERB_evictserver', + 'VERB_fileops', + 'VERB_hazard', + 'VERB_mutex', + 'VERB_read', + 'VERB_readserver', + 'VERB_reconcile', + 'VERB_salvage', + 'VERB_verify', + 'VERB_write' + ], + +################################################### +# Structure flag declarations +################################################### + 'conn' : [ 'SERVER_RUN' ], + 'session' : [ 'SESSION_INTERNAL', 'SESSION_SALVAGE_QUIET_ERR' ], +} diff --git a/dist/api_err.py b/dist/api_err.py new file mode 100644 index 00000000000..afc7d95e1f3 --- /dev/null +++ b/dist/api_err.py @@ -0,0 +1,84 @@ +# Output C #defines for errors into wiredtiger.in and the associated error +# message code in strerror.c. + +import re, textwrap + +import api_data +from dist import compare_srcfile + +# Update the #defines in the wiredtiger.in file. +tmp_file = '__tmp' +tfile = open(tmp_file, 'w') +skip = 0 +for line in open('../src/include/wiredtiger.in', 'r'): + if not skip: + tfile.write(line) + if line.count('Error return section: END'): + tfile.write(line) + skip = 0 + elif line.count('Error return section: BEGIN'): + tfile.write(' */\n') + skip = 1 + + # We don't want our error returns to conflict with any other + # package, so use an uncommon range, specifically, -31,800 to + # -31,999. + v = -31800 + for err in api_data.errors: + if 'undoc' in err.flags: + tfile.write('/*! @cond internal */\n') + tfile.write('/*! %s.%s */\n' % + (err.desc[0].upper() + err.desc[1:], + ''.join('\n * ' + l for l in textwrap.wrap(textwrap.dedent(err.long_desc).strip(), 77)) + '\n' if err.long_desc else '')) + tfile.write('#define\t%s\t%d\n' % (err.name, v)) + v -= 1 + if 'undoc' in err.flags: + tfile.write('/*! @endcond */\n') + tfile.write('/*\n') +tfile.close() +compare_srcfile(tmp_file, '../src/include/wiredtiger.in') + +# Output the wiredtiger_strerror code. +tmp_file = '__tmp' +tfile = open(tmp_file, 'w') +tfile.write('''/* DO NOT EDIT: automatically built by dist/api_err.py. */ + +#include "wt_internal.h" + +/* + * wiredtiger_strerror -- + * Return a string for any error value. + */ +const char * +wiredtiger_strerror(int error) +{ + static char errbuf[64]; + char *p; + + if (error == 0) + return ("Successful return: 0"); + + switch (error) { +''') + +for err in api_data.errors: + tfile.write('\tcase ' + err.name + ':\n') + tfile.write('\t\treturn ("' + err.name + ': ' + err.desc + '");\n') + +tfile.write('''\ + default: + if (error > 0 && (p = strerror(error)) != NULL) + return (p); + break; + } + + /* + * !!! + * Not thread-safe, but this is never supposed to happen. + */ + (void)snprintf(errbuf, sizeof(errbuf), "Unknown error: %d", error); + return (errbuf); +} +''') +tfile.close() +compare_srcfile(tmp_file, '../src/api/api_strerror.c') diff --git a/dist/api_flags.py b/dist/api_flags.py new file mode 100644 index 00000000000..34a440b071b --- /dev/null +++ b/dist/api_flags.py @@ -0,0 +1,80 @@ +# Read the api_flags file and output a C header file using the minimum +# number of distinct bits to ensure flags don't collide. +# +# The format of the api_flags file: +# Name (usually a method name) +# <tab> Flag + +import os, re, sys +from dist import compare_srcfile + +# Load the flags dictionary. +import api_data +flags = api_data.flags + +flag_cnt = {} # Dictionary [flag] : [reference count] +flag_name = {} # Dictionary [flag] : [name ...] +name_mask = {} # Dictionary [name] : [used flag mask] + +# Step through the flags dictionary and build our local dictionaries. +for method in flags.items(): + name_mask[method[0]] = 0x0 + for flag in method[1]: + if flag == '__NONE__': + continue + if flag not in flag_cnt: + flag_cnt[flag] = 1 + flag_name[flag] = [] + else: + flag_cnt[flag] += 1 + flag_name[flag].append(method[0]) + +# Create list of possible bit masks. +bits = [2 ** i for i in range(0, 32)] + +# Walk the list of flags in reverse, sorted-by-reference count order. For +# each flag, find a bit that's not currently in use by any method using the +# flag. +flag_bit = {} # Dictionary [flag] : [bit value] +for f in sorted(flag_cnt.items(),\ + key = lambda k_v : (k_v[1], k_v[0]), reverse = True): + mask = 0xffffffff + for m in flag_name[f[0]]: + mask &= ~name_mask[m] + if mask == 0: + print >>sys.stder, "api_flags: ran out of flags at " + m + " method", + sys.exit(1) + for b in bits: + if mask & b: + mask = b + break + flag_bit[f[0]] = mask + for m in flag_name[f[0]]: + name_mask[m] |= mask + +# Print out the flag masks in hex. +# Assumes tab stops set to 8 characters. +flag_info = '' +for f in sorted(flag_cnt.items()): + flag_info += "#define\tWT_%s%s%#010x\n" %\ + (f[0],\ + "\t" * max(1, 6 - int((len('WT_') + len(f[0])) / 8)),\ + flag_bit[f[0]]) + +# Update the wiredtiger.in file with the flags information. +tmp_file = '__tmp' +tfile = open(tmp_file, 'w') +skip = 0 +for line in open('../src/include/api.h', 'r'): + if skip: + if line.count('API flags section: END'): + tfile.write('/*\n' + line) + skip = 0 + else: + tfile.write(line) + if line.count('API flags section: BEGIN'): + skip = 1 + tfile.write(' */\n') + tfile.write(flag_info) +tfile.close() +compare_srcfile(tmp_file, '../src/include/api.h') diff --git a/dist/config.py b/dist/config.py new file mode 100644 index 00000000000..706e8c387e1 --- /dev/null +++ b/dist/config.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python + +import os, re, sys, textwrap +import api_data +from dist import compare_srcfile + +# Temporary file. +tmp_file = '__tmp' + +##################################################################### +# Update wiredtiger.in with doxygen comments +##################################################################### +f='../src/include/wiredtiger.in' +tfile = open(tmp_file, 'w') + +cbegin_re = re.compile(r'(\s*\*\s*)@config(?:empty|start)\{(.*?),.*\}') + +def gettype(c): + '''Derive the type of a config item''' + checks = c.flags + ctype = checks.get('type', None) + if not ctype and ('min' in checks or 'max' in checks): + ctype = 'int' + return ctype or 'string' + +def typedesc(c): + '''Descripe what type of value is expected for the given config item''' + checks = c.flags + cmin = str(checks.get('min', '')) + cmax = str(checks.get('max', '')) + choices = checks.get('choices', []) + ctype = gettype(c) + desc = { + 'boolean' : 'a boolean flag', + 'format' : 'a format string', + 'int' : 'an integer', + 'list' : 'a list', + 'string' : 'a string'}[ctype] + if cmin and cmax: + desc += ' between ' + cmin + ' and ' + cmax + elif cmin: + desc += ' greater than or equal to ' + cmin + elif cmax: + desc += ' no more than ' + cmax + if choices: + if ctype == 'list': + desc += ', with values chosen from the following options: ' + else: + desc += ', chosen from the following options: ' + desc += ', '.join('\\c "' + c + '"' for c in choices) + elif ctype == 'list': + desc += ' of strings' + return desc + +skip = False +for line in open(f, 'r'): + if skip: + if '@configend' in line: + skip = False + continue + + m = cbegin_re.match(line) + if not m: + tfile.write(line) + continue + + prefix, config_name = m.groups() + if config_name not in api_data.methods: + print >>sys.stderr, "Missing configuration for " + config_name + tfile.write(line) + continue + + skip = ('@configstart' in line) + + if not api_data.methods[config_name].config: + tfile.write(prefix + '@configempty{' + config_name + + ', see dist/api_data.py}\n') + continue + + tfile.write(prefix + '@configstart{' + config_name + + ', see dist/api_data.py}\n') + + w = textwrap.TextWrapper(width=80-len(prefix.expandtabs()), + break_on_hyphens=False) + lastname = None + for c in sorted(api_data.methods[config_name].config): + name = c.name + if '.' in name: + print >>sys.stderr, "Bad config key " + name + + # Deal with duplicates: with complex configurations (like + # WT_SESSION::create), it's simpler to deal with duplicates here than + # manually in api_data.py. + if name == lastname: + continue + lastname = name + desc = textwrap.dedent(c.desc) + '.' + desc = desc.replace(',', '\\,') + default = '\\c ' + str(c.default) if c.default or gettype(c) == 'int' \ + else 'empty' + tdesc = typedesc(c) + '; default ' + default + '.' + tdesc = tdesc.replace(',', '\\,') + output = '@config{' + ','.join((name, desc, tdesc)) + '}' + for l in w.wrap(output): + tfile.write(prefix + l + '\n') + + tfile.write(prefix + '@configend\n') + +tfile.close() +compare_srcfile(tmp_file, f) + +##################################################################### +# Create config_def.c with defaults for each config string +##################################################################### +f='../src/config/config_def.c' +tfile = open(tmp_file, 'w') + +tfile.write('''/* DO NOT EDIT: automatically built by dist/config.py. */ + +#include "wt_internal.h" +''') + +# Make a TextWrapper that can wrap at commas. +w = textwrap.TextWrapper(width=72, break_on_hyphens=False) +w.wordsep_re = w.wordsep_simple_re = re.compile(r'(,)') + +def checkstr(c): + '''Generate the JSON string used by __wt_config_check to validate the + config string''' + checks = c.flags + ctype = gettype(c) + cmin = str(checks.get('min', '')) + cmax = str(checks.get('max', '')) + choices = checks.get('choices', []) + result = [] + if ctype != 'string': + result.append('type=' + ctype) + if cmin: + result.append('min=' + cmin) + if cmax: + result.append('max=' + cmax) + if choices: + result.append('choices=' + '[' + + ','.join('\\"' + s + '\\"' for s in choices) + ']') + return ','.join(result) + +def get_default(c): + t = gettype(c) + if c.default or t == 'int': + return str(c.default) + elif t == 'string': + return '""' + else: + return '()' + +for name in sorted(api_data.methods.keys()): + ctype = api_data.methods[name].config + name = name.replace('.', '_') + tfile.write(''' +const char * +__wt_confdfl_%(name)s = +%(config)s; +''' % { + 'name' : name, + 'config' : '\n'.join(' "%s"' % line + for line in w.wrap(','.join('%s=%s' % (c.name, get_default(c)) + for c in sorted(ctype))) or [""]), +}) + tfile.write(''' +const char * +__wt_confchk_%(name)s = +%(check)s; +''' % { + 'name' : name, + 'check' : '\n'.join(' "%s"' % line + for line in w.wrap(','.join('%s=(%s)' % (c.name, checkstr(c)) + for c in sorted(ctype))) or [""]), +}) + +tfile.close() +compare_srcfile(tmp_file, f) diff --git a/dist/db.py b/dist/db.py new file mode 100644 index 00000000000..06a9484d1f9 --- /dev/null +++ b/dist/db.py @@ -0,0 +1,24 @@ +# A simple python script to build a file that can be bulk-loaded into a +# WiredTiger database for smoke-testing. + +import getopt, random, sys + +dmin = 7 # Minimum data size +dmax = 837 # Maximum data size + +seed = None # Random number seed +pairs = 100000 # Key/data pairs to output + +opts, args = getopt.getopt(sys.argv[1:], "m:n:s:") +for o, a in opts: + if o == "-m": + dmax = int(a) + elif o == "-n": + pairs = int(a) + elif o == "-s": + seed = int(a) + +random.seed(seed) +for i in range(pairs): + fmt = "%010d\ndata: %0" + str(random.randrange(dmin, dmax)) + "d" + print(fmt % (i, i)) diff --git a/dist/dist.py b/dist/dist.py new file mode 100644 index 00000000000..8dbe75cba09 --- /dev/null +++ b/dist/dist.py @@ -0,0 +1,58 @@ +import filecmp, os, re, shutil + +# source_files_list -- +# Return a list of the source file names in filelist. +def source_files_list(): + list=[] + file_re = re.compile(r'^(\w|/)+/((\w|.)+)') + for line in open('filelist', 'r'): + if file_re.match(line): + list.append(file_re.match(line).group(2)) + return sorted(list) + +# source_files -- +# Print a list of the source file names in filelist. +def source_files(): + for line in source_files_list(): + print(line) + +# source_paths_list -- +# Return a list of the source file paths in filelist. +def source_paths_list(): + list=[] + file_re = re.compile(r'^\w') + for line in open('filelist', 'r'): + if file_re.match(line): + list.append(line.rstrip()) + return sorted(list) + +# source_paths -- +# Print a list of the source file paths in filelist. +def source_paths(): + for line in source_paths_list(): + print(line) + +# directory_files_list -- +# Return a list of the directories in filelist. +def directory_files_list(): + dirs = {} + dir_re = re.compile(r'^((\w|/)+/)') + for line in open('filelist', 'r'): + if dir_re.match(line): + dirs[dir_re.match(line).group(1)] = 1 + return sorted(dirs) + +# directory_files -- +# Print a list of the directories in filelist. +def directory_files(): + for entry in directory_files_list(): + print(entry) + +# compare_srcfile -- +# Compare two files, and if they differ, update the source file. +def compare_srcfile(tmp, src): + if not os.path.isfile(src) or \ + not filecmp.cmp(tmp, src, False): + print('Updating ' + src) + shutil.copyfile(tmp, src) + os.remove(tmp) diff --git a/dist/filelist b/dist/filelist new file mode 100644 index 00000000000..0740542fc24 --- /dev/null +++ b/dist/filelist @@ -0,0 +1,111 @@ +# filelist -- +# List of source files for WiredTiger library. + +src/api/api_event.c +src/api/api_strerror.c +src/api/api_version.c +src/block/block_addr.c +src/block/block_alloc.c +src/block/block_cksum.c +src/block/block_mgr.c +src/block/block_open.c +src/block/block_read.c +src/block/block_slvg.c +src/block/block_vrfy.c +src/block/block_write.c +src/btree/bt_bulk.c +src/btree/bt_cache.c +src/btree/bt_cell.c +src/btree/bt_curnext.c +src/btree/bt_curprev.c +src/btree/bt_cursor.c +src/btree/bt_debug.c +src/btree/bt_discard.c +src/btree/bt_evict.c +src/btree/bt_handle.c +src/btree/bt_huffman.c +src/btree/bt_misc.c +src/btree/bt_ovfl.c +src/btree/bt_page.c +src/btree/bt_read.c +src/btree/bt_ret.c +src/btree/bt_root.c +src/btree/bt_slvg.c +src/btree/bt_stat.c +src/btree/bt_sync.c +src/btree/bt_upgrade.c +src/btree/bt_vrfy.c +src/btree/bt_vrfy_dsk.c +src/btree/bt_walk.c +src/btree/col_modify.c +src/btree/col_srch.c +src/btree/rec_evict.c +src/btree/rec_track.c +src/btree/rec_write.c +src/btree/row_key.c +src/btree/row_modify.c +src/btree/row_srch.c +src/config/config.c +src/config/config_check.c +src/config/config_collapse.c +src/config/config_concat.c +src/config/config_def.c +src/conn/conn_api.c +src/conn/conn_handle.c +src/conn/conn_open.c +src/conn/conn_stat.c +src/cursor/cur_bulk.c +src/cursor/cur_config.c +src/cursor/cur_dump.c +src/cursor/cur_file.c +src/cursor/cur_index.c +src/cursor/cur_stat.c +src/cursor/cur_std.c +src/cursor/cur_table.c +src/log/log.c +src/log/log_desc.c +src/os_posix/os_abort.c +src/os_posix/os_alloc.c +src/os_posix/os_dlopen.c +src/os_posix/os_errno.c +src/os_posix/os_exist.c +src/os_posix/os_filesize.c +src/os_posix/os_flock.c +src/os_posix/os_fsync.c +src/os_posix/os_ftruncate.c +src/os_posix/os_mtx.c +src/os_posix/os_open.c +src/os_posix/os_priv.c +src/os_posix/os_remove.c +src/os_posix/os_rename.c +src/os_posix/os_rw.c +src/os_posix/os_sleep.c +src/os_posix/os_thread.c +src/os_posix/os_yield.c +src/packing/packing.c +src/packing/packing_api.c +src/schema/schema_create.c +src/schema/schema_drop.c +src/schema/schema_list.c +src/schema/schema_open.c +src/schema/schema_plan.c +src/schema/schema_project.c +src/schema/schema_rename.c +src/schema/schema_table.c +src/schema/schema_track.c +src/schema/schema_truncate.c +src/schema/schema_util.c +src/schema/schema_worker.c +src/session/session_api.c +src/session/session_btree.c +src/support/err.c +src/support/filename.c +src/support/global.c +src/support/hazard.c +src/support/hex.c +src/support/huffman.c +src/support/pow.c +src/support/rand.c +src/support/scratch.c +src/support/sess_dump.c +src/support/stat.c diff --git a/dist/log.py b/dist/log.py new file mode 100644 index 00000000000..9b992b77074 --- /dev/null +++ b/dist/log.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python + +import os, re, sys, textwrap +import log_data +from dist import compare_srcfile + +# Temporary file. +tmp_file = '__tmp' + +# Map log record types to C +c_types = { + 'string' : 'const char *', +} + +# Map log record types to format strings +fmt_types = { + 'string' : 'S', +} + +##################################################################### +# Create log.i with inline functions for each log record type. +##################################################################### +f='../src/include/log.i' +tfile = open(tmp_file, 'w') + +tfile.write('/* DO NOT EDIT: automatically built by dist/log.py. */\n') + +for t in log_data.types: + tfile.write(''' +static inline int +__wt_logput_%(name)s(WT_SESSION_IMPL *session, %(param_decl)s) +{ + return (__wt_log_put(session, &__wt_logdesc_%(name)s, %(param_list)s)); +} +''' % { + 'name' : t.name, + 'param_decl' : ', '.join( + '%s %s' % (c_types.get(t, t), n) for t, n in t.fields), + 'param_list' : ', '.join(n for t, n in t.fields), +}) + +tfile.close() +compare_srcfile(tmp_file, f) + +##################################################################### +# Create log_desc.c with descriptors for each log record type. +##################################################################### +f='../src/log/log_desc.c' +tfile = open(tmp_file, 'w') + +tfile.write('''/* DO NOT EDIT: automatically built by dist/log.py. */ + +#include "wt_internal.h" +''') + +for t in log_data.types: + tfile.write(''' +WT_LOGREC_DESC +__wt_logdesc_%(name)s = +{ + "%(fmt)s", { %(field_list)s, NULL } +}; +''' % { + 'name' : t.name, + 'fmt' : ''.join(fmt_types[t] for t, n in t.fields), + 'field_list' : ', '.join('"%s"' % n for t, n in t.fields), +}) + +tfile.close() +compare_srcfile(tmp_file, f) diff --git a/dist/log_data.py b/dist/log_data.py new file mode 100644 index 00000000000..1e274e48991 --- /dev/null +++ b/dist/log_data.py @@ -0,0 +1,10 @@ +# Data for config.py, describes all configuration key / value pairs + +class LogRecordType: + def __init__(self, name, fields): + self.name = name + self.fields = fields + +types = [ + LogRecordType('debug', [('string', 'message')]) +] diff --git a/dist/s_all b/dist/s_all new file mode 100644 index 00000000000..1c98f4283e0 --- /dev/null +++ b/dist/s_all @@ -0,0 +1,78 @@ +#! /bin/sh + +# Run standard scripts. +t=__wt.$$ +trap 'rm -f $t *.pyc __tmp __wt.*' 0 1 2 3 13 15 + +# We require python which may not be installed. +type python > /dev/null 2>&1 || { + echo 's_all: python not found' + exit 1 +} + +run() +{ + printf "WiredTiger: $2..." + $1 > $t + + if `grep 'skipped' $t > /dev/null 2>&1`; then + printf " " && cat $t + elif `test -s $t`; then + echo + sed -e 's/^/ /' $t + else + echo ' OK' + fi + rm -f $t + return 0 +} + +echo 'dist/s_all run started...' + +reconf=0 +while : + do case "$1" in + -A) # Reconfigure the library build. + reconf=1 + shift;; + *) + break;; + esac +done + +run "sh ./s_version" "Updating files that include the package version" + +test "$reconf" -eq 0 || { + (cd ../build_posix && + run "sh ./reconf" "Rebuilding GNU tools library support") +} + +run "python api_err.py" "building error return API" +run "python api_flags.py" "building API flags" +run "python config.py" "building configuration code" +run "python log.py" "building logging layer" +run "python stat.py" "building statistics support" +run "python serial.py" "building serial function support" + +run "sh ./s_typedef -b" "building standard typedefs" +run "sh ./s_prototypes" "building function prototypes" +run "sh ./s_readme" "building README file" +run "sh ./s_tags" "building tags files" + +run "sh ./s_copyright" "checking copyright notices" +run "sh ./s_define" "checking for unused #defines" +run "sh ./s_funcs" "checking for unused functions" +run "sh ./s_stat" "checking for unused statistics fields" +run "sh ./s_getopt" "checking for incorrect getopt usage" +run "sh ./s_longlines" "checking for long lines" +run "sh ./s_printf" "checking for non-portable printf/scanf" +run "sh ./s_string" "checking string spelling" +run "sh ./s_style" "checking style" +run "sh ./s_symbols" "checking external symbol names" +run "sh ./s_typedef -c" "checking for unused typedefs" +run "sh ./s_types" "checking for old-style types" +run "sh ./s_whitespace" "checking whitespace" + +run "sh ./s_docs" "generating documentation" + +echo 'dist/s_all run finished' diff --git a/dist/s_copyright b/dist/s_copyright new file mode 100644 index 00000000000..1ab73970f87 --- /dev/null +++ b/dist/s_copyright @@ -0,0 +1,110 @@ +#! /bin/sh + +# It was a pain updating all the copyrights in the Berkeley DB tree: I'm not +# doing that again, the only files carrying copyrights in the WiredTiger tree +# are the source files, that is, *.[ch] and *.in. Check automatically to be +# sure the copyright is up-to-date. + +c1=__wt.1$$ +c2=__wt.2$$ +c3=__wt.3$$ +c4=__wt.4$$ +trap 'rm -f $c1 $c2 $c3 $c4; exit 0' 0 1 2 3 13 15 + +year=`date +%Y` + +cat > $c1 <<ENDOFTEXT + * Copyright (c) 2008-$year WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. +ENDOFTEXT + +# Copyright for files WiredTiger does not own. +cat > $c2 <<ENDOFTEXT + * Copyright (c) 2008-$year WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. +ENDOFTEXT + +cat > $c3 <<ENDOFTEXT +# Copyright (c) 2008-$year WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +ENDOFTEXT + +cat > $c4 <<ENDOFTEXT +# Copyright (c) 2008-$year WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +ENDOFTEXT + +check() +{ + # Skip auto-generated files, files in which WiredTiger holds no rights. + if `egrep "skip $1" s_copyright.list > /dev/null`; then + return; + fi + + # It's okay if the file doesn't exist: we may be running in a release + # tree with some files removed. + test -f ../$i || return + + # Check for a correct copyright header. + if `sed -e 2,4p -e 5q -e d ../$1 | diff - $c1 > /dev/null` ; then + return; + fi + if `sed -e 2,3p -e 4q -e d ../$1 | diff - $c2 > /dev/null` ; then + return; + fi + if `sed -e 3,5p -e 6q -e d ../$1 | diff - $c3 > /dev/null` ; then + return; + fi + if `sed -e 3,4p -e 5q -e d ../$1 | diff - $c4 > /dev/null` ; then + return; + fi + + echo "$1: copyright information is incorrect" +} + +l="`cd .. && \ + echo LICENSE \ + COPYING \ + examples/c/*.c \ + examples/python/*.py \ + lang/java/src/com/wiredtiger/*.java \ + lang/java/src/com/wiredtiger/*/*.java \ + lang/python/*.py \ + lang/python/*pack*.py \ + src/include/*.[hi] \ + src/include/*.in \ + src/utilities/*.[ch] \ + test/format/*.[ch] \ + test/salvage/*.[ch] \ + test/suite/*.py \ + test/thread/*.[ch]`" + +for i in $l `sed -e '/^[a-z]/! d' filelist`; do + check $i +done + +# The documentation copyright appears in two files. +s="Copyright (c) 2008-$year WiredTiger, Inc." +f="docs/build-javadoc.sh docs/style/footer.html" +for i in $f; do + if `grep "$s" ../$i > /dev/null`; then + continue; + fi + echo "$i: copyright information is incorrect" +done + +# The wt utility has a copyright it displays. +s="printf.*Copyright (c) 2008-$year WiredTiger, Inc." +f="src/utilities/util_cpyright.c" +for i in $f; do + if `grep "$s" ../$i > /dev/null`; then + continue; + fi + echo "$i: copyright information is incorrect" +done diff --git a/dist/s_copyright.list b/dist/s_copyright.list new file mode 100644 index 00000000000..685f4df9a37 --- /dev/null +++ b/dist/s_copyright.list @@ -0,0 +1,10 @@ +skip lang/python/setup.py +skip lang/python/wiredtiger.py +skip src/api/api_strerror.c +skip src/config/config_def.c +skip src/include/extern.h +skip src/include/log.i +skip src/include/queue.h +skip src/include/serial_funcs.i +skip src/log/log_desc.c +skip src/support/stat.c diff --git a/dist/s_define b/dist/s_define new file mode 100644 index 00000000000..0819a91f353 --- /dev/null +++ b/dist/s_define @@ -0,0 +1,32 @@ +#! /bin/sh + +# Complain about unused #defines. +t=__wt.$$ +trap 'rm -f $t; exit 0' 0 1 2 3 13 15 + +# List of files to search. +l=`sed -e 's,#.*,,' -e '/^$/d' -e 's,^,../,' filelist` +l="$l `echo ../src/include/*.i ../src/utilities/*.c `" + +( +# Copy out the list of #defines we don't use, but it's OK. +sed -e '/^$/d' -e '/^#/d' < s_define.list + +# Get the list of #defines. +# Ignore the list of statistic "keys" generated for applications. +search=`cat ../src/include/*.[hi] ../src/include/*.in | + sed -e '/Statistics section: BEGIN/,/Statistics section: END/d' | + egrep '^#define' | + sed 's/#define[ ][ ]*\([A-Za-z_][A-Za-z0-9_]*\).*/\1/' | + sort -u` + +# Print the list of macros, followed by the occurrences: we're looking for +# macros that only appear once. +echo "$search" +fgrep -who "$search" $l + +) | sort | uniq -u > $t + +test -s $t && cat $t + +exit 0 diff --git a/dist/s_define.list b/dist/s_define.list new file mode 100644 index 00000000000..a70ed526852 --- /dev/null +++ b/dist/s_define.list @@ -0,0 +1,92 @@ +# List of WiredTiger #defines that are "unused", but it's OK. +ALIGN_CHECK +API_CALL +API_CALL_NOCONF +API_SESSION_INIT +FLD_CLR +FLD_ISSET +HAVE_ATOMICS +LF_CLR +LF_SET +LLONG_MAX +LLONG_MIN +SIZE_CHECK +SPINLOCK_GCC +SPINLOCK_PTHREAD_MUTEX +WT_BARRIER +WT_BLOCK_DESC_SIZE +WT_INTPACK32_MAXSIZE +WT_MAX +WT_MIN +WT_PAUSE +WT_READ_BARRIER +WT_SKIP_PROBABILITY +WT_STAT +WT_STAT +WT_STAT_DECRV +WT_STAT_INCRV +__F +__WIREDTIGER_EXT_H_ +__WIREDTIGER_H_ +wiredtiger_err_printf +wiredtiger_scr_alloc +wiredtiger_scr_free + +# List of queue.h #defines that are "unused", but it's OK. +LIST_EMPTY +LIST_ENTRY +LIST_FIRST +LIST_FOREACH +LIST_HEAD +LIST_HEAD_INITIALIZER +LIST_INIT +LIST_INSERT_AFTER +LIST_INSERT_BEFORE +LIST_INSERT_HEAD +LIST_NEXT +LIST_REMOVE +QMD_TRACE_ELEM +QMD_TRACE_HEAD +QUEUE_MACRO_DEBUG +SLIST_EMPTY +SLIST_ENTRY +SLIST_FIRST +SLIST_FOREACH +SLIST_FOREACH_PREVPTR +SLIST_HEAD +SLIST_HEAD_INITIALIZER +SLIST_INIT +SLIST_INSERT_AFTER +SLIST_INSERT_HEAD +SLIST_NEXT +SLIST_REMOVE +SLIST_REMOVE_HEAD +STAILQ_CONCAT +STAILQ_EMPTY +STAILQ_ENTRY +STAILQ_FIRST +STAILQ_FOREACH +STAILQ_HEAD +STAILQ_HEAD_INITIALIZER +STAILQ_INIT +STAILQ_INSERT_AFTER +STAILQ_INSERT_HEAD +STAILQ_INSERT_TAIL +STAILQ_LAST +STAILQ_NEXT +STAILQ_REMOVE +STAILQ_REMOVE_HEAD +STAILQ_REMOVE_HEAD_UNTIL +TAILQ_CONCAT +TAILQ_EMPTY +TAILQ_FOREACH_REVERSE +TAILQ_HEAD +TAILQ_HEAD_INITIALIZER +TAILQ_INSERT_AFTER +TAILQ_INSERT_BEFORE +TAILQ_LAST +TAILQ_NEXT +TAILQ_PREV +TRACEBUF +TRASHIT +_DB_QUEUE_H_ diff --git a/dist/s_docs b/dist/s_docs new file mode 100755 index 00000000000..3852a5ff8db --- /dev/null +++ b/dist/s_docs @@ -0,0 +1,86 @@ +#! /bin/sh + +t=__wt.$$ +trap 'rm -f $t; exit 0' 0 1 2 3 13 15 + +# Skip this when building release packages: docs are built separately +test -n "$WT_RELEASE_BUILD" && exit 0 + +# We require doxygen which may not be installed. +type doxygen > /dev/null 2>&1 || { + echo 'skipped: doxygen not found' + exit 0 +} + +. ./RELEASE + +e=0 +spellchk() +{ + # If aspell has been installed, run a spell check. + type aspell > /dev/null 2>&1 || return + + (cd ../docs/src && + cat *.dox | aspell --mode=ccpp --lang=en --personal=./spell.ok list) | + sort -u > $t + test -s $t && { + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + echo 'Documentation spelling notes' + echo 'Update docs/src/spell.ok to remove warnings.' + sed -e 's/^/ /' < $t + echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" + e=1 + } +} + +build() +{ + # Run doxygen to generate warnings for the base HTML documentation. + # + # We omit Python because warnings are expected there (the code generated + # by swig does not have named arguments, but we want to document them + # as if they do. + (cd ../docs && + (eval cat Doxyfile $filter ; cat <<EOF +QUIET=YES +EOF +) | doxygen - + test -s doxygen.log && cat doxygen.log) > $t 2>&1 + test -s $t && { + cat $t + e=1 + } + + # Run again to generate the full documentation set (with Python). + (cd ../docs && (eval cat Doxyfile $filter ; cat <<EOF +QUIET=YES +INPUT+=../lang/python/wiredtiger.py +EOF +) | doxygen - + + # Fix up bad links doxygen generates in navtree.js + sed -i~ 's,/\.html,/,' navtree.js && rm -f navtree.js~ + ) +} + +filter="|sed '/PROJECT_NUMBER/s,=.*,=\"Version $WIREDTIGER_VERSION\",'" +while : + do case "$1" in + -p) # Generate PDFs + filter="$filter| sed '/GENERATE_LATEX/s,=.*,=YES,'" + shift;; + -t) # Include the TODO list + filter="$filter| sed '/GENERATE_TODOLIST/s,=.*,=YES,'" + shift;; + *) + break;; + esac +done + +# Spell check the documentation. +spellchk + +# Build the documentation. +build + +exit $e diff --git a/dist/s_funcs b/dist/s_funcs new file mode 100644 index 00000000000..e2619c407de --- /dev/null +++ b/dist/s_funcs @@ -0,0 +1,29 @@ +#! /bin/sh + +# Complain about unused functions +t=__wt.$$ +trap 'rm -f $t; exit 0' 0 1 2 3 13 15 + +# List of files to search. +l=`sed -e 's,#.*,,' -e '/^$/d' -e 's,^,../,' filelist` +l="$l `echo ../src/*/*.i`" + +( +# Copy out the functions we don't use, but it's OK. +sed -e '/^$/d' -e '/^#/d' < s_funcs.list + +# Get the list of functions +search=`egrep -h '^[a-zA-Z0-9_]*\(' $l | sed 's/(.*//' | sort -u` + +# Print the list of functions, followed by the occurrences: we're looking for +# functions that only appear once +echo "$search" +sed -n '/{/,/^}/p' $l | fgrep -wo "$search" + +sed -n '/^#define/,/[^\\]$/p' ../src/include/*.h ../src/include/*.in | + fgrep -who "$search" +) | sort | uniq -u > $t + +test -s $t && cat $t + +exit 0 diff --git a/dist/s_funcs.list b/dist/s_funcs.list new file mode 100644 index 00000000000..a39ec7a529e --- /dev/null +++ b/dist/s_funcs.list @@ -0,0 +1,15 @@ +# List of functions that aren't found by s_funcs, but that's OK. +__bit_ffs +__bit_nclr +__wt_bm_addr_stderr +__wt_btree_lex_compare +__wt_debug_addr +__wt_debug_tree +__wt_debug_tree_all +__wt_fsync +__wt_log_printf +__wt_nlpo2 +__wt_nlpo2_round +__wt_print_huffman_code +wiredtiger_struct_pack +wiredtiger_struct_unpack diff --git a/dist/s_getopt b/dist/s_getopt new file mode 100644 index 00000000000..91d658fcd21 --- /dev/null +++ b/dist/s_getopt @@ -0,0 +1,6 @@ +#! /bin/sh + +# Complain if someone breaks getopt usage in the wt utility. +egrep -w 'opterr|optind|optopt|optreset|optarg' ../src/utilities/*.c + +exit 0 diff --git a/dist/s_longlines b/dist/s_longlines new file mode 100644 index 00000000000..f5970cda544 --- /dev/null +++ b/dist/s_longlines @@ -0,0 +1,16 @@ +#! /bin/sh + +# Check for long lines +t=__wt.$$ +trap 'rm -f $t; exit 0' 0 1 2 3 13 15 + +l=`(sed -e '/^[a-z]/! d' -e 's,^,../,' filelist ; + ls ../examples/c/*.c \ + ../src/include/*.[hi] \ + ../src/include/*.in; + find ../test -name '*.[ch]' -print) | egrep -v 'support/stat\.c$'` + +for f in $l ; do + expand -t8 < $f | awk -- \ + "{if(length(\$0) > 80) printf(\"%s:%d\\n\", \"$f\", NR)}" +done diff --git a/dist/s_printf b/dist/s_printf new file mode 100644 index 00000000000..e039b6cffd2 --- /dev/null +++ b/dist/s_printf @@ -0,0 +1,21 @@ +#! /bin/sh + +t=__wt.$$ +trap 'rm -f $t; exit 0' 0 1 2 3 13 15 + +l="`sed -e '/^[a-z]/! d' -e 's,^,../,' filelist`" +l="$l `ls ../src/include/*.[hi] ../src/include/*.in`" +l="$l `find ../test -name '*.c' -print | egrep -v '/packing/|/insert/'`" + +# Look for '%l': that suggests we're trying to print a long value, and that's +# almost always wrong: we should use a portable PRI* macro to construct the +# format string instead. +(for f in $l ; do + sed -n -E '/WT_VERBOSE|printf|scanf|__wt_errx?/,/\);/{= +p +}' $f | + sed -e 'N' -e 's/\n/:/' -e "s,^,$f:," + done) | grep '%l' > $t +test -s $t && cat $t && exit 1 + +exit 0 diff --git a/dist/s_prototypes b/dist/s_prototypes new file mode 100755 index 00000000000..b52f2f67924 --- /dev/null +++ b/dist/s_prototypes @@ -0,0 +1,46 @@ +#! /bin/sh + +# Build a list of internal function and variable prototypes. +t=__wt.$$ +trap 'rm -f $t; exit 0' 0 1 2 3 13 15 + +( +cat <<EOF +/* DO NOT EDIT: automatically built by dist/s_prototypes. */ + +EOF + +for i in `sed -e '/^[a-z]/! d' filelist`; do + sed -n \ + -e '/^__wt_[a-z]/!{' \ + -e h \ + -e d \ + -e '}' \ + -e x \ + -e '/^static/d' \ + -e x \ + -e ': loop' \ + -e H \ + -e n \ + -e '/;/b end' \ + -e '/^{/!b loop' \ + -e ': end' \ + -e x \ + -e 's/ =.*$//' \ + -e '/#/!s/\n/ /g' \ + -e 's/\* /\*/g' \ + -e 's/ */ /g' \ + -e 's/^/extern /' \ + -e 's/WT_GCC_FUNC_/WT_GCC_/' \ + -e 's/$/;/p' \ + < ../$i +done) | awk '{ + + if (length() > 80) + gsub(", ", ",\n "); + print $0 +}' > $t + +f=../src/include/extern.h +cmp $t $f > /dev/null 2>&1 || + (echo "Building $f" && rm -f $f && cp $t $f) diff --git a/dist/s_readme b/dist/s_readme new file mode 100644 index 00000000000..c348baf4049 --- /dev/null +++ b/dist/s_readme @@ -0,0 +1,40 @@ +#! /bin/sh + +t=__wt.$$ +trap 'rm -f $t; exit 0' 0 1 2 3 13 15 + +f=../README + +. ./RELEASE + +# If the version number has changed, generate a new README file (don't generate +# a new one just because the date changed, that happens all the time). +cnt=`(sed -e q < $f; echo "$WIREDTIGER_VERSION_STRING") | + sed -e 's/:.*//' | sort -u | wc -l` +test $cnt -eq 1 && exit 0 + +cat << END_TEXT > $t +$WIREDTIGER_VERSION_STRING + +This is version $WIREDTIGER_VERSION_MAJOR.$WIREDTIGER_VERSION_MINOR.$WIREDTIGER_VERSION_PATCH of WiredTiger. + +See the file LICENSE for redistribution information. + +To view release and installation documentation, open docs/index.html +in your web browser. + +To build in a POSIX environment (for example, Linux or other UNIX-like +system), run: + + ./configure && make + +To install in the standard POSIX environment locations, enter: + + make install + +For more information on build and installation options, see the +WiredTiger documentation. +END_TEXT + +cmp $t $f > /dev/null 2>&1 || + (echo "Building $f" && rm -f $f && cp $t $f) diff --git a/dist/s_release b/dist/s_release new file mode 100755 index 00000000000..8682fe09d3d --- /dev/null +++ b/dist/s_release @@ -0,0 +1,40 @@ +#!/bin/sh +# Build a WiredTiger release package. + +set -e +RELEASE_DIR=../releases + +. ./RELEASE || exit 1 + +pkgver="$1" +if test -z "$pkgver" ; then + pkgver="$WIREDTIGER_VERSION" +fi +PKG="wiredtiger-$pkgver" +DEST="$RELEASE_DIR/$PKG" + +rm -rf "$DEST" + +echo "Running 'hg archive' to copy the tree" +hg archive \ + `sed -e '/^#/d' -e 's/^/--exclude ..\//' < s_release.list` \ + "$DEST" + +echo "Running 'dist/s_all' in the release tree" +(cd "$DEST/dist" && env WT_RELEASE_BUILD=yes sh s_all -A > /dev/null) + +echo "Running swig to generate the Python API" +(cd "$DEST/build_posix" && \ + ../configure --enable-python && \ + (cd lang/python && make ../../../lang/python/wiredtiger_wrap.c) && \ + make distclean) > /dev/null + +echo "Building documentation" +(cd "$DEST/dist" && sh s_docs > /dev/null) + +echo "Packing release into $RELEASE_DIR/$PKG.tar.bz2" +(cd "$RELEASE_DIR" && tar cf - $PKG | bzip2 -9 > $PKG.tar.bz2) + +echo "Packing documentation into $RELEASE_DIR/$PKG-docs.tar.bz2" +(cd "$RELEASE_DIR" && tar cf - $PKG/[A-Z][A-Z]* $PKG/docs | \ + bzip2 -9 > $PKG-docs.tar.bz2) diff --git a/dist/s_release.list b/dist/s_release.list new file mode 100644 index 00000000000..feeb32a8ae0 --- /dev/null +++ b/dist/s_release.list @@ -0,0 +1,11 @@ +# Exclusions from release packages. +# Each non-comment line is passed as an "--exclude" argument to "hg archive". +lang/java +lang/python/src +src/server +src/txn +test/format +test/insert +test/packing +test/salvage +test/thread diff --git a/dist/s_stat b/dist/s_stat new file mode 100644 index 00000000000..6eab264e7b6 --- /dev/null +++ b/dist/s_stat @@ -0,0 +1,28 @@ +#! /bin/sh + +# Complain about unused statistics fields. +t=__wt.$$ +trap 'rm -f $t; exit 0' 0 1 2 3 13 15 + +# List of files to search: skip stat.c, it lists all of the fields by +# definition. +l=`sed \ + -e '/src\/support\/stat.c/d' \ + -e 's,#.*,,' \ + -e '/^$/d' \ + -e 's,^,../,' filelist` +l="$l `echo ../src/include/*.i`" + +( +# Get the list of statistics fields. +search=`sed \ + -e 's/^ WT_STATS \([a-z_*]*\);$/\1/p' \ + -e d ../src/include/stat.h | + sort` + +echo "$search" +fgrep -who "$search" $l) | sort | uniq -u > $t + +test -s $t && cat $t + +exit 0 diff --git a/dist/s_string b/dist/s_string new file mode 100644 index 00000000000..daaf3fc6894 --- /dev/null +++ b/dist/s_string @@ -0,0 +1,29 @@ +#!/bin/sh - +# +# Check spelling in comments and quoted strings from the source files. + +t=__wt.$$ +trap 'rm -f $t; exit 0' 0 1 2 3 13 15 + +# If aspell has not been installed, quit +type aspell > /dev/null 2>&1 || { + echo 'skipped: aspell not found' + exit 0 +} + +check() { + aspell --mode=ccpp --lang=en list < ../$1 | + sort -u | + comm -23 /dev/stdin s_string.ok > $t + test -s $t && { + echo "==== $1" + cat $t + } +} + +l="`cd .. && echo src/include/*.[hi] src/include/*.in`" +for i in $l `sed -e '/^[a-z]/! d' filelist`; do + check $i +done + +exit 0 diff --git a/dist/s_string.ok b/dist/s_string.ok new file mode 100644 index 00000000000..49a0bffe646 --- /dev/null +++ b/dist/s_string.ok @@ -0,0 +1,511 @@ +AAAAA +AAAAAA +AAAAAAAAAA +AAAAAAAAAAAAA +AAAAAAAAAAAAAAAA +AAAAAAAAAAAAAAAAAA +ADDR +API +APIs +Alloc +Athlon +BBBBB +BBBBBB +BBBBBBBBBB +BBBBBBBBBBBBB +BBBBBBBBBBBBBBBB +BBBBBBBBBBBBBBBBBB +BSR +BTREE +BUF +BUFs +Bitfield +Bsearch +Btree +CAS +CELL's +CELLs +COL's +CONCAT +CONFIG +CPUs +CURSORs +CURSTD +Checksum +Config +DATAITEMs +DESC +Decrement +EB +EBUSY +EINTR +EINVAL +ENOENT +ENOMEM +ENOTSUP +ETIME +ETIMEDOUT +Enqueue +Env +Eron +FALLTHROUGH +FH +FLD +FOREACH +FREELIST +Fasttrack +FreeBSD +FreeBSD's +Freelist +GCC +IKEY +IMPL +IMPL's +INDX +INIT +INITIALIZER +INSERT's +JPEG +JSON +KV +Kanowski's +Kounavis +LF +LRU +LSB +LSN +LSN's +LSNs +LoadLoad +Lookup +MBUF +MEM +MERCHANTABILITY +MVCC +Marsaglia's +Mewhort +Mutex +Mutexes +NONINFRINGEMENT +NOTFOUND +NOTREACHED +NRECS +NUL +NULLs +NoAddr +OVFL +PADDR +PAGE's +POSIX +PSIZE +PTR +Pagesize +PlatformSDK +Pre +Qsort +READONLY +REF's +REQ +RET +RLE +RPC +Recurse +Redistributions +Resize +SCHED +SLIST +SSq +STAILQ +SYS +Skiplist +Solaris +Split's +StoreLoad +Subtree +Subtrees +TAILQ +TODO +TOOSMALL +UINT +URI +URIs +UTF +UnixLib +Vc +Vixie +VxWorks +WIREDTIGER +WinNT +WiredTiger +WiredTiger's +Wuninitialized +XP +abcdef +addfrag +addl +addr +addrs +alfred +alloc +allocator +allocsize +ap +api +arg +async +bigram +bitfield +bitfields +bitpos +bitstr +bitstring +bitwise +bm +bnd +boolean +br +breakpoint +bsearch +bt +btcur +btdsk +btmem +btree +buf +builtin +bytelock +bzip +calloc +catfmt +cfg +checkfrag +checksum +checksums +chk +cip +cksum +clr +cmp +colcheck +colgroup +colgroups +comparator +concat +cond +conf +config +conn +connectionp +crc +cref +ctype +curbtree +curbulk +curconfig +curdump +curfile +curindex +cursorp +curstat +curtable +cxa +data's +datalen +datasets +decrement +decrementing +del +delfmt +dequeue +dequeued +der +dereference +desc +dest +dlclose +dlh +dlopen +dlsym +dmsg +dsk +dst +dumpable +dumpfile +dup +encodings +endian +english +enqueue +enum's +eof +eop +errhandler +errno +errv +errx +esc +eventv +evictserver +exactp +extern +fcntl +ffc +ffs +fh +filename +fileops +filesize +filesystem +fixup +fmt +fp +free'd +freelist +fs +fsm +fstat +fsync +fsyncs +ftruncate +func +funcs +gcc +gdb +getone +getoneraw +getones +getonesn +getraw +getv +gobare +goesc +gostring +gostruct +goutf +havesize +hdr +hrow +html +huffman +iSh +ikey +imref +incr +incrementing +incrv +indices +indx +init +initn +initsize +inmem +intl +intpack +ints +inuse +io +ispo +kcell +keycmp +keyname +keyv +kv +len +lenp +lex +lexicographically +lfence +llll +logput +lookup +lookups +lru +lsn +lu +majorp +malloc +maxintlitem +maxintlpage +maxleafitem +maxleafpage +mb +mem +memalloc +membar +memcpy +memfree +memsize +metadata +mfence +minorp +minprefix +msg +mtx +multiprocess +multithread +multithreaded +mutex +mutexes +namespaces +nbits +nclr +negint +newname +nl +nlpo +nocase +nop +notsup +notyet +np +nset +nul +numSymbols +numbare +offpage +oldname +os +ovfl +packv +patchp +pfx +posint +posix +pre +prepended +presize +printf +priv +pthread +pv +py +qSS +qdown +qsort +qup +rdlock +readlock +readnear +readnext +readprev +readserver +realloc +recno +req +resizing +ret +retp +rle +rpc +rref +runlength +rwlock +rwlocks +rwunlock +savepoints +sb +schemas +schematab +scr +searchable +serializable +sessionp +setv +sfence +sizeof +sizev +skiplist +skiplists +slvg +snprintf +sp +spinlock +sprintf +src +srch +srvr +sset +stbar +stdarg +stderr +stdout +str +strdup +strerror +stringin +strncpy +strndup +strtouq +struct +structs +subgetraw +subgets +subinit +subtree +subtrees +superset +sys +t's +tV +tablename +tblock +tcursors +tempty +th +thazard +tinsert +toffpage +toverflow +tparent +transactional +trecno +treeconfig +trepeat +treplacement +trk +trk's +troot +tsplit +tstate +tt +ttracking +tupdate +tvalue +twrite +txn +typedef +uB +uint +unbare +uncompressing +undef +unesc +unescaped +uninstantiated +unjams +unlinked +unpackv +unreferenced +unsized +untyped +upd +uri +utf +va +vanishingly +vcell +versa +vpack +vsize +vslot +vunpack +vupdate +wiredtiger +wrapup +writelock +wrlock +wti +xff +xxxx +xxxxx +xxxxxx diff --git a/dist/s_style b/dist/s_style new file mode 100644 index 00000000000..76970a94ba0 --- /dev/null +++ b/dist/s_style @@ -0,0 +1,58 @@ +#! /bin/sh + +# General style correction and cleanup. +t=__wt.$$ +trap 'rm -f $t; exit 0' 0 1 2 3 13 15 + +extra=`cd .. && echo src/include/*.[hi] src/include/*.in dist/*.py` +( +for f in `sed -e '/^[a-z]/! d' filelist` $extra; do + f="../$f" + if grep "^[^}]*while (0);" $f > /dev/null; then + echo "$f: while (0) has trailing semi-colon" + fi + if grep "%dl|%ul|%xl" $f > /dev/null; then + echo "$f: bad printf format: %[dux]l" + fi + if grep "(unsigned)" $f > /dev/null; then + echo "$f: (unsigned) cast is wrong" + fi + + # Early exits from critical loops + sed -n -e '/API_CALL/,/API_END/{=;p;}' \ + -e '/va_start/,/va_end/{=;p;}' $f | \ + sed 'N;s/\n/:/' | \ + egrep 'return|WT_RET' | \ + sed -e "s,^,$f:," -e 's/$/ [return skips API_END call]/' + + # Bad code we can't easily fix + grep -Hn 'bzero|exit[ ]*\(1\)|^[ ]+[|&=+-]' $f + + tr -cd '[:alnum:][:space:][:punct:]' < $f | + unexpand | + sed -e 's/){/) {/' \ + -e 's/\([ ]\)exit (/\1exit(/g' \ + -e 's/\([ ]\)for(/\1for (/' \ + -e 's/\([ ]\)if(/\1if (/' \ + -e 's/\([ ]\)index(/\1strchr(/' \ + -e 's/\([ ]\)return(/\1return (/' \ + -e 's/^\([ ]+\)return \([^()]*\);/\1return (\2);/' \ + -e 's/\([ ]\)rindex(/\1strrchr(/' \ + -e 's/\([ ]\)sizeof (/\1sizeof(/g' \ + -e 's/\([ ]\)switch(/\1switch (/' \ + -e 's/\([ ]\)while(/\1while (/' \ + -e 's/\([ ,]\)uint\([ ,]\)/\1u_int\2/g' \ + -e 's/\([ ,]\)u_int8_t\([ ,]\)/\1uint8_t\2/g' \ + -e 's/\([ ,]\)u_int16_t\([ ,]\)/\1uint16_t\2/g' \ + -e 's/\([ ,]\)u_int32_t\([ ,]\)/\1uint32_t\2/g' \ + -e 's/\([ ,]\)u_int64_t\([ ,]\)/\1uint64_t\2/g' \ + -e 's/%\([dux]\)l/%l\1/' \ + -e 's/\([|&=+-]\) *\([^*]\)/\1 \2/' \ + -e 's/(void) /(void)/' \ + -e '/for /!s/;;$/;/' \ + -e 's/(unsigned)/(u_int)/' \ + -e 's/^#define /#define /' \ + -e 's/sizeof(WT_PAGE_DISK)/WT_PAGE_DISK_SIZE/g' >$t + + cmp $t $f > /dev/null 2>&1 || (echo "$f" && cp $t $f) +done) diff --git a/dist/s_symbols b/dist/s_symbols new file mode 100644 index 00000000000..b3e41844dd2 --- /dev/null +++ b/dist/s_symbols @@ -0,0 +1,41 @@ +#! /bin/sh + +# Check for illegal external symbols. +# +t=__a.c +trap 'rm -f $t; exit 0' 0 1 2 3 13 15 + +case `uname` in +Darwin) + NM='nm -gUo $f | egrep " T | D " | sed "s/ _/ /"' + ;; +*) + # We require GNU nm, which may not be installed. + type nm > /dev/null 2>&1 && + (nm --version | grep 'GNU nm') > /dev/null 2>&1 || { + echo 'skipped: GNU nm not found' + exit 0 + } + NM='nm --extern-only --defined-only --print-file-name $f' + ;; +esac + +# This check would normally be done after the library is built, but this way +# we don't forget about a symbol during development. Check the previously +# built library, if it exists. +f=../build_posix/.libs/libwiredtiger.a +test -f $f || { + echo "skipped: libwiredtiger.a not found" + exit 0 +} + +(sed -e '/^#/d' s_symbols.list && +eval $NM | +sed 's/.* //' | +egrep -v '^__wt') | +sort | +uniq -u > $t + +test -s $t && cat $t + +exit 0 diff --git a/dist/s_symbols.list b/dist/s_symbols.list new file mode 100644 index 00000000000..0bc487e25b3 --- /dev/null +++ b/dist/s_symbols.list @@ -0,0 +1,7 @@ +# List of OK external symbols. +wiredtiger_open +wiredtiger_strerror +wiredtiger_struct_pack +wiredtiger_struct_size +wiredtiger_struct_unpack +wiredtiger_version diff --git a/dist/s_tags b/dist/s_tags new file mode 100644 index 00000000000..65b8d917a05 --- /dev/null +++ b/dist/s_tags @@ -0,0 +1,42 @@ +#! /bin/sh + +# Build tags file. +# +t=__a.c +trap 'rm -f $t; exit 0' 0 1 2 3 13 15 + +# Skip this when building release packages +test -n "$WT_RELEASE_BUILD" && exit 0 + +# We require ctags which may not be installed. +type ctags > /dev/null 2>&1 || { + echo 'skipped: ctags not found' > $t + exit 0 +} + +# Test to see what flags this ctags binary supports. +# Use the -d, -t and -w flags to ctags if available. +flags="" +echo "f() { int a; }" > $t +for i in -d -t -w --language-force=C; do + if ctags $i $t 2>/dev/null; then + flags="$i $flags" + fi +done + +# Generate a tags file for the build directory +(cd ../build_posix +rm -f tags +ctags $flags ../src/include/*.in ../src/*/*.[chi] 2>/dev/null) + +# Put the shared tags file in the include directory, it's at the same level in +# the tree as the other source files. +(cd ../src/include +rm -f tags +ctags $flags ../include/*.in ../*/*.[chi] 2>/dev/null) + +# Link to the tags file from standard build and source directories. +dirs="`python -c 'from dist import directory_files; directory_files();'`" +for i in $dirs; do + (cd ../$i && rm -f tags && ln -s ../include/tags .) +done diff --git a/dist/s_typedef b/dist/s_typedef new file mode 100644 index 00000000000..42033dff4bc --- /dev/null +++ b/dist/s_typedef @@ -0,0 +1,70 @@ +#! /bin/sh + +t=__wt.$$ +trap 'rm -f $t; exit 0' 0 1 2 3 13 15 + +build() { + # Build the standard typedefs. + f=../src/include/wt_internal.in + (sed -e '/Forward structure declarations .*: BEGIN/{' \ + -e 'n' \ + -e 'q' \ + -e '}' < $f + + l=`echo ../src/include/*.[hi] ../src/include/*.in | + sed -e 's/wiredtiger.*\.in//' -e 's/queue.h//'` + egrep -h '^[ ]*(struct|union)[ ]*__.*[ ]*{$' $l | \ + sed -e 's/^[ ]*//' -e 's/[ ]*{$//' | sort | \ + while read t n; do + upper=`echo $n | sed -e 's/^__//' | tr a-z A-Z` + echo "$t $n;" + echo " typedef $t $n $upper;" + done + + echo '/*' + sed -e '/Forward structure declarations .*: END/,${' \ + -e 'p' \ + -e '}' \ + -e 'd' < $f) > $t + cmp $t $f > /dev/null 2>&1 || + (echo "Building $f" && rm -f $f && cp $t $f) +} + +check() { + # Complain about unused #typedefs. + # List of files to search. + l=`sed -e 's,#.*,,' -e '/^$/d' -e 's,^,../,' filelist` + l="$l `echo ../src/utilities/*.c`" + + ( + # Get the list of typedefs + search=`cat ../src/include/*.h ../src/include/*.in | + sed -e 's/^struct.*typedef.* \(.*\);$/\1/p' \ + -e 's/^union.*typedef.* \(.*\);$/\1/p' \ + -e d | + sort -u` + echo "$search" + fgrep -who "$search" $l + ) | sort | uniq -u > $t + + test -s $t && cat $t +} + +while : + do case "$1" in + -b) # -b builds the typedefs + build + shift;; + -c) # -c checks the typedefs + check + shift;; + *) + break;; + esac +done + +test "$#" -eq 0 || { + echo 'usage: s_typedef [-bc]' >&2 + exit 1 +} +exit 0 diff --git a/dist/s_types b/dist/s_types new file mode 100644 index 00000000000..274ae1d13fd --- /dev/null +++ b/dist/s_types @@ -0,0 +1,16 @@ +#! /bin/sh + +t=__wt.$$ +trap 'rm -f $t; exit 0' 0 1 2 3 13 15 + +l="`cd .. && echo src/include/*.[hi] src/include/*.in`" +l="`cd .. && find test -name '*.c' -print`" +l="`cd .. && echo */*.i`" +l="$l `sed -e '/^[a-z]/! d' filelist`" + +(cd .. && + egrep 'u_quad|u_int8_t|u_int16_t|u_int32_t|u_int64_t' $l | + sed '/@u_quad_decl@/d') > $t +test -s $t && cat $t && exit 1 + +exit 0 diff --git a/dist/s_version b/dist/s_version new file mode 100755 index 00000000000..6739fcef18e --- /dev/null +++ b/dist/s_version @@ -0,0 +1,39 @@ +#!/bin/sh + +# Propagate version changes to the necessary files. +. ./RELEASE + +m4dir=../build_posix/aclocal + +if test -f $m4dir/version.m4 -a -f $m4dir/version-set.m4 ; then + eval `grep '^VERSION_[A-Z]*=' $m4dir/version-set.m4` + if test x${WIREDTIGER_VERSION_MAJOR} = x${VERSION_MAJOR} -a \ + x${WIREDTIGER_VERSION_MINOR} = x${VERSION_MINOR} -a \ + x${WIREDTIGER_VERSION_PATCH} = x${VERSION_PATCH} ; then + exit 0 + fi +fi + +echo "Building $m4dir/version.m4" +cat <<EOF > $m4dir/version.m4 +dnl WiredTiger product version for AC_INIT. Maintained by dist/s_version +${WIREDTIGER_VERSION_MAJOR}.${WIREDTIGER_VERSION_MINOR}.${WIREDTIGER_VERSION_PATCH} +EOF + +echo "Building $m4dir/version.m4" +cat <<EOF > $m4dir/version-set.m4 +dnl build by dist/s_version + +VERSION_MAJOR=${WIREDTIGER_VERSION_MAJOR} +VERSION_MINOR=${WIREDTIGER_VERSION_MINOR} +VERSION_PATCH=${WIREDTIGER_VERSION_PATCH} +VERSION_STRING='"${WIREDTIGER_VERSION_STRING}"' + +AC_SUBST(VERSION_MAJOR) +AC_SUBST(VERSION_MINOR) +AC_SUBST(VERSION_PATCH) +AC_SUBST(VERSION_STRING) + +VERSION_NOPATCH=${WIREDTIGER_VERSION_MAJOR}.${WIREDTIGER_VERSION_MINOR} +AC_SUBST(VERSION_NOPATCH) +EOF diff --git a/dist/s_whitespace b/dist/s_whitespace new file mode 100644 index 00000000000..13fbacca484 --- /dev/null +++ b/dist/s_whitespace @@ -0,0 +1,15 @@ +#! /bin/sh + +# Single space and remove trailing whitespace from source files. +t=__wt.$$ +trap 'rm -f $t; exit 0' 0 1 2 3 13 15 + +inc=`cd .. && echo src/include/*.[hi] src/include/*.in dist/*.py dist/s_*` +( +for f in `sed -e '/^[a-z]/! d' filelist` $inc; do + f="../$f" + sed -e 's/[ ][ ]*$//' \ + -e '/^$/N' \ + -e '/\n$/D' < $f > $t + cmp $t $f > /dev/null 2>&1 || (echo "$f" && cp $t $f) +done) diff --git a/dist/serial.py b/dist/serial.py new file mode 100644 index 00000000000..9fb8d3fed69 --- /dev/null +++ b/dist/serial.py @@ -0,0 +1,175 @@ +# Output serialization functions. + +import textwrap +from dist import compare_srcfile + +class SerialArg: + def __init__(self, typestr, name, sized=0): + self.typestr = typestr + self.name = name + self.sized = sized + +class Serial: + def __init__(self, name, op, args): + self.name = name + self.op = op + self.args = args + +msgtypes = [ +Serial('col_append', 'WT_SERIAL_FUNC', [ + SerialArg('WT_INSERT_HEAD **', 'inshead'), + SerialArg('WT_INSERT ***', 'ins_stack'), + SerialArg('WT_INSERT_HEAD **', 'new_inslist', 1), + SerialArg('WT_INSERT_HEAD *', 'new_inshead', 1), + SerialArg('WT_INSERT *', 'new_ins', 1), + SerialArg('u_int', 'skipdepth'), + ]), + +Serial('evict_file', 'WT_SERIAL_EVICT', [ + SerialArg('int', 'close_method'), + ]), + +Serial('insert', 'WT_SERIAL_FUNC', [ + SerialArg('WT_PAGE *', 'page'), + SerialArg('uint32_t', 'write_gen'), + SerialArg('WT_INSERT_HEAD **', 'inshead'), + SerialArg('WT_INSERT ***', 'ins_stack'), + SerialArg('WT_INSERT_HEAD **', 'new_inslist', 1), + SerialArg('WT_INSERT_HEAD *', 'new_inshead', 1), + SerialArg('WT_INSERT *', 'new_ins', 1), + SerialArg('u_int', 'skipdepth'), + ]), + +Serial('row_key', 'WT_SERIAL_FUNC', [ + SerialArg('WT_PAGE *', 'page'), + SerialArg('WT_ROW *', 'row_arg'), + SerialArg('WT_IKEY *', 'ikey'), + ]), + +Serial('update', 'WT_SERIAL_FUNC', [ + SerialArg('WT_PAGE *', 'page'), + SerialArg('uint32_t', 'write_gen'), + SerialArg('WT_UPDATE **', 'srch_upd'), + SerialArg('WT_UPDATE **', 'new_upd', 1), + SerialArg('WT_UPDATE *', 'upd', 1), + ]), +] + +# decl -- +# Return a declaration for the variable. +def decl(l): + o = l.typestr + if o[-1] != '*': + o += ' ' + return o + l.name + +# decl_p -- +# Return a declaration for a reference to the variable, which requires +# another level of indirection. +def decl_p(l): + o = l.typestr + if o[-1] != '*': + o += ' ' + return o + '*' + l.name + 'p' + +# output -- +# Create serialized function calls. +def output(entry, f): + # structure declaration + f.write(''' +typedef struct { +''') + for l in entry.args: + f.write('\t' + decl(l) + ';\n') + if l.sized: + f.write('\tsize_t ' + l.name + '_size;\n') + f.write('\tint ' + l.name + '_taken;\n') + f.write('} __wt_' + entry.name + '_args;\n\n') + + # pack function + f.write('static inline int\n__wt_' + entry.name + '_serial(\n') + o = 'WT_SESSION_IMPL *session' + for l in entry.args: + if l.sized: + o += ', ' + decl_p(l) + ', size_t ' + l.name + '_size' + else: + o += ', ' + decl(l) + o += ')' + f.write('\n'.join('\t' + l for l in textwrap.wrap(o, 70))) + f.write(''' +{ +\t__wt_''' + entry.name + '''_args _args, *args = &_args; +\tint ret; + +''') + for l in entry.args: + if l.sized: + f.write('''\tif (''' + l.name + '''p == NULL) +\t\targs->''' + l.name + ''' = NULL; +\telse { +\t\targs->''' + l.name + ''' = *''' + l.name + '''p; +\t\t*''' + l.name + '''p = NULL; +\t\targs->''' + l.name + '''_size = ''' + l.name + '''_size; +\t} +\targs->''' + l.name + '''_taken = 0; + +''') + else: + f.write('\targs->' + l.name + ' = ' + l.name + ';\n\n') + f.write('\tret = __wt_session_serialize_func(session,\n') + f.write('\t ' + entry.op + + ', __wt_' + entry.name + '_serial_func, args);\n\n') + for l in entry.args: + if not l.sized: + continue + f.write('\tif (!args->' + l.name + '_taken)\n') + f.write('\t\t__wt_free(session, args->' + l.name + ');\n') + f.write('\treturn (ret);\n') + f.write('}\n\n') + + # unpack function + f.write('static inline void\n__wt_' + entry.name + '_unpack(\n') + o = 'WT_SESSION_IMPL *session' + for l in entry.args: + o += ', ' + decl_p(l) + o +=')' + f.write('\n'.join('\t' + l for l in textwrap.wrap(o, 70))) + f.write(''' +{ +\t__wt_''' + entry.name + '''_args *args = +\t (__wt_''' + entry.name + '''_args *)session->wq_args; + +''') + for l in entry.args: + f.write('\t*' + l.name + 'p = args->' + l.name + ';\n') + f.write('}\n') + + # taken functions + for l in entry.args: + if l.sized: + f.write(''' +static inline void\n__wt_''' + entry.name + '_' + l.name + '''_taken(WT_SESSION_IMPL *session, WT_PAGE *page) +{ +\t__wt_''' + entry.name + '''_args *args = +\t (__wt_''' + entry.name + '''_args *)session->wq_args; + +\targs->''' + l.name + '''_taken = 1; + +\tWT_ASSERT(session, args->''' + l.name + '''_size != 0); +\t__wt_cache_page_inmem_incr(session, page, args->''' + l.name + '''_size); +} +''') + +##################################################################### +# Update serial_funcs.i. +##################################################################### +tmp_file = '__tmp' +tfile = open(tmp_file, 'w') +tfile.write('/* DO NOT EDIT: automatically built by dist/serial.py. */\n') + +for entry in msgtypes: + output(entry, tfile) + +tfile.close() + +compare_srcfile(tmp_file, '../src/include/serial_funcs.i') diff --git a/dist/stat.py b/dist/stat.py new file mode 100644 index 00000000000..0af0e65c04b --- /dev/null +++ b/dist/stat.py @@ -0,0 +1,145 @@ +# Read the source files and output the statistics #defines and allocation code. + +import re, string, sys, textwrap +from operator import attrgetter +from dist import compare_srcfile +from dist import source_paths_list + +# Read the source files. +from stat_data import btree_stats, connection_stats + +# print_struct -- +# Print the structures for the stat.h file. +def print_struct(title, name, list): + f.write('/*\n') + f.write(' * Statistics entries for ' + title + ' handle.\n') + f.write(' */\n') + f.write('struct __wt_' + name + '_stats {\n') + + # Sort the structure fields by their description, so the eventual + # disply is sorted by string. + for l in sorted(list, key=attrgetter('desc')): + f.write('\tWT_STATS ' + l.name + ';\n') + f.write('};\n\n') + +# Update the #defines in the stat.h file. +tmp_file = '__tmp' +f = open(tmp_file, 'w') +skip = 0 +for line in open('../src/include/stat.h', 'r'): + if not skip: + f.write(line) + if line.count('Statistics section: END'): + f.write(line) + skip = 0 + elif line.count('Statistics section: BEGIN'): + f.write('\n') + skip = 1 + print_struct('BTREE', 'btree', btree_stats) + print_struct('CONNECTION', 'connection', connection_stats) +f.close() +compare_srcfile(tmp_file, '../src/include/stat.h') + +# print_define -- +# Print the #defines for the wiredtiger.in file. +def print_define(): + # Sort the structure fields by their description so they match + # the structure lists. + f.write(''' +/*! + * @name Statistics for connection handles + * @anchor statistics_keys + * @anchor statistics_conn + * Statistics in WiredTiger are accessed through cursors with \c "statistics:" + * URIs. Individual statistics can be queried through the cursor using the + * following keys. + * @{ + */ +''') + for v, l in enumerate(sorted(connection_stats, key=attrgetter('desc'))): + f.write('/*! %s */\n' % '\n * '.join(textwrap.wrap(l.desc, 70))) + f.write('#define\tWT_STAT_' + l.name + "\t" * + max(1, 6 - int((len('WT_STAT_') + len(l.name)) / 8)) + + str(v) + '\n') + f.write(''' +/*! + * @} + * @name Statistics for file objects + * @anchor statistics_file + * @{ + */ +''') + for v, l in enumerate(sorted(btree_stats, key=attrgetter('desc'))): + f.write('/*! %s */\n' % '\n * '.join(textwrap.wrap(l.desc, 70))) + f.write('#define\tWT_STAT_' + l.name + "\t" * + max(1, 6 - int((len('WT_STAT_') + len(l.name)) / 8)) + + str(v) + '\n') + f.write('/*! @} */\n') + +# Update the #defines in the wiredtiger.in file. +tmp_file = '__tmp' +f = open(tmp_file, 'w') +skip = 0 +for line in open('../src/include/wiredtiger.in', 'r'): + if not skip: + f.write(line) + if line.count('Statistics section: END'): + f.write(line) + skip = 0 + elif line.count('Statistics section: BEGIN'): + f.write(' */\n') + skip = 1 + print_define() + f.write('/*\n') +f.close() +compare_srcfile(tmp_file, '../src/include/wiredtiger.in') + +# print_func -- +# Print the functions for the stat.c file. +def print_func(name, list): + f.write(''' +int +__wt_stat_alloc_''' + name + '''_stats(WT_SESSION_IMPL *session, WT_''' + + name.upper() + '''_STATS **statsp) +{ +\tWT_''' + name.upper() + '''_STATS *stats; + +\tWT_RET(__wt_calloc_def(session, 1, &stats)); + +''') + + for l in sorted(list): + o = '\tstats->' + l.name + '.desc = "' + l.desc + '";\n' + if len(o) + 7 > 80: + o = o.replace('= ', '=\n\t ') + f.write(o) + f.write(''' +\t*statsp = stats; +\treturn (0); +} +''') + + f.write(''' +void +__wt_stat_clear_''' + name + '''_stats(WT_STATS *stats_arg) +{ +\tWT_''' + name.upper() + '''_STATS *stats; + +\tstats = (WT_''' + name.upper() + '''_STATS *)stats_arg; +''') + for l in sorted(list): + # Items marked permanent aren't cleared by the stat clear + # methods. + if not l.config.count('perm'): + f.write('\tstats->' + l.name + '.v = 0;\n'); + f.write('}\n') + +# Write the stat allocation and clear routines to the stat.c file. +f = open(tmp_file, 'w') +f.write('/* DO NOT EDIT: automatically built by dist/stat.py. */\n\n') +f.write('#include "wt_internal.h"\n') + +print_func('btree', btree_stats) +print_func('connection', connection_stats) +f.close() +compare_srcfile(tmp_file, '../src/support/stat.c') diff --git a/dist/stat_data.py b/dist/stat_data.py new file mode 100644 index 00000000000..a95c72c09a6 --- /dev/null +++ b/dist/stat_data.py @@ -0,0 +1,88 @@ +# Auto-generate statistics #defines, with allocation, clear and print functions. +# +# The XXX_stats dictionaries are a set of objects consisting of comma-separated +# configuration key words and a text description. The configuration key words +# are: +# perm -- Field is not cleared by the stat clear function. + +class Stat: + def __init__(self, name, desc, config=None): + self.name = name + self.desc = desc + self.config = config or [] + + def __cmp__(self, other): + return cmp(self.name, other.name) + +########################################## +# CONNECTION statistics +########################################## +connection_stats = [ + Stat('block_read', 'blocks read from a file'), + Stat('block_write', 'blocks written to a file'), + Stat('cache_bytes_inuse', 'cache: bytes currently held in the cache', 'perm'), + Stat('cache_bytes_max', 'cache: maximum bytes configured', 'perm'), + Stat('cache_evict_hazard', 'cache: pages selected for eviction not evicted because of a hazard reference'), + Stat('cache_evict_internal', 'cache: internal pages evicted'), + Stat('cache_evict_modified', 'cache: modified pages evicted'), + Stat('cache_evict_slow', 'cache: eviction server unable to reach eviction goal'), + Stat('cache_evict_unmodified', 'cache: unmodified pages evicted'), + Stat('cache_pages_inuse', 'cache: pages currently held in the cache', 'perm'), + Stat('cond_wait', 'condition wait calls'), + Stat('file_open', 'files currently open'), + Stat('memalloc', 'total memory allocations'), + Stat('memfree', 'total memory frees'), + Stat('rwlock_rdlock', 'rwlock readlock calls'), + Stat('rwlock_wrlock', 'rwlock writelock calls'), + Stat('total_read_io', 'total read I/Os'), + Stat('total_write_io', 'total write I/Os'), +] + +########################################## +# BTREE statistics +########################################## +btree_stats = [ + Stat('alloc', 'file: block allocations'), + Stat('cursor_inserts', 'cursor-inserts'), + Stat('cursor_read', 'cursor-read'), + Stat('cursor_read_near', 'cursor-read-near'), + Stat('cursor_read_next', 'cursor-read-next'), + Stat('cursor_read_prev', 'cursor-read-prev'), + Stat('cursor_resets', 'cursor-resets'), + Stat('cursor_removes', 'cursor-removes'), + Stat('cursor_updates', 'cursor-updates'), + Stat('extend', 'file: block allocations required file extension'), + Stat('file_allocsize', 'page size allocation unit'), + Stat('file_bulk_loaded', 'bulk-loaded entries'), + Stat('file_col_deleted', 'column-store deleted values'), + Stat('file_col_fix_pages', 'column-store fixed-size leaf pages'), + Stat('file_col_int_pages', 'column-store internal pages'), + Stat('file_col_var_pages', 'column-store variable-size leaf pages'), + Stat('file_entries', 'total entries'), + Stat('file_fixed_len', 'fixed-record size'), + Stat('file_freelist_bytes', 'number of bytes in the freelist'), + Stat('file_freelist_entries', 'number of entries in the freelist'), + Stat('file_magic', 'magic number'), + Stat('file_major', 'major version number'), + Stat('file_maxintlitem', 'maximum internal page item size'), + Stat('file_maxintlpage', 'maximum internal page size'), + Stat('file_maxleafitem', 'maximum leaf page item size'), + Stat('file_maxleafpage', 'maximum leaf page size'), + Stat('file_minor', 'minor version number'), + Stat('file_overflow', 'overflow pages'), + Stat('file_row_int_pages', 'row-store internal pages'), + Stat('file_row_leaf_pages', 'row-store leaf pages'), + Stat('file_size', 'file: size'), + Stat('free', 'file: block frees'), + Stat('overflow_read', 'file: overflow pages read from the file'), + Stat('page_read', 'file: pages read from the file'), + Stat('page_write', 'file: pages written to the file'), + Stat('rec_hazard', 'reconcile: unable to acquire hazard reference'), + Stat('rec_ovfl_key', 'reconcile: overflow key'), + Stat('rec_ovfl_value', 'reconcile: overflow value'), + Stat('rec_page_delete', 'reconcile: pages deleted'), + Stat('rec_page_merge', 'reconcile: deleted or temporary pages merged'), + Stat('rec_split_intl', 'reconcile: internal pages split'), + Stat('rec_split_leaf', 'reconcile: leaf pages split'), + Stat('rec_written', 'reconcile: pages written'), +] diff --git a/docs/Doxyfile b/docs/Doxyfile new file mode 100644 index 00000000000..0f1dfda9ed5 --- /dev/null +++ b/docs/Doxyfile @@ -0,0 +1,1781 @@ +# Doxyfile 1.7.4 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project +# +# All text after a hash (#) is considered a comment and will be ignored +# The format is: +# TAG = value [value, ...] +# For lists items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (" ") + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the config file +# that follow. The default is UTF-8 which is also the encoding used for all +# text before the first occurrence of this tag. Doxygen uses libiconv (or the +# iconv built into libc) for the transcoding. See +# http://www.gnu.org/software/libiconv for the list of possible encodings. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded +# by quotes) that should identify the project. + +PROJECT_NAME = WiredTiger + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. +# This could be handy for archiving the generated documentation or +# if some version control system is used. + +PROJECT_NUMBER = "Version 1.0" + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer +# a quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = + +# With the PROJECT_LOGO tag one can specify an logo or icon that is +# included in the documentation. The maximum height of the logo should not +# exceed 55 pixels and the maximum width should not exceed 200 pixels. +# Doxygen will copy the logo to the output directory. + +PROJECT_LOGO = images/LogoFinal-header.png + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) +# base path where the generated documentation will be put. +# If a relative path is entered, it will be relative to the location +# where doxygen was started. If left blank the current directory will be used. + +OUTPUT_DIRECTORY = + +# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create +# 4096 sub-directories (in 2 levels) under the output directory of each output +# format and will distribute the generated files over these directories. +# Enabling this option can be useful when feeding doxygen a huge amount of +# source files, where putting all generated files in the same directory would +# otherwise cause performance problems for the file system. + +CREATE_SUBDIRS = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# The default language is English, other supported languages are: +# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, +# Croatian, Czech, Danish, Dutch, Esperanto, Farsi, Finnish, French, German, +# Greek, Hungarian, Italian, Japanese, Japanese-en (Japanese with English +# messages), Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian, +# Polish, Portuguese, Romanian, Russian, Serbian, Serbian-Cyrillic, Slovak, +# Slovene, Spanish, Swedish, Ukrainian, and Vietnamese. + +OUTPUT_LANGUAGE = English + +# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will +# include brief member descriptions after the members that are listed in +# the file and class documentation (similar to JavaDoc). +# Set to NO to disable this. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend +# the brief description of a member or function before the detailed description. +# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator +# that is used to form the text in various listings. Each string +# in this list, if found as the leading text of the brief description, will be +# stripped from the text and the result after processing the whole list, is +# used as the annotated text. Otherwise, the brief description is used as-is. +# If left blank, the following values are used ("$name" is automatically +# replaced with the name of the entity): "The $name class" "The $name widget" +# "The $name file" "is" "provides" "specifies" "contains" +# "represents" "a" "an" "the" + +ABBREVIATE_BRIEF = "The $name class" \ + "The $name widget" \ + "The $name file" \ + is \ + provides \ + specifies \ + contains \ + represents \ + a \ + an \ + the + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# Doxygen will generate a detailed section even if there is only a brief +# description. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full +# path before files name in the file list and in the header files. If set +# to NO the shortest path that makes the file name unique will be used. + +FULL_PATH_NAMES = NO + +# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag +# can be used to strip a user-defined part of the path. Stripping is +# only done if one of the specified strings matches the left-hand part of +# the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the +# path to strip. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of +# the path mentioned in the documentation of a class, which tells +# the reader which header file to include in order to use a class. +# If left blank only the name of the header file containing the class +# definition is used. Otherwise one should specify the include paths that +# are normally passed to the compiler using the -I flag. + +STRIP_FROM_INC_PATH = ../src/include/ + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter +# (but less readable) file names. This can be useful if your file system +# doesn't support long names like on DOS, Mac, or CD-ROM. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen +# will interpret the first line (until the first dot) of a JavaDoc-style +# comment as the brief description. If set to NO, the JavaDoc +# comments will behave just like regular Qt-style comments +# (thus requiring an explicit @brief command for a brief description.) + +JAVADOC_AUTOBRIEF = YES + +# If the QT_AUTOBRIEF tag is set to YES then Doxygen will +# interpret the first line (until the first dot) of a Qt-style +# comment as the brief description. If set to NO, the comments +# will behave just like regular Qt-style comments (thus requiring +# an explicit \brief command for a brief description.) + +QT_AUTOBRIEF = YES + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen +# treat a multi-line C++ special comment block (i.e. a block of //! or /// +# comments) as a brief description. This used to be the default behaviour. +# The new default is to treat a multi-line C++ comment block as a detailed +# description. Set this tag to YES if you prefer the old behaviour instead. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented +# member inherits the documentation from any documented member that it +# re-implements. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce +# a new page for each member. If set to NO, the documentation of a member will +# be part of the file/class/namespace that contains it. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. +# Doxygen uses this value to replace tabs by spaces in code fragments. + +TAB_SIZE = 8 + +# This tag can be used to specify a number of aliases that acts +# as commands in the documentation. An alias has the form "name=value". +# For example adding "sideeffect=\par Side Effects:\n" will allow you to +# put the command \sideeffect (or @sideeffect) in the documentation, which +# will result in a user-defined paragraph with heading "Side Effects:". +# You can put \n's in the value part of an alias to insert newlines. + +ALIASES = "notyet{1}=<b>Not yet supported in WiredTiger.</b>\n@todo fix when \1 supported\n\n" \ + "errors=@returns zero on success and a non-zero error code on failure. See @ref error_returns \"Error Returns\" for details." \ + "ex_ref{1}=@ref \1 \"\1\", available in the source tree as \c examples/c/\1" \ + "hrow{1}=<tr><th>\1</th></tr>" \ + "hrow{2}=<tr><th>\1</th><th>\2</th></tr>" \ + "hrow{3}=<tr><th>\1</th><th>\2</th><th>\3</th></tr>" \ + "hrow{4}=<tr><th>\1</th><th>\2</th><th>\3</th><th>\4</th></tr>" \ + "hrow{5}=<tr><th>\1</th><th>\2</th><th>\3</th><th>\4</th><th>\5</th></tr>" \ + "row{1}=<tr><td>\1</td></tr>" \ + "row{2}=<tr><td>\1</td><td>\2</td></tr>" \ + "row{3}=<tr><td>\1</td><td>\2</td><td>\3</td></tr>" \ + "row{4}=<tr><td>\1</td><td>\2</td><td>\3</td><td>\4</td></tr>" \ + "row{5}=<tr><td>\1</td><td>\2</td><td>\3</td><td>\4</td><td>\5</td></tr>" \ + "configstart{2}=@param config\n Configuration string, see @ref config_strings. Permitted values:\n <table>@hrow{Name,Effect,Values}" \ + "config{3}= @row{<tt>\1</tt>,\2,\3}" \ + "configend= </table>" \ + "configempty{2}=@param config\n Configuration string, see @ref config_strings. No values currently permitted." + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C +# sources only. Doxygen will then generate output that is more tailored for C. +# For instance, some of the names that are used will be different. The list +# of all members will be omitted, etc. + +OPTIMIZE_OUTPUT_FOR_C = NO + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java +# sources only. Doxygen will then generate output that is more tailored for +# Java. For instance, namespaces will be presented as packages, qualified +# scopes will look different, etc. + +OPTIMIZE_OUTPUT_JAVA = YES + +# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran +# sources only. Doxygen will then generate output that is more tailored for +# Fortran. + +OPTIMIZE_FOR_FORTRAN = NO + +# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL +# sources. Doxygen will then generate output that is tailored for +# VHDL. + +OPTIMIZE_OUTPUT_VHDL = NO + +# Doxygen selects the parser to use depending on the extension of the files it +# parses. With this tag you can assign which parser to use for a given extension. +# Doxygen has a built-in mapping, but you can override or extend it using this +# tag. The format is ext=language, where ext is a file extension, and language +# is one of the parsers supported by doxygen: IDL, Java, Javascript, CSharp, C, +# C++, D, PHP, Objective-C, Python, Fortran, VHDL, C, C++. For instance to make +# doxygen treat .inc files as Fortran files (default is PHP), and .f files as C +# (default is Fortran), use: inc=Fortran f=C. Note that for custom extensions +# you also need to set FILE_PATTERNS otherwise the files are not read by doxygen. + +EXTENSION_MAPPING = + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want +# to include (a tag file for) the STL sources as input, then you should +# set this tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. +# func(std::string) {}). This also makes the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. + +BUILTIN_STL_SUPPORT = NO + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only. +# Doxygen will parse them like normal C++ but will assume all classes use public +# instead of private inheritance when no explicit protection keyword is present. + +SIP_SUPPORT = NO + +# For Microsoft's IDL there are propget and propput attributes to indicate getter +# and setter methods for a property. Setting this option to YES (the default) +# will make doxygen replace the get and set methods by a property in the +# documentation. This will only work if the methods are indeed getting or +# setting a simple type. If this is not the case, or you want to show the +# methods anyway, you should set this option to NO. + +IDL_PROPERTY_SUPPORT = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES, then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. + +DISTRIBUTE_GROUP_DOC = NO + +# Set the SUBGROUPING tag to YES (the default) to allow class member groups of +# the same type (for instance a group of public functions) to be put as a +# subgroup of that type (e.g. under the Public Functions section). Set it to +# NO to prevent subgrouping. Alternatively, this can be done per class using +# the \nosubgrouping command. + +SUBGROUPING = YES + +# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and +# unions are shown inside the group in which they are included (e.g. using +# @ingroup) instead of on a separate page (for HTML and Man pages) or +# section (for LaTeX and RTF). + +INLINE_GROUPED_CLASSES = NO + +# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum +# is documented as struct, union, or enum with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically +# be useful for C code in case the coding convention dictates that all compound +# types are typedef'ed and only the typedef is referenced, never the tag name. + +TYPEDEF_HIDES_STRUCT = YES + +# The SYMBOL_CACHE_SIZE determines the size of the internal cache use to +# determine which symbols to keep in memory and which to flush to disk. +# When the cache is full, less often used symbols will be written to disk. +# For small to medium size projects (<1000 input files) the default value is +# probably good enough. For larger projects a too small cache size can cause +# doxygen to be busy swapping symbols to and from disk most of the time +# causing a significant performance penalty. +# If the system has enough physical memory increasing the cache will improve the +# performance by keeping more symbols in memory. Note that the value works on +# a logarithmic scale so increasing the size by one will roughly double the +# memory usage. The cache size is given by this formula: +# 2^(16+SYMBOL_CACHE_SIZE). The valid range is 0..9, the default is 0, +# corresponding to a cache size of 2^16 = 65536 symbols + +SYMBOL_CACHE_SIZE = 0 + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in +# documentation are documented, even if no documentation was available. +# Private class members and static file members will be hidden unless +# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES + +EXTRACT_ALL = NO + +# If the EXTRACT_PRIVATE tag is set to YES all private members of a class +# will be included in the documentation. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_STATIC tag is set to YES all static members of a file +# will be included in the documentation. + +EXTRACT_STATIC = NO + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) +# defined locally in source files will be included in the documentation. +# If set to NO only classes defined in header files are included. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. When set to YES local +# methods, which are defined in the implementation section but not in +# the interface are included in the documentation. +# If set to NO (the default) only methods in the interface are included. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be +# extracted and appear in the documentation as a namespace called +# 'anonymous_namespace{file}', where file will be replaced with the base +# name of the file that contains the anonymous namespace. By default +# anonymous namespaces are hidden. + +EXTRACT_ANON_NSPACES = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all +# undocumented members of documented classes, files or namespaces. +# If set to NO (the default) these members will be included in the +# various overviews, but no documentation section is generated. +# This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. +# If set to NO (the default) these classes will be included in the various +# overviews. This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all +# friend (class|struct|union) declarations. +# If set to NO (the default) these declarations will be included in the +# documentation. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any +# documentation blocks found inside the body of a function. +# If set to NO (the default) these blocks will be appended to the +# function's detailed documentation block. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation +# that is typed after a \internal command is included. If the tag is set +# to NO (the default) then the documentation will be excluded. +# Set it to YES to include the internal documentation. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate +# file names in lower-case letters. If set to YES upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. + +CASE_SENSE_NAMES = NO + +# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen +# will show members with their full class and namespace scopes in the +# documentation. If set to YES the scope will be hidden. + +HIDE_SCOPE_NAMES = NO + +# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen +# will put a list of the files that are included by a file in the documentation +# of that file. + +SHOW_INCLUDE_FILES = NO + +# If the FORCE_LOCAL_INCLUDES tag is set to YES then Doxygen +# will list include files with double quotes in the documentation +# rather than with sharp brackets. + +FORCE_LOCAL_INCLUDES = NO + +# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] +# is inserted in the documentation for inline members. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen +# will sort the (detailed) documentation of file and class members +# alphabetically by member name. If set to NO the members will appear in +# declaration order. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the +# brief documentation of file, namespace and class members alphabetically +# by member name. If set to NO (the default) the members will appear in +# declaration order. + +SORT_BRIEF_DOCS = NO + +# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen +# will sort the (brief and detailed) documentation of class members so that +# constructors and destructors are listed first. If set to NO (the default) +# the constructors will appear in the respective orders defined by +# SORT_MEMBER_DOCS and SORT_BRIEF_DOCS. +# This tag will be ignored for brief docs if SORT_BRIEF_DOCS is set to NO +# and ignored for detailed docs if SORT_MEMBER_DOCS is set to NO. + +SORT_MEMBERS_CTORS_1ST = NO + +# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the +# hierarchy of group names into alphabetical order. If set to NO (the default) +# the group names will appear in their defined order. + +SORT_GROUP_NAMES = YES + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be +# sorted by fully-qualified names, including namespaces. If set to +# NO (the default), the class list will be sorted only by class name, +# not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the +# alphabetical list. + +SORT_BY_SCOPE_NAME = YES + +# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to +# do proper type resolution of all parameters of a function it will reject a +# match between the prototype and the implementation of a member function even +# if there is only one candidate or it is obvious which candidate to choose +# by doing a simple string match. By disabling STRICT_PROTO_MATCHING doxygen +# will still accept a match between prototype and implementation in such cases. + +STRICT_PROTO_MATCHING = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or +# disable (NO) the todo list. This list is created by putting \todo +# commands in the documentation. + +GENERATE_TODOLIST = NO + +# The GENERATE_TESTLIST tag can be used to enable (YES) or +# disable (NO) the test list. This list is created by putting \test +# commands in the documentation. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or +# disable (NO) the bug list. This list is created by putting \bug +# commands in the documentation. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or +# disable (NO) the deprecated list. This list is created by putting +# \deprecated commands in the documentation. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional +# documentation sections, marked by \if sectionname ... \endif. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines +# the initial value of a variable or macro consists of for it to appear in +# the documentation. If the initializer consists of more lines than specified +# here it will be hidden. Use a value of 0 to hide initializers completely. +# The appearance of the initializer of individual variables and macros in the +# documentation can be controlled using \showinitializer or \hideinitializer +# command in the documentation regardless of this setting. + +MAX_INITIALIZER_LINES = 0 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated +# at the bottom of the documentation of classes and structs. If set to YES the +# list will mention the files that were used to generate the documentation. + +SHOW_USED_FILES = NO + +# If the sources in your project are distributed over multiple directories +# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy +# in the documentation. The default is NO. + +SHOW_DIRECTORIES = NO + +# Set the SHOW_FILES tag to NO to disable the generation of the Files page. +# This will remove the Files entry from the Quick Index and from the +# Folder Tree View (if specified). The default is YES. + +SHOW_FILES = NO + +# Set the SHOW_NAMESPACES tag to NO to disable the generation of the +# Namespaces page. This will remove the Namespaces entry from the Quick Index +# and from the Folder Tree View (if specified). The default is YES. + +SHOW_NAMESPACES = NO + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from +# the version control system). Doxygen will invoke the program by executing (via +# popen()) the command <command> <input-file>, where <command> is the value of +# the FILE_VERSION_FILTER tag, and <input-file> is the name of an input file +# provided by doxygen. Whatever the program writes to standard output +# is used as the file version. See the manual for examples. + +FILE_VERSION_FILTER = + +# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed +# by doxygen. The layout file controls the global structure of the generated +# output files in an output format independent way. The create the layout file +# that represents doxygen's defaults, run doxygen with the -l option. +# You can optionally specify a file name after the option, if omitted +# DoxygenLayout.xml will be used as the name of the layout file. + +LAYOUT_FILE = style/DoxygenLayout.xml + +#--------------------------------------------------------------------------- +# configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated +# by doxygen. Possible values are YES and NO. If left blank NO is used. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated by doxygen. Possible values are YES and NO. If left blank +# NO is used. + +WARNINGS = YES + +# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings +# for undocumented members. If EXTRACT_ALL is set to YES then this flag will +# automatically be disabled. + +WARN_IF_UNDOCUMENTED = YES + +# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some +# parameters in a documented function, or documenting parameters that +# don't exist or using markup commands wrongly. + +WARN_IF_DOC_ERROR = YES + +# The WARN_NO_PARAMDOC option can be enabled to get warnings for +# functions that are documented, but have no documentation for their parameters +# or return value. If set to NO (the default) doxygen will only warn about +# wrong or incomplete parameter documentation, but not about the absence of +# documentation. + +WARN_NO_PARAMDOC = YES + +# The WARN_FORMAT tag determines the format of the warning messages that +# doxygen can produce. The string should contain the $file, $line, and $text +# tags, which will be replaced by the file and line number from which the +# warning originated and the warning text. Optionally the format may contain +# $version, which will be replaced by the version of the file (if it could +# be obtained via FILE_VERSION_FILTER) + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning +# and error messages should be written. If left blank the output is written +# to stderr. + +WARN_LOGFILE = doxygen.log + +#--------------------------------------------------------------------------- +# configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag can be used to specify the files and/or directories that contain +# documented source files. You may enter file names like "myfile.cpp" or +# directories like "/usr/src/myproject". Separate the files or directories +# with spaces. + +INPUT = ../src/include/wiredtiger.in \ + ../src/include/wiredtiger_ext.h \ + src + +# This tag can be used to specify the character encoding of the source files +# that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is +# also the default input encoding. Doxygen uses libiconv (or the iconv built +# into libc) for the transcoding. See http://www.gnu.org/software/libiconv for +# the list of possible encodings. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank the following patterns are tested: +# *.c *.cc *.cxx *.cpp *.c++ *.d *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh +# *.hxx *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.dox *.py +# *.f90 *.f *.for *.vhd *.vhdl + +FILE_PATTERNS = *.c \ + *.cc \ + *.cxx \ + *.cpp \ + *.c++ \ + *.d \ + *.java \ + *.ii \ + *.ixx \ + *.ipp \ + *.i++ \ + *.inl \ + *.h \ + *.hh \ + *.hxx \ + *.hpp \ + *.h++ \ + *.idl \ + *.odl \ + *.cs \ + *.php \ + *.php3 \ + *.inc \ + *.m \ + *.mm \ + *.dox \ + *.py \ + *.f90 \ + *.f \ + *.vhd \ + *.vhdl + +# The RECURSIVE tag can be used to turn specify whether or not subdirectories +# should be searched for input files as well. Possible values are YES and NO. +# If left blank NO is used. + +RECURSIVE = YES + +# The EXCLUDE tag can be used to specify files and/or directories that should +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. + +EXCLUDE = \ + src/architecture.dox \ + src/bdb-map.dox \ + src/design.dox \ + src/processes.dox \ + src/sql-map.dox + +# The EXCLUDE_SYMLINKS tag can be used select whether or not files or +# directories that are symbolic links (a Unix file system feature) are excluded +# from the input. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. Note that the wildcards are matched +# against the file with absolute path, so to exclude all test directories +# for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test + +EXCLUDE_SYMBOLS = __F + +# The EXAMPLE_PATH tag can be used to specify one or more files or +# directories that contain example code fragments that are included (see +# the \include command). + +EXAMPLE_PATH = ../examples/c + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank all files are included. + +EXAMPLE_PATTERNS = + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude +# commands irrespective of the value of the RECURSIVE tag. +# Possible values are YES and NO. If left blank NO is used. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or +# directories that contain image that are included in the documentation (see +# the \image command). + +IMAGE_PATH = images + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command <filter> <input-file>, where <filter> +# is the value of the INPUT_FILTER tag, and <input-file> is the name of an +# input file. Doxygen will then use the output that the filter program writes +# to standard output. If FILTER_PATTERNS is specified, this tag will be +# ignored. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: +# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further +# info on how filters are used. If FILTER_PATTERNS is empty or if +# non of the patterns match the file name, INPUT_FILTER is applied. + +FILTER_PATTERNS = *.py=tools/pyfilter + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will be used to filter the input files when producing source +# files to browse (i.e. when SOURCE_BROWSER is set to YES). + +FILTER_SOURCE_FILES = NO + +# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file +# pattern. A pattern will override the setting for FILTER_PATTERN (if any) +# and it is also possible to disable source filtering for a specific pattern +# using *.ext= (so without naming a filter). This option only has effect when +# FILTER_SOURCE_FILES is enabled. + +FILTER_SOURCE_PATTERNS = + +#--------------------------------------------------------------------------- +# configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will +# be generated. Documented entities will be cross-referenced with these sources. +# Note: To get rid of all source code in the generated output, make sure also +# VERBATIM_HEADERS is set to NO. + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body +# of functions and classes directly in the documentation. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct +# doxygen to hide any special comment blocks from generated source code +# fragments. Normal C and C++ comments will always remain visible. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES +# then for each documented function all documented +# functions referencing it will be listed. + +REFERENCED_BY_RELATION = NO + +# If the REFERENCES_RELATION tag is set to YES +# then for each documented function all documented entities +# called/used by that function will be listed. + +REFERENCES_RELATION = NO + +# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) +# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from +# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will +# link to the source code. Otherwise they will link to the documentation. + +REFERENCES_LINK_SOURCE = NO + +# If the USE_HTAGS tag is set to YES then the references to source code +# will point to the HTML generated by the htags(1) tool instead of doxygen +# built-in source browser. The htags tool is part of GNU's global source +# tagging system (see http://www.gnu.org/software/global/global.html). You +# will need version 4.8.6 or higher. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen +# will generate a verbatim copy of the header file for each class for +# which an include is specified. Set to NO to disable this. + +VERBATIM_HEADERS = NO + +#--------------------------------------------------------------------------- +# configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index +# of all compounds will be generated. Enable this if the project +# contains a lot of classes, structs, unions or interfaces. + +ALPHABETICAL_INDEX = NO + +# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then +# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns +# in which this list will be split (can be a number in the range [1..20]) + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all +# classes will be put under the same header in the alphabetical index. +# The IGNORE_PREFIX tag can be used to specify one or more prefixes that +# should be ignored while generating the index headers. + +IGNORE_PREFIX = WT_ + +#--------------------------------------------------------------------------- +# configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES (the default) Doxygen will +# generate HTML output. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `html' will be used as the default path. + +HTML_OUTPUT = . + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for +# each generated HTML page (for example: .htm,.php,.asp). If it is left blank +# doxygen will generate files with .html extension. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a personal HTML header for +# each generated HTML page. If it is left blank doxygen will generate a +# standard header. Note that when using a custom header you are responsible +# for the proper inclusion of any scripts and style sheets that doxygen +# needs, which is dependent on the configuration options used. +# It is adviced to generate a default header using "doxygen -w html +# header.html footer.html stylesheet.css YourConfigFile" and then modify +# that header. Note that the header is subject to change so you typically +# have to redo this when upgrading to a newer version of doxygen or when +# changing the value of configuration settings such as GENERATE_TREEVIEW! + +HTML_HEADER = style/header.html + +# The HTML_FOOTER tag can be used to specify a personal HTML footer for +# each generated HTML page. If it is left blank doxygen will generate a +# standard footer. + +HTML_FOOTER = style/footer.html + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading +# style sheet that is used by each HTML page. It can be used to +# fine-tune the look of the HTML output. If the tag is left blank doxygen +# will generate a default style sheet. Note that doxygen will try to copy +# the style sheet file to the HTML output directory, so don't put your own +# stylesheet in the HTML output directory as well, or it will be erased! + +HTML_STYLESHEET = style/wiredtiger.css + +# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the HTML output directory. Note +# that these files will be copied to the base HTML output directory. Use the +# $relpath$ marker in the HTML_HEADER and/or HTML_FOOTER files to load these +# files. In the HTML_STYLESHEET file, use the file name only. Also note that +# the files will be copied as-is; there are no commands or markers available. + +HTML_EXTRA_FILES = + +# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. +# Doxygen will adjust the colors in the stylesheet and background images +# according to this color. Hue is specified as an angle on a colorwheel, +# see http://en.wikipedia.org/wiki/Hue for more information. +# For instance the value 0 represents red, 60 is yellow, 120 is green, +# 180 is cyan, 240 is blue, 300 purple, and 360 is red again. +# The allowed range is 0 to 359. + +HTML_COLORSTYLE_HUE = 34 + +# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of +# the colors in the HTML output. For a value of 0 the output will use +# grayscales only. A value of 255 will produce the most vivid colors. + +HTML_COLORSTYLE_SAT = 81 + +# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to +# the luminance component of the colors in the HTML output. Values below +# 100 gradually make the output lighter, whereas values above 100 make +# the output darker. The value divided by 100 is the actual gamma applied, +# so 80 represents a gamma of 0.8, The value 220 represents a gamma of 2.2, +# and 100 does not change the gamma. + +HTML_COLORSTYLE_GAMMA = 96 + +# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML +# page will contain the date and time when the page was generated. Setting +# this to NO can help when comparing the output of multiple runs. + +HTML_TIMESTAMP = YES + +# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, +# files or namespaces will be aligned in HTML using tables. If set to +# NO a bullet list will be used. + +HTML_ALIGN_MEMBERS = YES + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. For this to work a browser that supports +# JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox +# Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari). + +HTML_DYNAMIC_SECTIONS = YES + +# If the GENERATE_DOCSET tag is set to YES, additional index files +# will be generated that can be used as input for Apple's Xcode 3 +# integrated development environment, introduced with OSX 10.5 (Leopard). +# To create a documentation set, doxygen will generate a Makefile in the +# HTML output directory. Running make will produce the docset in that +# directory and running "make install" will install the docset in +# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find +# it at startup. +# See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html +# for more information. + +GENERATE_DOCSET = NO + +# When GENERATE_DOCSET tag is set to YES, this tag determines the name of the +# feed. A documentation feed provides an umbrella under which multiple +# documentation sets from a single provider (such as a company or product suite) +# can be grouped. + +DOCSET_FEEDNAME = "Doxygen generated docs" + +# When GENERATE_DOCSET tag is set to YES, this tag specifies a string that +# should uniquely identify the documentation set bundle. This should be a +# reverse domain-name style string, e.g. com.mycompany.MyDocSet. Doxygen +# will append .docset to the name. + +DOCSET_BUNDLE_ID = org.doxygen.Project + +# When GENERATE_PUBLISHER_ID tag specifies a string that should uniquely identify +# the documentation publisher. This should be a reverse domain-name style +# string, e.g. com.mycompany.MyDocSet.documentation. + +DOCSET_PUBLISHER_ID = org.doxygen.Publisher + +# The GENERATE_PUBLISHER_NAME tag identifies the documentation publisher. + +DOCSET_PUBLISHER_NAME = Publisher + +# If the GENERATE_HTMLHELP tag is set to YES, additional index files +# will be generated that can be used as input for tools like the +# Microsoft HTML help workshop to generate a compiled HTML help file (.chm) +# of the generated HTML documentation. + +GENERATE_HTMLHELP = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can +# be used to specify the file name of the resulting .chm file. You +# can add a path in front of the file if the result should not be +# written to the html output directory. + +CHM_FILE = + +# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can +# be used to specify the location (absolute path including file name) of +# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run +# the HTML help compiler on the generated index.hhp. + +HHC_LOCATION = + +# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag +# controls if a separate .chi index file is generated (YES) or that +# it should be included in the master .chm file (NO). + +GENERATE_CHI = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_INDEX_ENCODING +# is used to encode HtmlHelp index (hhk), content (hhc) and project file +# content. + +CHM_INDEX_ENCODING = + +# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag +# controls whether a binary table of contents is generated (YES) or a +# normal table of contents (NO) in the .chm file. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members +# to the contents of the HTML help documentation and to the tree view. + +TOC_EXPAND = NO + +# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and +# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated +# that can be used as input for Qt's qhelpgenerator to generate a +# Qt Compressed Help (.qch) of the generated HTML documentation. + +GENERATE_QHP = NO + +# If the QHG_LOCATION tag is specified, the QCH_FILE tag can +# be used to specify the file name of the resulting .qch file. +# The path specified is relative to the HTML output folder. + +QCH_FILE = + +# The QHP_NAMESPACE tag specifies the namespace to use when generating +# Qt Help Project output. For more information please see +# http://doc.trolltech.com/qthelpproject.html#namespace + +QHP_NAMESPACE = org.doxygen.Project + +# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating +# Qt Help Project output. For more information please see +# http://doc.trolltech.com/qthelpproject.html#virtual-folders + +QHP_VIRTUAL_FOLDER = doc + +# If QHP_CUST_FILTER_NAME is set, it specifies the name of a custom filter to +# add. For more information please see +# http://doc.trolltech.com/qthelpproject.html#custom-filters + +QHP_CUST_FILTER_NAME = + +# The QHP_CUST_FILT_ATTRS tag specifies the list of the attributes of the +# custom filter to add. For more information please see +# <a href="http://doc.trolltech.com/qthelpproject.html#custom-filters"> +# Qt Help Project / Custom Filters</a>. + +QHP_CUST_FILTER_ATTRS = + +# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this +# project's +# filter section matches. +# <a href="http://doc.trolltech.com/qthelpproject.html#filter-attributes"> +# Qt Help Project / Filter Attributes</a>. + +QHP_SECT_FILTER_ATTRS = + +# If the GENERATE_QHP tag is set to YES, the QHG_LOCATION tag can +# be used to specify the location of Qt's qhelpgenerator. +# If non-empty doxygen will try to run qhelpgenerator on the generated +# .qhp file. + +QHG_LOCATION = + +# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files +# will be generated, which together with the HTML files, form an Eclipse help +# plugin. To install this plugin and make it available under the help contents +# menu in Eclipse, the contents of the directory containing the HTML and XML +# files needs to be copied into the plugins directory of eclipse. The name of +# the directory within the plugins directory should be the same as +# the ECLIPSE_DOC_ID value. After copying Eclipse needs to be restarted before +# the help appears. + +GENERATE_ECLIPSEHELP = NO + +# A unique identifier for the eclipse help plugin. When installing the plugin +# the directory name containing the HTML and XML files should also have +# this name. + +ECLIPSE_DOC_ID = org.doxygen.Project + +# The DISABLE_INDEX tag can be used to turn on/off the condensed index at +# top of each HTML page. The value NO (the default) enables the index and +# the value YES disables it. + +DISABLE_INDEX = NO + +# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values +# (range [0,1..20]) that doxygen will group on one line in the generated HTML +# documentation. Note that a value of 0 will completely suppress the enum +# values from appearing in the overview section. + +ENUM_VALUES_PER_LINE = 4 + +# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index +# structure should be generated to display hierarchical information. +# If the tag value is set to YES, a side panel will be generated +# containing a tree-like index structure (just like the one that +# is generated for HTML Help). For this to work a browser that supports +# JavaScript, DHTML, CSS and frames is required (i.e. any modern browser). +# Windows users are probably better off using the HTML help feature. + +GENERATE_TREEVIEW = YES + +# By enabling USE_INLINE_TREES, doxygen will generate the Groups, Directories, +# and Class Hierarchy pages using a tree view instead of an ordered list. + +USE_INLINE_TREES = YES + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be +# used to set the initial width (in pixels) of the frame in which the tree +# is shown. + +TREEVIEW_WIDTH = 200 + +# When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open +# links to external symbols imported via tag files in a separate window. + +EXT_LINKS_IN_WINDOW = NO + +# Use this tag to change the font size of Latex formulas included +# as images in the HTML documentation. The default is 10. Note that +# when you change the font size after a successful doxygen run you need +# to manually remove any form_*.png images from the HTML output directory +# to force them to be regenerated. + +FORMULA_FONTSIZE = 10 + +# Use the FORMULA_TRANPARENT tag to determine whether or not the images +# generated for formulas are transparent PNGs. Transparent PNGs are +# not supported properly for IE 6.0, but are supported on all modern browsers. +# Note that when changing this option you need to delete any form_*.png files +# in the HTML output before the changes have effect. + +FORMULA_TRANSPARENT = YES + +# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax +# (see http://www.mathjax.org) which uses client side Javascript for the +# rendering instead of using prerendered bitmaps. Use this if you do not +# have LaTeX installed or if you want to formulas look prettier in the HTML +# output. When enabled you also need to install MathJax separately and +# configure the path to it using the MATHJAX_RELPATH option. + +USE_MATHJAX = NO + +# When MathJax is enabled you need to specify the location relative to the +# HTML output directory using the MATHJAX_RELPATH option. The destination +# directory should contain the MathJax.js script. For instance, if the mathjax +# directory is located at the same level as the HTML output directory, then +# MATHJAX_RELPATH should be ../mathjax. The default value points to the +# mathjax.org site, so you can quickly see the result without installing +# MathJax, but it is strongly recommended to install a local copy of MathJax +# before deployment. + +MATHJAX_RELPATH = http://www.mathjax.org/mathjax + +# When the SEARCHENGINE tag is enabled doxygen will generate a search box +# for the HTML output. The underlying search engine uses javascript +# and DHTML and should work on any modern browser. Note that when using +# HTML help (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets +# (GENERATE_DOCSET) there is already a search function so this one should +# typically be disabled. For large projects the javascript based search engine +# can be slow, then enabling SERVER_BASED_SEARCH may provide a better solution. + +SEARCHENGINE = NO + +# When the SERVER_BASED_SEARCH tag is enabled the search engine will be +# implemented using a PHP enabled web server instead of at the web client +# using Javascript. Doxygen will generate the search PHP script and index +# file to put on the web server. The advantage of the server +# based approach is that it scales better to large projects and allows +# full text search. The disadvantages are that it is more difficult to setup +# and does not have live searching capabilities. + +SERVER_BASED_SEARCH = NO + +#--------------------------------------------------------------------------- +# configuration options related to the LaTeX output +#--------------------------------------------------------------------------- + +# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will +# generate Latex output. + +GENERATE_LATEX = NO + +# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `latex' will be used as the default path. + +LATEX_OUTPUT = latex + +# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be +# invoked. If left blank `latex' will be used as the default command name. +# Note that when enabling USE_PDFLATEX this option is only used for +# generating bitmaps for formulas in the HTML output, but not in the +# Makefile that is written to the output directory. + +LATEX_CMD_NAME = latex + +# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to +# generate index for LaTeX. If left blank `makeindex' will be used as the +# default command name. + +MAKEINDEX_CMD_NAME = makeindex + +# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact +# LaTeX documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_LATEX = NO + +# The PAPER_TYPE tag can be used to set the paper type that is used +# by the printer. Possible values are: a4, letter, legal and +# executive. If left blank a4wide will be used. + +PAPER_TYPE = a4wide + +# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX +# packages that should be included in the LaTeX output. + +EXTRA_PACKAGES = + +# The LATEX_HEADER tag can be used to specify a personal LaTeX header for +# the generated latex document. The header should contain everything until +# the first chapter. If it is left blank doxygen will generate a +# standard header. Notice: only use this tag if you know what you are doing! + +LATEX_HEADER = + +# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for +# the generated latex document. The footer should contain everything after +# the last chapter. If it is left blank doxygen will generate a +# standard footer. Notice: only use this tag if you know what you are doing! + +LATEX_FOOTER = + +# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated +# is prepared for conversion to pdf (using ps2pdf). The pdf file will +# contain links (just like the HTML output) instead of page references +# This makes the output suitable for online browsing using a pdf viewer. + +PDF_HYPERLINKS = YES + +# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of +# plain latex in the generated Makefile. Set this option to YES to get a +# higher quality PDF documentation. + +USE_PDFLATEX = YES + +# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. +# command to the generated LaTeX files. This will instruct LaTeX to keep +# running if errors occur, instead of asking the user for help. +# This option is also used when generating formulas in HTML. + +LATEX_BATCHMODE = NO + +# If LATEX_HIDE_INDICES is set to YES then doxygen will not +# include the index chapters (such as File Index, Compound Index, etc.) +# in the output. + +LATEX_HIDE_INDICES = YES + +# If LATEX_SOURCE_CODE is set to YES then doxygen will include +# source code with syntax highlighting in the LaTeX output. +# Note that which sources are shown also depends on other settings +# such as SOURCE_BROWSER. + +LATEX_SOURCE_CODE = YES + +#--------------------------------------------------------------------------- +# configuration options related to the RTF output +#--------------------------------------------------------------------------- + +# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output +# The RTF output is optimized for Word 97 and may not look very pretty with +# other RTF readers or editors. + +GENERATE_RTF = NO + +# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `rtf' will be used as the default path. + +RTF_OUTPUT = rtf + +# If the COMPACT_RTF tag is set to YES Doxygen generates more compact +# RTF documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_RTF = NO + +# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated +# will contain hyperlink fields. The RTF file will +# contain links (just like the HTML output) instead of page references. +# This makes the output suitable for online browsing using WORD or other +# programs which support those fields. +# Note: wordpad (write) and others do not support links. + +RTF_HYPERLINKS = NO + +# Load stylesheet definitions from file. Syntax is similar to doxygen's +# config file, i.e. a series of assignments. You only have to provide +# replacements, missing definitions are set to their default value. + +RTF_STYLESHEET_FILE = + +# Set optional variables used in the generation of an rtf document. +# Syntax is similar to doxygen's config file. + +RTF_EXTENSIONS_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES (the default) Doxygen will +# generate man pages + +GENERATE_MAN = NO + +# The MAN_OUTPUT tag is used to specify where the man pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `man' will be used as the default path. + +MAN_OUTPUT = man + +# The MAN_EXTENSION tag determines the extension that is added to +# the generated man pages (default is the subroutine's section .3) + +MAN_EXTENSION = .3 + +# If the MAN_LINKS tag is set to YES and Doxygen generates man output, +# then it will generate one additional man file for each entity +# documented in the real man page(s). These additional files +# only source the real man page, but without them the man command +# would be unable to find the correct page. The default is NO. + +MAN_LINKS = NO + +#--------------------------------------------------------------------------- +# configuration options related to the XML output +#--------------------------------------------------------------------------- + +# If the GENERATE_XML tag is set to YES Doxygen will +# generate an XML file that captures the structure of +# the code including all documentation. + +GENERATE_XML = NO + +# The XML_OUTPUT tag is used to specify where the XML pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `xml' will be used as the default path. + +XML_OUTPUT = xml + +# The XML_SCHEMA tag can be used to specify an XML schema, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_SCHEMA = + +# The XML_DTD tag can be used to specify an XML DTD, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_DTD = + +# If the XML_PROGRAMLISTING tag is set to YES Doxygen will +# dump the program listings (including syntax highlighting +# and cross-referencing information) to the XML output. Note that +# enabling this will significantly increase the size of the XML output. + +XML_PROGRAMLISTING = YES + +#--------------------------------------------------------------------------- +# configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- + +# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will +# generate an AutoGen Definitions (see autogen.sf.net) file +# that captures the structure of the code including all +# documentation. Note that this feature is still experimental +# and incomplete at the moment. + +GENERATE_AUTOGEN_DEF = NO + +#--------------------------------------------------------------------------- +# configuration options related to the Perl module output +#--------------------------------------------------------------------------- + +# If the GENERATE_PERLMOD tag is set to YES Doxygen will +# generate a Perl module file that captures the structure of +# the code including all documentation. Note that this +# feature is still experimental and incomplete at the +# moment. + +GENERATE_PERLMOD = NO + +# If the PERLMOD_LATEX tag is set to YES Doxygen will generate +# the necessary Makefile rules, Perl scripts and LaTeX code to be able +# to generate PDF and DVI output from the Perl module output. + +PERLMOD_LATEX = NO + +# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be +# nicely formatted so it can be parsed by a human reader. This is useful +# if you want to understand what is going on. On the other hand, if this +# tag is set to NO the size of the Perl module output will be much smaller +# and Perl will parse it just the same. + +PERLMOD_PRETTY = YES + +# The names of the make variables in the generated doxyrules.make file +# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. +# This is useful so different doxyrules.make files included by the same +# Makefile don't overwrite each other's variables. + +PERLMOD_MAKEVAR_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will +# evaluate all C-preprocessor directives found in the sources and include +# files. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro +# names in the source code. If set to NO (the default) only conditional +# compilation will be performed. Macro expansion can be done in a controlled +# way by setting EXPAND_ONLY_PREDEF to YES. + +MACRO_EXPANSION = YES + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES +# then the macro expansion is limited to the macros specified with the +# PREDEFINED and EXPAND_AS_DEFINED tags. + +EXPAND_ONLY_PREDEF = NO + +# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files +# pointed to by INCLUDE_PATH will be searched when a #include is found. + +SEARCH_INCLUDES = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by +# the preprocessor. + +INCLUDE_PATH = + +# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard +# patterns (like *.h and *.hpp) to filter out the header-files in the +# directories. If left blank, the patterns specified with FILE_PATTERNS will +# be used. + +INCLUDE_FILE_PATTERNS = + +# The PREDEFINED tag can be used to specify one or more macro names that +# are defined before the preprocessor is started (similar to the -D option of +# gcc). The argument of the tag is a list of macros of the form: name +# or name=definition (no spaces). If the definition and the = are +# omitted =1 is assumed. To prevent a macro definition from being +# undefined via #undef or recursively expanded use the := operator +# instead of the = operator. + +PREDEFINED = DOXYGEN \ + __wt_collator:=WT_COLLATOR \ + __wt_compressor:=WT_COMPRESSOR \ + __wt_connection:=WT_CONNECTION \ + __wt_cursor:=WT_CURSOR \ + __wt_cursor_type:=WT_CURSOR_TYPE \ + __wt_event_handler:=WT_EVENT_HANDLER \ + __wt_extension_api:=WT_EXTENSION_API \ + __wt_extractor:=WT_EXTRACTOR \ + __wt_item:=WT_ITEM \ + __wt_session:=WT_SESSION \ + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then +# this tag can be used to specify a list of macro names that should be expanded. +# The macro definition that is found in the sources will be used. +# Use the PREDEFINED tag if you want to use a different macro definition that +# overrules the definition found in the source code. + +EXPAND_AS_DEFINED = + +# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then +# doxygen's preprocessor will remove all references to function-like macros +# that are alone on a line, have an all uppercase name, and do not end with a +# semicolon, because these will confuse the parser if not removed. + +SKIP_FUNCTION_MACROS = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to external references +#--------------------------------------------------------------------------- + +# The TAGFILES option can be used to specify one or more tagfiles. +# Optionally an initial location of the external documentation +# can be added for each tagfile. The format of a tag file without +# this location is as follows: +# TAGFILES = file1 file2 ... +# Adding location for the tag files is done as follows: +# TAGFILES = file1=loc1 "file2 = loc2" ... +# where "loc1" and "loc2" can be relative or absolute paths or +# URLs. If a location is present for each tag, the installdox tool +# does not have to be run to correct the links. +# Note that each tag file must have a unique name +# (where the name does NOT include the path) +# If a tag file is not located in the directory in which doxygen +# is run, you must also specify the path to the tagfile here. + +TAGFILES = + +# When a file name is specified after GENERATE_TAGFILE, doxygen will create +# a tag file that is based on the input files it reads. + +GENERATE_TAGFILE = + +# If the ALLEXTERNALS tag is set to YES all external classes will be listed +# in the class index. If set to NO only the inherited external classes +# will be listed. + +ALLEXTERNALS = NO + +# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed +# in the modules index. If set to NO, only the current project's groups will +# be listed. + +EXTERNAL_GROUPS = YES + +# The PERL_PATH should be the absolute path and name of the perl script +# interpreter (i.e. the result of `which perl'). + +PERL_PATH = /usr/bin/perl + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will +# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base +# or super classes. Setting the tag to NO turns the diagrams off. Note that +# this option also works with HAVE_DOT disabled, but it is recommended to +# install and use dot, since it yields more powerful graphs. + +CLASS_DIAGRAMS = YES + +# You can define message sequence charts within doxygen comments using the \msc +# command. Doxygen will then run the mscgen tool (see +# http://www.mcternan.me.uk/mscgen/) to produce the chart and insert it in the +# documentation. The MSCGEN_PATH tag allows you to specify the directory where +# the mscgen tool resides. If left empty the tool is assumed to be found in the +# default search path. + +MSCGEN_PATH = + +# If set to YES, the inheritance and collaboration graphs will hide +# inheritance and usage relations if the target is undocumented +# or is not a class. + +HIDE_UNDOC_RELATIONS = YES + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz, a graph visualization +# toolkit from AT&T and Lucent Bell Labs. The other options in this section +# have no effect if this option is set to NO (the default) + +HAVE_DOT = NO + +# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is +# allowed to run in parallel. When set to 0 (the default) doxygen will +# base this on the number of processors available in the system. You can set it +# explicitly to a value larger than 0 to get control over the balance +# between CPU load and processing speed. + +DOT_NUM_THREADS = 0 + +# By default doxygen will write a font called Helvetica to the output +# directory and reference it in all dot files that doxygen generates. +# When you want a differently looking font you can specify the font name +# using DOT_FONTNAME. You need to make sure dot is able to find the font, +# which can be done by putting it in a standard location or by setting the +# DOTFONTPATH environment variable or by setting DOT_FONTPATH to the directory +# containing the font. + +DOT_FONTNAME = FreeSans.ttf + +# The DOT_FONTSIZE tag can be used to set the size of the font of dot graphs. +# The default size is 10pt. + +DOT_FONTSIZE = 10 + +# By default doxygen will tell dot to use the output directory to look for the +# FreeSans.ttf font (which doxygen will put there itself). If you specify a +# different font using DOT_FONTNAME you can set the path where dot +# can find it using this tag. + +DOT_FONTPATH = + +# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect inheritance relations. Setting this tag to YES will force the +# the CLASS_DIAGRAMS tag to NO. + +CLASS_GRAPH = YES + +# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect implementation dependencies (inheritance, containment, and +# class references variables) of the class with other documented classes. + +COLLABORATION_GRAPH = YES + +# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for groups, showing the direct groups dependencies + +GROUP_GRAPHS = YES + +# If the UML_LOOK tag is set to YES doxygen will generate inheritance and +# collaboration diagrams in a style similar to the OMG's Unified Modeling +# Language. + +UML_LOOK = NO + +# If set to YES, the inheritance and collaboration graphs will show the +# relations between templates and their instances. + +TEMPLATE_RELATIONS = NO + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT +# tags are set to YES then doxygen will generate a graph for each documented +# file showing the direct and indirect include dependencies of the file with +# other documented files. + +INCLUDE_GRAPH = YES + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and +# HAVE_DOT tags are set to YES then doxygen will generate a graph for each +# documented header file showing the documented files that directly or +# indirectly include this file. + +INCLUDED_BY_GRAPH = YES + +# If the CALL_GRAPH and HAVE_DOT options are set to YES then +# doxygen will generate a call dependency graph for every global function +# or class method. Note that enabling this option will significantly increase +# the time of a run. So in most cases it will be better to enable call graphs +# for selected functions only using the \callgraph command. + +CALL_GRAPH = NO + +# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then +# doxygen will generate a caller dependency graph for every global function +# or class method. Note that enabling this option will significantly increase +# the time of a run. So in most cases it will be better to enable caller +# graphs for selected functions only using the \callergraph command. + +CALLER_GRAPH = NO + +# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen +# will generate a graphical hierarchy of all classes instead of a textual one. + +GRAPHICAL_HIERARCHY = YES + +# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES +# then doxygen will show the dependencies a directory has on other directories +# in a graphical way. The dependency relations are determined by the #include +# relations between the files in the directories. + +DIRECTORY_GRAPH = YES + +# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images +# generated by dot. Possible values are svg, png, jpg, or gif. +# If left blank png will be used. + +DOT_IMAGE_FORMAT = png + +# The tag DOT_PATH can be used to specify the path where the dot tool can be +# found. If left blank, it is assumed the dot tool can be found in the path. + +DOT_PATH = + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the +# \dotfile command). + +DOTFILE_DIRS = + +# The MSCFILE_DIRS tag can be used to specify one or more directories that +# contain msc files that are included in the documentation (see the +# \mscfile command). + +MSCFILE_DIRS = + +# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of +# nodes that will be shown in the graph. If the number of nodes in a graph +# becomes larger than this value, doxygen will truncate the graph, which is +# visualized by representing a node as a red box. Note that doxygen if the +# number of direct children of the root node in a graph is already larger than +# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note +# that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. + +DOT_GRAPH_MAX_NODES = 50 + +# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the +# graphs generated by dot. A depth value of 3 means that only nodes reachable +# from the root by following a path via at most 3 edges will be shown. Nodes +# that lay further from the root node will be omitted. Note that setting this +# option to 1 or 2 may greatly reduce the computation time needed for large +# code bases. Also note that the size of a graph can be further restricted by +# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. + +MAX_DOT_GRAPH_DEPTH = 0 + +# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent +# background. This is disabled by default, because dot on Windows does not +# seem to support this out of the box. Warning: Depending on the platform used, +# enabling this option may lead to badly anti-aliased labels on the edges of +# a graph (i.e. they become hard to read). + +DOT_TRANSPARENT = NO + +# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output +# files in one run (i.e. multiple -o and -T options on the command line). This +# makes dot run faster, but since only newer versions of dot (>1.8.10) +# support this, this feature is disabled by default. + +DOT_MULTI_TARGETS = NO + +# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will +# generate a legend page explaining the meaning of the various boxes and +# arrows in the dot generated graphs. + +GENERATE_LEGEND = YES + +# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will +# remove the intermediate dot files that are used to generate +# the various graphs. + +DOT_CLEANUP = YES diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000000..80946074755 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,4 @@ +all: + @(cd ../dist && sh s_docs -t) + +.PHONY: all diff --git a/docs/build-javadoc.sh b/docs/build-javadoc.sh new file mode 100755 index 00000000000..c413db28d33 --- /dev/null +++ b/docs/build-javadoc.sh @@ -0,0 +1,12 @@ +DOCS=`dirname $0` +TOP=$DOCS/.. +. $TOP/config.sh + +CLASSPATH=$THRIFT_HOME/libthrift.jar:$SLF4J_JAR javadoc -public -d $DOCS/java \ + -source 1.5 \ + -sourcepath $TOP/lang/java/src:$TOP/src/server/gen-java \ + -stylesheetfile $DOCS/style/javadoc.css \ + -use -link http://java.sun.com/j2se/1.5.0/docs/api/ \ + -header '<b>WiredTiger API</b><br><font size="-1"> version '$WT_VERSION'</font>' \ + -windowtitle 'WiredTiger Java API' -bottom '<font size=1>Copyright (c) 2008-2012 WiredTiger, Inc. All rights reserved.</font>' \ + com.wiredtiger com.wiredtiger.util diff --git a/docs/build-pydoc.sh b/docs/build-pydoc.sh new file mode 100755 index 00000000000..5e6e3635be5 --- /dev/null +++ b/docs/build-pydoc.sh @@ -0,0 +1,6 @@ +DOCS=`dirname $0` +TOP=$DOCS/.. +. $TOP/config.sh + +cd python +PYTHONPATH=../../lang/python/src:$THRIFT_HOME/lib/python2.6/site-packages pydoc -w wiredtiger diff --git a/docs/images/LogoFace-watermark.png b/docs/images/LogoFace-watermark.png Binary files differnew file mode 100644 index 00000000000..7af37f88a7c --- /dev/null +++ b/docs/images/LogoFace-watermark.png diff --git a/docs/images/LogoFinal-header.png b/docs/images/LogoFinal-header.png Binary files differnew file mode 100644 index 00000000000..fcbdb27ae81 --- /dev/null +++ b/docs/images/LogoFinal-header.png diff --git a/docs/images/architecture.pdf b/docs/images/architecture.pdf Binary files differnew file mode 100644 index 00000000000..ff97ddff2b4 --- /dev/null +++ b/docs/images/architecture.pdf diff --git a/docs/images/architecture.png b/docs/images/architecture.png Binary files differnew file mode 100644 index 00000000000..c5b72bc05e0 --- /dev/null +++ b/docs/images/architecture.png diff --git a/docs/src/architecture.dox b/docs/src/architecture.dox new file mode 100644 index 00000000000..924b1364bf3 --- /dev/null +++ b/docs/src/architecture.dox @@ -0,0 +1,12 @@ +/*! @page architecture WiredTiger Architecture + +@todo explain the architecture + +@image html architecture.png +@image latex architecture.pdf "WiredTiger Architecture" height=12cm + +For more details about the WiredTiger architecture, see: + +- @subpage design +- @subpage fileformats + */ diff --git a/docs/src/basic-api.dox b/docs/src/basic-api.dox new file mode 100644 index 00000000000..d7098e9ab61 --- /dev/null +++ b/docs/src/basic-api.dox @@ -0,0 +1,115 @@ +/*! @page basic_api Getting Started with the API + +WiredTiger applications will generally use the following classes to access +and manage data: + + - a WT_CONNECTION represents a connection to a database. Most + applications will only open one connection to a database for each process. + All methods in WT_CONNECTION are thread safe. + + - a WT_SESSION represents a context in which database operations are + performed. Sessions are opened on a specified connection, and + applications must open a single session for each thread accessing the + database. + + - a WT_CURSOR represents a cursor over a collection of data. Cursors are + opened in the context of a session (which may have an associated + transaction), and can query and update records. In the common case, a + cursor is used to access records in a table. However, cursors can be used + on subsets of tables (such as a single column or a projection of multiple + columns), as an interface to statistics, configuration data or + application-specific data sources. + +Handles and operations are @ref config_strings "configured using strings", +which keeps the set of methods in the API relatively small and makes the +interface very similar regardless of the programming language used in the +application. WiredTiger supports the C, C++, Java and Python programming +languages (among others). + +By default, WiredTiger works as a traditional key/value store, where the +keys and values are raw byte arrays accessed using a WT_ITEM structure. +Keys and values may be up to (4GB - 512B) bytes in size, but depending +on how @ref WT_SESSION::create "maximum item sizes" are configured, +large key and value items will be stored on overflow pages. + +WiredTiger also supports a @ref schema "schema layer" so that keys and +values types can be chosen from a list, or composite keys or values made +up of columns with any combination of types. The size (4GB - 512B) byte +limit on keys and values still applies. + +All applications that use WiredTiger will be structured roughly as follows. +The code below is taken from the complete example program +@ex_ref{ex_access.c}. + +@section basic_connection Connecting to a database + +To access a database, first open a connection and create a session handle +for the single thread accessing the database: + +@snippet ex_access.c access example connection + +The configuration string @c "create" is passed to ::wiredtiger_open to +indicate the database should be created if it does not already exist. + +The code block above also shows simple error handling with +::wiredtiger_strerror (a function that returns a string describing an +error code passed as its argument). More complex error handling can be +configured by passing an implementation of WT_ERROR_HANDLER to +wiredtiger_open or WT_CONNECTION::open_session. + +@section basic_create_table Creating a table + +Create a table we can use to store data: + +@snippet ex_access.c access example table create + +This call creates a table called @c "access", configured to use strings +for its key and value columns. (See @ref schema for more information +on tables with other types of key and value columns.) + +@section basic_cursors Accessing data with cursors + +Now that we have a table, we open a cursor to perform some operations +on it: + +@snippet ex_access.c access example cursor open + +Here, the string @c "table:access" specifies that we are opening the +cursor on the table named @c "access". + +Then we insert a new row into the table. The WT_CURSOR::set_key and +WT_CURSOR::set_value calls put the application's data into the cursor. +The WT_CURSOR::insert call creates a record containing that data and +inserts it into the table. + +@snippet ex_access.c access example cursor insert + +Now we iterate through all of the records in the table, printing them out +as we go: + +@snippet ex_access.c access example cursor list + +Note that the key and value parts of the records are returned as C +strings because the table was created that way (even if it was created +by a previous run of the example). No data extraction or conversion is +required in the application. + +Because the cursor was positioned in the table after the WT_CURSOR::insert +call, we had to re-position it using the WT_CURSOR::first call; if we +weren't using the cursor for the call to WT_CURSOR::insert above, this loop +would simplify to: + +@code + while ((ret = cursor->next(cursor)) == 0) { + ... + } +@endcode + +@section basic_close Closing handles + +Lastly, we close the connection, which implicitly closes the cursor and +session handles: + +@snippet ex_access.c access example close + + */ diff --git a/docs/src/command-line.dox b/docs/src/command-line.dox new file mode 100644 index 00000000000..22d134c1bcb --- /dev/null +++ b/docs/src/command-line.dox @@ -0,0 +1,347 @@ +/*! @page command_line WiredTiger command line utility + +WiredTiger includes a command line utility, \c wt. + +@section Synopsis +<code>wt [-Vv] [-C config] [-h directory] command [command-specific arguments]</code> + +@section Description +The \c wt tool is a command-line utility that provides access to +various pieces of the WiredTiger functionality. + +@section Options +There are three global options: + +@par <code>-C config</code> +Specify configuration strings for the ::wiredtiger_open function. +@par <code>-h directory</code> +Specify a database home directory. +@par <code>-V</code> +Display WiredTiger version and exit. +@par <code>-v</code> +Set verbose output. + +Unless otherwise described by a \c wt command, the \c wt tool exits zero +on success and non-zero on error. + +The \c wt tool supports several commands. + +<hr> +@section utility_create wt create +Create a table or file. + +The \c create command creates the specified \c uri with the specified +configuration. It is equivalent to a call to WT_SESSION::create with +the specified string arguments. + +@subsection utility_create_synopsis Synopsis +<code>wt [-Vv] [-C config] [-h directory] create [-c config] uri</code> + +@subsection utility_create_options Options +The following are command-specific options for the \c create command: + +@par <code>-c</code> +Include a configuration string to be passed to WT_SESSION::create. + + +<hr> +@section utility_drop wt drop +Drop a table or file. + +The \c drop command drops the specified \c uri. It is equivalent to a +call to WT_SESSION::drop with the "force" configuration argument. + +@subsection utility_drop_synopsis Synopsis +<code>wt [-Vv] [-C config] [-h directory] drop uri</code> + +@subsection utility_drop_options Options +There are no command-specific options for the \c drop command. + + +<hr> +@section utility_dump wt dump +Export data in a text format. + +The \c dump command outputs the specified table in a portable format +which can be re-loaded into a new table using the \c load command. + +See @subpage dump_formats for details of the dump file formats. + +@subsection utility_dump_synopsis Synopsis +<code>wt [-Vv] [-C config] [-h directory] dump [-rx] [-f output] uri</code> + +@subsection utility_dump_options Options +The following are command-specific options for the \c dump command: + +@par <code>-f</code> +By default, the \c dump command output is written to the standard output; +the \c -f option re-directs the output to the specified file. + +@par <code>-r</code> +Dump in reverse order, from largest to smallest. + +@par <code>-x</code> +Dump all characters in a hexadecimal encoding (the default is to leave +printable characters unencoded). + + +<hr> +@section utility_dumpfile wt dumpfile +Dump a file in a debugging format. + +The \c dumpfile command dumps the specified physical file in a non-portable, +debugging format, exiting success if the file is correct, and failure if the +file is corrupted. + +@subsection utility_dumpfile_synopsis Synopsis +<code>wt [-Vv] [-C config] [-h directory] dumpfile [-f output] file</code> + +@subsection utility_dumpfile_options Options +The following are command-specific options for the \c dumpfile command: + +@par <code>-f</code> +By default, the \c dumpfile command output is written to the standard +output; the \c -f option re-directs the output to the specified +file. + + +<hr> +@section utility_read wt list +List the tables and files in the database. + +The \c list command prints out the URIs for tables and files stored in +the database. + +@subsection utility_list_synopsis Synopsis +<code>wt [-Vv] [-C config] [-h directory] list</code> + +@subsection utility_list_options Options +The \c list command has no command-specific options. + + +<hr> +@section utility_rename wt rename +Rename a table or file. + +The \c rename command renames the specified table or file. + +@subsection utility_rename_synopsis Synopsis +<code>wt [-Vv] [-C config] [-h directory] rename uri name</code> + +@subsection utility_rename_options Options +The \c rename command has no command-specific options. + + + +<hr> +@section utility_load wt load +Load a table or file from dump output. + +The \c load command reads the standard input for data and loads it into +a table or file, creating the table or file if it does not yet exist. +The data should be the format produced by the \c dump command; see @ref +dump_formats for details. + +By default, if the table or file already exists, data in the file or +table cannot be overwritten by the new data (use the \c -o option to +overwrite existing data). + +@subsection utility_load_synopsis Synopsis +<code>wt [-Vv] [-C config] [-h directory] load [-ao] [-f input] [-r name] [uri configuration ...]</code> + +@subsection utility_load_options Options +The following are command-specific options for the \c load command: + +@par <code>-a</code> +If the \c -a option is specified, record number keys in the input are +ignored and the data is appended to the object and assigned new record +number keys. The \c -a option is only applicable when loading an object +where the primary key is a record number. + +@par <code>-f</code> +By default, the \c load command reads from the standard input; the \c +-f option reads the input from the specified file. + +@par <code>-r</code> +By default, the \c load command uses the table or file name taken from +the input; the \c -r option renames the object. + +@par <code>-o</code> +By default, input data will not overwrite existing data where the +key/value pair already exists in the object, and the attempt will fail; +the \c -o option causes the \c load command to overwrite already +existing data. + +Additionally, \c uri and \c configuration pairs may be specified to the +\c load command. Each of these pairs may be used to modify the +configuration of an object in the table or file. For each of the pairs, +the configuration string will be appended to the WT_SESSION::create call +for the object matching the uri. + + +<hr> +@section utility_loadtext wt loadtext +Load text into a table or file. + +The \c loadtext command reads the standard input for text and loads it +into a table or file. The input data should be printable characters, +with newline delimiters for each key or value. + +The \c loadtext command does not create the file if it does not yet +exist. + +In the case of inserting values into a column-store table or file, each +value is appended to the table or file; in the case of inserting values +into a row-store table or file, lines are handled in pairs, where the +first line is the key and the second line is the value. If the +row-store table or file already exists, data in the table or file will +be overwritten by the new data. + +@subsection utility_loadtext_synopsis Synopsis +<code>wt [-Vv] [-C config] [-h directory] loadtext [-f input]</code> + +@subsection utility_loadtext_options Options +The following are command-specific options for the \c loadtext command: + +@par <code>-f</code> +By default, the \c loadtext command reads from the standard input; the +\c -f option reads the input from the specified file. + + +<hr> +@section utility_printlog wt printlog +Display the database log. + +The \c printlog command outputs the database log. + +@subsection utility_printlog_synopsis Synopsis +<code>wt [-Vv] [-C config] [-h directory] printlog [-p] [-f output]</code> + +@subsection utility_printlog_options Options +The following are command-specific options for the \c printlog command: + +@par <code>-f</code> +By default, the \c printlog command output is written to the standard +output; the \c -f option re-directs the output to the specified file. + +@par <code>-p</code> +Display the log in a printable format. + + +<hr> +@section utility_read wt read +Read records from a table or file. + +The \c read command prints out the records associated with the specified +keys from the specified object. + +The object must be configured with string or record number keys and +string values. + +The \c read command exits non-zero if a specified record is not found. + +@subsection utility_read_synopsis Synopsis +<code>wt [-Vv] [-C config] [-h directory] read uri key ...</code> + +@subsection utility_read_options Options +The \c read command has no command-specific options. + + +<hr> +@section utility_salvage wt salvage +Recover data from a corrupted file. + +The \c salvage command salvages the specified object, discarding any +data that cannot be recovered. Underlying files are re-written in +place, overwriting the original file contents. + +@subsection utility_salvage_synopsis Synopsis +<code>wt [-Vv] [-C config] [-h directory] salvage [-F force] uri</code> + +@subsection utility_salvage_options Options +The following are command-specific options for the \c salvage command: + +@par <code>-F</code> +By default, salvage will refuse to salvage files that fail basic tests +(for example, files that don't appear to be in a WiredTiger format). +The \c -F option forces the salvage of the file, regardless. + + +<hr> +@section utility_stat wt stat +Display database or object statistics. + +The \c stat command outputs run-time statistics for the WiredTiger +engine, or, if specified, for the command-line object. + +@subsection utility_stat_synopsis Synopsis +<code>wt [-Vv] [-C config] [-h directory] stat [uri]</code> + +@subsection utility_stat_options Options +The \c stat command has no command-specific options. + + +<hr> +@section utility_upgrade wt upgrade +Upgrade a table or file. + +The \c upgrade command upgrades the specified table or file, exiting +success if the object up-to-date, and failure if the object cannot be +upgraded. + +@subsection utility_upgrade_synopsis Synopsis +<code>wt [-Vv] [-C config] [-h directory] upgrade uri</code> + +@subsection utility_upgrade_options Options +The \c upgrade command has no command-specific options. + + +<hr> +@section utility_verify wt verify +Check the structural integrity of a table or file. + +The \c verify command verifies the specified table or file, exiting +success if the object is correct, and failure if the object is corrupted. + +@subsection utility_verify_synopsis Synopsis +<code>wt [-Vv] [-C config] [-h directory] verify uri</code> + +@subsection utility_verify_options Options +The \c verify command has no command-specific options. + + +<hr> +@section utility_write wt write +Write records to a table or file. + +The \c write command stores records into the specified object. + +The object must be configured with string or record number keys and +string values. + +If the \c write command is called with the \c -a option, each +command-line argument is a single value to be appended to the specified +column-store object. If the \c write command is not called with the \c +-a option, the command-line arguments are key/value pairs. + +Attempting to overwrite an already existing record will fail. + +@subsection utility_write_synopsis Synopsis +<code> +wt [-Vv] [-C config] [-h directory] write -a uri value ... +<br> +wt [-Vv] [-C config] [-h directory] write [-o] uri key value ... +</code> + +@subsection utility_write_options Options +The following are command-specific options for the \c write command: + +@par <code>-a</code> +Append each value as a new record in the object. + +@par <code>-o</code> +By default, attempting to overwrite an already existing record will +fail. The \c -o option changes \c write to overwrite previously +existing records. + +*/ diff --git a/docs/src/compression.dox b/docs/src/compression.dox new file mode 100644 index 00000000000..c2223f0a404 --- /dev/null +++ b/docs/src/compression.dox @@ -0,0 +1,47 @@ +/*! @page compression Compressors + +This section explains how to use compression engines with WiredTiger, including the builtin support for bzip2 and snappy. + +@section bzip2 Using bzip2 compression + +To use the builtin <a href="http://www.bzip.org/">bzip2</a> compression, first verify that bzip2 is installed on your system. On most UNIX and Linux variants, there will be a <code>bzlib.h</code> header file in the include directory for the compiler, as well as a library file available, often named <code>libbz2.so</code>, in <code>/usr/lib</code>. If these are available, you can simply enable bzip2 by using the \c --enable-bzip2 option when running configure. + +If you have installed your own version of bzip2 in a non-standard location, you'll need to modify the \c CPPFLAGS and \c LDFLAGS to indicate these locations. For example, with bzip2 includes and libraries installed in <code>/usr/local/include</code> and <code>/usr/local/lib</code>, run configure as: + +@code +cd build_posix +../dist/configure --enable-bzip2 CPPFLAGS="-I/usr/local/include" LDFLAGS="-L/usr/local/include" +@endcode + +After building, check that this compressor is built and working by running the compression part of the test suite: + +@code +cd build_posix +python ../test/suite/run.py compress +@endcode + +Verify that the bzip2 part of the test passes and was not skipped. + +@section snappy Using snappy compression + +Google's <a href="http://code.google.com/p/snappy/">snappy</a> compressor is generally not installed by default on UNIX and Linux distributions, so you will need to download, build and install it first. To configure WiredTiger to include snappy, use the \c --enable-snappy option along with \c CPPFLAGS and \c LDFLAGS. For example, with snappy includes and libraries installed in <code>/usr/local/include</code> and <code>/usr/local/lib</code>, run configure as: + +@code +cd build_posix +../dist/configure --enable-snappy CPPFLAGS="-I/usr/local/include" LDFLAGS="-L/usr/local/include" +@endcode + +After building, check that this compressor is built and working by running the compression part of the test suite: + +@code +cd build_posix +python ../test/suite/run.py compress +@endcode + +Verify that the snappy part of the test passes and was not skipped. + +@section custom Custom compression engines + +WiredTiger may be extended by adding custom compression engines; see @ref WT_COMPRESSOR for more information. + + */ diff --git a/docs/src/config-file.dox b/docs/src/config-file.dox new file mode 100644 index 00000000000..6490aca20e6 --- /dev/null +++ b/docs/src/config-file.dox @@ -0,0 +1,34 @@ +/*! @page config_file WiredTiger Home Directory Configuration File + +If a file named \c WiredTiger.config appears in the WiredTiger home +directory, it is read as a configuration string. Configuration values +specified in the config argument to the wiredtiger_open function +override configuration values specified in the \c WiredTiger.config file. + +The file is minimally parsed in order to build configuration strings +for the WiredTiger configuration parser: +<ul> + +<li>A backslash (<b><tt>\\</tt></b>) followed by any character other +than a newline character leaves both characters untouched; otherwise, +if a backslash is followed by a newline character, both the backslash +and the newline character are discarded. + +<li>Any text between double-quote pairs (<b><tt>"</tt></b>) is left +untouched, including newline and white-space characters. Backslash +characters escape double-quote characters: a backslash escaped +double-quote character can neither start or end a quoted string. + +<li>Comments are discarded. If the first non-white-space character +following an unquoted and unescaped newline character is a hash mark +(<b><tt>#</tt></b>), all characters up to the next newline character are +discarded. The ending newline character cannot be escaped or quoted, +once a comment line is identified, all characters to the next newline +are simply discarded. + +<li>Otherwise, all lines are concatenated and the newline characters +replaced with commas. + +</ul> + +*/ diff --git a/docs/src/config-strings.dox b/docs/src/config-strings.dox new file mode 100644 index 00000000000..65f78c19946 --- /dev/null +++ b/docs/src/config-strings.dox @@ -0,0 +1,80 @@ +/*! @page config_strings Configuration Strings + +@section config_intro Introduction + +Many operations in WiredTiger accept a string to configure options. These +strings all have the same format: + +<pre> + [key['='value]][','[key['='value]]]* +</pre> + +That is, they are simple comma-separated lists of +<code>"<key>=<value>"</code> pairs. If the <code>"=<value>"</code> part is +omitted, the value of 1 is assumed. + +To handle more complex configuration, such as specifying a schema, values +may be nested lists using parentheses. For example: + +<pre> + schema=(keyfmt=S,valuefmt=S,columns=(name,notes)) +</pre> + +Empty configuration strings may be represented in C or C++ by passing +<code>NULL</code>. + +Superfluous commas and whitespace in the configuration string are ignored +(including at the beginning and end of the string), so it is always safe to +combine two configuration strings by concatenating them with a comma in +between. + +Keys are processed in order from left to right, with later settings +overriding earlier ones unless multiple settings for a key are permitted. + +@section config_json JSON compatibility + +The parser for configuration strings will accept additional formatting as +follows: + +- parentheses may be round or square brackets or curly braces: + <code>'()'</code>, <code>'[]'</code> or <code>'{}'</code> +- the whole configuration string may optionally be wrapped in parentheses +- the key/value separator can be a colon: <code>':'</code> +- keys and values may be in double quotes: <code>"key" = "value"</code> +- quoted strings are interpreted as UTF-8 values + +The result of this relaxed parsing is that applications may pass strings +representing valid <a href="http://json.org/">JSON objects</a> wherever +configuration strings are required. + +For example, in Python, code might look as follows: + +\code + import json + config = json.dumps({ + "key_format" : "r", + "value_format" : "5sHQ", + "columns" : ("id", "country", "year", "population"), + "colgroup.population" : ["population"], + "index.country_year" : ["country", "year"] + }) +\endcode + +@section config_examples Code samples + +@todo improve the example + +Open a connection to a database, creating it if it does not exist and set a +cache size of 10MB, then open a session in the database: + +@snippet ex_config.c configure cache size + +Create a table that uses C language strings for keys and values: + +@snippet ex_config.c create a table + +Walk a transactional cursor through the table: + +@snippet ex_config.c transaction + +*/ diff --git a/docs/src/cursor-ops.dox b/docs/src/cursor-ops.dox new file mode 100644 index 00000000000..cd77621d43d --- /dev/null +++ b/docs/src/cursor-ops.dox @@ -0,0 +1,97 @@ +/*! @page cursor_ops Cursor operations + +Common operations in WiredTiger are performed using WT_CURSOR handles. +A cursor includes: + +- a position within a data source +- getter/setters for key and value fields +- encoding of fields to store in the data source +- methods to navigate within and iterate through the data + +@section cursor_opening Opening a cursor + +Cursors are created using the WT_SESSION::open_cursor method. The +following are examples from the example program @ex_ref{ex_cursor.c}: + +@snippet ex_cursor.c open cursor #1 +@snippet ex_cursor.c open cursor #2 + +In addition to traditional data sources, cursors in WiredTiger are used +to access projections and even created data sources such as the run-time +statistics: + +@snippet ex_cursor.c open cursor #3 + +See @ref cursors for more information on available cursor types. + +@section cursor_position Positioning a cursor + +Cursors may be positioned at the beginning of the data source, the end of +the data source, at an exact key within the data source, and near a key +within the data source. + +To invalidate the position of a cursor so that subsequent iterations start +from the beginning or end of the data source, use the WT_CURSOR::reset method: + +@snippet ex_cursor.c cursor reset + +To move a cursor forward or backward in the data source, use the cursor +WT_CURSOR::next and WT_CURSOR::prev methods: + +@snippet ex_cursor.c cursor next +@snippet ex_cursor.c cursor prev + +If the WT_CURSOR::next and WT_CURSOR::prev methods are called on cursors +without a position in the data source, they are positioned at the beginning +or end of the data source, respectively. + +To position a cursor at a specific location in the data source, use the +WT_CURSOR::search method: + +@snippet ex_cursor.c cursor search + +To position a cursor at or near a location in the data source, use the +WT_CURSOR::search_near method: + +@snippet ex_cursor.c cursor search near + +@section cursor_writes Inserting and updating + +To insert new data, and optionally update existing data, using a cursor, +use the WT_CURSOR::insert method: + +@snippet ex_cursor.c cursor insert + +By default, when inserting into a row-store, the WT_CURSOR::insert method +returns an error if the key already exists in the store, otherwise it +inserts a new key/value pair. If the <code>overwrite</code> configuration +string is specified to the WT_SESSION::open_cursor method, any previously +existing key/value pair is updated to the new value rather than returning +an error. + +By default, when updating an underlying column-store, the WT_CURSOR::insert +method ignores the application's key value, instead, it allocates an unused +record number in the store and returns that record number in the +application's key. If the <code>overwrite</code> configuration string is +specified to the WT_SESSION::open_cursor method, the application's key +value will be used to specify the record number being inserted or updated. + +To update existing data using a cursor, use the WT_CURSOR::update method: + +@snippet ex_cursor.c cursor update + +In all cases, calling WT_CURSOR::update where the key does not already +exist in the store will return an error. + +To remove existing data using a cursor, use the WT_CURSOR::remove method: + +@snippet ex_cursor.c cursor remove + +@section cursor_error Cursor position after error + +After any cursor handle method failure, the cursor's position is +undetermined. Applications that cannot re-position the cursor after +failure must duplicate the cursor before calling a cursor method that will +attempt to re-position the cursor. @notyet{cursor duplication} + + */ diff --git a/docs/src/cursors.dox b/docs/src/cursors.dox new file mode 100644 index 00000000000..735be7478a9 --- /dev/null +++ b/docs/src/cursors.dox @@ -0,0 +1,114 @@ +/*! @page cursors Cursors + +Common operations in WiredTiger are performed using WT_CURSOR handles. +A cursor includes: + +- a position within a data source +- getter/setters for key and value fields +- encoding of fields to store in the data source +- methods to navigate within and iterate through the data + +See @subpage cursor_ops for a description of how to use cursors. + +@section cursor_types Cursor types + +The following are the builtin cursor types: + +<table> + @hrow{URI, Type} + @row{<tt>colgroup:\<tablename\>.\<columnset\></tt>, +column group cursor} + @row{<tt>config:[\<uri\>]</tt>, +object configuration cursor (key=config string\, +value=config value} + @row{<tt>table:\<tablename\></tt>, +table cursor (key=table key\, value=table value)} + @row{<tt>file:\<filename\></tt>, +file cursor (key=file key\, value=file value)} + @row{<tt>index:\<tablename\>.\<indexname\></tt>, +index cursor (key=index key\, value=table value)} + @row{<tt>join:\<cursor1\>\&\<cursor2\>[&\<cursor3\>...]</tt>, +join cursor @notyet{join cursors}} + @row{<tt>statistics:[file</tt><tt>:\<filename\>]</tt>, + database or file statistics (key=(int)\, + value=(string)description\, (string)value\, (uint64_t)value)} +</table> + +@subsection cursor_index Index cursors + +When an index is created for a table, records are inserted into the index +whenever the table is updated. These records use a different key to the +primary table, as specified when the index is created with the +WT_SESSION::create method. + +A cursor opened on an index has the specified index columns as its key, +accessed by WT_CURSOR::set_key and WT_CURSOR::get_key. The value columns +default to returning the value columns from the table, but this can be +overridden by configuring a projection cursor (see @ref cursor_projections), +which can access the table key columns or a subset of the value columns. + +@subsection cursor_file File cursors + +WiredTiger's schema layer can be bypassed by opening cursors with a \c +"file:" URI, using the name of the underlying file. This can be useful for +seeing the contents of a column group or index without reading all of the +columns from the table. + +For example, if an index becomes inconsistent with its primary, a file +cursor can read from the index without errors (even though some of the keys +that are returned may not exist in the primary). + +@subsection cursor_statistics Statistics cursors + +Cursors can return run-time statistics about the WiredTiger engine as +well as statistics for the underlying row- and column-store files. Each +cursor iteration sets three separate values: a printable description of +the entry, a printable version of the entry's value, and the entry's +unsigned 64-bit integral value. + +The statistic key is an integer from the list of keys in @ref statistics_keys +"Statistics Keys". + +The following is an example of printing run-time statistics about the +WiredTiger engine: + +@snippet ex_stat.c statistics database function + +The following is an example of printing statistics about an underlying +file: + +@snippet ex_stat.c statistics file function + +Both examples can use a common display routine that iterates through the +statistics until the cursor returns the end of the list. + +@snippet ex_stat.c statistics display function + +@section cursor_projections Projections + +Cursors on tables, column groups and indices can return a subset of +columns. This is done by listing the column names in parenthesis in the +<code>uri</code> parameter to WT_SESSION::open_cursor. Only the fields +from the listed columns are returned by WT_CURSOR::get_value. + +This is particularly useful with index cursors, because if all columns in +the projection are available in the index (including primary key columns, +which are the values of the index), there is no need to access any column +groups. + +@section cursor_raw Raw mode + +Cursors can be configured for raw mode by specifying the \c "raw" config +keyword to WT_SESSION::open_cursor. In this mode, the methods +WT_CURSOR::get_key, WT_CURSOR::get_value, WT_CURSOR::set_key and +WT_CURSOR::set_value all take a single WT_ITEM in the variable-length +argument list instead of a separate argument for each column. + +For WT_CURSOR::get_key and WT_CURSOR::get_value in raw mode, the WT_ITEM +can be split into columns by calling ::wiredtiger_struct_unpack with the +cursor's \c key_format or \c value_format, respectively. For +WT_CURSOR::set_key and WT_CURSOR::set_value in raw mode, the WT_ITEM +should be equivalent to calling ::wiredtiger_struct_pack for the +cursor's \c key_format or \c value_format, respectively. + +*/ diff --git a/docs/src/dump-formats.dox b/docs/src/dump-formats.dox new file mode 100644 index 00000000000..0c021e3a29b --- /dev/null +++ b/docs/src/dump-formats.dox @@ -0,0 +1,51 @@ +/*! @page dump_formats Dump Formats + +The @ref utility_dump command produces a text representation of a table +that can be loaded by @ref utility_load. This page describes the output +format of the @ref utility_dump command. + +Dump files have three parts, a prefix, a header and a body. + +The dump prefix includes basic information about the dump including the +WiredTiger version that created the dump and the dump format. The dump +format consists of a line beginning with \c "Format=", and contains the +following information: + +<table> +@hrow{String, Meaning} +@row{hex, the dumped data is in a hexadecimal dump format} +@row{print, the dumped data is in a printable format} +</table> + +The dump header follows a single \c "Header" line in the file and +consists of paired key and value lines, where the key is the URI passed +to WT_SESSION::create and the value is corresponding configuration +string. The table or file can be recreated by calling +WT_SESSION::create for each pair of lines in the header. + +The dump body follows a single \c "Data" line in the file and consists +of a text representation of the records in the table. Each record is a +represented by a pair of lines: the first line is the key and the second +line is the value. These lines are encoded in one of two formats: a +printable format and a hexadecimal format. + +The printable format consists of literal printable characters, and +hexadecimal encoded non-printable characters. Encoded characters are +written as three separate characters: a backslash character followed by +two hexadecimal characters (first the high nibble and then the low +nibble). For example, a newline character in the ASCII character set +would be encoded as \c "\0a" and an escape character would be encoded +as \c "\1b". Backslash characters which do not precede a hexadecimal +encoding are paired, that is, the characters \c "\\" should be +interpreted as a single backslash character. + +The hexadecimal format consists of encoded characters, where each +literal character is written as a pair of characters (first the +high-nibble and then the low-nibble). For example, "0a" would be an +ASCII newline character and "1b" would be an ASCII escape character. + +Because the definition of "printable" may depend on the application's +locale, dump files in the printable output format may be less portable +than dump files in the hexadecimal output format. + + */ diff --git a/docs/src/examples.dox b/docs/src/examples.dox new file mode 100644 index 00000000000..d0a249aecb6 --- /dev/null +++ b/docs/src/examples.dox @@ -0,0 +1,44 @@ +/*! +@example ex_hello.c +This is an example of how to create and open a database. + +@example ex_access.c +Create, insert and access a simple table. + +@example ex_all.c +Demonstrate how to call every method in the API. + +@example ex_config.c +Shows how to configure some properties of the database and tables. + +@example ex_cursor.c +Shows some common cursor types and operations. + +@example ex_pack.c +Shows basic packing and unpacking of fields. + +@cond sharing +@notyet{inter-process sharing} +@example ex_process.c +Shows how to connect to a database from multiple processes. +@endcond + +@example ex_stat.c +Shows how to access database and table statistics. + +@example ex_schema.c +Shows how to create column-oriented data and access individual columns. + +@example ex_thread.c +Shows how to access a database with multiple threads. + +@example ex_transaction.c +Shows how to use transactions. @notyet{transactions} + +@example ex_call_center.c +A more complex schema based on a call center example, showing how to map some +SQL constructs onto the WiredTiger API. + +@example ex_extending.c +Shows how to extend WiredTiger with application-specific collations, extractors and cursor types. + */ diff --git a/docs/src/file-formats.dox b/docs/src/file-formats.dox new file mode 100644 index 00000000000..538709673d4 --- /dev/null +++ b/docs/src/file-formats.dox @@ -0,0 +1,72 @@ +/*! @page file_formats WiredTiger files + +@section formats File formats + +WiredTiger supports two underlying file formats: row-store and +column-store. WiredTiger row- and column-stores are both key/value +stores. + +In a row-store, both keys and data are variable-length byte strings. In +a column-store, the key is a 64-bit record number, and the data item is +either a variable- or fixed-length byte string. + +Generally, row-stores are faster for queries where a set of columns are +required by every lookup (because there's only a single set of meta-data +pages to go through, or read into the cache). Column-stores are faster +for queries where only a few of the columns are required for any lookup +(because only the columns being returned are present in the cache). + +Row-stores support three types of compression: prefix compression (where +any identical portion of each key byte string is only stored once), +Huffman encoding of individual key/value items, (see @subpage huffman +for details), and stream compression of the blocks in the file (see @ref +compression for details). + +Unlike some row-stores, WiredTiger does not support duplicate data +items, that is, for any single key, there can be only a single value, +and applications are responsible for creating unique key/value pairs. + +Column-stores with variable-length byte string values support three +types of compression: run-length encoding (where duplicate values are +only stored a single time), Huffman encoding of individual value items, +(see @ref huffman for details), and stream compression of the blocks in +the file (see @ref compression for details). + +Column-stores with fixed-length byte values support a single type of +compression: stream compression of the blocks in the file (see @ref +compression for details). + +In row-stores, keys and values too large to fit on a normal page are +stored as overflow items in the file. In variable-length column-stores, +values too large to fit on a normal page are stored as overflow items +in the file. + +WiredTiger allocates space from the underlying files in block units. +The minimum file allocation unit WiredTiger supports is 512B and the +maximum file allocation unit is 512MB. File block offsets are 64-bit +(meaning the maximum file size is very, very large). + +Variable-length column-store values, and row-store keys and values, can +be up to (4GB - 512B) in length. + +Fixed-length values are limited to 8-bits (that is, only values between +0 and 255 may be stored in fixed-length column-store files). + +@section remote Remote file systems + +WiredTiger objects may be stored on remote file systems if the remote +file system conforms to ISO/IEC 9945-1:1996 (POSIX.1). In the case of +read-only objects, multiple systems may access the objects +simultaneously; objects which are written cannot be accessed by more +than a single system. Because remote file systems are often slower than +local file systems, using a remote file system for storage may degrade +performance. + +@section file File permissions + +WiredTiger creates file system objects readable and writable by the +process owner, group and user, as modified by the process' umask value. +The group ownership of created file system objects may vary depending +on the system, and is not controlled by WiredTiger. + +*/ diff --git a/docs/src/home.dox b/docs/src/home.dox new file mode 100644 index 00000000000..222e0400f75 --- /dev/null +++ b/docs/src/home.dox @@ -0,0 +1,57 @@ +/*! @page home WiredTiger Home Directory + +WiredTiger file naming is based on the \c home argument specified to the +::wiredtiger_open function and the \c WIREDTIGER_HOME environment +variable. + +If a \c home argument is specified to the ::wiredtiger_open function, +its value is used as the database directory and the \c WIREDTIGER_HOME +environment variable is always ignored. + +If no \c home argument is specified to the ::wiredtiger_open function +and the WIREDTIGER_HOME environment variable is not set, the database +directory is the process' current working directory. No current working +directory path is maintained by the WiredTiger software and changing the +working directory after opening the WiredTiger database may cause +failure. + +If no \c home argument is specified to the ::wiredtiger_open function, +the WIREDTIGER_HOME environment variable is set, and the +::wiredtiger_open function was configured with the \c home_environment +string, the WIREDTIGER_HOME environment variable value is used as the +database directory. + +If no \c home argument is specified to the ::wiredtiger_open function, +the WIREDTIGER_HOME environment variable is set, the ::wiredtiger_open +function was configured with the \c home_environment_priv string, and +the user has appropriate privileges, the WIREDTIGER_HOME environment +value is used as the database directory. (The \c home_environment_priv +string is intended for applications that have or acquire special +privileges and wish to ensure an environment-specified home directory +is ignored unless the program is executed by a user with appropriate +permissions: if such an application does not configure the +::wiredtiger_open function with the \c home_environment_priv string, +setting an environment value will cause the open to fail unless the user +has appropriate permissions. On ISO/IEC 9945-1:1990 (POSIX.1) systems, +"appropriate permissions" is defined as a real user ID of 0.) + +If no \c home argument is specified to the ::wiredtiger_open function, +the WIREDTIGER_HOME environment variable is set, and the +::wiredtiger_open function was not configured with either the \c +home_environment or \c home_environment_priv strings, the open will +fail. + +Finally, consider security when configuring WiredTiger to use the +WIREDTIGER_HOME environment variable, especially in applications which +run with permissions other than the user's. Such applications are +potentially vulnerable to allowing users access to databases they could +not otherwise access. + +@section home_config Home directory configuration strings + +The WiredTiger home directory optionally includes a file named \c +WiredTiger.config. If this file exists when ::wiredtiger_open is +called, it is read for configuration strings. See @subpage config_file +for details. + +*/ diff --git a/docs/src/huffman.dox b/docs/src/huffman.dox new file mode 100644 index 00000000000..3365b68411e --- /dev/null +++ b/docs/src/huffman.dox @@ -0,0 +1,39 @@ +/*! @page huffman Huffman Encoding + +Keys in row-stores and variable-length values in either row- or +column-stores can be compressed with Huffman encoding. + +Huffman compression is maintained in memory as well as on disk, and can +increase the amount of usable data the cache can hold as well as +decrease the size of the data on disk. + +To specify Huffman encoding for the key in a row-store, specify \c +"btree_huffman_key=english" or \c "btree_huffman_key=<file>" in the +configuration passed to \c WT_SESSION::create. + +To specify Huffman encoding for a variable-length value in either a +row-store or a column-store, specify "btree_huffman_value=english" +or \c "btree_huffman_value=<file>" in the configuration passed to +\c WT_SESSION::create. + +Setting Huffman encoding to \c "english" configures WiredTiger to use +a built-in English language frequency table. + +Setting Huffman encoding to \c "<file>" configures WiredTiger to use the +frequency table read from the specified file. The format of the +frequency table file is lines containing pairs of unsigned integers +separated by whitespace. The first integer is the symbol value, the +second integer is the frequency value. Symbol values may be specified +as hexadecimal numbers (with a leading \c "0x" prefix), or as integers. +Symbol values must be unique and in the range of 0 to 65,535. Frequency +values do not need to be unique, but must be in the range of 0 to the +maximum 32-bit unsigned integer value (4,294,967,295), where the lower +the frequency value, the less likely the byte value is to occur. Any +unspecified symbol values are assumed to have frequencies of 0. + +Input containing symbol values that did not appear in the frequency +table (or appeared in the frequency table, but with frequency values of +0), are accepted, although will not compress as well as if they are +listed in the frequency table, with frequency values other than 0. + +*/ diff --git a/docs/src/install.dox b/docs/src/install.dox new file mode 100644 index 00000000000..ec824257647 --- /dev/null +++ b/docs/src/install.dox @@ -0,0 +1,181 @@ +/*! @page install How to build and install WiredTiger + +This section explains how to build and install the WiredTiger software. + +@section ports Ports + +WiredTiger has been tested on the following systems: +@par +Linux\n +FreeBSD\n +Mac OS X\n + +WiredTiger should be portable to any system that supports the ANSI C99, +POSIX 1003.1 and POSIX 1003.1c (threads extension) standards. + +@section building Building WiredTiger + +WiredTiger uses the <a href="http://gcc.gnu.org/">GNU Compiler +Collection</a> (GCC) toolchain for configuration and building (but does +not require the gcc compiler for building), including <a +href="http://www.gnu.org/software/autoconf/autoconf.html">autoconf</a> +and <a +href="http://www.gnu.org/software/libtool/libtool.html">libtool</a>. +The standard options for those tools can be specified when configuring +and building WiredTiger. + +To build the WiredTiger software on a POSIX-like system, change +directory to the top-level <code>build_posix</code> directory, +then configure and build the software: + +@code +cd build_posix +../configure +make +@endcode + +To rebuild from scratch, discard any previous configuration by cleaning +out the build area: + +@code +cd build_posix +make realclean +@endcode + +@section installing Installing WiredTiger + +The WiredTiger software consists of a library and a single standalone +utility. + +WiredTiger's distribution follows the GNU Coding Standards installation +guidelines, and by default WiredTiger builds and installs four versions +of the library in <code>/usr/local/lib</code>. For example: + +@code +file /usr/local/lib/libwiredtiger* +/usr/local/lib/libwiredtiger-1.0.0.so: ELF 64-bit LSB shared object, x86-64, version 1 (FreeBSD), dynamically linked, not stripped +/usr/local/lib/libwiredtiger.a: current ar archive +/usr/local/lib/libwiredtiger.la: libtool library file +/usr/local/lib/libwiredtiger.so: symbolic link to `libwiredtiger-1.0.0.so' +@endcode + +WiredTiger uses +<a href="http://www.gnu.org/software/libtool/libtool.html">libtool</a> to +build the libraries. By default, this tool builds both shared and +static libraries. To build only static libraries, configure WiredTiger +using the \c --disable-shared argument. To build only shared libraries, +configure using WiredTiger using the \c --disable-static argument. + +In addition, WiredTiger installs a standalone utility program named +<code>wt</code>. By default, this utility is installed in +<code>/usr/local/bin/wt</code>. + +To install WiredTiger: + +@code +cd build_posix +make install +@endcode + +To uninstall WiredTiger: + +@code +cd build_posix +make uninstall +@endcode + +To install WiredTiger's libraries or binaries into alternate locations, +use the configuration or installation options described in the +<a href="http://www.gnu.org/prep/standards/">GNU coding standards</a> +documentation. For example, to install the libraries and binaries into +a different location: + +@code +cd build_posix +../dist/configure --prefix=/c/wiredtiger +@endcode + +@section configure Configuring WiredTiger + +The WiredTiger software supports some additional configuration options: + +@par \c --enable-attach +Configure WiredTiger to sleep and wait for a debugger to attach on failure. +<b>DO NOT</b> configure this option in production environments. + +@par \c --enable-bzip2 +Configure WiredTiger for <a href="http://www.bzip.org/">bzip2</a> +compression; see @ref compression for more information. + +@par \c --enable-debug +Configure WiredTiger for debugging, including building with the +compiler's \c -g flag. +<b>DO NOT</b> configure this option in production environments. + +@par \c --enable-diagnostic +Configure WiredTiger to perform various run-time diagnostic tests. +<b>DO NOT</b> configure this option in production environments. + +@par \c --enable-python +Build the WiredTiger <a href="http://www.python.org">Python</a> API. + +@par \c --enable-snappy +Configure WiredTiger for <a href="http://code.google.com/p/snappy/">snappy</a> +compression; see @ref compression for more information. + +@par \c --enable-verbose +Configure WiredTiger to support the \c verbose configuration string to +::wiredtiger_open. + +@par <code>--with-spinlock</code> +Configure WiredTiger to use a specific mutex type, options are \c +pthread_mutex (the default, which configures WiredTiger to use POSIX 1003.1c +pthread mutexes for serialization) or \c gcc (which configures WiredTiger to +use gcc-based spinlocks for serialization). + +@section compiler Changing compiler or loader options + +To change the compiler or loader behavior during the build, use the +<code>CC</code>, <code>CFLAGS</code>, <code>LDFLAGS</code>, or +<code>LIBS</code> environment variables: + +@par \c CC +The compiler. +@par \c CFLAGS +Compiler flags. +@par \c LDFLAGS +Loader flags. +@par \c LIBS +Additional libraries. + +For example, to specify a different compiler: + +@code +cd build_posix +env CC=mygcc ../configure +@endcode + +By default, WiredTiger builds with the \c -O3 compiler optimization flag +unless the \c --enable-debug configuration option is specified, in which +case the \c -g compiler flag is used instead. For example, to specify +a different level of optimization: + +@code +cd build_posix +env CFLAGS=-Os ../configure +@endcode + +To specify a different set of include files: + +@code +cd build_posix +env CFLAGS=-I/usr/local/include ../configure +@endcode + +To specify an additional library: + +@code +cd build_posix +env LIBS="-lrtf -lmin" LDFLAGS=-L/usr/local/lib ../configure +@endcode + */ diff --git a/docs/src/introduction.dox b/docs/src/introduction.dox new file mode 100644 index 00000000000..634f8144475 --- /dev/null +++ b/docs/src/introduction.dox @@ -0,0 +1,33 @@ +/*! @mainpage Reference Guide + +WiredTiger is an extensible platform for data management. This +documentation describes the programming interface to WiredTiger used by +developers to construct applications. + +We follow SQL terminology: a database is set of tables managed together. +Tables consist of rows, where each row is a key and its associated +value. Tables may optionally have an associated schema, splitting the +value into a set of columns. Tables may also have associated indices, +each of which is ordered by one or more columns. + +In addition to the traditional row-oriented storage where all columns +of a row are stored together, WiredTiger supports column-oriented +storage, where one or more columns can be stored individually, allowing +more efficient access and storage. + +To download the WiredTiger source code, visit our +<a href="https://github.com/wiredtiger/wiredtiger"><b>Project page</b></a>. + +For more information about using WiredTiger, see: + +- @subpage install\n + +- @subpage using\n + +- @ref wt "WiredTiger API reference manual" + +- @subpage command_line\n + +- @subpage license\n + +*/ diff --git a/docs/src/keyvalue.dox b/docs/src/keyvalue.dox new file mode 100644 index 00000000000..f375886b030 --- /dev/null +++ b/docs/src/keyvalue.dox @@ -0,0 +1,19 @@ +/*! @page keyvalue Key/Value pairs + +Underlying WiredTiger files are key/value stores, that is, entries in +the file are paired keys and values. These keys and values can be +accessed directly, and a file cursor will only return the data from the +single file, without reference to any other related files or data. For +example, if an index were lost or to become inconsistent with its +primary, a file cursor could read from the index without problems (even +though some of the returned keys might not exist in the primary). + +In the case of a row-store file, keys and values are both +variable-length byte strings, up to (4GB - 512B) in length. + +In the case of a column-store file, keys are 64-bit record numbers, +where the first record in the file is record number 1. Values are +either variable-length byte strings, up to (4GB - 512B) in length, or +bit fields, where a bit field is 1 to 8 bits. + + */ diff --git a/docs/src/license.dox b/docs/src/license.dox new file mode 100644 index 00000000000..282b1da5ff6 --- /dev/null +++ b/docs/src/license.dox @@ -0,0 +1,52 @@ +/*! @page license WiredTiger license + +The WiredTiger software is Open Source software: you can redistribute +it and/or modify it under the superset of the terms of version 3 of the +<a href="http://www.gnu.org/licenses/gpl-3.0-standalone.html"> +<b>GNU General Public License</b></a> +as published by the Free Software Foundation, and the +<a href="http://www.opensource.org/licenses/BSD-3-Clause"> +<b>University of California, Berkeley (BSD) 3-Clause License</b></a>. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +<a href="http://www.gnu.org/licenses/gpl-3.0-standalone.html"> +<b>GNU General Public License</b></a> for details. + +For a license to use the WiredTiger software under conditions other than +those described above, or for technical support for this software, please +contact WiredTiger, Inc. at +<a mailto="info@wiredtiger.com">info@wiredtiger.com</a>. + +@section library 3rd party software included in the WiredTiger library + +The WiredTiger binary library build includes the following 3rd party +software, distributed under the following licenses: + +<table> +@hrow{File, License} +@row{src/include/bitstring.i, University of California\, Berkeley (BSD) 3-Clause License} +@row{src/include/queue.h, University of California\, Berkeley (BSD) 3-Clause License} +@row{src/utilities/util_getopt.c, University of California\, Berkeley (BSD) 3-Clause License} +</table> + +@section distribution 3rd party software included in the WiredTiger distribution + +The WiredTiger distribution includes additional 3rd party software +licensed under a variety of licenses. This software is not included +in the WiredTiger library binary as built by the distribution, it is +only used to build and test the WiredTiger documentation and software. +If you extract portions of the WiredTiger distribution for inclusion in +other packages, or redistribute selected portions of the WiredTiger +sources, please review the copyright notices and LICENSE files included +in the WiredTiger distribution for the terms and conditions of such +redistribution. + +@section pd Public domain software + +Portions of this program are public domain software. Public domain +files have copyright notices releasing the software into the public +domain and may be freely used. + +*/ diff --git a/docs/src/namespace.dox b/docs/src/namespace.dox new file mode 100644 index 00000000000..3ac7bfd6ec0 --- /dev/null +++ b/docs/src/namespace.dox @@ -0,0 +1,17 @@ +/*! @page name_space WiredTiger name spaces + +@section c C language name space + +WiredTiger's public function names begin with the string "wiredtiger". + +WiredTiger's public \c \#define and structure \c typedef declarations +begin with the string "WT_". + +WiredTiger's private function names begin with the string "__wt__". + +@section file File system name space + +WiredTiger's files begin with the string "WiredTiger"; applications +should not create files in the WiredTiger file system name space. + +*/ diff --git a/docs/src/packing.dox b/docs/src/packing.dox new file mode 100644 index 00000000000..7723e21bc79 --- /dev/null +++ b/docs/src/packing.dox @@ -0,0 +1,35 @@ +/*! @page packing Packing and Unpacking Data + +WiredTiger's data packing uses format strings similar to those specified in the +Python struct module: + http://docs.python.org/library/struct + +The first character of the format string can be used to indicate the byte +order, size and alignment of the packed data, according to the following +table: + +<table> +@hrow{Character, Byte order, Size, Alignment} +@row{<tt>.</tt>, big-endian, packed, none} +@row{<tt>></tt>, big-endian, standard, none} +@row{<tt><</tt>, little-endian, standard, none} +@row{<tt>\@</tt>, native, native, native} +</table> + +If the first character is not one of these, '.' (big-endian, packed) is +assumed: it naturally sorts in lexicographic order, and the packed format +uses variable-sized encoding of values to reduce the data size. + +@notyet{format types} <b>Only the default big-endian, packed format is currently supported.</b> + +The remaining characters in the format string specify the type of each field to be packed into or unpacked from a byte array. See @ref schema_types for the list of supported types. + +@todo Describe the variable-length integer packing in sufficient detail that it can be re-implemented in other programming languages or in network clients. + +@section config_examples Code samples + +The code below is taken from the complete example program @ex_ref{ex_pack.c}. It demonstrates how to pack three integer values into a buffer and then unpack them again. + +@snippet ex_pack.c packing + + */ diff --git a/docs/src/processes.dox b/docs/src/processes.dox new file mode 100644 index 00000000000..87f715792f9 --- /dev/null +++ b/docs/src/processes.dox @@ -0,0 +1,28 @@ +/*! @page processes Sharing Between Processes + +@notyet{inter-process sharing} + +WiredTiger includes a server that provides remote access to databases. The primary process to open a database can optionally start a server that handles requests from other processes. + +The remote interface is the way languages other than C/C++ are supported, and the interface for client processes in multiprocess C/C++ applications. + +The server can be embedded in an application or run from the command line. To embed the RPC server in your application, pass <code>"multiprocess"</code> in the configuration string to ::wiredtiger_open. Note that in this case, when your application exits, all client connections are forcibly closed. + +For details on running a standalone RPC server, see @ref utility. + +@section processes_sharing Opening connections from multiple processes + +When ::wiredtiger_open is called for a database, one of the following occurs: + +# no process has the database open, it was closed cleanly. In this case, the opening process becomes the primary process and the database is opened without recovery. +# no process has the database open, but it was not closed cleanly. In this case, the process becomes the primary and recovery is run before the database is opened. See @ref transaction_recovery for details. +# another process has the database open and is running the RPC server, in which case the opening process becomes a client. +# another process has the database open but is not running the RPC server, in which case the open fails. + +@section processes_example Code samples + +The code below is taken from the complete example program @ex_ref{ex_process.c}. + +@snippet ex_process.c processes + + */ diff --git a/docs/src/schema.dox b/docs/src/schema.dox new file mode 100644 index 00000000000..01b9572c0ff --- /dev/null +++ b/docs/src/schema.dox @@ -0,0 +1,206 @@ +/*! @page schema Schemas + +While many tables have simple key/value pairs for records, WiredTiger +also supports more complex data patterns. + +@section schema_intro Tables, rows and columns + +A table is a logical representation of data consisting of cells in rows and +columns. For example, a database might have this table. + +<table> +<tr><th>EmpId</th><th>Lastname</th><th>Firstname</th><th>Salary</th></tr> +<tr><td>1</td><td>Smith</td><td>Joe</td><td>40000</td></tr> +<tr><td>2</td><td>Jones</td><td>Mary</td><td>50000</td></tr> +<tr><td>3</td><td>Johnson</td><td>Cathy</td><td>44000</td></tr> +</table> + +This simple table includes an employee identifier, last name and first +name, and a salary. + +A row-oriented database would store all of the values in a row together, +then the values in the next row, and so on: + +<pre> + 1,Smith,Joe,40000; + 2,Jones,Mary,50000; + 3,Johnson,Cathy,44000; +</pre> + +A column-oriented database stores all of the values of a column together, +then the values of the next column, and so on: + +<pre> + 1,2,3; + Smith,Jones,Johnson; + Joe,Mary,Cathy; + 40000,50000,44000; +</pre> + +WiredTiger supports both storage formats, and can mix and match the storage +of columns within a logical table. + +Applications describe the format of their data by supplying a schema to +WT_SESSION::create. This specifies how the application's data can be split +into fields and mapped onto rows and columns. + +@section schema_types Column types + +By default, WiredTiger works as a traditional key/value store, where the +keys and values are raw byte arrays accessed using a WT_ITEM structure. +Keys and values may be up to (4GB - 512B) bytes in size, but depending +on how @ref WT_SESSION::create "maximum item sizes" are configured, +large key and value items will be stored on overflow pages. + +See @subpage keyvalue for more details on raw key / value items. + +The schema layer allows key and value types to be chosen from a list, +or composite keys or values made up of columns with any combination of +types. The size (4GB - 512B) byte limit on keys and values still +applies. + +WiredTiger's uses format strings similar to those specified in the Python +struct module to describe the types of columns in a table: + http://docs.python.org/library/struct + +@section schema_format_types Format types + +<table> +@hrow{Format, C Type, Java type, Python type, Standard size} +@row{x, pad byte, N/A, N/A, 1} +@row{b, signed char, byte, integer, 1} +@row{B, unsigned char, byte, integer, 1} +@row{h, short, short, integer, 2} +@row{H, unsigned short, short, integer, 2} +@row{i, int, int, integer, 4} +@row{I, unsigned int, int, integer, 4} +@row{l, long, int, integer, 4} +@row{L, unsigned long, int, integer, 4} +@row{q, long long, long, integer, 8} +@row{Q, unsigned long long, long, integer, 8} +@row{r, uint64_t, long, integer, 8} +@row{s, char[], String, string, fixed length} +@row{S, char[], String, string, variable} +@row{t, unsigned char, byte, integer, fixed bit length} +@row{u, WT_ITEM *, byte[], string, variable} +</table> + +The <code>'r'</code> type is used for record number keys in column stores. + +The <code>'S'</code> type is encoded as a C language string terminated by a +NUL character. + +The <code>'t'</code> type is used for fixed-length bit field values. If +it is preceded by a size, that indicates the number of bits to store, +between 1 and 8. That number of low-order bits will be stored in the +table. The default is a size of 1 bit: that is, a boolean. The +application must always use an <code>unsigned char</code> type (or +equivalently, <code>uint8_t</code>) for calls to WT_CURSOR::set_value, +and a pointer to the same for calls to WT_CURSOR::get_value. If a bit +field value is combined with other types in a packing format, it is +equivalent to <code>'B'</code>, and a full byte is used to store it. + +When referenced by a record number (that is, a key format of \c 'r'), +the <code>'t'</code> type will be stored in a fixed-length column-store, +and will not have a out-of-band value to indicate the record does not +exist. In this case, a 0 byte value is used to indicate the record does +not exist. This means removing a record with WT_CURSOR::remove is +equivalent to storing a value of 0 in the record with WT_CURSOR::update +(and storing a value of 0 in the record will cause cursor scans to +skip the record). Additionally, creating a record past the end of the +table or file implies the creation of any missing intermediate records, +with byte values of 0. + +The <code>'u'</code> type is for raw byte arrays: if it appears at the end +of a format string (including in the default <code>"u"</code> format for +untyped tables), the size is not stored explicitly. When <code>'u'</code> +appears within a format string, the size is stored as a 32-bit integer in +the same byte order as the rest of the format string, followed by the data. + +There is a default collator that gives lexicographic (byte-wise) +comparisons, and the default encoding is designed so that lexicographic +ordering of encoded keys is usually the expected ordering. For example, +the variable-length encoding of integers is designed so that they have the +natural integer ordering under the default collator. + +See @subpage packing for details of WiredTiger's packing format. + +WiredTiger can also be extended with @ref WT_COLLATOR. + +@section schema_data_access Columns in key and values + +Every table has a key format and a value format as describe in @ref +schema_types. These types are configured when the table is created by +passing \c key_format and \c value_format keys to WT_SESSION::create. + +Cursors for a table have the same key format as the table itself. The key +columns of a cursor are set with WT_CURSOR::set_key and accessed with +WT_CURSOR::get_key. WT_CURSOR::set_key is analogous to \c printf, and takes +a list of values in the order the key columns are configured in \c +key_format. The columns values are accessed with WT_CURSOR::get_key, which +is analogous to \c scanf, and takes a list of pointers to values in the same +order. + +Cursors for a table have the same value format as the table, unless a +projection is specified to WT_SESSION::open_cursor. WT_CURSOR::set_value is +used to set value columns, and WT_CURSOR::get_value is used to get value +columns. + +@section schema_columns Describing columns + +The columns in a table can be assigned names by passing a \c columns key to +WT_SESSION::create. The column names are assigned first to the columns in +the \c key_format, and then to the columns in \c value_format. There must be +a name for every column, and no column names may be repeated. + +@section schema_column_groups Storing groups of columns together + +Once column names are assigned, they can be used to configure column groups, +where groups of columns are stored in separate files. + +There are two steps involved in setting up column groups: first, pass a list +of names for the column groups in the \c colgroups configuration key to +WT_SESSION::create. Then make a call to WT_SESSION::create for each column +group, using the URI <code>colgroup:\<table\>:\<colgroup name\></code> and a +\c columns key in the configuration. Columns can be stored in multiple +column groups, but all value columns must appear in at least on column group. + +Column groups have the same key as the table. This is particularly useful +for column stores, because record numbers are not stored explicitly on disk, +so there is no repetition of keys across multiple files. Also note that key +columns cannot be stored in column group values: they can be retrieved with +WT_CURSOR::get_key. + +@section schema_indices Adding an index + +Schema columns can also be used to configure indices on tables. To create an +index, call WT_SESSION::create using the URI +<code>index:\<table\>:\<index name\></code> and include a \c columns key in +the configuration. WiredTiger updates all indices for a table whenever the +table is modified. + +A cursor can be opened on an index by passing the index URI to +WT_SESSION::open_cursor. Index cursors use the specified index key columns +for WT_CURSOR::get_key and WT_CURSOR::set_key, and by default return all of +the table value columns in WT_CURSOR::get_value. Index cursors are +read-only: they cannot be used to perform updates. + +@section schema_examples Code samples + +The code below is taken from the complete example program +@ex_ref{ex_schema.c}. + +@snippet ex_schema.c schema decl +@snippet ex_schema.c schema work + +The code below is taken from the complete example program +@ex_ref{ex_call_center.c}. + +@snippet ex_call_center.c call-center decl +@snippet ex_call_center.c call-center work + +@todo new section: schema_advanced Advanced Schemas +@todo non-relational data such as multiple index keys per row +@todo application-supplied extractors and collators need to be +registered before recovery can run. + */ diff --git a/docs/src/security.dox b/docs/src/security.dox new file mode 100644 index 00000000000..004a2661f54 --- /dev/null +++ b/docs/src/security.dox @@ -0,0 +1,24 @@ +/*! @page security WiredTiger security + +@section directory Database directory permissions + +The WiredTiger database directory should have its permissions set to +ensure database objects are not accessible to users without appropriate +permissions. See @ref home for more information. + +@section file File permissions + +WiredTiger creates file system objects readable and writable by the +process owner, group and user, as modified by the process' umask value. +The group ownership of created file system objects may vary depending +on the system, and is not controlled by WiredTiger. + +@section environment_variables Environment variables + +Consider security when configuring WiredTiger to use the WIREDTIGER_HOME +environment variable, especially in applications which run with +permissions other than the user's. Such applications are potentially +vulnerable to allowing users access to databases they could not +otherwise access. See @ref home for more information. + +*/ diff --git a/docs/src/signals.dox b/docs/src/signals.dox new file mode 100644 index 00000000000..1d05f0f2019 --- /dev/null +++ b/docs/src/signals.dox @@ -0,0 +1,11 @@ +/*! @page signals WiredTiger signal handling + +WiredTiger has no signal handlers and does not modify process signal handler behavior in any way. + +WiredTiger restarts any system calls interrupted by a signal. + +WiredTiger applications exiting as a result of a signal should exit WiredTiger gracefully, closing any open WiredTiger handles, before terminating the process. + +WiredTiger is not re-entrant, and applications using WiredTiger should not make WiredTiger API calls from signal handlers. + +*/ diff --git a/docs/src/spell.ok b/docs/src/spell.ok new file mode 100644 index 00000000000..8e8c6153635 --- /dev/null +++ b/docs/src/spell.ok @@ -0,0 +1,284 @@ +personal_ws-1.1 en 200 +APIs +Atomicity +BLOBs +CFLAGS +CPPFLAGS +DB's +DBTs +DbCursor +DbEnv +DbMultiple +EB +EmpId +FreeBSD +GCC +IEC +LDFLAGS +LIBS +LSB +MERCHANTABILITY +MVCC's +RepMgr +Rrx +TimesTen's +URIs +Vv +WiredTiger +WiredTiger's +aR +ack'ed +alloc +allocsize +ao +api +apiflags +ar +arg +autoconf +bdb +bdbmap +bindir +bitstring +bool +boolean +br +btree +builtin +bzip +cachesize +callbk +cd +cdb +cds +checksums +ckp +colgroup +colgroups +command's +comparator +cond +config +conn +const +control's +cpp +crashless +cursortype +dN +dNLen +dNOff +dT +dataN +dataitem +dbc +dbm +dbt +decl +del +desc +destructor +destructors +dlp +dontlock +dp +dsync +dumpfile +dup +dups +endcode +endcond +endian +endinternal +english +env +eof +erlang +errno +failchk +fd's +fieldname +fileformats +fileid +filename +filesystem +fillfactor +firstname +fput +freelist +gcc +gdbm +getopt +getter +gid +hb +hotbackup +href +hrow +hsearch +html +huffman +indices +init +intl +inuse +io +ip +je +jni +json +keyexist +keyfmt +keyname +keyvalue +lastname +len +li +libdir +libtool +libwiredtiger +lmin +loadtext +logc +lookup +lrtf +lsn +lt +mailto +malloc'd +marshalled +marshalling +maxintlitem +maxintlpage +maxleafitem +maxleafpage +memalloc +memfree +memp +metadata +minkey +mmap +mpool +mpoolfile +msg +msgs +multi +multiprocess +multithreading +multiversion +mutex +mutexes +mutexing +mvcc +mygcc +ndary +ndbm +newsite +nocase +nocasecoll +nodup +noflush +nolocking +nommap +nop +nosync +notgranted +notyet +nowait +nul +num +objectsin +ol +oltp +oob +ovfl +pcoll +pdf +pget +php +png +posix +pre +primary's +printf +printlog +printvalue +priv +pthread +pthreads +qnx +rdbms +rdlock +readlock +readonly +realclean +realloc +realloc'd +recno +recnoN +recnum +recs +rerequests +ret +ro +rpc +rwlock +rx +sHQ +scanf +schemas +selectable +seqname +serializable +spinlock +spinlocks +sql +src +startsync +strerror +struct +structs +subdatabases +subpage +superset +tablename +tcl +td +th +thang +timestamp +timestamps +todo +toolchain +tpc +tpcb +transactional +transactionally +tt +txn +txns +typedef +uint +ul +umask +unallocated +unencoded +unescaped +untyped +uri +useconds +usr +utf +util +valign +valuefmt +vec +versa +whitespace +wiredtiger +workQ +writelock +writelocks +wrlock +xa +yieldcpu diff --git a/docs/src/sql-map.dox b/docs/src/sql-map.dox new file mode 100644 index 00000000000..8f288565436 --- /dev/null +++ b/docs/src/sql-map.dox @@ -0,0 +1,32 @@ +/*! \page sql_mapping Mapping SQL onto the WiredTiger API + +@todo decide whether to keep this page, if so, fill it out + +An RDBMS has the SQL language, a parser and a query planner. All of them +sit on top of WiredTiger, as they would on top of any key-value store. + +The difference with WiredTiger's API is that the query plan is going to be +based on fundamental operations including: + +# sequential scan through a range; +# join (various flavors); +# sort (group by); +# projection, and so on. + +Those operations would be implemented as different WiredTiger cursor types, +using WT_CONNECTION::add_cursor_type. + +The cursor types could be implemented as one or more extensions, loaded +automatically whenever the database is opened for the first time. Then the +cursor types would always be there, and the query planner just asks for a +cursor that combines together data from various other cursors and comes out as +a set of key/data pairs it can deal with. + +With the right set of special cursor types are implemented, one cursor could +return the SQL result set. That is, combining cursors would implement a +query plan. And that could happen over a socket, mixing local processing +with server-side processing. + +This same interface could deal with many types of compression, transforming +data on the fly in various ways (like views), and so on. + */ diff --git a/docs/src/threads.dox b/docs/src/threads.dox new file mode 100644 index 00000000000..36c7a2c7f0c --- /dev/null +++ b/docs/src/threads.dox @@ -0,0 +1,37 @@ +/*! @page threads Multithreading + +WT_CONNECTION handles can be shared between threads, and applications +generally only open one connection to a given database per process. All +methods on WT_CONNECTION are thread safe. + +WT_SESSION and WT_CURSOR handles cannot be shared between threads concurrently: +the usual approach is for applications to open one WT_SESSION for each thread +that accesses a database. Applications may open multiple WT_CURSOR handles +within a session. + +Multiple threads must not access a session handle concurrently (including +accessing two or more cursor handles in the same session). However, +WT_SESSION handles may be accessed by different threads serially (for +example, from a pool of threads managed by the application with a set of +shared session handles). There is no thread-local state in WiredTiger, but +no built-in synchronization of session state either, so if multiple threads +access a session handle or dependencies such as cursors, the access must be +serialized by the application. + +@section threads_example Code samples + +The code below is taken from the complete example program +@ex_ref{ex_thread.c}. + +This is an example of a thread entry point. A new session is opened for +the thread and used for all operations within that thread. + +@snippet ex_thread.c thread scan + +Here is the main function that starts the threads. It opens a single +connection, shared between the threads, and closes the connection after +waiting for all of the threads to exit. + +@snippet ex_thread.c thread main + + */ diff --git a/docs/src/transactions.dox b/docs/src/transactions.dox new file mode 100644 index 00000000000..cee780a8fab --- /dev/null +++ b/docs/src/transactions.dox @@ -0,0 +1,82 @@ +/*! @page transactions Transactions + +@notyet{transactions} +<b>This page describes the expected behavior of the 2.X releases.</b> + +@section transactions_acid ACID properties + +Transactions provide a powerful abstraction for multiple threads to operate +on data concurrently because they have the following properties: + +- Atomicity: all or none of a transaction is completed. +- Consistency: if each transaction maintains some property when considered + separately, then the combined effect of executing the transactions + concurrently will maintain the same property. +- Isolation: developers can reason about transactions independently. +- Durability: once a transaction commits, its updates are saved. + +@section transactions_api Transactional API + +To configure for transactions, the database must be created with +transaction support enabled. This is done by passing the configuration +string <code>"transactional"</code> to ::wiredtiger_open when creating the +database. + +In WiredTiger, the transactional context is managed by the WT_SESSION +class. Applications call WT_SESSION::begin_transaction to start a new +transaction, which is only permitted when no cursors are open. Operations +performed with that WT_SESSION handle are then part of the transaction, and +their effects can be committed by calling WT_SESSION::commit_transaction or +WT_SESSION::rollback_transaction, both of which implicitly close any open +cursors. + +When transactions are used, operations may fail with additional errors such +as ::WT_DEADLOCK. + +@todo describe transaction error cases more fully + +@section transactions_cc Concurrency control + +WiredTiger uses an optimistic concurrency control algorithm. This avoids +the bottleneck of a centralized lock manager and expensive graph searching +to identify deadlock cycles. + +@section transaction_isolation Isolation levels + +The default isolation level is <code>serializable</code>, which means that +the concurrent execution of committed transactions is equivalent to +executing the transactions in some serial order. + +Weaker isolation levels are also provided, including <code>repeatable +read</code>, which permits phantoms, <code>snapshot isolation</code>, which +permits write skew, <code>read committed</code>, which permits lost updates +and always returns the most recently committed changes, and <code>read +uncommitted</code>, which always reads the most recent version of data, +regardless of whether it is committed. + +@section transaction_recovery Recovery + +Recovery is run automatically during ::wiredtiger_open when required. + +Recovery works by using a database log that contains a record of the +actions of all transactions. Recovery first finds the last complete +checkpoint, and then scans forward through the log from that point to +determine which transactions committed after the checkpoint. All actions +are rolled forward from the checkpoint so that the in-memory tree matches +its state when the crash occurred. Then the "losing" transactions (those +that did not commit) are rolled back to return the database to a consistent +state. + +This suggests the importance of regular checkpoints: they limit the amount +of work required during recovery, which speeds up the ::wiredtiger_open +call. See WT_SESSION::checkpoint for information about triggering +checkpoints. + +@section transaction_example Code samples + +The code below is taken from the complete example program +@ex_ref{ex_transaction.c}. + +@snippet ex_transaction.c transaction + + */ diff --git a/docs/src/tuning.dox b/docs/src/tuning.dox new file mode 100644 index 00000000000..6c08f5830ea --- /dev/null +++ b/docs/src/tuning.dox @@ -0,0 +1,120 @@ +/*! @page tuning Performance Tuning + +@section cache Cache size + +The cache size for the database is configurable by setting the \c +cache_size configuration string when calling the ::wiredtiger_open +function. + +The effectiveness of the cache can be measured by reviewing the page +eviction statistics for the database. + +An example of setting a cache size to 10MB: + +@snippet ex_config.c configure cache size + +@section page Page and overflow sizes + +There are four page and item size configuration values: \c internal_page_max, +\c internal_item_max, \c leaf_page_max and \c leaf_item_max. All four should +be specified to the WT_SESSION::create method, that is, they are configurable +on a per-file basis. + +The \c internal_page_max and \c leaf_page_max configuration values specify +the maximum size for Btree internal and leaf pages. That is, when an +internal or leaf page reaches the specified size, it splits into two pages. +Generally, internal pages should be sized to fit into the system's L1 or L2 +caches in order to minimize cache misses when searching the tree, while leaf +pages should be sized to maximize I/O performance (if reading from disk is +necessary, it is usually desirable to read a large amount of data, assuming +some locality of reference in the application's access pattern). + +The \c internal_item_max and \c leaf_item_max configuration values specify +the maximum size at which an object will be stored on-page. Larger items +will be stored separately in the file from the page where the item logically +appears. Referencing overflow items is more expensive than referencing +on-page items, requiring additional I/O if the object is not already cached. +For this reason, it is important to avoid creating large numbers of overflow +items that are repeatedly referenced, and the maximum item size should +probably be increased if many overflow items are being created. Because +pages must be large enough to store any item that is not an overflow item, +increasing the size of the overflow items may also require increasing the +page sizes. + +With respect to compression, page and item sizes do not necessarily reflect +the actual size of the page or item on disk, if block compression has been +configured. Block compression in WiredTiger happens within the disk I/O +subsystem, and so a page might split even if subsequent compression would +result in a resulting page size that would be small enough to leave as a +single page. In other words, page and overflow sizes are based on in-memory +sizes, not disk sizes. + +There are two other, related configuration values, also settable by the +WT_SESSION::create method. They are \c allocation_size, and \c split_pct. + +The \c allocation_size configuration value is the underlying unit of +allocation for the file. As the unit of file allocation, it has two +effects: first, it limits the ultimate size of the file, and second, it +determines how much space is wasted when storing overflow items. + +By limiting the size of the file, the allocation size limits the amount +of data that can be stored in a file. For example, if the allocation +size is set to the minimum possible (512B), the maximum file size is +2TB, that is, attempts to allocate new file blocks will fail when the +file reaches 2TB in size. If the allocation size is set to the maximum +possible (512MB), the maximum file size is 2EB. + +The unit of allocation also determines how much space is wasted when +storing overflow items. For example, if the allocation size were set +to the minimum value of 512B, an overflow item of 1100 bytes would +require 3 allocation sized file units, or 1536 bytes, wasting almost 500 +bytes. For this reason, as the allocation size increases, page sizes +and overflow item sizes will likely increase as well, to ensure that +significant space isn't wasted by overflow items. + +The last configuration value is \c split_pct, which configures the size +of a split page. When a page grows sufficiently large that it must be +split, the newly split page's size is \c split_pct percent of the +maximum page size. This value should be selected to avoid creating a +large number of tiny pages or repeatedly splitting whenever new entries +are inserted. For example, if the maximum page size is 1MB, a \c +split_pct value of 10% would potentially result in creating a large +number of 10KB pages, which may not be optimal for future I/O. Or, if +the maximum page size is 1MB, a \c split_pct value of 90% would +potentially result in repeatedly splitting pages as the split pages grow +to 1MB over and over. The default value for \c split_pct is 75%, +intended to keep large pages relatively large, while still giving split +pages room to grow. + +An example of configuring page sizes: + +@snippet ex_file.c file create + +@section statistics Performance monitoring with statistics + +WiredTiger maintains a variety of statistics that can be read with a +cursor. The statistics are at two levels: per-database statistics and +per-underlying file statistics. The database statistics are the +default; the underlying file statistics are available by specifying the +appropriate uri. + +The following is an example of printing run-time statistics about the +WiredTiger engine: + +@snippet ex_stat.c statistics database function + +The following is an example of printing statistics about an underlying +file: + +@snippet ex_stat.c statistics file function + +Both examples can use a common display routine that iterates through the +statistics until the cursor returns the end of the list. + +@snippet ex_stat.c statistics display function + +Note the raw value of the statistic is available from the \c value +field, as well as a printable string version of the value. + + + */ diff --git a/docs/src/using.dox b/docs/src/using.dox new file mode 100644 index 00000000000..1b0307bbfe1 --- /dev/null +++ b/docs/src/using.dox @@ -0,0 +1,23 @@ +/*! @page using Writing WiredTiger applications + +This section explains how to use WiredTiger by developing a sequence of +example programs: + +- @subpage basic_api +- @subpage config_strings +- @subpage schema +- @subpage cursors +- @subpage threads +- @subpage transactions + +Additional Notes: + +- @subpage home +- @subpage security +- @subpage file_formats +- @subpage compression +- @subpage name_space +- @subpage signals +- @subpage tuning + + */ diff --git a/docs/style/DoxygenLayout.xml b/docs/style/DoxygenLayout.xml new file mode 100644 index 00000000000..5654df6d7a7 --- /dev/null +++ b/docs/style/DoxygenLayout.xml @@ -0,0 +1,189 @@ +<doxygenlayout version="1.0"> + <!-- Navigation index tabs for HTML output --> + <navindex> + <tab type="mainpage" visible="yes" title=""/> + <tab type="pages" visible="yes" title="" intro=""/> + <tab type="modules" visible="yes" title="" intro=""/> + <tab type="namespaces" visible="yes" title=""> + <tab type="namespacelist" visible="yes" title="" intro=""/> + <tab type="namespacemembers" visible="yes" title="" intro=""/> + </tab> + <tab type="classes" visible="no" title=""> + <tab type="classlist" visible="yes" title="" intro=""/> + <tab type="classindex" visible="$ALPHABETICAL_INDEX" title=""/> + <tab type="hierarchy" visible="yes" title="" intro=""/> + <tab type="classmembers" visible="yes" title="" intro=""/> + </tab> + <tab type="files" visible="yes" title=""> + <tab type="filelist" visible="yes" title="" intro=""/> + <tab type="globals" visible="yes" title="" intro=""/> + </tab> + <tab type="dirs" visible="yes" title="" intro=""/> + <tab type="examples" visible="yes" title="" intro=""/> + <tab type="user" url="https://github.com/wiredtiger/wiredtiger/downloads/" visible="yes" title="Download"/> + </navindex> + + <!-- Layout definition for a class page --> + <class> + <briefdescription visible="yes"/> + <includes visible="$SHOW_INCLUDE_FILES"/> + <inheritancegraph visible="$CLASS_GRAPH"/> + <collaborationgraph visible="$COLLABORATION_GRAPH"/> + <allmemberslink visible="yes"/> + <memberdecl> + <nestedclasses visible="yes" title=""/> + <publictypes title=""/> + <publicslots title=""/> + <signals title=""/> + <publicmethods title=""/> + <publicstaticmethods title=""/> + <publicattributes title=""/> + <publicstaticattributes title=""/> + <protectedtypes title=""/> + <protectedslots title=""/> + <protectedmethods title=""/> + <protectedstaticmethods title=""/> + <protectedattributes title=""/> + <protectedstaticattributes title=""/> + <packagetypes title=""/> + <packagemethods title=""/> + <packagestaticmethods title=""/> + <packageattributes title=""/> + <packagestaticattributes title=""/> + <properties title=""/> + <events title=""/> + <privatetypes title=""/> + <privateslots title=""/> + <privatemethods title=""/> + <privatestaticmethods title=""/> + <privateattributes title=""/> + <privatestaticattributes title=""/> + <friends title=""/> + <related title="" subtitle=""/> + <membergroups visible="yes"/> + </memberdecl> + <detaileddescription title=""/> + <memberdef> + <inlineclasses title=""/> + <typedefs title=""/> + <enums title=""/> + <constructors title=""/> + <functions title=""/> + <related title=""/> + <variables title=""/> + <properties title=""/> + <events title=""/> + </memberdef> + <usedfiles visible="$SHOW_USED_FILES"/> + <authorsection visible="yes"/> + </class> + + <!-- Layout definition for a namespace page --> + <namespace> + <briefdescription visible="yes"/> + <memberdecl> + <nestednamespaces visible="yes" title=""/> + <classes visible="yes" title=""/> + <typedefs title=""/> + <enums title=""/> + <functions title=""/> + <variables title=""/> + <membergroups visible="yes"/> + </memberdecl> + <detaileddescription title=""/> + <memberdef> + <inlineclasses title=""/> + <typedefs title=""/> + <enums title=""/> + <functions title=""/> + <variables title=""/> + </memberdef> + <authorsection visible="yes"/> + </namespace> + + <!-- Layout definition for a file page --> + <file> + <briefdescription visible="yes"/> + <includes visible="$SHOW_INCLUDE_FILES"/> + <includegraph visible="$INCLUDE_GRAPH"/> + <includedbygraph visible="$INCLUDED_BY_GRAPH"/> + <sourcelink visible="yes"/> + <memberdecl> + <classes visible="yes" title=""/> + <namespaces visible="yes" title=""/> + <defines title=""/> + <typedefs title=""/> + <enums title=""/> + <functions title=""/> + <variables title=""/> + <membergroups visible="yes"/> + </memberdecl> + <detaileddescription title=""/> + <memberdef> + <inlineclasses title=""/> + <defines title=""/> + <typedefs title=""/> + <enums title=""/> + <functions title=""/> + <variables title=""/> + </memberdef> + <authorsection/> + </file> + + <!-- Layout definition for a group page --> + <group> + <briefdescription visible="yes"/> + <groupgraph visible="$GROUP_GRAPHS"/> + <memberdecl> + <classes visible="yes" title=""/> + <namespaces visible="yes" title=""/> + <dirs visible="yes" title=""/> + <nestedgroups visible="yes" title=""/> + <files visible="yes" title=""/> + <defines title=""/> + <typedefs title=""/> + <enums title=""/> + <enumvalues title=""/> + <functions title=""/> + <variables title=""/> + <signals title=""/> + <publicslots title=""/> + <protectedslots title=""/> + <privateslots title=""/> + <events title=""/> + <properties title=""/> + <friends title=""/> + <membergroups visible="yes"/> + </memberdecl> + <detaileddescription title=""/> + <memberdef> + <pagedocs/> + <inlineclasses title=""/> + <defines title=""/> + <typedefs title=""/> + <enums title=""/> + <enumvalues title=""/> + <functions title=""/> + <variables title=""/> + <signals title=""/> + <publicslots title=""/> + <protectedslots title=""/> + <privateslots title=""/> + <events title=""/> + <properties title=""/> + <friends title=""/> + </memberdef> + <authorsection visible="yes"/> + </group> + + <!-- Layout definition for a directory page --> + <directory> + <briefdescription visible="yes"/> + <directorygraph visible="yes"/> + <memberdecl> + <dirs visible="yes"/> + <files visible="yes"/> + </memberdecl> + <detaileddescription title=""/> + </directory> +</doxygenlayout> diff --git a/docs/style/background_navigation.png b/docs/style/background_navigation.png Binary files differnew file mode 100644 index 00000000000..d59e96ddef6 --- /dev/null +++ b/docs/style/background_navigation.png diff --git a/docs/style/doxygen.png b/docs/style/doxygen.png Binary files differnew file mode 100644 index 00000000000..f0a274bbaff --- /dev/null +++ b/docs/style/doxygen.png diff --git a/docs/style/footer.html b/docs/style/footer.html new file mode 100644 index 00000000000..cf0bb962e6f --- /dev/null +++ b/docs/style/footer.html @@ -0,0 +1,11 @@ +<!--BEGIN GENERATE_TREEVIEW--> + <li class="footer">Copyright (c) 2008-2012 WiredTiger, Inc. All rights reserved. Contact <a href="mailto:info@wiredtiger.com">info@wiredtiger.com</a> for more information.</li> + </ul> + </div> +<!--END GENERATE_TREEVIEW--> +<!--BEGIN !GENERATE_TREEVIEW--> +<hr class="footer"/><address class="footer"><small> +Copyright (c) 2008-2012 WiredTiger. All rights reserved. Contact <a href="mailto:info@wiredtiger.com">info@wiredtiger.com</a> for more information.</small></address> +<!--END !GENERATE_TREEVIEW--> +</body> +</html> diff --git a/docs/style/header.html b/docs/style/header.html new file mode 100644 index 00000000000..b498213ef9e --- /dev/null +++ b/docs/style/header.html @@ -0,0 +1,50 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml"> +<head> +<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/> +<meta http-equiv="X-UA-Compatible" content="IE=9"/> +<!--BEGIN PROJECT_NAME--><title>$projectname: $title</title><!--END PROJECT_NAME--> +<!--BEGIN !PROJECT_NAME--><title>$title</title><!--END !PROJECT_NAME--> +<link href="$relpath$tabs.css" rel="stylesheet" type="text/css"/> +<link href="$relpath$wiredtiger.css" rel="stylesheet" type="text/css" /> +$treeview +$search +$mathjax +</head> +<body> +<div id="top"><!-- do not remove this div! --> +<!--BEGIN TITLEAREA--> +<div id="titlearea"> +<table cellspacing="0" cellpadding="0"> + <tbody> + <tr style="height: 56px;"> + <!--BEGIN PROJECT_LOGO--> + <td id="projectlogo"> + <div class="logo"><a href="http://wiredtiger.com/"><img src="$relpath$images/LogoFinal-header.png" alt="WiredTiger" /></a></div> + </td> + <!--END PROJECT_LOGO--> + <!--BEGIN PROJECT_NAME--> + <td style="padding-left: 0.5em;"> + <div id="projectname"> + <!--BEGIN PROJECT_NUMBER--> <span id="projectnumber">$projectnumber</span><!--END PROJECT_NUMBER--> + </div> + <!--BEGIN PROJECT_BRIEF--><div id="projectbrief">$projectbrief</div><!--END PROJECT_BRIEF--> + </td> + <!--END PROJECT_NAME--> + <!--BEGIN !PROJECT_NAME--> + <!--BEGIN PROJECT_BRIEF--> + <td style="padding-left: 0.5em;"> + <div id="projectbrief">$projectbrief</div> + </td> + <!--END PROJECT_BRIEF--> + <!--END !PROJECT_NAME--> + <!--BEGIN DISABLE_INDEX--> + <!--BEGIN SEARCHENGINE--> + <td>$searchbox</td> + <!--END SEARCHENGINE--> + <!--END DISABLE_INDEX--> + </tr> + </tbody> +</table> +</div> +<!--END TITLEAREA--> diff --git a/docs/style/img_downArrow.png b/docs/style/img_downArrow.png Binary files differnew file mode 100644 index 00000000000..024a03c2654 --- /dev/null +++ b/docs/style/img_downArrow.png diff --git a/docs/style/javadoc.css b/docs/style/javadoc.css new file mode 100644 index 00000000000..f345b05b158 --- /dev/null +++ b/docs/style/javadoc.css @@ -0,0 +1,35 @@ +/* Javadoc style sheet */ + +/* Define colors, fonts and other style attributes here to override the defaults */ + +/* Page background color */ +body { background-color: #FFFFFF } + +/* Table colors */ +.TableHeadingColor { background: #CCCCFF } /* Dark mauve */ +.TableSubHeadingColor { background: #EEEEFF } /* Light mauve */ +.TableRowColor { background: #FFFFFF } /* White */ + +/* Font used in left-hand frame lists */ +.FrameTitleFont { font-size: 10pts; font-family: Helvetica, Arial, san-serif } +.FrameHeadingFont { font-size: 10pts; font-family: Helvetica, Arial, san-serif } +.FrameItemFont { font-size: 10pts; font-family: Helvetica, Arial, san-serif } + +/* Example of smaller, sans-serif font in frames */ +/* .FrameItemFont { font-size: 10pt; font-family: Helvetica, Arial, sans-serif } */ + +/* Navigation bar fonts and colors */ +.NavBarCell1 { background-color:#EEEEFF;} /* Red */ +.NavBarCell1Rev { background-color:#00008B;} /* Light red */ +.NavBarFont1 { font-family: Arial, Helvetica, sans-serif; color:#000000;} +.NavBarFont1Rev { font-family: Arial, Helvetica, sans-serif; color:#FFFFFF;} + +.NavBarCell2 { font-family: Arial, Helvetica, sans-serif; background-color:#FFFFFF;} +.NavBarCell3 { font-family: Arial, Helvetica, sans-serif; background-color:#FFFFFF;} + +pre.code +{ + border: solid thin; + padding: 2px; + background-color: #ffffcc; +} diff --git a/docs/style/tabs.css b/docs/style/tabs.css new file mode 100644 index 00000000000..c543eb00429 --- /dev/null +++ b/docs/style/tabs.css @@ -0,0 +1,128 @@ +/* tabs styles, based on http://www.alistapart.com/articles/slidingdoors */ + +div.navigation { + display:block; + float:left; + clear:both; + width:100%; + padding-top:10px; + padding-bottom:30px; + border-bottom:1px dotted #E0E0E0; + height: 60px; + background:url(background_navigation.png); +} + +div.contents { + float:left; + clear:both; + +} + +address { + display:none; +} + +DIV.tabs +{ + float : left; + width : 100%; + margin-bottom : 4px; +} + +DIV.tabs UL +{ + margin : 0px; + padding-left : 0px; + list-style : none; +} + +DIV.tabs LI, DIV.tabs FORM +{ + display : inline; + margin : 0px; + padding : 0px; + background-color:#FFFFFF; +} + +DIV.tabs FORM +{ + float : right; +} + +DIV.tabs A +{ + float : left; + font-size : 80%; + font-weight : bold; + text-decoration : none; +} + +DIV.tabs A:hover +{ + background-position: 100% -150px; +} + +DIV.tabs A:link, DIV.tabs A:visited, +DIV.tabs A:active, DIV.tabs A:hover +{ + color: #1A419D; +} + +DIV.tabs SPAN +{ + float : left; + display : block; + padding : 5px 9px; + white-space : nowrap; + border-top:1px dotted #E0E0E0; + border-bottom:1px dotted #E0E0E0; +} + +DIV.tabs #MSearchBox +{ + float : right; + display : inline; + font-size : 1em; +} + +DIV.tabs TD +{ + font-size : 80%; + font-weight : bold; + text-decoration : none; +} + + + +/* Commented Backslash Hack hides rule from IE5-Mac \*/ +DIV.tabs SPAN {float : none;} +/* End IE5-Mac hack */ + +DIV.tabs A:hover SPAN +{ + background-position: 0% -150px; +} + +DIV.tabs LI.current A +{ + background-position: 100% -150px; + border-width : 0px; + background-color: #F0F0F0; + background: #F0F0F0 url(img_downArrow.png) center bottom no-repeat; +} + +DIV.tabs LI.current SPAN +{ + background-position: 0% -150px; + padding-bottom : 6px; +} + +DIV.navpath +{ + background : none; + border : none; + border-bottom : 1px solid #84B0C7; + text-align : center; + margin : 2px; + padding : 2px; +} diff --git a/docs/style/wiredtiger.css b/docs/style/wiredtiger.css new file mode 100644 index 00000000000..000e0d9ad4c --- /dev/null +++ b/docs/style/wiredtiger.css @@ -0,0 +1,948 @@ +/* The standard CSS for doxygen */ + +body, table, div, p, dl { + font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; + color: #373737; + font-size: 13px; + line-height: 1.3; +} + +/* @group Heading Levels */ + +h1 { + font-size: 150%; +} + +.title { + font-size: 150%; + font-weight: bold; + margin: 10px 2px; +} + +h2 { + font-size: 120%; +} + +h3 { + font-size: 100%; +} + +dt { + font-weight: bold; +} + +div.multicol { + -moz-column-gap: 1em; + -webkit-column-gap: 1em; + -moz-column-count: 3; + -webkit-column-count: 3; +} + +p.startli, p.startdd, p.starttd { + margin-top: 2px; +} + +p.endli { + margin-bottom: 0px; +} + +p.enddd { + margin-bottom: 4px; +} + +p.endtd { + margin-bottom: 2px; +} + +/* @end */ + +caption { + font-weight: bold; +} + +span.legend { + font-size: 70%; + text-align: center; +} + +h3.version { + font-size: 90%; + text-align: center; +} + +div.qindex, div.navtab{ + background-color: #EBEFF6; + border: 1px solid #A3B4D7; + text-align: center; +} + +div.qindex, div.navpath { + width: 100%; + line-height: 140%; +} + +div.navtab { + margin-right: 15px; +} + +/* @group Link Styling */ + +a { + color: #3D578C; + font-weight: normal; + text-decoration: none; +} + +.contents a:visited { + color: #4665A2; +} + +a:hover { + text-decoration: underline; +} + +a.qindex { + font-weight: bold; +} + +a.qindexHL { + font-weight: bold; + background-color: #9CAFD4; + color: #ffffff; + border: 1px double #869DCA; +} + +.contents a.qindexHL:visited { + color: #ffffff; +} + +a.el { + font-weight: bold; +} + +a.elRef { +} + +a.code { + color: #4665A2; +} + +a.codeRef { + color: #4665A2; +} + +/* @end */ + +dl.el { + margin-left: -1cm; +} + +.fragment { + font-family: monospace, fixed; + font-size: 105%; +} + +pre.fragment { + border: 1px solid #C4CFE5; + background-color: #FBFCFD; + padding: 4px 6px; + margin: 4px 8px 4px 2px; + overflow: auto; + word-wrap: break-word; + font-size: 9pt; + line-height: 125%; +} + +div.ah { + background-color: black; + font-weight: bold; + color: #ffffff; + margin-bottom: 3px; + margin-top: 3px; + padding: 0.2em; + border: solid thin #333; + border-radius: 0.5em; + -webkit-border-radius: .5em; + -moz-border-radius: .5em; + box-shadow: 2px 2px 3px #999; + -webkit-box-shadow: 2px 2px 3px #999; + -moz-box-shadow: rgba(0, 0, 0, 0.15) 2px 2px 2px; + background-image: -webkit-gradient(linear, left top, left bottom, from(#eee), to(#000),color-stop(0.3, #444)); + background-image: -moz-linear-gradient(center top, #eee 0%, #444 40%, #000); +} + +div.groupHeader { + margin-left: 16px; + margin-top: 12px; + font-weight: bold; +} + +div.groupText { + margin-left: 16px; + font-style: italic; +} + +body { + background-color: white; + color: black; + margin: 0; +} + +div.contents { + margin-top: 10px; + margin-left: 8px; + margin-right: 8px; +} + +td.indexkey { + background-color: #EBEFF6; + font-weight: bold; + border: 1px solid #C4CFE5; + margin: 2px 0px 2px 0; + padding: 2px 10px; +} + +td.indexvalue { + background-color: #EBEFF6; + border: 1px solid #C4CFE5; + padding: 2px 10px; + margin: 2px 0px; +} + +tr.memlist { + background-color: #EEF1F7; +} + +p.formulaDsp { + text-align: center; +} + +img.formulaDsp { + +} + +img.formulaInl { + vertical-align: middle; +} + +div.center { + text-align: center; + margin-top: 0px; + margin-bottom: 0px; + padding: 0px; +} + +div.center img { + border: 0px; +} + +address.footer { + text-align: right; + padding-right: 12px; +} + +img.footer { + border: 0px; + vertical-align: middle; +} + +/* @group Code Colorization */ + +span.keyword { + color: #008000 +} + +span.keywordtype { + color: #604020 +} + +span.keywordflow { + color: #e08000 +} + +span.comment { + color: #800000 +} + +span.preprocessor { + color: #806020 +} + +span.stringliteral { + color: #002080 +} + +span.charliteral { + color: #008080 +} + +span.vhdldigit { + color: #ff00ff +} + +span.vhdlchar { + color: #000000 +} + +span.vhdlkeyword { + color: #700070 +} + +span.vhdllogic { + color: #ff0000 +} + +/* @end */ + +/* +.search { + color: #003399; + font-weight: bold; +} + +form.search { + margin-bottom: 0px; + margin-top: 0px; +} + +input.search { + font-size: 75%; + color: #000080; + font-weight: normal; + background-color: #e8eef2; +} +*/ + +td.tiny { + font-size: 75%; +} + +.dirtab { + padding: 4px; + border-collapse: collapse; + border: 1px solid #A3B4D7; +} + +th.dirtab { + background: #EBEFF6; + font-weight: bold; +} + +hr { + height: 0px; + border: none; + border-top: 1px solid #4A6AAA; +} + +hr.footer { + height: 1px; +} + +/* @group Member Descriptions */ + +table.memberdecls { + border-spacing: 0px; + padding: 0px; +} + +.mdescLeft, .mdescRight, +.memItemLeft, .memItemRight, +.memTemplItemLeft, .memTemplItemRight, .memTemplParams { + background-color: #F9FAFC; + border: none; + margin: 4px; + padding: 1px 0 0 8px; +} + +.mdescLeft, .mdescRight { + padding: 0px 8px 4px 8px; + color: #555; +} + +.memItemLeft, .memItemRight, .memTemplParams { + border-top: 1px solid #C4CFE5; +} + +.memItemLeft, .memTemplItemLeft { + white-space: nowrap; +} + +.memItemRight { + width: 100%; +} + +.memTemplParams { + color: #4665A2; + white-space: nowrap; +} + +/* @end */ + +/* @group Member Details */ + +/* Styles for detailed member documentation */ + +.memtemplate { + font-size: 80%; + color: #4665A2; + font-weight: normal; + margin-left: 9px; +} + +.memnav { + background-color: #EBEFF6; + border: 1px solid #A3B4D7; + text-align: center; + margin: 2px; + margin-right: 15px; + padding: 2px; +} + +.mempage { + width: 100%; +} + +.memitem { + padding: 0; + margin-bottom: 10px; + margin-right: 5px; +} + +.memname { + white-space: nowrap; + font-weight: bold; + margin-left: 6px; +} + +.memproto, dl.reflist dt { + border-top: 1px solid #A8B8D9; + border-left: 1px solid #A8B8D9; + border-right: 1px solid #A8B8D9; + padding: 6px 0px 6px 0px; + color: #253555; + font-weight: bold; + text-shadow: 0px 1px 1px rgba(255, 255, 255, 0.9); + /* opera specific markup */ + box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); + border-top-right-radius: 8px; + border-top-left-radius: 8px; + /* firefox specific markup */ + -moz-box-shadow: rgba(0, 0, 0, 0.15) 5px 5px 5px; + -moz-border-radius-topright: 8px; + -moz-border-radius-topleft: 8px; + /* webkit specific markup */ + -webkit-box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); + -webkit-border-top-right-radius: 8px; + -webkit-border-top-left-radius: 8px; + background-image:url('nav_f.png'); + background-repeat:repeat-x; + background-color: #EEE7DD; + +} + +.memdoc, dl.reflist dd { + border-bottom: 1px solid #A8B8D9; + border-left: 1px solid #A8B8D9; + border-right: 1px solid #A8B8D9; + padding: 2px 5px; + background-color: #FBFCFD; + border-top-width: 0; + /* opera specific markup */ + border-bottom-left-radius: 8px; + border-bottom-right-radius: 8px; + box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); + /* firefox specific markup */ + -moz-border-radius-bottomleft: 8px; + -moz-border-radius-bottomright: 8px; + -moz-box-shadow: rgba(0, 0, 0, 0.15) 5px 5px 5px; +/* background-image: -moz-linear-gradient(center top, #FFFFFF 0%, #FFFFFF 60%, #FBF9F7 95%, #F7F3EE); */ + /* webkit specific markup */ + -webkit-border-bottom-left-radius: 8px; + -webkit-border-bottom-right-radius: 8px; + -webkit-box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); +/* background-image: -webkit-gradient(linear,center top,center bottom,from(#FFFFFF), color-stop(0.6,#FFFFFF), color-stop(0.60,#FFFFFF), color-stop(0.95,#FBF9F7), to(#F7F3EE)); */ +} + +dl.reflist dt { + padding: 5px; +} + +dl.reflist dd { + margin: 0px 0px 10px 0px; + padding: 5px; +} + +.paramkey { + text-align: right; +} + +.paramtype { + white-space: nowrap; +} + +.paramname { + color: #602020; + white-space: nowrap; +} +.paramname em { + font-style: normal; +} + +.params, .retval, .exception, .tparams { + border-spacing: 6px 2px; +} + +.params .paramname, .retval .paramname { + font-weight: bold; + vertical-align: top; +} + +.params .paramtype { + font-style: italic; + vertical-align: top; +} + +.params .paramdir { + font-family: "courier new",courier,monospace; + vertical-align: top; +} + + + + +/* @end */ + +/* @group Directory (tree) */ + +/* for the tree view */ + +.ftvtree { + font-family: sans-serif; + margin: 0px; +} + +/* these are for tree view when used as main index */ + +.directory { + font-size: 9pt; + font-weight: bold; + margin: 5px; +} + +.directory h3 { + margin: 0px; + margin-top: 1em; + font-size: 11pt; +} + +/* +The following two styles can be used to replace the root node title +with an image of your choice. Simply uncomment the next two styles, +specify the name of your image and be sure to set 'height' to the +proper pixel height of your image. +*/ + +/* +.directory h3.swap { + height: 61px; + background-repeat: no-repeat; + background-image: url("yourimage.gif"); +} +.directory h3.swap span { + display: none; +} +*/ + +.directory > h3 { + margin-top: 0; +} + +.directory p { + margin: 0px; + white-space: nowrap; +} + +.directory div { + display: none; + margin: 0px; +} + +.directory img { + vertical-align: -30%; +} + +/* these are for tree view when not used as main index */ + +.directory-alt { + font-size: 100%; + font-weight: bold; +} + +.directory-alt h3 { + margin: 0px; + margin-top: 1em; + font-size: 11pt; +} + +.directory-alt > h3 { + margin-top: 0; +} + +.directory-alt p { + margin: 0px; + white-space: nowrap; +} + +.directory-alt div { + display: none; + margin: 0px; +} + +.directory-alt img { + vertical-align: -30%; +} + +/* @end */ + +div.dynheader { + margin-top: 8px; +} + +address { + font-style: normal; + color: #61482A; +} + +table.doxtable { + border-collapse:collapse; +} + +table.doxtable td, table.doxtable th { + border: 1px solid #684C2D; + padding: 3px 7px 2px; +} + +table.doxtable th { + background-color: #7F5E37; + color: #FFFFFF; + font-size: 110%; + padding-bottom: 4px; + padding-top: 5px; + text-align:left; +} + +table.fieldtable { + width: 100%; + margin-bottom: 10px; + border: 1px solid #D9C2A8; + border-spacing: 0px; + -moz-border-radius: 4px; + -webkit-border-radius: 4px; + border-radius: 4px; + -moz-box-shadow: rgba(0, 0, 0, 0.15) 2px 2px 2px; + -webkit-box-shadow: 2px 2px 2px rgba(0, 0, 0, 0.15); + box-shadow: 2px 2px 2px rgba(0, 0, 0, 0.15); +} + +.fieldtable td, .fieldtable th { + padding: 3px 7px 2px; +} + +.fieldtable td.fieldtype, .fieldtable td.fieldname { + white-space: nowrap; + border-right: 1px solid #D9C2A8; + border-bottom: 1px solid #D9C2A8; + vertical-align: top; +} + +.fieldtable td.fielddoc { + border-bottom: 1px solid #D9C2A8; + width: 100%; +} + +.fieldtable tr:last-child td { + border-bottom: none; +} + +.fieldtable th { + background-image:url('nav_f.png'); + background-repeat:repeat-x; + background-color: #F2EBE2; + font-size: 90%; + color: #553E25; + padding-bottom: 4px; + padding-top: 5px; + text-align:left; + -moz-border-radius-topleft: 4px; + -moz-border-radius-topright: 4px; + -webkit-border-top-left-radius: 4px; + -webkit-border-top-right-radius: 4px; + border-top-left-radius: 4px; + border-top-right-radius: 4px; + border-bottom: 1px solid #D9C2A8; +} + + +.tabsearch { + top: 0px; + left: 10px; + height: 36px; + background-image: url('tab_b.png'); + z-index: 101; + overflow: hidden; + font-size: 13px; +} + +.navpath ul +{ + font-size: 11px; + background-image:url('tab_b.png'); + background-repeat:repeat-x; + height:30px; + line-height:30px; + color:#CCAD8A; + border:solid 1px #E4D4C2; + overflow:hidden; + margin:0px; + padding:0px; +} + +.navpath li +{ + list-style-type:none; + float:left; + padding-left:10px; + padding-right:15px; + background-image:url('bc_s.png'); + background-repeat:no-repeat; + background-position:right; + color:#7C5B36; +} + +.navpath li.navelem a +{ + height:32px; + display:block; + text-decoration: none; + outline: none; +} + +.navpath li.navelem a:hover +{ + color:#BD9568; +} + +.navpath li.footer +{ + list-style-type:none; + float:right; + padding-left:10px; + padding-right:15px; + background-image:none; + background-repeat:no-repeat; + background-position:right; + color:#7C5B36; + font-size: 8pt; +} + + +div.summary +{ + float: right; + font-size: 8pt; + padding-right: 5px; + width: 50%; + text-align: right; +} + +div.summary a +{ + white-space: nowrap; +} + +div.ingroups +{ + margin-left: 5px; + font-size: 8pt; + padding-left: 5px; + width: 50%; + text-align: left; +} + +div.ingroups a +{ + white-space: nowrap; +} + +div.header +{ + background-image:url('nav_h.png'); + background-repeat:repeat-x; + background-color: #FCFBF9; + margin: 0px; + border-bottom: 1px solid #E5D6C4; +} + +div.headertitle, div.logo +{ + padding: 5px 5px 5px 10px; +} + +dl +{ + padding: 0 0 0 10px; +} + +dl.note, dl.warning, dl.attention, dl.pre, dl.post, dl.invariant, dl.deprecated, dl.todo, dl.test, dl.bug +{ + border-left:4px solid; + padding: 0 0 0 6px; +} + +dl.note +{ + border-color: #D0D000; +} + +dl.warning, dl.attention +{ + border-color: #FF0000; +} + +dl.pre, dl.post, dl.invariant +{ + border-color: #00D000; +} + +dl.deprecated +{ + border-color: #505050; +} + +dl.todo +{ + border-color: #00C0E0; +} + +dl.test +{ + border-color: #3030E0; +} + +dl.bug +{ + border-color: #5373B4; +} + +#projectlogo +{ + text-align: center; + vertical-align: bottom; + border-collapse: separate; +} + +#projectlogo img +{ + border: 0px none; +} + +#projectname +{ + font: 300% Tahoma, Arial,sans-serif; + margin: 0px; + padding: 2px 0px; +} + +#projectbrief +{ + font: 120% Tahoma, Arial,sans-serif; + margin: 0px; + padding: 0px; +} + +#projectnumber +{ + font: 50% Tahoma, Arial,sans-serif; + margin: 0px; + padding: 0px; +} + +#titlearea +{ + padding: 0px; + margin: 0px; + width: 100%; + border-bottom: 1px solid #B48753; +} + +.image +{ + text-align: center; +} + +.dotgraph +{ + text-align: center; +} + +.mscgraph +{ + text-align: center; +} + +.caption +{ + font-weight: bold; +} + +div.zoom +{ + border: 1px solid #CEB190; +} + +dl.citelist { + margin-bottom:50px; +} + +dl.citelist dt { + color:#755633; + float:left; + font-weight:bold; + margin-right:10px; + padding:5px; +} + +dl.citelist dd { + margin:2px 0; + padding:5px 0; +} + +@media print +{ + #top { display: none; } + #side-nav { display: none; } + #nav-path { display: none; } + body { overflow:visible; } + h1, h2, h3, h4, h5, h6 { page-break-after: avoid; } + .summary { display: none; } + .memitem { page-break-inside: avoid; } + #doc-content + { + margin-left:0 !important; + height:auto !important; + width:auto !important; + overflow:inherit; + display:inline; + } + pre.fragment + { + overflow: visible; + text-wrap: unrestricted; + white-space: -moz-pre-wrap; /* Moz */ + white-space: -pre-wrap; /* Opera 4-6 */ + white-space: -o-pre-wrap; /* Opera 7 */ + white-space: pre-wrap; /* CSS3 */ + word-wrap: break-word; /* IE 5.5+ */ + } +} + diff --git a/docs/tools/doxypy.py b/docs/tools/doxypy.py new file mode 100755 index 00000000000..54fef5f03a5 --- /dev/null +++ b/docs/tools/doxypy.py @@ -0,0 +1,414 @@ +#!/usr/bin/env python + +__applicationName__ = "doxypy" +__blurb__ = """ +doxypy is an input filter for Doxygen. It preprocesses python +files so that docstrings of classes and functions are reformatted +into Doxygen-conform documentation blocks. +""" + +__doc__ = __blurb__ + \ +""" +In order to make Doxygen preprocess files through doxypy, simply +add the following lines to your Doxyfile: + FILTER_SOURCE_FILES = YES + INPUT_FILTER = "python /path/to/doxypy.py" +""" + +__version__ = "0.4.2" +__date__ = "14th October 2009" +__website__ = "http://code.foosel.org/doxypy" + +__author__ = ( + "Philippe 'demod' Neumann (doxypy at demod dot org)", + "Gina 'foosel' Haeussge (gina at foosel dot net)" +) + +__licenseName__ = "GPL v2" +__license__ = """This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see <http://www.gnu.org/licenses/>. +""" + +import sys +import re + +from optparse import OptionParser, OptionGroup + +class FSM(object): + """Implements a finite state machine. + + Transitions are given as 4-tuples, consisting of an origin state, a target + state, a condition for the transition (given as a reference to a function + which gets called with a given piece of input) and a pointer to a function + to be called upon the execution of the given transition. + """ + + """ + @var transitions holds the transitions + @var current_state holds the current state + @var current_input holds the current input + @var current_transition hold the currently active transition + """ + + def __init__(self, start_state=None, transitions=[]): + self.transitions = transitions + self.current_state = start_state + self.current_input = None + self.current_transition = None + + def setStartState(self, state): + self.current_state = state + + def addTransition(self, from_state, to_state, condition, callback): + self.transitions.append([from_state, to_state, condition, callback]) + + def makeTransition(self, input): + """Makes a transition based on the given input. + + @param input input to parse by the FSM + """ + for transition in self.transitions: + [from_state, to_state, condition, callback] = transition + if from_state == self.current_state: + match = condition(input) + if match: + self.current_state = to_state + self.current_input = input + self.current_transition = transition + if options.debug: + print >>sys.stderr, "# FSM: executing (%s -> %s) for line '%s'" % (from_state, to_state, input) + callback(match) + return + +class Doxypy(object): + def __init__(self): + string_prefixes = "[uU]?[rR]?" + + self.start_single_comment_re = re.compile("^\s*%s(''')" % string_prefixes) + self.end_single_comment_re = re.compile("(''')\s*$") + + self.start_double_comment_re = re.compile("^\s*%s(\"\"\")" % string_prefixes) + self.end_double_comment_re = re.compile("(\"\"\")\s*$") + + self.single_comment_re = re.compile("^\s*%s(''').*(''')\s*$" % string_prefixes) + self.double_comment_re = re.compile("^\s*%s(\"\"\").*(\"\"\")\s*$" % string_prefixes) + + self.defclass_re = re.compile("^(\s*)(def .+:|class .+:)") + self.empty_re = re.compile("^\s*$") + self.hashline_re = re.compile("^\s*#.*$") + self.importline_re = re.compile("^\s*(import |from .+ import)") + + self.multiline_defclass_start_re = re.compile("^(\s*)(def|class)(\s.*)?$") + self.multiline_defclass_end_re = re.compile(":\s*$") + + ## Transition list format + # ["FROM", "TO", condition, action] + transitions = [ + ### FILEHEAD + + # single line comments + ["FILEHEAD", "FILEHEAD", self.single_comment_re.search, self.appendCommentLine], + ["FILEHEAD", "FILEHEAD", self.double_comment_re.search, self.appendCommentLine], + + # multiline comments + ["FILEHEAD", "FILEHEAD_COMMENT_SINGLE", self.start_single_comment_re.search, self.appendCommentLine], + ["FILEHEAD_COMMENT_SINGLE", "FILEHEAD", self.end_single_comment_re.search, self.appendCommentLine], + ["FILEHEAD_COMMENT_SINGLE", "FILEHEAD_COMMENT_SINGLE", self.catchall, self.appendCommentLine], + ["FILEHEAD", "FILEHEAD_COMMENT_DOUBLE", self.start_double_comment_re.search, self.appendCommentLine], + ["FILEHEAD_COMMENT_DOUBLE", "FILEHEAD", self.end_double_comment_re.search, self.appendCommentLine], + ["FILEHEAD_COMMENT_DOUBLE", "FILEHEAD_COMMENT_DOUBLE", self.catchall, self.appendCommentLine], + + # other lines + ["FILEHEAD", "FILEHEAD", self.empty_re.search, self.appendFileheadLine], + ["FILEHEAD", "FILEHEAD", self.hashline_re.search, self.appendFileheadLine], + ["FILEHEAD", "FILEHEAD", self.importline_re.search, self.appendFileheadLine], + ["FILEHEAD", "DEFCLASS", self.defclass_re.search, self.resetCommentSearch], + ["FILEHEAD", "DEFCLASS_MULTI", self.multiline_defclass_start_re.search, self.resetCommentSearch], + ["FILEHEAD", "DEFCLASS_BODY", self.catchall, self.appendFileheadLine], + + ### DEFCLASS + + # single line comments + ["DEFCLASS", "DEFCLASS_BODY", self.single_comment_re.search, self.appendCommentLine], + ["DEFCLASS", "DEFCLASS_BODY", self.double_comment_re.search, self.appendCommentLine], + + # multiline comments + ["DEFCLASS", "COMMENT_SINGLE", self.start_single_comment_re.search, self.appendCommentLine], + ["COMMENT_SINGLE", "DEFCLASS_BODY", self.end_single_comment_re.search, self.appendCommentLine], + ["COMMENT_SINGLE", "COMMENT_SINGLE", self.catchall, self.appendCommentLine], + ["DEFCLASS", "COMMENT_DOUBLE", self.start_double_comment_re.search, self.appendCommentLine], + ["COMMENT_DOUBLE", "DEFCLASS_BODY", self.end_double_comment_re.search, self.appendCommentLine], + ["COMMENT_DOUBLE", "COMMENT_DOUBLE", self.catchall, self.appendCommentLine], + + # other lines + ["DEFCLASS", "DEFCLASS", self.empty_re.search, self.appendDefclassLine], + ["DEFCLASS", "DEFCLASS", self.defclass_re.search, self.resetCommentSearch], + ["DEFCLASS", "DEFCLASS_MULTI", self.multiline_defclass_start_re.search, self.resetCommentSearch], + ["DEFCLASS", "DEFCLASS_BODY", self.catchall, self.stopCommentSearch], + + ### DEFCLASS_BODY + + ["DEFCLASS_BODY", "DEFCLASS", self.defclass_re.search, self.startCommentSearch], + ["DEFCLASS_BODY", "DEFCLASS_MULTI", self.multiline_defclass_start_re.search, self.startCommentSearch], + ["DEFCLASS_BODY", "DEFCLASS_BODY", self.catchall, self.appendNormalLine], + + ### DEFCLASS_MULTI + ["DEFCLASS_MULTI", "DEFCLASS", self.multiline_defclass_end_re.search, self.appendDefclassLine], + ["DEFCLASS_MULTI", "DEFCLASS_MULTI", self.catchall, self.appendDefclassLine], + ] + + self.fsm = FSM("FILEHEAD", transitions) + self.outstream = sys.stdout + + self.output = [] + self.comment = [] + self.filehead = [] + self.defclass = [] + self.indent = "" + + def __closeComment(self): + """Appends any open comment block and triggering block to the output.""" + + if options.autobrief: + if len(self.comment) == 1 \ + or (len(self.comment) > 2 and self.comment[1].strip() == ''): + self.comment[0] = self.__docstringSummaryToBrief(self.comment[0]) + + if self.comment: + block = self.makeCommentBlock() + self.output.extend(block) + + if self.defclass: + self.output.extend(self.defclass) + + def __docstringSummaryToBrief(self, line): + """Adds \\brief to the docstrings summary line. + + A \\brief is prepended, provided no other doxygen command is at the + start of the line. + """ + stripped = line.strip() + if stripped and not stripped[0] in ('@', '\\'): + return "\\brief " + line + else: + return line + + def __flushBuffer(self): + """Flushes the current outputbuffer to the outstream.""" + if self.output: + try: + if options.debug: + print >>sys.stderr, "# OUTPUT: ", self.output + print >>self.outstream, "\n".join(self.output) + self.outstream.flush() + except IOError: + # Fix for FS#33. Catches "broken pipe" when doxygen closes + # stdout prematurely upon usage of INPUT_FILTER, INLINE_SOURCES + # and FILTER_SOURCE_FILES. + pass + self.output = [] + + def catchall(self, input): + """The catchall-condition, always returns true.""" + return True + + def resetCommentSearch(self, match): + """Restarts a new comment search for a different triggering line. + + Closes the current commentblock and starts a new comment search. + """ + if options.debug: + print >>sys.stderr, "# CALLBACK: resetCommentSearch" + self.__closeComment() + self.startCommentSearch(match) + + def startCommentSearch(self, match): + """Starts a new comment search. + + Saves the triggering line, resets the current comment and saves + the current indentation. + """ + if options.debug: + print >>sys.stderr, "# CALLBACK: startCommentSearch" + self.defclass = [self.fsm.current_input] + self.comment = [] + self.indent = match.group(1) + + def stopCommentSearch(self, match): + """Stops a comment search. + + Closes the current commentblock, resets the triggering line and + appends the current line to the output. + """ + if options.debug: + print >>sys.stderr, "# CALLBACK: stopCommentSearch" + self.__closeComment() + + self.defclass = [] + self.output.append(self.fsm.current_input) + + def appendFileheadLine(self, match): + """Appends a line in the FILEHEAD state. + + Closes the open comment block, resets it and appends the current line. + """ + if options.debug: + print >>sys.stderr, "# CALLBACK: appendFileheadLine" + self.__closeComment() + self.comment = [] + self.output.append(self.fsm.current_input) + + def appendCommentLine(self, match): + """Appends a comment line. + + The comment delimiter is removed from multiline start and ends as + well as singleline comments. + """ + if options.debug: + print >>sys.stderr, "# CALLBACK: appendCommentLine" + (from_state, to_state, condition, callback) = self.fsm.current_transition + + # single line comment + if (from_state == "DEFCLASS" and to_state == "DEFCLASS_BODY") \ + or (from_state == "FILEHEAD" and to_state == "FILEHEAD"): + # remove comment delimiter from begin and end of the line + activeCommentDelim = match.group(1) + line = self.fsm.current_input + self.comment.append(line[line.find(activeCommentDelim)+len(activeCommentDelim):line.rfind(activeCommentDelim)]) + + if (to_state == "DEFCLASS_BODY"): + self.__closeComment() + self.defclass = [] + # multiline start + elif from_state == "DEFCLASS" or from_state == "FILEHEAD": + # remove comment delimiter from begin of the line + activeCommentDelim = match.group(1) + line = self.fsm.current_input + self.comment.append(line[line.find(activeCommentDelim)+len(activeCommentDelim):]) + # multiline end + elif to_state == "DEFCLASS_BODY" or to_state == "FILEHEAD": + # remove comment delimiter from end of the line + activeCommentDelim = match.group(1) + line = self.fsm.current_input + self.comment.append(line[0:line.rfind(activeCommentDelim)]) + if (to_state == "DEFCLASS_BODY"): + self.__closeComment() + self.defclass = [] + # in multiline comment + else: + # just append the comment line + self.comment.append(self.fsm.current_input) + + def appendNormalLine(self, match): + """Appends a line to the output.""" + if options.debug: + print >>sys.stderr, "# CALLBACK: appendNormalLine" + self.output.append(self.fsm.current_input) + + def appendDefclassLine(self, match): + """Appends a line to the triggering block.""" + if options.debug: + print >>sys.stderr, "# CALLBACK: appendDefclassLine" + self.defclass.append(self.fsm.current_input) + + def makeCommentBlock(self): + """Indents the current comment block with respect to the current + indentation level. + + @returns a list of indented comment lines + """ + doxyStart = "##" + commentLines = self.comment + + commentLines = map(lambda x: "%s# %s" % (self.indent, x), commentLines) + l = [self.indent + doxyStart] + l.extend(commentLines) + + return l + + def parse(self, input): + """Parses a python file given as input string and returns the doxygen- + compatible representation. + + @param input the python code to parse + @returns the modified python code + """ + lines = input.split("\n") + + for line in lines: + self.fsm.makeTransition(line) + + if self.fsm.current_state == "DEFCLASS": + self.__closeComment() + + return "\n".join(self.output) + + def parseFile(self, filename): + """Parses a python file given as input string and returns the doxygen- + compatible representation. + + @param input the python code to parse + @returns the modified python code + """ + f = open(filename, 'r') + + for line in f: + self.parseLine(line.rstrip('\r\n')) + if self.fsm.current_state == "DEFCLASS": + self.__closeComment() + self.__flushBuffer() + f.close() + + def parseLine(self, line): + """Parse one line of python and flush the resulting output to the + outstream. + + @param line the python code line to parse + """ + self.fsm.makeTransition(line) + self.__flushBuffer() + +def optParse(): + """Parses commandline options.""" + parser = OptionParser(prog=__applicationName__, version="%prog " + __version__) + + parser.set_usage("%prog [options] filename") + parser.add_option("--autobrief", + action="store_true", dest="autobrief", + help="use the docstring summary line as \\brief description" + ) + parser.add_option("--debug", + action="store_true", dest="debug", + help="enable debug output on stderr" + ) + + ## parse options + global options + (options, filename) = parser.parse_args() + + if not filename: + print >>sys.stderr, "No filename given." + sys.exit(-1) + + return filename[0] + +def main(): + """Starts the parser on the file given by the filename as the first + argument on the commandline. + """ + filename = optParse() + fsm = Doxypy() + fsm.parseFile(filename) + +if __name__ == "__main__": + main() diff --git a/docs/tools/fixlinks.py b/docs/tools/fixlinks.py new file mode 100755 index 00000000000..3fbcb01ce23 --- /dev/null +++ b/docs/tools/fixlinks.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# +# This Python script is run as part of generating the documentation for the +# WiredTiger Python API. It runs after doxypy, and turns method names in +# comments into references to the corresponding function in the C API. + +import re, sys + +def process(source): + # Replace standard struct wrapper wording + source = re.sub(r'(\s+#.*)Proxy of C (\w+) struct', + lambda m: ('%sPython wrapper around C ::%s%s@copydoc ::%s' % (m.group(1), m.group(2).upper(), m.group(1), m.group(2).upper())), source) + + # Replace lowercase class names with upper case + source = re.sub(r'(\s+#.*)(wt_\w+)::', + lambda m: ('%s%s::' % (m.group(1), m.group(2).upper())), source) + + # Replace "char" with "string" in comments + while True: + newsource = re.sub(r'(\s+#.*)\bchar\b', r'\1string', source) + if newsource == source: + break + source = newsource + + # Copy documentation + source = re.sub(r'(\s+# )(\w+)\(self, (connection|cursor|session).*', + lambda m: ('%s%s%s@copydoc WT_%s::%s' % + (m.group(0), m.group(1), m.group(1), m.group(3).upper(), m.group(2))), + source) + + # Replace "self, handle" with "self" -- these are typedef'ed away + source = re.sub(r'(\s+#.*self), (?:connection|cursor|session)', r'\1', source) + return source + +if __name__ == '__main__': + sys.stdout.write(process(sys.stdin.read())) diff --git a/docs/tools/pyfilter b/docs/tools/pyfilter new file mode 100755 index 00000000000..070d03b4697 --- /dev/null +++ b/docs/tools/pyfilter @@ -0,0 +1,5 @@ +#!/bin/sh + +tooldir=`dirname $0` + +python $tooldir/doxypy.py "$@" | python $tooldir/fixlinks.py diff --git a/examples/c/Makefile.am b/examples/c/Makefile.am new file mode 100644 index 00000000000..b4eae787749 --- /dev/null +++ b/examples/c/Makefile.am @@ -0,0 +1,22 @@ +LDADD = $(top_builddir)/libwiredtiger.la + +noinst_PROGRAMS = \ + ex_access \ + ex_all \ + ex_call_center \ + ex_config \ + ex_cursor \ + ex_extending \ + ex_file \ + ex_hello \ + ex_pack \ + ex_process \ + ex_schema \ + ex_stat \ + ex_thread \ + ex_transaction + +# The examples can be run with no arguments as simple smoke tests +TESTS = $(noinst_PROGRAMS) + +TESTS_ENVIRONMENT = rm -rf WT_TEST ; mkdir WT_TEST ; diff --git a/examples/c/ex_access.c b/examples/c/ex_access.c new file mode 100644 index 00000000000..f076f555e9e --- /dev/null +++ b/examples/c/ex_access.c @@ -0,0 +1,83 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * ex_access.c + * demonstrates how to create and access a simple table. + */ +#include <stdio.h> +#include <string.h> + +#include <wiredtiger.h> + +const char *home = "WT_TEST"; + +int main(void) +{ + /*! [access example connection] */ + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_SESSION *session; + const char *key, *value; + int ret; + + if ((ret = wiredtiger_open(home, NULL, "create", &conn)) != 0 || + (ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + fprintf(stderr, "Error connecting to %s: %s\n", + home, wiredtiger_strerror(ret)); + /*! [access example connection] */ + + /*! [access example table create] */ + ret = session->create(session, + "table:access", "key_format=S,value_format=S"); + /*! [access example table create] */ + + /*! [access example cursor open] */ + ret = session->open_cursor(session, + "table:access", NULL, NULL, &cursor); + /*! [access example cursor open] */ + + /*! [access example cursor insert] */ + cursor->set_key(cursor, "key1"); /* Insert a record. */ + cursor->set_value(cursor, "value1"); + ret = cursor->insert(cursor); + /*! [access example cursor insert] */ + + /*! [access example cursor list] */ + ret = cursor->reset(cursor); /* Restart the scan. */ + while ((ret = cursor->next(cursor)) == 0) { + ret = cursor->get_key(cursor, &key); + ret = cursor->get_value(cursor, &value); + + printf("Got record: %s : %s\n", key, value); + } + /*! [access example cursor list] */ + + /*! [access example close] */ + ret = conn->close(conn, NULL); + /*! [access example close] */ + + return (ret); +} diff --git a/examples/c/ex_all.c b/examples/c/ex_all.c new file mode 100644 index 00000000000..88d01cb467c --- /dev/null +++ b/examples/c/ex_all.c @@ -0,0 +1,652 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * ex_all.c + * Containing a call to every method in the WiredTiger API. + * + * It doesn't do anything very useful, just demonstrates how to call each + * method. This file is used to populate the API reference with code + * fragments. + */ + +#include <assert.h> +#include <errno.h> +#include <stdio.h> +#include <string.h> + +#include <wiredtiger.h> + +int add_collator(WT_CONNECTION *conn); +int add_compressor(WT_CONNECTION *conn); +int add_cursor_type(WT_CONNECTION *conn); +int add_extractor(WT_CONNECTION *conn); +int connection_ops(WT_CONNECTION *conn); +int cursor_ops(WT_SESSION *session); +int cursor_search_near(WT_CURSOR *cursor); +int session_ops(WT_SESSION *session); + +int +cursor_ops(WT_SESSION *session) +{ + WT_CURSOR *cursor, *other; + int ret; + + other = NULL; + + /*! [Open a cursor] */ + ret = session->open_cursor( + session, "table:mytable", NULL, NULL, &cursor); + /*! [Open a cursor] */ + + { + /*! [Get the cursor's string key] */ + const char *key; /* Get the cursor's string key. */ + ret = cursor->get_key(cursor, &key); + /*! [Get the cursor's string key] */ + } + + { + /*! [Get the cursor's record number key] */ + uint64_t recno; /* Get the cursor's record number key. */ + ret = cursor->get_key(cursor, &recno); + /*! [Get the cursor's record number key] */ + } + + { + /*! [Get the cursor's string value] */ + const char *value; /* Get the cursor's string value. */ + ret = cursor->get_value(cursor, &value); + /*! [Get the cursor's string value] */ + } + + { + /*! [Get the cursor's raw value] */ + WT_ITEM value; /* Get the cursor's raw value. */ + ret = cursor->get_value(cursor, &value); + /*! [Get the cursor's raw value] */ + } + + { + /*! [Set the cursor's string key] */ + /* Set the cursor's string key. */ + const char *key = "another key"; + cursor->set_key(cursor, key); + /*! [Set the cursor's string key] */ + } + + { + /*! [Set the cursor's record number key] */ + uint64_t recno = 37; /* Set the cursor's record number key. */ + cursor->set_key(cursor, recno); + /*! [Set the cursor's record number key] */ + } + + { + /*! [Set the cursor's string value] */ + /* Set the cursor's string value. */ + const char *value = "another value"; + cursor->set_value(cursor, value); + /*! [Set the cursor's string value] */ + } + { + /*! [Set the cursor's raw value] */ + WT_ITEM value; /* Set the cursor's raw value. */ + value.data = "another value"; + value.size = strlen("another value"); + cursor->set_value(cursor, &value); + /*! [Set the cursor's raw value] */ + } + + /*! [Return the next record] */ + ret = cursor->next(cursor); + /*! [Return the next record] */ + + /*! [Return the previous record] */ + ret = cursor->prev(cursor); + /*! [Return the previous record] */ + + /*! [Reset the cursor] */ + ret = cursor->reset(cursor); + /*! [Reset the cursor] */ + + /*! [Test cursor equality] */ + if (cursor->equals(cursor, other)) { + /* Take some action. */ + } + /*! [Test cursor equality] */ + + { + /*! [Search for an exact match] */ + const char *key = "some key"; + cursor->set_key(cursor, key); + ret = cursor->search(cursor); + /*! [Search for an exact match] */ + } + + cursor_search_near(cursor); + + { + /*! [Insert a new record] */ + /* Insert a new record. */ + const char *key = "some key"; + const char *value = "some value"; + cursor->set_key(cursor, key); + cursor->set_value(cursor, value); + ret = cursor->insert(cursor); + /*! [Insert a new record] */ + } + + { + /*! [Insert a new record or overwrite an existing record] */ + /* Insert a new record or overwrite an existing record. */ + const char *key = "some key"; + const char *value = "some value"; + ret = session->open_cursor( + session, "table:mytable", NULL, "overwrite", &cursor); + cursor->set_key(cursor, key); + cursor->set_value(cursor, value); + ret = cursor->insert(cursor); + /*! [Insert a new record or overwrite an existing record] */ + } + + { + /*! [Insert a new record and assign a record number] */ + /* Insert a new record and assign a record number. */ + uint64_t recno; + const char *value = "some value"; + ret = session->open_cursor( + session, "table:mytable", NULL, "append", &cursor); + cursor->set_value(cursor, value); + ret = cursor->insert(cursor); + if (ret == 0) + recno = cursor->get_key(cursor, &recno); + /*! [Insert a new record and assign a record number] */ + } + + { + /*! [Update an existing record] */ + const char *key = "some key"; + const char *value = "some value"; + cursor->set_key(cursor, key); + cursor->set_value(cursor, value); + ret = cursor->update(cursor); + /*! [Update an existing record] */ + } + + { + /*! [Remove a record] */ + const char *key = "some key"; + cursor->set_key(cursor, key); + ret = cursor->remove(cursor); + /*! [Remove a record] */ + } + + { + /*! [Display an error] */ + const char *key = "some key"; + cursor->set_key(cursor, key); + if ((ret = cursor->remove(cursor)) != 0) { + fprintf(stderr, + "cursor.remove: %s\n", wiredtiger_strerror(ret)); + return (ret); + } + /*! [Display an error] */ + } + + /*! [Close the cursor] */ + ret = cursor->close(cursor); + /*! [Close the cursor] */ + + return (ret); +} + +int +cursor_search_near(WT_CURSOR *cursor) +{ + int exact, ret; + const char *key = "some key"; + + /*! [Search for an exact or adjacent match] */ + cursor->set_key(cursor, key); + ret = cursor->search_near(cursor, &exact); + if (ret == 0) { + if (exact == 0) { + /* an exact match */ + } else if (exact < 0) { + /* returned smaller key */ + } else if (exact > 0) { + /* returned larger key */ + } + } + /*! [Search for an exact or adjacent match] */ + + /*! [Forward scan greater than or equal] */ + /* + * An example of a forward scan through the table, where all keys + * greater than or equal to a specified prefix are included in the + * scan. + */ + cursor->set_key(cursor, key); + ret = cursor->search_near(cursor, &exact); + if (ret == 0 && exact >= 0) { + /* include first key returned in the scan */ + } + + while ((ret = cursor->next(cursor)) == 0) { + /* the rest of the scan */ + } + /*! [Forward scan greater than or equal] */ + + /*! [Backward scan less than] */ + /* + * An example of a backward scan through the table, where all keys + * less than a specified prefix are included in the scan. + */ + cursor->set_key(cursor, key); + ret = cursor->search_near(cursor, &exact); + if (ret == 0 && exact < 0) { + /* include first key returned in the scan */ + } + + while ((ret = cursor->prev(cursor)) == 0) { + /* the rest of the scan */ + } + /*! [Backward scan less than] */ + + return (ret); +} + +int +session_ops(WT_SESSION *session) +{ + unsigned long mypid = 0; + int ret; + + cursor_ops(session); + + /*! [Create a table] */ + ret = session->create(session, "table:mytable", + "key_format=S,value_format=S"); + /*! [Create a table] */ + + /*! [session checkpoint] */ + ret = session->checkpoint(session, NULL); + /*! [session checkpoint] */ + + /*! [session drop] */ + ret = session->drop(session, "table:mytable", NULL); + /*! [session drop] */ + + /*! [session dumpfile] */ + ret = session->dumpfile(session, "file:myfile", NULL); + /*! [session dumpfile] */ + + /*! [session msg_printf] */ + ret = session->msg_printf(session, "process pid %lu", mypid); + /*! [session msg_printf] */ + + /*! [session rename] */ + ret = session->rename(session, "table:old", "table:new", NULL); + /*! [session rename] */ + + /*! [session salvage] */ + ret = session->salvage(session, "table:mytable", NULL); + /*! [session salvage] */ + + /*! [session sync] */ + ret = session->sync(session, "table:mytable", NULL); + /*! [session sync] */ + + /*! [session truncate] */ + ret = session->truncate(session, "table:mytable", NULL, NULL, NULL); + /*! [session truncate] */ + + { + /*! [session range truncate] */ + WT_CURSOR *start, *stop; + + ret = session->open_cursor( + session, "table:mytable", NULL, NULL, &start); + start->set_key(start, "June01"); + ret = start->search(start); + + ret = session->open_cursor( + session, "table:mytable", NULL, NULL, &stop); + stop->set_key(stop, "June30"); + ret = stop->search(stop); + + ret = session->truncate(session, NULL, start, stop, NULL); + /*! [session range truncate] */ + } + + /*! [session upgrade] */ + ret = session->upgrade(session, "table:mytable", NULL); + /*! [session upgrade] */ + + /*! [session verify] */ + ret = session->verify(session, "table:mytable", NULL); + /*! [session verify] */ + + /*! [session begin transaction] */ + ret = session->begin_transaction(session, NULL); + /*! [session begin transaction] */ + + /*! [session commit transaction] */ + ret = session->commit_transaction(session, NULL); + /*! [session commit transaction] */ + + /*! [session rollback transaction] */ + ret = session->rollback_transaction(session, NULL); + /*! [session rollback transaction] */ + + /*! [session close] */ + ret = session->close(session, NULL); + /*! [session close] */ + + return (ret); +} + +/*! [WT_CURSOR_TYPE size] */ +static int +my_cursor_size(WT_CURSOR_TYPE *ctype, const char *obj, size_t *sizep) +{ + (void)ctype; + (void)obj; + + *sizep = sizeof (WT_CURSOR); + return (0); +} +/*! [WT_CURSOR_TYPE size] */ + +/*! [WT_CURSOR_TYPE init] */ +static int +my_init_cursor(WT_CURSOR_TYPE *ctype, WT_SESSION *session, + const char *obj, WT_CURSOR *old_cursor, const char *config, + WT_CURSOR *new_cursor) +{ + /* Unused parameters */ + (void)ctype; + (void)session; + (void)obj; + (void)old_cursor; + (void)config; + (void)new_cursor; + + return (0); +} +/*! [WT_CURSOR_TYPE init] */ + +int +add_cursor_type(WT_CONNECTION *conn) +{ + int ret; + + /*! [WT_CURSOR_TYPE register] */ + static WT_CURSOR_TYPE my_ctype = { my_cursor_size, my_init_cursor }; + ret = conn->add_cursor_type(conn, NULL, &my_ctype, NULL); + /*! [WT_CURSOR_TYPE register] */ + + return (ret); +} + +/*! [Implement WT_COLLATOR] */ +/* + * A simple example of the collator API: compare the keys as strings. + */ +static int +my_compare(WT_COLLATOR *collator, WT_SESSION *session, + const WT_ITEM *value1, const WT_ITEM *value2, int *cmp) +{ + const char *p1, *p2; + + /* Unused parameters */ + (void)collator; + (void)session; + + p1 = (const char *)value1->data; + p2 = (const char *)value2->data; + while (*p1 != '\0' && *p1 == *p2) + p1++, p2++; + + *cmp = (int)*p2 - (int)*p1; + return (0); +} +/*! [Implement WT_COLLATOR] */ + +int +add_collator(WT_CONNECTION *conn) +{ + int ret; + + /*! [WT_COLLATOR register] */ + static WT_COLLATOR my_collator = { my_compare }; + ret = conn->add_collator(conn, "my_collator", &my_collator, NULL); + /*! [WT_COLLATOR register] */ + + return (ret); +} + +/*! [WT_COMPRESSOR compress] */ +/* + * A simple compression example that passes data through unchanged. + */ +static int +my_compress(WT_COMPRESSOR *compressor, WT_SESSION *session, + uint8_t *src, size_t src_len, + uint8_t *dst, size_t dst_len, + size_t *result_lenp, int *compression_failed) +{ + /* Unused parameters */ + (void)compressor; + (void)session; + + *compression_failed = 0; + if (dst_len < src_len) { + *compression_failed = 1; + return (0); + } + memcpy(dst, src, src_len); + *result_lenp = src_len; + return (0); +} +/*! [WT_COMPRESSOR compress] */ + +/*! [WT_COMPRESSOR decompress] */ +/* + * A simple decompression example that passes data through unchanged. + */ +static int +my_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, + uint8_t *src, size_t src_len, + uint8_t *dst, size_t dst_len, + size_t *result_lenp) +{ + /* Unused parameters */ + (void)compressor; + (void)session; + + if (dst_len < src_len) + return (ENOMEM); + + memcpy(dst, src, src_len); + *result_lenp = src_len; + return (0); +} +/*! [WT_COMPRESSOR decompress] */ + +/*! [WT_COMPRESSOR presize] */ +/* + * A simple pre-size example that returns the source length. + */ +static int +my_pre_size(WT_COMPRESSOR *compressor, WT_SESSION *session, + uint8_t *src, size_t src_len, + size_t *result_lenp) +{ + /* Unused parameters */ + (void)compressor; + (void)session; + (void)src; + + *result_lenp = src_len; + return (0); +} +/*! [WT_COMPRESSOR presize] */ + +int +add_compressor(WT_CONNECTION *conn) +{ + int ret; + + /*! [WT_COMPRESSOR register] */ + static WT_COMPRESSOR my_compressor = { + my_compress, my_decompress, my_pre_size }; + ret = conn->add_compressor(conn, "my_compress", &my_compressor, NULL); + /*! [WT_COMPRESSOR register] */ + + return (ret); +} + +/*! [WT_EXTRACTOR] */ +static int +my_extract(WT_EXTRACTOR *extractor, WT_SESSION *session, + const WT_ITEM *key, const WT_ITEM *value, + WT_ITEM *result) +{ + /* Unused parameters */ + (void)extractor; + (void)session; + (void)key; + + *result = *value; + return (0); +} +/*! [WT_EXTRACTOR] */ + +int +add_extractor(WT_CONNECTION *conn) +{ + int ret; + + /*! [WT_EXTRACTOR register] */ + static WT_EXTRACTOR my_extractor; + my_extractor.extract = my_extract; + ret = conn->add_extractor(conn, "my_extractor", &my_extractor, NULL); + /*! [WT_EXTRACTOR register] */ + + return (ret); +} + +int +connection_ops(WT_CONNECTION *conn) +{ + int ret; + + /*! [conn load extension] */ + ret = conn->load_extension(conn, "my_extension.dll", NULL); + /*! [conn load extension] */ + + add_cursor_type(conn); + add_collator(conn); + add_extractor(conn); + + /*! [conn close] */ + ret = conn->close(conn, NULL); + /*! [conn close] */ + + /*! [conn get_home] */ + printf("The database home is %s\n", conn->get_home(conn)); + /*! [conn get_home] */ + + /*! [is_new] */ + if (conn->is_new(conn)) { + /* First time initialization. */ + } + /*! [is_new] */ + + { + /*! [Open a session] */ + WT_SESSION *session; + ret = conn->open_session(conn, NULL, NULL, &session); + /*! [Open a session] */ + + session_ops(session); + } + + return (ret); +} + +int main(void) +{ + int ret; + + { + /*! [Open a connection] */ + WT_CONNECTION *conn; + const char *home = "WT_TEST"; + ret = wiredtiger_open(home, NULL, "create,transactional", &conn); + /*! [Open a connection] */ + } + + { + /*! [Get the packed size] */ + size_t size; + size = wiredtiger_struct_size("iSh", 42, "hello", -3); + assert(size < 100); + /*! [Get the packed size] */ + } + + { + /*! [Pack fields into a buffer] */ + char buf[100]; + ret = wiredtiger_struct_pack(buf, sizeof (buf), "iSh", 42, "hello", -3); + /*! [Pack fields into a buffer] */ + + { + /*! [Unpack fields from a buffer] */ + int i; + char *s; + short h; + ret = wiredtiger_struct_unpack(buf, sizeof (buf), "iSh", &i, &s, &h); + /*! [Unpack fields from a buffer] */ + } + } + + /*! [Get the WiredTiger library version #1] */ + printf("WiredTiger version %s\n", wiredtiger_version(NULL, NULL, NULL)); + /*! [Get the WiredTiger library version #1] */ + + { + /*! [Get the WiredTiger library version #2] */ + int major, minor, patch; + (void)wiredtiger_version(&major, &minor, &patch); + printf("WiredTiger version is %d, %d (patch %d)\n", + major, minor, patch); + /*! [Get the WiredTiger library version #2] */ + } + + return (ret); +} diff --git a/examples/c/ex_call_center.c b/examples/c/ex_call_center.c new file mode 100644 index 00000000000..84e1ee955c8 --- /dev/null +++ b/examples/c/ex_call_center.c @@ -0,0 +1,228 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * ex_call_center.c + * This is an example application that demonstrates how to map a + * moderately complex SQL application into WiredTiger. + */ + +#include <inttypes.h> +#include <stdio.h> +#include <string.h> + +#include <wiredtiger.h> + +const char *home = "WT_TEST"; + +/*! [call-center decl] */ +/* + * In SQL, the tables are described as follows: + * + * CREATE TABLE Customers(id INTEGER PRIMARY KEY, + * name VARCHAR(30), address VARCHAR(50), phone VARCHAR(15)) + * CREATE INDEX CustomersPhone ON Customers(phone) + * + * CREATE TABLE Calls(id INTEGER PRIMARY KEY, call_date DATE, + * cust_id INTEGER, emp_id INTEGER, call_type VARCHAR(12), + * notes VARCHAR(25)) + * CREATE INDEX CallsCustDate ON Calls(cust_id, call_date) + * + * In this example, both tables will use record numbers for their IDs, which + * will be the key. The C structs for the records are as follows. + */ + +/* Customer records. */ +typedef struct { + uint64_t id; + char *name; + char *address; + char *phone; +} CUSTOMER; + +/* Call records. */ +typedef struct { + uint64_t id; + uint64_t call_date; + uint64_t cust_id; + uint64_t emp_id; + char *call_type; + char *notes; +} CALL; +/*! [call-center decl] */ + +int main(void) +{ + int count, exact, ret; + WT_CONNECTION *conn; + WT_SESSION *session; + WT_CURSOR *cursor; + CUSTOMER cust; + CALL call; + + ret = wiredtiger_open(home, NULL, "create", &conn); + if (ret != 0) { + fprintf(stderr, "Error connecting to %s: %s\n", + home, wiredtiger_strerror(ret)); + return (1); + } + /* Note: further error checking omitted for clarity. */ + + /*! [call-center work] */ + ret = conn->open_session(conn, NULL, NULL, &session); + + /* + * Create the customers table, give names and types to the columns. + * The columns will be stored in two groups: "main" and "address", + * created below. + */ + ret = session->create(session, "table:customers", + "key_format=S," + "value_format=SSS," + "columns=(id,name,address,phone)," + "colgroups=(main,address)"); + + /* Create the main column group with value columns except address. */ + ret = session->create(session, + "colgroup:customers:main", "columns=(name,phone)"); + + /* Create the address column group with just the address. */ + ret = session->create(session, + "colgroup:customers:address", "columns=(address)"); + + /* Create an index on the customer table by phone number. */ + ret = session->create(session, + "index:customers:phone", "columns=(phone)"); + + /* + * Create the calls table, give names and types to the columns. + * All of the columns will be stored together, so no column groups are + * declared. + */ + ret = session->create(session, "table:calls", + "key_format=r," + "value_format=qrrSS," + "columns=(id,call_date,cust_id,emp_id,call_type,notes)"); + + /* + * Create an index on the calls table with a composite key of cust_id + * and call_date. + */ + ret = session->create(session, "index:calls:cust_date", + "columns=(cust_id,call_date)"); + + /* Populate the customers table with some data. */ + ret = session->open_cursor( + session, "table:customers", NULL, NULL, &cursor); + + cursor->set_key(cursor, "customer #1"); + cursor->set_value(cursor, + "Professor Oak", "LeafGreen Avenue", "123-456-7890"); + ret = cursor->insert(cursor); + + cursor->set_key(cursor, "customer #2"); + cursor->set_value(cursor, "Lorelei", "Sevii Islands", "098-765-4321"); + ret = cursor->insert(cursor); + + ret = cursor->close(cursor); + + /* + * First query: a call arrives. In SQL: + * + * SELECT id, name FROM Customers WHERE phone=? + * + * Use the cust_phone index, lookup by phone number to fill the + * customer record. The cursor will have a key format of "S" for a + * string because the cust_phone index has a single column ("phone"), + * which is of type "S". + * + * Specify the columns we want: the customer ID and the name. This + * means the cursor's value format will be "rS". + */ + ret = session->open_cursor(session, + "index:customers:phone(id,name)", + NULL, NULL, &cursor); + cursor->set_key(cursor, "212-555-1000"); + ret = cursor->search(cursor); + if (ret == 0) { + ret = cursor->get_value(cursor, &cust.id, &cust.name); + printf("Got customer record for %s\n", cust.name); + } + ret = cursor->close(cursor); + + /* + * Next query: get the recent order history. In SQL: + * + * SELECT * FROM Calls WHERE cust_id=? ORDER BY call_date DESC LIMIT 3 + * + * Use the call_cust_date index to find the matching calls. Since it is + * is in increasing order by date for a given customer, we want to start + * with the last record for the customer and work backwards. + * + * Specify a subset of columns to be returned. If these were all + * covered by the index, the primary would not be accessed. Stop after + * getting 3 records. + */ + ret = session->open_cursor(session, + "index:calls:cust_date(cust_id,call_type,notes)", + NULL, NULL, &cursor); + + /* + * The keys in the index are (cust_id,call_date) -- we want the largest + * call date for a given cust_id. Search for (cust_id+1,0), then work + * backwards. + */ + cursor->set_key(cursor, cust.id + 1, 0); + ret = cursor->search_near(cursor, &exact); + + /* + * If the table is empty, search_near will return WT_NOTFOUND. + * Otherwise the cursor will on a matching key if one exists, or on an + * adjacent key. If the key we find is equal or larger than the search + * key, go back one. + */ + if (ret == 0 && exact >= 0) + ret = cursor->prev(cursor); + if (ret == 0) + ret = cursor->get_value(cursor, + &call.cust_id, &call.call_type, &call.notes); + + count = 0; + while (ret == 0 && call.cust_id == cust.id) { + printf("Got call record on date %lu: type %s: %s\n", + (unsigned long)call.call_date, call.call_type, call.notes); + if (++count == 3) + break; + + ret = cursor->prev(cursor); + ret = cursor->get_value(cursor, + &call.cust_id, &call.call_type, &call.notes); + } + /*! [call-center work] */ + + ret = conn->close(conn, NULL); + + return (ret); +} diff --git a/examples/c/ex_config.c b/examples/c/ex_config.c new file mode 100644 index 00000000000..2b7f8d5d09f --- /dev/null +++ b/examples/c/ex_config.c @@ -0,0 +1,79 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * ex_config.c + * This is an example demonstrating how to configure various database and + * table properties. + */ + +#include <stdio.h> +#include <string.h> + +#include <wiredtiger.h> + +const char *home = "WT_TEST"; + +int main(void) +{ + int ret; + WT_CONNECTION *conn; + WT_SESSION *session; + WT_CURSOR *cursor; + const char *key, *value; + + /*! [configure cache size] */ + if ((ret = wiredtiger_open(home, NULL, + "create,cache_size=10M", &conn)) != 0) + fprintf(stderr, "Error connecting to %s: %s\n", + home, wiredtiger_strerror(ret)); + /*! [configure cache size] */ + + /*! [create a table] */ + ret = conn->open_session(conn, NULL, NULL, &session); + + ret = session->create(session, + "table:access", "key_format=S,value_format=S"); + /*! [create a table] */ + + /*! [transaction] */ + ret = session->begin_transaction(session, "priority=100,name=mytxn"); + + ret = session->open_cursor(session, "config:", NULL, NULL, &cursor); + + while ((ret = cursor->next(cursor)) == 0) { + cursor->get_key(cursor, &key); + cursor->get_value(cursor, &value); + printf("configuration value: %s = %s\n", key, value); + } + + ret = session->commit_transaction(session, NULL); + /*! [transaction] */ + + ret = conn->close(conn, NULL); + + return (ret); +} + diff --git a/examples/c/ex_cursor.c b/examples/c/ex_cursor.c new file mode 100644 index 00000000000..58efc7f8624 --- /dev/null +++ b/examples/c/ex_cursor.c @@ -0,0 +1,213 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * ex_cursor.c + * This is an example demonstrating some cursor types and operations. + */ + +#include <stdio.h> +#include <string.h> + +#include <wiredtiger.h> + +int cursor_reset(WT_CURSOR *cursor); +int cursor_forward_scan(WT_CURSOR *cursor); +int cursor_reverse_scan(WT_CURSOR *cursor); +int cursor_search(WT_CURSOR *cursor); +int cursor_search_near(WT_CURSOR *cursor); +int cursor_insert(WT_CURSOR *cursor); +int cursor_update(WT_CURSOR *cursor); +int cursor_remove(WT_CURSOR *cursor); + +const char *home = "WT_TEST"; + +/*! [cursor next] */ +int +cursor_forward_scan(WT_CURSOR *cursor) +{ + const char *key, *value; + int ret; + + while ((ret = cursor->next(cursor)) == 0) { + ret = cursor->get_key(cursor, &key); + ret = cursor->get_value(cursor, &value); + } + return (ret); +} +/*! [cursor next] */ + +/*! [cursor prev] */ +int +cursor_reverse_scan(WT_CURSOR *cursor) +{ + const char *key, *value; + int ret; + + while ((ret = cursor->prev(cursor)) == 0) { + ret = cursor->get_key(cursor, &key); + ret = cursor->get_value(cursor, &value); + } + return (ret); +} +/*! [cursor prev] */ + +/*! [cursor reset] */ +int +cursor_reset(WT_CURSOR *cursor) +{ + return (cursor->reset(cursor)); +} +/*! [cursor reset] */ + +/*! [cursor search] */ +int +cursor_search(WT_CURSOR *cursor) +{ + const char *value; + int ret; + + cursor->set_key(cursor, "foo"); + + if ((ret = cursor->search(cursor)) != 0) + ret = cursor->get_value(cursor, &value); + + return (ret); +} +/*! [cursor search] */ + +/*! [cursor search near] */ +int +cursor_search_near(WT_CURSOR *cursor) +{ + const char *key, *value; + int exact, ret; + + cursor->set_key(cursor, "foo"); + + if ((ret = cursor->search_near(cursor, &exact)) == 0) { + switch (exact) { + case -1: /* Returned key smaller than search key */ + ret = cursor->get_key(cursor, &key); + break; + case 0: /* Exact match found */ + break; + case 1: /* Returned key larger than search key */ + ret = cursor->get_key(cursor, &key); + break; + } + + ret = cursor->get_value(cursor, &value); + } + + return (ret); +} +/*! [cursor search near] */ + +/*! [cursor insert] */ +int +cursor_insert(WT_CURSOR *cursor) +{ + cursor->set_key(cursor, "foo"); + cursor->set_value(cursor, "bar"); + + return (cursor->insert(cursor)); +} +/*! [cursor insert] */ + +/*! [cursor update] */ +int +cursor_update(WT_CURSOR *cursor) +{ + cursor->set_key(cursor, "foo"); + cursor->set_value(cursor, "newbar"); + + return (cursor->update(cursor)); +} +/*! [cursor update] */ + +/*! [cursor remove] */ +int +cursor_remove(WT_CURSOR *cursor) +{ + cursor->set_key(cursor, "foo"); + return (cursor->remove(cursor)); +} +/*! [cursor remove] */ + +int main(void) +{ + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_SESSION *session; + int ret; + + /* Open a connection to the database, creating it if necessary. */ + if ((ret = wiredtiger_open(home, NULL, "create", &conn)) != 0) + fprintf(stderr, "Error connecting to %s: %s\n", + home, wiredtiger_strerror(ret)); + + /* Open a session for the current thread's work. */ + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + fprintf(stderr, "Error opening a session on %s: %s\n", + home, wiredtiger_strerror(ret)); + + ret = session->create(session, "table:world", + "key_format=r,value_format=5sii," + "columns=(id,country,population,area)"); + + /*! [open cursor #1] */ + ret = session->open_cursor(session, "table:world", NULL, NULL, &cursor); + /*! [open cursor #1] */ + + /*! [open cursor #2] */ + ret = session->open_cursor(session, + "table:world(country,population)", NULL, NULL, &cursor); + /*! [open cursor #2] */ + + /*! [open cursor #3] */ + ret = session->open_cursor(session, "statistics:", NULL, NULL, &cursor); + /*! [open cursor #3] */ + + /* Create a simple string table to illustrate basic operations. */ + ret = session->create(session, "table:map", + "key_format=S,value_format=S"); + ret = session->open_cursor(session, "table:map", NULL, NULL, &cursor); + ret = cursor_insert(cursor); + ret = cursor_reset(cursor); + ret = cursor_forward_scan(cursor); + ret = cursor_reset(cursor); + ret = cursor_reverse_scan(cursor); + ret = cursor_update(cursor); + ret = cursor_remove(cursor); + ret = cursor->close(cursor); + + /* Note: closing the connection implicitly closes open session(s). */ + if ((ret = conn->close(conn, NULL)) != 0) + fprintf(stderr, "Error connecting to %s: %s\n", + home, wiredtiger_strerror(ret)); + + return (ret); +} diff --git a/examples/c/ex_extending.c b/examples/c/ex_extending.c new file mode 100644 index 00000000000..e9eae61e9c7 --- /dev/null +++ b/examples/c/ex_extending.c @@ -0,0 +1,116 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * ex_extending.c + * This is an example demonstrating ways to extend WiredTiger with + * extractors, collators and loadable modules. + */ + +#include <stdio.h> +#include <string.h> + +#include <wiredtiger.h> + +const char *home = "WT_TEST"; + +/*! [case insensitive comparator] */ +/* A simple case insensitive comparator. */ +static int +__compare_nocase(WT_COLLATOR *collator, WT_SESSION *session, + const WT_ITEM *v1, const WT_ITEM *v2, int *cmp) +{ + const char *s1 = (const char *)v1->data; + const char *s2 = (const char *)v2->data; + + (void)session; /* unused */ + (void)collator; /* unused */ + + *cmp = strcasecmp(s1, s2); + return (0); +} + +static WT_COLLATOR nocasecoll = { __compare_nocase }; +/*! [case insensitive comparator] */ + +/*! [n character comparator] */ +/* + * Comparator that only compares the first N prefix characters of the string. + * This has associated data, so we need to extend WT_COLLATOR. + */ +typedef struct { + WT_COLLATOR iface; + uint32_t maxlen; +} PREFIX_COLLATOR; + +static int +__compare_prefixes(WT_COLLATOR *collator, WT_SESSION *session, + const WT_ITEM *v1, const WT_ITEM *v2, int *cmp) +{ + PREFIX_COLLATOR *pcoll = (PREFIX_COLLATOR *)collator; + const char *s1 = (const char *)v1->data; + const char *s2 = (const char *)v2->data; + + (void)session; /* unused */ + + *cmp = strncmp(s1, s2, pcoll->maxlen); + return (0); +} + +static PREFIX_COLLATOR pcoll10 = { {__compare_prefixes}, 10 }; +/*! [n character comparator] */ + +int main(void) +{ + int ret; + WT_CONNECTION *conn; + WT_SESSION *session; + + /* Open a connection to the database, creating it if necessary. */ + if ((ret = wiredtiger_open(home, NULL, "create", &conn)) != 0) + fprintf(stderr, "Error connecting to %s: %s\n", + home, wiredtiger_strerror(ret)); + + /*! [add collator nocase] */ + ret = conn->add_collator(conn, "nocase", &nocasecoll, NULL); + /*! [add collator nocase] */ + /*! [add collator prefix10] */ + ret = conn->add_collator(conn, "prefix10", &pcoll10.iface, NULL); + + /* Open a session for the current thread's work. */ + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + fprintf(stderr, "Error opening a session on %s: %s\n", + home, wiredtiger_strerror(ret)); + + /* XXX Do some work... */ + + /* Note: closing the connection implicitly closes open session(s). */ + if ((ret = conn->close(conn, NULL)) != 0) + /*! [add collator prefix10] */ + fprintf(stderr, "Error connecting to %s: %s\n", + home, wiredtiger_strerror(ret)); + + return (ret); +} diff --git a/examples/c/ex_file.c b/examples/c/ex_file.c new file mode 100644 index 00000000000..5ccbd01000b --- /dev/null +++ b/examples/c/ex_file.c @@ -0,0 +1,61 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * ex_file.c + * This is an example demonstrating how to configure an individual file. + */ + +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include <wiredtiger.h> + +const char *home = "WT_TEST"; + +int +main(void) +{ + WT_CONNECTION *conn; + WT_SESSION *session; + int ret; + + if ((ret = wiredtiger_open(home, NULL, "create", &conn)) != 0 || + (ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + fprintf(stderr, "Error connecting to %s: %s\n", + home, wiredtiger_strerror(ret)); + /* Note: further error checking omitted for clarity. */ + + /*! [file create] */ + ret = session->create(session, "file:example", + "key_format=u," + "internal_page_max=32KB,internal_item_max=1KB," + "leaf_page_max=1MB,leaf_item_max=32KB"); + /*! [file create] */ + + return (conn->close(conn, NULL) == 0 ? ret : EXIT_FAILURE); +} diff --git a/examples/c/ex_hello.c b/examples/c/ex_hello.c new file mode 100644 index 00000000000..0319cf1db97 --- /dev/null +++ b/examples/c/ex_hello.c @@ -0,0 +1,63 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * ex_hello.c + * This is an example demonstrating how to create and connect to a + * database. + */ + +#include <stdio.h> +#include <string.h> + +#include <wiredtiger.h> + +const char *home = "WT_TEST"; + +int main(void) +{ + int ret; + WT_CONNECTION *conn; + WT_SESSION *session; + + /* Open a connection to the database, creating it if necessary. */ + if ((ret = wiredtiger_open(home, NULL, "create", &conn)) != 0) + fprintf(stderr, "Error connecting to %s: %s\n", + home, wiredtiger_strerror(ret)); + + /* Open a session for the current thread's work. */ + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + fprintf(stderr, "Error opening a session on %s: %s\n", + home, wiredtiger_strerror(ret)); + + /* Do some work... */ + + /* Note: closing the connection implicitly closes open session(s). */ + if ((ret = conn->close(conn, NULL)) != 0) + fprintf(stderr, "Error connecting to %s: %s\n", + home, wiredtiger_strerror(ret)); + + return (ret); +} diff --git a/examples/c/ex_pack.c b/examples/c/ex_pack.c new file mode 100644 index 00000000000..7fe1c123181 --- /dev/null +++ b/examples/c/ex_pack.c @@ -0,0 +1,71 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * ex_pack.c + * This is an example demonstrating basic packing and unpacking of fields. + */ + +#include <stdio.h> +#include <string.h> + +#include <wiredtiger.h> + +const char *home = "WT_TEST"; + +int main(void) +{ + WT_CONNECTION *conn; + WT_SESSION *session; + char buf[50]; + size_t size; + int i, j, k, ret; + + /* Open a connection to the database, creating it if necessary. */ + if ((ret = wiredtiger_open(home, NULL, "create", &conn)) != 0) + fprintf(stderr, "Error connecting to %s: %s\n", + home, wiredtiger_strerror(ret)); + + /* Open a session for the current thread's work. */ + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + fprintf(stderr, "Error opening a session on %s: %s\n", + home, wiredtiger_strerror(ret)); + + /*! [packing] */ + size = wiredtiger_struct_size("iii", 42, 1000, -9); + if (size > sizeof (buf)) { + /* Allocate a bigger buffer. */ + } + wiredtiger_struct_pack(buf, size, "iii", 42, 1000, -9); + wiredtiger_struct_unpack(buf, size, "iii", &i, &j, &k); + /*! [packing] */ + + /* Note: closing the connection implicitly closes open session(s). */ + if ((ret = conn->close(conn, NULL)) != 0) + fprintf(stderr, "Error connecting to %s: %s\n", + home, wiredtiger_strerror(ret)); + + return (ret); +} diff --git a/examples/c/ex_process.c b/examples/c/ex_process.c new file mode 100644 index 00000000000..e203be00cbd --- /dev/null +++ b/examples/c/ex_process.c @@ -0,0 +1,66 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * ex_process.c + * This is an example demonstrating how to connect to a database from + * multiple processes. + */ + +#include <stdio.h> +#include <string.h> + +#include <wiredtiger.h> + +const char *home = "WT_TEST"; + +int main(void) +{ + int ret; + WT_CONNECTION *conn; + WT_SESSION *session; + + /*! [processes] */ + /* Open a connection to the database, creating it if necessary. */ + if ((ret = + wiredtiger_open(home, NULL, "create,multiprocess", &conn)) != 0) + fprintf(stderr, "Error connecting to %s: %s\n", + home, wiredtiger_strerror(ret)); + + /* Open a session for the current thread's work. */ + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + fprintf(stderr, "Error opening a session on %s: %s\n", + home, wiredtiger_strerror(ret)); + + /* XXX Do some work... */ + + /* Note: closing the connection implicitly closes open session(s). */ + if ((ret = conn->close(conn, NULL)) != 0) + fprintf(stderr, "Error connecting to %s: %s\n", + home, wiredtiger_strerror(ret)); + /*! [processes] */ + + return (ret); +} diff --git a/examples/c/ex_schema.c b/examples/c/ex_schema.c new file mode 100644 index 00000000000..6e89ebd0e4d --- /dev/null +++ b/examples/c/ex_schema.c @@ -0,0 +1,131 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * ex_schema.c + * This is an example application demonstrating how to create and access + * tables using a schema. + */ + +#include <stdio.h> +#include <string.h> + +#include <inttypes.h> +#include <wiredtiger.h> + +const char *home = "WT_TEST"; + +/*! [schema decl] */ +/* The C struct for the data we are storing with WiredTiger. */ +typedef struct { + char country[5]; + uint16_t year; + uint64_t population; +} POP_RECORD; +/*! [schema decl] */ + +POP_RECORD pop_data[] = { + { "USA", 1980, 226542250 }, + { "USA", 2009, 307006550 }, + { "UK", 2008, 61414062 }, + { "CAN", 2008, 33311400 }, + { "AU", 2008, 21431800 } +}; + +int main(void) +{ + int ret; + WT_CONNECTION *conn; + WT_SESSION *session; + WT_CURSOR *cursor; + POP_RECORD *p, *endp; + const char *country; + uint64_t recno; + uint16_t year; + + ret = wiredtiger_open(home, NULL, "create", &conn); + if (ret != 0) + fprintf(stderr, "Error connecting to %s: %s\n", + home, wiredtiger_strerror(ret)); + /* Note: error checking omitted for clarity. */ + + /*! [schema work] */ + ret = conn->open_session(conn, NULL, NULL, &session); + + /* + * Create the population table. + * Keys are record numbers, the format for values is + * (5-byte string, short, long). + * See ::wiredtiger_struct_pack for details of the format strings. + * + * If this program is run multiple times so the table already exists, + * this call will verify that the table exists. It is not required in + * that case, but is a safety check that the schema matches what the + * program expects. + */ + ret = session->create(session, "table:population", + "key_format=r," + "value_format=5sHQ," + "columns=(id,country,year,population)," + "colgroups=(main,population)"); + + /* Create the column groups to store population in its own file. */ + ret = session->create(session, "colgroup:population:main", + "columns=(country,year)"); + + ret = session->create(session, "colgroup:population:population", + "columns=(population)"); + + /* Create an index with composite key (country,year). */ + ret = session->create(session, "index:population:country_year", + "columns=(country,year)"); + + ret = session->open_cursor(session, "table:population", + NULL, "append", &cursor); + + endp = pop_data + (sizeof (pop_data) / sizeof (pop_data[0])); + for (p = pop_data; p < endp; p++) { + cursor->set_value(cursor, p->country, p->year, p->population); + ret = cursor->insert(cursor); + } + ret = cursor->close(cursor); + + /* Now just read through the countries we know about */ + ret = session->open_cursor(session, + "index:population:country_year(id)", + NULL, NULL, &cursor); + + while ((ret = cursor->next(cursor)) == 0) { + cursor->get_key(cursor, &country, &year); + cursor->get_value(cursor, &recno); + + printf("Got country %s : row ID %d\n", country, (int)recno); + } + + ret = conn->close(conn, NULL); + /*! [schema work] */ + + return (ret); +} diff --git a/examples/c/ex_stat.c b/examples/c/ex_stat.c new file mode 100644 index 00000000000..31b9266c062 --- /dev/null +++ b/examples/c/ex_stat.c @@ -0,0 +1,133 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * ex_stat.c + * This is an example demonstrating how to query a database's statistics. + */ + +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include <wiredtiger.h> + +int print_cursor(WT_CURSOR *); +int print_database_stats(WT_SESSION *); +int print_file_stats(WT_SESSION *); +int print_overflow_pages(WT_SESSION *); + +const char *home = "WT_TEST"; + +/*! [statistics display function] */ +int +print_cursor(WT_CURSOR *cursor) +{ + const char *desc, *pvalue; + uint64_t value; + int ret; + + while ( + (ret = cursor->next(cursor)) == 0 && + (ret = cursor->get_value(cursor, &desc, &pvalue, &value)) == 0) + printf("%s=%s\n", desc, pvalue); + + return (ret == WT_NOTFOUND ? 0 : ret); +} +/*! [statistics display function] */ + +int +print_database_stats(WT_SESSION *session) +{ + WT_CURSOR *cursor; + int ret; + + /*! [statistics database function] */ + if ((ret = session->open_cursor(session, + "statistics:", NULL, NULL, &cursor)) != 0) + return (ret); + + return (print_cursor(cursor)); + /*! [statistics database function] */ +} + +int +print_file_stats(WT_SESSION *session) +{ + WT_CURSOR *cursor; + int ret; + + /*! [statistics file function] */ + if ((ret = session->open_cursor(session, + "statistics:file:access.wt", NULL, NULL, &cursor)) != 0) + return (ret); + + return (print_cursor(cursor)); + /*! [statistics file function] */ +} + +int +print_overflow_pages(WT_SESSION *session) +{ + /*! [statistics retrieve by key] */ + WT_CURSOR *cursor; + const char *desc, *pvalue; + uint64_t value; + int ret; + + if ((ret = session->open_cursor(session, + "statistics:file:access.wt", NULL, NULL, &cursor)) != 0) + return (ret); + + cursor->set_key(cursor, WT_STAT_file_overflow); + ret = cursor->search(cursor); + ret = cursor->get_value(cursor, &desc, &pvalue, &value); + printf("%s=%s\n", desc, pvalue); + /*! [statistics retrieve by key] */ + + return (ret); +} + +int +main(void) +{ + WT_CONNECTION *conn; + WT_SESSION *session; + int ret; + + ret = wiredtiger_open(home, NULL, "create", &conn); + ret = conn->open_session(conn, NULL, NULL, &session); + ret = session->create( + session, "table:access", "key_format=S,value_format=S"); + + ret = print_database_stats(session); + + ret = print_file_stats(session); + + ret = print_overflow_pages(session); + + return (conn->close(conn, NULL) == 0 ? ret : EXIT_FAILURE); +} diff --git a/examples/c/ex_thread.c b/examples/c/ex_thread.c new file mode 100644 index 00000000000..ef65b6ca5cc --- /dev/null +++ b/examples/c/ex_thread.c @@ -0,0 +1,105 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * ex_thread.c + * This is an example demonstrating how to create and access a simple + * table from multiple threads. + */ + +#include <stdio.h> +#include <string.h> +#include <pthread.h> + +#include <wiredtiger.h> + +void *scan_thread(void *arg); + +const char *home = "WT_TEST"; +#define NUM_THREADS 10 + +WT_CONNECTION *conn; + +/*! [thread scan] */ +void * +scan_thread(void *arg) +{ + WT_SESSION *session; + WT_CURSOR *cursor; + const char *key, *value; + int ret; + + ret = conn->open_session(conn, NULL, NULL, &session); + ret = session->open_cursor(session, "table:access", + NULL, NULL, &cursor); + + /* Show all records. */ + while ((ret = cursor->next(cursor)) == 0) { + ret = cursor->get_key(cursor, &key); + ret = cursor->get_value(cursor, &value); + + printf("Got record: %s : %s\n", key, value); + } + + return (arg); +} +/*! [thread scan] */ + +/*! [thread main] */ +int +main(void) +{ + WT_SESSION *session; + WT_CURSOR *cursor; + pthread_t threads[NUM_THREADS]; + int i, ret; + + if ((ret = wiredtiger_open(home, NULL, + "create", &conn)) != 0) + fprintf(stderr, "Error connecting to %s: %s\n", + home, wiredtiger_strerror(ret)); + /* Note: further error checking omitted for clarity. */ + + ret = conn->open_session(conn, NULL, NULL, &session); + ret = session->create(session, "table:access", + "key_format=S,value_format=S"); + ret = session->open_cursor(session, "table:access", NULL, + "overwrite", &cursor); + cursor->set_key(cursor, "key1"); + cursor->set_value(cursor, "value1"); + ret = cursor->insert(cursor); + ret = session->close(session, NULL); + + for (i = 0; i < NUM_THREADS; i++) + ret = pthread_create(&threads[i], NULL, scan_thread, NULL); + + for (i = 0; i < NUM_THREADS; i++) + ret = pthread_join(threads[i], NULL); + + ret = conn->close(conn, NULL); + + return (ret); +} +/*! [thread main] */ diff --git a/examples/c/ex_transaction.c b/examples/c/ex_transaction.c new file mode 100644 index 00000000000..86254bb4a26 --- /dev/null +++ b/examples/c/ex_transaction.c @@ -0,0 +1,67 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * ex_transaction.c + * This is an example demonstrating how to use transactions. + */ + +#include <stdio.h> +#include <string.h> + +#include <wiredtiger.h> + +const char *home = "WT_TEST"; + +int main(void) +{ + int ret; + WT_CONNECTION *conn; + WT_SESSION *session; + + /* Open a connection to the database, creating it if necessary. */ + if ((ret = + wiredtiger_open(home, NULL, "create,transactional", &conn)) != 0) + fprintf(stderr, "Error connecting to %s: %s\n", + home, wiredtiger_strerror(ret)); + + /* Open a session for the current thread's work. */ + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + fprintf(stderr, "Error opening a session on %s: %s\n", + home, wiredtiger_strerror(ret)); + + /*! [transaction] */ + session->begin_transaction(session, NULL); + /* ... */ + session->commit_transaction(session, NULL); + /*! [transaction] */ + + /* Note: closing the connection implicitly closes open session(s). */ + if ((ret = conn->close(conn, NULL)) != 0) + fprintf(stderr, "Error connecting to %s: %s\n", + home, wiredtiger_strerror(ret)); + + return (ret); +} diff --git a/examples/python/ex_access.py b/examples/python/ex_access.py new file mode 100755 index 00000000000..5ae7f672651 --- /dev/null +++ b/examples/python/ex_access.py @@ -0,0 +1,64 @@ +#!/usr/bin/env PYTHONPATH=../../lang/python:../../lang/python/src python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ex_access.py +# demonstrates how to create and access a simple table. +# + +import wiredtiger +import sys + +home = 'WT_TEST' + +try: + conn = wiredtiger.wiredtiger_open(home, None, 'create') + print('connected: ' + `conn`); + session = conn.open_session(None, None) +except BaseException as e: + print('Error connecting to', (home + ':'), e); + sys.exit(1) + +# Note: further error checking omitted for clarity. + +session.create_table('access', 'key_format=S,value_format=S') +cursor = session.open_cursor('table:access', None, None) + +# Insert a record. +cursor.set_key('key1') +cursor.set_value('value1') + +# TODO: remove try block when cursor.insert works +try: + cursor.insert() +except BaseException as tuple: + print('Error cursor insert: ', tuple); + +for key, value in cursor: + print('Got record: ' + key + ' : ' + value) + +conn.close(None) +sys.exit(0) diff --git a/ext/collators/reverse/Makefile.am b/ext/collators/reverse/Makefile.am new file mode 100644 index 00000000000..33e52907692 --- /dev/null +++ b/ext/collators/reverse/Makefile.am @@ -0,0 +1,4 @@ +INCLUDES = -I$(top_builddir) -I$(top_srcdir)/src/include + +lib_LTLIBRARIES = reverse_collator.la +reverse_collator_la_LDFLAGS = -avoid-version -module diff --git a/ext/collators/reverse/reverse_collator.c b/ext/collators/reverse/reverse_collator.c new file mode 100644 index 00000000000..9a8a5eb59e7 --- /dev/null +++ b/ext/collators/reverse/reverse_collator.c @@ -0,0 +1,39 @@ +#include <errno.h> +#include <string.h> + +#include <wiredtiger_ext.h> + +WT_EXTENSION_API *wt_api; + +#define __UNUSED(v) ((void)(v)) + +static int +collate_reverse(WT_COLLATOR *collator, WT_SESSION *session, + const WT_ITEM *k1, const WT_ITEM *k2, int *cmp) +{ + size_t len; + + __UNUSED(collator); + __UNUSED(session); + + len = (k1->size < k2->size) ? k1->size : k2->size; + if ((*cmp = memcmp(k2->data, k1->data, len)) == 0) + *cmp = ((int)k1->size - (int)k2->size); + return (0); +} + +static WT_COLLATOR reverse_collator = { collate_reverse }; + +int +wiredtiger_extension_init( + WT_SESSION *session, WT_EXTENSION_API *api, const char *config) +{ + WT_CONNECTION *conn; + + __UNUSED(config); + + wt_api = api; + conn = session->connection; + + return (conn->add_collator(conn, "reverse", &reverse_collator, NULL)); +} diff --git a/ext/compressors/bzip2_compress/Makefile.am b/ext/compressors/bzip2_compress/Makefile.am new file mode 100644 index 00000000000..ac94ce1ef0a --- /dev/null +++ b/ext/compressors/bzip2_compress/Makefile.am @@ -0,0 +1,5 @@ +INCLUDES = -I$(top_builddir) -I$(top_srcdir)/src/include + +lib_LTLIBRARIES = bzip2_compress.la +bzip2_compress_la_LDFLAGS = -avoid-version -module +bzip2_compress_la_LIBADD = -lbz2 diff --git a/ext/compressors/bzip2_compress/bzip2_compress.c b/ext/compressors/bzip2_compress/bzip2_compress.c new file mode 100644 index 00000000000..46938d411aa --- /dev/null +++ b/ext/compressors/bzip2_compress/bzip2_compress.c @@ -0,0 +1,184 @@ +#include <bzlib.h> +#include <errno.h> +#include <stdlib.h> +#include <string.h> + +#include <wiredtiger.h> +#include <wiredtiger_ext.h> + +WT_EXTENSION_API *wt_api; + +static int +bzip2_compress(WT_COMPRESSOR *, + WT_SESSION *, uint8_t *, size_t, uint8_t *, size_t, size_t *, int *); +static int +bzip2_decompress(WT_COMPRESSOR *, + WT_SESSION *, uint8_t *, size_t, uint8_t *, size_t, size_t *); + +static WT_COMPRESSOR bzip2_compressor = { + bzip2_compress, bzip2_decompress, NULL }; + +#define __UNUSED(v) ((void)(v)) + +/* between 0-4: set the amount of verbosity to stderr */ +static int bz_verbosity = 0; + +/* between 1-9: set the block size to 100k x this number (compression only) */ +static int bz_blocksize100k = 1; + +/* + * between 0-250: workFactor: see bzip2 manual. 0 is a reasonable default + * (compression only) + */ +static int bz_workfactor = 0; + +/* if nonzero, decompress using less memory, but slower (decompression only) */ +static int bz_small = 0; + +int +wiredtiger_extension_init( + WT_SESSION *session, WT_EXTENSION_API *api, const char *config) +{ + WT_CONNECTION *conn; + + __UNUSED(config); + + wt_api = api; + conn = session->connection; + + return (conn->add_compressor( + conn, "bzip2_compress", &bzip2_compressor, NULL)); +} + +/* Bzip2 WT_COMPRESSOR implementation for WT_CONNECTION::add_compressor. */ +/* + * bzip2_error -- + * Output an error message, and return a standard error code. + */ +static int +bzip2_error(WT_SESSION *session, const char *call, int bzret) +{ + const char *msg; + + switch (bzret) { + case BZ_MEM_ERROR: + msg = "BZ_MEM_ERROR"; + break; + case BZ_OUTBUFF_FULL: + msg = "BZ_OUTBUFF_FULL"; + break; + case BZ_SEQUENCE_ERROR: + msg = "BZ_SEQUENCE_ERROR"; + break; + case BZ_PARAM_ERROR: + msg = "BZ_PARAM_ERROR"; + break; + case BZ_DATA_ERROR: + msg = "BZ_DATA_ERROR"; + break; + case BZ_DATA_ERROR_MAGIC: + msg = "BZ_DATA_ERROR_MAGIC"; + break; + case BZ_IO_ERROR: + msg = "BZ_IO_ERROR"; + break; + case BZ_UNEXPECTED_EOF: + msg = "BZ_UNEXPECTED_EOF"; + break; + case BZ_CONFIG_ERROR: + msg = "BZ_CONFIG_ERROR"; + break; + default: + msg = "unknown error"; + break; + } + + wiredtiger_err_printf( + session, "bzip2 error: %s: %s: %d", call, msg, bzret); + return (WT_ERROR); +} + +static void * +bzalloc(void *cookie, int number, int size) +{ + return (wiredtiger_scr_alloc(cookie, (size_t)number * size)); +} + +static void +bzfree(void *cookie, void *p) +{ + wiredtiger_scr_free(cookie, p); +} + +static int +bzip2_compress(WT_COMPRESSOR *compressor, WT_SESSION *session, + uint8_t *src, size_t src_len, + uint8_t *dst, size_t dst_len, + size_t *result_lenp, int *compression_failed) +{ + bz_stream bz; + int ret; + + __UNUSED(compressor); + + memset(&bz, 0, sizeof(bz)); + bz.bzalloc = bzalloc; + bz.bzfree = bzfree; + bz.opaque = session; + + if ((ret = BZ2_bzCompressInit(&bz, + bz_blocksize100k, bz_verbosity, bz_workfactor)) != BZ_OK) + return (bzip2_error(session, "BZ2_bzCompressInit", ret)); + + bz.next_in = src; + bz.avail_in = src_len; + bz.next_out = dst; + bz.avail_out = dst_len; + if ((ret = BZ2_bzCompress(&bz, BZ_FINISH)) == BZ_STREAM_END) { + *compression_failed = 0; + *result_lenp = dst_len - bz.avail_out; + } else + *compression_failed = 1; + + if ((ret = BZ2_bzCompressEnd(&bz)) != BZ_OK) + return (bzip2_error(session, "BZ2_bzCompressEnd", ret)); + + return (0); +} + +static int +bzip2_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, + uint8_t *src, size_t src_len, + uint8_t *dst, size_t dst_len, + size_t *result_lenp) +{ + bz_stream bz; + int ret, tret; + + __UNUSED(compressor); + + memset(&bz, 0, sizeof(bz)); + bz.bzalloc = bzalloc; + bz.bzfree = bzfree; + bz.opaque = session; + + if ((ret = BZ2_bzDecompressInit(&bz, bz_small, bz_verbosity)) != BZ_OK) + return (bzip2_error(session, "BZ2_bzDecompressInit", ret)); + + bz.next_in = src; + bz.avail_in = src_len; + bz.next_out = dst; + bz.avail_out = dst_len; + if ((ret = BZ2_bzDecompress(&bz)) == BZ_STREAM_END) { + *result_lenp = dst_len - bz.avail_out; + ret = 0; + } else + bzip2_error(session, "BZ2_bzDecompress", ret); + + if ((tret = BZ2_bzDecompressEnd(&bz)) != BZ_OK) + return (bzip2_error(session, "BZ2_bzDecompressEnd", tret)); + + return (ret == 0 ? + 0 : bzip2_error(session, "BZ2_bzDecompressEnd", ret)); +} +/* End Bzip2 WT_COMPRESSOR implementation for WT_CONNECTION::add_compressor. */ diff --git a/ext/compressors/nop_compress/Makefile.am b/ext/compressors/nop_compress/Makefile.am new file mode 100644 index 00000000000..13bd2af0734 --- /dev/null +++ b/ext/compressors/nop_compress/Makefile.am @@ -0,0 +1,4 @@ +INCLUDES = -I$(top_builddir) -I$(top_srcdir)/src/include + +lib_LTLIBRARIES = nop_compress.la +nop_compress_la_LDFLAGS = -avoid-version -module diff --git a/ext/compressors/nop_compress/nop_compress.c b/ext/compressors/nop_compress/nop_compress.c new file mode 100644 index 00000000000..dac303eab99 --- /dev/null +++ b/ext/compressors/nop_compress/nop_compress.c @@ -0,0 +1,73 @@ +#include <errno.h> +#include <string.h> + +#include <wiredtiger.h> +#include <wiredtiger_ext.h> + +WT_EXTENSION_API *wt_api; + +static int +nop_compress(WT_COMPRESSOR *, WT_SESSION *, + uint8_t *, size_t, uint8_t *, size_t, size_t *, int *); +static int +nop_decompress(WT_COMPRESSOR *, WT_SESSION *, + uint8_t *, size_t, uint8_t *, size_t, size_t *); + +static WT_COMPRESSOR nop_compressor = { nop_compress, nop_decompress, NULL }; + +#define __UNUSED(v) ((void)(v)) + +int +wiredtiger_extension_init( + WT_SESSION *session, WT_EXTENSION_API *api, const char *config) +{ + WT_CONNECTION *conn; + + __UNUSED(config); + + wt_api = api; + conn = session->connection; + + return ( + conn->add_compressor(conn, "nop_compress", &nop_compressor, NULL)); +} + +/* Implementation of WT_COMPRESSOR for WT_CONNECTION::add_compressor. */ +static int +nop_compress(WT_COMPRESSOR *compressor, WT_SESSION *session, + uint8_t *src, size_t src_len, + uint8_t *dst, size_t dst_len, + size_t *result_lenp, int *compression_failed) +{ + __UNUSED(compressor); + __UNUSED(session); + + *compression_failed = 0; + if (dst_len < src_len) { + *compression_failed = 1; + return (0); + } + + memcpy(dst, src, src_len); + *result_lenp = src_len; + + return (0); +} + +static int +nop_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, + uint8_t *src, size_t src_len, + uint8_t *dst, size_t dst_len, + size_t *result_lenp) +{ + __UNUSED(compressor); + __UNUSED(session); + + if (dst_len < src_len) + return (ENOMEM); + + memcpy(dst, src, src_len); + *result_lenp = src_len; + return (0); +} +/* End implementation of WT_COMPRESSOR. */ diff --git a/ext/compressors/snappy_compress/Makefile.am b/ext/compressors/snappy_compress/Makefile.am new file mode 100644 index 00000000000..4592b986f75 --- /dev/null +++ b/ext/compressors/snappy_compress/Makefile.am @@ -0,0 +1,5 @@ +INCLUDES = -I$(top_builddir) -I$(top_srcdir)/src/include + +lib_LTLIBRARIES = snappy_compress.la +snappy_compress_la_LDFLAGS = -avoid-version -module +snappy_compress_la_LIBADD = -lsnappy diff --git a/ext/compressors/snappy_compress/snappy_compress.c b/ext/compressors/snappy_compress/snappy_compress.c new file mode 100644 index 00000000000..a99bbffdff1 --- /dev/null +++ b/ext/compressors/snappy_compress/snappy_compress.c @@ -0,0 +1,159 @@ +#include <errno.h> +#include <snappy-c.h> +#include <stdlib.h> +#include <string.h> + +#include <wiredtiger.h> +#include <wiredtiger_ext.h> + +WT_EXTENSION_API *wt_api; + +static int +wt_snappy_compress(WT_COMPRESSOR *, WT_SESSION *, + uint8_t *, size_t, uint8_t *, size_t, size_t *, int *); +static int +wt_snappy_decompress(WT_COMPRESSOR *, WT_SESSION *, + uint8_t *, size_t, uint8_t *, size_t, size_t *); +static int +wt_snappy_pre_size(WT_COMPRESSOR *, WT_SESSION *, uint8_t *, size_t, size_t *); + +static WT_COMPRESSOR wt_snappy_compressor = { + wt_snappy_compress, wt_snappy_decompress, wt_snappy_pre_size }; + +#define __UNUSED(v) ((void)(v)) + +int +wiredtiger_extension_init( + WT_SESSION *session, WT_EXTENSION_API *api, const char *config) +{ + WT_CONNECTION *conn; + + __UNUSED(config); + + wt_api = api; + conn = session->connection; + + return (conn->add_compressor( + conn, "snappy_compress", &wt_snappy_compressor, NULL)); +} + +/* Snappy WT_COMPRESSOR for WT_CONNECTION::add_compressor. */ +/* + * wt_snappy_error -- + * Output an error message, and return a standard error code. + */ +static int +wt_snappy_error(WT_SESSION *session, const char *call, snappy_status snret) +{ + const char *msg; + + switch (snret) { + case SNAPPY_BUFFER_TOO_SMALL: + msg = "SNAPPY_BUFFER_TOO_SMALL"; + break; + case SNAPPY_INVALID_INPUT: + msg = "SNAPPY_INVALID_INPUT"; + break; + default: + msg = "unknown error"; + break; + } + + wiredtiger_err_printf( + session, "snappy error: %s: %s: %d", call, msg, snret); + return (WT_ERROR); +} + +static int +wt_snappy_compress(WT_COMPRESSOR *compressor, WT_SESSION *session, + uint8_t *src, size_t src_len, + uint8_t *dst, size_t dst_len, + size_t *result_lenp, int *compression_failed) +{ + snappy_status snret; + size_t snaplen; + char *snapbuf; + + __UNUSED(compressor); + + /* + * dst_len was computed in wt_snappy_pre_size, so we know it's big + * enough. Skip past the space we'll use to store the final count + * of compressed bytes. + */ + snaplen = dst_len - sizeof(size_t); + snapbuf = (char *)dst + sizeof(size_t); + + /* snaplen is an input and an output arg. */ + snret = snappy_compress((char *)src, src_len, snapbuf, &snaplen); + + if (snret == SNAPPY_OK) { + /* + * On decompression, snappy requires the exact compressed byte + * count (the current value of snaplen). WiredTiger does not + * preserve that value, so save snaplen at the beginning of the + * destination buffer. + */ + if (snaplen + sizeof(size_t) < src_len) { + *(size_t *)dst = snaplen; + *result_lenp = snaplen + sizeof(size_t); + *compression_failed = 0; + } else + /* The compressor failed to produce a smaller result. */ + *compression_failed = 1; + return (0); + } + return (wt_snappy_error(session, "snappy_compress", snret)); +} + +static int +wt_snappy_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, + uint8_t *src, size_t src_len, + uint8_t *dst, size_t dst_len, + size_t *result_lenp) +{ + snappy_status snret; + size_t snaplen; + + __UNUSED(compressor); + + /* retrieve the saved length */ + snaplen = *(size_t *)src; + if (snaplen + sizeof(size_t) > src_len) { + wiredtiger_err_printf( + session, + "wt_snappy_decompress: stored size exceeds buffer size"); + return (WT_ERROR); + } + + /* dst_len is an input and an output arg. */ + snret = snappy_uncompress( + (char *)src + sizeof(size_t), snaplen, (char *)dst, &dst_len); + + if (snret == SNAPPY_OK) { + *result_lenp = dst_len; + return (0); + } + + return (wt_snappy_error(session, "snappy_decompress", snret)); +} + +static int +wt_snappy_pre_size(WT_COMPRESSOR *compressor, WT_SESSION *session, + uint8_t *src, size_t src_len, + size_t *result_lenp) +{ + __UNUSED(compressor); + __UNUSED(session); + __UNUSED(src); + + /* + * Snappy requires the dest buffer be somewhat larger than the source. + * Fortunately, this is fast to compute, and will give us a dest buffer + * in wt_snappy_compress that we can compress to directly. We add space + * in the dest buffer to store the accurate compressed size. + */ + *result_lenp = snappy_max_compressed_length(src_len) + sizeof(size_t); + return (0); +} +/* End Snappy WT_COMPRESSOR for WT_CONNECTION::add_compressor. */ diff --git a/lang/python/LICENSE b/lang/python/LICENSE new file mode 100644 index 00000000000..d7a422fda11 --- /dev/null +++ b/lang/python/LICENSE @@ -0,0 +1,22 @@ +SWIG is free software: you can redistribute it and/or modify it +under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. See the LICENSE-GPL file for +the full terms of the GNU General Public license version 3. + +Portions of SWIG are also licensed under the terms of the licenses +in the file LICENSE-UNIVERSITIES. You must observe the terms of +these licenses, as well as the terms of the GNU General Public License, +when you distribute SWIG. + +The SWIG library and examples, under the Lib and Examples top level +directories, are distributed under the following terms: + + You may copy, modify, distribute, and make derivative works based on + this software, in source code or object code form, without + restriction. If you distribute the software to others, you may do + so according to the terms of your choice. This software is offered as + is, without warranty of any kind. + +See the COPYRIGHT file for a list of contributors to SWIG and their +copyright notices. diff --git a/lang/python/Makefile.am b/lang/python/Makefile.am new file mode 100644 index 00000000000..3a38be8a02d --- /dev/null +++ b/lang/python/Makefile.am @@ -0,0 +1,13 @@ +INCLUDES = -I$(abs_top_builddir) + +PYSRC = $(top_srcdir)/lang/python +if DEBUG +PY_SETUP_DEBUG = -g +endif +all-local: _wiredtiger.so +$(PYSRC)/wiredtiger_wrap.c: $(top_srcdir)/src/include/wiredtiger.in $(PYSRC)/wiredtiger.i + @(cd $(PYSRC) && \ + $(SWIG) -python -threads -I$(abs_top_builddir) wiredtiger.i) + +_wiredtiger.so: $(top_builddir)/libwiredtiger.la $(PYSRC)/wiredtiger_wrap.c + $(PYTHON) $(PYSRC)/setup.py build_ext -b . -t . -f $(PY_SETUP_DEBUG) diff --git a/lang/python/fpacking.py b/lang/python/fpacking.py new file mode 100644 index 00000000000..adabcfa807e --- /dev/null +++ b/lang/python/fpacking.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# +# WiredTiger fixed-size packing and unpacking functions, using the Python +# struct library. + +import struct + +def __wt2struct(fmt): + if not fmt: + return None, fmt + # Big endian with no alignment is the default + if fmt[0] in '@=<>!': + tfmt = fmt[0] + fmt = fmt[1:] + else: + tfmt = '>' + return tfmt, fmt.replace('r', 'Q') + +def unpack(fmt, s): + tfmt, fmt = __wt2struct(fmt) + if not fmt: + return () + result = () + pfmt = tfmt + sizebytes = 0 + for offset, f in enumerate(fmt): + if f.isdigit(): + sizebytes += 1 + # With a fixed size, everything is encoded as a string + if f in 'Su' and sizebytes > 0: + f = 's' + if f not in 'Su': + pfmt += f + sizebytes = 0 + continue + + # We've hit something that needs special handling, split any fixed-size + # values we've already passed + if len(pfmt) > 1: + size = struct.calcsize(pfmt) + result += struct.unpack_from(pfmt, s) + s = s[size:] + if f == 'S': + l = s.find('\0') + result += (s[:l],) + s = s[l+1:] + if f == 'u': + if offset == len(fmt) - 1: + result += (s,) + else: + l = struct.unpack_from(tfmt + 'l', s)[0] + s = s[struct.calcsize(tfmt + 'l'):] + result += (s[:l],) + s = s[l:] + pfmt = tfmt + sizebytes = 0 + + if len(pfmt) > 1: + result += struct.unpack(pfmt, s) + return result + +def pack(fmt, *values): + pfmt, fmt = __wt2struct(fmt) + if not fmt: + return '' + i = sizebytes = 0 + for offset, f in enumerate(fmt): + if f == 'S': + # Note: this code is being careful about embedded NUL characters + if sizebytes == 0: + l = values[i].find('\0') + 1 + if not l: + l = len(values[i]) + 1 + pfmt += str(l) + sizebytes = len(str(l)) + f = 's' + elif f == 'u': + if sizebytes == 0 and offset != len(fmt) - 1: + l = len(values[i]) + pfmt += 'l' + str(l) + values = values[:i] + (l,) + values[i:] + sizebytes = len(str(l)) + f = 's' + pfmt += f + if f.isdigit(): + sizebytes += 1 + continue + if f != 's' and sizebytes > 0: + i += int(pfmt[-sizebytes:]) + else: + i += 1 + sizebytes = 0 + return struct.pack(pfmt, *values) diff --git a/lang/python/intpack-test.py b/lang/python/intpack-test.py new file mode 100644 index 00000000000..02c5bf54701 --- /dev/null +++ b/lang/python/intpack-test.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. + +from intpacking import compress_int + +i = 1 +while i < 1 << 60: + print -i, ''.join('%02x' % ord(c) for c in compress_int(-i)) + print i, ''.join('%02x' % ord(c) for c in compress_int(i)) + i <<= 1 diff --git a/lang/python/intpacking.py b/lang/python/intpacking.py new file mode 100644 index 00000000000..3c94c1e7416 --- /dev/null +++ b/lang/python/intpacking.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. + +import math, struct + +# Variable-length integer packing +# need: up to 64 bits, both signed and unsigned +# +# Try hard for small values (up to ~2 bytes), after that, just encode the +# length in the first byte. +# +# First byte | Next | | +# byte | bytes| Min Value | Max Value +# ------------+------+------------------------+-------------------------------- +# [00 00xxxx] | free | N/A | N/A +# [00 01llll] | 8-l | -2^64 | -2^13 - 2^6 +# [00 1xxxxx] | 1 | -2^13 - 2^6 | -2^6 - 1 +# [01 xxxxxx] | 0 | -2^6 | -1 +# [10 xxxxxx] | 0 | 0 | 2^6 - 1 +# [11 0xxxxx] | 1 | 2^6 | 2^13 + 2^6 - 1 +# [11 10llll] | l | 2^14 + 2^7 | 2^64 - 1 +# [11 11xxxx] | free | N/A | N/A + +NEG_MULTI_MARKER = 0x10 +NEG_2BYTE_MARKER = 0x20 +NEG_1BYTE_MARKER = 0x40 +POS_1BYTE_MARKER = 0x80 +POS_2BYTE_MARKER = 0xc0 +POS_MULTI_MARKER = 0xe0 + +NEG_1BYTE_MIN = -2**6 +NEG_2BYTE_MIN = -2**13 + NEG_1BYTE_MIN +POS_1BYTE_MAX = 2**6 - 1 +POS_2BYTE_MAX = 2**13 + POS_1BYTE_MAX + +MINUS_BIT = -1 << 64 +UINT64_MASK = 0xffffffffffffffff + +def getbits(x, start, end=0): + '''return the least significant bits of x, from start to end''' + return (x & ((1 << start) - 1)) >> (end) + +def get_int(b, size): + r = 0; + for i in xrange(size): + r = (r << 8) | ord(b[i]) + return r + +def pack_int(x): + if x < NEG_2BYTE_MIN: + packed = struct.pack('>Q', x & UINT64_MASK) + while packed and packed[0] == '\xff': + packed = packed[1:] + return chr(NEG_MULTI_MARKER | getbits(8 - len(packed), 4)) + packed + elif x < NEG_1BYTE_MIN: + x -= NEG_2BYTE_MIN + return chr(NEG_2BYTE_MARKER | getbits(x, 13, 8)) + chr(getbits(x, 8)) + elif x < 0: + x -= NEG_1BYTE_MIN + return chr(NEG_1BYTE_MARKER | getbits(x, 6)) + elif x <= POS_1BYTE_MAX: + return chr(POS_1BYTE_MARKER | getbits(x, 6)) + elif x <= POS_2BYTE_MAX: + x -= (POS_1BYTE_MAX + 1) + return chr(POS_2BYTE_MARKER | getbits(x, 13, 8)) + chr(getbits(x, 8)) + else: + packed = struct.pack('>Q', x - (POS_2BYTE_MAX + 1)) + while packed and packed[0] == '\x00': + packed = packed[1:] + return chr(POS_MULTI_MARKER | getbits(len(packed), 4)) + packed + +def unpack_int(b): + marker = ord(b[0]) + if marker < NEG_2BYTE_MARKER: + sz = 8 - getbits(marker, 4) + return ((-1 << (sz << 3)) | get_int(b[1:], sz), b[sz+1:]) + elif marker < NEG_1BYTE_MARKER: + return (NEG_2BYTE_MIN + ((getbits(marker, 5) << 8) | ord(b[1])), b[2:]) + elif marker < POS_1BYTE_MARKER: + return (NEG_1BYTE_MIN + getbits(marker, 6), b[1:]) + elif marker < POS_2BYTE_MARKER: + return (getbits(marker, 6), b[1:]) + elif marker < POS_MULTI_MARKER: + return (POS_1BYTE_MAX + 1 + ((getbits(marker, 5) << 8) | ord(b[1])), b[2:]) + else: + sz = getbits(marker, 4) + return (POS_2BYTE_MAX + 1 + get_int(b[1:], sz), b[sz+1:]) + +# Sanity testing +if __name__ == '__main__': + import random + + for big in (100, 10000, 1 << 40, 1 << 64): + for i in xrange(1000): + r = random.randint(-big, big) + print "\rChecking %d" % r, + if unpack_int(pack_int(r))[0] != r: + print "\nFound a problem with %d" % r + break + + print + + for i in xrange(1000): + r1 = random.randint(-big, big) + r2 = random.randint(-big, big) + print "\rChecking %d, %d" % (r1, r2), + if cmp(r1, r2) != cmp(pack_int(r1), pack_int(r2)): + print "\nFound a problem with %d, %d" % (r1, r2) + break + + print diff --git a/lang/python/packing-test.py b/lang/python/packing-test.py new file mode 100644 index 00000000000..065e26e029b --- /dev/null +++ b/lang/python/packing-test.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. + +from packing import pack, unpack + +def check(fmt, *v): + print fmt, repr(v), ''.join('%02x' % ord(c) for c in pack(fmt, *v)) + +check('iii', 0, 101, -99) +check('3i', 0, 101, -99) +check('iS', 42, "forty two") +check('u', r"\x42" * 20) +check('uu', r"\x42" * 10, r"\x42" * 10) diff --git a/lang/python/packing.py b/lang/python/packing.py new file mode 100644 index 00000000000..a642f70ba75 --- /dev/null +++ b/lang/python/packing.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# +# WiredTiger variable-length packing and unpacking functions + +from intpacking import pack_int, unpack_int + +def __get_type(fmt): + if not fmt: + return None, fmt + # Variable-sized encoding is the default (and only supported format in v1) + if fmt[0] in '.@<>': + tfmt = fmt[0] + fmt = fmt[1:] + else: + tfmt = '.' + return tfmt, fmt + +def unpack(fmt, s): + tfmt, fmt = __get_type(fmt) + if not fmt: + return () + if tfmt != '.': + raise ValueError('Only variable-length encoding is currently supported') + result = [] + havesize = size = 0 + for offset, f in enumerate(fmt): + if f.isdigit(): + size = (size * 10) + int(f) + havesize = 1 + continue + elif f == 'x': + if not havesize: + size = 1 + s = s[size:] + # Note: no value, don't increment i + elif f in 'Ssu': + if not havesize: + if f == 's': + size = 1 + elif f == 'S': + size = s.find('\0') + elif f == 'u': + if offset == len(fmt) - 1: + size = len(s) + else: + size, s = unpack_int(s) + result.append(s[:size]) + if f == 'S' and not havesize: + size += 1 + s = s[size:] + elif f in 't': + # bit type, size is number of bits + if not havesize: + size = 1 + result.append(ord(s[0:1])) + s = s[1:] + else: + # integral type + if not havesize: + size = 1 + for j in xrange(size): + v, s = unpack_int(s) + result.append(v) + havesize = size = 0 + return result + +def pack(fmt, *values): + tfmt, fmt = __get_type(fmt) + if not fmt: + return () + if tfmt != '.': + raise ValueError('Only variable-length encoding is currently supported') + result = '' + havesize = i = size = 0 + for offset, f in enumerate(fmt): + if f.isdigit(): + size = (size * 10) + int(f) + havesize = 1 + continue + elif f == 'x': + if not havesize: + result += '\0' + else: + result += '\0' * size + # Note: no value, don't increment i + elif f in 'Ssu': + if f == 'S' and '\0' in values[i]: + l = values[i].find('\0') + else: + l = len(values[i]) + if havesize: + if l > size: + l = size + elif f == 's': + havesize = size = 1 + elif f == 'u' and offset != len(fmt) - 1: + result += pack_int(l) + result += values[i][:l] + if f == 'S' and not havesize: + result += '\0' + elif size > l: + result += '\0' * (size - l) + i += 1 + elif f in 't': + # bit type, size is number of bits + if not havesize: + size = 1 + if size > 8: + raise ValueError("bit count cannot be greater than 8 for 't' encoding") + mask = (1 << size) - 1 + val = values[i] + if (mask & val) != val: + raise ValueError("value out of range for 't' encoding") + result += chr(val) + i += 1 + else: + # integral type + if not havesize: + size = 1 + for j in xrange(size): + result += pack_int(values[i]) + i += 1 + havesize = size = 0 + return result diff --git a/lang/python/setup.py b/lang/python/setup.py new file mode 100644 index 00000000000..366f71532eb --- /dev/null +++ b/lang/python/setup.py @@ -0,0 +1,34 @@ +# +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. + +import re, os +from distutils.core import setup, Extension + +# OS X hack: turn off the Universal binary support that is built into the +# Python build machinery, just build for the default CPU architecture. +if not 'ARCHFLAGS' in os.environ: + os.environ['ARCHFLAGS'] = '' + +dir = os.path.dirname(__file__) + +# Read the version information from dist/RELEASE +dist = os.path.join(os.path.dirname(os.path.dirname(dir)), 'dist') +for l in open(os.path.join(dist, 'RELEASE')): + if re.match(r'WIREDTIGER_VERSION_(?:MAJOR|MINOR|PATCH)=', l): + exec(l) + +wt_ver = '%d.%d' % (WIREDTIGER_VERSION_MAJOR, WIREDTIGER_VERSION_MINOR) + +setup(name='wiredtiger', version=wt_ver, + ext_modules=[Extension('_wiredtiger', + [os.path.join(dir, 'wiredtiger_wrap.c')], + include_dirs=['../..'], + library_dirs=['../../.libs'], + libraries=['wiredtiger'], + )], + py_modules=['wiredtiger'], +) diff --git a/lang/python/src/server.py b/lang/python/src/server.py new file mode 100644 index 00000000000..6ec75ef8a24 --- /dev/null +++ b/lang/python/src/server.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# +# WiredTiger Python RPC server for testing and tutorials. + +import sys + +from wiredtiger.service import WiredTiger +from wiredtiger.service.ttypes import * + +from wiredtiger.impl import * + +from thrift.transport import TSocket +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol +from thrift.server import TServer + +class WiredTigerHandler: + +handler = WiredTigerHandler() +processor = WiredTiger.Processor(handler) +transport = TSocket.TServerSocket(9090) +tfactory = TTransport.TBufferedTransportFactory() +pfactory = TBinaryProtocol.TBinaryProtocolFactory() + +server = TServer.TSimpleServer(processor, transport, tfactory, pfactory) + +# We could do one of these for a multithreaded server +#server = TServer.TThreadedServer(processor, transport, tfactory, pfactory) +#server = TServer.TThreadPoolServer(processor, transport, tfactory, pfactory) + +print 'Starting the server...' +server.serve() +print 'done.' diff --git a/lang/python/src/wiredtiger/__init__.py b/lang/python/src/wiredtiger/__init__.py new file mode 100644 index 00000000000..149a95d5138 --- /dev/null +++ b/lang/python/src/wiredtiger/__init__.py @@ -0,0 +1,241 @@ +# +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# +# WiredTiger public interface + +''' +WiredTiger Python API. + +This module exports several functions and classes. +''' + +import struct +from urlparse import urlparse + +from wiredtiger.service import WiredTiger +from service.ttypes import WT_RECORD +from thrift.transport import TSocket, TTransport +from thrift.protocol import TBinaryProtocol + +def __wt2struct(fmt): + if not fmt: + return None, fmt + # Big endian with no alignment is the default + if fmt[0] in '@=<>!': + tfmt = fmt[0] + fmt = fmt[1:] + else: + tfmt = '>' + return tfmt, fmt.replace('r', 'Q') + +def unpack(fmt, s): + tfmt, fmt = __wt2struct(fmt) + if not fmt: + return () + result = () + pfmt = tfmt + sizebytes = 0 + for offset, f in enumerate(fmt): + if f.isdigit(): + sizebytes += 1 + # With a fixed size, everything is encoded as a string + if f in 'Su' and sizebytes > 0: + f = 's' + if f not in 'Su': + pfmt += f + sizebytes = 0 + continue + + # We've hit something that needs special handling, split any fixed-size + # values we've already passed + if len(pfmt) > 1: + size = struct.calcsize(pfmt) + result += struct.unpack_from(pfmt, s) + s = s[size:] + if f == 'S': + l = s.find('\0') + result += (s[:l],) + s = s[l+1:] + if f == 'u': + if offset == len(fmt) - 1: + result += (s,) + else: + l = struct.unpack_from(tfmt + 'l', s)[0] + s = s[struct.calcsize(tfmt + 'l'):] + result += (s[:l],) + s = s[l:] + pfmt = tfmt + sizebytes = 0 + + if len(pfmt) > 1: + result += struct.unpack(pfmt, s) + return result + +def pack(fmt, *values): + pfmt, fmt = __wt2struct(fmt) + if not fmt: + return '' + i = sizebytes = 0 + for offset, f in enumerate(fmt): + if f == 'S': + # Note: this code is being careful about embedded NUL characters + if sizebytes == 0: + l = values[i].find('\0') + 1 + if not l: + l = len(values[i]) + 1 + pfmt += str(l) + sizebytes = len(str(l)) + f = 's' + elif f == 'u': + if sizebytes == 0 and offset != len(fmt) - 1: + l = len(values[i]) + pfmt += 'l' + str(l) + values = values[:i] + (l,) + values[i:] + sizebytes = len(str(l)) + f = 's' + pfmt += f + if f.isdigit(): + sizebytes += 1 + continue + if f != 's' and sizebytes > 0: + i += int(pfmt[-sizebytes:]) + else: + i += 1 + sizebytes = 0 + return struct.pack(pfmt, *values) + +class Cursor: + def __init__(self, session, handle): + self.session = session + self.client = session.client + self.id = handle.id + self.keyfmt = handle.keyfmt + self.valuefmt = handle.valuefmt + + def close(self, config=''): + return self.client.close_cursor(self.id, config) + + def get_key(self): + return unpack(self.keyfmt, self.key) + + def get_value(self): + return unpack(self.keyfmt, self.value) + + def set_key(self, *args): + self.key = pack(self.keyfmt, *args) + + def set_value(self, *args): + self.value = pack(self.valuefmt, *args) + + def first(self): + result = self.client.move_first(self.id) + self.key, self.value = result.record.key, result.record.value + return result.exact + + def last(self): + result = self.client.move_last(self.id) + self.key, self.value = result.record.key, result.record.value + return result.exact + + def next(self): + result = self.client.move_next(self.id) + self.key, self.value = result.record.key, result.record.value + return result.exact + + def prev(self): + result = self.client.move_prev(self.id) + self.key, self.value = result.record.key, result.record.value + return result.exact + + def search(self): + result = self.client.search(self.id, WT_RECORD(self.key, self.value)) + self.key, self.value = result.record.key, result.record.value + return result.exact + + def insert(self): + self.key = self.client.insert_record(self.id, WT_RECORD(self.key, self.value)) + + def update(self): + return self.client.update_record(self.id, self.value) + + def delete(self): + return self.client.delete_record(self.id) + + +class Session: + def __init__(self, conn, id): + self.conn = conn + self.client = conn.client + self.id = id + + def close(self, config=''): + self.client.close_session(self.id, config) + + def open_cursor(self, uri, config=''): + return Cursor(self, self.client.open_cursor(self.id, uri, config)) + + def dup_cursor(self, c, config=''): + return Cursor(self, self.client.dup_cursor(self.id, c.id, config)) + + def create_table(self, name, config=''): + self.client.create_table(self.id, name, config) + + def rename_table(self, oldname, newname, config=''): + self.client.rename_table(self.id, oldname, newname, config) + + def drop_table(self, name, config=''): + self.client.drop_table(self.id, name, config) + + def truncate_table(self, name, start=None, end=None, config=''): + self.client.truncate_table(self.id, name, name, start and start.id or 0, end and end.id or 0, config) + + def verify_table(self, name, config=''): + self.client.verify_table(self.id, name, config) + + def begin_transaction(self, config=''): + self.client.begin_transaction(self.id, config) + + def commit_transaction(self): + self.client.begin_transaction(self.id) + + def rollback_transaction(self): + self.client.rollback_transaction(self.id) + + def checkpoint(self, config=''): + self.client.checkpoint(self.id, config) + + +class Connection: + def __init__(self, uri, config=''): + url = urlparse(uri) + parts = url[1].split(':') + host = parts[0] + if len(parts) > 1: + port = int(parts[1]) + else: + port = 9090 + home = url[2] + + socket = TSocket.TSocket(host, port) + self.transport = TTransport.TBufferedTransport(socket) + protocol = TBinaryProtocol.TBinaryProtocol(self.transport) + self.client = WiredTiger.Client(protocol) + self.transport.open() + + self.id = self.client.open(home, config) + + def close(self, config=''): + self.client.close_connection(self.id, config) + self.transport.close() + + def version(self): + v = self.client.version(self.id, config) + return v.version_string, v.major, v.minor, v.patch + + def open_session(self, config=''): + id = self.client.open_session(self.id, config) + return Session(self, id) diff --git a/lang/python/src/wiredtiger/impl/__init__.py b/lang/python/src/wiredtiger/impl/__init__.py new file mode 100644 index 00000000000..2078d90a72c --- /dev/null +++ b/lang/python/src/wiredtiger/impl/__init__.py @@ -0,0 +1,219 @@ +# +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. +# +# WiredTiger API implementation + +''' +WiredTiger Python API implementation. +''' + +import struct +from urlparse import urlparse +from wiredtiger import pack, unpack +from wiredtiger.util import parse_config + +# Import the BDB symbols with the 'db.' prefix, it avoids polluting the +# namespace of this package +from bsddb3 import db + +class Table: + def __init__(self, db, name, key_format='u', value_format='u', columns=(,), colgroups=(,), indices=(,)): + self.db = db + self.name = name + self.key_format = key_format + self.value_format = value_format + self.columns = columns + self.colgroups = colgroups + self.indices = indices + + def close(self): + self.db.close(db.DB_NOSYNC) + + def check_schema(self, key_format='u', value_format='u', columns=(,), colgroups=(,), indices=(,)): + if (self.key_format != key_format or + self.value_format != value_format or + self.columns != columns or + self.colgroups != colgroups or + self.indices != indices): + raise 'Schemas don\'t match for table "' + self.name + '"' + +class Cursor: + def __init__(self, session, table): + self.session = session + self.table = table + self.key_format = table.key_format + self.value_format = table.value_format + self.dbc = table.db.cursor() + + def close(self, config=''): + self.dbc.close() + self.session.cursors.remove(self) + + def get_key(self): + return unpack(self.key_format, self.key) + + def get_value(self): + return unpack(self.key_format, self.value) + + def set_key(self, *args): + self.key = pack(self.key_format, *args) + + def set_value(self, *args): + self.value = pack(self.value_format, *args) + + def first(self): + self.key, self.value = self.dbc.first() + + def last(self): + self.key, self.value = self.dbc.last() + + def next(self): + self.key, self.value = self.dbc.next() + + def prev(self): + self.key, self.value = self.dbc.prev() + + def search(self): + searchkey = self.key + self.key, self.value = self.dbc.set_range(self.key) + return (self.key == searchkey) + + def insert(self): + self.dbc.put(self.key, self.value) + return self.key + + def update(self): + self.dbc.put(self.key, self.value, db.DB_CURRENT) + + def delete(self): + self.dbc.delete() + + +class Session: + def __init__(self, conn, id): + self.conn = conn + self.cursors = [] + self.tables = {'schema' : conn.schematab} + + def _close_cursors(self): + # Work on a copy of the list because Cursor.close removes itself + for c in self.cursors[:]: + c.close() + + def close(self, config=''): + self._close_cursors() + self.conn.sessions.remove(self) + + def open_cursor(self, uri, config=''): + c = self.conn._open_cursor(self, uri, config) + self.cursors.append(c) + return c + + def dup_cursor(self, c, config=''): + dupc = c.dup() + self.cursors.append(dupc) + return dupc + + def _open_table(self, name): + schema_cur = Cursor(self, self.conn.schematab) + schema_cur.set_key(name) + if schema_cur.search(): + k, v, c, cset, idx = schema_cur.get_value() + c = tuple(parse_config(c)) + cset = tuple(parse_config(cset)) + idx = tuple(parse_config(idx)) + self.tables[name] = Table(k, v, c, cset, idx) + + def create_table(self, name, config=''): + schema = {} + for k, v in parse_config(config): + if k in ('key_format', 'value_format', 'columns'): + schema[k] = v + elif k.startswith('colgroup'): + schema['colgroup'] = schema.get('colgroup', (,)) + (k[len('colgroup')+1:], v) + elif k.startswith('index'): + schema['indices'] = schema.get('indices', (,)) + (k[len('index')+1:], v) + else: + raise 'Unknown configuration "' + k + '"' + if name in self.tables: + self.tables[name].check_schema(**schema) + # XXX else try to open the table and retry + + def rename_table(self, oldname, newname, config=''): + pass + + def drop_table(self, name, config=''): + pass + + def truncate_table(self, name, start=None, end=None, config=''): + pass + + def verify_table(self, name, config=''): + pass + + def begin_transaction(self, config=''): + if self.cursors: + raise 'Transactions cannot be started with cursors open' + + def commit_transaction(self): + self._close_cursors() + pass + + def rollback_transaction(self): + self._close_cursors() + pass + + def checkpoint(self, config=''): + pass + + +class Connection: + def __init__(self, uri, config=''): + url = urlparse(uri) + parts = url[1].split(':') + self.host = parts[0] + if len(parts) > 1: + port = int(parts[1]) + else: + port = 9090 + + self.home = url[2] + self.sessions = [] + + self.env = db.DBEnv(); + self.env.open(self.home, + db.DB_PRIVATE | db.DB_CREATE | db.DB_INIT_MPOOL | db.DB_THREAD) + schemadb = db.DB(self.env); + schemadb.open("__wt_schema.db", None, db.DB_BTREE, db.DB_CREATE) + + # The schema of the schema table. + self.schematab = Table(schemadb, key_format='S', value_format='SSSSS', + columns=('name', 'key_format', 'value_format', 'colgroups', 'indices')) + + def close(self, config=''): + # Work on a copy of the list because Session.close removes itself + for s in self.sessions[:]: + s.close() + self.schematab.close() + self.env.close() + + def version(self): + return ("WiredTiger Python API 0.0.1", 0, 0, 1) + + def open_session(self, config=''): + s = Session(self, config) + self.sessions.append(s) + return s + + def _open_cursor(self, session, uri, config): + if uri == 'table:': + return Cursor(session, self.schematab) + elif uri.startswith('table:'): + # XXX projections + return Cursor(session, session._get_table(uri[6:])) + # XXX application-specific cursor types? + raise 'Unknown cursor type for "' + uri + '"' diff --git a/lang/python/src/wiredtiger/service/WiredTiger-remote b/lang/python/src/wiredtiger/service/WiredTiger-remote new file mode 100755 index 00000000000..027792c9c55 --- /dev/null +++ b/lang/python/src/wiredtiger/service/WiredTiger-remote @@ -0,0 +1,267 @@ +#!/usr/bin/env python +# +# Autogenerated by Thrift +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# + +import sys +import pprint +from urlparse import urlparse +from thrift.transport import TTransport +from thrift.transport import TSocket +from thrift.transport import THttpClient +from thrift.protocol import TBinaryProtocol + +import WiredTiger +from ttypes import * + +if len(sys.argv) <= 1 or sys.argv[1] == '--help': + print '' + print 'Usage: ' + sys.argv[0] + ' [-h host:port] [-u url] [-f[ramed]] function [arg1 [arg2...]]' + print '' + print 'Functions:' + print ' string strerror(i32 err)' + print ' WT_VERSION version()' + print ' WT_HANDLE open(string home, string config)' + print ' WT_HANDLE open_session(WT_HANDLE connection, string config)' + print ' void close_connection(WT_HANDLE connection, string config)' + print ' bool is_new(WT_HANDLE connection)' + print ' void close_session(WT_HANDLE session, string config)' + print ' WT_CURSOR_HANDLE open_cursor(WT_HANDLE session, string uri, WT_HANDLE to_dup, string config)' + print ' void create_table(WT_HANDLE session, string name, string config)' + print ' void rename_table(WT_HANDLE session, string oldname, string newname, string config)' + print ' void drop_table(WT_HANDLE session, string name, string config)' + print ' void truncate_table(WT_HANDLE session, string name, WT_HANDLE cursor_start, WT_HANDLE cursor_end, string config)' + print ' void verify_table(WT_HANDLE session, string name, string config)' + print ' void begin_transaction(WT_HANDLE session, string config)' + print ' void commit_transaction(WT_HANDLE session, string config)' + print ' void rollback_transaction(WT_HANDLE session, string config)' + print ' void checkpoint(WT_HANDLE session, string config)' + print ' WT_MOVE_RESULT move_first(WT_HANDLE cursor)' + print ' WT_MOVE_RESULT move_last(WT_HANDLE cursor)' + print ' WT_MOVE_RESULT move_next(WT_HANDLE cursor)' + print ' WT_MOVE_RESULT move_prev(WT_HANDLE cursor)' + print ' WT_MOVE_RESULT search(WT_HANDLE cursor, WT_RECORD record)' + print ' WT_MOVE_RESULT search_near(WT_HANDLE cursor, WT_RECORD record)' + print ' string insert_record(WT_HANDLE cursor, WT_RECORD record)' + print ' void update_record(WT_HANDLE cursor, string value)' + print ' void delete_record(WT_HANDLE cursor)' + print ' void close_cursor(WT_HANDLE cursor, string config)' + print '' + sys.exit(0) + +pp = pprint.PrettyPrinter(indent = 2) +host = 'localhost' +port = 9090 +uri = '' +framed = False +http = False +argi = 1 + +if sys.argv[argi] == '-h': + parts = sys.argv[argi+1].split(':') + host = parts[0] + port = int(parts[1]) + argi += 2 + +if sys.argv[argi] == '-u': + url = urlparse(sys.argv[argi+1]) + parts = url[1].split(':') + host = parts[0] + if len(parts) > 1: + port = int(parts[1]) + else: + port = 80 + uri = url[2] + if url[4]: + uri += '?%s' % url[4] + http = True + argi += 2 + +if sys.argv[argi] == '-f' or sys.argv[argi] == '-framed': + framed = True + argi += 1 + +cmd = sys.argv[argi] +args = sys.argv[argi+1:] + +if http: + transport = THttpClient.THttpClient(host, port, uri) +else: + socket = TSocket.TSocket(host, port) + if framed: + transport = TTransport.TFramedTransport(socket) + else: + transport = TTransport.TBufferedTransport(socket) +protocol = TBinaryProtocol.TBinaryProtocol(transport) +client = WiredTiger.Client(protocol) +transport.open() + +if cmd == 'strerror': + if len(args) != 1: + print 'strerror requires 1 args' + sys.exit(1) + pp.pprint(client.strerror(eval(args[0]),)) + +elif cmd == 'version': + if len(args) != 0: + print 'version requires 0 args' + sys.exit(1) + pp.pprint(client.version()) + +elif cmd == 'open': + if len(args) != 2: + print 'open requires 2 args' + sys.exit(1) + pp.pprint(client.open(args[0],args[1],)) + +elif cmd == 'open_session': + if len(args) != 2: + print 'open_session requires 2 args' + sys.exit(1) + pp.pprint(client.open_session(eval(args[0]),args[1],)) + +elif cmd == 'close_connection': + if len(args) != 2: + print 'close_connection requires 2 args' + sys.exit(1) + pp.pprint(client.close_connection(eval(args[0]),args[1],)) + +elif cmd == 'is_new': + if len(args) != 1: + print 'is_new requires 1 args' + sys.exit(1) + pp.pprint(client.is_new(eval(args[0]),)) + +elif cmd == 'close_session': + if len(args) != 2: + print 'close_session requires 2 args' + sys.exit(1) + pp.pprint(client.close_session(eval(args[0]),args[1],)) + +elif cmd == 'open_cursor': + if len(args) != 4: + print 'open_cursor requires 4 args' + sys.exit(1) + pp.pprint(client.open_cursor(eval(args[0]),args[1],eval(args[2]),args[3],)) + +elif cmd == 'create_table': + if len(args) != 3: + print 'create_table requires 3 args' + sys.exit(1) + pp.pprint(client.create_table(eval(args[0]),args[1],args[2],)) + +elif cmd == 'rename_table': + if len(args) != 4: + print 'rename_table requires 4 args' + sys.exit(1) + pp.pprint(client.rename_table(eval(args[0]),args[1],args[2],args[3],)) + +elif cmd == 'drop_table': + if len(args) != 3: + print 'drop_table requires 3 args' + sys.exit(1) + pp.pprint(client.drop_table(eval(args[0]),args[1],args[2],)) + +elif cmd == 'truncate_table': + if len(args) != 5: + print 'truncate_table requires 5 args' + sys.exit(1) + pp.pprint(client.truncate_table(eval(args[0]),args[1],eval(args[2]),eval(args[3]),args[4],)) + +elif cmd == 'verify_table': + if len(args) != 3: + print 'verify_table requires 3 args' + sys.exit(1) + pp.pprint(client.verify_table(eval(args[0]),args[1],args[2],)) + +elif cmd == 'begin_transaction': + if len(args) != 2: + print 'begin_transaction requires 2 args' + sys.exit(1) + pp.pprint(client.begin_transaction(eval(args[0]),args[1],)) + +elif cmd == 'commit_transaction': + if len(args) != 2: + print 'commit_transaction requires 2 args' + sys.exit(1) + pp.pprint(client.commit_transaction(eval(args[0]),args[1],)) + +elif cmd == 'rollback_transaction': + if len(args) != 2: + print 'rollback_transaction requires 2 args' + sys.exit(1) + pp.pprint(client.rollback_transaction(eval(args[0]),args[1],)) + +elif cmd == 'checkpoint': + if len(args) != 2: + print 'checkpoint requires 2 args' + sys.exit(1) + pp.pprint(client.checkpoint(eval(args[0]),args[1],)) + +elif cmd == 'move_first': + if len(args) != 1: + print 'move_first requires 1 args' + sys.exit(1) + pp.pprint(client.move_first(eval(args[0]),)) + +elif cmd == 'move_last': + if len(args) != 1: + print 'move_last requires 1 args' + sys.exit(1) + pp.pprint(client.move_last(eval(args[0]),)) + +elif cmd == 'move_next': + if len(args) != 1: + print 'move_next requires 1 args' + sys.exit(1) + pp.pprint(client.move_next(eval(args[0]),)) + +elif cmd == 'move_prev': + if len(args) != 1: + print 'move_prev requires 1 args' + sys.exit(1) + pp.pprint(client.move_prev(eval(args[0]),)) + +elif cmd == 'search': + if len(args) != 2: + print 'search requires 2 args' + sys.exit(1) + pp.pprint(client.search(eval(args[0]),eval(args[1]),)) + +elif cmd == 'search_near': + if len(args) != 2: + print 'search_near requires 2 args' + sys.exit(1) + pp.pprint(client.search_near(eval(args[0]),eval(args[1]),)) + +elif cmd == 'insert_record': + if len(args) != 2: + print 'insert_record requires 2 args' + sys.exit(1) + pp.pprint(client.insert_record(eval(args[0]),eval(args[1]),)) + +elif cmd == 'update_record': + if len(args) != 2: + print 'update_record requires 2 args' + sys.exit(1) + pp.pprint(client.update_record(eval(args[0]),args[1],)) + +elif cmd == 'delete_record': + if len(args) != 1: + print 'delete_record requires 1 args' + sys.exit(1) + pp.pprint(client.delete_record(eval(args[0]),)) + +elif cmd == 'close_cursor': + if len(args) != 2: + print 'close_cursor requires 2 args' + sys.exit(1) + pp.pprint(client.close_cursor(eval(args[0]),args[1],)) + +else: + print 'Unrecognized method %s' % cmd + sys.exit(1) + +transport.close() diff --git a/lang/python/src/wiredtiger/service/WiredTiger.py b/lang/python/src/wiredtiger/service/WiredTiger.py new file mode 100644 index 00000000000..e6689cf2714 --- /dev/null +++ b/lang/python/src/wiredtiger/service/WiredTiger.py @@ -0,0 +1,5200 @@ +# +# Autogenerated by Thrift +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# + +from thrift.Thrift import * +from ttypes import * +from thrift.Thrift import TProcessor +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol, TProtocol +try: + from thrift.protocol import fastbinary +except: + fastbinary = None + + +class Iface: + def strerror(self, err): + """ + Parameters: + - err + """ + pass + + def version(self, ): + pass + + def open(self, home, config): + """ + Parameters: + - home + - config + """ + pass + + def open_session(self, connection, config): + """ + Parameters: + - connection + - config + """ + pass + + def close_connection(self, connection, config): + """ + Parameters: + - connection + - config + """ + pass + + def is_new(self, connection): + """ + Parameters: + - connection + """ + pass + + def close_session(self, session, config): + """ + Parameters: + - session + - config + """ + pass + + def open_cursor(self, session, uri, to_dup, config): + """ + Parameters: + - session + - uri + - to_dup + - config + """ + pass + + def create_table(self, session, name, config): + """ + Parameters: + - session + - name + - config + """ + pass + + def rename_table(self, session, oldname, newname, config): + """ + Parameters: + - session + - oldname + - newname + - config + """ + pass + + def drop_table(self, session, name, config): + """ + Parameters: + - session + - name + - config + """ + pass + + def truncate_table(self, session, name, cursor_start, cursor_end, config): + """ + Parameters: + - session + - name + - cursor_start + - cursor_end + - config + """ + pass + + def verify_table(self, session, name, config): + """ + Parameters: + - session + - name + - config + """ + pass + + def begin_transaction(self, session, config): + """ + Parameters: + - session + - config + """ + pass + + def commit_transaction(self, session, config): + """ + Parameters: + - session + - config + """ + pass + + def rollback_transaction(self, session, config): + """ + Parameters: + - session + - config + """ + pass + + def checkpoint(self, session, config): + """ + Parameters: + - session + - config + """ + pass + + def move_first(self, cursor): + """ + Parameters: + - cursor + """ + pass + + def move_last(self, cursor): + """ + Parameters: + - cursor + """ + pass + + def move_next(self, cursor): + """ + Parameters: + - cursor + """ + pass + + def move_prev(self, cursor): + """ + Parameters: + - cursor + """ + pass + + def search(self, cursor, record): + """ + Parameters: + - cursor + - record + """ + pass + + def search_near(self, cursor, record): + """ + Parameters: + - cursor + - record + """ + pass + + def insert_record(self, cursor, record): + """ + Parameters: + - cursor + - record + """ + pass + + def update_record(self, cursor, value): + """ + Parameters: + - cursor + - value + """ + pass + + def delete_record(self, cursor): + """ + Parameters: + - cursor + """ + pass + + def close_cursor(self, cursor, config): + """ + Parameters: + - cursor + - config + """ + pass + + +class Client(Iface): + def __init__(self, iprot, oprot=None): + self._iprot = self._oprot = iprot + if oprot != None: + self._oprot = oprot + self._seqid = 0 + + def strerror(self, err): + """ + Parameters: + - err + """ + self.send_strerror(err) + return self.recv_strerror() + + def send_strerror(self, err): + self._oprot.writeMessageBegin('strerror', TMessageType.CALL, self._seqid) + args = strerror_args() + args.err = err + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_strerror(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = strerror_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success != None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "strerror failed: unknown result"); + + def version(self, ): + self.send_version() + return self.recv_version() + + def send_version(self, ): + self._oprot.writeMessageBegin('version', TMessageType.CALL, self._seqid) + args = version_args() + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_version(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = version_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success != None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "version failed: unknown result"); + + def open(self, home, config): + """ + Parameters: + - home + - config + """ + self.send_open(home, config) + return self.recv_open() + + def send_open(self, home, config): + self._oprot.writeMessageBegin('open', TMessageType.CALL, self._seqid) + args = open_args() + args.home = home + args.config = config + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_open(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = open_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success != None: + return result.success + if result.err != None: + raise result.err + raise TApplicationException(TApplicationException.MISSING_RESULT, "open failed: unknown result"); + + def open_session(self, connection, config): + """ + Parameters: + - connection + - config + """ + self.send_open_session(connection, config) + return self.recv_open_session() + + def send_open_session(self, connection, config): + self._oprot.writeMessageBegin('open_session', TMessageType.CALL, self._seqid) + args = open_session_args() + args.connection = connection + args.config = config + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_open_session(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = open_session_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success != None: + return result.success + if result.err != None: + raise result.err + raise TApplicationException(TApplicationException.MISSING_RESULT, "open_session failed: unknown result"); + + def close_connection(self, connection, config): + """ + Parameters: + - connection + - config + """ + self.send_close_connection(connection, config) + self.recv_close_connection() + + def send_close_connection(self, connection, config): + self._oprot.writeMessageBegin('close_connection', TMessageType.CALL, self._seqid) + args = close_connection_args() + args.connection = connection + args.config = config + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_close_connection(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = close_connection_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.err != None: + raise result.err + return + + def is_new(self, connection): + """ + Parameters: + - connection + """ + self.send_is_new(connection) + return self.recv_is_new() + + def send_is_new(self, connection): + self._oprot.writeMessageBegin('is_new', TMessageType.CALL, self._seqid) + args = is_new_args() + args.connection = connection + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_is_new(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = is_new_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success != None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "is_new failed: unknown result"); + + def close_session(self, session, config): + """ + Parameters: + - session + - config + """ + self.send_close_session(session, config) + self.recv_close_session() + + def send_close_session(self, session, config): + self._oprot.writeMessageBegin('close_session', TMessageType.CALL, self._seqid) + args = close_session_args() + args.session = session + args.config = config + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_close_session(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = close_session_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.err != None: + raise result.err + return + + def open_cursor(self, session, uri, to_dup, config): + """ + Parameters: + - session + - uri + - to_dup + - config + """ + self.send_open_cursor(session, uri, to_dup, config) + return self.recv_open_cursor() + + def send_open_cursor(self, session, uri, to_dup, config): + self._oprot.writeMessageBegin('open_cursor', TMessageType.CALL, self._seqid) + args = open_cursor_args() + args.session = session + args.uri = uri + args.to_dup = to_dup + args.config = config + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_open_cursor(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = open_cursor_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success != None: + return result.success + if result.err != None: + raise result.err + raise TApplicationException(TApplicationException.MISSING_RESULT, "open_cursor failed: unknown result"); + + def create_table(self, session, name, config): + """ + Parameters: + - session + - name + - config + """ + self.send_create_table(session, name, config) + self.recv_create_table() + + def send_create_table(self, session, name, config): + self._oprot.writeMessageBegin('create_table', TMessageType.CALL, self._seqid) + args = create_table_args() + args.session = session + args.name = name + args.config = config + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_create_table(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = create_table_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.err != None: + raise result.err + return + + def rename_table(self, session, oldname, newname, config): + """ + Parameters: + - session + - oldname + - newname + - config + """ + self.send_rename_table(session, oldname, newname, config) + self.recv_rename_table() + + def send_rename_table(self, session, oldname, newname, config): + self._oprot.writeMessageBegin('rename_table', TMessageType.CALL, self._seqid) + args = rename_table_args() + args.session = session + args.oldname = oldname + args.newname = newname + args.config = config + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_rename_table(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = rename_table_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.err != None: + raise result.err + return + + def drop_table(self, session, name, config): + """ + Parameters: + - session + - name + - config + """ + self.send_drop_table(session, name, config) + self.recv_drop_table() + + def send_drop_table(self, session, name, config): + self._oprot.writeMessageBegin('drop_table', TMessageType.CALL, self._seqid) + args = drop_table_args() + args.session = session + args.name = name + args.config = config + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_drop_table(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = drop_table_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.err != None: + raise result.err + return + + def truncate_table(self, session, name, cursor_start, cursor_end, config): + """ + Parameters: + - session + - name + - cursor_start + - cursor_end + - config + """ + self.send_truncate_table(session, name, cursor_start, cursor_end, config) + self.recv_truncate_table() + + def send_truncate_table(self, session, name, cursor_start, cursor_end, config): + self._oprot.writeMessageBegin('truncate_table', TMessageType.CALL, self._seqid) + args = truncate_table_args() + args.session = session + args.name = name + args.cursor_start = cursor_start + args.cursor_end = cursor_end + args.config = config + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_truncate_table(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = truncate_table_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.err != None: + raise result.err + return + + def verify_table(self, session, name, config): + """ + Parameters: + - session + - name + - config + """ + self.send_verify_table(session, name, config) + self.recv_verify_table() + + def send_verify_table(self, session, name, config): + self._oprot.writeMessageBegin('verify_table', TMessageType.CALL, self._seqid) + args = verify_table_args() + args.session = session + args.name = name + args.config = config + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_verify_table(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = verify_table_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.err != None: + raise result.err + return + + def begin_transaction(self, session, config): + """ + Parameters: + - session + - config + """ + self.send_begin_transaction(session, config) + self.recv_begin_transaction() + + def send_begin_transaction(self, session, config): + self._oprot.writeMessageBegin('begin_transaction', TMessageType.CALL, self._seqid) + args = begin_transaction_args() + args.session = session + args.config = config + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_begin_transaction(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = begin_transaction_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.err != None: + raise result.err + return + + def commit_transaction(self, session, config): + """ + Parameters: + - session + - config + """ + self.send_commit_transaction(session, config) + self.recv_commit_transaction() + + def send_commit_transaction(self, session, config): + self._oprot.writeMessageBegin('commit_transaction', TMessageType.CALL, self._seqid) + args = commit_transaction_args() + args.session = session + args.config = config + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_commit_transaction(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = commit_transaction_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.err != None: + raise result.err + return + + def rollback_transaction(self, session, config): + """ + Parameters: + - session + - config + """ + self.send_rollback_transaction(session, config) + self.recv_rollback_transaction() + + def send_rollback_transaction(self, session, config): + self._oprot.writeMessageBegin('rollback_transaction', TMessageType.CALL, self._seqid) + args = rollback_transaction_args() + args.session = session + args.config = config + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_rollback_transaction(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = rollback_transaction_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.err != None: + raise result.err + return + + def checkpoint(self, session, config): + """ + Parameters: + - session + - config + """ + self.send_checkpoint(session, config) + self.recv_checkpoint() + + def send_checkpoint(self, session, config): + self._oprot.writeMessageBegin('checkpoint', TMessageType.CALL, self._seqid) + args = checkpoint_args() + args.session = session + args.config = config + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_checkpoint(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = checkpoint_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.err != None: + raise result.err + return + + def move_first(self, cursor): + """ + Parameters: + - cursor + """ + self.send_move_first(cursor) + return self.recv_move_first() + + def send_move_first(self, cursor): + self._oprot.writeMessageBegin('move_first', TMessageType.CALL, self._seqid) + args = move_first_args() + args.cursor = cursor + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_move_first(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = move_first_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success != None: + return result.success + if result.err != None: + raise result.err + raise TApplicationException(TApplicationException.MISSING_RESULT, "move_first failed: unknown result"); + + def move_last(self, cursor): + """ + Parameters: + - cursor + """ + self.send_move_last(cursor) + return self.recv_move_last() + + def send_move_last(self, cursor): + self._oprot.writeMessageBegin('move_last', TMessageType.CALL, self._seqid) + args = move_last_args() + args.cursor = cursor + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_move_last(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = move_last_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success != None: + return result.success + if result.err != None: + raise result.err + raise TApplicationException(TApplicationException.MISSING_RESULT, "move_last failed: unknown result"); + + def move_next(self, cursor): + """ + Parameters: + - cursor + """ + self.send_move_next(cursor) + return self.recv_move_next() + + def send_move_next(self, cursor): + self._oprot.writeMessageBegin('move_next', TMessageType.CALL, self._seqid) + args = move_next_args() + args.cursor = cursor + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_move_next(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = move_next_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success != None: + return result.success + if result.err != None: + raise result.err + raise TApplicationException(TApplicationException.MISSING_RESULT, "move_next failed: unknown result"); + + def move_prev(self, cursor): + """ + Parameters: + - cursor + """ + self.send_move_prev(cursor) + return self.recv_move_prev() + + def send_move_prev(self, cursor): + self._oprot.writeMessageBegin('move_prev', TMessageType.CALL, self._seqid) + args = move_prev_args() + args.cursor = cursor + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_move_prev(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = move_prev_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success != None: + return result.success + if result.err != None: + raise result.err + raise TApplicationException(TApplicationException.MISSING_RESULT, "move_prev failed: unknown result"); + + def search(self, cursor, record): + """ + Parameters: + - cursor + - record + """ + self.send_search(cursor, record) + return self.recv_search() + + def send_search(self, cursor, record): + self._oprot.writeMessageBegin('search', TMessageType.CALL, self._seqid) + args = search_args() + args.cursor = cursor + args.record = record + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_search(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = search_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success != None: + return result.success + if result.err != None: + raise result.err + raise TApplicationException(TApplicationException.MISSING_RESULT, "search failed: unknown result"); + + def search_near(self, cursor, record): + """ + Parameters: + - cursor + - record + """ + self.send_search_near(cursor, record) + return self.recv_search_near() + + def send_search_near(self, cursor, record): + self._oprot.writeMessageBegin('search_near', TMessageType.CALL, self._seqid) + args = search_near_args() + args.cursor = cursor + args.record = record + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_search_near(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = search_near_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success != None: + return result.success + if result.err != None: + raise result.err + raise TApplicationException(TApplicationException.MISSING_RESULT, "search_near failed: unknown result"); + + def insert_record(self, cursor, record): + """ + Parameters: + - cursor + - record + """ + self.send_insert_record(cursor, record) + return self.recv_insert_record() + + def send_insert_record(self, cursor, record): + self._oprot.writeMessageBegin('insert_record', TMessageType.CALL, self._seqid) + args = insert_record_args() + args.cursor = cursor + args.record = record + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_insert_record(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = insert_record_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success != None: + return result.success + if result.err != None: + raise result.err + raise TApplicationException(TApplicationException.MISSING_RESULT, "insert_record failed: unknown result"); + + def update_record(self, cursor, value): + """ + Parameters: + - cursor + - value + """ + self.send_update_record(cursor, value) + self.recv_update_record() + + def send_update_record(self, cursor, value): + self._oprot.writeMessageBegin('update_record', TMessageType.CALL, self._seqid) + args = update_record_args() + args.cursor = cursor + args.value = value + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_update_record(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = update_record_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.err != None: + raise result.err + return + + def delete_record(self, cursor): + """ + Parameters: + - cursor + """ + self.send_delete_record(cursor) + self.recv_delete_record() + + def send_delete_record(self, cursor): + self._oprot.writeMessageBegin('delete_record', TMessageType.CALL, self._seqid) + args = delete_record_args() + args.cursor = cursor + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_delete_record(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = delete_record_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.err != None: + raise result.err + return + + def close_cursor(self, cursor, config): + """ + Parameters: + - cursor + - config + """ + self.send_close_cursor(cursor, config) + self.recv_close_cursor() + + def send_close_cursor(self, cursor, config): + self._oprot.writeMessageBegin('close_cursor', TMessageType.CALL, self._seqid) + args = close_cursor_args() + args.cursor = cursor + args.config = config + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_close_cursor(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = close_cursor_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.err != None: + raise result.err + return + + +class Processor(Iface, TProcessor): + def __init__(self, handler): + self._handler = handler + self._processMap = {} + self._processMap["strerror"] = Processor.process_strerror + self._processMap["version"] = Processor.process_version + self._processMap["open"] = Processor.process_open + self._processMap["open_session"] = Processor.process_open_session + self._processMap["close_connection"] = Processor.process_close_connection + self._processMap["is_new"] = Processor.process_is_new + self._processMap["close_session"] = Processor.process_close_session + self._processMap["open_cursor"] = Processor.process_open_cursor + self._processMap["create_table"] = Processor.process_create_table + self._processMap["rename_table"] = Processor.process_rename_table + self._processMap["drop_table"] = Processor.process_drop_table + self._processMap["truncate_table"] = Processor.process_truncate_table + self._processMap["verify_table"] = Processor.process_verify_table + self._processMap["begin_transaction"] = Processor.process_begin_transaction + self._processMap["commit_transaction"] = Processor.process_commit_transaction + self._processMap["rollback_transaction"] = Processor.process_rollback_transaction + self._processMap["checkpoint"] = Processor.process_checkpoint + self._processMap["move_first"] = Processor.process_move_first + self._processMap["move_last"] = Processor.process_move_last + self._processMap["move_next"] = Processor.process_move_next + self._processMap["move_prev"] = Processor.process_move_prev + self._processMap["search"] = Processor.process_search + self._processMap["search_near"] = Processor.process_search_near + self._processMap["insert_record"] = Processor.process_insert_record + self._processMap["update_record"] = Processor.process_update_record + self._processMap["delete_record"] = Processor.process_delete_record + self._processMap["close_cursor"] = Processor.process_close_cursor + + def process(self, iprot, oprot): + (name, type, seqid) = iprot.readMessageBegin() + if name not in self._processMap: + iprot.skip(TType.STRUCT) + iprot.readMessageEnd() + x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name)) + oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid) + x.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + return + else: + self._processMap[name](self, seqid, iprot, oprot) + return True + + def process_strerror(self, seqid, iprot, oprot): + args = strerror_args() + args.read(iprot) + iprot.readMessageEnd() + result = strerror_result() + result.success = self._handler.strerror(args.err) + oprot.writeMessageBegin("strerror", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_version(self, seqid, iprot, oprot): + args = version_args() + args.read(iprot) + iprot.readMessageEnd() + result = version_result() + result.success = self._handler.version() + oprot.writeMessageBegin("version", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_open(self, seqid, iprot, oprot): + args = open_args() + args.read(iprot) + iprot.readMessageEnd() + result = open_result() + try: + result.success = self._handler.open(args.home, args.config) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("open", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_open_session(self, seqid, iprot, oprot): + args = open_session_args() + args.read(iprot) + iprot.readMessageEnd() + result = open_session_result() + try: + result.success = self._handler.open_session(args.connection, args.config) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("open_session", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_close_connection(self, seqid, iprot, oprot): + args = close_connection_args() + args.read(iprot) + iprot.readMessageEnd() + result = close_connection_result() + try: + self._handler.close_connection(args.connection, args.config) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("close_connection", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_is_new(self, seqid, iprot, oprot): + args = is_new_args() + args.read(iprot) + iprot.readMessageEnd() + result = is_new_result() + result.success = self._handler.is_new(args.connection) + oprot.writeMessageBegin("is_new", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_close_session(self, seqid, iprot, oprot): + args = close_session_args() + args.read(iprot) + iprot.readMessageEnd() + result = close_session_result() + try: + self._handler.close_session(args.session, args.config) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("close_session", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_open_cursor(self, seqid, iprot, oprot): + args = open_cursor_args() + args.read(iprot) + iprot.readMessageEnd() + result = open_cursor_result() + try: + result.success = self._handler.open_cursor(args.session, args.uri, args.to_dup, args.config) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("open_cursor", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_create_table(self, seqid, iprot, oprot): + args = create_table_args() + args.read(iprot) + iprot.readMessageEnd() + result = create_table_result() + try: + self._handler.create_table(args.session, args.name, args.config) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("create_table", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_rename_table(self, seqid, iprot, oprot): + args = rename_table_args() + args.read(iprot) + iprot.readMessageEnd() + result = rename_table_result() + try: + self._handler.rename_table(args.session, args.oldname, args.newname, args.config) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("rename_table", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_drop_table(self, seqid, iprot, oprot): + args = drop_table_args() + args.read(iprot) + iprot.readMessageEnd() + result = drop_table_result() + try: + self._handler.drop_table(args.session, args.name, args.config) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("drop_table", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_truncate_table(self, seqid, iprot, oprot): + args = truncate_table_args() + args.read(iprot) + iprot.readMessageEnd() + result = truncate_table_result() + try: + self._handler.truncate_table(args.session, args.name, args.cursor_start, args.cursor_end, args.config) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("truncate_table", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_verify_table(self, seqid, iprot, oprot): + args = verify_table_args() + args.read(iprot) + iprot.readMessageEnd() + result = verify_table_result() + try: + self._handler.verify_table(args.session, args.name, args.config) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("verify_table", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_begin_transaction(self, seqid, iprot, oprot): + args = begin_transaction_args() + args.read(iprot) + iprot.readMessageEnd() + result = begin_transaction_result() + try: + self._handler.begin_transaction(args.session, args.config) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("begin_transaction", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_commit_transaction(self, seqid, iprot, oprot): + args = commit_transaction_args() + args.read(iprot) + iprot.readMessageEnd() + result = commit_transaction_result() + try: + self._handler.commit_transaction(args.session, args.config) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("commit_transaction", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_rollback_transaction(self, seqid, iprot, oprot): + args = rollback_transaction_args() + args.read(iprot) + iprot.readMessageEnd() + result = rollback_transaction_result() + try: + self._handler.rollback_transaction(args.session, args.config) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("rollback_transaction", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_checkpoint(self, seqid, iprot, oprot): + args = checkpoint_args() + args.read(iprot) + iprot.readMessageEnd() + result = checkpoint_result() + try: + self._handler.checkpoint(args.session, args.config) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("checkpoint", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_move_first(self, seqid, iprot, oprot): + args = move_first_args() + args.read(iprot) + iprot.readMessageEnd() + result = move_first_result() + try: + result.success = self._handler.move_first(args.cursor) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("move_first", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_move_last(self, seqid, iprot, oprot): + args = move_last_args() + args.read(iprot) + iprot.readMessageEnd() + result = move_last_result() + try: + result.success = self._handler.move_last(args.cursor) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("move_last", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_move_next(self, seqid, iprot, oprot): + args = move_next_args() + args.read(iprot) + iprot.readMessageEnd() + result = move_next_result() + try: + result.success = self._handler.move_next(args.cursor) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("move_next", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_move_prev(self, seqid, iprot, oprot): + args = move_prev_args() + args.read(iprot) + iprot.readMessageEnd() + result = move_prev_result() + try: + result.success = self._handler.move_prev(args.cursor) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("move_prev", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_search(self, seqid, iprot, oprot): + args = search_args() + args.read(iprot) + iprot.readMessageEnd() + result = search_result() + try: + result.success = self._handler.search(args.cursor, args.record) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("search", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_search_near(self, seqid, iprot, oprot): + args = search_near_args() + args.read(iprot) + iprot.readMessageEnd() + result = search_near_result() + try: + result.success = self._handler.search_near(args.cursor, args.record) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("search_near", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_insert_record(self, seqid, iprot, oprot): + args = insert_record_args() + args.read(iprot) + iprot.readMessageEnd() + result = insert_record_result() + try: + result.success = self._handler.insert_record(args.cursor, args.record) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("insert_record", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_update_record(self, seqid, iprot, oprot): + args = update_record_args() + args.read(iprot) + iprot.readMessageEnd() + result = update_record_result() + try: + self._handler.update_record(args.cursor, args.value) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("update_record", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_delete_record(self, seqid, iprot, oprot): + args = delete_record_args() + args.read(iprot) + iprot.readMessageEnd() + result = delete_record_result() + try: + self._handler.delete_record(args.cursor) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("delete_record", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + def process_close_cursor(self, seqid, iprot, oprot): + args = close_cursor_args() + args.read(iprot) + iprot.readMessageEnd() + result = close_cursor_result() + try: + self._handler.close_cursor(args.cursor, args.config) + except WT_ERROR, err: + result.err = err + oprot.writeMessageBegin("close_cursor", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + + +# HELPER FUNCTIONS AND STRUCTURES + +class strerror_args: + """ + Attributes: + - err + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'err', None, None, ), # 1 + ) + + def __init__(self, err=None,): + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.err = iprot.readI32(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('strerror_args') + if self.err != None: + oprot.writeFieldBegin('err', TType.I32, 1) + oprot.writeI32(self.err) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class strerror_result: + """ + Attributes: + - success + """ + + thrift_spec = ( + (0, TType.STRING, 'success', None, None, ), # 0 + ) + + def __init__(self, success=None,): + self.success = success + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRING: + self.success = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('strerror_result') + if self.success != None: + oprot.writeFieldBegin('success', TType.STRING, 0) + oprot.writeString(self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class version_args: + + thrift_spec = ( + ) + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('version_args') + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class version_result: + """ + Attributes: + - success + """ + + thrift_spec = ( + (0, TType.STRUCT, 'success', (WT_VERSION, WT_VERSION.thrift_spec), None, ), # 0 + ) + + def __init__(self, success=None,): + self.success = success + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WT_VERSION() + self.success.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('version_result') + if self.success != None: + oprot.writeFieldBegin('success', TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class open_args: + """ + Attributes: + - home + - config + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRING, 'home', None, None, ), # 1 + (2, TType.STRING, 'config', None, None, ), # 2 + ) + + def __init__(self, home=None, config=None,): + self.home = home + self.config = config + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.home = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.config = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('open_args') + if self.home != None: + oprot.writeFieldBegin('home', TType.STRING, 1) + oprot.writeString(self.home) + oprot.writeFieldEnd() + if self.config != None: + oprot.writeFieldBegin('config', TType.STRING, 2) + oprot.writeString(self.config) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class open_result: + """ + Attributes: + - success + - err + """ + + thrift_spec = ( + (0, TType.I32, 'success', None, None, ), # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, success=None, err=None,): + self.success = success + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.I32: + self.success = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('open_result') + if self.success != None: + oprot.writeFieldBegin('success', TType.I32, 0) + oprot.writeI32(self.success) + oprot.writeFieldEnd() + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class open_session_args: + """ + Attributes: + - connection + - config + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'connection', None, None, ), # 1 + (2, TType.STRING, 'config', None, None, ), # 2 + ) + + def __init__(self, connection=None, config=None,): + self.connection = connection + self.config = config + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.connection = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.config = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('open_session_args') + if self.connection != None: + oprot.writeFieldBegin('connection', TType.I32, 1) + oprot.writeI32(self.connection) + oprot.writeFieldEnd() + if self.config != None: + oprot.writeFieldBegin('config', TType.STRING, 2) + oprot.writeString(self.config) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class open_session_result: + """ + Attributes: + - success + - err + """ + + thrift_spec = ( + (0, TType.I32, 'success', None, None, ), # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, success=None, err=None,): + self.success = success + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.I32: + self.success = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('open_session_result') + if self.success != None: + oprot.writeFieldBegin('success', TType.I32, 0) + oprot.writeI32(self.success) + oprot.writeFieldEnd() + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class close_connection_args: + """ + Attributes: + - connection + - config + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'connection', None, None, ), # 1 + (2, TType.STRING, 'config', None, None, ), # 2 + ) + + def __init__(self, connection=None, config=None,): + self.connection = connection + self.config = config + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.connection = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.config = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('close_connection_args') + if self.connection != None: + oprot.writeFieldBegin('connection', TType.I32, 1) + oprot.writeI32(self.connection) + oprot.writeFieldEnd() + if self.config != None: + oprot.writeFieldBegin('config', TType.STRING, 2) + oprot.writeString(self.config) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class close_connection_result: + """ + Attributes: + - err + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, err=None,): + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('close_connection_result') + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class is_new_args: + """ + Attributes: + - connection + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'connection', None, None, ), # 1 + ) + + def __init__(self, connection=None,): + self.connection = connection + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.connection = iprot.readI32(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('is_new_args') + if self.connection != None: + oprot.writeFieldBegin('connection', TType.I32, 1) + oprot.writeI32(self.connection) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class is_new_result: + """ + Attributes: + - success + """ + + thrift_spec = ( + (0, TType.BOOL, 'success', None, None, ), # 0 + ) + + def __init__(self, success=None,): + self.success = success + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.BOOL: + self.success = iprot.readBool(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('is_new_result') + if self.success != None: + oprot.writeFieldBegin('success', TType.BOOL, 0) + oprot.writeBool(self.success) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class close_session_args: + """ + Attributes: + - session + - config + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'session', None, None, ), # 1 + (2, TType.STRING, 'config', None, None, ), # 2 + ) + + def __init__(self, session=None, config=None,): + self.session = session + self.config = config + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.session = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.config = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('close_session_args') + if self.session != None: + oprot.writeFieldBegin('session', TType.I32, 1) + oprot.writeI32(self.session) + oprot.writeFieldEnd() + if self.config != None: + oprot.writeFieldBegin('config', TType.STRING, 2) + oprot.writeString(self.config) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class close_session_result: + """ + Attributes: + - err + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, err=None,): + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('close_session_result') + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class open_cursor_args: + """ + Attributes: + - session + - uri + - to_dup + - config + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'session', None, None, ), # 1 + (2, TType.STRING, 'uri', None, None, ), # 2 + (3, TType.I32, 'to_dup', None, None, ), # 3 + (4, TType.STRING, 'config', None, None, ), # 4 + ) + + def __init__(self, session=None, uri=None, to_dup=None, config=None,): + self.session = session + self.uri = uri + self.to_dup = to_dup + self.config = config + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.session = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.uri = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I32: + self.to_dup = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.config = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('open_cursor_args') + if self.session != None: + oprot.writeFieldBegin('session', TType.I32, 1) + oprot.writeI32(self.session) + oprot.writeFieldEnd() + if self.uri != None: + oprot.writeFieldBegin('uri', TType.STRING, 2) + oprot.writeString(self.uri) + oprot.writeFieldEnd() + if self.to_dup != None: + oprot.writeFieldBegin('to_dup', TType.I32, 3) + oprot.writeI32(self.to_dup) + oprot.writeFieldEnd() + if self.config != None: + oprot.writeFieldBegin('config', TType.STRING, 4) + oprot.writeString(self.config) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class open_cursor_result: + """ + Attributes: + - success + - err + """ + + thrift_spec = ( + (0, TType.STRUCT, 'success', (WT_CURSOR_HANDLE, WT_CURSOR_HANDLE.thrift_spec), None, ), # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, success=None, err=None,): + self.success = success + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WT_CURSOR_HANDLE() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('open_cursor_result') + if self.success != None: + oprot.writeFieldBegin('success', TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class create_table_args: + """ + Attributes: + - session + - name + - config + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'session', None, None, ), # 1 + (2, TType.STRING, 'name', None, None, ), # 2 + (3, TType.STRING, 'config', None, None, ), # 3 + ) + + def __init__(self, session=None, name=None, config=None,): + self.session = session + self.name = name + self.config = config + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.session = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.name = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.config = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('create_table_args') + if self.session != None: + oprot.writeFieldBegin('session', TType.I32, 1) + oprot.writeI32(self.session) + oprot.writeFieldEnd() + if self.name != None: + oprot.writeFieldBegin('name', TType.STRING, 2) + oprot.writeString(self.name) + oprot.writeFieldEnd() + if self.config != None: + oprot.writeFieldBegin('config', TType.STRING, 3) + oprot.writeString(self.config) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class create_table_result: + """ + Attributes: + - err + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, err=None,): + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('create_table_result') + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class rename_table_args: + """ + Attributes: + - session + - oldname + - newname + - config + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'session', None, None, ), # 1 + (2, TType.STRING, 'oldname', None, None, ), # 2 + (3, TType.STRING, 'newname', None, None, ), # 3 + (4, TType.STRING, 'config', None, None, ), # 4 + ) + + def __init__(self, session=None, oldname=None, newname=None, config=None,): + self.session = session + self.oldname = oldname + self.newname = newname + self.config = config + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.session = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.oldname = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.newname = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.config = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('rename_table_args') + if self.session != None: + oprot.writeFieldBegin('session', TType.I32, 1) + oprot.writeI32(self.session) + oprot.writeFieldEnd() + if self.oldname != None: + oprot.writeFieldBegin('oldname', TType.STRING, 2) + oprot.writeString(self.oldname) + oprot.writeFieldEnd() + if self.newname != None: + oprot.writeFieldBegin('newname', TType.STRING, 3) + oprot.writeString(self.newname) + oprot.writeFieldEnd() + if self.config != None: + oprot.writeFieldBegin('config', TType.STRING, 4) + oprot.writeString(self.config) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class rename_table_result: + """ + Attributes: + - err + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, err=None,): + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('rename_table_result') + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class drop_table_args: + """ + Attributes: + - session + - name + - config + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'session', None, None, ), # 1 + (2, TType.STRING, 'name', None, None, ), # 2 + (3, TType.STRING, 'config', None, None, ), # 3 + ) + + def __init__(self, session=None, name=None, config=None,): + self.session = session + self.name = name + self.config = config + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.session = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.name = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.config = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('drop_table_args') + if self.session != None: + oprot.writeFieldBegin('session', TType.I32, 1) + oprot.writeI32(self.session) + oprot.writeFieldEnd() + if self.name != None: + oprot.writeFieldBegin('name', TType.STRING, 2) + oprot.writeString(self.name) + oprot.writeFieldEnd() + if self.config != None: + oprot.writeFieldBegin('config', TType.STRING, 3) + oprot.writeString(self.config) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class drop_table_result: + """ + Attributes: + - err + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, err=None,): + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('drop_table_result') + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class truncate_table_args: + """ + Attributes: + - session + - name + - cursor_start + - cursor_end + - config + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'session', None, None, ), # 1 + (2, TType.STRING, 'name', None, None, ), # 2 + (3, TType.I32, 'cursor_start', None, None, ), # 3 + (4, TType.I32, 'cursor_end', None, None, ), # 4 + (5, TType.STRING, 'config', None, None, ), # 5 + ) + + def __init__(self, session=None, name=None, cursor_start=None, cursor_end=None, config=None,): + self.session = session + self.name = name + self.cursor_start = cursor_start + self.cursor_end = cursor_end + self.config = config + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.session = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.name = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I32: + self.cursor_start = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.cursor_end = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRING: + self.config = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('truncate_table_args') + if self.session != None: + oprot.writeFieldBegin('session', TType.I32, 1) + oprot.writeI32(self.session) + oprot.writeFieldEnd() + if self.name != None: + oprot.writeFieldBegin('name', TType.STRING, 2) + oprot.writeString(self.name) + oprot.writeFieldEnd() + if self.cursor_start != None: + oprot.writeFieldBegin('cursor_start', TType.I32, 3) + oprot.writeI32(self.cursor_start) + oprot.writeFieldEnd() + if self.cursor_end != None: + oprot.writeFieldBegin('cursor_end', TType.I32, 4) + oprot.writeI32(self.cursor_end) + oprot.writeFieldEnd() + if self.config != None: + oprot.writeFieldBegin('config', TType.STRING, 5) + oprot.writeString(self.config) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class truncate_table_result: + """ + Attributes: + - err + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, err=None,): + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('truncate_table_result') + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class verify_table_args: + """ + Attributes: + - session + - name + - config + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'session', None, None, ), # 1 + (2, TType.STRING, 'name', None, None, ), # 2 + (3, TType.STRING, 'config', None, None, ), # 3 + ) + + def __init__(self, session=None, name=None, config=None,): + self.session = session + self.name = name + self.config = config + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.session = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.name = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.config = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('verify_table_args') + if self.session != None: + oprot.writeFieldBegin('session', TType.I32, 1) + oprot.writeI32(self.session) + oprot.writeFieldEnd() + if self.name != None: + oprot.writeFieldBegin('name', TType.STRING, 2) + oprot.writeString(self.name) + oprot.writeFieldEnd() + if self.config != None: + oprot.writeFieldBegin('config', TType.STRING, 3) + oprot.writeString(self.config) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class verify_table_result: + """ + Attributes: + - err + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, err=None,): + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('verify_table_result') + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class begin_transaction_args: + """ + Attributes: + - session + - config + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'session', None, None, ), # 1 + (2, TType.STRING, 'config', None, None, ), # 2 + ) + + def __init__(self, session=None, config=None,): + self.session = session + self.config = config + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.session = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.config = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('begin_transaction_args') + if self.session != None: + oprot.writeFieldBegin('session', TType.I32, 1) + oprot.writeI32(self.session) + oprot.writeFieldEnd() + if self.config != None: + oprot.writeFieldBegin('config', TType.STRING, 2) + oprot.writeString(self.config) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class begin_transaction_result: + """ + Attributes: + - err + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, err=None,): + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('begin_transaction_result') + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class commit_transaction_args: + """ + Attributes: + - session + - config + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'session', None, None, ), # 1 + (2, TType.STRING, 'config', None, None, ), # 2 + ) + + def __init__(self, session=None, config=None,): + self.session = session + self.config = config + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.session = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.config = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('commit_transaction_args') + if self.session != None: + oprot.writeFieldBegin('session', TType.I32, 1) + oprot.writeI32(self.session) + oprot.writeFieldEnd() + if self.config != None: + oprot.writeFieldBegin('config', TType.STRING, 2) + oprot.writeString(self.config) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class commit_transaction_result: + """ + Attributes: + - err + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, err=None,): + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('commit_transaction_result') + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class rollback_transaction_args: + """ + Attributes: + - session + - config + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'session', None, None, ), # 1 + (2, TType.STRING, 'config', None, None, ), # 2 + ) + + def __init__(self, session=None, config=None,): + self.session = session + self.config = config + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.session = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.config = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('rollback_transaction_args') + if self.session != None: + oprot.writeFieldBegin('session', TType.I32, 1) + oprot.writeI32(self.session) + oprot.writeFieldEnd() + if self.config != None: + oprot.writeFieldBegin('config', TType.STRING, 2) + oprot.writeString(self.config) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class rollback_transaction_result: + """ + Attributes: + - err + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, err=None,): + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('rollback_transaction_result') + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class checkpoint_args: + """ + Attributes: + - session + - config + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'session', None, None, ), # 1 + (2, TType.STRING, 'config', None, None, ), # 2 + ) + + def __init__(self, session=None, config=None,): + self.session = session + self.config = config + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.session = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.config = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('checkpoint_args') + if self.session != None: + oprot.writeFieldBegin('session', TType.I32, 1) + oprot.writeI32(self.session) + oprot.writeFieldEnd() + if self.config != None: + oprot.writeFieldBegin('config', TType.STRING, 2) + oprot.writeString(self.config) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class checkpoint_result: + """ + Attributes: + - err + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, err=None,): + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('checkpoint_result') + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class move_first_args: + """ + Attributes: + - cursor + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'cursor', None, None, ), # 1 + ) + + def __init__(self, cursor=None,): + self.cursor = cursor + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.cursor = iprot.readI32(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('move_first_args') + if self.cursor != None: + oprot.writeFieldBegin('cursor', TType.I32, 1) + oprot.writeI32(self.cursor) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class move_first_result: + """ + Attributes: + - success + - err + """ + + thrift_spec = ( + (0, TType.STRUCT, 'success', (WT_MOVE_RESULT, WT_MOVE_RESULT.thrift_spec), None, ), # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, success=None, err=None,): + self.success = success + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WT_MOVE_RESULT() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('move_first_result') + if self.success != None: + oprot.writeFieldBegin('success', TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class move_last_args: + """ + Attributes: + - cursor + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'cursor', None, None, ), # 1 + ) + + def __init__(self, cursor=None,): + self.cursor = cursor + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.cursor = iprot.readI32(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('move_last_args') + if self.cursor != None: + oprot.writeFieldBegin('cursor', TType.I32, 1) + oprot.writeI32(self.cursor) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class move_last_result: + """ + Attributes: + - success + - err + """ + + thrift_spec = ( + (0, TType.STRUCT, 'success', (WT_MOVE_RESULT, WT_MOVE_RESULT.thrift_spec), None, ), # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, success=None, err=None,): + self.success = success + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WT_MOVE_RESULT() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('move_last_result') + if self.success != None: + oprot.writeFieldBegin('success', TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class move_next_args: + """ + Attributes: + - cursor + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'cursor', None, None, ), # 1 + ) + + def __init__(self, cursor=None,): + self.cursor = cursor + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.cursor = iprot.readI32(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('move_next_args') + if self.cursor != None: + oprot.writeFieldBegin('cursor', TType.I32, 1) + oprot.writeI32(self.cursor) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class move_next_result: + """ + Attributes: + - success + - err + """ + + thrift_spec = ( + (0, TType.STRUCT, 'success', (WT_MOVE_RESULT, WT_MOVE_RESULT.thrift_spec), None, ), # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, success=None, err=None,): + self.success = success + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WT_MOVE_RESULT() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('move_next_result') + if self.success != None: + oprot.writeFieldBegin('success', TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class move_prev_args: + """ + Attributes: + - cursor + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'cursor', None, None, ), # 1 + ) + + def __init__(self, cursor=None,): + self.cursor = cursor + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.cursor = iprot.readI32(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('move_prev_args') + if self.cursor != None: + oprot.writeFieldBegin('cursor', TType.I32, 1) + oprot.writeI32(self.cursor) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class move_prev_result: + """ + Attributes: + - success + - err + """ + + thrift_spec = ( + (0, TType.STRUCT, 'success', (WT_MOVE_RESULT, WT_MOVE_RESULT.thrift_spec), None, ), # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, success=None, err=None,): + self.success = success + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WT_MOVE_RESULT() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('move_prev_result') + if self.success != None: + oprot.writeFieldBegin('success', TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class search_args: + """ + Attributes: + - cursor + - record + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'cursor', None, None, ), # 1 + (2, TType.STRUCT, 'record', (WT_RECORD, WT_RECORD.thrift_spec), None, ), # 2 + ) + + def __init__(self, cursor=None, record=None,): + self.cursor = cursor + self.record = record + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.cursor = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.record = WT_RECORD() + self.record.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('search_args') + if self.cursor != None: + oprot.writeFieldBegin('cursor', TType.I32, 1) + oprot.writeI32(self.cursor) + oprot.writeFieldEnd() + if self.record != None: + oprot.writeFieldBegin('record', TType.STRUCT, 2) + self.record.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class search_result: + """ + Attributes: + - success + - err + """ + + thrift_spec = ( + (0, TType.STRUCT, 'success', (WT_MOVE_RESULT, WT_MOVE_RESULT.thrift_spec), None, ), # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, success=None, err=None,): + self.success = success + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WT_MOVE_RESULT() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('search_result') + if self.success != None: + oprot.writeFieldBegin('success', TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class search_near_args: + """ + Attributes: + - cursor + - record + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'cursor', None, None, ), # 1 + (2, TType.STRUCT, 'record', (WT_RECORD, WT_RECORD.thrift_spec), None, ), # 2 + ) + + def __init__(self, cursor=None, record=None,): + self.cursor = cursor + self.record = record + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.cursor = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.record = WT_RECORD() + self.record.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('search_near_args') + if self.cursor != None: + oprot.writeFieldBegin('cursor', TType.I32, 1) + oprot.writeI32(self.cursor) + oprot.writeFieldEnd() + if self.record != None: + oprot.writeFieldBegin('record', TType.STRUCT, 2) + self.record.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class search_near_result: + """ + Attributes: + - success + - err + """ + + thrift_spec = ( + (0, TType.STRUCT, 'success', (WT_MOVE_RESULT, WT_MOVE_RESULT.thrift_spec), None, ), # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, success=None, err=None,): + self.success = success + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRUCT: + self.success = WT_MOVE_RESULT() + self.success.read(iprot) + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('search_near_result') + if self.success != None: + oprot.writeFieldBegin('success', TType.STRUCT, 0) + self.success.write(oprot) + oprot.writeFieldEnd() + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class insert_record_args: + """ + Attributes: + - cursor + - record + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'cursor', None, None, ), # 1 + (2, TType.STRUCT, 'record', (WT_RECORD, WT_RECORD.thrift_spec), None, ), # 2 + ) + + def __init__(self, cursor=None, record=None,): + self.cursor = cursor + self.record = record + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.cursor = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.record = WT_RECORD() + self.record.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('insert_record_args') + if self.cursor != None: + oprot.writeFieldBegin('cursor', TType.I32, 1) + oprot.writeI32(self.cursor) + oprot.writeFieldEnd() + if self.record != None: + oprot.writeFieldBegin('record', TType.STRUCT, 2) + self.record.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class insert_record_result: + """ + Attributes: + - success + - err + """ + + thrift_spec = ( + (0, TType.STRING, 'success', None, None, ), # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, success=None, err=None,): + self.success = success + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.STRING: + self.success = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('insert_record_result') + if self.success != None: + oprot.writeFieldBegin('success', TType.STRING, 0) + oprot.writeString(self.success) + oprot.writeFieldEnd() + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class update_record_args: + """ + Attributes: + - cursor + - value + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'cursor', None, None, ), # 1 + (2, TType.STRING, 'value', None, None, ), # 2 + ) + + def __init__(self, cursor=None, value=None,): + self.cursor = cursor + self.value = value + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.cursor = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.value = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('update_record_args') + if self.cursor != None: + oprot.writeFieldBegin('cursor', TType.I32, 1) + oprot.writeI32(self.cursor) + oprot.writeFieldEnd() + if self.value != None: + oprot.writeFieldBegin('value', TType.STRING, 2) + oprot.writeString(self.value) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class update_record_result: + """ + Attributes: + - err + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, err=None,): + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('update_record_result') + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class delete_record_args: + """ + Attributes: + - cursor + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'cursor', None, None, ), # 1 + ) + + def __init__(self, cursor=None,): + self.cursor = cursor + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.cursor = iprot.readI32(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('delete_record_args') + if self.cursor != None: + oprot.writeFieldBegin('cursor', TType.I32, 1) + oprot.writeI32(self.cursor) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class delete_record_result: + """ + Attributes: + - err + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, err=None,): + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('delete_record_result') + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class close_cursor_args: + """ + Attributes: + - cursor + - config + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'cursor', None, None, ), # 1 + (2, TType.STRING, 'config', None, None, ), # 2 + ) + + def __init__(self, cursor=None, config=None,): + self.cursor = cursor + self.config = config + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.cursor = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.config = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('close_cursor_args') + if self.cursor != None: + oprot.writeFieldBegin('cursor', TType.I32, 1) + oprot.writeI32(self.cursor) + oprot.writeFieldEnd() + if self.config != None: + oprot.writeFieldBegin('config', TType.STRING, 2) + oprot.writeString(self.config) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class close_cursor_result: + """ + Attributes: + - err + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'err', (WT_ERROR, WT_ERROR.thrift_spec), None, ), # 1 + ) + + def __init__(self, err=None,): + self.err = err + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.err = WT_ERROR() + self.err.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('close_cursor_result') + if self.err != None: + oprot.writeFieldBegin('err', TType.STRUCT, 1) + self.err.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) diff --git a/lang/python/src/wiredtiger/service/__init__.py b/lang/python/src/wiredtiger/service/__init__.py new file mode 100644 index 00000000000..35e1b55e02b --- /dev/null +++ b/lang/python/src/wiredtiger/service/__init__.py @@ -0,0 +1 @@ +__all__ = ['ttypes', 'constants', 'WiredTiger'] diff --git a/lang/python/src/wiredtiger/service/constants.py b/lang/python/src/wiredtiger/service/constants.py new file mode 100644 index 00000000000..2f17ec34fee --- /dev/null +++ b/lang/python/src/wiredtiger/service/constants.py @@ -0,0 +1,9 @@ +# +# Autogenerated by Thrift +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# + +from thrift.Thrift import * +from ttypes import * + diff --git a/lang/python/src/wiredtiger/service/ttypes.py b/lang/python/src/wiredtiger/service/ttypes.py new file mode 100644 index 00000000000..6e7c7f6c2c3 --- /dev/null +++ b/lang/python/src/wiredtiger/service/ttypes.py @@ -0,0 +1,411 @@ +# +# Autogenerated by Thrift +# +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING +# + +from thrift.Thrift import * + +from thrift.transport import TTransport +from thrift.protocol import TBinaryProtocol, TProtocol +try: + from thrift.protocol import fastbinary +except: + fastbinary = None + + + +class WT_RECORD: + """ + Attributes: + - key + - value + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRING, 'key', None, None, ), # 1 + (2, TType.STRING, 'value', None, None, ), # 2 + ) + + def __init__(self, key=None, value=None,): + self.key = key + self.value = value + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.key = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.value = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('WT_RECORD') + if self.key != None: + oprot.writeFieldBegin('key', TType.STRING, 1) + oprot.writeString(self.key) + oprot.writeFieldEnd() + if self.value != None: + oprot.writeFieldBegin('value', TType.STRING, 2) + oprot.writeString(self.value) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class WT_MOVE_RESULT: + """ + Attributes: + - exact + - record + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'exact', None, None, ), # 1 + (2, TType.STRUCT, 'record', (WT_RECORD, WT_RECORD.thrift_spec), None, ), # 2 + ) + + def __init__(self, exact=None, record=None,): + self.exact = exact + self.record = record + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.exact = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.record = WT_RECORD() + self.record.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('WT_MOVE_RESULT') + if self.exact != None: + oprot.writeFieldBegin('exact', TType.I32, 1) + oprot.writeI32(self.exact) + oprot.writeFieldEnd() + if self.record != None: + oprot.writeFieldBegin('record', TType.STRUCT, 2) + self.record.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class WT_CURSOR_HANDLE: + """ + Attributes: + - id + - key_format + - value_format + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'id', None, None, ), # 1 + (2, TType.STRING, 'key_format', None, None, ), # 2 + (3, TType.STRING, 'value_format', None, None, ), # 3 + ) + + def __init__(self, id=None, key_format=None, value_format=None,): + self.id = id + self.key_format = key_format + self.value_format = value_format + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.id = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.key_format = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.value_format = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('WT_CURSOR_HANDLE') + if self.id != None: + oprot.writeFieldBegin('id', TType.I32, 1) + oprot.writeI32(self.id) + oprot.writeFieldEnd() + if self.key_format != None: + oprot.writeFieldBegin('key_format', TType.STRING, 2) + oprot.writeString(self.key_format) + oprot.writeFieldEnd() + if self.value_format != None: + oprot.writeFieldBegin('value_format', TType.STRING, 3) + oprot.writeString(self.value_format) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class WT_VERSION: + """ + Attributes: + - version_string + - major + - minor + - patch + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRING, 'version_string', None, None, ), # 1 + (2, TType.I32, 'major', None, None, ), # 2 + (3, TType.I32, 'minor', None, None, ), # 3 + (4, TType.I32, 'patch', None, None, ), # 4 + ) + + def __init__(self, version_string=None, major=None, minor=None, patch=None,): + self.version_string = version_string + self.major = major + self.minor = minor + self.patch = patch + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.version_string = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.major = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I32: + self.minor = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.patch = iprot.readI32(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('WT_VERSION') + if self.version_string != None: + oprot.writeFieldBegin('version_string', TType.STRING, 1) + oprot.writeString(self.version_string) + oprot.writeFieldEnd() + if self.major != None: + oprot.writeFieldBegin('major', TType.I32, 2) + oprot.writeI32(self.major) + oprot.writeFieldEnd() + if self.minor != None: + oprot.writeFieldBegin('minor', TType.I32, 3) + oprot.writeI32(self.minor) + oprot.writeFieldEnd() + if self.patch != None: + oprot.writeFieldBegin('patch', TType.I32, 4) + oprot.writeI32(self.patch) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class WT_ERROR(Exception): + """ + Attributes: + - err + - message + """ + + thrift_spec = ( + None, # 0 + (1, TType.I32, 'err', None, None, ), # 1 + (2, TType.STRING, 'message', None, None, ), # 2 + ) + + def __init__(self, err=None, message=None,): + self.err = err + self.message = message + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.err = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.message = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('WT_ERROR') + if self.err != None: + oprot.writeFieldBegin('err', TType.I32, 1) + oprot.writeI32(self.err) + oprot.writeFieldEnd() + if self.message != None: + oprot.writeFieldBegin('message', TType.STRING, 2) + oprot.writeString(self.message) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __str__(self): + return repr(self) + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) diff --git a/lang/python/src/wiredtiger/util.py b/lang/python/src/wiredtiger/util.py new file mode 100644 index 00000000000..b8e8f1d65a4 --- /dev/null +++ b/lang/python/src/wiredtiger/util.py @@ -0,0 +1,13 @@ +# +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# All rights reserved. +# +# See the file LICENSE for redistribution information. + +import re + +def parse_config(config): + for l in re.findall(r'(?:\(.*?\)|[^,])+', config): + for k, v in (l+"=yes").split('=')[0:2]: + yield (k.strip(), v.strip()) diff --git a/lang/python/wiredtiger.i b/lang/python/wiredtiger.i new file mode 100644 index 00000000000..01e0785720b --- /dev/null +++ b/lang/python/wiredtiger.i @@ -0,0 +1,362 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + * + * wiredtiger.i + * The SWIG interface file defining the wiredtiger python API. + */ + +%define DOCSTRING +"@defgroup wt_python WiredTiger Python API +Python wrappers aroung the WiredTiger C API. +@{ +@cond IGNORE" +%enddef + +%module(docstring=DOCSTRING) wiredtiger + +%feature("autodoc", "0"); + +%pythoncode %{ +from packing import pack, unpack +## @endcond +%} + +/* Set the input argument to point to a temporary variable */ +%typemap(in, numinputs=0) WT_CONNECTION ** (WT_CONNECTION *temp = NULL) { + $1 = &temp; +} +%typemap(in, numinputs=0) WT_SESSION ** (WT_SESSION *temp = NULL) { + $1 = &temp; +} +%typemap(in, numinputs=0) WT_CURSOR ** (WT_CURSOR *temp = NULL) { + $1 = &temp; +} + +/* Convert 'int *' to an output arg in search_near, wiredtiger_version */ +%apply int *OUTPUT { int * }; + +/* Event handlers are not supported in Python. */ +%typemap(in, numinputs=0) WT_EVENT_HANDLER * { $1 = NULL; } + +/* Set the return value to the returned connection, session, or cursor */ +%typemap(argout) WT_CONNECTION ** { + $result = SWIG_NewPointerObj(SWIG_as_voidptr(*$1), + SWIGTYPE_p___wt_connection, 0); +} +%typemap(argout) WT_SESSION ** { + $result = SWIG_NewPointerObj(SWIG_as_voidptr(*$1), + SWIGTYPE_p___wt_session, 0); +} + +%typemap(argout) WT_CURSOR ** { + $result = SWIG_NewPointerObj(SWIG_as_voidptr(*$1), + SWIGTYPE_p___wt_cursor, 0); + if (*$1 != NULL) { + (*$1)->flags |= WT_CURSTD_RAW; + PyObject_SetAttrString($result, "is_column", + PyBool_FromLong(strcmp((*$1)->key_format, "r") == 0)); + } +} + +/* 64 bit typemaps. */ +%typemap(in) uint64_t { + $1 = PyLong_AsUnsignedLongLong($input); +} +%typemap(out) uint64_t { + $result = PyLong_FromUnsignedLongLong($1); +} + +/* Throw away references after close. */ +%define DESTRUCTOR(class, method) +%feature("shadow") class::method %{ + def method(self, *args): + '''close(self, config) -> int + + @copydoc class::method''' + try: + return $action(self, *args) + finally: + self.this = None +%} +%enddef +DESTRUCTOR(__wt_connection, close) +DESTRUCTOR(__wt_cursor, close) +DESTRUCTOR(__wt_session, close) + +/* Don't require empty config strings. */ +%typemap(default) const char *config { $1 = NULL; } + +/* + * Error returns other than WT_NOTFOUND generate an exception. + * Use our own exception type, in future tailored to the kind + * of error. + */ +%header %{ +static PyObject *wtError; +%} + +%init %{ + /* + * Create an exception type and put it into the _wiredtiger module. + * First increment the reference count because PyModule_AddObject + * decrements it. Then note that "m" is the local variable for the + * module in the SWIG generated code. If there is a SWIG variable for + * this, I haven't found it. + */ + wtError = + PyErr_NewException("_wiredtiger.WiredTigerError", NULL, NULL); + Py_INCREF(wtError); + PyModule_AddObject(m, "WiredTigerError", wtError); +%} + +%pythoncode %{ +WiredTigerError = _wiredtiger.WiredTigerError + +## @cond DISABLE +# Implements the iterable contract +class IterableCursor: + def __init__(self, cursor): + self.cursor = cursor + + def __iter__(self): + return self + + def next(self): + if self.cursor.next() == WT_NOTFOUND: + raise StopIteration + return self.cursor.get_keys() + self.cursor.get_values() +## @endcond +%} + +%typemap(out) int { + if ($1 != 0 && $1 != WT_NOTFOUND) { + /* We could use PyErr_SetObject for more complex reporting. */ + SWIG_Python_SetErrorMsg(wtError, wiredtiger_strerror($1)); + SWIG_fail; + } + $result = SWIG_From_int((int)($1)); +} + +/* + * Extra 'self' elimination. + * The methods we're wrapping look like this: + * struct __wt_xxx { + * int method(WT_XXX *, ...otherargs...); + * }; + * To SWIG, that is equivalent to: + * int method(struct __wt_xxx *self, WT_XXX *, ...otherargs...); + * and we use consecutive argument matching of typemaps to convert two args to + * one. + */ +%define SELFHELPER(type) +%typemap(in) (type *self, type *) (void *argp = 0, int res = 0) %{ + res = SWIG_ConvertPtr($input, &argp, $descriptor, $disown | 0); + if (!SWIG_IsOK(res)) { + SWIG_exception_fail(SWIG_ArgError(res), + "in method '" "$symname" "', argument " "$argnum" + " of type '" "$type" "'"); + } + $2 = $1 = ($ltype)(argp); +%} +%enddef + +SELFHELPER(struct __wt_connection) +SELFHELPER(struct __wt_session) +SELFHELPER(struct __wt_cursor) + +/* WT_CURSOR customization. */ +/* First, replace the varargs get / set methods with Python equivalents. */ +%ignore __wt_cursor::get_key; +%ignore __wt_cursor::set_key; +%ignore __wt_cursor::get_value; +%ignore __wt_cursor::set_value; + +/* SWIG magic to turn Python byte strings into data / size. */ +%apply (char *STRING, int LENGTH) { (char *data, int size) }; + +%extend __wt_cursor { + /* Get / set keys and values */ + void _set_key(char *data, int size) { + WT_ITEM k; + k.data = data; + k.size = (uint32_t)size; + $self->set_key($self, &k); + } + + void _set_recno(uint64_t recno) { + WT_ITEM k; + uint8_t recno_buf[20]; + if (wiredtiger_struct_pack(recno_buf, sizeof (recno_buf), + "r", recno) == 0) { + k.data = recno_buf; + k.size = (uint32_t)wiredtiger_struct_size("q", recno); + $self->set_key($self, &k); + } + } + + void _set_value(char *data, int size) { + WT_ITEM v; + v.data = data; + v.size = (uint32_t)size; + $self->set_value($self, &v); + } + + PyObject *_get_key() { + WT_ITEM k; + int ret = $self->get_key($self, &k); + if (ret != 0) { + SWIG_Python_SetErrorMsg(wtError, + wiredtiger_strerror(ret)); + return (NULL); + } + return SWIG_FromCharPtrAndSize(k.data, k.size); + } + + PyObject *_get_recno() { + WT_ITEM k; + uint64_t r; + int ret = $self->get_key($self, &k); + if (ret == 0) + ret = wiredtiger_struct_unpack(k.data, k.size, "q", &r); + if (ret != 0) { + SWIG_Python_SetErrorMsg(wtError, + wiredtiger_strerror(ret)); + return (NULL); + } + return PyLong_FromUnsignedLongLong(r); + } + + PyObject *_get_value() { + WT_ITEM v; + int ret = $self->get_value($self, &v); + if (ret != 0) { + SWIG_Python_SetErrorMsg(wtError, + wiredtiger_strerror(ret)); + return (NULL); + } + return SWIG_FromCharPtrAndSize(v.data, v.size); + } + +%pythoncode %{ + def get_key(self): + '''get_key(self) -> object + + @copydoc WT_CURSOR::get_key + Returns only the first column.''' + return self.get_keys()[0] + + def get_keys(self): + '''get_keys(self) -> (object, ...) + + @copydoc WT_CURSOR::get_key''' + if self.is_column: + return [self._get_recno(),] + else: + return unpack(self.key_format, self._get_key()) + + def get_value(self): + '''get_value(self) -> object + + @copydoc WT_CURSOR::get_value + Returns only the first column.''' + return self.get_values()[0] + + def get_values(self): + '''get_values(self) -> (object, ...) + + @copydoc WT_CURSOR::get_value''' + return unpack(self.value_format, self._get_value()) + + def set_key(self, *args): + '''set_key(self) -> None + + @copydoc WT_CURSOR::set_key''' + if self.is_column: + self._set_recno(args[0]) + else: + # Keep the Python string pinned + self._key = pack(self.key_format, *args) + self._set_key(self._key) + + def set_value(self, *args): + '''set_value(self) -> None + + @copydoc WT_CURSOR::set_value''' + # Keep the Python string pinned + self._value = pack(self.value_format, *args) + self._set_value(self._value) + + def __iter__(self): + '''Cursor objects support iteration, equivalent to calling + WT_CURSOR::next until it returns ::WT_NOTFOUND.''' + if not hasattr(self, '_iterable'): + self._iterable = IterableCursor(self) + return self._iterable +%} +}; + +/* Remove / rename parts of the C API that we don't want in Python. */ +%immutable __wt_cursor::session; +%immutable __wt_cursor::uri; +%immutable __wt_cursor::key_format; +%immutable __wt_cursor::value_format; +%immutable __wt_session::connection; + +%ignore __wt_buf; +%ignore __wt_collator; +%ignore __wt_connection::add_collator; +%ignore __wt_compressor; +%ignore __wt_connection::add_compressor; +%ignore __wt_cursor_type; +%ignore __wt_connection::add_cursor_type; +%ignore __wt_event_handler; +%ignore __wt_extractor; +%ignore __wt_connection::add_extractor; +%ignore __wt_item; + +%ignore wiredtiger_struct_pack; +%ignore wiredtiger_struct_packv; +%ignore wiredtiger_struct_size; +%ignore wiredtiger_struct_sizev; +%ignore wiredtiger_struct_unpack; +%ignore wiredtiger_struct_unpackv; + +%ignore wiredtiger_extension_init; + +%rename(Cursor) __wt_cursor; +%rename(Session) __wt_session; +%rename(Connection) __wt_connection; + +%include "wiredtiger.h" + +%pythoncode %{ +## @} + +class stat: + """ a set of static defines used by statistics cursor """ + pass + +class filestat: + """ a set of static defines used by statistics cursor """ + pass + +import sys +# All names starting with 'WT_STAT_file_' are renamed to +# the wiredtiger.filestat class, those starting with 'WT_STAT_' are +# renamed to wiredtiger.stat . +def _rename_with_prefix(prefix, toclass): + curmodule = sys.modules[__name__] + for name in dir(curmodule): + if name.startswith(prefix): + shortname = name[len(prefix):] + setattr(toclass, shortname, getattr(curmodule, name)) + delattr(curmodule, name) + +_rename_with_prefix('WT_STAT_file_', filestat) +_rename_with_prefix('WT_STAT_', stat) +del _rename_with_prefix +%} + diff --git a/src/api/api_event.c b/src/api/api_event.c new file mode 100644 index 00000000000..5298767a0c9 --- /dev/null +++ b/src/api/api_event.c @@ -0,0 +1,71 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __handle_error_default -- + * Default WT_EVENT_HANDLER->handle_error implementation: send to stderr. + */ +static void +__handle_error_default(WT_EVENT_HANDLER *handler, int error, const char *errmsg) +{ + size_t len_err, len_errmsg; + const char *err; + + WT_UNUSED(handler); + + if (error != 0) { + err = wiredtiger_strerror(error); + len_err = strlen(err); + len_errmsg = strlen(errmsg); + if (len_err >= len_errmsg && + strcmp(errmsg + (len_errmsg - len_err), err) != 0) { + fprintf(stderr, + "%s: %s\n", errmsg, wiredtiger_strerror(error)); + return; + } + } + fprintf(stderr, "%s\n", errmsg); +} + +/* + * __handle_message_default -- + * Default WT_EVENT_HANDLER->handle_message implementation: send to stdout. + */ +static int +__handle_message_default(WT_EVENT_HANDLER *handler, const char *message) +{ + WT_UNUSED(handler); + + printf("%s\n", message); + + return (0); +} + +/* + * __handle_progress_default -- + * Default WT_EVENT_HANDLER->handle_progress implementation: ignore. + */ +static int +__handle_progress_default(WT_EVENT_HANDLER *handler, + const char *operation, uint64_t progress) +{ + WT_UNUSED(handler); + WT_UNUSED(operation); + WT_UNUSED(progress); + + return (0); +} + +static WT_EVENT_HANDLER __event_handler_default = { + __handle_error_default, + __handle_message_default, + __handle_progress_default +}; + +WT_EVENT_HANDLER *__wt_event_handler_default = &__event_handler_default; diff --git a/src/api/api_strerror.c b/src/api/api_strerror.c new file mode 100644 index 00000000000..0b9c927fc04 --- /dev/null +++ b/src/api/api_strerror.c @@ -0,0 +1,41 @@ +/* DO NOT EDIT: automatically built by dist/api_err.py. */ + +#include "wt_internal.h" + +/* + * wiredtiger_strerror -- + * Return a string for any error value. + */ +const char * +wiredtiger_strerror(int error) +{ + static char errbuf[64]; + char *p; + + if (error == 0) + return ("Successful return: 0"); + + switch (error) { + case WT_DEADLOCK: + return ("WT_DEADLOCK: conflict between concurrent operations"); + case WT_DUPLICATE_KEY: + return ("WT_DUPLICATE_KEY: attempt to insert an existing key"); + case WT_ERROR: + return ("WT_ERROR: non-specific WiredTiger error"); + case WT_NOTFOUND: + return ("WT_NOTFOUND: item not found"); + case WT_RESTART: + return ("WT_RESTART: restart the operation (internal)"); + default: + if (error > 0 && (p = strerror(error)) != NULL) + return (p); + break; + } + + /* + * !!! + * Not thread-safe, but this is never supposed to happen. + */ + (void)snprintf(errbuf, sizeof(errbuf), "Unknown error: %d", error); + return (errbuf); +} diff --git a/src/api/api_version.c b/src/api/api_version.c new file mode 100644 index 00000000000..e4bc3610d7a --- /dev/null +++ b/src/api/api_version.c @@ -0,0 +1,24 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * wiredtiger_version -- + * Return library version information. + */ +const char * +wiredtiger_version(int *majorp, int *minorp, int *patchp) +{ + if (majorp != NULL) + *majorp = WIREDTIGER_VERSION_MAJOR; + if (minorp != NULL) + *minorp = WIREDTIGER_VERSION_MINOR; + if (patchp != NULL) + *patchp = WIREDTIGER_VERSION_PATCH; + return ((char *)WIREDTIGER_VERSION_STRING); +} diff --git a/src/block/block_addr.c b/src/block/block_addr.c new file mode 100644 index 00000000000..f292c8afca0 --- /dev/null +++ b/src/block/block_addr.c @@ -0,0 +1,97 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_block_buffer_to_addr -- + * Convert a filesystem address cookie into its components. + */ +int +__wt_block_buffer_to_addr(WT_BLOCK *block, + const uint8_t *p, off_t *offsetp, uint32_t *sizep, uint32_t *cksump) +{ + uint64_t a; + + WT_RET(__wt_vunpack_uint(&p, 0, &a)); + if (offsetp != NULL) + *offsetp = (off_t)a * block->allocsize + WT_BLOCK_DESC_SECTOR; + WT_RET(__wt_vunpack_uint(&p, 0, &a)); + if (sizep != NULL) + *sizep = (uint32_t)a * block->allocsize; + + if (cksump != NULL) { + WT_RET(__wt_vunpack_uint(&p, 0, &a)); + *cksump = (uint32_t)a; + } + + return (0); +} + +/* + * __wt_block_addr_to_buffer -- + * Convert the filesystem components into its address cookie. + */ +int +__wt_block_addr_to_buffer(WT_BLOCK *block, + uint8_t **p, off_t offset, uint32_t size, uint32_t cksum) +{ + uint64_t a; + + a = (uint64_t)(offset - WT_BLOCK_DESC_SECTOR) / block->allocsize; + WT_RET(__wt_vpack_uint(p, 0, a)); + a = size / block->allocsize; + WT_RET(__wt_vpack_uint(p, 0, a)); + a = cksum; + WT_RET(__wt_vpack_uint(p, 0, a)); + return (0); +} + +/* + * __wt_block_addr_valid -- + * Return if an address cookie is valid. + */ +int +__wt_block_addr_valid(WT_SESSION_IMPL *session, + WT_BLOCK *block, const uint8_t *addr, uint32_t addr_size) +{ + off_t offset; + uint32_t size; + + WT_UNUSED(session); + WT_UNUSED(addr_size); + + /* Crack the cookie. */ + WT_RET(__wt_block_buffer_to_addr(block, addr, &offset, &size, NULL)); + + /* All we care about is if it's past the end of the file. */ + return (offset + size > block->fh->file_size ? 0 : 1); +} + +/* + * __wt_block_addr_string -- + * Return a printable string representation of an address cookie. + */ +int +__wt_block_addr_string(WT_SESSION_IMPL *session, + WT_BLOCK *block, WT_ITEM *buf, const uint8_t *addr, uint32_t addr_size) +{ + off_t offset; + uint32_t cksum, size; + + WT_UNUSED(addr_size); + + /* Crack the cookie. */ + WT_RET(__wt_block_buffer_to_addr(block, addr, &offset, &size, &cksum)); + + /* Printable representation. */ + WT_RET(__wt_buf_fmt(session, buf, + "[%" PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]", + (uintmax_t)offset, (uintmax_t)offset + size, size, cksum)); + + return (0); +} diff --git a/src/block/block_alloc.c b/src/block/block_alloc.c new file mode 100644 index 00000000000..28b604bc7ef --- /dev/null +++ b/src/block/block_alloc.c @@ -0,0 +1,752 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static int __block_extend(WT_SESSION_IMPL *, WT_BLOCK *, off_t *, off_t); +static int __block_truncate(WT_SESSION_IMPL *, WT_BLOCK *); + +/* + * __block_off_srch -- + * Search a by-offset skiplist. + */ +static void +__block_off_srch(WT_FREE **head, off_t off, WT_FREE ***stack, int skip_off) +{ + WT_FREE **fep; + int i; + + /* + * Start at the highest skip level, then go as far as possible at each + * level before stepping down to the next. + * + * Return a stack for an exact match or the next-largest item. + * + * The WT_FREE structure contains two skiplists, the primary one and the + * per-size bucket one: if the skip_off flag is set, offset the skiplist + * array by the depth specified in this particular structure. + */ + for (i = WT_SKIP_MAXDEPTH - 1, fep = &head[i]; i >= 0;) + if (*fep != NULL && (*fep)->off < off) + fep = &(*fep)->next[i + (skip_off ? (*fep)->depth : 0)]; + else + stack[i--] = fep--; +} + +/* + * __block_size_srch -- + * Search the by-size skiplist. + */ +static void +__block_size_srch(WT_SIZE **head, off_t size, WT_SIZE ***stack) +{ + WT_SIZE **szp; + int i; + + /* + * Start at the highest skip level, then go as far as possible at each + * level before stepping down to the next. + * + * Return a stack for an exact match or the next-largest item. + */ + for (i = WT_SKIP_MAXDEPTH - 1, szp = &head[i]; i >= 0;) + if (*szp != NULL && (*szp)->size < size) + szp = &(*szp)->next[i]; + else + stack[i--] = szp--; +} + +/* + * __block_off_pair_srch -- + * Search the primary by-offset skiplist for before/after records of the + * specified offset. + */ +static void +__block_off_pair_srch( + WT_FREE **head, off_t off, WT_FREE **beforep, WT_FREE **afterp) +{ + WT_FREE **fep; + int i; + + *beforep = *afterp = NULL; + + /* + * Start at the highest skip level, then go as far as possible at each + * level before stepping down to the next. + */ + for (i = WT_SKIP_MAXDEPTH - 1, fep = &head[i]; i >= 0;) { + if (*fep == NULL) { + --i; + --fep; + continue; + } + + if ((*fep)->off < off) { /* Keep going at this level */ + *beforep = *fep; + fep = &(*fep)->next[i]; + } else { /* Drop down a level */ + *afterp = *fep; + --i; + --fep; + } + } +} + +/* + * __block_off_last -- + * Return the last free range in the offset skiplist. + */ +static WT_FREE * +__block_off_last(WT_FREE **head) +{ + WT_FREE *fe, **fep; + int i; + + fe = NULL; + + for (i = WT_SKIP_MAXDEPTH - 1, fep = &head[i]; i >= 0;) { + if (*fep == NULL) { + --i; + --fep; + continue; + } + fe = *fep; + fep = &(*fep)->next[i]; + } + return (fe); +} + +/* + * __block_off_insert -- + * Insert a record into the system. + */ +static int +__block_off_insert(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_FREE *fe) +{ + WT_FREE **astack[WT_SKIP_MAXDEPTH]; + WT_SIZE *szp, **sstack[WT_SKIP_MAXDEPTH]; + u_int i; + + /* + * If we are inserting a new size onto the size skiplist, we'll need + * a new WT_FREE structure for that skiplist. + */ + __block_size_srch(block->fsize, fe->size, sstack); + szp = *sstack[0]; + if (szp == NULL || szp->size != fe->size) { + WT_RET(__wt_calloc(session, 1, + sizeof(WT_SIZE) + fe->depth * sizeof(WT_SIZE *), &szp)); + szp->size = fe->size; + szp->depth = fe->depth; + for (i = 0; i < fe->depth; ++i) { + szp->next[i] = *sstack[i]; + *sstack[i] = szp; + } + } + + /* Insert the new WT_FREE structure into the offset skiplist. */ + __block_off_srch(block->foff, fe->off, astack, 0); + for (i = 0; i < fe->depth; ++i) { + fe->next[i] = *astack[i]; + *astack[i] = fe; + } + + /* + * Insert the new WT_FREE structure into the size element's offset + * skiplist. + */ + __block_off_srch(szp->foff, fe->off, astack, 1); + for (i = 0; i < fe->depth; ++i) { + fe->next[i + fe->depth] = *astack[i]; + *astack[i] = fe; + } + + ++block->freelist_entries; + block->freelist_bytes += (uint64_t)fe->size; + block->freelist_dirty = 1; + + return (0); +} + +/* + * __block_off_remove -- + * Remove a record from the system. + */ +static int +__block_off_remove( + WT_SESSION_IMPL *session, WT_BLOCK *block, off_t off, WT_FREE **fep) +{ + WT_FREE *fe, **astack[WT_SKIP_MAXDEPTH]; + WT_SIZE *szp, **sstack[WT_SKIP_MAXDEPTH]; + u_int i; + + /* Find and remove the record from the by-offset skiplist. */ + __block_off_srch(block->foff, off, astack, 0); + fe = *astack[0]; + if (fe == NULL || fe->off != off) + return (EINVAL); + for (i = 0; i < fe->depth; ++i) + *astack[i] = fe->next[i]; + + /* + * Find and remove the record from the size's offset skiplist; if that + * empties the by-size skiplist entry, remove it as well. + */ + __block_size_srch(block->fsize, fe->size, sstack); + szp = *sstack[0]; + if (szp == NULL || szp->size != fe->size) + return (EINVAL); + __block_off_srch(szp->foff, off, astack, 1); + fe = *astack[0]; + if (fe == NULL || fe->off != off) + return (EINVAL); + for (i = 0; i < fe->depth; ++i) + *astack[i] = fe->next[i + fe->depth]; + if (szp->foff[0] == NULL) { + for (i = 0; i < szp->depth; ++i) + *sstack[i] = szp->next[i]; + __wt_free(session, szp); + } + + --block->freelist_entries; + block->freelist_bytes -= (uint64_t)fe->size; + block->freelist_dirty = 1; + + /* Return the record if our caller wants it, otherwise free it. */ + if (fep == NULL) + __wt_free(session, fe); + else + *fep = fe; + + return (0); +} + +/* + * __wt_block_alloc -- + * Alloc a chunk of space from the underlying file. + */ +int +__wt_block_alloc( + WT_SESSION_IMPL *session, WT_BLOCK *block, off_t *offsetp, off_t size) +{ + WT_FREE *fe; + WT_SIZE *szp, **sstack[WT_SKIP_MAXDEPTH]; + int ret; + + ret = 0; + + WT_BSTAT_INCR(session, alloc); + if (size % block->allocsize != 0) + WT_RET_MSG(session, EINVAL, + "cannot allocate a block size %" PRIdMAX " that is not " + "a multiple of the allocation size %" PRIu32, + (intmax_t)size, block->allocsize); + + __wt_spin_lock(session, &block->freelist_lock); + + /* + * Allocation is first-fit by size: search the by-size skiplist for the + * requested size and take the first entry on the by-size offset list. + * If we don't have anything large enough, extend the file. + */ + __block_size_srch(block->fsize, size, sstack); + szp = *sstack[0]; + if (szp == NULL) { + WT_ERR(__block_extend(session, block, offsetp, size)); + goto done; + } + + /* Remove the first record, and set the returned offset. */ + fe = szp->foff[0]; + WT_ERR(__block_off_remove(session, block, fe->off, &fe)); + *offsetp = fe->off; + + /* If doing a partial allocation, adjust the record and put it back. */ + if (fe->size > size) { + WT_VERBOSE(session, block, + "allocate %" PRIdMAX " from range %" PRIdMAX "-%" + PRIdMAX ", range shrinks to %" PRIdMAX "-%" PRIdMAX, + (intmax_t)size, + (intmax_t)fe->off, (intmax_t)(fe->off + fe->size), + (intmax_t)(fe->off + size), + (intmax_t)(fe->off + size + fe->size - size)); + + fe->off += size; + fe->size -= size; + WT_ERR(__block_off_insert(session, block,fe)); + } else { + WT_VERBOSE(session, block, + "allocate range %" PRIdMAX "-%" PRIdMAX, + (intmax_t)fe->off, (intmax_t)(fe->off + fe->size)); + + __wt_free(session, fe); + } + +done: err: + __wt_spin_unlock(session, &block->freelist_lock); + return (ret); +} + +/* + * __block_extend -- + * Extend the file to allocate space. + */ +static int +__block_extend( + WT_SESSION_IMPL *session, WT_BLOCK *block, off_t *offsetp, off_t size) +{ + WT_FH *fh; + + fh = block->fh; + + /* We should never be allocating from an empty file. */ + if (fh->file_size < WT_BLOCK_DESC_SECTOR) + WT_RET_MSG(session, EINVAL, + "cannot allocate from a file with no description " + "information"); + + /* + * Make sure we don't allocate past the maximum file size. There's no + * easy way to know the maximum off_t on a system, limit growth to 8B + * bits (we currently check an off_t is 8B in verify_build.h). I don't + * think we're likely to see anything bigger for awhile. + * + * XXX + * This isn't sufficient: if we grow the file to the end, there isn't + * enough room to write the free-list out when we close the file. It + * is vanishingly unlikely to happen (we use free blocks where they're + * available to write the free list), but if the free-list is a bunch + * of small blocks, each group of which are insufficient to hold the + * free list, and the file has been fully populated, file close will + * fail because we can't write the free list. + */ + if (fh->file_size > (off_t)INT64_MAX - size) + WT_RET_MSG(session, WT_ERROR, + "block allocation failed, file cannot grow further"); + + *offsetp = fh->file_size; + fh->file_size += size; + + WT_BSTAT_INCR(session, extend); + WT_VERBOSE(session, block, + "file extend %" PRIdMAX "B @ %" PRIdMAX, + (intmax_t)size, (intmax_t)*offsetp); + + return (0); +} + +/* + * __wt_block_free_buf -- + * Free a cookie-referenced chunk of space to the underlying file. + */ +int +__wt_block_free_buf(WT_SESSION_IMPL *session, + WT_BLOCK *block, const uint8_t *addr, uint32_t addr_size) +{ + off_t offset; + uint32_t size; + + WT_UNUSED(addr_size); + + /* Crack the cookie. */ + WT_RET(__wt_block_buffer_to_addr(block, addr, &offset, &size, NULL)); + + WT_RET(__wt_block_free(session, block, offset, (off_t)size)); + + return (0); +} + +/* + * __wt_block_free -- + * Free a chunk of space to the underlying file. + */ +int +__wt_block_free( + WT_SESSION_IMPL *session, WT_BLOCK *block, off_t off, off_t size) +{ + WT_FREE *fe, *after, *before; + u_int skipdepth; + int ret; + + ret = 0; + + WT_BSTAT_INCR(session, free); + WT_VERBOSE(session, block, + "free %" PRIdMAX "/%" PRIdMAX, (intmax_t)off, (intmax_t)size); + + __wt_spin_lock(session, &block->freelist_lock); + + /* + * Retrieve the records preceding/following the offset. If the records + * are contiguous with the free'd offset, combine records. + */ + __block_off_pair_srch(block->foff, off, &before, &after); + if (before != NULL) { + if (before->off + before->size > off) + WT_ERR_MSG(session, EINVAL, + "existing range %" PRIdMAX "-%" PRIdMAX " overlaps " + "with free'd range %" PRIdMAX "-%" PRIdMAX, + (intmax_t)before->off, + (intmax_t)(before->off + before->size), + (intmax_t)off, + (intmax_t)(off + size)); + if (before->off + before->size != off) + before = NULL; + } + if (after != NULL) { + if (off + size > after->off) + WT_ERR_MSG(session, EINVAL, + "free'd range %" PRIdMAX "-%" PRIdMAX " overlaps " + "with existing range %" PRIdMAX "-%" PRIdMAX, + (intmax_t)off, + (intmax_t)(off + size), + (intmax_t)after->off, + (intmax_t)(after->off + after->size)); + if (off + size != after->off) + after = NULL; + } + if (before == NULL && after == NULL) { + /* Allocate a new WT_FREE structure. */ + skipdepth = __wt_skip_choose_depth(); + WT_ERR(__wt_calloc(session, 1, + sizeof(WT_FREE) + skipdepth * 2 * sizeof(WT_FREE *), &fe)); + fe->off = off; + fe->size = size; + fe->depth = (uint8_t)skipdepth; + goto done; + } + + /* + * If the "before" offset range abuts, we'll use it as our new record; + * if the "after" offset range also abuts, include its size and remove + * it from the system. Else, only the "after" offset range abuts, use + * the "after" offset range as our new record. In either case, remove + * the record we're going to use, adjust it and re-insert it. + */ + if (before == NULL) { + WT_ERR(__block_off_remove(session, block, after->off, &fe)); + + WT_VERBOSE(session, block, + "free range grows from %" PRIdMAX "-%" PRIdMAX ", to %" + PRIdMAX "-%" PRIdMAX, + (intmax_t)fe->off, (intmax_t)(fe->off + fe->size), + (intmax_t)off, (intmax_t)(off + fe->size + size)); + + fe->off = off; + fe->size += size; + } else { + if (after != NULL) { + size += after->size; + WT_ERR(__block_off_remove( + session, block, after->off, NULL)); + } + WT_ERR(__block_off_remove(session, block, before->off, &fe)); + + WT_VERBOSE(session, block, + "free range grows from %" PRIdMAX "-%" PRIdMAX ", to %" + PRIdMAX "-%" PRIdMAX, + (intmax_t)fe->off, (intmax_t)(fe->off + fe->size), + (intmax_t)fe->off, (intmax_t)(fe->off + fe->size + size)); + + fe->size += size; + } + +done: WT_ERR(__block_off_insert(session, block,fe)); + +err: __wt_spin_unlock(session, &block->freelist_lock); + return (ret); +} + +/* + * __wt_block_freelist_open -- + * Initialize the free-list structures. + */ +void +__wt_block_freelist_open(WT_SESSION_IMPL *session, WT_BLOCK *block) +{ + __wt_spin_init(session, &block->freelist_lock); + + block->freelist_bytes = 0; + block->freelist_entries = 0; + block->freelist_dirty = 0; + + block->free_offset = WT_BLOCK_INVALID_OFFSET; + block->free_size = block->free_cksum = 0; +} + +/* + * __wt_block_freelist_read -- + * Read the free-list. + */ +int +__wt_block_freelist_read(WT_SESSION_IMPL *session, WT_BLOCK *block) +{ + WT_ITEM *tmp; + off_t offset, size; + uint8_t *p; + int ret; + + tmp = NULL; + ret = 0; + + /* See if there's a free-list to read. */ + if (block->free_offset == WT_BLOCK_INVALID_OFFSET) + return (0); + + WT_RET(__wt_scr_alloc(session, block->free_size, &tmp)); + WT_ERR(__wt_block_read(session, block, + tmp, block->free_offset, block->free_size, block->free_cksum)); + + /* + * The free-list is read before the file is verified, which means we + * need to be a little paranoid. We know the free-list chunk itself + * is entirely in the file because we checked when we first read the + * file's description structure. Confirm the offset/size pairs are + * valid, then insert them into the linked list. + */ + p = WT_BLOCK_HEADER_BYTE(tmp->mem); + +#define WT_FREELIST_READ(p, v) do { \ + (v) = *(off_t *)(p); \ + (p) += sizeof(off_t); \ +} while (0) + + WT_FREELIST_READ(p, offset); + WT_FREELIST_READ(p, size); + if (offset != WT_BLOCK_FREELIST_MAGIC || size != 0) + goto corrupted; + for (;;) { + WT_FREELIST_READ(p, offset); + WT_FREELIST_READ(p, size); + if (offset == WT_BLOCK_INVALID_OFFSET) + break; + + if ((offset - WT_BLOCK_DESC_SECTOR) % block->allocsize != 0 || + size % block->allocsize != 0 || + offset + size > block->fh->file_size) +corrupted: WT_ERR_MSG(session, WT_ERROR, + "file contains a corrupted free-list, range %" + PRIdMAX "-%" PRIdMAX " past end-of-file", + (intmax_t)offset, (intmax_t)(offset + size)); + + WT_ERR(__wt_block_free(session, block, offset, size)); + } + + /* + * Insert the free-list itself into the linked list, but don't clear + * the values, if the free-list is never modified, we don't write it. + */ + WT_ERR(__wt_block_free( + session, block, block->free_offset, (off_t)block->free_size)); + +err: __wt_scr_free(&tmp); + return (ret); +} + +/* + * __wt_block_freelist_close -- + * Write the free-list at the tail of the file. + */ +void +__wt_block_freelist_close(WT_SESSION_IMPL *session, WT_BLOCK *block) +{ + __wt_spin_destroy(session, &block->freelist_lock); +} + +/* + * __wt_block_freelist_write -- + * Write the free-list at the tail of the file. + */ +int +__wt_block_freelist_write(WT_SESSION_IMPL *session, WT_BLOCK *block) +{ + WT_ITEM *tmp; + WT_FREE *fe; + WT_PAGE_HEADER *dsk; + uint32_t datasize, size; + uint8_t *p; + int ret; + + tmp = NULL; + ret = 0; + + /* If the free-list hasn't changed, there's nothing to write. */ + if (block->freelist_dirty == 0) + return (0); + + /* If there aren't any free-list entries, we're done. */ + if (block->freelist_entries == 0) { + block->free_offset = WT_BLOCK_INVALID_OFFSET; + block->free_size = block->free_cksum = 0; + return (0); + } + + /* Truncate the file if possible. */ + WT_RET(__block_truncate(session, block)); + + WT_VERBOSE_CALL(session, block, __wt_block_dump(session, block)); + + /* + * Get a scratch buffer, clear the page's header and data, initialize + * the header. Allocate an allocation-sized aligned buffer so the + * block write function can zero-out unused bytes and write it without + * copying to something larger. + * + * Allocate room for the free-list entries, plus 2 additional entries: + * the initial WT_BLOCK_FREELIST_MAGIC/0 pair and the list-terminating + * WT_BLOCK_INVALID_OFFSET/0 pair. + */ + datasize = size = + (block->freelist_entries + 2) * WT_STORE_SIZE(sizeof(off_t) * 2); + WT_RET(__wt_block_write_size(session, block, &size)); + WT_RET(__wt_scr_alloc(session, size, &tmp)); + dsk = tmp->mem; + memset(dsk, 0, WT_BLOCK_HEADER_BYTE_SIZE); + dsk->u.datalen = WT_STORE_SIZE(datasize); + dsk->type = WT_PAGE_FREELIST; + tmp->size = WT_STORE_SIZE(WT_BLOCK_HEADER_BYTE_SIZE + datasize); + + /* Fill the page's data. */ + p = WT_BLOCK_HEADER_BYTE(dsk); + +#define WT_FREELIST_WRITE(p, v) do { \ + *(off_t *)(p) = (v); \ + (p) += sizeof(off_t); \ +} while (0) + + WT_FREELIST_WRITE(p, WT_BLOCK_FREELIST_MAGIC); /* Initial value */ + WT_FREELIST_WRITE(p, 0); + WT_FREE_FOREACH(fe, block->foff) { /* Free ranges */ + WT_FREELIST_WRITE(p, fe->off); + WT_FREELIST_WRITE(p, fe->size); + } + WT_FREELIST_WRITE(p, WT_BLOCK_INVALID_OFFSET); /* Ending value */ + WT_FREELIST_WRITE(p, 0); + + /* + * Discard the in-memory free-list: this has to happen before writing + * the free-list because the underlying block write function is going + * to allocate file space for the free-list block(s), and allocating + * from the blocks on the free-list we just wrote won't work out well. + * A workaround would be to not compress the free-list, which implies + * some kind of "write but don't compress" code path, and that's more + * complex than ordering these operations so the eventual allocation + * in the write code always extends the file. + */ + __wt_block_discard(session, block); + + /* Write the free list to disk and update the file's meta-data. */ + WT_ERR(__wt_block_write(session, block, tmp, + &block->free_offset, &block->free_size, &block->free_cksum)); + + WT_VERBOSE(session, block, + "free-list written %" PRIdMAX "/%" PRIu32, + (intmax_t)block->free_offset, block->free_size); + +err: __wt_scr_free(&tmp); + return (ret); +} + +/* + * __block_truncate -- + * Truncate the file if the last part of the file isn't in use. + */ +static int +__block_truncate(WT_SESSION_IMPL *session, WT_BLOCK *block) +{ + WT_FH *fh; + WT_FREE *fe; + + fh = block->fh; + + /* + * Check if the last free range is at the end of the file, and if so, + * truncate the file and discard the range. + */ + if ((fe = __block_off_last(block->foff)) == NULL) + return (0); + if (fe->off + fe->size != fh->file_size) + return (0); + + WT_VERBOSE(session, block, + "truncate file from %" PRIdMAX " to %" PRIdMAX, + (intmax_t)fh->file_size, (intmax_t)fe->off); + + fh->file_size = fe->off; + WT_RET(__wt_ftruncate(session, fh, fh->file_size)); + + WT_RET(__block_off_remove(session, block, fe->off, NULL)); + + return (0); +} + +/* + * __wt_block_discard -- + * Discard any free-list entries. + */ +void +__wt_block_discard(WT_SESSION_IMPL *session, WT_BLOCK *block) +{ + WT_FREE *fe, *nfe; + WT_SIZE *szp, *nszp; + + for (fe = block->foff[0]; fe != NULL; fe = nfe) { + nfe = fe->next[0]; + __wt_free(session, fe); + } + memset(block->foff, 0, sizeof(block->foff)); + for (szp = block->fsize[0]; szp != NULL; szp = nszp) { + nszp = szp->next[0]; + __wt_free(session, szp); + } + memset(block->fsize, 0, sizeof(block->fsize)); + + block->freelist_bytes = 0; + block->freelist_entries = 0; + block->freelist_dirty = 0; +} + +/* + * __wt_block_stat -- + * Free-list statistics. + */ +void +__wt_block_stat(WT_SESSION_IMPL *session, WT_BLOCK *block) +{ + WT_BSTAT_SET(session, file_freelist_bytes, block->freelist_bytes); + WT_BSTAT_SET(session, file_freelist_entries, block->freelist_entries); + WT_BSTAT_SET(session, file_size, block->fh->file_size); + WT_BSTAT_SET(session, file_magic, WT_BLOCK_MAGIC); + WT_BSTAT_SET(session, file_major, WT_BLOCK_MAJOR_VERSION); + WT_BSTAT_SET(session, file_minor, WT_BLOCK_MINOR_VERSION); +} + +#ifdef HAVE_VERBOSE +void +__wt_block_dump(WT_SESSION_IMPL *session, WT_BLOCK *block) +{ + WT_FREE *fe; + WT_SIZE *szp; + + WT_VERBOSE(session, block, "freelist by offset:"); + WT_FREE_FOREACH(fe, block->foff) + WT_VERBOSE(session, block, + "\t{%" PRIuMAX "/%" PRIuMAX "}", + (uintmax_t)fe->off, (uintmax_t)fe->size); + + WT_VERBOSE(session, block, "freelist by size:"); + WT_FREE_FOREACH(szp, block->fsize) { + WT_VERBOSE(session, block, + "\t{%" PRIuMAX "}", + (uintmax_t)szp->size); + WT_FREE_FOREACH_OFF(fe, szp->foff) + WT_VERBOSE(session, block, + "\t\t{%" PRIuMAX "/%" PRIuMAX "}", + (uintmax_t)fe->off, (uintmax_t)fe->size); + } +} +#endif diff --git a/src/block/block_cksum.c b/src/block/block_cksum.c new file mode 100644 index 00000000000..4ed521d0adb --- /dev/null +++ b/src/block/block_cksum.c @@ -0,0 +1,1160 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "wt_internal.h" + +static const uint32_t g_crc_slicing[8][256] = { +#ifdef WORDS_BIGENDIAN + /* + * Big endian tables have entries that are byte reversed from little + * endian tables. + */ + { + 0x00000000, 0x03836bf2, 0xf7703be1, 0xf4f35013, + 0x1f979ac7, 0x1c14f135, 0xe8e7a126, 0xeb64cad4, + 0xcf58d98a, 0xccdbb278, 0x3828e26b, 0x3bab8999, + 0xd0cf434d, 0xd34c28bf, 0x27bf78ac, 0x243c135e, + 0x6fc75e10, 0x6c4435e2, 0x98b765f1, 0x9b340e03, + 0x7050c4d7, 0x73d3af25, 0x8720ff36, 0x84a394c4, + 0xa09f879a, 0xa31cec68, 0x57efbc7b, 0x546cd789, + 0xbf081d5d, 0xbc8b76af, 0x487826bc, 0x4bfb4d4e, + 0xde8ebd20, 0xdd0dd6d2, 0x29fe86c1, 0x2a7ded33, + 0xc11927e7, 0xc29a4c15, 0x36691c06, 0x35ea77f4, + 0x11d664aa, 0x12550f58, 0xe6a65f4b, 0xe52534b9, + 0x0e41fe6d, 0x0dc2959f, 0xf931c58c, 0xfab2ae7e, + 0xb149e330, 0xb2ca88c2, 0x4639d8d1, 0x45bab323, + 0xaede79f7, 0xad5d1205, 0x59ae4216, 0x5a2d29e4, + 0x7e113aba, 0x7d925148, 0x8961015b, 0x8ae26aa9, + 0x6186a07d, 0x6205cb8f, 0x96f69b9c, 0x9575f06e, + 0xbc1d7b41, 0xbf9e10b3, 0x4b6d40a0, 0x48ee2b52, + 0xa38ae186, 0xa0098a74, 0x54fada67, 0x5779b195, + 0x7345a2cb, 0x70c6c939, 0x8435992a, 0x87b6f2d8, + 0x6cd2380c, 0x6f5153fe, 0x9ba203ed, 0x9821681f, + 0xd3da2551, 0xd0594ea3, 0x24aa1eb0, 0x27297542, + 0xcc4dbf96, 0xcfced464, 0x3b3d8477, 0x38beef85, + 0x1c82fcdb, 0x1f019729, 0xebf2c73a, 0xe871acc8, + 0x0315661c, 0x00960dee, 0xf4655dfd, 0xf7e6360f, + 0x6293c661, 0x6110ad93, 0x95e3fd80, 0x96609672, + 0x7d045ca6, 0x7e873754, 0x8a746747, 0x89f70cb5, + 0xadcb1feb, 0xae487419, 0x5abb240a, 0x59384ff8, + 0xb25c852c, 0xb1dfeede, 0x452cbecd, 0x46afd53f, + 0x0d549871, 0x0ed7f383, 0xfa24a390, 0xf9a7c862, + 0x12c302b6, 0x11406944, 0xe5b33957, 0xe63052a5, + 0xc20c41fb, 0xc18f2a09, 0x357c7a1a, 0x36ff11e8, + 0xdd9bdb3c, 0xde18b0ce, 0x2aebe0dd, 0x29688b2f, + 0x783bf682, 0x7bb89d70, 0x8f4bcd63, 0x8cc8a691, + 0x67ac6c45, 0x642f07b7, 0x90dc57a4, 0x935f3c56, + 0xb7632f08, 0xb4e044fa, 0x401314e9, 0x43907f1b, + 0xa8f4b5cf, 0xab77de3d, 0x5f848e2e, 0x5c07e5dc, + 0x17fca892, 0x147fc360, 0xe08c9373, 0xe30ff881, + 0x086b3255, 0x0be859a7, 0xff1b09b4, 0xfc986246, + 0xd8a47118, 0xdb271aea, 0x2fd44af9, 0x2c57210b, + 0xc733ebdf, 0xc4b0802d, 0x3043d03e, 0x33c0bbcc, + 0xa6b54ba2, 0xa5362050, 0x51c57043, 0x52461bb1, + 0xb922d165, 0xbaa1ba97, 0x4e52ea84, 0x4dd18176, + 0x69ed9228, 0x6a6ef9da, 0x9e9da9c9, 0x9d1ec23b, + 0x767a08ef, 0x75f9631d, 0x810a330e, 0x828958fc, + 0xc97215b2, 0xcaf17e40, 0x3e022e53, 0x3d8145a1, + 0xd6e58f75, 0xd566e487, 0x2195b494, 0x2216df66, + 0x062acc38, 0x05a9a7ca, 0xf15af7d9, 0xf2d99c2b, + 0x19bd56ff, 0x1a3e3d0d, 0xeecd6d1e, 0xed4e06ec, + 0xc4268dc3, 0xc7a5e631, 0x3356b622, 0x30d5ddd0, + 0xdbb11704, 0xd8327cf6, 0x2cc12ce5, 0x2f424717, + 0x0b7e5449, 0x08fd3fbb, 0xfc0e6fa8, 0xff8d045a, + 0x14e9ce8e, 0x176aa57c, 0xe399f56f, 0xe01a9e9d, + 0xabe1d3d3, 0xa862b821, 0x5c91e832, 0x5f1283c0, + 0xb4764914, 0xb7f522e6, 0x430672f5, 0x40851907, + 0x64b90a59, 0x673a61ab, 0x93c931b8, 0x904a5a4a, + 0x7b2e909e, 0x78adfb6c, 0x8c5eab7f, 0x8fddc08d, + 0x1aa830e3, 0x192b5b11, 0xedd80b02, 0xee5b60f0, + 0x053faa24, 0x06bcc1d6, 0xf24f91c5, 0xf1ccfa37, + 0xd5f0e969, 0xd673829b, 0x2280d288, 0x2103b97a, + 0xca6773ae, 0xc9e4185c, 0x3d17484f, 0x3e9423bd, + 0x756f6ef3, 0x76ec0501, 0x821f5512, 0x819c3ee0, + 0x6af8f434, 0x697b9fc6, 0x9d88cfd5, 0x9e0ba427, + 0xba37b779, 0xb9b4dc8b, 0x4d478c98, 0x4ec4e76a, + 0xa5a02dbe, 0xa623464c, 0x52d0165f, 0x51537dad + },{ + 0x00000000, 0x7798a213, 0xee304527, 0x99a8e734, + 0xdc618a4e, 0xabf9285d, 0x3251cf69, 0x45c96d7a, + 0xb8c3149d, 0xcf5bb68e, 0x56f351ba, 0x216bf3a9, + 0x64a29ed3, 0x133a3cc0, 0x8a92dbf4, 0xfd0a79e7, + 0x81f1c53f, 0xf669672c, 0x6fc18018, 0x1859220b, + 0x5d904f71, 0x2a08ed62, 0xb3a00a56, 0xc438a845, + 0x3932d1a2, 0x4eaa73b1, 0xd7029485, 0xa09a3696, + 0xe5535bec, 0x92cbf9ff, 0x0b631ecb, 0x7cfbbcd8, + 0x02e38b7f, 0x757b296c, 0xecd3ce58, 0x9b4b6c4b, + 0xde820131, 0xa91aa322, 0x30b24416, 0x472ae605, + 0xba209fe2, 0xcdb83df1, 0x5410dac5, 0x238878d6, + 0x664115ac, 0x11d9b7bf, 0x8871508b, 0xffe9f298, + 0x83124e40, 0xf48aec53, 0x6d220b67, 0x1abaa974, + 0x5f73c40e, 0x28eb661d, 0xb1438129, 0xc6db233a, + 0x3bd15add, 0x4c49f8ce, 0xd5e11ffa, 0xa279bde9, + 0xe7b0d093, 0x90287280, 0x098095b4, 0x7e1837a7, + 0x04c617ff, 0x735eb5ec, 0xeaf652d8, 0x9d6ef0cb, + 0xd8a79db1, 0xaf3f3fa2, 0x3697d896, 0x410f7a85, + 0xbc050362, 0xcb9da171, 0x52354645, 0x25ade456, + 0x6064892c, 0x17fc2b3f, 0x8e54cc0b, 0xf9cc6e18, + 0x8537d2c0, 0xf2af70d3, 0x6b0797e7, 0x1c9f35f4, + 0x5956588e, 0x2ecefa9d, 0xb7661da9, 0xc0febfba, + 0x3df4c65d, 0x4a6c644e, 0xd3c4837a, 0xa45c2169, + 0xe1954c13, 0x960dee00, 0x0fa50934, 0x783dab27, + 0x06259c80, 0x71bd3e93, 0xe815d9a7, 0x9f8d7bb4, + 0xda4416ce, 0xaddcb4dd, 0x347453e9, 0x43ecf1fa, + 0xbee6881d, 0xc97e2a0e, 0x50d6cd3a, 0x274e6f29, + 0x62870253, 0x151fa040, 0x8cb74774, 0xfb2fe567, + 0x87d459bf, 0xf04cfbac, 0x69e41c98, 0x1e7cbe8b, + 0x5bb5d3f1, 0x2c2d71e2, 0xb58596d6, 0xc21d34c5, + 0x3f174d22, 0x488fef31, 0xd1270805, 0xa6bfaa16, + 0xe376c76c, 0x94ee657f, 0x0d46824b, 0x7ade2058, + 0xf9fac3fb, 0x8e6261e8, 0x17ca86dc, 0x605224cf, + 0x259b49b5, 0x5203eba6, 0xcbab0c92, 0xbc33ae81, + 0x4139d766, 0x36a17575, 0xaf099241, 0xd8913052, + 0x9d585d28, 0xeac0ff3b, 0x7368180f, 0x04f0ba1c, + 0x780b06c4, 0x0f93a4d7, 0x963b43e3, 0xe1a3e1f0, + 0xa46a8c8a, 0xd3f22e99, 0x4a5ac9ad, 0x3dc26bbe, + 0xc0c81259, 0xb750b04a, 0x2ef8577e, 0x5960f56d, + 0x1ca99817, 0x6b313a04, 0xf299dd30, 0x85017f23, + 0xfb194884, 0x8c81ea97, 0x15290da3, 0x62b1afb0, + 0x2778c2ca, 0x50e060d9, 0xc94887ed, 0xbed025fe, + 0x43da5c19, 0x3442fe0a, 0xadea193e, 0xda72bb2d, + 0x9fbbd657, 0xe8237444, 0x718b9370, 0x06133163, + 0x7ae88dbb, 0x0d702fa8, 0x94d8c89c, 0xe3406a8f, + 0xa68907f5, 0xd111a5e6, 0x48b942d2, 0x3f21e0c1, + 0xc22b9926, 0xb5b33b35, 0x2c1bdc01, 0x5b837e12, + 0x1e4a1368, 0x69d2b17b, 0xf07a564f, 0x87e2f45c, + 0xfd3cd404, 0x8aa47617, 0x130c9123, 0x64943330, + 0x215d5e4a, 0x56c5fc59, 0xcf6d1b6d, 0xb8f5b97e, + 0x45ffc099, 0x3267628a, 0xabcf85be, 0xdc5727ad, + 0x999e4ad7, 0xee06e8c4, 0x77ae0ff0, 0x0036ade3, + 0x7ccd113b, 0x0b55b328, 0x92fd541c, 0xe565f60f, + 0xa0ac9b75, 0xd7343966, 0x4e9cde52, 0x39047c41, + 0xc40e05a6, 0xb396a7b5, 0x2a3e4081, 0x5da6e292, + 0x186f8fe8, 0x6ff72dfb, 0xf65fcacf, 0x81c768dc, + 0xffdf5f7b, 0x8847fd68, 0x11ef1a5c, 0x6677b84f, + 0x23bed535, 0x54267726, 0xcd8e9012, 0xba163201, + 0x471c4be6, 0x3084e9f5, 0xa92c0ec1, 0xdeb4acd2, + 0x9b7dc1a8, 0xece563bb, 0x754d848f, 0x02d5269c, + 0x7e2e9a44, 0x09b63857, 0x901edf63, 0xe7867d70, + 0xa24f100a, 0xd5d7b219, 0x4c7f552d, 0x3be7f73e, + 0xc6ed8ed9, 0xb1752cca, 0x28ddcbfe, 0x5f4569ed, + 0x1a8c0497, 0x6d14a684, 0xf4bc41b0, 0x8324e3a3 + },{ + 0x00000000, 0x7e9241a5, 0x0d526f4f, 0x73c02eea, + 0x1aa4de9e, 0x64369f3b, 0x17f6b1d1, 0x6964f074, + 0xc53e5138, 0xbbac109d, 0xc86c3e77, 0xb6fe7fd2, + 0xdf9a8fa6, 0xa108ce03, 0xd2c8e0e9, 0xac5aa14c, + 0x8a7da270, 0xf4efe3d5, 0x872fcd3f, 0xf9bd8c9a, + 0x90d97cee, 0xee4b3d4b, 0x9d8b13a1, 0xe3195204, + 0x4f43f348, 0x31d1b2ed, 0x42119c07, 0x3c83dda2, + 0x55e72dd6, 0x2b756c73, 0x58b54299, 0x2627033c, + 0x14fb44e1, 0x6a690544, 0x19a92bae, 0x673b6a0b, + 0x0e5f9a7f, 0x70cddbda, 0x030df530, 0x7d9fb495, + 0xd1c515d9, 0xaf57547c, 0xdc977a96, 0xa2053b33, + 0xcb61cb47, 0xb5f38ae2, 0xc633a408, 0xb8a1e5ad, + 0x9e86e691, 0xe014a734, 0x93d489de, 0xed46c87b, + 0x8422380f, 0xfab079aa, 0x89705740, 0xf7e216e5, + 0x5bb8b7a9, 0x252af60c, 0x56ead8e6, 0x28789943, + 0x411c6937, 0x3f8e2892, 0x4c4e0678, 0x32dc47dd, + 0xd98065c7, 0xa7122462, 0xd4d20a88, 0xaa404b2d, + 0xc324bb59, 0xbdb6fafc, 0xce76d416, 0xb0e495b3, + 0x1cbe34ff, 0x622c755a, 0x11ec5bb0, 0x6f7e1a15, + 0x061aea61, 0x7888abc4, 0x0b48852e, 0x75dac48b, + 0x53fdc7b7, 0x2d6f8612, 0x5eafa8f8, 0x203de95d, + 0x49591929, 0x37cb588c, 0x440b7666, 0x3a9937c3, + 0x96c3968f, 0xe851d72a, 0x9b91f9c0, 0xe503b865, + 0x8c674811, 0xf2f509b4, 0x8135275e, 0xffa766fb, + 0xcd7b2126, 0xb3e96083, 0xc0294e69, 0xbebb0fcc, + 0xd7dfffb8, 0xa94dbe1d, 0xda8d90f7, 0xa41fd152, + 0x0845701e, 0x76d731bb, 0x05171f51, 0x7b855ef4, + 0x12e1ae80, 0x6c73ef25, 0x1fb3c1cf, 0x6121806a, + 0x47068356, 0x3994c2f3, 0x4a54ec19, 0x34c6adbc, + 0x5da25dc8, 0x23301c6d, 0x50f03287, 0x2e627322, + 0x8238d26e, 0xfcaa93cb, 0x8f6abd21, 0xf1f8fc84, + 0x989c0cf0, 0xe60e4d55, 0x95ce63bf, 0xeb5c221a, + 0x4377278b, 0x3de5662e, 0x4e2548c4, 0x30b70961, + 0x59d3f915, 0x2741b8b0, 0x5481965a, 0x2a13d7ff, + 0x864976b3, 0xf8db3716, 0x8b1b19fc, 0xf5895859, + 0x9ceda82d, 0xe27fe988, 0x91bfc762, 0xef2d86c7, + 0xc90a85fb, 0xb798c45e, 0xc458eab4, 0xbacaab11, + 0xd3ae5b65, 0xad3c1ac0, 0xdefc342a, 0xa06e758f, + 0x0c34d4c3, 0x72a69566, 0x0166bb8c, 0x7ff4fa29, + 0x16900a5d, 0x68024bf8, 0x1bc26512, 0x655024b7, + 0x578c636a, 0x291e22cf, 0x5ade0c25, 0x244c4d80, + 0x4d28bdf4, 0x33bafc51, 0x407ad2bb, 0x3ee8931e, + 0x92b23252, 0xec2073f7, 0x9fe05d1d, 0xe1721cb8, + 0x8816eccc, 0xf684ad69, 0x85448383, 0xfbd6c226, + 0xddf1c11a, 0xa36380bf, 0xd0a3ae55, 0xae31eff0, + 0xc7551f84, 0xb9c75e21, 0xca0770cb, 0xb495316e, + 0x18cf9022, 0x665dd187, 0x159dff6d, 0x6b0fbec8, + 0x026b4ebc, 0x7cf90f19, 0x0f3921f3, 0x71ab6056, + 0x9af7424c, 0xe46503e9, 0x97a52d03, 0xe9376ca6, + 0x80539cd2, 0xfec1dd77, 0x8d01f39d, 0xf393b238, + 0x5fc91374, 0x215b52d1, 0x529b7c3b, 0x2c093d9e, + 0x456dcdea, 0x3bff8c4f, 0x483fa2a5, 0x36ade300, + 0x108ae03c, 0x6e18a199, 0x1dd88f73, 0x634aced6, + 0x0a2e3ea2, 0x74bc7f07, 0x077c51ed, 0x79ee1048, + 0xd5b4b104, 0xab26f0a1, 0xd8e6de4b, 0xa6749fee, + 0xcf106f9a, 0xb1822e3f, 0xc24200d5, 0xbcd04170, + 0x8e0c06ad, 0xf09e4708, 0x835e69e2, 0xfdcc2847, + 0x94a8d833, 0xea3a9996, 0x99fab77c, 0xe768f6d9, + 0x4b325795, 0x35a01630, 0x466038da, 0x38f2797f, + 0x5196890b, 0x2f04c8ae, 0x5cc4e644, 0x2256a7e1, + 0x0471a4dd, 0x7ae3e578, 0x0923cb92, 0x77b18a37, + 0x1ed57a43, 0x60473be6, 0x1387150c, 0x6d1554a9, + 0xc14ff5e5, 0xbfddb440, 0xcc1d9aaa, 0xb28fdb0f, + 0xdbeb2b7b, 0xa5796ade, 0xd6b94434, 0xa82b0591 + },{ + 0x00000000, 0xb8aa45dd, 0x812367bf, 0x39892262, + 0xf331227b, 0x4b9b67a6, 0x721245c4, 0xcab80019, + 0xe66344f6, 0x5ec9012b, 0x67402349, 0xdfea6694, + 0x1552668d, 0xadf82350, 0x94710132, 0x2cdb44ef, + 0x3db164e9, 0x851b2134, 0xbc920356, 0x0438468b, + 0xce804692, 0x762a034f, 0x4fa3212d, 0xf70964f0, + 0xdbd2201f, 0x637865c2, 0x5af147a0, 0xe25b027d, + 0x28e30264, 0x904947b9, 0xa9c065db, 0x116a2006, + 0x8b1425d7, 0x33be600a, 0x0a374268, 0xb29d07b5, + 0x782507ac, 0xc08f4271, 0xf9066013, 0x41ac25ce, + 0x6d776121, 0xd5dd24fc, 0xec54069e, 0x54fe4343, + 0x9e46435a, 0x26ec0687, 0x1f6524e5, 0xa7cf6138, + 0xb6a5413e, 0x0e0f04e3, 0x37862681, 0x8f2c635c, + 0x45946345, 0xfd3e2698, 0xc4b704fa, 0x7c1d4127, + 0x50c605c8, 0xe86c4015, 0xd1e56277, 0x694f27aa, + 0xa3f727b3, 0x1b5d626e, 0x22d4400c, 0x9a7e05d1, + 0xe75fa6ab, 0x5ff5e376, 0x667cc114, 0xded684c9, + 0x146e84d0, 0xacc4c10d, 0x954de36f, 0x2de7a6b2, + 0x013ce25d, 0xb996a780, 0x801f85e2, 0x38b5c03f, + 0xf20dc026, 0x4aa785fb, 0x732ea799, 0xcb84e244, + 0xdaeec242, 0x6244879f, 0x5bcda5fd, 0xe367e020, + 0x29dfe039, 0x9175a5e4, 0xa8fc8786, 0x1056c25b, + 0x3c8d86b4, 0x8427c369, 0xbdaee10b, 0x0504a4d6, + 0xcfbca4cf, 0x7716e112, 0x4e9fc370, 0xf63586ad, + 0x6c4b837c, 0xd4e1c6a1, 0xed68e4c3, 0x55c2a11e, + 0x9f7aa107, 0x27d0e4da, 0x1e59c6b8, 0xa6f38365, + 0x8a28c78a, 0x32828257, 0x0b0ba035, 0xb3a1e5e8, + 0x7919e5f1, 0xc1b3a02c, 0xf83a824e, 0x4090c793, + 0x51fae795, 0xe950a248, 0xd0d9802a, 0x6873c5f7, + 0xa2cbc5ee, 0x1a618033, 0x23e8a251, 0x9b42e78c, + 0xb799a363, 0x0f33e6be, 0x36bac4dc, 0x8e108101, + 0x44a88118, 0xfc02c4c5, 0xc58be6a7, 0x7d21a37a, + 0x3fc9a052, 0x8763e58f, 0xbeeac7ed, 0x06408230, + 0xccf88229, 0x7452c7f4, 0x4ddbe596, 0xf571a04b, + 0xd9aae4a4, 0x6100a179, 0x5889831b, 0xe023c6c6, + 0x2a9bc6df, 0x92318302, 0xabb8a160, 0x1312e4bd, + 0x0278c4bb, 0xbad28166, 0x835ba304, 0x3bf1e6d9, + 0xf149e6c0, 0x49e3a31d, 0x706a817f, 0xc8c0c4a2, + 0xe41b804d, 0x5cb1c590, 0x6538e7f2, 0xdd92a22f, + 0x172aa236, 0xaf80e7eb, 0x9609c589, 0x2ea38054, + 0xb4dd8585, 0x0c77c058, 0x35fee23a, 0x8d54a7e7, + 0x47eca7fe, 0xff46e223, 0xc6cfc041, 0x7e65859c, + 0x52bec173, 0xea1484ae, 0xd39da6cc, 0x6b37e311, + 0xa18fe308, 0x1925a6d5, 0x20ac84b7, 0x9806c16a, + 0x896ce16c, 0x31c6a4b1, 0x084f86d3, 0xb0e5c30e, + 0x7a5dc317, 0xc2f786ca, 0xfb7ea4a8, 0x43d4e175, + 0x6f0fa59a, 0xd7a5e047, 0xee2cc225, 0x568687f8, + 0x9c3e87e1, 0x2494c23c, 0x1d1de05e, 0xa5b7a583, + 0xd89606f9, 0x603c4324, 0x59b56146, 0xe11f249b, + 0x2ba72482, 0x930d615f, 0xaa84433d, 0x122e06e0, + 0x3ef5420f, 0x865f07d2, 0xbfd625b0, 0x077c606d, + 0xcdc46074, 0x756e25a9, 0x4ce707cb, 0xf44d4216, + 0xe5276210, 0x5d8d27cd, 0x640405af, 0xdcae4072, + 0x1616406b, 0xaebc05b6, 0x973527d4, 0x2f9f6209, + 0x034426e6, 0xbbee633b, 0x82674159, 0x3acd0484, + 0xf075049d, 0x48df4140, 0x71566322, 0xc9fc26ff, + 0x5382232e, 0xeb2866f3, 0xd2a14491, 0x6a0b014c, + 0xa0b30155, 0x18194488, 0x219066ea, 0x993a2337, + 0xb5e167d8, 0x0d4b2205, 0x34c20067, 0x8c6845ba, + 0x46d045a3, 0xfe7a007e, 0xc7f3221c, 0x7f5967c1, + 0x6e3347c7, 0xd699021a, 0xef102078, 0x57ba65a5, + 0x9d0265bc, 0x25a82061, 0x1c210203, 0xa48b47de, + 0x88500331, 0x30fa46ec, 0x0973648e, 0xb1d92153, + 0x7b61214a, 0xc3cb6497, 0xfa4246f5, 0x42e80328 + },{ + 0x00000000, 0xac6f1138, 0x58df2270, 0xf4b03348, + 0xb0be45e0, 0x1cd154d8, 0xe8616790, 0x440e76a8, + 0x910b67c5, 0x3d6476fd, 0xc9d445b5, 0x65bb548d, + 0x21b52225, 0x8dda331d, 0x796a0055, 0xd505116d, + 0xd361228f, 0x7f0e33b7, 0x8bbe00ff, 0x27d111c7, + 0x63df676f, 0xcfb07657, 0x3b00451f, 0x976f5427, + 0x426a454a, 0xee055472, 0x1ab5673a, 0xb6da7602, + 0xf2d400aa, 0x5ebb1192, 0xaa0b22da, 0x066433e2, + 0x57b5a81b, 0xfbdab923, 0x0f6a8a6b, 0xa3059b53, + 0xe70bedfb, 0x4b64fcc3, 0xbfd4cf8b, 0x13bbdeb3, + 0xc6becfde, 0x6ad1dee6, 0x9e61edae, 0x320efc96, + 0x76008a3e, 0xda6f9b06, 0x2edfa84e, 0x82b0b976, + 0x84d48a94, 0x28bb9bac, 0xdc0ba8e4, 0x7064b9dc, + 0x346acf74, 0x9805de4c, 0x6cb5ed04, 0xc0dafc3c, + 0x15dfed51, 0xb9b0fc69, 0x4d00cf21, 0xe16fde19, + 0xa561a8b1, 0x090eb989, 0xfdbe8ac1, 0x51d19bf9, + 0xae6a5137, 0x0205400f, 0xf6b57347, 0x5ada627f, + 0x1ed414d7, 0xb2bb05ef, 0x460b36a7, 0xea64279f, + 0x3f6136f2, 0x930e27ca, 0x67be1482, 0xcbd105ba, + 0x8fdf7312, 0x23b0622a, 0xd7005162, 0x7b6f405a, + 0x7d0b73b8, 0xd1646280, 0x25d451c8, 0x89bb40f0, + 0xcdb53658, 0x61da2760, 0x956a1428, 0x39050510, + 0xec00147d, 0x406f0545, 0xb4df360d, 0x18b02735, + 0x5cbe519d, 0xf0d140a5, 0x046173ed, 0xa80e62d5, + 0xf9dff92c, 0x55b0e814, 0xa100db5c, 0x0d6fca64, + 0x4961bccc, 0xe50eadf4, 0x11be9ebc, 0xbdd18f84, + 0x68d49ee9, 0xc4bb8fd1, 0x300bbc99, 0x9c64ada1, + 0xd86adb09, 0x7405ca31, 0x80b5f979, 0x2cdae841, + 0x2abedba3, 0x86d1ca9b, 0x7261f9d3, 0xde0ee8eb, + 0x9a009e43, 0x366f8f7b, 0xc2dfbc33, 0x6eb0ad0b, + 0xbbb5bc66, 0x17daad5e, 0xe36a9e16, 0x4f058f2e, + 0x0b0bf986, 0xa764e8be, 0x53d4dbf6, 0xffbbcace, + 0x5cd5a26e, 0xf0bab356, 0x040a801e, 0xa8659126, + 0xec6be78e, 0x4004f6b6, 0xb4b4c5fe, 0x18dbd4c6, + 0xcddec5ab, 0x61b1d493, 0x9501e7db, 0x396ef6e3, + 0x7d60804b, 0xd10f9173, 0x25bfa23b, 0x89d0b303, + 0x8fb480e1, 0x23db91d9, 0xd76ba291, 0x7b04b3a9, + 0x3f0ac501, 0x9365d439, 0x67d5e771, 0xcbbaf649, + 0x1ebfe724, 0xb2d0f61c, 0x4660c554, 0xea0fd46c, + 0xae01a2c4, 0x026eb3fc, 0xf6de80b4, 0x5ab1918c, + 0x0b600a75, 0xa70f1b4d, 0x53bf2805, 0xffd0393d, + 0xbbde4f95, 0x17b15ead, 0xe3016de5, 0x4f6e7cdd, + 0x9a6b6db0, 0x36047c88, 0xc2b44fc0, 0x6edb5ef8, + 0x2ad52850, 0x86ba3968, 0x720a0a20, 0xde651b18, + 0xd80128fa, 0x746e39c2, 0x80de0a8a, 0x2cb11bb2, + 0x68bf6d1a, 0xc4d07c22, 0x30604f6a, 0x9c0f5e52, + 0x490a4f3f, 0xe5655e07, 0x11d56d4f, 0xbdba7c77, + 0xf9b40adf, 0x55db1be7, 0xa16b28af, 0x0d043997, + 0xf2bff359, 0x5ed0e261, 0xaa60d129, 0x060fc011, + 0x4201b6b9, 0xee6ea781, 0x1ade94c9, 0xb6b185f1, + 0x63b4949c, 0xcfdb85a4, 0x3b6bb6ec, 0x9704a7d4, + 0xd30ad17c, 0x7f65c044, 0x8bd5f30c, 0x27bae234, + 0x21ded1d6, 0x8db1c0ee, 0x7901f3a6, 0xd56ee29e, + 0x91609436, 0x3d0f850e, 0xc9bfb646, 0x65d0a77e, + 0xb0d5b613, 0x1cbaa72b, 0xe80a9463, 0x4465855b, + 0x006bf3f3, 0xac04e2cb, 0x58b4d183, 0xf4dbc0bb, + 0xa50a5b42, 0x09654a7a, 0xfdd57932, 0x51ba680a, + 0x15b41ea2, 0xb9db0f9a, 0x4d6b3cd2, 0xe1042dea, + 0x34013c87, 0x986e2dbf, 0x6cde1ef7, 0xc0b10fcf, + 0x84bf7967, 0x28d0685f, 0xdc605b17, 0x700f4a2f, + 0x766b79cd, 0xda0468f5, 0x2eb45bbd, 0x82db4a85, + 0xc6d53c2d, 0x6aba2d15, 0x9e0a1e5d, 0x32650f65, + 0xe7601e08, 0x4b0f0f30, 0xbfbf3c78, 0x13d02d40, + 0x57de5be8, 0xfbb14ad0, 0x0f017998, 0xa36e68a0 + },{ + 0x00000000, 0x196b30ef, 0xc3a08cdb, 0xdacbbc34, + 0x7737f5b2, 0x6e5cc55d, 0xb4977969, 0xadfc4986, + 0x1f180660, 0x0673368f, 0xdcb88abb, 0xc5d3ba54, + 0x682ff3d2, 0x7144c33d, 0xab8f7f09, 0xb2e44fe6, + 0x3e300cc0, 0x275b3c2f, 0xfd90801b, 0xe4fbb0f4, + 0x4907f972, 0x506cc99d, 0x8aa775a9, 0x93cc4546, + 0x21280aa0, 0x38433a4f, 0xe288867b, 0xfbe3b694, + 0x561fff12, 0x4f74cffd, 0x95bf73c9, 0x8cd44326, + 0x8d16f485, 0x947dc46a, 0x4eb6785e, 0x57dd48b1, + 0xfa210137, 0xe34a31d8, 0x39818dec, 0x20eabd03, + 0x920ef2e5, 0x8b65c20a, 0x51ae7e3e, 0x48c54ed1, + 0xe5390757, 0xfc5237b8, 0x26998b8c, 0x3ff2bb63, + 0xb326f845, 0xaa4dc8aa, 0x7086749e, 0x69ed4471, + 0xc4110df7, 0xdd7a3d18, 0x07b1812c, 0x1edab1c3, + 0xac3efe25, 0xb555ceca, 0x6f9e72fe, 0x76f54211, + 0xdb090b97, 0xc2623b78, 0x18a9874c, 0x01c2b7a3, + 0xeb5b040e, 0xf23034e1, 0x28fb88d5, 0x3190b83a, + 0x9c6cf1bc, 0x8507c153, 0x5fcc7d67, 0x46a74d88, + 0xf443026e, 0xed283281, 0x37e38eb5, 0x2e88be5a, + 0x8374f7dc, 0x9a1fc733, 0x40d47b07, 0x59bf4be8, + 0xd56b08ce, 0xcc003821, 0x16cb8415, 0x0fa0b4fa, + 0xa25cfd7c, 0xbb37cd93, 0x61fc71a7, 0x78974148, + 0xca730eae, 0xd3183e41, 0x09d38275, 0x10b8b29a, + 0xbd44fb1c, 0xa42fcbf3, 0x7ee477c7, 0x678f4728, + 0x664df08b, 0x7f26c064, 0xa5ed7c50, 0xbc864cbf, + 0x117a0539, 0x081135d6, 0xd2da89e2, 0xcbb1b90d, + 0x7955f6eb, 0x603ec604, 0xbaf57a30, 0xa39e4adf, + 0x0e620359, 0x170933b6, 0xcdc28f82, 0xd4a9bf6d, + 0x587dfc4b, 0x4116cca4, 0x9bdd7090, 0x82b6407f, + 0x2f4a09f9, 0x36213916, 0xecea8522, 0xf581b5cd, + 0x4765fa2b, 0x5e0ecac4, 0x84c576f0, 0x9dae461f, + 0x30520f99, 0x29393f76, 0xf3f28342, 0xea99b3ad, + 0xd6b7081c, 0xcfdc38f3, 0x151784c7, 0x0c7cb428, + 0xa180fdae, 0xb8ebcd41, 0x62207175, 0x7b4b419a, + 0xc9af0e7c, 0xd0c43e93, 0x0a0f82a7, 0x1364b248, + 0xbe98fbce, 0xa7f3cb21, 0x7d387715, 0x645347fa, + 0xe88704dc, 0xf1ec3433, 0x2b278807, 0x324cb8e8, + 0x9fb0f16e, 0x86dbc181, 0x5c107db5, 0x457b4d5a, + 0xf79f02bc, 0xeef43253, 0x343f8e67, 0x2d54be88, + 0x80a8f70e, 0x99c3c7e1, 0x43087bd5, 0x5a634b3a, + 0x5ba1fc99, 0x42cacc76, 0x98017042, 0x816a40ad, + 0x2c96092b, 0x35fd39c4, 0xef3685f0, 0xf65db51f, + 0x44b9faf9, 0x5dd2ca16, 0x87197622, 0x9e7246cd, + 0x338e0f4b, 0x2ae53fa4, 0xf02e8390, 0xe945b37f, + 0x6591f059, 0x7cfac0b6, 0xa6317c82, 0xbf5a4c6d, + 0x12a605eb, 0x0bcd3504, 0xd1068930, 0xc86db9df, + 0x7a89f639, 0x63e2c6d6, 0xb9297ae2, 0xa0424a0d, + 0x0dbe038b, 0x14d53364, 0xce1e8f50, 0xd775bfbf, + 0x3dec0c12, 0x24873cfd, 0xfe4c80c9, 0xe727b026, + 0x4adbf9a0, 0x53b0c94f, 0x897b757b, 0x90104594, + 0x22f40a72, 0x3b9f3a9d, 0xe15486a9, 0xf83fb646, + 0x55c3ffc0, 0x4ca8cf2f, 0x9663731b, 0x8f0843f4, + 0x03dc00d2, 0x1ab7303d, 0xc07c8c09, 0xd917bce6, + 0x74ebf560, 0x6d80c58f, 0xb74b79bb, 0xae204954, + 0x1cc406b2, 0x05af365d, 0xdf648a69, 0xc60fba86, + 0x6bf3f300, 0x7298c3ef, 0xa8537fdb, 0xb1384f34, + 0xb0faf897, 0xa991c878, 0x735a744c, 0x6a3144a3, + 0xc7cd0d25, 0xdea63dca, 0x046d81fe, 0x1d06b111, + 0xafe2fef7, 0xb689ce18, 0x6c42722c, 0x752942c3, + 0xd8d50b45, 0xc1be3baa, 0x1b75879e, 0x021eb771, + 0x8ecaf457, 0x97a1c4b8, 0x4d6a788c, 0x54014863, + 0xf9fd01e5, 0xe096310a, 0x3a5d8d3e, 0x2336bdd1, + 0x91d2f237, 0x88b9c2d8, 0x52727eec, 0x4b194e03, + 0xe6e50785, 0xff8e376a, 0x25458b5e, 0x3c2ebbb1 + },{ + 0x00000000, 0xc82c0368, 0x905906d0, 0x587505b8, + 0xd1c5e0a5, 0x19e9e3cd, 0x419ce675, 0x89b0e51d, + 0x53fd2d4e, 0x9bd12e26, 0xc3a42b9e, 0x0b8828f6, + 0x8238cdeb, 0x4a14ce83, 0x1261cb3b, 0xda4dc853, + 0xa6fa5b9c, 0x6ed658f4, 0x36a35d4c, 0xfe8f5e24, + 0x773fbb39, 0xbf13b851, 0xe766bde9, 0x2f4abe81, + 0xf50776d2, 0x3d2b75ba, 0x655e7002, 0xad72736a, + 0x24c29677, 0xecee951f, 0xb49b90a7, 0x7cb793cf, + 0xbd835b3d, 0x75af5855, 0x2dda5ded, 0xe5f65e85, + 0x6c46bb98, 0xa46ab8f0, 0xfc1fbd48, 0x3433be20, + 0xee7e7673, 0x2652751b, 0x7e2770a3, 0xb60b73cb, + 0x3fbb96d6, 0xf79795be, 0xafe29006, 0x67ce936e, + 0x1b7900a1, 0xd35503c9, 0x8b200671, 0x430c0519, + 0xcabce004, 0x0290e36c, 0x5ae5e6d4, 0x92c9e5bc, + 0x48842def, 0x80a82e87, 0xd8dd2b3f, 0x10f12857, + 0x9941cd4a, 0x516dce22, 0x0918cb9a, 0xc134c8f2, + 0x7a07b77a, 0xb22bb412, 0xea5eb1aa, 0x2272b2c2, + 0xabc257df, 0x63ee54b7, 0x3b9b510f, 0xf3b75267, + 0x29fa9a34, 0xe1d6995c, 0xb9a39ce4, 0x718f9f8c, + 0xf83f7a91, 0x301379f9, 0x68667c41, 0xa04a7f29, + 0xdcfdece6, 0x14d1ef8e, 0x4ca4ea36, 0x8488e95e, + 0x0d380c43, 0xc5140f2b, 0x9d610a93, 0x554d09fb, + 0x8f00c1a8, 0x472cc2c0, 0x1f59c778, 0xd775c410, + 0x5ec5210d, 0x96e92265, 0xce9c27dd, 0x06b024b5, + 0xc784ec47, 0x0fa8ef2f, 0x57ddea97, 0x9ff1e9ff, + 0x16410ce2, 0xde6d0f8a, 0x86180a32, 0x4e34095a, + 0x9479c109, 0x5c55c261, 0x0420c7d9, 0xcc0cc4b1, + 0x45bc21ac, 0x8d9022c4, 0xd5e5277c, 0x1dc92414, + 0x617eb7db, 0xa952b4b3, 0xf127b10b, 0x390bb263, + 0xb0bb577e, 0x78975416, 0x20e251ae, 0xe8ce52c6, + 0x32839a95, 0xfaaf99fd, 0xa2da9c45, 0x6af69f2d, + 0xe3467a30, 0x2b6a7958, 0x731f7ce0, 0xbb337f88, + 0xf40e6ef5, 0x3c226d9d, 0x64576825, 0xac7b6b4d, + 0x25cb8e50, 0xede78d38, 0xb5928880, 0x7dbe8be8, + 0xa7f343bb, 0x6fdf40d3, 0x37aa456b, 0xff864603, + 0x7636a31e, 0xbe1aa076, 0xe66fa5ce, 0x2e43a6a6, + 0x52f43569, 0x9ad83601, 0xc2ad33b9, 0x0a8130d1, + 0x8331d5cc, 0x4b1dd6a4, 0x1368d31c, 0xdb44d074, + 0x01091827, 0xc9251b4f, 0x91501ef7, 0x597c1d9f, + 0xd0ccf882, 0x18e0fbea, 0x4095fe52, 0x88b9fd3a, + 0x498d35c8, 0x81a136a0, 0xd9d43318, 0x11f83070, + 0x9848d56d, 0x5064d605, 0x0811d3bd, 0xc03dd0d5, + 0x1a701886, 0xd25c1bee, 0x8a291e56, 0x42051d3e, + 0xcbb5f823, 0x0399fb4b, 0x5becfef3, 0x93c0fd9b, + 0xef776e54, 0x275b6d3c, 0x7f2e6884, 0xb7026bec, + 0x3eb28ef1, 0xf69e8d99, 0xaeeb8821, 0x66c78b49, + 0xbc8a431a, 0x74a64072, 0x2cd345ca, 0xe4ff46a2, + 0x6d4fa3bf, 0xa563a0d7, 0xfd16a56f, 0x353aa607, + 0x8e09d98f, 0x4625dae7, 0x1e50df5f, 0xd67cdc37, + 0x5fcc392a, 0x97e03a42, 0xcf953ffa, 0x07b93c92, + 0xddf4f4c1, 0x15d8f7a9, 0x4dadf211, 0x8581f179, + 0x0c311464, 0xc41d170c, 0x9c6812b4, 0x544411dc, + 0x28f38213, 0xe0df817b, 0xb8aa84c3, 0x708687ab, + 0xf93662b6, 0x311a61de, 0x696f6466, 0xa143670e, + 0x7b0eaf5d, 0xb322ac35, 0xeb57a98d, 0x237baae5, + 0xaacb4ff8, 0x62e74c90, 0x3a924928, 0xf2be4a40, + 0x338a82b2, 0xfba681da, 0xa3d38462, 0x6bff870a, + 0xe24f6217, 0x2a63617f, 0x721664c7, 0xba3a67af, + 0x6077affc, 0xa85bac94, 0xf02ea92c, 0x3802aa44, + 0xb1b24f59, 0x799e4c31, 0x21eb4989, 0xe9c74ae1, + 0x9570d92e, 0x5d5cda46, 0x0529dffe, 0xcd05dc96, + 0x44b5398b, 0x8c993ae3, 0xd4ec3f5b, 0x1cc03c33, + 0xc68df460, 0x0ea1f708, 0x56d4f2b0, 0x9ef8f1d8, + 0x174814c5, 0xdf6417ad, 0x87111215, 0x4f3d117d + },{ + 0x00000000, 0x277d3c49, 0x4efa7892, 0x698744db, + 0x6d821d21, 0x4aff2168, 0x237865b3, 0x040559fa, + 0xda043b42, 0xfd79070b, 0x94fe43d0, 0xb3837f99, + 0xb7862663, 0x90fb1a2a, 0xf97c5ef1, 0xde0162b8, + 0xb4097684, 0x93744acd, 0xfaf30e16, 0xdd8e325f, + 0xd98b6ba5, 0xfef657ec, 0x97711337, 0xb00c2f7e, + 0x6e0d4dc6, 0x4970718f, 0x20f73554, 0x078a091d, + 0x038f50e7, 0x24f26cae, 0x4d752875, 0x6a08143c, + 0x9965000d, 0xbe183c44, 0xd79f789f, 0xf0e244d6, + 0xf4e71d2c, 0xd39a2165, 0xba1d65be, 0x9d6059f7, + 0x43613b4f, 0x641c0706, 0x0d9b43dd, 0x2ae67f94, + 0x2ee3266e, 0x099e1a27, 0x60195efc, 0x476462b5, + 0x2d6c7689, 0x0a114ac0, 0x63960e1b, 0x44eb3252, + 0x40ee6ba8, 0x679357e1, 0x0e14133a, 0x29692f73, + 0xf7684dcb, 0xd0157182, 0xb9923559, 0x9eef0910, + 0x9aea50ea, 0xbd976ca3, 0xd4102878, 0xf36d1431, + 0x32cb001a, 0x15b63c53, 0x7c317888, 0x5b4c44c1, + 0x5f491d3b, 0x78342172, 0x11b365a9, 0x36ce59e0, + 0xe8cf3b58, 0xcfb20711, 0xa63543ca, 0x81487f83, + 0x854d2679, 0xa2301a30, 0xcbb75eeb, 0xecca62a2, + 0x86c2769e, 0xa1bf4ad7, 0xc8380e0c, 0xef453245, + 0xeb406bbf, 0xcc3d57f6, 0xa5ba132d, 0x82c72f64, + 0x5cc64ddc, 0x7bbb7195, 0x123c354e, 0x35410907, + 0x314450fd, 0x16396cb4, 0x7fbe286f, 0x58c31426, + 0xabae0017, 0x8cd33c5e, 0xe5547885, 0xc22944cc, + 0xc62c1d36, 0xe151217f, 0x88d665a4, 0xafab59ed, + 0x71aa3b55, 0x56d7071c, 0x3f5043c7, 0x182d7f8e, + 0x1c282674, 0x3b551a3d, 0x52d25ee6, 0x75af62af, + 0x1fa77693, 0x38da4ada, 0x515d0e01, 0x76203248, + 0x72256bb2, 0x555857fb, 0x3cdf1320, 0x1ba22f69, + 0xc5a34dd1, 0xe2de7198, 0x8b593543, 0xac24090a, + 0xa82150f0, 0x8f5c6cb9, 0xe6db2862, 0xc1a6142b, + 0x64960134, 0x43eb3d7d, 0x2a6c79a6, 0x0d1145ef, + 0x09141c15, 0x2e69205c, 0x47ee6487, 0x609358ce, + 0xbe923a76, 0x99ef063f, 0xf06842e4, 0xd7157ead, + 0xd3102757, 0xf46d1b1e, 0x9dea5fc5, 0xba97638c, + 0xd09f77b0, 0xf7e24bf9, 0x9e650f22, 0xb918336b, + 0xbd1d6a91, 0x9a6056d8, 0xf3e71203, 0xd49a2e4a, + 0x0a9b4cf2, 0x2de670bb, 0x44613460, 0x631c0829, + 0x671951d3, 0x40646d9a, 0x29e32941, 0x0e9e1508, + 0xfdf30139, 0xda8e3d70, 0xb30979ab, 0x947445e2, + 0x90711c18, 0xb70c2051, 0xde8b648a, 0xf9f658c3, + 0x27f73a7b, 0x008a0632, 0x690d42e9, 0x4e707ea0, + 0x4a75275a, 0x6d081b13, 0x048f5fc8, 0x23f26381, + 0x49fa77bd, 0x6e874bf4, 0x07000f2f, 0x207d3366, + 0x24786a9c, 0x030556d5, 0x6a82120e, 0x4dff2e47, + 0x93fe4cff, 0xb48370b6, 0xdd04346d, 0xfa790824, + 0xfe7c51de, 0xd9016d97, 0xb086294c, 0x97fb1505, + 0x565d012e, 0x71203d67, 0x18a779bc, 0x3fda45f5, + 0x3bdf1c0f, 0x1ca22046, 0x7525649d, 0x525858d4, + 0x8c593a6c, 0xab240625, 0xc2a342fe, 0xe5de7eb7, + 0xe1db274d, 0xc6a61b04, 0xaf215fdf, 0x885c6396, + 0xe25477aa, 0xc5294be3, 0xacae0f38, 0x8bd33371, + 0x8fd66a8b, 0xa8ab56c2, 0xc12c1219, 0xe6512e50, + 0x38504ce8, 0x1f2d70a1, 0x76aa347a, 0x51d70833, + 0x55d251c9, 0x72af6d80, 0x1b28295b, 0x3c551512, + 0xcf380123, 0xe8453d6a, 0x81c279b1, 0xa6bf45f8, + 0xa2ba1c02, 0x85c7204b, 0xec406490, 0xcb3d58d9, + 0x153c3a61, 0x32410628, 0x5bc642f3, 0x7cbb7eba, + 0x78be2740, 0x5fc31b09, 0x36445fd2, 0x1139639b, + 0x7b3177a7, 0x5c4c4bee, 0x35cb0f35, 0x12b6337c, + 0x16b36a86, 0x31ce56cf, 0x58491214, 0x7f342e5d, + 0xa1354ce5, 0x864870ac, 0xefcf3477, 0xc8b2083e, + 0xccb751c4, 0xebca6d8d, 0x824d2956, 0xa530151f + } +#else + { + 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, + 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb, + 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, + 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, + 0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b, + 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, + 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, + 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, + 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, + 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, + 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, + 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, + 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, + 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a, + 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, + 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, + 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, + 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, + 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, + 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, + 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, + 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, + 0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, + 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, + 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, + 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, + 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, + 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, + 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, + 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, + 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, + 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829, + 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, + 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, + 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, + 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, + 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, + 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, + 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, + 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, + 0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652, + 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, + 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, + 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, + 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, + 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, + 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, + 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, + 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, + 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, + 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, + 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, + 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, + 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, + 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, + 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, + 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, + 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, + 0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321, + 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, + 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, + 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, + 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, + 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351 + },{ + 0x00000000, 0x13a29877, 0x274530ee, 0x34e7a899, + 0x4e8a61dc, 0x5d28f9ab, 0x69cf5132, 0x7a6dc945, + 0x9d14c3b8, 0x8eb65bcf, 0xba51f356, 0xa9f36b21, + 0xd39ea264, 0xc03c3a13, 0xf4db928a, 0xe7790afd, + 0x3fc5f181, 0x2c6769f6, 0x1880c16f, 0x0b225918, + 0x714f905d, 0x62ed082a, 0x560aa0b3, 0x45a838c4, + 0xa2d13239, 0xb173aa4e, 0x859402d7, 0x96369aa0, + 0xec5b53e5, 0xfff9cb92, 0xcb1e630b, 0xd8bcfb7c, + 0x7f8be302, 0x6c297b75, 0x58ced3ec, 0x4b6c4b9b, + 0x310182de, 0x22a31aa9, 0x1644b230, 0x05e62a47, + 0xe29f20ba, 0xf13db8cd, 0xc5da1054, 0xd6788823, + 0xac154166, 0xbfb7d911, 0x8b507188, 0x98f2e9ff, + 0x404e1283, 0x53ec8af4, 0x670b226d, 0x74a9ba1a, + 0x0ec4735f, 0x1d66eb28, 0x298143b1, 0x3a23dbc6, + 0xdd5ad13b, 0xcef8494c, 0xfa1fe1d5, 0xe9bd79a2, + 0x93d0b0e7, 0x80722890, 0xb4958009, 0xa737187e, + 0xff17c604, 0xecb55e73, 0xd852f6ea, 0xcbf06e9d, + 0xb19da7d8, 0xa23f3faf, 0x96d89736, 0x857a0f41, + 0x620305bc, 0x71a19dcb, 0x45463552, 0x56e4ad25, + 0x2c896460, 0x3f2bfc17, 0x0bcc548e, 0x186eccf9, + 0xc0d23785, 0xd370aff2, 0xe797076b, 0xf4359f1c, + 0x8e585659, 0x9dface2e, 0xa91d66b7, 0xbabffec0, + 0x5dc6f43d, 0x4e646c4a, 0x7a83c4d3, 0x69215ca4, + 0x134c95e1, 0x00ee0d96, 0x3409a50f, 0x27ab3d78, + 0x809c2506, 0x933ebd71, 0xa7d915e8, 0xb47b8d9f, + 0xce1644da, 0xddb4dcad, 0xe9537434, 0xfaf1ec43, + 0x1d88e6be, 0x0e2a7ec9, 0x3acdd650, 0x296f4e27, + 0x53028762, 0x40a01f15, 0x7447b78c, 0x67e52ffb, + 0xbf59d487, 0xacfb4cf0, 0x981ce469, 0x8bbe7c1e, + 0xf1d3b55b, 0xe2712d2c, 0xd69685b5, 0xc5341dc2, + 0x224d173f, 0x31ef8f48, 0x050827d1, 0x16aabfa6, + 0x6cc776e3, 0x7f65ee94, 0x4b82460d, 0x5820de7a, + 0xfbc3faf9, 0xe861628e, 0xdc86ca17, 0xcf245260, + 0xb5499b25, 0xa6eb0352, 0x920cabcb, 0x81ae33bc, + 0x66d73941, 0x7575a136, 0x419209af, 0x523091d8, + 0x285d589d, 0x3bffc0ea, 0x0f186873, 0x1cbaf004, + 0xc4060b78, 0xd7a4930f, 0xe3433b96, 0xf0e1a3e1, + 0x8a8c6aa4, 0x992ef2d3, 0xadc95a4a, 0xbe6bc23d, + 0x5912c8c0, 0x4ab050b7, 0x7e57f82e, 0x6df56059, + 0x1798a91c, 0x043a316b, 0x30dd99f2, 0x237f0185, + 0x844819fb, 0x97ea818c, 0xa30d2915, 0xb0afb162, + 0xcac27827, 0xd960e050, 0xed8748c9, 0xfe25d0be, + 0x195cda43, 0x0afe4234, 0x3e19eaad, 0x2dbb72da, + 0x57d6bb9f, 0x447423e8, 0x70938b71, 0x63311306, + 0xbb8de87a, 0xa82f700d, 0x9cc8d894, 0x8f6a40e3, + 0xf50789a6, 0xe6a511d1, 0xd242b948, 0xc1e0213f, + 0x26992bc2, 0x353bb3b5, 0x01dc1b2c, 0x127e835b, + 0x68134a1e, 0x7bb1d269, 0x4f567af0, 0x5cf4e287, + 0x04d43cfd, 0x1776a48a, 0x23910c13, 0x30339464, + 0x4a5e5d21, 0x59fcc556, 0x6d1b6dcf, 0x7eb9f5b8, + 0x99c0ff45, 0x8a626732, 0xbe85cfab, 0xad2757dc, + 0xd74a9e99, 0xc4e806ee, 0xf00fae77, 0xe3ad3600, + 0x3b11cd7c, 0x28b3550b, 0x1c54fd92, 0x0ff665e5, + 0x759baca0, 0x663934d7, 0x52de9c4e, 0x417c0439, + 0xa6050ec4, 0xb5a796b3, 0x81403e2a, 0x92e2a65d, + 0xe88f6f18, 0xfb2df76f, 0xcfca5ff6, 0xdc68c781, + 0x7b5fdfff, 0x68fd4788, 0x5c1aef11, 0x4fb87766, + 0x35d5be23, 0x26772654, 0x12908ecd, 0x013216ba, + 0xe64b1c47, 0xf5e98430, 0xc10e2ca9, 0xd2acb4de, + 0xa8c17d9b, 0xbb63e5ec, 0x8f844d75, 0x9c26d502, + 0x449a2e7e, 0x5738b609, 0x63df1e90, 0x707d86e7, + 0x0a104fa2, 0x19b2d7d5, 0x2d557f4c, 0x3ef7e73b, + 0xd98eedc6, 0xca2c75b1, 0xfecbdd28, 0xed69455f, + 0x97048c1a, 0x84a6146d, 0xb041bcf4, 0xa3e32483 + },{ + 0x00000000, 0xa541927e, 0x4f6f520d, 0xea2ec073, + 0x9edea41a, 0x3b9f3664, 0xd1b1f617, 0x74f06469, + 0x38513ec5, 0x9d10acbb, 0x773e6cc8, 0xd27ffeb6, + 0xa68f9adf, 0x03ce08a1, 0xe9e0c8d2, 0x4ca15aac, + 0x70a27d8a, 0xd5e3eff4, 0x3fcd2f87, 0x9a8cbdf9, + 0xee7cd990, 0x4b3d4bee, 0xa1138b9d, 0x045219e3, + 0x48f3434f, 0xedb2d131, 0x079c1142, 0xa2dd833c, + 0xd62de755, 0x736c752b, 0x9942b558, 0x3c032726, + 0xe144fb14, 0x4405696a, 0xae2ba919, 0x0b6a3b67, + 0x7f9a5f0e, 0xdadbcd70, 0x30f50d03, 0x95b49f7d, + 0xd915c5d1, 0x7c5457af, 0x967a97dc, 0x333b05a2, + 0x47cb61cb, 0xe28af3b5, 0x08a433c6, 0xade5a1b8, + 0x91e6869e, 0x34a714e0, 0xde89d493, 0x7bc846ed, + 0x0f382284, 0xaa79b0fa, 0x40577089, 0xe516e2f7, + 0xa9b7b85b, 0x0cf62a25, 0xe6d8ea56, 0x43997828, + 0x37691c41, 0x92288e3f, 0x78064e4c, 0xdd47dc32, + 0xc76580d9, 0x622412a7, 0x880ad2d4, 0x2d4b40aa, + 0x59bb24c3, 0xfcfab6bd, 0x16d476ce, 0xb395e4b0, + 0xff34be1c, 0x5a752c62, 0xb05bec11, 0x151a7e6f, + 0x61ea1a06, 0xc4ab8878, 0x2e85480b, 0x8bc4da75, + 0xb7c7fd53, 0x12866f2d, 0xf8a8af5e, 0x5de93d20, + 0x29195949, 0x8c58cb37, 0x66760b44, 0xc337993a, + 0x8f96c396, 0x2ad751e8, 0xc0f9919b, 0x65b803e5, + 0x1148678c, 0xb409f5f2, 0x5e273581, 0xfb66a7ff, + 0x26217bcd, 0x8360e9b3, 0x694e29c0, 0xcc0fbbbe, + 0xb8ffdfd7, 0x1dbe4da9, 0xf7908dda, 0x52d11fa4, + 0x1e704508, 0xbb31d776, 0x511f1705, 0xf45e857b, + 0x80aee112, 0x25ef736c, 0xcfc1b31f, 0x6a802161, + 0x56830647, 0xf3c29439, 0x19ec544a, 0xbcadc634, + 0xc85da25d, 0x6d1c3023, 0x8732f050, 0x2273622e, + 0x6ed23882, 0xcb93aafc, 0x21bd6a8f, 0x84fcf8f1, + 0xf00c9c98, 0x554d0ee6, 0xbf63ce95, 0x1a225ceb, + 0x8b277743, 0x2e66e53d, 0xc448254e, 0x6109b730, + 0x15f9d359, 0xb0b84127, 0x5a968154, 0xffd7132a, + 0xb3764986, 0x1637dbf8, 0xfc191b8b, 0x595889f5, + 0x2da8ed9c, 0x88e97fe2, 0x62c7bf91, 0xc7862def, + 0xfb850ac9, 0x5ec498b7, 0xb4ea58c4, 0x11abcaba, + 0x655baed3, 0xc01a3cad, 0x2a34fcde, 0x8f756ea0, + 0xc3d4340c, 0x6695a672, 0x8cbb6601, 0x29faf47f, + 0x5d0a9016, 0xf84b0268, 0x1265c21b, 0xb7245065, + 0x6a638c57, 0xcf221e29, 0x250cde5a, 0x804d4c24, + 0xf4bd284d, 0x51fcba33, 0xbbd27a40, 0x1e93e83e, + 0x5232b292, 0xf77320ec, 0x1d5de09f, 0xb81c72e1, + 0xccec1688, 0x69ad84f6, 0x83834485, 0x26c2d6fb, + 0x1ac1f1dd, 0xbf8063a3, 0x55aea3d0, 0xf0ef31ae, + 0x841f55c7, 0x215ec7b9, 0xcb7007ca, 0x6e3195b4, + 0x2290cf18, 0x87d15d66, 0x6dff9d15, 0xc8be0f6b, + 0xbc4e6b02, 0x190ff97c, 0xf321390f, 0x5660ab71, + 0x4c42f79a, 0xe90365e4, 0x032da597, 0xa66c37e9, + 0xd29c5380, 0x77ddc1fe, 0x9df3018d, 0x38b293f3, + 0x7413c95f, 0xd1525b21, 0x3b7c9b52, 0x9e3d092c, + 0xeacd6d45, 0x4f8cff3b, 0xa5a23f48, 0x00e3ad36, + 0x3ce08a10, 0x99a1186e, 0x738fd81d, 0xd6ce4a63, + 0xa23e2e0a, 0x077fbc74, 0xed517c07, 0x4810ee79, + 0x04b1b4d5, 0xa1f026ab, 0x4bdee6d8, 0xee9f74a6, + 0x9a6f10cf, 0x3f2e82b1, 0xd50042c2, 0x7041d0bc, + 0xad060c8e, 0x08479ef0, 0xe2695e83, 0x4728ccfd, + 0x33d8a894, 0x96993aea, 0x7cb7fa99, 0xd9f668e7, + 0x9557324b, 0x3016a035, 0xda386046, 0x7f79f238, + 0x0b899651, 0xaec8042f, 0x44e6c45c, 0xe1a75622, + 0xdda47104, 0x78e5e37a, 0x92cb2309, 0x378ab177, + 0x437ad51e, 0xe63b4760, 0x0c158713, 0xa954156d, + 0xe5f54fc1, 0x40b4ddbf, 0xaa9a1dcc, 0x0fdb8fb2, + 0x7b2bebdb, 0xde6a79a5, 0x3444b9d6, 0x91052ba8 + },{ + 0x00000000, 0xdd45aab8, 0xbf672381, 0x62228939, + 0x7b2231f3, 0xa6679b4b, 0xc4451272, 0x1900b8ca, + 0xf64463e6, 0x2b01c95e, 0x49234067, 0x9466eadf, + 0x8d665215, 0x5023f8ad, 0x32017194, 0xef44db2c, + 0xe964b13d, 0x34211b85, 0x560392bc, 0x8b463804, + 0x924680ce, 0x4f032a76, 0x2d21a34f, 0xf06409f7, + 0x1f20d2db, 0xc2657863, 0xa047f15a, 0x7d025be2, + 0x6402e328, 0xb9474990, 0xdb65c0a9, 0x06206a11, + 0xd725148b, 0x0a60be33, 0x6842370a, 0xb5079db2, + 0xac072578, 0x71428fc0, 0x136006f9, 0xce25ac41, + 0x2161776d, 0xfc24ddd5, 0x9e0654ec, 0x4343fe54, + 0x5a43469e, 0x8706ec26, 0xe524651f, 0x3861cfa7, + 0x3e41a5b6, 0xe3040f0e, 0x81268637, 0x5c632c8f, + 0x45639445, 0x98263efd, 0xfa04b7c4, 0x27411d7c, + 0xc805c650, 0x15406ce8, 0x7762e5d1, 0xaa274f69, + 0xb327f7a3, 0x6e625d1b, 0x0c40d422, 0xd1057e9a, + 0xaba65fe7, 0x76e3f55f, 0x14c17c66, 0xc984d6de, + 0xd0846e14, 0x0dc1c4ac, 0x6fe34d95, 0xb2a6e72d, + 0x5de23c01, 0x80a796b9, 0xe2851f80, 0x3fc0b538, + 0x26c00df2, 0xfb85a74a, 0x99a72e73, 0x44e284cb, + 0x42c2eeda, 0x9f874462, 0xfda5cd5b, 0x20e067e3, + 0x39e0df29, 0xe4a57591, 0x8687fca8, 0x5bc25610, + 0xb4868d3c, 0x69c32784, 0x0be1aebd, 0xd6a40405, + 0xcfa4bccf, 0x12e11677, 0x70c39f4e, 0xad8635f6, + 0x7c834b6c, 0xa1c6e1d4, 0xc3e468ed, 0x1ea1c255, + 0x07a17a9f, 0xdae4d027, 0xb8c6591e, 0x6583f3a6, + 0x8ac7288a, 0x57828232, 0x35a00b0b, 0xe8e5a1b3, + 0xf1e51979, 0x2ca0b3c1, 0x4e823af8, 0x93c79040, + 0x95e7fa51, 0x48a250e9, 0x2a80d9d0, 0xf7c57368, + 0xeec5cba2, 0x3380611a, 0x51a2e823, 0x8ce7429b, + 0x63a399b7, 0xbee6330f, 0xdcc4ba36, 0x0181108e, + 0x1881a844, 0xc5c402fc, 0xa7e68bc5, 0x7aa3217d, + 0x52a0c93f, 0x8fe56387, 0xedc7eabe, 0x30824006, + 0x2982f8cc, 0xf4c75274, 0x96e5db4d, 0x4ba071f5, + 0xa4e4aad9, 0x79a10061, 0x1b838958, 0xc6c623e0, + 0xdfc69b2a, 0x02833192, 0x60a1b8ab, 0xbde41213, + 0xbbc47802, 0x6681d2ba, 0x04a35b83, 0xd9e6f13b, + 0xc0e649f1, 0x1da3e349, 0x7f816a70, 0xa2c4c0c8, + 0x4d801be4, 0x90c5b15c, 0xf2e73865, 0x2fa292dd, + 0x36a22a17, 0xebe780af, 0x89c50996, 0x5480a32e, + 0x8585ddb4, 0x58c0770c, 0x3ae2fe35, 0xe7a7548d, + 0xfea7ec47, 0x23e246ff, 0x41c0cfc6, 0x9c85657e, + 0x73c1be52, 0xae8414ea, 0xcca69dd3, 0x11e3376b, + 0x08e38fa1, 0xd5a62519, 0xb784ac20, 0x6ac10698, + 0x6ce16c89, 0xb1a4c631, 0xd3864f08, 0x0ec3e5b0, + 0x17c35d7a, 0xca86f7c2, 0xa8a47efb, 0x75e1d443, + 0x9aa50f6f, 0x47e0a5d7, 0x25c22cee, 0xf8878656, + 0xe1873e9c, 0x3cc29424, 0x5ee01d1d, 0x83a5b7a5, + 0xf90696d8, 0x24433c60, 0x4661b559, 0x9b241fe1, + 0x8224a72b, 0x5f610d93, 0x3d4384aa, 0xe0062e12, + 0x0f42f53e, 0xd2075f86, 0xb025d6bf, 0x6d607c07, + 0x7460c4cd, 0xa9256e75, 0xcb07e74c, 0x16424df4, + 0x106227e5, 0xcd278d5d, 0xaf050464, 0x7240aedc, + 0x6b401616, 0xb605bcae, 0xd4273597, 0x09629f2f, + 0xe6264403, 0x3b63eebb, 0x59416782, 0x8404cd3a, + 0x9d0475f0, 0x4041df48, 0x22635671, 0xff26fcc9, + 0x2e238253, 0xf36628eb, 0x9144a1d2, 0x4c010b6a, + 0x5501b3a0, 0x88441918, 0xea669021, 0x37233a99, + 0xd867e1b5, 0x05224b0d, 0x6700c234, 0xba45688c, + 0xa345d046, 0x7e007afe, 0x1c22f3c7, 0xc167597f, + 0xc747336e, 0x1a0299d6, 0x782010ef, 0xa565ba57, + 0xbc65029d, 0x6120a825, 0x0302211c, 0xde478ba4, + 0x31035088, 0xec46fa30, 0x8e647309, 0x5321d9b1, + 0x4a21617b, 0x9764cbc3, 0xf54642fa, 0x2803e842 + },{ + 0x00000000, 0x38116fac, 0x7022df58, 0x4833b0f4, + 0xe045beb0, 0xd854d11c, 0x906761e8, 0xa8760e44, + 0xc5670b91, 0xfd76643d, 0xb545d4c9, 0x8d54bb65, + 0x2522b521, 0x1d33da8d, 0x55006a79, 0x6d1105d5, + 0x8f2261d3, 0xb7330e7f, 0xff00be8b, 0xc711d127, + 0x6f67df63, 0x5776b0cf, 0x1f45003b, 0x27546f97, + 0x4a456a42, 0x725405ee, 0x3a67b51a, 0x0276dab6, + 0xaa00d4f2, 0x9211bb5e, 0xda220baa, 0xe2336406, + 0x1ba8b557, 0x23b9dafb, 0x6b8a6a0f, 0x539b05a3, + 0xfbed0be7, 0xc3fc644b, 0x8bcfd4bf, 0xb3debb13, + 0xdecfbec6, 0xe6ded16a, 0xaeed619e, 0x96fc0e32, + 0x3e8a0076, 0x069b6fda, 0x4ea8df2e, 0x76b9b082, + 0x948ad484, 0xac9bbb28, 0xe4a80bdc, 0xdcb96470, + 0x74cf6a34, 0x4cde0598, 0x04edb56c, 0x3cfcdac0, + 0x51eddf15, 0x69fcb0b9, 0x21cf004d, 0x19de6fe1, + 0xb1a861a5, 0x89b90e09, 0xc18abefd, 0xf99bd151, + 0x37516aae, 0x0f400502, 0x4773b5f6, 0x7f62da5a, + 0xd714d41e, 0xef05bbb2, 0xa7360b46, 0x9f2764ea, + 0xf236613f, 0xca270e93, 0x8214be67, 0xba05d1cb, + 0x1273df8f, 0x2a62b023, 0x625100d7, 0x5a406f7b, + 0xb8730b7d, 0x806264d1, 0xc851d425, 0xf040bb89, + 0x5836b5cd, 0x6027da61, 0x28146a95, 0x10050539, + 0x7d1400ec, 0x45056f40, 0x0d36dfb4, 0x3527b018, + 0x9d51be5c, 0xa540d1f0, 0xed736104, 0xd5620ea8, + 0x2cf9dff9, 0x14e8b055, 0x5cdb00a1, 0x64ca6f0d, + 0xccbc6149, 0xf4ad0ee5, 0xbc9ebe11, 0x848fd1bd, + 0xe99ed468, 0xd18fbbc4, 0x99bc0b30, 0xa1ad649c, + 0x09db6ad8, 0x31ca0574, 0x79f9b580, 0x41e8da2c, + 0xa3dbbe2a, 0x9bcad186, 0xd3f96172, 0xebe80ede, + 0x439e009a, 0x7b8f6f36, 0x33bcdfc2, 0x0badb06e, + 0x66bcb5bb, 0x5eadda17, 0x169e6ae3, 0x2e8f054f, + 0x86f90b0b, 0xbee864a7, 0xf6dbd453, 0xcecabbff, + 0x6ea2d55c, 0x56b3baf0, 0x1e800a04, 0x269165a8, + 0x8ee76bec, 0xb6f60440, 0xfec5b4b4, 0xc6d4db18, + 0xabc5decd, 0x93d4b161, 0xdbe70195, 0xe3f66e39, + 0x4b80607d, 0x73910fd1, 0x3ba2bf25, 0x03b3d089, + 0xe180b48f, 0xd991db23, 0x91a26bd7, 0xa9b3047b, + 0x01c50a3f, 0x39d46593, 0x71e7d567, 0x49f6bacb, + 0x24e7bf1e, 0x1cf6d0b2, 0x54c56046, 0x6cd40fea, + 0xc4a201ae, 0xfcb36e02, 0xb480def6, 0x8c91b15a, + 0x750a600b, 0x4d1b0fa7, 0x0528bf53, 0x3d39d0ff, + 0x954fdebb, 0xad5eb117, 0xe56d01e3, 0xdd7c6e4f, + 0xb06d6b9a, 0x887c0436, 0xc04fb4c2, 0xf85edb6e, + 0x5028d52a, 0x6839ba86, 0x200a0a72, 0x181b65de, + 0xfa2801d8, 0xc2396e74, 0x8a0ade80, 0xb21bb12c, + 0x1a6dbf68, 0x227cd0c4, 0x6a4f6030, 0x525e0f9c, + 0x3f4f0a49, 0x075e65e5, 0x4f6dd511, 0x777cbabd, + 0xdf0ab4f9, 0xe71bdb55, 0xaf286ba1, 0x9739040d, + 0x59f3bff2, 0x61e2d05e, 0x29d160aa, 0x11c00f06, + 0xb9b60142, 0x81a76eee, 0xc994de1a, 0xf185b1b6, + 0x9c94b463, 0xa485dbcf, 0xecb66b3b, 0xd4a70497, + 0x7cd10ad3, 0x44c0657f, 0x0cf3d58b, 0x34e2ba27, + 0xd6d1de21, 0xeec0b18d, 0xa6f30179, 0x9ee26ed5, + 0x36946091, 0x0e850f3d, 0x46b6bfc9, 0x7ea7d065, + 0x13b6d5b0, 0x2ba7ba1c, 0x63940ae8, 0x5b856544, + 0xf3f36b00, 0xcbe204ac, 0x83d1b458, 0xbbc0dbf4, + 0x425b0aa5, 0x7a4a6509, 0x3279d5fd, 0x0a68ba51, + 0xa21eb415, 0x9a0fdbb9, 0xd23c6b4d, 0xea2d04e1, + 0x873c0134, 0xbf2d6e98, 0xf71ede6c, 0xcf0fb1c0, + 0x6779bf84, 0x5f68d028, 0x175b60dc, 0x2f4a0f70, + 0xcd796b76, 0xf56804da, 0xbd5bb42e, 0x854adb82, + 0x2d3cd5c6, 0x152dba6a, 0x5d1e0a9e, 0x650f6532, + 0x081e60e7, 0x300f0f4b, 0x783cbfbf, 0x402dd013, + 0xe85bde57, 0xd04ab1fb, 0x9879010f, 0xa0686ea3 + },{ + 0x00000000, 0xef306b19, 0xdb8ca0c3, 0x34bccbda, + 0xb2f53777, 0x5dc55c6e, 0x697997b4, 0x8649fcad, + 0x6006181f, 0x8f367306, 0xbb8ab8dc, 0x54bad3c5, + 0xd2f32f68, 0x3dc34471, 0x097f8fab, 0xe64fe4b2, + 0xc00c303e, 0x2f3c5b27, 0x1b8090fd, 0xf4b0fbe4, + 0x72f90749, 0x9dc96c50, 0xa975a78a, 0x4645cc93, + 0xa00a2821, 0x4f3a4338, 0x7b8688e2, 0x94b6e3fb, + 0x12ff1f56, 0xfdcf744f, 0xc973bf95, 0x2643d48c, + 0x85f4168d, 0x6ac47d94, 0x5e78b64e, 0xb148dd57, + 0x370121fa, 0xd8314ae3, 0xec8d8139, 0x03bdea20, + 0xe5f20e92, 0x0ac2658b, 0x3e7eae51, 0xd14ec548, + 0x570739e5, 0xb83752fc, 0x8c8b9926, 0x63bbf23f, + 0x45f826b3, 0xaac84daa, 0x9e748670, 0x7144ed69, + 0xf70d11c4, 0x183d7add, 0x2c81b107, 0xc3b1da1e, + 0x25fe3eac, 0xcace55b5, 0xfe729e6f, 0x1142f576, + 0x970b09db, 0x783b62c2, 0x4c87a918, 0xa3b7c201, + 0x0e045beb, 0xe13430f2, 0xd588fb28, 0x3ab89031, + 0xbcf16c9c, 0x53c10785, 0x677dcc5f, 0x884da746, + 0x6e0243f4, 0x813228ed, 0xb58ee337, 0x5abe882e, + 0xdcf77483, 0x33c71f9a, 0x077bd440, 0xe84bbf59, + 0xce086bd5, 0x213800cc, 0x1584cb16, 0xfab4a00f, + 0x7cfd5ca2, 0x93cd37bb, 0xa771fc61, 0x48419778, + 0xae0e73ca, 0x413e18d3, 0x7582d309, 0x9ab2b810, + 0x1cfb44bd, 0xf3cb2fa4, 0xc777e47e, 0x28478f67, + 0x8bf04d66, 0x64c0267f, 0x507ceda5, 0xbf4c86bc, + 0x39057a11, 0xd6351108, 0xe289dad2, 0x0db9b1cb, + 0xebf65579, 0x04c63e60, 0x307af5ba, 0xdf4a9ea3, + 0x5903620e, 0xb6330917, 0x828fc2cd, 0x6dbfa9d4, + 0x4bfc7d58, 0xa4cc1641, 0x9070dd9b, 0x7f40b682, + 0xf9094a2f, 0x16392136, 0x2285eaec, 0xcdb581f5, + 0x2bfa6547, 0xc4ca0e5e, 0xf076c584, 0x1f46ae9d, + 0x990f5230, 0x763f3929, 0x4283f2f3, 0xadb399ea, + 0x1c08b7d6, 0xf338dccf, 0xc7841715, 0x28b47c0c, + 0xaefd80a1, 0x41cdebb8, 0x75712062, 0x9a414b7b, + 0x7c0eafc9, 0x933ec4d0, 0xa7820f0a, 0x48b26413, + 0xcefb98be, 0x21cbf3a7, 0x1577387d, 0xfa475364, + 0xdc0487e8, 0x3334ecf1, 0x0788272b, 0xe8b84c32, + 0x6ef1b09f, 0x81c1db86, 0xb57d105c, 0x5a4d7b45, + 0xbc029ff7, 0x5332f4ee, 0x678e3f34, 0x88be542d, + 0x0ef7a880, 0xe1c7c399, 0xd57b0843, 0x3a4b635a, + 0x99fca15b, 0x76ccca42, 0x42700198, 0xad406a81, + 0x2b09962c, 0xc439fd35, 0xf08536ef, 0x1fb55df6, + 0xf9fab944, 0x16cad25d, 0x22761987, 0xcd46729e, + 0x4b0f8e33, 0xa43fe52a, 0x90832ef0, 0x7fb345e9, + 0x59f09165, 0xb6c0fa7c, 0x827c31a6, 0x6d4c5abf, + 0xeb05a612, 0x0435cd0b, 0x308906d1, 0xdfb96dc8, + 0x39f6897a, 0xd6c6e263, 0xe27a29b9, 0x0d4a42a0, + 0x8b03be0d, 0x6433d514, 0x508f1ece, 0xbfbf75d7, + 0x120cec3d, 0xfd3c8724, 0xc9804cfe, 0x26b027e7, + 0xa0f9db4a, 0x4fc9b053, 0x7b757b89, 0x94451090, + 0x720af422, 0x9d3a9f3b, 0xa98654e1, 0x46b63ff8, + 0xc0ffc355, 0x2fcfa84c, 0x1b736396, 0xf443088f, + 0xd200dc03, 0x3d30b71a, 0x098c7cc0, 0xe6bc17d9, + 0x60f5eb74, 0x8fc5806d, 0xbb794bb7, 0x544920ae, + 0xb206c41c, 0x5d36af05, 0x698a64df, 0x86ba0fc6, + 0x00f3f36b, 0xefc39872, 0xdb7f53a8, 0x344f38b1, + 0x97f8fab0, 0x78c891a9, 0x4c745a73, 0xa344316a, + 0x250dcdc7, 0xca3da6de, 0xfe816d04, 0x11b1061d, + 0xf7fee2af, 0x18ce89b6, 0x2c72426c, 0xc3422975, + 0x450bd5d8, 0xaa3bbec1, 0x9e87751b, 0x71b71e02, + 0x57f4ca8e, 0xb8c4a197, 0x8c786a4d, 0x63480154, + 0xe501fdf9, 0x0a3196e0, 0x3e8d5d3a, 0xd1bd3623, + 0x37f2d291, 0xd8c2b988, 0xec7e7252, 0x034e194b, + 0x8507e5e6, 0x6a378eff, 0x5e8b4525, 0xb1bb2e3c + },{ + 0x00000000, 0x68032cc8, 0xd0065990, 0xb8057558, + 0xa5e0c5d1, 0xcde3e919, 0x75e69c41, 0x1de5b089, + 0x4e2dfd53, 0x262ed19b, 0x9e2ba4c3, 0xf628880b, + 0xebcd3882, 0x83ce144a, 0x3bcb6112, 0x53c84dda, + 0x9c5bfaa6, 0xf458d66e, 0x4c5da336, 0x245e8ffe, + 0x39bb3f77, 0x51b813bf, 0xe9bd66e7, 0x81be4a2f, + 0xd27607f5, 0xba752b3d, 0x02705e65, 0x6a7372ad, + 0x7796c224, 0x1f95eeec, 0xa7909bb4, 0xcf93b77c, + 0x3d5b83bd, 0x5558af75, 0xed5dda2d, 0x855ef6e5, + 0x98bb466c, 0xf0b86aa4, 0x48bd1ffc, 0x20be3334, + 0x73767eee, 0x1b755226, 0xa370277e, 0xcb730bb6, + 0xd696bb3f, 0xbe9597f7, 0x0690e2af, 0x6e93ce67, + 0xa100791b, 0xc90355d3, 0x7106208b, 0x19050c43, + 0x04e0bcca, 0x6ce39002, 0xd4e6e55a, 0xbce5c992, + 0xef2d8448, 0x872ea880, 0x3f2bddd8, 0x5728f110, + 0x4acd4199, 0x22ce6d51, 0x9acb1809, 0xf2c834c1, + 0x7ab7077a, 0x12b42bb2, 0xaab15eea, 0xc2b27222, + 0xdf57c2ab, 0xb754ee63, 0x0f519b3b, 0x6752b7f3, + 0x349afa29, 0x5c99d6e1, 0xe49ca3b9, 0x8c9f8f71, + 0x917a3ff8, 0xf9791330, 0x417c6668, 0x297f4aa0, + 0xe6ecfddc, 0x8eefd114, 0x36eaa44c, 0x5ee98884, + 0x430c380d, 0x2b0f14c5, 0x930a619d, 0xfb094d55, + 0xa8c1008f, 0xc0c22c47, 0x78c7591f, 0x10c475d7, + 0x0d21c55e, 0x6522e996, 0xdd279cce, 0xb524b006, + 0x47ec84c7, 0x2fefa80f, 0x97eadd57, 0xffe9f19f, + 0xe20c4116, 0x8a0f6dde, 0x320a1886, 0x5a09344e, + 0x09c17994, 0x61c2555c, 0xd9c72004, 0xb1c40ccc, + 0xac21bc45, 0xc422908d, 0x7c27e5d5, 0x1424c91d, + 0xdbb77e61, 0xb3b452a9, 0x0bb127f1, 0x63b20b39, + 0x7e57bbb0, 0x16549778, 0xae51e220, 0xc652cee8, + 0x959a8332, 0xfd99affa, 0x459cdaa2, 0x2d9ff66a, + 0x307a46e3, 0x58796a2b, 0xe07c1f73, 0x887f33bb, + 0xf56e0ef4, 0x9d6d223c, 0x25685764, 0x4d6b7bac, + 0x508ecb25, 0x388de7ed, 0x808892b5, 0xe88bbe7d, + 0xbb43f3a7, 0xd340df6f, 0x6b45aa37, 0x034686ff, + 0x1ea33676, 0x76a01abe, 0xcea56fe6, 0xa6a6432e, + 0x6935f452, 0x0136d89a, 0xb933adc2, 0xd130810a, + 0xccd53183, 0xa4d61d4b, 0x1cd36813, 0x74d044db, + 0x27180901, 0x4f1b25c9, 0xf71e5091, 0x9f1d7c59, + 0x82f8ccd0, 0xeafbe018, 0x52fe9540, 0x3afdb988, + 0xc8358d49, 0xa036a181, 0x1833d4d9, 0x7030f811, + 0x6dd54898, 0x05d66450, 0xbdd31108, 0xd5d03dc0, + 0x8618701a, 0xee1b5cd2, 0x561e298a, 0x3e1d0542, + 0x23f8b5cb, 0x4bfb9903, 0xf3feec5b, 0x9bfdc093, + 0x546e77ef, 0x3c6d5b27, 0x84682e7f, 0xec6b02b7, + 0xf18eb23e, 0x998d9ef6, 0x2188ebae, 0x498bc766, + 0x1a438abc, 0x7240a674, 0xca45d32c, 0xa246ffe4, + 0xbfa34f6d, 0xd7a063a5, 0x6fa516fd, 0x07a63a35, + 0x8fd9098e, 0xe7da2546, 0x5fdf501e, 0x37dc7cd6, + 0x2a39cc5f, 0x423ae097, 0xfa3f95cf, 0x923cb907, + 0xc1f4f4dd, 0xa9f7d815, 0x11f2ad4d, 0x79f18185, + 0x6414310c, 0x0c171dc4, 0xb412689c, 0xdc114454, + 0x1382f328, 0x7b81dfe0, 0xc384aab8, 0xab878670, + 0xb66236f9, 0xde611a31, 0x66646f69, 0x0e6743a1, + 0x5daf0e7b, 0x35ac22b3, 0x8da957eb, 0xe5aa7b23, + 0xf84fcbaa, 0x904ce762, 0x2849923a, 0x404abef2, + 0xb2828a33, 0xda81a6fb, 0x6284d3a3, 0x0a87ff6b, + 0x17624fe2, 0x7f61632a, 0xc7641672, 0xaf673aba, + 0xfcaf7760, 0x94ac5ba8, 0x2ca92ef0, 0x44aa0238, + 0x594fb2b1, 0x314c9e79, 0x8949eb21, 0xe14ac7e9, + 0x2ed97095, 0x46da5c5d, 0xfedf2905, 0x96dc05cd, + 0x8b39b544, 0xe33a998c, 0x5b3fecd4, 0x333cc01c, + 0x60f48dc6, 0x08f7a10e, 0xb0f2d456, 0xd8f1f89e, + 0xc5144817, 0xad1764df, 0x15121187, 0x7d113d4f + },{ + 0x00000000, 0x493c7d27, 0x9278fa4e, 0xdb448769, + 0x211d826d, 0x6821ff4a, 0xb3657823, 0xfa590504, + 0x423b04da, 0x0b0779fd, 0xd043fe94, 0x997f83b3, + 0x632686b7, 0x2a1afb90, 0xf15e7cf9, 0xb86201de, + 0x847609b4, 0xcd4a7493, 0x160ef3fa, 0x5f328edd, + 0xa56b8bd9, 0xec57f6fe, 0x37137197, 0x7e2f0cb0, + 0xc64d0d6e, 0x8f717049, 0x5435f720, 0x1d098a07, + 0xe7508f03, 0xae6cf224, 0x7528754d, 0x3c14086a, + 0x0d006599, 0x443c18be, 0x9f789fd7, 0xd644e2f0, + 0x2c1de7f4, 0x65219ad3, 0xbe651dba, 0xf759609d, + 0x4f3b6143, 0x06071c64, 0xdd439b0d, 0x947fe62a, + 0x6e26e32e, 0x271a9e09, 0xfc5e1960, 0xb5626447, + 0x89766c2d, 0xc04a110a, 0x1b0e9663, 0x5232eb44, + 0xa86bee40, 0xe1579367, 0x3a13140e, 0x732f6929, + 0xcb4d68f7, 0x827115d0, 0x593592b9, 0x1009ef9e, + 0xea50ea9a, 0xa36c97bd, 0x782810d4, 0x31146df3, + 0x1a00cb32, 0x533cb615, 0x8878317c, 0xc1444c5b, + 0x3b1d495f, 0x72213478, 0xa965b311, 0xe059ce36, + 0x583bcfe8, 0x1107b2cf, 0xca4335a6, 0x837f4881, + 0x79264d85, 0x301a30a2, 0xeb5eb7cb, 0xa262caec, + 0x9e76c286, 0xd74abfa1, 0x0c0e38c8, 0x453245ef, + 0xbf6b40eb, 0xf6573dcc, 0x2d13baa5, 0x642fc782, + 0xdc4dc65c, 0x9571bb7b, 0x4e353c12, 0x07094135, + 0xfd504431, 0xb46c3916, 0x6f28be7f, 0x2614c358, + 0x1700aeab, 0x5e3cd38c, 0x857854e5, 0xcc4429c2, + 0x361d2cc6, 0x7f2151e1, 0xa465d688, 0xed59abaf, + 0x553baa71, 0x1c07d756, 0xc743503f, 0x8e7f2d18, + 0x7426281c, 0x3d1a553b, 0xe65ed252, 0xaf62af75, + 0x9376a71f, 0xda4ada38, 0x010e5d51, 0x48322076, + 0xb26b2572, 0xfb575855, 0x2013df3c, 0x692fa21b, + 0xd14da3c5, 0x9871dee2, 0x4335598b, 0x0a0924ac, + 0xf05021a8, 0xb96c5c8f, 0x6228dbe6, 0x2b14a6c1, + 0x34019664, 0x7d3deb43, 0xa6796c2a, 0xef45110d, + 0x151c1409, 0x5c20692e, 0x8764ee47, 0xce589360, + 0x763a92be, 0x3f06ef99, 0xe44268f0, 0xad7e15d7, + 0x572710d3, 0x1e1b6df4, 0xc55fea9d, 0x8c6397ba, + 0xb0779fd0, 0xf94be2f7, 0x220f659e, 0x6b3318b9, + 0x916a1dbd, 0xd856609a, 0x0312e7f3, 0x4a2e9ad4, + 0xf24c9b0a, 0xbb70e62d, 0x60346144, 0x29081c63, + 0xd3511967, 0x9a6d6440, 0x4129e329, 0x08159e0e, + 0x3901f3fd, 0x703d8eda, 0xab7909b3, 0xe2457494, + 0x181c7190, 0x51200cb7, 0x8a648bde, 0xc358f6f9, + 0x7b3af727, 0x32068a00, 0xe9420d69, 0xa07e704e, + 0x5a27754a, 0x131b086d, 0xc85f8f04, 0x8163f223, + 0xbd77fa49, 0xf44b876e, 0x2f0f0007, 0x66337d20, + 0x9c6a7824, 0xd5560503, 0x0e12826a, 0x472eff4d, + 0xff4cfe93, 0xb67083b4, 0x6d3404dd, 0x240879fa, + 0xde517cfe, 0x976d01d9, 0x4c2986b0, 0x0515fb97, + 0x2e015d56, 0x673d2071, 0xbc79a718, 0xf545da3f, + 0x0f1cdf3b, 0x4620a21c, 0x9d642575, 0xd4585852, + 0x6c3a598c, 0x250624ab, 0xfe42a3c2, 0xb77edee5, + 0x4d27dbe1, 0x041ba6c6, 0xdf5f21af, 0x96635c88, + 0xaa7754e2, 0xe34b29c5, 0x380faeac, 0x7133d38b, + 0x8b6ad68f, 0xc256aba8, 0x19122cc1, 0x502e51e6, + 0xe84c5038, 0xa1702d1f, 0x7a34aa76, 0x3308d751, + 0xc951d255, 0x806daf72, 0x5b29281b, 0x1215553c, + 0x230138cf, 0x6a3d45e8, 0xb179c281, 0xf845bfa6, + 0x021cbaa2, 0x4b20c785, 0x906440ec, 0xd9583dcb, + 0x613a3c15, 0x28064132, 0xf342c65b, 0xba7ebb7c, + 0x4027be78, 0x091bc35f, 0xd25f4436, 0x9b633911, + 0xa777317b, 0xee4b4c5c, 0x350fcb35, 0x7c33b612, + 0x866ab316, 0xcf56ce31, 0x14124958, 0x5d2e347f, + 0xe54c35a1, 0xac704886, 0x7734cfef, 0x3e08b2c8, + 0xc451b7cc, 0x8d6dcaeb, 0x56294d82, 0x1f1530a5 + } +#endif +}; + +/* + * __wt_cksum -- + * Return a checksum for a chunk of memory. + * + * Slicing-by-8 algorithm by Michael E. Kounavis and Frank L. Berry from + * Intel Corp.: + * http://www.intel.com/technology/comms/perfnet/download/CRC_generators.pdf + * + * Based on Peter Kanowski's posting: + * http://www.strchr.com/crc32_popcnt + * + * The big endian version calculates the same result at each step, except the + * value of the crc is byte reversed from what it would be at that step for + * little endian. + */ +uint32_t +__wt_cksum(const void *chunk, size_t len) +{ + uint32_t crc, next; + size_t nqwords; + const uint8_t *p; + + crc = 0xffffffff; + + /* Checksum one byte at a time to the first 4B boundary. */ + for (p = chunk; + ((uintptr_t)p & (sizeof(uint32_t) - 1)) != 0 && + len > 0; ++p, --len) +#ifdef WORDS_BIGENDIAN + crc = g_crc_slicing[0][((crc >> 24) ^ *p) & 0xFF] ^ (crc << 8); +#else + crc = g_crc_slicing[0][(crc ^ *p) & 0xFF] ^ (crc >> 8); +#endif + + /* Checksum in 8B chunks. */ + for (nqwords = len / sizeof(uint64_t); nqwords; nqwords--) { + crc ^= *(uint32_t *)p; + p += sizeof(uint32_t); + next = *(uint32_t *)p; + p += sizeof(uint32_t); + crc = +#ifdef WORDS_BIGENDIAN + g_crc_slicing[4][(crc ) & 0xFF] ^ + g_crc_slicing[5][(crc >> 8) & 0xFF] ^ + g_crc_slicing[6][(crc >> 16) & 0xFF] ^ + g_crc_slicing[7][(crc >> 24)] ^ + g_crc_slicing[0][(next ) & 0xFF] ^ + g_crc_slicing[1][(next >> 8) & 0xFF] ^ + g_crc_slicing[2][(next >> 16) & 0xFF] ^ + g_crc_slicing[3][(next >> 24)]; +#else + g_crc_slicing[7][(crc ) & 0xFF] ^ + g_crc_slicing[6][(crc >> 8) & 0xFF] ^ + g_crc_slicing[5][(crc >> 16) & 0xFF] ^ + g_crc_slicing[4][(crc >> 24)] ^ + g_crc_slicing[3][(next ) & 0xFF] ^ + g_crc_slicing[2][(next >> 8) & 0xFF] ^ + g_crc_slicing[1][(next >> 16) & 0xFF] ^ + g_crc_slicing[0][(next >> 24)]; +#endif + } + + /* Checksum trailing bytes one byte at a time. */ +#ifdef WORDS_BIGENDIAN + for (len &= 0x7; len > 0; ++p, len--) + crc = g_crc_slicing[0][((crc >> 24) ^ *p) & 0xFF] ^ (crc << 8); + + /* Do final byte swap to produce a result identical to little endian */ + crc = + ((crc << 24) & 0xFF000000) | + ((crc << 8) & 0x00FF0000) | + ((crc >> 8) & 0x0000FF00) | + ((crc >> 24) & 0x000000FF); +#else + for (len &= 0x7; len > 0; ++p, len--) + crc = g_crc_slicing[0][(crc ^ *p) & 0xFF] ^ (crc >> 8); +#endif + return (~crc); +} diff --git a/src/block/block_mgr.c b/src/block/block_mgr.c new file mode 100644 index 00000000000..3391ce27210 --- /dev/null +++ b/src/block/block_mgr.c @@ -0,0 +1,309 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static int +__bm_invalid(WT_SESSION_IMPL *session) +{ + WT_RET_MSG(session, EINVAL, "invalid block manager handle"); +} + +/* + * __wt_bm_addr_valid -- + * Return if an address cookie is valid. + */ +int +__wt_bm_addr_valid( + WT_SESSION_IMPL *session, const uint8_t *addr, uint32_t addr_size) +{ + WT_BLOCK *block; + + if ((block = session->btree->block) == NULL) + return (__bm_invalid(session)); + + return (__wt_block_addr_valid(session, block, addr, addr_size)); +} + +#ifdef HAVE_DIAGNOSTIC +/* + * __wt_bm_addr_stderr -- + * Print an address on stderr. + */ +int +__wt_bm_addr_stderr( + WT_SESSION_IMPL *session, const uint8_t *addr, uint32_t addr_size) +{ + WT_BLOCK *block; + WT_ITEM *buf; + int ret; + + if ((block = session->btree->block) == NULL) + return (__bm_invalid(session)); + + WT_RET(__wt_scr_alloc(session, 0, &buf)); + ret = __wt_block_addr_string(session, block, buf, addr, addr_size); + if (ret == 0) + fprintf(stderr, "%s\n", (char *)buf->data); + __wt_scr_free(&buf); + return (ret); +} +#endif + +/* + * __wt_bm_addr_string + * Return a printable string representation of an address cookie. + */ +int +__wt_bm_addr_string(WT_SESSION_IMPL *session, + WT_ITEM *buf, const uint8_t *addr, uint32_t addr_size) +{ + WT_BLOCK *block; + + if ((block = session->btree->block) == NULL) + return (__bm_invalid(session)); + + return ( + __wt_block_addr_string(session, block, buf, addr, addr_size)); +} + +/* + * __wt_bm_create -- + * Create a new file. + */ +int +__wt_bm_create(WT_SESSION_IMPL *session, const char *filename) +{ + return (__wt_block_create(session, filename)); +} + +/* + * __wt_bm_open -- + * Open a file. + */ +int +__wt_bm_open(WT_SESSION_IMPL *session, + const char *filename, const char *config, const char *cfg[], int salvage) +{ + return (__wt_block_open( + session, filename, config, cfg, salvage, &session->btree->block)); +} + +/* + * __wt_bm_close -- + * Close a file. + */ +int +__wt_bm_close(WT_SESSION_IMPL *session) +{ + WT_BLOCK *block; + int ret; + + if ((block = session->btree->block) == NULL) + return (0); + + ret = __wt_block_close(session, block); + session->btree->block = NULL; + + return (ret); +} + +/* + * __wt_bm_truncate -- + * Truncate a file. + */ +int +__wt_bm_truncate(WT_SESSION_IMPL *session, const char *filename) +{ + return (__wt_block_truncate(session, filename)); +} + +/* + * __wt_bm_free -- + * Free a block of space to the underlying file. + */ +int +__wt_bm_free(WT_SESSION_IMPL *session, const uint8_t *addr, uint32_t addr_size) +{ + WT_BLOCK *block; + + if ((block = session->btree->block) == NULL) + return (__bm_invalid(session)); + + return (__wt_block_free_buf(session, block, addr, addr_size)); +} + +/* + * __wt_bm_read -- + * Read a address cookie-referenced block into a buffer. + */ +int +__wt_bm_read(WT_SESSION_IMPL *session, + WT_ITEM *buf, const uint8_t *addr, uint32_t addr_size) +{ + WT_BLOCK *block; + + if ((block = session->btree->block) == NULL) + return (__bm_invalid(session)); + + return (__wt_block_read_buf(session, block, buf, addr, addr_size)); +} + +/* + * __wt_bm_block_header -- + * Return the size of the block manager's header. + */ +int +__wt_bm_block_header(WT_SESSION_IMPL *session, uint32_t *headerp) +{ + WT_BLOCK *block; + + if ((block = session->btree->block) == NULL) + return (__bm_invalid(session)); + + return (__wt_block_header(session, block, headerp)); +} + +/* + * __wt_bm_write_size -- + * Return the buffer size required to write a block. + */ +int +__wt_bm_write_size(WT_SESSION_IMPL *session, uint32_t *sizep) +{ + WT_BLOCK *block; + + if ((block = session->btree->block) == NULL) + return (__bm_invalid(session)); + + return (__wt_block_write_size(session, block, sizep)); +} + +/* + * __wt_bm_write -- + * Write a buffer into a block, returning the block's address cookie. + */ +int +__wt_bm_write( + WT_SESSION_IMPL *session, WT_ITEM *buf, uint8_t *addr, uint32_t *addr_size) +{ + WT_BLOCK *block; + + if ((block = session->btree->block) == NULL) + return (__bm_invalid(session)); + + return (__wt_block_write_buf(session, block, buf, addr, addr_size)); +} + +/* + * __wt_bm_stat -- + * Block-manager statistics. + */ +int +__wt_bm_stat(WT_SESSION_IMPL *session) +{ + WT_BLOCK *block; + + if ((block = session->btree->block) == NULL) + return (__bm_invalid(session)); + + __wt_block_stat(session, block); + return (0); +} + +/* + * __wt_bm_salvage_start -- + * Start a block manager salvage. + */ +int +__wt_bm_salvage_start(WT_SESSION_IMPL *session) +{ + WT_BLOCK *block; + + if ((block = session->btree->block) == NULL) + return (__bm_invalid(session)); + + return (__wt_block_salvage_start(session, block)); +} + +/* + * __wt_bm_salvage_next -- + * Return the next block from the file. + */ +int +__wt_bm_salvage_next(WT_SESSION_IMPL *session, WT_ITEM *buf, + uint8_t *addr, uint32_t *addr_sizep, uint64_t *write_genp, int *eofp) +{ + WT_BLOCK *block; + + if ((block = session->btree->block) == NULL) + return (__bm_invalid(session)); + + return (__wt_block_salvage_next( + session, block, buf, addr, addr_sizep, write_genp, eofp)); +} + +/* + * __wt_bm_salvage_end -- + * End a block manager salvage. + */ +int +__wt_bm_salvage_end(WT_SESSION_IMPL *session, int success) +{ + WT_BLOCK *block; + + if ((block = session->btree->block) == NULL) + return (__bm_invalid(session)); + + return (__wt_block_salvage_end(session, block, success)); +} + +/* + * __wt_bm_verify_start -- + * Start a block manager salvage. + */ +int +__wt_bm_verify_start(WT_SESSION_IMPL *session, int *emptyp) +{ + WT_BLOCK *block; + + if ((block = session->btree->block) == NULL) + return (__bm_invalid(session)); + + return (__wt_block_verify_start(session, block, emptyp)); +} + +/* + * __wt_bm_verify_end -- + * End a block manager salvage. + */ +int +__wt_bm_verify_end(WT_SESSION_IMPL *session) +{ + WT_BLOCK *block; + + if ((block = session->btree->block) == NULL) + return (__bm_invalid(session)); + + return (__wt_block_verify_end(session, block)); +} + +/* + * __wt_bm_verify_addr -- + * Verify an address. + */ +int +__wt_bm_verify_addr(WT_SESSION_IMPL *session, + const uint8_t *addr, uint32_t addr_size) +{ + WT_BLOCK *block; + + if ((block = session->btree->block) == NULL) + return (__bm_invalid(session)); + + return (__wt_block_verify_addr(session, block, addr, addr_size)); +} diff --git a/src/block/block_open.c b/src/block/block_open.c new file mode 100644 index 00000000000..ffa2a01b0c3 --- /dev/null +++ b/src/block/block_open.c @@ -0,0 +1,323 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static int __desc_read(WT_SESSION_IMPL *, WT_BLOCK *, int); +static int __desc_update(WT_SESSION_IMPL *, WT_BLOCK *); + +/* + * __wt_block_truncate -- + * Truncate a file. + */ +int +__wt_block_truncate(WT_SESSION_IMPL *session, const char *filename) +{ + WT_FH *fh; + int ret; + + /* Open the underlying file handle. */ + WT_RET(__wt_open(session, filename, 0, &fh)); + + /* Truncate the file. */ + WT_ERR(__wt_ftruncate(session, fh, (off_t)0)); + + /* Write out the file's meta-data. */ + ret = __wt_desc_init(session, fh); + + /* Close the file handle. */ +err: WT_TRET(__wt_close(session, fh)); + + return (ret); +} + +/* + * __wt_block_create -- + * Create a file. + */ +int +__wt_block_create(WT_SESSION_IMPL *session, const char *filename) +{ + WT_FH *fh; + int exist, ret; + + /* Check to see if the file exists -- we don't want to overwrite it. */ + WT_RET(__wt_exist(session, filename, &exist)); + if (exist) + WT_RET_MSG(session, WT_ERROR, + "the file %s already exists; to re-create it, remove it " + "first, then create it", + filename); + + /* Open the underlying file handle. */ + WT_RET(__wt_open(session, filename, 1, &fh)); + + /* Write out the file's meta-data. */ + ret = __wt_desc_init(session, fh); + + /* Close the file handle. */ + WT_TRET(__wt_close(session, fh)); + + /* Undo any create on error. */ + if (ret != 0) + (void)__wt_remove(session, filename); + + return (ret); +} + +/* + * __wt_block_open -- + * Open a file. + */ +int +__wt_block_open(WT_SESSION_IMPL *session, const char *filename, + const char *config, const char *cfg[], int salvage, void *retp) +{ + WT_BLOCK *block; + WT_CONFIG_ITEM cval; + WT_CONNECTION_IMPL *conn; + WT_NAMED_COMPRESSOR *ncomp; + int ret; + + conn = S2C(session); + + /* + * Allocate the structure, connect (so error close works), copy the + * name. + */ + WT_RET(__wt_calloc_def(session, 1, &block)); + WT_ERR(__wt_strdup(session, filename, &block->name)); + + /* Initialize the free-list structures. */ + __wt_block_freelist_open(session, block); + + /* Open the underlying file handle. */ + WT_ERR(__wt_open(session, filename, 1, &block->fh)); + + /* Get the allocation size. */ + WT_ERR(__wt_config_getones(session, config, "allocation_size", &cval)); + block->allocsize = (uint32_t)cval.val; + + /* Check if configured for checksums. */ + WT_ERR(__wt_config_getones(session, config, "checksum", &cval)); + block->checksum = cval.val == 0 ? 0 : 1; + + /* Page compressor */ + WT_RET(__wt_config_getones(session, config, "block_compressor", &cval)); + if (cval.len > 0) { + TAILQ_FOREACH(ncomp, &conn->compqh, q) { + if (strncmp(ncomp->name, cval.str, cval.len) == 0) { + block->compressor = ncomp->compressor; + break; + } + } + if (block->compressor == NULL) + WT_ERR_MSG(session, EINVAL, + "unknown block_compressor '%.*s'", + (int)cval.len, cval.str); + } + + /* + * Normally we read the file's meta-data to see if this is a WiredTiger + * file. But, if it's a salvage operation and force is set, we ignore + * the file's format entirely. + */ + cval.val = 0; + if (salvage) { + ret = __wt_config_gets(session, cfg, "force", &cval); + if (ret != 0 && ret != WT_NOTFOUND) + WT_RET(ret); + } + if (cval.val == 0) + WT_ERR(__desc_read(session, block, salvage)); + + /* If not an open for a salvage operation, read the freelist. */ + if (!salvage) + WT_ERR(__wt_block_freelist_read(session, block)); + + F_SET(block, WT_BLOCK_OK); + *(void **)retp = block; + return (0); + +err: (void)__wt_block_close(session, block); + return (ret); +} + +/* + * __wt_block_close -- + * Close a file. + */ +int +__wt_block_close(WT_SESSION_IMPL *session, WT_BLOCK *block) +{ + int ret; + + ret = 0; + + /* + * If the file was active, write out the free-list and update the + * file's description. + */ + if (F_ISSET(block, WT_BLOCK_OK)) { + WT_TRET(__wt_block_freelist_write(session, block)); + WT_TRET(__desc_update(session, block)); + } + + if (block->name != NULL) + __wt_free(session, block->name); + + if (block->fh != NULL) + WT_RET(__wt_close(session, block->fh)); + + __wt_block_freelist_close(session, block); + + __wt_free(session, block->fragbits); + + __wt_free(session, block); + return (ret); +} + +/* + * __desc_read -- + * Read and verify the file's metadata. + */ +static int +__desc_read(WT_SESSION_IMPL *session, WT_BLOCK *block, int salvage) +{ + WT_BLOCK_DESC *desc; + uint32_t cksum; + uint8_t buf[WT_BLOCK_DESC_SECTOR]; + + /* + * We currently always do the verification step, because it's cheap + * and we only do it the first time a file is opened. + * + * Read the first sector. + */ + WT_RET(__wt_read(session, block->fh, (off_t)0, sizeof(buf), buf)); + + desc = (WT_BLOCK_DESC *)buf; + WT_VERBOSE(session, block, + "open: magic %" PRIu32 + ", major/minor: %" PRIu32 "/%" PRIu32 + ", checksum %#" PRIx32 + ", free offset/size %" PRIu64 "/%" PRIu32 + ", write-generation %" PRIu64, + desc->magic, + desc->majorv, desc->minorv, + desc->cksum, + desc->free_offset, desc->free_size, + desc->write_gen); + + /* + * We fail the open if the checksum fails, or the magic number is wrong + * or the major/minor numbers are unsupported for this version. This + * test is done even if the caller is verifying or salvaging the file: + * it makes sense for verify, and for salvage we don't overwrite files + * without some reason to believe they are WiredTiger files. The user + * may have entered the wrong file name, and is now frantically pounding + * their interrupt key. + */ + cksum = desc->cksum; + desc->cksum = 0; + if (desc->magic != WT_BLOCK_MAGIC || + cksum != __wt_cksum(buf, sizeof(buf))) + WT_RET_MSG(session, WT_ERROR, "%s %s%s", + "does not appear to be a WiredTiger file", + block->name, + salvage ? "; to salvage this file, configure the salvage " + "operation with the force flag" : ""); + + if (desc->majorv > WT_BLOCK_MAJOR_VERSION || + (desc->majorv == WT_BLOCK_MAJOR_VERSION && + desc->minorv > WT_BLOCK_MINOR_VERSION)) + WT_RET_MSG(session, WT_ERROR, + "%s is an unsupported version of a WiredTiger file", + block->name); + + block->write_gen = desc->write_gen; + + /* That's all we check for salvage. */ + if (salvage) + return (0); + + if ((desc->free_offset != WT_BLOCK_INVALID_OFFSET && + desc->free_offset + desc->free_size > + (uint64_t)block->fh->file_size)) + WT_RET_MSG(session, WT_ERROR, + "free list offset references non-existent file space"); + + block->free_offset = (off_t)desc->free_offset; + block->free_size = desc->free_size; + block->free_cksum = desc->free_cksum; + + return (0); +} + +/* + * __wt_desc_init -- + * Write an initial file's descriptor structure. + */ +int +__wt_desc_init(WT_SESSION_IMPL *session, WT_FH *fh) +{ + WT_BLOCK_DESC *desc; + uint8_t buf[WT_BLOCK_DESC_SECTOR]; + + memset(buf, 0, sizeof(buf)); + desc = (WT_BLOCK_DESC *)buf; + + desc->magic = WT_BLOCK_MAGIC; + desc->majorv = WT_BLOCK_MAJOR_VERSION; + desc->minorv = WT_BLOCK_MINOR_VERSION; + + desc->free_offset = WT_BLOCK_INVALID_OFFSET; + desc->free_size = desc->free_cksum = 0; + + /* Update the checksum. */ + desc->cksum = 0; + desc->cksum = __wt_cksum(buf, sizeof(buf)); + + return (__wt_write(session, fh, (off_t)0, sizeof(buf), buf)); +} + +/* + * __desc_update -- + * Update the file's descriptor structure. + */ +static int +__desc_update(WT_SESSION_IMPL *session, WT_BLOCK *block) +{ + WT_BLOCK_DESC *desc; + uint8_t buf[WT_BLOCK_DESC_SECTOR]; + + /* Read the first sector. */ + WT_RET(__wt_read(session, block->fh, (off_t)0, sizeof(buf), buf)); + desc = (WT_BLOCK_DESC *)buf; + + /* See if anything has changed. */ + if (desc->free_offset == (uint64_t)block->free_offset && + desc->free_size == block->free_size && + desc->write_gen == block->write_gen) + return (0); + + WT_VERBOSE(session, block, + "resetting free list [offset %" PRIuMAX ", size %" PRIu32 "]", + (uintmax_t)block->free_offset, block->free_size); + + desc->free_offset = (uint64_t)block->free_offset; + desc->free_size = block->free_size; + desc->free_cksum = block->free_cksum; + + desc->write_gen = block->write_gen; + + /* Update the checksum. */ + desc->cksum = 0; + desc->cksum = __wt_cksum(buf, sizeof(buf)); + + return (__wt_write(session, block->fh, (off_t)0, sizeof(buf), buf)); +} diff --git a/src/block/block_read.c b/src/block/block_read.c new file mode 100644 index 00000000000..76c56d266c2 --- /dev/null +++ b/src/block/block_read.c @@ -0,0 +1,162 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_block_read_buf -- + * Read filesystem cookie referenced block into a buffer. + */ +int +__wt_block_read_buf(WT_SESSION_IMPL *session, WT_BLOCK *block, + WT_ITEM *buf, const uint8_t *addr, uint32_t addr_size) +{ + WT_ITEM *tmp; + off_t offset; + uint32_t size, cksum; + int ret; + + ret = 0; + + /* Crack the cookie. */ + WT_RET(__wt_block_buffer_to_addr(block, addr, &offset, &size, &cksum)); + + /* Read the block. */ + WT_RET(__wt_block_read(session, block, buf, offset, size, cksum)); + + /* Optionally verify the page. */ + if (block->fragbits == NULL) + return (0); + + WT_RET(__wt_scr_alloc(session, 0, &tmp)); + WT_ERR(__wt_block_addr_string(session, block, tmp, addr, addr_size)); + WT_ERR(__wt_verify_dsk( + session, (char *)tmp->data, buf->mem, buf->size)); + +err: __wt_scr_free(&tmp); + + return (ret); +} + +/* + * __wt_block_read -- + * Read an addr/size pair referenced block into a buffer. + */ +int +__wt_block_read(WT_SESSION_IMPL *session, WT_BLOCK *block, + WT_ITEM *buf, off_t offset, uint32_t size, uint32_t cksum) +{ + WT_BLOCK_HEADER *blk; + WT_ITEM *tmp; + WT_PAGE_HEADER *dsk; + size_t result_len; + uint32_t page_cksum; + int ret; + + tmp = NULL; + ret = 0; + + WT_VERBOSE(session, read, + "off %" PRIuMAX ", size %" PRIu32 ", cksum %" PRIu32, + (uintmax_t)offset, size, cksum); + + /* + * If we're compressing the file blocks, place the initial read into a + * scratch buffer, we're going to have to re-allocate more memory for + * decompression. Else check the caller's buffer size and grow it as + * necessary, there will only be one buffer. + */ + if (block->compressor == NULL) { + WT_RET(__wt_buf_init(session, buf, size)); + buf->size = size; + dsk = buf->mem; + } else { + WT_RET(__wt_scr_alloc(session, size, &tmp)); + tmp->size = size; + dsk = tmp->mem; + } + + /* Read. */ + WT_ERR(__wt_read(session, block->fh, offset, size, dsk)); + blk = WT_BLOCK_HEADER_REF(dsk); + + /* Validate the checksum. */ + if (block->checksum && + cksum != WT_BLOCK_CHECKSUM_NOT_SET && + blk->cksum != WT_BLOCK_CHECKSUM_NOT_SET) { + blk->cksum = 0; + page_cksum = __wt_cksum(dsk, size); + if (page_cksum == WT_BLOCK_CHECKSUM_NOT_SET) + ++page_cksum; + if (cksum != page_cksum) { + if (!F_ISSET(session, WT_SESSION_SALVAGE_QUIET_ERR)) + __wt_errx(session, + "read checksum error [%" + PRIu32 "B @ %" PRIuMAX ", %" + PRIu32 " != %" PRIu32 "]", + size, (uintmax_t)offset, cksum, page_cksum); + WT_ERR(WT_ERROR); + } + } + + /* + * If the in-memory block size is larger than the on-disk block size, + * the block is compressed. Size the user's buffer, copy the skipped + * bytes of the original image into place, then decompress. + * + * If the in-memory block size is less than or equal to the on-disk + * block size, the block is not compressed. + */ + if (blk->disk_size < dsk->size) { + if (block->compressor == NULL) + WT_ERR(__wt_illegal_value(session)); + + WT_RET(__wt_buf_init(session, buf, dsk->size)); + buf->size = dsk->size; + + /* + * Note the source length is NOT the number of compressed bytes, + * it's the length of the block we just read (minus the skipped + * bytes). We don't store the number of compressed bytes: some + * compression engines need that length stored externally, they + * don't have markers in the stream to signal the end of the + * compressed bytes. Those engines must store the compressed + * byte length somehow, see the snappy compression extension for + * an example. + */ + memcpy(buf->mem, tmp->mem, WT_BLOCK_COMPRESS_SKIP); + WT_ERR(block->compressor->decompress( + block->compressor, &session->iface, + (uint8_t *)tmp->mem + WT_BLOCK_COMPRESS_SKIP, + tmp->size - WT_BLOCK_COMPRESS_SKIP, + (uint8_t *)buf->mem + WT_BLOCK_COMPRESS_SKIP, + dsk->size - WT_BLOCK_COMPRESS_SKIP, + &result_len)); + if (result_len != dsk->size - WT_BLOCK_COMPRESS_SKIP) + WT_ERR(__wt_illegal_value(session)); + } else + if (block->compressor == NULL) + buf->size = dsk->size; + else + /* + * We guessed wrong: there was a compressor, but this + * block was not compressed, and now the page is in the + * wrong buffer and the buffer may be of the wrong size. + * This should be rare, why configure a compressor that + * doesn't work? Allocate a buffer of the right size + * (we used a scratch buffer which might be large), and + * copy the data into place. + */ + WT_ERR( + __wt_buf_set(session, buf, tmp->data, dsk->size)); + + WT_BSTAT_INCR(session, page_read); + WT_CSTAT_INCR(session, block_read); + +err: __wt_scr_free(&tmp); + return (ret); +} diff --git a/src/block/block_slvg.c b/src/block/block_slvg.c new file mode 100644 index 00000000000..5d0e3a603f9 --- /dev/null +++ b/src/block/block_slvg.c @@ -0,0 +1,162 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_block_salvage_start -- + * Start a file salvage. + */ +int +__wt_block_salvage_start(WT_SESSION_IMPL *session, WT_BLOCK *block) +{ + off_t len; + uint32_t allocsize; + + /* + * Truncate the file to an initial sector plus N allocation size + * units (bytes trailing the last multiple of an allocation size + * unit must be garbage, by definition). + */ + if (block->fh->file_size > WT_BLOCK_DESC_SECTOR) { + allocsize = block->allocsize; + len = block->fh->file_size - WT_BLOCK_DESC_SECTOR; + len = (len / allocsize) * allocsize; + len += WT_BLOCK_DESC_SECTOR; + if (len != block->fh->file_size) + WT_RET(__wt_ftruncate(session, block->fh, len)); + } + + /* Reset the description sector. */ + WT_RET(__wt_desc_init(session, block->fh)); + + /* The first sector of the file is the description record, skip it. */ + block->slvg_off = WT_BLOCK_DESC_SECTOR; + + /* + * We don't currently need to do anything about the freelist because + * we don't read it for salvage operations. + */ + + return (0); +} + +/* + * __wt_block_salvage_end -- + * End a file salvage. + */ +int +__wt_block_salvage_end(WT_SESSION_IMPL *session, WT_BLOCK *block, int success) +{ + /* + * If not successful, discard the free-list, it's not useful, and + * don't write back an updated description block. + */ + if (!success) { + F_CLR(block, WT_BLOCK_OK); + __wt_block_discard(session, block); + } + return (0); +} + +/* + * __wt_block_salvage_next -- + * Return the next block from the file. + */ +int +__wt_block_salvage_next( + WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, + uint8_t *addr, uint32_t *addr_sizep, uint64_t *write_genp, int *eofp) +{ + WT_BLOCK_HEADER *blk; + WT_FH *fh; + off_t max, offset; + uint32_t allocsize, cksum, size; + uint8_t *endp; + + *eofp = 0; + + offset = block->slvg_off; + fh = block->fh; + allocsize = block->allocsize; + WT_RET(__wt_buf_initsize(session, buf, allocsize)); + + /* Read through the file, looking for pages with valid checksums. */ + for (max = fh->file_size;;) { + if (offset >= max) { /* Check eof. */ + *eofp = 1; + return (0); + } + + /* + * Read the start of a possible page (an allocation-size block), + * and get a page length from it. + */ + WT_RET(__wt_read(session, fh, offset, allocsize, buf->mem)); + blk = WT_BLOCK_HEADER_REF(buf->mem); + + /* + * The page can't be more than the min/max page size, or past + * the end of the file. + */ + size = blk->disk_size; + cksum = blk->cksum; + if (size == 0 || + size % allocsize != 0 || + size > WT_BTREE_PAGE_SIZE_MAX || + offset + (off_t)size > max) + goto skip; + + /* + * The page size isn't insane, read the entire page: reading the + * page validates the checksum and then decompresses the page as + * needed. If reading the page fails, it's probably corruption, + * we ignore this block. + */ + if (__wt_block_read(session, block, buf, offset, size, cksum)) { +skip: WT_VERBOSE(session, salvage, + "skipping %" PRIu32 "B at file offset %" PRIuMAX, + allocsize, (uintmax_t)offset); + + /* + * Free the block and make sure we don't return it more + * than once. + */ + WT_RET(__wt_block_free( + session, block, offset, (off_t)allocsize)); + block->slvg_off = offset += allocsize; + continue; + } + + /* + * Valid block, return to our caller. + * + * The buffer may have grown: make sure we read from the full + * page image. + */ + blk = WT_BLOCK_HEADER_REF(buf->mem); + break; + } + + /* + * Track the largest write-generation we've seen in the file so future + * writes, done after salvage completes, are preferred to these blocks. + */ + *write_genp = blk->write_gen; + if (block->write_gen < blk->write_gen) + block->write_gen = blk->write_gen; + + /* Re-create the address cookie that should reference this block. */ + endp = addr; + WT_RET(__wt_block_addr_to_buffer(block, &endp, offset, size, cksum)); + *addr_sizep = WT_PTRDIFF32(endp, addr); + + /* We're successfully returning the page, move past it. */ + block->slvg_off = offset + size; + + return (0); +} diff --git a/src/block/block_vrfy.c b/src/block/block_vrfy.c new file mode 100644 index 00000000000..e6a4cf52410 --- /dev/null +++ b/src/block/block_vrfy.c @@ -0,0 +1,211 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static int __verify_addfrag(WT_SESSION_IMPL *, WT_BLOCK *, off_t, off_t); +static int __verify_checkfrag(WT_SESSION_IMPL *, WT_BLOCK *); +static int __verify_freelist(WT_SESSION_IMPL *, WT_BLOCK *); + +/* + * __wt_block_verify_start -- + * Start file verification. + */ +int +__wt_block_verify_start(WT_SESSION_IMPL *session, WT_BLOCK *block, int *emptyp) +{ + off_t file_size; + + file_size = block->fh->file_size; + + /* + * We're done if the file has no data pages (this is what happens if + * we verify a file immediately after creation). + */ + if (file_size == WT_BLOCK_DESC_SECTOR) { + *emptyp = 1; + return (0); + } + *emptyp = 0; + + /* + * The file size should be a multiple of the allocsize, offset by the + * size of the descriptor sector, the first 512B of the file. + */ + if (file_size > WT_BLOCK_DESC_SECTOR) + file_size -= WT_BLOCK_DESC_SECTOR; + if (file_size % block->allocsize != 0) + WT_RET_MSG(session, WT_ERROR, + "the file size is not a multiple of the allocation size"); + + /* + * Allocate a bit array, where each bit represents a single allocation + * size piece of the file (this is how we track the parts of the file + * we've verified, and check for multiply referenced or unreferenced + * blocks). Storing this on the heap seems reasonable, verifying a 1TB + * file with an 512B allocation size would require a 256MB bit array: + * + * (((1 * 2^40) / 512) / 8) = 256 * 2^20 + * + * To verify larger files than we can handle in this way, we'd have to + * write parts of the bit array into a disk file. + * + * We also have a minimum maximum verifiable file size of 16TB because + * the underlying bit package takes a 32-bit count of bits to allocate: + * + * 2^32 * 512 * 8 = 16 * 2^40 + */ + if (file_size / block->allocsize > UINT32_MAX) + WT_RET_MSG( + session, WT_ERROR, "the file is too large to verify"); + + block->frags = (uint32_t)(file_size / block->allocsize); + WT_RET(__bit_alloc(session, block->frags, &block->fragbits)); + + /* Verify the free-list. */ + WT_RET(__verify_freelist(session, block)); + + return (0); +} + +/* + * __wt_block_verify_end -- + * End file verification. + */ +int +__wt_block_verify_end(WT_SESSION_IMPL *session, WT_BLOCK *block) +{ + int ret; + + /* Verify we read every file block. */ + ret = __verify_checkfrag(session, block); + + __wt_free(session, block->fragbits); + + return (ret); +} + +/* + * __wt_block_verify_addr -- + * Verify an address. + */ +int +__wt_block_verify_addr(WT_SESSION_IMPL *session, + WT_BLOCK *block, const uint8_t *addr, uint32_t addr_size) +{ + off_t offset; + uint32_t size; + + WT_UNUSED(addr_size); + + /* Crack the cookie. */ + WT_RET(__wt_block_buffer_to_addr(block, addr, &offset, &size, NULL)); + + WT_RET(__verify_addfrag(session, block, offset, (off_t)size)); + + return (0); +} + +/* + * __verify_freelist -- + * Add the freelist fragments to the list of verified fragments. + */ +static int +__verify_freelist(WT_SESSION_IMPL *session, WT_BLOCK *block) +{ + WT_FREE *fe; + int ret; + + ret = 0; + + WT_FREE_FOREACH(fe, block->foff) { + if (fe->off + (off_t)fe->size > block->fh->file_size) + WT_RET_MSG(session, WT_ERROR, + "free-list entry offset %" PRIuMAX "references " + "non-existent file pages", + (uintmax_t)fe->off); + + WT_VERBOSE(session, verify, + "free-list range %" PRIdMAX "-%" PRIdMAX, + (intmax_t)fe->off, (intmax_t)(fe->off + fe->size)); + + WT_TRET(__verify_addfrag(session, block, fe->off, fe->size)); + } + + return (ret); +} + +/* The bit list ignores the first sector: convert to/from an frag/offset. */ +#define WT_OFF_TO_FRAG(block, off) \ + (((off) - WT_BLOCK_DESC_SECTOR) / (block)->allocsize) +#define WT_FRAG_TO_OFF(block, frag) \ + (((off_t)(frag)) * (block)->allocsize + WT_BLOCK_DESC_SECTOR) + +/* + * __verify_addfrag -- + * Add the fragments to the list, and complain if we've already verified + * this chunk of the file. + */ +static int +__verify_addfrag( + WT_SESSION_IMPL *session, WT_BLOCK *block, off_t offset, off_t size) +{ + uint32_t frag, frags, i; + + frag = (uint32_t)WT_OFF_TO_FRAG(block, offset); + frags = (uint32_t)(size / block->allocsize); + + for (i = 0; i < frags; ++i) + if (__bit_test(block->fragbits, frag + i)) + WT_RET_MSG(session, WT_ERROR, + "file fragment at offset %" PRIuMAX + " already verified", + (uintmax_t)offset); + + __bit_nset(block->fragbits, frag, frag + (frags - 1)); + return (0); +} + +/* + * __verify_checkfrag -- + * Verify we've checked all the fragments in the file. + */ +static int +__verify_checkfrag(WT_SESSION_IMPL *session, WT_BLOCK *block) +{ + uint32_t first, last, frags; + uint8_t *fragbits; + int ret; + + fragbits = block->fragbits; + frags = block->frags; + ret = 0; + + /* + * Check for file fragments we haven't verified -- every time we find + * a bit that's clear, complain. We re-start the search each time + * after setting the clear bit(s) we found: it's simpler and this isn't + * supposed to happen a lot. + */ + for (;;) { + if (__bit_ffc(fragbits, frags, &first) != 0) + break; + __bit_set(fragbits, first); + for (last = first + 1; last < frags; ++last) { + if (__bit_test(fragbits, last)) + break; + __bit_set(fragbits, last); + } + + __wt_errx(session, + "file range %" PRIuMAX "-%" PRIuMAX " was never verified", + (uintmax_t)WT_FRAG_TO_OFF(block, first), + (uintmax_t)WT_FRAG_TO_OFF(block, last)); + ret = WT_ERROR; + } + return (ret); +} diff --git a/src/block/block_write.c b/src/block/block_write.c new file mode 100644 index 00000000000..f4194ccc460 --- /dev/null +++ b/src/block/block_write.c @@ -0,0 +1,246 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_block_header -- + * Return the size of the block-specific header. + */ +int +__wt_block_header(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t *headerp) +{ + WT_UNUSED(session); + WT_UNUSED(block); + + *headerp = WT_BLOCK_HEADER_SIZE; + return (0); +} + +/* + * __wt_block_write_size -- + * Return the buffer size required to write a block. + */ +int +__wt_block_write_size( + WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t *sizep) +{ + WT_UNUSED(session); + + *sizep = WT_ALIGN(*sizep + WT_BLOCK_HEADER_BYTE_SIZE, block->allocsize); + return (0); +} + +/* + * __wt_block_write_buf -- + * Write a buffer into a block, returning the block's address cookie. + */ +int +__wt_block_write_buf(WT_SESSION_IMPL *session, + WT_BLOCK *block, WT_ITEM *buf, uint8_t *addr, uint32_t *addr_size) +{ + off_t offset; + uint32_t size, cksum; + uint8_t *endp; + + WT_UNUSED(addr_size); + + WT_RET(__wt_block_write(session, block, buf, &offset, &size, &cksum)); + + endp = addr; + WT_RET(__wt_block_addr_to_buffer(block, &endp, offset, size, cksum)); + *addr_size = WT_PTRDIFF32(endp, addr); + + return (0); +} + +/* + * __wt_block_write -- + * Write a buffer into a block, returning the block's addr/size and + * checksum. + */ +int +__wt_block_write(WT_SESSION_IMPL *session, WT_BLOCK *block, + WT_ITEM *buf, off_t *offsetp, uint32_t *sizep, uint32_t *cksump) +{ + WT_BLOCK_HEADER *blk; + WT_PAGE_HEADER *dsk; + WT_ITEM *tmp; + off_t offset; + uint32_t align_size, size; + int compression_failed, ret; + uint8_t *src, *dst; + size_t len, src_len, dst_len, result_len; + + tmp = NULL; + ret = 0; + + /* + * Set the block's in-memory size. + * + * XXX + * Should be set by our caller, it's part of the WT_PAGE_HEADER? + */ + dsk = buf->mem; + dsk->size = buf->size; + + /* + * We're passed a table's page image: WT_ITEM->{mem,size} are the image + * and byte count. + * + * Diagnostics: verify the disk page: this violates layering, but it's + * the place we can ensure we never write a corrupted page. Note that + * we are verifying the free-list page, too. (We created a "page" for + * the free-list, it was simpler than creating another type of object + * in the file.) + */ +#ifdef HAVE_DIAGNOSTIC + WT_RET(__wt_verify_dsk(session, "[write-check]", buf->mem, buf->size)); +#endif + + /* + * Align the size to an allocation unit. + * + * The buffer must be big enough for us to zero to the next allocsize + * boundary, this is one of the reasons the btree layer must find out + * from the block-manager layer the maximum size of the eventual write. + */ + align_size = WT_ALIGN(buf->size, block->allocsize); + if (align_size > buf->memsize) + WT_RET_MSG(session, EINVAL, + "write buffer was incorrectly allocated"); + + /* + * Optionally stream-compress the data, but don't compress blocks that + * are already as small as they're going to get. + */ + if (block->compressor == NULL || align_size == block->allocsize) { +not_compressed: /* + * If not compressing the buffer, we need to zero out any unused + * bytes at the end. + */ + memset( + (uint8_t *)buf->mem + buf->size, 0, align_size - buf->size); + buf->size = align_size; + + /* + * Set the in-memory size to the on-page size (we check the size + * to decide if a block is compressed: if the sizes match, the + * block is NOT compressed). + */ + dsk = buf->mem; + } else { + /* Skip the first 32B of the source data. */ + src = (uint8_t *)buf->mem + WT_BLOCK_COMPRESS_SKIP; + src_len = buf->size - WT_BLOCK_COMPRESS_SKIP; + + /* + * Compute the size needed for the destination buffer. We only + * allocate enough memory for a copy of the original by default, + * if any compressed version is bigger than the original, we + * won't use it. However, some compression engines (snappy is + * one example), may need more memory because they don't stop + * just because there's no more memory into which to compress. + */ + if (block->compressor->pre_size == NULL) + len = src_len; + else + WT_ERR(block->compressor->pre_size(block->compressor, + &session->iface, src, src_len, &len)); + WT_RET(__wt_scr_alloc( + session, (uint32_t)len + WT_BLOCK_COMPRESS_SKIP, &tmp)); + + /* Skip the first 32B of the destination data. */ + dst = (uint8_t *)tmp->mem + WT_BLOCK_COMPRESS_SKIP; + dst_len = len; + + /* + * If compression fails, fallback to the original version. This + * isn't unexpected: if compression doesn't work for some chunk + * of bytes for some reason (noting there's likely additional + * format/header information which compressed output requires), + * it just means the uncompressed version is as good as it gets, + * and that's what we use. + */ + compression_failed = 0; + WT_ERR(block->compressor->compress(block->compressor, + &session->iface, + src, src_len, + dst, dst_len, + &result_len, &compression_failed)); + if (compression_failed) + goto not_compressed; + + /* + * Set the final data size and see if compression gave us back + * at least one allocation unit (if we don't get at least one + * file allocation unit, use the uncompressed version because + * it will be faster to read). + */ + tmp->size = (uint32_t)result_len + WT_BLOCK_COMPRESS_SKIP; + size = WT_ALIGN(tmp->size, block->allocsize); + if (size >= align_size) + goto not_compressed; + align_size = size; + + /* Copy in the skipped header bytes, zero out unused bytes. */ + memcpy(tmp->mem, buf->mem, WT_BLOCK_COMPRESS_SKIP); + memset( + (uint8_t *)tmp->mem + tmp->size, 0, align_size - tmp->size); + + dsk = tmp->mem; + } + + blk = WT_BLOCK_HEADER_REF(dsk); + + /* + * We increment the block's write generation so it's easy to identify + * newer versions of blocks during salvage: it's common in WiredTiger + * for multiple blocks to be internally consistent with identical + * first and last keys, so we need a way to know the most recent state + * of the block. (We could check to see which leaf is referenced by + * by the internal page, which implies salvaging internal pages (which + * I don't want to do), and it's not quite as good anyway, because the + * internal page may not have been written to disk after the leaf page + * was updated. So, write generations it is.) + */ + blk->write_gen = ++block->write_gen; + + blk->disk_size = align_size; + + /* + * Update the block's checksum: checksum the compressed contents, not + * the uncompressed contents. If the computed checksum happens to be + * equal to the special "not set" value, increment it. We do the same + * on the checking side. + */ + if (block->checksum) { + blk->cksum = 0; + blk->cksum = __wt_cksum(dsk, align_size); + if (blk->cksum == WT_BLOCK_CHECKSUM_NOT_SET) + ++blk->cksum; + } else + blk->cksum = WT_BLOCK_CHECKSUM_NOT_SET; + + /* Allocate space from the underlying file and write the block. */ + WT_ERR(__wt_block_alloc(session, block, &offset, (off_t)align_size)); + WT_ERR(__wt_write(session, block->fh, offset, align_size, dsk)); + + WT_BSTAT_INCR(session, page_write); + WT_CSTAT_INCR(session, block_write); + + WT_VERBOSE(session, write, + "off %" PRIuMAX ", size %" PRIu32 ", cksum %" PRIu32, + (uintmax_t)offset, align_size, blk->cksum); + + *offsetp = offset; + *sizep = align_size; + *cksump = blk->cksum; + +err: __wt_scr_free(&tmp); + return (ret); +} diff --git a/src/btree/bt_bulk.c b/src/btree/bt_bulk.c new file mode 100644 index 00000000000..c4865f3c034 --- /dev/null +++ b/src/btree/bt_bulk.c @@ -0,0 +1,152 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static int __bulk_row_keycmp_err(WT_CURSOR_BULK *); + +/* + * __wt_bulk_init -- + * Start a bulk load. + */ +int +__wt_bulk_init(WT_CURSOR_BULK *cbulk) +{ + WT_SESSION_IMPL *session; + int ret; + + session = (WT_SESSION_IMPL *)cbulk->cbt.iface.session; + + /* + * You can't bulk-load into existing trees. Check, and retrieve the + * leaf page we're going to use. + */ + if ((ret = __wt_btree_root_empty(session, &cbulk->leaf)) != 0) + WT_RET_MSG( + session, ret, "bulk-load is only possible for empty trees"); + + WT_RET(__wt_rec_bulk_init(cbulk)); + + return (0); +} + +/* + * __wt_bulk_insert -- + * Bulk insert, called once per item. + */ +int +__wt_bulk_insert(WT_CURSOR_BULK *cbulk) +{ + WT_BTREE *btree; + WT_CURSOR *cursor; + WT_SESSION_IMPL *session; + int cmp; + + session = (WT_SESSION_IMPL *)cbulk->cbt.iface.session; + btree = session->btree; + cursor = &cbulk->cbt.iface; + + switch (btree->type) { + case BTREE_COL_FIX: + WT_RET(__wt_rec_col_fix_bulk_insert(cbulk)); + break; + case BTREE_COL_VAR: + /* + * If this isn't the first value inserted, compare it against + * the last value and increment the RLE count. + * + * Instead of a "first time" variable, I'm using the RLE count, + * because it is set to 0 exactly once, the first time through + * the code. + */ + if (cbulk->rle != 0) { + if (cbulk->cmp.size == cursor->value.size && + memcmp(cbulk->cmp.data, + cursor->value.data, cursor->value.size) == 0) { + ++cbulk->rle; + break; + } + WT_RET(__wt_rec_col_var_bulk_insert(cbulk)); + } + WT_RET(__wt_buf_set(session, + &cbulk->cmp, cursor->value.data, cursor->value.size)); + cbulk->rle = 1; + break; + case BTREE_ROW: + /* + * If this isn't the first value inserted, compare it against + * the last key to ensure the application doesn't accidentally + * corrupt the table. + * + * Instead of a "first time" variable, I'm using the RLE count, + * because it is set to 0 exactly once, the first time through + * the code. + */ + if (cbulk->rle != 0) { + WT_RET(WT_BTREE_CMP(session, session->btree, + &cursor->key, &cbulk->cmp, cmp)); + if (cmp <= 0) + return (__bulk_row_keycmp_err(cbulk)); + } + WT_RET(__wt_buf_set(session, + &cbulk->cmp, cursor->key.data, cursor->key.size)); + cbulk->rle = 1; + + WT_RET(__wt_rec_row_bulk_insert(cbulk)); + break; + WT_ILLEGAL_VALUE(session); + } + + WT_BSTAT_INCR(session, file_bulk_loaded); + return (0); +} + +/* + * __wt_bulk_end -- + * Clean up after a bulk load. + */ +int +__wt_bulk_end(WT_CURSOR_BULK *cbulk) +{ + WT_SESSION_IMPL *session; + + session = (WT_SESSION_IMPL *)cbulk->cbt.iface.session; + + WT_RET(__wt_rec_bulk_wrapup(cbulk)); + + __wt_buf_free(session, &cbulk->cmp); + + return (0); +} + +/* + * __bulk_row_keycmp_err -- + * Error routine when keys inserted out-of-order. + */ +static int +__bulk_row_keycmp_err(WT_CURSOR_BULK *cbulk) +{ + WT_ITEM a, b; + WT_CURSOR *cursor; + WT_SESSION_IMPL *session; + + session = (WT_SESSION_IMPL *)cbulk->cbt.iface.session; + cursor = &cbulk->cbt.iface; + + WT_CLEAR(a); + WT_CLEAR(b); + + WT_RET(__wt_buf_set_printable( + session, &a, cursor->key.data, cursor->key.size)); + WT_RET(__wt_buf_set_printable( + session, &b, cbulk->cmp.data, cbulk->cmp.size)); + + WT_RET_MSG(session, EINVAL, + "bulk-load presented with out-of-order keys: %.*s compares smaller " + "than previously inserted key %.*s", + (int)a.size, (char *)a.data, (int)b.size, (char *)b.data); +} diff --git a/src/btree/bt_cache.c b/src/btree/bt_cache.c new file mode 100644 index 00000000000..44b63890d2b --- /dev/null +++ b/src/btree/bt_cache.c @@ -0,0 +1,106 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_cache_create -- + * Create the underlying cache. + */ +int +__wt_cache_create(WT_CONNECTION_IMPL *conn, const char *cfg[]) +{ + WT_CACHE *cache; + WT_CONFIG_ITEM cval; + WT_SESSION_IMPL *session; + int ret; + + session = &conn->default_session; + ret = 0; + + WT_RET(__wt_calloc_def(session, 1, &conn->cache)); + cache = conn->cache; + + /* Configure the cache. */ + WT_ERR(__wt_config_gets(session, cfg, "eviction_target", &cval)); + cache->eviction_target = (u_int)cval.val; + WT_ERR(__wt_config_gets(session, cfg, "eviction_trigger", &cval)); + cache->eviction_trigger = (u_int)cval.val; + + /* + * The target size must be lower than the trigger size or we will never + * get any work done. + */ + if (cache->eviction_target >= cache->eviction_trigger) + WT_ERR_MSG(session, EINVAL, + "eviction target must be lower than the eviction trigger"); + + WT_ERR(__wt_cond_alloc(session, + "cache eviction server", 1, &cache->evict_cond)); + __wt_spin_init(session, &cache->lru_lock); + + /* + * Allocate the eviction request array. We size it to allow one + * eviction request request per session. + */ + cache->max_evict_request = conn->session_size; + WT_ERR(__wt_calloc_def( + session, cache->max_evict_request, &cache->evict_request)); + + /* + * We pull some values from the cache statistics (rather than have two + * copies). Set them. + */ + __wt_cache_stats_update(conn); + + return (0); + +err: __wt_cache_destroy(conn); + return (ret); +} + +/* + * __wt_cache_stats_update -- + * Update the cache statistics for return to the application. + */ +void +__wt_cache_stats_update(WT_CONNECTION_IMPL *conn) +{ + WT_CACHE *cache; + + cache = conn->cache; + + WT_STAT_SET(conn->stats, cache_bytes_max, conn->cache_size); + WT_STAT_SET( + conn->stats, cache_bytes_inuse, __wt_cache_bytes_inuse(cache)); + WT_STAT_SET( + conn->stats, cache_pages_inuse, __wt_cache_pages_inuse(cache)); +} + +/* + * __wt_cache_destroy -- + * Discard the underlying cache. + */ +void +__wt_cache_destroy(WT_CONNECTION_IMPL *conn) +{ + WT_SESSION_IMPL *session; + WT_CACHE *cache; + + session = &conn->default_session; + cache = conn->cache; + + if (cache == NULL) + return; + + if (cache->evict_cond != NULL) + (void)__wt_cond_destroy(session, cache->evict_cond); + __wt_spin_destroy(session, &cache->lru_lock); + + __wt_free(session, cache->evict_request); + __wt_free(session, conn->cache); +} diff --git a/src/btree/bt_cell.c b/src/btree/bt_cell.c new file mode 100644 index 00000000000..e7b6fdd8802 --- /dev/null +++ b/src/btree/bt_cell.c @@ -0,0 +1,68 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_cell_copy -- + * Copy an on-page cell into a return buffer, processing as needed. + */ +int +__wt_cell_copy(WT_SESSION_IMPL *session, WT_CELL *cell, WT_ITEM *retb) +{ + WT_CELL_UNPACK *unpack, _unpack; + + unpack = &_unpack; + + __wt_cell_unpack(cell, unpack); + return (__wt_cell_unpack_copy(session, unpack, retb)); +} + +/* + * __wt_cell_unpack_copy -- + * Copy an unpacked cell into a return buffer, processing as needed. + */ +int +__wt_cell_unpack_copy( + WT_SESSION_IMPL *session, WT_CELL_UNPACK *unpack, WT_ITEM *retb) +{ + WT_BTREE *btree; + void *huffman; + + btree = session->btree; + + /* Get the cell's data. */ + switch (unpack->type) { + case WT_CELL_KEY: + case WT_CELL_VALUE: + WT_RET(__wt_buf_set(session, retb, unpack->data, unpack->size)); + break; + case WT_CELL_KEY_OVFL: + case WT_CELL_VALUE_OVFL: + WT_RET(__wt_ovfl_in(session, retb, unpack->data, unpack->size)); + break; + WT_ILLEGAL_VALUE(session); + } + + /* Select a Huffman encoding function. */ + switch (unpack->type) { + case WT_CELL_KEY: + case WT_CELL_KEY_OVFL: + if ((huffman = btree->huffman_key) == NULL) + return (0); + break; + case WT_CELL_VALUE: + case WT_CELL_VALUE_OVFL: + default: + if ((huffman = btree->huffman_value) == NULL) + return (0); + break; + } + + return (__wt_huffman_decode( + session, huffman, retb->data, retb->size, retb)); +} diff --git a/src/btree/bt_curnext.c b/src/btree/bt_curnext.c new file mode 100644 index 00000000000..77ef160a3e6 --- /dev/null +++ b/src/btree/bt_curnext.c @@ -0,0 +1,429 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __cursor_fix_append_next -- + * Return the next entry on the append list. + */ +static inline int +__cursor_fix_append_next(WT_CURSOR_BTREE *cbt, int newpage) +{ + WT_ITEM *val; + + val = &cbt->iface.value; + + if (newpage) { + if ((cbt->ins = WT_SKIP_FIRST(cbt->ins_head)) == NULL) + return (WT_NOTFOUND); + } else + if (cbt->recno == WT_INSERT_RECNO(cbt->ins) && + (cbt->ins = WT_SKIP_NEXT(cbt->ins)) == NULL) + return (WT_NOTFOUND); + + cbt->iface.recno = ++cbt->recno; + if (cbt->recno < WT_INSERT_RECNO(cbt->ins)) { + cbt->v = 0; + val->data = &cbt->v; + } else + val->data = WT_UPDATE_DATA(cbt->ins->upd); + val->size = 1; + return (0); +} + +/* + * __cursor_fix_next -- + * Move to the next, fixed-length column-store item. + */ +static inline int +__cursor_fix_next(WT_CURSOR_BTREE *cbt, int newpage) +{ + WT_BTREE *btree; + WT_INSERT *ins; + WT_ITEM *val; + WT_SESSION_IMPL *session; + uint64_t *recnop; + + session = (WT_SESSION_IMPL *)cbt->iface.session; + btree = session->btree; + + recnop = &cbt->iface.recno; + val = &cbt->iface.value; + + /* Initialize for each new page. */ + if (newpage) { + cbt->last_standard_recno = __col_last_recno(cbt->page); + if (cbt->last_standard_recno == 0) + return (WT_NOTFOUND); + cbt->recno = cbt->page->u.col_fix.recno; + goto new_page; + } + + /* Move to the next entry and return the item. */ + for (;;) { + if (cbt->recno >= cbt->last_standard_recno) + return (WT_NOTFOUND); + ++cbt->recno; +new_page: *recnop = cbt->recno; + + /* Check any insert list for a matching record. */ + if ((ins = __col_insert_search_match( + WT_COL_UPDATE_SINGLE(cbt->page), cbt->recno)) != NULL) { + val->data = WT_UPDATE_DATA(ins->upd); + val->size = 1; + return (0); + } + + cbt->v = __bit_getv_recno(cbt->page, cbt->recno, btree->bitcnt); + val->data = &cbt->v; + val->size = 1; + return (0); + } + /* NOTREACHED */ +} + +/* + * __cursor_var_append_next -- + * Return the next variable-length entry on the append list. + */ +static inline int +__cursor_var_append_next(WT_CURSOR_BTREE *cbt, int newpage) +{ + WT_ITEM *val; + + val = &cbt->iface.value; + + if (newpage) { + cbt->ins = WT_SKIP_FIRST(cbt->ins_head); + goto new_page; + } + + for (;;) { + if ((cbt->ins = WT_SKIP_NEXT(cbt->ins)) == NULL) + return (WT_NOTFOUND); + +new_page: cbt->iface.recno = WT_INSERT_RECNO(cbt->ins); + if (WT_UPDATE_DELETED_ISSET(cbt->ins->upd)) + continue; + val->data = WT_UPDATE_DATA(cbt->ins->upd); + val->size = cbt->ins->upd->size; + break; + } + return (0); +} + +/* + * __cursor_var_next -- + * Move to the next, variable-length column-store item. + */ +static inline int +__cursor_var_next(WT_CURSOR_BTREE *cbt, int newpage) +{ + WT_CELL *cell; + WT_CELL_UNPACK unpack; + WT_COL *cip; + WT_INSERT *ins; + WT_ITEM *val; + WT_SESSION_IMPL *session; + uint64_t *recnop; + + session = (WT_SESSION_IMPL *)cbt->iface.session; + recnop = &cbt->iface.recno; + val = &cbt->iface.value; + + /* Initialize for each new page. */ + if (newpage) { + cbt->last_standard_recno = __col_last_recno(cbt->page); + if (cbt->last_standard_recno == 0) + return (WT_NOTFOUND); + cbt->recno = cbt->page->u.col_var.recno; + goto new_page; + } + + /* Move to the next entry and return the item. */ + for (;;) { + if (cbt->recno >= cbt->last_standard_recno) + return (WT_NOTFOUND); + ++cbt->recno; +new_page: *recnop = cbt->recno; + + /* Find the matching WT_COL slot. */ + if ((cip = __col_var_search(cbt->page, cbt->recno)) == NULL) + return (WT_NOTFOUND); + + /* Check any insert list for a matching record. */ + if ((ins = __col_insert_search_match( + WT_COL_UPDATE(cbt->page, cip), cbt->recno)) != NULL) { + if (WT_UPDATE_DELETED_ISSET(ins->upd)) + continue; + val->data = WT_UPDATE_DATA(ins->upd); + val->size = ins->upd->size; + return (0); + } + + /* + * If we're at the same slot as the last reference and there's + * no matching insert list item, re-use the return information. + * Otherwise, unpack the cell and build the return information. + */ + if (cbt->cip_saved != cip) { + if ((cell = WT_COL_PTR(cbt->page, cip)) == NULL) + continue; + __wt_cell_unpack(cell, &unpack); + switch (unpack.type) { + case WT_CELL_DEL: + continue; + case WT_CELL_VALUE: + if (session->btree->huffman_value == NULL) { + cbt->tmp.data = unpack.data; + cbt->tmp.size = unpack.size; + break; + } + /* FALLTHROUGH */ + default: + WT_RET(__wt_cell_unpack_copy( + session, &unpack, &cbt->tmp)); + } + cbt->cip_saved = cip; + } + val->data = cbt->tmp.data; + val->size = cbt->tmp.size; + return (0); + } + /* NOTREACHED */ +} + +/* + * __cursor_row_next -- + * Move to the next row-store item. + */ +static inline int +__cursor_row_next(WT_CURSOR_BTREE *cbt, int newpage) +{ + WT_ITEM *key, *val; + WT_ROW *rip; + WT_UPDATE *upd; + + key = &cbt->iface.key; + val = &cbt->iface.value; + + /* + * For row-store pages, we need a single item that tells us the part + * of the page we're walking (otherwise switching from next to prev + * and vice-versa is just too complicated), so we map the WT_ROW and + * WT_INSERT_HEAD insert array slots into a single name space: slot 1 + * is the "smallest key insert list", slot 2 is WT_ROW[0], slot 3 is + * WT_INSERT_HEAD[0], and so on. This means WT_INSERT lists are + * odd-numbered slots, and WT_ROW array slots are even-numbered slots. + * + * New page configuration. + */ + if (newpage) { + cbt->ins_head = WT_ROW_INSERT_SMALLEST(cbt->page); + cbt->ins = WT_SKIP_FIRST(cbt->ins_head); + cbt->slot = 1; + goto new_insert; + } + + /* Move to the next entry and return the item. */ + for (;;) { + /* + * Continue traversing any insert list; maintain the insert list + * head reference and entry count in case we switch to a cursor + * previous movement. + */ + if (cbt->ins != NULL) + cbt->ins = WT_SKIP_NEXT(cbt->ins); + +new_insert: if (cbt->ins != NULL) { + upd = cbt->ins->upd; + if (WT_UPDATE_DELETED_ISSET(upd)) + continue; + key->data = WT_INSERT_KEY(cbt->ins); + key->size = WT_INSERT_KEY_SIZE(cbt->ins); + val->data = WT_UPDATE_DATA(upd); + val->size = upd->size; + return (0); + } + + /* Check for the end of the page. */ + if (cbt->slot >= cbt->page->entries * 2 + 1) + return (WT_NOTFOUND); + ++cbt->slot; + + /* + * Odd-numbered slots configure as WT_INSERT_HEAD entries, + * even-numbered slots configure as WT_ROW entries. + */ + if (cbt->slot & 0x01) { + cbt->ins_head = + WT_ROW_INSERT_SLOT(cbt->page, cbt->slot / 2 - 1); + cbt->ins = WT_SKIP_FIRST(cbt->ins_head); + goto new_insert; + } + cbt->ins_head = NULL; + cbt->ins = NULL; + + rip = &cbt->page->u.row.d[cbt->slot / 2 - 1]; + upd = WT_ROW_UPDATE(cbt->page, rip); + if (upd != NULL && WT_UPDATE_DELETED_ISSET(upd)) + continue; + + return (__cursor_row_slot_return(cbt, rip)); + } + /* NOTREACHED */ +} + +/* + * __wt_btcur_iterate_setup -- + * Initialize a cursor for iteration, usually based on a search. + */ +void +__wt_btcur_iterate_setup(WT_CURSOR_BTREE *cbt, int next) +{ + WT_PAGE *page; + + WT_UNUSED(next); + + /* + * We don't currently have to do any setup when we switch between next + * and prev calls, but I'm sure we will someday -- I'm leaving support + * here for both flags for that reason. + */ + F_SET(cbt, WT_CBT_ITERATE_NEXT | WT_CBT_ITERATE_PREV); + + /* + * If we don't have a search page, then we're done, we're starting at + * the beginning or end of the tree, not as a result of a search. + */ + if ((page = cbt->page) == NULL) + return; + + if (page->type == WT_PAGE_ROW_LEAF) { + /* + * For row-store pages, we need a single item that tells us the + * part of the page we're walking (otherwise switching from next + * to prev and vice-versa is just too complicated), so we map + * the WT_ROW and WT_INSERT_HEAD insert array slots into a + * single name space: slot 1 is the "smallest key insert list", + * slot 2 is WT_ROW[0], slot 3 is WT_INSERT_HEAD[0], and so on. + * This means WT_INSERT lists are odd-numbered slots, and WT_ROW + * array slots are even-numbered slots. + * + * !!! + * I'm re-using WT_CURSOR_BTREE->slot for this purpose, which + * means that WT_CURSOR_BTREE->slot is now useless outside of + * cursor next/prev. If that turns out to be a bad idea because + * we need the original value of WT_CURSOR_BTREE->slot after a + * next/prev call, switch to another field to hold the iteration + * slot. + */ + cbt->slot = (cbt->slot + 1) * 2; + if (cbt->ins_head != NULL) { + if (cbt->ins_head == WT_ROW_INSERT_SMALLEST(page)) + cbt->slot = 1; + else + cbt->slot += 1; + } + } else { + /* + * For column-store pages, calculate the largest record on the + * page. + */ + cbt->last_standard_recno = __col_last_recno(page); + + /* If we're traversing the append list, set the reference. */ + if (cbt->ins_head != NULL && + cbt->ins_head == WT_COL_APPEND(cbt->btree, page)) + F_SET(cbt, WT_CBT_ITERATE_APPEND); + } +} + +/* + * __wt_btcur_next -- + * Move to the next record in the tree. + */ +int +__wt_btcur_next(WT_CURSOR_BTREE *cbt) +{ + WT_SESSION_IMPL *session; + int newpage, ret; + + session = (WT_SESSION_IMPL *)cbt->iface.session; + WT_BSTAT_INCR(session, cursor_read_next); + + __cursor_func_init(cbt, 0); + + /* + * If we aren't already iterating in the right direction, there's + * some setup to do. + */ + if (!F_ISSET(cbt, WT_CBT_ITERATE_NEXT)) + __wt_btcur_iterate_setup(cbt, 1); + + /* + * Walk any page we're holding until the underlying call returns not- + * found. Then, move to the next page, until we reach the end of the + * file. + */ + for (newpage = 0;; newpage = 1) { + if (F_ISSET(cbt, WT_CBT_ITERATE_APPEND)) { + switch (cbt->page->type) { + case WT_PAGE_COL_FIX: + ret = __cursor_fix_append_next(cbt, newpage); + break; + case WT_PAGE_COL_VAR: + ret = __cursor_var_append_next(cbt, newpage); + break; + WT_ILLEGAL_VALUE(session); + } + if (ret == 0) + break; + F_CLR(cbt, WT_CBT_ITERATE_APPEND); + if (ret != WT_NOTFOUND) + break; + } else if (cbt->page != NULL) { + switch (cbt->page->type) { + case WT_PAGE_COL_FIX: + ret = __cursor_fix_next(cbt, newpage); + break; + case WT_PAGE_COL_VAR: + ret = __cursor_var_next(cbt, newpage); + break; + case WT_PAGE_ROW_LEAF: + ret = __cursor_row_next(cbt, newpage); + break; + WT_ILLEGAL_VALUE(session); + } + if (ret != WT_NOTFOUND) + break; + + /* + * The last page in a column-store has appended entries. + * We handle it separately from the usual cursor code: + * it's only that one page and it's in a simple format. + */ + if (cbt->page->type != WT_PAGE_ROW_LEAF && + (cbt->ins_head = + WT_COL_APPEND(cbt->btree, cbt->page)) != NULL) { + F_SET(cbt, WT_CBT_ITERATE_APPEND); + continue; + } + } + + do { + WT_ERR(__wt_tree_np(session, &cbt->page, 0, 1)); + WT_ERR_TEST(cbt->page == NULL, WT_NOTFOUND); + } while ( + cbt->page->type == WT_PAGE_COL_INT || + cbt->page->type == WT_PAGE_ROW_INT); + } + +err: __cursor_func_resolve(cbt, ret); + return (ret); +} diff --git a/src/btree/bt_curprev.c b/src/btree/bt_curprev.c new file mode 100644 index 00000000000..ffdecfe672a --- /dev/null +++ b/src/btree/bt_curprev.c @@ -0,0 +1,486 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * Walking backwards through skip lists. + * + * The skip list stack is an array of pointers set up by a search. It points + * to the position a node should go in the skip list. In other words, the skip + * list search stack always points *after* the search item (that is, into the + * search item's next array). + * + * Helper macros to go from a stack pointer at level i, pointing into a next + * array, to insert node containing that next array. + */ +#define PREV_ITEM(ins_head, insp, i) \ + (((insp) == &(ins_head)->head[i] || (insp) == NULL) ? NULL : \ + (WT_INSERT *)((char *)((insp) - (i)) - offsetof(WT_INSERT, next))) + +#define PREV_INS(cbt, i) \ + PREV_ITEM((cbt)->ins_head, (cbt)->ins_stack[(i)], (i)) + +/* + * __cursor_skip_prev -- + * Move back one position in a skip list stack (aka "finger"). + */ +static inline void +__cursor_skip_prev(WT_CURSOR_BTREE *cbt) +{ + WT_INSERT *current, *ins; + WT_ITEM key; + WT_SESSION_IMPL *session; + int i; + + session = (WT_SESSION_IMPL *)cbt->iface.session; + + /* + * If the search stack does not point at the current item, fill it in + * with a search. + */ + if ((current = cbt->ins) != PREV_INS(cbt, 0)) { + if (cbt->btree->type == BTREE_ROW) { + key.data = WT_INSERT_KEY(current); + key.size = WT_INSERT_KEY_SIZE(current); + cbt->ins = __wt_search_insert(session, + cbt, cbt->ins_head, &key); + } else + cbt->ins = __col_insert_search(cbt->ins_head, + cbt->ins_stack, WT_INSERT_RECNO(current)); + + /* Check that we found the expected item. */ + WT_ASSERT(session, cbt->ins == current); + WT_ASSERT(session, PREV_INS(cbt, 0) == current); + } + + /* + * Find the first node up the search stack that does not move. + * + * The depth of the current item must be at least this level, since we + * see it in that many levels of the stack. + * + * !!! Watch these loops carefully: they all rely on the value of i, + * and the exit conditions to end up with the right values are + * non-trivial. + */ + for (i = 0; i < WT_SKIP_MAXDEPTH - 1; i++) + if ((ins = PREV_INS(cbt, i + 1)) != current) + break; + + /* + * Find a starting point for the new search. That is either at the + * non-moving node if we found a valid node, or the beginning of the + * next list down that is not the current node. + * + * Since it is the beginning of a list, and we know the current node is + * has a skip depth at least this high, any node we find must sort + * before the current node. + */ + if (ins == NULL || ins == current) + for (; i >= 0; i--) { + cbt->ins_stack[i] = NULL; + ins = cbt->ins_head->head[i]; + if (ins != NULL && ins != current) + break; + } + + /* Walk any remaining levels until just before the current node. */ + while (i >= 0) { + WT_ASSERT(session, ins != NULL); + if (ins->next[i] != current) /* Stay at this level */ + ins = ins->next[i]; + else { /* Drop down a level */ + cbt->ins_stack[i] = &ins->next[i]; + --i; + } + } + + /* If we found a previous node, the next one must be current. */ + WT_ASSERT(session, + cbt->ins_stack[0] == NULL || *cbt->ins_stack[0] == current); + + cbt->ins = PREV_INS(cbt, 0); +} + +/* + * __cursor_fix_append_prev -- + * Return the previous fixed-length entry on the append list. + */ +static inline int +__cursor_fix_append_prev(WT_CURSOR_BTREE *cbt, int newpage) +{ + WT_ITEM *val; + + val = &cbt->iface.value; + + if (newpage) { + if ((cbt->ins = WT_SKIP_LAST(cbt->ins_head)) == NULL) + return (WT_NOTFOUND); + cbt->recno = WT_INSERT_RECNO(cbt->ins); + } else { + if (cbt->recno == WT_INSERT_RECNO(cbt->ins)) { + __cursor_skip_prev(cbt); + if (cbt->ins == NULL) + return (WT_NOTFOUND); + } + --cbt->recno; + } + + cbt->iface.recno = cbt->recno; + if (cbt->recno > WT_INSERT_RECNO(cbt->ins)) { + cbt->v = 0; + val->data = &cbt->v; + } else + val->data = WT_UPDATE_DATA(cbt->ins->upd); + val->size = 1; + return (0); +} + +/* + * __cursor_fix_prev -- + * Move to the previous, fixed-length column-store item. + */ +static inline int +__cursor_fix_prev(WT_CURSOR_BTREE *cbt, int newpage) +{ + WT_BTREE *btree; + WT_INSERT *ins; + WT_ITEM *val; + WT_SESSION_IMPL *session; + uint64_t *recnop; + + session = (WT_SESSION_IMPL *)cbt->iface.session; + btree = session->btree; + + recnop = &cbt->iface.recno; + val = &cbt->iface.value; + + /* Initialize for each new page. */ + if (newpage) { + cbt->last_standard_recno = __col_last_recno(cbt->page); + if (cbt->last_standard_recno == 0) + return (WT_NOTFOUND); + cbt->recno = cbt->last_standard_recno; + goto new_page; + } + + /* Move to the previous entry and return the item. */ + for (;;) { + if (cbt->recno == cbt->page->u.col_fix.recno) + return (WT_NOTFOUND); + --cbt->recno; +new_page: *recnop = cbt->recno; + + /* Check any insert list for a matching record. */ + if ((ins = cbt->ins = __col_insert_search_match( + WT_COL_UPDATE_SINGLE(cbt->page), cbt->recno)) != NULL) { + val->data = WT_UPDATE_DATA(ins->upd); + val->size = 1; + return (0); + } + + cbt->v = __bit_getv_recno(cbt->page, cbt->recno, btree->bitcnt); + val->data = &cbt->v; + val->size = 1; + return (0); + } + /* NOTREACHED */ +} + +/* + * __cursor_var_append_prev -- + * Return the previous variable-length entry on the append list. + */ +static inline int +__cursor_var_append_prev(WT_CURSOR_BTREE *cbt, int newpage) +{ + WT_ITEM *val; + + val = &cbt->iface.value; + + if (newpage) { + cbt->ins = WT_SKIP_LAST(cbt->ins_head); + goto new_page; + } + + for (;;) { + __cursor_skip_prev(cbt); +new_page: if (cbt->ins == NULL) + return (WT_NOTFOUND); + + cbt->iface.recno = WT_INSERT_RECNO(cbt->ins); + if (WT_UPDATE_DELETED_ISSET(cbt->ins->upd)) + continue; + val->data = WT_UPDATE_DATA(cbt->ins->upd); + val->size = cbt->ins->upd->size; + break; + } + return (0); +} + +/* + * __cursor_var_prev -- + * Move to the previous, variable-length column-store item. + */ +static inline int +__cursor_var_prev(WT_CURSOR_BTREE *cbt, int newpage) +{ + WT_CELL *cell; + WT_CELL_UNPACK unpack; + WT_COL *cip; + WT_INSERT *ins; + WT_ITEM *val; + WT_SESSION_IMPL *session; + uint64_t *recnop; + + session = (WT_SESSION_IMPL *)cbt->iface.session; + recnop = &cbt->iface.recno; + val = &cbt->iface.value; + + /* Initialize for each new page. */ + if (newpage) { + cbt->last_standard_recno = __col_last_recno(cbt->page); + if (cbt->last_standard_recno == 0) + return (WT_NOTFOUND); + cbt->recno = cbt->last_standard_recno; + goto new_page; + } + + /* Move to the previous entry and return the item. */ + for (;;) { + --cbt->recno; +new_page: *recnop = cbt->recno; + if (cbt->recno < cbt->page->u.col_var.recno) + return (WT_NOTFOUND); + + /* Find the matching WT_COL slot. */ + if ((cip = __col_var_search(cbt->page, cbt->recno)) == NULL) + return (WT_NOTFOUND); + + /* Check any insert list for a matching record. */ + if ((ins = __col_insert_search_match( + WT_COL_UPDATE(cbt->page, cip), cbt->recno)) != NULL) { + if (WT_UPDATE_DELETED_ISSET(ins->upd)) + continue; + cbt->ins = ins; + val->data = WT_UPDATE_DATA(ins->upd); + val->size = ins->upd->size; + return (0); + } + + /* + * If we're at the same slot as the last reference and there's + * no matching insert list item, re-use the return information. + * Otherwise, unpack the cell and build the return information. + */ + if (cbt->cip_saved != cip) { + if ((cell = WT_COL_PTR(cbt->page, cip)) == NULL) + continue; + __wt_cell_unpack(cell, &unpack); + switch (unpack.type) { + case WT_CELL_DEL: + continue; + case WT_CELL_VALUE: + if (session->btree->huffman_value == NULL) { + cbt->tmp.data = unpack.data; + cbt->tmp.size = unpack.size; + break; + } + /* FALLTHROUGH */ + default: + WT_RET(__wt_cell_unpack_copy( + session, &unpack, &cbt->tmp)); + } + cbt->cip_saved = cip; + } + val->data = cbt->tmp.data; + val->size = cbt->tmp.size; + return (0); + } + /* NOTREACHED */ +} + +/* + * __cursor_row_prev -- + * Move to the previous row-store item. + */ +static inline int +__cursor_row_prev(WT_CURSOR_BTREE *cbt, int newpage) +{ + WT_INSERT *ins; + WT_ITEM *key, *val; + WT_ROW *rip; + WT_SESSION_IMPL *session; + WT_UPDATE *upd; + + session = (WT_SESSION_IMPL *)cbt->iface.session; + key = &cbt->iface.key; + val = &cbt->iface.value; + + /* + * For row-store pages, we need a single item that tells us the part + * of the page we're walking (otherwise switching from next to prev + * and vice-versa is just too complicated), so we map the WT_ROW and + * WT_INSERT_HEAD insert array slots into a single name space: slot 1 + * is the "smallest key insert list", slot 2 is WT_ROW[0], slot 3 is + * WT_INSERT_HEAD[0], and so on. This means WT_INSERT lists are + * odd-numbered slots, and WT_ROW array slots are even-numbered slots. + * + * New page configuration. + */ + if (newpage) { + /* + * If we haven't instantiated keys on this page, do so, else it + * is a very, very slow traversal. + */ + if (!F_ISSET(cbt->page, WT_PAGE_BUILD_KEYS)) + WT_RET(__wt_row_leaf_keys(session, cbt->page)); + + if (cbt->page->entries == 0) + cbt->ins_head = WT_ROW_INSERT_SMALLEST(cbt->page); + else + cbt->ins_head = WT_ROW_INSERT_SLOT( + cbt->page, cbt->page->entries - 1); + cbt->slot = cbt->page->entries * 2 + 1; + cbt->ins = WT_SKIP_LAST(cbt->ins_head); + goto new_insert; + } + + /* Move to the previous entry and return the item. */ + for (;;) { + /* + * Continue traversing any insert list. Maintain the reference + * to the current insert element in case we switch to a cursor + * next movement. + */ + if (cbt->ins != NULL) + __cursor_skip_prev(cbt); + +new_insert: if ((ins = cbt->ins) != NULL) { + upd = ins->upd; + if (WT_UPDATE_DELETED_ISSET(upd)) + continue; + key->data = WT_INSERT_KEY(ins); + key->size = WT_INSERT_KEY_SIZE(ins); + val->data = WT_UPDATE_DATA(upd); + val->size = upd->size; + return (0); + } + + /* Check for the beginning of the page. */ + if (cbt->slot == 1) + return (WT_NOTFOUND); + --cbt->slot; + + /* + * Odd-numbered slots configure as WT_INSERT_HEAD entries, + * even-numbered slots configure as WT_ROW entries. + */ + if (cbt->slot & 0x01) { + cbt->ins_head = cbt->slot == 1 ? + WT_ROW_INSERT_SMALLEST(cbt->page) : + WT_ROW_INSERT_SLOT(cbt->page, cbt->slot / 2 - 1); + cbt->ins = WT_SKIP_LAST(cbt->ins_head); + goto new_insert; + } + cbt->ins_head = NULL; + cbt->ins = NULL; + + rip = &cbt->page->u.row.d[cbt->slot / 2 - 1]; + upd = WT_ROW_UPDATE(cbt->page, rip); + if (upd != NULL && WT_UPDATE_DELETED_ISSET(upd)) + continue; + + return (__cursor_row_slot_return(cbt, rip)); + } + /* NOTREACHED */ +} + +/* + * __wt_btcur_prev -- + * Move to the previous record in the tree. + */ +int +__wt_btcur_prev(WT_CURSOR_BTREE *cbt) +{ + WT_SESSION_IMPL *session; + int newpage, ret; + + session = (WT_SESSION_IMPL *)cbt->iface.session; + WT_BSTAT_INCR(session, cursor_read_prev); + + __cursor_func_init(cbt, 0); + + /* + * If we aren't already iterating in the right direction, there's + * some setup to do. + */ + if (!F_ISSET(cbt, WT_CBT_ITERATE_PREV)) + __wt_btcur_iterate_setup(cbt, 0); + + /* + * Walk any page we're holding until the underlying call returns not- + * found. Then, move to the previous page, until we reach the start + * of the file. + */ + for (newpage = 0;; newpage = 1) { + if (F_ISSET(cbt, WT_CBT_ITERATE_APPEND)) { + switch (cbt->page->type) { + case WT_PAGE_COL_FIX: + ret = __cursor_fix_append_prev(cbt, newpage); + break; + case WT_PAGE_COL_VAR: + ret = __cursor_var_append_prev(cbt, newpage); + break; + WT_ILLEGAL_VALUE_ERR(session); + } + if (ret == 0) + break; + F_CLR(cbt, WT_CBT_ITERATE_APPEND); + if (ret != WT_NOTFOUND) + break; + newpage = 1; + } + if (cbt->page != NULL) { + switch (cbt->page->type) { + case WT_PAGE_COL_FIX: + ret = __cursor_fix_prev(cbt, newpage); + break; + case WT_PAGE_COL_VAR: + ret = __cursor_var_prev(cbt, newpage); + break; + case WT_PAGE_ROW_LEAF: + ret = __cursor_row_prev(cbt, newpage); + break; + WT_ILLEGAL_VALUE_ERR(session); + } + if (ret != WT_NOTFOUND) + break; + } + + do { + WT_ERR(__wt_tree_np(session, &cbt->page, 0, 0)); + WT_ERR_TEST(cbt->page == NULL, WT_NOTFOUND); + } while ( + cbt->page->type == WT_PAGE_COL_INT || + cbt->page->type == WT_PAGE_ROW_INT); + + /* + * The last page in a column-store has appended entries. + * We handle it separately from the usual cursor code: + * it's only that one page and it's in a simple format. + */ + if (cbt->page->type != WT_PAGE_ROW_LEAF && + (cbt->ins_head = + WT_COL_APPEND(cbt->btree, cbt->page)) != NULL) + F_SET(cbt, WT_CBT_ITERATE_APPEND); + } + +err: __cursor_func_resolve(cbt, ret); + return (ret); +} diff --git a/src/btree/bt_cursor.c b/src/btree/bt_cursor.c new file mode 100644 index 00000000000..ab38a4e97d6 --- /dev/null +++ b/src/btree/bt_cursor.c @@ -0,0 +1,445 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __cursor_size_chk -- + * Return if an inserted item is too large. + */ +static inline int +__cursor_size_chk(WT_SESSION_IMPL *session, WT_ITEM *kv) +{ + WT_BTREE *btree; + + btree = session->btree; + + if (btree->type == BTREE_COL_FIX) { + /* Fixed-size column-stores take a single byte. */ + if (kv->size != 1) + WT_RET_MSG(session, EINVAL, + "item size of %" PRIu32 " does not match " + "fixed-length file requirement of 1 byte", + kv->size); + } else { + if (kv->size > WT_BTREE_MAX_OBJECT_SIZE) + WT_RET_MSG(session, EINVAL, + "item size of %" PRIu32 " exceeds the maximum " + "supported size of %" PRIu32, + kv->size, WT_BTREE_MAX_OBJECT_SIZE); + } + return (0); +} + +/* + * __cursor_fix_implicit -- + * Return if search went past the end of the tree. + */ +static inline int +__cursor_fix_implicit(WT_BTREE *btree, WT_CURSOR_BTREE *cbt) +{ + return (btree->type == BTREE_COL_FIX && + !F_ISSET(cbt, WT_CBT_MAX_RECORD) ? 1 : 0); +} + +/* + * __cursor_invalid -- + * Return if the cursor references an invalid K/V pair (either the pair + * doesn't exist at all because the tree is empty, or the pair was deleted). + */ +static inline int +__cursor_invalid(WT_CURSOR_BTREE *cbt) +{ + WT_BTREE *btree; + WT_CELL *cell; + WT_CELL_UNPACK unpack; + WT_COL *cip; + WT_INSERT *ins; + WT_PAGE *page; + + btree = cbt->btree; + ins = cbt->ins; + page = cbt->page; + + /* If we found an item on an insert list, check there. */ + if (ins != NULL) + return (WT_UPDATE_DELETED_ISSET(ins->upd) ? 1 : 0); + + /* The page may be empty, the search routine doesn't check. */ + if (page->entries == 0) + return (1); + + /* Otherwise, check for an update in the page's slots. */ + switch (btree->type) { + case BTREE_COL_FIX: + break; + case BTREE_COL_VAR: + cip = &page->u.col_var.d[cbt->slot]; + if ((cell = WT_COL_PTR(page, cip)) == NULL) + return (WT_NOTFOUND); + __wt_cell_unpack(cell, &unpack); + if (unpack.type == WT_CELL_DEL) + return (1); + break; + case BTREE_ROW: + if (page->u.row.upd != NULL && + page->u.row.upd[cbt->slot] != NULL && + WT_UPDATE_DELETED_ISSET(page->u.row.upd[cbt->slot])) + return (1); + break; + } + return (0); +} + +/* + * __wt_btcur_reset -- + * Invalidate the cursor position. + */ +int +__wt_btcur_reset(WT_CURSOR_BTREE *cbt) +{ + WT_SESSION_IMPL *session; + + session = (WT_SESSION_IMPL *)cbt->iface.session; + WT_BSTAT_INCR(session, cursor_resets); + + __cursor_func_init(cbt, 1); + __cursor_search_clear(cbt); + + return (0); +} + +/* + * __wt_btcur_search -- + * Search for a matching record in the tree. + */ +int +__wt_btcur_search(WT_CURSOR_BTREE *cbt) +{ + WT_BTREE *btree; + WT_CURSOR *cursor; + WT_ITEM *val; + WT_SESSION_IMPL *session; + int ret; + + btree = cbt->btree; + cursor = &cbt->iface; + session = (WT_SESSION_IMPL *)cursor->session; + WT_BSTAT_INCR(session, cursor_read); + + if (btree->type == BTREE_ROW) + WT_RET(__cursor_size_chk(session, &cursor->key)); + + __cursor_func_init(cbt, 1); + + WT_ERR(btree->type == BTREE_ROW ? + __wt_row_search(session, cbt, 0) : + __wt_col_search(session, cbt, 0)); + if (cbt->compare != 0 || __cursor_invalid(cbt)) { + /* + * Creating a record past the end of the tree in a fixed-length + * column-store implicitly fills the gap with empty records. + */ + if (__cursor_fix_implicit(btree, cbt)) { + cbt->v = 0; + val = &cbt->iface.value; + val->data = &cbt->v; + val->size = 1; + } else + ret = WT_NOTFOUND; + } else + ret = __wt_kv_return(session, cbt, 0); + +err: __cursor_func_resolve(cbt, ret); + + return (ret); +} + +/* + * __wt_btcur_search_near -- + * Search for a record in the tree. + */ +int +__wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exact) +{ + WT_BTREE *btree; + WT_ITEM *val; + WT_CURSOR *cursor; + WT_SESSION_IMPL *session; + int ret; + + btree = cbt->btree; + cursor = &cbt->iface; + session = (WT_SESSION_IMPL *)cursor->session; + WT_BSTAT_INCR(session, cursor_read_near); + + if (btree->type == BTREE_ROW) + WT_RET(__cursor_size_chk(session, &cursor->key)); + + __cursor_func_init(cbt, 1); + + WT_ERR(btree->type == BTREE_ROW ? + __wt_row_search(session, cbt, 0) : + __wt_col_search(session, cbt, 0)); + + /* + * Creating a record past the end of the tree in a fixed-length column- + * store implicitly fills the gap with empty records. In this case, we + * instantiate the empty record, it's an exact match. + * + * Else, if we find a valid key (one that wasn't deleted), return it. + * + * Else, if we found a deleted key, try to move to the next key in the + * tree (bias for prefix searches). Cursor next skips deleted records, + * so we don't have to test for them again. + * + * Else if there's no larger tree key, redo the search and try and find + * an earlier record. If that fails, quit, there's no record to return. + */ + if (cbt->compare != 0 && __cursor_fix_implicit(btree, cbt)) { + cbt->v = 0; + val = &cbt->iface.value; + val->data = &cbt->v; + val->size = 1; + *exact = 0; + } else if (!__cursor_invalid(cbt)) { + *exact = cbt->compare; + ret = __wt_kv_return(session, cbt, cbt->compare == 0 ? 0 : 1); + } else if ((ret = __wt_btcur_next(cbt)) != WT_NOTFOUND) + *exact = 1; + else { + WT_ERR(btree->type == BTREE_ROW ? + __wt_row_search(session, cbt, 0) : + __wt_col_search(session, cbt, 0)); + if (!__cursor_invalid(cbt)) { + *exact = cbt->compare; + ret = __wt_kv_return( + session, cbt, cbt->compare == 0 ? 0 : 1); + } else if ((ret = __wt_btcur_prev(cbt)) != WT_NOTFOUND) + *exact = -1; + } + +err: __cursor_func_resolve(cbt, ret); + return (ret); +} + +/* + * __wt_btcur_insert -- + * Insert a record into the tree. + */ +int +__wt_btcur_insert(WT_CURSOR_BTREE *cbt) +{ + WT_BTREE *btree; + WT_CURSOR *cursor; + WT_SESSION_IMPL *session; + int ret; + + btree = cbt->btree; + cursor = &cbt->iface; + session = (WT_SESSION_IMPL *)cursor->session; + WT_BSTAT_INCR(session, cursor_inserts); + + if (btree->type == BTREE_ROW) + WT_RET(__cursor_size_chk(session, &cursor->key)); + WT_RET(__cursor_size_chk(session, &cursor->value)); + +retry: __cursor_func_init(cbt, 1); + + switch (btree->type) { + case BTREE_COL_FIX: + case BTREE_COL_VAR: + /* + * If WT_CURSTD_APPEND set insert a new record (ignoring the + * application's record number), return the record number. + */ + if (F_ISSET(cursor, WT_CURSTD_APPEND)) { + if ((ret = + __wt_col_modify(session, cbt, 1)) == WT_RESTART) + goto retry; + cbt->iface.recno = cbt->recno; + break; + } + + WT_ERR(__wt_col_search(session, cbt, 1)); + + /* + * If WT_CURSTD_OVERWRITE set, insert/update the key/value pair. + * + * If WT_CURSTD_OVERWRITE not set, fail if the key exists, else + * insert the key/value pair. Creating a record past the end + * of the tree in a fixed-length column-store implicitly fills + * the gap with empty records. Fail in that case, the record + * exists. + */ + if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE) && + ((cbt->compare == 0 && !__cursor_invalid(cbt)) || + (cbt->compare != 0 && __cursor_fix_implicit(btree, cbt)))) { + ret = WT_DUPLICATE_KEY; + break; + } + if ((ret = __wt_col_modify(session, cbt, 3)) == WT_RESTART) + goto retry; + break; + case BTREE_ROW: + /* + * If WT_CURSTD_OVERWRITE not set, fail if the key exists, else + * insert the key/value pair. + * + * If WT_CURSTD_OVERWRITE set, insert/update the key/value pair. + */ + WT_ERR(__wt_row_search(session, cbt, 1)); + if (cbt->compare == 0 && + !__cursor_invalid(cbt) && + !F_ISSET(cursor, WT_CURSTD_OVERWRITE)) { + ret = WT_DUPLICATE_KEY; + break; + } + if ((ret = __wt_row_modify(session, cbt, 0)) == WT_RESTART) + goto retry; + break; + WT_ILLEGAL_VALUE(session); + } + +err: __cursor_func_resolve(cbt, ret); + + return (ret); +} + +/* + * __wt_btcur_remove -- + * Remove a record from the tree. + */ +int +__wt_btcur_remove(WT_CURSOR_BTREE *cbt) +{ + WT_BTREE *btree; + WT_CURSOR *cursor; + WT_SESSION_IMPL *session; + int ret; + + btree = cbt->btree; + cursor = &cbt->iface; + session = (WT_SESSION_IMPL *)cursor->session; + WT_BSTAT_INCR(session, cursor_removes); + + if (btree->type == BTREE_ROW) + WT_RET(__cursor_size_chk(session, &cursor->key)); + +retry: __cursor_func_init(cbt, 1); + + switch (btree->type) { + case BTREE_COL_FIX: + case BTREE_COL_VAR: + WT_ERR(__wt_col_search(session, cbt, 1)); + + /* + * Remove the record if it exists. Creating a record past the + * end of the tree in a fixed-length column-store implicitly + * fills the gap with empty records. Return success in that + * case, the record was deleted successfully. + */ + if (cbt->compare != 0 || __cursor_invalid(cbt)) + ret = + __cursor_fix_implicit(btree, cbt) ? 0 : WT_NOTFOUND; + else if ((ret = __wt_col_modify(session, cbt, 2)) == WT_RESTART) + goto retry; + break; + case BTREE_ROW: + /* Remove the record if it exists. */ + WT_ERR(__wt_row_search(session, cbt, 1)); + if (cbt->compare != 0 || __cursor_invalid(cbt)) + ret = WT_NOTFOUND; + else if ((ret = __wt_row_modify(session, cbt, 1)) == WT_RESTART) + goto retry; + break; + WT_ILLEGAL_VALUE(session); + } + +err: __cursor_func_resolve(cbt, ret); + + return (ret); +} + +/* + * __wt_btcur_update -- + * Update a record in the tree. + */ +int +__wt_btcur_update(WT_CURSOR_BTREE *cbt) +{ + WT_BTREE *btree; + WT_CURSOR *cursor; + WT_SESSION_IMPL *session; + int ret; + + btree = cbt->btree; + cursor = &cbt->iface; + session = (WT_SESSION_IMPL *)cursor->session; + WT_BSTAT_INCR(session, cursor_updates); + + if (btree->type == BTREE_ROW) + WT_RET(__cursor_size_chk(session, &cursor->key)); + WT_RET(__cursor_size_chk(session, &cursor->value)); + +retry: __cursor_func_init(cbt, 1); + + switch (btree->type) { + case BTREE_COL_FIX: + if (cursor->value.size != 1) + WT_RET_MSG(session, EINVAL, + "item size of %" PRIu32 " does not match " + "fixed-length file requirement of 1 byte", + cursor->value.size); + /* FALLTHROUGH */ + case BTREE_COL_VAR: + WT_ERR(__wt_col_search(session, cbt, 1)); + + /* + * Update the record if it exists. Creating a record past the + * end of the tree in a fixed-length column-store implicitly + * fills the gap with empty records. Update the record in that + * case, the record exists. + */ + if ((cbt->compare != 0 || __cursor_invalid(cbt)) && + !__cursor_fix_implicit(btree, cbt)) + ret = WT_NOTFOUND; + else if ((ret = __wt_col_modify(session, cbt, 3)) == WT_RESTART) + goto retry; + break; + case BTREE_ROW: + /* Update the record it it exists. */ + WT_ERR(__wt_row_search(session, cbt, 1)); + if (cbt->compare != 0 || __cursor_invalid(cbt)) + ret = WT_NOTFOUND; + else if ((ret = __wt_row_modify(session, cbt, 0)) == WT_RESTART) + goto retry; + break; + WT_ILLEGAL_VALUE(session); + } + +err: __cursor_func_resolve(cbt, ret); + + return (ret); +} + +/* + * __wt_btcur_close -- + * Close a btree cursor. + */ +int +__wt_btcur_close(WT_CURSOR_BTREE *cbt) +{ + WT_SESSION_IMPL *session; + + session = (WT_SESSION_IMPL *)cbt->iface.session; + + __cursor_func_init(cbt, 1); + __wt_buf_free(session, &cbt->tmp); + + return (0); +} diff --git a/src/btree/bt_debug.c b/src/btree/bt_debug.c new file mode 100644 index 00000000000..a3f9511a6ee --- /dev/null +++ b/src/btree/bt_debug.c @@ -0,0 +1,949 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +#ifdef HAVE_DIAGNOSTIC +/* + * We pass around a session handle and output information, group it together. + */ +typedef struct { + WT_SESSION_IMPL *session; /* Enclosing session */ + + /* + * When using the standard event handlers, the debugging output has to + * do its own message handling because its output isn't line-oriented. + */ + FILE *fp; /* Output file stream */ + WT_ITEM *msg; /* Buffered message */ + + WT_ITEM *tmp; /* Temporary space */ +} WT_DBG; + +/* Diagnostic output separator. */ +static const char *sep = "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=\n"; + +static int __debug_cell(WT_DBG *, WT_PAGE_HEADER *, WT_CELL_UNPACK *); +static int __debug_cell_data(WT_DBG *, const char *, WT_CELL_UNPACK *); +static void __debug_col_skip(WT_DBG *, WT_INSERT_HEAD *, const char *, int); +static int __debug_config(WT_SESSION_IMPL *, WT_DBG *, const char *); +static int __debug_dsk_cell(WT_DBG *, WT_PAGE_HEADER *); +static void __debug_dsk_col_fix(WT_DBG *, WT_PAGE_HEADER *); +static void __debug_ikey(WT_DBG *, WT_IKEY *); +static void __debug_item(WT_DBG *, const char *, const void *, size_t); +static int __debug_page(WT_DBG *, WT_PAGE *, uint32_t); +static void __debug_page_col_fix(WT_DBG *, WT_PAGE *); +static int __debug_page_col_int(WT_DBG *, WT_PAGE *, uint32_t); +static int __debug_page_col_var(WT_DBG *, WT_PAGE *); +static int __debug_page_hdr(WT_DBG *, WT_PAGE *); +static int __debug_page_modify(WT_DBG *, WT_PAGE *); +static int __debug_page_row_int(WT_DBG *, WT_PAGE *, uint32_t); +static int __debug_page_row_leaf(WT_DBG *, WT_PAGE *); +static int __debug_ref(WT_DBG *, WT_REF *, WT_PAGE *); +static void __debug_row_skip(WT_DBG *, WT_INSERT_HEAD *); +static int __debug_tree(WT_SESSION_IMPL *, WT_PAGE *, const char *, uint32_t); +static void __debug_update(WT_DBG *, WT_UPDATE *, int); +static void __dmsg(WT_DBG *, const char *, ...) + WT_GCC_ATTRIBUTE((format (printf, 2, 3))); +static void __dmsg_wrapup(WT_DBG *); + +/* + * __debug_hex_byte -- + * Output a single byte in hex. + */ +static inline void +__debug_hex_byte(WT_DBG *ds, uint8_t v) +{ + static const char hex[] = "0123456789abcdef"; + + __dmsg(ds, "#%c%c", hex[(v & 0xf0) >> 4], hex[v & 0x0f]); +} + +/* + * __debug_config -- + * Configure debugging output. + */ +static int +__debug_config(WT_SESSION_IMPL *session, WT_DBG *ds, const char *ofile) +{ + memset(ds, 0, sizeof(WT_DBG)); + + ds->session = session; + + WT_RET(__wt_scr_alloc(session, 512, &ds->tmp)); + + /* + * If we weren't given a file, we use the default event handler, and + * we'll have to buffer messages. + */ + if (ofile == NULL) + return (__wt_scr_alloc(session, 512, &ds->msg)); + + /* If we're using a file, flush on each line. */ + if ((ds->fp = fopen(ofile, "w")) == NULL) + WT_RET_MSG(session, __wt_errno(), "%s", ofile); + + (void)setvbuf(ds->fp, NULL, _IOLBF, 0); + return (0); +} + +/* + * __dmsg_wrapup -- + * Flush any remaining output, release resources. + */ +static void +__dmsg_wrapup(WT_DBG *ds) +{ + WT_SESSION_IMPL *session; + WT_ITEM *msg; + + session = ds->session; + msg = ds->msg; + + __wt_scr_free(&ds->tmp); + + /* + * Discard the buffer -- it shouldn't have anything in it, but might + * as well be cautious. + */ + if (msg != NULL) { + if (msg->size != 0) + __wt_msg(session, "%s", (char *)msg->mem); + __wt_scr_free(&ds->msg); + } + + /* Close any file we opened. */ + if (ds->fp != NULL) + (void)fclose(ds->fp); +} + +/* + * __dmsg -- + * Debug message. + */ +static void +__dmsg(WT_DBG *ds, const char *fmt, ...) +{ + va_list ap; + WT_ITEM *msg; + WT_SESSION_IMPL *session; + size_t len, space; + char *p; + + session = ds->session; + + /* + * Debug output chunks are not necessarily terminated with a newline + * character. It's easy if we're dumping to a stream, but if we're + * dumping to an event handler, which is line-oriented, we must buffer + * the output chunk, and pass it to the event handler once we see a + * terminating newline. + */ + if (ds->fp == NULL) { + msg = ds->msg; + for (;;) { + p = (char *)msg->mem + msg->size; + space = msg->memsize - msg->size; + va_start(ap, fmt); + len = (size_t)vsnprintf(p, space, fmt, ap); + va_end(ap); + + /* Check if there was enough space. */ + if (len < space) { + msg->size += (uint32_t)len; + break; + } + + /* + * There's not much to do on error without checking for + * an error return on every single printf. Anyway, it's + * pretty unlikely and this is debugging output, I'm not + * going to worry about it. + */ + if (__wt_buf_grow( + session, msg, msg->memsize + len + 128) != 0) + return; + } + if (((uint8_t *)msg->mem)[msg->size - 1] == '\n') { + ((uint8_t *)msg->mem)[msg->size - 1] = '\0'; + __wt_msg(session, "%s", (char *)msg->mem); + msg->size = 0; + } + } else { + va_start(ap, fmt); + (void)vfprintf(ds->fp, fmt, ap); + va_end(ap); + } +} + +/* + * __wt_debug_addr -- + * Read and dump a disk page in debugging mode. + */ +int +__wt_debug_addr( + WT_SESSION_IMPL *session, uint32_t addr, uint32_t size, const char *ofile) +{ + WT_ITEM *buf; + int ret; + + buf = NULL; + ret = 0; + + WT_RET(__wt_scr_alloc(session, size, &buf)); + WT_ERR(__wt_block_read( + session, session->btree->block, buf, addr, size, 0)); + ret = __wt_debug_disk(session, buf->mem, ofile); +err: __wt_scr_free(&buf); + + return (ret); +} + +/* + * __wt_debug_disk -- + * Dump a disk page in debugging mode. + */ +int +__wt_debug_disk( + WT_SESSION_IMPL *session, WT_PAGE_HEADER *dsk, const char *ofile) +{ + WT_DBG *ds, _ds; + int ret; + + ret = 0; + + ds = &_ds; + WT_RET(__debug_config(session, ds, ofile)); + + __dmsg(ds, "%s page", __wt_page_type_string(dsk->type)); + switch (dsk->type) { + case WT_PAGE_COL_FIX: + case WT_PAGE_COL_INT: + case WT_PAGE_COL_VAR: + __dmsg(ds, ", recno %" PRIu64, dsk->recno); + /* FALLTHROUGH */ + case WT_PAGE_ROW_INT: + case WT_PAGE_ROW_LEAF: + __dmsg(ds, ", entries %" PRIu32 "\n", dsk->u.entries); + break; + case WT_PAGE_OVFL: + __dmsg(ds, ", datalen %" PRIu32 "\n", dsk->u.datalen); + break; + WT_ILLEGAL_VALUE(session); + } + + switch (dsk->type) { + case WT_PAGE_COL_FIX: + __debug_dsk_col_fix(ds, dsk); + break; + case WT_PAGE_COL_INT: + case WT_PAGE_COL_VAR: + case WT_PAGE_ROW_INT: + case WT_PAGE_ROW_LEAF: + ret = __debug_dsk_cell(ds, dsk); + break; + default: + break; + } + + __dmsg_wrapup(ds); + + return (ret); +} + +/* + * __debug_dsk_col_fix -- + * Dump a WT_PAGE_COL_FIX page. + */ +static void +__debug_dsk_col_fix(WT_DBG *ds, WT_PAGE_HEADER *dsk) +{ + WT_BTREE *btree; + uint32_t i; + uint8_t v; + + btree = ds->session->btree; + + WT_FIX_FOREACH(btree, dsk, v, i) { + __dmsg(ds, "\t{"); + __debug_hex_byte(ds, v); + __dmsg(ds, "}\n"); + } +} + +/* + * __debug_dsk_cell -- + * Dump a page of WT_CELL's. + */ +static int +__debug_dsk_cell(WT_DBG *ds, WT_PAGE_HEADER *dsk) +{ + WT_BTREE *btree; + WT_CELL *cell; + WT_CELL_UNPACK *unpack, _unpack; + uint32_t i; + + btree = ds->session->btree; + unpack = &_unpack; + + WT_CELL_FOREACH(btree, dsk, cell, unpack, i) { + __wt_cell_unpack(cell, unpack); + WT_RET(__debug_cell(ds, dsk, unpack)); + } + return (0); +} + +#define WT_DEBUG_TREE_LEAF 0x01 /* Debug leaf pages */ +#define WT_DEBUG_TREE_WALK 0x02 /* Descend the tree */ + +/* + * __wt_debug_tree_all -- + * Dump the in-memory information for a tree, including leaf pages. + */ +int +__wt_debug_tree_all(WT_SESSION_IMPL *session, WT_PAGE *page, const char *ofile) +{ + return (__debug_tree( + session, page, ofile, WT_DEBUG_TREE_LEAF | WT_DEBUG_TREE_WALK)); +} + +/* + * __wt_debug_tree -- + * Dump the in-memory information for a tree, not including leaf pages. + */ +int +__wt_debug_tree(WT_SESSION_IMPL *session, WT_PAGE *page, const char *ofile) +{ + return (__debug_tree(session, page, ofile, WT_DEBUG_TREE_WALK)); +} + +/* + * __wt_debug_page -- + * Dump the in-memory information for a page. + */ +int +__wt_debug_page(WT_SESSION_IMPL *session, WT_PAGE *page, const char *ofile) +{ + WT_DBG *ds, _ds; + int ret; + + ds = &_ds; + WT_RET(__debug_config(session, ds, ofile)); + + ret = __debug_page(ds, page, WT_DEBUG_TREE_LEAF); + + __dmsg_wrapup(ds); + + return (ret); +} + +/* + * __debug_tree -- + * Dump the in-memory information for a tree. + */ +static int +__debug_tree( + WT_SESSION_IMPL *session, WT_PAGE *page, const char *ofile, uint32_t flags) +{ + WT_DBG *ds, _ds; + int ret; + + ds = &_ds; + WT_RET(__debug_config(session, ds, ofile)); + + /* A NULL page starts at the top of the tree -- it's a convenience. */ + if (page == NULL) + page = session->btree->root_page; + + ret = __debug_page(ds, page, flags); + + __dmsg_wrapup(ds); + + return (ret); +} + +/* + * __debug_page -- + * Dump the in-memory information for an in-memory page. + */ +static int +__debug_page(WT_DBG *ds, WT_PAGE *page, uint32_t flags) +{ + WT_SESSION_IMPL *session; + + session = ds->session; + + /* Dump the page header. */ + WT_RET(__debug_page_hdr(ds, page)); + + /* Dump the page modification structure. */ + WT_RET(__debug_page_modify(ds, page)); + + /* Dump the page. */ + switch (page->type) { + case WT_PAGE_COL_FIX: + if (LF_ISSET(WT_DEBUG_TREE_LEAF)) + __debug_page_col_fix(ds, page); + break; + case WT_PAGE_COL_INT: + WT_RET(__debug_page_col_int(ds, page, flags)); + break; + case WT_PAGE_COL_VAR: + if (LF_ISSET(WT_DEBUG_TREE_LEAF)) + WT_RET(__debug_page_col_var(ds, page)); + break; + case WT_PAGE_ROW_INT: + WT_RET(__debug_page_row_int(ds, page, flags)); + break; + case WT_PAGE_ROW_LEAF: + if (LF_ISSET(WT_DEBUG_TREE_LEAF)) + WT_RET(__debug_page_row_leaf(ds, page)); + break; + WT_ILLEGAL_VALUE(session); + } + + return (0); +} + +/* + * __debug_page_hdr -- + * Dump an in-memory page's metadata. + */ +static int +__debug_page_hdr(WT_DBG *ds, WT_PAGE *page) +{ + WT_SESSION_IMPL *session; + + session = ds->session; + + __dmsg(ds, "%p %s", + page, __wt_page_addr_string(session, ds->tmp, page)); + + switch (page->type) { + case WT_PAGE_COL_INT: + __dmsg(ds, " recno %" PRIu64, page->u.intl.recno); + break; + case WT_PAGE_COL_FIX: + __dmsg(ds, " recno %" PRIu64, page->u.col_fix.recno); + break; + case WT_PAGE_COL_VAR: + __dmsg(ds, " recno %" PRIu64, page->u.col_var.recno); + break; + case WT_PAGE_ROW_INT: + case WT_PAGE_ROW_LEAF: + break; + WT_ILLEGAL_VALUE(session); + } + + __dmsg(ds, ": %s", __wt_page_type_string(page->type)); + + __dmsg(ds, " (%s", __wt_page_is_modified(page) ? "dirty" : "clean"); + if (F_ISSET(page, WT_PAGE_FORCE_EVICT)) + __dmsg(ds, ", force-evict"); + if (F_ISSET(page, WT_PAGE_PINNED)) + __dmsg(ds, ", pinned"); + if (F_ISSET(page, WT_PAGE_REC_EMPTY)) + __dmsg(ds, ", empty"); + if (F_ISSET(page, WT_PAGE_REC_REPLACE)) + __dmsg(ds, ", replaced"); + if (F_ISSET(page, WT_PAGE_REC_SPLIT)) + __dmsg(ds, ", split"); + if (F_ISSET(page, WT_PAGE_REC_SPLIT_MERGE)) + __dmsg(ds, ", split-merge"); + __dmsg(ds, ")\n"); + + if (WT_PAGE_IS_ROOT(page)) + __dmsg(ds, "\troot"); + else + __dmsg(ds, "\tparent %p", page->parent); + __dmsg(ds, + ", disk %p, entries %" PRIu32 "\n", page->dsk, page->entries); + + return (0); +} + +/* + * __debug_page_modify -- + * Dump an in-memory page's modification structure. + */ +static int +__debug_page_modify(WT_DBG *ds, WT_PAGE *page) +{ + WT_PAGE_MODIFY *mod; + WT_PAGE_TRACK *track; + WT_SESSION_IMPL *session; + uint32_t i; + + session = ds->session; + + if ((mod = page->modify) == NULL) + return (0); + + __dmsg(ds, + "\t" "write/disk generations: %" PRIu32 "/%" PRIu32 "\n", + mod->write_gen, mod->disk_gen); + + switch (F_ISSET(page, WT_PAGE_REC_MASK)) { + case 0: + break; + case WT_PAGE_REC_EMPTY: + __dmsg(ds, "\t" "empty page\n"); + break; + case WT_PAGE_REC_REPLACE: + __dmsg(ds, "\t" "replacement %s\n", + __wt_addr_string(session, ds->tmp, + mod->u.replace.addr, mod->u.replace.size)); + break; + case WT_PAGE_REC_SPLIT: + __dmsg(ds, "\t" "split page %p\n", mod->u.split); + break; + case WT_PAGE_REC_SPLIT_MERGE: + __dmsg(ds, "\t" "split-merge page %p\n", mod->u.split); + break; + WT_ILLEGAL_VALUE(session); + } + + if (mod->track_entries != 0) + __dmsg(ds, "\t" "tracking list:\n"); + for (track = mod->track, i = 0; i < mod->track_entries; ++track, ++i) { + switch (track->type) { + case WT_PT_BLOCK: + __dmsg(ds, "\t\t" "block"); + break; + case WT_PT_BLOCK_EVICT: + __dmsg(ds, "\t\t" "block-evict"); + break; + case WT_PT_OVFL: + __dmsg(ds, "\t\t" "overflow (on)"); + break; + case WT_PT_OVFL_DISCARD: + __dmsg(ds, "\t\t" "overflow (off)"); + break; + case WT_PT_EMPTY: + continue; + WT_ILLEGAL_VALUE(session); + } + __dmsg(ds, " %s\n", __wt_addr_string( + session, ds->tmp, track->addr.addr, track->addr.size)); + } + + return (0); +} + +/* + * __debug_page_col_fix -- + * Dump an in-memory WT_PAGE_COL_FIX page. + */ +static void +__debug_page_col_fix(WT_DBG *ds, WT_PAGE *page) +{ + WT_BTREE *btree; + WT_INSERT *ins; + WT_PAGE_HEADER *dsk; + WT_SESSION_IMPL *session; + uint64_t recno; + uint32_t i; + uint8_t v; + + session = ds->session; + btree = session->btree; + dsk = page->dsk; + recno = page->u.col_fix.recno; + + if (dsk != NULL) { + ins = WT_SKIP_FIRST(WT_COL_UPDATE_SINGLE(page)); + WT_FIX_FOREACH(btree, dsk, v, i) { + __dmsg(ds, "\t%" PRIu64 "\t{", recno); + __debug_hex_byte(ds, v); + __dmsg(ds, "}\n"); + + /* Check for a match on the update list. */ + if (ins != NULL && WT_INSERT_RECNO(ins) == recno) { + __dmsg(ds, + "\tupdate %" PRIu64 "\n", + WT_INSERT_RECNO(ins)); + __debug_update(ds, ins->upd, 1); + ins = WT_SKIP_NEXT(ins); + } + ++recno; + } + } + + if (WT_COL_UPDATE_SINGLE(page) != NULL) { + __dmsg(ds, "%s", sep); + __debug_col_skip(ds, WT_COL_UPDATE_SINGLE(page), "update", 1); + } + if (WT_COL_APPEND(btree, page) != NULL) { + __dmsg(ds, "%s", sep); + __debug_col_skip(ds, WT_COL_APPEND(btree, page), "append", 1); + } +} + +/* + * __debug_page_col_int -- + * Dump an in-memory WT_PAGE_COL_INT page. + */ +static int +__debug_page_col_int(WT_DBG *ds, WT_PAGE *page, uint32_t flags) +{ + WT_REF *ref; + uint32_t i; + + WT_REF_FOREACH(page, ref, i) { + __dmsg(ds, "\trecno %" PRIu64 "\n", ref->u.recno); + WT_RET(__debug_ref(ds, ref, page)); + } + + if (LF_ISSET(WT_DEBUG_TREE_WALK)) + WT_REF_FOREACH(page, ref, i) + if (ref->state == WT_REF_MEM) { + __dmsg(ds, "\n"); + WT_RET(__debug_page(ds, ref->page, flags)); + } + + return (0); +} + +/* + * __debug_page_col_var -- + * Dump an in-memory WT_PAGE_COL_VAR page. + */ +static int +__debug_page_col_var(WT_DBG *ds, WT_PAGE *page) +{ + WT_BTREE *btree; + WT_CELL *cell; + WT_CELL_UNPACK *unpack, _unpack; + WT_COL *cip; + WT_INSERT_HEAD *update; + uint64_t recno, rle; + uint32_t i; + char tag[64]; + + btree = ds->session->btree; + unpack = &_unpack; + recno = page->u.col_var.recno; + + WT_COL_FOREACH(page, cip, i) { + if ((cell = WT_COL_PTR(page, cip)) == NULL) { + unpack = NULL; + rle = 1; + } else { + __wt_cell_unpack(cell, unpack); + rle = __wt_cell_rle(unpack); + } + snprintf(tag, sizeof(tag), "%" PRIu64 " %" PRIu64, recno, rle); + WT_RET(__debug_cell_data(ds, tag, unpack)); + + if ((update = WT_COL_UPDATE(page, cip)) != NULL) + __debug_col_skip(ds, update, "update", 0); + recno += rle; + } + + if (WT_COL_APPEND(btree, page) != NULL) { + __dmsg(ds, "%s", sep); + __debug_col_skip(ds, WT_COL_APPEND(btree, page), "append", 0); + } + + return (0); +} + +/* + * __debug_page_row_int -- + * Dump an in-memory WT_PAGE_ROW_INT page. + */ +static int +__debug_page_row_int(WT_DBG *ds, WT_PAGE *page, uint32_t flags) +{ + WT_REF *ref; + uint32_t i; + + WT_REF_FOREACH(page, ref, i) { + __debug_ikey(ds, ref->u.key); + WT_RET(__debug_ref(ds, ref, page)); + } + + if (LF_ISSET(WT_DEBUG_TREE_WALK)) + WT_REF_FOREACH(page, ref, i) + if (ref->state == WT_REF_MEM) { + __dmsg(ds, "\n"); + WT_RET(__debug_page(ds, ref->page, flags)); + } + return (0); +} + +/* + * __debug_page_row_leaf -- + * Dump an in-memory WT_PAGE_ROW_LEAF page. + */ +static int +__debug_page_row_leaf(WT_DBG *ds, WT_PAGE *page) +{ + WT_CELL *cell; + WT_CELL_UNPACK *unpack, _unpack; + WT_INSERT_HEAD *insert; + WT_ROW *rip; + WT_UPDATE *upd; + uint32_t i; + + unpack = &_unpack; + + /* + * Dump any K/V pairs inserted into the page before the first from-disk + * key on the page. + */ + if ((insert = WT_ROW_INSERT_SMALLEST(page)) != NULL) + __debug_row_skip(ds, insert); + + /* Dump the page's K/V pairs. */ + WT_ROW_FOREACH(page, rip, i) { + if (__wt_off_page(page, rip->key)) + __debug_ikey(ds, rip->key); + else { + __wt_cell_unpack(rip->key, unpack); + WT_RET(__debug_cell_data(ds, "K", unpack)); + } + + if ((cell = __wt_row_value(page, rip)) == NULL) + __dmsg(ds, "\tV {}\n"); + else { + __wt_cell_unpack(cell, unpack); + WT_RET(__debug_cell_data(ds, "V", unpack)); + } + + if ((upd = WT_ROW_UPDATE(page, rip)) != NULL) + __debug_update(ds, upd, 0); + + if ((insert = WT_ROW_INSERT(page, rip)) != NULL) + __debug_row_skip(ds, insert); + } + + return (0); +} + +/* + * __debug_col_skip -- + * Dump a column-store skiplist. + */ +static void +__debug_col_skip(WT_DBG *ds, WT_INSERT_HEAD *head, const char *tag, int hexbyte) +{ + WT_INSERT *ins; + + WT_SKIP_FOREACH(ins, head) { + __dmsg(ds, + "\t%s %" PRIu64 "\n", tag, WT_INSERT_RECNO(ins)); + __debug_update(ds, ins->upd, hexbyte); + } +} + +/* + * __debug_row_skip -- + * Dump an insert array. + */ +static void +__debug_row_skip(WT_DBG *ds, WT_INSERT_HEAD *head) +{ + WT_INSERT *ins; + + WT_SKIP_FOREACH(ins, head) { + __debug_item(ds, + "insert", WT_INSERT_KEY(ins), WT_INSERT_KEY_SIZE(ins)); + __debug_update(ds, ins->upd, 0); + } +} + +/* + * __debug_update -- + * Dump an update array. + */ +static void +__debug_update(WT_DBG *ds, WT_UPDATE *upd, int hexbyte) +{ + for (; upd != NULL; upd = upd->next) + if (WT_UPDATE_DELETED_ISSET(upd)) + __dmsg(ds, "\tvalue {deleted}\n"); + else if (hexbyte) { + __dmsg(ds, "\t{"); + __debug_hex_byte(ds, + ((uint8_t *)WT_UPDATE_DATA(upd))[0]); + __dmsg(ds, "}\n"); + } else + __debug_item(ds, + "value", WT_UPDATE_DATA(upd), upd->size); +} + +/* + * __debug_ref -- + * Dump a WT_REF structure. + */ +static int +__debug_ref(WT_DBG *ds, WT_REF *ref, WT_PAGE *page) +{ + WT_SESSION_IMPL *session; + uint32_t size; + const uint8_t *addr; + + session = ds->session; + + __dmsg(ds, "\t"); + switch (ref->state) { + case WT_REF_DISK: + __dmsg(ds, "disk"); + break; + case WT_REF_EVICTING: + __dmsg(ds, "evicting %p", ref->page); + break; + case WT_REF_LOCKED: + __dmsg(ds, "locked %p", ref->page); + break; + case WT_REF_MEM: + __dmsg(ds, "memory %p", ref->page); + break; + case WT_REF_READING: + __dmsg(ds, "reading"); + break; + WT_ILLEGAL_VALUE(session); + } + + if (ref->addr == NULL) + __dmsg(ds, " %s\n", "[NoAddr]"); + else { + __wt_get_addr(page, ref, &addr, &size); + __dmsg(ds, + " %s\n", __wt_addr_string(session, ds->tmp, addr, size)); + } + return (0); +} + +/* + * __debug_cell -- + * Dump a single unpacked WT_CELL. + */ +static int +__debug_cell(WT_DBG *ds, WT_PAGE_HEADER *dsk, WT_CELL_UNPACK *unpack) +{ + WT_ITEM *buf; + WT_SESSION_IMPL *session; + int ret; + + session = ds->session; + buf = NULL; + + __dmsg(ds, "\t%s: len %" PRIu32, + __wt_cell_type_string(unpack->raw), unpack->size); + + switch (unpack->type) { + case WT_CELL_DEL: + case WT_CELL_VALUE: + /* + * Column-store internal page value cells include a record + * number, column-store leaf page value cells include a RLE; + * row-store leaf page value cells have no associated value. + */ + if (unpack->v != 0) + __dmsg(ds, ", %s: %" PRIu64, + dsk->type == WT_PAGE_COL_INT ? "recno" : "rle", + unpack->v); + break; + case WT_CELL_KEY: + __dmsg(ds, ", pfx: %" PRIu8, unpack->prefix); + break; + case WT_CELL_ADDR: + case WT_CELL_KEY_OVFL: + case WT_CELL_VALUE_OVFL: + WT_RET(__wt_scr_alloc(session, 64, &buf)); + if ((ret = __wt_bm_addr_string( + session, buf, unpack->data, unpack->size)) == 0) + __dmsg(ds, ", %s %s", + unpack->type == WT_CELL_ADDR ? "addr" : "ovfl", + (char *)buf->data); + __wt_scr_free(&buf); + WT_RET(ret); + break; + WT_ILLEGAL_VALUE(session); + } + __dmsg(ds, "\n"); + + return (__debug_cell_data(ds, NULL, unpack)); +} + +/* + * __debug_cell_data -- + * Dump a single cell's data in debugging mode. + */ +static int +__debug_cell_data(WT_DBG *ds, const char *tag, WT_CELL_UNPACK *unpack) +{ + WT_ITEM *buf; + WT_SESSION_IMPL *session; + int ret; + + session = ds->session; + buf = NULL; + ret = 0; + + /* + * Column-store references to deleted cells return a NULL cell + * reference. + */ + if (unpack == NULL) + goto deleted; + + switch (unpack->type) { + case WT_CELL_ADDR: + __debug_item(ds, tag, "addr", strlen("addr")); + break; + case WT_CELL_DEL: +deleted: __debug_item(ds, tag, "deleted", strlen("deleted")); + break; + case WT_CELL_KEY: + case WT_CELL_KEY_OVFL: + case WT_CELL_VALUE: + case WT_CELL_VALUE_OVFL: + WT_RET(__wt_scr_alloc(session, 256, &buf)); + if ((ret = __wt_cell_unpack_copy(session, unpack, buf)) == 0) + __debug_item(ds, tag, buf->data, buf->size); + __wt_scr_free(&buf); + break; + WT_ILLEGAL_VALUE(session); + } + + return (ret); +} + +/* + * __debug_ikey -- + * Dump a single WT_IKEY in debugging mode, with an optional tag. + */ +static void +__debug_ikey(WT_DBG *ds, WT_IKEY *ikey) +{ + __debug_item(ds, "K", WT_IKEY_DATA(ikey), ikey->size); +} + +/* + * __debug_item -- + * Dump a single data/size pair, with an optional tag. + */ +static void +__debug_item(WT_DBG *ds, const char *tag, const void *data_arg, size_t size) +{ + const uint8_t *data; + int ch; + + __dmsg(ds, "\t%s%s{", tag == NULL ? "" : tag, tag == NULL ? "" : " "); + for (data = data_arg; size > 0; --size, ++data) { + ch = data[0]; + if (isprint(ch)) + __dmsg(ds, "%c", ch); + else + __debug_hex_byte(ds, data[0]); + } + __dmsg(ds, "}\n"); +} +#endif diff --git a/src/btree/bt_discard.c b/src/btree/bt_discard.c new file mode 100644 index 00000000000..7962ace7fb5 --- /dev/null +++ b/src/btree/bt_discard.c @@ -0,0 +1,306 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static void __free_page_col_fix(WT_SESSION_IMPL *, WT_PAGE *); +static void __free_page_col_int(WT_SESSION_IMPL *, WT_PAGE *); +static void __free_page_col_var(WT_SESSION_IMPL *, WT_PAGE *); +static void __free_page_row_int(WT_SESSION_IMPL *, WT_PAGE *); +static void __free_page_row_leaf(WT_SESSION_IMPL *, WT_PAGE *); +static void __free_skip_array(WT_SESSION_IMPL *, WT_INSERT_HEAD **, uint32_t); +static void __free_skip_list(WT_SESSION_IMPL *, WT_INSERT *); +static void __free_update(WT_SESSION_IMPL *, WT_UPDATE **, uint32_t); +static void __free_update_list(WT_SESSION_IMPL *, WT_UPDATE *); + +/* + * __wt_page_out -- + * Discard an in-memory page, freeing all memory associated with it. + */ +void +__wt_page_out(WT_SESSION_IMPL *session, WT_PAGE *page, uint32_t flags) +{ + /* + * When a page is discarded, it's been disconnected from its parent and + * parent's WT_REF structure may now point to a different page. Make + * sure we don't use any of that information by accident. + */ + page->parent = NULL; + page->ref = NULL; + + /* If not a split merged into its parent, the page must be clean. */ + WT_ASSERT(session, + !__wt_page_is_modified(page) || + F_ISSET(page, WT_PAGE_REC_SPLIT_MERGE)); + +#ifdef HAVE_DIAGNOSTIC + __wt_hazard_validate(session, page); +#endif + + /* + * If this page has a memory footprint associated with it, update + * the cache information. + */ + if (page->memory_footprint != 0) + __wt_cache_page_evict(session, page); + + switch (page->type) { + case WT_PAGE_COL_FIX: + __free_page_col_fix(session, page); + break; + case WT_PAGE_COL_INT: + __free_page_col_int(session, page); + break; + case WT_PAGE_COL_VAR: + __free_page_col_var(session, page); + break; + case WT_PAGE_ROW_INT: + __free_page_row_int(session, page); + break; + case WT_PAGE_ROW_LEAF: + __free_page_row_leaf(session, page); + break; + } + + if (!LF_ISSET(WT_PAGE_FREE_IGNORE_DISK)) /* Disk image */ + __wt_free(session, page->dsk); + + if (page->modify != NULL) { /* WT_PAGE_MODIFY */ + __wt_free(session, page->modify->track); + __wt_free(session, page->modify); + } + + __wt_free(session, page); +} + +/* + * __free_page_col_fix -- + * Discard a WT_PAGE_COL_FIX page. + */ +static void +__free_page_col_fix(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_BTREE *btree; + WT_INSERT_HEAD *append; + + btree = session->btree; + + /* Free the append array. */ + if ((append = WT_COL_APPEND(btree, page)) != NULL) { + __free_skip_list(session, WT_SKIP_FIRST(append)); + __wt_free(session, append); + __wt_free(session, btree->append); + } + + /* Free the update array. */ + if (page->modify != NULL && page->modify->update != NULL) + __free_skip_array(session, page->modify->update, 1); +} + +/* + * __free_page_col_int -- + * Discard a WT_PAGE_COL_INT page. + */ +static void +__free_page_col_int(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_REF *ref; + uint32_t i; + + /* + * For each referenced addr, see if the addr was an allocation, and if + * so, free it. + */ + WT_REF_FOREACH(page, ref, i) + if (ref->addr != NULL && + __wt_off_page(page, ref->addr)) { + __wt_free(session, ((WT_ADDR *)ref->addr)->addr); + __wt_free(session, ref->addr); + } + + /* Free the subtree-reference array. */ + __wt_free(session, page->u.intl.t); +} + +/* + * __free_page_col_var -- + * Discard a WT_PAGE_COL_VAR page. + */ +static void +__free_page_col_var(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_BTREE *btree; + WT_INSERT_HEAD *append; + + btree = session->btree; + + /* Free the in-memory index array. */ + __wt_free(session, page->u.col_var.d); + + /* Free the RLE lookup array. */ + __wt_free(session, page->u.col_var.repeats); + + /* Free the append array. */ + if ((append = WT_COL_APPEND(btree, page)) != NULL) { + __free_skip_list(session, WT_SKIP_FIRST(append)); + __wt_free(session, append); + __wt_free(session, btree->append); + } + + /* Free the insert array. */ + if (page->modify != NULL && page->modify->update != NULL) + __free_skip_array(session, page->modify->update, page->entries); +} + +/* + * __free_page_row_int -- + * Discard a WT_PAGE_ROW_INT page. + */ +static void +__free_page_row_int(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_IKEY *ikey; + WT_REF *ref; + uint32_t i; + + /* + * Free any allocated keys. + * + * For each referenced addr, see if the addr was an allocation, and if + * so, free it. + */ + WT_REF_FOREACH(page, ref, i) { + if ((ikey = ref->u.key) != NULL) + __wt_free(session, ikey); + if (ref->addr != NULL && + __wt_off_page(page, ref->addr)) { + __wt_free(session, ((WT_ADDR *)ref->addr)->addr); + __wt_free(session, ref->addr); + } + } + + /* Free the subtree-reference array. */ + __wt_free(session, page->u.intl.t); +} + +/* + * __free_page_row_leaf -- + * Discard a WT_PAGE_ROW_LEAF page. + */ +static void +__free_page_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_IKEY *ikey; + WT_ROW *rip; + uint32_t i; + + /* + * Free the in-memory index array. + * + * For each entry, see if the key was an allocation (that is, if it + * points somewhere other than the original page), and if so, free + * the memory. + */ + WT_ROW_FOREACH(page, rip, i) + if ((ikey = rip->key) != NULL && __wt_off_page(page, ikey)) + __wt_free(session, ikey); + __wt_free(session, page->u.row.d); + + /* + * Free the insert array. + * + * Row-store tables have one additional slot in the insert array (the + * insert array has an extra slot to hold keys that sort before keys + * found on the original page). + */ + if (page->u.row.ins != NULL) + __free_skip_array(session, page->u.row.ins, page->entries + 1); + + /* Free the update array. */ + if (page->u.row.upd != NULL) + __free_update(session, page->u.row.upd, page->entries); +} + +/* + * __free_skip_array -- + * Discard an array of skip list headers. + */ +static void +__free_skip_array( + WT_SESSION_IMPL *session, WT_INSERT_HEAD **head_arg, uint32_t entries) +{ + WT_INSERT_HEAD **head; + + /* + * For each non-NULL slot in the page's array of inserts, free the + * linked list anchored in that slot. + */ + for (head = head_arg; entries > 0; --entries, ++head) + if (*head != NULL) { + __free_skip_list(session, WT_SKIP_FIRST(*head)); + __wt_free(session, *head); + } + + /* Free the page's array of inserts. */ + __wt_free(session, head_arg); +} + +/* + * __free_skip_list -- + * Walk a WT_INSERT forward-linked list and free the per-thread combination + * of a WT_INSERT structure and its associated chain of WT_UPDATE structures. + */ +static void +__free_skip_list(WT_SESSION_IMPL *session, WT_INSERT *ins) +{ + WT_INSERT *next; + + do { + __free_update_list(session, ins->upd); + + next = WT_SKIP_NEXT(ins); + __wt_free(session, ins); + } while ((ins = next) != NULL); +} + +/* + * __free_update -- + * Discard the update array. + */ +static void +__free_update( + WT_SESSION_IMPL *session, WT_UPDATE **update_head, uint32_t entries) +{ + WT_UPDATE **updp; + + /* + * For each non-NULL slot in the page's array of updates, free the + * linked list anchored in that slot. + */ + for (updp = update_head; entries > 0; --entries, ++updp) + if (*updp != NULL) + __free_update_list(session, *updp); + + /* Free the page's array of updates. */ + __wt_free(session, update_head); +} + +/* + * __free_update_list -- + * Walk a WT_UPDATE forward-linked list and free the per-thread combination + * of a WT_UPDATE structure and its associated data. + */ +static void +__free_update_list(WT_SESSION_IMPL *session, WT_UPDATE *upd) +{ + WT_UPDATE *next; + + do { + next = upd->next; + __wt_free(session, upd); + } while ((upd = next) != NULL); +} diff --git a/src/btree/bt_evict.c b/src/btree/bt_evict.c new file mode 100644 index 00000000000..3a214882d50 --- /dev/null +++ b/src/btree/bt_evict.c @@ -0,0 +1,841 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static void __evict_dup_remove(WT_SESSION_IMPL *); +static int __evict_file(WT_SESSION_IMPL *, WT_EVICT_REQ *); +static int __evict_lru(WT_SESSION_IMPL *); +static int __evict_lru_cmp(const void *, const void *); +static void __evict_pages(WT_SESSION_IMPL *); +static int __evict_page_cmp(const void *, const void *); +static int __evict_request_walk(WT_SESSION_IMPL *); +static int __evict_walk(WT_SESSION_IMPL *); +static int __evict_walk_file(WT_SESSION_IMPL *, u_int *); +static int __evict_worker(WT_SESSION_IMPL *); + +/* + * Tuning constants: I hesitate to call this tuning, but we want to review some + * number of pages from each file's in-memory tree for each page we evict. + */ +#define WT_EVICT_GROUP 10 /* Evict N pages at a time */ +#define WT_EVICT_WALK_PER_TABLE 20 /* Pages to visit per file */ +#define WT_EVICT_WALK_BASE 100 /* Pages tracked across file visits */ + +/* + * WT_EVICT_REQ_FOREACH -- + * Walk a list of eviction requests. + */ +#define WT_EVICT_REQ_FOREACH(er, er_end, cache) \ + for ((er) = (cache)->evict_request, \ + (er_end) = (er) + (cache)->max_evict_request; \ + (er) < (er_end); ++(er)) + +/* + * __evict_clr -- + * Clear an entry in the eviction list. + */ +static inline void +__evict_clr(WT_EVICT_LIST *e) +{ + e->page = NULL; + e->btree = WT_DEBUG_POINT; +} + +/* + * __evict_req_set -- + * Set an entry in the eviction request list. + */ +static inline void +__evict_req_set( + WT_SESSION_IMPL *session, WT_EVICT_REQ *r, WT_PAGE *page, uint32_t flags) +{ + /* Should be empty */ + WT_ASSERT(session, r->session == NULL); + + WT_CLEAR(*r); + r->btree = session->btree; + r->page = page; + r->flags = flags; + + /* + * Publish: there must be a barrier to ensure the structure fields are + * set before the eviction thread can see the request. + */ + WT_PUBLISH(r->session, session); +} + +/* + * __evict_req_clr -- + * Clear an entry in the eviction request list. + */ +static inline void +__evict_req_clr(WT_SESSION_IMPL *session, WT_EVICT_REQ *r) +{ + WT_UNUSED(session); + + /* + * Publish; there must be a barrier to ensure the structure fields are + * set before the entry is made available for re-use. + */ + WT_PUBLISH(r->session, NULL); +} + +/* + * __wt_evict_server_wake -- + * Wake the eviction server thread. + */ +void +__wt_evict_server_wake(WT_SESSION_IMPL *session) +{ + WT_CACHE *cache; + WT_CONNECTION_IMPL *conn; + uint64_t bytes_inuse, bytes_max; + + conn = S2C(session); + cache = conn->cache; + bytes_inuse = __wt_cache_bytes_inuse(cache); + bytes_max = conn->cache_size; + + WT_VERBOSE(session, evictserver, + "waking, bytes inuse %s max (%" PRIu64 "MB %s %" PRIu64 "MB), ", + bytes_inuse <= bytes_max ? "<=" : ">", + bytes_inuse / WT_MEGABYTE, + bytes_inuse <= bytes_max ? "<=" : ">", + bytes_max / WT_MEGABYTE); + + __wt_cond_signal(session, cache->evict_cond); +} + +/* + * __evict_file_serial_func -- + * Eviction serialization function called when a tree is being flushed + * or closed. + */ +void +__wt_evict_file_serial_func(WT_SESSION_IMPL *session) +{ + WT_CACHE *cache; + WT_EVICT_REQ *er, *er_end; + int close_method; + + __wt_evict_file_unpack(session, &close_method); + + cache = S2C(session)->cache; + + /* Find an empty slot and enter the eviction request. */ + WT_EVICT_REQ_FOREACH(er, er_end, cache) + if (er->session == NULL) { + __evict_req_set(session, er, NULL, close_method ? + WT_EVICT_REQ_CLOSE : 0); + return; + } + + __wt_errx(session, "eviction server request table full"); + __wt_session_serialize_wrapup(session, NULL, WT_ERROR); +} + +/* + * __wt_evict_page_request -- + * Schedule a page for forced eviction due to a high volume of inserts or + * updates. + * + * NOTE: this function is called from inside serialized functions, so it + * is holding the serial lock. + */ +int +__wt_evict_page_request(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_CACHE *cache; + WT_EVICT_REQ *er, *er_end; + int owned; + + cache = S2C(session)->cache; + + /* + * Application threads request forced eviction of pages when they + * become too big. The application thread must hold a hazard reference + * when this function is called, which protects it from being freed. + * + * However, it is possible (but unlikely) that the page is already part + * way through the process of being evicted: a thread may have selected + * it from the LRU list but not yet checked its hazard references. + * + * To avoid a freed page pointer ending up on the request list, we + * check the page state here while holding the LRU lock. Since the + * state of page references in the eviction list is switched to + * WT_REF_EVICTING while holding the LRU lock, this check prevents a + * page from being evicted twice. + */ + owned = 0; + __wt_spin_lock(session, &cache->lru_lock); + if (!F_ISSET(page, WT_PAGE_FORCE_EVICT) && + WT_ATOMIC_CAS(page->ref->state, WT_REF_MEM, WT_REF_EVICTING)) { + F_SET(page, WT_PAGE_FORCE_EVICT); + owned = 1; + } + __wt_spin_unlock(session, &cache->lru_lock); + + /* + * If we didn't swap the page state, some other thread is already + * evicting it, which is fine. + */ + if (!owned) + return (0); + + /* Find an empty slot and enter the eviction request. */ + WT_EVICT_REQ_FOREACH(er, er_end, cache) + if (er->session == NULL) { + __evict_req_set(session, er, page, WT_EVICT_REQ_PAGE); + __wt_evict_server_wake(session); + return (0); + } + + __wt_errx(session, "eviction server request table full"); + return (WT_ERROR); +} + +/* + * __wt_cache_evict_server -- + * Thread to evict pages from the cache. + */ +void * +__wt_cache_evict_server(void *arg) +{ + WT_CONNECTION_IMPL *conn; + WT_SESSION_IMPL *session; + WT_CACHE *cache; + int ret; + + conn = arg; + cache = conn->cache; + ret = 0; + + /* + * We need a session handle because we're reading/writing pages. + * Start with the default session to keep error handling simple. + */ + session = &conn->default_session; + WT_ERR(__wt_open_session(conn, 1, NULL, NULL, &session)); + + while (F_ISSET(conn, WT_SERVER_RUN)) { + WT_VERBOSE(session, evictserver, "sleeping"); + __wt_cond_wait(session, cache->evict_cond); + if (!F_ISSET(conn, WT_SERVER_RUN)) + break; + WT_VERBOSE(session, evictserver, "waking"); + + /* Evict pages from the cache as needed. */ + WT_ERR(__evict_worker(session)); + } + + if (ret == 0) { + if (__wt_cache_bytes_inuse(cache) != 0) { + __wt_errx(session, + "cache server: exiting with %" PRIu64 " pages, " + "%" PRIu64 " bytes in use", + __wt_cache_pages_inuse(cache), + __wt_cache_bytes_inuse(cache)); + } + } else +err: __wt_err(session, ret, "eviction server error"); + + WT_VERBOSE(session, evictserver, "exiting"); + + __wt_free(session, cache->evict); + + if (session != &conn->default_session) + (void)session->iface.close(&session->iface, NULL); + + return (NULL); +} + +/* + * __evict_worker -- + * Evict pages from memory. + */ +static int +__evict_worker(WT_SESSION_IMPL *session) +{ + WT_CACHE *cache; + WT_CONNECTION_IMPL *conn; + uint64_t bytes_start, bytes_inuse, bytes_max; + int loop; + + conn = S2C(session); + cache = conn->cache; + + /* Evict pages from the cache. */ + for (loop = 0;; loop++) { + /* Walk the eviction-request queue. */ + WT_RET(__evict_request_walk(session)); + + /* Keep evicting until we hit 90% of the maximum cache size. */ + bytes_inuse = __wt_cache_bytes_inuse(cache); + bytes_max = conn->cache_size; + + /* + * Keep evicting until we hit the target cache usage. + */ + bytes_inuse = __wt_cache_bytes_inuse(cache); + bytes_max = conn->cache_size; + if (bytes_inuse < cache->eviction_target * (bytes_max / 100)) + break; + + WT_RET(__evict_lru(session)); + + /* + * If we're making progress, keep going; if we're not making + * any progress at all, go back to sleep, it's not something + * we can fix. + */ + bytes_start = bytes_inuse; + bytes_inuse = __wt_cache_bytes_inuse(cache); + if (bytes_start == bytes_inuse) { + if (loop == 10) { + WT_STAT_INCR(conn->stats, cache_evict_slow); + WT_VERBOSE(session, evictserver, + "unable to reach eviction goal"); + break; + } + } else + loop = 0; + } + return (0); +} + +/* + * __evict_request_walk -- + * Walk the eviction request queue. + */ +static int +__evict_request_walk(WT_SESSION_IMPL *session) +{ + WT_SESSION_IMPL *request_session; + WT_CACHE *cache; + WT_EVICT_REQ *er, *er_end; + int ret; + + cache = S2C(session)->cache; + + /* + * Walk the eviction request queue, looking for sync or close requests + * (defined by a valid WT_SESSION_IMPL handle). If we find a request, + * perform it, flush the result and clear the request slot, then wake + * up the requesting thread. + */ + WT_EVICT_REQ_FOREACH(er, er_end, cache) { + if ((request_session = er->session) == NULL) + continue; + + /* Reference the correct WT_BTREE handle. */ + WT_SET_BTREE_IN_SESSION(session, er->btree); + + /* + * Block out concurrent eviction while we are handling this + * request. + */ + __wt_spin_lock(session, &cache->lru_lock); + + /* + * The eviction candidate list might reference pages we are + * about to discard; clear it. + */ + memset(cache->evict, 0, cache->evict_allocated); + + /* + * If we're about to do a walk of the file tree (and possibly + * close the file), any page we're referencing won't be useful; + * if we pushing out a page, that page might be our eviction + * location. Regardless, discard any page we're holding and + * we can restart our walk as needed. + */ + session->btree->evict_page = NULL; + + if (F_ISSET(er, WT_EVICT_REQ_PAGE)) { + WT_VERBOSE(session, evictserver, + "forcing eviction of page %p", er->page); + for (;;) { + ret = __wt_rec_evict(session, er->page, 0); + if (ret != EBUSY) + break; + __wt_yield(); + } + } else + ret = __evict_file(session, er); + + /* Clear the reference to the btree handle. */ + WT_CLEAR_BTREE_IN_SESSION(session); + + __wt_spin_unlock(session, &cache->lru_lock); + + /* + * Resolve the request and clear the slot. + * + * !!! + * Page eviction is special: the requesting thread is already + * inside wrapup. + */ + if (!F_ISSET(er, WT_EVICT_REQ_PAGE)) + __wt_session_serialize_wrapup( + request_session, NULL, ret); + + __evict_req_clr(session, er); + } + return (0); +} + +/* + * __evict_file -- + * Flush pages for a specific file as part of a close/sync operation. + */ +static int +__evict_file(WT_SESSION_IMPL *session, WT_EVICT_REQ *er) +{ + WT_PAGE *next_page, *page; + + WT_VERBOSE(session, evictserver, + "file request: %s", + (F_ISSET(er, WT_EVICT_REQ_CLOSE) ? "close" : "sync")); + + /* + * We can't evict the page just returned to us, it marks our place in + * the tree. So, always stay one page ahead of the page being returned. + */ + next_page = NULL; + WT_RET(__wt_tree_np(session, &next_page, 1, 1)); + for (;;) { + if ((page = next_page) == NULL) + break; + WT_RET(__wt_tree_np(session, &next_page, 1, 1)); + + /* + * Close: discarding all of the file's pages from the cache. + * Sync: only dirty pages need to be written. + * + * First, write the dirty pages: if we're closing the file, we + * will be evicting all of the pages, and all child pages have + * to be in their final, clean state, to evict the parent. + * + * The specific problem this solves is an empty page, which is + * dirty because new material was added: reconciling it clears + * the empty flag, and then we evict it. + */ + if (__wt_page_is_modified(page)) + WT_RET(__wt_rec_write(session, page, NULL)); + if (!F_ISSET(er, WT_EVICT_REQ_CLOSE)) + continue; + + /* + * We do not attempt to evict pages expected to be merged into + * their parents, with the single exception that the root page + * can't be merged into anything, it must be written. + */ + if (WT_PAGE_IS_ROOT(page) || + !F_ISSET(page, WT_PAGE_REC_EMPTY | + WT_PAGE_REC_SPLIT | WT_PAGE_REC_SPLIT_MERGE)) + WT_RET(__wt_rec_evict(session, page, WT_REC_SINGLE)); + } + + return (0); +} + +/* + * __evict_lru -- + * Evict pages from the cache based on their read generation. + */ +static int +__evict_lru(WT_SESSION_IMPL *session) +{ + WT_CACHE *cache; + int ret; + + cache = S2C(session)->cache; + + __wt_spin_lock(session, &cache->lru_lock); + + /* Get some more pages to consider for eviction. */ + WT_ERR(__evict_walk(session)); + + /* Remove duplicates from the list. */ + __evict_dup_remove(session); + +err: __wt_spin_unlock(session, &cache->lru_lock); + + /* Reconcile and discard some pages. */ + if (ret == 0) + __evict_pages(session); + + return (ret); +} + +/* + * __evict_walk -- + * Fill in the array by walking the next set of pages. + */ +static int +__evict_walk(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + WT_BTREE *btree; + WT_CACHE *cache; + u_int elem, i; + int ret; + + conn = S2C(session); + cache = S2C(session)->cache; + + /* + * Resize the array in which we're tracking pages, as necessary, then + * get some pages from each underlying file. We hold a mutex for the + * entire time -- it's slow, but (1) how often do new files get added + * or removed to/from the system, and (2) it's all in-memory stuff, so + * it's not that slow. + */ + ret = 0; + __wt_spin_lock(session, &conn->spinlock); + + elem = WT_EVICT_WALK_BASE + (conn->btqcnt * WT_EVICT_WALK_PER_TABLE); + if (elem > cache->evict_entries) { + WT_ERR(__wt_realloc(session, &cache->evict_allocated, + elem * sizeof(WT_EVICT_LIST), &cache->evict)); + cache->evict_entries = elem; + } + cache->evict_current = cache->evict; + + i = WT_EVICT_WALK_BASE; + TAILQ_FOREACH(btree, &conn->btqh, q) { + /* Reference the correct WT_BTREE handle. */ + WT_SET_BTREE_IN_SESSION(session, btree); + + ret = __evict_walk_file(session, &i); + + WT_CLEAR_BTREE_IN_SESSION(session); + + if (ret != 0) + goto err; + } + +err: __wt_spin_unlock(session, &conn->spinlock); + return (ret); +} + +/* + * __evict_walk_file -- + * Get a few page eviction candidates from a single underlying file. + */ +static int +__evict_walk_file(WT_SESSION_IMPL *session, u_int *slotp) +{ + WT_BTREE *btree; + WT_CACHE *cache; + WT_PAGE *page; + int i, restarted_once; + + btree = session->btree; + cache = S2C(session)->cache; + + /* + * Get the next WT_EVICT_WALK_PER_TABLE entries. + * + * We can't evict the page just returned to us, it marks our place in + * the tree. So, always stay one page ahead of the page being returned. + */ + i = restarted_once = 0; + do { + if ((page = btree->evict_page) == NULL) + goto skip; + + /* + * Root and pinned pages can't be evicted. + * !!! + * It's still in flux if root pages are pinned or not, test for + * both cases for now. + */ + if (WT_PAGE_IS_ROOT(page) || F_ISSET(page, WT_PAGE_PINNED)) + goto skip; + + /* + * Skip locked pages: we would skip them later, and they just + * fill up the eviction list for no benefit. + */ + if (page->ref->state != WT_REF_MEM) + goto skip; + + /* + * Skip pages expected to be merged into their parents. The + * problem is if a parent and its child are both added to the + * eviction list and the child is merged into the parent when + * the parent is evicted, the child is left corrupted on the + * list (and might have already been selected for eviction by + * another thread). + */ + if (F_ISSET(page, WT_PAGE_REC_EMPTY | + WT_PAGE_REC_SPLIT | WT_PAGE_REC_SPLIT_MERGE)) + goto skip; + + WT_VERBOSE(session, evictserver, + "select: %p, size %" PRIu32, page, page->memory_footprint); + + ++i; + cache->evict[*slotp].page = page; + cache->evict[*slotp].btree = btree; + ++*slotp; + +skip: WT_RET(__wt_tree_np(session, &btree->evict_page, 1, 1)); + if (btree->evict_page == NULL && restarted_once++ == 1) + break; + } while (i < WT_EVICT_WALK_PER_TABLE); + + return (0); +} + +/* + * __evict_dup_remove -- + * Discard duplicates from the list of pages we collected. + */ +static void +__evict_dup_remove(WT_SESSION_IMPL *session) +{ + WT_CACHE *cache; + WT_EVICT_LIST *evict; + u_int elem, i, j; + + cache = S2C(session)->cache; + + /* + * We have an array of page eviction references that may contain NULLs, + * as well as duplicate entries. + * + * First, sort the array by WT_REF address, then delete any duplicates. + * The reason is because we might evict the page but leave a duplicate + * entry in the "saved" area of the array, and that would be a NULL + * dereference on the next run. (If someone ever tries to remove this + * duplicate cleanup for better performance, you can't fix it just by + * checking the WT_REF state -- that only works if you are discarding + * a page from a single level of the tree; if you are discarding a + * page and its parent, the duplicate of the page's WT_REF might have + * been free'd before a subsequent review of the eviction array.) + */ + evict = cache->evict; + elem = cache->evict_entries; + qsort(evict, (size_t)elem, sizeof(WT_EVICT_LIST), __evict_page_cmp); + for (i = 0; i < elem; i = j) { + /* + * Once we hit a NULL, we're done, the NULLs all sorted to the + * end of the array. + */ + if (evict[i].page == NULL) + break; + + for (j = i + 1; j < elem; ++j) { + /* Delete the second and any subsequent duplicates. */ + if (evict[i].page == evict[j].page) + __evict_clr(&evict[j]); + else + break; + } + } + + /* Sort the array by LRU, then evict the most promising candidates. */ + qsort(cache->evict, elem, sizeof(WT_EVICT_LIST), __evict_lru_cmp); +} + +/* + * __evict_get_page -- + * Get a page for eviction. + */ +static void +__evict_get_page( + WT_SESSION_IMPL *session, int is_app, WT_BTREE **btreep, WT_PAGE **pagep) +{ + WT_CACHE *cache; + WT_EVICT_LIST *evict; + WT_REF *ref; + + cache = S2C(session)->cache; + *btreep = NULL; + *pagep = NULL; + + if (__wt_spin_trylock(session, &cache->lru_lock) != 0) + return; + + evict = cache->evict_current; + if (evict != NULL && + evict >= cache->evict && evict < cache->evict + WT_EVICT_GROUP && + evict->page != NULL) { + WT_ASSERT(session, evict->btree != NULL); + + /* + * For now, application sessions can only evict pages from + * trees they have open. Otherwise, closing a different + * session handle could cause the tree we are evicting from + * to be closed underneath us. + */ + if (is_app && + __wt_session_has_btree(session, evict->btree) != 0) + goto done; + + /* Move to the next page queued for eviction. */ + ++cache->evict_current; + + /* + * If the page happens to be marked for forced eviction, ignore + * it: it will be sitting in the request queue. This is + * unlikely, and it is simpler to leave it for the eviction + * thread than trying to find it and clear the request. + */ + if (F_ISSET(evict->page, WT_PAGE_FORCE_EVICT)) + goto done; + + /* + * Set the page locked here while holding the eviction mutex to + * prevent multiple attempts to evict it. + */ + ref = evict->page->ref; + if (!WT_ATOMIC_CAS(ref->state, WT_REF_MEM, WT_REF_EVICTING)) + goto done; + + *pagep = evict->page; + *btreep = evict->btree; + + /* + * If we're evicting our current eviction point in the file, + * clear it and restart the walk. + */ + if (*pagep == evict->btree->evict_page) + evict->btree->evict_page = NULL; + + /* + * Paranoia: remove the entry so we never try and reconcile + * the same page on reconciliation error. + */ + __evict_clr(evict); + } + +done: __wt_spin_unlock(session, &cache->lru_lock); +} + +/* + * __wt_evict_lru_page -- + * Called by both eviction and application threads to evict a page. + */ +int +__wt_evict_lru_page(WT_SESSION_IMPL *session, int is_app) +{ + WT_BTREE *btree, *saved_btree; + WT_PAGE *page; + + __evict_get_page(session, is_app, &btree, &page); + if (page == NULL) + return (WT_NOTFOUND); + + /* Reference the correct WT_BTREE handle. */ + saved_btree = session->btree; + WT_SET_BTREE_IN_SESSION(session, btree); + + /* + * We don't care why eviction failed (maybe the page was dirty and we're + * out of disk space, or the page had an in-memory subtree already being + * evicted). Regardless, don't pick the same page every time. + */ + if (__wt_rec_evict(session, page, 0) != 0) { + page->read_gen = __wt_cache_read_gen(session); + + /* + * If the evicting state of the page was not cleared, clear it + * now to make the page available again. + */ + if (page->ref->state == WT_REF_EVICTING) + page->ref->state = WT_REF_MEM; + } + + WT_CLEAR_BTREE_IN_SESSION(session); + session->btree = saved_btree; + + return (0); +} + +/* + * __evict_page -- + * Reconcile and discard cache pages. + */ +static void +__evict_pages(WT_SESSION_IMPL *session) +{ + u_int i; + + for (i = 0; i < WT_EVICT_GROUP; i++) + if (__wt_evict_lru_page(session, 0) != 0) + break; +} + +/* + * __evict_page_cmp -- + * Qsort function: sort WT_EVICT_LIST array based on the page's address. + */ +static int +__evict_page_cmp(const void *a, const void *b) +{ + WT_PAGE *a_page, *b_page; + + /* + * There may be NULL references in the array; sort them as greater than + * anything else so they migrate to the end of the array. + */ + a_page = ((WT_EVICT_LIST *)a)->page; + b_page = ((WT_EVICT_LIST *)b)->page; + if (a_page == NULL) + return (b_page == NULL ? 0 : 1); + if (b_page == NULL) + return (-1); + + /* Sort the page address in ascending order. */ + return (a_page > b_page ? 1 : (a_page < b_page ? -1 : 0)); +} + +/* + * __evict_lru_cmp -- + * Qsort function: sort WT_EVICT_LIST array based on the page's read + * generation. + */ +static int +__evict_lru_cmp(const void *a, const void *b) +{ + WT_PAGE *a_page, *b_page; + uint64_t a_lru, b_lru; + + /* + * There may be NULL references in the array; sort them as greater than + * anything else so they migrate to the end of the array. + */ + a_page = ((WT_EVICT_LIST *)a)->page; + b_page = ((WT_EVICT_LIST *)b)->page; + if (a_page == NULL) + return (b_page == NULL ? 0 : 1); + if (b_page == NULL) + return (-1); + + /* Sort the LRU in ascending order. */ + a_lru = a_page->read_gen; + b_lru = b_page->read_gen; + + /* + * Bias in favor of leaf pages. Otherwise, we can waste time + * considering parent pages for eviction while their child pages are + * still in memory. + * + * Bump the LRU generation by a small fixed amount: the idea being that + * if we have enough good leaf page candidates, we should evict them + * first, but not completely ignore an old internal page. + */ + if (a_page->type == WT_PAGE_ROW_INT || a_page->type == WT_PAGE_COL_INT) + a_lru += WT_EVICT_GROUP; + if (b_page->type == WT_PAGE_ROW_INT || b_page->type == WT_PAGE_COL_INT) + b_lru += WT_EVICT_GROUP; + return (a_lru > b_lru ? 1 : (a_lru < b_lru ? -1 : 0)); +} diff --git a/src/btree/bt_handle.c b/src/btree/bt_handle.c new file mode 100644 index 00000000000..5a652932095 --- /dev/null +++ b/src/btree/bt_handle.c @@ -0,0 +1,683 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static int __btree_conf(WT_SESSION_IMPL *, const char *, uint32_t); +static int __btree_last(WT_SESSION_IMPL *); +static int __btree_page_sizes(WT_SESSION_IMPL *, const char *); +static int __btree_root_init_empty(WT_SESSION_IMPL *); +static int __btree_tree_init(WT_SESSION_IMPL *); + +static int pse1(WT_SESSION_IMPL *, const char *, uint32_t, uint32_t); +static int pse2(WT_SESSION_IMPL *, const char *, uint32_t, uint32_t, uint32_t); + +/* + * __wt_btree_create -- + * Create a Btree. + */ +int +__wt_btree_create(WT_SESSION_IMPL *session, const char *filename) +{ + return (__wt_bm_create(session, filename)); +} + +/* + * __wt_btree_truncate -- + * Truncate a Btree. + */ +int +__wt_btree_truncate(WT_SESSION_IMPL *session, const char *filename) +{ + return (__wt_bm_truncate(session, filename)); +} + +/* + * __wt_btree_open -- + * Open a Btree. + */ +int +__wt_btree_open(WT_SESSION_IMPL *session, + const char *name, const char *filename, + const char *config, const char *cfg[], uint32_t flags) +{ + WT_BTREE *btree; + WT_CONNECTION_IMPL *conn; + int matched, ret; + + conn = S2C(session); + + /* + * The file configuration string must point to allocated memory: it + * is stored in the returned btree handle and freed when the handle + * is closed. + */ + + /* Increment the reference count if we already have the btree open. */ + matched = 0; + __wt_spin_lock(session, &conn->spinlock); + TAILQ_FOREACH(btree, &conn->btqh, q) { + if (strcmp(filename, btree->filename) == 0) { + ++btree->refcnt; + session->btree = btree; + matched = 1; + break; + } + } + if (matched) { + __wt_spin_unlock(session, &conn->spinlock); + + /* Check that the handle is open. */ + __wt_readlock(session, btree->rwlock); + matched = F_ISSET(btree, WT_BTREE_OPEN); + __wt_rwunlock(session, btree->rwlock); + + if (!matched) { + __wt_writelock(session, btree->rwlock); + if (!F_ISSET(btree, WT_BTREE_OPEN)) { + /* We're going to overwrite the old config. */ + __wt_free(session, btree->config); + goto conf; + } + + /* It was opened while we waited. */ + __wt_rwunlock(session, btree->rwlock); + } + + /* The config string will not be needed: free it now. */ + __wt_free(session, config); + return (0); + } + + /* + * Allocate the WT_BTREE structure, its lock, and set the name so we + * can put the handle into the list. + */ + btree = NULL; + if ((ret = __wt_calloc_def(session, 1, &btree)) == 0 && + (ret = __wt_rwlock_alloc( + session, "btree handle", &btree->rwlock)) == 0 && + (ret = __wt_strdup(session, name, &btree->name)) == 0 && + (ret = __wt_strdup(session, filename, &btree->filename)) == 0) { + /* Lock the handle before it is inserted in the list. */ + __wt_writelock(session, btree->rwlock); + + /* Add to the connection list. */ + btree->refcnt = 1; + TAILQ_INSERT_TAIL(&conn->btqh, btree, q); + ++conn->btqcnt; + } + __wt_spin_unlock(session, &conn->spinlock); + + if (ret != 0) { + if (btree != NULL) { + if (btree->rwlock != NULL) + (void)__wt_rwlock_destroy( + session, btree->rwlock); + __wt_free(session, btree->name); + __wt_free(session, btree->filename); + } + __wt_free(session, btree); + return (ret); + } + + session->btree = btree; + + /* Initialize and configure the WT_BTREE structure. */ +conf: WT_ERR(__btree_conf(session, config, flags)); + + /* Open the underlying block object. */ + WT_ERR(__wt_bm_open(session, btree->filename, + btree->config, cfg, F_ISSET(btree, WT_BTREE_SALVAGE) ? 1 : 0)); + WT_ERR(__wt_bm_block_header(session, &btree->block_header)); + + /* Initialize the tree if not a special command. */ + if (!F_ISSET(btree, + WT_BTREE_SALVAGE | WT_BTREE_UPGRADE | WT_BTREE_VERIFY)) + WT_ERR(__btree_tree_init(session)); + + F_SET(btree, WT_BTREE_OPEN); + WT_STAT_INCR(conn->stats, file_open); + +err: __wt_rwunlock(session, btree->rwlock); + if (ret != 0) + (void)__wt_btree_close(session); + return (ret); +} + +/* + * __wt_btree_reopen -- + * Reset an open btree handle back to its initial state. + */ +int +__wt_btree_reopen(WT_SESSION_IMPL *session, uint32_t flags) +{ + WT_BTREE *btree; + + btree = session->btree; + + /* + * Clear any existing cache. The reason for this is because verify and + * salvage don't want to deal with in-memory trees, that is, reads must + * be satisfied from the disk. + */ + if (btree->root_page != NULL) + WT_RET(__wt_evict_file_serial(session, 1)); + WT_ASSERT(session, btree->root_page == NULL); + + /* After all pages are evicted, update the root's address. */ + if (btree->root_update) { + /* + * Release the original blocks held by the root, that is, + * the blocks listed in the schema file. + */ + WT_RET(__wt_btree_free_root(session)); + + WT_RET(__wt_btree_set_root( + session, btree->root_addr.addr, btree->root_addr.size)); + if (btree->root_addr.addr != NULL) + __wt_free(session, btree->root_addr.addr); + btree->root_update = 0; + } + + btree->flags = flags; /* XXX */ + + /* Initialize the tree if not a special command. */ + if (!F_ISSET(btree, + WT_BTREE_SALVAGE | WT_BTREE_UPGRADE | WT_BTREE_VERIFY)) + WT_RET(__btree_tree_init(session)); + + F_SET(btree, WT_BTREE_OPEN); /* XXX */ + return (0); +} + +/* + * __btree_conf -- + * Configure a WT_BTREE structure. + */ +static int +__btree_conf(WT_SESSION_IMPL *session, const char *config, uint32_t flags) +{ + WT_BTREE *btree; + WT_CONFIG_ITEM cval; + WT_CONNECTION_IMPL *conn; + WT_NAMED_COLLATOR *ncoll; + uint32_t bitcnt; + int fixed; + + btree = session->btree; + conn = S2C(session); + + /* Validate file types and check the data format plan. */ + WT_RET(__wt_config_getones(session, config, "key_format", &cval)); + WT_RET(__wt_struct_check(session, cval.str, cval.len, NULL, NULL)); + if (cval.len > 0 && strncmp(cval.str, "r", cval.len) == 0) + btree->type = BTREE_COL_VAR; + else + btree->type = BTREE_ROW; + WT_RET(__wt_strndup(session, cval.str, cval.len, &btree->key_format)); + + WT_RET(__wt_config_getones(session, config, "value_format", &cval)); + WT_RET(__wt_strndup(session, cval.str, cval.len, &btree->value_format)); + + /* Row-store key comparison and key gap for prefix compression. */ + if (btree->type == BTREE_ROW) { + WT_RET(__wt_config_getones( + session, config, "collator", &cval)); + if (cval.len > 0) { + TAILQ_FOREACH(ncoll, &conn->collqh, q) { + if (strncmp( + ncoll->name, cval.str, cval.len) == 0) { + btree->collator = ncoll->collator; + break; + } + } + if (btree->collator == NULL) + WT_RET_MSG(session, EINVAL, + "unknown collator '%.*s'", + (int)cval.len, cval.str); + } + WT_RET(__wt_config_getones(session, config, "key_gap", &cval)); + btree->key_gap = (uint32_t)cval.val; + } + /* Check for fixed-size data. */ + if (btree->type == BTREE_COL_VAR) { + WT_RET(__wt_struct_check( + session, cval.str, cval.len, &fixed, &bitcnt)); + if (fixed) { + if (bitcnt == 0 || bitcnt > 8) + WT_RET_MSG(session, EINVAL, + "fixed-width field sizes must be greater " + "than 0 and less than or equal to 8"); + btree->bitcnt = (uint8_t)bitcnt; + btree->type = BTREE_COL_FIX; + } + } + + /* Page sizes */ + WT_RET(__btree_page_sizes(session, config)); + + /* Huffman encoding */ + WT_RET(__wt_btree_huffman_open(session, config)); + + WT_RET(__wt_stat_alloc_btree_stats(session, &btree->stats)); + + /* Take the config string: it will be freed with the btree handle. */ + btree->config = config; + + /* Set the flags. */ + btree->flags = flags; + + return (0); +} + +/* + * __btree_tree_init -- + * Open the file in the block manager and read the root/last pages. + */ +static int +__btree_tree_init(WT_SESSION_IMPL *session) +{ + WT_ITEM *addr; + int ret; + + ret = 0; + + WT_RET(__wt_scr_alloc(session, 0, &addr)); + WT_ERR(__wt_btree_get_root(session, addr)); + + /* + * If there's a root page in the file, read it in and pin it. + * If there's no root page, create an empty in-memory page. + */ + if (addr->data == NULL) + WT_ERR(__btree_root_init_empty(session)); + else + WT_ERR(__wt_btree_root_init(session, addr)); + + /* Get the last page of the file. */ + WT_ERR(__btree_last(session)); + +err: __wt_scr_free(&addr); + + return (ret); +} + +/* + * __wt_btree_root_init -- + * Read in a tree from disk. + */ +int +__wt_btree_root_init(WT_SESSION_IMPL *session, WT_ITEM *addr) +{ + WT_BTREE *btree; + WT_ITEM tmp; + WT_PAGE *page; + int ret; + + btree = session->btree; + + /* Read the root into memory. */ + WT_CLEAR(tmp); + WT_RET(__wt_bm_read(session, &tmp, addr->data, addr->size)); + + /* Build the in-memory version of the page. */ + WT_ERR(__wt_page_inmem(session, NULL, NULL, tmp.mem, &page)); + + /* This page can never leave memory. */ + F_SET(page, WT_PAGE_PINNED); + + btree->root_page = page; + return (0); + +err: __wt_buf_free(session, &tmp); + return (ret); +} + +/* + * __btree_root_init_empty -- + * Create an empty in-memory tree. + */ +static int +__btree_root_init_empty(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + WT_PAGE *root, *leaf; + WT_REF *ref; + int ret; + + btree = session->btree; + root = leaf = NULL; + ret = 0; + + /* + * Create a leaf page -- this can be reconciled while the root stays + * pinned. + */ + WT_ERR(__wt_calloc_def(session, 1, &leaf)); + switch (btree->type) { + case BTREE_COL_FIX: + leaf->u.col_fix.recno = 1; + leaf->type = WT_PAGE_COL_FIX; + break; + case BTREE_COL_VAR: + leaf->u.col_var.recno = 1; + leaf->type = WT_PAGE_COL_VAR; + break; + case BTREE_ROW: + leaf->type = WT_PAGE_ROW_LEAF; + break; + } + leaf->entries = 0; + + /* + * A note about empty trees: the initial tree is a root page and a leaf + * page, neither of which are marked dirty. If evicted without being + * modified, that's OK, nothing will ever be written. + * + * Create the empty root page. + * + * !!! + * Be cautious about changing the order of updates in this code: to call + * __wt_page_out on error, we require a correct page setup at each point + * where we might fail. + */ + WT_ERR(__wt_calloc_def(session, 1, &root)); + switch (btree->type) { + case BTREE_COL_FIX: + case BTREE_COL_VAR: + root->type = WT_PAGE_COL_INT; + root->u.intl.recno = 1; + WT_ERR(__wt_calloc_def(session, 1, &root->u.intl.t)); + ref = root->u.intl.t; + ref->page = leaf; + ref->addr = NULL; + ref->state = WT_REF_MEM; + ref->u.recno = 1; + break; + case BTREE_ROW: + root->type = WT_PAGE_ROW_INT; + WT_ERR(__wt_calloc_def(session, 1, &root->u.intl.t)); + ref = root->u.intl.t; + ref->page = leaf; + ref->addr = NULL; + ref->state = WT_REF_MEM; + WT_ERR(__wt_row_ikey_alloc( + session, 0, "", 1, (WT_IKEY **)&(ref->u.key))); + break; + WT_ILLEGAL_VALUE(session); + } + root->entries = 1; + root->parent = NULL; + root->ref = NULL; + F_SET(root, WT_PAGE_PINNED); + + leaf->ref = ref; + leaf->parent = root; + + btree->root_page = root; + + /* + * Mark the child page dirty so that if it is evicted, the tree ends + * up sane. + */ + WT_ERR(__wt_page_modify_init(session, leaf)); + __wt_page_modify_set(leaf); + + return (0); + +err: if (leaf != NULL) + __wt_page_out(session, leaf, 0); + if (root != NULL) + __wt_page_out(session, root, 0); + return (ret); +} + +/* + * __wt_btree_root_empty -- + * Bulk loads only work on empty trees: check before doing a bulk load. + */ +int +__wt_btree_root_empty(WT_SESSION_IMPL *session, WT_PAGE **leafp) +{ + WT_BTREE *btree; + WT_PAGE *root, *child; + + btree = session->btree; + root = btree->root_page; + + if (root->entries != 1) + return (WT_ERROR); + + child = root->u.intl.t->page; + if (child->entries != 0) + return (WT_ERROR); + + *leafp = child; + return (0); +} + +/* + * __btree_last -- + * Read and pin the last page of the file. + */ +static int +__btree_last(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + WT_PAGE *page; + + btree = session->btree; + + if (btree->type == BTREE_ROW) + return (0); + + page = NULL; + WT_RET(__wt_tree_np(session, &page, 0, 0)); + if (page == NULL) + return (WT_NOTFOUND); + + btree->last_page = page; + btree->last_recno = __col_last_recno(page); + + F_SET(page, WT_PAGE_LAST_PAGE | WT_PAGE_PINNED); + + /* + * Publish: there must be a barrier to ensure the pinned flag is set + * before we discard our hazard reference. + */ + WT_WRITE_BARRIER(); + __wt_hazard_clear(session, page); + + return (0); +} + +/* + * __wt_btree_close -- + * Close a Btree. + */ +int +__wt_btree_close(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + WT_CONNECTION_IMPL *conn; + int inuse, ret; + + btree = session->btree; + conn = S2C(session); + ret = 0; + + /* Remove from the connection's list. */ + __wt_spin_lock(session, &conn->spinlock); + inuse = (--btree->refcnt > 0); + if (!inuse) { + TAILQ_REMOVE(&conn->btqh, btree, q); + --conn->btqcnt; + } + __wt_spin_unlock(session, &conn->spinlock); + if (inuse) + return (0); + + if (F_ISSET(btree, WT_BTREE_OPEN)) + WT_STAT_DECR(conn->stats, file_open); + + /* Clear any cache. */ + if (btree->root_page != NULL) + WT_TRET(__wt_evict_file_serial(session, 1)); + WT_ASSERT(session, btree->root_page == NULL); + + /* After all pages are evicted, update the root's address. */ + if (btree->root_update) { + /* + * Release the original blocks held by the root, that is, + * the blocks listed in the schema file. + */ + WT_RET(__wt_btree_free_root(session)); + + WT_RET(__wt_btree_set_root( + session, btree->root_addr.addr, btree->root_addr.size)); + if (btree->root_addr.addr != NULL) + __wt_free(session, btree->root_addr.addr); + btree->root_update = 0; + } + + /* Close the underlying block manager reference. */ + WT_TRET(__wt_bm_close(session)); + + /* Free allocated memory. */ + __wt_free(session, btree->name); + __wt_free(session, btree->filename); + __wt_free(session, btree->config); + __wt_free(session, btree->key_format); + __wt_free(session, btree->key_plan); + __wt_free(session, btree->idxkey_format); + __wt_free(session, btree->value_format); + __wt_free(session, btree->value_plan); + __wt_btree_huffman_close(session); + __wt_free(session, btree->stats); + + WT_TRET(__wt_rwlock_destroy(session, btree->rwlock)); + __wt_free(session, session->btree); + + return (ret); +} + +/* + * __btree_page_sizes -- + * Verify the page sizes. + */ +static int +__btree_page_sizes(WT_SESSION_IMPL *session, const char *config) +{ + WT_BTREE *btree; + WT_CONFIG_ITEM cval; + uint32_t intl_split_size, leaf_split_size, split_pct; + + btree = session->btree; + + WT_RET(__wt_config_getones(session, config, "allocation_size", &cval)); + btree->allocsize = (uint32_t)cval.val; + WT_RET( + __wt_config_getones(session, config, "internal_page_max", &cval)); + btree->maxintlpage = (uint32_t)cval.val; + WT_RET(__wt_config_getones( + session, config, "internal_item_max", &cval)); + btree->maxintlitem = (uint32_t)cval.val; + WT_RET(__wt_config_getones(session, config, "leaf_page_max", &cval)); + btree->maxleafpage = (uint32_t)cval.val; + WT_RET(__wt_config_getones( + session, config, "leaf_item_max", &cval)); + btree->maxleafitem = (uint32_t)cval.val; + + /* Allocation sizes must be a power-of-two, nothing else makes sense. */ + if (!__wt_ispo2(btree->allocsize)) + WT_RET_MSG(session, + EINVAL, "the allocation size must be a power of two"); + + /* All page sizes must be in units of the allocation size. */ + if (btree->maxintlpage < btree->allocsize || + btree->maxintlpage % btree->allocsize != 0 || + btree->maxleafpage < btree->allocsize || + btree->maxleafpage % btree->allocsize != 0) + WT_RET_MSG(session, EINVAL, + "page sizes must be a multiple of the page allocation " + "size (%" PRIu32 "B)", btree->allocsize); + + /* + * Set the split percentage: reconciliation splits to a + * smaller-than-maximum page size so we don't split every time a new + * entry is added. + */ + WT_RET(__wt_config_getones(session, config, "split_pct", &cval)); + split_pct = (uint32_t)cval.val; + intl_split_size = WT_SPLIT_PAGE_SIZE( + btree->maxintlpage, btree->allocsize, split_pct); + leaf_split_size = WT_SPLIT_PAGE_SIZE( + btree->maxleafpage, btree->allocsize, split_pct); + + /* + * Default values for internal and leaf page items: make sure at least + * 8 items fit on split pages. + */ + if (btree->maxintlitem == 0) + btree->maxintlitem = intl_split_size / 8; + if (btree->maxleafitem == 0) + btree->maxleafitem = leaf_split_size / 8; + /* Check we can fit at least 2 items on a page. */ + if (btree->maxintlitem > btree->maxintlpage / 2) + return (pse1(session, "internal", + btree->maxintlpage, btree->maxintlitem)); + if (btree->maxleafitem > btree->maxleafpage / 2) + return (pse1(session, "leaf", + btree->maxleafpage, btree->maxleafitem)); + + /* + * Take into account the size of a split page: + * + * Make it a separate error message so it's clear what went wrong. + */ + if (btree->maxintlitem > intl_split_size / 2) + return (pse2(session, "internal", + btree->maxintlpage, btree->maxintlitem, split_pct)); + if (btree->maxleafitem > leaf_split_size / 2) + return (pse2(session, "leaf", + btree->maxleafpage, btree->maxleafitem, split_pct)); + + /* + * Limit allocation units to 128MB, and page sizes to 512MB. There's + * no reason we couldn't support larger sizes (any sizes up to the + * smaller of an off_t and a size_t should work), but an application + * specifying larger allocation or page sizes would likely be making + * as mistake. The API checked this, but we assert it anyway. + */ + WT_ASSERT(session, btree->allocsize >= WT_BTREE_ALLOCATION_SIZE_MIN); + WT_ASSERT(session, btree->allocsize <= WT_BTREE_ALLOCATION_SIZE_MAX); + WT_ASSERT(session, btree->maxintlpage <= WT_BTREE_PAGE_SIZE_MAX); + WT_ASSERT(session, btree->maxleafpage <= WT_BTREE_PAGE_SIZE_MAX); + + return (0); +} + +static int +pse1(WT_SESSION_IMPL *session, const char *type, uint32_t max, uint32_t ovfl) +{ + WT_RET_MSG(session, EINVAL, + "%s page size (%" PRIu32 "B) too small for the maximum item size " + "(%" PRIu32 "B); the page must be able to hold at least 2 items", + type, max, ovfl); +} + +static int +pse2(WT_SESSION_IMPL *session, + const char *type, uint32_t max, uint32_t ovfl, uint32_t pct) +{ + WT_RET_MSG(session, EINVAL, + "%s page size (%" PRIu32 "B) too small for the maximum item size " + "(%" PRIu32 "B), because of the split percentage (%" PRIu32 + "%%); a split page must be able to hold at least 2 items", + type, max, ovfl, pct); +} diff --git a/src/btree/bt_huffman.c b/src/btree/bt_huffman.c new file mode 100644 index 00000000000..bd8ba2523a1 --- /dev/null +++ b/src/btree/bt_huffman.c @@ -0,0 +1,335 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * 7-bit ASCII, with English language frequencies. + * + * Based on "Case-sensitive letter and bigram frequency counts from large-scale + * English corpora" + * Michael N. Jones and D.J.K. Mewhort + * Queen's University, Kingston, Ontario, Canada + * Behavior Research Methods, Instruments, & Computers 2004, 36 (3), 388-396 + * + * Additionally supports space and tab characters; space is the most common + * character in text where it occurs, and tab appears about as frequently as + * 'a' and 'n' in text where it occurs. + */ +struct __wt_huffman_table { + uint32_t symbol; + uint32_t frequency; +}; +static const struct __wt_huffman_table __wt_huffman_nytenglish[] = { + /* nul */ { 0x00, 0 }, /* For an escape character. */ + /* ht */ { 0x09, 5263779 }, + /* sp */ { 0x20, 8000000 }, + /* ! */ { 0x21, 2178 }, + /* " */ { 0x22, 284671 }, + /* # */ { 0x23, 10 }, + /* $ */ { 0x24, 51572 }, + /* % */ { 0x25, 1993 }, + /* & */ { 0x26, 6523 }, + /* ' */ { 0x27, 204497 }, + /* ( */ { 0x28, 53398 }, + /* ) */ { 0x29, 53735 }, + /* * */ { 0x2a, 20716 }, + /* + */ { 0x2b, 309 }, + /* , */ { 0x2c, 984969 }, + /* - */ { 0x2d, 252302 }, + /* . */ { 0x2e, 946136 }, + /* / */ { 0x2f, 8161 }, + /* 0 */ { 0x30, 546233 }, + /* 1 */ { 0x31, 460946 }, + /* 2 */ { 0x32, 333499 }, + /* 3 */ { 0x33, 187606 }, + /* 4 */ { 0x34, 192528 }, + /* 5 */ { 0x35, 374413 }, + /* 6 */ { 0x36, 153865 }, + /* 7 */ { 0x37, 120094 }, + /* 8 */ { 0x38, 182627 }, + /* 9 */ { 0x39, 282364 }, + /* : */ { 0x3a, 54036 }, + /* ; */ { 0x3b, 36727 }, + /* < */ { 0x3c, 82 }, + /* = */ { 0x3d, 22 }, + /* > */ { 0x3e, 83 }, + /* ? */ { 0x3f, 12357 }, + /* @ */ { 0x40, 1 }, + /* A */ { 0x41, 280937 }, + /* B */ { 0x42, 169474 }, + /* C */ { 0x43, 229363 }, + /* D */ { 0x44, 129632 }, + /* E */ { 0x45, 138443 }, + /* F */ { 0x46, 100751 }, + /* G */ { 0x47, 93212 }, + /* H */ { 0x48, 123632 }, + /* I */ { 0x49, 223312 }, + /* J */ { 0x4a, 78706 }, + /* K */ { 0x4b, 46580 }, + /* L */ { 0x4c, 106984 }, + /* M */ { 0x4d, 259474 }, + /* N */ { 0x4e, 205409 }, + /* O */ { 0x4f, 105700 }, + /* P */ { 0x50, 144239 }, + /* Q */ { 0x51, 11659 }, + /* R */ { 0x52, 146448 }, + /* S */ { 0x53, 304971 }, + /* T */ { 0x54, 325462 }, + /* U */ { 0x55, 57488 }, + /* V */ { 0x56, 31053 }, + /* W */ { 0x57, 107195 }, + /* X */ { 0x58, 7578 }, + /* Y */ { 0x59, 94297 }, + /* Z */ { 0x5a, 5610 }, + /* [ */ { 0x5b, 1 }, + /* \ */ { 0x5c, 1 }, + /* ] */ { 0x5d, 1 }, + /* ^ */ { 0x5e, 1 }, + /* _ */ { 0x5f, 1 }, + /* ` */ { 0x60, 1 }, + /* a */ { 0x61, 5263779 }, + /* b */ { 0x62, 866156 }, + /* c */ { 0x63, 1960412 }, + /* d */ { 0x64, 2369820 }, + /* e */ { 0x65, 7741842 }, + /* f */ { 0x66, 1296925 }, + /* g */ { 0x67, 1206747 }, + /* h */ { 0x68, 2955858 }, + /* i */ { 0x69, 4527332 }, + /* j */ { 0x6a, 65856 }, + /* k */ { 0x6b, 460788 }, + /* l */ { 0x6c, 2553152 }, + /* m */ { 0x6d, 1467376 }, + /* n */ { 0x6e, 4535545 }, + /* o */ { 0x6f, 4729266 }, + /* p */ { 0x70, 1255579 }, + /* q */ { 0x71, 54221 }, + /* r */ { 0x72, 4137949 }, + /* s */ { 0x73, 4186210 }, + /* t */ { 0x74, 5507692 }, + /* u */ { 0x75, 1613323 }, + /* v */ { 0x76, 653370 }, + /* w */ { 0x77, 1015656 }, + /* x */ { 0x78, 123577 }, + /* y */ { 0x79, 1062040 }, + /* z */ { 0x7a, 66423 }, + /* { */ { 0x7b, 1 }, + /* | */ { 0x7c, 1 }, + /* } */ { 0x7d, 1 }, + /* ~ */ { 0x7e, 1 } +}; + +static int __wt_huffman_read(WT_SESSION_IMPL *, + WT_CONFIG_ITEM *, struct __wt_huffman_table **, u_int *, u_int *); + +/* + * __wt_btree_huffman_open -- + * Configure Huffman encoding for the tree. + */ +int +__wt_btree_huffman_open(WT_SESSION_IMPL *session, const char *config) +{ + WT_BTREE *btree; + u_int entries, numbytes; + struct __wt_huffman_table *table; + WT_CONFIG_ITEM key_conf, value_conf; + int ret; + + btree = session->btree; + + WT_RET(__wt_config_getones(session, config, "huffman_key", &key_conf)); + WT_RET(__wt_config_getones( + session, config, "huffman_value", &value_conf)); + if (key_conf.len == 0 && value_conf.len == 0) + return (0); + + switch (btree->type) { /* Check file type compatibility. */ + case BTREE_COL_FIX: + WT_RET_MSG(session, EINVAL, + "fixed-size column-store files may not be Huffman encoded"); + case BTREE_COL_VAR: + if (key_conf.len != 0) + WT_RET_MSG(session, EINVAL, + "the keys of variable-length column-store files " + "may not be Huffman encoded"); + break; + case BTREE_ROW: + break; + } + + if (strncasecmp(key_conf.str, "english", key_conf.len) == 0) { + struct __wt_huffman_table + copy[WT_ELEMENTS(__wt_huffman_nytenglish)]; + + memcpy(copy, + __wt_huffman_nytenglish, sizeof(__wt_huffman_nytenglish)); + WT_RET(__wt_huffman_open(session, copy, + WT_ELEMENTS(__wt_huffman_nytenglish), + 1, &btree->huffman_key)); + + /* Check for a shared key/value table. */ + if (strncasecmp( + value_conf.str, "english", value_conf.len) == 0) { + btree->huffman_value = btree->huffman_key; + return (0); + } + } else { + WT_RET(__wt_huffman_read( + session, &key_conf, &table, &entries, &numbytes)); + ret = __wt_huffman_open(session, table, + entries, numbytes, &btree->huffman_key); + __wt_free(session, table); + if (ret != 0) + return (ret); + + /* Check for a shared key/value table. */ + if (value_conf.len != 0 && key_conf.len == value_conf.len && + memcmp(key_conf.str, value_conf.str, key_conf.len) == 0) { + btree->huffman_value = btree->huffman_key; + return (0); + } + } + if (strncasecmp(value_conf.str, "english", value_conf.len) == 0) { + struct __wt_huffman_table + copy[WT_ELEMENTS(__wt_huffman_nytenglish)]; + + memcpy(copy, + __wt_huffman_nytenglish, sizeof(__wt_huffman_nytenglish)); + WT_RET(__wt_huffman_open(session, copy, + WT_ELEMENTS(__wt_huffman_nytenglish), + 1, &btree->huffman_value)); + } else { + WT_RET(__wt_huffman_read( + session, &value_conf, &table, &entries, &numbytes)); + ret = __wt_huffman_open(session, table, + entries, numbytes, &btree->huffman_value); + __wt_free(session, table); + if (ret != 0) + return (ret); + } + + return (0); +} + +/* + * __wt_huffman_read -- + * Read a Huffman table from a file. + */ +static int +__wt_huffman_read(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *ip, + struct __wt_huffman_table **tablep, u_int *entriesp, u_int *numbytesp) +{ + struct __wt_huffman_table *table, *tp; + FILE *fp; + uint64_t symbol, frequency; + u_int entries, lineno, max; + int ret; + char *file; + + ret = 0; + file = NULL; + table = NULL; + + /* + * UTF-8 table is 256 bytes, with a range of 0-255. + * UTF-16 is 128KB (2 * 65536) bytes, with a range of 0-65535. + */ + if (strncasecmp(ip->str, "utf8", 4) == 0) { + entries = UINT8_MAX; + max = UINT8_MAX - 1; + *numbytesp = 1; + WT_RET(__wt_calloc_def(session, entries, &table)); + + if (ip->len == 4) + WT_ERR_MSG(session, EINVAL, + "no Huffman table file name specified"); + WT_ERR(__wt_calloc_def(session, ip->len, &file)); + memcpy(file, ip->str + 4, ip->len - 4); + } else if (strncasecmp(ip->str, "utf16", 5) == 0) { + entries = UINT16_MAX; + max = UINT16_MAX - 1; + *numbytesp = 2; + WT_RET(__wt_calloc_def(session, entries, &table)); + + if (ip->len == 5) + WT_ERR_MSG(session, EINVAL, + "no Huffman table file name specified"); + WT_ERR(__wt_calloc_def(session, ip->len, &file)); + memcpy(file, ip->str + 5, ip->len - 5); + } else { + WT_ERR_MSG(session, EINVAL, + "unknown Huffman configuration value %.*s", + (int)ip->len, ip->str); + } + + if ((fp = fopen(file, "r")) == NULL) + WT_ERR_MSG(session, __wt_errno(), + "unable to read Huffman table file %.*s", + (int)ip->len, ip->str); + + for (tp = table, lineno = 1; (ret = + fscanf(fp, "%" SCNu64 " %" SCNu64, &symbol, &frequency)) != EOF; + ++tp, ++lineno) { + if (lineno > entries) + WT_ERR_MSG(session, EINVAL, + "Huffman table file %.*s is corrupted, " + "more than %" PRIu32 " entries", + (int)ip->len, ip->str, entries); + if (ret != 2) + WT_ERR_MSG(session, EINVAL, + "line %u of Huffman table file %.*s is corrupted: " + "expected two unsigned integral values", + lineno, (int)ip->len, ip->str); + if (symbol > max) + WT_ERR_MSG(session, EINVAL, + "line %u of Huffman table file %.*s is corrupted: " + "symbol larger than maximum value of %u", + lineno, (int)ip->len, ip->str, max); + if (frequency > UINT32_MAX) + WT_ERR_MSG(session, EINVAL, + "line %u of Huffman table file %.*s is corrupted: " + "frequency larger than maximum value of %" PRIu32, + lineno, (int)ip->len, ip->str, UINT32_MAX); + + tp->symbol = (uint32_t)symbol; + tp->frequency = (uint32_t)frequency; + } + + *entriesp = lineno - 1; + *tablep = table; + return (0); + +err: __wt_free(session, file); + __wt_free(session, table); + return (ret); +} + +/* + * __wt_btree_huffman_close -- + * Close the Huffman tables. + */ +void +__wt_btree_huffman_close(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + + btree = session->btree; + + if (btree->huffman_key != NULL) { + /* Key and data may use the same table, only close it once. */ + if (btree->huffman_value == btree->huffman_key) + btree->huffman_value = NULL; + + __wt_huffman_close(session, btree->huffman_key); + btree->huffman_key = NULL; + } + if (btree->huffman_value != NULL) { + __wt_huffman_close(session, btree->huffman_value); + btree->huffman_value = NULL; + } +} diff --git a/src/btree/bt_misc.c b/src/btree/bt_misc.c new file mode 100644 index 00000000000..9ed42431751 --- /dev/null +++ b/src/btree/bt_misc.c @@ -0,0 +1,109 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_page_type_string -- + * Return a string representing the page type. + */ +const char * +__wt_page_type_string(u_int type) +{ + switch (type) { + case WT_PAGE_INVALID: + return ("invalid"); + case WT_PAGE_COL_FIX: + return ("column-store fixed-length leaf"); + case WT_PAGE_COL_INT: + return ("column-store internal"); + case WT_PAGE_COL_VAR: + return ("column-store variable-length leaf"); + case WT_PAGE_OVFL: + return ("overflow"); + case WT_PAGE_ROW_INT: + return ("row-store internal"); + case WT_PAGE_ROW_LEAF: + return ("row-store leaf"); + case WT_PAGE_FREELIST: + return ("freelist"); + default: + return ("unknown"); + } + /* NOTREACHED */ +} + +/* + * __wt_cell_type_string -- + * Return a string representing the cell type. + */ +const char * +__wt_cell_type_string(uint8_t type) +{ + switch (type) { + case WT_CELL_ADDR: + return ("address"); + case WT_CELL_DEL: + return ("deleted"); + case WT_CELL_KEY: + return ("key"); + case WT_CELL_KEY_OVFL: + return ("key-overflow"); + case WT_CELL_KEY_SHORT: + return ("short-key"); + case WT_CELL_VALUE: + return ("value"); + case WT_CELL_VALUE_OVFL: + return ("value-overflow"); + case WT_CELL_VALUE_SHORT: + return ("short-value"); + default: + return ("illegal"); + } + /* NOTREACHED */ +} + +/* + * __wt_page_addr_string -- + * Figure out a page's "address" and load a buffer with a printable, + * nul-terminated representation of that address. + */ +const char * +__wt_page_addr_string(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_PAGE *page) +{ + uint32_t size; + const uint8_t *addr; + + if (WT_PAGE_IS_ROOT(page)) { + buf->data = "[Root]"; + buf->size = WT_STORE_SIZE(strlen("[Root]")); + return (buf->data); + } + + __wt_get_addr(page->parent, page->ref, &addr, &size); + + return (__wt_addr_string(session, buf, addr, size)); +} + +/* + * __wt_addr_string -- + * Load a buffer with a printable, nul-terminated representation of an + * address. + */ +const char * +__wt_addr_string( + WT_SESSION_IMPL *session, WT_ITEM *buf, const uint8_t *addr, uint32_t size) +{ + if (addr == NULL) { + buf->data = WT_NOADDR; + buf->size = WT_STORE_SIZE(strlen(WT_NOADDR)); + } else if (__wt_bm_addr_string(session, buf, addr, size) != 0) { + buf->data = "[Error]"; + buf->size = WT_STORE_SIZE(strlen("[Error]")); + } + return ((char *)buf->data); +} diff --git a/src/btree/bt_ovfl.c b/src/btree/bt_ovfl.c new file mode 100644 index 00000000000..4d90646bdbd --- /dev/null +++ b/src/btree/bt_ovfl.c @@ -0,0 +1,38 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_ovfl_in -- + * Read an overflow item from the disk. + */ +int +__wt_ovfl_in( + WT_SESSION_IMPL *session, WT_ITEM *store, const uint8_t *addr, uint32_t len) +{ + WT_BTREE *btree; + + btree = session->btree; + + WT_BSTAT_INCR(session, overflow_read); + + /* + * Read an overflow page, using an address from a page for which we + * (better) have a hazard reference. + * + * Overflow reads are synchronous. That may bite me at some point, but + * WiredTiger supports large page sizes, overflow items should be rare. + */ + WT_RET(__wt_bm_read(session, store, addr, len)); + + /* Reference the start of the data and set the data's length. */ + store->data = WT_PAGE_HEADER_BYTE(btree, store->mem); + store->size = ((WT_PAGE_HEADER *)store->mem)->u.datalen; + + return (0); +} diff --git a/src/btree/bt_page.c b/src/btree/bt_page.c new file mode 100644 index 00000000000..a9ef7f76dcb --- /dev/null +++ b/src/btree/bt_page.c @@ -0,0 +1,474 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static int __inmem_col_fix(WT_SESSION_IMPL *, WT_PAGE *); +static int __inmem_col_int(WT_SESSION_IMPL *, WT_PAGE *); +static int __inmem_col_var(WT_SESSION_IMPL *, WT_PAGE *); +static int __inmem_row_int(WT_SESSION_IMPL *, WT_PAGE *); +static int __inmem_row_leaf(WT_SESSION_IMPL *, WT_PAGE *); + +/* + * __wt_page_in -- + * Acquire a hazard reference to a page; if the page is not in-memory, + * read it from the disk and build an in-memory version. + */ +int +__wt_page_in_func( + WT_SESSION_IMPL *session, WT_PAGE *parent, WT_REF *ref +#ifdef HAVE_DIAGNOSTIC + , const char *file, int line +#endif + ) +{ + int read_lockout; + + for (;;) { + switch (ref->state) { + case WT_REF_DISK: + /* + * The page isn't in memory, attempt to set the + * state to WT_REF_READING. If successful, read it. + */ + __wt_eviction_check(session, &read_lockout); + if (read_lockout || !WT_ATOMIC_CAS( + ref->state, WT_REF_DISK, WT_REF_READING)) + break; + + WT_RET(__wt_cache_read(session, parent, ref)); + continue; + case WT_REF_EVICTING: + case WT_REF_LOCKED: + case WT_REF_READING: + /* + * The page is being read or considered for eviction -- + * wait for that to be resolved. + */ + break; + case WT_REF_MEM: + /* + * The page is in memory: get a hazard reference, update + * the page's LRU and return. The expected reason we + * can't get a hazard reference is because the page is + * being evicted; yield and try again. + */ + if (F_ISSET(ref->page, WT_PAGE_PINNED) || + __wt_hazard_set(session, ref +#ifdef HAVE_DIAGNOSTIC + , file, line +#endif + ) == 0) { + ref->page->read_gen = + __wt_cache_read_gen(session); + return (0); + } + break; + WT_ILLEGAL_VALUE(session); + } + + /* + * Find a page to evict -- if that succeeds, + * try again immediately. If it fails, we + * don't care why, but give up our slice before + * retrying. + */ + if (__wt_evict_lru_page(session, 1) != 0) + __wt_yield(); + } +} + +/* + * __wt_page_inmem -- + * Build in-memory page information. + */ +int +__wt_page_inmem(WT_SESSION_IMPL *session, + WT_PAGE *parent, WT_REF *parent_ref, WT_PAGE_HEADER *dsk, WT_PAGE **pagep) +{ + WT_PAGE *page; + int ret; + + WT_ASSERT_RET(session, dsk->u.entries > 0); + + *pagep = NULL; + + /* + * Allocate and initialize the WT_PAGE. + * Set the LRU so the page is not immediately selected for eviction. + */ + WT_RET(__wt_calloc_def(session, 1, &page)); + page->type = dsk->type; + page->parent = parent; + page->ref = parent_ref; + page->dsk = dsk; + /* + * Set the write generation to 1 (which can't match a search where the + * write generation wasn't set, that is, remained 0). + */ + page->read_gen = __wt_cache_read_gen(session); + + switch (page->type) { + case WT_PAGE_COL_FIX: + page->u.col_fix.recno = dsk->recno; + WT_ERR(__inmem_col_fix(session, page)); + break; + case WT_PAGE_COL_INT: + page->u.intl.recno = dsk->recno; + WT_ERR(__inmem_col_int(session, page)); + break; + case WT_PAGE_COL_VAR: + page->u.col_var.recno = dsk->recno; + WT_ERR(__inmem_col_var(session, page)); + break; + case WT_PAGE_ROW_INT: + WT_ERR(__inmem_row_int(session, page)); + break; + case WT_PAGE_ROW_LEAF: + WT_ERR(__inmem_row_leaf(session, page)); + break; + WT_ILLEGAL_VALUE(session); + } + + *pagep = page; + return (0); + +err: __wt_free(session, page); + return (ret); +} + +/* + * __wt_page_modify_init -- + * A page is about to be modified, allocate the modification structure. + */ +int +__wt_page_modify_init(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + if (page->modify == NULL) + WT_RET(__wt_calloc_def(session, 1, &page->modify)); + return (0); +} + +/* + * __inmem_col_fix -- + * Build in-memory index for fixed-length column-store leaf pages. + */ +static int +__inmem_col_fix(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_BTREE *btree; + WT_PAGE_HEADER *dsk; + + btree = session->btree; + dsk = page->dsk; + + page->u.col_fix.bitf = WT_PAGE_HEADER_BYTE(btree, dsk); + page->entries = dsk->u.entries; + return (0); +} + +/* + * __inmem_col_int -- + * Build in-memory index for column-store internal pages. + */ +static int +__inmem_col_int(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_BTREE *btree; + WT_CELL *cell; + WT_CELL_UNPACK *unpack, _unpack; + WT_PAGE_HEADER *dsk; + WT_REF *ref; + uint32_t i; + + btree = session->btree; + dsk = page->dsk; + unpack = &_unpack; + + /* + * Column-store page entries map one-to-one to the number of physical + * entries on the page (each physical entry is a offset object). + */ + WT_RET(__wt_calloc_def( + session, (size_t)dsk->u.entries, &page->u.intl.t)); + + /* + * Walk the page, building references: the page contains value items. + * The value items are on-page items (WT_CELL_VALUE). + */ + ref = page->u.intl.t; + WT_CELL_FOREACH(btree, dsk, cell, unpack, i) { + __wt_cell_unpack(cell, unpack); + ref->addr = cell; + ref->u.recno = unpack->v; + ++ref; + } + + page->entries = dsk->u.entries; + return (0); +} + +/* + * __inmem_col_var -- + * Build in-memory index for variable-length, data-only leaf pages in + * column-store trees. + */ +static int +__inmem_col_var(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_BTREE *btree; + WT_COL *cip; + WT_COL_RLE *repeats; + WT_CELL *cell; + WT_CELL_UNPACK *unpack, _unpack; + WT_PAGE_HEADER *dsk; + uint64_t recno, rle; + size_t bytes_allocated; + uint32_t i, indx, max_repeats, nrepeats; + + btree = session->btree; + dsk = page->dsk; + unpack = &_unpack; + repeats = NULL; + bytes_allocated = max_repeats = nrepeats = 0; + recno = page->u.col_var.recno; + + /* + * Column-store page entries map one-to-one to the number of physical + * entries on the page (each physical entry is a data item). + */ + WT_RET(__wt_calloc_def( + session, (size_t)dsk->u.entries, &page->u.col_var.d)); + + /* + * Walk the page, building references: the page contains unsorted value + * items. The value items are on-page (WT_CELL_VALUE), overflow items + * (WT_CELL_VALUE_OVFL) or deleted items (WT_CELL_DEL). + */ + cip = page->u.col_var.d; + indx = 0; + WT_CELL_FOREACH(btree, dsk, cell, unpack, i) { + __wt_cell_unpack(cell, unpack); + (cip++)->__value = WT_PAGE_DISK_OFFSET(page, cell); + + /* + * Add records with repeat counts greater than 1 to an array we + * use for fast lookups. + */ + rle = __wt_cell_rle(unpack); + if (rle > 1) { + if (nrepeats == max_repeats) { + max_repeats = (max_repeats == 0) ? + 10 : 2 * max_repeats; + WT_RET(__wt_realloc(session, &bytes_allocated, + max_repeats * sizeof(WT_COL_RLE), + &repeats)); + } + repeats[nrepeats].indx = indx; + repeats[nrepeats].recno = recno; + repeats[nrepeats++].rle = rle; + } + indx++; + recno += rle; + } + + page->u.col_var.repeats = repeats; + page->u.col_var.nrepeats = nrepeats; + page->entries = dsk->u.entries; + return (0); +} + +/* + * __inmem_row_int -- + * Build in-memory index for row-store internal pages. + */ +static int +__inmem_row_int(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_BTREE *btree; + WT_ITEM *current, *last, *tmp; + WT_CELL *cell; + WT_CELL_UNPACK *unpack, _unpack; + WT_PAGE_HEADER *dsk; + WT_REF *ref; + uint32_t i, nindx, prefix; + int ret; + void *huffman; + + btree = session->btree; + current = last = NULL; + unpack = &_unpack; + dsk = page->dsk; + ret = 0; + huffman = btree->huffman_key; + + WT_ERR(__wt_scr_alloc(session, 0, ¤t)); + WT_ERR(__wt_scr_alloc(session, 0, &last)); + + /* + * Internal row-store page entries map one-to-two to the number of + * physical entries on the page (each physical entry is a data item + * and offset object). + */ + nindx = dsk->u.entries / 2; + WT_RET((__wt_calloc_def(session, (size_t)nindx, &page->u.intl.t))); + + /* + * Set the number of elements now -- we're about to allocate memory, + * and if we fail in the middle of the page, we want to discard that + * memory properly. + */ + page->entries = nindx; + + /* + * Walk the page, instantiating keys: the page contains sorted key and + * offpage-reference pairs. Keys are row store internal pages with + * on-page/overflow (WT_CELL_KEY/KEY_OVFL) items, and offpage references + * are WT_CELL_OFF items. + */ + ref = page->u.intl.t; + WT_CELL_FOREACH(btree, dsk, cell, unpack, i) { + __wt_cell_unpack(cell, unpack); + switch (unpack->type) { + case WT_CELL_KEY: + case WT_CELL_KEY_OVFL: + break; + case WT_CELL_ADDR: + ref->addr = cell; + ++ref; + continue; + WT_ILLEGAL_VALUE(session); + } + + /* + * If Huffman decoding is required or it's an overflow record, + * use the heavy-weight __wt_cell_unpack_copy() call to build + * the key. Else, we can do it faster internally as we don't + * have to shuffle memory around as much. + */ + prefix = unpack->prefix; + if (huffman != NULL || unpack->ovfl) { + WT_RET(__wt_cell_unpack_copy(session, unpack, current)); + + /* + * If there's a prefix, make sure there's enough buffer + * space, then shift the decoded data past the prefix + * and copy the prefix into place. + */ + if (prefix != 0) { + WT_ERR(__wt_buf_grow( + session, current, prefix + current->size)); + memmove((uint8_t *)current->data + + prefix, current->data, current->size); + memcpy( + (void *)current->data, last->data, prefix); + current->size += prefix; + } + } else { + /* + * Get the cell's data/length and make sure we have + * enough buffer space. + */ + WT_ERR(__wt_buf_grow( + session, current, prefix + unpack->size)); + + /* Copy the prefix then the data into place. */ + if (prefix != 0) + memcpy((void *) + current->data, last->data, prefix); + memcpy((uint8_t *) + current->data + prefix, unpack->data, unpack->size); + current->size = prefix + unpack->size; + } + + /* + * Allocate and initialize the instantiated key. + */ + WT_ERR(__wt_row_ikey_alloc(session, + WT_PAGE_DISK_OFFSET(page, cell), + current->data, current->size, (WT_IKEY **)&ref->u.key)); + + /* + * Swap buffers if it's not an overflow key, we have a new + * prefix-compressed key. + */ + if (!unpack->ovfl) { + tmp = last; + last = current; + current = tmp; + } + } + +err: __wt_scr_free(¤t); + __wt_scr_free(&last); + return (ret); +} + +/* + * __inmem_row_leaf -- + * Build in-memory index for row-store leaf pages. + */ +static int +__inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_BTREE *btree; + WT_CELL *cell; + WT_CELL_UNPACK *unpack, _unpack; + WT_PAGE_HEADER *dsk; + WT_ROW *rip; + uint32_t i, nindx; + + btree = session->btree; + dsk = page->dsk; + unpack = &_unpack; + + /* + * Leaf row-store page entries map to a maximum of two-to-one to the + * number of physical entries on the page (each physical entry might + * be a key without any subsequent data item). + */ + WT_RET((__wt_calloc_def( + session, (size_t)dsk->u.entries * 2, &page->u.row.d))); + + /* + * Walk a row-store page of WT_CELLs, building indices and finding the + * end of the page. + * + * The page contains key/data pairs. Keys are on-page (WT_CELL_KEY) or + * overflow (WT_CELL_KEY_OVFL) items, data are either a single on-page + * (WT_CELL_VALUE) or overflow (WT_CELL_VALUE_OVFL) item. + */ + nindx = 0; + rip = page->u.row.d; + WT_CELL_FOREACH(btree, dsk, cell, unpack, i) { + __wt_cell_unpack(cell, unpack); + switch (unpack->type) { + case WT_CELL_KEY: + case WT_CELL_KEY_OVFL: + ++nindx; + if (rip->key != NULL) + ++rip; + rip->key = cell; + break; + case WT_CELL_VALUE: + case WT_CELL_VALUE_OVFL: + break; + WT_ILLEGAL_VALUE(session); + } + } + + page->entries = nindx; + + /* + * If the keys are Huffman encoded, instantiate some set of them. It + * doesn't matter if we are randomly searching the page or scanning a + * cursor through it, there isn't a fast-path to getting keys off the + * page. + */ + return (btree->huffman_key == NULL ? + 0 : __wt_row_leaf_keys(session, page)); +} diff --git a/src/btree/bt_read.c b/src/btree/bt_read.c new file mode 100644 index 00000000000..6e9691462ba --- /dev/null +++ b/src/btree/bt_read.c @@ -0,0 +1,54 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_cache_read -- + * Read a page from the file. + */ +int +__wt_cache_read(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_REF *ref) +{ + WT_ITEM tmp; + WT_PAGE *page; + uint32_t size; + const uint8_t *addr; + int ret; + + /* + * We don't pass in an allocated buffer, force allocation of new memory + * of the appropriate size. + */ + WT_CLEAR(tmp); + ret = 0; + + WT_ASSERT(session, ref->state == WT_REF_READING); + + /* Get the address. */ + __wt_get_addr(parent, ref, &addr, &size); + + /* Force allocation of new memory. */ + WT_ERR(__wt_bm_read(session, &tmp, addr, size)); + + /* Build the in-memory version of the page. */ + WT_ERR(__wt_page_inmem(session, parent, ref, tmp.mem, &page)); + + __wt_cache_page_read( + session, page, sizeof(WT_PAGE) + page->dsk->size); + + WT_VERBOSE(session, read, + "page %p, %s", page, __wt_page_type_string(page->type)); + + ref->page = page; + ref->state = WT_REF_MEM; + return (0); + +err: ref->state = WT_REF_DISK; + __wt_buf_free(session, &tmp); + return (ret); +} diff --git a/src/btree/bt_ret.c b/src/btree/bt_ret.c new file mode 100644 index 00000000000..0206cd1b0a5 --- /dev/null +++ b/src/btree/bt_ret.c @@ -0,0 +1,116 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_kv_return -- + * Return a page referenced key/value pair to the application. + */ +int +__wt_kv_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, int key_ret) +{ + WT_BTREE *btree; + WT_CELL *cell; + WT_CELL_UNPACK *unpack, _unpack; + WT_CURSOR *cursor; + WT_IKEY *ikey; + WT_PAGE *page; + WT_ROW *rip; + WT_UPDATE *upd; + uint8_t v; + + btree = session->btree; + unpack = &_unpack; + + page = cbt->page; + cursor = &cbt->iface; + + switch (page->type) { + case WT_PAGE_COL_FIX: + if (key_ret) + cursor->recno = cbt->recno; + + /* + * If the cursor references a WT_INSERT item, take the related + * WT_UPDATE item. + */ + if (cbt->ins != NULL) { + upd = cbt->ins->upd; + cursor->value.data = WT_UPDATE_DATA(upd); + cursor->value.size = upd->size; + return (0); + } + v = __bit_getv_recno(page, cbt->iface.recno, btree->bitcnt); + return (__wt_buf_set(session, &cursor->value, &v, 1)); + case WT_PAGE_COL_VAR: + if (key_ret) + cursor->recno = cbt->recno; + + /* + * If the cursor references a WT_INSERT item, take the related + * WT_UPDATE item. + */ + if (cbt->ins != NULL) { + upd = cbt->ins->upd; + cursor->value.data = WT_UPDATE_DATA(upd); + cursor->value.size = upd->size; + return (0); + } + cell = WT_COL_PTR(page, &page->u.col_var.d[cbt->slot]); + break; + case WT_PAGE_ROW_LEAF: + rip = &page->u.row.d[cbt->slot]; + + /* + * If the cursor references a WT_INSERT item, take the key and + * related WT_UPDATE item. Otherwise, take the key from the + * original page, and the value from any related WT_UPDATE item, + * or the page if the key was never updated. + */ + if (cbt->ins == NULL) { + if (key_ret) { + if (__wt_off_page(page, rip->key)) { + ikey = rip->key; + cursor->key.data = WT_IKEY_DATA(ikey); + cursor->key.size = ikey->size; + } else + WT_RET(__wt_row_key( + session, page, rip, &cursor->key)); + } + upd = WT_ROW_UPDATE(page, rip); + } else { + if (key_ret) { + cursor->key.data = WT_INSERT_KEY(cbt->ins); + cursor->key.size = WT_INSERT_KEY_SIZE(cbt->ins); + } + upd = cbt->ins->upd; + } + if (upd != NULL) { + cursor->value.data = WT_UPDATE_DATA(upd); + cursor->value.size = upd->size; + return (0); + } + + /* Take the original cell (which may be empty). */ + if ((cell = __wt_row_value(page, rip)) == NULL) { + cursor->value.size = 0; + return (0); + } + break; + WT_ILLEGAL_VALUE(session); + } + + /* It's a cell, unpack and expand it as necessary. */ + __wt_cell_unpack(cell, unpack); + if (btree->huffman_value == NULL && unpack->type == WT_CELL_VALUE) { + cursor->value.data = unpack->data; + cursor->value.size = unpack->size; + return (0); + } else + return (__wt_cell_unpack_copy(session, unpack, &cursor->value)); +} diff --git a/src/btree/bt_root.c b/src/btree/bt_root.c new file mode 100644 index 00000000000..2a4fa94bd0e --- /dev/null +++ b/src/btree/bt_root.c @@ -0,0 +1,324 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static int __btree_get_root(WT_SESSION_IMPL *, const char **, int *, int *); +static int __btree_get_turtle(WT_SESSION_IMPL *, const char **, int *, int *); +static int __btree_set_root(WT_SESSION_IMPL *, char *); +static int __btree_set_turtle(WT_SESSION_IMPL *, char *); + +#define WT_TURTLE_MSG "The turtle." + +#define WT_SCHEMA_TURTLE "WiredTiger.turtle" /* Schema root page */ +#define WT_SCHEMA_TURTLE_SET "WiredTiger.turtle.set" /* Schema root prep */ + +/* + * __wt_btree_get_root -- + * Get the file's root address. + */ +int +__wt_btree_get_root(WT_SESSION_IMPL *session, WT_ITEM *addr) +{ + WT_BTREE *btree; + uint32_t size; + int majorv, minorv, ret; + const char *v; + + btree = session->btree; + v = NULL; + ret = 0; + + /* If there's no root address, return a NULL with a size of 0. */ + addr->data = NULL; + addr->size = 0; + + /* + * If we don't find a file, we're creating a new one, at the current + * version. + */ + majorv = WT_BTREE_MAJOR_VERSION; + minorv = WT_BTREE_MINOR_VERSION; + + /* Get the root address and major/minor numbers. */ + WT_ERR(strcmp(btree->filename, WT_SCHEMA_FILENAME) == 0 ? + __btree_get_turtle(session, &v, &majorv, &minorv) : + __btree_get_root(session, &v, &majorv, &minorv)); + + if (majorv > WT_BTREE_MAJOR_VERSION || + (majorv == WT_BTREE_MAJOR_VERSION && + minorv > WT_BTREE_MINOR_VERSION)) + WT_ERR_MSG(session, EACCES, + "%s is an unsupported version of a WiredTiger file", + btree->filename); + + /* Nothing or "[NoAddr]" means no address. */ + if (v != NULL && strlen(v) != 0 && strcmp(v, WT_NOADDR) != 0) { + WT_ERR(__wt_hex_to_raw(session, (void *)v, (void *)v, &size)); + WT_ERR(__wt_buf_set(session, addr, v, size)); + } + +err: if (ret != 0) + __wt_errx(session, + "unable to find %s file's root address", btree->filename); + + __wt_free(session, v); + return (ret); +} + +/* + * __wt_btree_free_root -- + * Free the file's root address. + */ +int +__wt_btree_free_root(WT_SESSION_IMPL *session) +{ + WT_ITEM *addr, *as; + WT_BTREE *btree; + int ret; + + btree = session->btree; + addr = as = NULL; + + WT_RET(__wt_scr_alloc(session, WT_BM_MAX_ADDR_COOKIE, &addr)); + WT_ERR(__wt_btree_get_root(session, addr)); + if (addr->data != NULL) { + WT_RET(__wt_scr_alloc(session, 0, &as)); + WT_VERBOSE(session, verify, "free %s root %s", + btree->filename, + __wt_addr_string(session, as, addr->data, addr->size)); + + WT_ERR(__wt_bm_free(session, addr->data, addr->size)); + } + +err: __wt_scr_free(&addr); + __wt_scr_free(&as); + return (ret); +} + +/* + * __wt_btree_set_root -- + * Set the file's root address. + */ +int +__wt_btree_set_root(WT_SESSION_IMPL *session, uint8_t *addr, uint32_t size) +{ + WT_BTREE *btree; + WT_ITEM *v; + int ret; + + btree = session->btree; + v = NULL; + ret = 0; + + /* + * Every bytes is encoded as 2 bytes, plus a trailing nul byte, + * and it needs to hold the [NoAddr] string. + */ + WT_RET(__wt_scr_alloc( + session, size * 2 + WT_STORE_SIZE(strlen(WT_NOADDR)), &v)); + + WT_VERBOSE(session, verify, "set %s root %s", + btree->filename, __wt_addr_string(session, v, addr, size)); + + /* + * We're not using the WT_ITEM as a buffer going forward, but fill + * in the values anyway, just for safety. + */ + if (addr == NULL) { + v->data = WT_NOADDR; + v->size = WT_STORE_SIZE(strlen(WT_NOADDR)) + 1; + } else { + __wt_raw_to_hex(addr, v->mem, &size); + v->data = v->mem; + v->size = size; + } + + WT_ERR(strcmp(btree->filename, WT_SCHEMA_FILENAME) == 0 ? + __btree_set_turtle(session, (char *)v->data) : + __btree_set_root(session, (char *)v->data)); + +err: if (ret != 0) + __wt_errx(session, + "unable to update %s file's root address", btree->filename); + + __wt_scr_free(&v); + return (ret); +} + +/* + * __btree_get_turtle -- + * Get the schema file's root address. + */ +static int +__btree_get_turtle( + WT_SESSION_IMPL *session, const char **vp, int *majorp, int *minorp) +{ + FILE *fp; + int found_root, found_version, ret; + const char *path; + char *p, line[1024]; + + *vp = NULL; + + fp = NULL; + ret = 0; + path = NULL; + + found_root = found_version = 0; + + WT_ERR(__wt_filename(session, WT_SCHEMA_TURTLE, &path)); + WT_ERR_TEST((fp = fopen(path, "r")) == NULL, 0); + for (;;) { + if (fgets(line, (int)sizeof(line), fp) == NULL) { + if (ferror(fp)) { + ret = __wt_errno(); + goto format; + } + break; + } + if ((p = strchr(line, '\n')) == NULL) + goto format; + *p = '\0'; + + if (strcmp(line, WT_TURTLE_MSG) == 0) + continue; + if (strncmp(line, "root:", strlen("root:")) == 0) { + WT_ERR(__wt_strdup( + session, line + strlen("root:"), vp)); + found_root = 1; + continue; + } + if (strncmp(line, "version:", strlen("version:")) == 0) { + if (sscanf(line, + "version:major=%d,minor=%d", majorp, minorp) != 2) + goto format; + found_version = 1; + continue; + } + goto format; + } + + if (!found_root || !found_version) { +format: if (ret == 0) + ret = WT_ERROR; + __wt_errx(session, "the %s file is corrupted", path); + } + +err: if (fp != NULL) + WT_TRET(fclose(fp)); + __wt_free(session, path); + + return (ret); +} + +/* + * __btree_set_turtle -- + * Set the schema file's root address. + */ +static int +__btree_set_turtle(WT_SESSION_IMPL *session, char *v) +{ + WT_ITEM *buf; + FILE *fp; + size_t len; + int ret; + const char *path; + + buf = NULL; + ret = 0; + path = NULL; + + WT_ERR(__wt_filename(session, WT_SCHEMA_TURTLE_SET, &path)); + WT_ERR_TEST((fp = fopen(path, "w")) == NULL, WT_ERROR); + + WT_RET(__wt_scr_alloc(session, 0, &buf)); + WT_ERR(__wt_buf_fmt(session, buf, + "%s\n" + "root:%s\n" + "version:major=%d,minor=%d\n", + WT_TURTLE_MSG, v, WT_BTREE_MAJOR_VERSION, WT_BTREE_MINOR_VERSION)); + len = (size_t)fprintf(fp, "%s", (char *)buf->data); + if (len != buf->size) + ret = WT_ERROR; + + WT_TRET(fflush(fp)); + WT_TRET(fclose(fp)); + + if (ret == 0) + ret = __wt_rename( + session, WT_SCHEMA_TURTLE_SET, WT_SCHEMA_TURTLE); + else + (void)__wt_remove(session, WT_SCHEMA_TURTLE_SET); + +err: if (path != NULL) + __wt_free(session, path); + __wt_scr_free(&buf); + return (ret); +} + +/* + * __btree_get_root -- + * Get a non-schema file's root address. + */ +static int +__btree_get_root( + WT_SESSION_IMPL *session, const char **vp, int *majorp, int *minorp) +{ + WT_BTREE *btree; + WT_ITEM *key; + int ret; + const char *version; + + *vp = NULL; + + btree = session->btree; + key = NULL; + ret = 0; + version = NULL; + + WT_RET(__wt_scr_alloc(session, 0, &key)); + WT_ERR(__wt_buf_fmt(session, key, "root:%s", btree->filename)); + WT_ERR(__wt_schema_table_read(session, key->data, vp)); + + WT_ERR(__wt_buf_fmt(session, key, "version:%s", btree->filename)); + WT_ERR(__wt_schema_table_read(session, key->data, &version)); + if (sscanf(version, "major=%d,minor=%d", majorp, minorp) != 2) + WT_ERR_MSG(session, EINVAL, + "unable to find %s file's version number", btree->filename); + + __wt_free(session, version); +err: __wt_scr_free(&key); + + session->btree = btree; /* XXX: schema-read overwrites */ + return (ret == WT_NOTFOUND ? 0 : ret); +} + +/* + * __btree_set_root -- + * Set a non-schema file's root address. + */ +static int +__btree_set_root(WT_SESSION_IMPL *session, char *v) +{ + WT_BTREE *btree; + WT_ITEM *key; + int ret; + + btree = session->btree; + key = NULL; + ret = 0; + + WT_RET(__wt_scr_alloc(session, 0, &key)); + WT_ERR(__wt_buf_fmt(session, key, "root:%s", btree->filename)); + WT_ERR(__wt_schema_table_update(session, key->data, v)); + +err: __wt_scr_free(&key); + + session->btree = btree; /* XXX: schema-insert overwrites */ + return (ret); +} diff --git a/src/btree/bt_slvg.c b/src/btree/bt_slvg.c new file mode 100644 index 00000000000..5321d1a5bdc --- /dev/null +++ b/src/btree/bt_slvg.c @@ -0,0 +1,2297 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +struct __wt_stuff; typedef struct __wt_stuff WT_STUFF; +struct __wt_track; typedef struct __wt_track WT_TRACK; + +/* + * There's a bunch of stuff we pass around during salvage, group it together + * to make the code prettier. + */ +struct __wt_stuff { + WT_SESSION_IMPL *session; /* Salvage session */ + WT_BTREE *btree; /* Enclosing Btree */ + + WT_TRACK **pages; /* Pages */ + uint32_t pages_next; /* Next empty slot */ + size_t pages_allocated; /* Bytes allocated */ + + WT_TRACK **ovfl; /* Overflow pages */ + uint32_t ovfl_next; /* Next empty slot */ + size_t ovfl_allocated; /* Bytes allocated */ + + uint8_t page_type; /* Page type */ + + /* If need to free blocks backing merged page ranges. */ + int merge_free; + + WT_ITEM *tmp1; /* Verbose print buffer */ + WT_ITEM *tmp2; /* Verbose print buffer */ + + uint64_t fcnt; /* Progress counter */ +}; + +/* + * WT_TRACK -- + * Structure to track validated pages, one per page. + */ +struct __wt_track { + WT_STUFF *ss; /* Enclosing stuff */ + + WT_ADDR addr; /* Page address */ + uint32_t size; /* Page size */ + uint64_t gen; /* Page generation */ + + /* + * Pages that reference overflow pages contain a list of the overflow + * pages they reference. + */ + WT_ADDR *ovfl; /* Referenced overflow pages */ + uint32_t ovfl_cnt; /* Overflow list elements */ + + union { + struct { +#undef row_start +#define row_start u.row._row_start + WT_ITEM _row_start; /* Row-store start range */ +#undef row_stop +#define row_stop u.row._row_stop + WT_ITEM _row_stop; /* Row-store stop range */ + } row; + + struct { +#undef col_start +#define col_start u.col._col_start + uint64_t _col_start; /* Col-store start range */ +#undef col_stop +#define col_stop u.col._col_stop + uint64_t _col_stop; /* Col-store stop range */ +#undef col_missing +#define col_missing u.col._col_missing + uint64_t _col_missing; /* Col-store missing range */ + } col; + } u; + +#define WT_TRACK_CHECK_START 0x001 /* Initial key updated */ +#define WT_TRACK_CHECK_STOP 0x002 /* Last key updated */ +#define WT_TRACK_MERGE 0x004 /* Page requires merging */ +#define WT_TRACK_NO_FILE_BLOCKS 0x008 /* WT_TRACK w/o file blocks */ +#define WT_TRACK_OVFL_REFD 0x010 /* Overflow page referenced */ + uint32_t flags; +}; + + /* Flags to __slvg_trk_free() */ +#define WT_TRK_FREE_BLOCKS 0x01 /* Free any blocks */ +#define WT_TRK_FREE_OVFL 0x02 /* Free any overflow pages */ + +static int __salvage(WT_SESSION_IMPL *, const char **); +static int __slvg_cleanup(WT_SESSION_IMPL *, WT_STUFF *); +static int __slvg_col_build_internal(WT_SESSION_IMPL *, uint32_t, WT_STUFF *); +static int __slvg_col_build_leaf( + WT_SESSION_IMPL *, WT_TRACK *, WT_PAGE *, WT_REF *); +static int __slvg_col_merge_ovfl( + WT_SESSION_IMPL *, WT_TRACK *, WT_PAGE *, uint64_t, uint64_t); +static int __slvg_col_range(WT_SESSION_IMPL *, WT_STUFF *); +static void __slvg_col_range_missing(WT_SESSION_IMPL *, WT_STUFF *); +static int __slvg_col_range_overlap( + WT_SESSION_IMPL *, uint32_t, uint32_t, WT_STUFF *); +static void __slvg_col_trk_update_start(uint32_t, WT_STUFF *); +static int __slvg_merge_block_free(WT_SESSION_IMPL *, WT_STUFF *); +static int __slvg_ovfl_compare(const void *, const void *); +static int __slvg_ovfl_discard(WT_SESSION_IMPL *, WT_STUFF *); +static int __slvg_ovfl_reconcile(WT_SESSION_IMPL *, WT_STUFF *); +static int __slvg_read(WT_SESSION_IMPL *, WT_STUFF *); +static int __slvg_row_build_internal(WT_SESSION_IMPL *, uint32_t, WT_STUFF *); +static int __slvg_row_build_leaf(WT_SESSION_IMPL *, + WT_TRACK *, WT_PAGE *, WT_REF *, WT_STUFF *); +static int __slvg_row_merge_ovfl( + WT_SESSION_IMPL *, WT_TRACK *, WT_PAGE *, uint32_t, uint32_t); +static int __slvg_row_range(WT_SESSION_IMPL *, WT_STUFF *); +static int __slvg_row_range_overlap( + WT_SESSION_IMPL *, uint32_t, uint32_t, WT_STUFF *); +static int __slvg_row_trk_update_start( + WT_SESSION_IMPL *, WT_ITEM *, uint32_t, WT_STUFF *); +static int __slvg_trk_compare_addr(const void *, const void *); +static int __slvg_trk_compare_gen(const void *, const void *); +static int __slvg_trk_compare_key(const void *, const void *); +static int __slvg_trk_free(WT_SESSION_IMPL *, WT_TRACK **, uint32_t); +static int __slvg_trk_init(WT_SESSION_IMPL *, uint8_t *, + uint32_t, uint32_t, uint64_t, WT_STUFF *, WT_TRACK **); +static int __slvg_trk_leaf(WT_SESSION_IMPL *, + WT_PAGE_HEADER *, uint8_t *, uint32_t, uint64_t, WT_STUFF *); +static int __slvg_trk_leaf_ovfl( + WT_SESSION_IMPL *, WT_PAGE_HEADER *, WT_TRACK *); +static int __slvg_trk_ovfl(WT_SESSION_IMPL *, + WT_PAGE_HEADER *, uint8_t *, uint32_t, uint64_t, WT_STUFF *); + +/* + * __wt_salvage -- + * Salvage a Btree. + */ +int +__wt_salvage(WT_SESSION_IMPL *session, const char *cfg[]) +{ + return (__salvage(session, cfg)); +} + +/* + * __salvage -- + * Salvage a Btree. + */ +static int +__salvage(WT_SESSION_IMPL *session, const char *cfg[]) +{ + WT_BTREE *btree; + WT_STUFF *ss, stuff; + uint32_t i, leaf_cnt; + int ret, started; + + WT_UNUSED(cfg); + + btree = session->btree; + ret = started = 0; + + WT_CLEAR(stuff); + ss = &stuff; + ss->session = session; + ss->btree = btree; + ss->page_type = WT_PAGE_INVALID; + + /* Allocate temporary buffers. */ + WT_ERR(__wt_scr_alloc(session, 0, &ss->tmp1)); + WT_ERR(__wt_scr_alloc(session, 0, &ss->tmp2)); + + /* + * Step 1: + * Clear the salvaged file's root address, we're done with this file + * until it's salvaged. We do this first because salvage writes a + * root page when it wraps up, and the eviction of that page updates + * the root's address: if the root address were still set, eviction + * would also free the previous root page, which would collide with + * salvage freeing the previous root page when it reads those blocks + * from the file. + */ + WT_ERR(__wt_btree_set_root(session, NULL, 0)); + + /* + * Step 2: + * Inform the underlying block manager that we're salvaging the file. + */ + WT_ERR(__wt_bm_salvage_start(session)); + started = 1; + + /* + * Step 3: + * Read the file and build in-memory structures that reference any leaf + * or overflow page. Any pages other than leaf or overflow pages are + * added to the free list. + * + * Turn off read checksum and verification error messages while we're + * reading the file, we expect to see corrupted blocks. + */ + F_SET(session, WT_SESSION_SALVAGE_QUIET_ERR); + ret = __slvg_read(session, ss); + F_CLR(session, WT_SESSION_SALVAGE_QUIET_ERR); + WT_ERR(ret); + + /* + * Step 4: + * Review the relationships between the pages and the overflow items. + * + * Step 5: + * Add unreferenced overflow page blocks to the free list. + */ + if (ss->ovfl_next != 0) { + WT_ERR(__slvg_ovfl_reconcile(session, ss)); + WT_ERR(__slvg_ovfl_discard(session, ss)); + } + + /* + * Step 6: + * Walk the list of pages looking for overlapping ranges to resolve. + * If we find a range that needs to be resolved, set a global flag + * and a per WT_TRACK flag on the pages requiring modification. + * + * This requires sorting the page list by key, and secondarily by LSN. + * + * !!! + * It's vanishingly unlikely and probably impossible for fixed-length + * column-store files to have overlapping key ranges. It's possible + * for an entire key range to go missing (if a page is corrupted and + * lost), but because pages can't split, it shouldn't be possible to + * find pages where the key ranges overlap. That said, we check for + * it and clean up after it in reconciliation because it doesn't cost + * much and future column-store formats or operations might allow for + * fixed-length format ranges to overlap during salvage, and I don't + * want to have to retrofit the code later. + */ + qsort(ss->pages, + (size_t)ss->pages_next, sizeof(WT_TRACK *), __slvg_trk_compare_key); + if (ss->page_type == WT_PAGE_ROW_LEAF) + WT_ERR(__slvg_row_range(session, ss)); + else + WT_ERR(__slvg_col_range(session, ss)); + + /* + * Step 7: + * We may have lost key ranges in column-store databases, that is, some + * part of the record number space is gone. Look for missing ranges. + */ + switch (ss->page_type) { + case WT_PAGE_COL_FIX: + case WT_PAGE_COL_VAR: + __slvg_col_range_missing(session, ss); + break; + case WT_PAGE_ROW_LEAF: + break; + } + + /* + * Step 8: + * Build an internal page that references all of the leaf pages, + * and write it, as well as any merged pages, to the file. + * + * Count how many leaf pages we have (we could track this during the + * array shuffling/splitting, but that's a lot harder). + */ + for (leaf_cnt = i = 0; i < ss->pages_next; ++i) + if (ss->pages[i] != NULL) + ++leaf_cnt; + if (leaf_cnt != 0) + switch (ss->page_type) { + case WT_PAGE_COL_FIX: + case WT_PAGE_COL_VAR: + WT_ERR( + __slvg_col_build_internal(session, leaf_cnt, ss)); + break; + case WT_PAGE_ROW_LEAF: + WT_ERR( + __slvg_row_build_internal(session, leaf_cnt, ss)); + break; + } + + /* + * Step 9: + * If we had to merge key ranges, we have to do a final pass through + * the leaf page array and discard file pages used during key merges. + * We can't do it earlier: if we free'd the leaf pages we're merging as + * we merged them, the write of subsequent leaf pages or the internal + * page might allocate those free'd file blocks, and if the salvage run + * subsequently fails, we'd have overwritten pages used to construct the + * final key range. In other words, if the salvage run fails, we don't + * want to overwrite data the next salvage run might need. + */ + if (ss->merge_free) + WT_ERR(__slvg_merge_block_free(session, ss)); + + /* + * Step 11: + * Inform the underlying block manager that we're done. + */ +err: if (started) + WT_TRET(__wt_bm_salvage_end(session, ret == 0 ? 1 : 0)); + + /* Discard the leaf and overflow page memory. */ + WT_TRET(__slvg_cleanup(session, ss)); + + /* Discard temporary buffers. */ + __wt_scr_free(&ss->tmp1); + __wt_scr_free(&ss->tmp2); + + /* Wrap up reporting. */ + __wt_progress(session, NULL, ss->fcnt); + + return (ret); +} + +/* + * __slvg_read -- + * Read the file and build a table of the pages we can use. + */ +static int +__slvg_read(WT_SESSION_IMPL *session, WT_STUFF *ss) +{ + WT_ITEM *as, *buf; + WT_PAGE_HEADER *dsk; + uint64_t gen; + uint32_t addrbuf_size; + uint8_t addrbuf[WT_BM_MAX_ADDR_COOKIE]; + int eof, ret; + + ret = 0; + + as = buf = NULL; + WT_ERR(__wt_scr_alloc(session, 0, &as)); + WT_ERR(__wt_scr_alloc(session, 0, &buf)); + + for (;;) { + WT_ERR(__wt_bm_salvage_next( + session, buf, addrbuf, &addrbuf_size, &gen, &eof)); + if (eof) + break; + dsk = buf->mem; + + /* Report progress every 10 reads. */ + if (++ss->fcnt % 10 == 0) + __wt_progress(session, NULL, ss->fcnt); + + /* Create a printable version of the address. */ + WT_ERR(__wt_bm_addr_string(session, as, addrbuf, addrbuf_size)); + + /* + * Make sure it's an expected page type for the file. + * + * We only care about leaf and overflow pages from here on out; + * discard all of the others. We put them on the free list now, + * because we might as well overwrite them, we want the file to + * grow as little as possible, or shrink, and future salvage + * calls don't need them either. + */ + switch (dsk->type) { + case WT_PAGE_COL_INT: + case WT_PAGE_FREELIST: + case WT_PAGE_ROW_INT: + WT_VERBOSE(session, salvage, + "%s page ignored %s", + __wt_page_type_string(dsk->type), (char *)as->data); + WT_ERR(__wt_bm_free(session, addrbuf, addrbuf_size)); + continue; + } + + /* + * Next, verify the page. It's vanishingly unlikely a page + * could pass checksum and still be broken, but a degree of + * paranoia is healthy in salvage. Regardless, verify does + * return failure because it detects failures we'd expect to + * see in a corrupted file, like overflow references past the + * end of the file, might as well discard these pages now. + */ + if (__wt_verify_dsk(session, + (char *)as->data, buf->mem, buf->size) != 0) { + WT_VERBOSE(session, salvage, + "%s page failed verify %s", + __wt_page_type_string(dsk->type), (char *)as->data); + WT_ERR(__wt_bm_free(session, addrbuf, addrbuf_size)); + continue; + } + + WT_VERBOSE(session, salvage, + "tracking %s page, generation %" PRIu64 " %s", + __wt_page_type_string(dsk->type), gen, (char *)as->data); + + switch (dsk->type) { + case WT_PAGE_COL_FIX: + case WT_PAGE_COL_VAR: + case WT_PAGE_ROW_LEAF: + if (ss->page_type == WT_PAGE_INVALID) + ss->page_type = dsk->type; + if (ss->page_type != dsk->type) + WT_RET_MSG(session, WT_ERROR, + "file contains multiple file formats (both " + "%s and %s), and cannot be salvaged", + __wt_page_type_string(ss->page_type), + __wt_page_type_string(dsk->type)); + + WT_ERR(__slvg_trk_leaf( + session, dsk, addrbuf, addrbuf_size, gen, ss)); + break; + case WT_PAGE_OVFL: + WT_ERR(__slvg_trk_ovfl( + session, dsk, addrbuf, addrbuf_size, gen, ss)); + break; + } + } + +err: __wt_scr_free(&as); + __wt_scr_free(&buf); + + return (ret); +} + +/* + * __slvg_trk_init -- + * Initialize tracking information for a page. + */ +static int +__slvg_trk_init(WT_SESSION_IMPL *session, + uint8_t *addr, uint32_t addr_size, + uint32_t size, uint64_t gen, WT_STUFF *ss, WT_TRACK **retp) +{ + WT_TRACK *trk; + int ret; + + WT_RET(__wt_calloc_def(session, 1, &trk)); + trk->ss = ss; + + WT_ERR(__wt_strndup(session, (char *)addr, addr_size, &trk->addr.addr)); + trk->addr.size = addr_size; + trk->size = size; + trk->gen = gen; + + *retp = trk; + return (0); + +err: if (trk->addr.addr != NULL) + __wt_free(session, trk->addr.addr); + if (trk != NULL) + __wt_free(session, trk); + return (ret); +} + +/* + * __slvg_trk_leaf -- + * Track a leaf page. + */ +static int +__slvg_trk_leaf(WT_SESSION_IMPL *session, WT_PAGE_HEADER *dsk, + uint8_t *addr, uint32_t size, uint64_t gen, WT_STUFF *ss) +{ + WT_BTREE *btree; + WT_CELL *cell; + WT_CELL_UNPACK *unpack, _unpack; + WT_PAGE *page; + WT_TRACK *trk; + uint64_t stop_recno; + uint32_t i; + int ret; + + btree = session->btree; + unpack = &_unpack; + page = NULL; + trk = NULL; + ret = 0; + + /* Re-allocate the array of pages, as necessary. */ + if (ss->pages_next * sizeof(WT_TRACK *) == ss->pages_allocated) + WT_RET(__wt_realloc(session, &ss->pages_allocated, + (ss->pages_next + 1000) * sizeof(WT_TRACK *), &ss->pages)); + + /* Allocate a WT_TRACK entry for this new page and fill it in. */ + WT_RET(__slvg_trk_init(session, addr, size, dsk->size, gen, ss, &trk)); + + switch (dsk->type) { + case WT_PAGE_COL_FIX: + /* + * Column-store fixed-sized format: start and stop keys can be + * taken from the block's header, and doesn't contain overflow + * items. + */ + trk->col_start = dsk->recno; + trk->col_stop = dsk->recno + (dsk->u.entries - 1); + + WT_VERBOSE(session, salvage, + "%s records %" PRIu64 "-%" PRIu64, + __wt_addr_string( + session, ss->tmp1, trk->addr.addr, trk->addr.size), + trk->col_start, trk->col_stop); + break; + case WT_PAGE_COL_VAR: + /* + * Column-store variable-length format: the start key can be + * taken from the block's header, stop key requires walking + * the page. + */ + stop_recno = dsk->recno; + WT_CELL_FOREACH(btree, dsk, cell, unpack, i) { + __wt_cell_unpack(cell, unpack); + stop_recno += __wt_cell_rle(unpack); + } + + trk->col_start = dsk->recno; + trk->col_stop = stop_recno - 1; + + WT_VERBOSE(session, salvage, + "%s records %" PRIu64 "-%" PRIu64, + __wt_addr_string( + session, ss->tmp1, trk->addr.addr, trk->addr.size), + trk->col_start, trk->col_stop); + + /* Column-store pages can contain overflow items. */ + WT_ERR(__slvg_trk_leaf_ovfl(session, dsk, trk)); + break; + case WT_PAGE_ROW_LEAF: + /* + * Row-store format: copy the first and last keys on the page. + * Keys are prefix-compressed, the simplest and slowest thing + * to do is instantiate the in-memory page (which instantiates + * the prefix keys at specific split points), then instantiate + * and copy the full keys, then free the page. We do this + * on every leaf page, and if you need to speed up the salvage, + * it's probably a great place to start. + */ + WT_ERR(__wt_page_inmem(session, NULL, NULL, dsk, &page)); + WT_ERR(__wt_row_key(session, + page, &page->u.row.d[0], &trk->row_start)); + WT_ERR(__wt_row_key(session, + page, &page->u.row.d[page->entries - 1], + &trk->row_stop)); + + if (WT_VERBOSE_ISSET(session, salvage)) { + WT_ERR(__wt_buf_set_printable(session, ss->tmp1, + trk->row_start.data, trk->row_start.size)); + WT_VERBOSE(session, salvage, + "%s start key %.*s", + __wt_addr_string(session, + ss->tmp2, trk->addr.addr, trk->addr.size), + (int)ss->tmp1->size, (char *)ss->tmp1->data); + WT_ERR(__wt_buf_set_printable(session, ss->tmp1, + trk->row_stop.data, trk->row_stop.size)); + WT_VERBOSE(session, salvage, + "%s stop key %.*s", + __wt_addr_string(session, + ss->tmp2, trk->addr.addr, trk->addr.size), + (int)ss->tmp1->size, (char *)ss->tmp1->data); + } + + /* Row-store pages can contain overflow items. */ + WT_ERR(__slvg_trk_leaf_ovfl(session, dsk, trk)); + break; + } + ss->pages[ss->pages_next++] = trk; + + if (0) { +err: __wt_free(session, trk); + } + if (page != NULL) + __wt_page_out(session, page, WT_PAGE_FREE_IGNORE_DISK); + return (ret); +} + +/* + * __slvg_trk_ovfl -- + * Track an overflow page. + */ +static int +__slvg_trk_ovfl(WT_SESSION_IMPL *session, WT_PAGE_HEADER *dsk, + uint8_t *addr, uint32_t size, uint64_t gen, WT_STUFF *ss) +{ + WT_TRACK *trk; + + /* + * Reallocate the overflow page array as necessary, then save the + * page's location information. + */ + if (ss->ovfl_next * sizeof(WT_TRACK *) == ss->ovfl_allocated) + WT_RET(__wt_realloc(session, &ss->ovfl_allocated, + (ss->ovfl_next + 1000) * sizeof(WT_TRACK *), &ss->ovfl)); + + WT_RET(__slvg_trk_init(session, addr, size, dsk->size, gen, ss, &trk)); + ss->ovfl[ss->ovfl_next++] = trk; + + return (0); +} + +/* + * __slvg_trk_leaf_ovfl -- + * Search a leaf page for overflow items. + */ +static int +__slvg_trk_leaf_ovfl( + WT_SESSION_IMPL *session, WT_PAGE_HEADER *dsk, WT_TRACK *trk) +{ + WT_BTREE *btree; + WT_CELL *cell; + WT_CELL_UNPACK *unpack, _unpack; + uint32_t i, ovfl_cnt; + + btree = session->btree; + unpack = &_unpack; + + /* + * Two passes: count the overflow items, then copy them into an + * allocated array. + */ + ovfl_cnt = 0; + WT_CELL_FOREACH(btree, dsk, cell, unpack, i) { + __wt_cell_unpack(cell, unpack); + if (unpack->ovfl) + ++ovfl_cnt; + } + if (ovfl_cnt == 0) + return (0); + + WT_RET(__wt_calloc_def(session, ovfl_cnt, &trk->ovfl)); + trk->ovfl_cnt = ovfl_cnt; + + ovfl_cnt = 0; + WT_CELL_FOREACH(btree, dsk, cell, unpack, i) { + __wt_cell_unpack(cell, unpack); + if (unpack->ovfl) { + WT_RET(__wt_strndup(session, unpack->data, + unpack->size, &trk->ovfl[ovfl_cnt].addr)); + trk->ovfl[ovfl_cnt].size = unpack->size; + + WT_VERBOSE(session, salvage, + "%s overflow reference %s", + __wt_addr_string(session, + trk->ss->tmp1, trk->addr.addr, trk->addr.size), + __wt_addr_string(session, + trk->ss->tmp2, unpack->data, unpack->size)); + + if (++ovfl_cnt == trk->ovfl_cnt) + break; + } + } + + return (0); +} + +/* + * __slvg_col_range -- + * Figure out the leaf pages we need and free the leaf pages we don't. + * + * When pages split, the key range is split across multiple pages. If not all + * of the old versions of the page are overwritten, or not all of the new pages + * are written, or some of the pages are corrupted, salvage will read different + * pages with overlapping key ranges, at different LSNs. + * + * We salvage all of the key ranges we find, at the latest LSN value: this means + * we may resurrect pages of deleted items, as page deletion doesn't write leaf + * pages and salvage will read and instantiate the contents of an old version of + * the deleted page. + * + * The leaf page array is sorted in key order, and secondarily on LSN: what this + * means is that for each new key range, the first page we find is the best page + * for that key. The process is to walk forward from each page until we reach + * a page with a starting key after the current page's stopping key. + * + * For each of page, check to see if they overlap the current page's key range. + * If they do, resolve the overlap. Because WiredTiger rarely splits pages, + * overlap resolution usually means discarding a page because the key ranges + * are the same, and one of the pages is simply an old version of the other. + * + * However, it's possible more complex resolution is necessary. For example, + * here's an improbably complex list of page ranges and LSNs: + * + * Page Range LSN + * 30 A-G 3 + * 31 C-D 4 + * 32 B-C 5 + * 33 C-F 6 + * 34 C-D 7 + * 35 F-M 8 + * 36 H-O 9 + * + * We walk forward from each page reviewing all other pages in the array that + * overlap the range. For each overlap, the current or the overlapping + * page is updated so the page with the most recent information for any range + * "owns" that range. Here's an example for page 30. + * + * Review page 31: because page 31 has the range C-D and a higher LSN than page + * 30, page 30 would "split" into two ranges, A-C and E-G, conceding the C-D + * range to page 31. The new track element would be inserted into array with + * the following result: + * + * Page Range LSN + * 30 A-C 3 << Changed WT_TRACK element + * 31 C-D 4 + * 32 B-C 5 + * 33 C-F 6 + * 34 C-D 7 + * 30 E-G 3 << New WT_TRACK element + * 35 F-M 8 + * 36 H-O 9 + * + * Continue the review of the first element, using its new values. + * + * Review page 32: because page 31 has the range B-C and a higher LSN than page + * 30, page 30's A-C range would be truncated, conceding the B-C range to page + * 32. + * 30 A-B 3 + * E-G 3 + * 31 C-D 4 + * 32 B-C 5 + * 33 C-F 6 + * 34 C-D 7 + * + * Review page 33: because page 33 has a starting key (C) past page 30's ending + * key (B), we stop evaluating page 30's A-B range, as there can be no further + * overlaps. + * + * This process is repeated for each page in the array. + * + * When page 33 is processed, we'd discover that page 33's C-F range overlaps + * page 30's E-G range, and page 30's E-G range would be updated, conceding the + * E-F range to page 33. + * + * This is not computationally expensive because we don't walk far forward in + * the leaf array because it's sorted by starting key, and because WiredTiger + * splits are rare, the chance of finding the kind of range overlap requiring + * re-sorting the array is small. + */ +static int +__slvg_col_range(WT_SESSION_IMPL *session, WT_STUFF *ss) +{ + WT_TRACK *jtrk; + uint32_t i, j; + + /* + * DO NOT MODIFY THIS CODE WITHOUT REVIEWING THE CORRESPONDING ROW- OR + * COLUMN-STORE CODE: THEY ARE IDENTICAL OTHER THAN THE PAGES THAT ARE + * BEING HANDLED. + * + * Walk the page array looking for overlapping key ranges, adjusting + * the ranges based on the LSN until there are no overlaps. + * + * DO NOT USE POINTERS INTO THE ARRAY: THE ARRAY IS RE-SORTED IN PLACE + * AS ENTRIES ARE SPLIT, SO ARRAY REFERENCES MUST ALWAYS BE ARRAY BASE + * PLUS OFFSET. + */ + for (i = 0; i < ss->pages_next; ++i) { + if (ss->pages[i] == NULL) + continue; + + /* Check for pages that overlap our page. */ + for (j = i + 1; j < ss->pages_next; ++j) { + if (ss->pages[j] == NULL) + continue; + /* + * We're done if this page starts after our stop, no + * subsequent pages can overlap our page. + */ + if (ss->pages[j]->col_start > + ss->pages[i]->col_stop) + break; + + /* There's an overlap, fix it up. */ + jtrk = ss->pages[j]; + WT_RET(__slvg_col_range_overlap(session, i, j, ss)); + + /* + * If the overlap resolution changed the entry's start + * key, the entry might have moved and the page array + * re-sorted, and pages[j] would reference a different + * page. We don't move forward if that happened, we + * re-process the slot again (by decrementing j before + * the loop's increment). + */ + if (ss->pages[j] != NULL && jtrk != ss->pages[j]) + --j; + } + } + return (0); +} + +/* + * __slvg_col_range_overlap -- + * Two column-store key ranges overlap, deal with it. + */ +static int +__slvg_col_range_overlap( + WT_SESSION_IMPL *session, uint32_t a_slot, uint32_t b_slot, WT_STUFF *ss) +{ + WT_TRACK *a_trk, *b_trk, *new; + + /* + * DO NOT MODIFY THIS CODE WITHOUT REVIEWING THE CORRESPONDING ROW- OR + * COLUMN-STORE CODE: THEY ARE IDENTICAL OTHER THAN THE PAGES THAT ARE + * BEING HANDLED. + */ + a_trk = ss->pages[a_slot]; + b_trk = ss->pages[b_slot]; + + WT_VERBOSE(session, salvage, + "%s and %s range overlap", + __wt_addr_string( + session, ss->tmp1, a_trk->addr.addr, a_trk->addr.size), + __wt_addr_string( + session, ss->tmp2, b_trk->addr.addr, b_trk->addr.size)); + + /* + * The key ranges of two WT_TRACK pages in the array overlap -- choose + * the ranges we're going to take from each. + * + * We can think of the overlap possibilities as 11 different cases: + * + * AAAAAAAAAAAAAAAAAA + * #1 BBBBBBBBBBBBBBBBBB pages are the same + * #2 BBBBBBBBBBBBB overlaps the beginning + * #3 BBBBBBBBBBBBBBBB overlaps the end + * #4 BBBBB B is a prefix of A + * #5 BBBBBB B is middle of A + * #6 BBBBBBBBBB B is a suffix of A + * + * and: + * + * BBBBBBBBBBBBBBBBBB + * #7 AAAAAAAAAAAAAAAA same as #2 + * #8 AAAAAAAAAAAAA same as #3 + * #9 AAAAA A is a prefix of B + * #10 AAAAAAAAAA A is a suffix of B + * #11 AAAAAA A is middle of B + * + * Because the leaf page array was sorted by record number and a_trk + * appears earlier in that array than b_trk, cases #2/7, #10 and #11 + * are impossible. + * + * Finally, there's one additional complicating factor -- final ranges + * are assigned based on the page's LSN. + */ + if (a_trk->col_start == b_trk->col_start) { + /* + * Case #1, #4 and #9. + * + * The secondary sort of the leaf page array was the page's LSN, + * in high-to-low order, which means a_trk has a higher LSN, and + * is more desirable, than b_trk. In cases #1 and #4 and #9, + * where the start of the range is the same for the two pages, + * this simplifies things, it guarantees a_trk has a higher LSN + * than b_trk. + */ + if (a_trk->col_stop >= b_trk->col_stop) + /* + * Case #1, #4: a_trk is a superset of b_trk, and a_trk + * is more desirable -- discard b_trk. + */ + goto delete; + + /* + * Case #9: b_trk is a superset of a_trk, but a_trk is more + * desirable: keep both but delete a_trk's key range from + * b_trk. + */ + b_trk->col_start = a_trk->col_stop + 1; + __slvg_col_trk_update_start(b_slot, ss); + F_SET(b_trk, WT_TRACK_MERGE); + goto merge; + } + + if (a_trk->col_stop == b_trk->col_stop) { + /* Case #6. */ + if (a_trk->gen > b_trk->gen) + /* + * Case #6: a_trk is a superset of b_trk and a_trk is + * more desirable -- discard b_trk. + */ + goto delete; + + /* + * Case #6: a_trk is a superset of b_trk, but b_trk is more + * desirable: keep both but delete b_trk's key range from a_trk. + */ + a_trk->col_stop = b_trk->col_start - 1; + F_SET(a_trk, WT_TRACK_MERGE); + goto merge; + } + + if (a_trk->col_stop < b_trk->col_stop) { + /* Case #3/8. */ + if (a_trk->gen > b_trk->gen) { + /* + * Case #3/8: a_trk is more desirable, delete a_trk's + * key range from b_trk; + */ + b_trk->col_start = a_trk->col_stop + 1; + __slvg_col_trk_update_start(b_slot, ss); + F_SET(b_trk, WT_TRACK_MERGE); + } else { + /* + * Case #3/8: b_trk is more desirable, delete b_trk's + * key range from a_trk; + */ + a_trk->col_stop = b_trk->col_start - 1; + F_SET(a_trk, WT_TRACK_MERGE); + } + goto merge; + } + + /* + * Case #5: a_trk is a superset of b_trk and a_trk is more desirable -- + * discard b_trk. + */ + if (a_trk->gen > b_trk->gen) { +delete: WT_RET(__slvg_trk_free(session, + &ss->pages[b_slot], WT_TRK_FREE_BLOCKS | WT_TRK_FREE_OVFL)); + return (0); + } + + /* + * Case #5: b_trk is more desirable and is a middle chunk of a_trk. + * Split a_trk into two parts, the key range before b_trk and the + * key range after b_trk. + * + * First, create a copy of the original page's WT_TRACK information + * (same LSN, addr and size), that we'll use to reference the key + * range at the end of a_trk. + */ + WT_RET(__slvg_trk_init(session, a_trk->addr.addr, + a_trk->addr.size, a_trk->size, a_trk->gen, ss, &new)); + + /* + * Second, reallocate the array of pages if necessary, and then insert + * the new element into the array after the existing element (that's + * probably wrong, but we'll fix it up in a second). + */ + if (ss->pages_next * sizeof(WT_TRACK *) == ss->pages_allocated) + WT_RET(__wt_realloc(session, &ss->pages_allocated, + (ss->pages_next + 1000) * sizeof(WT_TRACK *), &ss->pages)); + memmove(ss->pages + a_slot + 1, ss->pages + a_slot, + (ss->pages_next - a_slot) * sizeof(*ss->pages)); + ss->pages[a_slot + 1] = new; + ++ss->pages_next; + + /* + * Third, set its start key to be the first key after the stop key of + * the middle chunk (that's b_trk), and its stop key to be the stop key + * of the original chunk, and call __slvg_col_trk_update_start. That + * function will re-sort the WT_TRACK array as necessary to move our + * new entry into the right sorted location. + */ + new->col_start = b_trk->col_stop + 1; + new->col_stop = a_trk->col_stop; + __slvg_col_trk_update_start(a_slot + 1, ss); + + /* + * Fourth, the new WT_TRACK information doesn't reference any file + * blocks (let the original a_trk structure reference file blocks). + */ + F_SET(new, WT_TRACK_MERGE | WT_TRACK_NO_FILE_BLOCKS); + + /* + * Finally, set the original WT_TRACK information to reference only + * the initial key space in the page, that is, everything up to the + * starting key of the middle chunk (that's b_trk). + */ + a_trk->col_stop = b_trk->col_start - 1; + F_SET(a_trk, WT_TRACK_MERGE); + +merge: WT_VERBOSE(session, salvage, + "%s and %s require merge", + __wt_addr_string( + session, ss->tmp1, a_trk->addr.addr, a_trk->addr.size), + __wt_addr_string( + session, ss->tmp2, b_trk->addr.addr, b_trk->addr.size)); + return (0); +} + +/* + * __slvg_col_trk_update_start -- + * Update a column-store page's start key after an overlap. + */ +static void +__slvg_col_trk_update_start(uint32_t slot, WT_STUFF *ss) +{ + WT_TRACK *trk; + uint32_t i; + + trk = ss->pages[slot]; + + /* + * If we deleted an initial piece of the WT_TRACK name space, it may no + * longer be in the right location. + * + * For example, imagine page #1 has the key range 30-50, it split, and + * we wrote page #2 with key range 30-40, and page #3 key range with + * 40-50, where pages #2 and #3 have larger LSNs than page #1. When the + * key ranges were sorted, page #2 came first, then page #1 (because of + * their earlier start keys than page #3), and page #2 came before page + * #1 because of its LSN. When we resolve the overlap between page #2 + * and page #1, we truncate the initial key range of page #1, and it now + * sorts after page #3, because it has the same starting key of 40, and + * a lower LSN. + * + * We have already updated b_trk's start key; what we may have to do is + * re-sort some number of elements in the list. + */ + for (i = slot + 1; i < ss->pages_next; ++i) { + if (ss->pages[i] == NULL) + continue; + if (ss->pages[i]->col_start > trk->col_stop) + break; + } + i -= slot; + if (i > 1) + qsort(ss->pages + slot, (size_t)i, + sizeof(WT_TRACK *), __slvg_trk_compare_key); +} + +/* + * __slvg_col_range_missing -- + * Detect missing ranges from column-store files. + */ +static void +__slvg_col_range_missing(WT_SESSION_IMPL *session, WT_STUFF *ss) +{ + WT_TRACK *trk; + uint64_t r; + uint32_t i; + + for (i = 0, r = 0; i < ss->pages_next; ++i) { + if ((trk = ss->pages[i]) == NULL) + continue; + if (trk->col_start != r + 1) { + WT_VERBOSE(session, salvage, + "%s column-store missing range from %" + PRIu64 " to %" PRIu64 " inclusive", + __wt_addr_string(session, + ss->tmp1, trk->addr.addr, trk->addr.size), + r + 1, trk->col_start - 1); + + /* + * We need to instantiate deleted items for the missing + * record range. + */ + trk->col_missing = r + 1; + F_SET(trk, WT_TRACK_MERGE); + } + r = trk->col_stop; + } +} + +/* + * __slvg_col_build_internal -- + * Build a column-store in-memory page that references all of the leaf + * pages we've found. + */ +static int +__slvg_col_build_internal( + WT_SESSION_IMPL *session, uint32_t leaf_cnt, WT_STUFF *ss) +{ + WT_ADDR *addr; + WT_PAGE *page; + WT_REF *ref; + WT_TRACK *trk; + uint32_t i; + int ret; + + /* Allocate a column-store internal page. */ + WT_RET(__wt_calloc_def(session, 1, &page)); + WT_ERR(__wt_calloc_def(session, (size_t)leaf_cnt, &page->u.intl.t)); + + /* Fill it in. */ + page->parent = NULL; /* Root page */ + page->ref = NULL; + page->read_gen = 0; + page->u.intl.recno = 1; + page->entries = leaf_cnt; + page->type = WT_PAGE_COL_INT; + WT_RET(__wt_page_modify_init(session, page)); + __wt_page_modify_set(page); + + for (ref = page->u.intl.t, i = 0; i < ss->pages_next; ++i) { + if ((trk = ss->pages[i]) == NULL) + continue; + + WT_ERR(__wt_calloc(session, 1, sizeof(WT_ADDR), &addr)); + WT_ERR(__wt_strndup(session, + (char *)trk->addr.addr, trk->addr.size, &addr->addr)); + addr->size = trk->addr.size; + ref->page = NULL; + ref->addr = addr; + ref->u.recno = trk->col_start; + ref->state = WT_REF_DISK; + + /* + * If the page's key range is unmodified from when we read it + * (in other words, we didn't merge part of this page with + * another page), we can use the page without change. If we + * did merge with another page, we must build a page reflecting + * the updated key range, and that requires an additional pass + * to free its backing blocks. + */ + if (F_ISSET(trk, WT_TRACK_MERGE)) { + ss->merge_free = 1; + + WT_ERR(__slvg_col_build_leaf(session, trk, page, ref)); + } + ++ref; + } + + /* Write the internal page to disk. */ + return (__wt_rec_evict(session, page, WT_REC_SINGLE)); + +err: __wt_page_out(session, page, 0); + return (ret); +} + +/* + * __slvg_col_build_leaf -- + * Build a column-store leaf page for a merged page. + */ +static int +__slvg_col_build_leaf( + WT_SESSION_IMPL *session, WT_TRACK *trk, WT_PAGE *parent, WT_REF *ref) +{ + WT_COL *save_col_var; + WT_PAGE *page; + WT_SALVAGE_COOKIE *cookie, _cookie; + uint64_t skip, take; + uint32_t save_entries; + int ret; + + cookie = &_cookie; + WT_CLEAR(*cookie); + ret = 0; + + /* Get the original page, including the full in-memory setup. */ + WT_RET(__wt_page_in(session, parent, ref)); + page = ref->page; + save_col_var = page->u.col_var.d; + save_entries = page->entries; + + /* + * Calculate the number of K/V entries we are going to skip, and + * the total number of K/V entries we'll take from this page. + */ + cookie->skip = skip = trk->col_start - page->u.col_var.recno; + cookie->take = take = (trk->col_stop - trk->col_start) + 1; + + WT_VERBOSE(session, salvage, + "%s merge discarding first %" PRIu64 " records, " + "then taking %" PRIu64 " records", + __wt_addr_string( + session, trk->ss->tmp1, trk->addr.addr, trk->addr.size), + skip, take); + + /* + * Discard backing overflow pages for any items being discarded that + * reference overflow pages. + */ + if (page->type == WT_PAGE_COL_VAR) + WT_ERR(__slvg_col_merge_ovfl(session, trk, page, skip, take)); + + /* + * If we're missing some part of the range, the real start range is in + * trk->col_missing, else, it's in trk->col_start. Update the parent's + * reference as well as the page itself. + */ + if (trk->col_missing == 0) + page->u.col_var.recno = trk->col_start; + else { + page->u.col_var.recno = trk->col_missing; + cookie->missing = trk->col_start - trk->col_missing; + + WT_VERBOSE(session, salvage, + "%s merge inserting %" PRIu64 " missing records", + __wt_addr_string( + session, trk->ss->tmp1, trk->addr.addr, trk->addr.size), + cookie->missing); + } + ref->u.recno = page->u.col_var.recno; + + /* + * We can't discard the original blocks associated with this page now. + * (The problem is we don't want to overwrite any original information + * until the salvage run succeeds -- if we free the blocks now, the next + * merge page we write might allocate those blocks and overwrite them, + * and should the salvage run eventually fail, the original information + * would have been lost.) Clear the reference addr so eviction doesn't + * free the underlying blocks. + */ + __wt_free(session, ((WT_ADDR *)ref->addr)->addr); + __wt_free(session, ref->addr); + ref->addr = NULL; + + /* Write the new version of the leaf page to disk. */ + WT_ERR(__wt_page_modify_init(session, page)); + __wt_page_modify_set(page); + WT_ERR(__wt_rec_write(session, page, cookie)); + + /* Reset the page. */ + page->u.col_var.d = save_col_var; + page->entries = save_entries; + + __wt_page_release(session, page); + ret = __wt_rec_evict(session, page, WT_REC_SINGLE); + + if (0) { +err: __wt_page_release(session, page); + } + + return (ret); +} + +/* + * __slvg_col_merge_ovfl -- + * Free file blocks referenced from keys discarded from merged pages. + */ +static int +__slvg_col_merge_ovfl(WT_SESSION_IMPL *session, + WT_TRACK *trk, WT_PAGE *page, uint64_t skip, uint64_t take) +{ + WT_CELL_UNPACK *unpack, _unpack; + WT_CELL *cell; + WT_COL *cip; + uint64_t recno, start, stop; + uint32_t i; + + unpack = &_unpack; + + recno = page->u.col_var.recno; + start = recno + skip; + stop = (recno + skip + take) - 1; + + WT_COL_FOREACH(page, cip, i) { + cell = WT_COL_PTR(page, cip); + __wt_cell_unpack(cell, unpack); + recno += __wt_cell_rle(unpack); + + if (unpack->type != WT_CELL_VALUE_OVFL) + continue; + if (recno >= start && recno <= stop) + continue; + + WT_VERBOSE(session, salvage, + "%s merge discard freed overflow reference %s", + __wt_addr_string(session, + trk->ss->tmp1, trk->addr.addr, trk->addr.size), + __wt_addr_string(session, + trk->ss->tmp2, unpack->data, unpack->size)); + + WT_RET(__wt_bm_free(session, unpack->data, unpack->size)); + } + return (0); +} + +/* + * __slvg_row_range -- + * Figure out the leaf pages we need and discard everything else. At the + * same time, tag the overflow pages they reference. + */ +static int +__slvg_row_range(WT_SESSION_IMPL *session, WT_STUFF *ss) +{ + WT_TRACK *jtrk; + WT_BTREE *btree; + uint32_t i, j; + int cmp; + + btree = session->btree; + + /* + * DO NOT MODIFY THIS CODE WITHOUT REVIEWING THE CORRESPONDING ROW- OR + * COLUMN-STORE CODE: THEY ARE IDENTICAL OTHER THAN THE PAGES THAT ARE + * BEING HANDLED. + * + * Walk the page array looking for overlapping key ranges, adjusting + * the ranges based on the LSN until there are no overlaps. + * + * DO NOT USE POINTERS INTO THE ARRAY: THE ARRAY IS RE-SORTED IN PLACE + * AS ENTRIES ARE SPLIT, SO ARRAY REFERENCES MUST ALWAYS BE ARRAY BASE + * PLUS OFFSET. + */ + for (i = 0; i < ss->pages_next; ++i) { + if (ss->pages[i] == NULL) + continue; + + /* Check for pages that overlap our page. */ + for (j = i + 1; j < ss->pages_next; ++j) { + if (ss->pages[j] == NULL) + continue; + /* + * We're done if this page starts after our stop, no + * subsequent pages can overlap our page. + */ + WT_RET(WT_BTREE_CMP(session, btree, + &ss->pages[j]->row_start, + &ss->pages[i]->row_stop, cmp)); + if (cmp > 0) + break; + + /* There's an overlap, fix it up. */ + jtrk = ss->pages[j]; + WT_RET(__slvg_row_range_overlap(session, i, j, ss)); + + /* + * If the overlap resolution changed the entry's start + * key, the entry might have moved and the page array + * re-sorted, and pages[j] would reference a different + * page. We don't move forward if that happened, we + * re-process the slot again (by decrementing j before + * the loop's increment). + */ + if (ss->pages[j] != NULL && jtrk != ss->pages[j]) + --j; + } + } + return (0); +} + +/* + * __slvg_row_range_overlap -- + * Two row-store key ranges overlap, deal with it. + */ +static int +__slvg_row_range_overlap( + WT_SESSION_IMPL *session, uint32_t a_slot, uint32_t b_slot, WT_STUFF *ss) +{ + WT_BTREE *btree; + WT_TRACK *a_trk, *b_trk, *new; + int cmp; + + /* + * DO NOT MODIFY THIS CODE WITHOUT REVIEWING THE CORRESPONDING ROW- OR + * COLUMN-STORE CODE: THEY ARE IDENTICAL OTHER THAN THE PAGES THAT ARE + * BEING HANDLED. + */ + btree = session->btree; + + a_trk = ss->pages[a_slot]; + b_trk = ss->pages[b_slot]; + + WT_VERBOSE(session, salvage, + "%s and %s range overlap", + __wt_addr_string( + session, ss->tmp1, a_trk->addr.addr, a_trk->addr.size), + __wt_addr_string( + session, ss->tmp2, b_trk->addr.addr, b_trk->addr.size)); + + /* + * The key ranges of two WT_TRACK pages in the array overlap -- choose + * the ranges we're going to take from each. + * + * We can think of the overlap possibilities as 11 different cases: + * + * AAAAAAAAAAAAAAAAAA + * #1 BBBBBBBBBBBBBBBBBB pages are the same + * #2 BBBBBBBBBBBBB overlaps the beginning + * #3 BBBBBBBBBBBBBBBB overlaps the end + * #4 BBBBB B is a prefix of A + * #5 BBBBBB B is middle of A + * #6 BBBBBBBBBB B is a suffix of A + * + * and: + * + * BBBBBBBBBBBBBBBBBB + * #7 AAAAAAAAAAAAAAAA same as #2 + * #8 AAAAAAAAAAAAA same as #3 + * #9 AAAAA A is a prefix of B + * #10 AAAAAAAAAA A is a suffix of B + * #11 AAAAAA A is middle of B + * + * Because the leaf page array was sorted by record number and a_trk + * appears earlier in that array than b_trk, cases #2/7, #10 and #11 + * are impossible. + * + * Finally, there's one additional complicating factor -- final ranges + * are assigned based on the page's LSN. + */ +#define A_TRK_START (&a_trk->row_start) +#define A_TRK_STOP (&a_trk->row_stop) +#define B_TRK_START (&b_trk->row_start) +#define B_TRK_STOP (&b_trk->row_stop) +#define SLOT_START(i) (&ss->pages[i]->row_start) +#define __slvg_key_copy(session, dst, src) \ + __wt_buf_set(session, dst, (src)->data, (src)->size) + + WT_RET(WT_BTREE_CMP(session, btree, A_TRK_START, B_TRK_START, cmp)); + if (cmp == 0) { + /* + * Case #1, #4 and #9. + * + * The secondary sort of the leaf page array was the page's LSN, + * in high-to-low order, which means a_trk has a higher LSN, and + * is more desirable, than b_trk. In cases #1 and #4 and #9, + * where the start of the range is the same for the two pages, + * this simplifies things, it guarantees a_trk has a higher LSN + * than b_trk. + */ + WT_RET( + WT_BTREE_CMP(session, btree, A_TRK_STOP, B_TRK_STOP, cmp)); + if (cmp >= 0) + /* + * Case #1, #4: a_trk is a superset of b_trk, and a_trk + * is more desirable -- discard b_trk. + */ + goto delete; + + /* + * Case #9: b_trk is a superset of a_trk, but a_trk is more + * desirable: keep both but delete a_trk's key range from + * b_trk. + */ + WT_RET(__slvg_row_trk_update_start( + session, A_TRK_STOP, b_slot, ss)); + F_SET(b_trk, WT_TRACK_CHECK_START | WT_TRACK_MERGE); + goto merge; + } + + WT_RET(WT_BTREE_CMP(session, btree, A_TRK_STOP, B_TRK_STOP, cmp)); + if (cmp == 0) { + /* Case #6. */ + if (a_trk->gen > b_trk->gen) + /* + * Case #6: a_trk is a superset of b_trk and a_trk is + * more desirable -- discard b_trk. + */ + goto delete; + + /* + * Case #6: a_trk is a superset of b_trk, but b_trk is more + * desirable: keep both but delete b_trk's key range from a_trk. + */ + WT_RET(__slvg_key_copy(session, A_TRK_STOP, B_TRK_START)); + F_SET(a_trk, WT_TRACK_CHECK_STOP | WT_TRACK_MERGE); + goto merge; + } + + WT_RET(WT_BTREE_CMP(session, btree, A_TRK_STOP, B_TRK_STOP, cmp)); + if (cmp < 0) { + /* Case #3/8. */ + if (a_trk->gen > b_trk->gen) { + /* + * Case #3/8: a_trk is more desirable, delete a_trk's + * key range from b_trk; + */ + WT_RET(__slvg_row_trk_update_start( + session, A_TRK_STOP, b_slot, ss)); + F_SET(b_trk, WT_TRACK_CHECK_START | WT_TRACK_MERGE); + } else { + /* + * Case #3/8: b_trk is more desirable, delete b_trk's + * key range from a_trk; + */ + WT_RET(__slvg_key_copy( + session, A_TRK_STOP, B_TRK_START)); + F_SET(a_trk, WT_TRACK_CHECK_STOP | WT_TRACK_MERGE); + } + goto merge; + } + + /* + * Case #5: a_trk is a superset of b_trk and a_trk is more desirable -- + * discard b_trk. + */ + if (a_trk->gen > b_trk->gen) { +delete: WT_RET(__slvg_trk_free(session, + &ss->pages[b_slot], WT_TRK_FREE_BLOCKS | WT_TRK_FREE_OVFL)); + return (0); + } + + /* + * Case #5: b_trk is more desirable and is a middle chunk of a_trk. + * Split a_trk into two parts, the key range before b_trk and the + * key range after b_trk. + * + * First, create a copy of the original page's WT_TRACK information + * (same LSN, addr and size), that we'll use to reference the key + * range at the end of a_trk. + */ + WT_RET(__slvg_trk_init(session, a_trk->addr.addr, + a_trk->addr.size, a_trk->size, a_trk->gen, ss, &new)); + + /* + * Second, reallocate the array of pages if necessary, and then insert + * the new element into the array after the existing element (that's + * probably wrong, but we'll fix it up in a second). + */ + if (ss->pages_next * sizeof(WT_TRACK *) == ss->pages_allocated) + WT_RET(__wt_realloc(session, &ss->pages_allocated, + (ss->pages_next + 1000) * sizeof(WT_TRACK *), &ss->pages)); + memmove(ss->pages + a_slot + 1, ss->pages + a_slot, + (ss->pages_next - a_slot) * sizeof(*ss->pages)); + ss->pages[a_slot + 1] = new; + ++ss->pages_next; + + /* + * Third, set its its stop key to be the stop key of the original chunk, + * and call __slvg_row_trk_update_start. That function will both set + * the start key to be the first key after the stop key of the middle + * chunk (that's b_trk), and re-sort the WT_TRACK array as necessary to + * move our new entry into the right sorted location. + */ + WT_RET(__slvg_key_copy(session, &new->row_stop, A_TRK_STOP)); + WT_RET( + __slvg_row_trk_update_start(session, B_TRK_STOP, a_slot + 1, ss)); + + /* + * Fourth, the new WT_TRACK information doesn't reference any file + * blocks (let the original a_trk structure reference file blocks). + */ + F_SET(new, + WT_TRACK_CHECK_START | WT_TRACK_MERGE | WT_TRACK_NO_FILE_BLOCKS); + + /* + * Finally, set the original WT_TRACK information to reference only + * the initial key space in the page, that is, everything up to the + * starting key of the middle chunk (that's b_trk). + */ + WT_RET(__slvg_key_copy(session, A_TRK_STOP, B_TRK_START)); + F_SET(a_trk, WT_TRACK_CHECK_STOP | WT_TRACK_MERGE); + +merge: WT_VERBOSE(session, salvage, + "%s and %s require merge", + __wt_addr_string( + session, ss->tmp1, a_trk->addr.addr, a_trk->addr.size), + __wt_addr_string( + session, ss->tmp2, b_trk->addr.addr, b_trk->addr.size)); + return (0); +} + +/* + * __slvg_row_trk_update_start -- + * Update a row-store page's start key after an overlap. + */ +static int +__slvg_row_trk_update_start( + WT_SESSION_IMPL *session, WT_ITEM *stop, uint32_t slot, WT_STUFF *ss) +{ + WT_BTREE *btree; + WT_IKEY *ikey; + WT_ITEM *dsk, *key, *item, _item; + WT_PAGE *page; + WT_ROW *rip; + WT_TRACK *trk; + uint32_t i; + int cmp, found, ret; + + btree = session->btree; + key = dsk = NULL; + page = NULL; + found = ret = 0; + + trk = ss->pages[slot]; + + /* + * If we deleted an initial piece of the WT_TRACK name space, it may no + * longer be in the right location. + * + * For example, imagine page #1 has the key range 30-50, it split, and + * we wrote page #2 with key range 30-40, and page #3 key range with + * 40-50, where pages #2 and #3 have larger LSNs than page #1. When the + * key ranges were sorted, page #2 came first, then page #1 (because of + * their earlier start keys than page #3), and page #2 came before page + * #1 because of its LSN. When we resolve the overlap between page #2 + * and page #1, we truncate the initial key range of page #1, and it now + * sorts after page #3, because it has the same starting key of 40, and + * a lower LSN. + * + * First, update the WT_TRACK start key based on the specified stop key. + * + * Read and instantiate the WT_TRACK page (we don't have to verify the + * page, nor do we have to be quiet on error, we've already read this + * page successfully). + */ + WT_RET(__wt_scr_alloc(session, trk->size, &dsk)); + WT_ERR(__wt_bm_read(session, dsk, trk->addr.addr, trk->addr.size)); + WT_ERR(__wt_page_inmem(session, NULL, NULL, dsk->mem, &page)); + + /* + * Walk the page, looking for a key sorting greater than the specified + * stop key -- that's our new start key. + */ + WT_RET(__wt_scr_alloc(session, 0, &key)); + WT_ROW_FOREACH(page, rip, i) { + if (__wt_off_page(page, rip->key)) { + ikey = rip->key; + _item.data = WT_IKEY_DATA(ikey); + _item.size = ikey->size; + item = &_item; + } else { + WT_ERR(__wt_row_key(session, page, rip, key)); + item = key; + } + WT_ERR(WT_BTREE_CMP(session, btree, item, stop, cmp)); + if (cmp > 0) { + found = 1; + break; + } + } + + /* + * We know that at least one key on the page sorts after the specified + * stop key, otherwise the page would have entirely overlapped and we + * would have discarded it, we wouldn't be here. Therefore, this test + * is safe. (But, it never hurts to check.) + */ + WT_RET_TEST(!found, WT_ERROR); + WT_RET(__slvg_key_copy(session, &trk->row_start, item)); + + /* + * We may need to re-sort some number of elements in the list. Walk + * forward in the list until reaching an entry which cannot overlap + * the adjusted entry. If it's more than a single slot, re-sort the + * entries. + */ + for (i = slot + 1; i < ss->pages_next; ++i) { + if (ss->pages[i] == NULL) + continue; + WT_ERR(WT_BTREE_CMP(session, btree, + SLOT_START(i), &trk->row_stop, cmp)); + if (cmp > 0) + break; + } + i -= slot; + if (i > 1) + qsort(ss->pages + slot, (size_t)i, + sizeof(WT_TRACK *), __slvg_trk_compare_key); + + if (page != NULL) + __wt_page_out(session, page, WT_PAGE_FREE_IGNORE_DISK); + +err: __wt_scr_free(&dsk); + __wt_scr_free(&key); + + return (ret); +} + +/* + * __slvg_row_build_internal -- + * Build a row-store in-memory page that references all of the leaf + * pages we've found. + */ +static int +__slvg_row_build_internal( + WT_SESSION_IMPL *session, uint32_t leaf_cnt, WT_STUFF *ss) +{ + WT_ADDR *addr; + WT_PAGE *page; + WT_REF *ref; + WT_TRACK *trk; + uint32_t i; + int ret; + + /* Allocate a row-store internal page. */ + WT_RET(__wt_calloc_def(session, 1, &page)); + WT_ERR(__wt_calloc_def(session, (size_t)leaf_cnt, &page->u.intl.t)); + + /* Fill it in. */ + page->parent = NULL; /* Root page */ + page->ref = NULL; + page->read_gen = 0; + page->entries = leaf_cnt; + page->type = WT_PAGE_ROW_INT; + WT_ERR(__wt_page_modify_init(session, page)); + __wt_page_modify_set(page); + + for (ref = page->u.intl.t, i = 0; i < ss->pages_next; ++i) { + if ((trk = ss->pages[i]) == NULL) + continue; + + WT_ERR(__wt_calloc(session, 1, sizeof(WT_ADDR), &addr)); + WT_ERR(__wt_strndup(session, + (char *)trk->addr.addr, trk->addr.size, &addr->addr)); + addr->size = trk->addr.size; + + ref->page = NULL; + ref->addr = addr; + ref->state = WT_REF_DISK; + + /* + * If the page's key range is unmodified from when we read it + * (in other words, we didn't merge part of this page with + * another page), we can use the page without change. If we + * did merge with another page, we must build a page reflecting + * the updated key range, and that requires an additional pass + * to free its backing blocks. + */ + if (F_ISSET(trk, WT_TRACK_MERGE)) { + ss->merge_free = 1; + + WT_ERR(__slvg_row_build_leaf( + session, trk, page, ref, ss)); + } else + WT_ERR(__wt_row_ikey_alloc(session, 0, + trk->row_start.data, + trk->row_start.size, + (WT_IKEY **)&ref->u.key)); + ++ref; + } + + /* Write the internal page to disk. */ + return (__wt_rec_evict(session, page, WT_REC_SINGLE)); + +err: __wt_page_out(session, page, 0); + return (ret); +} + +/* + * __slvg_row_build_leaf -- + * Build a row-store leaf page for a merged page. + */ +static int +__slvg_row_build_leaf(WT_SESSION_IMPL *session, + WT_TRACK *trk, WT_PAGE *parent, WT_REF *ref, WT_STUFF *ss) +{ + WT_BTREE *btree; + WT_IKEY *ikey; + WT_ITEM *item, _item, *key; + WT_PAGE *page; + WT_ROW *rip; + WT_SALVAGE_COOKIE *cookie, _cookie; + uint32_t i, skip_start, skip_stop; + int cmp, ret; + + btree = session->btree; + page = NULL; + + cookie = &_cookie; + WT_CLEAR(*cookie); + ret = 0; + + /* Allocate temporary space in which to instantiate the keys. */ + WT_RET(__wt_scr_alloc(session, 0, &key)); + + /* Get the original page, including the full in-memory setup. */ + WT_ERR(__wt_page_in(session, parent, ref)); + page = ref->page; + + /* + * Figure out how many page keys we want to take and how many we want + * to skip. + * + * If checking the starting range key, the key we're searching for will + * be equal to the starting range key. This is because we figured out + * the true merged-page start key as part of discarding initial keys + * from the page (see the __slvg_row_range_overlap function, and its + * calls to __slvg_row_trk_update_start for more information). + * + * If checking the stopping range key, we want the keys on the page that + * are less-than the stopping range key. This is because we copied a + * key from another page to define this page's stop range: that page is + * the page that owns the "equal to" range space. + */ + skip_start = skip_stop = 0; + if (F_ISSET(trk, WT_TRACK_CHECK_START)) + WT_ROW_FOREACH(page, rip, i) { + if (__wt_off_page(page, rip->key)) { + ikey = rip->key; + _item.data = WT_IKEY_DATA(ikey); + _item.size = ikey->size; + item = &_item; + } else { + WT_ERR(__wt_row_key(session, page, rip, key)); + item = key; + } + + /* + * >= is correct: see the comment above. + */ + WT_ERR(WT_BTREE_CMP(session, btree, + item, &trk->row_start, cmp)); + if (cmp >= 0) + break; + if (WT_VERBOSE_ISSET(session, salvage)) { + WT_ERR(__wt_buf_set_printable(session, + ss->tmp1, item->data, item->size)); + WT_VERBOSE(session, salvage, + "%s merge discarding leading key %.*s", + __wt_addr_string(session, + ss->tmp2, trk->addr.addr, trk->addr.size), + (int)ss->tmp1->size, + (char *)ss->tmp1->data); + } + ++skip_start; + } + if (F_ISSET(trk, WT_TRACK_CHECK_STOP)) + WT_ROW_FOREACH_REVERSE(page, rip, i) { + if (__wt_off_page(page, rip->key)) { + ikey = rip->key; + _item.data = WT_IKEY_DATA(ikey); + _item.size = ikey->size; + item = &_item; + } else { + WT_ERR(__wt_row_key(session, page, rip, key)); + item = key; + } + + /* + * < is correct: see the comment above. + */ + WT_ERR(WT_BTREE_CMP(session, btree, + item, &trk->row_stop, cmp)); + if (cmp < 0) + break; + if (WT_VERBOSE_ISSET(session, salvage)) { + WT_ERR(__wt_buf_set_printable(session, + ss->tmp1, item->data, item->size)); + WT_VERBOSE(session, salvage, + "%s merge discarding trailing key %.*s", + __wt_addr_string(session, + ss->tmp2, trk->addr.addr, trk->addr.size), + (int)ss->tmp1->size, + (char *)ss->tmp1->data); + } + ++skip_stop; + } + + /* + * I believe it's no longer possible for a salvaged page to be entirely + * empty, that is, if we selected the page for salvage, there is at + * least one cell on the page we want. This is a change from previous + * behavior, so I'm asserting it. + */ + WT_ASSERT_RET(session, skip_start + skip_stop < page->entries); + + /* + * Take a copy of this page's first key to define the start of + * its range. The key may require processing, otherwise, it's + * a copy from the page. + */ + rip = page->u.row.d + skip_start; + if (__wt_off_page(page, rip->key)) { + ikey = rip->key; + WT_ERR(__wt_row_ikey_alloc(session, 0, + WT_IKEY_DATA(ikey), ikey->size, (WT_IKEY **)&ref->u.key)); + } else { + WT_ERR(__wt_row_key(session, page, rip, key)); + WT_ERR(__wt_row_ikey_alloc(session, 0, + key->data, key->size, (WT_IKEY **)&ref->u.key)); + } + + /* + * Discard backing overflow pages for any items being discarded that + * reference overflow pages. + */ + WT_ERR(__slvg_row_merge_ovfl(session, trk, page, 0, skip_start)); + WT_ERR(__slvg_row_merge_ovfl( + session, trk, page, page->entries - skip_stop, page->entries)); + + /* + * If we take all of the keys, we don't write the page and we clear the + * merge flags so that the underlying blocks are not later freed (for + * merge pages re-written into the file, the underlying blocks have to + * be freed, but if this page never gets written, we shouldn't free the + * blocks). + */ + if (skip_start == 0 && skip_stop == 0) + F_CLR(trk, WT_TRACK_MERGE); + else { + /* + * Change the page to reflect the correct record count: there + * is no need to copy anything on the page itself, the entries + * value limits the number of page items. + */ + page->entries -= skip_stop; + cookie->skip = skip_start; + + /* + * We can't discard the original blocks associated with the page + * now. (The problem is we don't want to overwrite any original + * information until the salvage run succeeds -- if we free the + * blocks now, the next merge page we write might allocate those + * blocks and overwrite them, and should the salvage run fail, + * the original information would have been lost to subsequent + * salvage runs.) Clear the reference addr so eviction doesn't + * free the underlying blocks. + */ + __wt_free(session, ((WT_ADDR *)ref->addr)->addr); + __wt_free(session, ref->addr); + ref->addr = NULL; + + /* Write the new version of the leaf page to disk. */ + WT_ERR(__wt_page_modify_init(session, page)); + __wt_page_modify_set(page); + WT_ERR(__wt_rec_write(session, page, cookie)); + + /* Reset the page. */ + page->entries += skip_stop; + } + + /* + * Discard our hazard reference and evict the page, updating the + * parent's reference. + */ + __wt_page_release(session, page); + ret = __wt_rec_evict(session, page, WT_REC_SINGLE); + + if (0) { +err: __wt_page_release(session, page); + } + __wt_scr_free(&key); + + return (ret); +} + +/* + * __slvg_row_merge_ovfl -- + * Free file blocks referenced from keys discarded from merged pages. + */ +static int +__slvg_row_merge_ovfl(WT_SESSION_IMPL *session, + WT_TRACK *trk, WT_PAGE *page, uint32_t start, uint32_t stop) +{ + WT_CELL *cell; + WT_CELL_UNPACK *unpack, _unpack; + WT_ROW *rip; + + unpack = &_unpack; + + for (rip = page->u.row.d + start; start < stop; ++start) { + if (__wt_off_page(page, rip->key)) + cell = WT_PAGE_REF_OFFSET( + page, ((WT_IKEY *)rip->key)->cell_offset); + else + cell = rip->key; + __wt_cell_unpack(cell, unpack); + if (unpack->type == WT_CELL_KEY_OVFL) { + WT_VERBOSE(session, salvage, + "%s merge discard freed overflow reference %s", + __wt_addr_string(session, + trk->ss->tmp1, trk->addr.addr, trk->addr.size), + __wt_addr_string(session, + trk->ss->tmp2, unpack->data, unpack->size)); + + WT_RET(__wt_bm_free( + session, unpack->data, unpack->size)); + } + + if ((cell = __wt_row_value(page, rip)) == NULL) + continue; + __wt_cell_unpack(cell, unpack); + if (unpack->type == WT_CELL_VALUE_OVFL) { + WT_VERBOSE(session, salvage, + "%s merge discard freed overflow reference %s", + __wt_addr_string(session, + trk->ss->tmp1, trk->addr.addr, trk->addr.size), + __wt_addr_string(session, + trk->ss->tmp2, unpack->data, unpack->size)); + + WT_RET(__wt_bm_free( + session, unpack->data, unpack->size)); + } + } + return (0); +} + +/* + * __slvg_trk_compare_addr -- + * Compare two WT_TRACK array entries by address cookie. + */ +static int +__slvg_trk_compare_addr(const void *a, const void *b) +{ + WT_TRACK *a_trk, *b_trk; + uint32_t len; + int ret; + + a_trk = *(WT_TRACK **)a; + b_trk = *(WT_TRACK **)b; + + /* + * We don't care about the order because these are opaque cookies -- + * we're just sorting them so we can binary search instead of linear + * search. + */ + len = WT_MIN(a_trk->addr.size, b_trk->addr.size); + ret = memcmp(a_trk->addr.addr, b_trk->addr.addr, len); + if (ret == 0) + ret = a_trk->addr.size > b_trk->addr.size ? -1 : 1; + return (ret); +} + +/* + * __slvg_ovfl_compare -- + * Bsearch comparison routine for the overflow array. + */ +static int +__slvg_ovfl_compare(const void *a, const void *b) +{ + WT_ADDR *addr; + WT_TRACK *trk; + uint32_t len; + int ret; + + addr = (WT_ADDR *)a; + trk = *(WT_TRACK **)b; + + len = WT_MIN(trk->addr.size, addr->size); + ret = memcmp(addr->addr, trk->addr.addr, len); + if (ret == 0 && addr->size != trk->addr.size) + ret = addr->size < trk->addr.size ? -1 : 1; + return (ret); +} + +/* + * __slvg_ovfl_reconcile -- + * Review relationships between leaf pages and the overflow pages, delete + * leaf pages until there's a one-to-one relationship between leaf and overflow + * pages. + */ +static int +__slvg_ovfl_reconcile(WT_SESSION_IMPL *session, WT_STUFF *ss) +{ + WT_ADDR *addr; + WT_TRACK **searchp, *trk; + uint32_t i, j; + + /* + * Discard any page referencing a non-existent overflow page. We do + * this before checking overlapping key ranges on the grounds that a + * bad key range we can use is better than a terrific key range that + * references pages we don't have. + * + * An alternative would be to discard only the on-page item referencing + * the missing overflow item. We're not doing that because: (1) absent + * corruption, a missing overflow item is a strong argument the page was + * replaced (but admittedly, corruption is probably why we're here); (2) + * it's a lot of work, and as WiredTiger supports very large page sizes, + * overflow items simply shouldn't be common. + * + * If an overflow page is referenced more than once, discard leaf pages + * with the lowest LSNs until overflow pages are only referenced once. + * + * This requires sorting the page list by LSN, and the overflow array + * by address cookie. + */ + qsort(ss->pages, + (size_t)ss->pages_next, sizeof(WT_TRACK *), __slvg_trk_compare_gen); + qsort(ss->ovfl, + (size_t)ss->ovfl_next, sizeof(WT_TRACK *), __slvg_trk_compare_addr); + + /* + * Walk the list of pages and discard any pages referencing non-existent + * overflow pages or referencing overflow pages also referenced by pages + * with higher LSNs. Our caller sorted the page list by LSN, high to + * low, so we don't have to do explicit testing of the page LSNs, the + * first page to reference an overflow page is the best page to own it. + */ + for (i = 0; i < ss->pages_next; ++i) { + if ((trk = ss->pages[i]) == NULL) + continue; + for (j = 0; j < trk->ovfl_cnt; ++j) { + addr = &trk->ovfl[j]; + searchp = bsearch(addr, ss->ovfl, ss->ovfl_next, + sizeof(WT_TRACK *), __slvg_ovfl_compare); + + /* + * If the overflow page doesn't exist or its size does + * not match, or if another page has already claimed the + * overflow page, discard the leaf page. + */ + if (searchp != NULL && + !F_ISSET(*searchp, WT_TRACK_OVFL_REFD)) { + F_SET(*searchp, WT_TRACK_OVFL_REFD); + continue; + } + + /* + * This leaf page isn't usable. Discard the leaf page + * and clear the "referenced" flag for overflow pages + * already claimed by this page. I hate to repeat the + * searches, but the alternative is a pointer for each + * overflow page referenced by the leaf page and this + * is the only thing we'd use it for. + */ + while (j > 0) { + addr = &trk->ovfl[--j]; + searchp = + bsearch(addr, ss->ovfl, ss->ovfl_next, + sizeof(WT_TRACK *), __slvg_ovfl_compare); + F_CLR(*searchp, WT_TRACK_OVFL_REFD); + } + WT_VERBOSE(session, salvage, + "%s references unavailable overflow page %s", + __wt_addr_string(session, + ss->tmp1, trk->addr.addr, trk->addr.size), + __wt_addr_string(session, + ss->tmp2, addr->addr, addr->size)); + WT_RET(__slvg_trk_free( + session, &ss->pages[i], WT_TRK_FREE_BLOCKS)); + break; + } + } + return (0); +} + +/* + * __slvg_trk_compare_key -- + * Compare two WT_TRACK array entries by key, and secondarily, by LSN. + */ +static int +__slvg_trk_compare_key(const void *a, const void *b) +{ + WT_BTREE *btree; + WT_TRACK *a_trk, *b_trk; + uint64_t a_gen, a_recno, b_gen, b_recno; + int cmp; + + a_trk = *(WT_TRACK **)a; + b_trk = *(WT_TRACK **)b; + + if (a_trk == NULL) + return (b_trk == NULL ? 0 : 1); + if (b_trk == NULL) + return (-1); + + switch (a_trk->ss->page_type) { + case WT_PAGE_COL_FIX: + case WT_PAGE_COL_VAR: + a_recno = a_trk->col_start; + b_recno = b_trk->col_start; + if (a_recno == b_recno) + break; + if (a_recno > b_recno) + return (1); + if (a_recno < b_recno) + return (-1); + break; + case WT_PAGE_ROW_LEAF: + btree = a_trk->ss->btree; + (void)WT_BTREE_CMP(a_trk->ss->session, btree, + &a_trk->row_start, &b_trk->row_start, cmp); + if (cmp != 0) + return (cmp); + break; + } + + /* + * If the primary keys compare equally, differentiate based on LSN. + * Sort from highest LSN to lowest, that is, the earlier pages in + * the array are more desirable. + */ + a_gen = a_trk->gen; + b_gen = b_trk->gen; + return (a_gen > b_gen ? -1 : (a_gen < b_gen ? 1 : 0)); +} + +/* + * __slvg_trk_compare_gen -- + * Compare two WT_TRACK array entries by LSN. + */ +static int +__slvg_trk_compare_gen(const void *a, const void *b) +{ + WT_TRACK *a_trk, *b_trk; + uint64_t a_gen, b_gen; + + a_trk = *(WT_TRACK **)a; + b_trk = *(WT_TRACK **)b; + + /* + * Sort from highest LSN to lowest, that is, the earlier pages in the + * array are more desirable. + */ + a_gen = a_trk->gen; + b_gen = b_trk->gen; + return (a_gen > b_gen ? -1 : (a_gen < b_gen ? 1 : 0)); +} + +/* + * __slvg_merge_block_free -- + * Free file blocks for pages that had to be merged. + */ +static int +__slvg_merge_block_free(WT_SESSION_IMPL *session, WT_STUFF *ss) +{ + WT_TRACK *trk; + uint32_t i; + + /* + * Free any underlying file blocks for merged pages. We do not free + * referenced overflow pages: that had to be done when creating the + * merged pages because we chose the overflow pages to free based on + * the keys we retained or discarded. + */ + for (i = 0; i < ss->pages_next; ++i) { + if ((trk = ss->pages[i]) == NULL) + continue; + if (F_ISSET(trk, WT_TRACK_MERGE) && + !F_ISSET(trk, WT_TRACK_NO_FILE_BLOCKS)) + WT_RET(__slvg_trk_free( + session, &ss->pages[i], WT_TRK_FREE_BLOCKS)); + } + + return (0); +} + +/* + * __slvg_ovfl_discard -- + * Discard unused overflow pages. + */ +static int +__slvg_ovfl_discard(WT_SESSION_IMPL *session, WT_STUFF *ss) +{ + uint32_t i; + + /* + * Walk the overflow page array: if an overflow page isn't referenced, + * add its file blocks to the free list. + */ + for (i = 0; i < ss->ovfl_next; ++i) { + if (F_ISSET(ss->ovfl[i], WT_TRACK_OVFL_REFD)) + continue; + WT_VERBOSE(session, salvage, + "%s unused overflow page", + __wt_addr_string(session, + ss->tmp1, ss->ovfl[i]->addr.addr, ss->ovfl[i]->addr.size)); + WT_RET(__slvg_trk_free( + session, &ss->ovfl[i], WT_TRK_FREE_BLOCKS)); + } + + return (0); +} + +/* + * __slvg_free -- + * Discard memory allocated to the page and overflow arrays. + */ +static int +__slvg_cleanup(WT_SESSION_IMPL *session, WT_STUFF *ss) +{ + uint32_t i; + + /* Discard the leaf page array. */ + for (i = 0; i < ss->pages_next; ++i) + if (ss->pages[i] != NULL) + WT_RET(__slvg_trk_free(session, &ss->pages[i], 0)); + __wt_free(session, ss->pages); + + /* Discard the ovfl page array. */ + for (i = 0; i < ss->ovfl_next; ++i) + if (ss->ovfl[i] != NULL) + WT_RET(__slvg_trk_free(session, &ss->ovfl[i], 0)); + __wt_free(session, ss->ovfl); + + return (0); +} + +/* + * __slvg_trk_free -- + * Discard a WT_TRACK structure and (optionally) its underlying blocks. + */ +static int +__slvg_trk_free(WT_SESSION_IMPL *session, WT_TRACK **trkp, uint32_t flags) +{ + WT_ADDR *addr; + WT_TRACK *trk; + uint32_t i; + + trk = *trkp; + *trkp = NULL; + + /* + * If freeing underlying file blocks or overflow pages, this is a page + * we were tracking but eventually decided not to use. That merits a + * verbose description. + */ + if (LF_ISSET(WT_TRK_FREE_BLOCKS)) { + WT_VERBOSE(session, salvage, + "%s page discarded: discard freed file bytes %" PRIu32, + __wt_addr_string( + session, trk->ss->tmp1, trk->addr.addr, trk->addr.size), + trk->size); + WT_RET(__wt_bm_free(session, trk->addr.addr, trk->addr.size)); + } + __wt_free(session, trk->addr.addr); + + for (i = 0; i < trk->ovfl_cnt; ++i) { + addr = &trk->ovfl[i]; + if (LF_ISSET(WT_TRK_FREE_OVFL)) { + WT_VERBOSE(session, salvage, + "%s page discarded: discard freed overflow page %s", + __wt_addr_string(session, + trk->ss->tmp1, trk->addr.addr, trk->addr.size), + __wt_addr_string(session, + trk->ss->tmp2, addr->addr, addr->size)); + + WT_RET(__wt_bm_free(session, addr->addr, addr->size)); + } + __wt_free(session, addr->addr); + } + __wt_free(session, trk->ovfl); + + if (trk->ss->page_type == WT_PAGE_ROW_LEAF) { + __wt_buf_free(session, &trk->row_start); + __wt_buf_free(session, &trk->row_stop); + } + __wt_free(session, trk); + + return (0); +} diff --git a/src/btree/bt_stat.c b/src/btree/bt_stat.c new file mode 100644 index 00000000000..eba5d076c59 --- /dev/null +++ b/src/btree/bt_stat.c @@ -0,0 +1,177 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static int __stat_page(WT_SESSION_IMPL *, WT_PAGE *); +static int __stat_page_col_var(WT_SESSION_IMPL *, WT_PAGE *); +static int __stat_page_row_leaf(WT_SESSION_IMPL *, WT_PAGE *); + +/* + * __wt_btree_stat_init -- + * Initialize the Btree statistics. + */ +int +__wt_btree_stat_init(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + WT_PAGE *page; + int ret; + + btree = session->btree; + + WT_RET(__wt_bm_stat(session)); + + WT_BSTAT_SET(session, file_allocsize, btree->allocsize); + WT_BSTAT_SET(session, file_fixed_len, btree->bitcnt); + WT_BSTAT_SET(session, file_maxintlpage, btree->maxintlpage); + WT_BSTAT_SET(session, file_maxintlitem, btree->maxintlitem); + WT_BSTAT_SET(session, file_maxleafpage, btree->maxleafpage); + WT_BSTAT_SET(session, file_maxleafitem, btree->maxleafitem); + + page = NULL; + while ((ret = __wt_tree_np(session, &page, 0, 1)) == 0 && page != NULL) + WT_RET(__stat_page(session, page)); + return (ret == WT_NOTFOUND ? 0 : ret); +} + +/* + * __stat_page -- + * Stat any Btree page. + */ +static int +__stat_page(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + /* + * All internal pages and overflow pages are trivial, all we track is + * a count of the page type. + */ + switch (page->type) { + case WT_PAGE_COL_FIX: + WT_BSTAT_INCR(session, file_col_fix_pages); + WT_BSTAT_INCRV(session, file_entries, page->entries); + break; + case WT_PAGE_COL_INT: + WT_BSTAT_INCR(session, file_col_int_pages); + WT_BSTAT_INCRV(session, file_entries, page->entries); + break; + case WT_PAGE_COL_VAR: + WT_RET(__stat_page_col_var(session, page)); + break; + case WT_PAGE_OVFL: + WT_BSTAT_INCR(session, file_overflow); + break; + case WT_PAGE_ROW_INT: + WT_BSTAT_INCR(session, file_row_int_pages); + WT_BSTAT_INCRV(session, file_entries, page->entries); + break; + case WT_PAGE_ROW_LEAF: + WT_RET(__stat_page_row_leaf(session, page)); + break; + WT_ILLEGAL_VALUE(session); + } + return (0); +} + +/* + * __stat_page_col_var -- + * Stat a WT_PAGE_COL_VAR page. + */ +static int +__stat_page_col_var(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_CELL *cell; + WT_CELL_UNPACK *unpack, _unpack; + WT_COL *cip; + WT_INSERT *ins; + WT_UPDATE *upd; + uint32_t i; + int orig_deleted; + + unpack = &_unpack; + + WT_BSTAT_INCR(session, file_col_var_pages); + + /* + * Walk the page, counting regular and overflow data items, and checking + * to be sure any updates weren't deletions. If the item was updated, + * assume it was updated by an item of the same size (it's expensive to + * figure out if it will require the same space or not, especially if + * there's Huffman encoding). + */ + WT_COL_FOREACH(page, cip, i) { + if ((cell = WT_COL_PTR(page, cip)) == NULL) { + orig_deleted = 1; + WT_BSTAT_INCR(session, file_col_deleted); + } else { + orig_deleted = 0; + __wt_cell_unpack(cell, unpack); + WT_BSTAT_INCRV( + session, file_entries, __wt_cell_rle(unpack)); + } + + /* + * Walk the insert list, checking for changes. For each insert + * we find, correct the original count based on its state. + */ + WT_SKIP_FOREACH(ins, WT_COL_UPDATE(page, cip)) { + upd = ins->upd; + if (WT_UPDATE_DELETED_ISSET(upd)) { + if (orig_deleted) + continue; + WT_BSTAT_INCR(session, file_col_deleted); + WT_BSTAT_DECR(session, file_entries); + } else { + if (!orig_deleted) + continue; + WT_BSTAT_DECR(session, file_col_deleted); + WT_BSTAT_INCR(session, file_entries); + } + } + } + return (0); +} + +/* + * __stat_page_row_leaf -- + * Stat a WT_PAGE_ROW_LEAF page. + */ +static int +__stat_page_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_INSERT *ins; + WT_ROW *rip; + WT_UPDATE *upd; + uint32_t cnt, i; + + WT_BSTAT_INCR(session, file_row_leaf_pages); + + /* + * Stat any K/V pairs inserted into the page before the first from-disk + * key on the page. + */ + cnt = 0; + WT_SKIP_FOREACH(ins, WT_ROW_INSERT_SMALLEST(page)) + if (!WT_UPDATE_DELETED_ISSET(ins->upd)) + ++cnt; + + /* Stat the page's K/V pairs. */ + WT_ROW_FOREACH(page, rip, i) { + upd = WT_ROW_UPDATE(page, rip); + if (upd == NULL || !WT_UPDATE_DELETED_ISSET(upd)) + ++cnt; + + /* Stat inserted K/V pairs. */ + WT_SKIP_FOREACH(ins, WT_ROW_INSERT(page, rip)) + if (!WT_UPDATE_DELETED_ISSET(ins->upd)) + ++cnt; + } + + WT_BSTAT_INCRV(session, file_entries, cnt); + + return (0); +} diff --git a/src/btree/bt_sync.c b/src/btree/bt_sync.c new file mode 100644 index 00000000000..c4ee826bd79 --- /dev/null +++ b/src/btree/bt_sync.c @@ -0,0 +1,41 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_btree_sync -- + * Sync the tree. + */ +int +__wt_btree_sync(WT_SESSION_IMPL *session, const char *cfg[]) +{ + WT_UNUSED(cfg); + + /* + * Ask the eviction thread to flush any dirty pages. + * + * Reconciliation is just another reader of the page, so it's probably + * possible to do this work in the current thread, rather than poking + * the eviction thread. The trick is for the sync thread to acquire + * hazard references on each dirty page, which would block the eviction + * thread from attempting to evict the pages. + * + * I'm not doing it for a few reasons: (1) I don't want sync to update + * the page's read-generation number, and eviction already knows how to + * do that, (2) I don't want more than a single sync running at a time, + * and calling eviction makes it work that way without additional work, + * (3) sync and eviction cannot operate on the same page at the same + * time and I'd rather they didn't operate in the same file at the same + * time, and using eviction makes it work that way without additional + * synchronization, (4) eventually we'll have async write I/O, and this + * approach results in one less page writer in the system, (5) the code + * already works that way. None of these problems can't be fixed, but + * I don't see a reason to change at this time, either. + */ + return (__wt_evict_file_serial(session, 0)); +} diff --git a/src/btree/bt_upgrade.c b/src/btree/bt_upgrade.c new file mode 100644 index 00000000000..574a2335796 --- /dev/null +++ b/src/btree/bt_upgrade.c @@ -0,0 +1,22 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_upgrade -- + * Upgrade a file. + */ +int +__wt_upgrade(WT_SESSION_IMPL *session, const char *cfg[]) +{ + WT_UNUSED(session); + WT_UNUSED(cfg); + + /* There's nothing to upgrade, yet. */ + return (0); +} diff --git a/src/btree/bt_vrfy.c b/src/btree/bt_vrfy.c new file mode 100644 index 00000000000..0d78f9d0bbf --- /dev/null +++ b/src/btree/bt_vrfy.c @@ -0,0 +1,499 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * There's a bunch of stuff we pass around during verification, group it + * together to make the code prettier. + */ +typedef struct { + uint64_t record_total; /* Total record count */ + + WT_ITEM *max_key; /* Largest key */ + WT_ITEM *max_addr; /* Largest key page */ + + uint64_t fcnt; /* Progress counter */ + + int dumpfile; /* Dump file stream */ + + WT_ITEM *tmp1; /* Temporary buffer */ + WT_ITEM *tmp2; /* Temporary buffer */ +} WT_VSTUFF; + +static int __verify_int(WT_SESSION_IMPL *, int); +static int __verify_overflow( + WT_SESSION_IMPL *, const uint8_t *, uint32_t, WT_VSTUFF *); +static int __verify_overflow_cell(WT_SESSION_IMPL *, WT_PAGE *, WT_VSTUFF *); +static int __verify_row_int_key_order( + WT_SESSION_IMPL *, WT_PAGE *, WT_REF *, uint32_t, WT_VSTUFF *); +static int __verify_row_leaf_key_order( + WT_SESSION_IMPL *, WT_PAGE *, WT_VSTUFF *); +static int __verify_tree(WT_SESSION_IMPL *, WT_PAGE *, uint64_t, WT_VSTUFF *); + +/* + * __wt_verify -- + * Verify a file. + */ +int +__wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) +{ + WT_UNUSED(cfg); + + return (__verify_int(session, 0)); +} + +/* + * __wt_dumpfile -- + * Dump a file in debugging mode. + */ +int +__wt_dumpfile(WT_SESSION_IMPL *session, const char *cfg[]) +{ + WT_UNUSED(cfg); + +#ifdef HAVE_DIAGNOSTIC + /* + * We use the verification code to do debugging dumps because if we're + * dumping in debugging mode, we want to confirm the page is OK before + * walking it. + */ + return (__verify_int(session, 1)); +#else + WT_RET_MSG(session, ENOTSUP, + "the WiredTiger library was not built in diagnostic mode"); +#endif +} + +/* + * __verify_int -- + * Internal version of verify: verify a Btree, optionally dumping each + * page in debugging mode. + */ +static int +__verify_int(WT_SESSION_IMPL *session, int dumpfile) +{ + WT_BTREE *btree; + WT_VSTUFF *vs, _vstuff; + int empty, ret; + + btree = session->btree; + ret = 0; + + WT_RET(__wt_bm_verify_start(session, &empty)); + if (empty) + return (0); + + WT_CLEAR(_vstuff); + vs = &_vstuff; + + vs->dumpfile = dumpfile; + WT_ERR(__wt_scr_alloc(session, 0, &vs->max_key)); + WT_ERR(__wt_scr_alloc(session, 0, &vs->max_addr)); + WT_ERR(__wt_scr_alloc(session, 0, &vs->tmp1)); + WT_ERR(__wt_scr_alloc(session, 0, &vs->tmp2)); + + /* + * Verify the tree, starting at the root: if there is no root, that's + * still possibly a legal file, but all of the pages must be on the + * free-list. + */ + WT_ERR(__wt_btree_get_root(session, vs->tmp1)); + if (vs->tmp1->data != NULL) { + WT_ERR(__wt_bm_verify_addr( + session, vs->tmp1->data, vs->tmp1->size)); + WT_ERR(__wt_btree_root_init(session, vs->tmp1)); + WT_ERR( + __verify_tree(session, btree->root_page, (uint64_t)1, vs)); + } + + if (0) { +err: if (ret == 0) + ret = WT_ERROR; + } + + if (vs != NULL) { + /* Wrap up reporting. */ + __wt_progress(session, NULL, vs->fcnt); + + /* Free allocated memory. */ + __wt_scr_free(&vs->max_key); + __wt_scr_free(&vs->max_addr); + __wt_scr_free(&vs->tmp1); + __wt_scr_free(&vs->tmp2); + } + + WT_TRET(__wt_bm_verify_end(session)); + + return (ret); +} + +/* + * __verify_tree -- + * Verify a tree, recursively descending through it in depth-first fashion. + * The page argument was physically verified (so we know it's correctly formed), + * and the in-memory version built. Our job is to check logical relationships + * in the page and in the tree. + */ +static int +__verify_tree(WT_SESSION_IMPL *session, + WT_PAGE *page, uint64_t parent_recno, WT_VSTUFF *vs) +{ + WT_CELL *cell; + WT_CELL_UNPACK *unpack, _unpack; + WT_COL *cip; + WT_ITEM *tmp; + WT_REF *ref; + uint64_t recno; + const uint8_t *addr; + uint32_t entry, i, size; + int ret; + + ret = 0; + unpack = &_unpack; + + WT_VERBOSE(session, verify, "%p: %s %s", + page, __wt_page_addr_string(session, vs->tmp1, page), + __wt_page_type_string(page->type)); + + /* + * The page's physical structure was verified when it was read into + * memory by the read server thread, and then the in-memory version + * of the page was built. Now we make sure the page and tree are + * logically consistent. + * + * !!! + * The problem: (1) the read server has to build the in-memory version + * of the page because the read server is the thread that flags when + * any thread can access the page in the tree; (2) we can't build the + * in-memory version of the page until the physical structure is known + * to be OK, so the read server has to verify at least the physical + * structure of the page; (3) doing complete page verification requires + * reading additional pages (for example, overflow keys imply reading + * overflow pages in order to test the key's order in the page); (4) + * the read server cannot read additional pages because it will hang + * waiting on itself. For this reason, we split page verification + * into a physical verification, which allows the in-memory version + * of the page to be built, and then a subsequent logical verification + * which happens here. + * + * Report progress every 10 pages. + */ + if (++vs->fcnt % 10 == 0) + __wt_progress(session, NULL, vs->fcnt); + +#ifdef HAVE_DIAGNOSTIC + /* Optionally dump the page in debugging mode. */ + if (vs->dumpfile) { + WT_RET(__wt_debug_page(session, page, NULL)); + if (page->dsk != NULL) + WT_RET(__wt_debug_disk(session, page->dsk, NULL)); + } +#endif + + /* + * Column-store key order checks: check the starting record number, + * then update the total record count. + */ + switch (page->type) { + case WT_PAGE_COL_FIX: + recno = page->u.col_fix.recno; + goto recno_chk; + case WT_PAGE_COL_INT: + recno = page->u.intl.recno; + goto recno_chk; + case WT_PAGE_COL_VAR: + recno = page->u.col_var.recno; +recno_chk: if (parent_recno != recno) + WT_RET_MSG(session, WT_ERROR, + "page at %s has a starting record of %" PRIu64 + " when the expected starting record is %" PRIu64, + __wt_page_addr_string(session, vs->tmp1, page), + recno, parent_recno); + break; + } + switch (page->type) { + case WT_PAGE_COL_FIX: + vs->record_total += page->entries; + break; + case WT_PAGE_COL_VAR: + recno = 0; + WT_COL_FOREACH(page, cip, i) + if ((cell = WT_COL_PTR(page, cip)) == NULL) + ++recno; + else { + __wt_cell_unpack(cell, unpack); + recno += __wt_cell_rle(unpack); + } + vs->record_total += recno; + break; + } + + /* + * Row-store leaf page key order check: it's a depth-first traversal, + * the first key on this page should be larger than any key previously + * seen. + */ + switch (page->type) { + case WT_PAGE_ROW_LEAF: + WT_RET(__verify_row_leaf_key_order(session, page, vs)); + break; + } + + /* + * Check overflow pages. We check overflow cells separately from other + * tests that walk the page as it's simpler, and I don't care much how + * fast table verify runs. + */ + switch (page->type) { + case WT_PAGE_COL_VAR: + case WT_PAGE_ROW_INT: + case WT_PAGE_ROW_LEAF: + WT_RET(__verify_overflow_cell(session, page, vs)); + break; + } + + /* Check tree connections and recursively descend the tree. */ + switch (page->type) { + case WT_PAGE_COL_INT: + /* For each entry in an internal page, verify the subtree. */ + WT_REF_FOREACH(page, ref, i) { + /* + * It's a depth-first traversal: this entry's starting + * record number should be 1 more than the total records + * reviewed to this point. + */ + if (ref->u.recno != vs->record_total + 1) { + WT_RET(__wt_scr_alloc(session, 0, &tmp)); + __wt_cell_unpack(ref->addr, unpack); + ret = __wt_bm_addr_string( + session, tmp, unpack->data, unpack->size); + __wt_errx(session, "page at %s has a starting " + "record of %" PRIu64 " when the expected " + "starting record was %" PRIu64, + ret == 0 ? + (char *)tmp->data : "[Unknown address]", + ref->u.recno, vs->record_total + 1); + __wt_scr_free(&tmp); + return (WT_ERROR); + } + + /* ref references the subtree containing the record */ + __wt_get_addr(page, ref, &addr, &size); + WT_RET(__wt_bm_verify_addr(session, addr, size)); + WT_RET(__wt_page_in(session, page, ref)); + ret = + __verify_tree(session, ref->page, ref->u.recno, vs); + __wt_page_release(session, ref->page); + WT_RET(ret); + } + break; + case WT_PAGE_ROW_INT: + /* For each entry in an internal page, verify the subtree. */ + entry = 0; + WT_REF_FOREACH(page, ref, i) { + /* + * It's a depth-first traversal: this entry's starting + * key should be larger than the largest key previously + * reviewed. + * + * The 0th key of any internal page is magic, and we + * can't test against it. + */ + if (entry != 0) + WT_RET(__verify_row_int_key_order( + session, page, ref, entry, vs)); + ++entry; + + /* ref references the subtree containing the record */ + __wt_get_addr(page, ref, &addr, &size); + WT_RET(__wt_bm_verify_addr(session, addr, size)); + WT_RET(__wt_page_in(session, page, ref)); + ret = + __verify_tree(session, ref->page, (uint64_t)0, vs); + __wt_page_release(session, ref->page); + WT_RET(ret); + } + break; + } + return (0); +} + +/* + * __verify_row_int_key_order -- + * Compare a key on an internal page to the largest key we've seen so + * far; update the largest key we've seen so far to that key. + */ +static int +__verify_row_int_key_order(WT_SESSION_IMPL *session, + WT_PAGE *page, WT_REF *ref, uint32_t entry, WT_VSTUFF *vs) +{ + WT_BTREE *btree; + WT_IKEY *ikey; + WT_ITEM item; + int cmp; + + btree = session->btree; + + /* The maximum key is set, we updated it from a leaf page first. */ + WT_ASSERT(session, vs->max_addr->size != 0); + + /* Set up the key structure. */ + ikey = ref->u.key; + item.data = WT_IKEY_DATA(ikey); + item.size = ikey->size; + + /* Compare the key against the largest key we've seen so far. */ + WT_RET(WT_BTREE_CMP(session, btree, &item, vs->max_key, cmp)); + if (cmp <= 0) + WT_RET_MSG(session, WT_ERROR, + "the internal key in entry %" PRIu32 " on the page at %s " + "sorts before the last key appearing on page %s, earlier " + "in the tree", + entry, + __wt_page_addr_string(session, vs->tmp1, page), + (char *)vs->max_addr->data); + + /* Update the largest key we've seen to the key just checked. */ + WT_RET(__wt_buf_set(session, vs->max_key, item.data, item.size)); + (void)__wt_page_addr_string(session, vs->max_addr, page); + + return (0); +} + +/* + * __verify_row_leaf_key_order -- + * Compare the first key on a leaf page to the largest key we've seen so + * far; update the largest key we've seen so far to the last key on the page. + */ +static int +__verify_row_leaf_key_order( + WT_SESSION_IMPL *session, WT_PAGE *page, WT_VSTUFF *vs) +{ + WT_BTREE *btree; + int cmp; + + btree = session->btree; + + /* + * If a tree is empty (just created), it won't have keys; if there + * are no keys, we're done. + */ + if (page->entries == 0) + return (0); + + /* + * We visit our first leaf page before setting the maximum key (the 0th + * keys on the internal pages leading to the smallest leaf in the tree + * are all empty entries). + */ + if (vs->max_addr->size != 0) { + WT_RET(__wt_row_key(session, page, page->u.row.d, vs->tmp1)); + + /* + * Compare the key against the largest key we've seen so far. + * + * If we're comparing against a key taken from an internal page, + * we can compare equal (which is an expected path, the internal + * page key is often a copy of the leaf page's first key). But, + * in the case of the 0th slot on an internal page, the last key + * we've seen was a key from a previous leaf page, and it's not + * OK to compare equally in that case. + */ + WT_RET(WT_BTREE_CMP(session, + btree, vs->tmp1, (WT_ITEM *)vs->max_key, cmp)); + if (cmp < 0) + WT_RET_MSG(session, WT_ERROR, + "the first key on the page at %s sorts equal to or " + "less than a key appearing on the page at %s, " + "earlier in the tree", + __wt_page_addr_string(session, vs->tmp1, page), + (char *)vs->max_addr->data); + } + + /* Update the largest key we've seen to the last key on this page. */ + WT_RET(__wt_row_key(session, + page, page->u.row.d + (page->entries - 1), vs->max_key)); + (void)__wt_page_addr_string(session, vs->max_addr, page); + + return (0); +} + +/* + * __verify_overflow_cell -- + * Verify any overflow cells on the page. + */ +static int +__verify_overflow_cell(WT_SESSION_IMPL *session, WT_PAGE *page, WT_VSTUFF *vs) +{ + WT_BTREE *btree; + WT_CELL *cell; + WT_CELL_UNPACK *unpack, _unpack; + WT_PAGE_HEADER *dsk; + uint32_t cell_num, i; + int ret; + + btree = session->btree; + unpack = &_unpack; + ret = 0; + + /* + * If a tree is empty (just created), it won't have a disk image; + * if there is no disk image, we're done. + */ + if ((dsk = page->dsk) == NULL) + return (0); + + /* Walk the disk page, verifying pages referenced by overflow cells. */ + cell_num = 0; + WT_CELL_FOREACH(btree, dsk, cell, unpack, i) { + ++cell_num; + __wt_cell_unpack(cell, unpack); + switch (unpack->type) { + case WT_CELL_KEY_OVFL: + case WT_CELL_VALUE_OVFL: + WT_ERR(__verify_overflow( + session, unpack->data, unpack->size, vs)); + break; + } + } + return (0); + +err: WT_RET_MSG(session, ret, + "cell %" PRIu32 " on page at %s references an overflow item at %s " + "that failed verification", + cell_num - 1, + __wt_page_addr_string(session, vs->tmp1, page), + __wt_addr_string(session, vs->tmp2, unpack->data, unpack->size)); +} + +/* + * __verify_overflow -- + * Read in an overflow page and check it. + */ +static int +__verify_overflow(WT_SESSION_IMPL *session, + const uint8_t *addrbuf, uint32_t addrbuf_len, WT_VSTUFF *vs) +{ + WT_PAGE_HEADER *dsk; + + /* Read and verify the overflow item. */ + WT_RET(__wt_bm_verify_addr(session, addrbuf, addrbuf_len)); + WT_RET(__wt_bm_read(session, vs->tmp1, addrbuf, addrbuf_len)); + + /* + * The page has already been verified, but we haven't confirmed that + * it was an overflow page, only that it was a valid page. Confirm + * it's the type of page we expected. + */ + dsk = vs->tmp1->mem; + if (dsk->type != WT_PAGE_OVFL) + WT_RET_MSG(session, WT_ERROR, + "overflow referenced page at %s is not an overflow page", + __wt_addr_string(session, vs->tmp1, addrbuf, addrbuf_len)); + return (0); +} diff --git a/src/btree/bt_vrfy_dsk.c b/src/btree/bt_vrfy_dsk.c new file mode 100644 index 00000000000..610fce614a1 --- /dev/null +++ b/src/btree/bt_vrfy_dsk.c @@ -0,0 +1,604 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static int __err_cell_corrupted(WT_SESSION_IMPL *, uint32_t, const char *); +static int __err_cell_type( + WT_SESSION_IMPL *, uint32_t, const char *, uint8_t, uint8_t); +static int __err_eof(WT_SESSION_IMPL *, uint32_t, const char *); +static int __verify_dsk_chunk( + WT_SESSION_IMPL *, const char *, WT_PAGE_HEADER *, uint32_t); +static int __verify_dsk_col_fix( + WT_SESSION_IMPL *, const char *, WT_PAGE_HEADER *); +static int __verify_dsk_col_int( + WT_SESSION_IMPL *, const char *, WT_PAGE_HEADER *); +static int __verify_dsk_col_var( + WT_SESSION_IMPL *, const char *, WT_PAGE_HEADER *); +static int __verify_dsk_row( + WT_SESSION_IMPL *, const char *, WT_PAGE_HEADER *); + +#define WT_ERR_VRFY(session, ...) do { \ + if (!(F_ISSET(session, WT_SESSION_SALVAGE_QUIET_ERR))) \ + __wt_errx(session, __VA_ARGS__); \ + goto err; \ +} while (0) + +#define WT_RET_VRFY(session, ...) do { \ + if (!(F_ISSET(session, WT_SESSION_SALVAGE_QUIET_ERR))) \ + __wt_errx(session, __VA_ARGS__); \ + return (WT_ERROR); \ +} while (0) + +/* + * __wt_verify_dsk -- + * Verify a single Btree page as read from disk. + */ +int +__wt_verify_dsk(WT_SESSION_IMPL *session, + const char *addr, WT_PAGE_HEADER *dsk, uint32_t size) +{ + u_int i; + uint8_t *p; + + /* Check the page type. */ + switch (dsk->type) { + case WT_PAGE_COL_FIX: + case WT_PAGE_COL_INT: + case WT_PAGE_COL_VAR: + case WT_PAGE_FREELIST: + case WT_PAGE_OVFL: + case WT_PAGE_ROW_INT: + case WT_PAGE_ROW_LEAF: + break; + case WT_PAGE_INVALID: + default: + WT_RET_VRFY(session, + "page at %s has an invalid type of %" PRIu32, + addr, dsk->type); + } + + /* Check the page record number. */ + switch (dsk->type) { + case WT_PAGE_COL_FIX: + case WT_PAGE_COL_INT: + case WT_PAGE_COL_VAR: + if (dsk->recno != 0) + break; + WT_RET_VRFY(session, + "%s page at %s has a record number of zero", + __wt_page_type_string(dsk->type), addr); + case WT_PAGE_FREELIST: + case WT_PAGE_OVFL: + case WT_PAGE_ROW_INT: + case WT_PAGE_ROW_LEAF: + if (dsk->recno == 0) + break; + WT_RET_VRFY(session, + "%s page at %s has a non-zero record number", + __wt_page_type_string(dsk->type), addr); + } + + /* Check the in-memory size. */ + if (dsk->size != size) + WT_RET_VRFY(session, + "%s page at %s has an incorrect size (%" PRIu32 " != %" + PRIu32 ")", + __wt_page_type_string(dsk->type), addr, dsk->size, size); + + /* Unused bytes */ + for (p = dsk->unused, i = sizeof(dsk->unused); i > 0; --i) + if (*p != '\0') + WT_RET_VRFY(session, + "page at %s has non-zero unused page header bytes", + addr); + + /* Verify the items on the page. */ + switch (dsk->type) { + case WT_PAGE_COL_INT: + return (__verify_dsk_col_int(session, addr, dsk)); + case WT_PAGE_COL_FIX: + return (__verify_dsk_col_fix(session, addr, dsk)); + case WT_PAGE_COL_VAR: + return (__verify_dsk_col_var(session, addr, dsk)); + case WT_PAGE_ROW_INT: + case WT_PAGE_ROW_LEAF: + return (__verify_dsk_row(session, addr, dsk)); + case WT_PAGE_FREELIST: + case WT_PAGE_OVFL: + return (__verify_dsk_chunk(session, addr, dsk, dsk->u.datalen)); + WT_ILLEGAL_VALUE(session); + } + /* NOTREACHED */ +} + +/* + * __verify_dsk_row -- + * Walk a WT_PAGE_ROW_INT or WT_PAGE_ROW_LEAF disk page and verify it. + */ +static int +__verify_dsk_row( + WT_SESSION_IMPL *session, const char *addr, WT_PAGE_HEADER *dsk) +{ + WT_BTREE *btree; + WT_CELL *cell; + WT_CELL_UNPACK *unpack, _unpack; + WT_ITEM *current, *last, *last_pfx, *last_ovfl; + enum { FIRST, WAS_KEY, WAS_VALUE } last_cell_type; + void *huffman; + uint32_t cell_num, cell_type, i, prefix; + uint8_t *end; + int cmp, ret; + + btree = session->btree; + huffman = btree->huffman_key; + unpack = &_unpack; + ret = 0; + + current = last_pfx = last_ovfl = NULL; + WT_ERR(__wt_scr_alloc(session, 0, ¤t)); + WT_ERR(__wt_scr_alloc(session, 0, &last_pfx)); + WT_ERR(__wt_scr_alloc(session, 0, &last_ovfl)); + last = last_ovfl; + + end = (uint8_t *)dsk + dsk->size; + + last_cell_type = FIRST; + cell_num = 0; + WT_CELL_FOREACH(btree, dsk, cell, unpack, i) { + ++cell_num; + + /* Carefully unpack the cell. */ + if (__wt_cell_unpack_safe(cell, unpack, end) != 0) { + ret = __err_cell_corrupted(session, cell_num, addr); + goto err; + } + + /* + * Check the raw cell type, then collapse the short key/data + * types. + */ + WT_RET(__err_cell_type( + session, cell_num, addr, unpack->raw, dsk->type)); + cell_type = unpack->type; + + /* + * Check ordering relationships between the WT_CELL entries. + * For row-store internal pages, check for: + * two values in a row, + * two keys in a row, + * a value as the first cell on a page. + * For row-store leaf pages, check for: + * two values in a row, + * a value as the first cell on a page. + */ + switch (cell_type) { + case WT_CELL_KEY: + case WT_CELL_KEY_OVFL: + switch (last_cell_type) { + case FIRST: + case WAS_VALUE: + break; + case WAS_KEY: + if (dsk->type == WT_PAGE_ROW_LEAF) + break; + WT_ERR_VRFY(session, + "cell %" PRIu32 " on page at %s is the " + "first of two adjacent keys", + cell_num - 1, addr); + } + last_cell_type = WAS_KEY; + break; + case WT_CELL_ADDR: + case WT_CELL_VALUE: + case WT_CELL_VALUE_OVFL: + switch (last_cell_type) { + case FIRST: + WT_ERR_VRFY(session, + "page at %s begins with a value", addr); + case WAS_KEY: + break; + case WAS_VALUE: + WT_ERR_VRFY(session, + "cell %" PRIu32 " on page at %s is the " + "first of two adjacent values", + cell_num - 1, addr); + } + last_cell_type = WAS_VALUE; + break; + } + + /* Check if any referenced item is entirely in the file. */ + switch (cell_type) { + case WT_CELL_ADDR: + case WT_CELL_KEY_OVFL: + case WT_CELL_VALUE_OVFL: + if (!__wt_bm_addr_valid( + session, unpack->data, unpack->size)) + goto eof; + break; + } + + /* + * Remaining checks are for key order and prefix compression. + * If this cell isn't a key, we're done, move to the next cell. + * If this cell is an overflow item, instantiate the key and + * compare it with the last key. Otherwise, we have to deal + * with prefix compression. + */ + switch (cell_type) { + case WT_CELL_KEY: + break; + case WT_CELL_KEY_OVFL: + WT_ERR(__wt_cell_unpack_copy(session, unpack, current)); + goto key_compare; + default: + /* Not a key -- continue with the next cell. */ + continue; + } + + /* + * Prefix compression checks. + * + * Confirm the first non-overflow key on a page has a zero + * prefix compression count. + */ + prefix = unpack->prefix; + if (last_pfx->size == 0 && prefix != 0) + WT_ERR_VRFY(session, + "the %" PRIu32 " key on page at %s is the first " + "non-overflow key on the page and has a non-zero " + "prefix compression value", + cell_num, addr); + + /* Confirm the prefix compression count is possible. */ + if (cell_num > 1 && prefix > last->size) + WT_ERR_VRFY(session, + "key %" PRIu32 " on page at %s has a prefix " + "compression count of %" PRIu32 + ", larger than the length of the previous key, %" + PRIu32, + cell_num, addr, prefix, last->size); + + /* + * If Huffman decoding required, use the heavy-weight call to + * __wt_cell_unpack_copy() to build the key, up to the prefix. + * Else, we can do it faster internally because we don't have + * to shuffle memory around as much. + */ + if (huffman == NULL) { + /* + * Get the cell's data/length and make sure we have + * enough buffer space. + */ + WT_ERR(__wt_buf_grow( + session, current, prefix + unpack->size)); + + /* Copy the prefix then the data into place. */ + if (prefix != 0) + memcpy((void *) + current->data, last->data, prefix); + memcpy((uint8_t *) + current->data + prefix, unpack->data, unpack->size); + current->size = prefix + unpack->size; + } else { + WT_ERR(__wt_cell_unpack_copy(session, unpack, current)); + + /* + * If there's a prefix, make sure there's enough buffer + * space, then shift the decoded data past the prefix + * and copy the prefix into place. + */ + if (prefix != 0) { + WT_ERR(__wt_buf_grow( + session, current, prefix + current->size)); + memmove((uint8_t *)current->data + + prefix, current->data, current->size); + memcpy( + (void *)current->data, last->data, prefix); + current->size += prefix; + } + } + +key_compare: /* + * Compare the current key against the last key. + * + * Be careful about the 0th key on internal pages: we only store + * the first byte and custom collators may not be able to handle + * truncated keys. + */ + if ((dsk->type == WT_PAGE_ROW_INT && cell_num > 3) || + (dsk->type != WT_PAGE_ROW_INT && cell_num > 1)) { + WT_ERR( + WT_BTREE_CMP(session, btree, last, current, cmp)); + if (cmp >= 0) + WT_ERR_VRFY(session, + "the %" PRIu32 " and %" PRIu32 " keys on " + "page at %s are incorrectly sorted", + cell_num - 2, cell_num, addr); + } + + /* + * Swap the buffers: last always references the last key entry, + * last_pfx and last_ovfl reference the last prefix-compressed + * and last overflow key entries. Current gets pointed to the + * buffer we're not using this time around, which is where the + * next key goes. + */ + last = current; + if (cell_type == WT_CELL_KEY) { + current = last_pfx; + last_pfx = last; + } else { + current = last_ovfl; + last_ovfl = last; + } + WT_ASSERT(session, last != current); + } + + if (0) { +eof: ret = __err_eof(session, cell_num, addr); + } + + if (0) { +err: if (ret == 0) + ret = WT_ERROR; + } + __wt_scr_free(¤t); + __wt_scr_free(&last_pfx); + __wt_scr_free(&last_ovfl); + return (ret); +} + +/* + * __verify_dsk_col_int -- + * Walk a WT_PAGE_COL_INT disk page and verify it. + */ +static int +__verify_dsk_col_int( + WT_SESSION_IMPL *session, const char *addr, WT_PAGE_HEADER *dsk) +{ + WT_BTREE *btree; + WT_CELL *cell; + WT_CELL_UNPACK *unpack, _unpack; + uint32_t cell_num, i; + uint8_t *end; + + btree = session->btree; + unpack = &_unpack; + end = (uint8_t *)dsk + dsk->size; + + cell_num = 0; + WT_CELL_FOREACH(btree, dsk, cell, unpack, i) { + ++cell_num; + + /* Carefully unpack the cell. */ + if (__wt_cell_unpack_safe(cell, unpack, end) != 0) + return (__err_cell_corrupted(session, cell_num, addr)); + + /* Check the cell type. */ + WT_RET (__err_cell_type( + session, cell_num, addr, unpack->raw, dsk->type)); + + /* Check if any referenced item is entirely in the file. */ + if (!__wt_bm_addr_valid(session, unpack->data, unpack->size)) + return (__err_eof(session, cell_num, addr)); + } + + return (0); +} + +/* + * __verify_dsk_col_fix -- + * Walk a WT_PAGE_COL_FIX disk page and verify it. + */ +static int +__verify_dsk_col_fix( + WT_SESSION_IMPL *session, const char *addr, WT_PAGE_HEADER *dsk) +{ + WT_BTREE *btree; + uint32_t datalen; + + btree = session->btree; + + datalen = __bitstr_size(btree->bitcnt * dsk->u.entries); + return (__verify_dsk_chunk(session, addr, dsk, datalen)); +} + +/* + * __verify_dsk_col_var -- + * Walk a WT_PAGE_COL_VAR disk page and verify it. + */ +static int +__verify_dsk_col_var( + WT_SESSION_IMPL *session, const char *addr, WT_PAGE_HEADER *dsk) +{ + WT_BTREE *btree; + WT_CELL *cell; + WT_CELL_UNPACK *unpack, _unpack; + uint32_t cell_num, cell_type, i, last_size; + int last_deleted; + const uint8_t *last_data; + uint8_t *end; + + btree = session->btree; + unpack = &_unpack; + end = (uint8_t *)dsk + dsk->size; + + last_data = NULL; + last_size = 0; + last_deleted = 0; + + cell_num = 0; + WT_CELL_FOREACH(btree, dsk, cell, unpack, i) { + ++cell_num; + + /* Carefully unpack the cell. */ + if (__wt_cell_unpack_safe(cell, unpack, end) != 0) + return (__err_cell_corrupted(session, cell_num, addr)); + + /* + * Check the raw cell type, then collapse the short key/data + * types. + */ + WT_RET (__err_cell_type( + session, cell_num, addr, unpack->raw, dsk->type)); + cell_type = unpack->type; + + /* Check if any referenced item is entirely in the file. + */ + if (cell_type == WT_CELL_VALUE_OVFL && + !__wt_bm_addr_valid(session, unpack->data, unpack->size)) + return (__err_eof(session, cell_num, addr)); + + /* + * Compare the last two items and see if reconciliation missed + * a chance for RLE encoding. We don't have to care about data + * encoding or anything else, a byte comparison is enough. + */ + if (last_deleted == 1) { + if (cell_type == WT_CELL_DEL) + goto match_err; + } else + if (cell_type == WT_CELL_VALUE && + last_data != NULL && + last_size == unpack->size && + memcmp(last_data, unpack->data, last_size) == 0) +match_err: WT_RET_VRFY(session, + "data entries %" PRIu32 " and %" PRIu32 + " on page at %s are identical and should " + "have been run-length encoded", + cell_num - 1, cell_num, addr); + + switch (cell_type) { + case WT_CELL_DEL: + last_deleted = 1; + last_data = NULL; + break; + case WT_CELL_VALUE_OVFL: + last_deleted = 0; + last_data = NULL; + break; + case WT_CELL_VALUE: + last_deleted = 0; + last_data = unpack->data; + last_size = unpack->size; + break; + } + } + + return (0); +} + +/* + * __verify_dsk_chunk -- + * Verify a Chunk O' Data on a Btree page. + */ +static int +__verify_dsk_chunk(WT_SESSION_IMPL *session, + const char *addr, WT_PAGE_HEADER *dsk, uint32_t datalen) +{ + WT_BTREE *btree; + uint8_t *p, *end; + + btree = session->btree; + end = (uint8_t *)dsk + dsk->size; + + /* + * Fixed-length column-store, overflow and freelist pages are simple + * chunks of data. + */ + if (datalen == 0) + WT_RET_VRFY(session, + "%s page at %s has no data", + __wt_page_type_string(dsk->type), addr); + + /* Verify the data doesn't overflow the end of the page. */ + p = WT_PAGE_HEADER_BYTE(btree, dsk); + if (p + datalen > end) + WT_RET_VRFY(session, + "data on page at %s extends past the end of the page", + addr); + + /* Any bytes after the data chunk should be nul bytes. */ + for (p += datalen; p < end; ++p) + if (*p != '\0') + WT_RET_VRFY(session, + "%s page at %s has non-zero trailing bytes", + __wt_page_type_string(dsk->type), addr); + + return (0); +} + +/* + * __err_cell_corrupted -- + * Generic corrupted cell, we couldn't read it. + */ +static int +__err_cell_corrupted( + WT_SESSION_IMPL *session, uint32_t entry_num, const char *addr) +{ + WT_RET_VRFY(session, + "item %" PRIu32 " on page at %s is a corrupted cell", + entry_num, addr); +} + +/* + * __err_cell_type -- + * Generic illegal cell type for a particular page type error. + */ +static int +__err_cell_type(WT_SESSION_IMPL *session, + uint32_t entry_num, const char *addr, uint8_t cell_type, uint8_t dsk_type) +{ + switch (cell_type) { + case WT_CELL_ADDR: + if (dsk_type == WT_PAGE_COL_INT || + dsk_type == WT_PAGE_ROW_INT) + return (0); + break; + case WT_CELL_DEL: + if (dsk_type == WT_PAGE_COL_VAR) + return (0); + break; + case WT_CELL_KEY: + case WT_CELL_KEY_OVFL: + case WT_CELL_KEY_SHORT: + if (dsk_type == WT_PAGE_ROW_INT || + dsk_type == WT_PAGE_ROW_LEAF) + return (0); + break; + case WT_CELL_VALUE: + case WT_CELL_VALUE_SHORT: + case WT_CELL_VALUE_OVFL: + if (dsk_type == WT_PAGE_COL_VAR || + dsk_type == WT_PAGE_ROW_LEAF) + return (0); + break; + default: + break; + } + + WT_RET_VRFY(session, + "illegal cell and page type combination: cell %" PRIu32 + " on page at %s is a %s cell on a %s page", + entry_num, addr, + __wt_cell_type_string(cell_type), __wt_page_type_string(dsk_type)); +} + +/* + * __err_eof -- + * Generic item references non-existent file pages error. + */ +static int +__err_eof(WT_SESSION_IMPL *session, uint32_t entry_num, const char *addr) +{ + WT_RET_VRFY(session, + "off-page item %" PRIu32 + " on page at %s references non-existent file pages", + entry_num, addr); +} diff --git a/src/btree/bt_walk.c b/src/btree/bt_walk.c new file mode 100644 index 00000000000..d25195fca77 --- /dev/null +++ b/src/btree/bt_walk.c @@ -0,0 +1,116 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_tree_np -- + * Move to the next/previous page in the tree. + */ +int +__wt_tree_np(WT_SESSION_IMPL *session, WT_PAGE **pagep, int cacheonly, int next) +{ + WT_BTREE *btree; + WT_PAGE *page, *t; + WT_REF *ref; + uint32_t slot; + int ret; + + btree = session->btree; + + /* + * Take a copy of any returned page; we have a hazard reference on the + * page, by definition. + */ + page = *pagep; + *pagep = NULL; + + /* If no page is active, begin a walk from the start of the tree. */ + if (page == NULL) { + if ((page = btree->root_page) == NULL) + return (0); + slot = next ? 0 : page->entries - 1; + goto descend; + } + + /* If the active page was the root, we've reached the walk's end. */ + if (WT_PAGE_IS_ROOT(page)) + return (0); + + /* Figure out the current slot in the parent page's WT_REF array. */ + t = page->parent; + slot = (uint32_t)(page->ref - t->u.intl.t); + + /* + * Swap our hazard reference for the hazard reference of our parent, + * if it's not the root page (we could access it directly because we + * know it's in memory, but we need a hazard reference). Don't leave + * a hazard reference dangling on error. + * + * We're hazard-reference coupling up the tree and that's OK: first, + * hazard references can't deadlock, so there's none of the usual + * problems found when logically locking up a Btree; second, we don't + * release our current hazard reference until we have our parent's + * hazard reference. If the eviction thread tries to evict the active + * page, that fails because of our hazard reference. If eviction tries + * to evict our parent, that fails because the parent has a child page + * that can't be discarded. + */ + ret = (WT_PAGE_IS_ROOT(t) || cacheonly) ? + 0 : __wt_page_in(session, t, t->ref); + if (!cacheonly) { + __wt_page_release(session, page); + WT_RET(ret); + } + page = t; + + /* + * If we're at the last/first slot on the page, return this page in + * post-order traversal. Otherwise we move to the next/prev slot + * and left/right-most element in its subtree. + */ + for (;;) { + if ((!next && slot == 0) || + (next && slot == page->entries - 1)) { + *pagep = page; + return (0); + } + if (next) + ++slot; + else + --slot; + +descend: for (;;) { + if (page->type == WT_PAGE_ROW_INT || + page->type == WT_PAGE_COL_INT) + ref = &page->u.intl.t[slot]; + else { + *pagep = page; + return (0); + } + + /* We may only care about in-memory pages. */ + if (cacheonly) { + if (ref->state != WT_REF_MEM) + break; + } else { + /* + * Swap hazard references at each level (but + * don't leave a hazard reference dangling on + * error). + */ + ret = __wt_page_in(session, page, ref); + __wt_page_release(session, page); + WT_RET(ret); + } + + page = ref->page; + slot = next ? 0 : page->entries - 1; + } + } + /* NOTREACHED */ +} diff --git a/src/btree/col_modify.c b/src/btree/col_modify.c new file mode 100644 index 00000000000..468185a0704 --- /dev/null +++ b/src/btree/col_modify.c @@ -0,0 +1,299 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static int __col_insert_alloc( + WT_SESSION_IMPL *, uint64_t, u_int, WT_INSERT **, size_t *); + +/* + * __wt_col_modify -- + * Column-store delete, insert, and update. + */ +int +__wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, int op) +{ + WT_BTREE *btree; + WT_INSERT *ins, *ins_copy; + WT_INSERT_HEAD **inshead, *new_inshead, **new_inslist; + WT_ITEM *value, _value; + WT_PAGE *page; + WT_UPDATE *upd; + size_t ins_size, new_inshead_size, new_inslist_size, upd_size; + uint64_t recno; + u_int skipdepth; + int i, ret; + + btree = cbt->btree; + page = cbt->page; + + switch (op) { + case 1: /* Append */ + page = btree->last_page; + __cursor_search_clear(cbt); + + value = &cbt->iface.value; + recno = 0; /* Engine allocates */ + break; + case 2: /* Remove */ + if (btree->type == BTREE_COL_FIX) { + value = &_value; + value->data = ""; + value->size = 1; + } else + value = NULL; + recno = cbt->iface.recno; /* App specified */ + break; + case 3: /* Insert/Update */ + default: + value = &cbt->iface.value; + recno = cbt->iface.recno; /* App specified */ + + /* + * There's some chance the application specified a record past + * the last record on the page. If that's the case, and we're + * inserting a new WT_INSERT/WT_UPDATE pair, it goes on the + * append list, not the update list. + */ + if (recno > __col_last_recno(page)) + op = 1; + break; + } + + /* If we don't yet have a modify structure, we'll need one. */ + if (page->modify == NULL) + WT_RET(__wt_page_modify_init(session, page)); + + ins = NULL; + new_inshead = NULL; + new_inslist = NULL; + upd = NULL; + ret = 0; + + /* + * Delete, insert or update a column-store entry. + * + * If modifying a previously modified record, create a new WT_UPDATE + * entry and have a serialized function link it into an existing + * WT_INSERT entry's WT_UPDATE list. + * + * Else, allocate an insert array as necessary, build a WT_INSERT and + * WT_UPDATE structure pair, and call a serialized function to insert + * the WT_INSERT structure. + */ + if (cbt->compare == 0 && cbt->ins != NULL) { + WT_ERR(__wt_update_alloc(session, value, &upd, &upd_size)); + + /* Insert the WT_UPDATE structure. */ + ret = __wt_update_serial(session, page, + cbt->write_gen, &cbt->ins->upd, NULL, 0, &upd, upd_size); + } else { + /* There may be no insert list, allocate as necessary. */ + new_inshead_size = new_inslist_size = 0; + if (op == 1) { + if (btree->append == NULL) { + new_inslist_size = 1 * sizeof(WT_INSERT_HEAD *); + WT_ERR( + __wt_calloc_def(session, 1, &new_inslist)); + inshead = &new_inslist[0]; + } else + inshead = &btree->append[0]; + cbt->ins_head = *inshead; + } else if (page->type == WT_PAGE_COL_FIX) { + if (page->modify->update == NULL) { + new_inslist_size = 1 * sizeof(WT_INSERT_HEAD *); + WT_ERR( + __wt_calloc_def(session, 1, &new_inslist)); + inshead = &new_inslist[0]; + } else + inshead = &page->modify->update[0]; + } else { + if (page->modify->update == NULL) { + new_inslist_size = + page->entries * sizeof(WT_INSERT_HEAD *); + WT_ERR(__wt_calloc_def( + session, page->entries, &new_inslist)); + inshead = &new_inslist[cbt->slot]; + } else + inshead = &page->modify->update[cbt->slot]; + } + + /* There may be no WT_INSERT list, allocate as necessary. */ + if (*inshead == NULL) { + new_inshead_size = sizeof(WT_INSERT_HEAD); + WT_RET(__wt_calloc_def(session, 1, &new_inshead)); + for (i = 0; i < WT_SKIP_MAXDEPTH; i++) + cbt->ins_stack[i] = &new_inshead->head[i]; + cbt->ins_head = new_inshead; + } + + /* Choose a skiplist depth for this insert. */ + skipdepth = __wt_skip_choose_depth(); + + /* + * Allocate a WT_INSERT/WT_UPDATE pair, and update the cursor + * to reference it. + */ + WT_ERR(__col_insert_alloc( + session, recno, skipdepth, &ins, &ins_size)); + WT_ERR(__wt_update_alloc(session, value, &upd, &upd_size)); + ins->upd = upd; + ins_size += upd_size; + cbt->ins = ins; + + /* + * Insert or append the WT_INSERT structure. + */ + if (op == 1) { + /* + * The serialized function clears ins: take a copy of + * the pointer so we can look up the record number. + */ + ins_copy = ins; + + WT_ERR(__wt_col_append_serial(session, + inshead, cbt->ins_stack, + &new_inslist, new_inslist_size, + &new_inshead, new_inshead_size, + &ins, ins_size, skipdepth)); + + /* Set up the cursor for the inserted page and value. */ + cbt->page = btree->last_page; + cbt->recno = WT_INSERT_RECNO(ins_copy); + } else + WT_ERR(__wt_insert_serial(session, + page, cbt->write_gen, + inshead, cbt->ins_stack, + &new_inslist, new_inslist_size, + &new_inshead, new_inshead_size, + &ins, ins_size, skipdepth)); + } + + if (ret != 0) { +err: if (ins != NULL) + __wt_free(session, ins); + if (upd != NULL) + __wt_free(session, upd); + } + + __wt_free(session, new_inslist); + __wt_free(session, new_inshead); + + return (ret); +} + +/* + * __col_insert_alloc -- + * Column-store insert: allocate a WT_INSERT structure from the session's + * buffer and fill it in. + */ +static int +__col_insert_alloc(WT_SESSION_IMPL *session, + uint64_t recno, u_int skipdepth, WT_INSERT **insp, size_t *ins_sizep) +{ + WT_INSERT *ins; + size_t ins_size; + + /* + * Allocate the WT_INSERT structure and skiplist pointers, then copy + * the record number into place. + */ + ins_size = sizeof(WT_INSERT) + skipdepth * sizeof(WT_INSERT *); + WT_RET(__wt_calloc(session, 1, ins_size, &ins)); + + WT_INSERT_RECNO(ins) = recno; + + *insp = ins; + *ins_sizep = ins_size; + return (0); +} + +/* + * __wt_col_append_serial_func -- + * Server function to append an WT_INSERT entry to the tree. + */ +void +__wt_col_append_serial_func(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + WT_PAGE *page; + WT_INSERT *ins, *new_ins, ***ins_stack; + WT_INSERT_HEAD **inshead, **new_inslist, *new_inshead; + uint64_t recno; + u_int i, skipdepth; + int ret; + + btree = session->btree; + page = btree->last_page; + ret = 0; + + __wt_col_append_unpack(session, &inshead, &ins_stack, + &new_inslist, &new_inshead, &new_ins, &skipdepth); + + /* + * If the page does not yet have an insert array, our caller passed + * us one. + */ + if (btree->append == NULL) { + btree->append = new_inslist; + __wt_col_append_new_inslist_taken(session, page); + } + + /* + * If the insert head does not yet have an insert list, our caller + * passed us one. + */ + if (*inshead == NULL) { + *inshead = new_inshead; + __wt_col_append_new_inshead_taken(session, page); + } + + /* + * If the application specified a record number, there's a race: the + * application may have searched for the record, not found it, then + * called into the append code, and another thread might have added + * the record. Fortunately, we're in the right place because if the + * record didn't exist at some point, it can only have been created + * on this list. Search for the record, if specified. + */ + if ((recno = WT_INSERT_RECNO(new_ins)) == 0) + recno = WT_INSERT_RECNO(new_ins) = ++btree->last_recno; + ins = __col_insert_search(*inshead, ins_stack, recno); + + /* If we find the record number, there's been a race. */ + if (ins != NULL && WT_INSERT_RECNO(ins) == recno) { + ret = WT_RESTART; + goto done; + } + + /* + * If we don't find the record, check to see if we extended the file, + * and update the last record number. + */ + if (recno > btree->last_recno) + btree->last_recno = recno; + + /* + * Publish: First, point the new WT_INSERT item's skiplist references + * to the next elements in the insert list, then flush memory. Second, + * update the skiplist elements that reference the new WT_INSERT item, + * this ensures the list is never inconsistent. + */ + for (i = 0; i < skipdepth; i++) + new_ins->next[i] = *ins_stack[i]; + WT_WRITE_BARRIER(); + for (i = 0; i < skipdepth; i++) { + if ((*inshead)->tail[i] == NULL || + ins_stack[i] == &(*inshead)->tail[i]->next[i]) + (*inshead)->tail[i] = new_ins; + *ins_stack[i] = new_ins; + } + + __wt_col_append_new_ins_taken(session, page); + +done: __wt_session_serialize_wrapup(session, page, ret); +} diff --git a/src/btree/col_srch.c b/src/btree/col_srch.c new file mode 100644 index 00000000000..72c9673bd78 --- /dev/null +++ b/src/btree/col_srch.c @@ -0,0 +1,165 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_col_search -- + * Search a column-store tree for a specific record-based key. + */ +int +__wt_col_search(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, int is_modify) +{ + WT_BTREE *btree; + WT_COL *cip; + WT_INSERT *ins; + WT_INSERT_HEAD *ins_head; + WT_PAGE *page; + WT_REF *ref; + uint64_t recno; + uint32_t base, indx, limit; + int ret; + + __cursor_search_clear(cbt); + + recno = cbt->iface.recno; + + btree = session->btree; + ref = NULL; + + /* Search the internal pages of the tree. */ + for (page = btree->root_page; page->type == WT_PAGE_COL_INT;) { + WT_ASSERT(session, ref == NULL || + ref->u.recno == page->u.intl.recno); + + /* Binary search of internal pages. */ + for (base = 0, + limit = page->entries; limit != 0; limit >>= 1) { + indx = base + (limit >> 1); + ref = page->u.intl.t + indx; + + if (recno == ref->u.recno) + break; + if (recno < ref->u.recno) + continue; + base = indx + 1; + --limit; + } + WT_ASSERT(session, ref != NULL); + + /* + * Reference the slot used for next step down the tree. + * + * Base is the smallest index greater than recno and may be the + * (last + 1) index. The slot for descent is the one before + * base. + */ + if (recno != ref->u.recno) { + /* + * We don't have to correct for base == 0 because the + * only way for base to be 0 is if recno is the page's + * starting recno. + */ + WT_ASSERT(session, base > 0); + ref = page->u.intl.t + (base - 1); + } + + /* Swap the parent page for the child page. */ + WT_ERR(__wt_page_in(session, page, ref)); + __wt_page_release(session, page); + page = ref->page; + } + + /* + * Copy the leaf page's write generation value before reading the page. + * Use a read memory barrier to ensure we read the value before we read + * any of the page's contents. + */ + if (is_modify) { + /* Initialize the page's modification information */ + if (page->modify == NULL) + WT_RET(__wt_page_modify_init(session, page)); + + WT_ORDERED_READ(cbt->write_gen, page->modify->write_gen); + } + + cbt->page = page; + cbt->recno = recno; + cbt->compare = 0; + + /* + * Search the leaf page. We do not check in the search path for a + * record greater than the maximum record in the tree; in that case, + * we arrive here with a record that's impossibly large for the page. + */ + if (page->type == WT_PAGE_COL_FIX) { + if (recno >= page->u.col_fix.recno + page->entries) { + cbt->recno = page->u.col_fix.recno + page->entries; + goto past_end; + } else + ins_head = WT_COL_UPDATE_SINGLE(page); + } else + if ((cip = __col_var_search(page, recno)) == NULL) { + cbt->recno = __col_last_recno(page); + goto past_end; + } else { + cbt->slot = WT_COL_SLOT(page, cip); + ins_head = WT_COL_UPDATE_SLOT(page, cbt->slot); + } + + /* + * We have a match on the page, check for an update. Check the page's + * update list (fixed-length), or slot's update list (variable-length) + * for a better match. The only better match we can find is an exact + * match, otherwise the existing match on the page is the one we want. + * For that reason, don't set the cursor's WT_INSERT_HEAD/WT_INSERT pair + * until we know we have a useful entry. + */ + if ((ins = + __col_insert_search(ins_head, cbt->ins_stack, recno)) != NULL) + if (recno == WT_INSERT_RECNO(ins)) { + cbt->ins_head = ins_head; + cbt->ins = ins; + } + return (0); + +past_end: + /* + * A record past the end of the page's standard information. Check the + * append list; by definition, any record on the append list is closer + * than the last record on the page, so it's a better choice for return. + * This is a rarely used path: we normally find exact matches, because + * column-store files are dense, but in this case the caller searched + * past the end of the table. + */ + cbt->ins_head = WT_COL_APPEND(btree, page); + if ((cbt->ins = + __col_insert_search(cbt->ins_head, cbt->ins_stack, recno)) == NULL) + cbt->compare = -1; + else { + cbt->recno = WT_INSERT_RECNO(cbt->ins); + if (recno == cbt->recno) + cbt->compare = 0; + else if (recno < cbt->recno) + cbt->compare = 1; + else + cbt->compare = -1; + } + + /* + * Note if the record is past the maximum record in the tree, the cursor + * search functions need to know for fixed-length column-stores because + * appended records implicitly create any skipped records, and cursor + * search functions have to handle that case. + */ + if (cbt->compare == -1) + F_SET(cbt, WT_CBT_MAX_RECORD); + return (0); + +err: __wt_page_release(session, page); + return (ret); +} diff --git a/src/btree/rec_evict.c b/src/btree/rec_evict.c new file mode 100644 index 00000000000..5a441fb175c --- /dev/null +++ b/src/btree/rec_evict.c @@ -0,0 +1,575 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static int __hazard_exclusive(WT_SESSION_IMPL *, WT_REF *); +static int __rec_discard(WT_SESSION_IMPL *, WT_PAGE *); +static int __rec_discard_page(WT_SESSION_IMPL *, WT_PAGE *); +static int __rec_excl(WT_SESSION_IMPL *, WT_PAGE *, uint32_t); +static void __rec_excl_clear(WT_SESSION_IMPL *); +static int __rec_excl_page(WT_SESSION_IMPL *, WT_REF *, uint32_t); +static int __rec_page_clean_update(WT_SESSION_IMPL *, WT_PAGE *); +static int __rec_page_dirty_update(WT_SESSION_IMPL *, WT_PAGE *); +static int __rec_review(WT_SESSION_IMPL *, WT_PAGE *, uint32_t); +static int __rec_root_addr_update(WT_SESSION_IMPL *, uint8_t *, uint32_t); +static int __rec_root_clean_update(WT_SESSION_IMPL *, WT_PAGE *); +static int __rec_root_dirty_update(WT_SESSION_IMPL *, WT_PAGE *); + +/* + * __wt_rec_evict -- + * Reconciliation plus eviction. + */ +int +__wt_rec_evict(WT_SESSION_IMPL *session, WT_PAGE *page, uint32_t flags) +{ + WT_CONNECTION_IMPL *conn; + int ret; + + conn = S2C(session); + + WT_VERBOSE(session, evict, + "page %p (%s)", page, __wt_page_type_string(page->type)); + + /* + * We don't separately evict pages which are expected to be merged into + * their parents when the parent is evicted. The exception is root + * pages that split (the eviction code calls this function with a split + * page when closing a file where the root page has split). + */ + if (F_ISSET(page, WT_PAGE_REC_SPLIT_MERGE) || + (!WT_PAGE_IS_ROOT(page) && + F_ISSET(page, WT_PAGE_REC_EMPTY | WT_PAGE_REC_SPLIT))) + WT_ERR_MSG(session, EINVAL, + "attempt to evict an empty or split page that should " + "have been merged into its parent"); + + /* + * Get exclusive access to the page and review the page and its subtree + * for conditions that would block our eviction of the page. If the + * check fails (for example, we find a child page that can't be merged), + * we're done. We have to make this check for clean pages, too: while + * unlikely eviction would choose an internal page with children, it's + * not disallowed anywhere. + */ + WT_ERR(__rec_review(session, page, flags)); + + /* If the page is dirty, write it. */ + if (__wt_page_is_modified(page)) + WT_ERR(__wt_rec_write(session, page, NULL)); + + /* Count evictions of internal pages during normal operation. */ + if (!LF_ISSET(WT_REC_SINGLE) && + (page->type == WT_PAGE_COL_INT || page->type == WT_PAGE_ROW_INT)) + WT_STAT_INCR(conn->stats, cache_evict_internal); + + /* Update the parent and discard the page. */ + if (F_ISSET(page, WT_PAGE_REC_MASK) == 0) { + WT_STAT_INCR(conn->stats, cache_evict_unmodified); + + if (WT_PAGE_IS_ROOT(page)) + WT_ERR(__rec_root_clean_update(session, page)); + else + WT_ERR(__rec_page_clean_update(session, page)); + } else { + WT_STAT_INCR(conn->stats, cache_evict_modified); + + if (WT_PAGE_IS_ROOT(page)) + WT_ERR(__rec_root_dirty_update(session, page)); + else + WT_ERR(__rec_page_dirty_update(session, page)); + } + + return (0); + +err: /* + * If unable to evict this page, release exclusive reference(s) we've + * acquired. + */ + __rec_excl_clear(session); + return (ret); +} + +/* + * __rec_page_clean_update -- + * Update a page's reference for an evicted, clean page. + */ +static int +__rec_page_clean_update(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + /* + * Update the relevant WT_REF structure; no memory flush is needed, + * the state field is declared volatile. + */ + page->ref->page = NULL; + page->ref->state = WT_REF_DISK; + + return (__rec_discard_page(session, page)); +} + +/* + * __rec_root_clean_update -- + * Update a page's reference for an evicted, clean page. + */ +static int +__rec_root_clean_update(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_BTREE *btree; + + btree = session->btree; + + btree->root_page = NULL; + + return (__rec_discard_page(session, page)); +} + +/* + * __rec_page_dirty_update -- + * Update a page's reference for an evicted, dirty page. + */ +static int +__rec_page_dirty_update(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_PAGE_MODIFY *mod; + WT_REF *parent_ref; + + mod = page->modify; + parent_ref = page->ref; + + switch (F_ISSET(page, WT_PAGE_REC_MASK)) { + case WT_PAGE_REC_EMPTY: /* Page is empty */ + /* + * We're not going to evict this page after all, instead we'll + * merge it into its parent when that page is evicted. Release + * our exclusive reference to it, as well as any pages below it + * we locked down, and return it into use. + */ + __rec_excl_clear(session); + return (0); + case WT_PAGE_REC_REPLACE: /* 1-for-1 page swap */ + if (parent_ref->addr != NULL && + __wt_off_page(page->parent, parent_ref->addr)) { + __wt_free(session, ((WT_ADDR *)parent_ref->addr)->addr); + __wt_free(session, parent_ref->addr); + } + + /* + * Update the parent to reference the replacement page. + * + * Publish: a barrier to ensure the structure fields are set + * before the state change makes the page available to readers. + */ + WT_RET(__wt_calloc( + session, 1, sizeof(WT_ADDR), &parent_ref->addr)); + ((WT_ADDR *)parent_ref->addr)->addr = mod->u.replace.addr; + ((WT_ADDR *)parent_ref->addr)->size = mod->u.replace.size; + parent_ref->page = NULL; + WT_PUBLISH(parent_ref->state, WT_REF_DISK); + break; + case WT_PAGE_REC_SPLIT: /* Page split */ + /* + * Update the parent to reference new internal page(s). + * + * Publish: a barrier to ensure the structure fields are set + * before the state change makes the page available to readers. + */ + parent_ref->page = mod->u.split; + WT_PUBLISH(parent_ref->state, WT_REF_MEM); + + /* Clear the reference else discarding the page will free it. */ + mod->u.split = NULL; + break; + WT_ILLEGAL_VALUE(session); + } + + /* + * Discard pages which were merged into this page during reconciliation, + * then discard the page itself. + */ + WT_RET(__rec_discard(session, page)); + + return (0); +} + +/* + * __rec_root_addr_update -- + * Update the root page's address. + */ +static int +__rec_root_addr_update(WT_SESSION_IMPL *session, uint8_t *addr, uint32_t size) +{ + WT_ADDR *root_addr; + WT_BTREE *btree; + + btree = session->btree; + root_addr = &btree->root_addr; + + /* Free any previously created root addresses. */ + if (root_addr->addr != NULL) { + WT_RET(__wt_bm_free(session, root_addr->addr, root_addr->size)); + __wt_free(session, root_addr->addr); + } + btree->root_update = 1; + + root_addr->addr = addr; + root_addr->size = size; + + return (0); +} + +/* + * __rec_root_dirty_update -- + * Update the reference for an evicted, dirty root page. + */ +static int +__rec_root_dirty_update(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_BTREE *btree; + WT_PAGE *next; + WT_PAGE_MODIFY *mod; + + btree = session->btree; + mod = page->modify; + + next = NULL; + switch (F_ISSET(page, WT_PAGE_REC_MASK)) { + case WT_PAGE_REC_EMPTY: /* Page is empty */ + WT_VERBOSE(session, evict, "root page empty"); + + /* If the root page is empty, clear the root address. */ + WT_RET(__rec_root_addr_update(session, NULL, 0)); + btree->root_page = NULL; + break; + case WT_PAGE_REC_REPLACE: /* 1-for-1 page swap */ + WT_VERBOSE(session, evict, "root page replaced"); + + /* Update the root to its replacement. */ + WT_RET(__rec_root_addr_update( + session, mod->u.replace.addr, mod->u.replace.size)); + btree->root_page = NULL; + break; + case WT_PAGE_REC_SPLIT: /* Page split */ + WT_VERBOSE(session, evict, + "root page split %p -> %p", page, mod->u.split); + + /* Update the root to the split page. */ + next = mod->u.split; + + /* Clear the reference else discarding the page will free it. */ + mod->u.split = NULL; + break; + } + + /* + * Discard pages which were merged into this page during reconciliation, + * then discard the page itself. + */ + WT_RET(__rec_discard(session, page)); + + if (next == NULL) + return (0); + + /* + * Newly created internal pages are normally merged into their parent + * when the parent is evicted. Newly split root pages can't be merged, + * they have no parent and the new root page must be written. We also + * have to write the root page immediately, as the sync or close that + * triggered the split won't see our new root page during its traversal. + * + * Make the new root page look like a normal page that's been modified, + * write it out and discard it. Keep doing that and eventually we'll + * perform a simple replacement (as opposed to another level of split), + * allowing us to can update the tree's root information and quit. The + * only time we see multiple splits in here is when we've bulk-loaded + * something huge, and now we're evicting the index page referencing all + * of those leaf pages. + */ + WT_RET(__wt_page_modify_init(session, next)); + __wt_page_modify_set(next); + F_CLR(next, WT_PAGE_REC_MASK); + + WT_RET(__wt_rec_write(session, next, NULL)); + + return (__rec_root_dirty_update(session, next)); +} + +/* + * __rec_review -- + * Get exclusive access to the page and review the page and its subtree + * for conditions that would block our eviction of the page. + */ +static int +__rec_review(WT_SESSION_IMPL *session, WT_PAGE *page, uint32_t flags) +{ + session->excl_next = 0; /* Track the pages we lock. */ + + /* + * Get exclusive access to the page if our caller doesn't have the tree + * locked down. + */ + if (!LF_ISSET(WT_REC_SINGLE)) + WT_RET(__hazard_exclusive(session, page->ref)); + + /* Walk the page's subtree and make sure we can evict this page. */ + switch (page->type) { + case WT_PAGE_COL_INT: + case WT_PAGE_ROW_INT: + WT_RET(__rec_excl(session, page, flags)); + break; + default: + break; + } + + return (0); +} + +/* + * __rec_excl -- + * Walk an internal page's subtree, getting exclusive access as necessary, + * and checking if the subtree can be evicted. + */ +static int +__rec_excl(WT_SESSION_IMPL *session, WT_PAGE *parent, uint32_t flags) +{ + WT_PAGE *page; + WT_REF *ref; + uint32_t i; + + /* For each entry in the page... */ + WT_REF_FOREACH(parent, ref, i) { + switch (ref->state) { + case WT_REF_DISK: /* On-disk */ + continue; + case WT_REF_EVICTING: /* Being evaluated */ + case WT_REF_MEM: /* In-memory */ + break; + case WT_REF_LOCKED: /* Being evicted */ + case WT_REF_READING: /* Being read */ + return (WT_ERROR); + } + WT_RET(__rec_excl_page(session, ref, flags)); + + /* Recurse down the tree. */ + page = ref->page; + switch (page->type) { + case WT_PAGE_COL_INT: + case WT_PAGE_ROW_INT: + WT_RET(__rec_excl(session, page, flags)); + break; + default: + break; + } + } + return (0); +} + +/* + * __rec_excl_clear -- + * Discard exclusive access and return a page's subtree to availability. + */ +static void +__rec_excl_clear(WT_SESSION_IMPL *session) +{ + WT_REF *ref; + uint32_t i; + + for (i = 0; i < session->excl_next; ++i) { + if ((ref = session->excl[i]) == NULL) + break; + WT_ASSERT(session, ref->state == WT_REF_LOCKED); + ref->state = WT_REF_MEM; + } + session->excl_next = 0; +} + +/* + * __rec_excl_page -- + * Acquire exclusive access to a page as necessary, and check if the page + * can be evicted. + */ +static int +__rec_excl_page(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags) +{ + WT_PAGE *page; + + page = ref->page; + + /* + * If we find a page that can't be merged into its parent, we're done: + * you can't evict a page that references other in-memory pages, those + * pages must be evicted first. While the test is necessary, it should + * not happen much: reading an internal page increments its read + * generation, and so internal pages shouldn't be selected for eviction + * until after their children have been evicted. + * + * A cheap test: if the child page doesn't at least have a chance of a + * merge, we're done. + */ + if (!F_ISSET(page, + WT_PAGE_REC_EMPTY | WT_PAGE_REC_SPLIT | WT_PAGE_REC_SPLIT_MERGE)) + return (EBUSY); + + /* + * Next, if our caller doesn't have the tree locked down, get exclusive + * access to the page. + */ + if (!LF_ISSET(WT_REC_SINGLE)) + WT_RET(__hazard_exclusive(session, ref)); + + /* + * If the page is dirty, try and write it. This is because once a page + * is flagged for a merge into its parent, the eviction server no longer + * makes any attempt to evict it, it only attempts to evict its parent. + * If a parent page is blocked from eviction because of a dirty child + * page, we would never write the child page and never evict the parent. + * This prevents that from happening. + */ + if (__wt_page_is_modified(page)) + WT_RET(__wt_rec_write(session, page, NULL)); + + /* Repeat the cheap test: an empty page might no longer be "empty". */ + if (!F_ISSET(page, + WT_PAGE_REC_EMPTY | WT_PAGE_REC_SPLIT | WT_PAGE_REC_SPLIT_MERGE)) + return (EBUSY); + + /* + * Finally, a more careful test: merge-split pages are OK, no matter if + * they're clean or dirty, we can always merge them into the parent. + * Clean split or empty pages are OK too. Dirty split or empty pages + * are not OK, they must be written first so we know what they're going + * to look like to the parent. + */ + if (F_ISSET(page, WT_PAGE_REC_SPLIT_MERGE) || + (F_ISSET(page, WT_PAGE_REC_SPLIT | WT_PAGE_REC_EMPTY) && + !__wt_page_is_modified(page))) + return (0); + + return (EBUSY); +} + +/* + * __rec_discard -- + * Discard any pages merged into an evicted page, then the page itself. + */ +static int +__rec_discard(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_REF *ref; + uint32_t i; + + switch (page->type) { + case WT_PAGE_COL_INT: + case WT_PAGE_ROW_INT: + /* For each entry in the page... */ + WT_REF_FOREACH(page, ref, i) + if (ref->state != WT_REF_DISK) + WT_RET(__rec_discard(session, ref->page)); + /* FALLTHROUGH */ + default: + WT_RET(__rec_discard_page(session, page)); + break; + } + return (0); +} + +/* + * __rec_discard_page -- + * Process the page's list of tracked objects, and discard it. + */ +static int +__rec_discard_page(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_PAGE_MODIFY *mod; + + mod = page->modify; + + /* + * or if the page was split and later merged, discard it. + */ + if (mod != NULL) { + /* + * If the page has been modified and was tracking objects, + * resolve them. + */ + WT_RET(__wt_rec_track_wrapup(session, page, 1)); + + /* + * If the page was split and eventually merged into the parent, + * discard the split page; if the split page was promoted into + * a split-merge page, then the reference must be cleared before + * the page is discarded. + */ + if (F_ISSET( + page, WT_PAGE_REC_MASK) == WT_PAGE_REC_SPLIT && + mod->u.split != NULL) + __wt_page_out(session, mod->u.split, 0); + } + + /* Discard the page itself. */ + __wt_page_out(session, page, 0); + + return (0); +} + +/* + * __hazard_exclusive -- + * Request exclusive access to a page. + */ +static int +__hazard_exclusive(WT_SESSION_IMPL *session, WT_REF *ref) +{ + WT_CONNECTION_IMPL *conn; + uint32_t elem, i; + int ret, was_evicting; + + /* + * Hazard references are acquired down the tree, which means we can't + * deadlock. + * + * Request exclusive access to the page. It may be either in the + * evicting state (if this is the top-level page for this eviction + * operation), or a child page in memory. If another thread already + * has this page, give up. + * + * Keep track of the original state: if it was WT_REF_EVICTING and + * we cannot get it exclusive, restore it to that state. This ensures + * that application threads drain from a page that eviction is trying + * to force out. Without this, application threads can starve eviction + * and heap usage grows without bounds. + */ + was_evicting = 0; + if (WT_ATOMIC_CAS(ref->state, WT_REF_EVICTING, WT_REF_LOCKED)) + was_evicting = 1; + else if (!WT_ATOMIC_CAS(ref->state, WT_REF_MEM, WT_REF_LOCKED)) + return (EBUSY); /* We couldn't change the state. */ + + /* Walk the list of hazard references to search for a match. */ + conn = S2C(session); + elem = conn->session_size * conn->hazard_size; + for (i = 0; i < elem; ++i) + if (conn->hazard[i].page == ref->page) { + WT_BSTAT_INCR(session, rec_hazard); + WT_CSTAT_INCR(session, cache_evict_hazard); + + WT_VERBOSE(session, + evict, "page %p hazard request failed", ref->page); + WT_ERR(EBUSY); + } + + /* We have exclusive access, track that so we can unlock to clean up. */ + if (session->excl_next * sizeof(WT_REF *) == session->excl_allocated) + WT_ERR(__wt_realloc(session, &session->excl_allocated, + (session->excl_next + 50) * sizeof(WT_REF *), + &session->excl)); + session->excl[session->excl_next++] = ref; + + return (0); + + /* Restore to the original state on error. */ +err: ref->state = (was_evicting ? WT_REF_EVICTING : WT_REF_MEM); + return (ret); +} diff --git a/src/btree/rec_track.c b/src/btree/rec_track.c new file mode 100644 index 00000000000..db9394258ce --- /dev/null +++ b/src/btree/rec_track.c @@ -0,0 +1,399 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * A page in memory has a list of associated blocks and overflow items. For + * example, when an overflow item is modified, the original overflow blocks + * must be freed at some point. Or, when a page is split, then written again, + * the first split must be freed. This code tracks those objects: they are + * generally called from the routines in rec_write.c, which update the objects + * each time a page is reconciled. + */ + +#ifdef HAVE_VERBOSE +static void __track_dump(WT_SESSION_IMPL *, WT_PAGE *, const char *); +static void __track_msg(WT_SESSION_IMPL *, WT_PAGE *, const char *, WT_ADDR *); +static void __track_print(WT_SESSION_IMPL *, WT_PAGE *, WT_PAGE_TRACK *); +#endif + +/* + * __rec_track_clear -- + * Clear a track entry. + */ +static inline void +__rec_track_clear(WT_PAGE_TRACK *track) +{ + track->type = WT_PT_EMPTY; + track->data = NULL; + track->size = 0; + track->addr.addr = NULL; + track->addr.size = 0; +} + +/* + * __rec_track_extend -- + * Extend the list of objects we're tracking + */ +static int +__rec_track_extend(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_PAGE_MODIFY *mod; + size_t bytes_allocated; + + mod = page->modify; + + /* + * The __wt_realloc() function uses the "bytes allocated" value + * to figure out how much of the memory it needs to clear (see + * the function for an explanation of why the memory is cleared, + * it's a security thing). We can calculate the bytes allocated + * so far, which saves a size_t in the WT_PAGE_MODIFY structure. + * That's worth a little dance, we have one of them per modified + * page. + */ + bytes_allocated = mod->track_entries * sizeof(*mod->track); + WT_RET(__wt_realloc(session, &bytes_allocated, + (mod->track_entries + 20) * sizeof(*mod->track), &mod->track)); + mod->track_entries += 20; + return (0); +} + +/* + * __wt_rec_track_block -- + * Add an addr/size pair to the page's list of tracked objects. + */ +int +__wt_rec_track_block(WT_SESSION_IMPL *session, + __wt_pt_type_t type, WT_PAGE *page, const uint8_t *addr, uint32_t size) +{ + WT_PAGE_MODIFY *mod; + WT_PAGE_TRACK *empty, *track; + uint32_t i; + + mod = page->modify; + + /* + * There may be multiple requests to track a single block. For example, + * an internal page with an overflow key that references a page that's + * split: every time the page is written, we'll figure out the key's + * overflow pages are no longer useful because the underlying page has + * split, but we have no way to know that we've figured that same thing + * out several times already. Check for duplicates. + */ + empty = NULL; + for (track = mod->track, i = 0; i < mod->track_entries; ++track, ++i) { + if (track->type == WT_PT_EMPTY) { + empty = track; + continue; + } + if (track->type == type && + track->addr.size == size && + memcmp(addr, track->addr.addr, size) == 0) + return (0); + } + + /* Reallocate space as necessary. */ + if (empty == NULL) { + WT_RET(__rec_track_extend(session, page)); + empty = &mod->track[mod->track_entries - 1]; + } + + track = empty; + track->type = type; + track->data = NULL; + track->size = 0; + WT_RET(__wt_strndup(session, (char *)addr, size, &track->addr.addr)); + track->addr.size = size; + + WT_VERBOSE_CALL( + session, reconcile, __track_print(session, page, track)); + return (0); +} + +/* + * __wt_rec_track_ovfl -- + * Add an overflow object to the page's list of tracked objects. + */ +int +__wt_rec_track_ovfl(WT_SESSION_IMPL *session, WT_PAGE *page, + uint8_t *addr, uint32_t addr_size, const void *data, uint32_t data_size) +{ + WT_PAGE_MODIFY *mod; + WT_PAGE_TRACK *empty, *track; + uint8_t *p; + uint32_t i; + + WT_ASSERT(session, addr != NULL); + + mod = page->modify; + + empty = NULL; + for (track = mod->track, i = 0; i < mod->track_entries; ++track, ++i) + if (track->type == WT_PT_EMPTY) { + empty = track; + break; + } + + /* Reallocate space as necessary. */ + if (empty == NULL) { + WT_RET(__rec_track_extend(session, page)); + empty = &mod->track[mod->track_entries - 1]; + } + + /* + * Minor optimization: allocate a single chunk of space instead of two + * separate ones: be careful when it's freed. + */ + WT_RET(__wt_calloc_def(session, addr_size + data_size, &p)); + + track = empty; + track->type = WT_PT_OVFL; + track->addr.addr = p; + track->addr.size = addr_size; + memcpy(track->addr.addr, addr, addr_size); + + p += addr_size; + track->data = p; + track->size = data_size; + memcpy(track->data, data, data_size); + + WT_VERBOSE_CALL( + session, reconcile, __track_print(session, page, track)); + return (0); +} + +/* + * __wt_rec_track_ovfl_reuse -- + * Search for an overflow record and reactivate it. + */ +int +__wt_rec_track_ovfl_reuse(WT_SESSION_IMPL *session, WT_PAGE *page, + const void *data, uint32_t size, uint8_t **addrp, uint32_t *sizep) +{ + WT_PAGE_TRACK *track; + uint32_t i; + + WT_PAGE_MODIFY *mod; + + mod = page->modify; + for (track = mod->track, i = 0; i < mod->track_entries; ++track, ++i) { + /* Check for a match. */ + if (track->type != WT_PT_OVFL_DISCARD || + size != track->size || memcmp(data, track->data, size) != 0) + continue; + + /* Found a match, return the record to use. */ + track->type = WT_PT_OVFL; + + /* Return the block addr/size pair to our caller. */ + *addrp = track->addr.addr; + *sizep = track->addr.size; + + WT_VERBOSE_CALL(session, reconcile, __track_msg( + session, page, "reactivate overflow", &track->addr)); + return (1); + } + return (0); +} + +/* + * __wt_rec_track_init -- + * Initialize/Reset the tracking information when writing a page. + */ +int +__wt_rec_track_init(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_PAGE_MODIFY *mod; + WT_PAGE_TRACK *track; + uint32_t i; + + WT_VERBOSE_CALL( + session, reconcile, __track_dump(session, page, "reconcile init")); + + mod = page->modify; + + for (track = mod->track, i = 0; i < mod->track_entries; ++track, ++i) + switch (track->type) { + case WT_PT_BLOCK_EVICT: + /* + * We had a block we would have discarded, had the last + * reconciliation been the final one used to evict the + * page -- it wasn't, and we didn't. Clear the slot. + */ + __rec_track_clear(track); + break; + case WT_PT_OVFL: + /* + * An overflow item associated with this page: mark it + * "not in use", we'll reactivate any being re-used as + * we process the page. + */ + WT_VERBOSE_CALL(session, reconcile, __track_msg( + session, page, "set overflow OFF", &track->addr)); + track->type = WT_PT_OVFL_DISCARD; + break; + case WT_PT_EMPTY: + break; + case WT_PT_BLOCK: + case WT_PT_OVFL_DISCARD: + /* + * We shouldn't see WT_PT_BLOCK or WT_PT_OVFL_DISCARD, + * those blocks were discarded at the end of the last + * reconciliation of this page. + */ + /* FALLTHROUGH */ + WT_ILLEGAL_VALUE(session); + } + return (0); +} + +/* + * __wt_rec_track_wrapup -- + * Temporarily/Permanently resolve the page's list of tracked objects. + */ +int +__wt_rec_track_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page, int final) +{ + WT_PAGE_TRACK *track; + uint32_t i; + + WT_VERBOSE_CALL(session, reconcile, + __track_dump(session, + page, final ? "eviction wrapup" : "reconcile wrapup")); + + /* + * After a sync of a page, some of the objects we're tracking are no + * longer needed -- free what we can free. + * + * WT_PT_EMPTY: + * Empty slot. + * WT_PT_BLOCK: + * A discarded block, free when this reconciliation completes. + * WT_PT_BLOCK_EVICT: + * A discarded block based on this reconciliation; if the page is + * evicted based on this reconciliation, discard the block. (For + * example, an overflow key that references a deleted item will be + * discarded, but a subsequent reconciliation might find the key + * is once more in use because the item is no longer deleted.) + * WT_PT_OVFL: + * An overflow record that's in-use. Ignored after any particular + * reconciliation, because we need to track it for re-use in future + * reconciliations. When the page is evicted, discard its memory, + * leaving the underlying blocks alone. + * WT_PT_OVFL_DISCARD: + * An overflow record that's no longer in-use. Discard the memory + * and free the underlying blocks after reconciliation completes. + */ + for (track = page->modify->track, + i = 0; i < page->modify->track_entries; ++track, ++i) { + switch (track->type) { + case WT_PT_EMPTY: + continue; + case WT_PT_BLOCK_EVICT: + if (!final) + continue; + /* FALLTHROUGH */ + case WT_PT_BLOCK: + WT_VERBOSE_CALL(session, reconcile, __track_msg( + session, page, "discard block", &track->addr)); + WT_RET(__wt_bm_free( + session, track->addr.addr, track->addr.size)); + __wt_free(session, track->addr.addr); + break; + case WT_PT_OVFL: + WT_VERBOSE_CALL(session, reconcile, __track_msg( + session, page, "retain overflow", &track->addr)); + if (!final) + continue; + + /* Freeing WT_PAGE_TRACK->addr frees ->data, too. */ + __wt_free(session, track->addr.addr); + break; + case WT_PT_OVFL_DISCARD: + WT_VERBOSE_CALL(session, reconcile, __track_msg( + session, page, "discard overflow", &track->addr)); + WT_RET(__wt_bm_free( + session, track->addr.addr, track->addr.size)); + + /* Freeing WT_PAGE_TRACK->addr frees ->data, too. */ + __wt_free(session, track->addr.addr); + break; + } + + __rec_track_clear(track); + } + return (0); +} + +#ifdef HAVE_VERBOSE +/* + * __track_dump -- + * Dump the list of tracked objects. + */ +static void +__track_dump(WT_SESSION_IMPL *session, WT_PAGE *page, const char *tag) +{ + WT_PAGE_MODIFY *mod; + WT_PAGE_TRACK *track; + uint32_t i; + + mod = page->modify; + + if (mod->track_entries == 0) + return; + + WT_VERBOSE(session, + reconcile, "page %p tracking list at %s:", page, tag); + for (track = mod->track, i = 0; i < mod->track_entries; ++track, ++i) + __track_print(session, page, track); +} + +/* + * __track_print -- + * Display a tracked entry. + */ +static void +__track_print(WT_SESSION_IMPL *session, WT_PAGE *page, WT_PAGE_TRACK *track) +{ + switch (track->type) { + case WT_PT_BLOCK: + __track_msg(session, page, "block", &track->addr); + break; + case WT_PT_BLOCK_EVICT: + __track_msg(session, page, "block-evict", &track->addr); + return; + case WT_PT_OVFL: + __track_msg(session, page, "overflow (on)", &track->addr); + break; + case WT_PT_OVFL_DISCARD: + __track_msg(session, page, "overflow (off)", &track->addr); + break; + case WT_PT_EMPTY: + default: /* Not possible. */ + break; + } +} + +/* + * __track_msg -- + * Output a verbose message and associated page and address pair. + */ +static void +__track_msg( + WT_SESSION_IMPL *session, WT_PAGE *page, const char *msg, WT_ADDR *addr) +{ + WT_ITEM *buf; + + if (__wt_scr_alloc(session, 64, &buf)) + return; + WT_VERBOSE(session, reconcile, "page %p %s %s", page, msg, + __wt_addr_string(session, buf, addr->addr, addr->size)); + __wt_scr_free(&buf); +} +#endif diff --git a/src/btree/rec_write.c b/src/btree/rec_write.c new file mode 100644 index 00000000000..358fe75267f --- /dev/null +++ b/src/btree/rec_write.c @@ -0,0 +1,2972 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +struct __rec_kv; typedef struct __rec_kv WT_KV; +struct __rec_boundary; typedef struct __rec_boundary WT_BOUNDARY; + +/* + * Reconciliation is the process of taking an in-memory page, walking each entry + * in the page, building a backing disk image in a temporary buffer representing + * that information, and writing that buffer to disk. What could be simpler? + * + * WT_RECONCILE -- + * Information tracking a single page reconciliation. + */ +typedef struct { + WT_PAGE *page; /* Page being reconciled */ + + WT_ITEM dsk; /* Temporary disk-image buffer */ + + /* + * Reconciliation gets tricky if we have to split a page, that is, if + * the disk image we create exceeds the maximum size of disk images for + * this page type. First, the split sizes: reconciliation splits to a + * smaller-than-maximum page size when a split is required so we don't + * repeatedly split a packed page. + */ + uint32_t btree_split_pct; /* Split page percent */ + uint32_t page_size; /* Maximum page size */ + uint32_t split_size; /* Split page size */ + + /* + * The problem with splits is we've done a lot of work by the time we + * realize we're going to have to split, we don't want to start over. + * + * To keep from having to start over when we hit the maximum page size, + * we track the page information when we approach a split boundary. + * If we eventually have to split, we walk this structure and pretend + * we were splitting all along. After that, we continue to append to + * this structure, and eventually walk it to create a new internal page + * that references all of our split pages. + */ + struct __rec_boundary { + /* + * The start field records location in the initial split buffer, + * that is, the first byte of the split chunk recorded before we + * decide to split a page; the offset between the first byte of + * chunk[0] and the first byte of chunk[1] is chunk[0]'s length. + * + * Once we split a page, we stop filling in the start field, as + * we're writing the split chunks as we find them. + */ + uint8_t *start; /* Split's first byte */ + + /* + * The recno and entries fields are the starting record number + * of the split chunk (for column-store splits), and the number + * of entries in the split chunk. These fields are used both + * to write the split chunk, and to create a new internal page + * to reference the split pages. + */ + uint64_t recno; /* Split's starting record */ + uint32_t entries; /* Split's entries */ + + WT_ADDR addr; /* Split's written location */ + + /* + * The key for a row-store page; no column-store key is needed + * because the page's recno, stored in the recno field, is the + * column-store key. + */ + WT_ITEM key; /* Promoted row-store key */ + } *bnd; /* Saved boundaries */ + uint32_t bnd_next; /* Next boundary slot */ + uint32_t bnd_entries; /* Total boundary slots */ + size_t bnd_allocated; /* Bytes allocated */ + + /* + * We track the total number of page entries copied into split chunks + * so we can easily figure out how many entries in the current split + * chunk. + */ + uint32_t total_entries; /* Total entries in splits */ + + /* + * And there's state information as to where in this process we are: + * (1) tracking split boundaries because we can still fit more split + * chunks into the maximum page size, (2) tracking the maximum page + * size boundary because we can't fit any more split chunks into the + * maximum page size, (3) not performing boundary checks because it's + * either not useful with the current page size configuration, or + * because we've already been forced to split. + */ + enum { SPLIT_BOUNDARY=0, /* Next: a split page boundary */ + SPLIT_MAX=1, /* Next: the maximum page boundary */ + SPLIT_TRACKING_OFF=2 } /* No boundary checks */ + bnd_state; + + /* + * We track current information about the current record number, the + * number of entries copied into the temporary buffer, where we are + * in the temporary buffer, and how much memory remains. Those items + * are packaged here rather than passing pointers to stack locations + * around the code. + */ + uint64_t recno; /* Current record number */ + uint32_t entries; /* Current number of entries */ + uint8_t *first_free; /* Current first free byte */ + uint32_t space_avail; /* Remaining space in this chunk */ + + /* + * We don't need to keep the 0th key around on internal pages, the + * search code ignores them as nothing can sort less by definition. + * There's some trickiness here, see the code for comments on how + * these fields work. + */ + int cell_zero; /* Row-store internal page 0th key */ + WT_REF *merge_ref; /* Row-store merge correction key */ + + /* + * WT_KV-- + * An on-page key/value item we're building. + */ + struct __rec_kv { + WT_ITEM buf; /* Data */ + WT_CELL cell; /* Cell and cell's length */ + uint32_t cell_len; + uint32_t len; /* Total length of cell + data */ + } k, v; /* Key/Value being built */ + + WT_ITEM *cur, _cur; /* Key/Value being built */ + WT_ITEM *last, _last; /* Last key/value built */ + + int key_pfx_compress; /* If can prefix-compress next key */ + int key_pfx_compress_conf; /* If prefix compression configured */ + int key_sfx_compress; /* If can suffix-compress next key */ + int key_sfx_compress_conf; /* If suffix compression configured */ +} WT_RECONCILE; + +static void __rec_cell_build_addr( + WT_SESSION_IMPL *, const void *, uint32_t, uint64_t); +static int __rec_cell_build_key( + WT_SESSION_IMPL *, const void *, uint32_t, int, int *); +static int __rec_cell_build_ovfl( + WT_SESSION_IMPL *, WT_KV *, uint8_t, uint64_t); +static int __rec_cell_build_val( + WT_SESSION_IMPL *, const void *, uint32_t, uint64_t); +static int __rec_col_fix(WT_SESSION_IMPL *, WT_PAGE *); +static int __rec_col_fix_slvg( + WT_SESSION_IMPL *, WT_PAGE *, WT_SALVAGE_COOKIE *); +static int __rec_col_int(WT_SESSION_IMPL *, WT_PAGE *); +static int __rec_col_merge(WT_SESSION_IMPL *, WT_PAGE *); +static int __rec_col_var(WT_SESSION_IMPL *, WT_PAGE *, WT_SALVAGE_COOKIE *); +static int __rec_col_var_helper(WT_SESSION_IMPL *, + WT_SALVAGE_COOKIE *, WT_ITEM *, int, int, uint64_t); +static int __rec_row_int(WT_SESSION_IMPL *, WT_PAGE *); +static int __rec_row_leaf(WT_SESSION_IMPL *, WT_PAGE *, WT_SALVAGE_COOKIE *); +static int __rec_row_leaf_insert(WT_SESSION_IMPL *, WT_INSERT *); +static int __rec_row_merge(WT_SESSION_IMPL *, WT_PAGE *); +static int __rec_split(WT_SESSION_IMPL *session); +static int __rec_split_col(WT_SESSION_IMPL *, WT_PAGE *, WT_PAGE **); +static int __rec_split_finish(WT_SESSION_IMPL *); +static int __rec_split_fixup(WT_SESSION_IMPL *); +static int __rec_split_init(WT_SESSION_IMPL *, WT_PAGE *, uint64_t, uint32_t); +static int __rec_split_row(WT_SESSION_IMPL *, WT_PAGE *, WT_PAGE **); +static int __rec_split_row_promote(WT_SESSION_IMPL *, uint8_t); +static int __rec_split_write(WT_SESSION_IMPL *, WT_BOUNDARY *, WT_ITEM *); +static int __rec_write_init(WT_SESSION_IMPL *, WT_PAGE *); +static int __rec_write_wrapup(WT_SESSION_IMPL *, WT_PAGE *); + +/* + * __rec_track_cell -- + * If a cell references an overflow chunk, add it to the page's list. + */ +static inline int +__rec_track_cell( + WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL_UNPACK *unpack) +{ + return (unpack->ovfl ? __wt_rec_track_block(session, + WT_PT_BLOCK_EVICT, page, unpack->data, unpack->size) : 0); +} + +/* + * __wt_rec_write -- + * Reconcile an in-memory page into its on-disk format, and write it. + */ +int +__wt_rec_write( + WT_SESSION_IMPL *session, WT_PAGE *page, WT_SALVAGE_COOKIE *salvage) +{ + WT_VERBOSE(session, reconcile, + "page %p %s", page, __wt_page_type_string(page->type)); + + WT_BSTAT_INCR(session, rec_written); + + /* We're shouldn't get called with a clean page, that's an error. */ + WT_ASSERT(session, __wt_page_is_modified(page)); + + /* + * We can't do anything with a previously split page, that has to + * be merged into its parent. + */ + if (F_ISSET(page, WT_PAGE_REC_SPLIT_MERGE)) + return (0); + + /* Initialize the reconciliation structures for each new run. */ + WT_RET(__rec_write_init(session, page)); + + /* Initialize the overflow tracking information for each new run. */ + WT_RET(__wt_rec_track_init(session, page)); + + /* Reconcile the page. */ + switch (page->type) { + case WT_PAGE_COL_FIX: + if (salvage != NULL) + WT_RET(__rec_col_fix_slvg(session, page, salvage)); + else + WT_RET(__rec_col_fix(session, page)); + break; + case WT_PAGE_COL_INT: + WT_RET(__rec_col_int(session, page)); + break; + case WT_PAGE_COL_VAR: + WT_RET(__rec_col_var(session, page, salvage)); + break; + case WT_PAGE_ROW_INT: + WT_RET(__rec_row_int(session, page)); + break; + case WT_PAGE_ROW_LEAF: + WT_RET(__rec_row_leaf(session, page, salvage)); + break; + WT_ILLEGAL_VALUE(session); + } + + /* Wrap up the page's reconciliation. */ + WT_RET(__rec_write_wrapup(session, page)); + + /* Wrap up overflow tracking, discarding what we can. */ + WT_RET(__wt_rec_track_wrapup(session, page, 0)); + + /* + * If this page has a parent, mark the parent dirty. + * + * There's no chance we need to flush this write -- the eviction thread + * is the only thread that eventually cares if the page is dirty or not, + * and it's our update making the parent dirty. (Other threads do have + * to flush their set-page-modified update, of course). + * + * We don't care if we race with updates: the page will still be marked + * dirty and that's all we care about. + */ + if (!WT_PAGE_IS_ROOT(page)) { + WT_RET(__wt_page_modify_init(session, page->parent)); + __wt_page_modify_set(page->parent); + } + + return (0); +} + +/* + * __rec_write_init -- + * Initialize the reconciliation structure. + */ +static int +__rec_write_init(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_CONFIG_ITEM cval; + WT_RECONCILE *r; + + /* Update the disk generation before we read anything from the page. */ + WT_ORDERED_READ(page->modify->disk_gen, page->modify->write_gen); + + /* Allocate a reconciliation structure if we don't already have one. */ + if ((r = session->reconcile) == NULL) { + WT_RET(__wt_calloc_def(session, 1, &r)); + session->reconcile = r; + + /* Connect prefix compression pointers/buffers. */ + r->cur = &r->_cur; + r->last = &r->_last; + + /* Configuration. */ + WT_RET(__wt_config_getones(session, + session->btree->config, "split_pct", &cval)); + r->btree_split_pct = (uint32_t)cval.val; + + WT_RET(__wt_config_getones(session, + session->btree->config, + "internal_key_truncate", &cval)); + r->key_sfx_compress_conf = (cval.val != 0); + + WT_RET(__wt_config_getones(session, + session->btree->config, + "prefix_compression", &cval)); + r->key_pfx_compress_conf = (cval.val != 0); + } + + r->page = page; + + return (0); +} + +/* + * __rec_destroy -- + * Clean up the reconciliation structure. + */ +void +__wt_rec_destroy(WT_SESSION_IMPL *session) +{ + WT_BOUNDARY *bnd; + WT_RECONCILE *r; + uint32_t i; + + if ((r = session->reconcile) == NULL) + return; + + __wt_buf_free(session, &r->dsk); + + if (r->bnd != NULL) { + for (bnd = r->bnd, i = 0; i < r->bnd_entries; ++bnd, ++i) { + __wt_free(session, bnd->addr.addr); + __wt_buf_free(session, &bnd->key); + } + __wt_free(session, r->bnd); + } + + __wt_buf_free(session, &r->k.buf); + __wt_buf_free(session, &r->v.buf); + __wt_buf_free(session, &r->_cur); + __wt_buf_free(session, &r->_last); + + __wt_free(session, session->reconcile); +} + +/* + * __rec_incr -- + * Update the memory tracking structure for a set of new entries. + */ +static inline void +__rec_incr(WT_SESSION_IMPL *session, WT_RECONCILE *r, uint32_t v, uint32_t size) +{ + /* + * The buffer code is fragile and prone to off-by-one errors -- check + * for overflow in diagnostic mode. + */ + WT_ASSERT(session, r->space_avail >= size); + WT_ASSERT(session, + WT_BLOCK_FITS(r->first_free, size, r->dsk.mem, r->page_size)); + + r->entries += v; + r->space_avail -= size; + r->first_free += size; +} + +/* + * __rec_copy_incr -- + * Copy a key/value cell and buffer pair into the new image. + */ +static inline void +__rec_copy_incr(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_KV *kv) +{ + uint32_t len; + uint8_t *p, *t; + + /* + * If there's only one chunk of data to copy (because the cell and data + * are being copied from the original disk page), the cell length won't + * be set, the WT_ITEM data/length will reference the data to be copied. + * + * WT_CELLs are typically small, 1 or 2 bytes -- don't call memcpy, do + * the copy in-line. + */ + for (p = (uint8_t *)r->first_free, + t = (uint8_t *)&kv->cell, len = kv->cell_len; len > 0; --len) + *p++ = *t++; + + /* The data can be quite large -- call memcpy. */ + if (kv->buf.size != 0) + memcpy(p, kv->buf.data, kv->buf.size); + + WT_ASSERT(session, kv->len == kv->cell_len + kv->buf.size); + __rec_incr(session, r, 1, kv->len); +} + +/* + * __rec_key_state_update -- + * Update prefix and suffix compression based on the last key. + */ +static inline void +__rec_key_state_update(WT_RECONCILE *r, int ovfl_key) +{ + WT_ITEM *a; + + /* + * If writing an overflow key onto the page, don't update the "last key" + * value, and leave the state of prefix compression alone. (If we are + * currently doing prefix compression, we have a key state which will + * continue to work, we're just skipping the key just created because + * it's an overflow key and doesn't participate in prefix compression. + * If we are not currently doing prefix compression, we can't start, an + * overflow key doesn't give us any state.) + * + * Additionally, if we wrote an overflow key onto the page, turn off the + * suffix compression of row-store internal node keys. (When we split, + * "last key" is the largest key on the previous page, and "cur key" is + * the first key on the next page, which is being promoted. In some + * cases we can discard bytes from the "cur key" that are not needed to + * distinguish between the "last key" and "cur key", compressing the + * size of keys on internal nodes. If we just built an overflow key, + * we're not going to update the "last key", making suffix compression + * impossible for the next key. Alternatively, we could remember where + * the last key was on the page, detect it's an overflow key, read it + * from disk and do suffix compression, but that's too much work for an + * unlikely event.) + * + * If we're not writing an overflow key on the page, update the last-key + * value and turn on both prefix and suffix compression. + */ + if (ovfl_key) + r->key_sfx_compress = 0; + else { + a = r->cur; + r->cur = r->last; + r->last = a; + + r->key_pfx_compress = r->key_pfx_compress_conf; + r->key_sfx_compress = r->key_sfx_compress_conf; + } +} + +/* + * __rec_split_bnd_grow -- + * Grow the boundary array as necessary. + */ +static inline int +__rec_split_bnd_grow(WT_SESSION_IMPL *session) +{ + WT_RECONCILE *r; + + r = session->reconcile; + + /* + * Make sure there's enough room in which to save another boundary. + * + * The calculation is actually +1, because we save the start point one + * past the current entry -- make it +20 so we don't grow slot-by-slot. + */ + if (r->bnd_next + 1 >= r->bnd_entries) { + WT_RET(__wt_realloc(session, &r->bnd_allocated, + (r->bnd_entries + 20) * sizeof(*r->bnd), &r->bnd)); + r->bnd_entries += 20; + } + return (0); +} + +/* + * __rec_split_init -- + * Initialization for the reconciliation split functions. + */ +static int +__rec_split_init( + WT_SESSION_IMPL *session, WT_PAGE *page, uint64_t recno, uint32_t max) +{ + WT_BTREE *btree; + WT_PAGE_HEADER *dsk; + WT_RECONCILE *r; + + r = session->reconcile; + btree = session->btree; + + /* Ensure the scratch buffer is large enough. */ + WT_RET(__wt_bm_write_size(session, &max)); + WT_RET(__wt_buf_initsize(session, &r->dsk, (size_t)max)); + + /* + * Clear the header and set the page type (the type doesn't change, and + * setting it later requires additional code in a few different places). + */ + dsk = r->dsk.mem; + memset(dsk, 0, WT_PAGE_HEADER_SIZE); + dsk->type = page->type; + + /* + * If we have to split, we want to choose a smaller page size for the + * split pages, because otherwise we could end up splitting one large + * packed page over and over. We don't want to pick the minimum size + * either, because that penalizes an application that did a bulk load + * and subsequently inserted a few items into packed pages. Currently, + * I'm using 75%, but I have no empirical evidence that's a good value. + * We should leave this as a tuning variable, but probably undocumented. + * + * The maximum page size may be a multiple of the split page size (for + * example, there's a maximum page size of 128KB, but because the table + * is active and we don't want to split a lot, the split size is 20KB). + * The maximum page size may NOT be an exact multiple of the split page + * size. + * + * It's lots of work to build these pages and don't want to start over + * when we reach the maximum page size (it's painful to restart after + * creating overflow items and compacted data, for example, as those + * items have already been written to disk). So, the loop calls the + * helper functions when approaching a split boundary, and we save the + * information at that point. That allows us to go back and split the + * page at the boundary points if we eventually overflow the maximum + * page size. + * + * Finally, fixed-size column-store pages can split under (very) rare + * circumstances, but they're usually allocated at a fixed page size, + * never anything smaller. + */ + r->page_size = max; + r->split_size = page->type == WT_PAGE_COL_FIX ? + max : + WT_SPLIT_PAGE_SIZE(max, btree->allocsize, r->btree_split_pct); + + /* + * If the maximum page size is the same as the split page size, there + * is no need to maintain split boundaries within a larger page. + */ + r->bnd_state = + max == r->split_size ? SPLIT_TRACKING_OFF : SPLIT_BOUNDARY; + + /* + * Initialize the array of boundary items and set the initial record + * number and buffer address. + */ + r->bnd_next = 0; + WT_RET(__rec_split_bnd_grow(session)); + r->bnd[0].recno = recno; + r->bnd[0].start = WT_PAGE_HEADER_BYTE(btree, dsk); + + /* Initialize the total entries. */ + r->total_entries = 0; + + /* + * Set the caller's information and configure so the loop calls us + * when approaching the split boundary. + */ + r->recno = recno; + r->entries = 0; + r->first_free = WT_PAGE_HEADER_BYTE(btree, dsk); + r->space_avail = r->split_size - WT_PAGE_HEADER_BYTE_SIZE(btree); + + /* New page, compression off. */ + r->key_pfx_compress = r->key_sfx_compress = 0; + + return (0); +} + +/* + * __rec_split -- + * Handle the page reconciliation bookkeeping. (Did you know "bookkeeper" + * has 3 doubled letters in a row? Sweet-tooth does, too.) + */ +static int +__rec_split(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + WT_BOUNDARY *bnd; + WT_PAGE_HEADER *dsk; + WT_RECONCILE *r; + uint32_t current_len; + + /* + * Handle page-buffer size tracking; we have to do this work in every + * reconciliation loop, and I don't want to repeat the code that many + * times. + */ + r = session->reconcile; + btree = session->btree; + dsk = r->dsk.mem; + + /* + * There are 3 cases we have to handle. + * + * #1 + * Not done, and about to cross a split boundary, in which case we save + * away the current boundary information and return. + * + * #2 + * Not done, and about to cross the max boundary, in which case we have + * to physically split the page -- use the saved split information to + * write all the split pages. + * + * #3 + * Not done, and about to cross the split boundary, but we've already + * done the split thing when we approached the max boundary, in which + * case we write the page and keep going. + * + * Cases #1 and #2 are the hard ones: we're called when we're about to + * cross each split boundary, and we save information away so we can + * split if we have to. We're also called when we're about to cross + * the maximum page boundary: in that case, we do the actual split, + * clean things up, then keep going. + */ + switch (r->bnd_state) { + case SPLIT_BOUNDARY: /* Case #1 */ + /* + * Save the information about where we are when the split would + * have happened. + */ + WT_RET(__rec_split_bnd_grow(session)); + bnd = &r->bnd[r->bnd_next++]; + + /* Set the number of entries for the just finished chunk. */ + bnd->entries = r->entries - r->total_entries; + r->total_entries = r->entries; + + /* + * Set the starting record number, buffer address and promotion + * key for the next chunk, clear the entries (not required, but + * cleaner). + */ + ++bnd; + bnd->recno = r->recno; + bnd->start = r->first_free; + if (dsk->type == WT_PAGE_ROW_INT || + dsk->type == WT_PAGE_ROW_LEAF) + WT_RET(__rec_split_row_promote(session, dsk->type)); + bnd->entries = 0; + + /* + * Set the space available to another split-size chunk, if we + * have one. If we don't have room for another split chunk, + * add whatever space remains in the maximum page size, and + * hope it's enough. + */ + current_len = WT_PTRDIFF32(r->first_free, dsk); + if (current_len + r->split_size <= r->page_size) + r->space_avail = + r->split_size - WT_PAGE_HEADER_BYTE_SIZE(btree); + else { + r->bnd_state = SPLIT_MAX; + r->space_avail = (r->page_size - + WT_PAGE_HEADER_BYTE_SIZE(btree)) - current_len; + } + break; + case SPLIT_MAX: /* Case #2 */ + /* + * It didn't all fit into a single page. + * + * Cycle through the saved split-point information, writing the + * split chunks we have tracked. + */ + WT_RET(__rec_split_fixup(session)); + + /* We're done saving split chunks. */ + r->bnd_state = SPLIT_TRACKING_OFF; + break; + case SPLIT_TRACKING_OFF: /* Case #3 */ + WT_RET(__rec_split_bnd_grow(session)); + bnd = &r->bnd[r->bnd_next++]; + + /* + * It didn't all fit, but either we've already noticed it and + * are now processing the rest of the page at the split-size + * boundaries, or the split size was the same as the page size, + * so we never bothered with saving split-point information. + * + * Write the current disk image. + */ + dsk->recno = bnd->recno; + dsk->u.entries = r->entries; + r->dsk.size = WT_PTRDIFF32(r->first_free, dsk); + WT_RET(__rec_split_write(session, bnd, &r->dsk)); + + /* + * Set the starting record number and promotion key for the next + * chunk, clear the entries (not required, but cleaner). + */ + ++bnd; + bnd->recno = r->recno; + if (dsk->type == WT_PAGE_ROW_INT || + dsk->type == WT_PAGE_ROW_LEAF) + WT_RET(__rec_split_row_promote(session, dsk->type)); + bnd->entries = 0; + + /* + * Set the caller's entry count and buffer information for the + * next chunk. We only get here if we're not splitting or have + * already split, so it's split-size chunks from here on out. + */ + r->entries = 0; + r->first_free = WT_PAGE_HEADER_BYTE(btree, dsk); + r->space_avail = + r->split_size - WT_PAGE_HEADER_BYTE_SIZE(btree); + break; + } + return (0); +} + +/* + * __rec_split_finish -- + * Finish processing a split page. + */ +static int +__rec_split_finish(WT_SESSION_IMPL *session) +{ + WT_BOUNDARY *bnd; + WT_PAGE_HEADER *dsk; + WT_RECONCILE *r; + + r = session->reconcile; + + /* + * We're done reconciling a page. + * + * First, we only arrive here with no entries to write if the page was + * entirely empty (if the page wasn't empty, the only reason to split, + * resetting entries to 0, is because there's another entry to write, + * which then sets entries to 1). If the page was empty, we eventually + * delete it. + */ + if (r->entries == 0) { + WT_ASSERT_RET(session, r->bnd_next == 0); + return (0); + } + + /* + * Second, check our split status: + * + * If we have already split, put the remaining data in the next boundary + * slot. + * + * If we have not yet split, the reconciled page fit into a maximum page + * size, all of our boundary checking was wasted. Change the first + * boundary slot to represent the full page (the first boundary slot is + * largely correct, just update the number of entries). + */ + if (r->bnd_state == SPLIT_TRACKING_OFF) { + WT_RET(__rec_split_bnd_grow(session)); + bnd = &r->bnd[r->bnd_next++]; + } else { + r->bnd_next = 1; + bnd = &r->bnd[0]; + bnd->entries = r->entries; + } + + /* Write the remaining information. */ + dsk = r->dsk.mem; + dsk->recno = bnd->recno; + dsk->u.entries = r->entries; + r->dsk.size = WT_PTRDIFF32(r->first_free, dsk); + return (__rec_split_write(session, bnd, &r->dsk)); +} + +/* + * __rec_split_fixup -- + * Fix up after crossing the maximum page boundary. + */ +static int +__rec_split_fixup(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + WT_BOUNDARY *bnd; + WT_ITEM *tmp; + WT_PAGE_HEADER *dsk; + WT_RECONCILE *r; + uint32_t i, len; + uint8_t *dsk_start; + int ret; + + /* + * When we overflow physical limits of the page, we walk the list of + * split chunks we've created and write those pages out, then update + * the caller's information. + */ + r = session->reconcile; + btree = session->btree; + tmp = NULL; + ret = 0; + + /* + * The data isn't laid out on a page boundary or nul padded; copy it to + * a clean, aligned, padded buffer before writing it. + * + * Allocate a scratch buffer to hold the new disk image. Copy the + * WT_PAGE_HEADER header onto the scratch buffer, most of the header + * information remains unchanged between the pages. + */ + WT_RET(__wt_scr_alloc(session, r->split_size, &tmp)); + dsk = tmp->mem; + memcpy(dsk, r->dsk.mem, WT_PAGE_HEADER_SIZE); + + /* + * For each split chunk we've created, update the disk image and copy + * it into place. + */ + dsk_start = WT_PAGE_HEADER_BYTE(btree, dsk); + for (i = 0, bnd = r->bnd; i < r->bnd_next; ++i, ++bnd) { + /* Copy the page contents to the temporary buffer. */ + len = WT_PTRDIFF32((bnd + 1)->start, bnd->start); + memcpy(dsk_start, bnd->start, len); + + /* Write the page. */ + dsk->recno = bnd->recno; + dsk->u.entries = bnd->entries; + tmp->size = WT_PAGE_HEADER_BYTE_SIZE(btree) + len; + WT_ERR(__rec_split_write(session, bnd, tmp)); + } + + /* + * There is probably a remnant in the working buffer that didn't get + * written; copy it down to the beginning of the working buffer, and + * update the starting record number. + * + * Confirm the remnant is no larger than the available split buffer. + * + * Fix up our caller's information. + */ + len = WT_PTRDIFF32(r->first_free, bnd->start); + WT_ASSERT_RET( + session, len < r->split_size - WT_PAGE_HEADER_BYTE_SIZE(btree)); + + dsk = r->dsk.mem; + dsk_start = WT_PAGE_HEADER_BYTE(btree, dsk); + (void)memmove(dsk_start, bnd->start, len); + + r->entries -= r->total_entries; + r->first_free = dsk_start + len; + r->space_avail = + (r->split_size - WT_PAGE_HEADER_BYTE_SIZE(btree)) - len; + +err: __wt_scr_free(&tmp); + return (ret); +} + +/* + * __rec_split_write -- + * Write a disk block out for the split helper functions. + */ +static int +__rec_split_write(WT_SESSION_IMPL *session, WT_BOUNDARY *bnd, WT_ITEM *buf) +{ + WT_CELL *cell; + WT_PAGE_HEADER *dsk; + uint32_t size; + uint8_t addr[WT_BM_MAX_ADDR_COOKIE]; + + /* + * We always write an additional byte on row-store leaf pages after the + * key value pairs. The reason is that zero-length value items are not + * written on the page and they're detected by finding two adjacent key + * cells. If the last value item on a page is zero length, we need a + * key cell after it on the page to detect it. The row-store leaf page + * reconciliation code made sure we had a spare byte in the buffer, now + * write a trailing zero-length key cell. This isn't a valid key cell, + * but since it's not referenced by the entries on the page, no code but + * the code reading after the key cell, to find the key value, will ever + * see it. + */ +#define WT_TRAILING_KEY_CELL (sizeof(uint8_t)) + dsk = buf->mem; + if (dsk->type == WT_PAGE_ROW_LEAF) { + WT_ASSERT_RET(session, buf->size < buf->memsize); + + cell = (WT_CELL *)&(((uint8_t *)buf->data)[buf->size]); + __wt_cell_pack_key_empty(cell); + ++buf->size; + } + + /* Write the chunk and save the location information. */ + WT_VERBOSE(session, write, "%s", __wt_page_type_string(dsk->type)); + WT_RET(__wt_bm_write(session, buf, addr, &size)); + WT_RET(__wt_strndup(session, (char *)addr, size, &bnd->addr.addr)); + bnd->addr.size = size; + + return (0); +} + +/* + * __rec_split_row_promote -- + * Key promotion for a row-store. + */ +static int +__rec_split_row_promote(WT_SESSION_IMPL *session, uint8_t type) +{ + WT_BTREE *btree; + WT_CELL *cell; + WT_CELL_UNPACK *unpack, _unpack; + WT_RECONCILE *r; + uint32_t cnt, len, size; + const uint8_t *pa, *pb; + + r = session->reconcile; + btree = session->btree; + unpack = &_unpack; + + /* + * For a column-store, the promoted key is the recno and we already have + * a copy. For a row-store, it's the first key on the page, a variable- + * length byte string, get a copy. + * + * This function is called from __rec_split at each split boundary, but + * that means we're not called before the first boundary. It's painful, + * but we need to detect that case and copy the key from the page we're + * building. We could simplify this by grabbing a copy of the first key + * we put on a page, perhaps in the function building keys for a page, + * but that's going to be uglier than this. + */ + if (r->bnd_next == 1) { + /* + * The cell had better have a zero-length prefix: it's the first + * key on the page. (If it doesn't have a zero-length prefix, + * __wt_cell_update_copy() won't be sufficient any way, we'd + * only copy the non-prefix-compressed portion of the key.) + */ + cell = WT_PAGE_HEADER_BYTE(btree, r->dsk.mem); + __wt_cell_unpack(cell, unpack); + WT_ASSERT_RET(session, unpack->prefix == 0); + WT_RET( + __wt_cell_unpack_copy(session, unpack, &r->bnd[0].key)); + } + + /* + * For the current slot, take the last key we built, after doing suffix + * compression. + * + * Suffix compression is a hack to shorten keys on internal pages. We + * only need enough bytes in the promoted key to ensure searches go to + * the correct page: the promoted key has to be larger than the last key + * on the leaf page preceding it, but we don't need any more bytes than + * that. In other words, we can discard any suffix bytes not required + * to distinguish between the key being promoted and the last key on the + * leaf page preceding it. This can only be done for the first level of + * internal pages, you cannot repeat suffix truncation as you split up + * the tree, it loses too much information. + * + * The r->last key sorts before the r->cur key, so we'll either find a + * larger byte value in r->cur, or r->cur will be the longer key. One + * caveat: if the largest key on the previous page was an overflow key, + * we don't have a key against which to compare, and we can't do suffix + * compression. + */ + if (type == WT_PAGE_ROW_LEAF && r->key_sfx_compress) { + pa = r->last->data; + pb = r->cur->data; + len = WT_MIN(r->last->size, r->cur->size); + size = len + 1; + for (cnt = 1; len > 0; ++cnt, --len, ++pa, ++pb) + if (*pa != *pb) { + size = cnt; + break; + } + } else + size = r->cur->size; + return (__wt_buf_set( + session, &r->bnd[r->bnd_next].key, r->cur->data, size)); +} + +/* + * __wt_rec_bulk_init -- + * Bulk insert reconciliation initialization. + */ +int +__wt_rec_bulk_init(WT_CURSOR_BULK *cbulk) +{ + WT_BTREE *btree; + WT_PAGE *page; + WT_SESSION_IMPL *session; + uint64_t recno; + + session = (WT_SESSION_IMPL *)cbulk->cbt.iface.session; + btree = session->btree; + page = cbulk->leaf; + + WT_RET(__rec_write_init(session, page)); + + switch (btree->type) { + case BTREE_COL_FIX: + case BTREE_COL_VAR: + recno = 1; + break; + case BTREE_ROW: + recno = 0; + break; + WT_ILLEGAL_VALUE(session); + } + + WT_RET(__rec_split_init( + session, page, recno, session->btree->maxleafpage)); + + return (0); +} + +/* + * __wt_rec_bulk_wrapup -- + * Bulk insert reconciliation cleanup. + */ +int +__wt_rec_bulk_wrapup(WT_CURSOR_BULK *cbulk) +{ + WT_BTREE *btree; + WT_PAGE *page; + WT_RECONCILE *r; + WT_SESSION_IMPL *session; + + session = (WT_SESSION_IMPL *)cbulk->cbt.iface.session; + r = session->reconcile; + btree = session->btree; + + switch (btree->type) { + case BTREE_COL_FIX: + if (cbulk->entry != 0) + __rec_incr(session, r, cbulk->entry, + __bitstr_size(cbulk->entry * btree->bitcnt)); + break; + case BTREE_COL_VAR: + if (cbulk->rle != 0) + WT_RET(__wt_rec_col_var_bulk_insert(cbulk)); + break; + case BTREE_ROW: + break; + WT_ILLEGAL_VALUE(session); + } + + page = cbulk->leaf; + + WT_RET(__rec_split_finish(session)); + WT_RET(__rec_write_wrapup(session, page)); + + /* Mark the page's parent dirty. */ + WT_RET(__wt_page_modify_init(session, page->parent)); + __wt_page_modify_set(page->parent); + + return (0); +} + +/* + * __wt_rec_row_bulk_insert -- + * Row-store bulk insert. + */ +int +__wt_rec_row_bulk_insert(WT_CURSOR_BULK *cbulk) +{ + WT_CURSOR *cursor; + WT_KV *key, *val; + WT_RECONCILE *r; + WT_SESSION_IMPL *session; + int ovfl_key; + + session = (WT_SESSION_IMPL *)cbulk->cbt.iface.session; + r = session->reconcile; + + cursor = &cbulk->cbt.iface; + key = &r->k; + val = &r->v; + WT_RET(__rec_cell_build_key(session, /* Build key cell */ + cursor->key.data, cursor->key.size, 0, &ovfl_key)); + WT_RET(__rec_cell_build_val(session, /* Build value cell */ + cursor->value.data, cursor->value.size, (uint64_t)0)); + + /* + * Boundary, split or write the page. If the K/V pair doesn't + * fit: split the page, switch to the non-prefix-compressed key + * and turn off compression until a full key is written to the + * new page. + * + * We write a trailing key cell on the page after the K/V pairs + * (see WT_TRAILING_KEY_CELL for more information). + */ + while (key->len + val->len + WT_TRAILING_KEY_CELL > r->space_avail) { + WT_RET(__rec_split(session)); + + r->key_pfx_compress = 0; + if (!ovfl_key) + WT_RET(__rec_cell_build_key( + session, NULL, 0, 0, &ovfl_key)); + } + + /* Copy the key/value pair onto the page. */ + __rec_copy_incr(session, r, key); + if (val->len != 0) + __rec_copy_incr(session, r, val); + + /* Update compression state. */ + __rec_key_state_update(r, ovfl_key); + + return (0); +} + +/* + * __wt_rec_col_fix_bulk_insert -- + * Fixed-length column-store bulk insert. + */ +int +__wt_rec_col_fix_bulk_insert(WT_CURSOR_BULK *cbulk) +{ + WT_BTREE *btree; + WT_CURSOR *cursor; + WT_RECONCILE *r; + WT_SESSION_IMPL *session; + + session = (WT_SESSION_IMPL *)cbulk->cbt.iface.session; + r = session->reconcile; + btree = session->btree; + cursor = &cbulk->cbt.iface; + + if (cbulk->entry == cbulk->nrecs) { + if (cbulk->entry != 0) { + /* + * If everything didn't fit, update the counters and + * split. + * + * Boundary: split or write the page. + */ + __rec_incr(session, r, cbulk->entry, + __bitstr_size(cbulk->entry * btree->bitcnt)); + WT_RET(__rec_split(session)); + } + cbulk->entry = 0; + cbulk->nrecs = r->space_avail / btree->bitcnt; + } + + __bit_setv(r->first_free, + cbulk->entry, btree->bitcnt, ((uint8_t *)cursor->value.data)[0]); + ++cbulk->entry; + ++r->recno; + + return (0); +} + +/* + * __wt_rec_col_var_bulk_insert -- + * Variable-length column-store bulk insert. + */ +int +__wt_rec_col_var_bulk_insert(WT_CURSOR_BULK *cbulk) +{ + WT_SESSION_IMPL *session; + WT_KV *val; + WT_RECONCILE *r; + + session = (WT_SESSION_IMPL *)cbulk->cbt.iface.session; + r = session->reconcile; + + val = &r->v; + WT_RET(__rec_cell_build_val( + session, cbulk->cmp.data, cbulk->cmp.size, cbulk->rle)); + + /* Boundary: split or write the page. */ + while (val->len > r->space_avail) + WT_RET(__rec_split(session)); + + /* Copy the value onto the page. */ + __rec_copy_incr(session, r, val); + + /* Update the starting record number in case we split. */ + r->recno += cbulk->rle; + + return (0); +} + +/* + * __rec_col_int -- + * Reconcile a column-store internal page. + */ +static int +__rec_col_int(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_RET(__rec_split_init(session, + page, page->u.intl.recno, session->btree->maxintlpage)); + + /* + * Walking the row-store internal pages is complicated by the fact that + * we're taking keys from the underlying disk image for the top-level + * page and we're taking keys from in-memory structures for merge pages. + * Column-store is simpler because the only information we copy is the + * record number and address, and it comes from in-memory structures in + * both the top-level and merge cases. In short, both the top-level + * and merge page walks look the same, and we just call the merge page + * function on the top-level page. + */ + WT_RET(__rec_col_merge(session, page)); + + /* Write the remnant page. */ + return (__rec_split_finish(session)); +} + +/* + * __rec_col_merge -- + * Recursively walk a column-store internal tree of merge pages. + */ +static int +__rec_col_merge(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_KV *val; + WT_CELL_UNPACK *unpack, _unpack; + WT_PAGE *rp; + WT_RECONCILE *r; + WT_REF *ref; + uint32_t i; + int val_set; + + WT_BSTAT_INCR(session, rec_page_merge); + + r = session->reconcile; + unpack = &_unpack; + val = &r->v; + + /* For each entry in the page... */ + WT_REF_FOREACH(page, ref, i) { + /* Update the starting record number in case we split. */ + r->recno = ref->u.recno; + + /* + * The page may be deleted or internally created during a split. + * Deleted/split pages are merged into the parent and discarded. + */ + val_set = 0; + if (ref->state != WT_REF_DISK) { + rp = ref->page; + switch (F_ISSET(rp, WT_PAGE_REC_MASK)) { + case WT_PAGE_REC_EMPTY: + /* + * Column-store pages are almost never empty, as + * discarding a page would remove a chunk of the + * name space. The exceptions are pages created + * when the tree is created, and never filled. + */ + continue; + case WT_PAGE_REC_REPLACE: + __rec_cell_build_addr(session, + rp->modify->u.replace.addr, + rp->modify->u.replace.size, + ref->u.recno); + val_set = 1; + break; + case WT_PAGE_REC_SPLIT: + WT_RET(__rec_col_merge( + session, rp->modify->u.split)); + continue; + case WT_PAGE_REC_SPLIT_MERGE: + WT_RET(__rec_col_merge(session, rp)); + continue; + } + } + + /* + * Build the value cell. The child page address is in one of 3 + * places: if the page was replaced, the page's modify structure + * references it and we built the value cell just above in the + * switch statement. Else, the WT_REF->addr reference points to + * an on-page cell or an off-page WT_ADDR structure: if it's an + * on-page cell and we copy it from the page, else build a new + * cell. + */ + if (!val_set) { + if (__wt_off_page(page, ref->addr)) + __rec_cell_build_addr(session, + ((WT_ADDR *)ref->addr)->addr, + ((WT_ADDR *)ref->addr)->size, + ref->u.recno); + else { + __wt_cell_unpack(ref->addr, unpack); + val->buf.data = ref->addr; + val->buf.size = unpack->len; + val->cell_len = 0; + val->len = unpack->len; + } + } + + /* Boundary: split or write the page. */ + while (val->len > r->space_avail) + WT_RET(__rec_split(session)); + + /* Copy the value onto the page. */ + __rec_copy_incr(session, r, val); + } + + return (0); +} + +/* + * __rec_col_fix -- + * Reconcile a fixed-width, column-store leaf page. + */ +static int +__rec_col_fix(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_BTREE *btree; + WT_INSERT *ins; + WT_INSERT_HEAD *append; + WT_RECONCILE *r; + uint64_t recno; + uint32_t entry, nrecs; + + r = session->reconcile; + btree = session->btree; + + /* Update any changes to the original on-page data items. */ + WT_SKIP_FOREACH(ins, WT_COL_UPDATE_SINGLE(page)) + __bit_setv_recno( + page, WT_INSERT_RECNO(ins), btree->bitcnt, + ((uint8_t *)WT_UPDATE_DATA(ins->upd))[0]); + + /* Allocate the memory. */ + WT_RET(__rec_split_init(session, + page, page->u.col_fix.recno, btree->maxleafpage)); + + /* Copy the updated, disk-image bytes into place. */ + memcpy(r->first_free, page->u.col_fix.bitf, + __bitstr_size(page->entries * btree->bitcnt)); + + /* Calculate the number of entries per page remainder. */ + entry = page->entries; + nrecs = (r->space_avail / btree->bitcnt) - page->entries; + r->recno += entry; + + /* Walk any append list. */ + append = WT_COL_APPEND(btree, page); + WT_SKIP_FOREACH(ins, append) + for (;;) { + /* + * The application may have inserted records which left + * gaps in the name space. + */ + for (recno = WT_INSERT_RECNO(ins); + nrecs > 0 && r->recno < recno; + --nrecs, ++entry, ++r->recno) + __bit_setv( + r->first_free, entry, btree->bitcnt, 0); + + if (nrecs > 0) { + __bit_setv(r->first_free, entry, btree->bitcnt, + ((uint8_t *)WT_UPDATE_DATA(ins->upd))[0]); + --nrecs; + ++entry; + ++r->recno; + break; + } + + /* + * If everything didn't fit, update the counters and + * split. + * + * Boundary: split or write the page. + */ + __rec_incr(session, + r, entry, __bitstr_size(entry * btree->bitcnt)); + WT_RET(__rec_split(session)); + + /* Calculate the number of entries per page. */ + entry = 0; + nrecs = r->space_avail / btree->bitcnt; + } + + /* Update the counters. */ + __rec_incr(session, r, entry, __bitstr_size(entry * btree->bitcnt)); + + /* Write the remnant page. */ + return (__rec_split_finish(session)); +} + +/* + * __rec_col_fix -- + * Reconcile a fixed-width, column-store leaf page created during salvage. + */ +static int +__rec_col_fix_slvg( + WT_SESSION_IMPL *session, WT_PAGE *page, WT_SALVAGE_COOKIE *salvage) +{ + WT_BTREE *btree; + WT_RECONCILE *r; + uint64_t page_start, page_take; + uint32_t entry, nrecs; + + r = session->reconcile; + btree = session->btree; + + /* + * !!! + * It's vanishingly unlikely and probably impossible for fixed-length + * column-store files to have overlapping key ranges. It's possible + * for an entire key range to go missing (if a page is corrupted and + * lost), but because pages can't split, it shouldn't be possible to + * find pages where the key ranges overlap. That said, we check for + * it during salvage and clean up after it here because it doesn't + * cost much and future column-store formats or operations might allow + * for fixed-length format ranges to overlap during salvage, and I + * don't want to have to retrofit the code later. + */ + WT_RET(__rec_split_init(session, + page, page->u.col_fix.recno, btree->maxleafpage)); + + /* We may not be taking all of the entries on the original page. */ + page_take = salvage->take == 0 ? page->entries : salvage->take; + page_start = salvage->skip == 0 ? 0 : salvage->skip; + for (;;) { + /* Calculate the number of entries per page. */ + entry = 0; + nrecs = r->space_avail / btree->bitcnt; + + for (; nrecs > 0 && salvage->missing > 0; + --nrecs, --salvage->missing, ++entry) + __bit_setv(r->first_free, entry, btree->bitcnt, 0); + + for (; nrecs > 0 && page_take > 0; + --nrecs, --page_take, ++page_start, ++entry) + __bit_setv(r->first_free, entry, btree->bitcnt, + __bit_getv(page->u.col_fix.bitf, + (uint32_t)page_start, btree->bitcnt)); + + r->recno += entry; + __rec_incr( + session, r, entry, __bitstr_size(entry * btree->bitcnt)); + + /* + * If everything didn't fit, then we have to force a split and + * keep going. + * + * Boundary: split or write the page. + */ + if (salvage->missing == 0 && page_take == 0) + break; + WT_RET(__rec_split(session)); + } + + /* Write the remnant page. */ + return (__rec_split_finish(session)); +} + +/* + * __rec_col_var_helper -- + * Create a column-store variable length record cell and write it onto a + * page. + */ +static int +__rec_col_var_helper( + WT_SESSION_IMPL *session, WT_SALVAGE_COOKIE *salvage, + WT_ITEM *value, int deleted, int raw, uint64_t rle) +{ + WT_RECONCILE *r; + WT_KV *val; + + r = session->reconcile; + val = &r->v; + + /* + * Occasionally, salvage needs to discard records from the beginning or + * end of the page, and because the items may be part of a RLE cell, do + * the adjustments here. It's not a mistake we don't bother telling + * our caller we've handled all the records from the page we care about, + * and can quit processing the page: salvage is a rare operation and I + * don't want to complicate our caller's loop. + */ + if (salvage != NULL) { + if (salvage->done) + return (0); + if (salvage->skip != 0) { + if (rle <= salvage->skip) { + salvage->skip -= rle; + return (0); + } + salvage->skip = 0; + rle -= salvage->skip; + } + if (salvage->take != 0) { + if (rle <= salvage->take) + salvage->take -= rle; + else { + rle = salvage->take; + salvage->take = 0; + } + if (salvage->take == 0) + salvage->done = 1; + } + } + + if (deleted) { + val->cell_len = __wt_cell_pack_del(&val->cell, rle); + val->buf.size = 0; + val->len = val->cell_len; + } else if (raw) { + val->buf.data = value->data; + val->buf.size = value->size; + val->cell_len = 0; + val->len = val->buf.size; + } else + WT_RET(__rec_cell_build_val( + session, value->data, value->size, rle)); + + /* Boundary: split or write the page. */ + while (val->len > r->space_avail) + WT_RET(__rec_split(session)); + + /* Copy the value onto the page. */ + __rec_copy_incr(session, r, val); + + /* Update the starting record number in case we split. */ + r->recno += rle; + + return (0); +} + +/* + * __rec_col_var -- + * Reconcile a variable-width column-store leaf page. + */ +static int +__rec_col_var( + WT_SESSION_IMPL *session, WT_PAGE *page, WT_SALVAGE_COOKIE *salvage) +{ + WT_BTREE *btree; + WT_CELL *cell; + WT_CELL_UNPACK *unpack, _unpack; + WT_COL *cip; + WT_INSERT *ins; + WT_INSERT_HEAD *append; + WT_ITEM *last, orig; + WT_RECONCILE *r; + WT_UPDATE *upd; + uint64_t n, nrepeat, repeat_count, rle, slvg_missing, src_recno; + uint32_t i, size; + int can_compare, deleted, last_deleted, orig_deleted, ret; + const void *data; + + r = session->reconcile; + btree = session->btree; + last = r->last; + unpack = &_unpack; + + WT_CLEAR(orig); + data = NULL; + size = 0; + + WT_RET(__rec_split_init(session, + page, page->u.col_var.recno, session->btree->maxleafpage)); + + /* + * The salvage code may be calling us to reconcile a page where there + * were missing records in the column-store name space. In this case + * we write a single RLE element onto a new page, so we know it fits, + * then update the starting record number. + * + * Note that we DO NOT pass the salvage cookie to our helper function + * in this case, we're handling one of the salvage cookie fields on + * our own, and don't need assistance from the helper function. + */ + slvg_missing = salvage == NULL ? 0 : salvage->missing; + if (slvg_missing) + WT_ERR(__rec_col_var_helper( + session, NULL, NULL, 1, 0, slvg_missing)); + + /* + * We track two data items through this loop: the previous (last) item + * and the current item: if the last item is the same as the current + * item, we increment the RLE count for the last item; if the last item + * is different from the current item, we write the last item onto the + * page, and replace it with the current item. The r->recno counter + * tracks records written to the page, and is incremented by the helper + * function immediately after writing records to the page. The record + * number of our source record, that is, the current item, is maintained + * in src_recno. + */ + src_recno = r->recno; + + /* For each entry in the in-memory page... */ + rle = 0; + can_compare = deleted = last_deleted = 0; + WT_COL_FOREACH(page, cip, i) { + /* + * Review the original cell, and get its repeat count and + * insert list. + */ + cell = WT_COL_PTR(page, cip); + ins = WT_SKIP_FIRST(WT_COL_UPDATE(page, cip)); + if (cell == NULL) { + nrepeat = 1; + orig_deleted = 1; + } else { + __wt_cell_unpack(cell, unpack); + + /* + * The data may be Huffman encoded, which means we have + * to decode it in order to compare it with the last + * item we saw, which may have been an update string. + * This code guarantees we find every single pair of + * objects we can RLE encode, including the application + * inserting an update to an existing record where it + * happened (?) to match a Huffman-encoded value in the + * previous or next record. However, we try to avoid + * copying in overflow records: if there's a WT_INSERT + * entry inserting a new record into a reference counted + * overflow record, then we have to write copies of the + * overflow record, and we do the comparisons. But, we + * don't copy in the overflow record just to see if it + * matches records on either side. + */ + if (unpack->ovfl && ins == NULL) { + /* + * Write out any record we're tracking and turn + * off comparisons for the next item. + */ + if (can_compare) { + WT_ERR(__rec_col_var_helper( + session, salvage, + last, last_deleted, 0, rle)); + can_compare = 0; + } + + /* Write out the overflow cell as a raw cell. */ + last->data = cell; + last->size = unpack->len; + WT_ERR(__rec_col_var_helper( + session, salvage, + last, 0, 1, __wt_cell_rle(unpack))); + src_recno += __wt_cell_rle(unpack); + continue; + } + + nrepeat = __wt_cell_rle(unpack); + orig_deleted = unpack->type == WT_CELL_DEL ? 1 : 0; + + /* Get a copy of the cell. */ + if (!orig_deleted) + WT_ERR(__wt_cell_unpack_copy( + session, unpack, &orig)); + + /* + * If we're re-writing a cell's reference of an overflow + * value, free the underlying file space. + * + * !!! + * We could optimize here by using the original overflow + * information for some set of the column values. (For + * example, if column cells #10-17 reference overflow X, + * and cell #12 is updated with a new record: we could + * use the original overflow X for either cells #10-11 + * or cells #13-17.) We don't do that, instead we write + * new overflow records for both groups. I'm skipping + * that work because I don't want the complexity, and + * overflow records should be rare. + */ + WT_ERR(__rec_track_cell(session, page, unpack)); + } + + /* + * Generate on-page entries: loop repeat records, looking for + * WT_INSERT entries matching the record number. The WT_INSERT + * lists are in sorted order, so only need check the next one. + */ + for (n = 0; + n < nrepeat; n += repeat_count, src_recno += repeat_count) { + if (ins != NULL && WT_INSERT_RECNO(ins) == src_recno) { + upd = ins->upd; + ins = WT_SKIP_NEXT(ins); + + deleted = WT_UPDATE_DELETED_ISSET(upd); + if (!deleted) { + data = WT_UPDATE_DATA(upd); + size = upd->size; + } + + repeat_count = 1; + } else { + upd = NULL; + deleted = orig_deleted; + if (!deleted) { + data = orig.data; + size = orig.size; + } + + /* + * The repeat count is the number of records up + * to the next WT_INSERT record, or up to the + * end of the entry if we have no more WT_INSERT + * records. + */ + if (ins == NULL) + repeat_count = nrepeat - n; + else + repeat_count = + WT_INSERT_RECNO(ins) - src_recno; + } + + /* + * Handle RLE accounting and comparisons. + * + * If we don't have a record against which to compare, + * save this record for the purpose and continue. + * + * If we have a record against which to compare, and + * the records compare equal, increment the rle counter + * and continue. If the records don't compare equal, + * output the last record and swap the last and current + * buffers: do NOT update the starting record number, + * we've been doing that all along. + */ + if (can_compare) { + if ((deleted && last_deleted) || + (!last_deleted && !deleted && + last->size == size && + memcmp(last->data, data, size) == 0)) { + rle += repeat_count; + continue; + } + + WT_ERR(__rec_col_var_helper(session, + salvage, last, last_deleted, 0, rle)); + } + + /* + * Swap the current/last state. We can't always assign + * the data values to the buffer because they may come + * from a copy built based on an encoded cell. Check, + * because encoded cells aren't common and we'd like to + * avoid the copy. + */ + if (!deleted) { + if (data == orig.data) + WT_ERR(__wt_buf_set( + session, last, data, size)); + else { + last->data = data; + last->size = size; + } + } + last_deleted = deleted; + + /* Reset RLE counter and turn on comparisons. */ + rle = repeat_count; + can_compare = 1; + } + } + + /* Walk any append list. */ + append = WT_COL_APPEND(btree, page); + WT_SKIP_FOREACH(ins, append) + for (n = WT_INSERT_RECNO(ins); src_recno <= n; ++src_recno) { + /* + * The application may have inserted records which left + * gaps in the name space. + */ + if (src_recno < n) + deleted = 1; + else { + upd = ins->upd; + deleted = WT_UPDATE_DELETED_ISSET(upd); + if (!deleted) { + data = WT_UPDATE_DATA(upd); + size = upd->size; + } + } + + /* + * Handle RLE accounting and comparisons -- see comment + * above, this code fragment does the same thing. + */ + if (can_compare) { + if ((deleted && last_deleted) || + (!last_deleted && !deleted && + last->size == size && + memcmp(last->data, data, size) == 0)) { + ++rle; + continue; + } + + WT_ERR(__rec_col_var_helper(session, + salvage, last, last_deleted, 0, rle)); + } + + /* + * Swap the current/last state. We always assign the + * data values to the buffer because they can only be + * the data from a WT_UPDATE structure. + */ + if (!deleted) { + last->data = data; + last->size = size; + } + last_deleted = deleted; + + /* Reset RLE counter and turn on comparisons. */ + rle = 1; + can_compare = 1; + } + + /* If we were tracking a record, write it. */ + if (can_compare) + WT_ERR(__rec_col_var_helper( + session, salvage, last, last_deleted, 0, rle)); + + /* Write the remnant page. */ + ret = __rec_split_finish(session); + +err: __wt_buf_free(session, &orig); + return (ret); +} + +/* + * __rec_row_int -- + * Reconcile a row-store internal page. + */ +static int +__rec_row_int(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_CELL *cell; + WT_CELL_UNPACK *unpack, _unpack; + WT_IKEY *ikey; + WT_KV *key, *val; + WT_PAGE *rp; + WT_RECONCILE *r; + WT_REF *ref; + uint32_t i; + int onpage_ovfl, ovfl_key, val_set; + + r = session->reconcile; + unpack = &_unpack; + key = &r->k; + val = &r->v; + + WT_RET(__rec_split_init(session, + page, 0ULL, session->btree->maxintlpage)); + + /* + * Ideally, we'd never store the 0th key on row-store internal pages + * because it's never used during tree search and there's no reason + * to waste the space. The problem is how we do splits: when we split, + * we've potentially picked out several "split points" in the buffer + * which is overflowing the maximum page size, and when the overflow + * happens, we go back and physically split the buffer, at those split + * points, into new pages. It would be both difficult and expensive + * to re-process the 0th key at each split point to be an empty key, + * so we don't do that. However, we are reconciling an internal page + * for whatever reason, and the 0th key is known to be useless. We + * truncate the key to a single byte, instead of removing it entirely, + * it simplifies various things in other parts of the code (we don't + * have to special case transforming the page from its disk image to + * its in-memory version, for example). + */ + r->cell_zero = 1; + + /* For each entry in the in-memory page... */ + WT_REF_FOREACH(page, ref, i) { + /* + * Keys are always instantiated for row-store internal pages, + * set the WT_IKEY reference, and unpack the cell if the key + * references one. + */ + ikey = ref->u.key; + if (ikey->cell_offset == 0) { + cell = NULL; + /* + * We need to know if we're using on-page overflow cell + * in a few places below, initialize the unpacked cell's + * overflow value so there's an easy test. + */ + onpage_ovfl = 0; + } else { + cell = WT_PAGE_REF_OFFSET(page, ikey->cell_offset); + __wt_cell_unpack(cell, unpack); + onpage_ovfl = unpack->ovfl; + } + + /* + * The page may be deleted or internally created during a split. + * Deleted/split pages are merged into the parent and discarded. + * + * There's one special case we have to handle here: the internal + * page being merged has a potentially incorrect first key and + * we need to replace it with the one we have. The problem is + * caused by the fact that the page search algorithm coerces the + * 0th key on any internal page to be smaller than any search + * key. We do that because we don't want to have to update the + * internal pages every time a new "smallest" key is inserted + * into the tree. But, if a new "smallest" key is inserted into + * our split-created subtree, and we don't update the internal + * page, when we merge that internal page into its parent page, + * the key may be incorrect (or more likely, have been coerced + * to a single byte because it's an internal page's 0th key. + * Imagine the following tree: + * + * 2 5 40 internal page + * | + * 10 | 20 split-created internal page + * | + * 6 inserted smallest key + * + * after a simple merge, we'd have corruption: + * + * 2 10 20 40 merged internal page + * | + * 6 key sorts before parent's key + * + * To fix this problem, we take the higher-level page's key as + * our first key, because that key sorts before any possible + * key inserted into the subtree, and discard whatever 0th key + * is on the split-created internal page. + */ + val_set = 0; + if (ref->state != WT_REF_DISK) { + rp = ref->page; + switch (F_ISSET(rp, WT_PAGE_REC_MASK)) { + case WT_PAGE_REC_EMPTY: + /* + * Overflow keys referencing discarded pages are + * no longer useful. + */ + if (onpage_ovfl) + WT_RET(__rec_track_cell( + session, page, unpack)); + continue; + case WT_PAGE_REC_REPLACE: + __rec_cell_build_addr(session, + rp->modify->u.replace.addr, + rp->modify->u.replace.size, 0); + val_set = 1; + break; + case WT_PAGE_REC_SPLIT: + case WT_PAGE_REC_SPLIT_MERGE: + /* + * Overflow keys referencing split pages are no + * no longer useful (the interesting key is the + * key for the split page). + */ + if (onpage_ovfl) + WT_RET(__rec_track_cell( + session, page, unpack)); + + r->merge_ref = ref; + WT_RET(__rec_row_merge(session, + F_ISSET(rp, WT_PAGE_REC_MASK) == + WT_PAGE_REC_SPLIT_MERGE ? + rp : rp->modify->u.split)); + continue; + } + } + + /* + * Build key cell. + * + * If the key is an overflow item, assume prefix compression + * won't make things better, and simply copy it. + * + * XXX + * We have the key in-hand (we instantiate all internal page + * keys when the page is brought into memory), so it would be + * easy to check prefix compression, I'm just not bothering. + * If we did gain by prefix compression, we'd have to discard + * the old overflow key and write a new one, and this isn't a + * likely path anyway. + * + * Truncate any 0th key, internal pages don't need 0th keys. + */ + if (onpage_ovfl) { + key->buf.data = cell; + key->buf.size = unpack->len; + key->cell_len = 0; + key->len = key->buf.size; + ovfl_key = 1; + } else + WT_RET(__rec_cell_build_key(session, + WT_IKEY_DATA(ikey), + r->cell_zero ? 1 : ikey->size, 1, &ovfl_key)); + r->cell_zero = 0; + + /* + * Build the value cell. The child page address is in one of 3 + * places: if the page was replaced, the page's modify structure + * references it and we built the value cell just above in the + * switch statement. Else, the WT_REF->addr reference points to + * an on-page cell or an off-page WT_ADDR structure: if it's an + * on-page cell we copy it from the page, else build a new cell. + */ + if (!val_set) { + if (__wt_off_page(page, ref->addr)) + __rec_cell_build_addr(session, + ((WT_ADDR *)ref->addr)->addr, + ((WT_ADDR *)ref->addr)->size, 0); + else { + __wt_cell_unpack(ref->addr, unpack); + val->buf.data = ref->addr; + val->buf.size = unpack->len; + val->cell_len = 0; + val->len = unpack->len; + } + } + + /* + * Boundary, split or write the page. If the K/V pair doesn't + * fit: split the page, turn off compression (until a full key + * is written to the page), change to a non-prefix-compressed + * key. + */ + while (key->len + val->len > r->space_avail) { + /* + * We have to have a copy of any overflow key because + * we're about to promote it. + */ + if (ovfl_key && onpage_ovfl) + WT_RET(__wt_cell_copy(session, cell, r->cur)); + WT_RET(__rec_split(session)); + + r->key_pfx_compress = 0; + if (!ovfl_key) + WT_RET(__rec_cell_build_key( + session, NULL, 0, 1, &ovfl_key)); + } + + /* Copy the key and value onto the page. */ + __rec_copy_incr(session, r, key); + __rec_copy_incr(session, r, val); + + /* Update compression state. */ + __rec_key_state_update(r, ovfl_key); + } + + /* Write the remnant page. */ + return (__rec_split_finish(session)); +} + +/* + * __rec_row_merge -- + * Recursively walk a row-store internal tree of merge pages. + */ +static int +__rec_row_merge(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_CELL_UNPACK *unpack, _unpack; + WT_IKEY *ikey; + WT_KV *key, *val; + WT_PAGE *rp; + WT_RECONCILE *r; + WT_REF *ref; + uint32_t i; + int ovfl_key, val_set; + + WT_BSTAT_INCR(session, rec_page_merge); + + r = session->reconcile; + unpack = &_unpack; + key = &r->k; + val = &r->v; + + /* For each entry in the in-memory page... */ + WT_REF_FOREACH(page, ref, i) { + /* + * The page may be deleted or internally created during a split. + * Deleted/split pages are merged into the parent and discarded. + */ + val_set = 0; + if (ref->state != WT_REF_DISK) { + rp = ref->page; + switch (F_ISSET(rp, WT_PAGE_REC_MASK)) { + case WT_PAGE_REC_EMPTY: + continue; + case WT_PAGE_REC_REPLACE: + __rec_cell_build_addr(session, + rp->modify->u.replace.addr, + rp->modify->u.replace.size, 0); + val_set = 1; + break; + case WT_PAGE_REC_SPLIT: + case WT_PAGE_REC_SPLIT_MERGE: + /* + * If we have a merge key set, we're working our + * way down a merge tree. If we have not set a + * merge key, we're starting descent of a new + * merge tree, set the merge key. + */ + if (r->merge_ref == NULL) + r->merge_ref = ref; + WT_RET(__rec_row_merge(session, + F_ISSET(rp, WT_PAGE_REC_MASK) == + WT_PAGE_REC_SPLIT_MERGE ? + rp : rp->modify->u.split)); + continue; + } + } + + /* + * Build the key cell. If this is the first key in a "to be + * merged" subtree, use the merge correction key saved in the + * top-level parent page when this function was called. + * + * Truncate any 0th key, internal pages don't need 0th keys. + */ + ikey = r->merge_ref == NULL ? ref->u.key : r->merge_ref->u.key; + r->merge_ref = NULL; + WT_RET(__rec_cell_build_key(session, WT_IKEY_DATA(ikey), + r->cell_zero ? 1 : ikey->size, 1, &ovfl_key)); + r->cell_zero = 0; + + /* + * Build the value cell. The child page address is in one of 3 + * places: if the page was replaced, the page's modify structure + * references it and we built the value cell just above in the + * switch statement. Else, the WT_REF->addr reference points to + * an on-page cell or an off-page WT_ADDR structure: if it's an + * on-page cell we copy it from the page, else build a new cell. + */ + if (!val_set) { + if (__wt_off_page(page, ref->addr)) + __rec_cell_build_addr(session, + ((WT_ADDR *)ref->addr)->addr, + ((WT_ADDR *)ref->addr)->size, 0); + else { + __wt_cell_unpack(ref->addr, unpack); + val->buf.data = ref->addr; + val->buf.size = unpack->len; + val->cell_len = 0; + val->len = unpack->len; + } + } + + /* + * Boundary, split or write the page. If the K/V pair doesn't + * fit: split the page, turn off compression (until a full key + * is written to the page), change to a non-prefix-compressed + * key. + */ + while (key->len + val->len > r->space_avail) { + WT_RET(__rec_split(session)); + + r->key_pfx_compress = 0; + if (!ovfl_key) + WT_RET(__rec_cell_build_key( + session, NULL, 0, 1, &ovfl_key)); + } + + /* Copy the key and value onto the page. */ + __rec_copy_incr(session, r, key); + __rec_copy_incr(session, r, val); + + /* Update compression state. */ + __rec_key_state_update(r, ovfl_key); + } + + return (0); +} + +/* + * __rec_row_leaf -- + * Reconcile a row-store leaf page. + */ +static int +__rec_row_leaf( + WT_SESSION_IMPL *session, WT_PAGE *page, WT_SALVAGE_COOKIE *salvage) +{ + WT_CELL *cell, *val_cell; + WT_CELL_UNPACK *unpack, _unpack; + WT_IKEY *ikey; + WT_INSERT *ins; + WT_ITEM *tmpkey; + WT_KV *key, *val; + WT_RECONCILE *r; + WT_ROW *rip; + WT_UPDATE *upd; + uint64_t slvg_skip; + uint32_t i; + int ovfl_key, ret; + + r = session->reconcile; + tmpkey = NULL; + unpack = &_unpack; + slvg_skip = salvage == NULL ? 0 : salvage->skip; + ret = 0; + + key = &r->k; + val = &r->v; + + WT_RET(__rec_split_init(session, + page, 0ULL, session->btree->maxleafpage)); + + /* + * Write any K/V pairs inserted into the page before the first from-disk + * key on the page. + */ + if ((ins = WT_SKIP_FIRST(WT_ROW_INSERT_SMALLEST(page))) != NULL) + WT_RET(__rec_row_leaf_insert(session, ins)); + + /* + * A temporary buffer in which to instantiate any uninstantiated keys. + * From this point on, we need to jump to the err label on error so the + * buffer is discarded. + */ + WT_RET(__wt_scr_alloc(session, 0, &tmpkey)); + + /* For each entry in the page... */ + WT_ROW_FOREACH(page, rip, i) { + /* + * The salvage code, on some rare occasions, wants to reconcile + * a page but skip some leading records on the page. Because + * the row-store leaf reconciliation function copies keys from + * the original disk page, this is non-trivial -- just changing + * the in-memory pointers isn't sufficient, we have to change + * the WT_CELL structures on the disk page, too. It's ugly, but + * we pass in a value that tells us how many records to skip in + * this case. + */ + if (slvg_skip != 0) { + --slvg_skip; + continue; + } + + /* + * Set the WT_IKEY reference (if the key was instantiated), and + * the key cell reference. + */ + if (__wt_off_page(page, rip->key)) { + ikey = rip->key; + cell = WT_PAGE_REF_OFFSET(page, ikey->cell_offset); + } else { + ikey = NULL; + cell = rip->key; + } + + /* Build value cell. */ + if ((val_cell = __wt_row_value(page, rip)) != NULL) + __wt_cell_unpack(val_cell, unpack); + if ((upd = WT_ROW_UPDATE(page, rip)) == NULL) { + /* + * Copy the item off the page -- however, when the page + * was read into memory, there may not have been a value + * item, that is, it may have been zero length. + */ + if (val_cell == NULL) + val->buf.size = 0; + else { + val->buf.data = val_cell; + val->buf.size = unpack->len; + } + val->cell_len = 0; + val->len = val->buf.size; + } else { + /* + * If we updated an overflow value, free the underlying + * file space. + */ + if (val_cell != NULL) + WT_ERR(__rec_track_cell(session, page, unpack)); + + /* + * If this key/value pair was deleted, we're done. If + * we deleted an overflow key, free the underlying file + * space. + */ + if (WT_UPDATE_DELETED_ISSET(upd)) { + __wt_cell_unpack(cell, unpack); + WT_ERR(__rec_track_cell(session, page, unpack)); + + /* + * We skip creating the key, don't try to use + * the last valid key in prefix calculations. + */ + tmpkey->size = 0; + + goto leaf_insert; + } + + /* + * If no value, nothing needs to be copied. Otherwise, + * build the value's WT_CELL chunk from the most recent + * update value. + */ + if (upd->size == 0) + val->cell_len = val->len = val->buf.size = 0; + else + WT_ERR(__rec_cell_build_val( + session, WT_UPDATE_DATA(upd), + upd->size, (uint64_t)0)); + } + + /* + * Build key cell. + */ + __wt_cell_unpack(cell, unpack); + if (unpack->type == WT_CELL_KEY_OVFL) { + /* + * If the key is an overflow item, assume prefix + * compression won't make things better, and copy it. + */ + key->buf.data = cell; + key->buf.size = unpack->len; + key->cell_len = 0; + key->len = key->buf.size; + ovfl_key = 1; + + /* Don't try to use a prefix across an overflow key. */ + tmpkey->size = 0; + } else { + /* + * If the key is already instantiated, use it. + * Else, if the key is available from the page, use it. + * Else, if we can construct the key from a previous + * key, do so. + * Else, instantiate the key. + */ + if (ikey != NULL) { + tmpkey->data = WT_IKEY_DATA(ikey); + tmpkey->size = ikey->size; + } else if (session->btree->huffman_key == NULL && + unpack->type == WT_CELL_KEY && + unpack->prefix == 0) { + tmpkey->data = unpack->data; + tmpkey->size = unpack->size; + } else if (session->btree->huffman_key == NULL && + unpack->type == WT_CELL_KEY && + tmpkey->size >= unpack->prefix) { + /* + * If we previously built a prefix-compressed + * key in the temporary buffer, WT_ITEM->data + * will be the same as WT_ITEM->mem: grow the + * buffer if necessary and copy the suffix into + * place. + * + * If we previously pointed the temporary buffer + * at an on-page key, WT_ITEM->data will not be + * the same as WT_ITEM->mem: grow the buffer if + * necessary, copy the prefix into place, then + * re-point the WT_ITEM->data field to the newly + * constructed memory, and then copy the suffix + * into place. + */ + WT_ERR(__wt_buf_grow(session, + tmpkey, unpack->prefix + unpack->size)); + if (tmpkey->data != tmpkey->mem) { + memcpy(tmpkey->mem, tmpkey->data, + unpack->prefix); + tmpkey->data = tmpkey->mem; + } + memcpy((uint8_t *)tmpkey->data + unpack->prefix, + unpack->data, unpack->size); + tmpkey->size = unpack->prefix + unpack->size; + } else + WT_ERR( + __wt_row_key(session, page, rip, tmpkey)); + + WT_ERR(__rec_cell_build_key( + session, tmpkey->data, tmpkey->size, 0, &ovfl_key)); + } + + /* + * Boundary, split or write the page. If the K/V pair doesn't + * fit: split the page, switch to the non-prefix-compressed key + * and turn off compression until a full key is written to the + * new page. + * + * We write a trailing key cell on the page after the K/V pairs + * (see WT_TRAILING_KEY_CELL for more information). + */ + while (key->len + + val->len + WT_TRAILING_KEY_CELL > r->space_avail) { + /* + * We have to have a copy of any overflow key because + * we're about to promote it. + */ + if (ovfl_key && unpack->type == WT_CELL_KEY_OVFL) + WT_RET(__wt_cell_unpack_copy( + session, unpack, r->cur)); + WT_ERR(__rec_split(session)); + + r->key_pfx_compress = 0; + if (!ovfl_key) + WT_ERR(__rec_cell_build_key( + session, NULL, 0, 0, &ovfl_key)); + } + + /* Copy the key/value pair onto the page. */ + __rec_copy_incr(session, r, key); + if (val->len != 0) + __rec_copy_incr(session, r, val); + + /* Update compression state. */ + __rec_key_state_update(r, ovfl_key); + +leaf_insert: /* Write any K/V pairs inserted into the page after this key. */ + if ((ins = WT_SKIP_FIRST(WT_ROW_INSERT(page, rip))) != NULL) + WT_ERR(__rec_row_leaf_insert(session, ins)); + } + + /* Write the remnant page. */ + ret = __rec_split_finish(session); + +err: __wt_scr_free(&tmpkey); + return (ret); +} + +/* + * __rec_row_leaf_insert -- + * Walk an insert chain, writing K/V pairs. + */ +static int +__rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_INSERT *ins) +{ + WT_KV *key, *val; + WT_RECONCILE *r; + WT_UPDATE *upd; + int ovfl_key; + + r = session->reconcile; + key = &r->k; + val = &r->v; + + for (; ins != NULL; ins = WT_SKIP_NEXT(ins)) { + upd = ins->upd; /* Build value cell. */ + if (WT_UPDATE_DELETED_ISSET(upd)) + continue; + if (upd->size == 0) + val->len = 0; + else + WT_RET(__rec_cell_build_val(session, + WT_UPDATE_DATA(upd), upd->size, (uint64_t)0)); + + WT_RET(__rec_cell_build_key(session, /* Build key cell. */ + WT_INSERT_KEY(ins), WT_INSERT_KEY_SIZE(ins), 0, &ovfl_key)); + + /* + * Boundary, split or write the page. If the K/V pair doesn't + * fit: split the page, switch to the non-prefix-compressed key + * and turn off compression until a full key is written to the + * new page. + * + * We write a trailing key cell on the page after the K/V pairs + * (see WT_TRAILING_KEY_CELL for more information). + */ + while (key->len + + val->len + WT_TRAILING_KEY_CELL > r->space_avail) { + WT_RET(__rec_split(session)); + + r->key_pfx_compress = 0; + if (!ovfl_key) + WT_RET(__rec_cell_build_key( + session, NULL, 0, 0, &ovfl_key)); + } + + /* Copy the key/value pair onto the page. */ + __rec_copy_incr(session, r, key); + if (val->len != 0) + __rec_copy_incr(session, r, val); + + /* Update compression state. */ + __rec_key_state_update(r, ovfl_key); + } + + return (0); +} + +/* + * __rec_write_wrapup -- + * Finish the reconciliation. + */ +static int +__rec_write_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_BOUNDARY *bnd; + WT_PAGE_MODIFY *mod; + WT_RECONCILE *r; + WT_REF *ref; + uint32_t i, size; + int ret; + const uint8_t *addr; + + r = session->reconcile; + mod = page->modify; + ret = 0; + + /* + * This page may have previously been reconciled, and that information + * is now about to be replaced. Make sure it's discarded at some point, + * and clear the underlying modification information, we're creating a + * new reality. + */ + switch (F_ISSET(page, WT_PAGE_REC_MASK)) { + case 0: /* + * The page has never been reconciled before, track the original + * address blocks (if any). If we're splitting the root page + * we may schedule the same blocks to be freed repeatedly: that + * is OK, the track function checks for duplicates. + */ + if (!WT_PAGE_IS_ROOT(page) && page->ref->addr != NULL) { + __wt_get_addr(page->parent, page->ref, &addr, &size); + WT_RET(__wt_rec_track_block( + session, WT_PT_BLOCK, page, addr, size)); + } + break; + case WT_PAGE_REC_EMPTY: /* Page deleted */ + break; + case WT_PAGE_REC_REPLACE: /* 1-for-1 page swap */ + /* Discard the replacement leaf page's blocks. */ + WT_RET(__wt_rec_track_block(session, WT_PT_BLOCK, + page, mod->u.replace.addr, mod->u.replace.size)); + + /* Discard the replacement page's address. */ + __wt_free(session, mod->u.replace.addr); + mod->u.replace.addr = NULL; + mod->u.replace.size = 0; + break; + case WT_PAGE_REC_SPLIT: /* Page split */ + /* Discard the split page's leaf-page blocks. */ + WT_REF_FOREACH(mod->u.split, ref, i) + WT_RET(__wt_rec_track_block( + session, WT_PT_BLOCK, page, + ((WT_ADDR *)ref->addr)->addr, + ((WT_ADDR *)ref->addr)->size)); + + /* Discard the split page itself. */ + __wt_page_out(session, mod->u.split, 0); + mod->u.split = NULL; + break; + case WT_PAGE_REC_SPLIT_MERGE: /* Page split */ + /* + * We should never be here with a split-merge page: you cannot + * reconcile split-merge pages, they can only be merged into a + * parent. + */ + /* FALLTHROUGH */ + WT_ILLEGAL_VALUE(session); + } + F_CLR(page, WT_PAGE_REC_MASK); + + switch (r->bnd_next) { + case 0: /* Page delete */ + WT_VERBOSE(session, reconcile, "page %p empty", page); + + WT_BSTAT_INCR(session, rec_page_delete); + + /* + * If the page was empty, we want to discard it from the tree + * by discarding the parent's key when evicting the parent. + * Mark the page as deleted, then return success, leaving the + * page in memory. If the page is subsequently modified, that + * is OK, we'll just reconcile it again. + */ + F_SET(page, WT_PAGE_REC_EMPTY); + break; + case 1: /* 1-for-1 page swap */ + /* + * Because WiredTiger's pages grow without splitting, we're + * replacing a single page with another single page most of + * the time. + */ + bnd = &r->bnd[0]; +#ifdef HAVE_VERBOSE + if (WT_VERBOSE_ISSET(session, reconcile)) { + WT_ITEM *buf; + WT_RET(__wt_scr_alloc(session, 64, &buf)); + WT_VERBOSE(session, reconcile, "page %p written to %s", + page, __wt_addr_string( + session, buf, bnd->addr.addr, bnd->addr.size)); + __wt_scr_free(&buf); + } +#endif + mod->u.replace = bnd->addr; + bnd->addr.addr = NULL; + + F_SET(page, WT_PAGE_REC_REPLACE); + break; + default: /* Page split */ + WT_VERBOSE(session, reconcile, + "page %p split into %" PRIu32 " pages", + page, r->bnd_next); + + switch (page->type) { + case WT_PAGE_COL_INT: + case WT_PAGE_ROW_INT: + WT_BSTAT_INCR(session, rec_split_intl); + break; + case WT_PAGE_COL_FIX: + case WT_PAGE_COL_VAR: + case WT_PAGE_ROW_LEAF: + WT_BSTAT_INCR(session, rec_split_leaf); + break; + WT_ILLEGAL_VALUE(session); + } + +#ifdef HAVE_VERBOSE + if (WT_VERBOSE_ISSET(session, reconcile)) { + WT_ITEM *tkey; + if (page->type == WT_PAGE_ROW_INT || + page->type == WT_PAGE_ROW_LEAF) + WT_RET(__wt_scr_alloc(session, 0, &tkey)); + for (bnd = r->bnd, i = 0; i < r->bnd_next; ++bnd, ++i) + switch (page->type) { + case WT_PAGE_ROW_INT: + case WT_PAGE_ROW_LEAF: + WT_ERR(__wt_buf_set_printable( + session, tkey, + bnd->key.data, bnd->key.size)); + WT_VERBOSE(session, reconcile, + "split: starting key " + "%.*s", + (int)tkey->size, + (char *)tkey->data); + break; + case WT_PAGE_COL_FIX: + case WT_PAGE_COL_INT: + case WT_PAGE_COL_VAR: + WT_VERBOSE(session, reconcile, + "split: starting recno %" PRIu64, + bnd->recno); + break; + WT_ILLEGAL_VALUE(session); + } +err: if (page->type == WT_PAGE_ROW_INT || + page->type == WT_PAGE_ROW_LEAF) + __wt_scr_free(&tkey); + } +#endif + switch (page->type) { + case WT_PAGE_ROW_INT: + case WT_PAGE_ROW_LEAF: + WT_RET(__rec_split_row(session, page, &mod->u.split)); + break; + case WT_PAGE_COL_INT: + case WT_PAGE_COL_FIX: + case WT_PAGE_COL_VAR: + WT_RET(__rec_split_col(session, page, &mod->u.split)); + break; + WT_ILLEGAL_VALUE(session); + } + + F_SET(page, WT_PAGE_REC_SPLIT); + break; + } + + return (ret); +} + +/* + * __rec_split_row -- + * Split a row-store page, creating a new internal page. + */ +static int +__rec_split_row(WT_SESSION_IMPL *session, WT_PAGE *orig, WT_PAGE **splitp) +{ + WT_BOUNDARY *bnd; + WT_PAGE *page; + WT_RECONCILE *r; + WT_REF *ref; + uint32_t i; + int ret; + + r = session->reconcile; + ret = 0; + + /* Allocate a row-store internal page. */ + WT_RET(__wt_calloc_def(session, 1, &page)); + WT_ERR(__wt_calloc_def(session, (size_t)r->bnd_next, &page->u.intl.t)); + + /* Fill it in. */ + page->parent = orig->parent; + page->ref = orig->ref; + page->read_gen = __wt_cache_read_gen(session); + page->entries = r->bnd_next; + page->type = WT_PAGE_ROW_INT; + + /* + * we don't re-write parent pages when child pages split, which means + * we have only one slot to work with in the parent. When a leaf page + * splits, we create a new internal page referencing the split pages, + * and when the leaf page is evicted, we update the leaf's slot in the + * parent to reference the new internal page in the tree (deepening the + * tree by a level). We don't want the tree to deepen permanently, so + * we never write that new internal page to disk, we only merge it into + * the parent when the parent page is evicted. + * + * We set one flag (WT_PAGE_REC_SPLIT) on the original page so future + * reconciliations of its parent merge in the newly created split page. + * We set a different flag (WT_PAGE_REC_SPLIT_MERGE) on the created + * split page so after we evict the original page and replace it with + * the split page, the parent continues to merge in the split page. + * The flags are different because the original page can be evicted and + * its memory discarded, but the newly created split page cannot be + * evicted, it can only be merged into its parent. + */ + F_SET(page, WT_PAGE_REC_SPLIT_MERGE); + + /* Enter each split page into the new, internal page. */ + for (ref = page->u.intl.t, + bnd = r->bnd, i = 0; i < r->bnd_next; ++ref, ++bnd, ++i) { + WT_ERR(__wt_row_ikey_alloc(session, 0, + bnd->key.data, bnd->key.size, (WT_IKEY **)&ref->u.key)); + WT_ERR(__wt_calloc(session, 1, sizeof(WT_ADDR), &ref->addr)); + ((WT_ADDR *)ref->addr)->addr = bnd->addr.addr; + ((WT_ADDR *)ref->addr)->size = bnd->addr.size; + bnd->addr.addr = NULL; + + ref->page = NULL; + ref->state = WT_REF_DISK; + } + + *splitp = page; + return (0); + +err: __wt_page_out(session, page, 0); + return (ret); +} + +/* + * __rec_split_col -- + * Split a column-store page, creating a new internal page. + */ +static int +__rec_split_col(WT_SESSION_IMPL *session, WT_PAGE *orig, WT_PAGE **splitp) +{ + WT_BOUNDARY *bnd; + WT_PAGE *page; + WT_RECONCILE *r; + WT_REF *ref; + uint32_t i; + int ret; + + r = session->reconcile; + ret = 0; + + /* Allocate a column-store internal page. */ + WT_RET(__wt_calloc_def(session, 1, &page)); + WT_ERR(__wt_calloc_def(session, (size_t)r->bnd_next, &page->u.intl.t)); + + /* Fill it in. */ + page->parent = orig->parent; + page->ref = orig->ref; + page->read_gen = __wt_cache_read_gen(session); + page->u.intl.recno = r->bnd[0].recno; + page->entries = r->bnd_next; + page->type = WT_PAGE_COL_INT; + + /* + * See the comment above in __rec_split_row(). + */ + F_SET(page, WT_PAGE_REC_SPLIT_MERGE); + + /* Enter each split page into the new, internal page. */ + for (ref = page->u.intl.t, + bnd = r->bnd, i = 0; i < r->bnd_next; ++ref, ++bnd, ++i) { + ref->u.recno = bnd->recno; + WT_ERR(__wt_calloc(session, 1, sizeof(WT_ADDR), &ref->addr)); + ((WT_ADDR *)ref->addr)->addr = bnd->addr.addr; + ((WT_ADDR *)ref->addr)->size = bnd->addr.size; + bnd->addr.addr = NULL; + + ref->page = NULL; + ref->state = WT_REF_DISK; + } + + *splitp = page; + return (0); + +err: __wt_page_out(session, page, 0); + return (ret); +} + +/* + * __rec_cell_build_key -- + * Process a key and return a WT_CELL structure and byte string to be + * stored on the page. + */ +static int +__rec_cell_build_key(WT_SESSION_IMPL *session, + const void *data, uint32_t size, int is_internal, int *is_ovflp) +{ + WT_BTREE *btree; + WT_KV *key; + WT_RECONCILE *r; + uint32_t pfx_max; + uint8_t pfx; + const uint8_t *a, *b; + + r = session->reconcile; + btree = session->btree; + key = &r->k; + *is_ovflp = 0; + + pfx = 0; + if (data == NULL) + /* + * When data is NULL, our caller has a prefix compressed key + * they can't use (probably because they just crossed a split + * point). Use the full key saved when last called, instead. + */ + WT_RET(__wt_buf_set( + session, &key->buf, r->cur->data, r->cur->size)); + else { + /* + * Save a copy of the key for later reference: we use the full + * key for prefix-compression comparisons, and if we are, for + * any reason, unable to use the compressed key we generate. + */ + WT_RET(__wt_buf_set(session, r->cur, data, size)); + + /* + * Do prefix compression on the key. We know by definition the + * previous key sorts before the current key, which means the + * keys must differ and we just need to compare up to the + * shorter of the two keys. Also, we can't compress out more + * than 256 bytes, limit the comparison to that. + */ + if (r->key_pfx_compress) { + pfx_max = UINT8_MAX; + if (size < pfx_max) + pfx_max = size; + if (r->last->size < pfx_max) + pfx_max = r->last->size; + for (a = data, b = r->last->data; pfx < pfx_max; ++pfx) + if (*a++ != *b++) + break; + } + + /* Copy the non-prefix bytes into the key buffer. */ + WT_RET(__wt_buf_set( + session, &key->buf, (uint8_t *)data + pfx, size - pfx)); + } + + /* Optionally compress the value using the Huffman engine. */ + if (btree->huffman_key != NULL) + WT_RET(__wt_huffman_encode(session, btree->huffman_key, + key->buf.data, key->buf.size, &key->buf)); + + /* Create an overflow object if the data won't fit. */ + if (key->buf.size > + (is_internal ? btree->maxintlitem : btree->maxleafitem)) { + WT_BSTAT_INCR(session, rec_ovfl_key); + + /* + * Overflow objects aren't prefix compressed -- rebuild any + * object that was prefix compressed. + */ + if (pfx == 0) { + *is_ovflp = 1; + return (__rec_cell_build_ovfl( + session, key, WT_CELL_KEY_OVFL, (uint64_t)0)); + } + return (__rec_cell_build_key( + session, NULL, 0, is_internal, is_ovflp)); + } + + key->cell_len = __wt_cell_pack_key(&key->cell, pfx, key->buf.size); + key->len = key->cell_len + key->buf.size; + + return (0); +} + +/* + * __rec_cell_build_addr -- + * Process an address reference and return a WT_CELL structure to be stored + * on the page. + */ +static void +__rec_cell_build_addr( + WT_SESSION_IMPL *session, const void *data, uint32_t size, uint64_t recno) +{ + WT_KV *val; + WT_RECONCILE *r; + + r = session->reconcile; + val = &r->v; + + /* + * We don't copy the data into the buffer, it's not necessary; just + * re-point the buffer's data/length fields. + */ + val->buf.data = data; + val->buf.size = size; + val->cell_len = __wt_cell_pack_addr(&val->cell, recno, val->buf.size); + val->len = val->cell_len + val->buf.size; +} + +/* + * __rec_cell_build_val -- + * Process a data item and return a WT_CELL structure and byte string to + * be stored on the page. + */ +static int +__rec_cell_build_val( + WT_SESSION_IMPL *session, const void *data, uint32_t size, uint64_t rle) +{ + WT_BTREE *btree; + WT_KV *val; + WT_RECONCILE *r; + + r = session->reconcile; + btree = session->btree; + val = &r->v; + + /* + * We don't copy the data into the buffer, it's not necessary; just + * re-point the buffer's data/length fields. + */ + val->buf.data = data; + val->buf.size = size; + + /* Handle zero-length cells quickly. */ + if (size != 0) { + /* Optionally compress the data using the Huffman engine. */ + if (btree->huffman_value != NULL) + WT_RET(__wt_huffman_encode( + session, btree->huffman_value, + val->buf.data, val->buf.size, &val->buf)); + + /* Create an overflow object if the data won't fit. */ + if (val->buf.size > btree->maxleafitem) { + WT_BSTAT_INCR(session, rec_ovfl_value); + + return (__rec_cell_build_ovfl( + session, val, WT_CELL_VALUE_OVFL, rle)); + } + } + val->cell_len = __wt_cell_pack_data(&val->cell, rle, val->buf.size); + val->len = val->cell_len + val->buf.size; + + return (0); +} + +/* + * __rec_cell_build_ovfl -- + * Store overflow items in the file, returning the address cookie. + */ +static int +__rec_cell_build_ovfl( + WT_SESSION_IMPL *session, WT_KV *kv, uint8_t type, uint64_t rle) +{ + WT_BTREE *btree; + WT_ITEM *tmp; + WT_PAGE *page; + WT_PAGE_HEADER *dsk; + WT_RECONCILE *r; + uint32_t size; + int ret; + uint8_t *addr, buf[WT_BM_MAX_ADDR_COOKIE]; + + r = session->reconcile; + btree = session->btree; + page = r->page; + tmp = NULL; + ret = 0; + + /* + * See if this overflow record has already been written and reuse it if + * possible. Else, write a new overflow record. + */ + if (!__wt_rec_track_ovfl_reuse( + session, page, kv->buf.data, kv->buf.size, &addr, &size)) { + /* Allocate a buffer big enough to write the overflow record. */ + size = kv->buf.size; + WT_RET(__wt_bm_write_size(session, &size)); + WT_RET(__wt_scr_alloc(session, size, &tmp)); + + /* Initialize the buffer: disk header and overflow record. */ + dsk = tmp->mem; + memset(dsk, 0, WT_PAGE_HEADER_SIZE); + dsk->type = WT_PAGE_OVFL; + dsk->u.datalen = kv->buf.size; + memcpy(WT_PAGE_HEADER_BYTE(btree, dsk), + kv->buf.data, kv->buf.size); + tmp->size = WT_PAGE_HEADER_BYTE_SIZE(btree) + kv->buf.size; + + /* Write the buffer. */ + addr = buf; + WT_ERR(__wt_bm_write(session, tmp, addr, &size)); + + /* Track the overflow record. */ + WT_ERR(__wt_rec_track_ovfl( + session, page, addr, size, kv->buf.data, kv->buf.size)); + } + + /* Set the callers K/V to reference the overflow record's address. */ + WT_ERR(__wt_buf_set(session, &kv->buf, addr, size)); + + /* Build the cell and return. */ + kv->cell_len = __wt_cell_pack_ovfl(&kv->cell, type, rle, kv->buf.size); + kv->len = kv->cell_len + kv->buf.size; + +err: __wt_scr_free(&tmp); + return (ret); +} diff --git a/src/btree/row_key.c b/src/btree/row_key.c new file mode 100644 index 00000000000..9cff462ed22 --- /dev/null +++ b/src/btree/row_key.c @@ -0,0 +1,438 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static void __inmem_row_leaf_slots(uint8_t *, uint32_t, uint32_t, uint32_t); + +/* + * __wt_row_leaf_keys -- + * Instantiate the interesting keys for random search of a page. + */ +int +__wt_row_leaf_keys(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_BTREE *btree; + WT_ITEM *tmp; + WT_ROW *rip; + uint32_t i; + int ret; + + btree = session->btree; + ret = 0; + + if (page->entries == 0) { /* Just checking... */ + F_SET(page, WT_PAGE_BUILD_KEYS); + return (0); + } + + /* + * Row-store leaf pages are written as one big prefix-compressed chunk, + * that is, only the first key on the page is not prefix-compressed, and + * to instantiate the last key on the page, you have to take the first + * key on the page and roll it forward to the end of the page. We don't + * want to do that on every page access, of course, so we instantiate a + * set of keys, essentially creating prefix chunks on the page, where we + * can roll forward from the closest, previous, instantiated key. The + * complication is that not all keys on a page are equal: we're doing a + * binary search on the page, which means there are keys we look at a + * lot (every time we search the page), and keys we never look at unless + * they are actually being searched for. This function figures out the + * "interesting" keys on a page, and then we sequentially walk that list + * instantiating those keys. + * + * Allocate a bit array and figure out the set of "interesting" keys, + * marking up the array. + */ + WT_RET(__wt_scr_alloc(session, __bitstr_size(page->entries), &tmp)); + + __inmem_row_leaf_slots(tmp->mem, 0, page->entries, btree->key_gap); + + /* Instantiate the keys. */ + for (rip = page->u.row.d, i = 0; i < page->entries; ++rip, ++i) + if (__bit_test(tmp->mem, i)) + WT_ERR(__wt_row_key(session, page, rip, NULL)); + + F_SET(page, WT_PAGE_BUILD_KEYS); + +err: __wt_scr_free(&tmp); + return (ret); +} + +/* + * __inmem_row_leaf_slots -- + * Figure out the interesting slots of a page for random search, up to + * the specified depth. + */ +static void +__inmem_row_leaf_slots( + uint8_t *list, uint32_t base, uint32_t entries, uint32_t gap) +{ + uint32_t indx, limit; + + if (entries < gap) + return; + + /* + * !!! + * Don't clean this code up -- it deliberately looks like the binary + * search code. + * + * !!! + * There's got to be a function that would give me this information, but + * I don't see any performance reason we can't just do this recursively. + */ + limit = entries; + indx = base + (limit >> 1); + __bit_set(list, indx); + + __inmem_row_leaf_slots(list, base, limit >> 1, gap); + + base = indx + 1; + --limit; + __inmem_row_leaf_slots(list, base, limit >> 1, gap); +} + +/* + * __wt_row_key -- + * Copy an on-page key into a return buffer, or, if no return buffer + * is specified, instantiate the key into the in-memory page. + */ +int +__wt_row_key( + WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW *rip_arg, WT_ITEM *retb) +{ + enum { FORWARD, BACKWARD } direction; + WT_CELL_UNPACK *unpack, _unpack; + WT_IKEY *ikey; + WT_ITEM *tmp; + WT_ROW *rip; + int is_local, ret, slot_offset; + void *key; + + rip = rip_arg; + tmp = NULL; + unpack = &_unpack; + + /* + * If the caller didn't pass us a buffer, create one. We don't use + * an existing buffer because the memory will be attached to a page + * for semi-permanent use, and using an existing buffer might waste + * memory if the one allocated from the pool was larger than needed. + */ + is_local = 0; + if (retb == NULL) { + is_local = 1; + WT_ERR(__wt_scr_alloc(session, 0, &retb)); + } + + direction = BACKWARD; + for (slot_offset = 0;;) { + /* + * Multiple threads of control may be searching this page, which + * means the key may change underfoot, and here's where it gets + * tricky: first, copy the key. We don't need any barriers, the + * key is updated atomically, and we just need a valid copy. + */ + key = rip->key; + + /* + * Key copied. + * + * If another thread instantiated the key while we were doing + * that, we don't have any work to do. Figure this out using + * the key's value: + * + * If the key points off-page, another thread updated the key, + * we can just use it. + * + * If the key points on-page, we have a copy of a WT_CELL value + * that can be processed, regardless of what any other thread is + * doing. + * + * Overflow keys are not prefix-compressed, we don't want to + * read/write them during reconciliation simply because their + * prefix might change. That means we can't use instantiated + * overflow keys to figure out the prefix for other keys, + * specifically, in this code when we're looking for a key we + * can roll-forward to figure out the target key's prefix, + * instantiated overflow keys aren't useful. + * + * 1: the test for an on/off page reference. + */ + if (__wt_off_page(page, key)) { + ikey = key; + + /* + * If this is the key we originally wanted, we don't + * care if we're rolling forward or backward, or if + * it's an overflow key or not, it's what we wanted. + * Take a copy and wrap up. + */ + if (slot_offset == 0) { + WT_ERR(__wt_buf_set(session, + retb, WT_IKEY_DATA(ikey), ikey->size)); + break; + } + + __wt_cell_unpack(WT_PAGE_REF_OFFSET( + page, ikey->cell_offset), unpack); + + /* + * If we wanted a different key and this key is an + * overflow key: + * If we're rolling backward, this key is useless + * to us because it doesn't have a valid prefix: keep + * rolling backward. + * If we're rolling forward, there's no work to be + * done because prefixes skip overflow keys: keep + * rolling forward. + */ + if (unpack->ovfl) + goto next; + + /* + * If we wanted a different key and this key is not an + * overflow key, it has a valid prefix, we can use it. + * If rolling backward, take a copy of the key and + * switch directions, we can roll forward from this key. + * If rolling forward, replace the key we've been + * building with this key, it's what we would have built + * anyway. + * In short: if it's not an overflow key, take a copy + * and roll forward. + */ + WT_ERR(__wt_buf_set( + session, retb, WT_IKEY_DATA(ikey), ikey->size)); + direction = FORWARD; + goto next; + } + + /* Unpack the key's cell. */ + __wt_cell_unpack(key, unpack); + + /* 2: the test for an on-page reference to an overflow key. */ + if (unpack->type == WT_CELL_KEY_OVFL) { + /* + * If this is the key we wanted from the start, we don't + * care if it's an overflow key, get a copy and wrap up. + */ + if (slot_offset == 0) { + WT_ERR(__wt_cell_unpack_copy( + session, unpack, retb)); + break; + } + + /* + * If we wanted a different key and this key is an + * overflow key: + * If we're rolling backward, this key is useless + * to us because it doesn't have a valid prefix: keep + * rolling backward. + * If we're rolling forward, there's no work to be + * done because prefixes skip overflow keys: keep + * rolling forward. + */ + goto next; + } + + /* + * 3: the test for an on-page reference to a key that isn't + * prefix compressed. + */ + if (unpack->prefix == 0) { + /* + * If this is the key we originally wanted, we don't + * care if we're rolling forward or backward, it's + * what we want. Take a copy and wrap up. + * + * If we wanted a different key, this key has a valid + * prefix we can use it. + * If rolling backward, take a copy of the key and + * switch directions, we can roll forward from this key. + * If rolling forward there's a bug, we should have + * found this key while rolling backwards and switched + * directions then. + */ + WT_ERR(__wt_cell_unpack_copy(session, unpack, retb)); + if (slot_offset == 0) + break; + + WT_ASSERT(session, direction == BACKWARD); + direction = FORWARD; + goto next; + } + + /* + * 4: an on-page reference to a key that's prefix compressed. + * If rolling backward, keep looking for something we can + * use. + * If rolling forward, build the full key and keep rolling + * forward. + */ + if (direction == FORWARD) { + /* + * Get a copy of the current key; + * Ensure the buffer can hold the key plus the prefix; + * Append the key to the prefix (already in the buffer); + * Set the final size of the key. + */ + if (tmp == NULL) + WT_ERR(__wt_scr_alloc(session, 0, &tmp)); + WT_ERR(__wt_cell_unpack_copy(session, unpack, tmp)); + WT_ERR(__wt_buf_initsize( + session, retb, tmp->size + unpack->prefix)); + memcpy((uint8_t *) + retb->data + unpack->prefix, tmp->data, tmp->size); + + if (slot_offset == 0) + break; + } + +next: switch (direction) { + case BACKWARD: + --rip; + ++slot_offset; + break; + case FORWARD: + ++rip; + --slot_offset; + break; + } + } + + __wt_scr_free(&tmp); + + /* + * If a return buffer was specified, the caller just wants a copy and + * no further work is needed. + */ + if (!is_local) + return (0); + + /* + * Allocate and initialize a WT_IKEY structure, we're instantiating + * this key. + */ + WT_ERR(__wt_row_ikey_alloc(session, + WT_PAGE_DISK_OFFSET(page, rip_arg->key), + retb->data, retb->size, &ikey)); + + /* Serialize the swap of the key into place. */ + ret = __wt_row_key_serial(session, page, rip_arg, ikey); + + /* + * Free the WT_IKEY structure if the serialized call didn't use it for + * the key. + */ + if (rip_arg->key != ikey) + __wt_free(session, ikey); + + __wt_scr_free(&retb); + + return (ret); + +err: if (is_local && retb != NULL) + __wt_scr_free(&retb); + __wt_scr_free(&tmp); + return (ret); +} + +/* + * __wt_row_value -- + * Return a pointer to the value cell for a row-store leaf page key, or + * NULL if there isn't one. + */ +WT_CELL * +__wt_row_value(WT_PAGE *page, WT_ROW *rip) +{ + WT_CELL *cell; + WT_CELL_UNPACK *unpack, _unpack; + + unpack = &_unpack; + + /* + * Multiple threads of control may be searching this page, which means + * the key may change underfoot, and here's where it gets tricky: first, + * copy the key. + */ + cell = rip->key; + + /* + * Key copied. + * + * Now, cell either references a WT_IKEY structure that has a value-cell + * offset, or references the on-page key WT_CELL, and we can walk past + * that to find the value WT_CELL. Both can be processed regardless of + * what other threads are doing. + */ + if (__wt_off_page(page, cell)) + cell = WT_PAGE_REF_OFFSET(page, ((WT_IKEY *)cell)->cell_offset); + + /* + * Row-store leaf pages may have a single data cell between each key, or + * keys may be adjacent (when the data cell is empty). Move to the next + * key. The page reconciliation code guarantees there is always a key + * cell after an empty data cell, so this is safe. + */ + __wt_cell_unpack(cell, unpack); + cell = (WT_CELL *)((uint8_t *)cell + unpack->len); + if (__wt_cell_type(cell) == WT_CELL_KEY || + __wt_cell_type(cell) == WT_CELL_KEY_OVFL) + return (NULL); + return (cell); +} + +/* + * __wt_row_ikey_alloc -- + * Instantiate a key in a WT_IKEY structure. + */ +int +__wt_row_ikey_alloc(WT_SESSION_IMPL *session, + uint32_t cell_offset, const void *key, uint32_t size, WT_IKEY **ikeyp) +{ + WT_IKEY *ikey; + + /* + * Allocate the WT_IKEY structure and room for the value, then copy + * the value into place. + */ + WT_RET(__wt_calloc(session, 1, sizeof(WT_IKEY) + size, &ikey)); + ikey->size = size; + ikey->cell_offset = cell_offset; + memcpy(WT_IKEY_DATA(ikey), key, size); + + *ikeyp = ikey; + return (0); +} + +/* + * __wt_row_key_serial_func -- + * Server function to instantiate a key during a row-store search. + */ +void +__wt_row_key_serial_func(WT_SESSION_IMPL *session) +{ + WT_IKEY *ikey; + WT_PAGE *page; + WT_ROW *rip; + + __wt_row_key_unpack(session, &page, &rip, &ikey); + + /* + * We don't care about the page's write generation -- there's a simpler + * test, if the key we're interested in still needs to be instantiated, + * because it can only be in one of two states. + */ + if (!__wt_off_page(page, rip->key)) { + rip->key = ikey; + __wt_cache_page_inmem_incr( + session, page, sizeof(WT_IKEY) + ikey->size); + } + + __wt_session_serialize_wrapup(session, NULL, 0); +} diff --git a/src/btree/row_modify.c b/src/btree/row_modify.c new file mode 100644 index 00000000000..44909954952 --- /dev/null +++ b/src/btree/row_modify.c @@ -0,0 +1,321 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_row_modify -- + * Row-store insert, update and delete. + */ +int +__wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, int is_remove) +{ + WT_INSERT *ins; + WT_INSERT_HEAD **inshead, *new_inshead, **new_inslist; + WT_ITEM *key, *value; + WT_PAGE *page; + WT_UPDATE **new_upd, *upd, **upd_entry; + size_t ins_size, upd_size; + size_t new_inshead_size, new_inslist_size, new_upd_size; + uint32_t ins_slot; + u_int skipdepth; + int i, ret; + + key = &cbt->iface.key; + value = is_remove ? NULL : &cbt->iface.value; + + page = cbt->page; + + ins = NULL; + new_inshead = NULL; + new_inslist = NULL; + new_upd = NULL; + upd = NULL; + ret = 0; + + /* + * Modify: allocate an update array as necessary, build a WT_UPDATE + * structure, and call a serialized function to insert the WT_UPDATE + * structure. + * + * Insert: allocate an insert array as necessary, build a WT_INSERT + * and WT_UPDATE structure pair, and call a serialized function to + * insert the WT_INSERT structure. + */ + if (cbt->compare == 0) { + new_upd_size = 0; + if (cbt->ins == NULL) { + /* + * Allocate an update array as necessary. + * + * Set the WT_UPDATE array reference. + */ + if (page->u.row.upd == NULL) { + WT_ERR(__wt_calloc_def( + session, page->entries, &new_upd)); + new_upd_size = + page->entries * sizeof(WT_UPDATE *); + upd_entry = &new_upd[cbt->slot]; + } else + upd_entry = &page->u.row.upd[cbt->slot]; + } else + upd_entry = &cbt->ins->upd; + + /* Allocate room for the new value from per-thread memory. */ + WT_ERR(__wt_update_alloc(session, value, &upd, &upd_size)); + + /* Insert the WT_UPDATE structure. */ + ret = __wt_update_serial(session, page, cbt->write_gen, + upd_entry, &new_upd, new_upd_size, &upd, upd_size); + } else { + /* + * Allocate insert array if necessary, and set the array + * reference. + * + * We allocate an additional insert array slot for insert keys + * sorting less than any key on the page. The test to select + * that slot is baroque: if the search returned the first page + * slot, we didn't end up processing an insert list, and the + * comparison value indicates the search key was smaller than + * the returned slot, then we're using the smallest-key insert + * slot. That's hard, so we set a flag. + */ + ins_slot = F_ISSET( + cbt, WT_CBT_SEARCH_SMALLEST) ? page->entries : cbt->slot; + + new_inshead_size = new_inslist_size = 0; + if (page->u.row.ins == NULL) { + WT_ERR(__wt_calloc_def( + session, page->entries + 1, &new_inslist)); + new_inslist_size = + (page->entries + 1) * sizeof(WT_INSERT_HEAD *); + inshead = &new_inslist[ins_slot]; + } else + inshead = &page->u.row.ins[ins_slot]; + + /* + * Allocate a new insert list head as necessary. + * + * If allocating a new insert list head, we have to initialize + * the cursor's insert list stack and insert head reference as + * well, search couldn't have. + */ + if (*inshead == NULL) { + new_inshead_size = sizeof(WT_INSERT_HEAD); + WT_ERR(__wt_calloc_def(session, 1, &new_inshead)); + for (i = 0; i < WT_SKIP_MAXDEPTH; i++) + cbt->ins_stack[i] = &new_inshead->head[i]; + cbt->ins_head = new_inshead; + } + + /* Choose a skiplist depth for this insert. */ + skipdepth = __wt_skip_choose_depth(); + + /* + * Allocate a WT_INSERT/WT_UPDATE pair, and update the cursor + * to reference it. + */ + WT_ERR(__wt_row_insert_alloc( + session, key, skipdepth, &ins, &ins_size)); + WT_ERR(__wt_update_alloc(session, value, &upd, &upd_size)); + ins->upd = upd; + ins_size += upd_size; + cbt->ins = ins; + + /* Insert the WT_INSERT structure. */ + ret = __wt_insert_serial(session, page, cbt->write_gen, + inshead, cbt->ins_stack, + &new_inslist, new_inslist_size, + &new_inshead, new_inshead_size, + &ins, ins_size, skipdepth); + } + + if (ret != 0) { +err: if (ins != NULL) + __wt_free(session, ins); + if (upd != NULL) + __wt_free(session, upd); + } + + /* Free any insert, update arrays. */ + __wt_free(session, new_inslist); + __wt_free(session, new_inshead); + __wt_free(session, new_upd); + + return (ret); +} + +/* + * __wt_row_insert_alloc -- + * Row-store insert: allocate a WT_INSERT structure from the session's + * buffer and fill it in. + */ +int +__wt_row_insert_alloc(WT_SESSION_IMPL *session, + WT_ITEM *key, u_int skipdepth, WT_INSERT **insp, size_t *ins_sizep) +{ + WT_INSERT *ins; + size_t ins_size; + + /* + * Allocate the WT_INSERT structure, next pointers for the skip list, + * and room for the key. Then copy the key into place. + */ + ins_size = sizeof(WT_INSERT) + + skipdepth * sizeof(WT_INSERT *) + key->size; + WT_RET(__wt_calloc(session, 1, ins_size, &ins)); + + ins->u.key.offset = WT_STORE_SIZE(ins_size - key->size); + WT_INSERT_KEY_SIZE(ins) = key->size; + memcpy(WT_INSERT_KEY(ins), key->data, key->size); + + *insp = ins; + if (ins_sizep != NULL) + *ins_sizep = ins_size; + return (0); +} + +/* + * __wt_insert_serial_func -- + * Server function to add an WT_INSERT entry to the page. + */ +void +__wt_insert_serial_func(WT_SESSION_IMPL *session) +{ + WT_INSERT *new_ins, ***ins_stack; + WT_INSERT_HEAD **inshead, **new_inslist, *new_inshead; + WT_PAGE *page; + uint32_t write_gen; + u_int i, skipdepth; + int ret; + + ret = 0; + + __wt_insert_unpack(session, &page, &write_gen, &inshead, + &ins_stack, &new_inslist, &new_inshead, &new_ins, &skipdepth); + + /* Check the page's write-generation. */ + WT_ERR(__wt_page_write_gen_check(page, write_gen)); + + /* + * If the page does not yet have an insert array, our caller passed + * us one. + */ + if (page->type == WT_PAGE_ROW_LEAF) { + if (page->u.row.ins == NULL) { + page->u.row.ins = new_inslist; + __wt_insert_new_inslist_taken(session, page); + } + } else + if (page->modify->update == NULL) { + page->modify->update = new_inslist; + __wt_insert_new_inslist_taken(session, page); + } + + /* + * If the insert head does not yet have an insert list, our caller + * passed us one. + */ + if (*inshead == NULL) { + *inshead = new_inshead; + __wt_insert_new_inshead_taken(session, page); + } + + /* + * Publish: First, point the new WT_INSERT item's skiplist references + * to the next elements in the insert list, then flush memory. Second, + * update the skiplist elements that reference the new WT_INSERT item, + * this ensures the list is never inconsistent. + */ + for (i = 0; i < skipdepth; i++) + new_ins->next[i] = *ins_stack[i]; + WT_WRITE_BARRIER(); + for (i = 0; i < skipdepth; i++) { + if ((*inshead)->tail[i] == NULL || + ins_stack[i] == &(*inshead)->tail[i]->next[i]) + (*inshead)->tail[i] = new_ins; + *ins_stack[i] = new_ins; + } + + __wt_insert_new_ins_taken(session, page); + +err: __wt_session_serialize_wrapup(session, page, ret); +} + +/* + * __wt_update_alloc -- + * Allocate a WT_UPDATE structure and associated value from the session's + * buffer and fill it in. + */ +int +__wt_update_alloc(WT_SESSION_IMPL *session, + WT_ITEM *value, WT_UPDATE **updp, size_t *sizep) +{ + WT_UPDATE *upd; + size_t size; + + /* + * Allocate the WT_UPDATE structure and room for the value, then copy + * the value into place. + */ + size = value == NULL ? 0 : value->size; + WT_RET(__wt_calloc(session, 1, sizeof(WT_UPDATE) + size, &upd)); + if (value == NULL) + WT_UPDATE_DELETED_SET(upd); + else { + upd->size = WT_STORE_SIZE(size); + memcpy(WT_UPDATE_DATA(upd), value->data, size); + } + + *updp = upd; + if (sizep != NULL) + *sizep = sizeof(WT_UPDATE) + size; + return (0); +} + +/* + * __wt_update_serial_func -- + * Server function to add an WT_UPDATE entry in the page array. + */ +void +__wt_update_serial_func(WT_SESSION_IMPL *session) +{ + WT_PAGE *page; + WT_UPDATE **new_upd, *upd, **upd_entry; + uint32_t write_gen; + int ret; + + ret = 0; + + __wt_update_unpack( + session, &page, &write_gen, &upd_entry, &new_upd, &upd); + + /* Check the page's write-generation. */ + WT_ERR(__wt_page_write_gen_check(page, write_gen)); + + /* + * If the page needs an update array (column-store pages and inserts on + * row-store pages do not use the update array), our caller passed us + * one of the correct size. Check the page still needs one (the write + * generation test should have caught that, though). + */ + if (new_upd != NULL && page->u.row.upd == NULL) { + page->u.row.upd = new_upd; + __wt_update_new_upd_taken(session, page); + } + + upd->next = *upd_entry; + /* + * Publish: there must be a barrier to ensure the new entry's next + * pointer is set before we update the linked list. + */ + WT_PUBLISH(*upd_entry, upd); + + __wt_update_upd_taken(session, page); + +err: __wt_session_serialize_wrapup(session, page, ret); +} diff --git a/src/btree/row_srch.c b/src/btree/row_srch.c new file mode 100644 index 00000000000..c0f38f36b74 --- /dev/null +++ b/src/btree/row_srch.c @@ -0,0 +1,288 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_search_insert -- + * Search a row-store insert list, creating a skiplist stack as we go. + */ +WT_INSERT * +__wt_search_insert(WT_SESSION_IMPL *session, + WT_CURSOR_BTREE *cbt, WT_INSERT_HEAD *inshead, WT_ITEM *srch_key) +{ + WT_BTREE *btree; + WT_INSERT **insp, *ret_ins; + WT_ITEM insert_key; + int cmp, i; + + /* If there's no insert chain to search, we're done. */ + if ((ret_ins = WT_SKIP_LAST(inshead)) == NULL) + return (NULL); + + btree = session->btree; + + /* Fast-path appends. */ + insert_key.data = WT_INSERT_KEY(ret_ins); + insert_key.size = WT_INSERT_KEY_SIZE(ret_ins); + (void)WT_BTREE_CMP(session, btree, srch_key, &insert_key, cmp); + if (cmp >= 0) { + for (i = WT_SKIP_MAXDEPTH - 1; i >= 0; i--) + cbt->ins_stack[i] = (inshead->tail[i] != NULL) ? + &inshead->tail[i]->next[i] : + &inshead->head[i]; + cbt->compare = -cmp; + return (ret_ins); + } + + /* + * The insert list is a skip list: start at the highest skip level, then + * go as far as possible at each level before stepping down to the next. + */ + ret_ins = NULL; + for (i = WT_SKIP_MAXDEPTH - 1, insp = &inshead->head[i]; i >= 0; ) { + if (*insp == NULL) { + cbt->ins_stack[i--] = insp--; + continue; + } + + /* + * Comparisons may be repeated as we drop down skiplist levels; + * don't repeat comparisons, they might be expensive. + */ + if (ret_ins != *insp) { + ret_ins = *insp; + insert_key.data = WT_INSERT_KEY(ret_ins); + insert_key.size = WT_INSERT_KEY_SIZE(ret_ins); + (void)WT_BTREE_CMP(session, btree, + srch_key, &insert_key, cmp); + } + + if (cmp > 0) /* Keep going at this level */ + insp = &ret_ins->next[i]; + else if (cmp == 0) + for (; i >= 0; i--) + cbt->ins_stack[i] = &ret_ins->next[i]; + else /* Drop down a level */ + cbt->ins_stack[i--] = insp--; + } + + /* + * For every insert element we review, we're getting closer to a better + * choice; update the compare field to its new value. + */ + cbt->compare = -cmp; + return (ret_ins); +} + +/* + * __wt_row_search -- + * Search a row-store tree for a specific key. + */ +int +__wt_row_search(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, int is_modify) +{ + WT_BTREE *btree; + WT_CELL_UNPACK *unpack, _unpack; + WT_IKEY *ikey; + WT_ITEM *item, _item, *srch_key; + WT_PAGE *page; + WT_REF *ref; + WT_ROW *rip; + uint32_t base, indx, limit; + int cmp, ret; + void *key; + + __cursor_search_clear(cbt); + + srch_key = &cbt->iface.key; + + btree = session->btree; + unpack = &_unpack; + rip = NULL; + + cmp = -1; /* Assume we don't match. */ + + /* Search the internal pages of the tree. */ + item = &_item; + for (page = btree->root_page; page->type == WT_PAGE_ROW_INT;) { + /* Binary search of internal pages. */ + for (base = 0, ref = NULL, + limit = page->entries; limit != 0; limit >>= 1) { + indx = base + (limit >> 1); + ref = page->u.intl.t + indx; + + /* + * If we're about to compare an application key with the + * 0th index on an internal page, pretend the 0th index + * sorts less than any application key. This test is so + * we don't have to update internal pages if the + * application stores a new, "smallest" key in the tree. + */ + if (indx != 0) { + ikey = ref->u.key; + item->data = WT_IKEY_DATA(ikey); + item->size = ikey->size; + + WT_RET(WT_BTREE_CMP( + session, btree, srch_key, item, cmp)); + if (cmp == 0) + break; + if (cmp < 0) + continue; + } + base = indx + 1; + --limit; + } + WT_ASSERT(session, ref != NULL); + + /* + * Reference the slot used for next step down the tree. + * + * Base is the smallest index greater than key and may be the + * (last + 1) index. (Base cannot be the 0th index as the 0th + * index always sorts less than any application key). The slot + * for descent is the one before base. + */ + if (cmp != 0) + ref = page->u.intl.t + (base - 1); + + /* Swap the parent page for the child page. */ + WT_ERR(__wt_page_in(session, page, ref)); + __wt_page_release(session, page); + page = ref->page; + } + + /* + * Copy the leaf page's write generation value before reading the page. + * Use a read memory barrier to ensure we read the value before we read + * any of the page's contents. + */ + if (is_modify) { + /* Initialize the page's modification information */ + if (page->modify == NULL) + WT_RET(__wt_page_modify_init(session, page)); + + WT_ORDERED_READ(cbt->write_gen, page->modify->write_gen); + } + cbt->page = page; + + /* Do a binary search of the leaf page. */ + for (base = 0, limit = page->entries; limit != 0; limit >>= 1) { + indx = base + (limit >> 1); + rip = page->u.row.d + indx; + +retry: /* + * Multiple threads of control may be searching this page, which + * means the key may change underfoot, and here's where it gets + * tricky: first, copy the key. We don't need any barriers, the + * key is updated atomically, and we just need a valid copy. + */ + key = rip->key; + + /* + * Key copied. + * + * If another thread instantiated the key, we don't have any + * work to do. Figure this out using the key's value: + * + * If the key points off-page, the key has been instantiated, + * just use it. + * + * If the key points on-page, we have a copy of a WT_CELL value + * that can be processed, regardless of what any other thread is + * doing. + */ + if (__wt_off_page(page, key)) { + ikey = key; + _item.data = WT_IKEY_DATA(ikey); + _item.size = ikey->size; + item = &_item; + } else { + if (btree->huffman_key != NULL) { + WT_ERR(__wt_row_key(session, page, rip, NULL)); + goto retry; + } + __wt_cell_unpack(key, unpack); + if (unpack->type == WT_CELL_KEY && + unpack->prefix == 0) { + _item.data = unpack->data; + _item.size = unpack->size; + item = &_item; + } else { + WT_ERR(__wt_row_key(session, page, rip, NULL)); + goto retry; + } + } + + WT_RET(WT_BTREE_CMP(session, btree, srch_key, item, cmp)); + if (cmp == 0) + break; + if (cmp < 0) + continue; + + base = indx + 1; + --limit; + } + + /* + * The best case is finding an exact match in the page's WT_ROW slot + * array, which is probable for any read-mostly workload. In that + * case, we're not doing any kind of insert, all we can do is update + * an existing entry. Check that case and get out fast. + */ + if (cmp == 0) { + WT_ASSERT(session, rip != NULL); + cbt->compare = 0; + cbt->slot = WT_ROW_SLOT(page, rip); + return (0); + } + + /* + * We didn't find an exact match in the WT_ROW array. + * + * Base is the smallest index greater than key and may be the 0th index + * or the (last + 1) index. Set the WT_ROW reference to be the largest + * index less than the key if that's possible (if base is the 0th index + * it means the application is inserting a key before any key found on + * the page). + */ + rip = page->u.row.d; + if (base == 0) + cbt->compare = 1; + else { + rip += base - 1; + cbt->compare = -1; + } + + /* + * It's still possible there is an exact match, but it's on an insert + * list. Figure out which insert chain to search, and do the initial + * setup of the return information for the insert chain (we'll correct + * it as needed depending on what we find.) + * + * If inserting a key smaller than any key found in the WT_ROW array, + * use the extra slot of the insert array, otherwise insert lists map + * one-to-one to the WT_ROW array. + */ + cbt->slot = WT_ROW_SLOT(page, rip); + if (base == 0) { + F_SET(cbt, WT_CBT_SEARCH_SMALLEST); + cbt->ins_head = WT_ROW_INSERT_SMALLEST(page); + } else + cbt->ins_head = WT_ROW_INSERT_SLOT(page, cbt->slot); + + /* + * Search the insert list for a match; __wt_search_insert sets the + * return insert information appropriately. + */ + cbt->ins = __wt_search_insert(session, cbt, cbt->ins_head, srch_key); + return (0); + +err: __wt_page_release(session, page); + return (ret); +} diff --git a/src/config/config.c b/src/config/config.c new file mode 100644 index 00000000000..2905aed4042 --- /dev/null +++ b/src/config/config.c @@ -0,0 +1,660 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __config_err -- + * Error message and return for config string parse failures. + */ +static int +__config_err(WT_CONFIG *conf, const char *msg, int err) +{ + WT_RET_MSG(conf->session, err == 0 ? EINVAL : err, + "Error parsing '%.*s' at byte %u: %s", + (int)(conf->end - conf->orig), conf->orig, + (u_int)(conf->cur - conf->orig), msg); +} + +/* + * __wt_config_initn -- + * Initialize a config handle, used to iterate through a config string of + * specified length. + */ +int +__wt_config_initn( + WT_SESSION_IMPL *session, WT_CONFIG *conf, const char *str, size_t len) +{ + conf->session = session; + conf->orig = conf->cur = str; + conf->end = str + len; + conf->depth = 0; + conf->top = -1; + conf->go = NULL; + + return (0); +} + +/* + * __wt_config_init -- + * Initialize a config handle, used to iterate through a NUL-terminated + * config string. + */ +int +__wt_config_init(WT_SESSION_IMPL *session, WT_CONFIG *conf, const char *str) +{ + size_t len; + + len = (str == NULL) ? 0 : strlen(str); + + return (__wt_config_initn(session, conf, str, len)); +} + +/* + * __wt_config_subinit -- + * Initialize a config handle, used to iterate through a config string + * extracted from another config string (used for parsing nested + * structures). + */ +int +__wt_config_subinit( + WT_SESSION_IMPL *session, WT_CONFIG *conf, WT_CONFIG_ITEM *item) +{ + return (__wt_config_initn(session, conf, item->str, item->len)); +} + +#define PUSH(i, t) do { \ + if (conf->top == -1) \ + conf->top = conf->depth; \ + if (conf->depth == conf->top) { \ + if (out->len > 0) \ + return __config_err(conf, \ + "New value starts without a separator", 0); \ + out->type = (t); \ + out->str = (conf->cur + (i)); \ + } \ +} while (0) + +#define CAP(i) do { \ + if (conf->depth == conf->top) \ + out->len = (size_t)((conf->cur + (i) + 1) - out->str); \ +} while (0) + +typedef enum { + A_LOOP, A_BAD, A_DOWN, A_UP, A_VALUE, A_NEXT, A_QDOWN, A_QUP, + A_ESC, A_UNESC, A_BARE, A_NUMBARE, A_UNBARE, A_UTF8_2, + A_UTF8_3, A_UTF8_4, A_UTF_CONTINUE +} CONFIG_ACTION; + +/* + * static void *gostruct[] = { + * [0 ... 255] = &&l_bad, + * ['\t'] = &&l_loop, [' '] = &&l_loop, + * ['\r'] = &&l_loop, ['\n'] = &&l_loop, + * ['"'] = &&l_qup, + * [':'] = &&l_value, ['='] = &&l_value, + * [','] = &&l_next, + * // tracking [] and {} individually would allow fuller + * // validation but is really messy + * ['('] = &&l_up, [')'] = &&l_down, + * ['['] = &&l_up, [']'] = &&l_down, + * ['{'] = &&l_up, ['}'] = &&l_down, + * // bare identifiers + * ['-'] = &&l_numbare, + * ['0' ... '9'] = &&l_numbare, + * ['_'] = &&l_bare, + * ['A' ... 'Z'] = &&l_bare, ['a' ... 'z'] = &&l_bare, + * }; + */ +static int8_t gostruct[256] = { + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_LOOP, A_LOOP, A_BAD, A_BAD, A_LOOP, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_LOOP, A_BAD, A_QUP, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_UP, A_DOWN, A_BAD, A_BAD, + A_NEXT, A_NUMBARE, A_BAD, A_BAD, A_NUMBARE, A_NUMBARE, + A_NUMBARE, A_NUMBARE, A_NUMBARE, A_NUMBARE, A_NUMBARE, + A_NUMBARE, A_NUMBARE, A_NUMBARE, A_VALUE, A_BAD, A_BAD, + A_VALUE, A_BAD, A_BAD, A_BAD, A_BARE, A_BARE, A_BARE, A_BARE, + A_BARE, A_BARE, A_BARE, A_BARE, A_BARE, A_BARE, A_BARE, A_BARE, + A_BARE, A_BARE, A_BARE, A_BARE, A_BARE, A_BARE, A_BARE, A_BARE, + A_BARE, A_BARE, A_BARE, A_BARE, A_BARE, A_BARE, A_UP, A_BAD, + A_DOWN, A_BAD, A_BARE, A_BAD, A_BARE, A_BARE, A_BARE, A_BARE, + A_BARE, A_BARE, A_BARE, A_BARE, A_BARE, A_BARE, A_BARE, A_BARE, + A_BARE, A_BARE, A_BARE, A_BARE, A_BARE, A_BARE, A_BARE, A_BARE, + A_BARE, A_BARE, A_BARE, A_BARE, A_BARE, A_BARE, A_UP, A_BAD, + A_DOWN, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD +}; + +/* + * static void *gobare[] = + * { + * [0 ... 31] = &&l_bad, + * // could be more pedantic/validation-checking + * [32 ... 126] = &&l_loop, + * ['\t'] = &&l_unbare, [' '] = &&l_unbare, + * ['\r'] = &&l_unbare, ['\n'] = &&l_unbare, + * [':'] = &&l_unbare, ['='] = &&l_unbare, + * [','] = &&l_unbare, + * [')'] = &&l_unbare, [']'] = &&l_unbare, ['}'] = &&l_unbare, + * [127 ... 255] = &&l_bad + * }; + */ +static int8_t gobare[256] = { + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_UNBARE, A_UNBARE, A_BAD, A_BAD, A_UNBARE, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_UNBARE, + A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, + A_UNBARE, A_LOOP, A_LOOP, A_UNBARE, A_LOOP, A_LOOP, A_LOOP, + A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, + A_LOOP, A_LOOP, A_UNBARE, A_LOOP, A_LOOP, A_UNBARE, A_LOOP, + A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, + A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, + A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, + A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_UNBARE, + A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, + A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, + A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, + A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, + A_UNBARE, A_LOOP, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD +}; + +/* + * static void *gostring[] = + * { + * [0 ... 31] = &&l_bad, [127] = &&l_bad, + * [32 ... 126] = &&l_loop, + * ['\\'] = &&l_esc, ['"'] = &&l_qdown, + * [128 ... 191] = &&l_bad, + * [192 ... 223] = &&l_utf8_2, + * [224 ... 239] = &&l_utf8_3, + * [240 ... 247] = &&l_utf8_4, + * [248 ... 255] = &&l_bad + * }; + */ +static int8_t gostring[256] = { + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_LOOP, A_LOOP, A_QDOWN, + A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, + A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, + A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, + A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, + A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, + A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, + A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, + A_LOOP, A_ESC, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, + A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, + A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, + A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_LOOP, + A_LOOP, A_LOOP, A_LOOP, A_LOOP, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_UTF8_2, + A_UTF8_2, A_UTF8_2, A_UTF8_2, A_UTF8_2, A_UTF8_2, A_UTF8_2, + A_UTF8_2, A_UTF8_2, A_UTF8_2, A_UTF8_2, A_UTF8_2, A_UTF8_2, + A_UTF8_2, A_UTF8_2, A_UTF8_2, A_UTF8_2, A_UTF8_2, A_UTF8_2, + A_UTF8_2, A_UTF8_2, A_UTF8_2, A_UTF8_2, A_UTF8_2, A_UTF8_2, + A_UTF8_2, A_UTF8_2, A_UTF8_2, A_UTF8_2, A_UTF8_2, A_UTF8_2, + A_UTF8_2, A_UTF8_3, A_UTF8_3, A_UTF8_3, A_UTF8_3, A_UTF8_3, + A_UTF8_3, A_UTF8_3, A_UTF8_3, A_UTF8_3, A_UTF8_3, A_UTF8_3, + A_UTF8_3, A_UTF8_3, A_UTF8_3, A_UTF8_3, A_UTF8_3, A_UTF8_4, + A_UTF8_4, A_UTF8_4, A_UTF8_4, A_UTF8_4, A_UTF8_4, A_UTF8_4, + A_UTF8_4, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD +}; + +/* + * static void *goutf8_continue[] = + * { + * [0 ... 127] = &&l_bad, + * [128 ... 191] = &&l_utf_continue, + * [192 ... 255] = &&l_bad + * }; + */ +static int8_t goutf8_continue[256] = { + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, + A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, + A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, + A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, + A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, + A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, + A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, + A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, + A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, + A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, + A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, + A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, + A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, + A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, + A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, + A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, A_UTF_CONTINUE, + A_UTF_CONTINUE, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD +}; + +/* + * static void *goesc[] = + * { + * [0 ... 255] = &&l_bad, + * ['"'] = &&l_unesc, ['\\'] = &&l_unesc, + * ['/'] = &&l_unesc, ['b'] = &&l_unesc, + * ['f'] = &&l_unesc, ['n'] = &&l_unesc, + * ['r'] = &&l_unesc, ['t'] = &&l_unesc, ['u'] = &&l_unesc + * }; + */ +static int8_t goesc[256] = { + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_UNESC, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_UNESC, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_UNESC, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_UNESC, A_BAD, A_BAD, A_BAD, A_UNESC, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_UNESC, A_BAD, + A_BAD, A_BAD, A_UNESC, A_BAD, A_UNESC, A_UNESC, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, + A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD +}; + +/* + * __process_value + * Deal with special config values like true / false. + */ +static int +__process_value(WT_CONFIG *conf, WT_CONFIG_ITEM *value) +{ + char *endptr; + + /* Empty values are okay: we can't do anything interesting with them. */ + if (value->len == 0) + return (0); + + if (value->type == ITEM_ID) { + if (strncasecmp(value->str, "true", value->len) == 0) { + value->type = ITEM_NUM; + value->val = 1; + } else if (strncasecmp(value->str, "false", value->len) == 0) { + value->type = ITEM_NUM; + value->val = 0; + } + } else if (value->type == ITEM_NUM) { + errno = 0; + value->val = strtoll(value->str, &endptr, 10); + if (errno == ERANGE) + return ( + __config_err(conf, "Number out of range", ERANGE)); + + /* Check any leftover characters. */ + while (endptr < value->str + value->len) + switch (*endptr++) { + case 'b': + case 'B': + /* Byte: no change. */ + break; + case 'k': + case 'K': + value->val <<= 10; + break; + case 'm': + case 'M': + value->val <<= 20; + break; + case 'g': + case 'G': + value->val <<= 30; + break; + case 't': + case 'T': + value->val <<= 40; + break; + case 'p': + case 'P': + value->val <<= 50; + break; + default: + /* + * We didn't get a well-formed number. That + * might be okay, the required type will be + * checked by __wt_config_check. + */ + value->type = ITEM_ID; + break; + } + } + + return (0); +} + +/* + * __wt_config_next -- + * Get the next config item in the string. + */ +int +__wt_config_next(WT_CONFIG *conf, WT_CONFIG_ITEM *key, WT_CONFIG_ITEM *value) +{ + WT_CONFIG_ITEM *out = key; + int utf8_remain = 0; + static WT_CONFIG_ITEM default_value = { + "", 0, 1, ITEM_NUM + }; + + key->len = 0; + *value = default_value; + + if (conf->go == NULL) + conf->go = gostruct; + + while (conf->cur < conf->end) { + switch (conf->go[(int)*conf->cur]) { + case A_LOOP: + break; + + case A_BAD: + return __config_err(conf, "Unexpected character", 0); + + case A_DOWN: + --conf->depth; + CAP(0); + break; + + case A_UP: + if (conf->top == -1) + conf->top = 1; + PUSH(0, ITEM_STRUCT); + ++conf->depth; + break; + + case A_VALUE: + if (conf->depth == conf->top) { + /* + * Special case: ':' is permitted in unquoted + * values. + */ + if (out == value && *conf->cur != ':') + return __config_err(conf, + "Value already complete", 0); + out = value; + } + break; + + case A_NEXT: + /* + * If we're at the top level and we have a complete + * key (and optional value), we're done. + */ + if (conf->depth == conf->top && key->len > 0) { + ++conf->cur; + goto val; + } else + break; + + case A_QDOWN: + CAP(-1); + conf->go = gostruct; + break; + + case A_QUP: + PUSH(1, ITEM_STRING); + conf->go = gostring; + break; + + case A_ESC: + conf->go = goesc; + break; + + case A_UNESC: + conf->go = gostring; + break; + + case A_BARE: + PUSH(0, ITEM_ID); + conf->go = gobare; + break; + + case A_NUMBARE: + PUSH(0, ITEM_NUM); + conf->go = gobare; + break; + + case A_UNBARE: + CAP(-1); + conf->go = gostruct; + continue; + + case A_UTF8_2: + conf->go = goutf8_continue; + utf8_remain = 1; + break; + + case A_UTF8_3: + conf->go = goutf8_continue; + utf8_remain = 2; + break; + + case A_UTF8_4: + conf->go = goutf8_continue; + utf8_remain = 3; + break; + + case A_UTF_CONTINUE: + if (!--utf8_remain) + conf->go = gostring; + break; + } + + conf->cur++; + } + + /* Might have a trailing key/value without a closing brace */ + if (conf->go == gobare) { + CAP(-1); + conf->go = gostruct; + } + + /* Did we find something? */ + if (conf->depth <= conf->top && key->len > 0) +val: return (__process_value(conf, value)); + + /* We're either at the end of the string or we failed to parse. */ + if (conf->depth == 0) + return (WT_NOTFOUND); + + return __config_err(conf, + "Closing brackets missing from config string", 0); +} + +/* + * __wt_config_getraw -- + * Given a config parser, find the final value for a given key. + */ +int +__wt_config_getraw( + WT_CONFIG *cparser, WT_CONFIG_ITEM *key, WT_CONFIG_ITEM *value) +{ + WT_CONFIG_ITEM k, v; + int ret; + + while ((ret = __wt_config_next(cparser, &k, &v)) == 0) { + if ((k.type == ITEM_STRING || k.type == ITEM_ID) && + key->len == k.len && + strncasecmp(key->str, k.str, k.len) == 0) { + *value = v; + return (0); + } + } + + return (ret); +} + +/* + * __wt_config_get -- + * Given a NULL-terminated list of configuration strings, find + * the final value for a given key. + */ +int +__wt_config_get(WT_SESSION_IMPL *session, + const char **cfg, WT_CONFIG_ITEM *key, WT_CONFIG_ITEM *value) +{ + WT_CONFIG cparser; + int found, ret; + + for (found = 0; *cfg != NULL; cfg++) { + WT_RET(__wt_config_init(session, &cparser, *cfg)); + if ((ret = __wt_config_getraw(&cparser, key, value)) == 0) + found = 1; + else if (ret != WT_NOTFOUND) + return (ret); + } + + return (found ? 0 : WT_NOTFOUND); +} + +/* + * __wt_config_gets -- + * Given a NULL-terminated list of configuration strings, find the final + * value for a given string key. + */ +int +__wt_config_gets(WT_SESSION_IMPL *session, + const char **cfg, const char *key, WT_CONFIG_ITEM *value) +{ + WT_CONFIG_ITEM key_item; + + key_item.type = ITEM_STRING; + key_item.str = key; + key_item.len = strlen(key); + + return (__wt_config_get(session, cfg, &key_item, value)); +} + +/* + * __wt_config_getone -- + * Get the value for a given key from a single config string. + */ + int +__wt_config_getone(WT_SESSION_IMPL *session, + const char *cfg, WT_CONFIG_ITEM *key, WT_CONFIG_ITEM *value) +{ + const char *cfgs[] = { cfg, NULL }; + return (__wt_config_get(session, cfgs, key, value)); +} + +/* + * __wt_config_getones -- + * Get the value for a given string key from a single config string. + */ + int +__wt_config_getones(WT_SESSION_IMPL *session, + const char *cfg, const char *key, WT_CONFIG_ITEM *value) +{ + const char *cfgs[] = { cfg, NULL }; + return (__wt_config_gets(session, cfgs, key, value)); +} + +/* + * __wt_config_subgetraw -- + * Get the value for a given key from a config string in a WT_CONFIG_ITEM. + * This is useful for dealing with nested structs in config strings. + */ + int +__wt_config_subgetraw(WT_SESSION_IMPL *session, + WT_CONFIG_ITEM *cfg, WT_CONFIG_ITEM *key, WT_CONFIG_ITEM *value) +{ + WT_CONFIG cparser; + + WT_RET(__wt_config_initn(session, &cparser, cfg->str, cfg->len)); + return (__wt_config_getraw(&cparser, key, value)); +} + +/* + * __wt_config_subgets -- + * Get the value for a given key from a config string in a WT_CONFIG_ITEM. + * This is useful for dealing with nested structs in config strings. + */ + int +__wt_config_subgets(WT_SESSION_IMPL *session, + WT_CONFIG_ITEM *cfg, const char *key, WT_CONFIG_ITEM *value) +{ + WT_CONFIG_ITEM key_item; + + key_item.str = key; + key_item.len = strlen(key); + + return (__wt_config_subgetraw(session, cfg, &key_item, value)); +} diff --git a/src/config/config_check.c b/src/config/config_check.c new file mode 100644 index 00000000000..8285a9444a6 --- /dev/null +++ b/src/config/config_check.c @@ -0,0 +1,120 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_config_check-- + * Check that all keys in an application-supplied config string match + * what is specified in the check string. + * + * All check strings are generated by dist/config.py from the constraints given + * in dist/api_data.py + */ +int +__wt_config_check(WT_SESSION_IMPL *session, + const char *checks, const char *config) +{ + WT_CONFIG parser, cparser, sparser; + WT_CONFIG_ITEM k, v, chk, ck, cv, dummy; + int found, ret; + + /* It is always okay to pass NULL. */ + if (config == NULL) + return (0); + + WT_RET(__wt_config_init(session, &parser, config)); + while ((ret = __wt_config_next(&parser, &k, &v)) == 0) { + if (k.type != ITEM_STRING && k.type != ITEM_ID) + WT_RET_MSG(session, EINVAL, + "Invalid configuration key found: '%.*s'", + (int)k.len, k.str); + + if ((ret = __wt_config_getone(session, + checks, &k, &chk)) != 0) { + if (ret == WT_NOTFOUND) + WT_RET_MSG(session, EINVAL, + "Unknown configuration key found: '%.*s'", + (int)k.len, k.str); + return (ret); + } + + WT_RET(__wt_config_subinit(session, &cparser, &chk)); + while ((ret = __wt_config_next(&cparser, &ck, &cv)) == 0) { + if (strncmp(ck.str, "type", ck.len) == 0) { + if ((strncmp(cv.str, "int", cv.len) == 0 && + v.type != ITEM_NUM) || + (strncmp(cv.str, "boolean", cv.len) == 0 && + (v.type != ITEM_NUM || + (v.val != 0 && v.val != 1))) || + (strncmp(cv.str, "list", cv.len) == 0 && + v.len > 0 && v.type != ITEM_STRUCT)) + WT_RET_MSG(session, EINVAL, + "Invalid value type " + "for key '%.*s': expected a %.*s", + (int)k.len, k.str, + (int)cv.len, cv.str); + } else if (strncmp(ck.str, "min", ck.len) == 0) { + if (v.val < cv.val) + WT_RET_MSG(session, EINVAL, + "Value too small for key '%.*s' " + "the minimum is %.*s", + (int)k.len, k.str, + (int)cv.len, cv.str); + } else if (strncmp(ck.str, "max", ck.len) == 0) { + if (v.val > cv.val) + WT_RET_MSG(session, EINVAL, + "Value too large for key '%.*s' " + "the maximum is %.*s", + (int)k.len, k.str, + (int)cv.len, cv.str); + } else if (strncmp(ck.str, "choices", ck.len) == 0) { + if (v.len == 0) + WT_RET_MSG(session, EINVAL, + "Key '%.*s' requires a value", + (int)k.len, k.str); + if (v.type == ITEM_STRUCT) { + /* + * Handle the 'verbose' case of a list + * containing restricted choices. + */ + WT_RET(__wt_config_subinit(session, + &sparser, &v)); + found = 1; + while (found && + (ret = __wt_config_next(&sparser, + &v, &dummy)) == 0) { + ret = __wt_config_subgetraw( + session, &cv, &v, &dummy); + found = (ret == 0); + } + } else { + ret = __wt_config_subgetraw(session, + &cv, &v, &dummy); + found = (ret == 0); + } + + if (ret != 0 && ret != WT_NOTFOUND) + return (ret); + if (!found) + WT_RET_MSG(session, EINVAL, + "Value '%.*s' not a " + "permitted choice for key '%.*s'", + (int)v.len, v.str, + (int)k.len, k.str); + } else + WT_RET_MSG(session, EINVAL, + "unexpected configuration description " + "keyword %.*s", (int)ck.len, ck.str); + } + } + + if (ret == WT_NOTFOUND) + ret = 0; + + return (ret); +} diff --git a/src/config/config_collapse.c b/src/config/config_collapse.c new file mode 100644 index 00000000000..ca0c863ea55 --- /dev/null +++ b/src/config/config_collapse.c @@ -0,0 +1,66 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_config_collapse -- + * Given a NULL-terminated list of configuration strings, where the first + * one contains all the defaults, collapse them into a newly allocated + * buffer. + */ +int +__wt_config_collapse(WT_SESSION_IMPL *session, + const char **cfg, const char **config_ret) +{ + WT_CONFIG cparser; + WT_CONFIG_ITEM k, v; + WT_ITEM buf; + int ret; + + WT_CLEAR(buf); + + WT_RET(__wt_config_init(session, &cparser, cfg[0])); + while ((ret = __wt_config_next(&cparser, &k, &v)) == 0) { + if (k.type != ITEM_STRING && k.type != ITEM_ID) + WT_RET_MSG(session, EINVAL, + "Invalid configuration key found: '%s'\n", k.str); + WT_ERR(__wt_config_get(session, cfg, &k, &v)); + /* Include the quotes around string keys/values. */ + if (k.type == ITEM_STRING) { + --k.str; + k.len += 2; + } + if (v.type == ITEM_STRING) { + --v.str; + v.len += 2; + } + WT_ERR(__wt_buf_catfmt(session, &buf, "%.*s=%.*s,", + (int)k.len, k.str, (int)v.len, v.str)); + } + + if (ret != WT_NOTFOUND) + goto err; + + /* + * If the caller passes us no valid configuration strings, we end up + * here with no allocated memory to return. Check the final buffer + * size: empty configuration strings are possible, and paranoia is + * good. + */ + if (buf.size == 0) + WT_RET(__wt_buf_initsize(session, &buf, 1)); + + /* Strip the trailing comma and NUL-terminate */ + ((char *)buf.data)[buf.size - 1] = '\0'; + + *config_ret = buf.data; + return (0); + +err: __wt_buf_free(session, &buf); + return (ret); +} diff --git a/src/config/config_concat.c b/src/config/config_concat.c new file mode 100644 index 00000000000..6bbee5ee77e --- /dev/null +++ b/src/config/config_concat.c @@ -0,0 +1,75 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_config_concat -- + * Given a NULL-terminated list of configuration strings, concatenate them + * into a newly allocated buffer. Nothing special is assumed about any + * of the config strings, they are simply combined in order. + * + * This code deals with the case where some of the config strings are + * wrapped in brackets but others aren't: the resulting string does not + * have brackets. + */ +int +__wt_config_concat( + WT_SESSION_IMPL *session, const char **cfg, const char **config_ret) +{ + WT_CONFIG cparser; + WT_CONFIG_ITEM k, v; + WT_ITEM buf; + int ret; + const char **cp; + + WT_CLEAR(buf); + ret = 0; + + for (cp = cfg; *cp != NULL; ++cp) { + WT_ERR(__wt_config_init(session, &cparser, *cp)); + while ((ret = __wt_config_next(&cparser, &k, &v)) == 0) { + if (k.type != ITEM_STRING && k.type != ITEM_ID) + WT_ERR_MSG(session, EINVAL, + "Invalid configuration key found: '%s'\n", + k.str); + /* Include the quotes around string keys/values. */ + if (k.type == ITEM_STRING) { + --k.str; + k.len += 2; + } + if (v.type == ITEM_STRING) { + --v.str; + v.len += 2; + } + WT_ERR(__wt_buf_catfmt(session, &buf, "%.*s%s%.*s,", + (int)k.len, k.str, + (v.len > 0) ? "=" : "", + (int)v.len, v.str)); + } + if (ret != WT_NOTFOUND) + goto err; + } + + /* + * If the caller passes us no valid configuration strings, we end up + * here with no allocated memory to return. Check the final buffer + * size: empty configuration strings are possible, and paranoia is + * good. + */ + if (buf.size == 0) + WT_RET(__wt_buf_initsize(session, &buf, 1)); + + /* Strip the trailing comma and NUL-terminate */ + ((char *)buf.data)[buf.size - 1] = '\0'; + + *config_ret = buf.data; + return (0); + +err: __wt_buf_free(session, &buf); + return (ret); +} diff --git a/src/config/config_def.c b/src/config/config_def.c new file mode 100644 index 00000000000..69e60a81c76 --- /dev/null +++ b/src/config/config_def.c @@ -0,0 +1,283 @@ +/* DO NOT EDIT: automatically built by dist/config.py. */ + +#include "wt_internal.h" + +const char * +__wt_confdfl_colgroup_meta = + "columns=(),filename="""; + +const char * +__wt_confchk_colgroup_meta = + "columns=(type=list),filename=()"; + +const char * +__wt_confdfl_connection_add_collator = + ""; + +const char * +__wt_confchk_connection_add_collator = + ""; + +const char * +__wt_confdfl_connection_add_compressor = + ""; + +const char * +__wt_confchk_connection_add_compressor = + ""; + +const char * +__wt_confdfl_connection_add_cursor_type = + ""; + +const char * +__wt_confchk_connection_add_cursor_type = + ""; + +const char * +__wt_confdfl_connection_add_extractor = + ""; + +const char * +__wt_confchk_connection_add_extractor = + ""; + +const char * +__wt_confdfl_connection_close = + ""; + +const char * +__wt_confchk_connection_close = + ""; + +const char * +__wt_confdfl_connection_load_extension = + "entry=wiredtiger_extension_init,prefix="""; + +const char * +__wt_confchk_connection_load_extension = + "entry=(),prefix=()"; + +const char * +__wt_confdfl_connection_open_session = + ""; + +const char * +__wt_confchk_connection_open_session = + ""; + +const char * +__wt_confdfl_cursor_close = + ""; + +const char * +__wt_confchk_cursor_close = + ""; + +const char * +__wt_confdfl_file_meta = + "allocation_size=512B,block_compressor="",checksum=true,collator=""," + "columns=(),huffman_key="",huffman_value="",internal_item_max=0," + "internal_key_truncate=true,internal_page_max=2KB,key_format=u,key_gap=10" + ",leaf_item_max=0,leaf_page_max=1MB,prefix_compression=true,split_pct=75," + "type=btree,value_format=u"; + +const char * +__wt_confchk_file_meta = + "allocation_size=(type=int,min=512B,max=128MB),block_compressor=()," + "checksum=(type=boolean),collator=(),columns=(type=list),huffman_key=()," + "huffman_value=(),internal_item_max=(type=int,min=0)," + "internal_key_truncate=(type=boolean),internal_page_max=(type=int," + "min=512B,max=512MB),key_format=(type=format),key_gap=(type=int,min=0)," + "leaf_item_max=(type=int,min=0),leaf_page_max=(type=int,min=512B," + "max=512MB),prefix_compression=(type=boolean),split_pct=(type=int,min=25," + "max=100),type=(choices=[\"btree\"]),value_format=(type=format)"; + +const char * +__wt_confdfl_index_meta = + "columns=(),filename="""; + +const char * +__wt_confchk_index_meta = + "columns=(type=list),filename=()"; + +const char * +__wt_confdfl_session_begin_transaction = + "isolation=read-committed,name="",priority=0,sync=full"; + +const char * +__wt_confchk_session_begin_transaction = + "isolation=(choices=[\"serializable\",\"snapshot\",\"read-committed\"," + "\"read-uncommitted\"]),name=(),priority=(type=int,min=-100,max=100)," + "sync=(choices=[\"full\",\"flush\",\"write\",\"none\"])"; + +const char * +__wt_confdfl_session_checkpoint = + "archive=false,flush_cache=true,flush_log=true,force=false,log_size=0," + "timeout=0"; + +const char * +__wt_confchk_session_checkpoint = + "archive=(type=boolean),flush_cache=(type=boolean)," + "flush_log=(type=boolean),force=(type=boolean),log_size=(type=int,min=0)," + "timeout=(type=int,min=0)"; + +const char * +__wt_confdfl_session_close = + ""; + +const char * +__wt_confchk_session_close = + ""; + +const char * +__wt_confdfl_session_commit_transaction = + ""; + +const char * +__wt_confchk_session_commit_transaction = + ""; + +const char * +__wt_confdfl_session_create = + "allocation_size=512B,block_compressor="",checksum=true,colgroups=()," + "collator="",columns=(),columns=(),exclusive=false,filename=""," + "huffman_key="",huffman_value="",internal_item_max=0," + "internal_key_truncate=true,internal_page_max=2KB,key_format=u," + "key_format=u,key_gap=10,leaf_item_max=0,leaf_page_max=1MB," + "prefix_compression=true,split_pct=75,type=btree,value_format=u," + "value_format=u"; + +const char * +__wt_confchk_session_create = + "allocation_size=(type=int,min=512B,max=128MB),block_compressor=()," + "checksum=(type=boolean),colgroups=(type=list),collator=()," + "columns=(type=list),columns=(type=list),exclusive=(type=boolean)," + "filename=(),huffman_key=(),huffman_value=(),internal_item_max=(type=int," + "min=0),internal_key_truncate=(type=boolean),internal_page_max=(type=int," + "min=512B,max=512MB),key_format=(type=format),key_format=(type=format)," + "key_gap=(type=int,min=0),leaf_item_max=(type=int,min=0)," + "leaf_page_max=(type=int,min=512B,max=512MB)," + "prefix_compression=(type=boolean),split_pct=(type=int,min=25,max=100)," + "type=(choices=[\"btree\"]),value_format=(type=format)," + "value_format=(type=format)"; + +const char * +__wt_confdfl_session_drop = + "force=false"; + +const char * +__wt_confchk_session_drop = + "force=(type=boolean)"; + +const char * +__wt_confdfl_session_dumpfile = + ""; + +const char * +__wt_confchk_session_dumpfile = + ""; + +const char * +__wt_confdfl_session_log_printf = + ""; + +const char * +__wt_confchk_session_log_printf = + ""; + +const char * +__wt_confdfl_session_open_cursor = + "append=false,bulk=false,clear_on_close=false,dump=""," + "isolation=read-committed,overwrite=false,raw=false,statistics=false"; + +const char * +__wt_confchk_session_open_cursor = + "append=(type=boolean),bulk=(type=boolean),clear_on_close=(type=boolean)," + "dump=(choices=[\"hex\",\"print\"]),isolation=(choices=[\"snapshot\"," + "\"read-committed\",\"read-uncommitted\"]),overwrite=(type=boolean)," + "raw=(type=boolean),statistics=(type=boolean)"; + +const char * +__wt_confdfl_session_rename = + ""; + +const char * +__wt_confchk_session_rename = + ""; + +const char * +__wt_confdfl_session_rollback_transaction = + ""; + +const char * +__wt_confchk_session_rollback_transaction = + ""; + +const char * +__wt_confdfl_session_salvage = + "force=false"; + +const char * +__wt_confchk_session_salvage = + "force=(type=boolean)"; + +const char * +__wt_confdfl_session_sync = + ""; + +const char * +__wt_confchk_session_sync = + ""; + +const char * +__wt_confdfl_session_truncate = + ""; + +const char * +__wt_confchk_session_truncate = + ""; + +const char * +__wt_confdfl_session_upgrade = + ""; + +const char * +__wt_confchk_session_upgrade = + ""; + +const char * +__wt_confdfl_session_verify = + ""; + +const char * +__wt_confchk_session_verify = + ""; + +const char * +__wt_confdfl_table_meta = + "colgroups=(),columns=(),key_format=u,value_format=u"; + +const char * +__wt_confchk_table_meta = + "colgroups=(type=list),columns=(type=list),key_format=(type=format)," + "value_format=(type=format)"; + +const char * +__wt_confdfl_wiredtiger_open = + "cache_size=100MB,create=false,error_prefix="",eviction_target=80," + "eviction_trigger=95,extensions=(),hazard_max=30,home_environment=false," + "home_environment_priv=false,logging=false,multiprocess=false," + "session_max=50,transactional=false,verbose=()"; + +const char * +__wt_confchk_wiredtiger_open = + "cache_size=(type=int,min=1MB,max=10TB),create=(type=boolean)," + "error_prefix=(),eviction_target=(type=int,min=10,max=99)," + "eviction_trigger=(type=int,min=10,max=99),extensions=(type=list)," + "hazard_max=(type=int,min=15),home_environment=(type=boolean)," + "home_environment_priv=(type=boolean),logging=(type=boolean)," + "multiprocess=(type=boolean),session_max=(type=int,min=1)," + "transactional=(type=boolean),verbose=(type=list,choices=[\"block\"," + "\"evict\",\"evictserver\",\"fileops\",\"hazard\",\"mutex\",\"read\"," + "\"readserver\",\"reconcile\",\"salvage\",\"verify\",\"write\"])"; diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c new file mode 100644 index 00000000000..f93c431e144 --- /dev/null +++ b/src/conn/conn_api.c @@ -0,0 +1,801 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static int __conn_config(WT_CONNECTION_IMPL *, const char **, WT_ITEM **); +static int __conn_home(WT_CONNECTION_IMPL *, const char *, const char **); +static int __conn_single(WT_CONNECTION_IMPL *, const char **); + +/* + * api_err_printf -- + * Extension API call to print to the error stream. + */ +static void +__api_err_printf(WT_SESSION *wt_session, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + __wt_eventv((WT_SESSION_IMPL *)wt_session, 0, 0, NULL, 0, fmt, ap); + va_end(ap); +} + +static WT_EXTENSION_API __api = { + __api_err_printf, + __wt_scr_alloc_ext, + __wt_scr_free_ext +}; + +/* + * __conn_load_extension -- + * WT_CONNECTION->load_extension method. + */ +static int +__conn_load_extension( + WT_CONNECTION *wt_conn, const char *path, const char *config) +{ + WT_CONFIG_ITEM cval; + WT_CONNECTION_IMPL *conn; + WT_DLH *dlh; + WT_SESSION_IMPL *session; + int (*entry)(WT_SESSION *, WT_EXTENSION_API *, const char *); + int ret; + const char *entry_name; + + dlh = NULL; + + conn = (WT_CONNECTION_IMPL *)wt_conn; + CONNECTION_API_CALL(conn, session, load_extension, config, cfg); + + entry_name = NULL; + WT_ERR(__wt_config_gets(session, cfg, "entry", &cval)); + WT_ERR(__wt_strndup(session, cval.str, cval.len, &entry_name)); + + /* + * This assumes the underlying shared libraries are reference counted, + * that is, that re-opening a shared library simply increments a ref + * count, and closing it simply decrements the ref count, and the last + * close discards the reference entirely -- in other words, we do not + * check to see if we've already opened this shared library. + */ + WT_ERR(__wt_dlopen(session, path, &dlh)); + WT_ERR(__wt_dlsym(session, dlh, entry_name, &entry)); + + /* Call the entry function. */ + WT_ERR(entry(&session->iface, &__api, config)); + + /* Link onto the environment's list of open libraries. */ + __wt_spin_lock(session, &conn->spinlock); + TAILQ_INSERT_TAIL(&conn->dlhqh, dlh, q); + __wt_spin_unlock(session, &conn->spinlock); + + if (0) { +err: if (dlh != NULL) + (void)__wt_dlclose(session, dlh); + } + __wt_free(session, entry_name); + + API_END(session); + + return (ret); +} + +/* + * __conn_add_cursor_type -- + * WT_CONNECTION->add_cursor_type method. + */ +static int +__conn_add_cursor_type(WT_CONNECTION *wt_conn, + const char *prefix, WT_CURSOR_TYPE *ctype, const char *config) +{ + WT_CONNECTION_IMPL *conn; + WT_SESSION_IMPL *session; + int ret; + + WT_UNUSED(prefix); + WT_UNUSED(ctype); + ret = ENOTSUP; + + conn = (WT_CONNECTION_IMPL *)wt_conn; + CONNECTION_API_CALL(conn, session, add_cursor_type, config, cfg); + WT_UNUSED(cfg); +err: API_END(session); + + return (ret); +} + +/* + * __conn_add_collator -- + * WT_CONNECTION->add_collator method. + */ +static int +__conn_add_collator(WT_CONNECTION *wt_conn, + const char *name, WT_COLLATOR *collator, const char *config) +{ + WT_CONNECTION_IMPL *conn; + WT_NAMED_COLLATOR *ncoll; + WT_SESSION_IMPL *session; + int ret; + + conn = (WT_CONNECTION_IMPL *)wt_conn; + CONNECTION_API_CALL(conn, session, add_collator, config, cfg); + WT_UNUSED(cfg); + + WT_ERR(__wt_calloc_def(session, 1, &ncoll)); + WT_ERR(__wt_strdup(session, name, &ncoll->name)); + ncoll->collator = collator; + + __wt_spin_lock(session, &conn->spinlock); + TAILQ_INSERT_TAIL(&conn->collqh, ncoll, q); + __wt_spin_unlock(session, &conn->spinlock); + ncoll = NULL; +err: API_END(session); + __wt_free(session, ncoll); + + return (ret); +} + +/* + * __conn_add_compressor -- + * WT_CONNECTION->add_compressor method. + */ +static int +__conn_add_compressor(WT_CONNECTION *wt_conn, + const char *name, WT_COMPRESSOR *compressor, const char *config) +{ + WT_CONNECTION_IMPL *conn; + WT_SESSION_IMPL *session; + WT_NAMED_COMPRESSOR *ncomp; + int ret; + + WT_UNUSED(name); + WT_UNUSED(compressor); + + conn = (WT_CONNECTION_IMPL *)wt_conn; + CONNECTION_API_CALL(conn, session, add_compressor, config, cfg); + WT_UNUSED(cfg); + + WT_ERR(__wt_calloc_def(session, 1, &ncomp)); + WT_ERR(__wt_strdup(session, name, &ncomp->name)); + ncomp->compressor = compressor; + + __wt_spin_lock(session, &conn->spinlock); + TAILQ_INSERT_TAIL(&conn->compqh, ncomp, q); + __wt_spin_unlock(session, &conn->spinlock); + ncomp = NULL; +err: API_END(session); + __wt_free(session, ncomp); + return (ret); +} + +/* + * __conn_remove_collator -- + * remove collator added by WT_CONNECTION->add_collator, + * only used internally. + */ +static void +__conn_remove_collator(WT_CONNECTION_IMPL *conn, WT_NAMED_COLLATOR *ncoll) +{ + WT_SESSION_IMPL *session; + + session = &conn->default_session; + + /* Remove from the connection's list. */ + TAILQ_REMOVE(&conn->collqh, ncoll, q); + + /* Free associated memory */ + __wt_free(session, ncoll->name); + __wt_free(session, ncoll); +} + +/* + * __conn_remove_compressor -- + * remove compressor added by WT_CONNECTION->add_compressor, + * only used internally. + */ +static void +__conn_remove_compressor(WT_CONNECTION_IMPL *conn, WT_NAMED_COMPRESSOR *ncomp) +{ + WT_SESSION_IMPL *session; + + session = &conn->default_session; + + /* Remove from the connection's list. */ + TAILQ_REMOVE(&conn->compqh, ncomp, q); + + /* Free associated memory */ + __wt_free(session, ncomp->name); + __wt_free(session, ncomp); +} + +/* + * __conn_add_extractor -- + * WT_CONNECTION->add_extractor method. + */ +static int +__conn_add_extractor(WT_CONNECTION *wt_conn, + const char *name, WT_EXTRACTOR *extractor, const char *config) +{ + WT_CONNECTION_IMPL *conn; + WT_SESSION_IMPL *session; + int ret; + + WT_UNUSED(name); + WT_UNUSED(extractor); + ret = ENOTSUP; + + conn = (WT_CONNECTION_IMPL *)wt_conn; + CONNECTION_API_CALL(conn, session, add_extractor, config, cfg); + WT_UNUSED(cfg); +err: API_END(session); + + return (ret); +} + +static const char * +__conn_get_home(WT_CONNECTION *wt_conn) +{ + return (((WT_CONNECTION_IMPL *)wt_conn)->home); +} + +/* + * __conn_is_new -- + * WT_CONNECTION->is_new method. + */ +static int +__conn_is_new(WT_CONNECTION *wt_conn) +{ + return (((WT_CONNECTION_IMPL *)wt_conn)->is_new); +} + +/* + * __conn_close -- + * WT_CONNECTION->close method. + */ +static int +__conn_close(WT_CONNECTION *wt_conn, const char *config) +{ + WT_CONNECTION_IMPL *conn; + WT_SESSION_IMPL *s, *session, **tp; + WT_SESSION *wt_session; + WT_NAMED_COLLATOR *ncoll; + WT_NAMED_COMPRESSOR *ncomp; + int ret; + + ret = 0; + conn = (WT_CONNECTION_IMPL *)wt_conn; + + CONNECTION_API_CALL(conn, session, close, config, cfg); + WT_UNUSED(cfg); + + /* Close open sessions. */ + for (tp = conn->sessions; (s = *tp) != NULL;) { + if (!F_ISSET(s, WT_SESSION_INTERNAL)) { + wt_session = &s->iface; + WT_TRET(wt_session->close(wt_session, config)); + + /* + * We closed a session, which has shuffled pointers + * around. Restart the search. + */ + tp = conn->sessions; + } else + ++tp; + } + + /* Free memory for collators */ + while ((ncoll = TAILQ_FIRST(&conn->collqh)) != NULL) + __conn_remove_collator(conn, ncoll); + + /* Free memory for compressors */ + while ((ncomp = TAILQ_FIRST(&conn->compqh)) != NULL) + __conn_remove_compressor(conn, ncomp); + + WT_TRET(__wt_connection_close(conn)); + /* We no longer have a session, don't try to update it. */ + session = NULL; +err: API_END(session); + + return (ret); +} + +/* + * __conn_open_session -- + * WT_CONNECTION->open_session method. + */ +static int +__conn_open_session(WT_CONNECTION *wt_conn, + WT_EVENT_HANDLER *event_handler, const char *config, + WT_SESSION **wt_sessionp) +{ + WT_CONNECTION_IMPL *conn; + WT_SESSION_IMPL *session, *session_ret; + int ret; + + conn = (WT_CONNECTION_IMPL *)wt_conn; + session_ret = NULL; + ret = 0; + + CONNECTION_API_CALL(conn, session, open_session, config, cfg); + WT_UNUSED(cfg); + + WT_ERR(__wt_open_session(conn, 0, event_handler, config, &session_ret)); + + *wt_sessionp = &session_ret->iface; +err: API_END(session); + + return (ret); +} + +/* + * wiredtiger_open -- + * Main library entry point: open a new connection to a WiredTiger + * database. + */ +int +wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, + const char *config, WT_CONNECTION **wt_connp) +{ + static WT_CONNECTION stdc = { + __conn_load_extension, + __conn_add_cursor_type, + __conn_add_collator, + __conn_add_compressor, + __conn_add_extractor, + __conn_close, + __conn_get_home, + __conn_is_new, + __conn_open_session + }; + static struct { + const char *vname; + uint32_t vflag; + } *vt, verbtypes[] = { + { "block", WT_VERB_block }, + { "evict", WT_VERB_evict }, + { "evictserver",WT_VERB_evictserver }, + { "fileops", WT_VERB_fileops }, + { "hazard", WT_VERB_hazard }, + { "mutex", WT_VERB_mutex }, + { "read", WT_VERB_read }, + { "readserver", WT_VERB_readserver }, + { "reconcile", WT_VERB_reconcile }, + { "salvage", WT_VERB_salvage }, + { "verify", WT_VERB_verify }, + { "write", WT_VERB_write }, + { NULL, 0 } + }; + WT_CONFIG subconfig; + WT_CONFIG_ITEM cval, skey, sval; + WT_CONNECTION_IMPL *conn; + WT_ITEM *cbuf, expath, exconfig; + WT_SESSION_IMPL *session; + int ret; + const char *cfg[] = + { __wt_confdfl_wiredtiger_open, config, NULL, NULL }; + + *wt_connp = NULL; + session = NULL; + cbuf = NULL; + WT_CLEAR(expath); + WT_CLEAR(exconfig); + + WT_RET(__wt_library_init()); + + WT_RET(__wt_calloc_def(NULL, 1, &conn)); + conn->iface = stdc; + + /* + * Immediately link the structure into the connection structure list: + * the only thing ever looked at on that list is the database name, + * and a NULL value is fine. + */ + __wt_spin_lock(NULL, &__wt_process.spinlock); + TAILQ_INSERT_TAIL(&__wt_process.connqh, conn, q); + __wt_spin_unlock(NULL, &__wt_process.spinlock); + + session = &conn->default_session; + session->iface.connection = &conn->iface; + session->name = "wiredtiger_open"; + + /* + * Configure event handling as soon as possible so errors are handled + * correctly. If the application didn't configure an event handler, + * use the default one, and use default entries for any entries not + * set by the application. + */ + if (event_handler == NULL) + event_handler = __wt_event_handler_default; + else { + if (event_handler->handle_error == NULL) + event_handler->handle_error = + __wt_event_handler_default->handle_error; + if (event_handler->handle_message == NULL) + event_handler->handle_message = + __wt_event_handler_default->handle_message; + if (event_handler->handle_progress == NULL) + event_handler->handle_progress = + __wt_event_handler_default->handle_progress; + } + session->event_handler = event_handler; + + /* Remaining basic initialization of the connection structure. */ + WT_ERR(__wt_connection_init(conn)); + + /* Check the configuration strings. */ + WT_ERR( + __wt_config_check(session, __wt_confchk_wiredtiger_open, config)); + + /* Get the database home. */ + WT_ERR(__conn_home(conn, home, cfg)); + + /* Read the database-home configuration file. */ + WT_ERR(__conn_config(conn, cfg, &cbuf)); + + /* Make sure no other thread of control already owns this database. */ + WT_ERR(__conn_single(conn, cfg)); + + WT_ERR(__wt_config_gets(session, cfg, "cache_size", &cval)); + conn->cache_size = cval.val; + WT_ERR(__wt_config_gets(session, cfg, "hazard_max", &cval)); + conn->hazard_size = (uint32_t)cval.val; + WT_ERR(__wt_config_gets(session, cfg, "session_max", &cval)); + conn->session_size = (uint32_t)cval.val; + + /* Configure verbose flags. */ + conn->verbose = 0; +#ifdef HAVE_VERBOSE + WT_ERR(__wt_config_gets(session, cfg, "verbose", &cval)); + for (vt = verbtypes; vt->vname != NULL; vt++) { + WT_ERR(__wt_config_subinit(session, &subconfig, &cval)); + skey.str = vt->vname; + skey.len = strlen(vt->vname); + ret = __wt_config_getraw(&subconfig, &skey, &sval); + if (ret == 0 && sval.val) + FLD_SET(conn->verbose, vt->vflag); + else if (ret != WT_NOTFOUND) + goto err; + } +#endif + + WT_ERR(__wt_config_gets(session, cfg, "logging", &cval)); + if (cval.val != 0) + WT_ERR(__wt_open(session, WT_LOG_FILENAME, 1, &conn->log_fh)); + + /* Load any extensions referenced in the config. */ + WT_ERR(__wt_config_gets(session, cfg, "extensions", &cval)); + WT_ERR(__wt_config_subinit(session, &subconfig, &cval)); + while ((ret = __wt_config_next(&subconfig, &skey, &sval)) == 0) { + WT_ERR(__wt_buf_fmt( + session, &expath, "%.*s", (int)skey.len, skey.str)); + if (sval.len > 0) + WT_ERR(__wt_buf_fmt(session, &exconfig, + "entry=%.*s\n", (int)sval.len, sval.str)); + WT_ERR(conn->iface.load_extension(&conn->iface, + expath.data, (sval.len > 0) ? exconfig.data : NULL)); + } + if (ret == WT_NOTFOUND) + ret = 0; + WT_ERR(ret); + + /* + * Open the connection; if that fails, the connection handle has been + * destroyed by the time the open function returns. + */ + if ((ret = __wt_connection_open(conn, cfg)) != 0) { + conn = NULL; + WT_ERR(ret); + } + + STATIC_ASSERT(offsetof(WT_CONNECTION_IMPL, iface) == 0); + *wt_connp = &conn->iface; + + /* + * Destroying the connection on error will destroy our session handle, + * cleanup using the session handle first, then discard the connection. + */ +err: if (cbuf != NULL) + __wt_buf_free(session, cbuf); + __wt_buf_free(session, &expath); + __wt_buf_free(session, &exconfig); + + if (ret != 0 && conn != NULL) + __wt_connection_destroy(conn); + + return (ret); +} + +/* + * __conn_home -- + * Set the database home directory. + */ +static int +__conn_home(WT_CONNECTION_IMPL *conn, const char *home, const char **cfg) +{ + WT_CONFIG_ITEM cval; + WT_SESSION_IMPL *session; + + session = &conn->default_session; + + /* If the application specifies a home directory, use it. */ + if (home != NULL) + goto copy; + + /* If there's no WIREDTIGER_HOME environment variable, use ".". */ + if ((home = getenv("WIREDTIGER_HOME")) == NULL) { + home = "."; + goto copy; + } + + /* + * Security stuff: + * + * If the "home_environment" configuration string is set, use the + * environment variable for all processes. + */ + WT_RET(__wt_config_gets(session, cfg, "home_environment", &cval)); + if (cval.val != 0) + goto copy; + + /* + * If the "home_environment_priv" configuration string is set, use the + * environment variable if the process has appropriate privileges. + */ + WT_RET(__wt_config_gets(session, cfg, "home_environment_priv", &cval)); + if (cval.val == 0) + WT_RET_MSG(session, WT_ERROR, "%s", + "WIREDTIGER_HOME environment variable set but WiredTiger " + "not configured to use that environment variable"); + + if (!__wt_has_priv()) + WT_RET_MSG(session, WT_ERROR, "%s", + "WIREDTIGER_HOME environment variable set but process " + "lacks privileges to use that environment variable"); + +copy: return (__wt_strdup(session, home, &conn->home)); +} + +/* + * __conn_single -- + * Confirm that no other thread of control is using this database. + */ +static int +__conn_single(WT_CONNECTION_IMPL *conn, const char **cfg) +{ + WT_CONFIG_ITEM cval; + WT_CONNECTION_IMPL *t; + WT_SESSION_IMPL *session; + off_t size; + uint32_t len; + int created, ret; + char buf[256]; + + session = &conn->default_session; + +#define WT_FLAGFILE "WiredTiger" + /* + * Optionally create the wiredtiger flag file if it doesn't already + * exist. We don't actually care if we create it or not, the "am I the + * only locker" tests are all that matter. + */ + WT_RET(__wt_config_gets(session, cfg, "create", &cval)); + WT_RET(__wt_open(session, + WT_FLAGFILE, cval.val == 0 ? 0 : 1, &conn->lock_fh)); + + /* + * Lock a byte of the file: if we don't get the lock, some other process + * is holding it, we're done. Note the file may be zero-length length, + * and that's OK, the underlying call supports acquisition of locks past + * the end-of-file. + */ + if (__wt_bytelock(conn->lock_fh, (off_t)0, 1) != 0) + WT_ERR_MSG(session, EBUSY, "%s", + "WiredTiger database is already being managed by another " + "process"); + + /* Check to see if another thread of control has this database open. */ + ret = 0; + __wt_spin_lock(session, &__wt_process.spinlock); + TAILQ_FOREACH(t, &__wt_process.connqh, q) + if (t->home != NULL && + t != conn && strcmp(t->home, conn->home) == 0) { + ret = EBUSY; + break; + } + __wt_spin_unlock(session, &__wt_process.spinlock); + if (ret != 0) + WT_ERR_MSG(session, EBUSY, "%s", + "WiredTiger database is already being managed by another " + "thread in this process"); + + /* + * If the size of the file is 0, we created it (or we're racing with + * the thread that created it, it doesn't matter), write some bytes + * into the file. Strictly speaking, this isn't even necessary, but + * zero-length files always make me nervous. + */ + WT_ERR(__wt_filesize(session, conn->lock_fh, &size)); + if (size == 0) { + len = (uint32_t)snprintf(buf, sizeof(buf), "%s\n%s\n", + WT_FLAGFILE, wiredtiger_version(NULL, NULL, NULL)); + WT_ERR(__wt_write( + session, conn->lock_fh, (off_t)0, (uint32_t)len, buf)); + created = 1; + } else + created = 0; + + /* + * If we found a zero-length WiredTiger lock file, and eventually ended + * as the database owner, return that we created the database. (There + * is a theoretical chance that another process created the WiredTiger + * lock file but we won the race to add the WT_CONNECTION_IMPL structure + * to the process' list. It doesn't much matter, only one thread will + * be told it created the database.) + */ + conn->is_new = created; + + return (0); + +err: if (conn->lock_fh != NULL) { + (void)__wt_close(session, conn->lock_fh); + conn->lock_fh = NULL; + } + return (ret); +} + +/* + * __conn_config -- + * Read in any WiredTiger_config file in the home directory. + */ +static int +__conn_config(WT_CONNECTION_IMPL *conn, const char **cfg, WT_ITEM **cbufp) +{ + WT_ITEM *cbuf; + WT_FH *fh; + WT_SESSION_IMPL *session; + off_t size; + uint32_t len; + int exist, quoted, ret; + uint8_t *p, *t; + + *cbufp = NULL; /* Returned buffer */ + + cbuf = NULL; + fh = NULL; + session = &conn->default_session; + ret = 0; + + /* Check for an optional configuration file. */ +#define WT_CONFIGFILE "WiredTiger.config" + WT_RET(__wt_exist(session, WT_CONFIGFILE, &exist)); + if (!exist) + return (0); + + /* Open the configuration file. */ + WT_RET(__wt_open(session, WT_CONFIGFILE, 0, &fh)); + WT_ERR(__wt_filesize(session, fh, &size)); + if (size == 0) + goto err; + + /* + * Sanity test: a 100KB configuration file would be insane. (There's + * no practical reason to limit the file size, but I can either limit + * the file size to something rational, or I can add code to test if + * the off_t size is larger than a uint32_t, which is more complicated + * and a waste of time.) + */ + if (size > 100 * 1024) + WT_ERR_MSG(session, EFBIG, WT_CONFIGFILE); + len = (uint32_t)size; + + /* + * Copy the configuration file into memory, with a little slop, I'm not + * interested in debugging off-by-ones. + * + * The beginning of a file is the same as if we run into an unquoted + * newline character, simplify the parsing loop by pretending that's + * what we're doing. + */ + WT_ERR(__wt_scr_alloc(session, len + 10, &cbuf)); + WT_ERR( + __wt_read(session, fh, (off_t)0, len, ((uint8_t *)cbuf->mem) + 1)); + ((uint8_t *)cbuf->mem)[0] = '\n'; + cbuf->size = len + 1; + + /* + * Collapse the file's lines into a single string: newline characters + * are replaced with commas unless the newline is quoted or backslash + * escaped. Comment lines (an unescaped newline where the next non- + * white-space character is a hash), are discarded. + */ + for (quoted = 0, p = t = cbuf->mem; len > 0;) { + /* + * Backslash pairs pass through untouched, unless immediately + * preceding a newline, in which case both the backslash and + * the newline are discarded. Backslash characters escape + * quoted characters, too, that is, a backslash followed by a + * quote doesn't start or end a quoted string. + */ + if (*p == '\\' && len > 1) { + if (p[1] != '\n') { + *t++ = p[0]; + *t++ = p[1]; + } + p += 2; + len -= 2; + continue; + } + + /* + * If we're in a quoted string, or starting a quoted string, + * take all characters, including white-space and newlines. + */ + if (quoted || *p == '"') { + if (*p == '"') + quoted = !quoted; + *t++ = *p++; + --len; + continue; + } + + /* Everything else gets taken, except for newline characters. */ + if (*p != '\n') { + *t++ = *p++; + --len; + continue; + } + + /* + * Replace any newline characters with commas (and strings of + * commas are safe). + * + * After any newline, skip to a non-white-space character; if + * the next character is a hash mark, skip to the next newline. + */ + for (;;) { + for (*t++ = ','; --len > 0 && isspace(*++p);) + ; + if (len == 0) + break; + if (*p != '#') + break; + while (--len > 0 && *++p != '\n') + ; + if (len == 0) + break; + } + } + *t = '\0'; + +#if 0 + fprintf(stderr, "file config: {%s}\n", (char *)cbuf->data); + exit(0); +#endif + + /* Check the configuration string. */ + WT_ERR(__wt_config_check( + session, __wt_confchk_wiredtiger_open, cbuf->data)); + + /* + * The configuration file falls between the default configuration and + * the wiredtiger_open() configuration, overriding the defaults but not + * overriding the wiredtiger_open() configuration. + */ + cfg[2] = cfg[1]; + cfg[1] = cbuf->data; + + *cbufp = cbuf; + + if (0) { +err: if (cbuf != NULL) + __wt_buf_free(session, cbuf); + } + if (fh != NULL) + WT_TRET(__wt_close(session, fh)); + return (ret); +} diff --git a/src/conn/conn_handle.c b/src/conn/conn_handle.c new file mode 100644 index 00000000000..a55e4ebe3e0 --- /dev/null +++ b/src/conn/conn_handle.c @@ -0,0 +1,81 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_connection_init -- + * Structure initialization for a just-created WT_CONNECTION_IMPL handle. + */ +int +__wt_connection_init(WT_CONNECTION_IMPL *conn) +{ + WT_SESSION_IMPL *session; + + session = &conn->default_session; + + TAILQ_INIT(&conn->btqh); /* WT_BTREE list */ + TAILQ_INIT(&conn->dlhqh); /* Library list */ + TAILQ_INIT(&conn->fhqh); /* File list */ + TAILQ_INIT(&conn->collqh); /* Collator list */ + TAILQ_INIT(&conn->compqh); /* Compressor list */ + + /* Statistics. */ + WT_RET(__wt_stat_alloc_connection_stats(session, &conn->stats)); + + /* Serialized function call spinlock. */ + __wt_spin_init(session, &conn->serial_lock); + + /* Connection spinlock. */ + __wt_spin_init(session, &conn->spinlock); + + return (0); +} + +/* + * __wt_connection_destroy -- + * Destroy the connection's underlying WT_CONNECTION_IMPL structure. + */ +void +__wt_connection_destroy(WT_CONNECTION_IMPL *conn) +{ + WT_SESSION_IMPL *session; + + session = &conn->default_session; + + /* Check there's something to destroy. */ + if (conn == NULL) + return; + + /* + * Close remaining open files (before discarding the mutex, the + * underlying file-close code uses the mutex to guard lists of + * open files. + */ + if (conn->lock_fh != NULL) + (void)__wt_close(session, conn->lock_fh); + + if (conn->log_fh != NULL) + (void)__wt_close(session, conn->log_fh); + + /* Remove from the list of connections. */ + __wt_spin_lock(session, &__wt_process.spinlock); + TAILQ_REMOVE(&__wt_process.connqh, conn, q); + __wt_spin_unlock(session, &__wt_process.spinlock); + + __wt_spin_destroy(session, &conn->serial_lock); + __wt_spin_destroy(session, &conn->spinlock); + + /* Free allocated memory. */ + __wt_free(session, conn->home); + __wt_free(session, conn->sessions); + __wt_free(session, conn->session_array); + __wt_free(session, conn->hazard); + __wt_free(session, conn->stats); + + __wt_free(NULL, conn); +} diff --git a/src/conn/conn_open.c b/src/conn/conn_open.c new file mode 100644 index 00000000000..fb6833ae249 --- /dev/null +++ b/src/conn/conn_open.c @@ -0,0 +1,122 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_connection_open -- + * Open a connection. + */ +int +__wt_connection_open(WT_CONNECTION_IMPL *conn, const char *cfg[]) +{ + WT_SESSION_IMPL *session; + int ret; + + /* Default session. */ + conn->default_session.iface.connection = &conn->iface; + + session = &conn->default_session; + ret = 0; + + /* WT_SESSION_IMPL and hazard arrays. */ + WT_ERR(__wt_calloc(session, + conn->session_size, sizeof(WT_SESSION_IMPL *), &conn->sessions)); + WT_ERR(__wt_calloc(session, + conn->session_size, sizeof(WT_SESSION_IMPL), + &conn->session_array)); + WT_ERR(__wt_calloc(session, + conn->session_size * conn->hazard_size, sizeof(WT_HAZARD), + &conn->hazard)); + + /* Create the cache. */ + WT_ERR(__wt_cache_create(conn, cfg)); + + /* + * Publish: there must be a barrier to ensure the connection structure + * fields are set before other threads read from the pointer. + */ + WT_WRITE_BARRIER(); + + /* Start worker threads. */ + F_SET(conn, WT_SERVER_RUN); + WT_ERR(__wt_thread_create( + &conn->cache_evict_tid, __wt_cache_evict_server, conn)); + + return (0); + +err: (void)__wt_connection_close(conn); + return (ret); +} + +/* + * __wt_connection_close -- + * Close a connection handle. + */ +int +__wt_connection_close(WT_CONNECTION_IMPL *conn) +{ + WT_BTREE *btree; + WT_SESSION_IMPL *session; + WT_DLH *dlh; + WT_FH *fh; + int ret; + + session = &conn->default_session; + ret = 0; + + /* Complain if WT_BTREE handles weren't closed. */ + while ((btree = TAILQ_FIRST(&conn->btqh)) != NULL) { + if (F_ISSET(btree, WT_BTREE_OPEN)) + __wt_errx(session, + "Connection has an open btree handle: %s", + btree->name); + WT_SET_BTREE_IN_SESSION(session, btree); + + /* + * XXX + * This won't work, you can't close a btree handle with the + * default session. + */ + WT_TRET(__wt_btree_close(session)); + } + + /* + * Complain if files weren't closed (ignoring the lock and logging + * files, we'll close them in a minute. + */ + TAILQ_FOREACH(fh, &conn->fhqh, q) { + if (fh == conn->lock_fh || fh == conn->log_fh) + continue; + + __wt_errx(session, + "Connection has open file handles: %s", fh->name); + WT_TRET(__wt_close(session, fh)); + fh = TAILQ_FIRST(&conn->fhqh); + } + + /* Shut down the server threads. */ + F_CLR(conn, WT_SERVER_RUN); + if (conn->cache_evict_tid != 0) { + __wt_evict_server_wake(session); + WT_TRET(__wt_thread_join(conn->cache_evict_tid)); + } + + /* Discard the cache. */ + __wt_cache_destroy(conn); + + /* Close extensions. */ + while ((dlh = TAILQ_FIRST(&conn->dlhqh)) != NULL) { + TAILQ_REMOVE(&conn->dlhqh, dlh, q); + WT_TRET(__wt_dlclose(session, dlh)); + } + + /* Destroy the handle. */ + __wt_connection_destroy(conn); + + return (ret); +} diff --git a/src/conn/conn_stat.c b/src/conn/conn_stat.c new file mode 100644 index 00000000000..451803f4e5a --- /dev/null +++ b/src/conn/conn_stat.c @@ -0,0 +1,22 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_conn_stat_init -- + * Initialize the per-connection statistics. + */ +void +__wt_conn_stat_init(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + + conn = S2C(session); + + __wt_cache_stats_update(conn); +} diff --git a/src/cursor/cur_bulk.c b/src/cursor/cur_bulk.c new file mode 100644 index 00000000000..474a7e0b413 --- /dev/null +++ b/src/cursor/cur_bulk.c @@ -0,0 +1,74 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __curbulk_insert -- + * WT_CURSOR->insert for the bulk cursor type. + */ +static int +__curbulk_insert(WT_CURSOR *cursor) +{ + WT_BTREE *btree; + WT_CURSOR_BULK *cbulk; + WT_SESSION_IMPL *session; + int ret; + + cbulk = (WT_CURSOR_BULK *)cursor; + btree = cbulk->cbt.btree; + CURSOR_API_CALL(cursor, session, insert, btree); + if (btree->type == BTREE_ROW) + WT_CURSOR_NEEDKEY(cursor); + WT_CURSOR_NEEDVALUE(cursor); + WT_ERR(__wt_bulk_insert(cbulk)); +err: API_END(session); + + return (ret); +} + +/* + * __curbulk_close -- + * WT_CURSOR->close for the bulk cursor type. + */ +static int +__curbulk_close(WT_CURSOR *cursor) +{ + WT_BTREE *btree; + WT_CURSOR_BULK *cbulk; + WT_SESSION_IMPL *session; + int ret; + + cbulk = (WT_CURSOR_BULK *)cursor; + btree = cbulk->cbt.btree; + + CURSOR_API_CALL(cursor, session, close, btree); + WT_TRET(__wt_bulk_end(cbulk)); + if (session->btree != NULL) + WT_TRET(__wt_session_release_btree(session)); + /* The URI is owned by the btree handle. */ + cursor->uri = NULL; + WT_TRET(__wt_cursor_close(cursor)); + API_END(session); + + return (ret); +} + +/* + * __wt_curbulk_init -- + * Initialize a bulk cursor. + */ +int +__wt_curbulk_init(WT_CURSOR_BULK *cbulk) +{ + WT_CURSOR *c = &cbulk->cbt.iface; + + c->insert = __curbulk_insert; + c->close = __curbulk_close; + + return (__wt_bulk_init(cbulk)); +} diff --git a/src/cursor/cur_config.c b/src/cursor/cur_config.c new file mode 100644 index 00000000000..c4a64d2387d --- /dev/null +++ b/src/cursor/cur_config.c @@ -0,0 +1,77 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __curconfig_close -- + * WT_CURSOR->close method for the config cursor type. + */ +static int +__curconfig_close(WT_CURSOR *cursor) +{ + return (__wt_cursor_close(cursor)); +} + +/* + * __wt_curconfig_open -- + * WT_SESSION->open_cursor method for config cursors. + */ +int +__wt_curconfig_open(WT_SESSION_IMPL *session, + const char *uri, const char *cfg[], WT_CURSOR **cursorp) +{ + static WT_CURSOR iface = { + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, /* equals */ + __wt_cursor_notsup, /* next */ + __wt_cursor_notsup, /* prev */ + __wt_cursor_notsup, /* reset */ + __wt_cursor_notsup, /* search */ + /* search-near */ + (int (*)(WT_CURSOR *, int *))__wt_cursor_notsup, + __wt_cursor_notsup, /* insert */ + __wt_cursor_notsup, /* update */ + __wt_cursor_notsup, /* remove */ + __curconfig_close, + { NULL, NULL }, /* TAILQ_ENTRY q */ + 0, /* recno key */ + { 0 }, /* recno raw buffer */ + { NULL, 0, 0, NULL, 0 },/* WT_ITEM key */ + { NULL, 0, 0, NULL, 0 },/* WT_ITEM value */ + 0, /* int saved_err */ + 0 /* uint32_t flags */ + }; + WT_CURSOR_CONFIG *cconfig; + WT_CURSOR *cursor; + int ret; + + WT_UNUSED(uri); + + WT_RET(__wt_calloc_def(session, 1, &cconfig)); + + cursor = &cconfig->iface; + *cursor = iface; + cursor->session = &session->iface; + cursor->key_format = cursor->value_format = "S"; + + STATIC_ASSERT(offsetof(WT_CURSOR_CONFIG, iface) == 0); + WT_ERR(__wt_cursor_init(cursor, uri, 0, 1, cfg)); + *cursorp = cursor; + + if (0) { +err: __wt_free(session, cconfig); + } + return (ret); +} diff --git a/src/cursor/cur_dump.c b/src/cursor/cur_dump.c new file mode 100644 index 00000000000..477bb70e4a5 --- /dev/null +++ b/src/cursor/cur_dump.c @@ -0,0 +1,278 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __raw_to_dump -- + * We have a buffer where the data item contains a raw value, convert it + * to a printable string. + */ +static int +__raw_to_dump( + WT_SESSION_IMPL *session, WT_ITEM *from, WT_ITEM *to, int hexonly) +{ + WT_ITEM *tmp; + uint32_t size; + + /* + * In the worst case, every character takes up 3 spaces, plus a + * trailing nul byte. + */ + WT_RET(__wt_scr_alloc(session, from->size * 3 + 10, &tmp)); + size = from->size; + if (hexonly) + __wt_raw_to_hex(from->data, tmp->mem, &size); + else + __wt_raw_to_esc_hex(from->data, tmp->mem, &size); + tmp->size = size; + + __wt_buf_swap(to, tmp); + __wt_scr_free(&tmp); + + return (0); +} + +/* + * __dump_to_raw -- + * We have a scratch buffer where the data item contains a dump string, + * convert it to a raw value. + */ +static int +__dump_to_raw( + WT_SESSION_IMPL *session, const char *src_arg, WT_ITEM *item, int hexonly) +{ + uint32_t size; + + /* + * XXX + * Overwrite the string in place: the underlying cursor set_key and + * set_value functions are going to use the cursor's key and value + * buffers, which means we can't. This should probably be fixed by + * layering the dump cursor on top of other cursors and then we can + * use the dump cursor's key/value buffers. + */ + if (hexonly) + WT_RET(__wt_hex_to_raw( + session, (void *)src_arg, (void *)src_arg, &size)); + else + WT_RET(__wt_esc_hex_to_raw( + session, (void *)src_arg, (void *)src_arg, &size)); + + memset(item, 0, sizeof(WT_ITEM)); + item->data = src_arg; + item->size = size; + return (0); +} + +/* + * __curdump_get_key -- + * WT_CURSOR->get_key for dump cursors. + */ +static int +__curdump_get_key(WT_CURSOR *cursor, ...) +{ + WT_ITEM item, *itemp; + WT_SESSION_IMPL *session; + int ret; + uint64_t recno; + va_list ap; + + CURSOR_API_CALL(cursor, session, get_key, NULL); + + if (WT_CURSOR_RECNO(cursor) && !F_ISSET(cursor, WT_CURSTD_RAW)) { + if (F_ISSET(cursor, WT_CURSTD_TABLE)) + WT_ERR(__wt_curtable_get_key(cursor, &recno)); + else + WT_ERR(__wt_cursor_get_key(cursor, &recno)); + + WT_ERR(__wt_buf_fmt(session, &cursor->key, "%" PRIu64, recno)); + } else { + if (F_ISSET(cursor, WT_CURSTD_TABLE)) + WT_ERR(__wt_curtable_get_key(cursor, &item)); + else + WT_ERR(__wt_cursor_get_key(cursor, &item)); + + WT_ERR(__raw_to_dump(session, &item, + &cursor->key, F_ISSET(cursor, WT_CURSTD_DUMP_HEX) ? 1 : 0)); + } + + va_start(ap, cursor); + if (F_ISSET(cursor, WT_CURSTD_RAW)) { + itemp = va_arg(ap, WT_ITEM *); + itemp->data = cursor->key.data; + itemp->size = cursor->key.size; + } else + *va_arg(ap, const char **) = cursor->key.data; + va_end(ap); + +err: API_END(session); + return (ret); +} + +/* + * str2recno -- + * Convert a string to a record number. + */ +static int +str2recno(WT_SESSION_IMPL *session, const char *p, uint64_t *recnop) +{ + uint64_t recno; + char *endptr; + + /* + * strtouq takes lots of things like hex values, signs and so on and so + * forth -- none of them are OK with us. Check the string starts with + * digit, that turns off the special processing. + */ + if (!isdigit(p[0])) + goto format; + + errno = 0; + recno = strtouq(p, &endptr, 0); + if (recno == ULLONG_MAX && errno == ERANGE) + WT_RET_MSG(session, ERANGE, "%s: invalid record number", p); + if (endptr[0] != '\0') +format: WT_RET_MSG(session, EINVAL, "%s: invalid record number", p); + + *recnop = recno; + return (0); +} + +/* + * __curdump_set_key -- + * WT_CURSOR->set_key for dump cursors. + */ +static void +__curdump_set_key(WT_CURSOR *cursor, ...) +{ + WT_ITEM item; + WT_SESSION_IMPL *session; + uint64_t recno; + va_list ap; + const char *p; + int ret; + + CURSOR_API_CALL(cursor, session, set_key, NULL); + + va_start(ap, cursor); + if (F_ISSET(cursor, WT_CURSTD_RAW)) + p = va_arg(ap, WT_ITEM *)->data; + else + p = va_arg(ap, const char *); + va_end(ap); + + if (WT_CURSOR_RECNO(cursor) && !F_ISSET(cursor, WT_CURSTD_RAW)) { + WT_ERR(str2recno(session, p, &recno)); + + if (F_ISSET(cursor, WT_CURSTD_TABLE)) + __wt_curtable_set_key(cursor, recno); + else + __wt_cursor_set_key(cursor, recno); + } else { + WT_ERR(__dump_to_raw(session, + p, &item, F_ISSET(cursor, WT_CURSTD_DUMP_HEX) ? 1 : 0)); + + if (F_ISSET(cursor, WT_CURSTD_TABLE)) + __wt_curtable_set_key(cursor, &item); + else + __wt_cursor_set_key(cursor, &item); + } + + if (0) { +err: cursor->saved_err = ret; + F_CLR(cursor, WT_CURSTD_KEY_SET); + } + + API_END(session); +} + +/* + * __curdump_get_value -- + * WT_CURSOR->get_value for dump cursors. + */ +static int +__curdump_get_value(WT_CURSOR *cursor, ...) +{ + WT_ITEM item, *itemp; + WT_SESSION_IMPL *session; + va_list ap; + int ret; + + CURSOR_API_CALL(cursor, session, get_value, NULL); + + if (F_ISSET(cursor, WT_CURSTD_TABLE)) + WT_ERR(__wt_curtable_get_value(cursor, &item)); + else + WT_ERR(__wt_cursor_get_value(cursor, &item)); + + WT_ERR(__raw_to_dump(session, &item, + &cursor->value, F_ISSET(cursor, WT_CURSTD_DUMP_HEX) ? 1 : 0)); + + va_start(ap, cursor); + if (F_ISSET(cursor, WT_CURSTD_RAW)) { + itemp = va_arg(ap, WT_ITEM *); + itemp->data = cursor->value.data; + itemp->size = cursor->value.size; + } else + *va_arg(ap, const char **) = cursor->value.data; + va_end(ap); + +err: API_END(session); + return (ret); +} + +/* + * __curdump_set_value -- + * WT_CURSOR->set_value for dump cursors. + */ +static void +__curdump_set_value(WT_CURSOR *cursor, ...) +{ + WT_ITEM item; + WT_SESSION_IMPL *session; + va_list ap; + int ret; + const char *p; + + CURSOR_API_CALL(cursor, session, set_value, NULL); + + va_start(ap, cursor); + if (F_ISSET(cursor, WT_CURSTD_RAW)) + p = va_arg(ap, WT_ITEM *)->data; + else + p = va_arg(ap, const char *); + va_end(ap); + + WT_ERR(__dump_to_raw(session, + p, &item, F_ISSET(cursor, WT_CURSTD_DUMP_HEX) ? 1 : 0)); + + if (F_ISSET(cursor, WT_CURSTD_TABLE)) + __wt_curtable_set_value(cursor, &item); + else + __wt_cursor_set_value(cursor, &item); + + if (0) { +err: cursor->saved_err = ret; + F_CLR(cursor, WT_CURSTD_VALUE_SET); + } + + API_END(session); +} + +/* + * __wt_curdump_init -- + * initialize a dump cursor. + */ +void +__wt_curdump_init(WT_CURSOR *cursor) +{ + cursor->get_key = __curdump_get_key; + cursor->get_value = __curdump_get_value; + cursor->set_key = __curdump_set_key; + cursor->set_value = __curdump_set_value; +} diff --git a/src/cursor/cur_file.c b/src/cursor/cur_file.c new file mode 100644 index 00000000000..06ffb47328c --- /dev/null +++ b/src/cursor/cur_file.c @@ -0,0 +1,317 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __curfile_next -- + * WT_CURSOR->next method for the btree cursor type. + */ +static int +__curfile_next(WT_CURSOR *cursor) +{ + WT_CURSOR_BTREE *cbt; + WT_SESSION_IMPL *session; + int ret; + + cbt = (WT_CURSOR_BTREE *)cursor; + CURSOR_API_CALL(cursor, session, next, cbt->btree); + ret = __wt_btcur_next((WT_CURSOR_BTREE *)cursor); + API_END(session); + + return (ret); +} + +/* + * __curfile_prev -- + * WT_CURSOR->prev method for the btree cursor type. + */ +static int +__curfile_prev(WT_CURSOR *cursor) +{ + WT_CURSOR_BTREE *cbt; + WT_SESSION_IMPL *session; + int ret; + + cbt = (WT_CURSOR_BTREE *)cursor; + CURSOR_API_CALL(cursor, session, prev, cbt->btree); + ret = __wt_btcur_prev((WT_CURSOR_BTREE *)cursor); + API_END(session); + + return (ret); +} + +/* + * __curfile_reset -- + * WT_CURSOR->reset method for the btree cursor type. + */ +static int +__curfile_reset(WT_CURSOR *cursor) +{ + WT_CURSOR_BTREE *cbt; + WT_SESSION_IMPL *session; + int ret; + + cbt = (WT_CURSOR_BTREE *)cursor; + CURSOR_API_CALL(cursor, session, reset, cbt->btree); + ret = __wt_btcur_reset(cbt); + API_END(session); + + return (ret); +} + +/* + * __curfile_search -- + * WT_CURSOR->search method for the btree cursor type. + */ +static int +__curfile_search(WT_CURSOR *cursor) +{ + WT_CURSOR_BTREE *cbt; + WT_SESSION_IMPL *session; + int ret; + + cbt = (WT_CURSOR_BTREE *)cursor; + CURSOR_API_CALL(cursor, session, search, cbt->btree); + WT_CURSOR_NEEDKEY(cursor); + ret = __wt_btcur_search(cbt); +err: API_END(session); + + return (ret); +} + +/* + * __curfile_search_near -- + * WT_CURSOR->search_near method for the btree cursor type. + */ +static int +__curfile_search_near(WT_CURSOR *cursor, int *exact) +{ + WT_CURSOR_BTREE *cbt; + WT_SESSION_IMPL *session; + int ret; + + cbt = (WT_CURSOR_BTREE *)cursor; + CURSOR_API_CALL(cursor, session, search_near, cbt->btree); + WT_CURSOR_NEEDKEY(cursor); + ret = __wt_btcur_search_near(cbt, exact); +err: API_END(session); + + return (ret); +} + +/* + * __curfile_insert -- + * WT_CURSOR->insert method for the btree cursor type. + */ +static int +__curfile_insert(WT_CURSOR *cursor) +{ + WT_CURSOR_BTREE *cbt; + WT_SESSION_IMPL *session; + int ret; + + cbt = (WT_CURSOR_BTREE *)cursor; + CURSOR_API_CALL(cursor, session, insert, cbt->btree); + if (!F_ISSET(cursor, WT_CURSTD_APPEND)) + WT_CURSOR_NEEDKEY(cursor); + WT_CURSOR_NEEDVALUE(cursor); + ret = __wt_btcur_insert((WT_CURSOR_BTREE *)cursor); +err: API_END(session); + + return (ret); +} + +/* + * __curfile_update -- + * WT_CURSOR->update method for the btree cursor type. + */ +static int +__curfile_update(WT_CURSOR *cursor) +{ + WT_CURSOR_BTREE *cbt; + WT_SESSION_IMPL *session; + int ret; + + cbt = (WT_CURSOR_BTREE *)cursor; + CURSOR_API_CALL(cursor, session, update, cbt->btree); + WT_CURSOR_NEEDKEY(cursor); + WT_CURSOR_NEEDVALUE(cursor); + ret = __wt_btcur_update((WT_CURSOR_BTREE *)cursor); +err: API_END(session); + + return (ret); +} + +/* + * __curfile_remove -- + * WT_CURSOR->remove method for the btree cursor type. + */ +static int +__curfile_remove(WT_CURSOR *cursor) +{ + WT_CURSOR_BTREE *cbt; + WT_SESSION_IMPL *session; + int ret; + + cbt = (WT_CURSOR_BTREE *)cursor; + CURSOR_API_CALL(cursor, session, remove, cbt->btree); + WT_CURSOR_NEEDKEY(cursor); + ret = __wt_btcur_remove((WT_CURSOR_BTREE *)cursor); +err: API_END(session); + + return (ret); +} + +/* + * __curfile_close -- + * WT_CURSOR->close method for the btree cursor type. + */ +static int +__curfile_close(WT_CURSOR *cursor) +{ + WT_CURSOR_BTREE *cbt; + WT_SESSION_IMPL *session; + int ret; + + cbt = (WT_CURSOR_BTREE *)cursor; + CURSOR_API_CALL(cursor, session, close, cbt->btree); + WT_TRET(__wt_btcur_close(cbt)); + if (session->btree != NULL) + WT_TRET(__wt_session_release_btree(session)); + /* The URI is owned by the btree handle. */ + cursor->uri = NULL; + WT_TRET(__wt_cursor_close(cursor)); + API_END(session); + + return (ret); +} + +/* + * __wt_curfile_create -- + * Open a cursor for a given btree handle. + */ +int +__wt_curfile_create(WT_SESSION_IMPL *session, + const char *cfg[], WT_CURSOR **cursorp) +{ + static WT_CURSOR iface = { + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + __curfile_next, + __curfile_prev, + __curfile_reset, + __curfile_search, + __curfile_search_near, + __curfile_insert, + __curfile_update, + __curfile_remove, + __curfile_close, + { NULL, NULL }, /* TAILQ_ENTRY q */ + 0, /* recno key */ + { 0 }, /* recno raw buffer */ + { NULL, 0, 0, NULL, 0 },/* WT_ITEM key */ + { NULL, 0, 0, NULL, 0 },/* WT_ITEM value */ + 0, /* int saved_err */ + 0 /* uint32_t flags */ + }; + WT_BTREE *btree; + WT_CONFIG_ITEM cval; + WT_CURSOR *cursor; + WT_CURSOR_BTREE *cbt; + size_t csize; + int bulk, ret; + + cbt = NULL; + ret = 0; + + btree = session->btree; + WT_ASSERT(session, btree != NULL); + + WT_RET(__wt_config_gets(session, cfg, "bulk", &cval)); + bulk = (cval.val != 0); + + /* Lock the handle while the cursor is using it. */ + WT_RET(__wt_session_lock_btree(session, + NULL, bulk ? WT_BTREE_EXCLUSIVE | WT_BTREE_BULK : 0)); + + csize = bulk ? sizeof(WT_CURSOR_BULK) : sizeof(WT_CURSOR_BTREE); + WT_RET(__wt_calloc(session, 1, csize, &cbt)); + + cursor = &cbt->iface; + *cursor = iface; + cursor->session = &session->iface; + cursor->uri = btree->name; + cursor->key_format = btree->key_format; + cursor->value_format = btree->value_format; + + cbt->btree = session->btree; + if (bulk) + WT_ERR(__wt_curbulk_init((WT_CURSOR_BULK *)cbt)); + + /* The append flag is only relevant to column stores. */ + if (WT_CURSOR_RECNO(cursor)) { + WT_ERR(__wt_config_gets(session, cfg, "append", &cval)); + if (cval.val != 0) + F_SET(cursor, WT_CURSTD_APPEND); + } + + WT_ERR(__wt_config_gets(session, cfg, "dump", &cval)); + if (cval.len != 0) { + __wt_curdump_init(cursor); + F_SET(cursor, + strncmp(cval.str, "print", cval.len) == 0 ? + WT_CURSTD_DUMP_PRINT : WT_CURSTD_DUMP_HEX); + } + + WT_ERR(__wt_config_gets(session, cfg, "raw", &cval)); + if (cval.val != 0) + F_SET(cursor, WT_CURSTD_RAW); + + WT_ERR(__wt_config_gets(session, cfg, "overwrite", &cval)); + if (cval.val != 0) + F_SET(cursor, WT_CURSTD_OVERWRITE); + + STATIC_ASSERT(offsetof(WT_CURSOR_BTREE, iface) == 0); + WT_ERR(__wt_cursor_init(cursor, cursor->uri, 1, 0, cfg)); + *cursorp = cursor; + + if (0) { +err: __wt_free(session, cbt); + } + + return (ret); +} + +/* + * __wt_curfile_open -- + * WT_SESSION->open_cursor method for the btree cursor type. + */ +int +__wt_curfile_open(WT_SESSION_IMPL *session, + const char *uri, const char *cfg[], WT_CURSOR **cursorp) +{ + /* TODO: handle projections. */ + + if (WT_PREFIX_MATCH(uri, "colgroup:")) + WT_RET(__wt_schema_get_btree(session, + uri, strlen(uri), NULL, WT_BTREE_NO_LOCK)); + else if (WT_PREFIX_MATCH(uri, "file:")) + WT_RET(__wt_session_get_btree(session, + uri, uri, NULL, NULL, WT_BTREE_NO_LOCK)); + else + return (EINVAL); + + return (__wt_curfile_create(session, cfg, cursorp)); +} diff --git a/src/cursor/cur_index.c b/src/cursor/cur_index.c new file mode 100644 index 00000000000..05762e44592 --- /dev/null +++ b/src/cursor/cur_index.c @@ -0,0 +1,437 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __curindex_get_value -- + * WT_CURSOR->get_value implementation for index cursors. + */ +static int +__curindex_get_value(WT_CURSOR *cursor, ...) +{ + WT_CURSOR_INDEX *cindex; + WT_ITEM *item; + WT_SESSION_IMPL *session; + va_list ap; + int ret; + + cindex = (WT_CURSOR_INDEX *)cursor; + CURSOR_API_CALL(cursor, session, get_value, NULL); + WT_CURSOR_NEEDVALUE(cursor); + + va_start(ap, cursor); + if (F_ISSET(cursor, WT_CURSTD_RAW)) { + ret = __wt_schema_project_merge(session, + cindex->cg_cursors, cindex->value_plan, + cursor->value_format, &cursor->value); + if (ret == 0) { + item = va_arg(ap, WT_ITEM *); + item->data = cursor->value.data; + item->size = cursor->value.size; + } + } else + ret = __wt_schema_project_out(session, + cindex->cg_cursors, cindex->value_plan, ap); + va_end(ap); +err: API_END(session); + + return (ret); +} + +/* + * __curindex_set_value -- + * WT_CURSOR->set_value implementation for index cursors. + */ +static void +__curindex_set_value(WT_CURSOR *cursor, ...) +{ + WT_SESSION_IMPL *session; + int ret; + + CURSOR_API_CALL(cursor, session, set_value, NULL); + WT_UNUSED(ret); + cursor->saved_err = ENOTSUP; + F_CLR(cursor, WT_CURSTD_VALUE_SET); + API_END(session); +} + +/* + * __curindex_move -- + * When an index cursor changes position, set the primary key in the + * associated column groups and update their positions to match. + */ +static int +__curindex_move(WT_CURSOR_INDEX *cindex) +{ + WT_CURSOR **cp, *first; + WT_SESSION_IMPL *session; + int i; + + session = (WT_SESSION_IMPL *)cindex->cbt.iface.session; + first = NULL; + + for (i = 0, cp = cindex->cg_cursors; + i < WT_COLGROUPS(cindex->table); + i++, cp++) { + if (*cp == NULL) + continue; + if (first == NULL) { + /* + * Set the primary key -- note that we need the primary + * key columns, so we have to use the full key format, + * not just the public columns. + */ + WT_RET(__wt_schema_project_slice(session, + cp, cindex->cbt.btree->key_plan, + 1, cindex->cbt.btree->key_format, + &cindex->cbt.iface.key)); + first = *cp; + } else { + (*cp)->key.data = first->key.data; + (*cp)->key.size = first->key.size; + (*cp)->recno = first->recno; + } + F_SET(*cp, WT_CURSTD_KEY_SET); + WT_RET((*cp)->search(*cp)); + } + + return (0); +} + +/* + * __curindex_next -- + * WT_CURSOR->next method for index cursors. + */ +static int +__curindex_next(WT_CURSOR *cursor) +{ + WT_CURSOR_INDEX *cindex; + WT_SESSION_IMPL *session; + int ret; + + cindex = (WT_CURSOR_INDEX *)cursor; + CURSOR_API_CALL(cursor, session, next, cindex->cbt.btree); + if ((ret = __wt_btcur_next(&cindex->cbt)) == 0) + ret = __curindex_move(cindex); + API_END(session); + + return (ret); +} + +/* + * __curindex_prev -- + * WT_CURSOR->prev method for index cursors. + */ +static int +__curindex_prev(WT_CURSOR *cursor) +{ + WT_CURSOR_INDEX *cindex; + WT_SESSION_IMPL *session; + int ret; + + cindex = (WT_CURSOR_INDEX *)cursor; + CURSOR_API_CALL(cursor, session, prev, cindex->cbt.btree); + if ((ret = __wt_btcur_prev(&cindex->cbt)) == 0) + ret = __curindex_move(cindex); + API_END(session); + + return (ret); +} + +/* + * __curindex_reset -- + * WT_CURSOR->reset method for index cursors. + */ +static int +__curindex_reset(WT_CURSOR *cursor) +{ + WT_CURSOR **cp; + WT_CURSOR_INDEX *cindex; + WT_SESSION_IMPL *session; + int i, ret; + + cindex = (WT_CURSOR_INDEX *)cursor; + CURSOR_API_CALL(cursor, session, reset, cindex->cbt.btree); + WT_TRET(__wt_btcur_reset(&cindex->cbt)); + + for (i = 0, cp = cindex->cg_cursors; + i < WT_COLGROUPS(cindex->table); + i++, cp++) { + if (*cp == NULL) + continue; + WT_TRET((*cp)->reset(*cp)); + } + API_END(session); + + return (ret); +} + +/* + * __curindex_search -- + * WT_CURSOR->search method for index cursors. + */ +static int +__curindex_search(WT_CURSOR *cursor) +{ + WT_CURSOR_INDEX *cindex; + WT_ITEM *oldkeyp; + WT_SESSION_IMPL *session; + int exact, ret; + + cindex = (WT_CURSOR_INDEX *)cursor; + CURSOR_API_CALL(cursor, session, search, cindex->cbt.btree); + + /* + * XXX + * A very odd corner case is an index with a recno key. + * The only way to get here is by creating an index on a column store + * using only the primary's recno as the index key. Disallow that for + * now. + */ + WT_ASSERT(session, strcmp(cursor->key_format, "r") != 0); + + /* + * We expect partial matches, but we want the smallest item that + * matches the prefix. Fail if there is no matching item. + */ + + /* + * Take a copy of the search key. + * XXX + * We can avoid this with a cursor flag indicating when the application + * owns the data. + */ + WT_ERR(__wt_scr_alloc(session, cursor->key.size, &oldkeyp)); + memcpy(oldkeyp->mem, cursor->key.data, cursor->key.size); + oldkeyp->size = cursor->key.size; + + WT_ERR(cursor->search_near(cursor, &exact)); + + /* + * We expect partial matches, and want the smallest record with a key + * greater than or equal to the search key. The only way for the key + * to be equal is if there is an index on the primary key, because + * otherwise the primary key columns will be appended to the index key, + * but we don't disallow that (odd) case. + */ + if (exact < 0) + WT_ERR(cursor->next(cursor)); + + if (cursor->key.size < oldkeyp->size || + memcmp(oldkeyp->data, cursor->key.data, oldkeyp->size) != 0) { + ret = WT_NOTFOUND; + goto err; + } + + WT_ERR(__curindex_move(cindex)); + +err: __wt_scr_free(&oldkeyp); + API_END(session); + + return (ret); +} + +/* + * __curindex_search_near -- + * WT_CURSOR->search_near method for index cursors. + */ +static int +__curindex_search_near(WT_CURSOR *cursor, int *exact) +{ + WT_CURSOR_INDEX *cindex; + WT_SESSION_IMPL *session; + int ret; + + cindex = (WT_CURSOR_INDEX *)cursor; + CURSOR_API_CALL(cursor, session, search_near, cindex->cbt.btree); + if ((ret = __wt_btcur_search_near(&cindex->cbt, exact)) == 0) + ret = __curindex_move(cindex); + API_END(session); + + return (ret); +} + +/* + * __curindex_close -- + * WT_CURSOR->close method for index cursors. + */ +static int +__curindex_close(WT_CURSOR *cursor) +{ + WT_BTREE *btree; + WT_CURSOR_INDEX *cindex; + WT_CURSOR **cp; + WT_SESSION_IMPL *session; + int i, ret; + + cindex = (WT_CURSOR_INDEX *)cursor; + btree = cindex->cbt.btree; + + CURSOR_API_CALL(cursor, session, close, btree); + + for (i = 0, cp = (cindex)->cg_cursors; + i < WT_COLGROUPS(cindex->table); i++, cp++) + if (*cp != NULL) { + WT_TRET((*cp)->close(*cp)); + *cp = NULL; + } + + __wt_free(session, cindex->cg_cursors); + if (cindex->key_plan != btree->key_plan) + __wt_free(session, cindex->key_plan); + if (cindex->value_plan != btree->value_plan) + __wt_free(session, cindex->value_plan); + if (cursor->value_format != cindex->table->value_format) + __wt_free(session, cindex->value_plan); + + WT_TRET(__wt_btcur_close(&cindex->cbt)); + WT_TRET(__wt_session_release_btree(session)); + /* The URI is owned by the btree handle. */ + cursor->uri = NULL; + WT_TRET(__wt_cursor_close(cursor)); + API_END(session); + + return (ret); +} + +static int +__curindex_open_colgroups( + WT_SESSION_IMPL *session, WT_CURSOR_INDEX *cindex, const char *cfg[]) +{ + WT_TABLE *table; + WT_CURSOR **cp; + char *proj; + uint32_t arg; + + table = cindex->table; + WT_RET(__wt_calloc_def(session, WT_COLGROUPS(table), &cp)); + cindex->cg_cursors = cp; + + /* Work out which column groups we need. */ + for (proj = (char *)cindex->value_plan; *proj != '\0'; proj++) { + arg = (uint32_t)strtoul(proj, &proj, 10); + if ((*proj != WT_PROJ_KEY && *proj != WT_PROJ_VALUE) || + cp[arg] != NULL) + continue; + session->btree = table->colgroup[arg]; + WT_RET(__wt_curfile_create(session, cfg, &cp[arg])); + } + + return (0); +} + +/* + * __wt_curindex_open -- + * WT_SESSION->open_cursor method for index cursors. + */ +int +__wt_curindex_open(WT_SESSION_IMPL *session, + const char *uri, const char *cfg[], WT_CURSOR **cursorp) +{ + WT_ITEM fmt, plan; + static WT_CURSOR iface = { + NULL, + NULL, + NULL, + NULL, + NULL, + __curindex_get_value, + NULL, + __curindex_set_value, + NULL, + __curindex_next, + __curindex_prev, + __curindex_reset, + __curindex_search, + __curindex_search_near, + __wt_cursor_notsup, /* insert */ + __wt_cursor_notsup, /* update */ + __wt_cursor_notsup, /* remove */ + __curindex_close, + { NULL, NULL }, /* TAILQ_ENTRY q */ + 0, /* recno key */ + { 0 }, /* recno raw buffer */ + { NULL, 0, 0, NULL, 0 },/* WT_ITEM key */ + { NULL, 0, 0, NULL, 0 },/* WT_ITEM value */ + 0, /* int saved_err */ + 0 /* uint32_t flags */ + }; + WT_CURSOR_INDEX *cindex; + WT_CURSOR_BTREE *cbt; + WT_CURSOR *cursor; + WT_TABLE *table; + const char *columns, *idxname, *tablename; + size_t namesize; + int ret; + + ret = 0; + + tablename = uri; + if (!WT_PREFIX_SKIP(tablename, "index:") || + (idxname = strchr(tablename, ':')) == NULL) + WT_RET_MSG(session, EINVAL, "Invalid cursor URI: '%s'", uri); + namesize = (size_t)(idxname - tablename); + ++idxname; + + if ((ret = __wt_schema_get_table(session, + tablename, namesize, &table)) != 0) { + if (ret == WT_NOTFOUND) + WT_RET_MSG(session, EINVAL, + "Cannot open cursor '%s' on unknown table", uri); + return (ret); + } + + columns = strchr(idxname, '('); + if (columns == NULL) + namesize = strlen(idxname); + else + namesize = (size_t)(columns - idxname); + + WT_RET(__wt_schema_open_index(session, table, idxname, namesize)); + WT_RET(__wt_session_lock_btree(session, NULL, 0)); + WT_RET(__wt_calloc_def(session, 1, &cindex)); + + cbt = &cindex->cbt; + cursor = &cbt->iface; + *cursor = iface; + cursor->session = &session->iface; + cbt->btree = session->btree; + + cindex->table = table; + cindex->key_plan = session->btree->key_plan; + cindex->value_plan = session->btree->value_plan; + + cursor->uri = cbt->btree->name; + cursor->key_format = cbt->btree->idxkey_format; + cursor->value_format = table->value_format; + + /* Handle projections. */ + if (columns != NULL) { + WT_CLEAR(fmt); + WT_ERR(__wt_struct_reformat(session, table, + columns, strlen(columns), NULL, 0, &fmt)); + cursor->value_format = __wt_buf_steal(session, &fmt, NULL); + + WT_CLEAR(plan); + WT_ERR(__wt_struct_plan(session, table, + columns, strlen(columns), 0, &plan)); + cindex->value_plan = __wt_buf_steal(session, &plan, NULL); + } + + /* Open the column groups needed for this index cursor. */ + WT_ERR(__curindex_open_colgroups(session, cindex, cfg)); + + WT_ERR(__wt_cursor_init(cursor, cursor->uri, 1, 1, cfg)); + *cursorp = cursor; + + if (0) { +err: (void)__curindex_close(cursor); + } + + return (ret); +} diff --git a/src/cursor/cur_stat.c b/src/cursor/cur_stat.c new file mode 100644 index 00000000000..48fc25730b2 --- /dev/null +++ b/src/cursor/cur_stat.c @@ -0,0 +1,411 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static int __curstat_next(WT_CURSOR *cursor); +static int __curstat_prev(WT_CURSOR *cursor); + +/* + * __curstat_print_value -- + * Convert statistics cursor value to printable format. + */ +static int +__curstat_print_value(WT_SESSION_IMPL *session, uint64_t v, WT_ITEM *buf) +{ + if (v >= WT_BILLION) + WT_RET(__wt_buf_fmt(session, buf, + "%" PRIu64 "B (%" PRIu64 ")", v / WT_BILLION, v)); + else if (v >= WT_MILLION) + WT_RET(__wt_buf_fmt(session, buf, + "%" PRIu64 "M (%" PRIu64 ")", v / WT_MILLION, v)); + else + WT_RET(__wt_buf_fmt(session, buf, "%" PRIu64, v)); + + return (0); +} + +/* + * __curstat_get_key -- + * WT_CURSOR->get_key for statistics cursors. + */ +static int +__curstat_get_key(WT_CURSOR *cursor, ...) +{ + WT_CURSOR_STAT *cst; + WT_ITEM *item; + WT_SESSION_IMPL *session; + size_t size; + va_list ap; + int ret; + + ret = 0; + CURSOR_API_CALL(cursor, session, get_key, NULL); + cst = (WT_CURSOR_STAT *)cursor; + va_start(ap, cursor); + + WT_CURSOR_NEEDKEY(cursor); + + if (F_ISSET(cursor, WT_CURSTD_RAW)) { + size = __wt_struct_size(session, cursor->key_format, cst->key); + WT_ERR(__wt_buf_initsize(session, &cursor->key, size)); + WT_ERR(__wt_struct_pack(session, cursor->key.mem, size, + cursor->key_format, cst->key)); + + item = va_arg(ap, WT_ITEM *); + item->data = cursor->key.data; + item->size = cursor->key.size; + } else + *va_arg(ap, int *) = cst->key; + +err: va_end(ap); + API_END(session); + return (ret); +} + +/* + * __curstat_get_value -- + * WT_CURSOR->get_value for statistics cursors. + */ +static int +__curstat_get_value(WT_CURSOR *cursor, ...) +{ + WT_CURSOR_STAT *cst; + WT_ITEM *item; + WT_SESSION_IMPL *session; + va_list ap; + size_t size; + int ret; + + ret = 0; + CURSOR_API_CALL(cursor, session, get_value, NULL); + cst = (WT_CURSOR_STAT *)cursor; + va_start(ap, cursor); + + WT_CURSOR_NEEDVALUE(cursor); + + if (F_ISSET(cursor, WT_CURSTD_RAW)) { + size = __wt_struct_size(session, cursor->value_format, + cst->stats_first[cst->key].desc, cst->pv.data, cst->v); + WT_ERR(__wt_buf_initsize(session, &cursor->value, size)); + WT_ERR(__wt_struct_pack(session, cursor->value.mem, size, + cursor->value_format, + cst->stats_first[cst->key].desc, cst->pv.data, cst->v)); + + item = va_arg(ap, WT_ITEM *); + item->data = cursor->value.data; + item->size = cursor->value.size; + } else { + *va_arg(ap, const char **) = cst->stats_first[cst->key].desc; + *va_arg(ap, const char **) = cst->pv.data; + *va_arg(ap, uint64_t *) = cst->v; + } + +err: va_end(ap); + API_END(session); + return (ret); +} + +/* + * __curstat_set_key -- + * WT_CURSOR->set_key for statistics cursors. + */ +static void +__curstat_set_key(WT_CURSOR *cursor, ...) +{ + WT_CURSOR_STAT *cst; + WT_ITEM *item; + WT_SESSION_IMPL *session; + va_list ap; + int ret; + + ret = 0; + CURSOR_API_CALL(cursor, session, set_key, NULL); + cst = (WT_CURSOR_STAT *)cursor; + + va_start(ap, cursor); + if (F_ISSET(cursor, WT_CURSTD_RAW)) { + item = va_arg(ap, WT_ITEM *); + ret = __wt_struct_unpack(session, item->data, item->size, + cursor->key_format, &cst->key); + } else + cst->key = va_arg(ap, int); + va_end(ap); + + if ((cursor->saved_err = ret) == 0) + F_SET(cursor, WT_CURSTD_KEY_SET); + else + F_CLR(cursor, WT_CURSTD_KEY_SET); + + API_END(session); +} + +/* + * __curstat_set_value -- + * WT_CURSOR->set_value for statistics cursors. + */ +static void +__curstat_set_value(WT_CURSOR *cursor, ...) +{ + WT_UNUSED(cursor); + return; +} + +/* + * __curstat_next -- + * WT_CURSOR->next method for the statistics cursor type. + */ +static int +__curstat_next(WT_CURSOR *cursor) +{ + WT_CURSOR_STAT *cst; + WT_SESSION_IMPL *session; + int ret; + + ret = 0; + CURSOR_API_CALL(cursor, session, next, NULL); + cst = (WT_CURSOR_STAT *)cursor; + + /* Move to the next item. */ + if (cst->notpositioned) { + cst->notpositioned = 0; + cst->key = 0; + } else if (cst->key < cst->stats_count - 1) + ++cst->key; + else { + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + WT_ERR(WT_NOTFOUND); + } + cst->v = cst->stats_first[cst->key].v; + WT_ERR(__curstat_print_value(session, cst->v, &cst->pv)); + F_SET(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + +err: API_END(session); + return (ret); +} + +/* + * __curstat_prev -- + * WT_CURSOR->prev method for the statistics cursor type. + */ +static int +__curstat_prev(WT_CURSOR *cursor) +{ + WT_CURSOR_STAT *cst; + WT_SESSION_IMPL *session; + int ret; + + ret = 0; + CURSOR_API_CALL(cursor, session, prev, NULL); + cst = (WT_CURSOR_STAT *)cursor; + + /* Move to the previous item. */ + if (cst->notpositioned) { + cst->notpositioned = 0; + cst->key = cst->stats_count - 1; + } else if (cst->key > 0) + --cst->key; + else { + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + WT_ERR(WT_NOTFOUND); + } + + cst->v = cst->stats_first[cst->key].v; + WT_ERR(__curstat_print_value(session, cst->v, &cst->pv)); + F_SET(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + +err: API_END(session); + return (ret); +} + +/* + * __curstat_reset -- + * WT_CURSOR->reset method for the statistics cursor type. + */ +static int +__curstat_reset(WT_CURSOR *cursor) +{ + WT_CURSOR_STAT *cst; + + cst = (WT_CURSOR_STAT *)cursor; + cst->notpositioned = 1; + return (0); +} + +/* + * __curstat_search -- + * WT_CURSOR->search method for the statistics cursor type. + */ +static int +__curstat_search(WT_CURSOR *cursor) +{ + WT_CURSOR_STAT *cst; + WT_SESSION_IMPL *session; + int ret; + + ret = 0; + CURSOR_API_CALL(cursor, session, search, NULL); + cst = (WT_CURSOR_STAT *)cursor; + + WT_CURSOR_NEEDKEY(cursor); + F_CLR(cursor, WT_CURSTD_VALUE_SET); + + if (cst->key < 0 || cst->key >= cst->stats_count) + WT_ERR(WT_NOTFOUND); + + cst->v = cst->stats_first[cst->key].v; + WT_ERR(__curstat_print_value(session, cst->v, &cst->pv)); + F_SET(cursor, WT_CURSTD_VALUE_SET); + +err: API_END(session); + return (ret); +} + +/* + * __curstat_close -- + * WT_CURSOR->close method for the statistics cursor type. + */ +static int +__curstat_close(WT_CURSOR *cursor) +{ + WT_CURSOR_STAT *cst; + WT_SESSION_IMPL *session; + int ret; + + ret = 0; + CURSOR_API_CALL(cursor, session, close, NULL); + cst = (WT_CURSOR_STAT *)cursor; + + if (ret == 0 && cst->clear_func) + cst->clear_func(cst->stats_first); + + __wt_buf_free(session, &cst->pv); + + if (cst->btree != NULL) { + session->btree = cst->btree; + WT_TRET(__wt_session_release_btree(session)); + session->btree = NULL; + } + + WT_TRET(__wt_cursor_close(cursor)); + + API_END(session); + return (ret); +} + +/* + * __wt_curstat_open -- + * WT_SESSION->open_cursor method for the statistics cursor type. + */ +int +__wt_curstat_open(WT_SESSION_IMPL *session, + const char *uri, const char *cfg[], WT_CURSOR **cursorp) +{ + static WT_CURSOR iface = { + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + __curstat_next, + __curstat_prev, + __curstat_reset, + __curstat_search, + /* search-near */ + (int (*)(WT_CURSOR *, int *))__wt_cursor_notsup, + __wt_cursor_notsup, /* insert */ + __wt_cursor_notsup, /* update */ + __wt_cursor_notsup, /* remove */ + __curstat_close, + { NULL, NULL }, /* TAILQ_ENTRY q */ + 0, /* recno key */ + { 0 }, /* recno raw buffer */ + { NULL, 0, 0, NULL, 0 },/* WT_ITEM key */ + { NULL, 0, 0, NULL, 0 },/* WT_ITEM value */ + 0, /* int saved_err */ + 0 /* uint32_t flags */ + }; + WT_BTREE *btree; + WT_CURSOR_STAT *cst; + WT_CONFIG_ITEM cval; + WT_CURSOR *cursor; + WT_STATS *stats_first; + void (*clear_func)(WT_STATS *); + int clear_on_close, raw, ret, stats_count; + + btree = NULL; + clear_func = NULL; + cst = NULL; + ret = 0; + + WT_RET(__wt_config_gets(session, cfg, "clear_on_close", &cval)); + clear_on_close = (cval.val != 0); + + WT_RET(__wt_config_gets(session, cfg, "raw", &cval)); + raw = (cval.val != 0); + + if (!WT_PREFIX_SKIP(uri, "statistics:")) + return (EINVAL); + if (WT_PREFIX_MATCH(uri, "file:")) { + WT_ERR( + __wt_session_get_btree(session, uri, uri, NULL, NULL, 0)); + btree = session->btree; + WT_ERR(__wt_btree_stat_init(session)); + stats_first = (WT_STATS *)session->btree->stats; + stats_count = sizeof(WT_BTREE_STATS) / sizeof(WT_STATS); + if (clear_on_close) + clear_func = __wt_stat_clear_btree_stats; + } else { + __wt_conn_stat_init(session); + stats_first = (WT_STATS *)S2C(session)->stats; + stats_count = sizeof(WT_CONNECTION_STATS) / sizeof(WT_STATS); + if (clear_on_close) + clear_func = __wt_stat_clear_connection_stats; + } + + WT_ERR(__wt_calloc_def(session, 1, &cst)); + cst->stats_first = stats_first; + cst->stats_count = stats_count; + cst->notpositioned = 1; + cst->clear_func = clear_func; + + cursor = &cst->iface; + *cursor = iface; + cursor->session = &session->iface; + + cursor->get_key = __curstat_get_key; + cursor->set_key = __curstat_set_key; + cursor->get_value = __curstat_get_value; + cursor->set_value = __curstat_set_value; + + cst->btree = btree; + if (raw) + F_SET(cursor, WT_CURSTD_RAW); + + STATIC_ASSERT(offsetof(WT_CURSOR_STAT, iface) == 0); + WT_ERR(__wt_cursor_init(cursor, uri, 0, 1, cfg)); + + /* + * We return the statistics field's offset as the key, and a string + * description, a string value, and a uint64_t value as the value + * columns. + */ + cursor->key_format = "i"; + cursor->value_format = "SSq"; + + *cursorp = cursor; + + if (0) { +err: __wt_free(session, cst); + } + + return (ret); +} diff --git a/src/cursor/cur_std.c b/src/cursor/cur_std.c new file mode 100644 index 00000000000..1619f6cf9b9 --- /dev/null +++ b/src/cursor/cur_std.c @@ -0,0 +1,373 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_cursor_notsup -- + * Unsupported cursor actions. + */ +int +__wt_cursor_notsup(WT_CURSOR *cursor) +{ + WT_UNUSED(cursor); + + return (ENOTSUP); +} + +/* + * __wt_cursor_get_key -- + * WT_CURSOR->get_key default implementation. + */ +int +__wt_cursor_get_key(WT_CURSOR *cursor, ...) +{ + va_list ap; + int ret; + + va_start(ap, cursor); + ret = __wt_cursor_get_keyv(cursor, cursor->flags, ap); + va_end(ap); + return (ret); +} + +/* + * __wt_cursor_get_keyv -- + * WT_CURSOR->get_key worker function. + */ +int +__wt_cursor_get_keyv(WT_CURSOR *cursor, uint32_t flags, va_list ap) +{ + WT_ITEM *key; + WT_SESSION_IMPL *session; + const char *fmt; + int ret; + + CURSOR_API_CALL(cursor, session, get_key, NULL); + WT_CURSOR_NEEDKEY(cursor); + + if (WT_CURSOR_RECNO(cursor)) { + if (LF_ISSET(WT_CURSTD_RAW)) { + key = va_arg(ap, WT_ITEM *); + key->data = cursor->raw_recno_buf; + key->size = (uint32_t) + __wt_struct_size(session, "q", cursor->recno); + ret = __wt_struct_pack(session, cursor->raw_recno_buf, + sizeof(cursor->raw_recno_buf), "q", cursor->recno); + } else + *va_arg(ap, uint64_t *) = cursor->recno; + } else { + fmt = cursor->key_format; + if (LF_ISSET( + WT_CURSTD_DUMP_HEX | WT_CURSTD_DUMP_PRINT | WT_CURSTD_RAW)) + fmt = "u"; + ret = __wt_struct_unpackv( + session, cursor->key.data, cursor->key.size, fmt, ap); + } + +err: API_END(session); + return (ret); +} + +/* + * __wt_cursor_get_value -- + * WT_CURSOR->get_value default implementation. + */ +int +__wt_cursor_get_value(WT_CURSOR *cursor, ...) +{ + WT_SESSION_IMPL *session; + const char *fmt; + va_list ap; + int ret; + + CURSOR_API_CALL(cursor, session, get_value, NULL); + WT_CURSOR_NEEDVALUE(cursor); + + va_start(ap, cursor); + fmt = F_ISSET(cursor, + WT_CURSTD_DUMP_HEX | WT_CURSTD_DUMP_PRINT | WT_CURSTD_RAW) ? + "u" : cursor->value_format; + ret = __wt_struct_unpackv(session, + cursor->value.data, cursor->value.size, fmt, ap); + va_end(ap); + +err: API_END(session); + return (ret); +} + +/* + * __wt_cursor_set_keyv -- + * WT_CURSOR->set_key default implementation. + */ +void +__wt_cursor_set_key(WT_CURSOR *cursor, ...) +{ + va_list ap; + + va_start(ap, cursor); + __wt_cursor_set_keyv(cursor, cursor->flags, ap); + va_end(ap); +} + +/* + * __wt_cursor_set_keyv -- + * WT_CURSOR->set_key default implementation. + */ +void +__wt_cursor_set_keyv(WT_CURSOR *cursor, uint32_t flags, va_list ap) +{ + WT_SESSION_IMPL *session; + WT_ITEM *buf, *item; + va_list ap_copy; + const char *fmt, *str; + size_t sz; + int ret; + + CURSOR_API_CALL(cursor, session, set_key, NULL); + + /* Fast path some common cases: single strings or byte arrays. */ + if (WT_CURSOR_RECNO(cursor)) { + if (LF_ISSET(WT_CURSTD_RAW)) { + item = va_arg(ap, WT_ITEM *); + WT_ERR(__wt_struct_unpack(session, + item->data, item->size, "q", &cursor->recno)); + } else + cursor->recno = va_arg(ap, uint64_t); + if (cursor->recno == 0) + WT_ERR_MSG(session, EINVAL, + "Record numbers must be greater than zero"); + cursor->key.data = &cursor->recno; + sz = sizeof(cursor->recno); + } else { + fmt = cursor->key_format; + if (LF_ISSET( + WT_CURSTD_DUMP_HEX | WT_CURSTD_DUMP_PRINT | WT_CURSTD_RAW)) + fmt = "u"; + if (strcmp(fmt, "S") == 0) { + str = va_arg(ap, const char *); + sz = strlen(str) + 1; + cursor->key.data = (void *)str; + } else if (strcmp(fmt, "u") == 0) { + item = va_arg(ap, WT_ITEM *); + sz = item->size; + cursor->key.data = (void *)item->data; + } else { + buf = &cursor->key; + + va_copy(ap_copy, ap); + sz = __wt_struct_sizev( + session, cursor->key_format, ap_copy); + va_end(ap_copy); + + WT_ERR(__wt_buf_initsize(session, buf, sz)); + WT_ERR(__wt_struct_packv( + session, buf->mem, sz, cursor->key_format, ap)); + } + } + if (sz == 0) + WT_ERR_MSG(session, EINVAL, "Empty keys not permitted"); + else if ((uint32_t)sz != sz) + WT_ERR_MSG(session, EINVAL, + "Key size (%" PRIu64 ") out of range", (uint64_t)sz); + cursor->saved_err = 0; + cursor->key.size = WT_STORE_SIZE(sz); + F_SET(cursor, WT_CURSTD_KEY_SET); + if (0) { +err: cursor->saved_err = ret; + F_CLR(cursor, WT_CURSTD_KEY_SET); + } + + API_END(session); +} + +/* + * __wt_cursor_set_value -- + * WT_CURSOR->set_value default implementation. + */ +void +__wt_cursor_set_value(WT_CURSOR *cursor, ...) +{ + WT_SESSION_IMPL *session; + WT_ITEM *buf, *item; + const char *fmt, *str; + size_t sz; + va_list ap; + int ret; + + CURSOR_API_CALL(cursor, session, set_value, NULL); + + va_start(ap, cursor); + fmt = F_ISSET(cursor, + WT_CURSTD_DUMP_HEX | WT_CURSTD_DUMP_PRINT | WT_CURSTD_RAW) ? + "u" : cursor->value_format; + /* Fast path some common cases: single strings or byte arrays. */ + if (strcmp(fmt, "S") == 0) { + str = va_arg(ap, const char *); + sz = strlen(str) + 1; + cursor->value.data = str; + } else if (strcmp(fmt, "u") == 0) { + item = va_arg(ap, WT_ITEM *); + sz = item->size; + cursor->value.data = item->data; + } else { + buf = &cursor->value; + sz = __wt_struct_sizev(session, cursor->value_format, ap); + va_end(ap); + va_start(ap, cursor); + if ((ret = __wt_buf_initsize(session, buf, sz)) != 0 || + (ret = __wt_struct_packv(session, buf->mem, sz, + cursor->value_format, ap)) != 0) { + cursor->saved_err = ret; + F_CLR(cursor, WT_CURSTD_VALUE_SET); + goto err; + } + cursor->value.data = buf->mem; + } + F_SET(cursor, WT_CURSTD_VALUE_SET); + cursor->value.size = WT_STORE_SIZE(sz); + va_end(ap); + +err: API_END(session); +} + +/* + * __cursor_search -- + * WT_CURSOR->search default implementation. + */ +static int +__cursor_search(WT_CURSOR *cursor) +{ + int exact; + + WT_RET(cursor->search_near(cursor, &exact)); + return ((exact == 0) ? 0 : WT_NOTFOUND); +} + +/* + * __cursor_equals -- + * WT_CURSOR->equals default implementation. + */ +static int +__cursor_equals(WT_CURSOR *cursor, WT_CURSOR *other) +{ + WT_SESSION_IMPL *session; + int ret; + + CURSOR_API_CALL(cursor, session, equals, NULL); + + /* Both cursors must refer to the same source. */ + if (other == NULL || strcmp(cursor->uri, other->uri) != 0) + goto done; + + /* Check that both have keys set and the keys match. */ + if (F_ISSET(cursor, WT_CURSTD_KEY_SET) && + F_ISSET(other, WT_CURSTD_KEY_SET)) { + if (WT_CURSOR_RECNO(cursor)) + ret = (cursor->recno == other->recno); + else if (cursor->key.size == other->key.size) + ret = (memcmp(cursor->key.data, other->key.data, + cursor->key.size) == 0); + } + +done: API_END(session); + return (ret); +} + +/* + * __wt_cursor_close -- + * WT_CURSOR->close default implementation. + */ +int +__wt_cursor_close(WT_CURSOR *cursor) +{ + WT_SESSION_IMPL *session; + int ret; + + CURSOR_API_CALL(cursor, session, close, NULL); + + __wt_buf_free(session, &cursor->key); + __wt_buf_free(session, &cursor->value); + + if (F_ISSET(cursor, WT_CURSTD_PUBLIC)) + TAILQ_REMOVE(&session->public_cursors, cursor, q); + else if (F_ISSET(cursor, WT_CURSTD_FILE)) + TAILQ_REMOVE(&session->file_cursors, cursor, q); + + __wt_free(session, cursor->uri); + __wt_free(session, cursor); + + API_END(session); + return (ret); +} + +/* + * __wt_cursor_init -- + * Default cursor initialization. + * + * Most cursors are "public", and added to the list in the session + * to be closed when the cursor is closed. However, some cursors are + * opened for internal use, or are opened inside another cursor (such + * as column groups or indices within a table cursor), and adding those + * cursors to the list introduces ordering dependencies into + * WT_SESSION->close that we prefer to avoid. + */ +int +__wt_cursor_init(WT_CURSOR *cursor, + const char *uri, int is_file, int is_public, const char *cfg[]) + WT_GCC_FUNC_ATTRIBUTE((warn_unused_result)) +{ + WT_SESSION_IMPL *session; + + WT_UNUSED(cfg); + session = (WT_SESSION_IMPL *)cursor->session; + + if (cursor->get_key == NULL) + cursor->get_key = __wt_cursor_get_key; + if (cursor->get_value == NULL) + cursor->get_value = __wt_cursor_get_value; + if (cursor->set_key == NULL) + cursor->set_key = __wt_cursor_set_key; + if (cursor->set_value == NULL) + cursor->set_value = __wt_cursor_set_value; + if (cursor->equals == NULL) + cursor->equals = __cursor_equals; + if (cursor->search == NULL) + cursor->search = __cursor_search; + + if (cursor->uri == NULL) + WT_RET(__wt_strdup(session, uri, &cursor->uri)); + + WT_CLEAR(cursor->key); + WT_CLEAR(cursor->value); + + if (is_file) { + F_SET(cursor, WT_CURSTD_FILE); + TAILQ_INSERT_HEAD(&session->file_cursors, cursor, q); + } else if (is_public) { + F_SET(cursor, WT_CURSTD_PUBLIC); + TAILQ_INSERT_HEAD(&session->public_cursors, cursor, q); + } + + return (0); +} + +/* + * __wt_cursor_kv_not_set -- + * Standard error message for key/values not set. + */ +int +__wt_cursor_kv_not_set(WT_CURSOR *cursor, int key) +{ + WT_SESSION_IMPL *session; + + session = (WT_SESSION_IMPL *)cursor->session; + + WT_RET_MSG(session, + cursor->saved_err == 0 ? EINVAL : cursor->saved_err, + "requires %s be set", key ? "key" : "value"); +} diff --git a/src/cursor/cur_table.c b/src/cursor/cur_table.c new file mode 100644 index 00000000000..68659efd6bf --- /dev/null +++ b/src/cursor/cur_table.c @@ -0,0 +1,640 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static int __curtable_open_indices(WT_CURSOR_TABLE *ctable); +static int __curtable_update(WT_CURSOR *cursor); + +#define APPLY_CG(ctable, f) do { \ + WT_CURSOR **__cp; \ + int __i; \ + for (__i = 0, __cp = ctable->cg_cursors; \ + __i < WT_COLGROUPS(ctable->table); \ + __i++, __cp++) \ + WT_TRET((*__cp)->f(*__cp)); \ +} while (0) + +#define APPLY_IDX(ctable, f) do { \ + WT_BTREE *btree; \ + WT_CURSOR **__cp; \ + int __i; \ + WT_ERR(__curtable_open_indices(ctable)); \ + __cp = (ctable)->idx_cursors; \ + for (__i = 0; __i < ctable->table->nindices; __i++, __cp++) { \ + btree = ((WT_CURSOR_BTREE *)*__cp)->btree; \ + WT_ERR(__wt_schema_project_merge(session, \ + ctable->cg_cursors, \ + btree->key_plan, btree->key_format, &(*__cp)->key));\ + F_SET(*__cp, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); \ + WT_ERR((*__cp)->f(*__cp)); \ + } \ +} while (0) + +/* + * __wt_curtable_get_key -- + * WT_CURSOR->get_key implementation for tables. + */ +int +__wt_curtable_get_key(WT_CURSOR *cursor, ...) +{ + WT_CURSOR *primary; + WT_CURSOR_TABLE *ctable; + va_list ap; + int ret; + + ctable = (WT_CURSOR_TABLE *)cursor; + primary = *ctable->cg_cursors; + + va_start(ap, cursor); + ret = __wt_cursor_get_keyv(primary, cursor->flags, ap); + va_end(ap); + + return (ret); +} + +/* + * __wt_curtable_get_value -- + * WT_CURSOR->get_value implementation for tables. + */ +int +__wt_curtable_get_value(WT_CURSOR *cursor, ...) +{ + WT_CURSOR *primary; + WT_CURSOR_TABLE *ctable; + WT_ITEM *item; + WT_SESSION_IMPL *session; + va_list ap; + int ret; + + ctable = (WT_CURSOR_TABLE *)cursor; + primary = *ctable->cg_cursors; + CURSOR_API_CALL(cursor, session, get_value, NULL); + WT_CURSOR_NEEDVALUE(primary); + + va_start(ap, cursor); + if (F_ISSET(cursor, + WT_CURSTD_DUMP_HEX | WT_CURSTD_DUMP_PRINT | WT_CURSTD_RAW)) { + ret = __wt_schema_project_merge(session, + ctable->cg_cursors, ctable->plan, + cursor->value_format, &cursor->value); + if (ret == 0) { + item = va_arg(ap, WT_ITEM *); + item->data = cursor->value.data; + item->size = cursor->value.size; + } + } else + ret = __wt_schema_project_out(session, + ctable->cg_cursors, ctable->plan, ap); + va_end(ap); +err: API_END(session); + + return (ret); +} + +/* + * __wt_curtable_set_key -- + * WT_CURSOR->set_key implementation for tables. + */ +void +__wt_curtable_set_key(WT_CURSOR *cursor, ...) +{ + WT_CURSOR **cp, *primary; + WT_CURSOR_TABLE *ctable; + va_list ap; + int i; + + ctable = (WT_CURSOR_TABLE *)cursor; + cp = ctable->cg_cursors; + primary = *cp++; + + va_start(ap, cursor); + __wt_cursor_set_keyv(primary, cursor->flags, ap); + va_end(ap); + + if (!F_ISSET(primary, WT_CURSTD_KEY_SET)) + return; + + /* Copy the primary key to the other cursors. */ + for (i = 1; i < WT_COLGROUPS(ctable->table); i++, cp++) { + (*cp)->recno = primary->recno; + (*cp)->key.data = primary->key.data; + (*cp)->key.size = primary->key.size; + F_SET(*cp, WT_CURSTD_KEY_SET); + } +} + +/* + * __wt_curtable_set_value -- + * WT_CURSOR->set_value implementation for tables. + */ +void +__wt_curtable_set_value(WT_CURSOR *cursor, ...) +{ + WT_CURSOR **cp; + WT_CURSOR_TABLE *ctable; + WT_ITEM *item; + WT_SESSION_IMPL *session; + va_list ap; + int i, ret; + + ctable = (WT_CURSOR_TABLE *)cursor; + CURSOR_API_CALL(cursor, session, set_value, NULL); + + va_start(ap, cursor); + if (F_ISSET(cursor, + WT_CURSTD_DUMP_HEX | WT_CURSTD_DUMP_PRINT | WT_CURSTD_RAW)) { + item = va_arg(ap, WT_ITEM *); + cursor->value.data = item->data; + cursor->value.size = item->size; + ret = __wt_schema_project_slice(session, + ctable->cg_cursors, ctable->plan, 0, + cursor->value_format, &cursor->value); + } else + ret = __wt_schema_project_in(session, + ctable->cg_cursors, ctable->plan, ap); + va_end(ap); + + for (i = 0, cp = ctable->cg_cursors; + i < WT_COLGROUPS(ctable->table); + i++, cp++) + if (ret == 0) + F_SET(*cp, WT_CURSTD_VALUE_SET); + else { + (*cp)->saved_err = ret; + F_CLR(*cp, WT_CURSTD_VALUE_SET); + } + + API_END(session); +} + +/* + * __curtable_next -- + * WT_CURSOR->next method for the table cursor type. + */ +static int +__curtable_next(WT_CURSOR *cursor) +{ + WT_CURSOR_TABLE *ctable; + WT_SESSION_IMPL *session; + int ret; + + ctable = (WT_CURSOR_TABLE *)cursor; + CURSOR_API_CALL(cursor, session, next, NULL); + APPLY_CG(ctable, next); + API_END(session); + + return (ret); +} + +/* + * __curtable_prev -- + * WT_CURSOR->prev method for the table cursor type. + */ +static int +__curtable_prev(WT_CURSOR *cursor) +{ + WT_CURSOR_TABLE *ctable; + WT_SESSION_IMPL *session; + int ret; + + ctable = (WT_CURSOR_TABLE *)cursor; + CURSOR_API_CALL(cursor, session, prev, NULL); + APPLY_CG(ctable, prev); + API_END(session); + + return (ret); +} + +/* + * __curtable_reset -- + * WT_CURSOR->reset method for the table cursor type. + */ +static int +__curtable_reset(WT_CURSOR *cursor) +{ + WT_CURSOR_TABLE *ctable; + WT_SESSION_IMPL *session; + int ret; + + ctable = (WT_CURSOR_TABLE *)cursor; + CURSOR_API_CALL(cursor, session, reset, NULL); + APPLY_CG(ctable, reset); + API_END(session); + + return (ret); +} + +/* + * __curtable_search -- + * WT_CURSOR->search method for the table cursor type. + */ +static int +__curtable_search(WT_CURSOR *cursor) +{ + WT_CURSOR_TABLE *ctable; + WT_SESSION_IMPL *session; + int ret; + + ctable = (WT_CURSOR_TABLE *)cursor; + CURSOR_API_CALL(cursor, session, search, NULL); + APPLY_CG(ctable, search); + API_END(session); + + return (ret); +} + +/* + * __curtable_search_near -- + * WT_CURSOR->search_near method for the table cursor type. + */ +static int +__curtable_search_near(WT_CURSOR *cursor, int *exact) +{ + WT_CURSOR_TABLE *ctable; + WT_CURSOR *primary, **cp; + WT_SESSION_IMPL *session; + int i, ret; + + ctable = (WT_CURSOR_TABLE *)cursor; + CURSOR_API_CALL(cursor, session, search_near, NULL); + cp = ctable->cg_cursors; + primary = *cp; + WT_ERR(primary->search_near(primary, exact)); + + for (i = 1, ++cp; i < WT_COLGROUPS(ctable->table); i++) { + (*cp)->key.data = primary->key.data; + (*cp)->key.size = primary->key.size; + (*cp)->recno = primary->recno; + WT_ERR((*cp)->search(*cp)); + } +err: API_END(session); + + return (ret); +} + +/* + * __curtable_insert -- + * WT_CURSOR->insert method for the table cursor type. + */ +static int +__curtable_insert(WT_CURSOR *cursor) +{ + WT_CURSOR_TABLE *ctable; + WT_CURSOR *primary, **cp; + WT_SESSION_IMPL *session; + int i, ret; + + ctable = (WT_CURSOR_TABLE *)cursor; + CURSOR_API_CALL(cursor, session, insert, NULL); + cp = ctable->cg_cursors; + + /* + * Split out the first insert, it may be allocating a recno, and this + * is also the point at which we discover whether this is an overwrite. + */ + primary = *cp++; + if ((ret = primary->insert(primary)) != 0) { + if (ret == WT_DUPLICATE_KEY && + F_ISSET(cursor, WT_CURSTD_OVERWRITE)) { + /* + * !!! The insert failure clears these flags, but does + * not touch the items. We could make a copy every time + * for overwrite cursors, but for now we just reset the + * flags. + */ + F_SET(primary, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + ret = __curtable_update(cursor); + } + goto err; + } + + for (i = 1; i < WT_COLGROUPS(ctable->table); i++, cp++) { + (*cp)->recno = primary->recno; + WT_ERR((*cp)->insert(*cp)); + } + + APPLY_IDX(ctable, insert); +err: API_END(session); + + return (ret); +} + +/* + * __curtable_update -- + * WT_CURSOR->update method for the table cursor type. + */ +static int +__curtable_update(WT_CURSOR *cursor) +{ + WT_CURSOR_TABLE *ctable; + WT_SESSION_IMPL *session; + int ret; + + ctable = (WT_CURSOR_TABLE *)cursor; + CURSOR_API_CALL(cursor, session, update, NULL); + WT_ERR(__curtable_open_indices(ctable)); + /* + * If the table has indices, first delete any old index keys, then + * update the primary, then insert the new index keys. This is + * complicated by the fact that we need the old value to generate the + * old index keys, so we make a temporary copy of the new value. + */ + if (ctable->idx_cursors != NULL) { + WT_ERR(__wt_schema_project_merge(session, + ctable->cg_cursors, ctable->plan, + cursor->value_format, &cursor->value)); + APPLY_CG(ctable, search); + WT_ERR(ret); + APPLY_IDX(ctable, remove); + WT_ERR(__wt_schema_project_slice(session, + ctable->cg_cursors, ctable->plan, 0, + cursor->value_format, &cursor->value)); + } + APPLY_CG(ctable, update); + WT_ERR(ret); + if (ctable->idx_cursors != NULL) + APPLY_IDX(ctable, insert); +err: API_END(session); + + return (ret); +} + +/* + * __curtable_remove -- + * WT_CURSOR->remove method for the table cursor type. + */ +static int +__curtable_remove(WT_CURSOR *cursor) +{ + WT_CURSOR_TABLE *ctable; + WT_SESSION_IMPL *session; + int ret; + + ctable = (WT_CURSOR_TABLE *)cursor; + CURSOR_API_CALL(cursor, session, remove, NULL); + + /* Find the old record so it can be removed from indices */ + WT_ERR(__curtable_open_indices(ctable)); + if (ctable->table->nindices > 0) { + APPLY_CG(ctable, search); + WT_ERR(ret); + APPLY_IDX(ctable, remove); + } + + APPLY_CG(ctable, remove); +err: API_END(session); + + return (ret); +} + +/* + * __curtable_close -- + * WT_CURSOR->close method for the table cursor type. + */ +static int +__curtable_close(WT_CURSOR *cursor) +{ + WT_CURSOR_TABLE *ctable; + WT_CURSOR **cp; + WT_SESSION_IMPL *session; + int i, ret; + + ctable = (WT_CURSOR_TABLE *)cursor; + CURSOR_API_CALL(cursor, session, close, NULL); + + for (i = 0, cp = (ctable)->cg_cursors; + i < WT_COLGROUPS(ctable->table); i++, cp++) + if (*cp != NULL) { + WT_TRET((*cp)->close(*cp)); + *cp = NULL; + } + + if (ctable->idx_cursors != NULL) + for (i = 0, cp = (ctable)->idx_cursors; + i < ctable->table->nindices; i++, cp++) + if (*cp != NULL) { + WT_TRET((*cp)->close(*cp)); + *cp = NULL; + } + + if (ctable->plan != ctable->table->plan) + __wt_free(session, ctable->plan); + __wt_free(session, ctable->cg_cursors); + __wt_free(session, ctable->idx_cursors); + /* The URI is owned by the table. */ + cursor->uri = NULL; + WT_TRET(__wt_cursor_close(cursor)); + API_END(session); + + return (ret); +} + +static int +__curtable_open_colgroups(WT_CURSOR_TABLE *ctable, const char *cfg[]) +{ + WT_SESSION_IMPL *session; + WT_TABLE *table; + WT_CURSOR **cp; + const char *cfg_no_overwrite[4]; + int i; + + session = (WT_SESSION_IMPL *)ctable->iface.session; + table = ctable->table; + + /* Underlying column groups are always opened without overwrite */ + cfg_no_overwrite[0] = cfg[0]; + cfg_no_overwrite[1] = cfg[1]; + cfg_no_overwrite[2] = "overwrite=false"; + cfg_no_overwrite[3] = NULL; + + if (!table->cg_complete) + WT_RET_MSG(session, EINVAL, + "Can't use '%s' until all column groups are created", + table->name); + + WT_RET(__wt_calloc_def(session, + WT_COLGROUPS(table), &ctable->cg_cursors)); + + for (i = 0, cp = ctable->cg_cursors; + i < WT_COLGROUPS(table); + i++, cp++) { + session->btree = table->colgroup[i]; + WT_RET(__wt_curfile_create(session, cfg_no_overwrite, cp)); + } + return (0); +} + +static int +__curtable_open_indices(WT_CURSOR_TABLE *ctable) +{ + WT_CURSOR **cp, *primary; + WT_SESSION_IMPL *session; + WT_TABLE *table; + const char *cfg[] = API_CONF_DEFAULTS(session, open_cursor, NULL); + int i; + + session = (WT_SESSION_IMPL *)ctable->iface.session; + table = ctable->table; + + if (!ctable->table->idx_complete) + WT_RET(__wt_schema_open_index(session, table, NULL, 0)); + if (table->nindices == 0 || ctable->idx_cursors != NULL) + return (0); + /* Check for bulk cursors. */ + primary = *ctable->cg_cursors; + if (F_ISSET(((WT_CURSOR_BTREE *)primary)->btree, WT_BTREE_BULK)) + WT_RET_MSG(session, ENOTSUP, + "Bulk load is not supported for tables with indices"); + WT_RET(__wt_calloc_def(session, table->nindices, &ctable->idx_cursors)); + for (i = 0, cp = ctable->idx_cursors; i < table->nindices; i++, cp++) { + session->btree = table->index[i]; + WT_RET(__wt_curfile_create(session, cfg, cp)); + } + return (0); +} + +/* + * __wt_curtable_open -- + * WT_SESSION->open_cursor method for table cursors. + */ +int +__wt_curtable_open(WT_SESSION_IMPL *session, + const char *uri, const char *cfg[], WT_CURSOR **cursorp) +{ + static WT_CURSOR iface = { + NULL, + NULL, + NULL, + NULL, + __wt_curtable_get_key, + __wt_curtable_get_value, + __wt_curtable_set_key, + __wt_curtable_set_value, + NULL, + __curtable_next, + __curtable_prev, + __curtable_reset, + __curtable_search, + __curtable_search_near, + __curtable_insert, + __curtable_update, + __curtable_remove, + __curtable_close, + { NULL, NULL }, /* TAILQ_ENTRY q */ + 0, /* recno key */ + { 0 }, /* raw recno buffer */ + { NULL, 0, 0, NULL, 0 },/* WT_ITEM key */ + { NULL, 0, 0, NULL, 0 },/* WT_ITEM value */ + 0, /* int saved_err */ + 0 /* uint32_t flags */ + }; + WT_CONFIG_ITEM cval; + WT_CURSOR *cursor; + WT_CURSOR_TABLE *ctable; + WT_ITEM fmt, plan; + WT_TABLE *table; + size_t size; + int ret; + const char *tablename, *columns; + + WT_CLEAR(fmt); + WT_CLEAR(plan); + ctable = NULL; + + tablename = uri; + if (!WT_PREFIX_SKIP(tablename, "table:")) + return (EINVAL); + columns = strchr(tablename, '('); + if (columns == NULL) + size = strlen(tablename); + else + size = columns - tablename; + if ((ret = __wt_schema_get_table(session, + tablename, size, &table)) != 0) { + if (ret == WT_NOTFOUND) + WT_RET_MSG(session, EINVAL, + "Cannot open cursor '%s' on unknown table", uri); + return (ret); + } + + if (!table->cg_complete) + WT_RET_MSG(session, EINVAL, + "Cannot open cursor '%s' on incomplete table", uri); + if (table->is_simple) { + /* + * The returned cursor should be public: it is not part of a + * table cursor. + */ + session->btree = table->colgroup[0]; + return (__wt_curfile_create(session, cfg, cursorp)); + } + + WT_RET(__wt_calloc_def(session, 1, &ctable)); + + cursor = &ctable->iface; + *cursor = iface; + cursor->session = &session->iface; + cursor->uri = table->name; + cursor->key_format = table->key_format; + cursor->value_format = table->value_format; + F_SET(cursor, WT_CURSTD_TABLE); + + ctable->table = table; + ctable->plan = table->plan; + + /* Handle projections. */ + if (columns != NULL) { + WT_ERR(__wt_struct_reformat(session, table, + columns, strlen(columns), NULL, 1, &fmt)); + cursor->value_format = __wt_buf_steal(session, &fmt, NULL); + + WT_ERR(__wt_struct_plan(session, table, + columns, strlen(columns), 0, &plan)); + ctable->plan = __wt_buf_steal(session, &plan, NULL); + } + + /* The append flag is only relevant to column stores. */ + if (WT_CURSOR_RECNO(cursor)) { + WT_ERR(__wt_config_gets(session, cfg, "append", &cval)); + if (cval.val != 0) + F_SET(cursor, WT_CURSTD_APPEND); + } + + WT_ERR(__wt_config_gets(session, cfg, "dump", &cval)); + if (cval.len != 0) { + __wt_curdump_init(cursor); + F_SET(cursor, + strncmp(cval.str, "print", cval.len) == 0 ? + WT_CURSTD_DUMP_PRINT : WT_CURSTD_DUMP_HEX); + } + + WT_ERR(__wt_config_gets(session, cfg, "raw", &cval)); + if (cval.val != 0) + F_SET(cursor, WT_CURSTD_RAW); + + WT_ERR(__wt_config_gets(session, cfg, "overwrite", &cval)); + if (cval.val != 0) + F_SET(cursor, WT_CURSTD_OVERWRITE); + + /* + * Open the colgroup cursors immediately: we're going to need them for + * any operation. We defer opening index cursors until we need them + * for an update. + */ + WT_ERR(__curtable_open_colgroups(ctable, cfg)); + + STATIC_ASSERT(offsetof(WT_CURSOR_TABLE, iface) == 0); + WT_ERR(__wt_cursor_init(cursor, cursor->uri, 0, 1, cfg)); + *cursorp = cursor; + + if (0) { +err: (void)__curtable_close(cursor); + } + + return (ret); +} diff --git a/src/include/api.h b/src/include/api.h new file mode 100644 index 00000000000..355f74ea65f --- /dev/null +++ b/src/include/api.h @@ -0,0 +1,280 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* + * WT_PROCESS -- + * Per-process information for the library. + */ +struct __wt_process { + WT_SPINLOCK spinlock; /* Per-process spinlock */ + + /* Locked: connection queue */ + TAILQ_HEAD(__wt_connection_impl_qh, __wt_connection_impl) connqh; +}; + +/******************************************* + * Implementation of WT_SESSION + *******************************************/ +/* + * WT_BTREE_SESSION -- + * Per-session cache of btree handles to avoid synchronization when + * opening cursors. + */ +struct __wt_btree_session { + WT_BTREE *btree; + + TAILQ_ENTRY(__wt_btree_session) q; +}; + +/* + * WT_HAZARD -- + * A hazard reference. + */ +struct __wt_hazard { + WT_PAGE *page; /* Page address */ +#ifdef HAVE_DIAGNOSTIC + const char *file; /* File/line where hazard acquired */ + int line; +#endif +}; + +typedef enum { + WT_SERIAL_NONE=0, /* No request */ + WT_SERIAL_FUNC=1, /* Function, then return */ + WT_SERIAL_EVICT=2, /* Function, then schedule evict */ +} wq_state_t; + +/* Get the connection implementation for a session */ +#define S2C(session) ((WT_CONNECTION_IMPL *)(session)->iface.connection) + +/* + * WT_SESSION_IMPL -- + * Implementation of WT_SESSION. + */ +struct __wt_session_impl { + WT_SESSION iface; + + WT_CONDVAR *cond; /* Condition variable */ + + const char *name; /* Name */ + WT_EVENT_HANDLER *event_handler; + + WT_BTREE *btree; /* Current file */ + TAILQ_HEAD(__btrees, __wt_btree_session) btrees; + + WT_CURSOR *cursor; /* Current cursor */ + /* All file cursors */ + TAILQ_HEAD(__file_cursors, __wt_cursor) file_cursors; + /* Cursors closed with the session */ + TAILQ_HEAD(__public_cursors, __wt_cursor) public_cursors; + + WT_BTREE *schematab; /* Schema tables */ + TAILQ_HEAD(__tables, __wt_table) tables; + + WT_ITEM logrec_buf; /* Buffer for log records */ + WT_ITEM logprint_buf; /* Buffer for debug log records */ + + WT_ITEM **scratch; /* Temporary memory for any function */ + u_int scratch_alloc; /* Currently allocated */ + + /* Serialized operation state */ + void *wq_args; /* Operation arguments */ + int wq_sleeping; /* Thread is blocked */ + int wq_ret; /* Return value */ + + WT_HAZARD *hazard; /* Hazard reference array */ + + void *reconcile; /* Reconciliation information */ + + WT_REF **excl; /* Eviction exclusive list */ + u_int excl_next; /* Next empty slot */ + size_t excl_allocated; /* Bytes allocated */ + + void *schema_track; /* Tracking schema operations */ + u_int schema_track_entries; /* Currently allocated */ + + uint32_t flags; +}; + +/******************************************* + * Implementation of WT_CONNECTION + *******************************************/ +/* + * WT_NAMED_COLLATOR -- + * A collator list entry + */ +struct __wt_named_collator { + const char *name; /* Name of collator */ + WT_COLLATOR *collator; /* User supplied object */ + TAILQ_ENTRY(__wt_named_collator) q; /* Linked list of collators */ +}; + +/* + * WT_NAMED_COMPRESSOR -- + * A compressor list entry + */ +struct __wt_named_compressor { + const char *name; /* Name of compressor */ + WT_COMPRESSOR *compressor; /* User supplied callbacks */ + TAILQ_ENTRY(__wt_named_compressor) q; /* Linked list of compressors */ +}; + +/* + * WT_CONNECTION_IMPL -- + * Implementation of WT_CONNECTION + */ +struct __wt_connection_impl { + WT_CONNECTION iface; + + WT_SESSION_IMPL default_session;/* For operations without an + application-supplied session */ + + WT_SPINLOCK spinlock; /* Connection spinlock */ + /* Connection queue */ + TAILQ_ENTRY(__wt_connection_impl) q; + + const char *home; /* Database home */ + int is_new; /* Connection created database */ + + WT_FH *lock_fh; /* Lock file handle */ + + WT_SPINLOCK serial_lock; /* Serial function call spinlock */ + pthread_t cache_evict_tid; /* Cache eviction server thread ID */ + pthread_t cache_read_tid; /* Cache read server thread ID */ + + /* Locked: btree list */ + TAILQ_HEAD(__wt_btree_qh, __wt_btree) btqh; + + /* Locked: file list */ + TAILQ_HEAD(__wt_fh_qh, __wt_fh) fhqh; + + /* Locked: library list */ + TAILQ_HEAD(__wt_dlh_qh, __wt_dlh) dlhqh; + + u_int btqcnt; /* Locked: btree count */ + u_int next_file_id; /* Locked: file ID counter */ + + /* + * WiredTiger allocates space for 50 simultaneous sessions (threads of + * control) by default. Growing the number of threads dynamically is + * possible, but tricky since server threads are walking the array + * without locking it. + * + * There's an array of WT_SESSION_IMPL pointers that reference the + * allocated array; we do it that way because we want an easy way for + * the server thread code to avoid walking the entire array when only a + * few threads are running. + */ + WT_SESSION_IMPL **sessions; /* Session reference */ + void *session_array; /* Session array */ + uint32_t session_cnt; /* Session count */ + + /* + * WiredTiger allocates space for 15 hazard references in each thread of + * control, by default. There's no code path that requires more than 15 + * pages at a time (and if we find one, the right change is to increase + * the default). + * + * The hazard array is separate from the WT_SESSION_IMPL array because + * we need to easily copy and search it when evicting pages from memory. + */ + WT_HAZARD *hazard; /* Hazard references array */ + uint32_t hazard_size; + uint32_t session_size; + + WT_CACHE *cache; /* Page cache */ + uint64_t cache_size; + + WT_CONNECTION_STATS *stats; /* Connection statistics */ + + WT_FH *log_fh; /* Logging file handle */ + + /* Locked: collator list */ + TAILQ_HEAD(__wt_coll_qh, __wt_named_collator) collqh; + + /* Locked: compressor list */ + TAILQ_HEAD(__wt_comp_qh, __wt_named_compressor) compqh; + + FILE *msgfile; + void (*msgcall)(const WT_CONNECTION_IMPL *, const char *); + + uint32_t verbose; + + uint32_t flags; +}; + +/* Standard entry points to the API: declares/initializes local variables. */ +#define API_CONF_DEFAULTS(h, n, cfg) \ + { __wt_confdfl_##h##_##n, (cfg), NULL } + +#define API_SESSION_INIT(s, h, n, cur, bt) \ + WT_BTREE *__oldbtree = (s)->btree; \ + const char *__oldname = (s)->name; \ + (s)->cursor = (cur); \ + (s)->btree = (bt); \ + (s)->name = #h "." #n; \ + ret = 0; + +#define API_CALL_NOCONF(s, h, n, cur, bt) do { \ + API_SESSION_INIT(s, h, n, cur, bt); + +#define API_CALL(s, h, n, cur, bt, cfg, cfgvar) do { \ + const char *cfgvar[] = API_CONF_DEFAULTS(h, n, cfg); \ + API_SESSION_INIT(s, h, n, cur, bt); \ + WT_ERR(((cfg) != NULL) ? \ + __wt_config_check((s), __wt_confchk_##h##_##n, (cfg)) : 0) + +#define API_END(s) \ + if ((s) != NULL) { \ + (s)->btree = __oldbtree; \ + (s)->name = __oldname; \ + } \ +} while (0) + +#define SESSION_API_CALL(s, n, cfg, cfgvar) \ + API_CALL(s, session, n, NULL, NULL, cfg, cfgvar); + +#define CONNECTION_API_CALL(conn, s, n, cfg, cfgvar) \ + s = &conn->default_session; \ + API_CALL(s, connection, n, NULL, NULL, cfg, cfgvar); \ + +#define CURSOR_API_CALL(cur, s, n, bt) \ + (s) = (WT_SESSION_IMPL *)(cur)->session; \ + API_CALL_NOCONF(s, cursor, n, (cur), bt); \ + +/******************************************* + * Global variables. + *******************************************/ +extern WT_EVENT_HANDLER *__wt_event_handler_default; +extern WT_EVENT_HANDLER *__wt_event_handler_verbose; +extern WT_PROCESS __wt_process; + +/* + * DO NOT EDIT: automatically built by dist/api_flags.py. + * API flags section: BEGIN + */ +#define WT_PAGE_FREE_IGNORE_DISK 0x00000001 +#define WT_REC_SINGLE 0x00000001 +#define WT_SERVER_RUN 0x00000001 +#define WT_SESSION_INTERNAL 0x00000002 +#define WT_SESSION_SALVAGE_QUIET_ERR 0x00000001 +#define WT_VERB_block 0x00000800 +#define WT_VERB_evict 0x00000400 +#define WT_VERB_evictserver 0x00000200 +#define WT_VERB_fileops 0x00000100 +#define WT_VERB_hazard 0x00000080 +#define WT_VERB_mutex 0x00000040 +#define WT_VERB_read 0x00000020 +#define WT_VERB_readserver 0x00000010 +#define WT_VERB_reconcile 0x00000008 +#define WT_VERB_salvage 0x00000004 +#define WT_VERB_verify 0x00000002 +#define WT_VERB_write 0x00000001 +/* + * API flags section: END + * DO NOT EDIT: automatically built by dist/api_flags.py. + */ diff --git a/src/include/bitstring.i b/src/include/bitstring.i new file mode 100644 index 00000000000..4f788a625b5 --- /dev/null +++ b/src/include/bitstring.i @@ -0,0 +1,327 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/*- + * Copyright (c) 1989, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Paul Vixie. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD: src/sys/sys/bitstring.h,v 1.5 2005/01/07 02:29:23 imp Exp $ + */ + + /* byte of the bitstring bit is in */ +#define __bit_byte(bit) ((bit) >> 3) + + /* mask for the bit within its byte */ +#define __bit_mask(bit) (1 << ((bit) & 0x7)) + +/* + * __bitstr_size -- + * Return the bytes in a bitstring of nbits. + */ +static inline uint32_t +__bitstr_size(uint32_t nbits) +{ + return (((nbits) + 7) >> 3); +} + +/* + * __bit_alloc -- + * Allocate a bitstring. + */ +static inline int +__bit_alloc(WT_SESSION_IMPL *session, uint32_t nbits, void *retp) +{ + return (__wt_calloc( + session, (size_t)__bitstr_size(nbits), sizeof(uint8_t), retp)); +} + +/* + * __bit_test -- + * Test one bit in name. + */ +static inline int +__bit_test(uint8_t *bitf, uint32_t bit) +{ + return (bitf[__bit_byte(bit)] & __bit_mask(bit) ? 1 : 0); +} + +/* + * __bit_set -- + * Set one bit in name. + */ +static inline void +__bit_set(uint8_t *bitf, uint32_t bit) +{ + bitf[__bit_byte(bit)] |= __bit_mask(bit); +} + +/* + * __bit_clear -- + * Clear one bit in name. + */ +static inline void +__bit_clear(uint8_t *bitf, uint32_t bit) +{ + bitf[__bit_byte(bit)] &= ~__bit_mask(bit); +} + +#ifdef __NOT_CURRENTLY_USED +/* + * __bit_nclr -- + * Clear bits start-to-stop in name. + */ +static inline void +__bit_nclr(uint8_t *bitf, uint32_t start, uint32_t stop) +{ + uint32_t startbyte, stopbyte; + + startbyte = __bit_byte(start); + stopbyte = __bit_byte(stop); + + if (startbyte == stopbyte) + bitf[startbyte] &= + ((0xff >> (8 - (start & 0x7))) | + (0xff << ((stop & 0x7) + 1))); + else { + bitf[startbyte] &= 0xff >> (8 - (start & 0x7)); + while (++startbyte < stopbyte) + bitf[startbyte] = 0; + bitf[stopbyte] &= 0xff << ((stop & 0x7) + 1); + } +} +#endif + +/* + * __bit_nset -- + * Set bits start-to-stop in name. + */ +static inline void +__bit_nset(uint8_t *bitf, uint32_t start, uint32_t stop) +{ + uint32_t startbyte, stopbyte; + + startbyte = __bit_byte(start); + stopbyte = __bit_byte(stop); + if (startbyte == stopbyte) + bitf[startbyte] |= + ((0xff << (start & 0x7)) & (0xff >> (7 - (stop & 0x7)))); + else { + bitf[startbyte] |= 0xff << (start & 0x7); + while (++startbyte < stopbyte) + bitf[startbyte] = 0xff; + bitf[stopbyte] |= 0xff >> (7 - (stop & 0x7)); + } +} + +/* + * __bit_ffc -- + * Find first clear bit in name, return 0 on success, -1 on no bit clear. + */ +static inline int +__bit_ffc(uint8_t *bitf, uint32_t nbits, uint32_t *retp) +{ + uint8_t lb; + uint32_t byte, stopbyte, value; + + value = 0; /* -Wuninitialized */ + + if (nbits == 0) + return (-1); + + for (byte = 0, + stopbyte = __bit_byte(nbits - 1); byte <= stopbyte; ++byte) + if (bitf[byte] != 0xff) { + value = byte << 3; + for (lb = bitf[byte]; lb & 0x01; ++value, lb >>= 1) + ; + break; + } + + if (byte > stopbyte || value >= nbits) + return (-1); + + *retp = value; + return (0); +} + +#ifdef __NOT_CURRENTLY_USED +/* + * __bit_ffs -- + * Find first set bit in name, return 0 on success, -1 on no bit set. + */ +static inline int +__bit_ffs(uint8_t *bitf, uint32_t nbits, uint32_t *retp) +{ + uint8_t lb; + uint32_t byte, stopbyte, value; + + if (nbits == 0) + return (-1); + + for (byte = 0, + stopbyte = __bit_byte(nbits - 1); byte <= stopbyte; ++byte) + if (bitf[byte] != 0) { + value = byte << 3; + for (lb = bitf[byte]; !(lb & 0x01); ++value, lb >>= 1) + ; + break; + } + + if (byte > stopbyte || value >= nbits) + return (-1); + + *retp = value; + return (0); +} +#endif + +/* + * __bit_getv -- + * Return a fixed-length column store bit-field value. + */ +static inline uint8_t +__bit_getv(uint8_t *bitf, uint32_t entry, uint8_t width) +{ + uint8_t value; + uint32_t bit; + +#define __BIT_GET(len, mask) \ + case len: \ + if (__bit_test(bitf, bit)) \ + value |= mask; \ + ++bit \ + /* FALLTHROUGH */ + + value = 0; + bit = entry * width; + + /* + * Fast-path single bytes, do repeated tests for the rest: we could + * slice-and-dice instead, but the compiler is probably going to do + * a better job than I will. + */ + switch (width) { + case 8: + return (bitf[__bit_byte(bit)]); + __BIT_GET(7, 0x40); + __BIT_GET(6, 0x20); + __BIT_GET(5, 0x10); + __BIT_GET(4, 0x08); + __BIT_GET(3, 0x04); + __BIT_GET(2, 0x02); + __BIT_GET(1, 0x01); + } + return (value); +} + +/* + * __bit_getv_recno -- + * Return a record number's bit-field value. + */ +static inline uint8_t +__bit_getv_recno(WT_PAGE *page, uint64_t recno, uint8_t width) +{ + return (__bit_getv(page->u.col_fix.bitf, + (uint32_t)(recno - page->u.col_fix.recno), width)); +} + +/* + * __bit_setv -- + * Set a fixed-length column store bit-field value. + */ +static inline void +__bit_setv(uint8_t *bitf, uint32_t entry, uint8_t width, uint8_t value) +{ + uint32_t bit; + +#define __BIT_SET(len, mask) \ + case len: \ + if (value & (mask)) \ + __bit_set(bitf, bit); \ + else \ + __bit_clear(bitf, bit); \ + ++bit \ + /* FALLTHROUGH */ + + bit = entry * width; + + /* + * Fast-path single bytes, do repeated tests for the rest: we could + * slice-and-dice instead, but the compiler is probably going to do + * a better job than I will. + */ + switch (width) { + case 8: + bitf[__bit_byte(bit)] = value; + return; + __BIT_SET(7, 0x40); + __BIT_SET(6, 0x20); + __BIT_SET(5, 0x10); + __BIT_SET(4, 0x08); + __BIT_SET(3, 0x04); + __BIT_SET(2, 0x02); + __BIT_SET(1, 0x01); + } +} + +/* + * __bit_setv_recno -- + * Set a record number's bit-field value. + */ +static inline void +__bit_setv_recno(WT_PAGE *page, uint64_t recno, uint8_t width, uint8_t value) +{ + return (__bit_setv(page->u.col_fix.bitf, + (uint32_t)(recno - page->u.col_fix.recno), width, value)); +} diff --git a/src/include/block.h b/src/include/block.h new file mode 100644 index 00000000000..d217a5ff496 --- /dev/null +++ b/src/include/block.h @@ -0,0 +1,237 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* + * WiredTiger's block manager interface. + */ +#define WT_BM_MAX_ADDR_COOKIE 255 /* Maximum address cookie */ + +/* + * The file's description is written into the first 512B of the file, which + * means we can use an offset of 0 as an invalid offset. + */ +#define WT_BLOCK_DESC_SECTOR 512 +#define WT_BLOCK_INVALID_OFFSET 0 + +/* + * The block allocator maintains two primary skiplists: first, the by-offset + * list linking WT_FREE elements and sorted by file offset (low-to-high): + * this list has an entry for every free chunk in the file. The second primary + * skiplist is the by-size list linking WT_SIZE elements and sorted by chunk + * size (low-to-high). This list has an entry for every free chunk size seen + * since the list was created. + * Additionally, each WT_SIZE element has a skiplist of its own, linking + * WT_FREE elements and sorted by file offset (low-to-high). This list has an + * entry for every free chunk in the file of a particular size. + * The trickiness is that each individual WT_FREE element appears on two + * skiplists. In order to minimize allocation calls, we allocate a single + * array of WT_FREE pointers at the end of the WT_FREE structure, for both + * skiplists, and store the depth of the skiplist in the WT_FREE structure. + * The skiplist entries for the offset skiplist start at WT_FREE.next[0] and + * the entries for the size skiplist start at WT_FREE.next[WT_FREE.depth]. + * + * WT_FREE -- + * Encapsulation of a free chunk of space. + */ +struct __wt_free { + off_t off; /* File offset */ + off_t size; /* Size */ + + uint8_t depth; /* Skip list depth */ + + /* + * Variable-length array, sized by the number of skiplist elements. + * The first depth array entries are the offset skiplist elements, + * the second depth array entries are the size skiplist. + */ + WT_FREE *next[0]; /* Offset, size skiplists */ +}; + +/* + * WT_SIZE -- + * Encapsulation of a block size skiplist entry. + */ +struct __wt_size { + off_t size; /* Size */ + + uint8_t depth; /* Skip list depth */ + + WT_FREE *foff[WT_SKIP_MAXDEPTH]; /* Per-size offset skiplist */ + + /* Variable-length array, sized by the number of skiplist elements. */ + WT_SIZE *next[0]; /* Size skiplist */ +}; + +/* + * WT_FREE_FOREACH -- + * Walk a block manager skiplist. + * WT_FREE_FOREACH_OFF -- + * Walk a block manager skiplist where the WT_FREE.next entries are offset + * by the depth. + */ +#define WT_FREE_FOREACH(skip, head) \ + for ((skip) = (head)[0]; \ + (skip) != NULL; (skip) = (skip)->next[0]) +#define WT_FREE_FOREACH_OFF(skip, head) \ + for ((skip) = (head)[0]; \ + (skip) != NULL; (skip) = (skip)->next[(skip)->depth]) + +/* + * WT_BLOCK -- + * Encapsulation of the standard WiredTiger block manager. + */ +struct __wt_block { + const char *name; /* Name */ + + WT_FH *fh; /* Backing file handle */ + + uint64_t write_gen; /* Write generation */ + + uint32_t allocsize; /* Allocation size */ + int checksum; /* If checksums configured */ + + WT_COMPRESSOR *compressor; /* Page compressor */ + + /* Freelist support */ + WT_SPINLOCK freelist_lock; /* Lock to protect the freelist. */ + + uint64_t freelist_bytes; /* Freelist byte count */ + uint32_t freelist_entries; /* Freelist entry count */ + int freelist_dirty; /* Freelist has been modified */ + + /* Freelist offset/size skiplists */ + WT_FREE *foff[WT_SKIP_MAXDEPTH]; + WT_SIZE *fsize[WT_SKIP_MAXDEPTH]; + + off_t free_offset; /* Freelist file location */ + uint32_t free_size; + uint32_t free_cksum; + + /* Salvage support */ + off_t slvg_off; /* Salvage file offset */ + + /* Verification support */ + uint32_t frags; /* Total frags */ + uint8_t *fragbits; /* Frag tracking bit list */ + +#define WT_BLOCK_OK 0x01 /* File successfully opened */ + uint32_t flags; +}; + +/* + * WT_BLOCK_DESC -- + * The file's description. + */ +struct __wt_block_desc { +#define WT_BLOCK_MAGIC 120897 + uint32_t magic; /* 00-03: Magic number */ +#define WT_BLOCK_MAJOR_VERSION 1 + uint16_t majorv; /* 04-05: Major version */ +#define WT_BLOCK_MINOR_VERSION 0 + uint16_t minorv; /* 06-07: Minor version */ + + uint32_t cksum; /* 08-11: Description block checksum */ + + uint32_t unused; /* 12-15: Padding */ +#define WT_BLOCK_FREELIST_MAGIC 071002 + uint64_t free_offset; /* 16-23: Free list page offset */ + uint32_t free_size; /* 24-27: Free list page length */ + uint32_t free_cksum; /* 28-31: Free list page checksum */ + + /* + * We maintain page write-generations in the non-transactional case + * (where, instead of a transactional LSN, the value is a counter), + * as that's how salvage can determine the most recent page between + * pages overlapping the same key range. The value has to persist, + * so it's included in the file's metadata. + */ + uint64_t write_gen; /* 32-39: Write generation */ +}; +/* + * WT_BLOCK_DESC_SIZE is the expected structure size -- we verify the build to + * ensure the compiler hasn't inserted padding (padding won't cause failure, + * since we reserve the first sector of the file for this information, but it + * would be worth investigation, regardless). + */ +#define WT_BLOCK_DESC_SIZE 40 + +/* + * WT_BLOCK_HEADER -- + * Blocks have a common header, a WT_PAGE_HEADER structure followed by a + * block-manager specific structure: WT_BLOCK_HEADER is WiredTiger's default. + */ +struct __wt_block_header { + /* + * We maintain page write-generations in the non-transactional case + * (where, instead of a transactional LSN, the value is a counter), + * as that's how salvage can determine the most recent page between + * pages overlapping the same key range. + * + * !!! + * The write-generation is "owned" by the btree layer, but it's easier + * to set it (when physically writing blocks), to persist it (in the + * WT_BLOCK_DESC structure, rather than the schema file), and restore + * it during salvage, in the block-manager layer. + */ + uint64_t write_gen; /* 00-07: write generation */ + + /* + * We write the page size in the on-disk page header because it makes + * salvage easier. (If we don't know the expected page length, we'd + * have to read increasingly larger chunks from the file until we find + * one that checksums, and that's going to be harsh given WiredTiger's + * potentially large page sizes.) + */ + uint32_t disk_size; /* 08-11: on-disk page size */ + + /* + * Page checksums are stored in two places. First, a page's checksum + * is in the internal page that references a page as part of the + * address cookie. This is done to improve the chances that we detect + * corruption (e.g., overwriting a page with another valid page image). + * Second, a page's checksum is stored in the disk header. This is for + * salvage, so that salvage knows when it has found a page that may be + * useful. + * + * Applications can turn off checksums, which is a promise that the + * file can never become corrupted, but people sometimes make promises + * they can't keep. If no checksums are configured, we use a pattern + * of alternating bits as the checksum, as that is unlikely to occur as + * the result of corruption in the file. If a page happens to checksum + * to this special bit pattern, we bump it by one during reads and + * writes to avoid ambiguity. + */ +#define WT_BLOCK_CHECKSUM_NOT_SET 0xA5C35A3C + uint32_t cksum; /* 12-15: checksum */ +}; +/* + * WT_BLOCK_HEADER_SIZE is the number of bytes we allocate for the structure: if + * the compiler inserts padding it will break the world. + */ +#define WT_BLOCK_HEADER_SIZE 16 + +/* + * WT_BLOCK_HEADER_BYTE + * WT_BLOCK_HEADER_BYTE_SIZE -- + * The first usable data byte on the block (past the combined headers). + */ +#define WT_BLOCK_HEADER_BYTE_SIZE \ + (WT_PAGE_HEADER_SIZE + WT_BLOCK_HEADER_SIZE) +#define WT_BLOCK_HEADER_BYTE(dsk) \ + ((void *)((uint8_t *)(dsk) + WT_BLOCK_HEADER_BYTE_SIZE)) + +/* + * Don't compress the block's WT_PAGE_HEADER and WT_BLOCK_HEADER structures. + * We need the WT_PAGE_HEADER in-memory size, and the WT_BLOCK_HEADER checksum + * and on-disk size to be immediately available without decompression. We use + * the on-disk size and checksum during salvage to figure out where the blocks + * are, and the in-memory size tells us how large a buffer we need to decompress + * the block. We could skip less than 64B, but a 64B boundary may offer better + * alignment for the underlying compression engine, and skipping 64B won't make + * a difference in terms of compression efficiency. + */ +#define WT_BLOCK_COMPRESS_SKIP 64 diff --git a/src/include/btmem.h b/src/include/btmem.h new file mode 100644 index 00000000000..8e230ff8c2f --- /dev/null +++ b/src/include/btmem.h @@ -0,0 +1,661 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* + * WT_PAGE_HEADER -- + * Blocks have a common header, a WT_PAGE_HEADER structure followed by a + * block-manager specific structure. + */ +struct __wt_page_header { + /* + * The record number of the first record of the page is stored on disk + * so we can figure out where the column-store leaf page fits into the + * key space during salvage. + */ + uint64_t recno; /* 00-07: column-store starting recno */ + + uint32_t size; /* 08-11: page size */ + + union { + uint32_t entries; /* 12-15: number of cells on page */ + uint32_t datalen; /* 12-15: overflow data length */ + } u; + + uint8_t type; /* 16: page type */ + + /* + * End the WT_PAGE_HEADER structure with 3 bytes of padding: it wastes + * space, but it leaves the WT_PAGE_HEADER structure 32-bit aligned and + * having a small amount of space to play with in the future can't hurt. + */ + uint8_t unused[3]; /* 17-19: unused padding */ +}; +/* + * WT_PAGE_HEADER_SIZE is the number of bytes we allocate for the structure: if + * the compiler inserts padding it will break the world. + */ +#define WT_PAGE_HEADER_SIZE 20 + +/* + * The block-manager specific information immediately follows the WT_PAGE_DISK + * structure. + */ +#define WT_BLOCK_HEADER_REF(dsk) \ + ((void *)((uint8_t *)(dsk) + WT_PAGE_HEADER_SIZE)) + +/* + * WT_PAGE_HEADER_BYTE -- + * WT_PAGE_HEADER_BYTE_SIZE -- + * The first usable data byte on the block (past the combined headers). + */ +#define WT_PAGE_HEADER_BYTE_SIZE(btree) \ + ((u_int)(WT_PAGE_HEADER_SIZE + (btree)->block_header)) +#define WT_PAGE_HEADER_BYTE(btree, dsk) \ + ((void *)((uint8_t *)(dsk) + WT_PAGE_HEADER_BYTE_SIZE(btree))) + +/* + * WT_ADDR -- + * A block location. + */ +#define WT_NOADDR "[NoAddr]" /* No address */ +struct __wt_addr { + uint8_t *addr; /* Cookie */ + uint32_t size; /* Cookie length */ +}; + +/* + * WT_PAGE_MODIFY -- + * When a page is modified, there's additional information maintained as it + * is written to disk. + */ +typedef enum { + WT_PT_EMPTY=0, /* Unused slot */ + WT_PT_BLOCK, /* Block: inactive */ + WT_PT_BLOCK_EVICT, /* Block: inactive on eviction */ + WT_PT_OVFL, /* Overflow: active */ + WT_PT_OVFL_DISCARD /* Overflow: inactive */ +} __wt_pt_type_t; + +struct __wt_page_modify { + /* + * The write generation is incremented after a page is modified. That + * is, it tracks page versions. + * + * The write generation value is used to detect changes scheduled based + * on out-of-date information. Two threads of control updating the same + * page could both search the page in state A. When the updates are + * performed serially, one of the changes will happen after the page is + * modified, and the search state for the other thread might no longer + * be applicable. To avoid this race, page write generations are copied + * into the search stack whenever a page is read, and check when a + * modification is serialized. The serialized function compares each + * page's current write generation to the generation copied in the + * read/search; if the two values match, the search occurred on a + * current version of the page and the modification can proceed. If the + * two generations differ, the serialized call returns an error and the + * operation must be restarted. + * + * The write-generation value could be stored on a per-entry basis if + * there's sufficient contention for the page as a whole. + * + * The write-generation is not declared volatile: write-generation is + * written by a serialized function when modifying a page, and must be + * flushed in order as the serialized updates are flushed. + * + * !!! + * 32-bit values are probably more than is needed: at some point we may + * need to clean up pages once there have been sufficient modifications + * to make our linked lists of inserted cells too slow to search, or as + * soon as enough memory is allocated in service of page modifications + * (although we should be able to release memory from the MVCC list as + * soon as there's no running thread/txn which might want that version + * of the data). I've used 32-bit types instead of 16-bit types as I + * am less confident a 16-bit write to memory will invariably be atomic. + */ + uint32_t write_gen; + + /* + * The disk generation tracks page versions written to disk. When a + * page is reconciled and written to disk, the thread doing that work + * is just another reader of the page, and other readers and writers + * can access the page at the same time. For this reason, the thread + * reconciling the page logs the write generation of the page it read. + */ + uint32_t disk_gen; + + union { + WT_PAGE *split; /* Resulting split */ + WT_ADDR replace; /* Resulting replacement */ + } u; + + /* + * Updated items in column-stores: variable-length RLE entries can + * expand to multiple entries which requires some kind of list we can + * expand on demand. Updated items in fixed-length files could be done + * based on an WT_UPDATE array as in row-stores, but there can be a + * very large number of bits on a single page, and the cost of the + * WT_UPDATE array would be huge. + */ + WT_INSERT_HEAD **update; /* Updated items */ + + /* + * Track pages, blocks to discard: as pages are reconciled, overflow + * K/V items are discarded along with their underlying blocks, and as + * pages are evicted, split and emptied pages are merged into their + * parents and discarded. If an overflow item was discarded and page + * reconciliation then failed, the in-memory tree would be corrupted. + * To keep the tree correct until we're sure page reconciliation has + * succeeded, we track the objects we'll discard when the reconciled + * page is evicted. + * + * Track overflow objects: if pages are reconciled more than once, an + * overflow item might be written repeatedly. Instead, when overflow + * items are written we save a copy and resulting location so we only + * write them once. + */ + struct __wt_page_track { + __wt_pt_type_t type; /* Type */ + + WT_ADDR addr; /* Overflow or block location */ + + uint8_t *data; /* Overflow data reference */ + uint32_t size; /* Overflow data length */ + } *track; /* Array of tracked objects */ + uint32_t track_entries; /* Total track slots */ +}; + +/* + * WT_PAGE -- + * The WT_PAGE structure describes the in-memory page information. + */ +struct __wt_page { + /* + * Two links to the parent: the physical parent page, and the internal + * page's reference structure used to find this page. + */ +#define WT_PAGE_IS_ROOT(page) \ + ((page)->parent == NULL) + WT_PAGE *parent; /* Page's parent */ + WT_REF *ref; /* Parent reference */ + + /* Per page-type information. */ + union { + /* + * Column- and row-store internal page. The recno is only used + * by column-store, but having the WT_REF array in the same page + * location makes some things simpler, and it doesn't cost us + * any memory, other structures in this union are still larger. + */ + struct { + uint64_t recno; /* Starting recno */ + WT_REF *t; /* Subtree */ + } intl; + + /* Row-store leaf page. */ + struct { + WT_ROW *d; /* K/V object pairs */ + + /* + * The column-store leaf page modification structures + * live in the WT_PAGE_MODIFY structure to keep the + * WT_PAGE structure as small as possible for read-only + * pages. For consistency, we could move the row-store + * modification structures into WT_PAGE_MODIFY too, but + * it doesn't shrink WT_PAGE any further, and avoiding + * ugly naming in WT_PAGE_MODIFY to avoid growing it + * won't be pretty. So far, avoiding ugly naming has + * overridden consistency. + */ + WT_INSERT_HEAD **ins; /* Inserts */ + WT_UPDATE **upd; /* Updates */ + } row; + + /* Fixed-length column-store leaf page. */ + struct { + uint64_t recno; /* Starting recno */ + uint8_t *bitf; /* COL_FIX items */ + } col_fix; + + /* Variable-length column-store leaf page. */ + struct { + uint64_t recno; /* Starting recno */ + WT_COL *d; /* COL_VAR items */ + + /* + * Variable-length column-store files maintain a list of + * RLE entries on the page so it's unnecessary to walk + * the page counting records to find a specific entry. + */ + WT_COL_RLE *repeats; /* RLE array for lookups */ + uint32_t nrepeats; /* Number of repeat slots. */ + } col_var; + } u; + + /* Page's on-disk representation: NULL for pages created in memory. */ + WT_PAGE_HEADER *dsk; + + /* If/when the page is modified, we need lots more information. */ + WT_PAGE_MODIFY *modify; + + /* + * The read generation is incremented each time the page is searched, + * and acts as an LRU value for each page in the tree; it is read by + * the eviction server thread to select pages to be discarded from the + * in-memory tree. + * + * The read generation is a 64-bit value; incremented every time the + * page is searched, a 32-bit value could overflow. + * + * The read-generation is not declared volatile: read-generation is set + * a lot (on every access), and we don't want to write it that much. + */ + uint64_t read_gen; + + /* + * In-memory pages optionally reference a number of entries originally + * read from disk and sizes the allocated arrays that describe the page. + */ + uint32_t entries; + + /* + * Memory attached to the page (although not exact or complete), used + * to force eviction of a page tying too much memory down. + */ + uint32_t memory_footprint; + +#define WT_PAGE_INVALID 0 /* Invalid page */ +#define WT_PAGE_COL_FIX 1 /* Col-store fixed-len leaf */ +#define WT_PAGE_COL_INT 2 /* Col-store internal page */ +#define WT_PAGE_COL_VAR 3 /* Col-store var-length leaf page */ +#define WT_PAGE_OVFL 4 /* Overflow page */ +#define WT_PAGE_ROW_INT 5 /* Row-store internal page */ +#define WT_PAGE_ROW_LEAF 6 /* Row-store leaf page */ +#define WT_PAGE_FREELIST 7 /* Free-list page */ + uint8_t type; /* Page type */ + + /* + * The flags are divided into two sets: flags set initially, before more + * than a single thread accesses the page, and the reconciliation flags. + * The alternative would be to move the WT_PAGE_REC_XXX flags into the + * WT_PAGE_MODIFY structure, but that costs more memory. Obviously, it + * is important not to add other flags that can be set at run-time, else + * the threads could race. + */ +#define WT_PAGE_BUILD_KEYS 0x001 /* Keys have been built in memory */ +#define WT_PAGE_FORCE_EVICT 0x002 /* Waiting for forced eviction */ +#define WT_PAGE_LAST_PAGE 0x004 /* Page is pinned */ +#define WT_PAGE_PINNED 0x008 /* Page is pinned */ +#define WT_PAGE_REC_EMPTY 0x010 /* Reconciliation: page empty */ +#define WT_PAGE_REC_REPLACE 0x020 /* Reconciliation: page replaced */ +#define WT_PAGE_REC_SPLIT 0x040 /* Reconciliation: page split */ +#define WT_PAGE_REC_SPLIT_MERGE 0x080 /* Reconciliation: page split merge */ + uint8_t flags; /* Page flags */ +}; + +#define WT_PAGE_REC_MASK \ + (WT_PAGE_REC_EMPTY | \ + WT_PAGE_REC_REPLACE | WT_PAGE_REC_SPLIT | WT_PAGE_REC_SPLIT_MERGE) + +/* + * WT_PAGE_DISK_OFFSET, WT_PAGE_REF_OFFSET -- + * Return the offset/pointer of a pointer/offset in a page disk image. + */ +#define WT_PAGE_DISK_OFFSET(page, p) \ + WT_PTRDIFF32(p, (page)->dsk) +#define WT_PAGE_REF_OFFSET(page, o) \ + ((void *)((uint8_t *)((page)->dsk) + (o))) + +/* + * WT_REF -- + * A single in-memory page and the state information used to determine if + * it's OK to dereference the pointer to the page. + */ +struct __wt_ref { + WT_PAGE *page; /* In-memory page */ + + void *addr; /* On-page cell or off_page WT_ADDR */ + + union { + uint64_t recno; /* Column-store: starting recno */ + void *key; /* Row-store: on-page cell or WT_IKEY */ + } u; + + /* + * Page state. + * + * WT_REF_DISK has a value of 0, the default state after allocating + * cleared memory. + * + * Synchronization is based on the WT_REF->state field, which has 5 + * possible states: + * + * WT_REF_DISK: + * The initial setting before a page is brought into memory, and + * set as a result of page eviction; the page is on disk, and must be + * read into into memory before use. + * + * WT_REF_EVICTING: + * Set by eviction when a page is about to be locked; prevents a + * page from being evicted multiple times concurrently. + * + * WT_REF_LOCKED: + * Set by eviction; an eviction thread has selected this page or + * a parent for eviction; once hazard references are checked, the page + * will be evicted. + * + * WT_REF_MEM: + * Set by a reading thread once the page has been read from disk; + * the page is in the cache and the page reference is OK. + * + * WT_REF_READING: + * Set by a reading thread before reading a page from disk; other + * readers of the page wait until the read completes. + * + * The life cycle of a typical page goes like this: pages are read into + * memory from disk and their state set to WT_REF_MEM. When the page is + * selected for eviction, the page state is set to WT_REF_LOCKED. In + * all cases, evicting threads reset the page's state when finished with + * the page: if eviction was successful (a clean page was discarded, and + * a dirty page was written to disk and then discarded), the page state + * is set to WT_REF_DISK; if eviction failed because the page was busy, + * page state is reset to WT_REF_MEM. + * + * Readers check the state field and if it's WT_REF_MEM, they set a + * hazard reference to the page, flush memory and re-confirm the page + * state. If the page state is unchanged, the reader has a valid + * reference and can proceed. + * + * When an evicting thread wants to discard a page from the tree, it + * sets the WT_REF_LOCKED state, flushes memory, then checks hazard + * references. If a hazard reference is found, state is reset to + * WT_REF_MEM, restoring the page to the readers. If the evicting + * thread does not find a hazard reference, the page is evicted. + */ + volatile enum { + WT_REF_DISK=0, /* Page is on disk */ + WT_REF_EVICTING, /* Page being evaluated for eviction */ + WT_REF_LOCKED, /* Page being evicted */ + WT_REF_MEM, /* Page is in cache and valid */ + WT_REF_READING /* Page being read */ + } state; +}; + +/* + * WT_REF_FOREACH -- + * Walk the subtree array of an in-memory internal page. + */ +#define WT_REF_FOREACH(page, ref, i) \ + for ((i) = (page)->entries, \ + (ref) = (page)->u.intl.t; (i) > 0; ++(ref), --(i)) + +/* + * WT_ROW -- + * Each in-memory page row-store leaf page has an array of WT_ROW structures: + * this is created from on-page data when a page is read from the file. It's + * sorted by key, fixed in size, and references data on the page. + */ +struct __wt_row { + void *key; /* On-page cell or off-page WT_IKEY */ +}; + +/* + * WT_ROW_FOREACH -- + * Walk the entries of an in-memory row-store leaf page. + */ +#define WT_ROW_FOREACH(page, rip, i) \ + for ((i) = (page)->entries, \ + (rip) = (page)->u.row.d; (i) > 0; ++(rip), --(i)) +#define WT_ROW_FOREACH_REVERSE(page, rip, i) \ + for ((i) = (page)->entries, \ + (rip) = (page)->u.row.d + ((page)->entries - 1); \ + (i) > 0; --(rip), --(i)) + +/* + * WT_ROW_SLOT -- + * Return the 0-based array offset based on a WT_ROW reference. + */ +#define WT_ROW_SLOT(page, rip) \ + ((uint32_t)(((WT_ROW *)rip) - (page)->u.row.d)) + +/* + * WT_COL -- + * Each in-memory variable-length column-store leaf page has an array of WT_COL + * structures: this is created from on-page data when a page is read from the + * file. It's fixed in size, and references data on the page. + */ +struct __wt_col { + /* + * Variable-length column-store data references are page offsets, not + * pointers (we boldly re-invent short pointers). The trade-off is 4B + * per K/V pair on a 64-bit machine vs. a single cycle for the addition + * of a base pointer. The on-page data is a WT_CELL (same as row-store + * pages). + * + * If the value is 0, it's a single, deleted record. + * + * Obscure the field name, code shouldn't use WT_COL->value, the public + * interface is WT_COL_PTR. + */ + uint32_t __value; +}; + +/* + * WT_COL_RLE -- + * In variable-length column store leaf pages, we build an array of entries + * with RLE counts greater than 1 when reading the page. We can do a binary + * search in this array, then an offset calculation to find the cell. + */ +struct __wt_col_rle { + uint64_t recno; /* Record number of first repeat. */ + uint64_t rle; /* Repeat count. */ + uint32_t indx; /* Slot of entry in col_var.d */ +} WT_GCC_ATTRIBUTE((packed)); + +/* + * WT_COL_PTR -- + * Return a pointer corresponding to the data offset -- if the item doesn't + * exist on the page, return a NULL. + */ +#define WT_COL_PTR(page, cip) \ + ((cip)->__value == 0 ? NULL : WT_PAGE_REF_OFFSET(page, (cip)->__value)) + +/* + * WT_COL_FOREACH -- + * Walk the entries of variable-length column-store leaf page. + */ +#define WT_COL_FOREACH(page, cip, i) \ + for ((i) = (page)->entries, \ + (cip) = (page)->u.col_var.d; (i) > 0; ++(cip), --(i)) + +/* + * WT_COL_SLOT -- + * Return the 0-based array offset based on a WT_COL reference. + */ +#define WT_COL_SLOT(page, cip) \ + ((uint32_t)(((WT_COL *)cip) - (page)->u.col_var.d)) + +/* + * WT_IKEY -- + * Instantiated key: row-store keys are usually prefix compressed and sometimes + * Huffman encoded or overflow objects. Normally, a row-store page in-memory + * key points to the on-page WT_CELL, but in some cases, we instantiate the key + * in memory, in which case the row-store page in-memory key points to a WT_IKEY + * structure. + */ +struct __wt_ikey { + uint32_t size; /* Key length */ + + /* + * If we no longer point to the key's on-page WT_CELL, we can't find its + * related value. Save the offset of the key cell in the page. + * + * Row-store cell references are page offsets, not pointers (we boldly + * re-invent short pointers). The trade-off is 4B per K/V pair on a + * 64-bit machine vs. a single cycle for the addition of a base pointer. + */ + uint32_t cell_offset; + + /* The key bytes immediately follow the WT_IKEY structure. */ +#define WT_IKEY_DATA(ikey) \ + ((void *)((uint8_t *)(ikey) + sizeof(WT_IKEY))) +}; + +/* + * WT_UPDATE -- + * Entries on leaf pages can be updated, either modified or deleted. Updates + * to entries referenced from the WT_ROW and WT_COL arrays are stored in the + * page's WT_UPDATE array. When the first element on a page is updated, the + * WT_UPDATE array is allocated, with one slot for every existing element in + * the page. A slot points to a WT_UPDATE structure; if more than one update + * is done for an entry, WT_UPDATE structures are formed into a forward-linked + * list. + */ +struct __wt_update { + WT_UPDATE *next; /* forward-linked list */ + + /* + * We use the maximum size as an is-deleted flag, which means we can't + * store 4GB objects; I'd rather do that than increase the size of this + * structure for a flag bit. + */ +#define WT_UPDATE_DELETED_ISSET(upd) ((upd)->size == UINT32_MAX) +#define WT_UPDATE_DELETED_SET(upd) ((upd)->size = UINT32_MAX) + uint32_t size; /* update length */ + + /* The untyped value immediately follows the WT_UPDATE structure. */ +#define WT_UPDATE_DATA(upd) \ + ((void *)((uint8_t *)(upd) + sizeof(WT_UPDATE))) +}; + +/* + * WT_INSERT -- + * + * Row-store leaf pages support inserts of new K/V pairs. When the first K/V + * pair is inserted, the WT_INSERT_HEAD array is allocated, with one slot for + * every existing element in the page, plus one additional slot. A slot points + * to a WT_INSERT_HEAD structure for the items which sort after the WT_ROW + * element that references it and before the subsequent WT_ROW element; the + * skiplist structure has a randomly chosen depth of next pointers in each + * inserted node. + * + * The additional slot is because it's possible to insert items smaller than any + * existing key on the page: for that reason, the first slot of the insert array + * holds keys smaller than any other key on the page. + * + * In column-store variable-length run-length encoded pages, a single indx + * entry may reference a large number of records, because there's a single + * on-page entry representing many identical records. (We don't expand those + * entries when the page comes into memory, as that would require resources as + * pages are moved to/from the cache, including read-only files.) Instead, a + * single indx entry represents all of the identical records originally found + * on the page. + * + * Modifying (or deleting) run-length encoded column-store records is hard + * because the page's entry no longer references a set of identical items. We + * handle this by "inserting" a new entry into the insert array, with its own + * record number. (This is the only case where it's possible to insert into a + * column-store: only appends are allowed, as insert requires re-numbering + * subsequent records. Berkeley DB did support mutable records, but it won't + * scale and it isn't useful enough to re-implement, IMNSHO.) + */ +struct __wt_insert { + WT_UPDATE *upd; /* value */ + + union { + uint64_t recno; /* column-store record number */ + struct { + uint32_t offset; /* row-store key data start */ + uint32_t size; /* row-store key data size */ + } key; + } u; + +#define WT_INSERT_KEY_SIZE(ins) ((ins)->u.key.size) +#define WT_INSERT_KEY(ins) \ + ((void *)((uint8_t *)(ins) + (ins)->u.key.offset)) +#define WT_INSERT_RECNO(ins) ((ins)->u.recno) + + WT_INSERT *next[0]; /* forward-linked skip list */ +}; + +/* + * Skiplist helper macros. + */ +#define WT_SKIP_FIRST(ins_head) \ + (((ins_head) == NULL) ? NULL : (ins_head)->head[0]) +#define WT_SKIP_LAST(ins_head) \ + (((ins_head) == NULL) ? NULL : (ins_head)->tail[0]) +#define WT_SKIP_NEXT(ins) ((ins)->next[0]) +#define WT_SKIP_FOREACH(ins, ins_head) \ + for ((ins) = WT_SKIP_FIRST(ins_head); \ + (ins) != NULL; \ + (ins) = WT_SKIP_NEXT(ins)) + +/* + * WT_INSERT_HEAD -- + * The head of a skiplist of WT_INSERT items. + */ +struct __wt_insert_head { + WT_INSERT *head[WT_SKIP_MAXDEPTH]; /* first item on skiplists */ + WT_INSERT *tail[WT_SKIP_MAXDEPTH]; /* last item on skiplists */ +}; + +/* + * The row-store leaf page insert lists are arrays of pointers to structures, + * and may not exist. The following macros return an array entry if the array + * of pointers and the specific structure exist, else NULL. + */ +#define WT_ROW_INSERT_SLOT(page, slot) \ + ((page)->u.row.ins == NULL ? NULL : (page)->u.row.ins[slot]) +#define WT_ROW_INSERT(page, ip) \ + WT_ROW_INSERT_SLOT(page, WT_ROW_SLOT(page, ip)) +#define WT_ROW_UPDATE(page, ip) \ + ((page)->u.row.upd == NULL ? \ + NULL : (page)->u.row.upd[WT_ROW_SLOT(page, ip)]) +/* + * WT_ROW_INSERT_SMALLEST references an additional slot past the end of the + * the "one per WT_ROW slot" insert array. That's because the insert array + * requires an extra slot to hold keys that sort before any key found on the + * original page. + */ +#define WT_ROW_INSERT_SMALLEST(page) \ + ((page)->u.row.ins == NULL ? NULL : (page)->u.row.ins[(page)->entries]) + +/* + * The column-store leaf page update lists are arrays of pointers to structures, + * and may not exist. The following macros return an array entry if the array + * of pointers and the specific structure exist, else NULL. + */ +#define WT_COL_UPDATE_SLOT(page, slot) \ + ((page)->modify == NULL || (page)->modify->update == NULL ? \ + NULL : (page)->modify->update[slot]) +#define WT_COL_UPDATE(page, ip) \ + WT_COL_UPDATE_SLOT(page, WT_COL_SLOT(page, ip)) + +/* + * WT_COL_UPDATE_SINGLE is a single WT_INSERT list, used for any fixed-length + * column-store updates for a page. + */ +#define WT_COL_UPDATE_SINGLE(page) \ + WT_COL_UPDATE_SLOT(page, 0) + +/* + * WT_COL_APPEND is a single WT_INSERT list, used for fixed- and variable-length + * appends. + */ +#define WT_COL_APPEND(btree, page) \ + (F_ISSET((page), WT_PAGE_LAST_PAGE) && \ + (btree)->append != NULL ? (btree)->append[0] : NULL) + +/* WT_FIX_FOREACH walks fixed-length bit-fields on a disk page. */ +#define WT_FIX_FOREACH(btree, dsk, v, i) \ + for ((i) = 0, \ + (v) = (i) < (dsk)->u.entries ? \ + __bit_getv( \ + WT_PAGE_HEADER_BYTE(btree, dsk), 0, (btree)->bitcnt) : 0; \ + (i) < (dsk)->u.entries; ++(i), \ + (v) = __bit_getv( \ + WT_PAGE_HEADER_BYTE(btree, dsk), i, (btree)->bitcnt)) diff --git a/src/include/btree.h b/src/include/btree.h new file mode 100644 index 00000000000..d3d7992627b --- /dev/null +++ b/src/include/btree.h @@ -0,0 +1,147 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#define WT_BTREE_MAJOR_VERSION 1 /* Version */ +#define WT_BTREE_MINOR_VERSION 0 + +/* + * The minimum btree leaf and internal page sizes are 512B, the maximum 512MB. + * (The maximum of 512MB is enforced by the software, it could be set as high + * as 4GB.) + */ +#define WT_BTREE_ALLOCATION_SIZE_MIN 512 +#define WT_BTREE_ALLOCATION_SIZE_MAX (128 * WT_MEGABYTE) +#define WT_BTREE_PAGE_SIZE_MAX (512 * WT_MEGABYTE) + +/* + * Variable-length value items and row-store key/value item lengths are stored + * in 32-bit unsigned integers, meaning the largest theoretical key/value item + * is 4GB. However, in the WT_UPDATE structure we use the UINT32_MAX size as a + * "deleted" flag. Limit the size of a single object to 4GB - 512B: it's a few + * additional bytes if we ever want to store a small structure length plus the + * object size in 32 bits, or if we ever need more denoted values. Storing 4GB + * objects in a Btree borders on clinical insanity, anyway. + * + * Record numbers are stored in 64-bit unsigned integers, meaning the largest + * record number is "really, really big". + */ +#define WT_BTREE_MAX_OBJECT_SIZE (UINT32_MAX - 512) + +/* + * Split page size calculation -- we don't want to repeatedly split every time + * a new entry is added, so we split to a smaller-than-maximum page size. + */ +#define WT_SPLIT_PAGE_SIZE(pagesize, allocsize, pct) \ + WT_ALIGN(((uintmax_t)(pagesize) * (pct)) / 100, allocsize) + +/* + * XXX + * The server threads use their own WT_SESSION_IMPL handles because they may + * want to block (for example, the eviction server calls reconciliation, and + * some of the reconciliation diagnostic code reads pages), and the user's + * session handle is already blocking on a server thread. The problem is the + * server thread needs to reference the correct btree handle, and that's + * hanging off the application's thread of control. For now, I'm just making + * it obvious where that's getting done. + */ +#define WT_SET_BTREE_IN_SESSION(s, b) ((s)->btree = (b)) +#define WT_CLEAR_BTREE_IN_SESSION(s) ((s)->btree = NULL) + +/* + * WT_BTREE -- + * A btree handle. + */ +struct __wt_btree { + WT_RWLOCK *rwlock; /* Lock for shared/exclusive ops */ + uint32_t refcnt; /* Sessions using this tree. */ + TAILQ_ENTRY(__wt_btree) q; /* Linked list of handles */ + + const char *name; /* Logical name */ + const char *filename; /* File name */ + const char *config; /* Configuration string */ + + enum { BTREE_COL_FIX=1, /* Fixed-length column store */ + BTREE_COL_VAR=2, /* Variable-length column store */ + BTREE_ROW=3 /* Row-store */ + } type; /* Type */ + + const char *key_format; /* Key format */ + const char *key_plan; /* Key projection plan */ + const char *idxkey_format; /* Index key format (hides primary) */ + const char *value_format; /* Value format */ + const char *value_plan; /* Value projection plan */ + uint8_t bitcnt; /* Fixed-length field size in bits */ + + /* Row-store comparison function */ + WT_COLLATOR *collator; /* Comparison function */ + + uint32_t key_gap; /* Row-store prefix key gap */ + + uint32_t allocsize; /* Allocation size */ + uint32_t maxintlpage; /* Internal page max size */ + uint32_t maxintlitem; /* Internal page max item size */ + uint32_t maxleafpage; /* Leaf page max size */ + uint32_t maxleafitem; /* Leaf page max item size */ + + void *huffman_key; /* Key huffman encoding */ + void *huffman_value; /* Value huffman encoding */ + + /* + * Column-store: track the last record in the file, and keep the last + * page pinned in memory for fast appends, to a skiplist of appended + * entries. + */ + WT_PAGE *last_page; /* Col-store append, last page */ + uint64_t last_recno; /* Col-store append, last recno */ + WT_INSERT_HEAD **append; /* Appended items */ + + WT_PAGE *root_page; /* Root page */ + WT_ADDR root_addr; /* Replacement root address */ + int root_update; /* 0: free original root blocks + 1: free saved root blocks and + update on close */ + + void *block; /* Block manager */ + u_int block_header; /* Block manager header length */ + + WT_PAGE *evict_page; /* Eviction thread's location */ + + WT_BTREE_STATS *stats; /* Btree statistics */ + +#define WT_BTREE_BULK 0x01 /* Bulk-load handle */ +#define WT_BTREE_EXCLUSIVE 0x02 /* Need exclusive access to handle */ +#define WT_BTREE_NO_LOCK 0x04 /* Do not lock the handle */ +#define WT_BTREE_OPEN 0x08 /* Handle is open */ +#define WT_BTREE_SALVAGE 0x10 /* Handle is for salvage */ +#define WT_BTREE_UPGRADE 0x20 /* Handle is for upgrade */ +#define WT_BTREE_VERIFY 0x40 /* Handle is for verify */ + uint32_t flags; +}; + +/* + * In diagnostic mode we track the locations from which hazard references + * were acquired. + */ +#ifdef HAVE_DIAGNOSTIC +#define __wt_page_in(a, b, c) \ + __wt_page_in_func(a, b, c, __FILE__, __LINE__) +#else +#define __wt_page_in(a, b, c) \ + __wt_page_in_func(a, b, c) +#endif + +/* + * WT_SALVAGE_COOKIE -- + * Encapsulation of salvage information for reconciliation. + */ +struct __wt_salvage_cookie { + uint64_t missing; /* Initial items to create */ + uint64_t skip; /* Initial items to skip */ + uint64_t take; /* Items to take */ + + int done; /* Ignore the rest */ +}; diff --git a/src/include/btree.i b/src/include/btree.i new file mode 100644 index 00000000000..a720111304a --- /dev/null +++ b/src/include/btree.i @@ -0,0 +1,261 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* + * __wt_cache_page_inmem_incr -- + * Increment a page's memory footprint in the cache. + */ +static inline void +__wt_cache_page_inmem_incr( + WT_SESSION_IMPL *session, WT_PAGE *page, size_t size) +{ + WT_CACHE *cache; + + cache = S2C(session)->cache; + + WT_ATOMIC_ADD(cache->bytes_inmem, size); + WT_ATOMIC_ADD(page->memory_footprint, WT_STORE_SIZE(size)); +} + +/* + * __wt_cache_page_read -- + * Read pages into the cache. + */ +static inline void +__wt_cache_page_read(WT_SESSION_IMPL *session, WT_PAGE *page, size_t size) +{ + WT_CACHE *cache; + + cache = S2C(session)->cache; + + WT_ASSERT(session, size != 0); + + WT_ATOMIC_ADD(cache->pages_read, 1); + WT_ATOMIC_ADD(cache->bytes_read, size); + WT_ATOMIC_ADD(page->memory_footprint, WT_STORE_SIZE(size)); +} + +/* + * __wt_cache_page_evict -- + * Evict pages from the cache. + */ +static inline void +__wt_cache_page_evict(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_CACHE *cache; + + cache = S2C(session)->cache; + + WT_ASSERT(session, page->memory_footprint != 0); + + WT_ATOMIC_ADD(cache->pages_evict, 1); + WT_ATOMIC_ADD(cache->bytes_evict, page->memory_footprint); + + page->memory_footprint = 0; +} + +static inline uint64_t +__wt_cache_read_gen(WT_SESSION_IMPL *session) +{ + return (++S2C(session)->cache->read_gen); +} + +/* + * __wt_cache_pages_inuse -- + * Return the number of pages in use. + */ +static inline uint64_t +__wt_cache_pages_inuse(WT_CACHE *cache) +{ + uint64_t pages_in, pages_out; + + /* + * Reading 64-bit fields, potentially on 32-bit machines, and other + * threads of control may be modifying them. Check them for sanity + * (although "interesting" corruption is vanishingly unlikely, these + * values just increment over time). + */ + pages_in = cache->pages_read; + pages_out = cache->pages_evict; + return (pages_in > pages_out ? pages_in - pages_out : 0); +} + +/* + * __wt_cache_bytes_inuse -- + * Return the number of bytes in use. + */ +static inline uint64_t +__wt_cache_bytes_inuse(WT_CACHE *cache) +{ + uint64_t bytes_in, bytes_out; + + /* + * Reading 64-bit fields, potentially on 32-bit machines, and other + * threads of control may be modifying them. Check them for sanity + * (although "interesting" corruption is vanishingly unlikely, these + * values just increment over time). + */ + bytes_in = cache->bytes_read + cache->bytes_inmem; + bytes_out = cache->bytes_evict; + return (bytes_in > bytes_out ? bytes_in - bytes_out : 0); +} + +/* + * __wt_page_modify_set -- + * Mark the page dirty. + */ +static inline void +__wt_page_modify_set(WT_PAGE *page) +{ + /* + * Publish: there must be a barrier to ensure all changes to the page + * are flushed before we update the page's write generation, otherwise + * a thread searching the page might see the page's write generation + * update before the changes to the page, which breaks the protocol. + */ + WT_WRITE_BARRIER(); + + /* The page is dirty if the disk and write generations differ. */ + ++page->modify->write_gen; +} + +/* + * __wt_page_is_modified -- + * Return if the page is dirty. + */ +static inline int +__wt_page_is_modified(WT_PAGE *page) +{ + return (page->modify == NULL || + page->modify->write_gen == page->modify->disk_gen ? 0 : 1); +} + +/* + * __wt_page_write_gen_check -- + * Confirm the page's write generation number is correct. + */ +static inline int +__wt_page_write_gen_check(WT_PAGE *page, uint32_t write_gen) +{ + return (page->modify->write_gen == write_gen ? 0 : WT_RESTART); +} + +/* + * __wt_off_page -- + * Return if a pointer references off-page data. + */ +static inline int +__wt_off_page(WT_PAGE *page, const void *p) +{ + /* + * There may be no underlying page, in which case the reference is + * off-page by definition. + * + * We use the page's disk size, not the page parent's reference disk + * size for a reason: the page may already be disconnected from the + * parent reference (when being discarded), or not yet be connected + * to the parent reference (when being created). + */ + return (page->dsk == NULL || + p < (void *)page->dsk || + p >= (void *)((uint8_t *)page->dsk + page->dsk->size)); +} + +/* + * __wt_get_addr -- + * Return the addr/size pair for a reference. + */ +static inline void +__wt_get_addr( + WT_PAGE *page, WT_REF *ref, const uint8_t **addrp, uint32_t *sizep) +{ + WT_CELL_UNPACK *unpack, _unpack; + + unpack = &_unpack; + + /* + * If NULL, there is no location. + * If off-page, the pointer references a WT_ADDR structure. + * If on-page, the pointer references a cell. + */ + if (ref->addr == NULL) { + *addrp = NULL; + *sizep = 0; + } else if (__wt_off_page(page, ref->addr)) { + *addrp = ((WT_ADDR *)(ref->addr))->addr; + *sizep = ((WT_ADDR *)(ref->addr))->size; + } else { + __wt_cell_unpack(ref->addr, unpack); + *addrp = unpack->data; + *sizep = unpack->size; + } +} + +/* + * __wt_page_release -- + * Release a reference to a page, unless it's pinned into memory, in which + * case we never acquired a hazard reference. + */ +static inline void +__wt_page_release(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + if (page != NULL && !F_ISSET(page, WT_PAGE_PINNED)) + __wt_hazard_clear(session, page); +} + +/* + * __wt_skip_choose_depth -- + * Randomly choose a depth for a skiplist insert. + */ +static inline u_int +__wt_skip_choose_depth(void) +{ + u_int d; + + for (d = 1; d < WT_SKIP_MAXDEPTH && + __wt_random() < WT_SKIP_PROBABILITY; d++) + ; + return (d); +} + +/* + * __wt_btree_lex_compare -- + * Lexicographic comparison routine. + * + * Returns: + * < 0 if user_item is lexicographically < tree_item + * = 0 if user_item is lexicographically = tree_item + * > 0 if user_item is lexicographically > tree_item + * + * We use the names "user" and "tree" so it's clear which the application is + * looking at when we call its comparison func. + */ +static inline int +__wt_btree_lex_compare(const WT_ITEM *user_item, const WT_ITEM *tree_item) +{ + const uint8_t *userp, *treep; + uint32_t len, usz, tsz; + + usz = user_item->size; + tsz = tree_item->size; + len = WT_MIN(usz, tsz); + + for (userp = user_item->data, treep = tree_item->data; + len > 0; + --len, ++userp, ++treep) + if (*userp != *treep) + return (*userp < *treep ? -1 : 1); + + /* Contents are equal up to the smallest length. */ + return ((usz == tsz) ? 0 : (usz < tsz) ? -1 : 1); +} + +#define WT_BTREE_CMP(s, bt, k1, k2, cmp) \ + (((bt)->collator == NULL) ? \ + (((cmp) = __wt_btree_lex_compare((k1), (k2))), 0) : \ + (bt)->collator->compare((bt)->collator, &(s)->iface, \ + (k1), (k2), &(cmp))) diff --git a/src/include/cache.h b/src/include/cache.h new file mode 100644 index 00000000000..63a545e5dc3 --- /dev/null +++ b/src/include/cache.h @@ -0,0 +1,71 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* + * WT_EVICT_LIST -- + * Encapsulation of an eviction candidate. + */ +struct __wt_evict_list { + WT_BTREE *btree; /* File object */ + WT_PAGE *page; /* Page */ +}; + +/* + * WT_EVICT_REQ -- + * Encapsulation of a eviction request. + */ +struct __wt_evict_req { + WT_SESSION_IMPL *session; /* Requesting thread */ + WT_BTREE *btree; /* Btree */ + WT_PAGE *page; /* Single page to flush */ + +#define WT_EVICT_REQ_CLOSE 0x1 /* Discard pages */ +#define WT_EVICT_REQ_PAGE 0x2 /* Force out a page */ + uint32_t flags; +}; + +/* + * WiredTiger cache structure. + */ +struct __wt_cache { + /* + * Different threads read/write pages to/from the cache and create pages + * in the cache, so we cannot know precisely how much memory is in use + * at any specific time. However, even though the values don't have to + * be exact, they can't be garbage, we track what comes in and what goes + * out and calculate the difference as needed. + */ + uint64_t bytes_read; /* Bytes/pages read by read server */ + uint64_t pages_read; + uint64_t bytes_inmem; /* Bytes/pages created in memory */ + uint64_t bytes_evict; /* Bytes/pages discarded by eviction */ + uint64_t pages_evict; + + /* + * Read information. + */ + uint32_t read_gen; /* Page read generation (LRU) */ + + /* + * Eviction thread information. + */ + WT_CONDVAR *evict_cond; /* Cache eviction server mutex */ + + WT_SPINLOCK lru_lock; /* Manage the eviction list. */ + + WT_EVICT_LIST *evict; /* Pages being tracked for eviction */ + WT_EVICT_LIST *evict_current; /* Current page to be evicted */ + size_t evict_allocated; /* Bytes allocated */ + uint32_t evict_entries; /* Total evict slots */ + + u_int eviction_trigger; /* Percent to trigger eviction. */ + u_int eviction_target; /* Percent to end eviction. */ + + WT_EVICT_REQ *evict_request; /* Eviction requests: + slot available if session is NULL */ + uint32_t max_evict_request; /* Size of the evict request array */ +}; diff --git a/src/include/cache.i b/src/include/cache.i new file mode 100644 index 00000000000..2145a85d256 --- /dev/null +++ b/src/include/cache.i @@ -0,0 +1,67 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* + * __wt_eviction_check -- + * Wake the eviction server if necessary. + */ +static inline void +__wt_eviction_check(WT_SESSION_IMPL *session, int *read_lockoutp) +{ + WT_CACHE *cache; + WT_CONNECTION_IMPL *conn; + uint64_t bytes_inuse, bytes_max; + + conn = S2C(session); + cache = conn->cache; + + /* + * If we're over the maximum cache, shut out reads (which + * include page allocations) until we evict to back under the + * maximum cache. Eviction will keep pushing out pages so we + * don't run on the edge all the time. + */ + bytes_inuse = __wt_cache_bytes_inuse(cache); + bytes_max = conn->cache_size; + if (read_lockoutp != NULL) + *read_lockoutp = (bytes_inuse > bytes_max); + + /* Wake eviction when we're over the trigger cache size. */ + if (bytes_inuse > cache->eviction_trigger * (bytes_max / 100)) + __wt_evict_server_wake(session); +} + +/* + * __wt_eviction_page_check -- + * Check if a page is too big and wake the eviction server if necessary. + */ +static inline int +__wt_eviction_page_check(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_CONNECTION_IMPL *conn; + + conn = S2C(session); + + /* + * If the page is pathologically large, force eviction. + * Otherwise, if the cache is more than 95% full, wake up the eviction + * thread. + */ + if (page != NULL && + !F_ISSET(page, WT_PAGE_FORCE_EVICT | WT_PAGE_PINNED) && + (((int64_t)page->memory_footprint > conn->cache_size / 2) || + (page->memory_footprint > 20 * session->btree->maxleafpage))) { + /* + * We're already inside a serialized function, so we need to + * take some care. + */ + WT_RET(__wt_evict_page_request(session, page)); + } else + __wt_eviction_check(session, NULL); + + return (0); +} diff --git a/src/include/cell.i b/src/include/cell.i new file mode 100644 index 00000000000..f379a06543c --- /dev/null +++ b/src/include/cell.i @@ -0,0 +1,428 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* + * WT_CELL -- + * Variable-length cell type. + * + * Pages containing variable-length data (WT_PAGE_ROW_INT, WT_PAGE_ROW_LEAF, + * and WT_PAGE_COL_VAR page types), have cells after the page header. + * + * There are 4 basic cell types: keys and data (each of which has an overflow + * form), deleted cells and off-page references. The cell is usually followed + * by additional data, varying by type: a key or data cell is followed by a set + * of bytes, an address cookie follows overflow or off-page cells. + * + * Deleted cells are place-holders for column-store files, where entries cannot + * be removed in order to preserve the record count. + * + * Here's cell use by page type: + * + * WT_PAGE_ROW_INT (row-store internal page): + * Keys and offpage-reference pairs (a WT_CELL_KEY or WT_CELL_KEY_OVFL + * cell followed by a WT_CELL_ADDR cell). + * + * WT_PAGE_ROW_LEAF (row-store leaf page): + * Keys with optional data cells (a WT_CELL_KEY or WT_CELL_KEY_OVFL cell, + * optionally followed by a WT_CELL_VALUE or WT_CELL_VALUE_OVFL cell). + * + * Both WT_PAGE_ROW_INT and WT_PAGE_ROW_LEAF pages prefix compress keys, using + * a single byte immediately following the cell. + * + * WT_PAGE_COL_INT (Column-store internal page): + * Off-page references (a WT_CELL_ADDR cell). + * + * WT_PAGE_COL_VAR (Column-store leaf page storing variable-length cells): + * Data cells (a WT_CELL_VALUE or WT_CELL_VALUE_OVFL cell), and deleted + * cells (a WT_CELL_DEL cell). + * + * Cell descriptor byte: + * + * Bits 1 and 2 are reserved for "short" key and data cells. If bit 1 (but not + * bit 2) is set, it's a short data item, less than 128 bytes in length, and the + * other 7 bits are the length. If bit 2 is set (but not bit 1), it's a short + * key, less than 64 bytes in length, and the other 6 bits are the length. The + * 0x03 bit combination (setting both 0x01 and 0x02) is unused, but will require + * code changes. + * + * Bit 3 marks variable-length column store data with an associated run-length + * counter or a record number: there is a uint64_t value immediately after the + * cell description byte. + */ +#define WT_CELL_VALUE_SHORT 0x001 /* Short data */ +#define WT_CELL_KEY_SHORT 0x002 /* Short key */ +#define WT_CELL_64V 0x004 /* RLE or recno */ +#define WT_CELL_UNUSED_BIT4 0x008 /* Unused */ +#define WT_CELL_UNUSED_BIT5 0x010 /* Unused */ + +/* + * Bits 6-8 are for other cell types (there are currently 6 cell types). + */ +#define WT_CELL_ADDR (0 << 5) /* Location reference */ +#define WT_CELL_DEL (1 << 5) /* Deleted */ +#define WT_CELL_KEY (2 << 5) /* Key */ +#define WT_CELL_KEY_OVFL (3 << 5) /* Key overflow */ +#define WT_CELL_VALUE (4 << 5) /* Value */ +#define WT_CELL_VALUE_OVFL (5 << 5) /* Value overflow */ +#define WT_CELL_UNUSED_TYPE6 (6 << 5) /* Unused */ +#define WT_CELL_TYPE_MASK (7 << 5) + +/* + * WT_CELL -- + * Variable-length, on-page cell header. + */ +struct __wt_cell { + /* + * Maximum of 16 bytes: + * 1: type + 64V flag (recno/rle) + * 1: prefix compression count + * 9: optional RLE or recno (uint64_t encoding, max 9 bytes) + * 5: optional data length (uint32_t encoding, max 5 bytes) + * + * The prefix compression count and 64V value overlap, this calculation + * is pessimistic, but it isn't worth optimizing. + */ + uint8_t __chunk[1 + 1 + WT_INTPACK64_MAXSIZE + WT_INTPACK32_MAXSIZE]; +}; + +/* + * WT_CELL_UNPACK -- + * Unpacked cell. + */ +struct __wt_cell_unpack { + uint64_t v; /* RLE count or recno */ + + const void *data; /* Data */ + uint32_t size; /* Data size */ + + uint32_t len; /* Cell + data total length */ + + uint8_t raw; /* Raw cell type (include "shorts") */ + uint8_t type; /* Cell type */ + uint8_t prefix; /* Cell prefix */ + uint8_t ovfl; /* Cell is an overflow */ +}; + +/* + * WT_CELL_FOREACH -- + * Walk the cells on a page. + */ +#define WT_CELL_FOREACH(btree, dsk, cell, unpack, i) \ + for ((cell) = \ + WT_PAGE_HEADER_BYTE(btree, dsk), (i) = (dsk)->u.entries; \ + (i) > 0; \ + (cell) = (WT_CELL *)((uint8_t *)(cell) + (unpack)->len), --(i)) + +/* + * __wt_cell_pack_addr -- + * Pack an address cell. + */ +static inline uint32_t +__wt_cell_pack_addr(WT_CELL *cell, uint64_t recno, uint32_t size) +{ + uint8_t *p; + + p = cell->__chunk + 1; + if (recno == 0) /* Type + recno */ + cell->__chunk[0] = WT_CELL_ADDR; + else { + cell->__chunk[0] = WT_CELL_ADDR | WT_CELL_64V; + (void)__wt_vpack_uint(&p, 0, recno); + } + /* Length */ + (void)__wt_vpack_uint(&p, 0, (uint64_t)size); + return (WT_PTRDIFF32(p, cell)); +} + +/* + * __wt_cell_pack_data -- + * Set a data item's WT_CELL contents. + */ +static inline uint32_t +__wt_cell_pack_data(WT_CELL *cell, uint64_t rle, uint32_t size) +{ + uint8_t byte, *p; + + /* + * Short data cells have 7-bits of length in the descriptor byte and no + * length bytes. + * + * Bit 0 is the WT_CELL_VALUE_SHORT flag; the other 7 bits are the size. + */ + if (rle == 0 && size <= 0x7f) { + byte = (uint8_t)size; + cell->__chunk[0] = (byte << 1) | WT_CELL_VALUE_SHORT; + return (1); + } + + p = cell->__chunk + 1; + if (rle < 2) /* Type + RLE */ + cell->__chunk[0] = WT_CELL_VALUE; + else { + cell->__chunk[0] = WT_CELL_VALUE | WT_CELL_64V; + (void)__wt_vpack_uint(&p, 0, rle); + } + /* Length */ + (void)__wt_vpack_uint(&p, 0, (uint64_t)size); + + return (WT_PTRDIFF32(p, cell)); +} + +/* + * __wt_cell_pack_del -- + * Write a deleted value cell. + */ +static inline uint32_t +__wt_cell_pack_del(WT_CELL *cell, uint64_t rle) +{ + uint8_t *p; + + p = cell->__chunk + 1; + if (rle < 2) { /* Type + RLE */ + cell->__chunk[0] = WT_CELL_DEL; + return (1); + } + + cell->__chunk[0] = WT_CELL_DEL | WT_CELL_64V; + (void)__wt_vpack_uint(&p, 0, rle); + + return (WT_PTRDIFF32(p, cell)); +} + +/* + * __wt_cell_pack_key -- + * Set a key's WT_CELL contents. + */ +static inline uint32_t +__wt_cell_pack_key(WT_CELL *cell, uint8_t prefix, uint32_t size) +{ + uint8_t byte, *p; + + /* + * Short keys have 6-bits of length in the descriptor byte and no length + * bytes. + * + * Bit 0 is 0, bit 1 is the WT_CELL_KEY_SHORT flag; the other 6 bits are + * the size. + */ + if (size <= 0x3f) { + byte = (uint8_t)size; + cell->__chunk[0] = (byte << 2) | WT_CELL_KEY_SHORT; + cell->__chunk[1] = prefix; + return (2); + } + + cell->__chunk[0] = WT_CELL_KEY; /* Type */ + cell->__chunk[1] = prefix; /* Prefix */ + + p = cell->__chunk + 2; /* Length */ + (void)__wt_vpack_uint(&p, 0, (uint64_t)size); + + return (WT_PTRDIFF32(p, cell)); +} + +/* + * __wt_cell_pack_key_empty -- + * Write an empty key cell. + */ +static inline void +__wt_cell_pack_key_empty(WT_CELL *cell) +{ + /* + * At the end of a row-store leaf page we have to write an empty key to + * act as a marker in case the last value on the page is zero-length. + * See the caller of this function for details. + */ + cell->__chunk[0] = WT_CELL_KEY; +} + +/* + * __wt_cell_pack_ovfl -- + * Pack an overflow cell. + */ +static inline uint32_t +__wt_cell_pack_ovfl(WT_CELL *cell, uint8_t type, uint64_t rle, uint32_t size) +{ + uint8_t *p; + + p = cell->__chunk + 1; + if (rle < 2) /* Type + RLE */ + cell->__chunk[0] = type; + else { + cell->__chunk[0] = type | WT_CELL_64V; + (void)__wt_vpack_uint(&p, 0, rle); + } + /* Length */ + (void)__wt_vpack_uint(&p, 0, (uint64_t)size); + + return (WT_PTRDIFF32(p, cell)); +} + +/* + * __wt_cell_rle -- + * Return the cell's RLE value. + */ +static inline uint64_t +__wt_cell_rle(WT_CELL_UNPACK *unpack) +{ + /* + * Any item with only 1 occurrence is stored with an RLE of 0, that is, + * without any RLE at all. This code is a single place to handle that + * correction, for simplicity. + */ + return (unpack->v < 2 ? 1 : unpack->v); +} + +/* + * __wt_cell_type -- + * Return the cell's type (collapsing "short" types). + */ +static inline u_int +__wt_cell_type(WT_CELL *cell) +{ + /* + * NOTE: WT_CELL_VALUE_SHORT MUST BE CHECKED BEFORE WT_CELL_KEY_SHORT. + */ + if (cell->__chunk[0] & WT_CELL_VALUE_SHORT) + return (WT_CELL_VALUE); + if (cell->__chunk[0] & WT_CELL_KEY_SHORT) + return (WT_CELL_KEY); + return (cell->__chunk[0] & WT_CELL_TYPE_MASK); +} + +/* + * __wt_cell_unpack_safe -- + * Unpack a WT_CELL into a structure during verification. + */ +static inline int +__wt_cell_unpack_safe(WT_CELL *cell, WT_CELL_UNPACK *unpack, uint8_t *end) +{ + uint64_t v; + const uint8_t *p; + + /* + * The verification code specifies an end argument, a pointer to 1 past + * the end-of-page. In that case, make sure we don't go past the end + * of the page when reading. If an error occurs, we simply return the + * error code, the verification code takes care of complaining (and, in + * the case of salvage, it won't complain at all, it's OK to fail). + */ +#undef CHK +#define CHK(p, len) do { \ + if (end != NULL && (((uint8_t *)p) + (len)) > end) \ + return (WT_ERROR); \ +} while (0) + + memset(unpack, 0, sizeof(*unpack)); + + /* + * Check the cell description byte, then get the cell type. + * + * NOTE: WT_CELL_VALUE_SHORT MUST BE CHECKED BEFORE WT_CELL_KEY_SHORT. + */ + CHK(cell, 0); + if (cell->__chunk[0] & WT_CELL_VALUE_SHORT) { + unpack->type = WT_CELL_VALUE; + unpack->raw = WT_CELL_VALUE_SHORT; + } else if (cell->__chunk[0] & WT_CELL_KEY_SHORT) { + unpack->type = WT_CELL_KEY; + unpack->raw = WT_CELL_KEY_SHORT; + } else + unpack->type = + unpack->raw = cell->__chunk[0] & WT_CELL_TYPE_MASK; + + /* + * Handle cells with neither an RLE count or data length: short key/data + * cells have 6- or 7-bits of data length in the descriptor byte and no + * RLE count or length bytes. Off-page cells have known sizes, and no + * RLE count or length bytes. + */ + switch (unpack->raw) { + case WT_CELL_KEY_SHORT: + /* + * Check the prefix byte that follows the cell descriptor byte. + */ + CHK(cell, 1); + unpack->prefix = cell->__chunk[1]; + + unpack->data = cell->__chunk + 2; + unpack->size = cell->__chunk[0] >> 2; + unpack->len = 2 + unpack->size; + goto done; + case WT_CELL_VALUE_SHORT: + /* + * Not reading any more memory, no further checks until the + * final check of the complete cell and its associated data. + */ + unpack->data = cell->__chunk + 1; + unpack->size = cell->__chunk[0] >> 1; + unpack->len = 1 + unpack->size; + goto done; + } + + /* + * The rest of the cell types optionally have an RLE count and/or a + * data length. + */ + p = (uint8_t *)cell + 1; /* skip cell */ + + if (unpack->raw == WT_CELL_KEY) { + /* + * Check the prefix byte that follows the cell descriptor byte. + */ + ++p; /* skip prefix */ + CHK(p, 0); + unpack->prefix = cell->__chunk[1]; + } + + if (cell->__chunk[0] & WT_CELL_64V) /* skip RLE/recno */ + WT_RET(__wt_vunpack_uint( + &p, end == NULL ? 0 : (size_t)(end - p), &unpack->v)); + + /* + * Deleted cells have known sizes, and no length bytes. + * Key/data and overflow cells have data length bytes. + */ + switch (unpack->raw) { + case WT_CELL_DEL: + unpack->len = WT_PTRDIFF32(p, cell); + break; + case WT_CELL_KEY_OVFL: + case WT_CELL_VALUE_OVFL: + unpack->ovfl = 1; + /* FALLTHROUGH */ + case WT_CELL_ADDR: + case WT_CELL_KEY: + case WT_CELL_VALUE: + WT_RET(__wt_vunpack_uint( + &p, end == NULL ? 0 : (size_t)(end - p), &v)); + unpack->data = p; + unpack->size = WT_STORE_SIZE(v); + unpack->len = WT_PTRDIFF32(p + unpack->size, cell); + break; + default: + return (WT_ERROR); /* Unknown cell type. */ + } + + /* + * Check the original cell against the full cell length (this is a + * diagnostic as well, we may be copying the cell from the page and + * we need the right length. + */ +done: CHK(cell, unpack->len); + return (0); +} + +/* + * __wt_cell_unpack -- + * Unpack a WT_CELL into a structure. + */ +static inline void +__wt_cell_unpack(WT_CELL *cell, WT_CELL_UNPACK *unpack) +{ + (void)__wt_cell_unpack_safe(cell, unpack, NULL); +} diff --git a/src/include/column.i b/src/include/column.i new file mode 100644 index 00000000000..83b2783dbf0 --- /dev/null +++ b/src/include/column.i @@ -0,0 +1,184 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* + * __col_insert_search_match -- + * Search an column-store insert list for an exact match. + */ +static inline WT_INSERT * +__col_insert_search_match(WT_INSERT_HEAD *inshead, uint64_t recno) +{ + WT_INSERT **insp, *ret_ins; + uint64_t ins_recno; + int cmp, i; + + /* If there's no insert chain to search, we're done. */ + if ((ret_ins = WT_SKIP_LAST(inshead)) == NULL) + return (NULL); + + /* Fast path the check for values at the end of the skiplist. */ + if (recno > WT_INSERT_RECNO(ret_ins)) + return (NULL); + else if (recno == WT_INSERT_RECNO(ret_ins)) + return (ret_ins); + + /* + * The insert list is a skip list: start at the highest skip level, then + * go as far as possible at each level before stepping down to the next. + */ + for (i = WT_SKIP_MAXDEPTH - 1, insp = &inshead->head[i]; i >= 0; ) { + if (*insp == NULL) { + --i; + --insp; + continue; + } + + ins_recno = WT_INSERT_RECNO(*insp); + cmp = (recno == ins_recno) ? 0 : (recno < ins_recno) ? -1 : 1; + + if (cmp == 0) /* Exact match: return */ + return (*insp); + else if (cmp > 0) /* Keep going at this level */ + insp = &(*insp)->next[i]; + else { /* Drop down a level */ + --i; + --insp; + } + } + + return (NULL); +} + +/* + * __col_insert_search -- + * Search a column-store insert list, creating a skiplist stack as we go. + */ +static inline WT_INSERT * +__col_insert_search( + WT_INSERT_HEAD *inshead, WT_INSERT ***ins_stack, uint64_t recno) +{ + WT_INSERT **insp, *ret_ins; + uint64_t ins_recno; + int cmp, i; + + /* If there's no insert chain to search, we're done. */ + if ((ret_ins = WT_SKIP_LAST(inshead)) == NULL) + return (NULL); + + /* Fast path appends. */ + if (recno >= WT_INSERT_RECNO(ret_ins)) { + for (i = 0; i < WT_SKIP_MAXDEPTH; i++) + ins_stack[i] = (inshead->tail[i] != NULL) ? + &inshead->tail[i]->next[i] : &inshead->head[i]; + return (ret_ins); + } + + /* + * The insert list is a skip list: start at the highest skip level, then + * go as far as possible at each level before stepping down to the next. + */ + for (i = WT_SKIP_MAXDEPTH - 1, insp = &inshead->head[i]; i >= 0; ) { + if (*insp == NULL) { + ins_stack[i--] = insp--; + continue; + } + + ret_ins = *insp; + ins_recno = WT_INSERT_RECNO(ret_ins); + cmp = (recno == ins_recno) ? 0 : (recno < ins_recno) ? -1 : 1; + + if (cmp > 0) /* Keep going at this level */ + insp = &(*insp)->next[i]; + else if (cmp == 0) /* Exact match: return */ + for (; i >= 0; i--) + ins_stack[i] = &ret_ins->next[i]; + else /* Drop down a level */ + ins_stack[i--] = insp--; + } + return (ret_ins); +} + +/* + * __col_last_recno -- + * Return the last record number for a column-store page. + */ +static inline uint64_t +__col_last_recno(WT_PAGE *page) +{ + WT_COL_RLE *repeat; + + /* + * If there's an append list (the last page), then there may be more + * records on the page. This function ignores those records, so our + * callers have to handle that explicitly, if they care. + * + * WT_PAGE_COL_FIX pages don't have a repeat array, so this works for + * fixed-length column-stores without any further check. + */ + if (page->u.col_var.nrepeats == 0) + return (page->entries == 0 ? 0 : + page->u.col_var.recno + (page->entries - 1)); + + repeat = &page->u.col_var.repeats[page->u.col_var.nrepeats - 1]; + return ( + (repeat->recno + repeat->rle) - 1 + + (page->entries - (repeat->indx + 1))); +} + +/* + * __col_var_search -- + * Search a variable-length column-store page for a record. + */ +static inline WT_COL * +__col_var_search(WT_PAGE *page, uint64_t recno) +{ + WT_COL_RLE *repeat; + uint64_t start_recno; + uint32_t base, indx, limit, start_indx; + + /* + * Find the matching slot. + * + * This is done in two stages: first, we do a binary search among any + * repeating records to find largest repeating less than the search key. + * Once there, we can do a simple offset calculation to find the correct + * slot for this record number, because we know any intervening records + * have repeat counts of 1. + */ + for (base = 0, + limit = page->u.col_var.nrepeats; limit != 0; limit >>= 1) { + indx = base + (limit >> 1); + + repeat = page->u.col_var.repeats + indx; + if (recno >= repeat->recno && + recno < repeat->recno + repeat->rle) + return (page->u.col_var.d + repeat->indx); + if (recno < repeat->recno) + continue; + base = indx + 1; + --limit; + } + + /* + * We didn't find an exact match, move forward from the largest repeat + * less than the search key. + */ + if (base == 0) { + start_indx = 0; + start_recno = page->u.col_var.recno; + } else { + repeat = page->u.col_var.repeats + (base - 1); + start_indx = repeat->indx + 1; + start_recno = repeat->recno + repeat->rle; + } + + if (recno >= start_recno + (page->entries - start_indx)) + return (NULL); + + return (page->u.col_var.d + + start_indx + (uint32_t)(recno - start_recno)); +} diff --git a/src/include/config.h b/src/include/config.h new file mode 100644 index 00000000000..c6c29f126c4 --- /dev/null +++ b/src/include/config.h @@ -0,0 +1,23 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +struct __wt_config { + WT_SESSION_IMPL *session; + const char *orig; + const char *end; + const char *cur; + + int depth, top; + int8_t *go; +}; + +struct __wt_config_item { + const char *str; + size_t len; + int64_t val; + enum { ITEM_STRING, ITEM_ID, ITEM_NUM, ITEM_STRUCT } type; +}; diff --git a/src/include/cursor.h b/src/include/cursor.h new file mode 100644 index 00000000000..d858505a99e --- /dev/null +++ b/src/include/cursor.h @@ -0,0 +1,166 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +struct __wt_cursor_btree { + WT_CURSOR iface; + + WT_BTREE *btree; /* Enclosing btree */ + + /* + * The following fields are set by the search functions as a precursor + * to page modification: we have a page, a WT_COL/WT_ROW slot on the + * page, an insert head, insert list and a skiplist stack (the stack of + * skiplist entries leading to the insert point). The search functions + * also return the relationship of the search key to the found key and + * a write-generation for the leaf page. + */ + WT_PAGE *page; /* Current page */ + uint32_t slot; /* WT_COL/WT_ROW 0-based slot */ + + WT_INSERT_HEAD *ins_head; /* Insert chain head */ + WT_INSERT *ins; /* Current insert node */ + /* Search stack */ + WT_INSERT **ins_stack[WT_SKIP_MAXDEPTH]; + + uint64_t recno; /* Record number */ + uint32_t write_gen; /* Saved leaf page's write generation */ + + /* + * The search function sets compare to: + * < 1 if the found key is less than the specified key + * 0 if the found key matches the specified key + * > 1 if the found key is larger than the specified key + */ + int compare; + + /* + * It's relatively expensive to calculate the last record on a variable- + * length column-store page because of the repeat values. Calculate it + * once per page and cache it. This value doesn't include the skiplist + * of appended entries on the last page. + */ + uint64_t last_standard_recno; + + /* + * Variable-length column-store values are run-length encoded and may + * be overflow values or Huffman encoded. To avoid repeatedly reading + * overflow values or decompressing encoded values, process it once and + * store the result in a temporary buffer. The cip_saved field is used + * to determine if we've switched columns since our last cursor call. + */ + WT_COL *cip_saved; /* Last iteration reference */ + + /* + * We don't instantiate prefix-compressed keys on pages where there's no + * Huffman encoding because we don't want to waste memory if only moving + * a cursor through the page, and it's faster to build keys while moving + * through the page than to roll-forward from a previously instantiated + * key (we don't instantiate all of the keys, just the ones at binary + * search points). We can't use the application's WT_CURSOR key field + * as a copy of the last-returned key because it may have been altered + * by the API layer, for example, dump cursors. Instead we store the + * last-returned key in a temporary buffer. The rip_saved field is used + * to determine if the key in the temporary buffer has the prefix needed + * for building the current key. + */ + WT_ROW *rip_saved; /* Last-returned key reference */ + + /* + * A temporary buffer with two uses: caching RLE values for column-store + * files, and caching the last-returned keys for row-store files. + */ + WT_ITEM tmp; + + /* + * Fixed-length column-store items are a single byte, and it's simpler + * and cheaper to allocate the space for it now than keep checking to + * see if we need to grow the buffer. + */ + uint8_t v; /* Fixed-length return value */ + +#define WT_CBT_ITERATE_APPEND 0x01 /* Col-store: iterating append list */ +#define WT_CBT_ITERATE_NEXT 0x02 /* Next iteration configuration */ +#define WT_CBT_ITERATE_PREV 0x04 /* Prev iteration configuration */ +#define WT_CBT_MAX_RECORD 0x08 /* Col-store: past end-of-table */ +#define WT_CBT_SEARCH_SMALLEST 0x10 /* Row-store: small-key insert list */ + uint8_t flags; +}; + +struct __wt_cursor_bulk { + WT_CURSOR_BTREE cbt; + + WT_PAGE *leaf; /* The leaf page */ + + /* + * Variable-length column store compares values during bulk load as + * part of RLE compression, row-store compares keys during bulk load + * to avoid corruption. + */ + WT_ITEM cmp; /* Comparison buffer */ + + /* + * Variable-length column-store RLE counter (also overloaded to mean + * the first time through the bulk-load insert routine, when set to 0). + */ + uint64_t rle; + + /* + * Fixed-length column-store current entry in memory chunk count, and + * the maximum number of records per chunk. + */ + uint32_t entry; /* Entry count */ + uint32_t nrecs; /* Max records per chunk */ +}; + +struct __wt_cursor_config { + WT_CURSOR iface; +}; + +struct __wt_cursor_index { + WT_CURSOR_BTREE cbt; + + WT_TABLE *table; + const char *key_plan, *value_plan; + WT_CURSOR **cg_cursors; +}; + +struct __wt_cursor_stat { + WT_CURSOR iface; + + WT_STATS *stats_first; /* First stats reference */ + int stats_count; /* Count of stats elements */ + + int notpositioned; /* Cursor not positioned */ + + int key; /* Current stats key */ + uint64_t v; /* Current stats value */ + WT_ITEM pv; /* Current stats value (string) */ + + void (*clear_func)(WT_STATS *); /* Function to clear stats. */ + WT_BTREE *btree; /* Pinned btree handle. */ +}; + +struct __wt_cursor_table { + WT_CURSOR iface; + + WT_TABLE *table; + const char *plan; + WT_CURSOR **cg_cursors; + WT_CURSOR **idx_cursors; +}; + +#define WT_CURSOR_RECNO(cursor) (strcmp((cursor)->key_format, "r") == 0) + +#define WT_CURSOR_NEEDKEY(cursor) do { \ + if (!F_ISSET(cursor, WT_CURSTD_KEY_SET)) \ + WT_ERR(__wt_cursor_kv_not_set(cursor, 1)); \ +} while (0) + +#define WT_CURSOR_NEEDVALUE(cursor) do { \ + if (!F_ISSET(cursor, WT_CURSTD_VALUE_SET)) \ + WT_ERR(__wt_cursor_kv_not_set(cursor, 0)); \ +} while (0) diff --git a/src/include/cursor.i b/src/include/cursor.i new file mode 100644 index 00000000000..cea5ac359f9 --- /dev/null +++ b/src/include/cursor.i @@ -0,0 +1,198 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* + * __cursor_search_clear -- + * Reset the cursor's state for a search. + */ +static inline void +__cursor_search_clear(WT_CURSOR_BTREE *cbt) +{ + /* Our caller should have released any page held by this cursor. */ + cbt->page = NULL; + cbt->slot = UINT32_MAX; /* Fail big */ + + cbt->ins_head = NULL; + cbt->ins = NULL; + cbt->ins_stack[0] = NULL; + /* We don't bother clearing the insert stack, that's more expensive. */ + + cbt->recno = 0; /* Illegal value */ + cbt->write_gen = 0; + + cbt->compare = 2; /* Illegal value */ + + cbt->cip_saved = NULL; + cbt->rip_saved = NULL; + + cbt->flags = 0; +} + +/* + * __cursor_func_init -- + * Reset the cursor's state for a new call. + */ +static inline void +__cursor_func_init(WT_CURSOR_BTREE *cbt, int page_release) +{ + WT_CURSOR *cursor; + WT_SESSION_IMPL *session; + + cursor = &cbt->iface; + session = (WT_SESSION_IMPL *)cursor->session; + + /* Optionally release any page references we're holding. */ + if (page_release && cbt->page != NULL) { + __wt_page_release(session, cbt->page); + cbt->page = NULL; + } + + /* Reset the returned key/value state. */ + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); +} + +/* + * __cursor_func_resolve -- + * Resolve the cursor's state for return. + */ +static inline void +__cursor_func_resolve(WT_CURSOR_BTREE *cbt, int ret) +{ + WT_CURSOR *cursor; + + cursor = &cbt->iface; + + /* + * On success, we're returning a key/value pair, and can iterate. + * On error, we're not returning anything, we can't iterate, and + * we should release any page references we're holding. + */ + if (ret == 0) + F_SET(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + else { + __cursor_func_init(cbt, 1); + __cursor_search_clear(cbt); + } +} + +/* + * __cursor_row_slot_return -- + * Return a WT_ROW slot's K/V pair. + */ +static inline int +__cursor_row_slot_return(WT_CURSOR_BTREE *cbt, WT_ROW *rip) +{ + WT_BTREE *btree; + WT_ITEM *kb, *vb; + WT_CELL *cell; + WT_CELL_UNPACK *unpack, _unpack; + WT_IKEY *ikey; + WT_SESSION_IMPL *session; + WT_UPDATE *upd; + void *key; + + session = (WT_SESSION_IMPL *)cbt->iface.session; + btree = session->btree; + unpack = &_unpack; + + kb = &cbt->iface.key; + vb = &cbt->iface.value; + + /* + * Return the WT_ROW slot's K/V pair. + * + * Multiple threads of control may be searching this page, which + * means the key may change underfoot, and here's where it gets + * tricky: first, copy the key. We don't need any barriers, the + * key is updated atomically, and we just need a valid copy. + */ + key = rip->key; + + /* + * Key copied. + * + * If another thread instantiated the key, we don't have any work to do. + * Figure this out using the key's value: + * + * If the key points off-page, the key has been instantiated, just use + * it. + * + * If the key points on-page, we have a copy of a WT_CELL value that can + * be processed, regardless of what any other thread is doing. + */ + if (__wt_off_page(cbt->page, key)) { + ikey = key; + kb->data = WT_IKEY_DATA(ikey); + kb->size = ikey->size; + } else { + /* + * If the key is simple and on-page, just reference it. + * Else, if the key is simple, prefix-compressed and on-page, + * and we have the previous expanded key in the cursor buffer, + * build the key quickly. + * Else, instantiate the key and do it all the hard way. + */ + if (btree->huffman_key != NULL) + goto slow; + __wt_cell_unpack(key, unpack); + if (unpack->type == WT_CELL_KEY && unpack->prefix == 0) { + kb->data = cbt->tmp.data = unpack->data; + kb->size = cbt->tmp.size = unpack->size; + cbt->rip_saved = rip; + } else if (unpack->type == WT_CELL_KEY && + cbt->rip_saved != NULL && cbt->rip_saved == rip - 1) { + /* + * If we previously built a prefix-compressed key in the + * temporary buffer, the WT_ITEM->data field will be the + * same as the WT_ITEM->mem field: grow the buffer if + * necessary and copy the suffix into place. If we + * previously pointed the temporary buffer at an on-page + * key, the WT_ITEM->data field will not be the same as + * the WT_ITEM->mem field: grow the buffer if necessary, + * copy the prefix into place, and then re-point the + * WT_ITEM->data field to the newly constructed memory. + */ + WT_RET(__wt_buf_grow( + session, &cbt->tmp, unpack->prefix + unpack->size)); + if (cbt->tmp.data != cbt->tmp.mem) { + memcpy((uint8_t *)cbt->tmp.mem, + cbt->tmp.data, unpack->prefix); + cbt->tmp.data = cbt->tmp.mem; + } + memcpy((uint8_t *)cbt->tmp.data + + unpack->prefix, unpack->data, unpack->size); + cbt->tmp.size = unpack->prefix + unpack->size; + kb->data = cbt->tmp.data; + kb->size = cbt->tmp.size; + cbt->rip_saved = rip; + } else +slow: WT_RET(__wt_row_key(session, cbt->page, rip, kb)); + } + + /* + * If the item was ever modified, use the WT_UPDATE data. + * Else, check for empty data. + * Else, use the value from the original disk image. + */ + if ((upd = WT_ROW_UPDATE(cbt->page, rip)) != NULL) { + vb->data = WT_UPDATE_DATA(upd); + vb->size = upd->size; + } else if ((cell = __wt_row_value(cbt->page, rip)) == NULL) { + vb->data = ""; + vb->size = 0; + } else { + __wt_cell_unpack(cell, unpack); + if (unpack->type == WT_CELL_VALUE && + btree->huffman_value == NULL) { + vb->data = unpack->data; + vb->size = unpack->size; + } else + WT_RET(__wt_cell_unpack_copy(session, unpack, vb)); + } + + return (0); +} diff --git a/src/include/dlh.h b/src/include/dlh.h new file mode 100644 index 00000000000..6c3aba944b2 --- /dev/null +++ b/src/include/dlh.h @@ -0,0 +1,13 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +struct __wt_dlh { + TAILQ_ENTRY(__wt_dlh) q; /* List of open libraries. */ + + void *handle; /* Handle returned by dlopen. */ + char *name; +}; diff --git a/src/include/error.h b/src/include/error.h new file mode 100644 index 00000000000..042dac73f66 --- /dev/null +++ b/src/include/error.h @@ -0,0 +1,73 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#define WT_DEBUG_POINT ((void *)0xdeadbeef) + +/* Return and branch-to-err-label cases for switch statements. */ +#define WT_ILLEGAL_VALUE(session) \ + default: \ + return (__wt_illegal_value(session)) +#define WT_ILLEGAL_VALUE_ERR(session) \ + default: \ + ret = __wt_illegal_value(session); \ + goto err + +/* Set "ret" and branch-to-err-label tests. */ +#define WT_ERR(a) do { \ + if ((ret = (a)) != 0) \ + goto err; \ +} while (0) +#define WT_ERR_MSG(session, v, ...) do { \ + ret = (v); \ + __wt_err(session, ret, __VA_ARGS__); \ + goto err; \ +} while (0) +#define WT_ERR_TEST(a, v) do { \ + if (a) { \ + ret = (v); \ + goto err; \ + } \ +} while (0) + +/* Return tests. */ +#define WT_RET(a) do { \ + int __ret; \ + if ((__ret = (a)) != 0) \ + return (__ret); \ +} while (0) +#define WT_RET_TEST(a, v) do { \ + if (a) \ + return (v); \ +} while (0) +#define WT_RET_MSG(session, v, ...) do { \ + int __ret = (v); \ + __wt_err(session, __ret, __VA_ARGS__); \ + return (__ret); \ +} while (0) + +/* Set "ret" if not already set. */ +#define WT_TRET(a) do { \ + int __ret; \ + if ((__ret = (a)) != 0 && ret == 0) \ + ret = __ret; \ +} while (0) + +/* + * WT_ASSERT, WT_ASSERT_RET -- + * Assert an expression, abort in diagnostic mode, otherwise, optionally + * return an error. + */ +#define WT_ASSERT(session, exp) do { \ + if (!(exp)) \ + (void)__wt_assert( \ + session, 0, __FILE__, __LINE__, "%s", #exp); \ +} while (0) +#define WT_ASSERT_RET(session, exp) do { \ + if (!(exp)) \ + return (__wt_assert( \ + session, WT_ERROR, __FILE__, __LINE__, "%s", #exp));\ +} while (0) diff --git a/src/include/extern.h b/src/include/extern.h new file mode 100644 index 00000000000..607dbf95889 --- /dev/null +++ b/src/include/extern.h @@ -0,0 +1,874 @@ +/* DO NOT EDIT: automatically built by dist/s_prototypes. */ + +extern int __wt_block_buffer_to_addr(WT_BLOCK *block, + const uint8_t *p, + off_t *offsetp, + uint32_t *sizep, + uint32_t *cksump); +extern int __wt_block_addr_to_buffer(WT_BLOCK *block, + uint8_t **p, + off_t offset, + uint32_t size, + uint32_t cksum); +extern int __wt_block_addr_valid(WT_SESSION_IMPL *session, + WT_BLOCK *block, + const uint8_t *addr, + uint32_t addr_size); +extern int __wt_block_addr_string(WT_SESSION_IMPL *session, + WT_BLOCK *block, + WT_ITEM *buf, + const uint8_t *addr, + uint32_t addr_size); +extern int __wt_block_alloc( WT_SESSION_IMPL *session, + WT_BLOCK *block, + off_t *offsetp, + off_t size); +extern int __wt_block_free_buf(WT_SESSION_IMPL *session, + WT_BLOCK *block, + const uint8_t *addr, + uint32_t addr_size); +extern int __wt_block_free( WT_SESSION_IMPL *session, + WT_BLOCK *block, + off_t off, + off_t size); +extern void __wt_block_freelist_open(WT_SESSION_IMPL *session, WT_BLOCK *block); +extern int __wt_block_freelist_read(WT_SESSION_IMPL *session, WT_BLOCK *block); +extern void __wt_block_freelist_close(WT_SESSION_IMPL *session, + WT_BLOCK *block); +extern int __wt_block_freelist_write(WT_SESSION_IMPL *session, WT_BLOCK *block); +extern void __wt_block_discard(WT_SESSION_IMPL *session, WT_BLOCK *block); +extern void __wt_block_stat(WT_SESSION_IMPL *session, WT_BLOCK *block); +extern void __wt_block_dump(WT_SESSION_IMPL *session, WT_BLOCK *block); +extern uint32_t __wt_cksum(const void *chunk, size_t len); +extern int __wt_bm_addr_valid( WT_SESSION_IMPL *session, + const uint8_t *addr, + uint32_t addr_size); +extern int __wt_bm_addr_stderr( WT_SESSION_IMPL *session, + const uint8_t *addr, + uint32_t addr_size); +extern int __wt_bm_addr_string(WT_SESSION_IMPL *session, + WT_ITEM *buf, + const uint8_t *addr, + uint32_t addr_size); +extern int __wt_bm_create(WT_SESSION_IMPL *session, const char *filename); +extern int __wt_bm_open(WT_SESSION_IMPL *session, + const char *filename, + const char *config, + const char *cfg[], + int salvage); +extern int __wt_bm_close(WT_SESSION_IMPL *session); +extern int __wt_bm_truncate(WT_SESSION_IMPL *session, const char *filename); +extern int __wt_bm_free(WT_SESSION_IMPL *session, + const uint8_t *addr, + uint32_t addr_size); +extern int __wt_bm_read(WT_SESSION_IMPL *session, + WT_ITEM *buf, + const uint8_t *addr, + uint32_t addr_size); +extern int __wt_bm_block_header(WT_SESSION_IMPL *session, uint32_t *headerp); +extern int __wt_bm_write_size(WT_SESSION_IMPL *session, uint32_t *sizep); +extern int __wt_bm_write( WT_SESSION_IMPL *session, + WT_ITEM *buf, + uint8_t *addr, + uint32_t *addr_size); +extern int __wt_bm_stat(WT_SESSION_IMPL *session); +extern int __wt_bm_salvage_start(WT_SESSION_IMPL *session); +extern int __wt_bm_salvage_next(WT_SESSION_IMPL *session, + WT_ITEM *buf, + uint8_t *addr, + uint32_t *addr_sizep, + uint64_t *write_genp, + int *eofp); +extern int __wt_bm_salvage_end(WT_SESSION_IMPL *session, int success); +extern int __wt_bm_verify_start(WT_SESSION_IMPL *session, int *emptyp); +extern int __wt_bm_verify_end(WT_SESSION_IMPL *session); +extern int __wt_bm_verify_addr(WT_SESSION_IMPL *session, + const uint8_t *addr, + uint32_t addr_size); +extern int __wt_block_truncate(WT_SESSION_IMPL *session, const char *filename); +extern int __wt_block_create(WT_SESSION_IMPL *session, const char *filename); +extern int __wt_block_open(WT_SESSION_IMPL *session, + const char *filename, + const char *config, + const char *cfg[], + int salvage, + void *retp); +extern int __wt_block_close(WT_SESSION_IMPL *session, WT_BLOCK *block); +extern int __wt_desc_init(WT_SESSION_IMPL *session, WT_FH *fh); +extern int __wt_block_read_buf(WT_SESSION_IMPL *session, + WT_BLOCK *block, + WT_ITEM *buf, + const uint8_t *addr, + uint32_t addr_size); +extern int __wt_block_read(WT_SESSION_IMPL *session, + WT_BLOCK *block, + WT_ITEM *buf, + off_t offset, + uint32_t size, + uint32_t cksum); +extern int __wt_block_salvage_start(WT_SESSION_IMPL *session, WT_BLOCK *block); +extern int __wt_block_salvage_end(WT_SESSION_IMPL *session, + WT_BLOCK *block, + int success); +extern int __wt_block_salvage_next( WT_SESSION_IMPL *session, + WT_BLOCK *block, + WT_ITEM *buf, + uint8_t *addr, + uint32_t *addr_sizep, + uint64_t *write_genp, + int *eofp); +extern int __wt_block_verify_start(WT_SESSION_IMPL *session, + WT_BLOCK *block, + int *emptyp); +extern int __wt_block_verify_end(WT_SESSION_IMPL *session, WT_BLOCK *block); +extern int __wt_block_verify_addr(WT_SESSION_IMPL *session, + WT_BLOCK *block, + const uint8_t *addr, + uint32_t addr_size); +extern int __wt_block_header(WT_SESSION_IMPL *session, + WT_BLOCK *block, + uint32_t *headerp); +extern int __wt_block_write_size( WT_SESSION_IMPL *session, + WT_BLOCK *block, + uint32_t *sizep); +extern int __wt_block_write_buf(WT_SESSION_IMPL *session, + WT_BLOCK *block, + WT_ITEM *buf, + uint8_t *addr, + uint32_t *addr_size); +extern int __wt_block_write(WT_SESSION_IMPL *session, + WT_BLOCK *block, + WT_ITEM *buf, + off_t *offsetp, + uint32_t *sizep, + uint32_t *cksump); +extern int __wt_bulk_init(WT_CURSOR_BULK *cbulk); +extern int __wt_bulk_insert(WT_CURSOR_BULK *cbulk); +extern int __wt_bulk_end(WT_CURSOR_BULK *cbulk); +extern int __wt_cache_create(WT_CONNECTION_IMPL *conn, const char *cfg[]); +extern void __wt_cache_stats_update(WT_CONNECTION_IMPL *conn); +extern void __wt_cache_destroy(WT_CONNECTION_IMPL *conn); +extern int __wt_cell_copy(WT_SESSION_IMPL *session, + WT_CELL *cell, + WT_ITEM *retb); +extern int __wt_cell_unpack_copy( WT_SESSION_IMPL *session, + WT_CELL_UNPACK *unpack, + WT_ITEM *retb); +extern void __wt_btcur_iterate_setup(WT_CURSOR_BTREE *cbt, int next); +extern int __wt_btcur_next(WT_CURSOR_BTREE *cbt); +extern int __wt_btcur_prev(WT_CURSOR_BTREE *cbt); +extern int __wt_btcur_reset(WT_CURSOR_BTREE *cbt); +extern int __wt_btcur_search(WT_CURSOR_BTREE *cbt); +extern int __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exact); +extern int __wt_btcur_insert(WT_CURSOR_BTREE *cbt); +extern int __wt_btcur_remove(WT_CURSOR_BTREE *cbt); +extern int __wt_btcur_update(WT_CURSOR_BTREE *cbt); +extern int __wt_btcur_close(WT_CURSOR_BTREE *cbt); +extern int __wt_debug_addr( WT_SESSION_IMPL *session, + uint32_t addr, + uint32_t size, + const char *ofile); +extern int __wt_debug_disk( WT_SESSION_IMPL *session, + WT_PAGE_HEADER *dsk, + const char *ofile); +extern int __wt_debug_tree_all(WT_SESSION_IMPL *session, + WT_PAGE *page, + const char *ofile); +extern int __wt_debug_tree(WT_SESSION_IMPL *session, + WT_PAGE *page, + const char *ofile); +extern int __wt_debug_page(WT_SESSION_IMPL *session, + WT_PAGE *page, + const char *ofile); +extern void __wt_page_out(WT_SESSION_IMPL *session, + WT_PAGE *page, + uint32_t flags); +extern void __wt_evict_server_wake(WT_SESSION_IMPL *session); +extern void __wt_evict_file_serial_func(WT_SESSION_IMPL *session); +extern int __wt_evict_page_request(WT_SESSION_IMPL *session, WT_PAGE *page); +extern void *__wt_cache_evict_server(void *arg); +extern int __wt_evict_lru_page(WT_SESSION_IMPL *session, int is_app); +extern int __wt_btree_create(WT_SESSION_IMPL *session, const char *filename); +extern int __wt_btree_truncate(WT_SESSION_IMPL *session, const char *filename); +extern int __wt_btree_open(WT_SESSION_IMPL *session, + const char *name, + const char *filename, + const char *config, + const char *cfg[], + uint32_t flags); +extern int __wt_btree_reopen(WT_SESSION_IMPL *session, uint32_t flags); +extern int __wt_btree_root_init(WT_SESSION_IMPL *session, WT_ITEM *addr); +extern int __wt_btree_root_empty(WT_SESSION_IMPL *session, WT_PAGE **leafp); +extern int __wt_btree_close(WT_SESSION_IMPL *session); +extern int __wt_btree_huffman_open(WT_SESSION_IMPL *session, + const char *config); +extern void __wt_btree_huffman_close(WT_SESSION_IMPL *session); +extern const char *__wt_page_type_string(u_int type); +extern const char *__wt_cell_type_string(uint8_t type); +extern const char *__wt_page_addr_string(WT_SESSION_IMPL *session, + WT_ITEM *buf, + WT_PAGE *page); +extern const char *__wt_addr_string( WT_SESSION_IMPL *session, + WT_ITEM *buf, + const uint8_t *addr, + uint32_t size); +extern int __wt_ovfl_in( WT_SESSION_IMPL *session, + WT_ITEM *store, + const uint8_t *addr, + uint32_t len); +extern int +__wt_page_in_func( + WT_SESSION_IMPL *session, WT_PAGE *parent, WT_REF *ref +#ifdef HAVE_DIAGNOSTIC + , const char *file, int line +#endif + ); +extern int __wt_page_inmem(WT_SESSION_IMPL *session, + WT_PAGE *parent, + WT_REF *parent_ref, + WT_PAGE_HEADER *dsk, + WT_PAGE **pagep); +extern int __wt_page_modify_init(WT_SESSION_IMPL *session, WT_PAGE *page); +extern int __wt_cache_read(WT_SESSION_IMPL *session, + WT_PAGE *parent, + WT_REF *ref); +extern int __wt_kv_return(WT_SESSION_IMPL *session, + WT_CURSOR_BTREE *cbt, + int key_ret); +extern int __wt_btree_get_root(WT_SESSION_IMPL *session, WT_ITEM *addr); +extern int __wt_btree_free_root(WT_SESSION_IMPL *session); +extern int __wt_btree_set_root(WT_SESSION_IMPL *session, + uint8_t *addr, + uint32_t size); +extern int __wt_salvage(WT_SESSION_IMPL *session, const char *cfg[]); +extern int __wt_btree_stat_init(WT_SESSION_IMPL *session); +extern int __wt_btree_sync(WT_SESSION_IMPL *session, const char *cfg[]); +extern int __wt_upgrade(WT_SESSION_IMPL *session, const char *cfg[]); +extern int __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]); +extern int __wt_dumpfile(WT_SESSION_IMPL *session, const char *cfg[]); +extern int __wt_verify_dsk(WT_SESSION_IMPL *session, + const char *addr, + WT_PAGE_HEADER *dsk, + uint32_t size); +extern int __wt_tree_np(WT_SESSION_IMPL *session, + WT_PAGE **pagep, + int cacheonly, + int next); +extern int __wt_col_modify(WT_SESSION_IMPL *session, + WT_CURSOR_BTREE *cbt, + int op); +extern void __wt_col_append_serial_func(WT_SESSION_IMPL *session); +extern int __wt_col_search(WT_SESSION_IMPL *session, + WT_CURSOR_BTREE *cbt, + int is_modify); +extern int __wt_rec_evict(WT_SESSION_IMPL *session, + WT_PAGE *page, + uint32_t flags); +extern int __wt_rec_track_block(WT_SESSION_IMPL *session, + __wt_pt_type_t type, + WT_PAGE *page, + const uint8_t *addr, + uint32_t size); +extern int __wt_rec_track_ovfl(WT_SESSION_IMPL *session, + WT_PAGE *page, + uint8_t *addr, + uint32_t addr_size, + const void *data, + uint32_t data_size); +extern int __wt_rec_track_ovfl_reuse(WT_SESSION_IMPL *session, + WT_PAGE *page, + const void *data, + uint32_t size, + uint8_t **addrp, + uint32_t *sizep); +extern int __wt_rec_track_init(WT_SESSION_IMPL *session, WT_PAGE *page); +extern int __wt_rec_track_wrapup(WT_SESSION_IMPL *session, + WT_PAGE *page, + int final); +extern int __wt_rec_write( WT_SESSION_IMPL *session, + WT_PAGE *page, + WT_SALVAGE_COOKIE *salvage); +extern void __wt_rec_destroy(WT_SESSION_IMPL *session); +extern int __wt_rec_bulk_init(WT_CURSOR_BULK *cbulk); +extern int __wt_rec_bulk_wrapup(WT_CURSOR_BULK *cbulk); +extern int __wt_rec_row_bulk_insert(WT_CURSOR_BULK *cbulk); +extern int __wt_rec_col_fix_bulk_insert(WT_CURSOR_BULK *cbulk); +extern int __wt_rec_col_var_bulk_insert(WT_CURSOR_BULK *cbulk); +extern int __wt_row_leaf_keys(WT_SESSION_IMPL *session, WT_PAGE *page); +extern int __wt_row_key( WT_SESSION_IMPL *session, + WT_PAGE *page, + WT_ROW *rip_arg, + WT_ITEM *retb); +extern WT_CELL *__wt_row_value(WT_PAGE *page, WT_ROW *rip); +extern int __wt_row_ikey_alloc(WT_SESSION_IMPL *session, + uint32_t cell_offset, + const void *key, + uint32_t size, + WT_IKEY **ikeyp); +extern void __wt_row_key_serial_func(WT_SESSION_IMPL *session); +extern int __wt_row_modify(WT_SESSION_IMPL *session, + WT_CURSOR_BTREE *cbt, + int is_remove); +extern int __wt_row_insert_alloc(WT_SESSION_IMPL *session, + WT_ITEM *key, + u_int skipdepth, + WT_INSERT **insp, + size_t *ins_sizep); +extern void __wt_insert_serial_func(WT_SESSION_IMPL *session); +extern int __wt_update_alloc(WT_SESSION_IMPL *session, + WT_ITEM *value, + WT_UPDATE **updp, + size_t *sizep); +extern void __wt_update_serial_func(WT_SESSION_IMPL *session); +extern WT_INSERT *__wt_search_insert(WT_SESSION_IMPL *session, + WT_CURSOR_BTREE *cbt, + WT_INSERT_HEAD *inshead, + WT_ITEM *srch_key); +extern int __wt_row_search(WT_SESSION_IMPL *session, + WT_CURSOR_BTREE *cbt, + int is_modify); +extern int __wt_config_initn( WT_SESSION_IMPL *session, + WT_CONFIG *conf, + const char *str, + size_t len); +extern int __wt_config_init(WT_SESSION_IMPL *session, + WT_CONFIG *conf, + const char *str); +extern int __wt_config_subinit( WT_SESSION_IMPL *session, + WT_CONFIG *conf, + WT_CONFIG_ITEM *item); +extern int __wt_config_next(WT_CONFIG *conf, + WT_CONFIG_ITEM *key, + WT_CONFIG_ITEM *value); +extern int __wt_config_getraw( WT_CONFIG *cparser, + WT_CONFIG_ITEM *key, + WT_CONFIG_ITEM *value); +extern int __wt_config_get(WT_SESSION_IMPL *session, + const char **cfg, + WT_CONFIG_ITEM *key, + WT_CONFIG_ITEM *value); +extern int __wt_config_gets(WT_SESSION_IMPL *session, + const char **cfg, + const char *key, + WT_CONFIG_ITEM *value); +extern int __wt_config_getone(WT_SESSION_IMPL *session, + const char *cfg, + WT_CONFIG_ITEM *key, + WT_CONFIG_ITEM *value); +extern int __wt_config_getones(WT_SESSION_IMPL *session, + const char *cfg, + const char *key, + WT_CONFIG_ITEM *value); +extern int __wt_config_subgetraw(WT_SESSION_IMPL *session, + WT_CONFIG_ITEM *cfg, + WT_CONFIG_ITEM *key, + WT_CONFIG_ITEM *value); +extern int __wt_config_subgets(WT_SESSION_IMPL *session, + WT_CONFIG_ITEM *cfg, + const char *key, + WT_CONFIG_ITEM *value); +extern int __wt_config_check(WT_SESSION_IMPL *session, + const char *checks, + const char *config); +extern int __wt_config_collapse(WT_SESSION_IMPL *session, + const char **cfg, + const char **config_ret); +extern int __wt_config_concat( WT_SESSION_IMPL *session, + const char **cfg, + const char **config_ret); +extern const char *__wt_confdfl_colgroup_meta; +extern const char *__wt_confchk_colgroup_meta; +extern const char *__wt_confdfl_connection_add_collator; +extern const char *__wt_confchk_connection_add_collator; +extern const char *__wt_confdfl_connection_add_compressor; +extern const char *__wt_confchk_connection_add_compressor; +extern const char *__wt_confdfl_connection_add_cursor_type; +extern const char *__wt_confchk_connection_add_cursor_type; +extern const char *__wt_confdfl_connection_add_extractor; +extern const char *__wt_confchk_connection_add_extractor; +extern const char *__wt_confdfl_connection_close; +extern const char *__wt_confchk_connection_close; +extern const char *__wt_confdfl_connection_load_extension; +extern const char *__wt_confchk_connection_load_extension; +extern const char *__wt_confdfl_connection_open_session; +extern const char *__wt_confchk_connection_open_session; +extern const char *__wt_confdfl_cursor_close; +extern const char *__wt_confchk_cursor_close; +extern const char *__wt_confdfl_file_meta; +extern const char *__wt_confchk_file_meta; +extern const char *__wt_confdfl_index_meta; +extern const char *__wt_confchk_index_meta; +extern const char *__wt_confdfl_session_begin_transaction; +extern const char *__wt_confchk_session_begin_transaction; +extern const char *__wt_confdfl_session_checkpoint; +extern const char *__wt_confchk_session_checkpoint; +extern const char *__wt_confdfl_session_close; +extern const char *__wt_confchk_session_close; +extern const char *__wt_confdfl_session_commit_transaction; +extern const char *__wt_confchk_session_commit_transaction; +extern const char *__wt_confdfl_session_create; +extern const char *__wt_confchk_session_create; +extern const char *__wt_confdfl_session_drop; +extern const char *__wt_confchk_session_drop; +extern const char *__wt_confdfl_session_dumpfile; +extern const char *__wt_confchk_session_dumpfile; +extern const char *__wt_confdfl_session_log_printf; +extern const char *__wt_confchk_session_log_printf; +extern const char *__wt_confdfl_session_open_cursor; +extern const char *__wt_confchk_session_open_cursor; +extern const char *__wt_confdfl_session_rename; +extern const char *__wt_confchk_session_rename; +extern const char *__wt_confdfl_session_rollback_transaction; +extern const char *__wt_confchk_session_rollback_transaction; +extern const char *__wt_confdfl_session_salvage; +extern const char *__wt_confchk_session_salvage; +extern const char *__wt_confdfl_session_sync; +extern const char *__wt_confchk_session_sync; +extern const char *__wt_confdfl_session_truncate; +extern const char *__wt_confchk_session_truncate; +extern const char *__wt_confdfl_session_upgrade; +extern const char *__wt_confchk_session_upgrade; +extern const char *__wt_confdfl_session_verify; +extern const char *__wt_confchk_session_verify; +extern const char *__wt_confdfl_table_meta; +extern const char *__wt_confchk_table_meta; +extern const char *__wt_confdfl_wiredtiger_open; +extern const char *__wt_confchk_wiredtiger_open; +extern int __wt_connection_init(WT_CONNECTION_IMPL *conn); +extern void __wt_connection_destroy(WT_CONNECTION_IMPL *conn); +extern int __wt_connection_open(WT_CONNECTION_IMPL *conn, const char *cfg[]); +extern int __wt_connection_close(WT_CONNECTION_IMPL *conn); +extern void __wt_conn_stat_init(WT_SESSION_IMPL *session); +extern int __wt_curbulk_init(WT_CURSOR_BULK *cbulk); +extern int __wt_curconfig_open(WT_SESSION_IMPL *session, + const char *uri, + const char *cfg[], + WT_CURSOR **cursorp); +extern void __wt_curdump_init(WT_CURSOR *cursor); +extern int __wt_curfile_create(WT_SESSION_IMPL *session, + const char *cfg[], + WT_CURSOR **cursorp); +extern int __wt_curfile_open(WT_SESSION_IMPL *session, + const char *uri, + const char *cfg[], + WT_CURSOR **cursorp); +extern int __wt_curindex_open(WT_SESSION_IMPL *session, + const char *uri, + const char *cfg[], + WT_CURSOR **cursorp); +extern int __wt_curstat_open(WT_SESSION_IMPL *session, + const char *uri, + const char *cfg[], + WT_CURSOR **cursorp); +extern int __wt_cursor_notsup(WT_CURSOR *cursor); +extern int __wt_cursor_get_key(WT_CURSOR *cursor, ...); +extern int __wt_cursor_get_keyv(WT_CURSOR *cursor, uint32_t flags, va_list ap); +extern int __wt_cursor_get_value(WT_CURSOR *cursor, ...); +extern void __wt_cursor_set_key(WT_CURSOR *cursor, ...); +extern void __wt_cursor_set_keyv(WT_CURSOR *cursor, uint32_t flags, va_list ap); +extern void __wt_cursor_set_value(WT_CURSOR *cursor, ...); +extern int __wt_cursor_close(WT_CURSOR *cursor); +extern int __wt_cursor_init(WT_CURSOR *cursor, + const char *uri, + int is_file, + int is_public, + const char *cfg[]) WT_GCC_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_kv_not_set(WT_CURSOR *cursor, int key); +extern int __wt_curtable_get_key(WT_CURSOR *cursor, ...); +extern int __wt_curtable_get_value(WT_CURSOR *cursor, ...); +extern void __wt_curtable_set_key(WT_CURSOR *cursor, ...); +extern void __wt_curtable_set_value(WT_CURSOR *cursor, ...); +extern int __wt_curtable_open(WT_SESSION_IMPL *session, + const char *uri, + const char *cfg[], + WT_CURSOR **cursorp); +extern int __wt_log_put(WT_SESSION_IMPL *session, WT_LOGREC_DESC *recdesc, ...); +extern int __wt_log_vprintf(WT_SESSION_IMPL *session, + const char *fmt, + va_list ap); +extern int __wt_log_printf(WT_SESSION_IMPL *session, + const char *fmt, + ...) WT_GCC_ATTRIBUTE((format (printf, + 2, + 3))); +extern WT_LOGREC_DESC __wt_logdesc_debug; +extern void __wt_abort(WT_SESSION_IMPL *session); +extern int __wt_calloc(WT_SESSION_IMPL *session, + size_t number, + size_t size, + void *retp); +extern int __wt_realloc(WT_SESSION_IMPL *session, + size_t *bytes_allocated_ret, + size_t bytes_to_allocate, + void *retp); +extern int __wt_strndup(WT_SESSION_IMPL *session, + const char *str, + size_t len, + void *retp); +extern int __wt_strdup(WT_SESSION_IMPL *session, const char *str, void *retp); +extern void __wt_free_int(WT_SESSION_IMPL *session, void *p_arg); +extern int __wt_dlopen(WT_SESSION_IMPL *session, + const char *path, + WT_DLH **dlhp); +extern int __wt_dlsym( WT_SESSION_IMPL *session, + WT_DLH *dlh, + const char *name, + void *sym_ret); +extern int __wt_dlclose(WT_SESSION_IMPL *session, WT_DLH *dlh); +extern int __wt_errno(void); +extern int __wt_exist(WT_SESSION_IMPL *session, const char *name, int *existp); +extern int __wt_filesize(WT_SESSION_IMPL *session, WT_FH *fh, off_t *sizep); +extern int __wt_bytelock(WT_FH *fhp, off_t byte, int lock); +extern int __wt_fsync(WT_SESSION_IMPL *session, WT_FH *fh); +extern int __wt_ftruncate(WT_SESSION_IMPL *session, WT_FH *fh, off_t len); +extern int __wt_cond_alloc(WT_SESSION_IMPL *session, + const char *name, + int is_locked, + WT_CONDVAR **condp); +extern void __wt_cond_wait(WT_SESSION_IMPL *session, WT_CONDVAR *cond); +extern void __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond); +extern int __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR *cond); +extern int __wt_rwlock_alloc( WT_SESSION_IMPL *session, + const char *name, + WT_RWLOCK **rwlockp); +extern void __wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock); +extern int __wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock); +extern void __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock); +extern void __wt_rwunlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock); +extern int __wt_rwlock_destroy(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock); +extern int __wt_open(WT_SESSION_IMPL *session, + const char *name, + int create, + WT_FH **fhp); +extern int __wt_close(WT_SESSION_IMPL *session, WT_FH *fh); +extern int __wt_has_priv(void); +extern int __wt_remove(WT_SESSION_IMPL *session, const char *name); +extern int __wt_rename(WT_SESSION_IMPL *session, + const char *from, + const char *to); +extern int __wt_read(WT_SESSION_IMPL *session, + WT_FH *fh, + off_t offset, + uint32_t bytes, + void *buf); +extern int __wt_write(WT_SESSION_IMPL *session, + WT_FH *fh, + off_t offset, + uint32_t bytes, + const void *buf); +extern void __wt_sleep(long seconds, long micro_seconds); +extern int __wt_thread_create(pthread_t *tidret, + void *(*func)(void *), + void *arg); +extern int __wt_thread_join(pthread_t tid); +extern void __wt_yield(void); +extern int __wt_struct_check(WT_SESSION_IMPL *session, + const char *fmt, + size_t len, + int *fixedp, + uint32_t *fixed_lenp); +extern size_t __wt_struct_sizev(WT_SESSION_IMPL *session, + const char *fmt, + va_list ap); +extern size_t __wt_struct_size(WT_SESSION_IMPL *session, const char *fmt, ...); +extern int __wt_struct_packv(WT_SESSION_IMPL *session, + void *buffer, + size_t size, + const char *fmt, + va_list ap); +extern int __wt_struct_pack(WT_SESSION_IMPL *session, + void *buffer, + size_t size, + const char *fmt, + ...); +extern int __wt_struct_unpackv(WT_SESSION_IMPL *session, + const void *buffer, + size_t size, + const char *fmt, + va_list ap); +extern int __wt_struct_unpack(WT_SESSION_IMPL *session, + const void *buffer, + size_t size, + const char *fmt, + ...); +extern int __wt_create_file(WT_SESSION_IMPL *session, + const char *name, + const char *fileuri, + const char *config); +extern int __wt_schema_create( WT_SESSION_IMPL *session, + const char *name, + const char *config); +extern int __wt_schema_drop(WT_SESSION_IMPL *session, + const char *uri, + const char *cfg[]); +extern int __wt_schema_add_table( WT_SESSION_IMPL *session, WT_TABLE *table); +extern int __wt_schema_find_table(WT_SESSION_IMPL *session, + const char *name, + size_t namelen, + WT_TABLE **tablep); +extern int __wt_schema_get_table(WT_SESSION_IMPL *session, + const char *name, + size_t namelen, + WT_TABLE **tablep); +extern void __wt_schema_destroy_table(WT_SESSION_IMPL *session, + WT_TABLE *table); +extern int __wt_schema_remove_table( WT_SESSION_IMPL *session, WT_TABLE *table); +extern int __wt_schema_close_tables(WT_SESSION_IMPL *session); +extern void __wt_schema_detach_tree(WT_SESSION_IMPL *session, WT_BTREE *btree); +extern int __wt_schema_colgroup_name(WT_SESSION_IMPL *session, + WT_TABLE *table, + const char *cgname, + size_t len, + char **namebufp); +extern int __wt_schema_get_btree(WT_SESSION_IMPL *session, + const char *objname, + size_t len, + const char *cfg[], + uint32_t flags); +extern int __wt_schema_open_colgroups(WT_SESSION_IMPL *session, + WT_TABLE *table); +extern int __wt_schema_open_index( WT_SESSION_IMPL *session, + WT_TABLE *table, + const char *idxname, + size_t len); +extern int __wt_schema_open_table(WT_SESSION_IMPL *session, + const char *name, + size_t namelen, + WT_TABLE **tablep); +extern int __wt_schema_colcheck(WT_SESSION_IMPL *session, + const char *key_format, + const char *value_format, + WT_CONFIG_ITEM *colconf, + int *kcolsp, + int *vcolsp); +extern int __wt_table_check(WT_SESSION_IMPL *session, WT_TABLE *table); +extern int __wt_struct_plan(WT_SESSION_IMPL *session, + WT_TABLE *table, + const char *columns, + size_t len, + int value_only, + WT_ITEM *plan); +extern int __wt_struct_reformat(WT_SESSION_IMPL *session, + WT_TABLE *table, + const char *columns, + size_t len, + const char *extra_cols, + int value_only, + WT_ITEM *format); +extern int __wt_struct_truncate(WT_SESSION_IMPL *session, + const char *input_fmt, + u_int ncols, + WT_ITEM *format); +extern int __wt_schema_project_in(WT_SESSION_IMPL *session, + WT_CURSOR **cp, + const char *proj_arg, + va_list ap); +extern int __wt_schema_project_out(WT_SESSION_IMPL *session, + WT_CURSOR **cp, + const char *proj_arg, + va_list ap); +extern int __wt_schema_project_slice(WT_SESSION_IMPL *session, + WT_CURSOR **cp, + const char *proj_arg, + int key_only, + const char *vformat, + WT_ITEM *value); +extern int __wt_schema_project_merge(WT_SESSION_IMPL *session, + WT_CURSOR **cp, + const char *proj_arg, + const char *vformat, + WT_ITEM *value); +extern int __wt_schema_rename(WT_SESSION_IMPL *session, + const char *uri, + const char *newuri, + const char *cfg[]); +extern int __wt_schema_table_cursor( WT_SESSION_IMPL *session, + const char *config, + WT_CURSOR **cursorp); +extern int __wt_schema_table_insert( WT_SESSION_IMPL *session, + const char *key, + const char *value); +extern int __wt_schema_table_update( WT_SESSION_IMPL *session, + const char *key, + const char *value); +extern int __wt_schema_table_remove(WT_SESSION_IMPL *session, const char *key); +extern int __wt_schema_table_read( WT_SESSION_IMPL *session, + const char *key, + const char **valuep); +extern int __wt_schema_table_track_on(WT_SESSION_IMPL *session); +extern int __wt_schema_table_track_off(WT_SESSION_IMPL *session, int unroll); +extern int __wt_schema_table_track_insert(WT_SESSION_IMPL *session, + const char *key); +extern int __wt_schema_table_track_update(WT_SESSION_IMPL *session, + const char *key); +extern int __wt_schema_table_track_fileop( WT_SESSION_IMPL *session, + const char *oldname, + const char *newname); +extern int __wt_schema_truncate( WT_SESSION_IMPL *session, + const char *uri, + const char *cfg[]); +extern int __wt_schema_name_check(WT_SESSION_IMPL *session, const char *uri); +extern int __wt_schema_worker(WT_SESSION_IMPL *session, + const char *uri, + const char *cfg[], + int (*func)(WT_SESSION_IMPL *, + const char *[]), + uint32_t open_flags); +extern int __wt_open_session(WT_CONNECTION_IMPL *conn, + int internal, + WT_EVENT_HANDLER *event_handler, + const char *config, + WT_SESSION_IMPL **sessionp); +extern int __wt_session_add_btree( WT_SESSION_IMPL *session, + WT_BTREE_SESSION **btree_sessionp); +extern int __wt_session_has_btree(WT_SESSION_IMPL *session, WT_BTREE *btree); +extern int __wt_session_lock_btree( WT_SESSION_IMPL *session, + const char *cfg[], + uint32_t flags); +extern int __wt_session_release_btree(WT_SESSION_IMPL *session); +extern int __wt_session_find_btree(WT_SESSION_IMPL *session, + const char *filename, + size_t namelen, + const char *cfg[], + uint32_t flags, + WT_BTREE_SESSION **btree_sessionp); +extern int __wt_session_get_btree(WT_SESSION_IMPL *session, + const char *name, + const char *fileuri, + const char *tconfig, + const char *cfg[], + uint32_t flags); +extern int __wt_session_remove_btree( WT_SESSION_IMPL *session, + WT_BTREE_SESSION *btree_session); +extern int __wt_session_close_any_open_btree(WT_SESSION_IMPL *session, + const char *name); +extern void __wt_eventv(WT_SESSION_IMPL *session, + int msg_event, + int error, + const char *file_name, + int line_number, + const char *fmt, + va_list ap); +extern void __wt_err(WT_SESSION_IMPL *session, + int error, + const char *fmt, + ...) WT_GCC_ATTRIBUTE((format (printf, + 3, + 4))); +extern void __wt_errx(WT_SESSION_IMPL *session, + const char *fmt, + ...) WT_GCC_ATTRIBUTE((format (printf, + 2, + 3))); +extern void __wt_msgv(WT_SESSION_IMPL *session, const char *fmt, va_list ap); +extern void __wt_verbose(WT_SESSION_IMPL *session, + const char *fmt, + ...) WT_GCC_ATTRIBUTE((format (printf, + 2, + 3))); +extern void __wt_msg(WT_SESSION_IMPL *session, + const char *fmt, + ...) WT_GCC_ATTRIBUTE((format (printf, + 2, + 3))); +extern int __wt_assert(WT_SESSION_IMPL *session, + int error, + const char *file_name, + int line_number, + const char *fmt, + ...) WT_GCC_ATTRIBUTE((format (printf, + 5, + 6))); +extern int __wt_illegal_value(WT_SESSION_IMPL *session); +extern int __wt_unknown_object_type(WT_SESSION_IMPL *session, const char *uri); +extern int __wt_filename(WT_SESSION_IMPL *session, + const char *name, + const char **path); +extern int __wt_library_init(void); +extern int __wt_breakpoint(void); +extern void __wt_attach(WT_SESSION_IMPL *session); +extern int +__wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref +#ifdef HAVE_DIAGNOSTIC + , const char *file, int line +#endif + ); +extern void __wt_hazard_clear(WT_SESSION_IMPL *session, WT_PAGE *page); +extern void __wt_hazard_empty(WT_SESSION_IMPL *session); +extern void __wt_hazard_validate(WT_SESSION_IMPL *session, WT_PAGE *page); +extern void __wt_raw_to_hex(const void *from, void *to, uint32_t *sizep); +extern void __wt_raw_to_esc_hex(const void *from, void *to, uint32_t *sizep); +extern int __wt_hex_to_raw( WT_SESSION_IMPL *session, + void *from, + void *to, + uint32_t *sizep); +extern int __wt_esc_hex_to_raw( WT_SESSION_IMPL *session, + void *from, + void *to, + uint32_t *sizep); +extern int __wt_huffman_open(WT_SESSION_IMPL *session, + void *symbol_frequency_array, + u_int symcnt, + u_int numbytes, + void *retp); +extern void __wt_huffman_close(WT_SESSION_IMPL *session, void *huffman_arg); +extern int __wt_print_huffman_code(void *huffman_arg, uint16_t symbol); +extern int __wt_huffman_encode(WT_SESSION_IMPL *session, + void *huffman_arg, + const uint8_t *from_arg, + uint32_t from_len, + WT_ITEM *to_buf); +extern int __wt_huffman_decode(WT_SESSION_IMPL *session, + void *huffman_arg, + const uint8_t *from_arg, + uint32_t from_len, + WT_ITEM *to_buf); +extern uint32_t __wt_nlpo2_round(uint32_t v); +extern uint32_t __wt_nlpo2(uint32_t v); +extern int __wt_ispo2(uint32_t v); +extern uint32_t __wt_random(void); +extern int __wt_buf_init(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size); +extern int __wt_buf_initsize(WT_SESSION_IMPL *session, + WT_ITEM *buf, + size_t size); +extern int __wt_buf_grow(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size); +extern int __wt_buf_set( WT_SESSION_IMPL *session, + WT_ITEM *buf, + const void *data, + size_t size); +extern int __wt_buf_set_printable( WT_SESSION_IMPL *session, + WT_ITEM *buf, + const void *from_arg, + size_t size); +extern void *__wt_buf_steal(WT_SESSION_IMPL *session, + WT_ITEM *buf, + uint32_t *sizep); +extern void __wt_buf_swap(WT_ITEM *a, WT_ITEM *b); +extern void __wt_buf_free(WT_SESSION_IMPL *session, WT_ITEM *buf); +extern int __wt_buf_fmt(WT_SESSION_IMPL *session, + WT_ITEM *buf, + const char *fmt, + ...) WT_GCC_ATTRIBUTE((format (printf, + 3, + 4))); +extern int __wt_buf_catfmt(WT_SESSION_IMPL *session, + WT_ITEM *buf, + const char *fmt, + ...) WT_GCC_ATTRIBUTE((format (printf, + 3, + 4))); +extern int __wt_scr_alloc(WT_SESSION_IMPL *session, + uint32_t size, + WT_ITEM **scratchp); +extern void __wt_scr_free(WT_ITEM **bufp); +extern void __wt_scr_discard(WT_SESSION_IMPL *session); +extern void *__wt_scr_alloc_ext(WT_SESSION *wt_session, size_t size); +extern void __wt_scr_free_ext(WT_SESSION *wt_session, void *p); +extern void __wt_session_dump_all(WT_SESSION_IMPL *session); +extern void __wt_session_dump(WT_SESSION_IMPL *session); +extern int __wt_stat_alloc_btree_stats(WT_SESSION_IMPL *session, + WT_BTREE_STATS **statsp); +extern void __wt_stat_clear_btree_stats(WT_STATS *stats_arg); +extern int __wt_stat_alloc_connection_stats(WT_SESSION_IMPL *session, + WT_CONNECTION_STATS **statsp); +extern void __wt_stat_clear_connection_stats(WT_STATS *stats_arg); diff --git a/src/include/intpack.i b/src/include/intpack.i new file mode 100644 index 00000000000..dc67b1f745d --- /dev/null +++ b/src/include/intpack.i @@ -0,0 +1,361 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* + * Variable-length integer encoding. + * We need up to 64 bits, signed and unsigned. Further, we want the packed + * representation to have the same lexicographic ordering as the integer + * values. This avoids the need for special-purpose comparison code. + * + * Try hard to keep small values small (up to ~2 bytes): that gives the biggest + * benefit for common cases storing small values. After that, just encode the + * length in the first byte: we could squeeze in a couple of extra bits, but + * the marginal benefit is small, and we want this code to be relatively + * easy to implement in client code or scripting APIs. + * + * First byte | Next | | + * byte | bytes| Min Value | Max Value + * ------------+------+------------------------+-------------------------------- + * [00 00xxxx] | free | N/A | N/A + * [00 01llll] | llll | -2^64 | -2^13 - 2^6 + * [00 1xxxxx] | 1 | -2^13 - 2^6 | -2^6 - 1 + * [01 xxxxxx] | 0 | -2^6 | -1 + * [10 xxxxxx] | 0 | 0 | 2^6 - 1 + * [11 0xxxxx] | 1 | 2^6 | 2^13 + 2^6 - 1 + * [11 10llll] | llll | 2^13 + 2^6 | 2^64 - 1 + * [11 11xxxx] | free | N/A | N/A + */ + +#define NEG_MULTI_MARKER (uint8_t)0x10 +#define NEG_2BYTE_MARKER (uint8_t)0x20 +#define NEG_1BYTE_MARKER (uint8_t)0x40 +#define POS_1BYTE_MARKER (uint8_t)0x80 +#define POS_2BYTE_MARKER (uint8_t)0xc0 +#define POS_MULTI_MARKER (uint8_t)0xe0 + +#define NEG_1BYTE_MIN ((-1) << 6) +#define NEG_2BYTE_MIN (((-1) << 13) + NEG_1BYTE_MIN) +#define POS_1BYTE_MAX ((1 << 6) - 1) +#define POS_2BYTE_MAX ((1 << 13) + POS_1BYTE_MAX) + +/* Extract bits <start> to <end> from a value (counting from LSB == 0). */ +#define GET_BITS(x, start, end) (((x) & ((1 << (start)) - 1)) >> (end)) + +#define WT_SIZE_CHECK(l, maxl) \ + WT_RET_TEST((maxl) != 0 && (size_t)(l) > (maxl), ENOMEM) + +/* Count the leading zero bytes. */ +#ifdef __GNUC__ +#define WT_LEADING_ZEROS(x, i) \ + (i = (x == 0) ? (int)sizeof (x) : __builtin_clzll(x) >> 3) +#else +#define WT_LEADING_ZEROS(x, i) do { \ + uint64_t __x = (x); \ + uint64_t __m = 0xff << 56; \ + for (i = 0; !(__x & __m) && i != 8; i++) \ + __m >>= 8; \ +} while (0) +#endif + +/* + * __wt_vpack_posint -- + * Packs a positive variable-length integer in the specified location. + */ +static inline int +__wt_vpack_posint(uint8_t **pp, size_t maxlen, uint64_t x) +{ + uint8_t *p; + int len, lz, shift; + + WT_LEADING_ZEROS(x, lz); + len = (int)sizeof (x) - lz; + WT_SIZE_CHECK(len + 1, maxlen); + p = *pp; + + /* There are four bits we can use in the first byte. */ + *p++ |= (len & 0xf); + + for (shift = (len - 1) << 3; len != 0; --len, shift -= 8) + *p++ = (x >> shift); + + *pp = p; + return (0); +} + +/* + * __wt_vpack_negint -- + * Packs a negative variable-length integer in the specified location. + */ +static inline int +__wt_vpack_negint(uint8_t **pp, size_t maxlen, uint64_t x) +{ + uint8_t *p; + int len, lz, shift; + + WT_LEADING_ZEROS(~x, lz); + len = (int)sizeof (x) - lz; + WT_SIZE_CHECK(len + 1, maxlen); + p = *pp; + + /* + * There are four size bits we can use in the first byte. + * For negative numbers, we store the number of leading 0xff bytes + * to maintain ordering (if this is not obvious, it may help to + * remember that -1 is the largest negative number). + */ + *p++ |= (lz & 0xf); + + for (shift = (len - 1) << 3; len != 0; shift -= 8, --len) + *p++ = (x >> shift); + + *pp = p; + return (0); +} + +/* + * __wt_vunpack_posint -- + * Reads a variable-length positive integer from the specified location. + */ +static inline int +__wt_vunpack_posint(const uint8_t **pp, size_t maxlen, uint64_t *retp) +{ + uint64_t x; + const uint8_t *p; + uint8_t len; + + /* There are four length bits in the first byte. */ + p = *pp; + len = (*p++ & 0xf); + WT_SIZE_CHECK(len + 1, maxlen); + + for (x = 0; len != 0; --len) + x = (x << 8) | *p++; + + *retp = x; + *pp = p; + return (0); +} + +/* + * __wt_vunpack_negint -- + * Reads a variable-length negative integer from the specified location. + */ +static inline int +__wt_vunpack_negint(const uint8_t **pp, size_t maxlen, uint64_t *retp) +{ + uint64_t x; + const uint8_t *p; + uint8_t len; + + /* There are four length bits in the first byte. */ + p = *pp; + len = (int)sizeof (x) - (*p++ & 0xf); + WT_SIZE_CHECK(len + 1, maxlen); + + for (x = UINT64_MAX; len != 0; --len) + x = (x << 8) | *p++; + + *retp = x; + *pp = p; + return (0); +} + +/* + * __wt_vpack_uint + * Variable-sized packing for unsigned integers + */ +static inline int +__wt_vpack_uint(uint8_t **pp, size_t maxlen, uint64_t x) +{ + uint8_t *p; + + WT_SIZE_CHECK(1, maxlen); + p = *pp; + if (x <= POS_1BYTE_MAX) + *p++ = POS_1BYTE_MARKER | GET_BITS(x, 6, 0); + else if (x <= POS_2BYTE_MAX) { + WT_SIZE_CHECK(2, maxlen); + x -= POS_1BYTE_MAX + 1; + *p++ = POS_2BYTE_MARKER | GET_BITS(x, 13, 8); + *p++ = GET_BITS(x, 8, 0); + } else { + x -= POS_2BYTE_MAX + 1; + *p = POS_MULTI_MARKER; + return (__wt_vpack_posint(pp, maxlen, x)); + } + + *pp = p; + return (0); +} + +/* + * __wt_vpack_int + * Variable-sized packing for signed integers + */ +static inline int +__wt_vpack_int(uint8_t **pp, size_t maxlen, int64_t x) +{ + uint8_t *p; + + WT_SIZE_CHECK(1, maxlen); + p = *pp; + if (x < NEG_2BYTE_MIN) { + *p = NEG_MULTI_MARKER; + return (__wt_vpack_negint(pp, maxlen, (uint64_t)x)); + } else if (x < NEG_1BYTE_MIN) { + WT_SIZE_CHECK(2, maxlen); + x -= NEG_2BYTE_MIN; + *p++ = NEG_2BYTE_MARKER | GET_BITS(x, 13, 8); + *p++ = GET_BITS(x, 8, 0); + } else if (x < 0) { + x -= NEG_1BYTE_MIN; + *p++ = NEG_1BYTE_MARKER | GET_BITS(x, 6, 0); + } else + /* For non-negative values, use the unsigned code above. */ + return (__wt_vpack_uint(pp, maxlen, (uint64_t)x)); + + *pp = p; + return (0); +} + +/* + * __wt_vunpack_uint + * Variable-sized unpacking for unsigned integers + */ +static inline int +__wt_vunpack_uint(const uint8_t **pp, size_t maxlen, uint64_t *xp) +{ + const uint8_t *p; + + WT_SIZE_CHECK(1, maxlen); + p = *pp; + switch (*p & 0xf0) { + case POS_1BYTE_MARKER: + case POS_1BYTE_MARKER | 0x10: + case POS_1BYTE_MARKER | 0x20: + case POS_1BYTE_MARKER | 0x30: + *xp = GET_BITS(*p, 6, 0); + p += 1; + break; + case POS_2BYTE_MARKER: + case POS_2BYTE_MARKER | 0x10: + WT_SIZE_CHECK(2, maxlen); + *xp = GET_BITS(*p++, 5, 0) << 8; + *xp |= *p++; + *xp += POS_1BYTE_MAX + 1; + break; + case POS_MULTI_MARKER: + WT_RET(__wt_vunpack_posint(pp, maxlen, xp)); + *xp += POS_2BYTE_MAX + 1; + return (0); + default: + return (EINVAL); + } + + *pp = p; + return (0); +} + +/* + * __wt_vunpack_int + * Variable-sized packing for signed integers + */ +static inline int +__wt_vunpack_int(const uint8_t **pp, size_t maxlen, int64_t *xp) +{ + const uint8_t *p; + + WT_SIZE_CHECK(1, maxlen); + p = *pp; + switch (*p & 0xf0) { + case NEG_MULTI_MARKER: + WT_RET(__wt_vunpack_negint(pp, maxlen, (uint64_t *)xp)); + return (0); + case NEG_2BYTE_MARKER: + case NEG_2BYTE_MARKER | 0x10: + WT_SIZE_CHECK(2, maxlen); + *xp = GET_BITS(*p++, 5, 0) << 8; + *xp |= *p++; + *xp += NEG_2BYTE_MIN; + p += 2; + break; + case NEG_1BYTE_MARKER: + case NEG_1BYTE_MARKER | 0x10: + case NEG_1BYTE_MARKER | 0x20: + case NEG_1BYTE_MARKER | 0x30: + *xp = NEG_1BYTE_MIN + GET_BITS(*p, 6, 0); + p += 1; + break; + default: + /* Identical to the unsigned case. */ + return (__wt_vunpack_uint(pp, maxlen, (uint64_t *)xp)); + } + + *pp = p; + return (0); +} + +/* + * __wt_vsize_posint -- + * Return the packed size of a positive variable-length integer. + */ +static inline size_t +__wt_vsize_posint(uint64_t x) +{ + int lz; + + WT_LEADING_ZEROS(x, lz); + return (WT_INTPACK64_MAXSIZE - lz); +} + +/* + * __wt_vsize_negint -- + * Return the packed size of a negative variable-length integer. + */ +static inline size_t +__wt_vsize_negint(uint64_t x) +{ + int lz; + + WT_LEADING_ZEROS(~x, lz); + return (size_t)(WT_INTPACK64_MAXSIZE - lz); +} + +/* + * __wt_vsize_uint + * Return the packed size of an unsigned integer. + */ +static inline size_t +__wt_vsize_uint(uint64_t x) +{ + if (x <= POS_1BYTE_MAX) + return (1); + else if (x <= POS_2BYTE_MAX) { + return (2); + } else { + x -= POS_2BYTE_MAX + 1; + return (__wt_vsize_posint(x)); + } +} + +/* + * __wt_vsize_int + * Return the packed size of a signed integer. + */ +static inline size_t +__wt_vsize_int(int64_t x) +{ + if (x < NEG_2BYTE_MIN) { + return (__wt_vsize_negint((uint64_t)x)); + } else if (x < NEG_1BYTE_MIN) { + return (2); + } else if (x < 0) { + return (1); + } else + /* For non-negative values, use the unsigned code above. */ + return (__wt_vsize_uint((uint64_t)x)); +} diff --git a/src/include/log.h b/src/include/log.h new file mode 100644 index 00000000000..d1ef702fb74 --- /dev/null +++ b/src/include/log.h @@ -0,0 +1,24 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#define WT_LOG_FILENAME "WiredTiger.log" /* Log file name */ + +/* Logging subsystem declarations. */ +typedef enum { + WT_LOGREC_INT16, + WT_LOGREC_UINT16, + WT_LOGREC_INT32, + WT_LOGREC_UINT32, + WT_LOGREC_INT64, + WT_LOGREC_UINT64, + WT_LOGREC_STRING, +} WT_LOGREC_FIELDTYPE; + +typedef struct { + const char *fmt; + const char *fields[]; +} WT_LOGREC_DESC; diff --git a/src/include/log.i b/src/include/log.i new file mode 100644 index 00000000000..9ac5e04e9de --- /dev/null +++ b/src/include/log.i @@ -0,0 +1,7 @@ +/* DO NOT EDIT: automatically built by dist/log.py. */ + +static inline int +__wt_logput_debug(WT_SESSION_IMPL *session, const char * message) +{ + return (__wt_log_put(session, &__wt_logdesc_debug, message)); +} diff --git a/src/include/misc.h b/src/include/misc.h new file mode 100644 index 00000000000..850e38a874e --- /dev/null +++ b/src/include/misc.h @@ -0,0 +1,137 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* Basic constants. */ +#define WT_BILLION (1000000000) +#define WT_MEGABYTE (1048576) +#define WT_MILLION (1000000) + +/* + * Sizes that cannot be larger than 2**32 are stored in uint32_t fields in + * common structures to save space. To minimize conversions from size_t to + * uint32_t through the code, we use the following macros. + */ +#define WT_STORE_SIZE(s) ((uint32_t)(s)) +#define WT_PTRDIFF(end, begin) \ + ((size_t)((uint8_t *)(end) - (uint8_t *)(begin))) +#define WT_PTRDIFF32(end, begin) \ + WT_STORE_SIZE(WT_PTRDIFF((end), (begin))) +#define WT_BLOCK_FITS(p, len, begin, maxlen) \ + ((uint8_t *)(p) >= (uint8_t *)(begin) && \ + ((uint8_t *)(p) + (len) <= (uint8_t *)(begin) + (maxlen))) +#define WT_PTR_IN_RANGE(p, begin, maxlen) \ + WT_BLOCK_FITS((p), 1, (begin), (maxlen)) + +/* + * Align an unsigned value of any type to a specified power-of-2, including the + * offset result of a pointer subtraction. Do the calculation using the largest + * unsigned integer type available, which results in conversion complaints; cast + * the result to a uint32_t because that's the size of a piece of data in the + * WiredTiger engine. + */ +#define WT_ALIGN(n, v) \ + ((uint32_t)((((uintmax_t)(n)) + ((v) - 1)) & ~(((uintmax_t)(v)) - 1))) + +/* Min, max. */ +#define WT_MIN(a, b) ((a) < (b) ? (a) : (b)) +#define WT_MAX(a, b) ((a) < (b) ? (b) : (a)) + +/* Elements in an array. */ +#define WT_ELEMENTS(a) (sizeof(a) / sizeof(a[0])) + +/* 10 level skip lists, 1/2 have a link to the next element. */ +#define WT_SKIP_MAXDEPTH 10 +#define WT_SKIP_PROBABILITY (UINT32_MAX >> 2) + +/* + * Quiet compiler warnings about unused parameters. + */ +#define WT_UNUSED(var) (void)(var) + +/* Add GCC-specific attributes to types and function declarations. */ +#ifdef __GNUC__ +#define WT_GCC_ATTRIBUTE(x) __attribute__(x) +#else +#define WT_GCC_ATTRIBUTE(x) +#endif + +/* + * Attribute are only permitted on function declarations, not definitions. + * This macro is a marker for function definitions that is rewritten by + * dist/s_prototypes to create extern.h. + */ +#define WT_GCC_FUNC_ATTRIBUTE(x) + +/* + * __wt_calloc_def -- + * Simple calls don't need separate sizeof arguments. + */ +#define __wt_calloc_def(a, b, c) \ + __wt_calloc(a, (size_t)(b), sizeof(**(c)), c) +/* + * Our internal free function clears the underlying address atomically so there + * is a smaller chance of racing threads seeing intermediate results while a + * structure is being free'd. (That would be a bug, of course, but I'd rather + * not drop core, just the same.) That's a non-standard "free" API, and the + * resulting bug is a mother to find -- make sure we get it right, don't make + * the caller remember to put the & operator on the pointer. + */ +#define __wt_free(a, b) __wt_free_int(a, &(b)) + +/* + * Flag set, clear and test. + * + * They come in 3 flavors: F_XXX (handles a field named "flags" in the structure + * referenced by its argument), LF_XXX (handles a local variable named "flags"), + * and FLD_XXX (handles any variable, anywhere. + * + * Flags are unsigned 32-bit values -- we cast to keep the compiler quiet (the + * hex constant might be a negative integer), and to ensure the hex constant is + * the correct size before applying the bitwise not operator. + */ +#define F_CLR(p, mask) ((p)->flags &= ~((uint32_t)(mask))) +#define F_ISSET(p, mask) ((p)->flags & ((uint32_t)(mask))) +#define F_SET(p, mask) ((p)->flags |= ((uint32_t)(mask))) + +#define LF_CLR(mask) ((flags) &= ~((uint32_t)(mask))) +#define LF_ISSET(mask) ((flags) & ((uint32_t)(mask))) +#define LF_SET(mask) ((flags) |= ((uint32_t)(mask))) + +#define FLD_CLR(field, mask) ((field) &= ~((uint32_t)(mask))) +#define FLD_ISSET(field, mask) ((field) & ((uint32_t)(mask))) +#define FLD_SET(field, mask) ((field) |= ((uint32_t)(mask))) + +/* Output a verbose message. */ +#ifdef HAVE_VERBOSE +#define WT_VERBOSE_ISSET(session, f) \ + (FLD_ISSET(S2C(session)->verbose, WT_VERB_##f)) +#define WT_VERBOSE(session, f, ...) do { \ + if (WT_VERBOSE_ISSET(session, f)) \ + __wt_verbose(session, #f ": " __VA_ARGS__); \ +} while (0) +#define WT_VERBOSE_CALL(session, f, func) do { \ + if (WT_VERBOSE_ISSET(session, f)) \ + func; \ +} while (0) +#else +#define WT_VERBOSE_ISSET(session, f) 0 +#define WT_VERBOSE(session, f, ...) +#define WT_VERBOSE_CALL(session, f, func) +#endif + +/* Clear a structure. */ +#define WT_CLEAR(s) \ + memset(&(s), 0, sizeof(s)) + +/* Check if a string matches a prefix. */ +#define WT_PREFIX_MATCH(str, pre) \ + (strncmp((str), (pre), strlen(pre)) == 0) + +/* Check if a string matches a prefix, and move past it. */ +#define WT_PREFIX_SKIP(str, pre) \ + ((strncmp((str), (pre), strlen(pre)) == 0) ? \ + ((str) += strlen(pre), 1) : 0) diff --git a/src/include/mutex.h b/src/include/mutex.h new file mode 100644 index 00000000000..539795ea404 --- /dev/null +++ b/src/include/mutex.h @@ -0,0 +1,184 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* + * Atomic writes: + * + * WiredTiger requires pointers (void *) and some variables to be read/written + * atomically, that is, in a single cycle. This is not write ordering -- to be + * clear, the requirement is that no partial value can ever be read or written. + * For example, if 8-bits of a 32-bit quantity were written, then the rest of + * the 32-bits were written, and another thread of control was able to read the + * memory location after the first 8-bits were written and before the subsequent + * 24-bits were written, WiredTiger would break. Or, if two threads of control + * attempt to write the same location simultaneously, the result must be one or + * the other of the two values, not some combination of both. + * + * To reduce memory requirements, we use a 32-bit type on 64-bit machines, which + * is OK if the compiler doesn't accumulate two adjacent 32-bit variables into a + * single 64-bit write, that is, there needs to be a single load/store of the 32 + * bits, not a load/store of 64 bits, where the 64 bits is comprised of two + * adjacent 32-bit locations. The problem is when two threads are cooperating + * (thread X finds 32-bits set to 0, writes in a new value, flushes memory; + * thread Y reads 32-bits that are non-zero, does some operation, resets the + * memory location to 0 and flushes). If thread X were to read the 32 bits + * adjacent to a different 32 bits, and write them both, the two threads could + * race. If that can happen, you must increase the size of the memory type to + * a type guaranteed to be written atomically in a single cycle, without writing + * an adjacent memory location. + * + * WiredTiger doesn't require atomic writes for any 64-bit memory locations and + * can run on machines with a 32-bit memory bus. + * + * We don't depend on writes across cache lines being atomic, and to make sure + * that never happens, we check address alignment: we know of no architectures + * with cache lines other than a multiple of 4 bytes in size, so aligned 4-byte + * accesses will always be in a single cache line. + * + * Atomic writes are often associated with memory barriers, implemented by the + * WT_READ_BARRIER and WT_WRITE_BARRIER macros. WiredTiger's requirement as + * described by the Solaris membar_enter description: + * + * No stores from after the memory barrier will reach visibility and + * no loads from after the barrier will be resolved before the lock + * acquisition reaches global visibility + * + * In other words, the WT_WRITE_BARRIER macro must ensure that memory stores by + * the processor, made before the WT_WRITE_BARRIER call, be visible to all + * processors in the system before any memory stores by the processor, made + * after the WT_WRITE_BARRIER call, are visible to any processor. The + * WT_READ_BARRIER macro ensures that all loads before the barrier are complete + * before any loads after the barrier. The compiler cannot reorder or cache + * values across a barrier. + * + * Lock and unlock operations imply both read and write barriers. In other + * words, barriers are not required for values protected by locking. + * + * Data locations may also be marked volatile, forcing the compiler to re-load + * the data on each access. This is a weaker semantic than barriers provide, + * only ensuring that the compiler will not cache values. It makes no ordering + * guarantees and may have no effect on systems with weaker cache guarantees. + * + * In summary, locking > barriers > volatile. + * + * To avoid locking shared data structures such as statistics and to permit + * atomic state changes, we rely on the WT_ATOMIC_ADD and WT_ATOMIC_CAS + * (compare and swap) operations. + */ +#if defined(_lint) +#define WT_ATOMIC_ADD(v, val) +#define WT_ATOMIC_CAS(v, oldv, newv) \ + ((v) == (oldv) || (v) == (newv) ? 1 : 0) +#define WT_FULL_BARRIER() +#define WT_READ_BARRIER() +#define WT_WRITE_BARRIER() +#define HAVE_ATOMICS 1 +#elif defined(__GNUC__) +#define WT_ATOMIC_ADD(v, val) \ + __sync_add_and_fetch(&(v), val) +#define WT_ATOMIC_CAS(v, oldv, newv) \ + __sync_bool_compare_and_swap(&(v), oldv, newv) + +#if defined(x86_64) || defined(__x86_64__) +#define WT_FULL_BARRIER() do { \ + asm volatile ("mfence" ::: "memory"); \ +} while (0) +#define WT_READ_BARRIER() do { \ + asm volatile ("lfence" ::: "memory"); \ +} while (0) +#define WT_WRITE_BARRIER() do { \ + asm volatile ("sfence" ::: "memory"); \ +} while (0) +#define HAVE_ATOMICS 1 +#elif defined(i386) || defined(__i386__) +#define WT_FULL_BARRIER() do { \ + asm volatile ("lock; addl $0, 0(%%esp)" ::: "memory"); \ +} while (0); +#define WT_READ_BARRIER() WT_FULL_BARRIER() +#define WT_WRITE_BARRIER() WT_FULL_BARRIER() +#define HAVE_ATOMICS 1 +#endif +#endif + +#ifndef HAVE_ATOMICS +#error "No write barrier implementation for this platform" +#endif + +/* + * Publish a value to a shared location. All previous stores must complete + * before the value is made public. + */ +#define WT_PUBLISH(v, val) do { \ + WT_WRITE_BARRIER(); \ + (v) = (val); \ +} while (0) + +/* + * Read a shared location and guarantee that subsequent reads do not see any + * earlier state. + */ +#define WT_ORDERED_READ(v, val) do { \ + (v) = (val); \ + WT_READ_BARRIER(); \ +} while (0) + +/* + * Condition variables: + * + * WiredTiger uses standard pthread condition variables to signal between + * threads, and for locking operations that are expected to block. + */ +struct __wt_condvar { + const char *name; /* Mutex name for debugging */ + + pthread_mutex_t mtx; /* Mutex */ + pthread_cond_t cond; /* Condition variable */ + + int locked; /* Mutex is locked */ +}; + +/* + * Read/write locks: + * + * WiredTiger uses standard pthread rwlocks to get shared and exclusive access + * to resources. + */ +struct __wt_rwlock { + const char *name; /* Lock name for debugging */ + + pthread_rwlock_t rwlock; /* Read/write lock */ +}; + +/* Compile read-write barrier */ +#define WT_BARRIER() asm volatile("" ::: "memory") + +/* Pause instruction to prevent excess processor bus usage */ +#define WT_PAUSE() asm volatile("pause\n" ::: "memory") + +/* + * Spin locks: + * + * These used for cases where fast mutual exclusion is needed (where operations + * done while holding the spin lock are expected to complete in a small number + * of instructions. + */ +#define SPINLOCK_GCC 0 +#define SPINLOCK_PTHREAD_MUTEX 1 + +#if SPINLOCK_TYPE == SPINLOCK_GCC + +typedef volatile int WT_SPINLOCK; + +#elif SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX + +typedef pthread_mutex_t WT_SPINLOCK; + +#else + +#error Unknown spinlock type + +#endif diff --git a/src/include/mutex.i b/src/include/mutex.i new file mode 100644 index 00000000000..b249d65d695 --- /dev/null +++ b/src/include/mutex.i @@ -0,0 +1,102 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* + * Spin locks: + * + * These used for cases where fast mutual exclusion is needed (where operations + * done while holding the spin lock are expected to complete in a small number + * of instructions. + */ + +#if SPINLOCK_TYPE == SPINLOCK_GCC + +static inline void +__wt_spin_init(WT_SESSION_IMPL *session, WT_SPINLOCK *t) +{ + WT_UNUSED(session); + + *(t) = 0; +} + +static inline void +__wt_spin_destroy(WT_SESSION_IMPL *session, WT_SPINLOCK *t) +{ + WT_UNUSED(session); + WT_UNUSED(t); +} + +static inline void +__wt_spin_lock(WT_SESSION_IMPL *session, WT_SPINLOCK *t) +{ + WT_UNUSED(session); + + while (__sync_lock_test_and_set(t, 1)) + while (*t) + WT_PAUSE(); +} + +static inline int +__wt_spin_trylock(WT_SESSION_IMPL *session, WT_SPINLOCK *t) +{ + WT_UNUSED(session); + + return (!__sync_lock_test_and_set(t, 1)); +} + +static inline void +__wt_spin_unlock(WT_SESSION_IMPL *session, WT_SPINLOCK *t) +{ + WT_UNUSED(session); + + __sync_lock_release(t); +} + +#elif SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX + +static inline void +__wt_spin_init(WT_SESSION_IMPL *session, WT_SPINLOCK *t) +{ + WT_UNUSED(session); + + (void)pthread_mutex_init(t, NULL); +} + +static inline void +__wt_spin_destroy(WT_SESSION_IMPL *session, WT_SPINLOCK *t) +{ + WT_UNUSED(session); + + (void)pthread_mutex_destroy(t); +} + +static inline void +__wt_spin_lock(WT_SESSION_IMPL *session, WT_SPINLOCK *t) +{ + WT_UNUSED(session); + pthread_mutex_lock(t); +} + +static inline int +__wt_spin_trylock(WT_SESSION_IMPL *session, WT_SPINLOCK *t) +{ + WT_UNUSED(session); + return (pthread_mutex_trylock(t)); +} + +static inline void +__wt_spin_unlock(WT_SESSION_IMPL *session, WT_SPINLOCK *t) +{ + WT_UNUSED(session); + pthread_mutex_unlock(t); +} + +#else + +#error Unknown spinlock type + +#endif diff --git a/src/include/os.h b/src/include/os.h new file mode 100644 index 00000000000..6e925d6923d --- /dev/null +++ b/src/include/os.h @@ -0,0 +1,41 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#define WT_SYSCALL_RETRY(call, ret) do { \ + int __retry; \ + for (__retry = 0; __retry < 10; ++__retry) { \ + if ((call) == 0) { \ + (ret) = 0; \ + break; \ + } \ + switch ((ret) = __wt_errno()) { \ + case EAGAIN: \ + case EBUSY: \ + case EINTR: \ + case EIO: \ + case EMFILE: \ + case ENFILE: \ + case ENOSPC: \ + __wt_sleep(0L, 500000L); \ + continue; \ + default: \ + break; \ + } \ + break; \ + } \ +} while (0) + +struct __wt_fh { + TAILQ_ENTRY(__wt_fh) q; /* List of open handles */ + + off_t file_size; /* File size */ + + char *name; /* File name */ + int fd; /* POSIX file handle */ + + u_int refcnt; /* Reference count */ +}; diff --git a/src/include/packing.i b/src/include/packing.i new file mode 100644 index 00000000000..e896acc6558 --- /dev/null +++ b/src/include/packing.i @@ -0,0 +1,441 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* + * Throughout this code we have to be aware of default argument conversion. + * + * Refer to Chapter 8 of "Expert C Programming" by Peter van der Linden for the + * gory details. The short version is that we have less cases to deal with + * because the compiler promotes shorter types to int or unsigned int. + */ + +typedef struct { + WT_SESSION_IMPL *session; + const char *cur, *end, *orig; + unsigned long repeats; +} WT_PACK; + +typedef struct { + union { + int64_t i; + uint64_t u; + const char *s; + WT_ITEM item; + } u; + uint32_t size; + int8_t havesize; + char type; +} WT_PACK_VALUE; + +static inline int +__pack_initn( + WT_SESSION_IMPL *session, WT_PACK *pack, const char *fmt, size_t len) +{ + if (*fmt == '@' || *fmt == '<' || *fmt == '>') + return (EINVAL); + if (*fmt == '.') + ++fmt; + + pack->session = session; + pack->cur = pack->orig = fmt; + pack->end = fmt + len; + pack->repeats = 0; + return (0); +} + +static inline int +__pack_init(WT_SESSION_IMPL *session, WT_PACK *pack, const char *fmt) +{ + return (__pack_initn(session, pack, fmt, strlen(fmt))); +} + +static inline int +__pack_next(WT_PACK *pack, WT_PACK_VALUE *pv) +{ + char *endsize; + + if (pack->repeats > 0) { + --pack->repeats; + return (0); + } + +next: if (pack->cur == pack->end) + return (WT_NOTFOUND); + + pv->size = WT_STORE_SIZE(strtoul(pack->cur, &endsize, 10)); + pv->havesize = (endsize > pack->cur); + if (!pv->havesize) + pv->size = 1; + pack->cur = endsize; + pack->repeats = 0; + pv->type = *pack->cur++; + + switch (pv->type) { + case 'S': + case 's': + case 'x': + return (0); + case 't': + if (pv->size < 1 || pv->size > 8) + WT_RET_MSG(pack->session, EINVAL, + "Bitfield sizes must be between 1 and 8 bits " + "in format '%.*s'", + (int)(pack->end - pack->orig), pack->orig); + return (0); + case 'u': + case 'U': + /* Special case for items with a size prefix. */ + pv->type = (!pv->havesize && *pack->cur != '\0') ? 'U' : 'u'; + return (0); + case 'b': + case 'h': + case 'i': + case 'B': + case 'H': + case 'I': + case 'l': + case 'L': + case 'q': + case 'Q': + case 'r': + case 'R': + /* Integral types repeat <size> times. */ + if (pv->size == 0) + goto next; + pack->repeats = pv->size - 1; + return (0); + default: + WT_RET_MSG(pack->session, EINVAL, + "Invalid type '%c' found in format '%.*s'", + pv->type, (int)(pack->end - pack->orig), pack->orig); + } + +} + +#define WT_PACK_GET(session, pv, ap) do { \ + switch (pv.type) { \ + case 'x': \ + break; \ + case 's': \ + case 'S': \ + pv.u.s = va_arg(ap, const char *); \ + break; \ + case 'U': \ + case 'u': \ + pv.u.item = *va_arg(ap, WT_ITEM *); \ + break; \ + case 'b': \ + case 'h': \ + case 'i': \ + pv.u.i = va_arg(ap, int); \ + break; \ + case 'B': \ + case 'H': \ + case 'I': \ + case 't': \ + pv.u.u = va_arg(ap, unsigned int); \ + break; \ + case 'l': \ + pv.u.i = va_arg(ap, long); \ + break; \ + case 'L': \ + pv.u.u = va_arg(ap, unsigned long); \ + break; \ + case 'q': \ + pv.u.i = va_arg(ap, int64_t); \ + break; \ + case 'Q': \ + case 'r': \ + case 'R': \ + pv.u.u = va_arg(ap, uint64_t); \ + break; \ + default: \ + WT_ASSERT(session, pv.type != pv.type); \ + break; \ + } \ +} while (0) + +static inline size_t +__pack_size(WT_SESSION_IMPL *session, WT_PACK_VALUE *pv) +{ + const char *p, *start, *end; + size_t s, pad; + + switch (pv->type) { + case 'x': + return (pv->size); + case 's': + case 'S': + /* + * XXX if pv->havesize, only want to know if there is a + * '\0' in the first pv->size characters. + */ + if (pv->type == 's' || pv->havesize) { + p = start = (const char *)pv->u.s; + end = start + pv->size; + while (p < end && *p != '\0') + p++; + s = (size_t)(p - start); + pad = pv->size - s; + } else { + s = strlen(pv->u.s); + pad = 1; + } + return (s + pad); + case 'U': + case 'u': + s = pv->u.item.size; + pad = 0; + if (pv->havesize && pv->size < s) + s = pv->size; + else if (pv->havesize) + pad = pv->size - s; + if (pv->type == 'U') + s += __wt_vsize_uint(s + pad); + return (s + pad); + case 'b': + case 'B': + case 't': + return (1); + case 'h': + case 'i': + case 'l': + case 'q': + return (__wt_vsize_int(pv->u.i)); + case 'H': + case 'I': + case 'L': + case 'Q': + case 'r': + return (__wt_vsize_uint(pv->u.u)); + case 'R': + return (sizeof (uint64_t)); + } + + WT_RET_MSG( + session, EINVAL, "unknown pack-value type: %c", (int)pv->type); +} + +static inline int +__pack_write( + WT_SESSION_IMPL *session, WT_PACK_VALUE *pv, uint8_t **pp, size_t maxlen) +{ + uint8_t *oldp; + size_t s, pad; + + switch (pv->type) { + case 'x': + WT_SIZE_CHECK(pv->size, maxlen); + memset(*pp, 0, pv->size); + *pp += pv->size; + break; + case 's': + case 'S': + /* + * XXX if pv->havesize, only want to know if there is a + * '\0' in the first pv->size characters. + */ + s = strlen(pv->u.s); + if ((pv->type == 's' || pv->havesize) && pv->size < s) { + s = pv->size; + pad = 0; + } else if (pv->havesize) + pad = pv->size - s; + else + pad = 1; + WT_SIZE_CHECK(s + pad, maxlen); + if (s > 0) + memcpy(*pp, pv->u.s, s); + *pp += s; + if (pad > 0) { + memset(*pp, 0, pad); + *pp += pad; + } + break; + case 'U': + case 'u': + s = pv->u.item.size; + pad = 0; + if (pv->havesize && pv->size < s) + s = pv->size; + else if (pv->havesize) + pad = pv->size - s; + if (pv->type == 'U') { + oldp = *pp; + WT_RET(__wt_vpack_uint(pp, maxlen, s + pad)); + maxlen -= (size_t)(*pp - oldp); + } + WT_SIZE_CHECK(s + pad, maxlen); + if (s > 0) + memcpy(*pp, pv->u.item.data, s); + *pp += s; + if (pad > 0) { + memset(*pp, 0, pad); + *pp += pad; + } + break; + case 'b': + /* Translate to maintain ordering with the sign bit. */ + WT_SIZE_CHECK(1, maxlen); + **pp = (uint8_t)(pv->u.i + 0x80); + *pp += 1; + break; + case 'B': + case 't': + WT_SIZE_CHECK(1, maxlen); + **pp = (uint8_t)pv->u.u; + *pp += 1; + break; + case 'h': + case 'i': + case 'l': + case 'q': + WT_RET(__wt_vpack_int(pp, maxlen, pv->u.i)); + break; + case 'H': + case 'I': + case 'L': + case 'Q': + case 'r': + WT_RET(__wt_vpack_uint(pp, maxlen, pv->u.u)); + break; + case 'R': + WT_SIZE_CHECK(sizeof (uint64_t), maxlen); + *(uint64_t *)*pp = pv->u.u; + *pp += sizeof(uint64_t); + break; + default: + WT_RET_MSG(session, EINVAL, + "unknown pack-value type: %c", (int)pv->type); + } + + return (0); +} + +static inline int +__unpack_read(WT_SESSION_IMPL *session, + WT_PACK_VALUE *pv, const uint8_t **pp, size_t maxlen) +{ + size_t s; + + switch (pv->type) { + case 'x': + WT_SIZE_CHECK(pv->size, maxlen); + *pp += pv->size; + break; + case 's': + case 'S': + if (pv->type == 's' || pv->havesize) + s = pv->size; + else + s = strlen((const char *)*pp) + 1; + if (s > 0) + pv->u.s = (const char *)*pp; + WT_SIZE_CHECK(s, maxlen); + *pp += s; + break; + case 'U': + WT_RET(__wt_vunpack_uint(pp, maxlen, &pv->u.u)); + /* FALLTHROUGH */ + case 'u': + if (pv->havesize) + s = pv->size; + else if (pv->type == 'U') + s = (size_t)pv->u.u; + else + s = maxlen; + WT_SIZE_CHECK(s, maxlen); + pv->u.item.data = *pp; + pv->u.item.size = WT_STORE_SIZE(s); + *pp += s; + break; + case 'b': + /* Translate to maintain ordering with the sign bit. */ + WT_SIZE_CHECK(1, maxlen); + pv->u.i = (int8_t)(**pp++ - 0x80); + break; + case 'B': + case 't': + WT_SIZE_CHECK(1, maxlen); + pv->u.u = **pp++; + break; + case 'h': + case 'i': + case 'l': + case 'q': + WT_RET(__wt_vunpack_int(pp, maxlen, &pv->u.i)); + break; + case 'H': + case 'I': + case 'L': + case 'Q': + case 'r': + WT_RET(__wt_vunpack_uint(pp, maxlen, &pv->u.u)); + break; + case 'R': + WT_SIZE_CHECK(sizeof (uint64_t), maxlen); + pv->u.u = *(uint64_t *)*pp; + *pp += sizeof(uint64_t); + break; + default: + WT_RET_MSG(session, EINVAL, + "unknown pack-value type: %c", (int)pv->type); + } + + return (0); +} + +#define WT_UNPACK_PUT(session, pv, ap) do { \ + switch (pv.type) { \ + case 'x': \ + break; \ + case 's': \ + case 'S': \ + *va_arg(ap, const char **) = pv.u.s; \ + break; \ + case 'U': \ + case 'u': \ + *va_arg(ap, WT_ITEM *) = pv.u.item; \ + break; \ + case 'b': \ + *va_arg(ap, int8_t *) = (int8_t)pv.u.i; \ + break; \ + case 'h': \ + *va_arg(ap, short *) = (short)pv.u.i; \ + break; \ + case 'i': \ + *va_arg(ap, int *) = (int)pv.u.i; \ + break; \ + case 'l': \ + *va_arg(ap, long *) = (long)pv.u.i; \ + break; \ + case 'q': \ + *va_arg(ap, int64_t *) = pv.u.i; \ + break; \ + case 'B': \ + case 't': \ + *va_arg(ap, uint8_t *) = (uint8_t)pv.u.u; \ + break; \ + case 'H': \ + *va_arg(ap, unsigned short *) = (unsigned short)pv.u.u; \ + break; \ + case 'I': \ + *va_arg(ap, unsigned int *) = (unsigned int)pv.u.u; \ + break; \ + case 'L': \ + *va_arg(ap, unsigned long *) = (unsigned long)pv.u.u; \ + break; \ + case 'Q': \ + case 'r': \ + case 'R': \ + *va_arg(ap, uint64_t *) = pv.u.u; \ + break; \ + default: \ + WT_ASSERT(session, pv.type != pv.type); \ + break; \ + } \ +} while (0) diff --git a/src/include/posix.h b/src/include/posix.h new file mode 100644 index 00000000000..db6327e84f2 --- /dev/null +++ b/src/include/posix.h @@ -0,0 +1,17 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* Some systems don't configure 64-bit MIN/MAX by default. */ +#ifndef ULLONG_MAX +#define ULLONG_MAX 0xffffffffffffffffULL +#endif +#ifndef LLONG_MAX +#define LLONG_MAX 0x7fffffffffffffffLL +#endif +#ifndef LLONG_MIN +#define LLONG_MIN (-0x7fffffffffffffffLL - 1) +#endif diff --git a/src/include/progress.i b/src/include/progress.i new file mode 100644 index 00000000000..315eeabc6cd --- /dev/null +++ b/src/include/progress.i @@ -0,0 +1,23 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* + * __wt_progress -- + * Send a progress message to stdout. + */ +static inline void +__wt_progress(WT_SESSION_IMPL *session, const char *s, uint64_t v) +{ + WT_EVENT_HANDLER *handler; + + if (s == NULL) + s = session->name; + + handler = session->event_handler; + if (handler->handle_progress != NULL) + (void)handler->handle_progress(handler, s, v); +} diff --git a/src/include/queue.h b/src/include/queue.h new file mode 100644 index 00000000000..42e736e7b09 --- /dev/null +++ b/src/include/queue.h @@ -0,0 +1,559 @@ +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)queue.h 8.5 (Berkeley) 8/20/94 + * $FreeBSD: src/sys/sys/queue.h,v 1.54 2002/08/05 05:18:43 alfred Exp $ + */ + +#ifndef _DB_QUEUE_H_ +#define _DB_QUEUE_H_ + +#if defined(__cplusplus) +extern "C" { +#endif + +/* + * This file defines four types of data structures: singly-linked lists, + * singly-linked tail queues, lists and tail queues. + * + * A singly-linked list is headed by a single forward pointer. The elements + * are singly linked for minimum space and pointer manipulation overhead at + * the expense of O(n) removal for arbitrary elements. New elements can be + * added to the list after an existing element or at the head of the list. + * Elements being removed from the head of the list should use the explicit + * macro for this purpose for optimum efficiency. A singly-linked list may + * only be traversed in the forward direction. Singly-linked lists are ideal + * for applications with large datasets and few or no removals or for + * implementing a LIFO queue. + * + * A singly-linked tail queue is headed by a pair of pointers, one to the + * head of the list and the other to the tail of the list. The elements are + * singly linked for minimum space and pointer manipulation overhead at the + * expense of O(n) removal for arbitrary elements. New elements can be added + * to the list after an existing element, at the head of the list, or at the + * end of the list. Elements being removed from the head of the tail queue + * should use the explicit macro for this purpose for optimum efficiency. + * A singly-linked tail queue may only be traversed in the forward direction. + * Singly-linked tail queues are ideal for applications with large datasets + * and few or no removals or for implementing a FIFO queue. + * + * A list is headed by a single forward pointer (or an array of forward + * pointers for a hash table header). The elements are doubly linked + * so that an arbitrary element can be removed without a need to + * traverse the list. New elements can be added to the list before + * or after an existing element or at the head of the list. A list + * may only be traversed in the forward direction. + * + * A tail queue is headed by a pair of pointers, one to the head of the + * list and the other to the tail of the list. The elements are doubly + * linked so that an arbitrary element can be removed without a need to + * traverse the list. New elements can be added to the list before or + * after an existing element, at the head of the list, or at the end of + * the list. A tail queue may be traversed in either direction. + * + * For details on the use of these macros, see the queue(3) manual page. + * + * + * SLIST LIST STAILQ TAILQ + * _HEAD + + + + + * _HEAD_INITIALIZER + + + + + * _ENTRY + + + + + * _INIT + + + + + * _EMPTY + + + + + * _FIRST + + + + + * _NEXT + + + + + * _PREV - - - + + * _LAST - - + + + * _FOREACH + + + + + * _FOREACH_REVERSE - - - + + * _INSERT_HEAD + + + + + * _INSERT_BEFORE - + - + + * _INSERT_AFTER + + + + + * _INSERT_TAIL - - + + + * _CONCAT - - + + + * _REMOVE_HEAD + - + - + * _REMOVE + + + + + * + */ + +/* + * XXX + * We #undef all of the macros because there are incompatible versions of this + * file and these macros on various systems. What makes the problem worse is + * they are included and/or defined by system include files which we may have + * already loaded into Berkeley DB before getting here. For example, FreeBSD's + * <rpc/rpc.h> includes its system <sys/queue.h>, and VxWorks UnixLib.h defines + * several of the LIST_XXX macros. Visual C.NET 7.0 also defines some of these + * same macros in Vc7\PlatformSDK\Include\WinNT.h. Make sure we use ours. + */ +#undef LIST_EMPTY +#undef LIST_ENTRY +#undef LIST_FIRST +#undef LIST_FOREACH +#undef LIST_HEAD +#undef LIST_HEAD_INITIALIZER +#undef LIST_INIT +#undef LIST_INSERT_AFTER +#undef LIST_INSERT_BEFORE +#undef LIST_INSERT_HEAD +#undef LIST_NEXT +#undef LIST_REMOVE +#undef QMD_TRACE_ELEM +#undef QMD_TRACE_HEAD +#undef QUEUE_MACRO_DEBUG +#undef SLIST_EMPTY +#undef SLIST_ENTRY +#undef SLIST_FIRST +#undef SLIST_FOREACH +#undef SLIST_FOREACH_PREVPTR +#undef SLIST_HEAD +#undef SLIST_HEAD_INITIALIZER +#undef SLIST_INIT +#undef SLIST_INSERT_AFTER +#undef SLIST_INSERT_HEAD +#undef SLIST_NEXT +#undef SLIST_REMOVE +#undef SLIST_REMOVE_HEAD +#undef STAILQ_CONCAT +#undef STAILQ_EMPTY +#undef STAILQ_ENTRY +#undef STAILQ_FIRST +#undef STAILQ_FOREACH +#undef STAILQ_HEAD +#undef STAILQ_HEAD_INITIALIZER +#undef STAILQ_INIT +#undef STAILQ_INSERT_AFTER +#undef STAILQ_INSERT_HEAD +#undef STAILQ_INSERT_TAIL +#undef STAILQ_LAST +#undef STAILQ_NEXT +#undef STAILQ_REMOVE +#undef STAILQ_REMOVE_HEAD +#undef STAILQ_REMOVE_HEAD_UNTIL +#undef TAILQ_CONCAT +#undef TAILQ_EMPTY +#undef TAILQ_ENTRY +#undef TAILQ_FIRST +#undef TAILQ_FOREACH +#undef TAILQ_FOREACH_REVERSE +#undef TAILQ_HEAD +#undef TAILQ_HEAD_INITIALIZER +#undef TAILQ_INIT +#undef TAILQ_INSERT_AFTER +#undef TAILQ_INSERT_BEFORE +#undef TAILQ_INSERT_HEAD +#undef TAILQ_INSERT_TAIL +#undef TAILQ_LAST +#undef TAILQ_NEXT +#undef TAILQ_PREV +#undef TAILQ_REMOVE +#undef TRACEBUF +#undef TRASHIT + +#define QUEUE_MACRO_DEBUG 0 +#if QUEUE_MACRO_DEBUG +/* Store the last 2 places the queue element or head was altered */ +struct qm_trace { + char * lastfile; + int lastline; + char * prevfile; + int prevline; +}; + +#define TRACEBUF struct qm_trace trace; +#define TRASHIT(x) do {(x) = (void *)-1;} while (0) + +#define QMD_TRACE_HEAD(head) do { \ + (head)->trace.prevline = (head)->trace.lastline; \ + (head)->trace.prevfile = (head)->trace.lastfile; \ + (head)->trace.lastline = __LINE__; \ + (head)->trace.lastfile = __FILE__; \ +} while (0) + +#define QMD_TRACE_ELEM(elem) do { \ + (elem)->trace.prevline = (elem)->trace.lastline; \ + (elem)->trace.prevfile = (elem)->trace.lastfile; \ + (elem)->trace.lastline = __LINE__; \ + (elem)->trace.lastfile = __FILE__; \ +} while (0) + +#else +#define QMD_TRACE_ELEM(elem) +#define QMD_TRACE_HEAD(head) +#define TRACEBUF +#define TRASHIT(x) +#endif /* QUEUE_MACRO_DEBUG */ + +/* + * Singly-linked List declarations. + */ +#define SLIST_HEAD(name, type) \ +struct name { \ + struct type *slh_first; /* first element */ \ +} + +#define SLIST_HEAD_INITIALIZER(head) \ + { NULL } + +#define SLIST_ENTRY(type) \ +struct { \ + struct type *sle_next; /* next element */ \ +} + +/* + * Singly-linked List functions. + */ +#define SLIST_EMPTY(head) ((head)->slh_first == NULL) + +#define SLIST_FIRST(head) ((head)->slh_first) + +#define SLIST_FOREACH(var, head, field) \ + for ((var) = SLIST_FIRST((head)); \ + (var); \ + (var) = SLIST_NEXT((var), field)) + +#define SLIST_FOREACH_PREVPTR(var, varp, head, field) \ + for ((varp) = &SLIST_FIRST((head)); \ + ((var) = *(varp)) != NULL; \ + (varp) = &SLIST_NEXT((var), field)) + +#define SLIST_INIT(head) do { \ + SLIST_FIRST((head)) = NULL; \ +} while (0) + +#define SLIST_INSERT_AFTER(slistelm, elm, field) do { \ + SLIST_NEXT((elm), field) = SLIST_NEXT((slistelm), field); \ + SLIST_NEXT((slistelm), field) = (elm); \ +} while (0) + +#define SLIST_INSERT_HEAD(head, elm, field) do { \ + SLIST_NEXT((elm), field) = SLIST_FIRST((head)); \ + SLIST_FIRST((head)) = (elm); \ +} while (0) + +#define SLIST_NEXT(elm, field) ((elm)->field.sle_next) + +#define SLIST_REMOVE(head, elm, type, field) do { \ + if (SLIST_FIRST((head)) == (elm)) { \ + SLIST_REMOVE_HEAD((head), field); \ + } \ + else { \ + struct type *curelm = SLIST_FIRST((head)); \ + while (SLIST_NEXT(curelm, field) != (elm)) \ + curelm = SLIST_NEXT(curelm, field); \ + SLIST_NEXT(curelm, field) = \ + SLIST_NEXT(SLIST_NEXT(curelm, field), field); \ + } \ +} while (0) + +#define SLIST_REMOVE_HEAD(head, field) do { \ + SLIST_FIRST((head)) = SLIST_NEXT(SLIST_FIRST((head)), field); \ +} while (0) + +/* + * Singly-linked Tail queue declarations. + */ +#define STAILQ_HEAD(name, type) \ +struct name { \ + struct type *stqh_first;/* first element */ \ + struct type **stqh_last;/* addr of last next element */ \ +} + +#define STAILQ_HEAD_INITIALIZER(head) \ + { NULL, &(head).stqh_first } + +#define STAILQ_ENTRY(type) \ +struct { \ + struct type *stqe_next; /* next element */ \ +} + +/* + * Singly-linked Tail queue functions. + */ +#define STAILQ_CONCAT(head1, head2) do { \ + if (!STAILQ_EMPTY((head2))) { \ + *(head1)->stqh_last = (head2)->stqh_first; \ + (head1)->stqh_last = (head2)->stqh_last; \ + STAILQ_INIT((head2)); \ + } \ +} while (0) + +#define STAILQ_EMPTY(head) ((head)->stqh_first == NULL) + +#define STAILQ_FIRST(head) ((head)->stqh_first) + +#define STAILQ_FOREACH(var, head, field) \ + for ((var) = STAILQ_FIRST((head)); \ + (var); \ + (var) = STAILQ_NEXT((var), field)) + +#define STAILQ_INIT(head) do { \ + STAILQ_FIRST((head)) = NULL; \ + (head)->stqh_last = &STAILQ_FIRST((head)); \ +} while (0) + +#define STAILQ_INSERT_AFTER(head, tqelm, elm, field) do { \ + if ((STAILQ_NEXT((elm), field) = STAILQ_NEXT((tqelm), field)) == NULL)\ + (head)->stqh_last = &STAILQ_NEXT((elm), field); \ + STAILQ_NEXT((tqelm), field) = (elm); \ +} while (0) + +#define STAILQ_INSERT_HEAD(head, elm, field) do { \ + if ((STAILQ_NEXT((elm), field) = STAILQ_FIRST((head))) == NULL) \ + (head)->stqh_last = &STAILQ_NEXT((elm), field); \ + STAILQ_FIRST((head)) = (elm); \ +} while (0) + +#define STAILQ_INSERT_TAIL(head, elm, field) do { \ + STAILQ_NEXT((elm), field) = NULL; \ + *(head)->stqh_last = (elm); \ + (head)->stqh_last = &STAILQ_NEXT((elm), field); \ +} while (0) + +#define STAILQ_LAST(head, type, field) \ + (STAILQ_EMPTY((head)) ? \ + NULL : \ + ((struct type *) \ + ((char *)((head)->stqh_last) - __offsetof(struct type, field)))) + +#define STAILQ_NEXT(elm, field) ((elm)->field.stqe_next) + +#define STAILQ_REMOVE(head, elm, type, field) do { \ + if (STAILQ_FIRST((head)) == (elm)) { \ + STAILQ_REMOVE_HEAD((head), field); \ + } \ + else { \ + struct type *curelm = STAILQ_FIRST((head)); \ + while (STAILQ_NEXT(curelm, field) != (elm)) \ + curelm = STAILQ_NEXT(curelm, field); \ + if ((STAILQ_NEXT(curelm, field) = \ + STAILQ_NEXT(STAILQ_NEXT(curelm, field), field)) == NULL)\ + (head)->stqh_last = &STAILQ_NEXT((curelm), field);\ + } \ +} while (0) + +#define STAILQ_REMOVE_HEAD(head, field) do { \ + if ((STAILQ_FIRST((head)) = \ + STAILQ_NEXT(STAILQ_FIRST((head)), field)) == NULL) \ + (head)->stqh_last = &STAILQ_FIRST((head)); \ +} while (0) + +#define STAILQ_REMOVE_HEAD_UNTIL(head, elm, field) do { \ + if ((STAILQ_FIRST((head)) = STAILQ_NEXT((elm), field)) == NULL) \ + (head)->stqh_last = &STAILQ_FIRST((head)); \ +} while (0) + +/* + * List declarations. + */ +#define LIST_HEAD(name, type) \ +struct name { \ + struct type *lh_first; /* first element */ \ +} + +#define LIST_HEAD_INITIALIZER(head) \ + { NULL } + +#define LIST_ENTRY(type) \ +struct { \ + struct type *le_next; /* next element */ \ + struct type **le_prev; /* address of previous next element */ \ +} + +/* + * List functions. + */ + +#define LIST_EMPTY(head) ((head)->lh_first == NULL) + +#define LIST_FIRST(head) ((head)->lh_first) + +#define LIST_FOREACH(var, head, field) \ + for ((var) = LIST_FIRST((head)); \ + (var); \ + (var) = LIST_NEXT((var), field)) + +#define LIST_INIT(head) do { \ + LIST_FIRST((head)) = NULL; \ +} while (0) + +#define LIST_INSERT_AFTER(listelm, elm, field) do { \ + if ((LIST_NEXT((elm), field) = LIST_NEXT((listelm), field)) != NULL)\ + LIST_NEXT((listelm), field)->field.le_prev = \ + &LIST_NEXT((elm), field); \ + LIST_NEXT((listelm), field) = (elm); \ + (elm)->field.le_prev = &LIST_NEXT((listelm), field); \ +} while (0) + +#define LIST_INSERT_BEFORE(listelm, elm, field) do { \ + (elm)->field.le_prev = (listelm)->field.le_prev; \ + LIST_NEXT((elm), field) = (listelm); \ + *(listelm)->field.le_prev = (elm); \ + (listelm)->field.le_prev = &LIST_NEXT((elm), field); \ +} while (0) + +#define LIST_INSERT_HEAD(head, elm, field) do { \ + if ((LIST_NEXT((elm), field) = LIST_FIRST((head))) != NULL) \ + LIST_FIRST((head))->field.le_prev = &LIST_NEXT((elm), field);\ + LIST_FIRST((head)) = (elm); \ + (elm)->field.le_prev = &LIST_FIRST((head)); \ +} while (0) + +#define LIST_NEXT(elm, field) ((elm)->field.le_next) + +#define LIST_REMOVE(elm, field) do { \ + if (LIST_NEXT((elm), field) != NULL) \ + LIST_NEXT((elm), field)->field.le_prev = \ + (elm)->field.le_prev; \ + *(elm)->field.le_prev = LIST_NEXT((elm), field); \ +} while (0) + +/* + * Tail queue declarations. + */ +#define TAILQ_HEAD(name, type) \ +struct name { \ + struct type *tqh_first; /* first element */ \ + struct type **tqh_last; /* addr of last next element */ \ + TRACEBUF \ +} + +#define TAILQ_HEAD_INITIALIZER(head) \ + { NULL, &(head).tqh_first } + +#define TAILQ_ENTRY(type) \ +struct { \ + struct type *tqe_next; /* next element */ \ + struct type **tqe_prev; /* address of previous next element */ \ + TRACEBUF \ +} + +/* + * Tail queue functions. + */ +#define TAILQ_CONCAT(head1, head2, field) do { \ + if (!TAILQ_EMPTY(head2)) { \ + *(head1)->tqh_last = (head2)->tqh_first; \ + (head2)->tqh_first->field.tqe_prev = (head1)->tqh_last; \ + (head1)->tqh_last = (head2)->tqh_last; \ + TAILQ_INIT((head2)); \ + QMD_TRACE_HEAD(head); \ + QMD_TRACE_HEAD(head2); \ + } \ +} while (0) + +#define TAILQ_EMPTY(head) ((head)->tqh_first == NULL) + +#define TAILQ_FIRST(head) ((head)->tqh_first) + +#define TAILQ_FOREACH(var, head, field) \ + for ((var) = TAILQ_FIRST((head)); \ + (var); \ + (var) = TAILQ_NEXT((var), field)) + +#define TAILQ_FOREACH_REVERSE(var, head, headname, field) \ + for ((var) = TAILQ_LAST((head), headname); \ + (var); \ + (var) = TAILQ_PREV((var), headname, field)) + +#define TAILQ_INIT(head) do { \ + TAILQ_FIRST((head)) = NULL; \ + (head)->tqh_last = &TAILQ_FIRST((head)); \ + QMD_TRACE_HEAD(head); \ +} while (0) + +#define TAILQ_INSERT_AFTER(head, listelm, elm, field) do { \ + if ((TAILQ_NEXT((elm), field) = TAILQ_NEXT((listelm), field)) != NULL)\ + TAILQ_NEXT((elm), field)->field.tqe_prev = \ + &TAILQ_NEXT((elm), field); \ + else { \ + (head)->tqh_last = &TAILQ_NEXT((elm), field); \ + QMD_TRACE_HEAD(head); \ + } \ + TAILQ_NEXT((listelm), field) = (elm); \ + (elm)->field.tqe_prev = &TAILQ_NEXT((listelm), field); \ + QMD_TRACE_ELEM(&(elm)->field); \ + QMD_TRACE_ELEM(&listelm->field); \ +} while (0) + +#define TAILQ_INSERT_BEFORE(listelm, elm, field) do { \ + (elm)->field.tqe_prev = (listelm)->field.tqe_prev; \ + TAILQ_NEXT((elm), field) = (listelm); \ + *(listelm)->field.tqe_prev = (elm); \ + (listelm)->field.tqe_prev = &TAILQ_NEXT((elm), field); \ + QMD_TRACE_ELEM(&(elm)->field); \ + QMD_TRACE_ELEM(&listelm->field); \ +} while (0) + +#define TAILQ_INSERT_HEAD(head, elm, field) do { \ + if ((TAILQ_NEXT((elm), field) = TAILQ_FIRST((head))) != NULL) \ + TAILQ_FIRST((head))->field.tqe_prev = \ + &TAILQ_NEXT((elm), field); \ + else \ + (head)->tqh_last = &TAILQ_NEXT((elm), field); \ + TAILQ_FIRST((head)) = (elm); \ + (elm)->field.tqe_prev = &TAILQ_FIRST((head)); \ + QMD_TRACE_HEAD(head); \ + QMD_TRACE_ELEM(&(elm)->field); \ +} while (0) + +#define TAILQ_INSERT_TAIL(head, elm, field) do { \ + TAILQ_NEXT((elm), field) = NULL; \ + (elm)->field.tqe_prev = (head)->tqh_last; \ + *(head)->tqh_last = (elm); \ + (head)->tqh_last = &TAILQ_NEXT((elm), field); \ + QMD_TRACE_HEAD(head); \ + QMD_TRACE_ELEM(&(elm)->field); \ +} while (0) + +#define TAILQ_LAST(head, headname) \ + (*(((struct headname *)((head)->tqh_last))->tqh_last)) + +#define TAILQ_NEXT(elm, field) ((elm)->field.tqe_next) + +#define TAILQ_PREV(elm, headname, field) \ + (*(((struct headname *)((elm)->field.tqe_prev))->tqh_last)) + +#define TAILQ_REMOVE(head, elm, field) do { \ + if ((TAILQ_NEXT((elm), field)) != NULL) \ + TAILQ_NEXT((elm), field)->field.tqe_prev = \ + (elm)->field.tqe_prev; \ + else { \ + (head)->tqh_last = (elm)->field.tqe_prev; \ + QMD_TRACE_HEAD(head); \ + } \ + *(elm)->field.tqe_prev = TAILQ_NEXT((elm), field); \ + TRASHIT((elm)->field.tqe_next); \ + TRASHIT((elm)->field.tqe_prev); \ + QMD_TRACE_ELEM(&(elm)->field); \ +} while (0) + +#if defined(__cplusplus) +} +#endif +#endif /* !_DB_QUEUE_H_ */ diff --git a/src/include/schema.h b/src/include/schema.h new file mode 100644 index 00000000000..b7467fb18bb --- /dev/null +++ b/src/include/schema.h @@ -0,0 +1,48 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#define WT_SCHEMA_FILENAME "WiredTiger.wt" /* Schema file name */ +#define WT_SCHEMA_URI "file:WiredTiger.wt" /* Schema file URI */ + +#define WT_SCHEMA_VERSION "WiredTiger version" /* Version keys */ +#define WT_SCHEMA_VERSION_STR "WiredTiger version string" + +/* Character constants for projection plans. */ +#define WT_PROJ_KEY 'k' /* Go to key in cursor <arg>. */ +#define WT_PROJ_NEXT 'n' /* Process the next item (<arg> repeats). */ +#define WT_PROJ_REUSE 'r' /* Reuse the previous item (<arg> repeats). */ +#define WT_PROJ_SKIP 's' /* Skip a column in the cursor (<arg> repeats). */ +#define WT_PROJ_VALUE 'v' /* Go to the value in cursor <arg>. */ + +/* + * WT_TABLE -- + * Handle for a logical table. A table consists of one or more column + * groups, each of which holds some set of columns all sharing a primary + * key; and zero or more indices, each of which holds some set of columns + * in an index key that can be used to reconstruct the primary key. + */ +struct __wt_table { + const char *name, *config, *plan; + const char *key_format, *value_format; + + WT_CONFIG_ITEM cgconf, colconf; + + WT_BTREE **colgroup; + WT_BTREE **index; + size_t index_alloc; + + TAILQ_ENTRY(__wt_table) q; + + int cg_complete, idx_complete, is_simple; + int ncolgroups, nindices, nkey_columns; +}; + +/* + * Tables without explicit column groups have a single default column group + * containing all of the columns. + */ +#define WT_COLGROUPS(t) WT_MAX((t)->ncolgroups, 1) diff --git a/src/include/serial.i b/src/include/serial.i new file mode 100644 index 00000000000..c74d683e333 --- /dev/null +++ b/src/include/serial.i @@ -0,0 +1,89 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* + * Serialization: serialization support allows scheduling operations requiring + * serialized access to a piece of memory, normally by a different thread of + * control. This includes updating and evicting pages from trees. + * + * __wt_session_serialize_func -- + * Schedule a serialization request, and block or spin until it completes. + */ +static inline int +__wt_session_serialize_func(WT_SESSION_IMPL *session, + wq_state_t op, void (*func)(WT_SESSION_IMPL *), void *args) +{ + WT_CONNECTION_IMPL *conn; + + conn = S2C(session); + + /* + * Threads serializing access to data using a function: + * call the function while holding a spinlock + * update the session sleeping state, and + * if necessary, block until an async action completes. + */ + session->wq_args = args; + session->wq_sleeping = (op == WT_SERIAL_EVICT); + + /* Functions are serialized by holding a spinlock. */ + __wt_spin_lock(session, &conn->serial_lock); + + func(session); + + __wt_spin_unlock(session, &conn->serial_lock); + + switch (op) { + case WT_SERIAL_EVICT: + __wt_evict_server_wake(session); + break; + default: + break; + } + + /* + * If we are waiting on a server thread, block on the session condition + * variable: when the operation is complete, this will be notified and + * we can continue. + */ + if (session->wq_sleeping) + __wt_cond_wait(session, session->cond); + return (session->wq_ret); +} + +/* + * __wt_session_serialize_wrapup -- + * Server function cleanup. + */ +static inline void +__wt_session_serialize_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page, int ret) +{ + if (ret == 0 && page != NULL) { + /* + * If passed a page and the return value is OK, we modified the + * page. Wake the eviction server as necessary if the page + * has become too large. + */ + __wt_page_modify_set(page); + (void)__wt_eviction_page_check(session, page); + } + + /* + * Publish: there must be a barrier to ensure the return value is set + * before the calling thread can see its results, and the page's new + * write generation makes it to memory. The latter isn't a correctness + * issue, the write generation just needs to be updated so that readers + * get credit for reading the right version of the page, otherwise, + * they will have to retry their update for reading an old version of + * the page. + */ + WT_PUBLISH(session->wq_ret, ret); + + /* If the calling thread is sleeping, wake it up. */ + if (session->wq_sleeping) + __wt_cond_signal(session, session->cond); +} diff --git a/src/include/serial_funcs.i b/src/include/serial_funcs.i new file mode 100644 index 00000000000..3f30c199f36 --- /dev/null +++ b/src/include/serial_funcs.i @@ -0,0 +1,419 @@ +/* DO NOT EDIT: automatically built by dist/serial.py. */ + +typedef struct { + WT_INSERT_HEAD **inshead; + WT_INSERT ***ins_stack; + WT_INSERT_HEAD **new_inslist; + size_t new_inslist_size; + int new_inslist_taken; + WT_INSERT_HEAD *new_inshead; + size_t new_inshead_size; + int new_inshead_taken; + WT_INSERT *new_ins; + size_t new_ins_size; + int new_ins_taken; + u_int skipdepth; +} __wt_col_append_args; + +static inline int +__wt_col_append_serial( + WT_SESSION_IMPL *session, WT_INSERT_HEAD **inshead, WT_INSERT + ***ins_stack, WT_INSERT_HEAD ***new_inslistp, size_t new_inslist_size, + WT_INSERT_HEAD **new_insheadp, size_t new_inshead_size, WT_INSERT + **new_insp, size_t new_ins_size, u_int skipdepth) +{ + __wt_col_append_args _args, *args = &_args; + int ret; + + args->inshead = inshead; + + args->ins_stack = ins_stack; + + if (new_inslistp == NULL) + args->new_inslist = NULL; + else { + args->new_inslist = *new_inslistp; + *new_inslistp = NULL; + args->new_inslist_size = new_inslist_size; + } + args->new_inslist_taken = 0; + + if (new_insheadp == NULL) + args->new_inshead = NULL; + else { + args->new_inshead = *new_insheadp; + *new_insheadp = NULL; + args->new_inshead_size = new_inshead_size; + } + args->new_inshead_taken = 0; + + if (new_insp == NULL) + args->new_ins = NULL; + else { + args->new_ins = *new_insp; + *new_insp = NULL; + args->new_ins_size = new_ins_size; + } + args->new_ins_taken = 0; + + args->skipdepth = skipdepth; + + ret = __wt_session_serialize_func(session, + WT_SERIAL_FUNC, __wt_col_append_serial_func, args); + + if (!args->new_inslist_taken) + __wt_free(session, args->new_inslist); + if (!args->new_inshead_taken) + __wt_free(session, args->new_inshead); + if (!args->new_ins_taken) + __wt_free(session, args->new_ins); + return (ret); +} + +static inline void +__wt_col_append_unpack( + WT_SESSION_IMPL *session, WT_INSERT_HEAD ***insheadp, WT_INSERT + ****ins_stackp, WT_INSERT_HEAD ***new_inslistp, WT_INSERT_HEAD + **new_insheadp, WT_INSERT **new_insp, u_int *skipdepthp) +{ + __wt_col_append_args *args = + (__wt_col_append_args *)session->wq_args; + + *insheadp = args->inshead; + *ins_stackp = args->ins_stack; + *new_inslistp = args->new_inslist; + *new_insheadp = args->new_inshead; + *new_insp = args->new_ins; + *skipdepthp = args->skipdepth; +} + +static inline void +__wt_col_append_new_inslist_taken(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + __wt_col_append_args *args = + (__wt_col_append_args *)session->wq_args; + + args->new_inslist_taken = 1; + + WT_ASSERT(session, args->new_inslist_size != 0); + __wt_cache_page_inmem_incr(session, page, args->new_inslist_size); +} + +static inline void +__wt_col_append_new_inshead_taken(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + __wt_col_append_args *args = + (__wt_col_append_args *)session->wq_args; + + args->new_inshead_taken = 1; + + WT_ASSERT(session, args->new_inshead_size != 0); + __wt_cache_page_inmem_incr(session, page, args->new_inshead_size); +} + +static inline void +__wt_col_append_new_ins_taken(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + __wt_col_append_args *args = + (__wt_col_append_args *)session->wq_args; + + args->new_ins_taken = 1; + + WT_ASSERT(session, args->new_ins_size != 0); + __wt_cache_page_inmem_incr(session, page, args->new_ins_size); +} + +typedef struct { + int close_method; +} __wt_evict_file_args; + +static inline int +__wt_evict_file_serial( + WT_SESSION_IMPL *session, int close_method) +{ + __wt_evict_file_args _args, *args = &_args; + int ret; + + args->close_method = close_method; + + ret = __wt_session_serialize_func(session, + WT_SERIAL_EVICT, __wt_evict_file_serial_func, args); + + return (ret); +} + +static inline void +__wt_evict_file_unpack( + WT_SESSION_IMPL *session, int *close_methodp) +{ + __wt_evict_file_args *args = + (__wt_evict_file_args *)session->wq_args; + + *close_methodp = args->close_method; +} + +typedef struct { + WT_PAGE *page; + uint32_t write_gen; + WT_INSERT_HEAD **inshead; + WT_INSERT ***ins_stack; + WT_INSERT_HEAD **new_inslist; + size_t new_inslist_size; + int new_inslist_taken; + WT_INSERT_HEAD *new_inshead; + size_t new_inshead_size; + int new_inshead_taken; + WT_INSERT *new_ins; + size_t new_ins_size; + int new_ins_taken; + u_int skipdepth; +} __wt_insert_args; + +static inline int +__wt_insert_serial( + WT_SESSION_IMPL *session, WT_PAGE *page, uint32_t write_gen, + WT_INSERT_HEAD **inshead, WT_INSERT ***ins_stack, WT_INSERT_HEAD + ***new_inslistp, size_t new_inslist_size, WT_INSERT_HEAD + **new_insheadp, size_t new_inshead_size, WT_INSERT **new_insp, size_t + new_ins_size, u_int skipdepth) +{ + __wt_insert_args _args, *args = &_args; + int ret; + + args->page = page; + + args->write_gen = write_gen; + + args->inshead = inshead; + + args->ins_stack = ins_stack; + + if (new_inslistp == NULL) + args->new_inslist = NULL; + else { + args->new_inslist = *new_inslistp; + *new_inslistp = NULL; + args->new_inslist_size = new_inslist_size; + } + args->new_inslist_taken = 0; + + if (new_insheadp == NULL) + args->new_inshead = NULL; + else { + args->new_inshead = *new_insheadp; + *new_insheadp = NULL; + args->new_inshead_size = new_inshead_size; + } + args->new_inshead_taken = 0; + + if (new_insp == NULL) + args->new_ins = NULL; + else { + args->new_ins = *new_insp; + *new_insp = NULL; + args->new_ins_size = new_ins_size; + } + args->new_ins_taken = 0; + + args->skipdepth = skipdepth; + + ret = __wt_session_serialize_func(session, + WT_SERIAL_FUNC, __wt_insert_serial_func, args); + + if (!args->new_inslist_taken) + __wt_free(session, args->new_inslist); + if (!args->new_inshead_taken) + __wt_free(session, args->new_inshead); + if (!args->new_ins_taken) + __wt_free(session, args->new_ins); + return (ret); +} + +static inline void +__wt_insert_unpack( + WT_SESSION_IMPL *session, WT_PAGE **pagep, uint32_t *write_genp, + WT_INSERT_HEAD ***insheadp, WT_INSERT ****ins_stackp, WT_INSERT_HEAD + ***new_inslistp, WT_INSERT_HEAD **new_insheadp, WT_INSERT **new_insp, + u_int *skipdepthp) +{ + __wt_insert_args *args = + (__wt_insert_args *)session->wq_args; + + *pagep = args->page; + *write_genp = args->write_gen; + *insheadp = args->inshead; + *ins_stackp = args->ins_stack; + *new_inslistp = args->new_inslist; + *new_insheadp = args->new_inshead; + *new_insp = args->new_ins; + *skipdepthp = args->skipdepth; +} + +static inline void +__wt_insert_new_inslist_taken(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + __wt_insert_args *args = + (__wt_insert_args *)session->wq_args; + + args->new_inslist_taken = 1; + + WT_ASSERT(session, args->new_inslist_size != 0); + __wt_cache_page_inmem_incr(session, page, args->new_inslist_size); +} + +static inline void +__wt_insert_new_inshead_taken(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + __wt_insert_args *args = + (__wt_insert_args *)session->wq_args; + + args->new_inshead_taken = 1; + + WT_ASSERT(session, args->new_inshead_size != 0); + __wt_cache_page_inmem_incr(session, page, args->new_inshead_size); +} + +static inline void +__wt_insert_new_ins_taken(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + __wt_insert_args *args = + (__wt_insert_args *)session->wq_args; + + args->new_ins_taken = 1; + + WT_ASSERT(session, args->new_ins_size != 0); + __wt_cache_page_inmem_incr(session, page, args->new_ins_size); +} + +typedef struct { + WT_PAGE *page; + WT_ROW *row_arg; + WT_IKEY *ikey; +} __wt_row_key_args; + +static inline int +__wt_row_key_serial( + WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW *row_arg, WT_IKEY + *ikey) +{ + __wt_row_key_args _args, *args = &_args; + int ret; + + args->page = page; + + args->row_arg = row_arg; + + args->ikey = ikey; + + ret = __wt_session_serialize_func(session, + WT_SERIAL_FUNC, __wt_row_key_serial_func, args); + + return (ret); +} + +static inline void +__wt_row_key_unpack( + WT_SESSION_IMPL *session, WT_PAGE **pagep, WT_ROW **row_argp, WT_IKEY + **ikeyp) +{ + __wt_row_key_args *args = + (__wt_row_key_args *)session->wq_args; + + *pagep = args->page; + *row_argp = args->row_arg; + *ikeyp = args->ikey; +} + +typedef struct { + WT_PAGE *page; + uint32_t write_gen; + WT_UPDATE **srch_upd; + WT_UPDATE **new_upd; + size_t new_upd_size; + int new_upd_taken; + WT_UPDATE *upd; + size_t upd_size; + int upd_taken; +} __wt_update_args; + +static inline int +__wt_update_serial( + WT_SESSION_IMPL *session, WT_PAGE *page, uint32_t write_gen, WT_UPDATE + **srch_upd, WT_UPDATE ***new_updp, size_t new_upd_size, WT_UPDATE + **updp, size_t upd_size) +{ + __wt_update_args _args, *args = &_args; + int ret; + + args->page = page; + + args->write_gen = write_gen; + + args->srch_upd = srch_upd; + + if (new_updp == NULL) + args->new_upd = NULL; + else { + args->new_upd = *new_updp; + *new_updp = NULL; + args->new_upd_size = new_upd_size; + } + args->new_upd_taken = 0; + + if (updp == NULL) + args->upd = NULL; + else { + args->upd = *updp; + *updp = NULL; + args->upd_size = upd_size; + } + args->upd_taken = 0; + + ret = __wt_session_serialize_func(session, + WT_SERIAL_FUNC, __wt_update_serial_func, args); + + if (!args->new_upd_taken) + __wt_free(session, args->new_upd); + if (!args->upd_taken) + __wt_free(session, args->upd); + return (ret); +} + +static inline void +__wt_update_unpack( + WT_SESSION_IMPL *session, WT_PAGE **pagep, uint32_t *write_genp, + WT_UPDATE ***srch_updp, WT_UPDATE ***new_updp, WT_UPDATE **updp) +{ + __wt_update_args *args = + (__wt_update_args *)session->wq_args; + + *pagep = args->page; + *write_genp = args->write_gen; + *srch_updp = args->srch_upd; + *new_updp = args->new_upd; + *updp = args->upd; +} + +static inline void +__wt_update_new_upd_taken(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + __wt_update_args *args = + (__wt_update_args *)session->wq_args; + + args->new_upd_taken = 1; + + WT_ASSERT(session, args->new_upd_size != 0); + __wt_cache_page_inmem_incr(session, page, args->new_upd_size); +} + +static inline void +__wt_update_upd_taken(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + __wt_update_args *args = + (__wt_update_args *)session->wq_args; + + args->upd_taken = 1; + + WT_ASSERT(session, args->upd_size != 0); + __wt_cache_page_inmem_incr(session, page, args->upd_size); +} diff --git a/src/include/stat.h b/src/include/stat.h new file mode 100644 index 00000000000..31ead1a84a7 --- /dev/null +++ b/src/include/stat.h @@ -0,0 +1,121 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +struct __wt_stats { + const char *desc; /* text description */ + uint64_t v; /* 64-bit value */ +}; + +#define WT_STAT(stats, fld) \ + (stats)->fld.v +#define WT_STAT_DECR(stats, fld) do { \ + --(stats)->fld.v; \ +} while (0) +#define WT_STAT_INCR(stats, fld) do { \ + ++(stats)->fld.v; \ +} while (0) +#define WT_STAT_DECRV(stats, fld, value) do { \ + (stats)->fld.v -= (value); \ +} while (0) +#define WT_STAT_INCRV(stats, fld, value) do { \ + (stats)->fld.v += (value); \ +} while (0) +#define WT_STAT_SET(stats, fld, value) do { \ + (stats)->fld.v = (uint64_t)(value); \ +} while (0) + +#define WT_BSTAT_INCR(session, fld) \ + WT_STAT_INCR((session)->btree->stats, fld) +#define WT_BSTAT_INCRV(session, fld, v) \ + WT_STAT_INCRV((session)->btree->stats, fld, v) +#define WT_BSTAT_DECR(session, fld) \ + WT_STAT_DECR((session)->btree->stats, fld) +#define WT_BSTAT_SET(session, fld, v) \ + WT_STAT_SET((session)->btree->stats, fld, v) + +#define WT_CSTAT_INCR(session, fld) \ + WT_STAT_INCR(S2C(session)->stats, fld) + +/* + * DO NOT EDIT: automatically built by dist/stat.py. + */ +/* Statistics section: BEGIN */ + +/* + * Statistics entries for BTREE handle. + */ +struct __wt_btree_stats { + WT_STATS file_bulk_loaded; + WT_STATS file_col_deleted; + WT_STATS file_col_fix_pages; + WT_STATS file_col_int_pages; + WT_STATS file_col_var_pages; + WT_STATS cursor_inserts; + WT_STATS cursor_read; + WT_STATS cursor_read_near; + WT_STATS cursor_read_next; + WT_STATS cursor_read_prev; + WT_STATS cursor_removes; + WT_STATS cursor_resets; + WT_STATS cursor_updates; + WT_STATS alloc; + WT_STATS extend; + WT_STATS free; + WT_STATS overflow_read; + WT_STATS page_read; + WT_STATS page_write; + WT_STATS file_size; + WT_STATS file_fixed_len; + WT_STATS file_magic; + WT_STATS file_major; + WT_STATS file_maxintlitem; + WT_STATS file_maxintlpage; + WT_STATS file_maxleafitem; + WT_STATS file_maxleafpage; + WT_STATS file_minor; + WT_STATS file_freelist_bytes; + WT_STATS file_freelist_entries; + WT_STATS file_overflow; + WT_STATS file_allocsize; + WT_STATS rec_page_merge; + WT_STATS rec_split_intl; + WT_STATS rec_split_leaf; + WT_STATS rec_ovfl_key; + WT_STATS rec_ovfl_value; + WT_STATS rec_page_delete; + WT_STATS rec_written; + WT_STATS rec_hazard; + WT_STATS file_row_int_pages; + WT_STATS file_row_leaf_pages; + WT_STATS file_entries; +}; + +/* + * Statistics entries for CONNECTION handle. + */ +struct __wt_connection_stats { + WT_STATS block_read; + WT_STATS block_write; + WT_STATS cache_bytes_inuse; + WT_STATS cache_evict_slow; + WT_STATS cache_evict_internal; + WT_STATS cache_bytes_max; + WT_STATS cache_evict_modified; + WT_STATS cache_pages_inuse; + WT_STATS cache_evict_hazard; + WT_STATS cache_evict_unmodified; + WT_STATS cond_wait; + WT_STATS file_open; + WT_STATS rwlock_rdlock; + WT_STATS rwlock_wrlock; + WT_STATS memalloc; + WT_STATS memfree; + WT_STATS total_read_io; + WT_STATS total_write_io; +}; + +/* Statistics section: END */ diff --git a/src/include/verify_build.h b/src/include/verify_build.h new file mode 100644 index 00000000000..c1a22f63b29 --- /dev/null +++ b/src/include/verify_build.h @@ -0,0 +1,67 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* + * NOTE: If you see a compile failure in this file, your compiler is laying out + * structs in memory in a way WiredTiger does not expect. Please refer to the + * build instructions in the documentation (docs/html/install.html) for more + * information. + */ + +/* + * Compile time assertions. + * + * If the argument to STATIC_ASSERT is zero, the macro evaluates to: + * + * (void)sizeof(char[-1]) + * + * which fails to compile (which is what we want, the assertion failed). + * If the value of the argument to STATIC_ASSERT is non-zero, then the macro + * evaluates to: + * + * (void)sizeof(char[1]); + * + * which compiles with no warnings, and produces no code. + * + * For more details about why this works, see + * http://scaryreasoner.wordpress.com/2009/02/28/ + */ +#define STATIC_ASSERT(cond) (void)sizeof(char[1 - 2 * !(cond)]) + +#define SIZE_CHECK(type, e) do { \ + char __check_##type[1 - 2 * !(sizeof(type) == (e))]; \ + (void)__check_##type; \ +} while (0) + +#define ALIGN_CHECK(type, a) \ + STATIC_ASSERT(WT_ALIGN(sizeof(type), (a)) == sizeof(type)) + +static inline void +__wt_verify_build(void) +{ + /* On-disk structures should not be padded. */ + SIZE_CHECK(WT_BLOCK_DESC, WT_BLOCK_DESC_SIZE); + + /* + * We mix-and-match 32-bit unsigned values and size_t's, mostly because + * we allocate and handle 32-bit objects, and lots of the underlying C + * library expects size_t values for the length of memory objects. We + * check, just to be sure. + */ + STATIC_ASSERT(sizeof(size_t) >= sizeof(uint32_t)); + + /* + * We require an off_t fit into an 8B chunk because 8B is the largest + * integral value we can encode into an address cookie. + * + * WiredTiger has never been tested on a system with 4B off_t types, + * disallow them for now. + */ + STATIC_ASSERT(sizeof(off_t) == sizeof(int64_t)); +} + +#undef ALIGN_CHECK diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in new file mode 100644 index 00000000000..802d17695f7 --- /dev/null +++ b/src/include/wiredtiger.in @@ -0,0 +1,1644 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#ifndef __WIREDTIGER_H_ +#define __WIREDTIGER_H_ + +#if defined(__cplusplus) +extern "C" { +#endif + +/******************************************* + * Version information + *******************************************/ +#define WIREDTIGER_VERSION_MAJOR @VERSION_MAJOR@ +#define WIREDTIGER_VERSION_MINOR @VERSION_MINOR@ +#define WIREDTIGER_VERSION_PATCH @VERSION_PATCH@ +#define WIREDTIGER_VERSION_STRING @VERSION_STRING@ + +/******************************************* + * Required includes + *******************************************/ +@wiredtiger_includes_decl@ + +/******************************************* + * Portable type names + *******************************************/ +@int8_decl@ +@u_int8_decl@ +@int16_decl@ +@u_int16_decl@ +@int32_decl@ +@u_int32_decl@ +@int64_decl@ +@u_int64_decl@ + +@u_char_decl@ +@u_short_decl@ +@u_int_decl@ +@u_long_decl@ +@u_quad_decl@ + +@uintmax_t_decl@ +@uintptr_t_decl@ + +#if defined(DOXYGEN) || defined(SWIG) +#define __F(func) func +#else +#define __F(func) (*func) +#endif + +#ifdef SWIG +%{ +#include <wiredtiger.h> +%} +#endif + +/*! @defgroup wt WiredTiger API + * The functions, handles and methods applications use to access and manage + * data with WiredTiger. + * + * @{ + */ + +/******************************************* + * Public forward structure declarations + *******************************************/ +struct __wt_collator; typedef struct __wt_collator WT_COLLATOR; +struct __wt_compressor; typedef struct __wt_compressor WT_COMPRESSOR; +struct __wt_connection; typedef struct __wt_connection WT_CONNECTION; +struct __wt_cursor; typedef struct __wt_cursor WT_CURSOR; +struct __wt_cursor_type; typedef struct __wt_cursor_type WT_CURSOR_TYPE; +struct __wt_event_handler; typedef struct __wt_event_handler WT_EVENT_HANDLER; +struct __wt_extension_api; typedef struct __wt_extension_api WT_EXTENSION_API; +struct __wt_extractor; typedef struct __wt_extractor WT_EXTRACTOR; +struct __wt_item; typedef struct __wt_item WT_ITEM; +struct __wt_session; typedef struct __wt_session WT_SESSION; + +/*! + * A raw item of data to be managed. Data items have a pointer to the data and + * a length (limited to 4GB for items stored in tables). + */ +struct __wt_item { + /*! + * The memory reference of the data item. + * + * For items returned by a WT_CURSOR, the pointer is only valid until + * the next operation on that cursor. Applications that need to keep + * an item across multiple cursor operations must make a copy. + */ + const void *data; + + /*! + * The number of bytes in the data item. + */ + uint32_t size; + +#ifndef DOXYGEN +#define WT_ITEM_INUSE 0x00000001 + /* This appears in the middle of the struct to avoid padding. */ + /*! Object flags (internal use). */ + uint32_t flags; + + /*! Managed memory chunk (internal use). */ + void *mem; + /*! Managed memory size (internal use). */ + size_t memsize; +#endif +}; + +/*! + * The maximum packed size of a 64-bit integer. The ::wiredtiger_struct_pack + * function will pack single long integers into at most this many bytes. + */ +#define WT_INTPACK64_MAXSIZE ((int)sizeof (int64_t) + 1) + +/*! + * The maximum packed size of a 32-bit integer. The ::wiredtiger_struct_pack + * function will pack single integers into at most this many bytes. + */ +#define WT_INTPACK32_MAXSIZE ((int)sizeof (int32_t) + 1) + +/*! + * A WT_CURSOR handle is the interface to a cursor. + * + * Cursors allow data to be searched, stepped through and updated: the + * so-called CRUD operations (create, read, update and delete). + * + * Raw data is represented by key/value pairs of WT_ITEM structures, but + * cursors can also provide access to fields within the key and value if the + * formats are described in the WT_SESSION::create method. + * + * A cursor can be positioned in a collection of data. Cursors are opened in + * the context of a session (which may have an associated transaction), and can + * query and update records. In the common case, a cursor is used to access + * records in a table. However, cursors can be used on subsets of tables (such + * as a single column or a projection of multiple columns), as an interface to + * statistics, configuration data or application-specific data sources. See + * WT_SESSION::open_cursor for more information. + * + * <b>Thread safety:</b> A WT_CURSOR handle cannot be shared between threads: + * it may only be used within the same thread as the encapsulating WT_SESSION. + */ +struct __wt_cursor { + WT_SESSION *session; /*!< The session handle for this cursor. */ + + /*! + * The name of the data source for the cursor, matches the \c uri + * parameter to WT_SESSION::open_cursor used to open the cursor. + */ + const char *uri; + + /*! + * The format of the data packed into key items. See @ref packing for + * details. If not set, a default value of "u" is assumed, and + * applications must use WT_ITEM structures to manipulate untyped byte + * arrays. + */ + const char *key_format; + + /*! + * The format of the data packed into value items. See @ref packing + * for details. If not set, a default value of "u" is assumed, and + * applications must use WT_ITEM structures to manipulate untyped byte + * arrays. + */ + const char *value_format; + + /*! @name Data access + * @{ + */ + /*! Get the key for the current record. + * + * @snippet ex_all.c Get the cursor's string key + * + * @snippet ex_all.c Get the cursor's record number key + * + * @param cursor the cursor handle + * @errors + */ + int __F(get_key)(WT_CURSOR *cursor, ...); + + /*! Get the value for the current record. + * + * @snippet ex_all.c Get the cursor's string value + * + * @snippet ex_all.c Get the cursor's raw value + * + * @param cursor the cursor handle + * @errors + */ + int __F(get_value)(WT_CURSOR *cursor, ...); + + /*! Set the key for the next operation. + * + * @snippet ex_all.c Set the cursor's string key + * + * @snippet ex_all.c Set the cursor's record number key + * + * @param cursor the cursor handle + * + * If an error occurs during this operation, a flag will be set in the + * cursor, and the next operation to access the key will fail. This + * simplifies error handling in applications. + */ + void __F(set_key)(WT_CURSOR *cursor, ...); + + /*! Set the value for the next operation. + * + * @snippet ex_all.c Set the cursor's string value + * + * @snippet ex_all.c Set the cursor's raw value + * + * @param cursor the cursor handle + * + * If an error occurs during this operation, a flag will be set in the + * cursor, and the next operation to access the value will fail. This + * simplifies error handling in applications. + */ + void __F(set_value)(WT_CURSOR *cursor, ...); + /*! @} */ + + /*! @name Cursor positioning + * @{ + */ + /*! Test whether two cursors refer to the same item. To be equal, + * both cursors must have the same data source, have valid keys, and + * the keys must be equal. + * + * @snippet ex_all.c Test cursor equality + * + * @param cursor the cursor handle + * @param other another cursor handle + * @returns true (non-zero) if both cursors reference the same item, + * false (zero) otherwise + */ + int __F(equals)(WT_CURSOR *cursor, WT_CURSOR *other); + + /*! Return the next record. + * + * @snippet ex_all.c Return the next record + * + * @param cursor the cursor handle + * @errors + */ + int __F(next)(WT_CURSOR *cursor); + + /*! Return the previous record. + * + * @snippet ex_all.c Return the previous record + * + * @param cursor the cursor handle + * @errors + */ + int __F(prev)(WT_CURSOR *cursor); + + /*! Reset the position of the cursor. Any resources held by the cursor + * are released, and the cursor position is no longer valid. Subsequent + * iteration with WT_CURSOR::next will move to the first record, or + * with WT_CURSOR::prev will move to the last record. + * + * @snippet ex_all.c Reset the cursor + * + * @param cursor the cursor handle + * @errors + */ + int __F(reset)(WT_CURSOR *cursor); + + /*! Move to the record matching the key. The key must first be set. + * + * @snippet ex_all.c Search for an exact match + * + * @param cursor the cursor handle + * @errors + */ + int __F(search)(WT_CURSOR *cursor); + + /*! Move to the record matching the key if it exists, or a record that + * would be adjacent. Either the smallest record larger than the key + * or the largest record smaller than the key (in other words, a + * logically adjacent key). The key must first be set. + * + * @snippet ex_all.c Search for an exact or adjacent match + * + * @snippet ex_all.c Forward scan greater than or equal + * + * @snippet ex_all.c Backward scan less than + * + * @param cursor the cursor handle + * @param exactp the status of the search: 0 if an exact match is + * found, < 0 if a smaller key is returned, > 0 if a larger key is + * returned + * @errors + */ + int __F(search_near)(WT_CURSOR *cursor, int *exactp); + /*! @} */ + + /*! @name Data modification + * @{ + */ + /*! Insert a record, and optionally overwrite an existing record. + * If the cursor was not configured with "append" or "overwrite", both + * the key and value must be set and the record must not already exist; + * the record will be inserted. + * + * If the cursor was configured with "overwrite", both the key and value + * must be set; if the record already exists, the key's value will be + * updated, otherwise, the record will be inserted. + * + * In a cursor with record number keys was configured with "append", + * the value must be set; a new record will be appended and the record + * number set as the cursor key value. + * + * Inserting a new record after the current maximum record in a + * fixed-length column-store implicitly creates skipped records as + * records with a value of 0. + * + * @snippet ex_all.c Insert a new record + * + * @snippet ex_all.c Insert a new record or overwrite an existing record + * + * @snippet ex_all.c Insert a new record and assign a record number + * + * @param cursor the cursor handle + * @errors + */ + int __F(insert)(WT_CURSOR *cursor); + + /*! Update a record. Both key and value must be set, the key must + * exist, and the value of the key's record will be updated. + * + * @snippet ex_all.c Update an existing record + * + * @param cursor the cursor handle + * @errors + * In particular, if no record with the specified key exists, + * ::WT_NOTFOUND is returned. + */ + int __F(update)(WT_CURSOR *cursor); + + /*! Remove a record. The key must be set, and the key's record will be + * removed. + * + * Removing a record in a fixed-length column-store is identical to + * setting the record's value to 0. + * + * @snippet ex_all.c Remove a record + * + * @param cursor the cursor handle + * @errors + * In particular, if no record with the specified key exists, + * ::WT_NOTFOUND is returned. + */ + int __F(remove)(WT_CURSOR *cursor); + /*! @} */ + + /*! Close the cursor. + * + * This releases the resources associated with the cursor handle. + * Cursors are closed implicitly by ending the enclosing transaction or + * closing the session in which they were opened. + * + * @snippet ex_all.c Close the cursor + * + * @param cursor the cursor handle + * @errors + */ + int __F(close)(WT_CURSOR *cursor); + + /* + * Protected fields, only to be used by cursor implementations. + */ + +#if !defined(SWIG) && !defined(DOXYGEN) + /* + * !!! + * Explicit representations of structures from queue.h. + * TAILQ_ENTRY(wt_cursor) q; + */ + struct { + WT_CURSOR *tqe_next; + WT_CURSOR **tqe_prev; + } q; /* Linked list of WT_CURSORs. */ + + uint64_t recno; + uint8_t raw_recno_buf[WT_INTPACK64_MAXSIZE]; + /* Holds a recno in raw mode. */ + WT_ITEM key, value; + int saved_err; /* Saved error in set_{key,value}. */ + +#define WT_CURSTD_APPEND 0x0001 +#define WT_CURSTD_DUMP_HEX 0x0002 +#define WT_CURSTD_DUMP_PRINT 0x0004 +#define WT_CURSTD_FILE 0x0008 +#define WT_CURSTD_KEY_SET 0x0010 +#define WT_CURSTD_OVERWRITE 0x0020 +#define WT_CURSTD_PUBLIC 0x0040 +#define WT_CURSTD_RAW 0x0080 +#define WT_CURSTD_TABLE 0x0100 +#define WT_CURSTD_VALUE_SET 0x0200 + uint32_t flags; +#endif +}; + +/*! + * All data operations are performed in the context of a WT_SESSION. This + * encapsulates the thread and transactional context of the operation. + * + * <b>Thread safety:</b> A WT_SESSION handle cannot be shared between threads: + * it may only be used within a single thread. Each thread accessing a + * database should open a separate WT_SESSION handle. + */ +struct __wt_session { + /*! The connection for this session. */ + WT_CONNECTION *connection; + + /*! Close the session handle. + * + * This will release the resources associated with the session handle, + * including rolling any active transactions and closing any cursors + * that remain open in the session. + * + * @snippet ex_all.c session close + * + * @param session the session handle + * @configempty{session.close, see dist/api_data.py} + * @errors + */ + int __F(close)(WT_SESSION *session, const char *config); + + /*! @name Cursor handles + * @{ + */ + + /*! Open a cursor. + * + * Open a new cursor on the specified data source. Cursor handles + * should be discarded by calling WT_CURSOR::close. + * + * Cursors are light-weight objects but may hold references to + * heavier-weight objects. Applications should usually not cache + * cursors as a performance optimization, instead, a new cursor + * should be opened for new groups of operations. + * + * Cursors are opened in the context of the current transaction. The + * cursor must be closed before the transaction can end. If + * WT_SESSION::commit_transaction or WT_SESSION::rollback_transaction + * are called with cursors open in the transaction, the cursor handles + * will be closed implicitly and must not be accessed again. + * + * An existing cursor can be duplicated by passing it as the \c to_dup + * parameter, otherwise the \c to_dup parameter should be NULL. + * @notyet{to_dup} + * + * @snippet ex_all.c Open a cursor + * + * @param session the session handle + * @param uri the data source on which the cursor operates; + * cursors are usually opened on tables, however, cursors can be + * opened on any data source, regardless of whether it is ultimately + * stored in a table. Some cursor types may have limited + * functionality (for example, be read-only or not support + * transactional updates). See @ref cursor_types for more + * information. + * <br> + * The following are the builtin cursor types: + * <table> + * @hrow{URI, Type, Key/Value types} + * @row{<tt>colgroup:\<tablename\>.\<columnset\></tt>, + * column group cursor, + * table key\, column group value(s)} + * @row{<tt>table:\<tablename\></tt>, + * table cursor, + * table key\, table value(s)} + * @row{<tt>file:\<filename\></tt>, + * file cursor, + * file key\, file value(s)} + * @row{<tt>index:\<tablename\>.\<indexname\></tt>, + * index cursor, + * key=index key\, value=table value(s)} + * @row{<tt>statistics:[file</tt><tt>:\<filename\>]</tt>, + * database or file statistics cursor, + * key=<code>int id</code>\, value=(<code>string description\, + * string value\, uint64_t value</code>)\, + * see @ref cursor_statistics for details} + * </table> + * @param to_dup a cursor to duplicate + * @param session the session handle + * @configstart{session.open_cursor, see dist/api_data.py} + * @config{append, only supported by cursors with record number keys: + * append the value as a new record\, creating a new record number + * key.,a boolean flag; default \c false.} + * @config{bulk, configure the cursor for bulk loads; bulk-load is a + * fast load path for empty objects\, only empty objects may be + * bulk-loaded.,a boolean flag; default \c false.} + * @config{clear_on_close, for statistics cursors\, reset statistics + * counters when the cursor is closed.,a boolean flag; default \c + * false.} + * @config{dump, configure the cursor for dump format inputs and + * outputs: "hex" selects a simple hexadecimal format\, "print" selects + * a format where only non-printing characters are hexadecimal encoded. + * The cursor dump format is compatible with the @ref utility_dump and + * @ref utility_load commands.,a string\, chosen from the following + * options: \c "hex"\, \c "print"; default empty.} + * @config{isolation, the isolation level for this cursor\, ignored for + * transactional cursors.,a string\, chosen from the following options: + * \c "snapshot"\, \c "read-committed"\, \c "read-uncommitted"; default + * \c read-committed.} + * @config{overwrite, change the behavior of the cursor's insert method + * to overwrite previously existing values.,a boolean flag; default \c + * false.} + * @config{raw, ignore the encodings for the key and value\, manage data + * as if the formats were \c "u". See @ref cursor_raw for more + * details.,a boolean flag; default \c false.} + * @config{statistics, configure the cursor for statistics.,a boolean + * flag; default \c false.} + * @configend + * @param cursorp a pointer to the newly opened cursor + * @errors + */ + int __F(open_cursor)(WT_SESSION *session, + const char *uri, WT_CURSOR *to_dup, + const char *config, WT_CURSOR **cursorp); + /*! @} */ + + /*! @name Table operations + * @{ + */ + /*! Create a table, column group, index or file. + * + * @snippet ex_all.c Create a table + * + * @param session the session handle + * @param name the URI of the object to create, such as \c "table:stock" + * @configstart{session.create, see dist/api_data.py} + * @config{allocation_size, the file unit allocation size\, in bytes\, + * must a power-of-two; smaller values decrease the file space required + * by overflow items\, and the default value of 512B is a good choice + * absent requirements from the operating system or storage device.,an + * integer between 512B and 128MB; default \c 512B.} + * @config{block_compressor, configure a compressor for file blocks. + * Permitted values are empty (off) or \c "bzip2"\, \c "snappy" or + * custom compression engine \c "name" created with + * WT_CONNECTION::add_compressor. See @ref compression for more + * information.,a string; default empty.} + * @config{checksum, configure file block checksums; if false\, the + * block manager is free to not write or check block checksums. This can + * increase performance in applications where compression provides + * checksum functionality or read-only applications where blocks require + * no verification.,a boolean flag; default \c true.} + * @config{colgroups, comma-separated list of names of column groups. + * Each column group is stored separately\, keyed by the primary key of + * the table. If no column groups are specified\, all columns are + * stored together in a single file. All value columns in the table + * must appear in at least one column group. Each column group must be + * created with a separate call to WT_SESSION::create.,a list of + * strings; default empty.} + * @config{collator, configure custom collation for keys. Value must be + * a collator name created with WT_CONNECTION::add_collator.,a string; + * default empty.} + * @config{columns, list of the column names. Comma-separated list of + * the form <code>(column[\,...])</code>. For tables\, the number of + * entries must match the total number of values in \c key_format and \c + * value_format. For colgroups and indices\, all column names must + * appear in the list of columns for the table.,a list of strings; + * default empty.} + * @config{exclusive, fail if the object exists. When false (the + * default)\, if the object exists\, check that its settings match the + * specified configuration.,a boolean flag; default \c false.} + * @config{filename, override the default filename derived from the + * object name.,a string; default empty.} + * @config{huffman_key, configure Huffman encoding for keys. Permitted + * values are empty (off)\, \c "english" or \c "<filename>". See @ref + * huffman for more information.,a string; default empty.} + * @config{huffman_value, configure Huffman encoding for values. + * Permitted values are empty (off)\, \c "english" or \c "<filename>". + * See @ref huffman for more information.,a string; default empty.} + * @config{internal_item_max, the maximum key size stored on internal + * nodes\, in bytes. If zero\, a maximum is calculated to permit at + * least 8 keys per internal page.,an integer greater than or equal to + * 0; default \c 0.} + * @config{internal_key_truncate, configure internal key truncation\, + * discarding unnecessary trailing bytes on internal keys.,a boolean + * flag; default \c true.} + * @config{internal_page_max, the maximum page size for internal nodes\, + * in bytes; the size must be a multiple of the allocation size and is + * significant for applications wanting to avoid excessive L2 cache + * misses while searching the tree.,an integer between 512B and 512MB; + * default \c 2KB.} + * @config{key_format, the format of the data packed into key items. + * See @ref schema_format_types for details. By default\, the + * key_format is \c 'u' and applications use WT_ITEM structures to + * manipulate raw byte arrays. By default\, records are stored in + * row-store files: keys of type \c 'r' are record numbers and records + * referenced by record number are stored in column-store files.,a + * format string; default \c u.} + * @config{key_gap, the maximum gap between instantiated keys in a Btree + * leaf page\, constraining the number of keys processed to instantiate + * a random Btree leaf page key.,an integer greater than or equal to 0; + * default \c 10.} + * @config{leaf_item_max, the maximum key or value size stored on leaf + * nodes\, in bytes. If zero\, a size is calculated to permit at least 8 + * items (values or row store keys) per leaf page.,an integer greater + * than or equal to 0; default \c 0.} + * @config{leaf_page_max, the maximum page size for leaf nodes\, in + * bytes; the size must be a multiple of the allocation size\, and is + * significant for applications wanting to maximize sequential data + * transfer from a storage device.,an integer between 512B and 512MB; + * default \c 1MB.} + * @config{prefix_compression, configure row-store format key prefix + * compression.,a boolean flag; default \c true.} + * @config{split_pct, the Btree page split size as a percentage of the + * maximum Btree page size\, that is\, when a Btree page is split\, it + * will be split into smaller pages\, where each page is the specified + * percentage of the maximum Btree page size.,an integer between 25 and + * 100; default \c 75.} + * @config{type, the file type.,a string\, chosen from the following + * options: \c "btree"; default \c btree.} + * @config{value_format, the format of the data packed into value items. + * See @ref schema_format_types for details. By default\, the + * value_format is \c 'u' and applications use a WT_ITEM structure to + * manipulate raw byte arrays. Value items of type 't' are bitfields\, + * and when configured with record number type keys\, will be stored + * using a fixed-length store.,a format string; default \c u.} + * @configend + * @errors + */ + int __F(create)(WT_SESSION *session, + const char *name, const char *config); + + /*! Drop (delete) an object. + * + * @snippet ex_all.c session drop + * + * @param session the session handle + * @param name the URI of the object to drop, such as \c "table:stock" + * @configstart{session.drop, see dist/api_data.py} + * @config{force, return success if the object does not exist.,a boolean + * flag; default \c false.} + * @configend + * @errors + */ + int __F(drop)(WT_SESSION *session, + const char *name, const char *config); + + /*! Rename an object. + * + * @snippet ex_all.c session rename + * + * @param session the session handle + * @param oldname the current URI of the object, such as \c "table:old" + * @param newname the new name of the object, such as \c "table:new" + * @configempty{session.rename, see dist/api_data.py} + * @errors + */ + int __F(rename)(WT_SESSION *session, + const char *oldname, const char *newname, const char *config); + + /*! Salvage a file or table + * + * Salvage rebuilds the file, or files of which a table is comprised, + * discarding any corrupted file blocks. + * + * Previously deleted records may re-appear, and inserted records may + * disappear, when salvage is done, so salvage should not be run + * unless it is known to be necessary. Normally, salvage should be + * called after a file or table has been corrupted, as reported by the + * WT_SESSION::verify method. + * + * Files are rebuilt in place, the salvage method overwrites the + * existing files. + * + * @snippet ex_all.c session salvage + * + * @param session the session handle + * @param name the URI of the file or table to salvage + * @configstart{session.salvage, see dist/api_data.py} + * @config{force, force salvage even of files that do not appear to be + * WiredTiger files.,a boolean flag; default \c false.} + * @configend + * @errors + */ + int __F(salvage)(WT_SESSION *session, + const char *name, const char *config); + + /*! Sync a file or table. + * + * Flush dirty pages from a table to stable storage. Note that not + * all pages are necessarily flushed (pages pinned in memory, or in + * use by other threads of control may not be written until all open + * session handles for the table are closed). + * + * @snippet ex_all.c session sync + * + * @param session the session handle + * @param name the URI of the file or table to sync + * @configempty{session.sync, see dist/api_data.py} + * @errors + */ + int __F(sync)(WT_SESSION *session, + const char *name, const char *config); + + /*! Truncate a file, table or a cursor range. + * + * @snippet ex_all.c session truncate + * + * @snippet ex_all.c session range truncate + * + * @param session the session handle + * @param name the URI of the file or table to truncate + * @param start optional cursor marking the first record discarded; + * if <code>NULL</code>, the truncate starts from the beginning of + * the table + * @param stop optional cursor marking the last record discarded; + * if <code>NULL</code>, the truncate continues to the end of the + * table + * @configempty{session.truncate, see dist/api_data.py} + * @errors + */ + int __F(truncate)(WT_SESSION *session, + const char *name, + WT_CURSOR *start, WT_CURSOR *stop, const char *config); + + /*! Upgrade a file or table. + * + * Upgrade upgrades a file, or the files of which a table is comprised. + * + * @snippet ex_all.c session upgrade + * + * @param session the session handle + * @param name the URI of the file or table to upgrade + * @configempty{session.upgrade, see dist/api_data.py} + * @errors + */ + int __F(upgrade)(WT_SESSION *session, + const char *name, const char *config); + + /*! Verify a file or table. + * + * Verify reports if a file, or the files of which a table is + * comprised, have been corrupted. The WT_SESSION::salvage method + * can be used to repair a corrupted file, + * + * @snippet ex_all.c session verify + * + * @param session the session handle + * @param name the URI of the file or table to verify + * @configempty{session.verify, see dist/api_data.py} + * @errors + */ + int __F(verify)(WT_SESSION *session, + const char *name, const char *config); + /*! @} */ + + /*! @name Transactions + * @{ + */ + /*! Start a transaction in this session. @notyet{transactions} + * + * All cursors opened in this session that support transactional + * semantics will operate in the context of the transaction. The + * transaction remains active until ended with + * WT_SESSION::commit_transaction or WT_SESSION::rollback_transaction. + * + * Ignored if a transaction is in progress. + * + * @todo describe nested transactions / savepoints + * + * @snippet ex_all.c session begin transaction + * + * @param session the session handle + * @configstart{session.begin_transaction, see dist/api_data.py} + * @config{isolation, the isolation level for this transaction.,a + * string\, chosen from the following options: \c "serializable"\, \c + * "snapshot"\, \c "read-committed"\, \c "read-uncommitted"; default \c + * read-committed.} + * @config{name, name of the transaction for tracing and debugging.,a + * string; default empty.} + * @config{priority, priority of the transaction for resolving + * conflicts. Transactions with higher values are less likely to + * abort.,an integer between -100 and 100; default \c 0.} + * @config{sync, how to sync log records when the transaction commits.,a + * string\, chosen from the following options: \c "full"\, \c "flush"\, + * \c "write"\, \c "none"; default \c full.} + * @configend + * @errors + */ + int __F(begin_transaction)(WT_SESSION *session, const char *config); + + /*! Commit the current transaction. @notyet{transactions} + * + * Any cursors opened during the transaction will be closed before + * the commit is processed. + * + * Ignored if no transaction is in progress. + * + * @snippet ex_all.c session commit transaction + * + * @param session the session handle + * @configempty{session.commit_transaction, see dist/api_data.py} + * @errors + */ + int __F(commit_transaction)(WT_SESSION *session, const char *config); + + /*! Roll back the current transaction. @notyet{transactions} + * + * Any cursors opened during the transaction will be closed before + * the rollback is processed. + * + * Ignored if no transaction is in progress. + * + * @snippet ex_all.c session rollback transaction + * + * @param session the session handle + * @configempty{session.rollback_transaction, see dist/api_data.py} + * @errors + */ + int __F(rollback_transaction)(WT_SESSION *session, const char *config); + + /*! Flush the cache and/or the log and optionally archive log files. + * @notyet{checkpoint} + * + * @snippet ex_all.c session checkpoint + * + * @param session the session handle + * @configstart{session.checkpoint, see dist/api_data.py} + * @config{archive, remove log files no longer required for + * transactional durability.,a boolean flag; default \c false.} + * @config{flush_cache, flush the cache.,a boolean flag; default \c + * true.} + * @config{flush_log, flush the log to disk.,a boolean flag; default \c + * true.} + * @config{force, write a new checkpoint even if nothing has changed + * since the last one.,a boolean flag; default \c false.} + * @config{log_size, only proceed if more than the specified number of + * bytes of log records have been written since the last checkpoint.,an + * integer greater than or equal to 0; default \c 0.} + * @config{timeout, only proceed if more than the specified number of + * milliseconds have elapsed since the last checkpoint.,an integer + * greater than or equal to 0; default \c 0.} + * @configend + * @errors + */ + int __F(checkpoint)(WT_SESSION *session, const char *config); + + /*! @} */ + /*! @name Debugging + * @{ + */ + + /*! Dump a physical file in debugging mode. + * + * The specified file is displayed in a non-portable debugging mode to + * the application's standard output. + * + * @snippet ex_all.c session dumpfile + * + * @param session the session handle + * @param name the URI of the file to dump + * @configempty{session.dumpfile, see dist/api_data.py} + * @errors + */ + int __F(dumpfile)(WT_SESSION *session, + const char *name, const char *config); + + /*! Send a string to the message handler for debugging. + * + * @snippet ex_all.c session msg_printf + * + * @param session the session handle + * @param fmt a printf-like format specification + * @errors + */ + int __F(msg_printf)(WT_SESSION *session, const char *fmt, ...); + + /*! @} */ +}; + +/*! + * A connection to a WiredTiger database. The connection may be opened within + * the same address space as the caller or accessed over a socket connection. + * + * Most applications will open a single connection to a database for each + * process. The first process to open a connection to a database will access + * the database in its own address space. Subsequent connections (if allowed) + * will communicate with the first process over a socket connection to perform + * their operations. + */ +struct __wt_connection { + /*! Load an extension. + * + * @snippet ex_all.c conn load extension + * + * @param connection the connection handle + * @param path the filename of the extension module + * @configstart{connection.load_extension, see dist/api_data.py} + * @config{entry, the entry point of the extension.,a string; default \c + * wiredtiger_extension_init.} + * @config{prefix, a prefix for all names registered by this extension + * (e.g.\, to make namespaces distinct or during upgrades.,a string; + * default empty.} + * @configend + * @errors + */ + int __F(load_extension)(WT_CONNECTION *connection, + const char *path, const char *config); + + /*! Add a new type of cursor. @notyet{custom cursors} + * + * The application must first implement the WT_CURSOR_TYPE interface + * and then register the implementation with WiredTiger: + * + * @snippet ex_all.c WT_CURSOR_TYPE register + * + * @param connection the connection handle + * @param prefix the prefix for location strings passed to + * WT_SESSION::open_cursor + * @param ctype the application-supplied code to manage cursors of + * this type + * @configempty{connection.add_cursor_type, see dist/api_data.py} + * @errors + */ + int __F(add_cursor_type)(WT_CONNECTION *connection, + const char *prefix, WT_CURSOR_TYPE *ctype, const char *config); + + /*! Add a custom collation function. @notyet{custom collation} + * + * The application must first implement the WT_COLLATOR interface and + * then register the implementation with WiredTiger: + * + * @snippet ex_all.c WT_COLLATOR register + * + * @param connection the connection handle + * @param name the name of the collation to be used in calls to + * WT_SESSION::create + * @param collator the application-supplied collation handler + * @configempty{connection.add_collator, see dist/api_data.py} + * @errors + */ + int __F(add_collator)(WT_CONNECTION *connection, + const char *name, WT_COLLATOR *collator, const char *config); + + /*! Add a compression function. + * + * The application must first implement the WT_COMPRESSOR interface + * and then register the implementation with WiredTiger: + * + * @snippet ex_all.c WT_COMPRESSOR register + * + * @param connection the connection handle + * @param name the name of the compression function to be used in calls + * to WT_SESSION::create + * @param compressor the application-supplied compression handler + * @configempty{connection.add_compressor, see dist/api_data.py} + * @errors + */ + int __F(add_compressor)(WT_CONNECTION *connection, + const char *name, WT_COMPRESSOR *compressor, const char *config); + + /*! Add a custom extractor for index keys or column groups. + * @notyet{custom extractors} + * + * The application must first implement the WT_EXTRACTOR interface and + * then register the implementation with WiredTiger: + * + * @snippet ex_all.c WT_EXTRACTOR register + * + * @param connection the connection handle + * @param name the name of the extractor to be used in calls to + * WT_SESSION::create + * @param extractor the application-supplied extractor + * @configempty{connection.add_extractor, see dist/api_data.py} + * @errors + */ + int __F(add_extractor)(WT_CONNECTION *connection, const char *name, + WT_EXTRACTOR *extractor, const char *config); + + /*! Close a connection. + * + * Any open sessions will be closed. + * + * @snippet ex_all.c conn close + * + * @param connection the connection handle + * @configempty{connection.close, see dist/api_data.py} + * @errors + */ + int __F(close)(WT_CONNECTION *connection, const char *config); + + /*! The home directory of the connection. + * + * @snippet ex_all.c conn get_home + * + * @param connection the connection handle + * @returns a pointer to a string naming the home directory + */ + const char *__F(get_home)(WT_CONNECTION *connection); + + /*! Return if opening this handle created the database. + * + * @snippet ex_all.c is_new + * + * @param connection the connection handle + * @returns false (zero) if the connection existed before the call to + * ::wiredtiger_open, true (non-zero) if it was created by opening this + * handle. + */ + int __F(is_new)(WT_CONNECTION *connection); + + /*! Open a session. + * + * @snippet ex_all.c Open a session + * + * @param connection the connection handle + * @param errhandler An error handler. If <code>NULL</code>, the + * connection's error handler is used + * @configempty{connection.open_session, see dist/api_data.py} + * @param sessionp the new session handle + * @errors + */ + int __F(open_session)(WT_CONNECTION *connection, + WT_EVENT_HANDLER *errhandler, const char *config, + WT_SESSION **sessionp); +}; + +/*! Open a connection to a database. + * + * @snippet ex_all.c Open a connection + * + * @param home The path to the database home directory. See @ref home + * for more information. + * @param errhandler An error handler. If <code>NULL</code>, a builtin error + * handler is installed that writes error messages to stderr + * @configstart{wiredtiger_open, see dist/api_data.py} + * @config{cache_size, maximum heap memory to allocate for the cache.,an integer + * between 1MB and 10TB; default \c 100MB.} + * @config{create, create the database if it does not exist.,a boolean flag; + * default \c false.} + * @config{error_prefix, prefix string for error messages.,a string; default + * empty.} + * @config{eviction_target, continue evicting until the cache becomes less full + * than this (as a percentage). Must be less than \c eviction_trigger.,an + * integer between 10 and 99; default \c 80.} + * @config{eviction_trigger, trigger eviction when the cache becomes this full + * (as a percentage).,an integer between 10 and 99; default \c 95.} + * @config{extensions, list of extensions to load. Optional values are passed + * as the \c config parameter to WT_CONNECTION::load_extension. Complex paths + * may need quoting\, for example\, + * <code>extensions=("/path/to/ext.so"="entry=my_entry")</code>.,a list of + * strings; default empty.} + * @config{hazard_max, number of simultaneous hazard references per session + * handle.,an integer greater than or equal to 15; default \c 30.} + * @config{home_environment, use the \c WIREDTIGER_HOME environment variable for + * naming unless the process is running with special privileges. See @ref home + * for more information.,a boolean flag; default \c false.} + * @config{home_environment_priv, use the \c WIREDTIGER_HOME environment + * variable for naming regardless of whether or not the process is running with + * special privileges. See @ref home for more information.,a boolean flag; + * default \c false.} + * @config{logging, enable logging.,a boolean flag; default \c false.} + * @config{multiprocess, permit sharing between processes (will automatically + * start an RPC server for primary processes and use RPC for secondary + * processes).,a boolean flag; default \c false.} + * @config{session_max, maximum expected number of sessions (including server + * threads).,an integer greater than or equal to 1; default \c 50.} + * @config{transactional, support transactional semantics.,a boolean flag; + * default \c false.} + * @config{verbose, enable messages for various events. Options are given as a + * list\, such as <code>"verbose=[evictserver\,read]"</code>.,a list\, with + * values chosen from the following options: \c "block"\, \c "evict"\, \c + * "evictserver"\, \c "fileops"\, \c "hazard"\, \c "mutex"\, \c "read"\, \c + * "readserver"\, \c "reconcile"\, \c "salvage"\, \c "verify"\, \c "write"; + * default empty.} + * @configend + * Additionally, if a file named \c WiredTiger.config appears in the WiredTiger + * home directory, it is read for configuration values (see @ref config_file + * for details). Configuration values specified in the \c config argument to + * the ::wiredtiger_open function override configuration values specified in + * the \c WiredTiger.config file. + * @param connectionp A pointer to the newly opened connection handle + * @errors + */ +int wiredtiger_open(const char *home, + WT_EVENT_HANDLER *errhandler, const char *config, + WT_CONNECTION **connectionp); + +/*! Return information about an error as a string; wiredtiger_strerror is a + * superset of the ISO C99/POSIX 1003.1-2001 function strerror. + * + * @snippet ex_all.c Display an error + * + * @param err a return value from a WiredTiger, C library or POSIX function + * @returns a string representation of the error + */ +const char *wiredtiger_strerror(int err); + +/*! + * The interface implemented by applications in order to handle error messages, + * information messages and progress. Entries set to NULL are ignored and will + * continue to use the default handlers. + */ +struct __wt_event_handler { + /*! + * Callback to handle error messages; by default, error messages are + * written to the stderr stream. + */ + void (*handle_error)(WT_EVENT_HANDLER *handler, + int error, const char *errmsg); + + /*! + * Callback to handle informational messages; by default, informational + * messages are written to the stdout stream. + */ + int (*handle_message)(WT_EVENT_HANDLER *handler, const char *message); + + /*! + * Callback to handle progress messages; by default, progress messages + * are ignored. + */ + int (*handle_progress)(WT_EVENT_HANDLER *handler, + const char *operation, uint64_t progress); +}; + +/*! Pack a structure into a buffer. + * + * See @ref packing for a description of the permitted format strings. + * + * @section pack_examples Packing Examples + * + * For example, the string <code>"iSh"</code> will pack a 32-bit integer + * followed by a NUL-terminated string, followed by a 16-bit integer. The + * default, big-endian encoding will be used, with no alignment. This could + * be used in C as follows: + * + * @snippet ex_all.c Pack fields into a buffer + * + * Then later, the values can be unpacked as follows: + * + * @snippet ex_all.c Unpack fields from a buffer + * + * @param buffer a pointer to a packed byte array + * @param size the number of valid bytes in the buffer + * @param format the data format, see ::wiredtiger_struct_pack + * @errors + */ +int wiredtiger_struct_pack(void *buffer, size_t size, const char *format, ...); + +/*! Calculate the size required to pack a structure. + * + * Note that for variable-sized fields including variable-sized strings and + * integers, the calculated sized merely reflects the expected sizes specified + * in the format string itself. + * + * @snippet ex_all.c Get the packed size + * + * @param format the data format, see ::wiredtiger_struct_pack + * @returns the number of bytes needed for the matching call to + * ::wiredtiger_struct_pack + */ +size_t wiredtiger_struct_size(const char *format, ...); + +/*! Unpack a structure from a buffer. + * + * Reverse of ::wiredtiger_struct_pack: gets values out of a packed byte string. + * + * @snippet ex_all.c Unpack fields from a buffer + * + * @param buffer a pointer to a packed byte array + * @param size the number of valid bytes in the buffer + * @param format the data format, see ::wiredtiger_struct_pack + * @errors + */ +int wiredtiger_struct_unpack(const void *buffer, size_t size, + const char *format, ...); + +/*! Get version information. + * + * @snippet ex_all.c Get the WiredTiger library version #1 + * @snippet ex_all.c Get the WiredTiger library version #2 + * + * @param majorp a location where the major version number is returned + * @param minorp a location where the minor version number is returned + * @param patchp a location where the patch version number is returned + * @returns a string representation of the version + */ +const char *wiredtiger_version(int *majorp, int *minorp, int *patchp); + +/******************************************* + * Error returns + *******************************************/ +/*! + * @anchor error_returns + * @name Error returns + * Most functions and methods in WiredTiger return an integer code indicating + * whether the operation succeeded or failed. A return of zero indicates + * success, all non-zero return values indicate some kind of failure. + * + * WiredTiger reserves all values from -31,800 to -31,999 to itself as possible + * error values. In addition, C99/POSIX error codes such as \c ENOMEM, + * \c EINVAL and \c ENOTSUP may also be returned, with the usual meanings. + * + * The following are all of the WiredTiger-specific error returns: + * @{ + */ +/* + * DO NOT EDIT: automatically built by dist/api_err.py. + * Error return section: BEGIN + */ +/*! Conflict between concurrent operations. + * This error is generated when an operation cannot be completed due to a + * conflict with concurrent operations. The operation should be retried. If a + * transaction is in progress, it should be rolled back and the operation + * retried in a new transaction. + */ +#define WT_DEADLOCK -31800 +/*! Attempt to insert an existing key. + * This error is generated when the application attempts to insert a record with + * the same key as an existing record without the 'overwrite' configuration to + * WT_SESSION::open_cursor. + */ +#define WT_DUPLICATE_KEY -31801 +/*! Non-specific WiredTiger error. + * This error is generated for cases that are not covered by specific error + * returns. + */ +#define WT_ERROR -31802 +/*! Item not found. + * This return value indicates that a search operation did not find a record + * matching the application's search key. This includes implicit search + * operations in WT_CURSOR::update or WT_CURSOR::remove operations. + */ +#define WT_NOTFOUND -31803 +/*! @cond internal */ +/*! Restart the operation (internal). */ +#define WT_RESTART -31804 +/*! @endcond */ +/* + * Error return section: END + * DO NOT EDIT: automatically built by dist/api_err.py. + */ +/*! @} */ + +/*! @} */ + +/*! @defgroup wt_ext WiredTiger Extension API + * The functions and interfaces applications use to customize and extend the + * behavior of WiredTiger. + * @{ + */ + +/*! + * The interface implemented by applications to provide custom ordering of + * records. + * + * Applications register their implementation with WiredTiger by calling + * WT_CONNECTION::add_collator. + * + * @snippet ex_extending.c add collator nocase + * + * @snippet ex_extending.c add collator prefix10 + */ +struct __wt_collator { + /*! Callback to compare keys. + * + * @param[out] cmp set to -1 if <code>key1 < key2</code>, + * 0 if <code>key1 == key2</code>, + * 1 if <code>key1 > key2</code>. + * @returns zero for success, non-zero to indicate an error. + * + * @snippet ex_all.c Implement WT_COLLATOR + * + * @snippet ex_extending.c case insensitive comparator + * + * @snippet ex_extending.c n character comparator + */ + int (*compare)(WT_COLLATOR *collator, WT_SESSION *session, + const WT_ITEM *key1, const WT_ITEM *key2, int *cmp); +}; + +/*! + * The interface implemented by applications to provide custom compression. + * + * Compressors must implement the WT_COMPRESSOR interface: the + * WT_COMPRESSOR::compress and WT_COMPRESSOR::decompress callbacks must be + * specified, and WT_COMPRESSOR::pre_size is optional. To build your own + * compressor, use one of the compressors in \c ext/compressors as a template: + * \c ext/nop_compress is a simple compressor that passes through data + * unchanged, and is a reasonable starting point. + * + * Applications register their implementation with WiredTiger by calling + * WT_CONNECTION::add_compressor. + * + * @snippet ex_all.c WT_COMPRESSOR register + */ +struct __wt_compressor { + /*! Callback to compress a chunk of data. + * + * WT_COMPRESSOR::compress is given a source buffer and a destination + * buffer, by default of the same size. If the callback can compress + * the buffer to a smaller size in the destination, it does so, sets + * the \c compression_failed return to 0 and returns 0. If compression + * does not produce a smaller result, the callback sets the + * \c compression_failed return to 1 and returns 0. If another + * error occurs, it returns an errno or WiredTiger error code. + * + * On entry, \c src will point to memory, with the length of the memory + * in \c src_len. After successful completion, the callback should + * return \c 0 and set \c result_lenp to the number of bytes required + * for the compressed representation. + * + * If compression would not shrink the data or the \c dst buffer is not + * large enough to hold the compressed data, the callback should set + * \c compression_failed to a non-zero value and return 0. + * + * @param[in] src the data to compress + * @param[in] src_len the length of the data to compress + * @param[in] dst the destination buffer + * @param[in] dst_len the length of the destination buffer + * @param[out] result_lenp the length of the compressed data + * @param[out] compression_failed non-zero if compression did not + * decrease the length of the data (compression may not have completed) + * @returns zero for success, non-zero to indicate an error. + * + * @snippet ex_all.c WT_COMPRESSOR compress + */ + int (*compress)(WT_COMPRESSOR *compressor, WT_SESSION *session, + uint8_t *src, size_t src_len, + uint8_t *dst, size_t dst_len, + size_t *result_lenp, int *compression_failed); + + /*! Callback to decompress a chunk of data. + * + * WT_COMPRESSOR::decompress is given a source buffer and a destination + * buffer. The contents are switched from \c compress: the + * source buffer is the compressed value, and the destination buffer is + * sized to be the original size. If the callback successfully + * decompresses the source buffer to the destination buffer, it returns + * 0. If an error occurs, it returns an errno or WiredTiger error code. + * The source buffer that WT_COMPRESSOR::decompress is given may have a + * size that is rounded up from the size originally produced by + * WT_COMPRESSOR::compress, with the remainder of the buffer set to + * zeroes. Most compressors do not care about this difference if the + * size to be decompressed can be implicitly discovered from the + * compressed data. If your compressor cares, you may need to allocate + * space for, and store, the actual size in the compressed buffer. See + * the source code for the included snappy compressor for an example. + * + * On entry, \c src will point to memory, with the length of the memory + * in \c src_len. After successful completion, the callback should + * return \c 0 and set \c result_lenp to the number of bytes required + * for the decompressed representation. + * + * If the \c dst buffer is not big enough to hold the decompressed + * data, the callback should return an error. + * + * @param[in] src the data to decompress + * @param[in] src_len the length of the data to decompress + * @param[in] dst the destination buffer + * @param[in] dst_len the length of the destination buffer + * @param[out] result_lenp the length of the decompressed data + * @returns zero for success, non-zero to indicate an error. + * + * @snippet ex_all.c WT_COMPRESSOR decompress + */ + int (*decompress)(WT_COMPRESSOR *compressor, WT_SESSION *session, + uint8_t *src, size_t src_len, + uint8_t *dst, size_t dst_len, + size_t *result_lenp); + + /*! Callback to size a destination buffer for compression + * + * WT_COMPRESSOR::pre_size is an optional callback that, given the + * source buffer and size, produces the size of the destination buffer + * to be given to WT_COMPRESSOR::compress. This is useful for + * compressors that assume that the output buffer is sized for the + * worst case and thus no overrun checks are made. If your compressor + * works like this, WT_COMPRESSOR::pre_size will need to be defined. + * See the source code for the snappy compressor for an example. + * However, if your compressor detects and avoids overruns against its + * target buffer, you will not need to define WT_COMPRESSOR::pre_size. + * When WT_COMPRESSOR::pre_size is set to NULL, the destination buffer + * is sized the same as the source buffer. This is always sufficient, + * since a compression result that is larger than the source buffer is + * discarded by WiredTiger. + * + * If not NULL, this callback is called before each call to + * WT_COMPRESS::compress to determine the size of the destination + * buffer to provide. If the callback is NULL, the destination + * buffer will be the same size as the source buffer. + * + * The callback should set \c result_lenp to a suitable buffer size + * for compression, typically the maximum length required by + * WT_COMPRESSOR::compress. + * + * This callback function is for compressors that require an output + * buffer larger than the source buffer (for example, that do not + * check for buffer overflow during compression). + * + * @param[in] src the data to compress + * @param[in] src_len the length of the data to compress + * @param[out] result_lenp the required destination buffer size + * @returns zero for success, non-zero to indicate an error. + * + * @snippet ex_all.c WT_COMPRESSOR presize + */ + int (*pre_size)(WT_COMPRESSOR *compressor, WT_SESSION *session, + uint8_t *src, size_t src_len, size_t *result_lenp); +}; + +/*! + * Applications can extend WiredTiger by providing new implementations of the + * WT_CURSOR class. + * + * <b>Thread safety:</b> WiredTiger may invoke methods on the WT_CURSOR_TYPE + * interface from multiple threads concurrently. It is the responsibility of + * the implementation to protect any shared data. + * + * Applications register their implementation with WiredTiger by calling + * WT_CONNECTION::add_cursor_type. + * + * @snippet ex_all.c WT_CURSOR_TYPE register + */ +struct __wt_cursor_type { + /*! Callback to determine how much space to allocate for a cursor. + * + * If the callback is NULL, no additional space is allocated in the + * WT_CURSOR implementation. + * + * @errors + * + * @snippet ex_all.c WT_CURSOR_TYPE size + */ + int (*cursor_size)(WT_CURSOR_TYPE *ctype, + const char *obj, size_t *sizep); + + /*! Callback to initialize a cursor. + * + * @snippet ex_all.c WT_CURSOR_TYPE init + */ + int (*init_cursor)(WT_CURSOR_TYPE *ctype, + WT_SESSION *session, const char *obj, WT_CURSOR *old_cursor, + const char *config, WT_CURSOR *new_cursor); +}; + +/*! + * The interface implemented by applications to provide custom extraction of + * index keys or column group values. + * + * Applications register implementations with WiredTiger by calling + * WT_CONNECTION::add_extractor. + * + * @snippet ex_all.c WT_EXTRACTOR register + */ +struct __wt_extractor { + /*! Callback to extract a value for an index or column group. + * + * @errors + * + * @snippet ex_all.c WT_EXTRACTOR + */ + int (*extract)(WT_EXTRACTOR *extractor, WT_SESSION *session, + const WT_ITEM *key, const WT_ITEM *value, + WT_ITEM *result); +}; + +/*! Entry point to an extension, implemented by loadable modules. + * + * @param session the session handle + * @param api entry points for WiredTiger functions exported to extensions + * @param config the config string passed to WT_CONNECTION::load_extension + * @errors + */ +extern int wiredtiger_extension_init(WT_SESSION *session, + WT_EXTENSION_API *api, const char *config); + +/*! @} */ + +/******************************************* + * Statistic reference. + *******************************************/ +/*! @addtogroup wt + * @{ + */ +/* + * DO NOT EDIT: automatically built by dist/api_stat.py. + * Statistics section: BEGIN + */ + +/*! + * @name Statistics for connection handles + * @anchor statistics_keys + * @anchor statistics_conn + * Statistics in WiredTiger are accessed through cursors with \c "statistics:" + * URIs. Individual statistics can be queried through the cursor using the + * following keys. + * @{ + */ +/*! blocks read from a file */ +#define WT_STAT_block_read 0 +/*! blocks written to a file */ +#define WT_STAT_block_write 1 +/*! cache: bytes currently held in the cache */ +#define WT_STAT_cache_bytes_inuse 2 +/*! cache: eviction server unable to reach eviction goal */ +#define WT_STAT_cache_evict_slow 3 +/*! cache: internal pages evicted */ +#define WT_STAT_cache_evict_internal 4 +/*! cache: maximum bytes configured */ +#define WT_STAT_cache_bytes_max 5 +/*! cache: modified pages evicted */ +#define WT_STAT_cache_evict_modified 6 +/*! cache: pages currently held in the cache */ +#define WT_STAT_cache_pages_inuse 7 +/*! cache: pages selected for eviction not evicted because of a hazard + * reference */ +#define WT_STAT_cache_evict_hazard 8 +/*! cache: unmodified pages evicted */ +#define WT_STAT_cache_evict_unmodified 9 +/*! condition wait calls */ +#define WT_STAT_cond_wait 10 +/*! files currently open */ +#define WT_STAT_file_open 11 +/*! rwlock readlock calls */ +#define WT_STAT_rwlock_rdlock 12 +/*! rwlock writelock calls */ +#define WT_STAT_rwlock_wrlock 13 +/*! total memory allocations */ +#define WT_STAT_memalloc 14 +/*! total memory frees */ +#define WT_STAT_memfree 15 +/*! total read I/Os */ +#define WT_STAT_total_read_io 16 +/*! total write I/Os */ +#define WT_STAT_total_write_io 17 + +/*! + * @} + * @name Statistics for file objects + * @anchor statistics_file + * @{ + */ +/*! bulk-loaded entries */ +#define WT_STAT_file_bulk_loaded 0 +/*! column-store deleted values */ +#define WT_STAT_file_col_deleted 1 +/*! column-store fixed-size leaf pages */ +#define WT_STAT_file_col_fix_pages 2 +/*! column-store internal pages */ +#define WT_STAT_file_col_int_pages 3 +/*! column-store variable-size leaf pages */ +#define WT_STAT_file_col_var_pages 4 +/*! cursor-inserts */ +#define WT_STAT_cursor_inserts 5 +/*! cursor-read */ +#define WT_STAT_cursor_read 6 +/*! cursor-read-near */ +#define WT_STAT_cursor_read_near 7 +/*! cursor-read-next */ +#define WT_STAT_cursor_read_next 8 +/*! cursor-read-prev */ +#define WT_STAT_cursor_read_prev 9 +/*! cursor-removes */ +#define WT_STAT_cursor_removes 10 +/*! cursor-resets */ +#define WT_STAT_cursor_resets 11 +/*! cursor-updates */ +#define WT_STAT_cursor_updates 12 +/*! file: block allocations */ +#define WT_STAT_alloc 13 +/*! file: block allocations required file extension */ +#define WT_STAT_extend 14 +/*! file: block frees */ +#define WT_STAT_free 15 +/*! file: overflow pages read from the file */ +#define WT_STAT_overflow_read 16 +/*! file: pages read from the file */ +#define WT_STAT_page_read 17 +/*! file: pages written to the file */ +#define WT_STAT_page_write 18 +/*! file: size */ +#define WT_STAT_file_size 19 +/*! fixed-record size */ +#define WT_STAT_file_fixed_len 20 +/*! magic number */ +#define WT_STAT_file_magic 21 +/*! major version number */ +#define WT_STAT_file_major 22 +/*! maximum internal page item size */ +#define WT_STAT_file_maxintlitem 23 +/*! maximum internal page size */ +#define WT_STAT_file_maxintlpage 24 +/*! maximum leaf page item size */ +#define WT_STAT_file_maxleafitem 25 +/*! maximum leaf page size */ +#define WT_STAT_file_maxleafpage 26 +/*! minor version number */ +#define WT_STAT_file_minor 27 +/*! number of bytes in the freelist */ +#define WT_STAT_file_freelist_bytes 28 +/*! number of entries in the freelist */ +#define WT_STAT_file_freelist_entries 29 +/*! overflow pages */ +#define WT_STAT_file_overflow 30 +/*! page size allocation unit */ +#define WT_STAT_file_allocsize 31 +/*! reconcile: deleted or temporary pages merged */ +#define WT_STAT_rec_page_merge 32 +/*! reconcile: internal pages split */ +#define WT_STAT_rec_split_intl 33 +/*! reconcile: leaf pages split */ +#define WT_STAT_rec_split_leaf 34 +/*! reconcile: overflow key */ +#define WT_STAT_rec_ovfl_key 35 +/*! reconcile: overflow value */ +#define WT_STAT_rec_ovfl_value 36 +/*! reconcile: pages deleted */ +#define WT_STAT_rec_page_delete 37 +/*! reconcile: pages written */ +#define WT_STAT_rec_written 38 +/*! reconcile: unable to acquire hazard reference */ +#define WT_STAT_rec_hazard 39 +/*! row-store internal pages */ +#define WT_STAT_file_row_int_pages 40 +/*! row-store leaf pages */ +#define WT_STAT_file_row_leaf_pages 41 +/*! total entries */ +#define WT_STAT_file_entries 42 +/*! @} */ +/* + * Statistics section: END + * DO NOT EDIT: automatically built by dist/api_stat.py. + */ +/*! @} */ +/*! @} */ + +#undef __F + +#if defined(__cplusplus) +} +#endif +#endif /* __WIREDTIGER_H_ */ diff --git a/src/include/wiredtiger_ext.h b/src/include/wiredtiger_ext.h new file mode 100644 index 00000000000..88ccd8385b0 --- /dev/null +++ b/src/include/wiredtiger_ext.h @@ -0,0 +1,49 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#ifndef __WIREDTIGER_EXT_H_ +#define __WIREDTIGER_EXT_H_ + +#include <wiredtiger.h> + +#if defined(__cplusplus) +extern "C" { +#endif + +/*! @addtogroup wt_ext + * @{ + */ + +/*! Table of WiredTiger extension functions. + * + * This structure is used to avoid the need to link extension modules with the + * library. + */ +struct __wt_extension_api { +/* !!! To maintain backwards compatibility, this structure is append-only. */ + /*! Put an error message on the WiredTiger error stream. */ + void (*err_printf)(WT_SESSION *, const char *fmt, ...); +#define wiredtiger_err_printf wt_api->err_printf + + /*! Allocate short-term use scratch memory. */ + void *(*scr_alloc)(WT_SESSION *, size_t); +#define wiredtiger_scr_alloc wt_api->scr_alloc + + /*! Free short-term use scratch memory. */ + void (*scr_free)(WT_SESSION *, void *); +#define wiredtiger_scr_free wt_api->scr_free +}; + +/*! This global variable must appear in each extension module. */ +extern WT_EXTENSION_API *wt_api; + +/*! @} */ + +#if defined(__cplusplus) +} +#endif +#endif /* __WIREDTIGER_EXT_H_ */ diff --git a/src/include/wt_internal.in b/src/include/wt_internal.in new file mode 100644 index 00000000000..d2b6aa7f96f --- /dev/null +++ b/src/include/wt_internal.in @@ -0,0 +1,192 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#if defined(__cplusplus) +extern "C" { +#endif + +/******************************************* + * WiredTiger public include file, and configuration control. + *******************************************/ +#include "wiredtiger_config.h" +#include "wiredtiger_ext.h" + +/******************************************* + * WiredTiger system include files. + *******************************************/ +#include <sys/stat.h> +#include <sys/uio.h> + +#include <ctype.h> +#include <dlfcn.h> +#include <errno.h> +#include <fcntl.h> +#include <inttypes.h> +#include <limits.h> +#include <pthread.h> +#include <stddef.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +/******************************************* + * WiredTiger externally maintained include files. + *******************************************/ +#include "queue.h" + +/******************************************* + * Forward structure declarations for internal structures. + *******************************************/ +/* + * DO NOT EDIT: automatically built by dist/s_typedef. + * Forward structure declarations for internal structures: BEGIN + */ +struct __wt_addr; + typedef struct __wt_addr WT_ADDR; +struct __wt_block; + typedef struct __wt_block WT_BLOCK; +struct __wt_block_desc; + typedef struct __wt_block_desc WT_BLOCK_DESC; +struct __wt_block_header; + typedef struct __wt_block_header WT_BLOCK_HEADER; +struct __wt_btree; + typedef struct __wt_btree WT_BTREE; +struct __wt_btree_session; + typedef struct __wt_btree_session WT_BTREE_SESSION; +struct __wt_btree_stats; + typedef struct __wt_btree_stats WT_BTREE_STATS; +struct __wt_cache; + typedef struct __wt_cache WT_CACHE; +struct __wt_cell; + typedef struct __wt_cell WT_CELL; +struct __wt_cell_unpack; + typedef struct __wt_cell_unpack WT_CELL_UNPACK; +struct __wt_col; + typedef struct __wt_col WT_COL; +struct __wt_col_rle; + typedef struct __wt_col_rle WT_COL_RLE; +struct __wt_condvar; + typedef struct __wt_condvar WT_CONDVAR; +struct __wt_config; + typedef struct __wt_config WT_CONFIG; +struct __wt_config_item; + typedef struct __wt_config_item WT_CONFIG_ITEM; +struct __wt_connection_impl; + typedef struct __wt_connection_impl WT_CONNECTION_IMPL; +struct __wt_connection_stats; + typedef struct __wt_connection_stats WT_CONNECTION_STATS; +struct __wt_cursor_btree; + typedef struct __wt_cursor_btree WT_CURSOR_BTREE; +struct __wt_cursor_bulk; + typedef struct __wt_cursor_bulk WT_CURSOR_BULK; +struct __wt_cursor_config; + typedef struct __wt_cursor_config WT_CURSOR_CONFIG; +struct __wt_cursor_index; + typedef struct __wt_cursor_index WT_CURSOR_INDEX; +struct __wt_cursor_stat; + typedef struct __wt_cursor_stat WT_CURSOR_STAT; +struct __wt_cursor_table; + typedef struct __wt_cursor_table WT_CURSOR_TABLE; +struct __wt_dlh; + typedef struct __wt_dlh WT_DLH; +struct __wt_evict_list; + typedef struct __wt_evict_list WT_EVICT_LIST; +struct __wt_evict_req; + typedef struct __wt_evict_req WT_EVICT_REQ; +struct __wt_fh; + typedef struct __wt_fh WT_FH; +struct __wt_free; + typedef struct __wt_free WT_FREE; +struct __wt_hazard; + typedef struct __wt_hazard WT_HAZARD; +struct __wt_ikey; + typedef struct __wt_ikey WT_IKEY; +struct __wt_insert; + typedef struct __wt_insert WT_INSERT; +struct __wt_insert_head; + typedef struct __wt_insert_head WT_INSERT_HEAD; +struct __wt_named_collator; + typedef struct __wt_named_collator WT_NAMED_COLLATOR; +struct __wt_named_compressor; + typedef struct __wt_named_compressor WT_NAMED_COMPRESSOR; +struct __wt_page; + typedef struct __wt_page WT_PAGE; +struct __wt_page_header; + typedef struct __wt_page_header WT_PAGE_HEADER; +struct __wt_page_modify; + typedef struct __wt_page_modify WT_PAGE_MODIFY; +struct __wt_page_track; + typedef struct __wt_page_track WT_PAGE_TRACK; +struct __wt_process; + typedef struct __wt_process WT_PROCESS; +struct __wt_ref; + typedef struct __wt_ref WT_REF; +struct __wt_row; + typedef struct __wt_row WT_ROW; +struct __wt_rwlock; + typedef struct __wt_rwlock WT_RWLOCK; +struct __wt_salvage_cookie; + typedef struct __wt_salvage_cookie WT_SALVAGE_COOKIE; +struct __wt_session_impl; + typedef struct __wt_session_impl WT_SESSION_IMPL; +struct __wt_size; + typedef struct __wt_size WT_SIZE; +struct __wt_stats; + typedef struct __wt_stats WT_STATS; +struct __wt_table; + typedef struct __wt_table WT_TABLE; +struct __wt_update; + typedef struct __wt_update WT_UPDATE; +/* + * Forward structure declarations for internal structures: END + * DO NOT EDIT: automatically built by dist/s_typedef. + */ + +/******************************************* + * WiredTiger internal include files. + *******************************************/ +#include "posix.h" +#include "misc.h" +#include "mutex.h" + +#include "block.h" +#include "btmem.h" +#include "btree.h" +#include "cache.h" +#include "config.h" +#include "dlh.h" +#include "error.h" +#include "log.h" +#include "os.h" +#include "stat.h" + +#include "api.h" +#include "cursor.h" +#include "schema.h" + +#include "extern.h" +#include "verify_build.h" + +/* Required by cell.i */ +#include "intpack.i" +#include "cell.i" + +#include "bitstring.i" +#include "btree.i" +#include "cache.i" +#include "column.i" +#include "cursor.i" +#include "log.i" +#include "mutex.i" +#include "packing.i" +#include "progress.i" +#include "serial.i" +#include "serial_funcs.i" + +#if defined(__cplusplus) +} +#endif diff --git a/src/log/log.c b/src/log/log.c new file mode 100644 index 00000000000..45de4d7beaa --- /dev/null +++ b/src/log/log.c @@ -0,0 +1,91 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static int +__log_record_size(WT_SESSION_IMPL *session, + WT_LOGREC_DESC *recdesc, va_list ap, size_t *sizep) +{ + WT_UNUSED(session); + + *sizep = wiredtiger_struct_size(recdesc->fmt, ap); + return (0); +} + +int +__wt_log_put(WT_SESSION_IMPL *session, WT_LOGREC_DESC *recdesc, ...) +{ + WT_ITEM *buf; + va_list ap; + size_t size; + int ret; + + buf = &session->logrec_buf; + + va_start(ap, recdesc); + WT_ERR(__log_record_size(session, recdesc, ap, &size)); + va_end(ap); + + WT_RET(__wt_buf_initsize(session, buf, size)); + + va_start(ap, recdesc); + WT_ERR(__wt_struct_packv(session, buf->mem, size, recdesc->fmt, ap)); +err: va_end(ap); + + return (ret); +} + +int +__wt_log_vprintf(WT_SESSION_IMPL *session, const char *fmt, va_list ap) +{ + WT_CONNECTION_IMPL *conn; + WT_ITEM *buf; + va_list ap_copy; + size_t len; + + conn = S2C(session); + + if (conn->log_fh == NULL) + return (0); + + buf = &session->logprint_buf; + + va_copy(ap_copy, ap); + len = (size_t)vsnprintf(NULL, 0, fmt, ap_copy) + 2; + va_end(ap_copy); + + WT_RET(__wt_buf_initsize(session, buf, len)); + + (void)vsnprintf(buf->mem, len, fmt, ap); + + /* + * For now, just dump the text into the file. Later, we will use + * __wt_logput_debug to wrap this in a log header. + */ +#if 1 + strcpy((char *)buf->mem + len - 2, "\n"); + return ((write(conn->log_fh->fd, buf->mem, len - 1) == + (ssize_t)len - 1) ? 0 : WT_ERROR); +#else + return (__wt_logput_debug(session, (char *)buf->mem)); +#endif +} + +int +__wt_log_printf(WT_SESSION_IMPL *session, const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 2, 3))) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = __wt_log_vprintf(session, fmt, ap); + va_end(ap); + + return (ret); +} diff --git a/src/log/log_desc.c b/src/log/log_desc.c new file mode 100644 index 00000000000..5ab03ac98ee --- /dev/null +++ b/src/log/log_desc.c @@ -0,0 +1,9 @@ +/* DO NOT EDIT: automatically built by dist/log.py. */ + +#include "wt_internal.h" + +WT_LOGREC_DESC +__wt_logdesc_debug = +{ + "S", { "message", NULL } +}; diff --git a/src/os_posix/os_abort.c b/src/os_posix/os_abort.c new file mode 100644 index 00000000000..810455453fd --- /dev/null +++ b/src/os_posix/os_abort.c @@ -0,0 +1,25 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_abort -- + * Abort the process, dropping core. + */ +void +__wt_abort(WT_SESSION_IMPL *session) +{ + __wt_errx(session, "aborting WiredTiger library"); + +#ifdef HAVE_DIAGNOSTIC + __wt_attach(session); +#endif + + abort(); + /* NOTREACHED */ +} diff --git a/src/os_posix/os_alloc.c b/src/os_posix/os_alloc.c new file mode 100644 index 00000000000..c4f9dd97cc4 --- /dev/null +++ b/src/os_posix/os_alloc.c @@ -0,0 +1,168 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * There's no malloc interface, WiredTiger never calls malloc. + * + * The problem is an application might allocate memory, write secret stuff in + * it, free the memory, then WiredTiger allocates the memory and uses it for a + * file page or log record, then writes it to disk, without having overwritten + * it fully. That results in the secret stuff being protected by WiredTiger's + * permission mechanisms, potentially inappropriate for the secret stuff. + */ + +/* + * __wt_calloc -- + * ANSI calloc function. + */ +int +__wt_calloc(WT_SESSION_IMPL *session, size_t number, size_t size, void *retp) +{ + void *p; + + /* + * !!! + * This function MUST handle a NULL WT_SESSION_IMPL handle. + */ + WT_ASSERT(session, number != 0 && size != 0); + + if (session != NULL && S2C(session)->stats != NULL) + WT_CSTAT_INCR(session, memalloc); + + if ((p = calloc(number, size)) == NULL) + WT_RET_MSG(session, __wt_errno(), "memory allocation"); + + *(void **)retp = p; + return (0); +} + +/* + * __wt_realloc -- + * ANSI realloc function. + */ +int +__wt_realloc(WT_SESSION_IMPL *session, + size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp) +{ + void *p; + size_t bytes_allocated; + + /* + * !!! + * This function MUST handle a NULL WT_SESSION_IMPL handle. + */ + WT_ASSERT(session, bytes_to_allocate != 0); + + if (session != NULL && S2C(session)->stats != NULL) + WT_CSTAT_INCR(session, memalloc); + + p = *(void **)retp; + + /* + * Sometimes we're allocating memory and we don't care about the + * final length -- bytes_allocated_ret may be NULL. + */ + bytes_allocated = + bytes_allocated_ret == NULL ? 0 : *bytes_allocated_ret; + WT_ASSERT(session, bytes_allocated < bytes_to_allocate); + + if ((p = realloc(p, bytes_to_allocate)) == NULL) + WT_RET_MSG(session, __wt_errno(), "memory allocation"); + + /* + * Clear the allocated memory -- an application might: allocate memory, + * write secret stuff into it, free the memory, then we re-allocate the + * memory and use it for a file page or log record, and then write it to + * disk. That would result in the secret stuff being protected by the + * WiredTiger permission mechanisms, potentially inappropriate for the + * secret stuff. + */ + memset((uint8_t *) + p + bytes_allocated, 0, bytes_to_allocate - bytes_allocated); + + /* Update caller's bytes allocated value. */ + if (bytes_allocated_ret != NULL) + *bytes_allocated_ret = bytes_to_allocate; + + *(void **)retp = p; + return (0); +} + +/* + * __wt_strndup -- + * Duplicate a string of a given length (and NUL-terminate). + */ +int +__wt_strndup(WT_SESSION_IMPL *session, const char *str, size_t len, void *retp) +{ + void *p; + + if (str == NULL) { + *(void **)retp = NULL; + return (0); + } + + /* + * !!! + * This function MUST handle a NULL WT_SESSION_IMPL handle. + */ + if (session != NULL && S2C(session)->stats != NULL) + WT_CSTAT_INCR(session, memalloc); + + WT_RET(__wt_calloc(session, len + 1, 1, &p)); + + /* + * Don't change this to strncpy, we rely on this function to duplicate + * "strings" that contain nul bytes. + */ + memcpy(p, str, len); + + *(void **)retp = p; + return (0); +} + +/* + * __wt_strdup -- + * ANSI strdup function. + */ +int +__wt_strdup(WT_SESSION_IMPL *session, const char *str, void *retp) +{ + return (__wt_strndup(session, + str, (str == NULL) ? 0 : strlen(str), retp)); +} + +/* + * __wt_free_int -- + * ANSI free function. + */ +void +__wt_free_int(WT_SESSION_IMPL *session, void *p_arg) +{ + void *p; + + /* + * !!! + * This function MUST handle a NULL WT_SESSION_IMPL handle. + */ + if (session != NULL && S2C(session)->stats != NULL) + WT_CSTAT_INCR(session, memfree); + + /* + * If there's a serialization bug we might race with another thread. + * We can't avoid the race (and we aren't willing to flush memory), + * but we minimize the window by clearing the free address atomically, + * hoping a racing thread will see, and won't free, a NULL pointer. + */ + p = *(void **)p_arg; + *(void **)p_arg = NULL; + + if (p != NULL) /* ANSI C free semantics */ + free(p); +} diff --git a/src/os_posix/os_dlopen.c b/src/os_posix/os_dlopen.c new file mode 100644 index 00000000000..caf0c1d3d26 --- /dev/null +++ b/src/os_posix/os_dlopen.c @@ -0,0 +1,81 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_dlopen -- + * Open a dynamic library. + */ +int +__wt_dlopen(WT_SESSION_IMPL *session, const char *path, WT_DLH **dlhp) +{ + WT_DLH *dlh; + int ret; + + WT_RET(__wt_calloc_def(session, 1, &dlh)); + WT_ERR(__wt_strdup(session, path, &dlh->name)); + + if ((dlh->handle = dlopen(path, RTLD_LAZY)) == NULL) + WT_ERR_MSG( + session, __wt_errno(), "dlopen(%s): %s", path, dlerror()); + + *dlhp = dlh; + if (0) { +err: __wt_free(session, dlh->name); + __wt_free(session, dlh); + } + return (ret); +} + +/* + * __wt_dlsym -- + * Lookup a symbol in a dynamic library. + */ +int +__wt_dlsym( + WT_SESSION_IMPL *session, WT_DLH *dlh, const char *name, void *sym_ret) +{ + void *sym; + + if ((sym = dlsym(dlh->handle, name)) == NULL) + WT_RET_MSG(session, __wt_errno(), + "dlsym(%s in %s): %s", name, dlh->name, dlerror()); + + *(void **)sym_ret = sym; + return (0); +} + +/* + * __wt_dlclose -- + * Close a dynamic library + */ +int +__wt_dlclose(WT_SESSION_IMPL *session, WT_DLH *dlh) +{ + int ret; + + ret = 0; + + /* + * FreeBSD dies inside __cxa_finalize when closing handles. + * + * For now, just skip the dlclose: this may leak some resources until + * the process exits, but that is preferable to hard-to-debug crashes + * during exit. + */ +#ifndef __FreeBSD__ + if (dlclose(dlh->handle) != 0) { + ret = __wt_errno(); + __wt_err(session, ret, "dlclose: %s", dlerror()); + } +#endif + + __wt_free(session, dlh->name); + __wt_free(session, dlh); + return (ret); +} diff --git a/src/os_posix/os_errno.c b/src/os_posix/os_errno.c new file mode 100644 index 00000000000..778d114748e --- /dev/null +++ b/src/os_posix/os_errno.c @@ -0,0 +1,22 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_errno -- + * Return errno, or WT_ERROR if errno not set. + */ +int +__wt_errno(void) +{ + /* + * Called when we know an error occurred, and we want the system + * error code, but there's some chance it's not set. + */ + return (errno == 0 ? WT_ERROR : errno); +} diff --git a/src/os_posix/os_exist.c b/src/os_posix/os_exist.c new file mode 100644 index 00000000000..04c395223ff --- /dev/null +++ b/src/os_posix/os_exist.c @@ -0,0 +1,37 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_exist -- + * Return if the file exists. + */ +int +__wt_exist(WT_SESSION_IMPL *session, const char *name, int *existp) +{ + const char *path; + struct stat sb; + int ret; + + WT_RET(__wt_filename(session, name, &path)); + + WT_SYSCALL_RETRY(stat(path, &sb), ret); + + __wt_free(session, path); + + if (ret == 0) { + *existp = 1; + return (0); + } + if (ret == ENOENT) { + *existp = 0; + return (0); + } + + WT_RET_MSG(session, ret, "%s: fstat", name); +} diff --git a/src/os_posix/os_filesize.c b/src/os_posix/os_filesize.c new file mode 100644 index 00000000000..42daf86cd86 --- /dev/null +++ b/src/os_posix/os_filesize.c @@ -0,0 +1,29 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_filesize -- + * Get the size of a file in bytes. + */ +int +__wt_filesize(WT_SESSION_IMPL *session, WT_FH *fh, off_t *sizep) +{ + struct stat sb; + int ret; + + WT_VERBOSE(session, fileops, "%s: fstat", fh->name); + + WT_SYSCALL_RETRY(fstat(fh->fd, &sb), ret); + if (ret == 0) { + *sizep = sb.st_size; + return (0); + } + + WT_RET_MSG(session, ret, "%s: fstat", fh->name); +} diff --git a/src/os_posix/os_flock.c b/src/os_posix/os_flock.c new file mode 100644 index 00000000000..8b317654105 --- /dev/null +++ b/src/os_posix/os_flock.c @@ -0,0 +1,37 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_bytelock -- + * Lock/unlock a byte in a file. + */ +int +__wt_bytelock(WT_FH *fhp, off_t byte, int lock) +{ + struct flock fl; + int ret; + + /* + * WiredTiger requires this function be able to acquire locks past + * the end of file. + * + * Note we're using fcntl(2) locking: all fcntl locks associated with a + * file for a given process are removed when any file descriptor for the + * file is closed by the process, even if a lock was never requested for + * that file descriptor. + */ + fl.l_start = byte; + fl.l_len = 1; + fl.l_type = lock ? F_WRLCK : F_UNLCK; + fl.l_whence = SEEK_SET; + + WT_SYSCALL_RETRY(fcntl(fhp->fd, F_SETLK, &fl), ret); + + return (ret); +} diff --git a/src/os_posix/os_fsync.c b/src/os_posix/os_fsync.c new file mode 100644 index 00000000000..3874510d946 --- /dev/null +++ b/src/os_posix/os_fsync.c @@ -0,0 +1,26 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_fsync -- + * Flush a file handle. + */ +int +__wt_fsync(WT_SESSION_IMPL *session, WT_FH *fh) +{ + int ret; + + WT_VERBOSE(session, fileops, "%s: fsync", fh->name); + + WT_SYSCALL_RETRY(fsync(fh->fd), ret); + if (ret == 0) + return (0); + + WT_RET_MSG(session, ret, "%s fsync error", fh->name); +} diff --git a/src/os_posix/os_ftruncate.c b/src/os_posix/os_ftruncate.c new file mode 100644 index 00000000000..6c98aecd852 --- /dev/null +++ b/src/os_posix/os_ftruncate.c @@ -0,0 +1,26 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_ftruncate -- + * Truncate a file. + */ +int +__wt_ftruncate(WT_SESSION_IMPL *session, WT_FH *fh, off_t len) +{ + int ret; + + WT_SYSCALL_RETRY(ftruncate(fh->fd, len), ret); + if (ret == 0) { + fh->file_size = len; + return (0); + } + + WT_RET_MSG(session, ret, "%s ftruncate error", fh->name); +} diff --git a/src/os_posix/os_mtx.c b/src/os_posix/os_mtx.c new file mode 100644 index 00000000000..f749d5d79f3 --- /dev/null +++ b/src/os_posix/os_mtx.c @@ -0,0 +1,262 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_cond_alloc -- + * Allocate and initialize a condition variable. + */ +int +__wt_cond_alloc(WT_SESSION_IMPL *session, + const char *name, int is_locked, WT_CONDVAR **condp) +{ + WT_CONDVAR *cond; + + /* + * !!! + * This function MUST handle a NULL session handle. + */ + WT_RET(__wt_calloc(session, 1, sizeof(WT_CONDVAR), &cond)); + + /* Initialize the mutex. */ + if (pthread_mutex_init(&cond->mtx, NULL) != 0) + goto err; + + /* Initialize the condition variable to permit self-blocking. */ + if (pthread_cond_init(&cond->cond, NULL) != 0) + goto err; + + cond->name = name; + cond->locked = is_locked; + + *condp = cond; + return (0); + +err: __wt_free(session, cond); + return (WT_ERROR); +} + +/* + * __wt_lock + * Lock a mutex. + */ +void +__wt_cond_wait(WT_SESSION_IMPL *session, WT_CONDVAR *cond) +{ + int ret; + + /* + * !!! + * This function MUST handle a NULL session handle. + */ + if (session != NULL) + WT_VERBOSE( + session, mutex, "lock %s mutex (%p)", cond->name, cond); + + WT_ERR(pthread_mutex_lock(&cond->mtx)); + + /* + * Check pthread_cond_wait() return for EINTR, ETIME and ETIMEDOUT, + * it's known to return these errors on some systems. + */ + while (cond->locked) { + ret = pthread_cond_wait(&cond->cond, &cond->mtx); + if (ret != 0 && + ret != EINTR && +#ifdef ETIME + ret != ETIME && +#endif + ret != ETIMEDOUT) { + (void)pthread_mutex_unlock(&cond->mtx); + goto err; + } + } + + cond->locked = 1; + if (session != NULL) + WT_CSTAT_INCR(session, cond_wait); + + WT_ERR(pthread_mutex_unlock(&cond->mtx)); + return; + +err: __wt_err(session, ret, "mutex lock failed"); + __wt_abort(session); +} + +/* + * __wt_cond_signal -- + * Signal a waiting thread. + */ +void +__wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond) +{ + int ret; + + /* + * !!! + * This function MUST handle a NULL session handle. + */ + if (session != NULL) + WT_VERBOSE( + session, mutex, "signal %s cond (%p)", cond->name, cond); + + ret = 0; + WT_ERR(pthread_mutex_lock(&cond->mtx)); + if (cond->locked) { + cond->locked = 0; + WT_ERR(pthread_cond_signal(&cond->cond)); + } + WT_ERR(pthread_mutex_unlock(&cond->mtx)); + return; + +err: __wt_err(session, ret, "mutex unlock failed"); + __wt_abort(session); +} + +/* + * __wt_cond_destroy -- + * Destroy a condition variable. + */ +int +__wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR *cond) +{ + int ret; + + ret = pthread_cond_destroy(&cond->cond); + WT_TRET(pthread_mutex_destroy(&cond->mtx)); + + __wt_free(session, cond); + + return ((ret == 0) ? 0 : WT_ERROR); + +} + +/* + * __wt_rwlock_alloc -- + * Allocate and initialize a read/write lock. + */ +int +__wt_rwlock_alloc( + WT_SESSION_IMPL *session, const char *name, WT_RWLOCK **rwlockp) +{ + WT_RWLOCK *rwlock; + int ret; + + WT_RET(__wt_calloc(session, 1, sizeof(WT_RWLOCK), &rwlock)); + ret = 0; + WT_ERR_TEST(pthread_rwlock_init(&rwlock->rwlock, NULL), WT_ERROR); + + rwlock->name = name; + *rwlockp = rwlock; + if (0) { +err: __wt_free(session, rwlock); + } + return (ret); +} + +/* + * __wt_readlock + * Get a shared lock. + */ +void +__wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) +{ + int ret; + + WT_VERBOSE(session, mutex, + "readlock %s rwlock (%p)", rwlock->name, rwlock); + + WT_ERR(pthread_rwlock_rdlock(&rwlock->rwlock)); + WT_CSTAT_INCR(session, rwlock_rdlock); + + if (0) { +err: __wt_err(session, ret, "rwlock readlock failed"); + __wt_abort(session); + } +} + +/* + * __wt_try_writelock + * Try to get an exclusive lock, or fail immediately if unavailable. + */ +int +__wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) +{ + int ret; + + WT_VERBOSE(session, mutex, + "try_writelock %s rwlock (%p)", rwlock->name, rwlock); + + if ((ret = pthread_rwlock_trywrlock(&rwlock->rwlock)) == 0) + WT_CSTAT_INCR(session, rwlock_wrlock); + else if (ret != EBUSY) { + __wt_err(session, ret, "rwlock try_writelock failed"); + __wt_abort(session); + } + + return (ret); +} + +/* + * __wt_writelock + * Wait to get an exclusive lock. + */ +void +__wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) +{ + int ret; + + WT_VERBOSE(session, mutex, + "writelock %s rwlock (%p)", rwlock->name, rwlock); + + WT_ERR(pthread_rwlock_wrlock(&rwlock->rwlock)); + WT_CSTAT_INCR(session, rwlock_wrlock); + + if (0) { +err: __wt_err(session, ret, "rwlock writelock failed"); + __wt_abort(session); + } +} + +/* + * __wt_rwunlock -- + * Release a read/write lock. + */ +void +__wt_rwunlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) +{ + int ret; + + WT_VERBOSE(session, mutex, + "unlock %s rwlock (%p)", rwlock->name, rwlock); + + WT_ERR(pthread_rwlock_unlock(&rwlock->rwlock)); + + if (0) { +err: __wt_err(session, ret, "rwlock unlock failed"); + __wt_abort(session); + } +} + +/* + * __wt_rwlock_destroy -- + * Destroy a mutex. + */ +int +__wt_rwlock_destroy(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) +{ + int ret; + + ret = pthread_rwlock_destroy(&rwlock->rwlock); + if (ret == EBUSY) + ret = 0; + WT_ASSERT(session, ret == 0); + __wt_free(session, rwlock); + + return (ret); +} diff --git a/src/os_posix/os_open.c b/src/os_posix/os_open.c new file mode 100644 index 00000000000..b82d88987a0 --- /dev/null +++ b/src/os_posix/os_open.c @@ -0,0 +1,132 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_open -- + * Open a file handle. + */ +int +__wt_open(WT_SESSION_IMPL *session, const char *name, int create, WT_FH **fhp) +{ + const char *path; + WT_CONNECTION_IMPL *conn; + WT_FH *fh; + mode_t mode; + int f, fd, matched, ret; + + conn = S2C(session); + fh = NULL; + fd = -1; + ret = 0; + + WT_VERBOSE(session, fileops, "%s: open", name); + + /* Increment the reference count if we already have the file open. */ + matched = 0; + __wt_spin_lock(session, &conn->spinlock); + TAILQ_FOREACH(fh, &conn->fhqh, q) { + if (strcmp(name, fh->name) == 0) { + ++fh->refcnt; + *fhp = fh; + matched = 1; + break; + } + } + __wt_spin_unlock(session, &conn->spinlock); + if (matched) + return (0); + + WT_RET(__wt_filename(session, name, &path)); + + f = O_RDWR; +#ifdef O_BINARY + /* Windows clones: we always want to treat the file as a binary. */ + f |= O_BINARY; +#endif + if (create) { + f |= O_CREAT; + mode = 0666; + } else + mode = 0; + + WT_SYSCALL_RETRY(((fd = open(path, f, mode)) == -1 ? 1 : 0), ret); + if (ret != 0) + WT_ERR_MSG(session, ret, "%s", name); + + WT_ERR(__wt_calloc(session, 1, sizeof(WT_FH), &fh)); + WT_ERR(__wt_strdup(session, name, &fh->name)); + +#if defined(HAVE_FCNTL) && defined(FD_CLOEXEC) + /* + * Security: + * The application may spawn a new process, and we don't want another + * process to have access to our file handles. There's an obvious + * race here... + */ + if ((f = fcntl(fd, F_GETFD)) == -1 || + fcntl(fd, F_SETFD, f | FD_CLOEXEC) == -1) + WT_ERR_MSG(session, __wt_errno(), "%s: fcntl", name); +#endif + + fh->fd = fd; + fh->refcnt = 1; + + /* Set the file's size. */ + WT_ERR(__wt_filesize(session, fh, &fh->file_size)); + + /* Link onto the environment's list of files. */ + __wt_spin_lock(session, &conn->spinlock); + TAILQ_INSERT_TAIL(&conn->fhqh, fh, q); + __wt_spin_unlock(session, &conn->spinlock); + + *fhp = fh; + + if (0) { +err: if (fh != NULL) { + __wt_free(session, fh->name); + __wt_free(session, fh); + } + if (fd != -1) + (void)close(fd); + } + + __wt_free(session, path); + return (ret); +} + +/* + * __wt_close -- + * Close a file handle. + */ +int +__wt_close(WT_SESSION_IMPL *session, WT_FH *fh) +{ + WT_CONNECTION_IMPL *conn; + int ret; + + conn = S2C(session); + ret = 0; + + if (fh == NULL || fh->refcnt == 0 || --fh->refcnt > 0) + return (0); + + /* Remove from the list and discard the memory. */ + __wt_spin_lock(session, &conn->spinlock); + TAILQ_REMOVE(&conn->fhqh, fh, q); + __wt_spin_unlock(session, &conn->spinlock); + + if (close(fh->fd) != 0) { + ret = __wt_errno(); + __wt_err(session, ret, "%s", fh->name); + } + + __wt_free(session, fh->name); + __wt_free(session, fh); + return (ret); +} diff --git a/src/os_posix/os_priv.c b/src/os_posix/os_priv.c new file mode 100644 index 00000000000..8ec056cbe58 --- /dev/null +++ b/src/os_posix/os_priv.c @@ -0,0 +1,18 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_has_priv -- + * Return if the process has special privileges. + */ +int +__wt_has_priv(void) +{ + return (getuid() == 0 ? 1 : 0); +} diff --git a/src/os_posix/os_remove.c b/src/os_posix/os_remove.c new file mode 100644 index 00000000000..1da53ad1425 --- /dev/null +++ b/src/os_posix/os_remove.c @@ -0,0 +1,48 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_remove -- + * Remove a file. + */ +int +__wt_remove(WT_SESSION_IMPL *session, const char *name) +{ + const char *path; + WT_CONNECTION_IMPL *conn; + WT_FH *fh; + int ret; + + conn = S2C(session); + fh = NULL; + + WT_VERBOSE(session, fileops, "%s: remove", name); + + /* If the file is open, close/free it. */ + __wt_spin_lock(session, &conn->spinlock); + TAILQ_FOREACH(fh, &conn->fhqh, q) { + if (strcmp(name, fh->name) == 0) + break; + } + __wt_spin_unlock(session, &conn->spinlock); + + /* This should be caught at a higher level. */ + WT_ASSERT(session, fh == NULL); + + WT_RET(__wt_filename(session, name, &path)); + + WT_SYSCALL_RETRY(remove(path), ret); + + __wt_free(session, path); + + if (ret == 0) + return (0); + + WT_RET_MSG(session, ret, "%s: remove", name); +} diff --git a/src/os_posix/os_rename.c b/src/os_posix/os_rename.c new file mode 100644 index 00000000000..67aaed1357a --- /dev/null +++ b/src/os_posix/os_rename.c @@ -0,0 +1,34 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_rename -- + * Rename a file. + */ +int +__wt_rename(WT_SESSION_IMPL *session, const char *from, const char *to) +{ + int ret; + const char *from_path, *to_path; + + WT_VERBOSE(session, fileops, "rename %s to %s", from, to); + + WT_RET(__wt_filename(session, from, &from_path)); + WT_RET(__wt_filename(session, to, &to_path)); + + WT_SYSCALL_RETRY(rename(from_path, to_path), ret); + + __wt_free(session, from_path); + __wt_free(session, to_path); + + if (ret == 0) + return (0); + + WT_RET_MSG(session, ret, "rename %s to %s", from, to); +} diff --git a/src/os_posix/os_rw.c b/src/os_posix/os_rw.c new file mode 100644 index 00000000000..8666ebe7eaa --- /dev/null +++ b/src/os_posix/os_rw.c @@ -0,0 +1,54 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_read -- + * Read a chunk. + */ +int +__wt_read(WT_SESSION_IMPL *session, + WT_FH *fh, off_t offset, uint32_t bytes, void *buf) +{ + WT_CSTAT_INCR(session, total_read_io); + + WT_VERBOSE(session, fileops, + "%s: read %" PRIu32 " bytes at offset %" PRIuMAX, + fh->name, bytes, (uintmax_t)offset); + + if (pread(fh->fd, buf, (size_t)bytes, offset) == (ssize_t)bytes) + return (0); + + WT_RET_MSG(session, __wt_errno(), + "%s read error: failed to read %" PRIu32 " bytes at offset %" + PRIuMAX, + fh->name, bytes, (uintmax_t)offset); +} + +/* + * __wt_write -- + * Write a chunk. + */ +int +__wt_write(WT_SESSION_IMPL *session, + WT_FH *fh, off_t offset, uint32_t bytes, const void *buf) +{ + WT_CSTAT_INCR(session, total_write_io); + + WT_VERBOSE(session, fileops, + "%s: write %" PRIu32 " bytes at offset %" PRIuMAX, + fh->name, bytes, (uintmax_t)offset); + + if (pwrite(fh->fd, buf, (size_t)bytes, offset) == (ssize_t)bytes) + return (0); + + WT_RET_MSG(session, __wt_errno(), + "%s write error: failed to write %" PRIu32 " bytes at offset %" + PRIuMAX, + fh->name, bytes, (uintmax_t)offset); +} diff --git a/src/os_posix/os_sleep.c b/src/os_posix/os_sleep.c new file mode 100644 index 00000000000..715869344c1 --- /dev/null +++ b/src/os_posix/os_sleep.c @@ -0,0 +1,23 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_sleep -- + * Pause the thread of control. + */ +void +__wt_sleep(long seconds, long micro_seconds) +{ + struct timeval t; + + t.tv_sec = seconds + micro_seconds / 1000000; + t.tv_usec = (suseconds_t)(micro_seconds % 1000000); + + (void)select(0, NULL, NULL, NULL, &t); +} diff --git a/src/os_posix/os_thread.c b/src/os_posix/os_thread.c new file mode 100644 index 00000000000..4d309f06244 --- /dev/null +++ b/src/os_posix/os_thread.c @@ -0,0 +1,29 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_thread_create -- + * Create a new thread of control. + */ +int +__wt_thread_create(pthread_t *tidret, void *(*func)(void *), void *arg) +{ + /* Spawn a new thread of control. */ + return ((pthread_create(tidret, NULL, func, arg) == 0) ? 0 : WT_ERROR); +} + +/* + * __wt_thread_join -- + * Wait for a thread of control to exit. + */ +int +__wt_thread_join(pthread_t tid) +{ + return (pthread_join(tid, NULL) == 0 ? 0 : WT_ERROR); +} diff --git a/src/os_posix/os_yield.c b/src/os_posix/os_yield.c new file mode 100644 index 00000000000..6c7c7de3918 --- /dev/null +++ b/src/os_posix/os_yield.c @@ -0,0 +1,18 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_yield -- + * Yield the thread of control. + */ +void +__wt_yield(void) +{ + sched_yield(); +} diff --git a/src/packing/packing.c b/src/packing/packing.c new file mode 100644 index 00000000000..7fe751f6d44 --- /dev/null +++ b/src/packing/packing.c @@ -0,0 +1,185 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_struct_check -- + * Check that the specified packing format is valid, and whether it fits + * into a fixed-sized bitfield. + */ +int +__wt_struct_check(WT_SESSION_IMPL *session, + const char *fmt, size_t len, int *fixedp, uint32_t *fixed_lenp) +{ + WT_PACK pack; + WT_PACK_VALUE pv; + int fields, ret; + + WT_CLEAR(pv); /* -Wuninitialized. */ + + WT_RET(__pack_initn(session, &pack, fmt, len)); + + for (fields = 0; (ret = __pack_next(&pack, &pv)) == 0; fields++) + ; + + if (ret != WT_NOTFOUND) + return (ret); + + if (fixedp != NULL && fixed_lenp != NULL) { + if (fields == 0) { + *fixedp = 1; + *fixed_lenp = 0; + } else if (fields == 1 && pv.type == 't') { + *fixedp = 1; + *fixed_lenp = pv.size; + } else + *fixedp = 0; + } + + return (0); +} + +/* + * __wt_struct_sizev -- + * Calculate the size of a packed byte string (va_list version). + */ +size_t +__wt_struct_sizev(WT_SESSION_IMPL *session, const char *fmt, va_list ap) +{ + WT_PACK pack; + WT_PACK_VALUE pv; + size_t total; + + WT_CLEAR(pv); /* -Wuninitialized */ + + if (__pack_init(session, &pack, fmt) != 0) + return ((size_t)-1); + + for (total = 0; __pack_next(&pack, &pv) == 0;) { + WT_PACK_GET(session, pv, ap); + total += __pack_size(session, &pv); + } + return (total); +} + +/* + * __wt_struct_size -- + * Calculate the size of a packed byte string. + */ +size_t +__wt_struct_size(WT_SESSION_IMPL *session, const char *fmt, ...) +{ + va_list ap; + size_t size; + + va_start(ap, fmt); + size = __wt_struct_sizev(session, fmt, ap); + va_end(ap); + + return (size); +} + +/* + * __wt_struct_packv -- + * Pack a byte string (va_list version). + */ +int +__wt_struct_packv(WT_SESSION_IMPL *session, + void *buffer, size_t size, const char *fmt, va_list ap) +{ + WT_PACK pack; + WT_PACK_VALUE pv; + uint8_t *p, *end; + int ret; + + WT_CLEAR(pv); /* -Wuninitialized */ + + WT_RET(__pack_init(session, &pack, fmt)); + + p = buffer; + end = p + size; + + while ((ret = __pack_next(&pack, &pv)) == 0) { + WT_PACK_GET(session, pv, ap); + WT_RET(__pack_write(session, &pv, &p, (size_t)(end - p))); + } + + WT_ASSERT(session, p <= end); + + if (ret != WT_NOTFOUND) + return (ret); + + return (0); +} + +/* + * __wt_struct_pack -- + * Pack a byte string. + */ +int +__wt_struct_pack(WT_SESSION_IMPL *session, + void *buffer, size_t size, const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = __wt_struct_packv(session, buffer, size, fmt, ap); + va_end(ap); + + return (ret); +} + +/* + * __wt_struct_unpackv -- + * Unpack a byte string (va_list version). + */ +int +__wt_struct_unpackv(WT_SESSION_IMPL *session, + const void *buffer, size_t size, const char *fmt, va_list ap) +{ + WT_PACK pack; + WT_PACK_VALUE pv; + const uint8_t *p, *end; + int ret; + + WT_RET(__pack_init(session, &pack, fmt)); + + p = buffer; + end = p + size; + + while ((ret = __pack_next(&pack, &pv)) == 0) { + WT_RET(__unpack_read(session, &pv, &p, (size_t)(end - p))); + WT_UNPACK_PUT(session, pv, ap); + } + + WT_ASSERT(session, p <= end); + + if (ret != WT_NOTFOUND) + return (ret); + + return (0); +} + +/* + * __wt_struct_unpack -- + * Unpack a byte string. + */ +int +__wt_struct_unpack(WT_SESSION_IMPL *session, + const void *buffer, size_t size, const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = __wt_struct_unpackv(session, buffer, size, fmt, ap); + va_end(ap); + + return (ret); +} diff --git a/src/packing/packing_api.c b/src/packing/packing_api.c new file mode 100644 index 00000000000..ddfad1dd669 --- /dev/null +++ b/src/packing/packing_api.c @@ -0,0 +1,71 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * wiredtiger_struct_size -- + * Calculate the size of a packed byte string. + */ +size_t +wiredtiger_struct_size(const char *fmt, ...) +{ + WT_SESSION_IMPL session; + va_list ap; + size_t size; + + WT_CLEAR(session); + session.event_handler = __wt_event_handler_default; + + va_start(ap, fmt); + size = __wt_struct_sizev(&session, fmt, ap); + va_end(ap); + + return (size); +} + +/* + * wiredtiger_struct_pack -- + * Pack a byte string. + */ +int +wiredtiger_struct_pack(void *buffer, size_t size, const char *fmt, ...) +{ + WT_SESSION_IMPL session; + va_list ap; + int ret; + + WT_CLEAR(session); + session.event_handler = __wt_event_handler_default; + + va_start(ap, fmt); + ret = __wt_struct_packv(&session, buffer, size, fmt, ap); + va_end(ap); + + return (ret); +} + +/* + * wiredtiger_struct_unpack -- + * Unpack a byte string. + */ +int +wiredtiger_struct_unpack(const void *buffer, size_t size, const char *fmt, ...) +{ + WT_SESSION_IMPL session; + va_list ap; + int ret; + + WT_CLEAR(session); + session.event_handler = __wt_event_handler_default; + + va_start(ap, fmt); + ret = __wt_struct_unpackv(&session, buffer, size, fmt, ap); + va_end(ap); + + return (ret); +} diff --git a/src/schema/schema_create.c b/src/schema/schema_create.c new file mode 100644 index 00000000000..8cd053c758a --- /dev/null +++ b/src/schema/schema_create.c @@ -0,0 +1,372 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +int +__wt_create_file(WT_SESSION_IMPL *session, + const char *name, const char *fileuri, const char *config) +{ + WT_ITEM *key, *val; + const char *cfg[] = API_CONF_DEFAULTS(session, create, config); + const char *filecfg[] = API_CONF_DEFAULTS(file, meta, config); + const char *filename, *treeconf; + int is_schema, vmajor, vminor, vpatch, ret; + + key = val = NULL; + treeconf = NULL; + ret = 0; + + filename = fileuri; + if (!WT_PREFIX_SKIP(filename, "file:")) + WT_RET_MSG( + session, EINVAL, "Expecting a 'file:' URI: %s", fileuri); + + /* + * Opening the schema table is a special case, use the config + * string we were passed to open the file. + */ + is_schema = (strcmp(filename, WT_SCHEMA_FILENAME) == 0); + + /* If the file exists, don't try to recreate it. */ + if ((ret = __wt_session_get_btree(session, name, fileuri, + is_schema ? config : NULL, NULL, WT_BTREE_NO_LOCK)) != WT_NOTFOUND) + return (ret); + + WT_RET(__wt_btree_create(session, filename)); + WT_ERR(__wt_schema_table_track_fileop(session, NULL, filename)); + + /* Insert WiredTiger version numbers into the schema file. */ + WT_ERR(__wt_scr_alloc(session, 0, &val)); + if (is_schema) { + WT_ERR(__wt_schema_table_insert( + session, WT_SCHEMA_VERSION_STR, + wiredtiger_version(&vmajor, &vminor, &vpatch))); + WT_ERR(__wt_buf_fmt(session, val, + "major=%d,minor=%d,patch=%d", vmajor, vminor, vpatch)); + WT_ERR(__wt_schema_table_insert( + session, WT_SCHEMA_VERSION, val->data)); + } + + /* + * Insert Btree version numbers into the schema file (including for + * the schema file itself, although the schema file version numbers + * can never be trusted, we have to get them from the turtle file). + */ + WT_ERR(__wt_scr_alloc(session, 0, &key)); + WT_ERR(__wt_buf_fmt(session, key, "version:%s", filename)); + WT_ERR(__wt_buf_fmt(session, val, "major=%d,minor=%d", + WT_BTREE_MAJOR_VERSION, WT_BTREE_MINOR_VERSION)); + WT_ERR(__wt_schema_table_insert(session, key->data, val->data)); + + if (is_schema) + WT_ERR(__wt_strdup(session, config, &treeconf)); + else + WT_ERR(__wt_config_collapse(session, filecfg, &treeconf)); + WT_ERR(__wt_schema_table_insert(session, fileuri, treeconf)); + + /* Allocate a WT_BTREE handle, and open the underlying file. */ + WT_ERR(__wt_btree_open(session, name, filename, treeconf, cfg, 0)); + treeconf = NULL; + WT_ERR(__wt_session_add_btree(session, NULL)); + + /* If something goes wrong, throw away anything we created. */ +err: __wt_scr_free(&key); + __wt_scr_free(&val); + __wt_free(session, treeconf); + return (ret); +} + +static int +__create_colgroup( + WT_SESSION_IMPL *session, const char *name, const char *config) +{ + WT_CONFIG_ITEM cval; + WT_ITEM fmt, namebuf, uribuf; + WT_TABLE *table; + const char *cfg[] = { __wt_confdfl_colgroup_meta, config, NULL, NULL }; + const char *filecfg[] = { config, NULL, NULL }; + const char **cfgp; + const char *cgconf, *cgname, *fileconf, *filename, *fileuri, *tablename; + size_t tlen; + int ret; + + cgconf = fileconf = NULL; + WT_CLEAR(fmt); + WT_CLEAR(namebuf); + WT_CLEAR(uribuf); + + tablename = name; + if (!WT_PREFIX_SKIP(tablename, "colgroup:")) + return (EINVAL); + cgname = strchr(tablename, ':'); + if (cgname != NULL) { + tlen = (size_t)(cgname - tablename); + ++cgname; + } else + tlen = strlen(tablename); + + if ((ret = + __wt_schema_get_table(session, tablename, tlen, &table)) != 0) + WT_RET_MSG(session, (ret == WT_NOTFOUND) ? ENOENT : ret, + "Can't create '%s' for non-existent table %.*s", + name, (int)tlen, tablename); + + /* Make sure the column group is referenced from the table. */ + if (cgname != NULL && (ret = + __wt_config_subgets(session, &table->cgconf, cgname, &cval)) != 0) + WT_RET_MSG(session, EINVAL, + "Column group '%s' not found in table '%.*s'", + cgname, (int)tlen, tablename); + + /* Find the first NULL entry in the cfg stack. */ + for (cfgp = &cfg[1]; *cfgp; cfgp++) + ; + + /* Add the filename to the colgroup config before collapsing. */ + if (__wt_config_getones(session, config, "filename", &cval) == 0) { + WT_ERR(__wt_buf_fmt( + session, &namebuf, "%.*s", (int)cval.len, cval.str)); + filename = namebuf.data; + } else { + if (cgname == NULL) + WT_ERR(__wt_buf_fmt(session, &namebuf, + "filename=%.*s.wt", (int)tlen, tablename)); + else + WT_ERR(__wt_buf_fmt(session, &namebuf, + "filename=%.*s_%s.wt", (int)tlen, tablename, + cgname)); + *cfgp++ = filename = namebuf.data; + (void)WT_PREFIX_SKIP(filename, "filename="); + } + + WT_ERR(__wt_config_collapse(session, cfg, &cgconf)); + + /* Calculate the key/value formats -- these go into the file config. */ + WT_ERR(__wt_buf_fmt(session, &fmt, "key_format=%s", table->key_format)); + if (cgname == NULL) + WT_ERR(__wt_buf_catfmt + (session, &fmt, ",value_format=%s", table->value_format)); + else { + if (__wt_config_getones(session, config, "columns", &cval) != 0) + WT_ERR_MSG(session, EINVAL, + "No 'columns' configuration for '%s'", name); + WT_ERR(__wt_buf_catfmt(session, &fmt, ",value_format=")); + WT_ERR(__wt_struct_reformat(session, + table, cval.str, cval.len, NULL, 1, &fmt)); + } + filecfg[1] = fmt.data; + WT_ERR(__wt_config_concat(session, filecfg, &fileconf)); + + WT_ERR(__wt_buf_fmt(session, &uribuf, "file:%s", filename)); + fileuri = uribuf.data; + + WT_ERR(__wt_schema_table_insert(session, name, cgconf)); + WT_ERR(__wt_create_file(session, name, fileuri, fileconf)); + + WT_ERR(__wt_schema_open_colgroups(session, table)); + +err: __wt_free(session, cgconf); + __wt_free(session, fileconf); + __wt_buf_free(session, &fmt); + __wt_buf_free(session, &namebuf); + __wt_buf_free(session, &uribuf); + return (ret); +} + +static int +__create_index(WT_SESSION_IMPL *session, const char *name, const char *config) +{ + WT_CONFIG pkcols; + WT_CONFIG_ITEM ckey, cval, icols; + WT_ITEM extra_cols, fmt, namebuf, uribuf; + WT_TABLE *table; + const char *cfg[] = { __wt_confdfl_index_meta, config, NULL, NULL }; + const char *filecfg[] = { config, NULL, NULL }; + const char *fileconf, *filename, *fileuri, *idxconf, *idxname; + const char *tablename; + size_t tlen; + int i, ret; + + idxconf = fileconf = NULL; + WT_CLEAR(fmt); + WT_CLEAR(extra_cols); + WT_CLEAR(namebuf); + WT_CLEAR(uribuf); + + tablename = name; + if (!WT_PREFIX_SKIP(tablename, "index:")) + return (EINVAL); + idxname = strchr(tablename, ':'); + if (idxname == NULL) + WT_RET_MSG(session, EINVAL, "Invalid index name, " + "should be <table name>:<index name>: %s", name); + + tlen = (size_t)(idxname++ - tablename); + if ((ret = + __wt_schema_get_table(session, tablename, tlen, &table)) != 0) + WT_RET_MSG(session, ret, + "Can't create an index for a non-existent table: %.*s", + (int)tlen, tablename); + + /* Add the filename to the index config before collapsing. */ + if (__wt_config_getones(session, config, "filename", &cval) == 0) { + WT_ERR(__wt_buf_fmt(session, + &namebuf, "%.*s", (int)cval.len, cval.str)); + filename = namebuf.data; + } else { + WT_ERR(__wt_buf_fmt(session, &namebuf, + "filename=%.*s_%s.wti", (int)tlen, tablename, idxname)); + cfg[2] = filename = namebuf.data; + (void)WT_PREFIX_SKIP(filename, "filename="); + } + WT_ERR(__wt_config_collapse(session, cfg, &idxconf)); + + /* Calculate the key/value formats -- these go into the file config. */ + if (__wt_config_getones(session, config, "columns", &icols) != 0) + WT_ERR_MSG(session, EINVAL, + "No 'columns' configuration for '%s'", name); + + /* + * The key format for an index is somewhat subtle: the application + * specifies a set of columns that it will use for the key, but the + * engine usually adds some hidden columns in order to derive the + * primary key. These hidden columns are part of the file's + * key_format, which we are calculating now, but not part of an index + * cursor's key_format. + */ + WT_ERR(__wt_config_subinit(session, &pkcols, &table->colconf)); + for (i = 0; i < table->nkey_columns && + (ret = __wt_config_next(&pkcols, &ckey, &cval)) == 0; + i++) { + /* + * If the primary key column is already in the secondary key, + * don't add it again. + */ + if (__wt_config_subgetraw(session, &icols, &ckey, &cval) == 0) + continue; + WT_ERR(__wt_buf_catfmt( + session, &extra_cols, "%.*s,", (int)ckey.len, ckey.str)); + } + if (ret != 0 && ret != WT_NOTFOUND) + goto err; + /* Index values are empty: all columns are packed into the index key. */ + WT_ERR(__wt_buf_fmt(session, &fmt, "value_format=,key_format=")); + WT_ERR(__wt_struct_reformat(session, table, + icols.str, icols.len, (const char *)extra_cols.data, 0, &fmt)); + filecfg[1] = fmt.data; + WT_ERR(__wt_config_concat(session, filecfg, &fileconf)); + + WT_ERR(__wt_buf_fmt(session, &uribuf, "file:%s", filename)); + fileuri = uribuf.data; + + WT_ERR(__wt_create_file(session, name, fileuri, fileconf)); + WT_ERR(__wt_schema_table_insert(session, name, idxconf)); + +err: __wt_free(session, fileconf); + __wt_free(session, idxconf); + __wt_buf_free(session, &fmt); + __wt_buf_free(session, &extra_cols); + __wt_buf_free(session, &namebuf); + __wt_buf_free(session, &uribuf); + return (ret); +} + +static int +__create_table(WT_SESSION_IMPL *session, const char *name, const char *config) +{ + WT_CONFIG conf; + WT_CONFIG_ITEM cgkey, cgval, cval; + WT_TABLE *table; + const char *cfg[] = { __wt_confdfl_table_meta, config, NULL, NULL }; + const char *tableconf, *tablename; + char *cgname; + size_t cgsize; + int ncolgroups, ret; + + cgname = NULL; + table = NULL; + tableconf = NULL; + + tablename = name; + if (!WT_PREFIX_SKIP(tablename, "table:")) + return (EINVAL); + + if ((ret = __wt_schema_get_table(session, + tablename, strlen(tablename), &table)) == 0) { + if (__wt_config_getones(session, + config, "exclusive", &cval) == 0 && cval.val) + WT_RET_MSG( + session, EEXIST, "Table exists: %s", tablename); + return (0); + } + if (ret != WT_NOTFOUND) + return (ret); + + WT_RET(__wt_config_gets(session, cfg, "colgroups", &cval)); + WT_RET(__wt_config_subinit(session, &conf, &cval)); + for (ncolgroups = 0; + (ret = __wt_config_next(&conf, &cgkey, &cgval)) == 0; + ncolgroups++) + ; + if (ret != WT_NOTFOUND) + return (ret); + + WT_RET(__wt_config_collapse(session, cfg, &tableconf)); + WT_ERR(__wt_schema_table_insert(session, name, tableconf)); + + /* Attempt to open the table now to catch any errors. */ + WT_ERR(__wt_schema_get_table( + session, tablename, strlen(tablename), &table)); + + if (ncolgroups == 0) { + cgsize = strlen("colgroup:") + strlen(tablename) + 1; + WT_ERR(__wt_calloc_def(session, cgsize, &cgname)); + snprintf(cgname, cgsize, "colgroup:%s", tablename); + WT_ERR(__create_colgroup(session, cgname, config)); + } + + if (0) { +err: if (table != NULL) + (void)__wt_schema_remove_table(session, table); + } + __wt_free(session, cgname); + __wt_free(session, tableconf); + return (ret); +} + +int +__wt_schema_create( + WT_SESSION_IMPL *session, const char *name, const char *config) +{ + + int ret; + + /* Disallow objects in the WiredTiger name space. */ + WT_RET(__wt_schema_name_check(session, name)); + + /* + * We track rename operations, if we fail in the middle, we want to + * back it all out. + */ + WT_RET(__wt_schema_table_track_on(session)); + + if (WT_PREFIX_MATCH(name, "colgroup:")) + ret = __create_colgroup(session, name, config); + else if (WT_PREFIX_MATCH(name, "file:")) + ret = __wt_create_file(session, name, name, config); + else if (WT_PREFIX_MATCH(name, "index:")) + ret = __create_index(session, name, config); + else if (WT_PREFIX_MATCH(name, "table:")) + ret = __create_table(session, name, config); + else + ret = __wt_unknown_object_type(session, name); + + WT_TRET(__wt_schema_table_track_off(session, ret != 0)); + + return (ret); +} diff --git a/src/schema/schema_drop.c b/src/schema/schema_drop.c new file mode 100644 index 00000000000..abe0c0edd85 --- /dev/null +++ b/src/schema/schema_drop.c @@ -0,0 +1,268 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __drop_file -- + * Drop a file. + */ +static int +__drop_file(WT_SESSION_IMPL *session, const char *uri, int force) +{ + static const char *list[] = { "file", "root", "version", NULL }; + WT_ITEM *buf; + int exist, ret; + const char *filename, **lp; + + buf = NULL; + + filename = uri; + if (!WT_PREFIX_SKIP(filename, "file:")) + return (EINVAL); + + /* If open, close the btree handle. */ + WT_RET(__wt_session_close_any_open_btree(session, filename)); + + /* Remove all of the schema table entries for this file. */ + WT_ERR(__wt_scr_alloc(session, 0, &buf)); + for (lp = list; *lp != NULL; ++lp) { + WT_ERR(__wt_buf_fmt(session, buf, "%s:%s", *lp, filename)); + + /* Remove the schema table entry (ignore missing items). */ + WT_TRET(__wt_schema_table_remove(session, buf->data)); + if (force && ret == WT_NOTFOUND) + ret = 0; + } + + /* Remove the underlying physical file. */ + WT_ERR(__wt_exist(session, filename, &exist)); + if (exist) + WT_TRET(__wt_remove(session, filename)); + +err: __wt_scr_free(&buf); + return (ret); +} + +/* + * __drop_tree -- + * Drop an index or colgroup reference. + */ +static int +__drop_tree(WT_SESSION_IMPL *session, WT_BTREE *btree, int force) +{ + WT_ITEM *buf; + int ret; + + ret = 0; + + /* Remove the schema table entry (ignore missing items). */ + WT_TRET(__wt_schema_table_remove(session, btree->name)); + if (force && ret == WT_NOTFOUND) + ret = 0; + + /* + * Drop the file. + * __drop_file closes the WT_BTREE handle, so we copy the + * WT_BTREE->filename field to make a URI. + */ + WT_ERR(__wt_scr_alloc(session, 0, &buf)); + WT_ERR(__wt_buf_fmt(session, buf, "file:%s", btree->filename)); + WT_ERR(__drop_file(session, buf->data, force)); + +err: __wt_scr_free(&buf); + + return (ret); +} + +/* + * __drop_colgroup -- + * WT_SESSION::drop for a colgroup. + */ +static int +__drop_colgroup( + WT_SESSION_IMPL *session, const char *uri, int force, const char *cfg[]) +{ + WT_BTREE *btree; + WT_TABLE *table; + const char *cgname, *tablename; + size_t tlen; + int i, ret; + + tablename = uri; + if (!WT_PREFIX_SKIP(tablename, "colgroup:")) + return (EINVAL); + cgname = strchr(tablename, ':'); + if (cgname != NULL) { + tlen = (size_t)(cgname - tablename); + ++cgname; + } else + tlen = strlen(tablename); + + /* + * Try to get the btree handle. Ideally, we would use an exclusive + * lock here to prevent access to the table while we are dropping it, + * but conflicts with the exclusive lock taken by + * __wt_session_close_any_open_btree. If two threads race dropping + * the same object, it will be caught there. + * + * If we can't get a tree, try to remove it from the schema table. + */ + if ((ret = __wt_schema_get_btree( + session, uri, strlen(uri), cfg, WT_BTREE_NO_LOCK)) != 0) { + (void)__wt_schema_table_remove(session, uri); + return (ret); + } + btree = session->btree; + + /* If we can get the table, detach the colgroup from it. */ + if ((ret = __wt_schema_get_table( + session, tablename, tlen, &table)) == 0) { + for (i = 0; i < WT_COLGROUPS(table); i++) { + if (table->colgroup[i] == btree) { + table->colgroup[i] = NULL; + table->cg_complete = 0; + break; + } + } + } else if (ret != WT_NOTFOUND) + WT_TRET(ret); + + WT_TRET(__drop_tree(session, btree, force)); + + return (ret); +} + +/* + * __drop_index -- + * WT_SESSION::drop for a colgroup. + */ +static int +__drop_index( + WT_SESSION_IMPL *session, const char *uri, int force, const char *cfg[]) +{ + WT_BTREE *btree; + WT_TABLE *table; + const char *idxname, *tablename; + size_t tlen; + int i, ret; + + tablename = uri; + if (!WT_PREFIX_SKIP(tablename, "index:") || + (idxname = strchr(tablename, ':')) == NULL) + return (EINVAL); + tlen = (size_t)(idxname - tablename); + ++idxname; + + /* + * Try to get the btree handle. Ideally, we would use an exclusive + * lock here to prevent access to the table while we are dropping it, + * but conflicts with the exclusive lock taken by + * __wt_session_close_any_open_btree. If two threads race dropping + * the same object, it will be caught there. + * + * If we can't get a tree, try to remove it from the schema table. + */ + if ((ret = __wt_schema_get_btree( + session, uri, strlen(uri), cfg, WT_BTREE_NO_LOCK)) != 0) { + (void)__wt_schema_table_remove(session, uri); + return (ret); + } + btree = session->btree; + + /* If we can get the table, detach the index from it. */ + if ((ret = __wt_schema_get_table( + session, tablename, tlen, &table)) == 0 && + (ret = __wt_schema_open_index( + session, table, idxname, strlen(idxname))) == 0) { + for (i = 0; i < table->nindices; i++) + if (table->index[i] == btree) { + table->index[i] = NULL; + table->idx_complete = 0; + } + } else if (ret != WT_NOTFOUND) + WT_TRET(ret); + + WT_TRET(__drop_tree(session, btree, force)); + + return (ret); +} + +/* + * __drop_table -- + * WT_SESSION::drop for a table. + */ +static int +__drop_table(WT_SESSION_IMPL *session, const char *uri, int force) +{ + WT_BTREE *btree; + WT_TABLE *table; + int i, ret; + const char *name; + + ret = 0; + + name = uri; + (void)WT_PREFIX_SKIP(name, "table:"); + + WT_RET(__wt_schema_get_table(session, name, strlen(name), &table)); + + /* Drop the column groups. */ + for (i = 0; i < WT_COLGROUPS(table); i++) { + if ((btree = table->colgroup[i]) == NULL) + continue; + table->colgroup[i] = NULL; + WT_TRET(__drop_tree(session, btree, force)); + } + + /* Drop the indices. */ + WT_TRET(__wt_schema_open_index(session, table, NULL, 0)); + for (i = 0; i < table->nindices; i++) { + btree = table->index[i]; + table->index[i] = NULL; + WT_TRET(__drop_tree(session, btree, force)); + } + + WT_TRET(__wt_schema_remove_table(session, table)); + + /* Remove the schema table entry (ignore missing items). */ + WT_TRET(__wt_schema_table_remove(session, uri)); + if (force && ret == WT_NOTFOUND) + ret = 0; + + return (ret); +} + +int +__wt_schema_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +{ + WT_CONFIG_ITEM cval; + int force, ret; + + cval.val = 0; + ret = __wt_config_gets(session, cfg, "force", &cval); + if (ret != 0 && ret != WT_NOTFOUND) + WT_RET(ret); + force = cval.val == 0 ? 0 : 1; + + /* Disallow drops from the WiredTiger name space. */ + WT_RET(__wt_schema_name_check(session, uri)); + + if (WT_PREFIX_MATCH(uri, "colgroup:")) + ret = __drop_colgroup(session, uri, force, cfg); + else if (WT_PREFIX_MATCH(uri, "file:")) + ret = __drop_file(session, uri, force); + else if (WT_PREFIX_MATCH(uri, "index:")) + ret = __drop_index(session, uri, force, cfg); + else if (WT_PREFIX_MATCH(uri, "table:")) + ret = __drop_table(session, uri, force); + else + return (__wt_unknown_object_type(session, uri)); + + /* If we didn't find a schema file entry, map that error to ENOENT. */ + return (ret == WT_NOTFOUND ? ENOENT : ret); +} diff --git a/src/schema/schema_list.c b/src/schema/schema_list.c new file mode 100644 index 00000000000..acc38880a19 --- /dev/null +++ b/src/schema/schema_list.c @@ -0,0 +1,145 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_schema_add_table -- + * Add a table handle to the session's cache. + */ +int +__wt_schema_add_table( + WT_SESSION_IMPL *session, WT_TABLE *table) +{ + TAILQ_INSERT_HEAD(&session->tables, table, q); + + return (0); +} + +/* + * __wt_schema_get_table -- + * Get the table handle for the named table. + */ +int +__wt_schema_find_table(WT_SESSION_IMPL *session, + const char *name, size_t namelen, WT_TABLE **tablep) +{ + WT_TABLE *table; + const char *tablename; + + TAILQ_FOREACH(table, &session->tables, q) { + tablename = table->name; + (void)WT_PREFIX_SKIP(tablename, "table:"); + if (strncmp(tablename, name, namelen) == 0 && + tablename[namelen] == '\0') { + *tablep = table; + return (0); + } + } + + return (WT_NOTFOUND); +} + +/* + * __wt_schema_get_table -- + * Get the table handle for the named table. + */ +int +__wt_schema_get_table(WT_SESSION_IMPL *session, + const char *name, size_t namelen, WT_TABLE **tablep) +{ + int ret; + + ret = __wt_schema_find_table(session, name, namelen, tablep); + + if (ret == WT_NOTFOUND) { + WT_RET(__wt_schema_open_table(session, name, namelen, tablep)); + ret = __wt_schema_add_table(session, *tablep); + } + + return (ret); +} + +/* + * __wt_schema_destroy_table -- + * Free a table handle. + */ +void +__wt_schema_destroy_table(WT_SESSION_IMPL *session, WT_TABLE *table) +{ + __wt_free(session, table->name); + __wt_free(session, table->config); + __wt_free(session, table->plan); + __wt_free(session, table->key_format); + __wt_free(session, table->value_format); + __wt_free(session, table->colgroup); + __wt_free(session, table->index); + __wt_free(session, table); +} + +/* + * __wt_schema_remove_table -- + * Remove the table handle from the session, closing if necessary. + */ +int +__wt_schema_remove_table( + WT_SESSION_IMPL *session, WT_TABLE *table) +{ + TAILQ_REMOVE(&session->tables, table, q); + __wt_schema_destroy_table(session, table); + + return (0); +} + +/* + * __wt_schema_close_tables -- + * Close all of the tables in a session. + */ +int +__wt_schema_close_tables(WT_SESSION_IMPL *session) +{ + WT_TABLE *table; + int ret; + + ret = 0; + while ((table = TAILQ_FIRST(&session->tables)) != NULL) + WT_TRET(__wt_schema_remove_table(session, table)); + + return (ret); +} + +/* + * __wt_schema_detach tree -- + * Remove any references to a tree from a table in the session. + * + * Note: this function should be called with an exclusive lock on the btree + * handle to prevent races. + */ +void +__wt_schema_detach_tree(WT_SESSION_IMPL *session, WT_BTREE *btree) +{ + WT_TABLE *table; + int i; + + TAILQ_FOREACH(table, &session->tables, q) { + /* Check the column groups. */ + for (i = 0; i < WT_COLGROUPS(table); i++) + if (table->colgroup[i] == btree) { + table->colgroup[i] = NULL; + table->cg_complete = 0; + return; + } + + /* Check the indices. */ + for (i = 0; i < table->nindices; i++) + if (table->index[i] == btree) { + table->index[i] = NULL; + table->idx_complete = 0; + return; + } + } +} diff --git a/src/schema/schema_open.c b/src/schema/schema_open.c new file mode 100644 index 00000000000..fd8f43b46ca --- /dev/null +++ b/src/schema/schema_open.c @@ -0,0 +1,426 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_schema_colgroup_name -- + * Get the URI for a column group. This is used for schema table lookups. + * The only complexity here is that simple tables (with a single column + * group) use a simpler naming scheme. + */ +int +__wt_schema_colgroup_name(WT_SESSION_IMPL *session, + WT_TABLE *table, const char *cgname, size_t len, char **namebufp) +{ + const char *tablename; + char *namebuf; + size_t namesize; + + namebuf = *namebufp; + tablename = table->name; + (void)WT_PREFIX_SKIP(tablename, "table:"); + + /* The primary filename is in the table config. */ + if (table->ncolgroups == 0) { + namesize = strlen("colgroup:") + strlen(tablename) + 1; + WT_RET(__wt_realloc(session, NULL, namesize, &namebuf)); + snprintf(namebuf, namesize, "colgroup:%s", tablename); + } else { + namesize = strlen("colgroup::") + strlen(tablename) + len + 1; + WT_RET(__wt_realloc(session, NULL, namesize, &namebuf)); + snprintf(namebuf, namesize, "colgroup:%s:%.*s", + tablename, (int)len, cgname); + } + + *namebufp = namebuf; + return (0); +} + +/* + * __wt_schema_get_btree -- + * Get the btree (into session->btree) for the named schema object + * (either a column group or an index). + */ +int +__wt_schema_get_btree(WT_SESSION_IMPL *session, + const char *objname, size_t len, const char *cfg[], uint32_t flags) +{ + WT_ITEM uribuf; + WT_CONFIG_ITEM cval; + WT_CURSOR *cursor; + const char *fileuri, *name, *objconf; + int ret; + + cursor = NULL; + WT_CLEAR(uribuf); + + name = objname; + if (len != strlen(objname)) + WT_ERR(__wt_strndup(session, objname, len, &name)); + + WT_ERR(__wt_schema_table_cursor(session, NULL, &cursor)); + cursor->set_key(cursor, name); + WT_ERR(cursor->search(cursor)); + WT_ERR(cursor->get_value(cursor, &objconf)); + + /* Get the filename from the schema table. */ + WT_ERR(__wt_config_getones(session, objconf, "filename", &cval)); + WT_ERR(__wt_buf_fmt( + session, &uribuf, "file:%.*s", (int)cval.len, cval.str)); + fileuri = uribuf.data; + + /* !!! Close the schema cursor first, this overwrites session->btree. */ + ret = cursor->close(cursor); + cursor = NULL; + if (ret != 0) + goto err; + + ret = __wt_session_get_btree(session, name, fileuri, NULL, cfg, flags); + if (ret == ENOENT) + __wt_errx(session, + "%s created but '%s' is missing", objname, fileuri); + +err: __wt_buf_free(session, &uribuf); + if (name != objname) + __wt_free(session, name); + if (cursor != NULL) + WT_TRET(cursor->close(cursor)); + return (ret); +} + +/* + * __wt_schema_open_colgroups -- + * Open the column groups for a table. + */ +int +__wt_schema_open_colgroups(WT_SESSION_IMPL *session, WT_TABLE *table) +{ + WT_CONFIG cparser; + WT_CONFIG_ITEM ckey, cval; + WT_ITEM plan; + char *cgname; + const char *fileconf; + int i, ret; + + if (table->cg_complete) + return (0); + + fileconf = NULL; + cgname = NULL; + ret = 0; + + WT_RET(__wt_config_subinit(session, &cparser, &table->cgconf)); + + /* Open each column group. */ + for (i = 0; i < WT_COLGROUPS(table); i++) { + if (table->ncolgroups > 0) + WT_ERR(__wt_config_next(&cparser, &ckey, &cval)); + else + WT_CLEAR(ckey); + if (table->colgroup[i] != NULL) + continue; + + WT_ERR(__wt_schema_colgroup_name(session, table, + ckey.str, ckey.len, &cgname)); + ret = __wt_schema_get_btree(session, + cgname, strlen(cgname), NULL, WT_BTREE_NO_LOCK); + if (ret != 0) { + /* It is okay if the table is not yet complete. */ + if (ret == WT_NOTFOUND) + ret = 0; + goto err; + } + table->colgroup[i] = session->btree; + } + + if (!table->is_simple) { + WT_ERR(__wt_table_check(session, table)); + + WT_CLEAR(plan); + WT_ERR(__wt_struct_plan(session, + table, table->colconf.str, table->colconf.len, 1, &plan)); + table->plan = __wt_buf_steal(session, &plan, NULL); + } + + table->cg_complete = 1; + +err: __wt_free(session, cgname); + __wt_free(session, fileconf); + return (ret); +} + +/* + * ___open_index -- + * Open an index. + */ +static int +__open_index(WT_SESSION_IMPL *session, WT_TABLE *table, + const char *uri, const char *idxconf, WT_BTREE **btreep) +{ + WT_BTREE *btree; + WT_CONFIG colconf; + WT_CONFIG_ITEM ckey, cval, icols; + WT_ITEM cols, fmt, plan, uribuf; + const char *fileuri; + u_int cursor_key_cols; + int i, ret; + + ret = 0; + WT_CLEAR(uribuf); + + /* Get the filename from the index config. */ + WT_ERR(__wt_config_getones(session, idxconf, "filename", &cval)); + WT_ERR(__wt_buf_fmt( + session, &uribuf, "file:%.*s", (int)cval.len, cval.str)); + fileuri = uribuf.data; + + ret = __wt_session_get_btree(session, uri, fileuri, + NULL, NULL, WT_BTREE_NO_LOCK); + if (ret == ENOENT) + __wt_errx(session, + "Index '%s' created but '%s' is missing", uri, fileuri); + /* Other errors will already have generated an error message. */ + if (ret != 0) + goto err; + + btree = session->btree; + + /* + * The key format for an index is somewhat subtle: the application + * specifies a set of columns that it will use for the key, but the + * engine usually adds some hidden columns in order to derive the + * primary key. These hidden columns are part of the file's key. + * + * The file's key_format is stored persistently, we need to calculate + * the index cursor key format (which will usually omit some of those + * keys). + */ + WT_ERR(__wt_config_getones(session, idxconf, "columns", &icols)); + + /* Start with the declared index columns. */ + WT_ERR(__wt_config_subinit(session, &colconf, &icols)); + WT_CLEAR(cols); + cursor_key_cols = 0; + while ((ret = __wt_config_next(&colconf, &ckey, &cval)) == 0) { + WT_ERR(__wt_buf_catfmt( + session, &cols, "%.*s,", (int)ckey.len, ckey.str)); + ++cursor_key_cols; + } + if (ret != 0 && ret != WT_NOTFOUND) + goto err; + + /* + * Now add any primary key columns from the table that are not + * already part of the index key. + */ + WT_ERR(__wt_config_subinit(session, &colconf, &table->colconf)); + for (i = 0; i < table->nkey_columns && + (ret = __wt_config_next(&colconf, &ckey, &cval)) == 0; + i++) { + /* + * If the primary key column is already in the secondary key, + * don't add it again. + */ + if (__wt_config_subgetraw(session, &icols, &ckey, &cval) == 0) + continue; + WT_ERR(__wt_buf_catfmt( + session, &cols, "%.*s,", (int)ckey.len, ckey.str)); + } + if (ret != 0 && ret != WT_NOTFOUND) + goto err; + + WT_CLEAR(plan); + WT_ERR(__wt_struct_plan(session, + table, cols.data, cols.size, 0, &plan)); + btree->key_plan = __wt_buf_steal(session, &plan, NULL); + + /* Set up the cursor key format (the visible columns). */ + WT_CLEAR(fmt); + WT_ERR(__wt_struct_truncate(session, + btree->key_format, cursor_key_cols, &fmt)); + btree->idxkey_format = __wt_buf_steal(session, &fmt, NULL); + + /* By default, index cursor values are the table value columns. */ + /* TODO Optimize to use index columns in preference to table lookups. */ + WT_ERR(__wt_struct_plan(session, + table, table->colconf.str, table->colconf.len, 1, &plan)); + btree->value_plan = __wt_buf_steal(session, &plan, NULL); + + *btreep = btree; + +err: __wt_buf_free(session, &cols); + __wt_buf_free(session, &uribuf); + + return (ret); +} + +/* + * __wt_schema_open_indices -- + * Open the indices for a table. + */ +int +__wt_schema_open_index( + WT_SESSION_IMPL *session, WT_TABLE *table, const char *idxname, size_t len) +{ + WT_CURSOR *cursor; + const char *idxconf, *name, *tablename, *uri; + int i, match, ret, skipped; + + cursor = NULL; + skipped = 0; + idxconf = NULL; + tablename = table->name; + (void)WT_PREFIX_SKIP(tablename, "table:"); + + if (len == 0 && table->idx_complete) + return (0); + + /* + * XXX Do a full scan through the schema table to find all matching + * indices. This scan be optimized when we have cursor search + next. + */ + WT_RET(__wt_schema_table_cursor(session, NULL, &cursor)); + + /* Open each index. */ + for (i = 0; (ret = cursor->next(cursor)) == 0;) { + WT_ERR(cursor->get_key(cursor, &uri)); + name = uri; + if (!WT_PREFIX_SKIP(name, "index:") || + !WT_PREFIX_SKIP(name, tablename) || + !WT_PREFIX_SKIP(name, ":")) + continue; + + /* Is this the index we are looking for? */ + match = (len > 0 && + strncmp(name, idxname, len) == 0 && name[len] == '\0'); + + if (i * sizeof(WT_BTREE *) >= table->index_alloc) + WT_ERR(__wt_realloc(session, &table->index_alloc, + WT_MAX(10 * sizeof(WT_BTREE *), + 2 * table->index_alloc), + &table->index)); + + if (table->index[i] == NULL) { + if (len == 0 || match) { + WT_ERR(cursor->get_value(cursor, &idxconf)); + WT_ERR(__open_index(session, + table, uri, idxconf, &table->index[i])); + } else + skipped = 1; + } + + if (match) { + ret = cursor->close(cursor); + cursor = NULL; + session->btree = table->index[i]; + break; + } + i++; + } + + /* Did we make it all the way through? */ + if (ret == WT_NOTFOUND) { + ret = 0; + if (!skipped) { + table->nindices = i; + table->idx_complete = 1; + } + } + +err: if (cursor != NULL) + WT_TRET(cursor->close(cursor)); + return (ret); +} + +/* + * __wt_schema_open_table -- + * Open a named table. + */ +int +__wt_schema_open_table(WT_SESSION_IMPL *session, + const char *name, size_t namelen, WT_TABLE **tablep) +{ + WT_CONFIG cparser; + WT_CONFIG_ITEM ckey, cval; + WT_CURSOR *cursor; + WT_ITEM buf; + WT_TABLE *table; + const char *tconfig; + char *tablename; + int ret; + + cursor = NULL; + table = NULL; + + WT_CLEAR(buf); + WT_RET(__wt_buf_fmt(session, &buf, "table:%.*s", (int)namelen, name)); + tablename = __wt_buf_steal(session, &buf, NULL); + + WT_ERR(__wt_schema_table_cursor(session, NULL, &cursor)); + cursor->set_key(cursor, tablename); + WT_ERR(cursor->search(cursor)); + WT_ERR(cursor->get_value(cursor, &tconfig)); + + WT_ERR(__wt_calloc_def(session, 1, &table)); + table->name = tablename; + tablename = NULL; + + WT_ERR(__wt_config_getones(session, tconfig, "columns", &cval)); + + WT_ERR(__wt_config_getones(session, tconfig, "key_format", &cval)); + WT_ERR(__wt_strndup(session, cval.str, cval.len, &table->key_format)); + WT_ERR(__wt_config_getones(session, tconfig, "value_format", &cval)); + WT_ERR(__wt_strndup(session, cval.str, cval.len, &table->value_format)); + WT_ERR(__wt_strdup(session, tconfig, &table->config)); + + /* Point to some items in the copy to save re-parsing. */ + WT_ERR(__wt_config_getones(session, table->config, + "columns", &table->colconf)); + + /* + * Count the number of columns: tables are "simple" if the columns + * are not named. + */ + WT_ERR(__wt_config_subinit(session, &cparser, &table->colconf)); + table->is_simple = 1; + while ((ret = __wt_config_next(&cparser, &ckey, &cval)) == 0) + table->is_simple = 0; + if (ret != WT_NOTFOUND) + goto err; + + /* Check that the columns match the key and value formats. */ + if (!table->is_simple) + WT_ERR(__wt_schema_colcheck(session, + table->key_format, table->value_format, &table->colconf, + &table->nkey_columns, NULL)); + + WT_ERR(__wt_config_getones(session, table->config, + "colgroups", &table->cgconf)); + + /* Count the number of column groups. */ + WT_ERR(__wt_config_subinit(session, &cparser, &table->cgconf)); + table->ncolgroups = 0; + while ((ret = __wt_config_next(&cparser, &ckey, &cval)) == 0) + ++table->ncolgroups; + if (ret != WT_NOTFOUND) + goto err; + + WT_ERR(__wt_calloc_def(session, WT_COLGROUPS(table), &table->colgroup)); + WT_ERR(__wt_schema_open_colgroups(session, table)); + + *tablep = table; + + if (0) { +err: if (table != NULL) + __wt_schema_destroy_table(session, table); + } + if (cursor != NULL) + WT_TRET(cursor->close(cursor)); + __wt_free(session, tablename); + return (ret); +} diff --git a/src/schema/schema_plan.c b/src/schema/schema_plan.c new file mode 100644 index 00000000000..0c1ef1d01c9 --- /dev/null +++ b/src/schema/schema_plan.c @@ -0,0 +1,364 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static int +__find_next_col(WT_SESSION_IMPL *session, WT_TABLE *table, + WT_CONFIG_ITEM *colname, int *cgnump, int *colnump, char *coltype) +{ + WT_BTREE *cgtree; + WT_CONFIG conf; + WT_CONFIG_ITEM cval, k, v; + int cg, col, foundcg, foundcol, getnext; + + foundcg = foundcol = -1; + + getnext = 1; + for (cg = 0; cg < WT_COLGROUPS(table); cg++) { + if ((cgtree = table->colgroup[cg]) == NULL) + continue; + /* + * If there is only one column group, we just scan through all + * of the columns. For tables with multiple column groups, we + * look at the key columns once, then go through the value + * columns for each group. + */ + if (cg == 0) { + cval = table->colconf; + col = 0; + } else { +cgcols: WT_RET(__wt_config_getones(session, + cgtree->config, "columns", &cval)); + col = table->nkey_columns; + } + WT_RET(__wt_config_subinit(session, &conf, &cval)); + for (; __wt_config_next(&conf, &k, &v) == 0; col++) { + if (cg == *cgnump && col == *colnump) + getnext = 1; + if (getnext && k.len == colname->len && + strncmp(colname->str, k.str, k.len) == 0) { + foundcg = cg; + foundcol = col; + getnext = 0; + } + if (cg == 0 && table->ncolgroups > 0 && + col == table->nkey_columns - 1) + goto cgcols; + } + } + + if (foundcg == -1) + return (WT_NOTFOUND); + + *cgnump = foundcg; + if (foundcol < table->nkey_columns) { + *coltype = WT_PROJ_KEY; + *colnump = foundcol; + } else { + *coltype = WT_PROJ_VALUE; + *colnump = foundcol - table->nkey_columns; + } + return (0); +} + +/* + * __wt_schema_colcheck -- + * Check that a list of columns matches a (key,value) format pair. + */ +int +__wt_schema_colcheck(WT_SESSION_IMPL *session, + const char *key_format, const char *value_format, WT_CONFIG_ITEM *colconf, + int *kcolsp, int *vcolsp) +{ + WT_CONFIG conf; + WT_CONFIG_ITEM k, v; + WT_PACK pack; + WT_PACK_VALUE pv; + int kcols, ncols, ret, vcols; + + WT_RET(__pack_init(session, &pack, key_format)); + for (kcols = 0; (ret = __pack_next(&pack, &pv)) == 0; kcols++) + ; + WT_RET_TEST(ret != WT_NOTFOUND, ret); + + WT_RET(__pack_init(session, &pack, value_format)); + for (vcols = 0; (ret = __pack_next(&pack, &pv)) == 0; vcols++) + ; + WT_RET_TEST(ret != WT_NOTFOUND, ret); + + /* Walk through the named columns. */ + WT_RET(__wt_config_subinit(session, &conf, colconf)); + for (ncols = 0; (ret = __wt_config_next(&conf, &k, &v)) == 0; ncols++) + ; + + if (ncols != 0 && ncols != kcols + vcols) + WT_RET_MSG(session, EINVAL, "Number of columns in '%.*s' " + "does not match key format '%s' plus value format '%s'", + (int)colconf->len, colconf->str, key_format, value_format); + + if (kcolsp != NULL) + *kcolsp = kcols; + if (vcolsp != NULL) + *vcolsp = vcols; + + return (0); +} + +/* + * __wt_table_check -- + * Make sure all columns appear in a column group. + */ +int +__wt_table_check(WT_SESSION_IMPL *session, WT_TABLE *table) +{ + WT_CONFIG conf; + WT_CONFIG_ITEM k, v; + int cg, col, i, ret; + char coltype; + + if (table->is_simple) + return (0); + + /* Walk through the columns. */ + WT_RET(__wt_config_subinit(session, &conf, &table->colconf)); + + /* Skip over the key columns. */ + for (i = 0; i < table->nkey_columns; i++) + WT_RET(__wt_config_next(&conf, &k, &v)); + cg = col = 0; + while ((ret = __wt_config_next(&conf, &k, &v)) == 0) { + if (__find_next_col( + session, table, &k, &cg, &col, &coltype) != 0) + WT_RET_MSG(session, EINVAL, + "Column '%.*s' in '%s' does not appear in a " + "column group", + (int)k.len, k.str, table->name); + /* + * Column groups can't store key columns in their value: + * __wt_struct_reformat should have already detected this case. + */ + WT_ASSERT(session, coltype == WT_PROJ_VALUE); + + } + if (ret != WT_NOTFOUND) + return (ret); + + return (0); +} + +/* + * __wt_struct_plan -- + * Given a table cursor containing a complete table, build the "projection + * plan" to distribute the columns to dependent stores. A string + * representing the plan will be appended to the plan buffer. + */ +int +__wt_struct_plan(WT_SESSION_IMPL *session, WT_TABLE *table, + const char *columns, size_t len, int value_only, WT_ITEM *plan) +{ + WT_CONFIG conf; + WT_CONFIG_ITEM k, v; + int cg, col, current_cg, current_col, start_cg, start_col; + int i, have_it; + char coltype, current_coltype; + + start_cg = start_col = -1; /* -Wuninitialized */ + + /* Work through the value columns by skipping over the key columns. */ + WT_RET(__wt_config_initn(session, &conf, columns, len)); + + if (value_only) + for (i = 0; i < table->nkey_columns; i++) + WT_RET(__wt_config_next(&conf, &k, &v)); + + current_cg = cg = 0; + current_col = col = INT_MAX; + current_coltype = coltype = WT_PROJ_KEY; /* Keep lint quiet. */ + while (__wt_config_next(&conf, &k, &v) == 0) { + have_it = 0; + + while (__find_next_col(session, table, + &k, &cg, &col, &coltype) == 0 && + (!have_it || cg != start_cg || col != start_col)) { + /* + * First we move to the column. If that is in a + * different column group to the last column we + * accessed, or before the last column in the same + * column group, or moving from the key to the value, + * we need to switch column groups or rewind. + */ + if (current_cg != cg || current_col > col || + current_coltype != coltype) { + WT_ASSERT(session, !value_only || + coltype == WT_PROJ_VALUE); + WT_RET(__wt_buf_catfmt( + session, plan, "%d%c", cg, coltype)); + + /* + * Set the current column group and column + * within the table. + */ + current_cg = cg; + current_col = 0; + current_coltype = coltype; + } + /* Now move to the column we want. */ + if (current_col < col) { + if (col - current_col > 1) + WT_RET(__wt_buf_catfmt(session, + plan, "%d", col - current_col)); + WT_RET(__wt_buf_catfmt(session, + plan, "%c", WT_PROJ_SKIP)); + } + /* + * Now copy the value in / out. In the common case, + * where each value is used in one column, we do a + * "next" operation. If the value is used again, we do + * a "reuse" operation to avoid making another copy. + */ + if (!have_it) { + WT_RET(__wt_buf_catfmt(session, + plan, "%c", WT_PROJ_NEXT)); + + start_cg = cg; + start_col = col; + have_it = 1; + } else + WT_RET(__wt_buf_catfmt(session, + plan, "%c", WT_PROJ_REUSE)); + current_col = col + 1; + } + } + + return (0); +} + +static int +__find_column_format(WT_SESSION_IMPL *session, + WT_TABLE *table, WT_CONFIG_ITEM *colname, int value_only, WT_PACK_VALUE *pv) +{ + WT_CONFIG conf; + WT_CONFIG_ITEM k, v; + WT_PACK pack; + int inkey, ret; + + WT_RET(__wt_config_subinit(session, &conf, &table->colconf)); + WT_RET(__pack_init(session, &pack, table->key_format)); + inkey = 1; + + while ((ret = __wt_config_next(&conf, &k, &v)) == 0) { + if ((ret = __pack_next(&pack, pv)) == WT_NOTFOUND && inkey) { + ret = __pack_init(session, &pack, table->value_format); + if (ret == 0) + ret = __pack_next(&pack, pv); + inkey = 0; + } + if (ret != 0) + return (ret); + + if (k.len == colname->len && + strncmp(colname->str, k.str, k.len) == 0) { + if (value_only && inkey) + return (EINVAL); + return (0); + } + } + + return (ret); +} + +/* + * __wt_struct_reformat -- + * Given a table and a list of columns (which could be values in a column + * group or index keys), calculate the resulting new format string. + * The result will be appended to the format buffer. + */ +int +__wt_struct_reformat(WT_SESSION_IMPL *session, WT_TABLE *table, + const char *columns, size_t len, const char *extra_cols, int value_only, + WT_ITEM *format) +{ + WT_CONFIG config; + WT_CONFIG_ITEM k, next_k, next_v; + WT_PACK_VALUE pv; + int have_next, ret; + + WT_CLEAR(pv); /* -Wuninitialized */ + + WT_RET(__wt_config_initn(session, &config, columns, len)); + WT_RET(__wt_config_next(&config, &next_k, &next_v)); + do { + k = next_k; + ret = __wt_config_next(&config, &next_k, &next_v); + if (ret != 0 && ret != WT_NOTFOUND) + return (ret); + have_next = (ret == 0); + + if (!have_next && extra_cols != NULL) { + WT_RET(__wt_config_init(session, &config, extra_cols)); + WT_RET(__wt_config_next(&config, &next_k, &next_v)); + have_next = 1; + extra_cols = NULL; + } + + if ((ret = __find_column_format(session, + table, &k, value_only, &pv)) != 0) { + if (value_only && ret == EINVAL) + WT_RET_MSG(session, EINVAL, + "A column group cannot store key column " + "'%.*s' in its value", (int)k.len, k.str); + WT_RET_MSG(session, EINVAL, + "Column '%.*s' not found", (int)k.len, k.str); + } + + /* + * Check whether we're moving an unsized WT_ITEM from the end + * to the middle, or vice-versa. This determines whether the + * size needs to be prepended. This is the only case where the + * destination size can be larger than the source size. + */ + if (pv.type == 'u' && !pv.havesize && have_next) + pv.type = 'U'; + else if (pv.type == 'U' && !have_next) + pv.type = 'u'; + + if (pv.havesize) + WT_RET(__wt_buf_catfmt( + session, format, "%d%c", (int)pv.size, pv.type)); + else + WT_RET(__wt_buf_catfmt(session, format, "%c", pv.type)); + } while (have_next); + + return (0); +} + +/* + * __wt_struct_truncate -- + * Return a packing string for the first N columns in a value. + */ +int +__wt_struct_truncate(WT_SESSION_IMPL *session, + const char *input_fmt, u_int ncols, WT_ITEM *format) +{ + WT_PACK pack; + WT_PACK_VALUE pv; + + WT_CLEAR(pv); /* -Wuninitialized */ + + WT_RET(__pack_init(session, &pack, input_fmt)); + while (ncols-- > 0) { + WT_RET(__pack_next(&pack, &pv)); + if (pv.havesize) + WT_RET(__wt_buf_catfmt( + session, format, "%d%c", (int)pv.size, pv.type)); + else + WT_RET(__wt_buf_catfmt(session, format, "%c", pv.type)); + } + + return (0); +} diff --git a/src/schema/schema_project.c b/src/schema/schema_project.c new file mode 100644 index 00000000000..72240f3c57d --- /dev/null +++ b/src/schema/schema_project.c @@ -0,0 +1,467 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_schema_project_in -- + * Given list of cursors and a projection, read columns from the + * application into the dependent cursors. + */ +int +__wt_schema_project_in(WT_SESSION_IMPL *session, + WT_CURSOR **cp, const char *proj_arg, va_list ap) +{ + WT_CURSOR *c; + WT_ITEM *buf; + WT_PACK pack; + WT_PACK_VALUE pv; + char *proj; + uint8_t *p, *end; + const uint8_t *next; + size_t len, offset, old_len; + uint32_t arg; + + WT_CLEAR(pack); /* -Wuninitialized */ + buf = NULL; /* -Wuninitialized */ + p = end = NULL; /* -Wuninitialized */ + + /* Reset any of the buffers we will be setting. */ + for (proj = (char *)proj_arg; *proj != '\0'; proj++) { + arg = (uint32_t)strtoul(proj, &proj, 10); + if (*proj == WT_PROJ_KEY) { + c = cp[arg]; + WT_RET(__wt_buf_init(session, &c->key, 0)); + } else if (*proj == WT_PROJ_VALUE) { + c = cp[arg]; + WT_RET(__wt_buf_init(session, &c->value, 0)); + } + } + + for (proj = (char *)proj_arg; *proj != '\0'; proj++) { + arg = (uint32_t)strtoul(proj, &proj, 10); + + switch (*proj) { + case WT_PROJ_KEY: + c = cp[arg]; + if (WT_CURSOR_RECNO(c)) { + c->key.data = &c->recno; + c->key.size = sizeof(c->recno); + WT_RET(__pack_init(session, &pack, "R")); + } else + WT_RET(__pack_init( + session, &pack, c->key_format)); + buf = &c->key; + p = (uint8_t *)buf->data; + end = p + buf->size; + continue; + + case WT_PROJ_VALUE: + c = cp[arg]; + WT_RET(__pack_init(session, &pack, c->value_format)); + buf = &c->value; + p = (uint8_t *)buf->data; + end = p + buf->size; + continue; + } + + /* + * Otherwise, the argument is a count, where a missing + * count means a count of 1. + */ + for (arg = (arg == 0) ? 1 : arg; arg > 0; arg--) { + switch (*proj) { + case WT_PROJ_NEXT: + case WT_PROJ_SKIP: + WT_RET(__pack_next(&pack, &pv)); + if (*proj == WT_PROJ_SKIP) { + /* + * A nasty case: if we are inserting + * out-of-order, we may reach the end + * of the data. That's okay: we want + * to append in that case, and we're + * positioned to do that. + */ + if (p == end) { + /* Set up an empty value. */ + WT_CLEAR(pv.u); + if (pv.type == 'S' || + pv.type == 's') + pv.u.s = ""; + + len = __pack_size(session, &pv); + WT_RET(__wt_buf_grow(session, + buf, buf->size + len)); + p = (uint8_t *)buf->data + + buf->size; + WT_RET(__pack_write( + session, &pv, &p, len)); + buf->size += WT_STORE_SIZE(len); + end = (uint8_t *)buf->data + + buf->size; + } else if (*proj == WT_PROJ_SKIP) + WT_RET(__unpack_read(session, + &pv, (const uint8_t **)&p, + (size_t)(end - p))); + + break; + } + WT_PACK_GET(session, pv, ap); + /* FALLTHROUGH */ + + case WT_PROJ_REUSE: + /* Read the item we're about to overwrite. */ + next = p; + if (p < end) + WT_RET(__unpack_read(session, &pv, + &next, (size_t)(end - p))); + old_len = (size_t)(next - p); + + len = __pack_size(session, &pv); + offset = WT_PTRDIFF(p, buf->data); + WT_RET(__wt_buf_grow(session, + buf, buf->size + len)); + p = (uint8_t *)buf->data + offset; + end = (uint8_t *)buf->data + buf->size + len; + /* Make room if we're inserting out-of-order. */ + if (offset + old_len < buf->size) + memmove(p + len, p + old_len, + buf->size - (offset + old_len)); + WT_RET(__pack_write(session, &pv, &p, len)); + buf->size += WT_STORE_SIZE(len); + break; + + default: + WT_RET_MSG(session, EINVAL, + "unexpected projection plan: %c", + (int)*proj); + } + } + } + + return (0); +} + +/* + * __wt_schema_project_out -- + * Given list of cursors and a projection, read columns from the + * dependent cursors and return them to the application. + */ +int +__wt_schema_project_out(WT_SESSION_IMPL *session, + WT_CURSOR **cp, const char *proj_arg, va_list ap) +{ + WT_CURSOR *c; + WT_PACK pack; + WT_PACK_VALUE pv; + char *proj; + uint8_t *p, *end; + uint32_t arg; + + WT_CLEAR(pack); /* -Wuninitialized */ + WT_CLEAR(pv); /* -Wuninitialized */ + p = end = NULL; /* -Wuninitialized */ + + for (proj = (char *)proj_arg; *proj != '\0'; proj++) { + arg = (uint32_t)strtoul(proj, &proj, 10); + + switch (*proj) { + case WT_PROJ_KEY: + c = cp[arg]; + p = (uint8_t *)c->key.data; + end = p + c->key.size; + if (WT_CURSOR_RECNO(c)) { + c->key.data = &c->recno; + c->key.size = sizeof(c->recno); + WT_RET(__pack_init(session, &pack, "R")); + } else + WT_RET(__pack_init( + session, &pack, c->key_format)); + continue; + + case WT_PROJ_VALUE: + c = cp[arg]; + p = (uint8_t *)c->value.data; + end = p + c->value.size; + WT_RET(__pack_init(session, &pack, c->value_format)); + continue; + } + + /* + * Otherwise, the argument is a count, where a missing + * count means a count of 1. + */ + for (arg = (arg == 0) ? 1 : arg; arg > 0; arg--) { + switch (*proj) { + case WT_PROJ_NEXT: + case WT_PROJ_SKIP: + WT_RET(__pack_next(&pack, &pv)); + WT_RET(__unpack_read(session, &pv, + (const uint8_t **)&p, (size_t)(end - p))); + if (*proj == WT_PROJ_SKIP) + break; + WT_UNPACK_PUT(session, pv, ap); + /* FALLTHROUGH */ + + case WT_PROJ_REUSE: + /* Don't copy out the same value twice. */ + break; + } + } + } + + return (0); +} + +/* + * __wt_schema_project_slice -- + * Given list of cursors and a projection, read columns from the + * a raw buffer. + */ +int +__wt_schema_project_slice(WT_SESSION_IMPL *session, WT_CURSOR **cp, + const char *proj_arg, int key_only, const char *vformat, WT_ITEM *value) +{ + WT_CURSOR *c; + WT_ITEM *buf; + WT_PACK pack, vpack; + WT_PACK_VALUE pv, vpv; + char *proj; + uint8_t *end, *p; + const uint8_t *next, *vp, *vend; + size_t len, offset, old_len; + uint32_t arg; + int skip; + + WT_CLEAR(pack); /* -Wuninitialized */ + WT_CLEAR(vpv); /* -Wuninitialized */ + buf = NULL; /* -Wuninitialized */ + p = end = NULL; /* -Wuninitialized */ + + WT_RET(__pack_init(session, &vpack, vformat)); + vp = (uint8_t *)value->data; + vend = vp + value->size; + + /* Reset any of the buffers we will be setting. */ + for (proj = (char *)proj_arg; *proj != '\0'; proj++) { + arg = (uint32_t)strtoul(proj, &proj, 10); + if (*proj == WT_PROJ_KEY) { + c = cp[arg]; + WT_RET(__wt_buf_init(session, &c->key, 0)); + } else if (*proj == WT_PROJ_VALUE && !key_only) { + c = cp[arg]; + WT_RET(__wt_buf_init(session, &c->value, 0)); + } + } + + skip = key_only; + for (proj = (char *)proj_arg; *proj != '\0'; proj++) { + arg = (uint32_t)strtoul(proj, &proj, 10); + + switch (*proj) { + case WT_PROJ_KEY: + skip = 0; + c = cp[arg]; + if (WT_CURSOR_RECNO(c)) { + c->key.data = &c->recno; + c->key.size = sizeof(c->recno); + WT_RET(__pack_init(session, &pack, "R")); + } else + WT_RET(__pack_init( + session, &pack, c->key_format)); + buf = &c->key; + p = (uint8_t *)buf->data; + end = p + buf->size; + continue; + + case WT_PROJ_VALUE: + if ((skip = key_only) != 0) + continue; + c = cp[arg]; + WT_RET(__pack_init(session, &pack, c->value_format)); + buf = &c->value; + p = (uint8_t *)buf->data; + end = p + buf->size; + continue; + } + + /* + * Otherwise, the argument is a count, where a missing + * count means a count of 1. + */ + for (arg = (arg == 0) ? 1 : arg; arg > 0; arg--) { + switch (*proj) { + case WT_PROJ_NEXT: + case WT_PROJ_SKIP: + if (!skip) { + WT_RET(__pack_next(&pack, &pv)); + + /* + * A nasty case: if we are inserting + * out-of-order, append a zero value + * to keep the buffer in the correct + * format. + */ + if (*proj == WT_PROJ_SKIP && + p == end) { + /* Set up an empty value. */ + WT_CLEAR(pv.u); + if (pv.type == 'S' || + pv.type == 's') + pv.u.s = ""; + + len = __pack_size(session, &pv); + WT_RET(__wt_buf_grow(session, + buf, buf->size + len)); + p = (uint8_t *)buf->data + + buf->size; + WT_RET(__pack_write( + session, &pv, &p, len)); + end = p; + buf->size += WT_STORE_SIZE(len); + } else if (*proj == WT_PROJ_SKIP) + WT_RET(__unpack_read(session, + &pv, (const uint8_t **)&p, + (size_t)(end - p))); + } + if (*proj == WT_PROJ_SKIP) + break; + WT_RET(__pack_next(&vpack, &vpv)); + WT_RET(__unpack_read(session, &vpv, + &vp, (size_t)(vend - vp))); + /* FALLTHROUGH */ + case WT_PROJ_REUSE: + if (skip) + break; + + /* Read the item we're about to overwrite. */ + next = p; + if (p < end) + WT_RET(__unpack_read(session, &pv, + &next, (size_t)(end - p))); + old_len = (size_t)(next - p); + + /* + * There is subtlety here: the value format + * may not exactly match the cursor's format. + * In particular, we need lengths with raw + * columns in the middle of a packed struct, + * but not if they are at the end of a column. + */ + pv.u = vpv.u; + + len = __pack_size(session, &pv); + offset = WT_PTRDIFF(p, buf->data); + WT_RET(__wt_buf_grow(session, + buf, buf->size + len - old_len)); + p = (uint8_t *)buf->data + offset; + /* Make room if we're inserting out-of-order. */ + if (offset + old_len < buf->size) + memmove(p + len, p + old_len, + buf->size - (offset + old_len)); + WT_RET(__pack_write(session, &pv, &p, len)); + buf->size += WT_STORE_SIZE(len - old_len); + end = (uint8_t *)buf->data + buf->size; + break; + default: + WT_RET_MSG(session, EINVAL, + "unexpected projection plan: %c", + (int)*proj); + } + } + } + + return (0); +} + +/* + * __wt_schema_project_merge -- + * Given list of cursors and a projection, build a buffer containing the + * column values read from the cursors. + */ +int +__wt_schema_project_merge(WT_SESSION_IMPL *session, + WT_CURSOR **cp, const char *proj_arg, const char *vformat, WT_ITEM *value) +{ + WT_CURSOR *c; + WT_ITEM *buf; + WT_PACK pack, vpack; + WT_PACK_VALUE pv, vpv; + char *proj; + uint8_t *p, *end, *vp; + size_t len; + uint32_t arg; + + WT_CLEAR(pack); /* -Wuninitialized */ + WT_CLEAR(pv); /* -Wuninitialized */ + WT_CLEAR(vpv); /* -Wuninitialized */ + p = end = NULL; /* -Wuninitialized */ + + WT_RET(__wt_buf_init(session, value, 0)); + WT_RET(__pack_init(session, &vpack, vformat)); + + for (proj = (char *)proj_arg; *proj != '\0'; proj++) { + arg = (uint32_t)strtoul(proj, &proj, 10); + + switch (*proj) { + case WT_PROJ_KEY: + c = cp[arg]; + if (WT_CURSOR_RECNO(c)) { + c->key.data = &c->recno; + c->key.size = sizeof(c->recno); + WT_RET(__pack_init(session, &pack, "R")); + } else + WT_RET(__pack_init( + session, &pack, c->key_format)); + buf = &c->key; + p = (uint8_t *)buf->data; + end = p + buf->size; + continue; + + case WT_PROJ_VALUE: + c = cp[arg]; + WT_RET(__pack_init(session, &pack, c->value_format)); + buf = &c->value; + p = (uint8_t *)buf->data; + end = p + buf->size; + continue; + } + + /* + * Otherwise, the argument is a count, where a missing + * count means a count of 1. + */ + for (arg = (arg == 0) ? 1 : arg; arg > 0; arg--) { + switch (*proj) { + case WT_PROJ_NEXT: + case WT_PROJ_SKIP: + WT_RET(__pack_next(&pack, &pv)); + WT_RET(__unpack_read(session, &pv, + (const uint8_t **)&p, + (size_t)(end - p))); + if (*proj == WT_PROJ_SKIP) + break; + + WT_RET(__pack_next(&vpack, &vpv)); + vpv.u = pv.u; + len = __pack_size(session, &vpv); + WT_RET(__wt_buf_grow(session, + value, value->size + len)); + vp = (uint8_t *)value->data + value->size; + WT_RET(__pack_write(session, &vpv, &vp, len)); + value->size += WT_STORE_SIZE(len); + /* FALLTHROUGH */ + + case WT_PROJ_REUSE: + /* Don't copy the same value twice. */ + break; + } + } + } + + return (0); +} diff --git a/src/schema/schema_rename.c b/src/schema/schema_rename.c new file mode 100644 index 00000000000..ccae7a7ec2f --- /dev/null +++ b/src/schema/schema_rename.c @@ -0,0 +1,233 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __rename_file -- + * WT_SESSION::rename for a file. + */ +static int +__rename_file( + WT_SESSION_IMPL *session, const char *oldname, const char *newname) +{ + static const char *list[] = { "file", "root", "version", NULL }; + WT_ITEM *buf; + int exist, ret; + const char *value, **lp; + + buf = NULL; + value = NULL; + ret = 0; + + /* If open, close the btree handle. */ + WT_RET(__wt_session_close_any_open_btree(session, oldname)); + + /* + * Check to see if the proposed name is already in use, in either + * the schema table or the filesystem. + */ + WT_ERR(__wt_scr_alloc(session, 0, &buf)); + WT_ERR(__wt_buf_fmt(session, buf, "file:%s", newname)); + switch (ret = __wt_schema_table_read(session, buf->data, &value)) { + case 0: + WT_ERR_MSG(session, EEXIST, "%s", (char *)buf->data); + case WT_NOTFOUND: + ret = 0; + break; + default: + WT_ERR(ret); + } + WT_ERR(__wt_exist(session, newname, &exist)); + if (exist) + WT_ERR_MSG(session, EEXIST, "%s", newname); + + /* Replace the old file entries with new file entries. */ + for (lp = list; *lp != NULL; ++lp) { + WT_ERR(__wt_buf_fmt(session, buf, "%s:%s", *lp, oldname)); + WT_ERR(__wt_schema_table_read(session, buf->data, &value)); + WT_ERR(__wt_schema_table_remove(session, buf->data)); + WT_ERR(__wt_buf_fmt(session, buf, "%s:%s", *lp, newname)); + WT_ERR(__wt_schema_table_insert(session, buf->data, value)); + } + + /* Rename the underlying file. */ + WT_ERR(__wt_schema_table_track_fileop(session, oldname, newname)); + WT_ERR(__wt_rename(session, oldname, newname)); + +err: __wt_scr_free(&buf); + __wt_free(session, value); + + return (ret); +} + +/* + * __rename_tree -- + * Rename an index or colgroup reference. + */ +static int +__rename_tree(WT_SESSION_IMPL *session, WT_BTREE *btree, const char *newname) +{ + WT_ITEM *of, *nf, *nk, *nv; + int ret; + const char *p, *t, *value; + + nf = nk = nv = of = NULL; + ret = 0; + + /* Read the old schema value. */ + WT_ERR(__wt_schema_table_read(session, btree->name, &value)); + + /* + * Create the new file name, new schema key, new schema value. + * + * Names are of the form "prefix.oldname:suffix", where suffix is + * optional; we need prefix and suffix. + */ + if ((p = strchr(btree->name, ':')) == NULL) + WT_ERR_MSG(session, EINVAL, + "invalid index or column-group name: %s", btree->name); + t = strchr(p + 1, ':'); + + WT_ERR(__wt_scr_alloc(session, 0, &nf)); + WT_ERR(__wt_buf_fmt(session, nf, "%s%s%s.wt", newname, + t == NULL ? "" : "_", t == NULL ? "" : t + 1)); + + WT_ERR(__wt_scr_alloc(session, 0, &nk)); + WT_ERR(__wt_buf_fmt(session, nk, "%.*s:%s%s%s", + (int)WT_PTRDIFF(p, btree->name), btree->name, + newname, t == NULL ? "" : ":", t == NULL ? "" : t + 1)); + + WT_ERR(__wt_scr_alloc(session, 0, &nv)); + if ((p = strstr(value, "filename=")) == NULL) + WT_ERR_MSG(session, EINVAL, + "index or column-group value has no file name: %s", value); + t = strchr(p, ','); + WT_ERR(__wt_buf_fmt(session, nv, "%.*s" "filename=%s%s", + (int)WT_PTRDIFF(p, value), value, + (char *)nf->data, t == NULL ? "" : t)); + + /* + * Remove the old schema table entry + * Insert the new schema table entry + */ + WT_ERR(__wt_schema_table_remove(session, btree->name)); + WT_ERR(__wt_schema_table_insert(session, nk->data, nv->data)); + + /* + * Rename the file. + * __rename_file closes the WT_BTREE handle, so we have to have a local + * copy of the WT_BTREE->filename field. + */ + WT_ERR(__wt_scr_alloc(session, 0, &of)); + WT_ERR(__wt_buf_set( + session, of, btree->filename, strlen(btree->filename) + 1)); + WT_ERR(__rename_file(session, of->data, nf->data)); + +err: __wt_scr_free(&nf); + __wt_scr_free(&nk); + __wt_scr_free(&nv); + __wt_scr_free(&of); + __wt_free(session, value); + return (ret); +} + +/* + * __rename_table -- + * WT_SESSION::rename for a table. + */ +static int +__rename_table( + WT_SESSION_IMPL *session, const char *oldname, const char *newname) +{ + WT_BTREE *btree; + WT_ITEM *buf; + WT_TABLE *table; + int i, ret; + const char *value; + + buf = NULL; + ret = 0; + + WT_RET( + __wt_schema_get_table(session, oldname, strlen(oldname), &table)); + + /* Rename the column groups. */ + for (i = 0; i < WT_COLGROUPS(table); i++) { + if ((btree = table->colgroup[i]) == NULL) + continue; + table->colgroup[i] = NULL; + WT_RET(__rename_tree(session, btree, newname)); + } + + /* Rename the indices. */ + WT_RET(__wt_schema_open_index(session, table, NULL, 0)); + for (i = 0; i < table->nindices; i++) { + btree = table->index[i]; + table->index[i] = NULL; + WT_RET(__rename_tree(session, btree, newname)); + } + + WT_RET(__wt_schema_remove_table(session, table)); + + /* Rename the table. */ + WT_ERR(__wt_scr_alloc(session, 0, &buf)); + WT_ERR(__wt_buf_fmt(session, buf, "table:%s", oldname)); + WT_ERR(__wt_schema_table_read(session, buf->data, &value)); + WT_ERR(__wt_schema_table_remove(session, buf->data)); + WT_ERR(__wt_buf_fmt(session, buf, "table:%s", newname)); + WT_ERR(__wt_schema_table_insert(session, buf->data, value)); + +err: __wt_scr_free(&buf); + return (ret); +} + +/* + * __wt_schema_rename -- + * WT_SESSION::rename. + */ +int +__wt_schema_rename(WT_SESSION_IMPL *session, + const char *uri, const char *newuri, const char *cfg[]) +{ + const char *oldname, *newname; + int ret; + + WT_UNUSED(cfg); + + /* Disallow renames to/from the WiredTiger name space. */ + WT_RET(__wt_schema_name_check(session, uri)); + WT_RET(__wt_schema_name_check(session, newuri)); + + /* + * We track rename operations, if we fail in the middle, we want to + * back it all out. + */ + WT_RET(__wt_schema_table_track_on(session)); + + oldname = uri; + newname = newuri; + if (WT_PREFIX_SKIP(oldname, "file:")) { + if (!WT_PREFIX_SKIP(newname, "file:")) + WT_RET_MSG(session, EINVAL, + "rename target type must match URI: %s to %s", + uri, newuri); + ret = __rename_file(session, oldname, newname); + } else if (WT_PREFIX_SKIP(oldname, "table:")) { + if (!WT_PREFIX_SKIP(newname, "table:")) + WT_RET_MSG(session, EINVAL, + "rename target type must match URI: %s to %s", + uri, newuri); + ret = __rename_table(session, oldname, newname); + } else + return (__wt_unknown_object_type(session, uri)); + + WT_TRET(__wt_schema_table_track_off(session, ret != 0)); + + /* If we didn't find a schema file entry, map that error to ENOENT. */ + return (ret == WT_NOTFOUND ? ENOENT : ret); +} diff --git a/src/schema/schema_table.c b/src/schema/schema_table.c new file mode 100644 index 00000000000..a0666cb8480 --- /dev/null +++ b/src/schema/schema_table.c @@ -0,0 +1,155 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static const char *schematab_config = "key_format=S,value_format=S"; + +/* + * __open_schema_table -- + * Opens the schema table, sets session->schematab. + */ +static inline int +__open_schema_table(WT_SESSION_IMPL *session) +{ + const char *cfg[] = API_CONF_DEFAULTS(file, meta, schematab_config); + const char *schemaconf; + int ret, tracking; + + ret = 0; + + if (session->schematab != NULL) + return (0); + + WT_RET(__wt_config_collapse(session, cfg, &schemaconf)); + + /* + * Turn off tracking when creating the schema file: this is always done + * before any other schema operations and there is no going back. + */ + tracking = (session->schema_track != NULL); + if (tracking) + __wt_schema_table_track_off(session, 0); + WT_ERR(__wt_create_file(session, + "file:" WT_SCHEMA_FILENAME, + "file:" WT_SCHEMA_FILENAME, schemaconf)); + session->schematab = session->btree; +err: __wt_free(session, schemaconf); + if (tracking) + WT_TRET(__wt_schema_table_track_on(session)); + return (ret); +} + +/* + * __wt_schema_table_cursor -- + * Opens a cursor on the schema table. + */ +int +__wt_schema_table_cursor( + WT_SESSION_IMPL *session, const char *config, WT_CURSOR **cursorp) +{ + const char *cfg[] = API_CONF_DEFAULTS(session, open_cursor, config); + + WT_RET(__open_schema_table(session)); + session->btree = session->schematab; + return (__wt_curfile_create(session, cfg, cursorp)); +} + +/* + * __wt_schema_table_insert -- + * Insert a row into the schema table. + */ +int +__wt_schema_table_insert( + WT_SESSION_IMPL *session, const char *key, const char *value) +{ + WT_CURSOR *cursor; + int ret; + + ret = 0; + + if (session->schema_track != NULL) /* Optional tracking */ + WT_RET(__wt_schema_table_track_insert(session, key)); + + WT_RET(__wt_schema_table_cursor(session, NULL, &cursor)); + cursor->set_key(cursor, key); + cursor->set_value(cursor, value); + WT_TRET(cursor->insert(cursor)); + WT_TRET(cursor->close(cursor)); + return (ret); +} + +/* + * __wt_schema_table_update -- + * Update a row in the schema table. + */ +int +__wt_schema_table_update( + WT_SESSION_IMPL *session, const char *key, const char *value) +{ + WT_CURSOR *cursor; + int ret; + + ret = 0; + + if (session->schema_track != NULL) /* Optional tracking */ + WT_RET(__wt_schema_table_track_update(session, key)); + + WT_RET(__wt_schema_table_cursor(session, "overwrite", &cursor)); + cursor->set_key(cursor, key); + cursor->set_value(cursor, value); + WT_TRET(cursor->insert(cursor)); + WT_TRET(cursor->close(cursor)); + return (ret); +} + +/* + * __wt_schema_table_remove -- + * Removes a row from the schema table. + */ +int +__wt_schema_table_remove(WT_SESSION_IMPL *session, const char *key) +{ + WT_CURSOR *cursor; + int ret; + + ret = 0; + + if (session->schema_track != NULL) /* Optional tracking */ + WT_RET(__wt_schema_table_track_update(session, key)); + + WT_RET(__wt_schema_table_cursor(session, NULL, &cursor)); + cursor->set_key(cursor, key); + WT_TRET(cursor->remove(cursor)); + WT_TRET(cursor->close(cursor)); + return (ret); +} + +/* + * __wt_schema_table_read -- + * Reads and copies a row from the schema table. + * The caller is responsible for freeing the allocated memory. + */ +int +__wt_schema_table_read( + WT_SESSION_IMPL *session, const char *key, const char **valuep) +{ + WT_CURSOR *cursor; + const char *value; + int ret; + + ret = 0; + + WT_RET(__wt_schema_table_cursor(session, NULL, &cursor)); + cursor->set_key(cursor, key); + WT_ERR(cursor->search(cursor)); + WT_ERR(cursor->get_value(cursor, &value)); + WT_ERR(__wt_strdup(session, value, valuep)); + +err: WT_TRET(cursor->close(cursor)); + return (ret); +} diff --git a/src/schema/schema_track.c b/src/schema/schema_track.c new file mode 100644 index 00000000000..e903cb9e5ec --- /dev/null +++ b/src/schema/schema_track.c @@ -0,0 +1,227 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * WT_SCHEMA_TRACK -- + * A tracked schema operation: a non-transactional log, maintained to make + * it easy to unroll simple schema table and filesystem operations. + */ +typedef struct __wt_schema_track { + enum { + WT_ST_EMPTY=0, /* Unused slot */ + WT_ST_FILEOP=1, /* File operation */ + WT_ST_REMOVE=2, /* Remove a schema table entry */ + WT_ST_SET=3 /* Reset a schema table entry */ + } op; + const char *a, *b; /* Strings */ +} WT_SCHEMA_TRACK; + +/* + * __schema_table_track_next -- + * Return the next slot, and extend the list of operations we're tracking, + * as necessary. + */ +static int +__schema_table_track_next(WT_SESSION_IMPL *session, WT_SCHEMA_TRACK **trkp) +{ + WT_SCHEMA_TRACK *trk; + size_t bytes_allocated; + u_int i; + + /* + * Slow, but we don't care -- it's a schema table op, searching an array + * of maybe 20 items. + */ + for (trk = session->schema_track, + i = 0; i < session->schema_track_entries; ++trk, ++i) + if (trk->op == WT_ST_EMPTY) { + if (trkp != NULL) + *trkp = trk; + return (0); + } + + /* + * The __wt_realloc() function uses the "bytes allocated" value + * to figure out how much of the memory it needs to clear (see + * the function for an explanation of why the memory is cleared, + * it's a security thing). + */ + bytes_allocated = + session->schema_track_entries * sizeof(WT_SCHEMA_TRACK); + WT_RET(__wt_realloc(session, &bytes_allocated, + (session->schema_track_entries + 20) * sizeof(WT_SCHEMA_TRACK), + &session->schema_track)); + if (trkp != NULL) + *trkp = &((WT_SCHEMA_TRACK *) + session->schema_track)[session->schema_track_entries]; + session->schema_track_entries += 20; + return (0); +} + +/* + * __wt_schema_table_track_on -- + * Turn on schema table tracking. + */ +int +__wt_schema_table_track_on(WT_SESSION_IMPL *session) +{ + return (__schema_table_track_next(session, NULL)); +} + +/* + * __wt_schema_table_track_off -- + * Turn off schema table tracking, unrolling on error. + */ +int +__wt_schema_table_track_off(WT_SESSION_IMPL *session, int unroll) +{ + WT_SCHEMA_TRACK *trk, *trk_orig; + int ret, tret; + + ret = 0; + + if (session->schema_track == NULL || session->schema_track_entries == 0) + return (0); + + trk_orig = session->schema_track; + trk = &trk_orig[session->schema_track_entries - 1]; + + /* Turn off tracking for unroll. */ + session->schema_track = NULL; + session->schema_track_entries = 0; + + for (;; --trk) { + if (unroll) + switch (trk->op) { + case WT_ST_EMPTY: /* Unused slot */ + break; + case WT_ST_FILEOP: /* File operation */ + /* + * For renames, both a and b are set. + * For creates, a is NULL. + * For removes, b is NULL. + */ + if (trk->a != NULL && trk->b != NULL && + (tret = __wt_rename( + session, trk->b, trk->a)) != 0) { + __wt_err(session, tret, + "schema table unroll rename " + "%s to %s", + trk->b, trk->a); + WT_TRET(tret); + } else if (trk->a == NULL && + ((tret = __wt_session_close_any_open_btree( + session, trk->b)) != 0 || (tret = + __wt_remove(session, trk->b)) != 0)) { + __wt_err(session, tret, + "schema table unroll create %s", + trk->b); + WT_TRET(tret); + } + /* + * We can't undo removes yet: that would imply + * some kind of temporary rename and remove in + * roll forward. + */ + break; + case WT_ST_REMOVE: /* Remove trk.a */ + if ((tret = __wt_schema_table_remove( + session, trk->a)) != 0) { + __wt_err(session, ret, + "schema table unroll remove: %s", + trk->a); + WT_TRET(tret); + } + break; + case WT_ST_SET: /* Set trk.a to trk.b */ + if ((tret = __wt_schema_table_update( + session, trk->a, trk->b)) != 0) { + __wt_err(session, ret, + "schema table unroll update " + "%s to %s", + trk->a, trk->b); + WT_TRET(tret); + } + break; + WT_ILLEGAL_VALUE(session); + } + + __wt_free(session, trk->a); + __wt_free(session, trk->b); + + if (trk == trk_orig) + break; + } + __wt_free(session, trk_orig); + return (ret); +} + +/* + * __wt_schema_table_track_insert -- + * Track an insert operation. + */ +int +__wt_schema_table_track_insert(WT_SESSION_IMPL *session, const char *key) +{ + WT_SCHEMA_TRACK *trk; + + WT_RET(__schema_table_track_next(session, &trk)); + + trk->op = WT_ST_REMOVE; + WT_RET(__wt_strdup(session, key, &trk->a)); + + return (0); +} + +/* + * __wt_schema_table_track_update -- + * Track a schema table update operation. + */ +int +__wt_schema_table_track_update(WT_SESSION_IMPL *session, const char *key) +{ + WT_SCHEMA_TRACK *trk; + int ret; + + WT_RET(__schema_table_track_next(session, &trk)); + + trk->op = WT_ST_SET; + WT_RET(__wt_strdup(session, key, &trk->a)); + + /* + * If there was a previous value, keep it around -- if not, then this + * "update" is really an insert. + */ + if ((ret = + __wt_schema_table_read(session, key, &trk->b)) == WT_NOTFOUND) { + trk->op = WT_ST_REMOVE; + ret = 0; + } + return (ret); +} + +/* + * __wt_schema_table_track_fs_rename -- + * Track a filesystem rename operation. + */ +int +__wt_schema_table_track_fileop( + WT_SESSION_IMPL *session, const char *oldname, const char *newname) +{ + WT_SCHEMA_TRACK *trk; + + WT_RET(__schema_table_track_next(session, &trk)); + + trk->op = WT_ST_FILEOP; + if (oldname != NULL) + WT_RET(__wt_strdup(session, oldname, &trk->a)); + if (newname != NULL) + WT_RET(__wt_strdup(session, newname, &trk->b)); + return (0); +} diff --git a/src/schema/schema_truncate.c b/src/schema/schema_truncate.c new file mode 100644 index 00000000000..5402fdf3454 --- /dev/null +++ b/src/schema/schema_truncate.c @@ -0,0 +1,116 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __truncate_file -- + * WT_SESSION::truncate for a file. + */ +static int +__truncate_file(WT_SESSION_IMPL *session, const char *name) +{ + WT_ITEM *uribuf; + int ret; + + uribuf = NULL; + ret = 0; + + /* If open, close the btree handle. */ + WT_RET(__wt_session_close_any_open_btree(session, name)); + + /* Delete the root address and truncate the file. */ + WT_ERR(__wt_scr_alloc(session, 0, &uribuf)); + WT_ERR(__wt_buf_fmt(session, uribuf, "root:%s", name)); + WT_ERR(__wt_schema_table_remove(session, uribuf->data)); + + WT_ERR(__wt_btree_truncate(session, name)); + +err: __wt_scr_free(&uribuf); + return (ret); +} + +/* + * __truncate_table -- + * WT_SESSION::truncate for a table. + */ +static int +__truncate_table(WT_SESSION_IMPL *session, const char *name) +{ + WT_BTREE *btree; + WT_ITEM *buf; + WT_TABLE *table; + int i, ret; + + ret = 0; + WT_RET(__wt_scr_alloc(session, 0, &buf)); + + WT_RET(__wt_schema_get_table(session, name, strlen(name), &table)); + /* + * We are closing the column groups, they must be reopened for future + * accesses to the table. + */ + table->cg_complete = 0; + + /* Truncate the column groups. */ + for (i = 0; i < WT_COLGROUPS(table); i++) { + if ((btree = table->colgroup[i]) == NULL) + continue; + table->colgroup[i] = NULL; + /* + * Take a copy of the file name: it will be freed when the + * handle is closed. + */ + WT_ERR(__wt_buf_set(session, buf, + btree->filename, strlen(btree->filename) + 1)); + WT_TRET(__truncate_file(session, buf->data)); + } + + /* Truncate the indices. */ + WT_TRET(__wt_schema_open_index(session, table, NULL, 0)); + for (i = 0; i < table->nindices; i++) { + btree = table->index[i]; + table->index[i] = NULL; + /* + * Take a copy of the file name: it will be freed when the + * handle is closed. + */ + WT_ERR(__wt_buf_set(session, buf, + btree->filename, strlen(btree->filename) + 1)); + WT_TRET(__truncate_file(session, buf->data)); + } + + /* Reopen the column groups. */ + if (ret == 0) + ret = __wt_schema_open_colgroups(session, table); + +err: __wt_scr_free(&buf); + return (ret); +} + +/* + * __wt_schema_truncate -- + * WT_SESSION::truncate. + */ +int +__wt_schema_truncate( + WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +{ + int ret; + + WT_UNUSED(cfg); + + if (WT_PREFIX_SKIP(uri, "file:")) + ret = __truncate_file(session, uri); + else if (WT_PREFIX_SKIP(uri, "table:")) + ret = __truncate_table(session, uri); + else + return (__wt_unknown_object_type(session, uri)); + + /* If we didn't find a schema file entry, map that error to ENOENT. */ + return (ret == WT_NOTFOUND ? ENOENT : ret); +} diff --git a/src/schema/schema_util.c b/src/schema/schema_util.c new file mode 100644 index 00000000000..39053782d7d --- /dev/null +++ b/src/schema/schema_util.c @@ -0,0 +1,51 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_schema_name_check -- + * Disallow any use of the WiredTiger name space. + */ +int +__wt_schema_name_check(WT_SESSION_IMPL *session, const char *uri) +{ + const char *name, *sep; + + /* + * Check if name is somewhere in the WiredTiger name space: it would be + * "bad" if the application truncated the schema file. We get passed + * both objects and simple strings, skip any leading URI prefix. + */ + name = uri; + if (WT_PREFIX_SKIP(name, "colgroup:") || + WT_PREFIX_SKIP(name, "index:")) { + /* These URIs normally reference a table name. */ + if ((sep = strchr(name, ':')) != NULL) + name = sep + 1; + } else if (!WT_PREFIX_SKIP(name, "table:") && + !WT_PREFIX_SKIP(name, "file:")) + return (__wt_unknown_object_type(session, uri)); + + if (WT_PREFIX_MATCH(name, "WiredTiger")) + WT_RET_MSG(session, EINVAL, + "%s: the \"WiredTiger\" name space may not be used by " + "applications", + name); + + /* + * Disallow JSON quoting characters -- the config string parsing code + * supports quoted strings, but there's no good reason to use them in + * names and we're not going to do the testing. + */ + if (strpbrk(name, "{},:[]\\\"'") != NULL) + WT_RET_MSG(session, EINVAL, + "%s: WiredTiger objects should not include grouping " + "characters in their names", + name); + return (0); +} diff --git a/src/schema/schema_worker.c b/src/schema/schema_worker.c new file mode 100644 index 00000000000..908c9e8ecdf --- /dev/null +++ b/src/schema/schema_worker.c @@ -0,0 +1,60 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_schema_worker -- + * Get Btree handles for the object and cycle through calls to an + * underlying worker function with each handle. + */ +int +__wt_schema_worker(WT_SESSION_IMPL *session, + const char *uri, const char *cfg[], + int (*func)(WT_SESSION_IMPL *, const char *[]), uint32_t open_flags) +{ + WT_BTREE *cg; + WT_TABLE *table; + const char *tablename; + int i, ret; + + tablename = uri; + ret = 0; + + /* Get the btree handle(s) and call the underlying function. */ + if (WT_PREFIX_MATCH(uri, "file:")) { + WT_RET(__wt_session_get_btree( + session, uri, uri, NULL, cfg, open_flags)); + ret = func(session, cfg); + WT_TRET(__wt_session_release_btree(session)); + WT_RET(ret); + } else if ( + WT_PREFIX_SKIP(uri, "colgroup:") || WT_PREFIX_SKIP(uri, "index:")) { + WT_RET(__wt_schema_get_btree( + session, uri, strlen(uri), cfg, 0)); + ret = func(session, cfg); + WT_TRET(__wt_session_release_btree(session)); + WT_RET(ret); + } else if (WT_PREFIX_SKIP(tablename, "table:")) { + WT_RET(__wt_schema_get_table(session, + tablename, strlen(tablename), &table)); + + for (i = 0; i < WT_COLGROUPS(table); i++) { + if ((cg = table->colgroup[i]) == NULL) + continue; + + WT_TRET(__wt_schema_get_btree(session, + cg->name, strlen(cg->name), cfg, 0)); + ret = func(session, cfg); + WT_TRET(__wt_session_release_btree(session)); + WT_RET(ret); + } + } else + return (__wt_unknown_object_type(session, uri)); + + return (ret); +} diff --git a/src/session/session_api.c b/src/session/session_api.c new file mode 100644 index 00000000000..368a446b0b0 --- /dev/null +++ b/src/session/session_api.c @@ -0,0 +1,504 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __session_close -- + * WT_SESSION->close method. + */ +static int +__session_close(WT_SESSION *wt_session, const char *config) +{ + WT_BTREE_SESSION *btree_session; + WT_CONNECTION_IMPL *conn; + WT_CURSOR *cursor; + WT_SESSION_IMPL *session, **tp; + int ret; + + conn = (WT_CONNECTION_IMPL *)wt_session->connection; + session = (WT_SESSION_IMPL *)wt_session; + ret = 0; + + SESSION_API_CALL(session, close, config, cfg); + WT_UNUSED(cfg); + + /* + * We first close all public cursors, then all remaining file cursors. + * File cursors are special because the btree code looks inside them + * to work out whether pages are in use. + */ + while ((cursor = TAILQ_FIRST(&session->public_cursors)) != NULL) + WT_TRET(cursor->close(cursor)); + + while ((cursor = TAILQ_FIRST(&session->file_cursors)) != NULL) + WT_TRET(cursor->close(cursor)); + + while ((btree_session = TAILQ_FIRST(&session->btrees)) != NULL) + WT_TRET(__wt_session_remove_btree(session, btree_session)); + + WT_TRET(__wt_schema_close_tables(session)); + + __wt_spin_lock(session, &conn->spinlock); + + /* Discard scratch buffers. */ + __wt_scr_discard(session); + + /* Confirm we're not holding any hazard references. */ + __wt_hazard_empty(session); + + /* Free the reconciliation information. */ + __wt_rec_destroy(session); + + /* Free the eviction exclusive-lock information. */ + __wt_free(session, session->excl); + + /* Destroy the thread's mutex. */ + if (session->cond != NULL) + (void)__wt_cond_destroy(session, session->cond); + + /* + * Replace the session reference we're closing with the last entry in + * the table, then clear the last entry. As far as the walk of the + * server threads is concerned, it's OK if the session appears twice, + * or if it doesn't appear at all, so these lines can race all they + * want. + */ + for (tp = conn->sessions; *tp != session; ++tp) + ; + --conn->session_cnt; + *tp = conn->sessions[conn->session_cnt]; + conn->sessions[conn->session_cnt] = NULL; + + /* + * Publish, making the session array entry available for re-use. There + * must be a barrier here to ensure the cleanup above completes before + * the entry is re-used. + */ + WT_PUBLISH(session->iface.connection, NULL); + + session = &conn->default_session; + __wt_spin_unlock(session, &conn->spinlock); +err: API_END(session); + + return (ret); +} + +/* + * __session_open_cursor -- + * WT_SESSION->open_cursor method. + */ +static int +__session_open_cursor(WT_SESSION *wt_session, + const char *uri, WT_CURSOR *to_dup, const char *config, WT_CURSOR **cursorp) +{ + WT_SESSION_IMPL *session; + int ret; + + WT_UNUSED(to_dup); + + session = (WT_SESSION_IMPL *)wt_session; + SESSION_API_CALL(session, open_cursor, config, cfg); + + if (WT_PREFIX_MATCH(uri, "colgroup:")) + ret = __wt_curfile_open(session, uri, cfg, cursorp); + else if (WT_PREFIX_MATCH(uri, "config:")) + ret = __wt_curconfig_open(session, uri, cfg, cursorp); + else if (WT_PREFIX_MATCH(uri, "file:")) + ret = __wt_curfile_open(session, uri, cfg, cursorp); + else if (WT_PREFIX_MATCH(uri, "index:")) + ret = __wt_curindex_open(session, uri, cfg, cursorp); + else if (WT_PREFIX_MATCH(uri, "statistics:")) + ret = __wt_curstat_open(session, uri, cfg, cursorp); + else if (WT_PREFIX_MATCH(uri, "table:")) + ret = __wt_curtable_open(session, uri, cfg, cursorp); + else { + __wt_err(session, EINVAL, "Unknown cursor type '%s'", uri); + ret = EINVAL; + } + +err: API_END(session); + return (ret); +} + +/* + * __session_create -- + * WT_SESSION->create method. + */ +static int +__session_create(WT_SESSION *wt_session, const char *name, const char *config) +{ + WT_SESSION_IMPL *session; + int ret; + + session = (WT_SESSION_IMPL *)wt_session; + ret = 0; + SESSION_API_CALL(session, create, config, cfg); + WT_UNUSED(cfg); + WT_ERR(__wt_schema_create(session, name, config)); +err: API_END(session); + return (ret); +} + +/* + * __session_rename -- + * WT_SESSION->rename method. + */ +static int +__session_rename(WT_SESSION *wt_session, + const char *uri, const char *newname, const char *config) +{ + WT_SESSION_IMPL *session; + int ret; + + session = (WT_SESSION_IMPL *)wt_session; + SESSION_API_CALL(session, rename, config, cfg); + ret = __wt_schema_rename(session, uri, newname, cfg); +err: API_END(session); + return (ret); +} + +/* + * __session_drop -- + * WT_SESSION->drop method. + */ +static int +__session_drop(WT_SESSION *wt_session, const char *name, const char *config) +{ + WT_SESSION_IMPL *session; + WT_CONFIG_ITEM cval; + int force, ret; + + force = 0; + + session = (WT_SESSION_IMPL *)wt_session; + + SESSION_API_CALL(session, drop, config, cfg); + + WT_ERR(__wt_config_gets(session, cfg, "force", &cval)); + force = (cval.val != 0); + + ret = __wt_schema_drop(session, name, cfg); +err: API_END(session); + + return (force ? 0 : ret); +} + +/* + * __session_dumpfile -- + * WT_SESSION->dumpfile method. + */ +static int +__session_dumpfile(WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_SESSION_IMPL *session; + int ret; + + session = (WT_SESSION_IMPL *)wt_session; + ret = 0; + + SESSION_API_CALL(session, dumpfile, config, cfg); + ret = __wt_schema_worker(session, uri, cfg, + __wt_dumpfile, WT_BTREE_EXCLUSIVE | WT_BTREE_VERIFY); +err: API_END(session); + return (ret); +} + +/* + * __session_salvage -- + * WT_SESSION->salvage method. + */ +static int +__session_salvage(WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_SESSION_IMPL *session; + int ret; + + session = (WT_SESSION_IMPL *)wt_session; + + SESSION_API_CALL(session, salvage, config, cfg); + ret = __wt_schema_worker(session, uri, cfg, + __wt_salvage, WT_BTREE_EXCLUSIVE | WT_BTREE_SALVAGE); +err: API_END(session); + return (ret); +} + +/* + * __session_sync -- + * WT_SESSION->sync method. + */ +static int +__session_sync(WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_SESSION_IMPL *session; + int ret; + + session = (WT_SESSION_IMPL *)wt_session; + + SESSION_API_CALL(session, sync, config, cfg); + ret = __wt_schema_worker(session, uri, cfg, __wt_btree_sync, 0); +err: API_END(session); + return (ret); +} + +/* + * __session_truncate -- + * WT_SESSION->truncate method. + */ +static int +__session_truncate(WT_SESSION *wt_session, + const char *uri, WT_CURSOR *start, WT_CURSOR *stop, const char *config) +{ + WT_SESSION_IMPL *session; + int ret; + + session = (WT_SESSION_IMPL *)wt_session; + + SESSION_API_CALL(session, truncate, config, cfg); + /* + * If the URI is specified, we don't need a start/stop, if start/stop + * is specified, we don't need a URI. + */ + if ((uri == NULL && start == NULL && stop == NULL) || + (uri != NULL && (start != NULL || stop != NULL))) + WT_ERR_MSG(session, EINVAL, + "the truncate method should be passed either a URI or " + "start/stop cursors, but not both"); + if (uri == NULL) { + /* + * From a starting/stopping cursor to the begin/end of the + * object is easy, walk the object. + */ + if (start == NULL) + for (;;) { + WT_ERR(stop->remove(stop)); + if ((ret = stop->prev(stop)) != 0) { + if (ret == WT_NOTFOUND) + ret = 0; + break; + } + } + else + for (;;) { + WT_ERR(start->remove(start)); + if (stop != NULL && + start->equals(start, stop)) + break; + if ((ret = start->next(start)) != 0) { + if (ret == WT_NOTFOUND) + ret = 0; + break; + } + } + } else + ret = __wt_schema_truncate(session, uri, cfg); +err: API_END(session); + return (ret); +} + +/* + * __session_upgrade -- + * WT_SESSION->upgrade method. + */ +static int +__session_upgrade(WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_SESSION_IMPL *session; + int ret; + + session = (WT_SESSION_IMPL *)wt_session; + + SESSION_API_CALL(session, upgrade, config, cfg); + ret = __wt_schema_worker(session, uri, cfg, + __wt_upgrade, WT_BTREE_EXCLUSIVE | WT_BTREE_UPGRADE); +err: API_END(session); + return (ret); +} + +/* + * __session_verify -- + * WT_SESSION->verify method. + */ +static int +__session_verify(WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_SESSION_IMPL *session; + int ret; + + session = (WT_SESSION_IMPL *)wt_session; + + SESSION_API_CALL(session, verify, config, cfg); + ret = __wt_schema_worker(session, uri, cfg, + __wt_verify, WT_BTREE_EXCLUSIVE | WT_BTREE_VERIFY); +err: API_END(session); + return (ret); +} + +/* + * __session_begin_transaction -- + * WT_SESSION->begin_transaction method. + */ +static int +__session_begin_transaction(WT_SESSION *wt_session, const char *config) +{ + WT_UNUSED(wt_session); + WT_UNUSED(config); + + return (ENOTSUP); +} + +/* + * __session_commit_transaction -- + * WT_SESSION->commit_transaction method. + */ +static int +__session_commit_transaction(WT_SESSION *wt_session, const char *config) +{ + WT_UNUSED(wt_session); + WT_UNUSED(config); + + return (ENOTSUP); +} + +/* + * __session_rollback_transaction -- + * WT_SESSION->rollback_transaction method. + */ +static int +__session_rollback_transaction(WT_SESSION *wt_session, const char *config) +{ + WT_UNUSED(wt_session); + WT_UNUSED(config); + + return (ENOTSUP); +} + +/* + * __session_checkpoint -- + * WT_SESSION->checkpoint method. + */ +static int +__session_checkpoint(WT_SESSION *wt_session, const char *config) +{ + WT_UNUSED(wt_session); + WT_UNUSED(config); + + return (ENOTSUP); +} + +/* + * __session_msg_printf -- + * WT_SESSION->msg_printf method. + */ +static int +__session_msg_printf(WT_SESSION *wt_session, const char *fmt, ...) +{ + WT_SESSION_IMPL *session; + va_list ap; + + session = (WT_SESSION_IMPL *)wt_session; + + va_start(ap, fmt); + __wt_msgv(session, fmt, ap); + va_end(ap); + + return (0); +} + +/* + * __wt_open_session -- + * Allocate a session handle. The internal parameter is used for sessions + * opened by WiredTiger for its own use. + */ +int +__wt_open_session(WT_CONNECTION_IMPL *conn, int internal, + WT_EVENT_HANDLER *event_handler, const char *config, + WT_SESSION_IMPL **sessionp) +{ + static WT_SESSION stds = { + NULL, + __session_close, + __session_open_cursor, + __session_create, + __session_drop, + __session_rename, + __session_salvage, + __session_sync, + __session_truncate, + __session_upgrade, + __session_verify, + __session_begin_transaction, + __session_commit_transaction, + __session_rollback_transaction, + __session_checkpoint, + __session_dumpfile, + __session_msg_printf + }; + WT_SESSION_IMPL *session, *session_ret; + uint32_t slot; + int ret; + + WT_UNUSED(config); + ret = 0; + session = &conn->default_session; + session_ret = NULL; + + __wt_spin_lock(session, &conn->spinlock); + + /* Check to see if there's an available session slot. */ + if (conn->session_cnt == conn->session_size - 1) + WT_ERR_MSG(session, WT_ERROR, + "WiredTiger only configured to support %d thread contexts", + conn->session_size); + + /* + * The session reference list is compact, the session array is not. + * Find the first empty session slot. + */ + for (slot = 0, session_ret = conn->session_array; + session_ret->iface.connection != NULL; + ++session_ret, ++slot) + ; + + /* Session entries are re-used, clear the old contents. */ + WT_CLEAR(*session_ret); + + WT_ERR(__wt_cond_alloc(session, "session", 1, &session_ret->cond)); + session_ret->iface = stds; + session_ret->iface.connection = &conn->iface; + WT_ASSERT(session, session->event_handler != NULL); + session_ret->event_handler = session->event_handler; + session_ret->hazard = conn->hazard + slot * conn->hazard_size; + + TAILQ_INIT(&session_ret->file_cursors); + TAILQ_INIT(&session_ret->public_cursors); + TAILQ_INIT(&session_ret->btrees); + if (event_handler != NULL) + session_ret->event_handler = event_handler; + + /* + * Public sessions are automatically closed during WT_CONNECTION->close. + * If the session handles for internal threads were to go on the public + * list, there would be complex ordering issues during close. Set a + * flag to avoid this: internal sessions are not closed automatically. + */ + if (internal) + F_SET(session_ret, WT_SESSION_INTERNAL); + + /* + * Publish: make the entry visible to server threads. There must be a + * barrier to ensure the structure fields are set before any other + * thread can see the session. + */ + WT_PUBLISH(conn->sessions[conn->session_cnt++], session_ret); + + STATIC_ASSERT(offsetof(WT_CONNECTION_IMPL, iface) == 0); + *sessionp = session_ret; + +err: __wt_spin_unlock(session, &conn->spinlock); + return (ret); +} diff --git a/src/session/session_btree.c b/src/session/session_btree.c new file mode 100644 index 00000000000..f8df5843b62 --- /dev/null +++ b/src/session/session_btree.c @@ -0,0 +1,232 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_session_add_btree -- + * Add a btree handle to the session's cache. + */ +int +__wt_session_add_btree( + WT_SESSION_IMPL *session, WT_BTREE_SESSION **btree_sessionp) +{ + WT_BTREE_SESSION *btree_session; + + WT_RET(__wt_calloc_def(session, 1, &btree_session)); + btree_session->btree = session->btree; + + TAILQ_INSERT_HEAD(&session->btrees, btree_session, q); + + if (btree_sessionp != NULL) + *btree_sessionp = btree_session; + + return (0); +} + +/* + * __wt_session_has_btree -- + * Check whether a tree is already open in a session's cache. + */ +int +__wt_session_has_btree(WT_SESSION_IMPL *session, WT_BTREE *btree) +{ + WT_BTREE_SESSION *btree_session; + + TAILQ_FOREACH(btree_session, &session->btrees, q) + if (btree_session->btree == btree) + return (0); + + return (WT_NOTFOUND); +} + +/* + * __wt_session_lock_btree -- + * Lock a btree handle. + */ +int +__wt_session_lock_btree( + WT_SESSION_IMPL *session, const char *cfg[], uint32_t flags) +{ + WT_BTREE *btree; + + btree = session->btree; + WT_ASSERT(session, btree != NULL); + + if (LF_ISSET(WT_BTREE_EXCLUSIVE)) { + /* + * Try to get an exclusive handle lock and fail immediately if + * it unavailable. We don't expect exclusive operations on + * trees to be mixed with ordinary cursor access, but if there + * is a use case in the future, we could make blocking here + * configurable. + */ + WT_RET(__wt_try_writelock(session, btree->rwlock)); + + /* + * Check if the handle needs to be reopened. If the handle was + * just opened, cfg is NULL, so there is no need to reopen in + * that case. + * + * We do need to pick up the flags anyway, for example to set + * WT_BTREE_BULK so the handle is closed correctly. + */ + if (cfg != NULL && LF_ISSET(WT_BTREE_BULK | + WT_BTREE_SALVAGE | WT_BTREE_UPGRADE | WT_BTREE_VERIFY)) + return (__wt_btree_reopen(session, flags)); + else + F_SET(btree, flags); + } else if (!LF_ISSET(WT_BTREE_NO_LOCK)) + __wt_readlock(session, btree->rwlock); + + return (0); +} + +/* + * __wt_session_unlock_btree -- + * Unlock a btree handle. + */ +int +__wt_session_release_btree(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + int ret; + + btree = session->btree; + WT_ASSERT(session, btree != NULL); + ret = 0; + + /* + * If we had exclusive access, reopen the tree without special flags so + * that other threads can use it. + */ + if (F_ISSET(btree, WT_BTREE_BULK | + WT_BTREE_SALVAGE | WT_BTREE_UPGRADE | WT_BTREE_VERIFY)) { + WT_ASSERT(session, F_ISSET(btree, WT_BTREE_EXCLUSIVE)); + ret = __wt_btree_reopen(session, 0); + } else if (F_ISSET(btree, WT_BTREE_EXCLUSIVE)) + F_CLR(btree, WT_BTREE_EXCLUSIVE); + + __wt_rwunlock(session, session->btree->rwlock); + + return (ret); +} + +/* + * __wt_session_find_btree -- + * Find an open btree handle for the named table. + */ +int +__wt_session_find_btree(WT_SESSION_IMPL *session, + const char *filename, size_t namelen, const char *cfg[], uint32_t flags, + WT_BTREE_SESSION **btree_sessionp) +{ + WT_BTREE *btree; + WT_BTREE_SESSION *btree_session; + + TAILQ_FOREACH(btree_session, &session->btrees, q) { + btree = btree_session->btree; + if (strncmp(filename, btree->filename, namelen) == 0 && + btree->filename[namelen] == '\0') { + if (btree_sessionp != NULL) + *btree_sessionp = btree_session; + session->btree = btree; + return (__wt_session_lock_btree(session, cfg, flags)); + } + } + + return (WT_NOTFOUND); +} + +/* + * __wt_session_get_btree -- + * Get a btree handle for the given name, set session->btree. + */ +int +__wt_session_get_btree(WT_SESSION_IMPL *session, + const char *name, const char *fileuri, const char *tconfig, + const char *cfg[], uint32_t flags) +{ + WT_BTREE_SESSION *btree_session; + const char *filename, *treeconf; + int exist, ret; + + filename = fileuri; + if (!WT_PREFIX_SKIP(filename, "file:")) + WT_RET_MSG( + session, EINVAL, "Expected a 'file:' URI: %s", fileuri); + + if ((ret = __wt_session_find_btree(session, + filename, strlen(filename), cfg, flags, &btree_session)) == 0) { + WT_ASSERT(session, btree_session->btree != NULL); + session->btree = btree_session->btree; + return (0); + } + if (ret != WT_NOTFOUND) + return (ret); + + WT_RET(__wt_exist(session, filename, &exist)); + if (!exist) + return (WT_NOTFOUND); + + /* + * A fixed configuration is passed in for special files, such + * as the schema table itself. + */ + if (tconfig != NULL) + WT_RET(__wt_strdup(session, tconfig, &treeconf)); + else + WT_RET(__wt_schema_table_read(session, fileuri, &treeconf)); + WT_RET(__wt_btree_open(session, name, filename, treeconf, cfg, flags)); + WT_RET(__wt_session_lock_btree(session, NULL, flags)); + WT_RET(__wt_session_add_btree(session, NULL)); + + return (0); +} + +/* + * __wt_session_remove_btree -- + * Remove the btree handle from the session, closing if necessary. + */ +int +__wt_session_remove_btree( + WT_SESSION_IMPL *session, WT_BTREE_SESSION *btree_session) +{ + TAILQ_REMOVE(&session->btrees, btree_session, q); + session->btree = btree_session->btree; + __wt_free(session, btree_session); + + return (__wt_btree_close(session)); +} + +/* + * __wt_session_close_any_open_btree -- + * If open, close the btree handle. + */ +int +__wt_session_close_any_open_btree(WT_SESSION_IMPL *session, const char *name) +{ + WT_BTREE_SESSION *btree_session; + int ret; + + if ((ret = __wt_session_find_btree(session, name, strlen(name), + NULL, WT_BTREE_EXCLUSIVE, &btree_session)) == 0) { + /* + * XXX + * We have an exclusive lock, which means there are no cursors + * open but some other thread may have the handle cached. + * Fixing this will mean adding additional synchronization to + * the cursor open path. + */ + WT_ASSERT(session, btree_session->btree->refcnt == 1); + __wt_schema_detach_tree(session, btree_session->btree); + ret = __wt_session_remove_btree(session, btree_session); + } else if (ret == WT_NOTFOUND) + ret = 0; + + return (ret); +} diff --git a/src/support/err.c b/src/support/err.c new file mode 100644 index 00000000000..aac2f8ab3ea --- /dev/null +++ b/src/support/err.c @@ -0,0 +1,187 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_eventv -- + * Report a message to an event handler. + */ +void +__wt_eventv(WT_SESSION_IMPL *session, int msg_event, + int error, + const char *file_name, int line_number, const char *fmt, va_list ap) +{ + WT_EVENT_HANDLER *handler; + const char *prefix1, *prefix2; + char *end, *p; + + /* + * !!! + * SECURITY: + * Buffer placed at the end of the stack in case snprintf overflows. + */ + char s[2048]; + + p = s; + end = s + sizeof(s); + + prefix1 = (session->btree != NULL) ? session->btree->name : NULL; + prefix2 = session->name; + + if (prefix1 != NULL && prefix2 != NULL && p < end) + p += snprintf(p, (size_t)(end - p), + "%s [%s]: ", prefix1, prefix2); + else if (prefix1 != NULL && p < end) + p += snprintf(p, (size_t)(end - p), "%s: ", prefix1); + else if (prefix2 != NULL && p < end) + p += snprintf(p, (size_t)(end - p), "%s: ", prefix2); + if (file_name != NULL && p < end) + p += snprintf(p, (size_t)(end - p), + "%s, %d: ", file_name, line_number); + if (p < end) + p += vsnprintf(p, (size_t)(end - p), fmt, ap); + if (error != 0 && p < end) + p += snprintf(p, + (size_t)(end - p), ": %s", wiredtiger_strerror(error)); + + handler = session->event_handler; + if (msg_event) + (void)handler->handle_message(handler, s); + else + handler->handle_error(handler, error, s); +} + +/* + * __wt_err -- + * Report an error. + */ +void +__wt_err(WT_SESSION_IMPL *session, int error, const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 3, 4))) +{ + va_list ap; + + va_start(ap, fmt); + __wt_eventv(session, 0, error, NULL, 0, fmt, ap); + va_end(ap); +} + +/* + * __wt_errx -- + * Report an error with no error code. + */ +void +__wt_errx(WT_SESSION_IMPL *session, const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 2, 3))) +{ + va_list ap; + + va_start(ap, fmt); + __wt_eventv(session, 0, 0, NULL, 0, fmt, ap); + va_end(ap); +} + +/* + * __wt_msg_call -- + * Pass a message to an event handler. + */ +void +__wt_msgv(WT_SESSION_IMPL *session, const char *fmt, va_list ap) +{ + WT_EVENT_HANDLER *handler; + + /* + * !!! + * SECURITY: + * Buffer placed at the end of the stack in case snprintf overflows. + */ + char s[2048]; + + (void)vsnprintf(s, sizeof(s), fmt, ap); + + handler = session->event_handler; + (void)handler->handle_message(handler, s); +} + +/* + * __wt_verbose -- + * Verbose message. + */ +void +__wt_verbose(WT_SESSION_IMPL *session, const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 2, 3))) +{ + va_list ap; + + va_start(ap, fmt); + __wt_eventv(session, 1, 0, NULL, 0, fmt, ap); + va_end(ap); +} + +/* + * __wt_msg -- + * Report a message. + */ +void +__wt_msg(WT_SESSION_IMPL *session, const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 2, 3))) +{ + va_list ap; + + va_start(ap, fmt); + __wt_msgv(session, fmt, ap); + va_end(ap); +} + +/* + * __wt_assert -- + * Assert and other unexpected failures, includes file/line information + * for debugging. + */ +int +__wt_assert(WT_SESSION_IMPL *session, + int error, const char *file_name, int line_number, const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 5, 6))) +{ + va_list ap; + + va_start(ap, fmt); + __wt_eventv(session, 0, error, file_name, line_number, fmt, ap); + va_end(ap); + +#ifdef HAVE_DIAGNOSTIC + __wt_abort(session); + /* NOTREACHED */ +#endif + return (error); +} + +/* + * __wt_illegal_value -- + * Print a standard error message when we detect an illegal value. + */ +int +__wt_illegal_value(WT_SESSION_IMPL *session) +{ + WT_RET_MSG(session, WT_ERROR, + "encountered an illegal file format or internal value; restart " + "the system and verify the underlying files, if corruption is " + "detected use the WT_SESSION salvage method or the wt utility's " + "salvage command to repair the file"); +} + +/* + * __wt_unknown_object_type -- + * Print a standard error message when given an unknown object type. + */ +int +__wt_unknown_object_type(WT_SESSION_IMPL *session, const char *uri) +{ + WT_RET_MSG(session, EINVAL, + "unknown or unsupported object type: %s", uri); +} diff --git a/src/support/filename.c b/src/support/filename.c new file mode 100644 index 00000000000..a886b3f09f9 --- /dev/null +++ b/src/support/filename.c @@ -0,0 +1,30 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_filename -- + * Build a filename in a scratch buffer. + */ +int +__wt_filename(WT_SESSION_IMPL *session, const char *name, const char **path) +{ + WT_CONNECTION_IMPL *conn; + size_t len; + char *buf; + + conn = S2C(session); + *path = NULL; + + len = strlen(conn->home) + 1 + strlen(name) + 1; + WT_RET(__wt_calloc(session, 1, len, &buf)); + snprintf(buf, len, "%s/%s", conn->home, name); + + *path = buf; + return (0); +} diff --git a/src/support/global.c b/src/support/global.c new file mode 100644 index 00000000000..7f78b620006 --- /dev/null +++ b/src/support/global.c @@ -0,0 +1,82 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +WT_PROCESS __wt_process; /* Per-process structure */ +static int __wt_pthread_once_failed; /* If initialization failed */ + +static void +__wt_pthread_once(void) +{ + __wt_spin_init(NULL, &__wt_process.spinlock); + + TAILQ_INIT(&__wt_process.connqh); + +#ifdef HAVE_DIAGNOSTIC + /* Load debugging code the compiler might optimize out. */ + (void)__wt_breakpoint(); +#endif +} + +/* + * __wt_library_init -- + * Some things to do, before we do anything else. + */ +int +__wt_library_init(void) +{ + static pthread_once_t once_control = PTHREAD_ONCE_INIT; + static int first = 1; + int ret; + + /* + * Do per-process initialization once, before anything else, but only + * once. I don't know how heavy_weight pthread_once might be, so I'm + * front-ending it with a local static and only using pthread_once to + * avoid a race. + */ + if (first) { + if ((ret = pthread_once(&once_control, __wt_pthread_once)) != 0) + __wt_pthread_once_failed = ret; + first = 0; + } + return (__wt_pthread_once_failed); +} + +#ifdef HAVE_DIAGNOSTIC +/* + * __wt_breakpoint -- + * A simple place to put a breakpoint, if you need one. + */ +int +__wt_breakpoint(void) +{ + return (0); +} + +/* + * __wt_attach -- + * A routine to wait for the debugging to attach. + */ +void +__wt_attach(WT_SESSION_IMPL *session) +{ +#ifdef HAVE_ATTACH + __wt_session_dump_all(session); + + __wt_errx(session, "process ID %" PRIdMAX + ": waiting for debugger...", (intmax_t)getpid()); + + /* Sleep forever, the debugger will interrupt us when it attaches. */ + for (;;) + __wt_sleep(100, 0); +#else + WT_UNUSED(session); +#endif +} +#endif diff --git a/src/support/hazard.c b/src/support/hazard.c new file mode 100644 index 00000000000..db34bd6014d --- /dev/null +++ b/src/support/hazard.c @@ -0,0 +1,215 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +#ifdef HAVE_DIAGNOSTIC +static void __hazard_dump(WT_SESSION_IMPL *); +#endif + +/* + * __wt_hazard_set -- + * Set a hazard reference. + */ +int +__wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref +#ifdef HAVE_DIAGNOSTIC + , const char *file, int line +#endif + ) +{ + WT_CONNECTION_IMPL *conn; + WT_HAZARD *hp; + + conn = S2C(session); + + /* + * Do the dance: + * + * The memory location which makes a page "real" is the WT_REF's state + * of WT_REF_MEM, which can be set to WT_REF_LOCKED at any time by the + * page eviction server. + * + * Add the WT_REF reference to the session's hazard list and flush the + * write, then see if the state field is still WT_REF_MEM. If it's + * still WT_REF_MEM, we can use the page because the page eviction + * server will see our hazard reference before it discards the buffer + * (the eviction server sets the state to WT_REF_LOCKED, then flushes + * memory and checks the hazard references). + */ + for (hp = session->hazard; + hp < session->hazard + conn->hazard_size; ++hp) { + if (hp->page != NULL) + continue; + + hp->page = ref->page; +#ifdef HAVE_DIAGNOSTIC + hp->file = file; + hp->line = line; +#endif + /* Publish the hazard reference before reading page's state. */ + WT_FULL_BARRIER(); + + /* + * Check to see if the page state is still valid (where valid + * means a state of WT_REF_MEM). + */ + if (ref->state == WT_REF_MEM) { + WT_VERBOSE(session, hazard, + "session %p hazard %p: set", session, ref->page); + return (0); + } + + /* + * The page isn't available, it's being considered for eviction + * (or being evicted, for all we know). If the eviction server + * sees our hazard reference before evicting the page, it will + * return the page to use, no harm done, if it doesn't, it will + * go ahead and complete the eviction. + * + * We don't bother publishing this update: the worst case is we + * prevent the page from being evicted, but that's not going to + * happen repeatedly, the eviction thread gives up and won't try + * again until it loops around through the tree. + */ + hp->page = NULL; + return (EBUSY); + } + + __wt_errx(session, + "there are no more hazard reference slots in the session"); + +#ifdef HAVE_DIAGNOSTIC + __hazard_dump(session); +#endif + + return (ENOMEM); +} + +/* + * __wt_hazard_clear -- + * Clear a hazard reference. + */ +void +__wt_hazard_clear(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_CONNECTION_IMPL *conn; + WT_HAZARD *hp; + + conn = S2C(session); + + /* + * The default value for a WT_HAZARD slot is NULL, but clearing a + * NULL reference isn't a good idea. + */ + WT_ASSERT(session, page != NULL); + + WT_VERBOSE(session, hazard, + "session %p hazard %p: clr", session, page); + + /* Clear the caller's hazard pointer. */ + for (hp = session->hazard; + hp < session->hazard + conn->hazard_size; ++hp) + if (hp->page == page) { + hp->page = NULL; + /* + * We don't have to flush memory here for correctness; + * it would give the page server thread faster access + * to the block were the block selected to be evicted, + * but the generation number was just set which makes + * it unlikely to be selected for eviction. + */ + return; + } + __wt_errx(session, "hazard reference not found"); +} + +/* + * __wt_hazard_empty -- + * Verify that no hazard references are set. + */ +void +__wt_hazard_empty(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + WT_HAZARD *hp; + + conn = S2C(session); + + /* + * Check for a set hazard reference and complain if we find one. Clear + * any we find because it's not a correctness problem (any hazard ref + * we find can't be real because the session is being closed when we're + * called). We do this work because it's not expensive, and we don't + * want to let a hazard reference lie around, keeping a page from being + * evicted. + */ +#ifdef HAVE_DIAGNOSTIC + __hazard_dump(session); +#endif + + for (hp = session->hazard; + hp < session->hazard + conn->hazard_size; ++hp) + if (hp->page != NULL) { + hp->page = NULL; + + __wt_errx(session, + "unexpected hazard reference at session.close"); + } +} + +#ifdef HAVE_DIAGNOSTIC +/* + * __hazard_dump -- + * Display the list of hazard references. + */ +static void +__hazard_dump(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + WT_HAZARD *hp; + int fail; + + conn = S2C(session); + + fail = 0; + for (hp = session->hazard; + hp < session->hazard + conn->hazard_size; ++hp) + if (hp->page != NULL) { + __wt_errx(session, + "hazard reference: (%p: %s, line %d)", + hp->page, hp->file, hp->line); + fail = 1; + } + + if (fail) + __wt_errx(session, "unexpected hazard reference"); +} + +/* + * __wt_hazard_validate -- + * Confirm that a page isn't on the hazard list. + */ +void +__wt_hazard_validate(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_CONNECTION_IMPL *conn; + WT_HAZARD *hp; + WT_SESSION_IMPL **tp; + + conn = S2C(session); + + for (tp = conn->sessions; (session = *tp) != NULL; ++tp) + for (hp = session->hazard; + hp < session->hazard + S2C(session)->hazard_size; ++hp) + if (hp->page == page) + __wt_errx(session, + "discarded page has hazard reference: " + "(%p: %s, line %d)", + hp->page, hp->file, hp->line); +} +#endif diff --git a/src/support/hex.c b/src/support/hex.c new file mode 100644 index 00000000000..e23118d2a22 --- /dev/null +++ b/src/support/hex.c @@ -0,0 +1,165 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static const u_char hex[] = "0123456789abcdef"; + +/* + * __wt_raw_to_hex -- + * Convert a chunk of data to a printable hex string. + */ +void +__wt_raw_to_hex(const void *from, void *to, uint32_t *sizep) +{ + uint32_t i; + const uint8_t *p; + uint8_t *t, *t_start; + + for (p = from, t = t_start = to, i = *sizep; i > 0; --i, ++p) { + *t++ = hex[(*p & 0xf0) >> 4]; + *t++ = hex[*p & 0x0f]; + } + *t++ = '\0'; + *sizep = WT_PTRDIFF32(t, t_start); +} + +/* + * __wt_raw_to_esc_hex -- + * Convert a chunk of data to an printable string using escaped hex as + * necessary. + */ +void +__wt_raw_to_esc_hex(const void *from, void *to, uint32_t *sizep) +{ + uint32_t i; + const uint8_t *p; + uint8_t *t, *t_start; + + /* + * In the worst case, every character takes up 3 spaces, plus a + * trailing nul byte. + */ + for (p = from, t = t_start = to, i = *sizep; i > 0; --i, ++p) + if (isprint((int)*p)) { + if (*p == '\\') + *t++ = '\\'; + *t++ = *p; + } else { + *t++ = '\\'; + *t++ = hex[(*p & 0xf0) >> 4]; + *t++ = hex[*p & 0x0f]; + } + *t++ = '\0'; + *sizep = WT_PTRDIFF32(t, t_start); +} + +/* + * hex2byte -- + * Convert a pair of hex characters into a byte. + */ +static inline int +hex2byte(uint8_t *from, uint8_t *to) +{ + uint8_t byte; + + switch (from[0]) { + case '0': byte = 0; break; + case '1': byte = 1 << 4; break; + case '2': byte = 2 << 4; break; + case '3': byte = 3 << 4; break; + case '4': byte = 4 << 4; break; + case '5': byte = 5 << 4; break; + case '6': byte = 6 << 4; break; + case '7': byte = 7 << 4; break; + case '8': byte = 8 << 4; break; + case '9': byte = 9 << 4; break; + case 'a': byte = 10 << 4; break; + case 'b': byte = 11 << 4; break; + case 'c': byte = 12 << 4; break; + case 'd': byte = 13 << 4; break; + case 'e': byte = 14 << 4; break; + case 'f': byte = 15 << 4; break; + default: + return (1); + } + + switch (from[1]) { + case '0': break; + case '1': byte |= 1; break; + case '2': byte |= 2; break; + case '3': byte |= 3; break; + case '4': byte |= 4; break; + case '5': byte |= 5; break; + case '6': byte |= 6; break; + case '7': byte |= 7; break; + case '8': byte |= 8; break; + case '9': byte |= 9; break; + case 'a': byte |= 10; break; + case 'b': byte |= 11; break; + case 'c': byte |= 12; break; + case 'd': byte |= 13; break; + case 'e': byte |= 14; break; + case 'f': byte |= 15; break; + default: + return (1); + } + *to = byte; + return (0); +} + +static int +__hex_fmterr(WT_SESSION_IMPL *session) +{ + WT_RET_MSG(session, EINVAL, "Invalid format in hexadecimal string"); +} + +/* + * __wt_hex_to_raw -- + * Convert a printable hex string to a chunk of data. + */ +int +__wt_hex_to_raw( + WT_SESSION_IMPL *session, void *from, void *to, uint32_t *sizep) +{ + uint8_t *p, *t, *t_start; + + if (strlen(from) % 2 != 0) + return (__hex_fmterr(session)); + + for (p = from, t = t_start = to; *p != '\0'; p += 2, ++t) + if (hex2byte(p, t)) + return (__hex_fmterr(session)); + + *sizep = WT_PTRDIFF32(t, t_start); + return (0); +} + +/* + * __wt_esc_hex_to_raw -- + * Convert a printable string using escaped hex as necessary to a chunk + * of data. + */ +int +__wt_esc_hex_to_raw( + WT_SESSION_IMPL *session, void *from, void *to, uint32_t *sizep) +{ + uint8_t *p, *t, *t_start; + + for (p = from, t = t_start = to; *p != '\0'; ++p, ++t) { + if ((*t = *p) != '\\') + continue; + ++p; + if (p[0] != '\\') { + if (p[0] == '\0' || p[1] == '\0' || hex2byte(p, t)) + return (__hex_fmterr(session)); + ++p; + } + } + *sizep = WT_PTRDIFF32(t, t_start); + return (0); +} diff --git a/src/support/huffman.c b/src/support/huffman.c new file mode 100644 index 00000000000..7b66b321b9c --- /dev/null +++ b/src/support/huffman.c @@ -0,0 +1,902 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +#define __HUFFMAN_DETAIL 0 /* Set to 1 for debugging output. */ + +/* Length of header in compressed message, in bits. */ +#define WT_HUFFMAN_HEADER 3 + +/* + * Maximum allowed length of Huffman code words, which otherwise can range up + * to (#symbols - 1) bits long. Lower value to use less memory for tables, + * higher value for better compression. Max value = 16 (or 32-7=25 or 64-7=57 + * if adjust data types). FYI, JPEG uses 16. A side effect of limiting max + * code length is that the worst case compression (a message of the least + * frequent symbols) is shorter. + */ +#define MAX_CODE_LENGTH 16 + +typedef struct __wt_freqtree_node { + /* + * Data structure representing a node of the huffman tree. It holds a + * 64-bit weight and pointers to the left and right child nodes. The + * node either has two child nodes or none. + */ + uint8_t symbol; /* only used in leaf nodes */ + uint64_t weight; + struct __wt_freqtree_node *left; /* bit 0 */ + struct __wt_freqtree_node *right; /* bit 1 */ +} WT_FREQTREE_NODE; + +typedef struct __wt_huffman_code { + uint16_t pattern; /* requirement: length of field's type + * in bits >= MAX_CODE_LENGTH. + */ + uint8_t length; +} WT_HUFFMAN_CODE; + +typedef struct __wt_huffman_obj { + /* + * Data structure here defines specific instance of the encoder/decoder. + */ + u_int numSymbols; /* Symbols: UINT16_MAX or UINT8_MAX */ + + uint16_t max_depth, min_depth; /* Tree max/min depths */ + + /* + * use: codes[symbol] = struct with pattern and length. + * Used in encoding and decoding. + * memory: codes[0-to-(number of symbols - 1)] + */ + WT_HUFFMAN_CODE *codes; + + /* + * use: code2symbol[Huffman_code] = symbol. + * Used in decoding. + * memory: code2symbol[1 << max_code_length] + */ + uint8_t *code2symbol; + +} WT_HUFFMAN_OBJ; + +/* + * Queue element data structure. + * + * Consists of a pointer to a huffman tree node, and a pointer to the next + * element in the queue. + */ +typedef struct node_queue_elem { + WT_FREQTREE_NODE *node; + struct node_queue_elem *next; +} NODE_QUEUE_ELEM; + +/* + * Queue of huffman tree nodes. + * + * Contains a pointer to the beginning and the end of the queue, which is + * implemented as a linked list. + */ +typedef struct node_queue { + NODE_QUEUE_ELEM *first; + NODE_QUEUE_ELEM *last; +} NODE_QUEUE; + +/* + * Internal data structure used to preserve the symbol when rearranging the + * frequency array. + */ +typedef struct __indexed_byte { + uint32_t symbol; /* not uint8_t: match external data structure */ + uint32_t frequency; +} INDEXED_SYMBOL; + +static int indexed_freq_compare(const void *, const void *); +static int indexed_symbol_compare(const void *, const void *); +static void make_table( + WT_SESSION_IMPL *, uint8_t *, uint16_t, WT_HUFFMAN_CODE *, u_int); +static void node_queue_close(WT_SESSION_IMPL *, NODE_QUEUE *); +static void node_queue_dequeue( + WT_SESSION_IMPL *, NODE_QUEUE *, WT_FREQTREE_NODE **); +static int node_queue_enqueue( + WT_SESSION_IMPL *, NODE_QUEUE *, WT_FREQTREE_NODE *); +static uint32_t profile_tree( + WT_FREQTREE_NODE *, uint16_t, uint16_t *, uint16_t *); +static void recursive_free_node(WT_SESSION_IMPL *, WT_FREQTREE_NODE *); +static void set_codes(WT_FREQTREE_NODE *, WT_HUFFMAN_CODE *, uint16_t, uint8_t); + +#define node_queue_is_empty(queue) \ + ((queue) == NULL || (queue)->first == NULL) + +/* + * indexed_symbol_compare -- + * Qsort comparator to order the table by symbol, lowest to highest. + */ +static int +indexed_symbol_compare(const void *a, const void *b) +{ + return (((INDEXED_SYMBOL *)a)->symbol > + ((INDEXED_SYMBOL *)b)->symbol ? 1 : + (((INDEXED_SYMBOL *)a)->symbol < + ((INDEXED_SYMBOL *)b)->symbol ? -1 : 0)); +} + +/* + * indexed_freq_compare -- + * Qsort comparator to order the table by frequency (the most frequent + * symbols will be at the end of the array). + */ +static int +indexed_freq_compare(const void *a, const void *b) +{ + return (((INDEXED_SYMBOL *)a)->frequency > + ((INDEXED_SYMBOL *)b)->frequency ? 1 : + (((INDEXED_SYMBOL *)a)->frequency < + ((INDEXED_SYMBOL *)b)->frequency ? -1 : 0)); +} + +/* + * profile_tree -- + * Traverses tree to determine #leaves under each node, max depth, min + * depth of leaf. + */ +static uint32_t +profile_tree(WT_FREQTREE_NODE *node, + uint16_t len, uint16_t *max_depth, uint16_t *min_depth) +{ + uint32_t leaf_cnt; + + if (node->left == NULL && node->right == NULL) { /* leaf */ + leaf_cnt = 1; + if (*max_depth < len) + *max_depth = len; + if (*min_depth > len) + *min_depth = len; + } else { + /* + * internal node -- way tree constructed internal always has + * left and right children + */ + leaf_cnt = + profile_tree(node->left, len + 1, max_depth, min_depth) + + profile_tree(node->right, len + 1, max_depth, min_depth); + } + node->weight = leaf_cnt; /* abuse weight field */ + return (leaf_cnt); +} + +/* + * set_codes -- + * Computes Huffman code for each symbol in tree. + * + * Method is standard way in the literature, except that limits maximum code + * length. A known max code length is important for limiting memory use by + * the tables and for knowing how large data types need to be such as the field + * that holds the code pattern. + */ +static void +set_codes(WT_FREQTREE_NODE *node, + WT_HUFFMAN_CODE *codes, uint16_t pattern, uint8_t len) +{ + WT_HUFFMAN_CODE *code; + uint16_t patternleft, patternright, half; + uint8_t remaining; + + if (node->left == NULL && node->right == NULL) { + code = &codes[node->symbol]; + code->pattern = pattern; + code->length = len; +#if __HUFFMAN_DETAIL + printf("%" PRIx16 ": code %" PRIx16 ", len %" PRIu8 "\n", + node->symbol, pattern, len); +#endif + } else { + /* + * Check each subtree individually to see if can afford to split + * up bits into possibly shorter codes, or if need to employ all + * remaining bits up to MAX_CODE_LENGTH to consecutively number + * leaves. + */ + remaining = MAX_CODE_LENGTH - len; + /* + * If not already in "low-bit mode", but need to be, open up + * lower-order bits for consecutive numbering. + */ + if (len < MAX_CODE_LENGTH && + ((half = 1 << (remaining - 1)) < node->left->weight || + half < node->right->weight)) { + pattern = pattern << remaining; + len = MAX_CODE_LENGTH; + } + + if (len < MAX_CODE_LENGTH) { + patternleft = (pattern << 1) | 0; + patternright = (pattern << 1) | 1; + len++; + } else { /* "low bit mode" */ + patternleft = pattern; + patternright = pattern + node->left->weight; + /* len unchanged */ + } + + set_codes(node->left, codes, patternleft, len); + set_codes(node->right, codes, patternright, len); + } +} + +/* + * make_table -- + * Computes Huffman table used for subsequent lookups in encoding and + * decoding. With the table, encoding from a symbol to Huffman code and + * decoding from a code to a symbol are simple array lookups. + */ +static void +make_table(WT_SESSION_IMPL *session, uint8_t *code2symbol, + uint16_t max_depth, WT_HUFFMAN_CODE *codes, u_int symcnt) +{ + uint32_t j, c1, c2; /* Exceeds uint16_t bounds at loop boundary. */ + uint16_t c, i; + uint8_t len, shift; + + /* Zero out, for assertion below. */ + for (j = 0, c2 = (1U << max_depth); j < c2; j++) + code2symbol[j] = 0; + + /* + * Here's the magic: flood all bit patterns for lower-order bits to + * point to same symbol. + */ + for (i = 0; i < symcnt; i++) { + if ((len = codes[i].length) == 0) + continue; + + /* + * The size of the array index should be enough to hold largest + * index into symbol table. Pre-existing symbols were packed + * 0-255, so 8 bits is enough. Don't want to make it larger + * than necessary, we allocate (2 ^ max-code-length) of them. + */ + c = codes[i].pattern; + shift = max_depth - len; + c1 = c << shift; + c2 = (c + 1) << shift; + for (j = c1; j < c2; j++) { + WT_ASSERT(session, code2symbol[j] == 0); + code2symbol[j] = i; + } + } +} + +/* + * recursive_free_node -- + * Recursively free the huffman frequency tree's nodes. + */ +static void +recursive_free_node(WT_SESSION_IMPL *session, WT_FREQTREE_NODE *node) +{ + if (node != NULL) { + recursive_free_node(session, node->left); + recursive_free_node(session, node->right); + __wt_free(session, node); + } +} + +/* + * __wt_huffman_open -- + * Take a frequency table and return a pointer to a descriptor object. + */ +int +__wt_huffman_open(WT_SESSION_IMPL *session, + void *symbol_frequency_array, u_int symcnt, u_int numbytes, void *retp) +{ + INDEXED_SYMBOL *indexed_freqs, *sym; + NODE_QUEUE *combined_nodes, *leaves; + WT_FREQTREE_NODE *node, *node2, **refnode, *tempnode; + WT_HUFFMAN_OBJ *huffman; + uint64_t w1, w2; + uint16_t i; + int ret; + + indexed_freqs = symbol_frequency_array; + + combined_nodes = leaves = NULL; + node = node2 = tempnode = NULL; + ret = 0; + + WT_RET(__wt_calloc_def(session, 1, &huffman)); + + /* + * The frequency table is 4B pairs of symbol and frequency. The symbol + * is either 1 or 2 bytes and the frequency ranges from 1 to UINT32_MAX + * (a frequency of 0 means the value is never expected to appear in the + * input). Validate the symbols are within range. + */ + if (numbytes != 1 && numbytes != 2) + WT_ERR_MSG(session, EINVAL, + "illegal number of symbol bytes specified for a huffman " + "table"); + + if (symcnt == 0) + WT_ERR_MSG(session, EINVAL, + "illegal number of symbols specified for a huffman table"); + + huffman->numSymbols = numbytes == 2 ? UINT16_MAX : UINT8_MAX; + + /* + * Order the array by symbol and check for invalid symbols and + * duplicates. + */ + qsort((void *)indexed_freqs, + symcnt, sizeof(INDEXED_SYMBOL), indexed_symbol_compare); + for (i = 0; i < symcnt; ++i) { + if (i > 0 && + indexed_freqs[i].symbol == indexed_freqs[i - 1].symbol) + WT_ERR_MSG(session, EINVAL, + "duplicate symbol %" PRIx16 + " specified in a huffman table", + indexed_freqs[i].symbol); + if (indexed_freqs[i].symbol > huffman->numSymbols) + WT_ERR_MSG(session, EINVAL, + "illegal symbol %" PRIx16 + " specified in a huffman table", + indexed_freqs[i].symbol); + } + + /* + * Massage frequencies. + */ + indexed_freqs = NULL; + WT_ERR(__wt_calloc_def(session, 256, &indexed_freqs)); + + /* + * Minimum of frequency==1 so everybody gets a Huffman code, in case + * data evolves and we need to represent this value. + */ + for (i = 0; i < 256; i++) { + sym = &indexed_freqs[i]; + sym->symbol = i; + sym->frequency = 1; + } + /* + * Avoid large tables by splitting UTF-16 frequencies into high byte + * and low byte. + */ + for (i = 0; i < symcnt; i++) { + sym = &((INDEXED_SYMBOL *)symbol_frequency_array)[i]; + indexed_freqs[sym->symbol & 0xff].frequency += sym->frequency; + if (numbytes == 2) + indexed_freqs[(sym->symbol >> 8) & 0xff].frequency += + sym->frequency; + } + huffman->numSymbols = symcnt = 256; + + /* + * The array must be sorted by frequency to be able to use a linear time + * construction algorithm. + */ + qsort((void *)indexed_freqs, + symcnt, sizeof(INDEXED_SYMBOL), indexed_freq_compare); + + /* We need two node queues to build the tree. */ + WT_ERR(__wt_calloc_def(session, 1, &leaves)); + WT_ERR(__wt_calloc_def(session, 1, &combined_nodes)); + + /* + * Adding the leaves to the queue. + * + * Discard symbols with a frequency of 0; this assumes these symbols + * never occur in the source stream, and the purpose is to reduce the + * huffman tree's size. + */ + for (i = 0; i < symcnt; ++i) + if (indexed_freqs[i].frequency > 0) { + WT_ERR(__wt_calloc_def(session, 1, &tempnode)); + tempnode->symbol = (uint8_t)indexed_freqs[i].symbol; + tempnode->weight = indexed_freqs[i].frequency; + WT_ERR(node_queue_enqueue(session, leaves, tempnode)); + tempnode = NULL; + } + + while (!node_queue_is_empty(leaves) || + !node_queue_is_empty(combined_nodes)) { + /* + * We have to get the node with the smaller weight, examining + * both queues' first element. We are collecting pairs of these + * items, by alternating between node and node2: + */ + refnode = !node ? &node : &node2; + + /* + * To decide which queue must be used, we get the weights of + * the first items from both: + */ + w1 = node_queue_is_empty(leaves) ? + UINT64_MAX : leaves->first->node->weight; + w2 = node_queue_is_empty(combined_nodes) ? + UINT64_MAX : combined_nodes->first->node->weight; + + /* + * Based on the two weights we finally can dequeue the smaller + * element and place it to the alternating target node pointer: + */ + if (w1 < w2) + node_queue_dequeue(session, leaves, refnode); + else + node_queue_dequeue(session, combined_nodes, refnode); + + /* + * In every second run, we have both node and node2 initialized. + */ + if (node != NULL && node2 != NULL) { + WT_ERR(__wt_calloc_def(session, 1, &tempnode)); + + /* The new weight is the sum of the two weights. */ + tempnode->weight = node->weight + node2->weight; + tempnode->left = node; + tempnode->right = node2; + + /* Enqueue it to the combined nodes queue */ + WT_ERR(node_queue_enqueue( + session, combined_nodes, tempnode)); + tempnode = NULL; + + /* Reset the state pointers */ + node = node2 = NULL; + } + } + + /* + * The remaining node is in the node variable, this is the root of the + * tree. Calculate how many bytes it takes to hold numSymbols bytes + * bits. + */ + huffman->max_depth = 0; + huffman->min_depth = MAX_CODE_LENGTH; + (void)profile_tree(node, 0, &huffman->max_depth, &huffman->min_depth); + if (huffman->max_depth > MAX_CODE_LENGTH) + huffman->max_depth = MAX_CODE_LENGTH; + + WT_ERR(__wt_calloc_def(session, huffman->numSymbols, &huffman->codes)); + set_codes(node, huffman->codes, 0, 0); + + WT_ERR(__wt_calloc_def( + session, 1U << huffman->max_depth, &huffman->code2symbol)); + make_table(session, huffman->code2symbol, + huffman->max_depth, huffman->codes, huffman->numSymbols); + +#if __HUFFMAN_DETAIL + { + uint8_t symbol; + uint32_t weighted_length; + + printf("leaf depth %" PRIu16 "..%" PRIu16 ", memory use: " + "codes %u# * %uB + code2symbol %u# * %uB\n", + huffman->min_depth, huffman->max_depth, + huffman->numSymbols, (u_int)sizeof(WT_HUFFMAN_CODE), + 1U << huffman->max_depth, (u_int)sizeof(uint16_t)); + + /* + * measure quality of computed Huffman codes, for different max bit + * lengths (say, 16 vs 24 vs 32) + */ + weighted_length = 0; + for (i = 0; i < symcnt; i++) { + symbol = indexed_freqs[i].symbol; + weighted_length += + indexed_freqs[i].frequency * huffman->codes[symbol].length; + printf( + "\t%" PRIu16 "->%" PRIu16 ". %" PRIu32 " * %" PRIu8 "\n", + i, symbol, + indexed_freqs[i].frequency, huffman->codes[symbol].length); + } + printf("weighted length of all codes (the smaller the better): " + "%" PRIu32 "\n", weighted_length); + } +#endif + + *(void **)retp = huffman; + + if (0) { +err: if (ret == 0) + ret = WT_ERROR; + } + __wt_free(session, indexed_freqs); + if (leaves != NULL) + node_queue_close(session, leaves); + if (combined_nodes != NULL) + node_queue_close(session, combined_nodes); + if (node != NULL) + recursive_free_node(session, node); + if (node2 != NULL) + recursive_free_node(session, node2); + __wt_free(session, tempnode); + if (ret != 0) + __wt_huffman_close(session, huffman); + return (ret); +} + +/* + * __wt_huffman_close -- + * Discard a Huffman descriptor object. + */ +void +__wt_huffman_close(WT_SESSION_IMPL *session, void *huffman_arg) +{ + WT_HUFFMAN_OBJ *huffman; + + huffman = huffman_arg; + + __wt_free(session, huffman->code2symbol); + __wt_free(session, huffman->codes); + __wt_free(session, huffman); +} + +#if __HUFFMAN_DETAIL +/* + * __wt_print_huffman_code -- + * Prints a symbol's Huffman code. + */ +int +__wt_print_huffman_code(void *huffman_arg, uint16_t symbol) +{ + WT_HUFFMAN_CODE code; + WT_HUFFMAN_OBJ *huffman; + + huffman = huffman_arg; + + if (symbol >= huffman->numSymbols) + printf("symbol %" PRIu16 " out of range\n", symbol); + else { + code = huffman->codes[symbol]; + if (code.length == 0) + printf( + "symbol %" PRIu16 " not defined -- 0 frequency\n", + symbol); + else + /* should print code as binary */ + printf( + "%" PRIu16 " -> code pattern " + "%" PRIx16 ", length %" PRIu8 "\n", + symbol, code.pattern, code.length); + } + + return (0); +} +#endif + +/* + * __wt_huffman_encode -- + * Take a byte string, encode it into the target. + * + * Translation from symbol to Huffman code is a simple array lookup. + * + * WT_HUFFMAN_OBJ contains an array called 'codes' with one WT_HUFFMAN_CODE per + * symbol. Then, given a symbol: + * pattern = codes[symbol].pattern; + * length = codes[symbol].length; + * + * To encode byte-string, we iterate over the input symbols. For each symbol, + * look it up via table, shift bits onto a shift register (an int long enough + * to hold the longest code word + up to 7 bits remaining from the previous), + * then drain out full bytes. Finally, at the end flush remaining bits + * and write header bits. + */ +int +__wt_huffman_encode(WT_SESSION_IMPL *session, void *huffman_arg, + const uint8_t *from_arg, uint32_t from_len, WT_ITEM *to_buf) +{ + WT_HUFFMAN_CODE code; + WT_HUFFMAN_OBJ *huffman; + WT_ITEM *tmp; + uint64_t bitpos; + uint32_t max_len, outlen, bytes; + const uint8_t *from; + uint8_t len, *out, padding_info, symbol; + int ret; + + /* + * Shift register to accumulate bits from input. + * Should be >= (MAX_CODE_LENGTH + 7), but also efficient to shift bits + * and preferably in a machine register. + */ + uint32_t bits; + + /* Count of bits in shift register ('bits' above). */ + uint8_t valid; + + huffman = huffman_arg; + from = from_arg; + tmp = NULL; + ret = 0; + + /* + * We don't want to find all of our callers and ensure they don't pass + * 0-length byte strings, but there's no reason to do any work. + */ + if (from_len == 0) { + to_buf->size = 0; + return (0); + } + + /* + * Compute the largest compressed output size, which is if all symbols + * are least frequent and so have largest Huffman codes, and compressed + * output may be larger than the input size. This way we don't have to + * worry about resizing the buffer during compression. Use the shared + * system buffer while compressing, then allocate a new buffer of the + * right size and copy the result into it. + */ + max_len = (WT_HUFFMAN_HEADER + + from_len * huffman->max_depth + 7 /* round up to full byte */) / 8; + WT_ERR(__wt_scr_alloc(session, max_len, &tmp)); + + /* + * Leave the first 3 bits of the encoded value empty, it holds the + * number of bits actually used in the last byte of the encoded value. + */ + bits = 0; + bitpos = WT_HUFFMAN_HEADER; + valid = WT_HUFFMAN_HEADER; + out = tmp->mem; + for (bytes = 0; bytes < from_len; bytes++) { + WT_ASSERT(session, WT_PTR_IN_RANGE(from, from_arg, from_len)); + + symbol = *from++; + + /* Translate symbol into Huffman code and stuff into buffer. */ + code = huffman->codes[symbol]; + len = code.length; + bits = (bits << len) | code.pattern; + valid += len; + bitpos += len; + while (valid >= 8) { + WT_ASSERT(session, + WT_PTR_IN_RANGE(out, tmp->mem, tmp->memsize)); + *out++ = (uint8_t)(bits >> (valid - 8)); + valid -= 8; + } + } + if (valid > 0) { /* Flush shift register. */ + WT_ASSERT(session, + WT_PTR_IN_RANGE(out, tmp->mem, tmp->memsize)); + *out = (uint8_t)(bits << (8 - valid)); + } + + /* + * At this point, bitpos is the total number of used bits (including + * the 3 bits at the beginning of the buffer, which we'll set now to + * the number of bits used in the last byte). Note if the number of + * bits used in the last byte is 8, we set the 3 bits to 0, in other + * words, the first 3 bits of the encoded value are the number of bits + * used in the last byte, unless they're 0, in which case there are 8 + * bits used in the last byte. + */ + padding_info = (bitpos % 8) << (8 - WT_HUFFMAN_HEADER); + ((uint8_t *)tmp->mem)[0] |= padding_info; + + /* Copy result of exact known size into caller's buffer. */ + outlen = (uint32_t)((bitpos + 7) / 8); + WT_ERR(__wt_buf_initsize(session, to_buf, outlen)); + memcpy(to_buf->mem, tmp->mem, outlen); + +#if __HUFFMAN_DETAIL + printf("encode: worst case %" PRIu32 " bytes -> actual %" PRIu32 "\n", + max_len, outlen); +#endif + +err: __wt_scr_free(&tmp); + return (ret); + +} + +/* + * __wt_huffman_decode -- + * Take a byte string, decode it into the target. + * + * Translation from Huffman code to symbol is a simple array lookup. + * + * WT_HUFFMAN_OBJ contains an array called 'code2symbol' indexed by code word + * and whose value is the corresponding symbol. + * From the symbol, we index into the 'codes' array to get the code length. + * + * When decoding a message, we don't know where the boundaries are between + * codes. The trick is that we collect enough bits for the longest code word, + * and construct the table such that for codes with fewer bits we flood the + * table with all of the bit patterns in the lower order bits. This works + * because the Huffman code is a unique prefix, and by the flooding we are + * treating bits beyond the unique prefix as don't care bits. + * + * For example, we have table of length 2^max_code_length (1<<max_code_length). + * For a code of length, max_code_length, the position code2symbol[code] = + * symbol. + * For a code word of (max_length - 1), we fill code2symbol[code << 1] = symbol, + * as well as code2symbol[(code << 1) | 1] = symbol. + * And so on, so in general we fill: + * code2symbol[(code) << shift inclusive .. (code+1) << shift exclusive]. + * + * To decode a message, we read in enough bits from input to fill the shift + * register with at least MAX_CODE_LENGTH bits. + * We look up in the table code2symbol to obtain the symbol. + * We look up the symbol in 'codes' to obtain the code length + * Finally, subtract off these bits from the shift register. + */ +int +__wt_huffman_decode(WT_SESSION_IMPL *session, void *huffman_arg, + const uint8_t *from_arg, uint32_t from_len, WT_ITEM *to_buf) +{ + WT_ITEM *tmp; + WT_HUFFMAN_OBJ *huffman; + uint64_t from_len_bits; + uint32_t bits, from_bytes, len, mask, max, max_len, outlen; + uint16_t pattern; + const uint8_t *from; + uint8_t padding_info, symbol, *to, valid; + int ret; + + huffman = huffman_arg; + from = from_arg; + tmp = NULL; + ret = 0; + + /* + * We don't want to find all of our callers and ensure they don't pass + * 0-length byte strings, but there's no reason to do any work. + */ + if (from_len == 0) { + to_buf->size = 0; + return (0); + } + + /* + * The first 3 bits are the number of used bits in the last byte, unless + * they're 0, in which case there are 8 bits used in the last byte. + */ + padding_info = (*from & 0xE0) >> (8 - WT_HUFFMAN_HEADER); + from_len_bits = from_len * 8; + if (padding_info != 0) + from_len_bits -= 8 - padding_info; + + /* Number of bits that have codes. */ + from_len_bits -= WT_HUFFMAN_HEADER; + + /* + * Compute largest uncompressed output size, which is if all symbols are + * most frequent and so have smallest Huffman codes and therefore + * largest expansion. Use the shared system buffer while uncompressing, + * then allocate a new buffer of exactly the right size and copy the + * result into it. + */ + max_len = (uint32_t)(from_len_bits / huffman->min_depth); + WT_ERR(__wt_scr_alloc(session, max_len, &tmp)); + to = tmp->mem; + + /* The first byte of input is a special case because of header bits. */ + bits = *from++; + valid = 8 - WT_HUFFMAN_HEADER; + from_bytes = from_len - 1; + + max = huffman->max_depth; + mask = (1U << max) - 1; + for (outlen = 0; from_len_bits > 0; outlen++) { + while (valid < max && from_bytes > 0) { + WT_ASSERT(session, + WT_PTR_IN_RANGE(from, from_arg, from_len)); + bits = (bits << 8) | *from++; + valid += 8; + from_bytes--; + } + pattern = valid >= max ? /* short patterns near end */ + (bits >> (valid - max)) : (bits << (max - valid)); + symbol = huffman->code2symbol[pattern & mask]; + len = huffman->codes[symbol].length; + valid -= len; + WT_ASSERT(session, from_len_bits >= len); + from_len_bits -= len; + + WT_ASSERT(session, + WT_PTR_IN_RANGE(to, tmp->mem, tmp->memsize)); + *to++ = symbol; + } + + /* Return the number of bytes used. */ + WT_ERR(__wt_buf_initsize(session, to_buf, outlen)); + memcpy(to_buf->mem, tmp->mem, outlen); + +#if __HUFFMAN_DETAIL + printf("decode: worst case %" PRIu32 " bytes -> actual %" PRIu32 "\n", + max_len, outlen); +#endif + +err: __wt_scr_free(&tmp); + return (ret); +} + +/* + * node_queue_close -- + * Delete a queue from memory. + * + * It does not delete the pointed huffman tree nodes! + */ +static void +node_queue_close(WT_SESSION_IMPL *session, NODE_QUEUE *queue) +{ + NODE_QUEUE_ELEM *elem, *next_elem; + + /* Freeing each element of the queue's linked list. */ + for (elem = queue->first; elem != NULL; elem = next_elem) { + next_elem = elem->next; + __wt_free(session, elem); + } + + /* Freeing the queue record itself. */ + __wt_free(session, queue); +} + +/* + * node_queue_enqueue -- + * Push a tree node to the end of the queue. + */ +static int +node_queue_enqueue( + WT_SESSION_IMPL *session, NODE_QUEUE *queue, WT_FREQTREE_NODE *node) +{ + NODE_QUEUE_ELEM *elem; + + /* Allocating a new linked list element */ + WT_RET(__wt_calloc_def(session, 1, &elem)); + + /* It holds the tree node, and has no next element yet */ + elem->node = node; + elem->next = NULL; + + /* If the queue is empty, the first element will be the new one. */ + if (queue->first == NULL) + queue->first = elem; + + /* + * If the queue is not empty, the last element's next pointer must be + * updated. + */ + if (queue->last != NULL) + queue->last->next = elem; + + /* The last element is the new one */ + queue->last = elem; + + return (0); +} + +/* + * node_queue_dequeue -- + * Removes a node from the beginning of the queue and copies the node's + * pointer to the location referred by the retp parameter. + */ +static void +node_queue_dequeue( + WT_SESSION_IMPL *session, NODE_QUEUE *queue, WT_FREQTREE_NODE **retp) +{ + NODE_QUEUE_ELEM *first_elem; + + /* + * Getting the first element of the queue and updating it to point to + * the next element as first. + */ + first_elem = queue->first; + *retp = first_elem->node; + queue->first = first_elem->next; + + /* + * If the last element was the dequeued element, we have to update it + * to NULL. + */ + if (queue->last == first_elem) + queue->last = NULL; + + /* Freeing the linked list element that has been dequeued */ + __wt_free(session, first_elem); +} diff --git a/src/support/pow.c b/src/support/pow.c new file mode 100644 index 00000000000..25bd587e26d --- /dev/null +++ b/src/support/pow.c @@ -0,0 +1,73 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +#ifdef __WIREDTIGER_UNUSED__ +/* + * __wt_nlpo2_round -- + * Round up to the next-largest power-of-two for a 32-bit unsigned value. + * + * In 12 operations, this code computes the next highest power of 2 for a 32-bit + * integer. The result may be expressed by the formula 1U << (lg(v - 1) + 1). + * Note that in the edge case where v is 0, it returns 0, which isn't a power of + * 2; you might append the expression v += (v == 0) to remedy this if it + * matters. It would be faster by 2 operations to use the formula and the + * log base 2 method that uses a lookup table, but in some situations, lookup + * tables are not suitable, so the above code may be best. (On a Athlon XP 2100+ + * I've found the above shift-left and then OR code is as fast as using a single + * BSR assembly language instruction, which scans in reverse to find the highest + * set bit.) It works by copying the highest set bit to all of the lower bits, + * and then adding one, which results in carries that set all of the lower bits + * to 0 and one bit beyond the highest set bit to 1. If the original number was + * a power of 2, then the decrement will reduce it to one less, so that we round + * up to the same original value. Devised by Sean Anderson, September 14, 2001. + * Pete Hart pointed me to a couple newsgroup posts by him and William Lewis in + * February of 1997, where they arrive at the same algorithm. + * http://graphics.stanford.edu/~seander/bithacks.html + * Sean Eron Anderson, seander@cs.stanford.edu + */ +uint32_t +__wt_nlpo2_round(uint32_t v) +{ + v--; /* If v is a power-of-two, return it. */ + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + return (v + 1); +} + +uint32_t +__wt_nlpo2(uint32_t v) +{ + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + return (v + 1); +} +#endif /* __WIREDTIGER_UNUSED__ */ + +/* + * __wt_ispo2 -- + * Return if a number is a power-of-two. + */ +int +__wt_ispo2(uint32_t v) +{ + /* + * Only numbers that are powers of two will satisfy the relationship + * (v & (v - 1) == 0). + * + * However n must be positive, this returns 0 as a power of 2; to fix + * that, use: (! (v & (v - 1)) && v) + */ + return ((v & (v - 1)) == 0); +} diff --git a/src/support/rand.c b/src/support/rand.c new file mode 100644 index 00000000000..f9202d757dc --- /dev/null +++ b/src/support/rand.c @@ -0,0 +1,47 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "wt_internal.h" + +/* + * __wt_random -- + * Return a 32-bit pseudo-random number. + * + * This is an implementation of George Marsaglia's multiply-with-carry pseudo- + * random number generator. Computationally fast, with reasonable randomness + * properties. + */ +uint32_t +__wt_random(void) +{ + static uint32_t m_w = 521288629; + static uint32_t m_z = 362436069; + + m_z = 36969 * (m_z & 65535) + (m_z >> 16); + m_w = 18000 * (m_w & 65535) + (m_w >> 16); + return (m_z << 16) + (m_w & 65535); +} diff --git a/src/support/scratch.c b/src/support/scratch.c new file mode 100644 index 00000000000..107f6516def --- /dev/null +++ b/src/support/scratch.c @@ -0,0 +1,441 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +static void __wt_buf_clear(WT_ITEM *); + +/* + * __wt_buf_clear -- + * Clear a buffer. + */ +static void +__wt_buf_clear(WT_ITEM *buf) +{ + buf->data = NULL; + buf->size = 0; + + buf->mem = NULL; + buf->memsize = 0; + + /* Note: don't clear the flags, the buffer remains marked in-use. */ +} + +/* + * __wt_buf_init -- + * Initialize a buffer at a specific size. + */ +int +__wt_buf_init(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size) +{ + WT_ASSERT(session, size <= UINT32_MAX); + + if (size > buf->memsize) + WT_RET(__wt_realloc(session, &buf->memsize, size, &buf->mem)); + + buf->data = buf->mem; + buf->size = 0; + + return (0); +} + +/* + * __wt_buf_initsize -- + * Initialize a buffer at a specific size, and set the data length. + */ +int +__wt_buf_initsize(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size) +{ + WT_RET(__wt_buf_init(session, buf, size)); + + buf->size = WT_STORE_SIZE(size); /* Set the data length. */ + + return (0); +} + +/* + * __wt_buf_grow -- + * Grow a buffer that's currently in-use. + */ +int +__wt_buf_grow(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size) +{ + size_t offset; + int set_data; + + WT_ASSERT(session, size <= UINT32_MAX); + + if (size > buf->memsize) { + /* + * Grow the buffer's memory: if the data reference is not set or + * references the buffer's memory, maintain it. + * + * This test is complicated so it won't fail if buf->mem is non- + * NULL and buf->memsize is 0. That shouldn't be possible, but + * I don't want to find out I'm wrong, either. + */ + if (buf->data == NULL || buf->data == buf->mem) { + offset = 0; + set_data = 1; + } else if (buf->data >= buf->mem && + (uint8_t *)buf->data < + (uint8_t *)buf->mem + buf->memsize) { + offset = WT_PTRDIFF(buf->data, buf->mem); + set_data = 1; + } else { + offset = 0; + set_data = 0; + } + + WT_RET(__wt_realloc(session, &buf->memsize, size, &buf->mem)); + + if (set_data) + buf->data = (uint8_t *)buf->mem + offset; + } + return (0); +} + +/* + * __wt_buf_set -- + * Set the contents of the buffer. + */ +int +__wt_buf_set( + WT_SESSION_IMPL *session, WT_ITEM *buf, const void *data, size_t size) +{ + /* Ensure the buffer is large enough. */ + WT_RET(__wt_buf_initsize(session, buf, size)); + + memcpy(buf->mem, data, size); + + return (0); +} + +/* + * __wt_buf_set_printable -- + * Set the contents of the buffer to a printable representation of a + * byte string. + */ +int +__wt_buf_set_printable( + WT_SESSION_IMPL *session, WT_ITEM *buf, const void *from_arg, size_t size) +{ + uint32_t u32size; + + /* + * In the worst case, every character takes up 3 spaces, plus a + * trailing nul byte. + */ + WT_RET(__wt_buf_init(session, buf, size * 3 + 10)); + + u32size = (uint32_t)size; + __wt_raw_to_esc_hex(from_arg, buf->mem, &u32size); + buf->size = u32size; + + return (0); +} + +/* + * __wt_buf_steal -- + * Steal a buffer for another purpose. + */ +void * +__wt_buf_steal(WT_SESSION_IMPL *session, WT_ITEM *buf, uint32_t *sizep) +{ + void *retp; + + /* + * Sometimes we steal a buffer for a different purpose, for example, + * we've read in an overflow item, and now it's going to become a key + * on an in-memory page, eventually freed when the page is discarded. + * + * First, correct for the possibility the data field doesn't point to + * the start of memory (if we only have a single memory reference, it + * must point to the start of the memory chunk, otherwise freeing the + * memory isn't going to work out). This is possibly a common case: + * it happens when we read in overflow items and we want to skip over + * the page header, so buf->data references a location past buf->mem. + */ + if (buf->data != buf->mem) { + WT_ASSERT(session, + buf->data > buf->mem && + (uint8_t *)buf->data < + (uint8_t *)buf->mem + buf->memsize && + (uint8_t *)buf->data + buf->size <= + (uint8_t *)buf->mem + buf->memsize); + memmove(buf->mem, buf->data, buf->size); + } + + /* Second, give our caller the buffer's memory. */ + retp = buf->mem; + if (sizep != NULL) + *sizep = buf->size; + + /* Third, discard the buffer's memory. */ + __wt_buf_clear(buf); + + return (retp); +} + +/* + * __wt_buf_swap -- + * Swap a pair of buffers. + */ +void +__wt_buf_swap(WT_ITEM *a, WT_ITEM *b) +{ + WT_ITEM tmp; + + tmp = *a; + *a = *b; + *b = tmp; +} + +/* + * __wt_buf_free -- + * Free a buffer. + */ +void +__wt_buf_free(WT_SESSION_IMPL *session, WT_ITEM *buf) +{ + __wt_free(session, buf->mem); + __wt_buf_clear(buf); +} + +/* + * __wt_buf_fmt -- + * Grow a buffer to accommodate a formatted string. + */ +int +__wt_buf_fmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 3, 4))) +{ + va_list ap; + size_t len; + + for (;;) { + va_start(ap, fmt); + len = + (size_t)vsnprintf(buf->mem, (size_t)buf->memsize, fmt, ap); + va_end(ap); + + /* Check if there was enough space. */ + if (len < buf->memsize) { + buf->data = buf->mem; + buf->size = WT_STORE_SIZE(len); + return (0); + } + + /* + * If not, double the size of the buffer: we're dealing with + * strings, and we don't expect these numbers to get huge. + */ + WT_RET(__wt_buf_grow( + session, buf, WT_MAX(len + 1, buf->memsize * 2))); + } +} + +/* + * __wt_buf_catfmt -- + * Grow a buffer to append a formatted string. + */ +int +__wt_buf_catfmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 3, 4))) +{ + va_list ap; + size_t len, space; + char *p; + + for (;;) { + va_start(ap, fmt); + p = (char *)((uint8_t *)buf->mem + buf->size); + WT_ASSERT(session, buf->memsize >= buf->size); + space = buf->memsize - buf->size; + len = (size_t)vsnprintf(p, (size_t)space, fmt, ap); + va_end(ap); + + /* Check if there was enough space. */ + if (len < space) { + buf->size += WT_STORE_SIZE(len); + return (0); + } + + /* + * If not, double the size of the buffer: we're dealing with + * strings, and we don't expect these numbers to get huge. + */ + WT_RET(__wt_buf_grow(session, buf, + WT_MAX(buf->size + len + 1, buf->memsize * 2))); + } +} + +/* + * __wt_scr_alloc -- + * Scratch buffer allocation function. + */ +int +__wt_scr_alloc(WT_SESSION_IMPL *session, uint32_t size, WT_ITEM **scratchp) +{ + WT_ITEM *buf, **p, *small, **slot; + size_t allocated; + u_int i; + int ret; + + /* Don't risk the caller not catching the error. */ + *scratchp = NULL; + + /* + * Each WT_SESSION_IMPL has an array of scratch buffers available for + * use by any function. We use WT_ITEM structures for scratch memory + * because we already have functions that do variable-length allocation + * on a WT_ITEM. Scratch buffers are allocated only by a single thread + * of control, so no locking is necessary. + * + * Walk the array, looking for a buffer we can use. + */ + for (i = 0, small = NULL, slot = NULL, + p = session->scratch; i < session->scratch_alloc; ++i, ++p) { + /* If we find an empty slot, remember it. */ + if ((buf = *p) == NULL) { + if (slot == NULL) + slot = p; + continue; + } + + if (F_ISSET(buf, WT_ITEM_INUSE)) + continue; + + /* + * If we find a buffer that's not in-use, check its size: if it + * is large enough, and we'd only waste 4KB by taking it, take + * it. If we don't want this one, remember it -- if we have two + * buffers we can "remember", then remember the smallest one. + */ + if (buf->memsize >= size && + (buf->memsize - size) < 4 * 1024) { + WT_ERR(__wt_buf_init(session, buf, size)); + F_SET(buf, WT_ITEM_INUSE); + *scratchp = buf; + return (0); + } + if (small == NULL) + small = buf; + else if (small->memsize > buf->memsize) + small = buf; + } + + /* + * If small is non-NULL, we found a buffer but it wasn't large enough. + * Try and grow it. + */ + if (small != NULL) { + WT_ERR(__wt_buf_init(session, small, size)); + F_SET(small, WT_ITEM_INUSE); + + *scratchp = small; + return (0); + } + + /* + * If slot is non-NULL, we found an empty slot, try and allocate a + * buffer and call recursively to find and grow the buffer. + */ + if (slot != NULL) { + WT_ERR(__wt_calloc_def(session, 1, slot)); + return (__wt_scr_alloc(session, size, scratchp)); + } + + /* + * Resize the array, we need more scratch buffers, then call recursively + * to find the empty slot, and so on and so forth. + */ + allocated = session->scratch_alloc * sizeof(WT_ITEM *); + WT_ERR(__wt_realloc(session, &allocated, + (session->scratch_alloc + 10) * sizeof(WT_ITEM *), + &session->scratch)); + session->scratch_alloc += 10; + return (__wt_scr_alloc(session, size, scratchp)); + +err: WT_RET_MSG(session, ret, + "session unable to allocate more scratch buffers"); +} + +/* + * __wt_scr_free -- + * Release a scratch buffer. + */ +void +__wt_scr_free(WT_ITEM **bufp) +{ + if (*bufp == NULL) + return; + F_CLR(*bufp, WT_ITEM_INUSE); + *bufp = NULL; +} + +/* + * __wt_scr_discard -- + * Free all memory associated with the scratch buffers. + */ +void +__wt_scr_discard(WT_SESSION_IMPL *session) +{ + WT_ITEM **bufp; + u_int i; + + for (i = 0, + bufp = session->scratch; i < session->scratch_alloc; ++i, ++bufp) { + if (*bufp != NULL) + __wt_buf_free(session, *bufp); + __wt_free(session, *bufp); + } + + __wt_free(session, session->scratch); +} + +/* + * __wt_scr_alloc_ext -- + * Allocate a scratch buffer, and return the memory reference. + */ +void * +__wt_scr_alloc_ext(WT_SESSION *wt_session, size_t size) +{ + WT_ITEM *buf; + WT_SESSION_IMPL *session; + + session = (WT_SESSION_IMPL *)wt_session; + + return (__wt_scr_alloc( + session, (uint32_t)size, &buf) == 0 ? buf->mem : NULL); +} + +/* + * __wt_scr_free_ext -- + * Free a scratch buffer based on the memory reference. + */ +void +__wt_scr_free_ext(WT_SESSION *wt_session, void *p) +{ + WT_ITEM **bufp; + WT_SESSION_IMPL *session; + u_int i; + + session = (WT_SESSION_IMPL *)wt_session; + + for (i = 0, + bufp = session->scratch; i < session->scratch_alloc; ++i, ++bufp) + if (*bufp != NULL && (*bufp)->mem == p) { + /* + * Do NOT call __wt_scr_free() here, it clears the + * caller's pointer, which would truncate the list. + */ + F_CLR(*bufp, WT_ITEM_INUSE); + return; + } + __wt_errx(session, "extension free'd non-existent scratch buffer"); +} diff --git a/src/support/sess_dump.c b/src/support/sess_dump.c new file mode 100644 index 00000000000..16bc1e702e7 --- /dev/null +++ b/src/support/sess_dump.c @@ -0,0 +1,67 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +#ifdef HAVE_DIAGNOSTIC +/* + * __wt_session_dump_all -- + * Dump information about all open sessions. + */ +void +__wt_session_dump_all(WT_SESSION_IMPL *session) +{ + WT_SESSION_IMPL **tp; + + if (session == NULL) + return; + + for (tp = S2C(session)->sessions; *tp != NULL; ++tp) + __wt_session_dump(*tp); +} + +/* + * __wt_session_dump -- + * Dump information about a session. + */ +void +__wt_session_dump(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + WT_CURSOR *cursor; + WT_HAZARD *hp; + int first; + + conn = S2C(session); + + __wt_msg(session, "session: %s%s%p", + session->name == NULL ? "" : session->name, + session->name == NULL ? "" : " ", session); + + first = 0; + TAILQ_FOREACH(cursor, &session->file_cursors, q) { + if (++first == 1) + __wt_msg(session, "\tcursors:"); + __wt_msg(session, "\t\t%p", cursor); + } + + first = 0; + for (hp = session->hazard; + hp < session->hazard + conn->hazard_size; ++hp) { + if (hp->page == NULL) + continue; + if (++first == 1) + __wt_msg(session, "\thazard references:"); +#ifdef HAVE_DIAGNOSTIC + __wt_msg(session, + "\t\t%p (%s, line %d)", hp->page, hp->file, hp->line); +#else + __wt_msg(session, "\t\t%p", hp->page); +#endif + } +} +#endif diff --git a/src/support/stat.c b/src/support/stat.c new file mode 100644 index 00000000000..bbd13b8b839 --- /dev/null +++ b/src/support/stat.c @@ -0,0 +1,170 @@ +/* DO NOT EDIT: automatically built by dist/stat.py. */ + +#include "wt_internal.h" + +int +__wt_stat_alloc_btree_stats(WT_SESSION_IMPL *session, WT_BTREE_STATS **statsp) +{ + WT_BTREE_STATS *stats; + + WT_RET(__wt_calloc_def(session, 1, &stats)); + + stats->alloc.desc = "file: block allocations"; + stats->cursor_inserts.desc = "cursor-inserts"; + stats->cursor_read.desc = "cursor-read"; + stats->cursor_read_near.desc = "cursor-read-near"; + stats->cursor_read_next.desc = "cursor-read-next"; + stats->cursor_read_prev.desc = "cursor-read-prev"; + stats->cursor_removes.desc = "cursor-removes"; + stats->cursor_resets.desc = "cursor-resets"; + stats->cursor_updates.desc = "cursor-updates"; + stats->extend.desc = "file: block allocations required file extension"; + stats->file_allocsize.desc = "page size allocation unit"; + stats->file_bulk_loaded.desc = "bulk-loaded entries"; + stats->file_col_deleted.desc = "column-store deleted values"; + stats->file_col_fix_pages.desc = "column-store fixed-size leaf pages"; + stats->file_col_int_pages.desc = "column-store internal pages"; + stats->file_col_var_pages.desc = + "column-store variable-size leaf pages"; + stats->file_entries.desc = "total entries"; + stats->file_fixed_len.desc = "fixed-record size"; + stats->file_freelist_bytes.desc = "number of bytes in the freelist"; + stats->file_freelist_entries.desc = + "number of entries in the freelist"; + stats->file_magic.desc = "magic number"; + stats->file_major.desc = "major version number"; + stats->file_maxintlitem.desc = "maximum internal page item size"; + stats->file_maxintlpage.desc = "maximum internal page size"; + stats->file_maxleafitem.desc = "maximum leaf page item size"; + stats->file_maxleafpage.desc = "maximum leaf page size"; + stats->file_minor.desc = "minor version number"; + stats->file_overflow.desc = "overflow pages"; + stats->file_row_int_pages.desc = "row-store internal pages"; + stats->file_row_leaf_pages.desc = "row-store leaf pages"; + stats->file_size.desc = "file: size"; + stats->free.desc = "file: block frees"; + stats->overflow_read.desc = "file: overflow pages read from the file"; + stats->page_read.desc = "file: pages read from the file"; + stats->page_write.desc = "file: pages written to the file"; + stats->rec_hazard.desc = + "reconcile: unable to acquire hazard reference"; + stats->rec_ovfl_key.desc = "reconcile: overflow key"; + stats->rec_ovfl_value.desc = "reconcile: overflow value"; + stats->rec_page_delete.desc = "reconcile: pages deleted"; + stats->rec_page_merge.desc = + "reconcile: deleted or temporary pages merged"; + stats->rec_split_intl.desc = "reconcile: internal pages split"; + stats->rec_split_leaf.desc = "reconcile: leaf pages split"; + stats->rec_written.desc = "reconcile: pages written"; + + *statsp = stats; + return (0); +} + +void +__wt_stat_clear_btree_stats(WT_STATS *stats_arg) +{ + WT_BTREE_STATS *stats; + + stats = (WT_BTREE_STATS *)stats_arg; + stats->alloc.v = 0; + stats->cursor_inserts.v = 0; + stats->cursor_read.v = 0; + stats->cursor_read_near.v = 0; + stats->cursor_read_next.v = 0; + stats->cursor_read_prev.v = 0; + stats->cursor_removes.v = 0; + stats->cursor_resets.v = 0; + stats->cursor_updates.v = 0; + stats->extend.v = 0; + stats->file_allocsize.v = 0; + stats->file_bulk_loaded.v = 0; + stats->file_col_deleted.v = 0; + stats->file_col_fix_pages.v = 0; + stats->file_col_int_pages.v = 0; + stats->file_col_var_pages.v = 0; + stats->file_entries.v = 0; + stats->file_fixed_len.v = 0; + stats->file_freelist_bytes.v = 0; + stats->file_freelist_entries.v = 0; + stats->file_magic.v = 0; + stats->file_major.v = 0; + stats->file_maxintlitem.v = 0; + stats->file_maxintlpage.v = 0; + stats->file_maxleafitem.v = 0; + stats->file_maxleafpage.v = 0; + stats->file_minor.v = 0; + stats->file_overflow.v = 0; + stats->file_row_int_pages.v = 0; + stats->file_row_leaf_pages.v = 0; + stats->file_size.v = 0; + stats->free.v = 0; + stats->overflow_read.v = 0; + stats->page_read.v = 0; + stats->page_write.v = 0; + stats->rec_hazard.v = 0; + stats->rec_ovfl_key.v = 0; + stats->rec_ovfl_value.v = 0; + stats->rec_page_delete.v = 0; + stats->rec_page_merge.v = 0; + stats->rec_split_intl.v = 0; + stats->rec_split_leaf.v = 0; + stats->rec_written.v = 0; +} + +int +__wt_stat_alloc_connection_stats(WT_SESSION_IMPL *session, WT_CONNECTION_STATS **statsp) +{ + WT_CONNECTION_STATS *stats; + + WT_RET(__wt_calloc_def(session, 1, &stats)); + + stats->block_read.desc = "blocks read from a file"; + stats->block_write.desc = "blocks written to a file"; + stats->cache_bytes_inuse.desc = + "cache: bytes currently held in the cache"; + stats->cache_bytes_max.desc = "cache: maximum bytes configured"; + stats->cache_evict_hazard.desc = + "cache: pages selected for eviction not evicted because of a hazard reference"; + stats->cache_evict_internal.desc = "cache: internal pages evicted"; + stats->cache_evict_modified.desc = "cache: modified pages evicted"; + stats->cache_evict_slow.desc = + "cache: eviction server unable to reach eviction goal"; + stats->cache_evict_unmodified.desc = "cache: unmodified pages evicted"; + stats->cache_pages_inuse.desc = + "cache: pages currently held in the cache"; + stats->cond_wait.desc = "condition wait calls"; + stats->file_open.desc = "files currently open"; + stats->memalloc.desc = "total memory allocations"; + stats->memfree.desc = "total memory frees"; + stats->rwlock_rdlock.desc = "rwlock readlock calls"; + stats->rwlock_wrlock.desc = "rwlock writelock calls"; + stats->total_read_io.desc = "total read I/Os"; + stats->total_write_io.desc = "total write I/Os"; + + *statsp = stats; + return (0); +} + +void +__wt_stat_clear_connection_stats(WT_STATS *stats_arg) +{ + WT_CONNECTION_STATS *stats; + + stats = (WT_CONNECTION_STATS *)stats_arg; + stats->block_read.v = 0; + stats->block_write.v = 0; + stats->cache_evict_hazard.v = 0; + stats->cache_evict_internal.v = 0; + stats->cache_evict_modified.v = 0; + stats->cache_evict_slow.v = 0; + stats->cache_evict_unmodified.v = 0; + stats->cond_wait.v = 0; + stats->file_open.v = 0; + stats->memalloc.v = 0; + stats->memfree.v = 0; + stats->rwlock_rdlock.v = 0; + stats->rwlock_wrlock.v = 0; + stats->total_read_io.v = 0; + stats->total_write_io.v = 0; +} diff --git a/src/txn/interleave.py b/src/txn/interleave.py new file mode 100755 index 00000000000..164a0c8a0df --- /dev/null +++ b/src/txn/interleave.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +# identical transactions +t1 = ["r1(x)", "w1(x)"] +t2 = ["r2(x)", "w2(x)"] +t3 = ["r3(x)", "w3(x)"] +t4 = ["r4(x)", "w4(x)"] + +def interleave(T1, T2): + """Given lists of operations as input, return all possible interleavings""" + if not T1: + return [T2] + elif not T2: + return [T1] + else: + return [T1[0:1] + l for l in interleave(T1[1:], T2)] + [T2[0:1] + l for l in interleave(T1, T2[1:])] + +for l1 in interleave(t1, t2): + for l2 in interleave(l1, t3): + for l in interleave(l2, t4): + # timestamps of item x + readts = 0 + writets = 0 + skip = False + failure = '' + + for op in l: + ts = int(op[1]) + # Check whether the operation is valid: + if op[0] == 'r': + if writets < ts and readts < ts: + readts = ts + elif op[0] == 'w': + if writets > ts or readts > ts: + if ts <= 2 and l.index("r" + str(ts+2) + "(x)") < l.index("w" + str(ts+1) + "(x)"): + skip = True + break + failure += ' ' + op + elif op[0] == 'c': + pass + if skip: + continue + elif failure: + print '%s: failed at%s' % (' '.join(l), failure) + else: + print '%s: passed' % (' '.join(l),) diff --git a/src/utilities/util.h b/src/utilities/util.h new file mode 100644 index 00000000000..0f3b6ed407e --- /dev/null +++ b/src/utilities/util.h @@ -0,0 +1,61 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include <wt_internal.h> + +#define UTIL_COLGROUP_OK 0x01 /* colgroup: prefix OK */ +#define UTIL_FILE_OK 0x02 /* file: prefix OK */ +#define UTIL_INDEX_OK 0x04 /* index: prefix OK */ +#define UTIL_TABLE_OK 0x08 /* table: prefix OK */ + +/* all known prefixes OK */ +#define UTIL_ALL_OK \ + (UTIL_COLGROUP_OK | UTIL_FILE_OK | UTIL_INDEX_OK | UTIL_TABLE_OK) + +typedef struct { + void *mem; /* Managed memory chunk */ + size_t memsize; /* Managed memory size */ +} ULINE; + +extern const char *progname; /* Program name */ +extern const char *usage_prefix; /* Global arguments */ +extern int verbose; /* Verbose flag */ + +extern WT_EVENT_HANDLER *verbose_handler; + +/* + * We compile in own version of getopt, it's simpler than figuring out what the + * system has. + */ +extern int util_opterr; /* if error message should be printed */ +extern int util_optind; /* index into parent argv vector */ +extern int util_optopt; /* character checked for validity */ +extern int util_optreset; /* reset getopt */ +extern char *util_optarg; /* argument associated with option */ + +int util_cerr(const char *, const char *, int); +void util_copyright(void); +int util_create(WT_SESSION *, int, char *[]); +int util_drop(WT_SESSION *, int, char *[]); +int util_dump(WT_SESSION *, int, char *[]); +int util_dumpfile(WT_SESSION *, int, char *[]); +int util_err(int, const char *, ...); +int util_getopt(int, char * const *, const char *); +int util_list(WT_SESSION *, int, char *[]); +int util_load(WT_SESSION *, int, char *[]); +int util_loadtext(WT_SESSION *, int, char *[]); +char *util_name(const char *, const char *, u_int); +int util_printlog(WT_SESSION *, int, char *[]); +int util_read(WT_SESSION *, int, char *[]); +int util_read_line(ULINE *, int, int *); +int util_rename(WT_SESSION *, int, char *[]); +int util_salvage(WT_SESSION *, int, char *[]); +int util_stat(WT_SESSION *, int, char *[]); +int util_str2recno(const char *p, uint64_t *recnop); +int util_upgrade(WT_SESSION *, int, char *[]); +int util_verify(WT_SESSION *, int, char *[]); +int util_write(WT_SESSION *, int, char *[]); diff --git a/src/utilities/util_cpyright.c b/src/utilities/util_cpyright.c new file mode 100644 index 00000000000..7297dff19f5 --- /dev/null +++ b/src/utilities/util_cpyright.c @@ -0,0 +1,35 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +void +util_copyright(void) +{ + printf("%s\n", "Copyright (c) 2008-2012 WiredTiger, Inc."); + printf("%s\n\n", "All rights reserved."); + + printf("%s\n\n", + "This program is free software: you can redistribute it and/or\n" + "modify it under the terms of version 3 of the GNU General\n" + "Public License as published by the Free Software Foundation."); + + printf("%s\n\n", + "This program is distributed in the hope that it will be useful,\n" + "but WITHOUT ANY WARRANTY; without even the implied warranty of\n" + "MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n" + "GNU General Public License for more details:"); + + printf("\t%s\n\n", + "http://www.gnu.org/licenses/gpl-3.0-standalone.html"); + + printf("%s\n", + "For a license to use the WiredTiger software under conditions\n" + "other than those described by the GNU General Public License,\n" + "or for technical support for this software, contact WiredTiger,\n" + "Inc. at info@wiredtiger.com."); +} diff --git a/src/utilities/util_create.c b/src/utilities/util_create.c new file mode 100644 index 00000000000..106a49d3537 --- /dev/null +++ b/src/utilities/util_create.c @@ -0,0 +1,52 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int usage(void); + +int +util_create(WT_SESSION *session, int argc, char *argv[]) +{ + int ch, ret; + const char *config, *uri; + + config = NULL; + while ((ch = util_getopt(argc, argv, "c:")) != EOF) + switch (ch) { + case 'c': /* command-line configuration */ + config = util_optarg; + break; + case '?': + default: + return (usage()); + } + + argc -= util_optind; + argv += util_optind; + + /* The remaining argument is the uri. */ + if (argc != 1) + return (usage()); + + if ((uri = util_name(*argv, "table", UTIL_ALL_OK)) == NULL) + return (1); + + if ((ret = session->create(session, uri, config)) != 0) + return (util_err(ret, "%s: session.create", uri)); + return (0); +} + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "create [-c configuration] uri\n", + progname, usage_prefix); + return (1); +} diff --git a/src/utilities/util_drop.c b/src/utilities/util_drop.c new file mode 100644 index 00000000000..efd69c98841 --- /dev/null +++ b/src/utilities/util_drop.c @@ -0,0 +1,45 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int usage(void); + +int +util_drop(WT_SESSION *session, int argc, char *argv[]) +{ + int ch; + const char *name; + + while ((ch = util_getopt(argc, argv, "")) != EOF) + switch (ch) { + case '?': + default: + return (usage()); + } + + argc -= util_optind; + argv += util_optind; + + /* The remaining argument is the uri. */ + if (argc != 1) + return (usage()); + if ((name = util_name(*argv, "table", UTIL_ALL_OK)) == NULL) + return (1); + + return (session->drop(session, name, "force")); +} + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "drop uri\n", + progname, usage_prefix); + return (1); +} diff --git a/src/utilities/util_dump.c b/src/utilities/util_dump.c new file mode 100644 index 00000000000..2c946c2568b --- /dev/null +++ b/src/utilities/util_dump.c @@ -0,0 +1,336 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int dump_prefix(int); +static int dump_suffix(void); +static int schema(WT_SESSION *, const char *); +static int schema_file(WT_CURSOR *, const char *); +static int schema_table(WT_CURSOR *, const char *); +static int usage(void); + +static inline int +dump_forward(WT_CURSOR *cursor, const char *name) +{ + const char *key, *value; + int ret; + + while ((ret = cursor->next(cursor)) == 0) { + if ((ret = cursor->get_key(cursor, &key)) != 0) + return (util_cerr(name, "get_key", ret)); + if ((ret = cursor->get_value(cursor, &value)) != 0) + return (util_cerr(name, "get_value", ret)); + if (printf("%s\n%s\n", key, value) < 0) + return (util_err(EIO, NULL)); + } + return (ret == WT_NOTFOUND ? 0 : util_cerr(name, "next", ret)); +} + +static inline int +dump_reverse(WT_CURSOR *cursor, const char *name) +{ + const char *key, *value; + int ret; + + while ((ret = cursor->prev(cursor)) == 0) { + if ((ret = cursor->get_key(cursor, &key)) != 0) + return (util_cerr(name, "get_key", ret)); + if ((ret = cursor->get_value(cursor, &value)) != 0) + return (util_cerr(name, "get_value", ret)); + if (printf("%s\n%s\n", key, value) < 0) + return (util_err(EIO, NULL)); + } + return (ret == WT_NOTFOUND ? 0 : util_cerr(name, "prev", ret)); +} + +int +util_dump(WT_SESSION *session, int argc, char *argv[]) +{ + WT_CURSOR *cursor; + int ch, hex, ret, reverse; + char *name; + + hex = reverse = 0; + name = NULL; + while ((ch = util_getopt(argc, argv, "f:rx")) != EOF) + switch (ch) { + case 'f': /* output file */ + if (freopen(util_optarg, "w", stdout) == NULL) + return ( + util_err(errno, "%s: reopen", util_optarg)); + break; + case 'r': + reverse = 1; + break; + case 'x': + hex = 1; + break; + case '?': + default: + return (usage()); + } + argc -= util_optind; + argv += util_optind; + + /* The remaining argument is the uri. */ + if (argc != 1) + return (usage()); + if ((name = + util_name(*argv, "table", UTIL_FILE_OK | UTIL_TABLE_OK)) == NULL) + goto err; + + if (dump_prefix(hex) != 0 || + schema(session, name) != 0 || + dump_suffix() != 0) + goto err; + + if ((ret = session->open_cursor(session, + name, NULL, hex ? "dump=hex" : "dump=print", &cursor)) != 0) { + fprintf(stderr, "%s: cursor open(%s) failed: %s\n", + progname, name, wiredtiger_strerror(ret)); + goto err; + } + + if (reverse) + ret = dump_reverse(cursor, name); + else + ret = dump_forward(cursor, name); + + if (0) { +err: ret = 1; + } + + if (name != NULL) + free(name); + + return (ret); +} + +/* + * schema -- + * Dump the schema for the uri. + */ +static int +schema(WT_SESSION *session, const char *uri) +{ + WT_CURSOR *cursor; + int ret, tret; + + ret = 0; + + /* Open the schema file. */ + if ((ret = session->open_cursor( + session, WT_SCHEMA_URI, NULL, NULL, &cursor)) != 0) { + fprintf(stderr, "%s: %s: session.open_cursor: %s\n", + progname, WT_SCHEMA_URI, wiredtiger_strerror(ret)); + return (1); + } + + /* Dump the schema. */ + if (strncmp(uri, "table:", strlen("table:")) == 0) + ret = schema_table(cursor, uri); + else + ret = schema_file(cursor, uri); + + if ((tret = cursor->close(cursor)) != 0 && ret == 0) + ret = tret; + + return (ret); +} + +/* + * schema_table -- + * Dump the schema for a table. + */ +static int +schema_table(WT_CURSOR *cursor, const char *uri) +{ + struct { + char *key; /* Schema key */ + char *value; /* Schema value */ + } *list; + int i, elem, list_elem, ret; + const char *key, *name, *value; + char *buf, *filename, *p, *t, *sep; + + ret = 0; + + /* Get the name. */ + if ((name = strchr(uri, ':')) == NULL) { + fprintf(stderr, "%s: %s: corrupted uri\n", progname, uri); + return (1); + } + ++name; + + list = NULL; + elem = list_elem = 0; + for (; (ret = cursor->next(cursor)) == 0; free(buf)) { + /* Get the key and duplicate it, we want to overwrite it. */ + if ((ret = cursor->get_key(cursor, &key)) != 0) + return (util_cerr(uri, "get_key", ret)); + if ((buf = strdup(key)) == NULL) + return (util_err(errno, NULL)); + + /* Check for the dump table's column groups or indices. */ + if ((p = strchr(buf, ':')) == NULL) + continue; + *p++ = '\0'; + if (strcmp(buf, "index") != 0 && strcmp(buf, "colgroup") != 0) + continue; + if ((t = strchr(p, ':')) == NULL) + continue; + *t++ = '\0'; + if (strcmp(p, name) != 0) + continue; + + /* Found one, save it for review. */ + if ((ret = cursor->get_value(cursor, &value)) != 0) + return (util_cerr(uri, "get_value", ret)); + if (elem == list_elem && (list = realloc(list, + (size_t)(list_elem += 20) * sizeof(*list))) == NULL) + return (util_err(errno, NULL)); + if ((list[elem].key = strdup(key)) == NULL) + return (util_err(errno, NULL)); + if ((list[elem].value = strdup(value)) == NULL) + return (util_err(errno, NULL)); + ++elem; + } + if (ret != WT_NOTFOUND) + return (util_cerr(uri, "next", ret)); + ret = 0; + + /* + * Dump out the schema information: first, dump the uri entry itself + * (requires a lookup). + */ + cursor->set_key(cursor, uri); + if ((ret = cursor->search(cursor)) != 0) + return (util_cerr(uri, "search", ret)); + if ((ret = cursor->get_key(cursor, &key)) != 0) + return (util_cerr(uri, "get_key", ret)); + if ((ret = cursor->get_value(cursor, &value)) != 0) + return (util_cerr(uri, "get_value", ret)); + if (printf("%s\n%s\n", key, value) < 0) + return (util_err(EIO, NULL)); + + /* + * Second, dump the column group and index key/value pairs: for each + * one, look up the related file information and append it to the base + * record. + */ + for (i = 0; i < elem; ++i) { + if ((filename = strstr(list[i].value, "filename=")) == NULL) { + fprintf(stderr, + "%s: %s: has no underlying file configuration\n", + progname, list[i].key); + return (1); + } + + /* + * Nul-terminate the filename if necessary, create the file + * URI, then look it up. + */ + if ((sep = strchr(filename, ',')) != NULL) + *sep = '\0'; + if ((t = strdup(filename)) == NULL) + return (util_err(errno, NULL)); + if (sep != NULL) + *sep = ','; + p = t + strlen("filename="); + p -= strlen("file:"); + memcpy(p, "file:", strlen("file:")); + cursor->set_key(cursor, p); + if ((ret = cursor->search(cursor)) != 0) { + fprintf(stderr, + "%s: %s: unable to find schema reference for the " + "underlying file %s\n", + progname, list[i].key, p); + return (1); + } + if ((ret = cursor->get_value(cursor, &value)) != 0) + return (util_cerr(uri, "get_value", ret)); + + /* + * The dumped configuration string is the original key plus the + * file's configuration. + */ + if (printf( + "%s\n%s,%s\n", list[i].key, list[i].value, value) < 0) + return (util_err(EIO, NULL)); + } + + /* Leak the memory, I don't care. */ + return (0); +} + +/* + * schema_file -- + * Dump the schema for a file. + */ +static int +schema_file(WT_CURSOR *cursor, const char *uri) +{ + const char *key, *value; + int ret; + + ret = 0; + + cursor->set_key(cursor, uri); + if ((ret = cursor->search(cursor)) != 0) + return (util_cerr(uri, "search", ret)); + if ((ret = cursor->get_key(cursor, &key)) != 0) + return (util_cerr(uri, "get_key", ret)); + if ((ret = cursor->get_value(cursor, &value)) != 0) + return (util_cerr(uri, "get_value", ret)); + if (printf("%s\n%s\n", key, value) < 0) + return (util_err(EIO, NULL)); + + return (0); +} + +/* + * dump_prefix -- + * Output the dump file header prefix. + */ +static int +dump_prefix(int hex) +{ + int vmajor, vminor, vpatch; + + (void)wiredtiger_version(&vmajor, &vminor, &vpatch); + + if (printf( + "WiredTiger Dump (WiredTiger Version %d.%d.%d)\n", + vmajor, vminor, vpatch) < 0 || + printf("Format=%s\n", hex ? "hex" : "print") < 0 || + printf("Header\n") < 0) + return (util_err(EIO, NULL)); + return (0); +} + +/* + * dump_suffix -- + * Output the dump file header suffix. + */ +static int +dump_suffix(void) +{ + if (printf("Data\n") < 0) + return (util_err(EIO, NULL)); + return (0); +} + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "dump [-rx] [-f output-file] uri\n", + progname, usage_prefix); + return (1); +} diff --git a/src/utilities/util_dumpfile.c b/src/utilities/util_dumpfile.c new file mode 100644 index 00000000000..c415bf9f1ba --- /dev/null +++ b/src/utilities/util_dumpfile.c @@ -0,0 +1,67 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int usage(void); + +int +util_dumpfile(WT_SESSION *session, int argc, char *argv[]) +{ + int ch, ret; + char *name; + + name = NULL; + while ((ch = util_getopt(argc, argv, "f:")) != EOF) + switch (ch) { + case 'f': /* output file */ + if (freopen(util_optarg, "w", stdout) == NULL) { + fprintf(stderr, "%s: %s: %s\n", + progname, util_optarg, strerror(errno)); + return (1); + } + break; + case '?': + default: + return (usage()); + } + argc -= util_optind; + argv += util_optind; + + /* The remaining argument is the file name. */ + if (argc != 1) + return (usage()); + if ((name = util_name(*argv, "file", UTIL_FILE_OK)) == NULL) + return (1); + + if ((ret = session->dumpfile(session, name, NULL)) != 0) { + fprintf(stderr, "%s: dumpfile(%s): %s\n", + progname, name, wiredtiger_strerror(ret)); + goto err; + } + if (verbose) + printf("\n"); + + if (0) { +err: ret = 1; + } + + if (name != NULL) + free(name); + + return (ret); +} + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "dumpfile [-f output-file] file\n", + progname, usage_prefix); + return (1); +} diff --git a/src/utilities/util_getopt.c b/src/utilities/util_getopt.c new file mode 100644 index 00000000000..df494ceab16 --- /dev/null +++ b/src/utilities/util_getopt.c @@ -0,0 +1,146 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/* $NetBSD: getopt.c,v 1.26 2003/08/07 16:43:40 agc Exp $ */ + +/* + * Copyright (c) 1987, 1993, 1994 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "util.h" + +int util_opterr = 1, /* if error message should be printed */ + util_optind = 1, /* index into parent argv vector */ + util_optopt, /* character checked for validity */ + util_optreset; /* reset getopt */ +char *util_optarg; /* argument associated with option */ + +#define BADCH (int)'?' +#define BADARG (int)':' +#define EMSG "" + +/* + * getopt -- + * Parse argc/argv argument vector. + */ +int +util_getopt(int nargc, char * const *nargv, const char *ostr) +{ + static const char *place = EMSG; /* option letter processing */ + const char *oli; /* option letter list index */ + + if (util_optreset || *place == 0) { /* update scanning pointer */ + util_optreset = 0; + place = nargv[util_optind]; + if (util_optind >= nargc || *place++ != '-') { + /* Argument is absent or is not an option */ + place = EMSG; + return (-1); + } + util_optopt = *place++; + if (util_optopt == '-' && *place == 0) { + /* "--" => end of options */ + ++util_optind; + place = EMSG; + return (-1); + } + if (util_optopt == 0) { + /* Solitary '-', treat as a '-' option + if the program (eg su) is looking for it. */ + place = EMSG; + if (strchr(ostr, '-') == NULL) + return (-1); + util_optopt = '-'; + } + } else + util_optopt = *place++; + + /* See if option letter is one the caller wanted... */ + if (util_optopt == ':' || (oli = strchr(ostr, util_optopt)) == NULL) { + if (*place == 0) + ++util_optind; + if (util_opterr && *ostr != ':') + (void)fprintf(stderr, + "%s: illegal option -- %c\n", progname, + util_optopt); + return (BADCH); + } + + /* Does this option need an argument? */ + if (oli[1] != ':') { + /* don't need argument */ + util_optarg = NULL; + if (*place == 0) + ++util_optind; + } else { + /* Option-argument is either the rest of this argument or the + entire next argument. */ + if (*place) + util_optarg = (char *)place; + else if (nargc > ++util_optind) + util_optarg = nargv[util_optind]; + else { + /* option-argument absent */ + place = EMSG; + if (*ostr == ':') + return (BADARG); + if (util_opterr) + (void)fprintf(stderr, + "%s: option requires an argument -- %c\n", + progname, util_optopt); + return (BADCH); + } + place = EMSG; + ++util_optind; + } + return (util_optopt); /* return option letter */ +} diff --git a/src/utilities/util_list.c b/src/utilities/util_list.c new file mode 100644 index 00000000000..2c5ad58ddbe --- /dev/null +++ b/src/utilities/util_list.c @@ -0,0 +1,87 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int list_print(WT_SESSION *); +static int usage(void); + +int +util_list(WT_SESSION *session, int argc, char *argv[]) +{ + int ch; + + while ((ch = util_getopt(argc, argv, "")) != EOF) + switch (ch) { + case '?': + default: + return (usage()); + } + argc -= util_optind; + argv += util_optind; + + if (argc != 0) + return (usage()); + + return (list_print(session)); +} + +/* + * list_print -- + * List the high-level objects in the database. + */ +static int +list_print(WT_SESSION *session) +{ + WT_CURSOR *cursor; + int ret; + const char *key; + + ret = 0; + + /* Open the schema file. */ + if ((ret = session->open_cursor( + session, WT_SCHEMA_URI, NULL, NULL, &cursor)) != 0) { + /* + * If there is no schema (yet), this will return WT_NOTFOUND. + * Treat that the same as an empty schema. + */ + if (ret == WT_NOTFOUND) + return (0); + + fprintf(stderr, "%s: %s: session.open_cursor: %s\n", + progname, WT_SCHEMA_URI, wiredtiger_strerror(ret)); + return (1); + } + +#define MATCH(s, tag) \ + (strncmp(s, tag, strlen(tag)) == 0) + + while ((ret = cursor->next(cursor)) == 0) { + /* Get the key. */ + if ((ret = cursor->get_key(cursor, &key)) != 0) + return (util_cerr("schema", "get_key", ret)); + + /* All we care about are top-level objects (files or tables). */ + if (MATCH(key, "table:") || MATCH(key, "file:")) + printf("%s\n", key); + } + if (ret != WT_NOTFOUND) + return (util_cerr("schema", "next", ret)); + + return (0); +} + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "list\n", + progname, usage_prefix); + return (1); +} diff --git a/src/utilities/util_load.c b/src/utilities/util_load.c new file mode 100644 index 00000000000..0d562ebfe6d --- /dev/null +++ b/src/utilities/util_load.c @@ -0,0 +1,435 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int format(void); +static int insert(WT_CURSOR *, const char *); +static int load_dump(WT_SESSION *); +static int schema_read(char ***, int *); +static int schema_rename(char **, const char *); +static int schema_update(WT_SESSION *, char **); +static int usage(void); + +static int append; /* -a append (ignore record number keys) */ +static char *cmdname; /* -r rename */ +static char **cmdconfig; /* configuration pairs */ +static int overwrite; /* -o overwrite existing data */ + +int +util_load(WT_SESSION *session, int argc, char *argv[]) +{ + int ch; + + while ((ch = util_getopt(argc, argv, "af:r:o")) != EOF) + switch (ch) { + case 'a': /* append (ignore record number keys) */ + append = 1; + break; + case 'f': /* input file */ + if (freopen(util_optarg, "r", stdin) == NULL) + return ( + util_err(errno, "%s: reopen", util_optarg)); + break; + case 'r': /* -r rename */ + cmdname = util_optarg; + break; + case 'o': /* -o (overwrite existing data) */ + overwrite = 1; + break; + case '?': + default: + return (usage()); + } + argc -= util_optind; + argv += util_optind; + + /* -a and -o are mutually exclusive. */ + if (append == 1 && overwrite == 1) + return (util_err(EINVAL, + "the -a (append) and -o (overwrite) flags are mutually " + "exclusive")); + + /* The remaining arguments are configuration uri/string pairs. */ + if (argc != 0) { + if (argc % 2 != 0) + return (usage()); + cmdconfig = argv; + } + + return (load_dump(session)); +} + +/* + * load_dump -- + * Load from the WiredTiger dump format. + */ +static int +load_dump(WT_SESSION *session) +{ + WT_CURSOR *cursor; + int hex, ret, tret; + char **entry, **list, *p, *uri, config[64]; + + list = NULL; /* -Wuninitialized */ + hex = 0; /* -Wuninitialized */ + + /* Read the schema file. */ + if ((ret = schema_read(&list, &hex)) != 0) + return (ret); + + /* + * Search for a table name -- if we find one, then it's table dump, + * otherwise, it's a single file dump. + */ + for (entry = list; *entry != NULL; ++entry) + if (strncmp(*entry, "table:", strlen("table:")) == 0) + break; + if (*entry == NULL) { + /* + * Single file dumps can only have two lines, the file name and + * the configuration information. + */ + if (list[0] == NULL || list[1] == NULL || list[2] != NULL || + strncmp(list[0], "file:", strlen("file:")) != 0) + return (format()); + + entry = list; + } + + /* + * Make sure the table key/value pair comes first, then we can just + * run through the array in order. (We already checked that we had + * a multiple of 2 entries, so this is safe.) + */ + if (entry != list) { + p = list[0]; list[0] = entry[0]; entry[0] = p; + p = list[1]; list[1] = entry[1]; entry[1] = p; + } + + /* Update the schema based on any command-line configuration. */ + if ((ret = schema_update(session, list)) != 0) + return (ret); + + uri = list[0]; + for (entry = list; *entry != NULL; entry += 2) + if ((ret = session->create(session, entry[0], entry[1])) != 0) + return (util_err(ret, "%s: session.create", entry[0])); + + /* Open the insert cursor. */ + (void)snprintf(config, sizeof(config), + "dump=%s%s%s", + hex ? "hex" : "print", + append ? ",append" : "", overwrite ? ",overwrite" : ""); + if ((ret = session->open_cursor( + session, uri, NULL, config, &cursor)) != 0) + return(util_err(ret, "%s: session.open", uri)); + + /* + * Check the append flag (it only applies to objects where the primary + * key is a record number). + */ + if (append && strcmp(cursor->key_format, "r") != 0) { + fprintf(stderr, + "%s: %s: -a option illegal unless the primary key is a " + "record number\n", + progname, uri); + ret = 1; + } else + ret = insert(cursor, uri); + + /* + * Technically, we don't have to close the cursor because the session + * handle will do it for us, but I'd like to see the flush to disk and + * the close succeed, it's better to fail early when loading files. + */ + if ((tret = cursor->close(cursor)) != 0) { + tret = util_err(tret, "%s: cursor.close", uri); + if (ret == 0) + ret = tret; + } + if (ret == 0 && (ret = session->sync(session, uri, NULL)) != 0) + ret = util_err(ret, "%s: session.sync", uri); + + return (ret == 0 ? 0 : 1); +} + +/* + * schema_read -- + * Read the schema lines and do some basic validation. + */ +static int +schema_read(char ***listp, int *hexp) +{ + ULINE l; + int entry, eof, max_entry; + const char *s; + char **list; + + memset(&l, 0, sizeof(l)); + + /* Header line #1: "WiredTiger Dump" and a WiredTiger version. */ + if (util_read_line(&l, 0, &eof)) + return (1); + s = "WiredTiger Dump "; + if (strncmp(l.mem, s, strlen(s)) != 0) + return (format()); + + /* Header line #2: "Format={hex,print}". */ + if (util_read_line(&l, 0, &eof)) + return (1); + if (strcmp(l.mem, "Format=print") == 0) + *hexp = 0; + else if (strcmp(l.mem, "Format=hex") == 0) + *hexp = 1; + else + return (format()); + + /* Header line #3: "Header". */ + if (util_read_line(&l, 0, &eof)) + return (1); + if (strcmp(l.mem, "Header") != 0) + return (format()); + + /* Now, read in lines until we get to the end of the headers. */ + for (entry = max_entry = 0, list = NULL;; ++entry) { + if (util_read_line(&l, 0, &eof)) + return (1); + if (strcmp(l.mem, "Data") == 0) + break; + + /* Grow the array of header lines as necessary. */ + if ((max_entry == 0 || entry == max_entry - 1) && + (list = realloc(list, + (size_t)(max_entry += 100) * sizeof(char *))) == NULL) + return (util_err(errno, NULL)); + if ((list[entry] = strdup(l.mem)) == NULL) + return (util_err(errno, NULL)); + } + list[entry] = NULL; + *listp = list; + + /* The lines are supposed to be in pairs. */ + if (entry / 2 == 0) + return (format()); + + /* Leak the memory, I don't care. */ + return (0); +} + +/* + * schema_update -- + * Reconcile and update the command line configuration against the + * schema we found. + */ +static int +schema_update(WT_SESSION *session, char **list) +{ + int found; + const char *cfg[] = { NULL, NULL, NULL }; + char **configp, **listp, *p, *t; + +#define MATCH(s, tag) \ + (strncmp(s, tag, strlen(tag)) == 0) + + /* + * If the object has been renamed, replace all of the column group, + * index, file and table names with the new name. + */ + if (cmdname != NULL) { + for (listp = list; *listp != NULL; listp += 2) + if (MATCH(*listp, "colgroup:") || + MATCH(*listp, "file:") || + MATCH(*listp, "index:") || + MATCH(*listp, "table:")) + if (schema_rename(listp, cmdname)) + return (1); + + /* + * If the object was renamed, and there are configuration pairs, + * rename the configuration pairs as well, because we don't know + * if the user used the old or new names for the pair's URI. + */ + for (configp = cmdconfig; + cmdconfig != NULL && *configp != NULL; configp += 2) + if (schema_rename(configp, cmdname)) + return (1); + } + + /* + * Remove all "filename=" configurations from the values, new filenames + * are chosen as part of table load. + */ + for (listp = list; *listp != NULL; listp += 2) + if ((p = strstr(listp[1], "filename=")) != NULL) { + if ((t = strchr(p, ',')) == NULL) + *p = '\0'; + else + strcpy(p, t + 1); + } + + /* + * It's possible to update everything except the key/value formats. + * If there were command-line configuration pairs, walk the list of + * command-line configuration strings, and check. + */ + for (configp = cmdconfig; + cmdconfig != NULL && *configp != NULL; configp += 2) + if (strstr(configp[1], "key_format=") || + strstr(configp[1], "value_format=")) + return (util_err(0, + "the command line configuration string may not " + "modify the object's key or value format")); + + /* + * If there were command-line configuration pairs, walk the list of + * command-line URIs and find a matching dump URI. For each match, + * rewrite the dump configuration as described by the command-line + * configuration. It is an error if a command-line URI doesn't find + * a single, exact match, that's likely a mistake. + */ + for (configp = cmdconfig; + cmdconfig != NULL && *configp != NULL; configp += 2) { + found = 0; + for (listp = list; *listp != NULL; listp += 2) { + if (strncmp(*configp, listp[0], strlen(*configp)) != 0) + continue; + /* + * !!! + * We support JSON configuration strings, which leads to + * configuration strings with brackets. Unfortunately, + * that implies we can't simply append new configuration + * strings to existing ones. We call an unpublished + * WiredTiger API to do the concatentation: if anyone + * else ever needs it we can make it public, but I think + * that's unlikely. We're also playing fast and loose + * with types, but it should work. + */ + cfg[0] = listp[1]; + cfg[1] = configp[1]; + if (__wt_config_concat( + (WT_SESSION_IMPL *)session, cfg, + (const char **)&listp[1]) != 0) + return (1); + ++found; + } + switch (found) { + case 0: + return (util_err(0, + "the command line object name %s was not matched " + "by any loaded object name", *configp)); + case 1: + break; + default: + return (util_err(0, + "the command line object name %s was not unique, " + "matching more than a single loaded object name", + *configp)); + } + } + + /* Leak the memory, I don't care. */ + return (0); +} + +/* + * schema_rename -- + * Update the URI name. + */ +static int +schema_rename(char **urip, const char *name) +{ + size_t len; + char *buf, *p; + + /* Allocate room. */ + len = strlen(*urip) + strlen(name) + 10; + if ((buf = malloc(len)) == NULL) + return (util_err(errno, NULL)); + + /* + * Find the separating colon characters, but not the trailing one may + * not be there. + */ + if ((p = strchr(*urip, ':')) == NULL) + return (format()); + *p = '\0'; + p = strchr(p + 1, ':'); + snprintf(buf, len, "%s:%s%s", *urip, name, p == NULL ? "" : p); + *urip = buf; + + return (0); +} + +/* + * format -- + * The input doesn't match the dump format. + */ +static int +format(void) +{ + return (util_err(0, "input does not match WiredTiger dump format")); +} + +/* + * insert -- + * Read and insert data. + */ +static int +insert(WT_CURSOR *cursor, const char *name) +{ + ULINE key, value; + uint64_t insert_count; + int eof, ret; + + memset(&key, 0, sizeof(key)); + memset(&value, 0, sizeof(value)); + + /* Read key/value pairs and insert them into the file. */ + for (insert_count = 0;;) { + /* + * Three modes: in row-store, we always read a key and use it, + * in column-store, we might read it (a dump), we might read + * and ignore it (a dump with "append" set), or not read it at + * all (flat-text load). + */ + if (util_read_line(&key, 1, &eof)) + return (1); + if (eof == 1) + break; + if (!append) + cursor->set_key(cursor, key.mem); + + if (util_read_line(&value, 0, &eof)) + return (1); + cursor->set_value(cursor, value.mem); + + if ((ret = cursor->insert(cursor)) != 0) + return (util_err(ret, "%s: cursor.insert", name)); + + /* Report on progress every 100 inserts. */ + if (verbose && ++insert_count % 100 == 0) { + printf("\r\t%s: %" PRIu64, name, insert_count); + fflush(stdout); + } + } + + if (verbose) + printf("\r\t%s: %" PRIu64 "\n", name, insert_count); + + return (0); +} + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "load [-as] [-f input-file] [-r name] [object configuration ...]\n", + progname, usage_prefix); + return (1); +} diff --git a/src/utilities/util_loadtext.c b/src/utilities/util_loadtext.c new file mode 100644 index 00000000000..1121a02c31f --- /dev/null +++ b/src/utilities/util_loadtext.c @@ -0,0 +1,156 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int insert(WT_CURSOR *, const char *, int); +static int text(WT_SESSION *, const char *); +static int usage(void); + +int +util_loadtext(WT_SESSION *session, int argc, char *argv[]) +{ + int ch; + const char *uri; + + while ((ch = util_getopt(argc, argv, "f:")) != EOF) + switch (ch) { + case 'f': /* input file */ + if (freopen(util_optarg, "r", stdin) == NULL) + return ( + util_err(errno, "%s: reopen", util_optarg)); + break; + case '?': + default: + return (usage()); + } + argc -= util_optind; + argv += util_optind; + + /* The remaining argument is the uri. */ + if (argc != 1) + return (usage()); + if ((uri = + util_name(*argv, "table", UTIL_FILE_OK | UTIL_TABLE_OK)) == NULL) + return (1); + + return (text(session, uri)); +} + +/* + * text -- + * Load flat-text into a file/table. + */ +static int +text(WT_SESSION *session, const char *uri) +{ + WT_CURSOR *cursor; + int readkey, ret, tret; + + /* + * Open the cursor, configured to append new records (in the case of + * column-store objects), or to overwrite existing strings (in the + * case of row-store objects). The two flags are mutually exclusive, + * but the library doesn't currently care that we set both of them. + */ + if ((ret = session->open_cursor( + session, uri, NULL, "append,overwrite", &cursor)) != 0) + return (util_err(ret, "%s: session.open", uri)); + + /* + * We're about to load strings, make sure the formats match. + * + * Row-store tables have key/value pairs, column-store tables only have + * values. + */ + if (strcmp(cursor->value_format, "S") != 0 || + (strcmp(cursor->key_format, "S") != 0 && + strcmp(cursor->key_format, "r") != 0)) + return (util_err(EINVAL, + "the loadtext command can only load objects configured " + "for record number or string keys, and string values")); + readkey = strcmp(cursor->key_format, "r") == 0 ? 0 : 1; + + /* Insert the records */ + ret = insert(cursor, uri, readkey); + + /* + * Technically, we don't have to close the cursor because the session + * handle will do it for us, but I'd like to see the flush to disk and + * the close succeed, it's better to fail early when loading files. + */ + if ((tret = cursor->close(cursor)) != 0) { + tret = util_err(tret, "%s: cursor.close", uri); + if (ret == 0) + ret = tret; + } + if (ret == 0 && (ret = session->sync(session, uri, NULL)) != 0) + ret = util_err(ret, "%s: session.sync", uri); + + return (ret == 0 ? 0 : 1); +} + +/* + * insert -- + * Read and insert data. + */ +static int +insert(WT_CURSOR *cursor, const char *name, int readkey) +{ + ULINE key, value; + uint64_t insert_count; + int eof, ret; + + memset(&key, 0, sizeof(key)); + memset(&value, 0, sizeof(value)); + + /* Read key/value pairs and insert them into the file. */ + for (insert_count = 0;;) { + /* + * Three modes: in row-store, we always read a key and use it, + * in column-store, we might read it (a dump), we might read + * and ignore it (a dump with "append" set), or not read it at + * all (flat-text load). + */ + if (readkey) { + if (util_read_line(&key, 1, &eof)) + return (1); + if (eof == 1) + break; + cursor->set_key(cursor, key.mem); + } + if (util_read_line(&value, readkey ? 0 : 1, &eof)) + return (1); + if (eof == 1) + break; + cursor->set_value(cursor, value.mem); + + if ((ret = cursor->insert(cursor)) != 0) + return (util_err(ret, "%s: cursor.insert", name)); + + /* Report on progress every 100 inserts. */ + if (verbose && ++insert_count % 100 == 0) { + printf("\r\t%s: %" PRIu64, name, insert_count); + fflush(stdout); + } + } + + if (verbose) + printf("\r\t%s: %" PRIu64 "\n", name, insert_count); + + return (0); +} + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "loadtext [-f input-file] uri\n", + progname, usage_prefix); + return (1); +} diff --git a/src/utilities/util_main.c b/src/utilities/util_main.c new file mode 100644 index 00000000000..7fe3015f1b9 --- /dev/null +++ b/src/utilities/util_main.c @@ -0,0 +1,275 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +const char *progname; /* Program name */ + /* Global arguments */ +const char *usage_prefix = "[-Vv] [-C config] [-h home]"; +int verbose; /* Verbose flag */ + +static const char *command; /* Command name */ + +static int usage(void); + +int +main(int argc, char *argv[]) +{ + WT_CONNECTION *conn; + WT_SESSION *session; + int ch, major_v, minor_v, ret, tret; + const char *config; + + conn = NULL; + ret = 0; + + /* Get the program name. */ + if ((progname = strrchr(argv[0], '/')) == NULL) + progname = argv[0]; + else + ++progname; + command = ""; + + /* Check the version against the library build. */ + (void)wiredtiger_version(&major_v, & minor_v, NULL); + if (major_v != WIREDTIGER_VERSION_MAJOR || + minor_v != WIREDTIGER_VERSION_MINOR) { + fprintf(stderr, + "%s: program build version %d.%d does not match " + "library build version %d.%d\n", + progname, + WIREDTIGER_VERSION_MAJOR, WIREDTIGER_VERSION_MINOR, + major_v, minor_v); + return (EXIT_FAILURE); + } + + /* Check for standard options. */ + config = NULL; + while ((ch = util_getopt(argc, argv, "C:h:Vv")) != EOF) + switch (ch) { + case 'C': /* wiredtiger_open config */ + config = util_optarg; + break; + case 'h': /* home directory */ + if (chdir(util_optarg) != 0) { + fprintf(stderr, "%s: chdir %s: %s\n", + progname, util_optarg, strerror(errno)); + return (EXIT_FAILURE); + } + break; + case 'V': /* version */ + printf("%s\n", wiredtiger_version(NULL, NULL, NULL)); + return (EXIT_SUCCESS); + case 'v': /* version */ + verbose = 1; + break; + case '?': + default: + return (usage()); + } + argc -= util_optind; + argv += util_optind; + + /* The next argument is the command name. */ + if (argc < 1) + return (usage()); + command = argv[0]; + + /* Reset getopt. */ + util_optreset = 1; + util_optind = 1; + + /* The copyright option doesn't require a database. */ + switch (command[0]) { + case 'c': + if (strcmp(command, "copyright") == 0) { + util_copyright(); + return (EXIT_SUCCESS); + } + break; + } + + /* The "create" and "load" commands can create the database. */ + if (config == NULL && + (strcmp(command, "create") == 0 || strcmp(command, "load") == 0)) + config = "create"; + + if ((ret = wiredtiger_open(".", + verbose ? verbose_handler : NULL, config, &conn)) != 0) + goto err; + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + goto err; + + switch (command[0]) { + case 'c': + if (strcmp(command, "create") == 0) + ret = util_create(session, argc, argv); + else + ret = usage(); + break; + case 'd': + if (strcmp(command, "drop") == 0) + ret = util_drop(session, argc, argv); + else if (strcmp(command, "dump") == 0) + ret = util_dump(session, argc, argv); + else if (strcmp(command, "dumpfile") == 0) + ret = util_dumpfile(session, argc, argv); + else + ret = usage(); + break; + case 'l': + if (strcmp(command, "list") == 0) + ret = util_list(session, argc, argv); + else if (strcmp(command, "load") == 0) + ret = util_load(session, argc, argv); + else if (strcmp(command, "loadtext") == 0) + ret = util_loadtext(session, argc, argv); + else + ret = usage(); + break; + case 'p': + if (strcmp(command, "printlog") == 0) + ret = util_printlog(session, argc, argv); + else + ret = usage(); + break; + case 'r': + if (strcmp(command, "read") == 0) + ret = util_read(session, argc, argv); + else if (strcmp(command, "rename") == 0) + ret = util_rename(session, argc, argv); + else + ret = usage(); + break; + case 's': + if (strcmp(command, "salvage") == 0) + ret = util_salvage(session, argc, argv); + else if (strcmp(command, "stat") == 0) + ret = util_stat(session, argc, argv); + else + ret = usage(); + break; + case 'v': + if (strcmp(command, "verify") == 0) + ret = util_verify(session, argc, argv); + else + ret = usage(); + break; + case 'u': + if (strcmp(command, "upgrade") == 0) + ret = util_upgrade(session, argc, argv); + else + ret = usage(); + break; + case 'w': + if (strcmp(command, "write") == 0) + ret = util_write(session, argc, argv); + else + ret = usage(); + break; + default: + ret = usage(); + break; + } + +err: if (conn != NULL && (tret = conn->close(conn, NULL)) != 0 && ret == 0) + ret = tret; + + return (ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE); +} + +static int +usage(void) +{ + fprintf(stderr, + "WiredTiger Data Engine (version %d.%d)\n", + WIREDTIGER_VERSION_MAJOR, WIREDTIGER_VERSION_MINOR); + fprintf(stderr, + "global options:\n" + "\t-C\twiredtiger_open configuration\n" + "\t-h\tdatabase directory\n" + "\t-V\tdisplay library version and exit\n" + "\t-v\tverbose\n"); + fprintf(stderr, + "commands:\n" + "\tcopyright copyright information\n" + "\tcreate\t create an object\n" + "\tdrop\t drop an object\n" + "\tdump\t dump an object\n" + "\tdumpfile dump a physical file in debugging format\n" + "\tlist\t list database objects\n" + "\tload\t load an object\n" + "\tprintlog display the database log\n" + "\tread\t read values from an object\n" + "\trename\t rename an object\n" + "\tsalvage\t salvage a file\n" + "\tstat\t display statistics for an object\n" + "\tupgrade\t upgrade an object\n" + "\tverify\t verify an object\n" + "\twrite\t write values to an object\n"); + + return (EXIT_FAILURE); +} + +/* + * util_name -- + * Build a name. + */ +char * +util_name(const char *s, const char *type, u_int flags) +{ + size_t len; + int copy; + char *name; + + copy = 0; + if (WT_PREFIX_MATCH(s, "colgroup:")) { + if (!(flags & UTIL_COLGROUP_OK)) { + fprintf(stderr, + "%s: %s: \"colgroup\" type not supported\n", + progname, command); + return (NULL); + } + copy = 1; + } else if (WT_PREFIX_MATCH(s, "file:")) { + if (!(flags & UTIL_FILE_OK)) { + fprintf(stderr, + "%s: %s: \"file\" type not supported\n", + progname, command); + return (NULL); + } + copy = 1; + } else if (WT_PREFIX_MATCH(s, "index:")) { + if (!(flags & UTIL_INDEX_OK)) { + fprintf(stderr, + "%s: %s: \"index\" type not supported\n", + progname, command); + return (NULL); + } + copy = 1; + } else if (WT_PREFIX_MATCH(s, "table:")) { + if (!(flags & UTIL_TABLE_OK)) { + fprintf(stderr, + "%s: %s: \"table\" type not supported\n", + progname, command); + return (NULL); + } + copy = 1; + } + + len = strlen(type) + strlen(s) + 2; + if ((name = calloc(len, 1)) == NULL) { + fprintf(stderr, "%s: %s\n", progname, strerror(errno)); + return (NULL); + } + + if (copy) + strcpy(name, s); + else + snprintf(name, len, "%s:%s", type, s); + return (name); +} diff --git a/src/utilities/util_misc.c b/src/utilities/util_misc.c new file mode 100644 index 00000000000..cf5f3e9c690 --- /dev/null +++ b/src/utilities/util_misc.c @@ -0,0 +1,115 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +int +util_cerr(const char *uri, const char *op, int ret) +{ + return (util_err(ret, "%s: cursor.%s", uri, op)); +} + +/* + * util_err -- + * Report an error. + */ +int +util_err(int e, const char *fmt, ...) +{ + va_list ap; + + (void)fprintf(stderr, "%s: ", progname); + if (fmt != NULL) { + va_start(ap, fmt); + (void)vfprintf(stderr, fmt, ap); + va_end(ap); + if (e != 0) + (void)fprintf(stderr, ": "); + } + if (e != 0) + (void)fprintf(stderr, "%s", wiredtiger_strerror(e)); + (void)fprintf(stderr, "\n"); + return (1); +} + +/* + * util_read_line -- + * Read a line from stdin into a ULINE. + */ +int +util_read_line(ULINE *l, int eof_expected, int *eofp) +{ + static unsigned long long line = 0; + uint32_t len; + int ch; + + ++line; + *eofp = 0; + + for (len = 0;; ++len) { + if ((ch = getchar()) == EOF) { + if (len == 0) { + if (eof_expected) { + *eofp = 1; + return (0); + } + return (util_err(0, + "line %llu: unexpected end-of-file", line)); + } + return (util_err(0, + "line %llu: no newline terminator", line)); + } + if (ch == '\n') + break; + /* + * We nul-terminate the string so it's easier to convert the + * line into a record number, that means we always need one + * extra byte at the end. + */ + if (l->memsize == 0 || len >= l->memsize - 1) { + if ((l->mem = + realloc(l->mem, l->memsize + 1024)) == NULL) + return (util_err(errno, NULL)); + l->memsize += 1024; + } + ((uint8_t *)l->mem)[len] = (uint8_t)ch; + } + + ((uint8_t *)l->mem)[len] = '\0'; /* nul-terminate */ + + return (0); +} + +/* + * util_str2recno -- + * Convert a string to a record number. + */ +int +util_str2recno(const char *p, uint64_t *recnop) +{ + uint64_t recno; + char *endptr; + + /* + * strtouq takes lots of things like hex values, signs and so on and so + * forth -- none of them are OK with us. Check the string starts with + * digit, that turns off the special processing. + */ + if (!isdigit(p[0])) + goto format; + + errno = 0; + recno = strtouq(p, &endptr, 0); + if (recno == ULLONG_MAX && errno == ERANGE) + return (util_err(ERANGE, "%s: invalid record number", p)); + + if (endptr[0] != '\0') +format: return (util_err(EINVAL, "%s: invalid record number", p)); + + *recnop = recno; + return (0); +} diff --git a/src/utilities/util_printlog.c b/src/utilities/util_printlog.c new file mode 100644 index 00000000000..300cedd9daf --- /dev/null +++ b/src/utilities/util_printlog.c @@ -0,0 +1,85 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int usage(void); + +int +util_printlog(WT_SESSION *session, int argc, char *argv[]) +{ + WT_CURSOR *cursor; + WT_ITEM key, value; + int ch, printable, ret; + + printable = 0; + while ((ch = util_getopt(argc, argv, "f:p")) != EOF) + switch (ch) { + case 'f': /* output file */ + if (freopen(util_optarg, "w", stdout) == NULL) { + fprintf(stderr, "%s: %s: reopen: %s\n", + progname, util_optarg, strerror(errno)); + return (1); + } + break; + case 'p': + printable = 1; + break; + case '?': + default: + return (usage()); + } + argc -= util_optind; + argv += util_optind; + + /* There should not be any more arguments. */ + if (argc != 0) + return (usage()); + + if ((ret = session->open_cursor(session, "log", + NULL, printable ? "printable" : "raw", &cursor)) != 0) { + fprintf(stderr, "%s: cursor open(log) failed: %s\n", + progname, wiredtiger_strerror(ret)); + goto err; + } + + while ((ret = cursor->next(cursor)) == 0) { + if ((ret = cursor->get_key(cursor, &key)) != 0) + break; + if ((ret = cursor->get_value(cursor, &value)) != 0) + break; + if (fwrite(key.data, 1, key.size, stdout) != key.size || + fwrite("\n", 1, 1, stdout) != 1 || + fwrite(value.data, 1, value.size, stdout) != value.size || + fwrite("\n", 1, 1, stdout) != 1) { + ret = errno; + break; + } + } + if (ret == WT_NOTFOUND) + ret = 0; + else { + fprintf(stderr, "%s: cursor get(log) failed: %s\n", + progname, wiredtiger_strerror(ret)); + goto err; + } + + if (0) { +err: ret = 1; + } + return (ret); +} + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "printlog [-p] [-f output-file]\n", + progname, usage_prefix); + return (1); +} diff --git a/src/utilities/util_read.c b/src/utilities/util_read.c new file mode 100644 index 00000000000..c3d10368105 --- /dev/null +++ b/src/utilities/util_read.c @@ -0,0 +1,102 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int usage(void); + +int +util_read(WT_SESSION *session, int argc, char *argv[]) +{ + WT_CURSOR *cursor; + uint64_t recno; + int ch, rkey, ret, rval; + const char *uri, *value; + + ret = 0; + + while ((ch = util_getopt(argc, argv, "")) != EOF) + switch (ch) { + case '?': + default: + return (usage()); + } + argc -= util_optind; + argv += util_optind; + + /* The remaining arguments are a uri followed by a list of keys. */ + if (argc < 2) + return (usage()); + if ((uri = util_name(*argv, "table", UTIL_ALL_OK)) == NULL) + return (1); + + /* Open the object. */ + if ((ret = session->open_cursor( + session, uri, NULL, NULL, &cursor)) != 0) + return (util_err(ret, "%s: session.open", uri)); + + /* + * A simple search only makes sense if the key format is a string or a + * record number, and the value format is a single string. + */ + if (strcmp(cursor->key_format, "r") != 0 && + strcmp(cursor->key_format, "S") != 0) { + fprintf(stderr, + "%s: read command only possible when the key format is " + "a record number or string\n", + progname); + return (1); + } + rkey = strcmp(cursor->key_format, "r") == 0 ? 1 : 0; + if (strcmp(cursor->value_format, "S") != 0) { + fprintf(stderr, + "%s: read command only possible when the value format is " + "a string\n", + progname); + return (1); + } + + /* + * Run through the keys, returning non-zero on error or if any requested + * key isn't found. + */ + for (rval = 0; *++argv != NULL;) { + if (rkey) { + if (util_str2recno(*argv, &recno)) + return (1); + cursor->set_key(cursor, recno); + } else + cursor->set_key(cursor, *argv); + + switch (ret = cursor->search(cursor)) { + case 0: + if ((ret = cursor->get_value(cursor, &value)) != 0) + return (util_cerr(uri, "get_value", ret)); + if (printf("%s\n", value) < 0) + return (util_err(EIO, NULL)); + break; + case WT_NOTFOUND: + (void)util_err(0, "%s: not found", *argv); + rval = 1; + break; + default: + return (util_cerr(uri, "search", ret)); + } + } + + return (rval); +} + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "read uri key ...\n", + progname, usage_prefix); + return (1); +} diff --git a/src/utilities/util_rename.c b/src/utilities/util_rename.c new file mode 100644 index 00000000000..22e5067a460 --- /dev/null +++ b/src/utilities/util_rename.c @@ -0,0 +1,62 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int usage(void); + +int +util_rename(WT_SESSION *session, int argc, char *argv[]) +{ + int ch, ret; + char *uri, *newname; + + uri = NULL; + while ((ch = util_getopt(argc, argv, "")) != EOF) + switch (ch) { + case '?': + default: + return (usage()); + } + argc -= util_optind; + argv += util_optind; + + /* The remaining arguments are the object uri and new name. */ + if (argc != 2) + return (usage()); + if ((uri = util_name( + *argv, "table", UTIL_FILE_OK | UTIL_TABLE_OK)) == NULL) + return (1); + newname = argv[1]; + + if ((ret = session->rename(session, uri, newname, NULL)) != 0) { + fprintf(stderr, "%s: rename %s to %s: %s\n", + progname, uri, newname, wiredtiger_strerror(ret)); + goto err; + } + if (verbose) + printf("\n"); + + if (0) { +err: ret = 1; + } + + if (uri != NULL) + free(uri); + + return (ret); +} + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "rename uri newname\n", + progname, usage_prefix); + return (1); +} diff --git a/src/utilities/util_salvage.c b/src/utilities/util_salvage.c new file mode 100644 index 00000000000..8f61fa3a906 --- /dev/null +++ b/src/utilities/util_salvage.c @@ -0,0 +1,65 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int usage(void); + +int +util_salvage(WT_SESSION *session, int argc, char *argv[]) +{ + int ch, ret; + const char *force; + char *name; + + force = NULL; + name = NULL; + while ((ch = util_getopt(argc, argv, "F")) != EOF) + switch (ch) { + case 'F': + force = "force"; + break; + case '?': + default: + return (usage()); + } + argc -= util_optind; + argv += util_optind; + + /* The remaining argument is the file name. */ + if (argc != 1) + return (usage()); + if ((name = util_name(*argv, "file", UTIL_FILE_OK)) == NULL) + return (1); + + if ((ret = session->salvage(session, name, force)) != 0) { + fprintf(stderr, "%s: salvage(%s): %s\n", + progname, name, wiredtiger_strerror(ret)); + goto err; + } + if (verbose) + printf("\n"); + + if (0) { +err: ret = 1; + } + + if (name != NULL) + free(name); + + return (ret); +} + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "salvage [-F] file\n", + progname, usage_prefix); + return (1); +} diff --git a/src/utilities/util_stat.c b/src/utilities/util_stat.c new file mode 100644 index 00000000000..988ebba86b6 --- /dev/null +++ b/src/utilities/util_stat.c @@ -0,0 +1,97 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int usage(void); + +int +util_stat(WT_SESSION *session, int argc, char *argv[]) +{ + WT_CURSOR *cursor; + uint64_t v; + size_t urilen; + int ch, objname_free, ret; + const char *pval, *desc; + char *objname, *uri; + + objname_free = 0; + objname = uri = NULL; + while ((ch = util_getopt(argc, argv, "")) != EOF) + switch (ch) { + case '?': + default: + return (usage()); + } + argc -= util_optind; + argv += util_optind; + + /* + * If there are no arguments, the statistics cursor operates on the + * connection, otherwise, the optional remaining argument is a file + * name. + */ + switch (argc) { + case 0: + objname = (char *)""; + break; + case 1: + if ((objname = util_name(*argv, "file", UTIL_FILE_OK)) == NULL) + return (1); + objname_free = 1; + break; + default: + return (usage()); + } + + urilen = strlen("statistics:") + strlen(objname) + 1; + if ((uri = calloc(urilen, 1)) == NULL) { + fprintf(stderr, "%s: %s\n", progname, strerror(errno)); + return (1); + } + snprintf(uri, urilen, "statistics:%s", objname); + + if ((ret = session->open_cursor( + session, uri, NULL, NULL, &cursor)) != 0) { + fprintf(stderr, "%s: cursor open(%s) failed: %s\n", + progname, uri, wiredtiger_strerror(ret)); + goto err; + } + + /* List the statistics. */ + while ( + (ret = cursor->next(cursor)) == 0 && + (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) + if (printf("%s=%s\n", desc, pval) < 0) { + ret = errno; + break; + } + if (ret == WT_NOTFOUND) + ret = 0; + + if (ret != 0) { + fprintf(stderr, "%s: cursor get(%s) failed: %s\n", + progname, objname, wiredtiger_strerror(ret)); + goto err; + } + +err: if (objname_free) + free(objname); + free(uri); + + return (ret); +} + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "stat [uri]\n", + progname, usage_prefix); + return (1); +} diff --git a/src/utilities/util_upgrade.c b/src/utilities/util_upgrade.c new file mode 100644 index 00000000000..9f8ac6c8949 --- /dev/null +++ b/src/utilities/util_upgrade.c @@ -0,0 +1,61 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int usage(void); + +int +util_upgrade(WT_SESSION *session, int argc, char *argv[]) +{ + int ch, ret; + char *name; + + name = NULL; + while ((ch = util_getopt(argc, argv, "")) != EOF) + switch (ch) { + case '?': + default: + return (usage()); + } + argc -= util_optind; + argv += util_optind; + + /* The remaining argument is the table name. */ + if (argc != 1) + return (usage()); + if ((name = util_name( + *argv, "table", UTIL_FILE_OK | UTIL_TABLE_OK)) == NULL) + return (1); + + if ((ret = session->upgrade(session, name, NULL)) != 0) { + fprintf(stderr, "%s: upgrade(%s): %s\n", + progname, name, wiredtiger_strerror(ret)); + goto err; + } + if (verbose) + printf("\n"); + + if (0) { +err: ret = 1; + } + + if (name != NULL) + free(name); + + return (ret); +} + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "upgrade file\n", + progname, usage_prefix); + return (1); +} diff --git a/src/utilities/util_verbose.c b/src/utilities/util_verbose.c new file mode 100644 index 00000000000..6e3703201e5 --- /dev/null +++ b/src/utilities/util_verbose.c @@ -0,0 +1,56 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +/* + * __handle_error_verbose -- + * Verbose WT_EVENT_HANDLER->handle_error implementation: send to stderr. + */ +static void +__handle_error_verbose(WT_EVENT_HANDLER *handler, int error, const char *errmsg) +{ + WT_UNUSED(handler); + WT_UNUSED(error); + + fprintf(stderr, "%s\n", errmsg); +} + +/* + * __handle_message_verbose -- + * Verbose WT_EVENT_HANDLER->handle_message implementation: send to stdout. + */ +static int +__handle_message_verbose(WT_EVENT_HANDLER *handler, const char *message) +{ + WT_UNUSED(handler); + + (void)printf("%s\n", message); + return (0); +} + +/* + * __handle_progress_verbose -- + * Default WT_EVENT_HANDLER->handle_progress implementation: ignore. + */ +static int +__handle_progress_verbose(WT_EVENT_HANDLER *handler, + const char *operation, uint64_t progress) +{ + WT_UNUSED(handler); + + (void)printf("\r\t%s %-20" PRIu64, operation, progress); + return (0); +} + +static WT_EVENT_HANDLER __event_handler_verbose = { + __handle_error_verbose, + __handle_message_verbose, + __handle_progress_verbose +}; + +WT_EVENT_HANDLER *verbose_handler = &__event_handler_verbose; diff --git a/src/utilities/util_verify.c b/src/utilities/util_verify.c new file mode 100644 index 00000000000..1efa143cedd --- /dev/null +++ b/src/utilities/util_verify.c @@ -0,0 +1,61 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int usage(void); + +int +util_verify(WT_SESSION *session, int argc, char *argv[]) +{ + int ch, ret; + char *name; + + name = NULL; + while ((ch = util_getopt(argc, argv, "")) != EOF) + switch (ch) { + case '?': + default: + return (usage()); + } + argc -= util_optind; + argv += util_optind; + + /* The remaining argument is the table name. */ + if (argc != 1) + return (usage()); + if ((name = util_name( + *argv, "table", UTIL_FILE_OK | UTIL_TABLE_OK)) == NULL) + return (1); + + if ((ret = session->verify(session, name, NULL)) != 0) { + fprintf(stderr, "%s: verify(%s): %s\n", + progname, name, wiredtiger_strerror(ret)); + goto err; + } + if (verbose) + printf("\n"); + + if (0) { +err: ret = 1; + } + + if (name != NULL) + free(name); + + return (ret); +} + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "verify file\n", + progname, usage_prefix); + return (1); +} diff --git a/src/utilities/util_write.c b/src/utilities/util_write.c new file mode 100644 index 00000000000..26244993ea6 --- /dev/null +++ b/src/utilities/util_write.c @@ -0,0 +1,108 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int usage(void); + +int +util_write(WT_SESSION *session, int argc, char *argv[]) +{ + WT_CURSOR *cursor; + uint64_t recno; + int append, ch, overwrite, rkey, ret; + const char *uri; + char config[100]; + + append = overwrite = ret = 0; + + while ((ch = util_getopt(argc, argv, "ao")) != EOF) + switch (ch) { + case 'a': + append = 1; + break; + case 'o': + overwrite = 1; + break; + case '?': + default: + return (usage()); + } + argc -= util_optind; + argv += util_optind; + + /* + * The remaining arguments are a uri followed by a list of values (if + * append is set), or key/value pairs (if append is not set). + */ + if (append) { + if (argc < 2) + return (usage()); + } else + if (argc < 3 || ((argc - 1) % 2 != 0)) + return (usage()); + if ((uri = + util_name(*argv, "table", UTIL_FILE_OK | UTIL_TABLE_OK)) == NULL) + return (1); + + /* Open the object. */ + (void)snprintf(config, sizeof(config), "%s,%s", + append ? "append=true" : "", overwrite ? "overwrite=true" : ""); + if ((ret = session->open_cursor( + session, uri, NULL, config, &cursor)) != 0) + return (util_err(ret, "%s: session.open", uri)); + + /* + * A simple search only makes sense if the key format is a string or a + * record number, and the value format is a single string. + */ + if (strcmp(cursor->key_format, "r") != 0 && + strcmp(cursor->key_format, "S") != 0) { + fprintf(stderr, + "%s: write command only possible when the key format is " + "a record number or string\n", + progname); + return (1); + } + rkey = strcmp(cursor->key_format, "r") == 0 ? 1 : 0; + if (strcmp(cursor->value_format, "S") != 0) { + fprintf(stderr, + "%s: write command only possible when the value format is " + "a string\n", + progname); + return (1); + } + + /* Run through the values or key/value pairs. */ + while (*++argv != NULL) { + if (!append) { + if (rkey) { + if (util_str2recno(*argv, &recno)) + return (1); + cursor->set_key(cursor, recno); + } else + cursor->set_key(cursor, *argv); + ++argv; + } + cursor->set_value(cursor, *argv); + + if ((ret = cursor->insert(cursor)) != 0) + return (util_cerr(uri, "search", ret)); + } + + return (0); +} + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "read [-ao] uri key ...\n", + progname, usage_prefix); + return (1); +} diff --git a/test/3rdparty/discover-0.4.0/PKG-INFO b/test/3rdparty/discover-0.4.0/PKG-INFO new file mode 100644 index 00000000000..3bcd1661643 --- /dev/null +++ b/test/3rdparty/discover-0.4.0/PKG-INFO @@ -0,0 +1,164 @@ +Metadata-Version: 1.0 +Name: discover +Version: 0.4.0 +Summary: Test discovery for unittest. Backported from Python 2.7 for Python 2.4+ +Home-page: http://pypi.python.org/pypi/discover/ +Author: Michael Foord +Author-email: michael@voidspace.org.uk +License: UNKNOWN +Description: This is the test discovery mechanism and ``load_tests`` protocol for unittest + backported from Python 2.7 to work with Python 2.4 or more recent (including + Python 3). + + .. note:: + + Test discovery is just part of what is new in unittest in Python 2.7. All + of the new features have been backported to run on Python 2.4-2.6, including + test discovery. This is the + `unittest2 package <http://pypi.python.org/pypi/unittest2>`_. + + discover can be installed with pip or easy_install. After installing switch the + current directory to the top level directory of your project and run:: + + python -m discover + python discover.py + + (If you have setuptools or `distribute <http://pypi.python.org/pypi/distribute>`_ + installed you will also have a ``discover`` script available.) + + This will discover all tests (with certain restrictions) from the current + directory. The discover module has several options to control its behavior (full + usage options are displayed with ``python -m discover -h``):: + + Usage: discover.py [options] + + Options: + -v, --verbose Verbose output + -s directory Directory to start discovery ('.' default) + -p pattern Pattern to match test files ('test*.py' default) + -t directory Top level directory of project (default to + start directory) + + For test discovery all test modules must be importable from the top + level directory of the project. + + For example to use a different pattern for matching test modules run:: + + python -m discover -p '*test.py' + + (For UNIX-like shells like Bash you need to put quotes around the test pattern + or the shell will attempt to expand the pattern instead of passing it through to + discover. On Windows you must *not* put quotes around the pattern as the + Windows shell will pass the quotes to discover as well.) + + Test discovery is implemented in ``discover.DiscoveringTestLoader.discover``. As + well as using discover as a command line script you can import + ``DiscoveringTestLoader``, which is a subclass of ``unittest.TestLoader``, and + use it in your test framework. + + This method finds and returns all test modules from the specified start + directory, recursing into subdirectories to find them. Only test files that + match *pattern* will be loaded. (Using shell style pattern matching.) + + All test modules must be importable from the top level of the project. If + the start directory is not the top level directory then the top level + directory must be specified separately. + + The ``load_tests`` protocol allows test modules and packages to customize how + they are loaded. This is implemented in + ``discover.DiscoveringTestLoader.loadTestsFromModule``. If a test module defines + a ``load_tests`` function then tests are loaded from the module by calling + ``load_tests`` with three arguments: `loader`, `standard_tests`, `None`. + + If a test package name (directory with `__init__.py`) matches the + pattern then the package will be checked for a ``load_tests`` + function. If this exists then it will be called with *loader*, *tests*, + *pattern*. + + .. note:: + + The default pattern for matching tests is ``test*.py``. The '.py' means + that it will match test files and *not* match package names. You can + change this by changing the pattern using a command line option like + ``-p 'test*'``. + + If ``load_tests`` exists then discovery does *not* recurse into the package, + ``load_tests`` is responsible for loading all tests in the package. + + The pattern is deliberately not stored as a loader attribute so that + packages can continue discovery themselves. *top_level_dir* is stored so + ``load_tests`` does not need to pass this argument in to + ``loader.discover()``. + + discover.py is maintained in a google code project (where bugs and feature + requests should be posted): http://code.google.com/p/unittest-ext/ + + The latest development version of discover.py can be found at: + http://code.google.com/p/unittest-ext/source/browse/trunk/discover.py + + + CHANGELOG + ========= + + + 2010/06/11 0.4.0 + ---------------- + + * Addition of a setuptools compatible test collector. Set + "test_suite = 'discover.collector'" in setup.py. "setup.py test" will start + test discovery with default parameters from the same directory as the setup.py. + * Allow test discovery using dotted module names instead of a path. + * Addition of a setuptools compatible entrypoint for the discover script. + * A faulty load_tests function will not halt test discovery. A failing test + is created to report the error. + * If test discovery imports a module from the wrong location (usually because + the module is globally installed and the user is expecting to run tests + against a development version in a different location) then discovery halts + with an ImportError and the problem is reported. + * Matching files during test discovery is done in + ``DiscoveringTestLoader._match_path``. This method can be overriden in + subclasses to, for example, match on the full file path or use regular + expressions for matching. + * Tests for discovery ported from unittest2. (The tests require unittest2 to + run.) + + Feature parity with the ``TestLoader`` in Python 2.7 RC 1. + + + 2010/02/07 0.3.2 + ---------------- + + * If ``load_tests`` exists it is passed the standard tests as a ``TestSuite`` + rather than a list of tests. + + 2009/09/13 0.3.1 + ---------------- + + * Fixed a problem when a package directory matches the discovery pattern. + + 2009/08/20 0.3.0 + ---------------- + + * Failing to import a file (e.g. due to a syntax error) no longer halts + discovery but is reported as a failure. + * Discovery will not attempt to import test files whose names are not valid Python + identifiers, even if they match the pattern. +Keywords: unittest,testing,tests +Platform: UNKNOWN +Classifier: Development Status :: 4 - Beta +Classifier: Environment :: Console +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2.4 +Classifier: Programming Language :: Python :: 2.5 +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.0 +Classifier: Programming Language :: Python :: 3.1 +Classifier: Programming Language :: Python :: 3.2 +Classifier: Operating System :: OS Independent +Classifier: Topic :: Software Development :: Libraries +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Software Development :: Testing diff --git a/test/3rdparty/discover-0.4.0/README.txt b/test/3rdparty/discover-0.4.0/README.txt new file mode 100644 index 00000000000..8e6b4e11803 --- /dev/null +++ b/test/3rdparty/discover-0.4.0/README.txt @@ -0,0 +1,137 @@ +This is the test discovery mechanism and ``load_tests`` protocol for unittest +backported from Python 2.7 to work with Python 2.4 or more recent (including +Python 3). + +.. note:: + + Test discovery is just part of what is new in unittest in Python 2.7. All + of the new features have been backported to run on Python 2.4-2.6, including + test discovery. This is the + `unittest2 package <http://pypi.python.org/pypi/unittest2>`_. + +discover can be installed with pip or easy_install. After installing switch the +current directory to the top level directory of your project and run:: + + python -m discover + python discover.py + +(If you have setuptools or `distribute <http://pypi.python.org/pypi/distribute>`_ +installed you will also have a ``discover`` script available.) + +This will discover all tests (with certain restrictions) from the current +directory. The discover module has several options to control its behavior (full +usage options are displayed with ``python -m discover -h``):: + + Usage: discover.py [options] + + Options: + -v, --verbose Verbose output + -s directory Directory to start discovery ('.' default) + -p pattern Pattern to match test files ('test*.py' default) + -t directory Top level directory of project (default to + start directory) + + For test discovery all test modules must be importable from the top + level directory of the project. + +For example to use a different pattern for matching test modules run:: + + python -m discover -p '*test.py' + +(For UNIX-like shells like Bash you need to put quotes around the test pattern +or the shell will attempt to expand the pattern instead of passing it through to +discover. On Windows you must *not* put quotes around the pattern as the +Windows shell will pass the quotes to discover as well.) + +Test discovery is implemented in ``discover.DiscoveringTestLoader.discover``. As +well as using discover as a command line script you can import +``DiscoveringTestLoader``, which is a subclass of ``unittest.TestLoader``, and +use it in your test framework. + +This method finds and returns all test modules from the specified start +directory, recursing into subdirectories to find them. Only test files that +match *pattern* will be loaded. (Using shell style pattern matching.) + +All test modules must be importable from the top level of the project. If +the start directory is not the top level directory then the top level +directory must be specified separately. + +The ``load_tests`` protocol allows test modules and packages to customize how +they are loaded. This is implemented in +``discover.DiscoveringTestLoader.loadTestsFromModule``. If a test module defines +a ``load_tests`` function then tests are loaded from the module by calling +``load_tests`` with three arguments: `loader`, `standard_tests`, `None`. + +If a test package name (directory with `__init__.py`) matches the +pattern then the package will be checked for a ``load_tests`` +function. If this exists then it will be called with *loader*, *tests*, +*pattern*. + +.. note:: + + The default pattern for matching tests is ``test*.py``. The '.py' means + that it will match test files and *not* match package names. You can + change this by changing the pattern using a command line option like + ``-p 'test*'``. + +If ``load_tests`` exists then discovery does *not* recurse into the package, +``load_tests`` is responsible for loading all tests in the package. + +The pattern is deliberately not stored as a loader attribute so that +packages can continue discovery themselves. *top_level_dir* is stored so +``load_tests`` does not need to pass this argument in to +``loader.discover()``. + +discover.py is maintained in a google code project (where bugs and feature +requests should be posted): http://code.google.com/p/unittest-ext/ + +The latest development version of discover.py can be found at: +http://code.google.com/p/unittest-ext/source/browse/trunk/discover.py + + +CHANGELOG +========= + + +2010/06/11 0.4.0 +---------------- + +* Addition of a setuptools compatible test collector. Set + "test_suite = 'discover.collector'" in setup.py. "setup.py test" will start + test discovery with default parameters from the same directory as the setup.py. +* Allow test discovery using dotted module names instead of a path. +* Addition of a setuptools compatible entrypoint for the discover script. +* A faulty load_tests function will not halt test discovery. A failing test + is created to report the error. +* If test discovery imports a module from the wrong location (usually because + the module is globally installed and the user is expecting to run tests + against a development version in a different location) then discovery halts + with an ImportError and the problem is reported. +* Matching files during test discovery is done in + ``DiscoveringTestLoader._match_path``. This method can be overriden in + subclasses to, for example, match on the full file path or use regular + expressions for matching. +* Tests for discovery ported from unittest2. (The tests require unittest2 to + run.) + +Feature parity with the ``TestLoader`` in Python 2.7 RC 1. + + +2010/02/07 0.3.2 +---------------- + +* If ``load_tests`` exists it is passed the standard tests as a ``TestSuite`` + rather than a list of tests. + +2009/09/13 0.3.1 +---------------- + +* Fixed a problem when a package directory matches the discovery pattern. + +2009/08/20 0.3.0 +---------------- + +* Failing to import a file (e.g. due to a syntax error) no longer halts + discovery but is reported as a failure. +* Discovery will not attempt to import test files whose names are not valid Python + identifiers, even if they match the pattern.
\ No newline at end of file diff --git a/test/3rdparty/discover-0.4.0/discover.py b/test/3rdparty/discover-0.4.0/discover.py new file mode 100755 index 00000000000..c1e20273bcb --- /dev/null +++ b/test/3rdparty/discover-0.4.0/discover.py @@ -0,0 +1,480 @@ +#! /usr/bin/env python +# Copyright Michael Foord 2009-2010 +# discover.py +# Test discovery for unittest +# Compatible with Python 2.4-2.6 and 3.0-3.1 +# Licensed under the BSD License +# See: http://pypi.python.org/pypi/discover + +import optparse +import os +import re +import sys +import traceback +import types +import unittest + +from fnmatch import fnmatch + +__version__ = '0.4.0' +__all__ = ['DiscoveringTestLoader', 'main', 'defaultTestLoader'] + + +if hasattr(types, 'ClassType'): + class_types = (types.ClassType, type) +else: + # for Python 3.0 compatibility + class_types = type + +def _CmpToKey(mycmp): + 'Convert a cmp= function into a key= function' + class K(object): + def __init__(self, obj): + self.obj = obj + def __lt__(self, other): + return mycmp(self.obj, other.obj) == -1 + return K + +try: + from types import UnboundMethodType +except ImportError: + # Python 3 compatibility + UnboundMethodType = types.FunctionType + +# what about .pyc or .pyo (etc) +# we would need to avoid loading the same tests multiple times +# from '.py', '.pyc' *and* '.pyo' +VALID_MODULE_NAME = re.compile(r'[_a-z]\w*\.py$', re.IGNORECASE) + + +def _make_failed_import_test(name, suiteClass): + message = 'Failed to import test module: %s' % name + if hasattr(traceback, 'format_exc'): + # Python 2.3 compatibility + # format_exc returns two frames of discover.py as well + message += '\n%s' % traceback.format_exc() + return _make_failed_test('ModuleImportFailure', name, ImportError(message), + suiteClass) + +def _make_failed_load_tests(name, exception, suiteClass): + return _make_failed_test('LoadTestsFailure', name, exception, suiteClass) + +def _make_failed_test(classname, methodname, exception, suiteClass): + def testFailure(self): + raise exception + attrs = {methodname: testFailure} + TestClass = type(classname, (unittest.TestCase,), attrs) + return suiteClass((TestClass(methodname),)) + +try: + cmp +except NameError: + @staticmethod + def cmp(x, y): + """Return -1 if x < y, 0 if x == y and 1 if x > y""" + return (x > y) - (x < y) + + +class DiscoveringTestLoader(unittest.TestLoader): + """ + This class is responsible for loading tests according to various criteria + and returning them wrapped in a TestSuite + """ + testMethodPrefix = 'test' + sortTestMethodsUsing = cmp + suiteClass = unittest.TestSuite + _top_level_dir = None + + def loadTestsFromTestCase(self, testCaseClass): + """Return a suite of all tests cases contained in testCaseClass""" + if issubclass(testCaseClass, unittest.TestSuite): + raise TypeError("Test cases should not be derived from TestSuite." + " Maybe you meant to derive from TestCase?") + testCaseNames = self.getTestCaseNames(testCaseClass) + if not testCaseNames and hasattr(testCaseClass, 'runTest'): + testCaseNames = ['runTest'] + loaded_suite = self.suiteClass(map(testCaseClass, testCaseNames)) + return loaded_suite + + def loadTestsFromModule(self, module, use_load_tests=True): + """Return a suite of all tests cases contained in the given module""" + tests = [] + for name in dir(module): + obj = getattr(module, name) + if isinstance(obj, type) and issubclass(obj, unittest.TestCase): + tests.append(self.loadTestsFromTestCase(obj)) + + load_tests = getattr(module, 'load_tests', None) + tests = self.suiteClass(tests) + if use_load_tests and load_tests is not None: + try: + return load_tests(self, tests, None) + except: + ExceptionClass, e = sys.exc_info()[:2] + if not isinstance(e, Exception): + # for BaseException exceptions + raise + return _make_failed_load_tests(module.__name__, e, + self.suiteClass) + return tests + + def loadTestsFromName(self, name, module=None): + """Return a suite of all tests cases given a string specifier. + + The name may resolve either to a module, a test case class, a + test method within a test case class, or a callable object which + returns a TestCase or TestSuite instance. + + The method optionally resolves the names relative to a given module. + """ + parts = name.split('.') + if module is None: + parts_copy = parts[:] + while parts_copy: + try: + module = __import__('.'.join(parts_copy)) + break + except ImportError: + del parts_copy[-1] + if not parts_copy: + raise + parts = parts[1:] + obj = module + for part in parts: + parent, obj = obj, getattr(obj, part) + + if isinstance(obj, types.ModuleType): + return self.loadTestsFromModule(obj) + elif isinstance(obj, type) and issubclass(obj, unittest.TestCase): + return self.loadTestsFromTestCase(obj) + elif (isinstance(obj, UnboundMethodType) and + isinstance(parent, type) and + issubclass(parent, unittest.TestCase)): + name = obj.__name__ + inst = parent(name) + # static methods follow a different path + if not isinstance(getattr(inst, name), types.FunctionType): + return self.suiteClass([inst]) + elif isinstance(obj, unittest.TestSuite): + return obj + if hasattr(obj, '__call__'): + test = obj() + if isinstance(test, unittest.TestSuite): + return test + elif isinstance(test, unittest.TestCase): + return self.suiteClass([test]) + else: + raise TypeError("calling %s returned %s, not a test" % + (obj, test)) + else: + raise TypeError("don't know how to make test from: %s" % obj) + + def loadTestsFromNames(self, names, module=None): + """Return a suite of all tests cases found using the given sequence + of string specifiers. See 'loadTestsFromName()'. + """ + suites = [self.loadTestsFromName(name, module) for name in names] + return self.suiteClass(suites) + + def getTestCaseNames(self, testCaseClass): + """Return a sorted sequence of method names found within testCaseClass + """ + def isTestMethod(attrname, testCaseClass=testCaseClass, + prefix=self.testMethodPrefix): + return attrname.startswith(prefix) and \ + hasattr(getattr(testCaseClass, attrname), '__call__') + testFnNames = list(filter(isTestMethod, dir(testCaseClass))) + if self.sortTestMethodsUsing: + testFnNames.sort(key=_CmpToKey(self.sortTestMethodsUsing)) + return testFnNames + + def discover(self, start_dir, pattern='test*.py', top_level_dir=None): + """Find and return all test modules from the specified start + directory, recursing into subdirectories to find them. Only test files + that match the pattern will be loaded. (Using shell style pattern + matching.) + + All test modules must be importable from the top level of the project. + If the start directory is not the top level directory then the top + level directory must be specified separately. + + If a test package name (directory with '__init__.py') matches the + pattern then the package will be checked for a 'load_tests' function. If + this exists then it will be called with loader, tests, pattern. + + If load_tests exists then discovery does *not* recurse into the package, + load_tests is responsible for loading all tests in the package. + + The pattern is deliberately not stored as a loader attribute so that + packages can continue discovery themselves. top_level_dir is stored so + load_tests does not need to pass this argument in to loader.discover(). + """ + set_implicit_top = False + if top_level_dir is None and self._top_level_dir is not None: + # make top_level_dir optional if called from load_tests in a package + top_level_dir = self._top_level_dir + elif top_level_dir is None: + set_implicit_top = True + top_level_dir = start_dir + + top_level_dir = os.path.abspath(top_level_dir) + + if not top_level_dir in sys.path: + # all test modules must be importable from the top level directory + # should we *unconditionally* put the start directory in first + # in sys.path to minimise likelihood of conflicts between installed + # modules and development versions? + sys.path.insert(0, top_level_dir) + self._top_level_dir = top_level_dir + + is_not_importable = False + if os.path.isdir(os.path.abspath(start_dir)): + start_dir = os.path.abspath(start_dir) + if start_dir != top_level_dir: + is_not_importable = not os.path.isfile(os.path.join(start_dir, '__init__.py')) + else: + # support for discovery from dotted module names + try: + __import__(start_dir) + except ImportError: + is_not_importable = True + else: + the_module = sys.modules[start_dir] + top_part = start_dir.split('.')[0] + start_dir = os.path.abspath(os.path.dirname((the_module.__file__))) + if set_implicit_top: + self._top_level_dir = os.path.abspath(os.path.dirname(os.path.dirname(sys.modules[top_part].__file__))) + sys.path.remove(top_level_dir) + + if is_not_importable: + raise ImportError('Start directory is not importable: %r' % start_dir) + + tests = list(self._find_tests(start_dir, pattern)) + return self.suiteClass(tests) + + def _get_name_from_path(self, path): + path = os.path.splitext(os.path.normpath(path))[0] + + _relpath = relpath(path, self._top_level_dir) + assert not os.path.isabs(_relpath), "Path must be within the project" + assert not _relpath.startswith('..'), "Path must be within the project" + + name = _relpath.replace(os.path.sep, '.') + return name + + def _get_module_from_name(self, name): + __import__(name) + return sys.modules[name] + + def _match_path(self, path, full_path, pattern): + # override this method to use alternative matching strategy + return fnmatch(path, pattern) + + def _find_tests(self, start_dir, pattern): + """Used by discovery. Yields test suites it loads.""" + paths = os.listdir(start_dir) + + for path in paths: + full_path = os.path.join(start_dir, path) + if os.path.isfile(full_path): + if not VALID_MODULE_NAME.match(path): + # valid Python identifiers only + continue + if not self._match_path(path, full_path, pattern): + continue + # if the test file matches, load it + name = self._get_name_from_path(full_path) + try: + module = self._get_module_from_name(name) + except: + yield _make_failed_import_test(name, self.suiteClass) + else: + mod_file = os.path.abspath(getattr(module, '__file__', full_path)) + realpath = os.path.splitext(mod_file)[0] + fullpath_noext = os.path.splitext(full_path)[0] + if realpath.lower() != fullpath_noext.lower(): + module_dir = os.path.dirname(realpath) + mod_name = os.path.splitext(os.path.basename(full_path))[0] + expected_dir = os.path.dirname(full_path) + msg = ("%r module incorrectly imported from %r. Expected %r. " + "Is this module globally installed?") + raise ImportError(msg % (mod_name, module_dir, expected_dir)) + yield self.loadTestsFromModule(module) + elif os.path.isdir(full_path): + if not os.path.isfile(os.path.join(full_path, '__init__.py')): + continue + + load_tests = None + tests = None + if fnmatch(path, pattern): + # only check load_tests if the package directory itself matches the filter + name = self._get_name_from_path(full_path) + package = self._get_module_from_name(name) + load_tests = getattr(package, 'load_tests', None) + tests = self.loadTestsFromModule(package, use_load_tests=False) + + if load_tests is None: + if tests is not None: + # tests loaded from package file + yield tests + # recurse into the package + for test in self._find_tests(full_path, pattern): + yield test + else: + try: + yield load_tests(self, tests, pattern) + except: + ExceptionClass, e = sys.exc_info()[:2] + if not isinstance(e, Exception): + # for BaseException exceptions + raise + yield _make_failed_load_tests(package.__name__, e, + self.suiteClass) + + +############################################## +# relpath implementation taken from Python 2.7 + +if not hasattr(os.path, 'relpath'): + if os.path is sys.modules.get('ntpath'): + def relpath(path, start=os.path.curdir): + """Return a relative version of a path""" + + if not path: + raise ValueError("no path specified") + start_list = os.path.abspath(start).split(os.path.sep) + path_list = os.path.abspath(path).split(os.path.sep) + if start_list[0].lower() != path_list[0].lower(): + unc_path, rest = os.path.splitunc(path) + unc_start, rest = os.path.splitunc(start) + if bool(unc_path) ^ bool(unc_start): + raise ValueError("Cannot mix UNC and non-UNC paths (%s and %s)" + % (path, start)) + else: + raise ValueError("path is on drive %s, start on drive %s" + % (path_list[0], start_list[0])) + # Work out how much of the filepath is shared by start and path. + for i in range(min(len(start_list), len(path_list))): + if start_list[i].lower() != path_list[i].lower(): + break + else: + i += 1 + + rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:] + if not rel_list: + return os.path.curdir + return os.path.join(*rel_list) + + else: + # default to posixpath definition + def relpath(path, start=os.path.curdir): + """Return a relative version of a path""" + + if not path: + raise ValueError("no path specified") + + start_list = os.path.abspath(start).split(os.path.sep) + path_list = os.path.abspath(path).split(os.path.sep) + + # Work out how much of the filepath is shared by start and path. + i = len(os.path.commonprefix([start_list, path_list])) + + rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:] + if not rel_list: + return os.path.curdir + return os.path.join(*rel_list) +else: + from os.path import relpath + +############################################# + + +USAGE = """\ +Usage: discover.py [options] + +Options: + -v, --verbose Verbose output + -s directory Directory to start discovery ('.' default) + -p pattern Pattern to match test files ('test*.py' default) + -t directory Top level directory of project (default to + start directory) + +For test discovery all test modules must be importable from the top +level directory of the project. +""" + +def _usage_exit(msg=None): + if msg: + print (msg) + print (USAGE) + sys.exit(2) + + +def _do_discovery(argv, verbosity, Loader): + # handle command line args for test discovery + parser = optparse.OptionParser() + parser.add_option('-v', '--verbose', dest='verbose', default=False, + help='Verbose output', action='store_true') + parser.add_option('-s', '--start-directory', dest='start', default='.', + help="Directory to start discovery ('.' default)") + parser.add_option('-p', '--pattern', dest='pattern', default='test*.py', + help="Pattern to match tests ('test*.py' default)") + parser.add_option('-t', '--top-level-directory', dest='top', default=None, + help='Top level directory of project (defaults to start directory)') + + options, args = parser.parse_args(argv) + if len(args) > 3: + _usage_exit() + + for name, value in zip(('start', 'pattern', 'top'), args): + setattr(options, name, value) + + if options.verbose: + verbosity = 2 + + start_dir = options.start + pattern = options.pattern + top_level_dir = options.top + + loader = Loader() + return loader.discover(start_dir, pattern, top_level_dir), verbosity + + +def _run_tests(tests, testRunner, verbosity, exit): + if isinstance(testRunner, class_types): + try: + testRunner = testRunner(verbosity=verbosity) + except TypeError: + # didn't accept the verbosity argument + testRunner = testRunner() + result = testRunner.run(tests) + if exit: + sys.exit(not result.wasSuccessful()) + return result + + +def main(argv=None, testRunner=None, testLoader=None, exit=True, verbosity=1): + if testLoader is None: + testLoader = DiscoveringTestLoader + if testRunner is None: + testRunner = unittest.TextTestRunner + if argv is None: + argv = sys.argv[1:] + + tests, verbosity = _do_discovery(argv, verbosity, testLoader) + return _run_tests(tests, testRunner, verbosity, exit) + +defaultTestLoader = DiscoveringTestLoader() + +def collector(): + # import __main__ triggers code re-execution + __main__ = sys.modules['__main__'] + setupDir = os.path.abspath(os.path.dirname(__main__.__file__)) + return defaultTestLoader.discover(setupDir) + +if __name__ == '__main__': + if sys.argv[0] is None: + # fix for weird behaviour when run with python -m + # from a zipped egg. + sys.argv[0] = 'discover.py' + main() diff --git a/test/3rdparty/discover-0.4.0/setup.cfg b/test/3rdparty/discover-0.4.0/setup.cfg new file mode 100644 index 00000000000..51964789b3d --- /dev/null +++ b/test/3rdparty/discover-0.4.0/setup.cfg @@ -0,0 +1,2 @@ +[sdist] +force-manifest = 1
\ No newline at end of file diff --git a/test/3rdparty/discover-0.4.0/setup.py b/test/3rdparty/discover-0.4.0/setup.py new file mode 100644 index 00000000000..2a0a0e7c218 --- /dev/null +++ b/test/3rdparty/discover-0.4.0/setup.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +# setup.py +# Install script for discover.py +# Copyright (C) 2009-2010 Michael Foord +# E-mail: michael AT voidspace DOT org DOT uk + +# This software is licensed under the terms of the BSD license. +# http://www.voidspace.org.uk/python/license.shtml + +import sys +from distutils.core import setup +from discover import __version__ as VERSION + + +NAME = 'discover' +MODULES = ('discover',) +DESCRIPTION = 'Test discovery for unittest. Backported from Python 2.7 for Python 2.4+' +URL = 'http://pypi.python.org/pypi/discover/' +CLASSIFIERS = [ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.4', + 'Programming Language :: Python :: 2.5', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.0', + 'Programming Language :: Python :: 3.1', + 'Programming Language :: Python :: 3.2', + 'Operating System :: OS Independent', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Topic :: Software Development :: Testing', +] +AUTHOR = 'Michael Foord' +AUTHOR_EMAIL = 'michael@voidspace.org.uk' +KEYWORDS = "unittest, testing, tests".split(', ') +LONG_DESCRIPTION = open('README.txt').read() + + +params = dict( + name=NAME, + version=VERSION, + description=DESCRIPTION, + long_description=LONG_DESCRIPTION, + author=AUTHOR, + author_email=AUTHOR_EMAIL, + url=URL, + py_modules=MODULES, + classifiers=CLASSIFIERS, + keywords=KEYWORDS +) + + +try: + from setuptools import setup +except ImportError: + from distutils.core import setup +else: + params.update(dict( + entry_points = { + 'console_scripts': [ + 'discover = discover:main', + ], + }, + )) + params['test_suite'] = 'discover.collector' + +setup(**params) + + diff --git a/test/3rdparty/testscenarios-0.2/.bzrignore b/test/3rdparty/testscenarios-0.2/.bzrignore new file mode 100644 index 00000000000..336aaca369d --- /dev/null +++ b/test/3rdparty/testscenarios-0.2/.bzrignore @@ -0,0 +1,5 @@ +TAGS +tags +lib/testtools +MANIFEST +dist diff --git a/test/3rdparty/testscenarios-0.2/Apache-2.0 b/test/3rdparty/testscenarios-0.2/Apache-2.0 new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/test/3rdparty/testscenarios-0.2/Apache-2.0 @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/test/3rdparty/testscenarios-0.2/BSD b/test/3rdparty/testscenarios-0.2/BSD new file mode 100644 index 00000000000..0e75db647b3 --- /dev/null +++ b/test/3rdparty/testscenarios-0.2/BSD @@ -0,0 +1,26 @@ +Copyright (c) Robert Collins and Testscenarios contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of Robert Collins nor the names of Subunit contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY ROBERT COLLINS AND SUBUNIT CONTRIBUTORS ``AS IS'' +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. diff --git a/test/3rdparty/testscenarios-0.2/COPYING b/test/3rdparty/testscenarios-0.2/COPYING new file mode 100644 index 00000000000..ee16c4ecaf9 --- /dev/null +++ b/test/3rdparty/testscenarios-0.2/COPYING @@ -0,0 +1,31 @@ +Testscenarios is licensed under two licenses, the Apache License, Version 2.0 +or the 3-clause BSD License. You may use this project under either of these +licenses - choose the one that works best for you. + +We require contributions to be licensed under both licenses. The primary +difference between them is that the Apache license takes care of potential +issues with Patents and other intellectual property concerns that some users +or contributors may find important. + +Generally every source file in Testscenarios needs a license grant under both +these licenses. As the code is shipped as a single unit, a brief form is used: +---- +Copyright (c) [yyyy][,yyyy]* [name or 'Testscenarios Contributors'] + +Licensed under either the Apache License, Version 2.0 or the BSD 3-clause +license at the users choice. A copy of both licenses are available in the +project source as Apache-2.0 and BSD. You may not use this file except in +compliance with one of these two licences. + +Unless required by applicable law or agreed to in writing, software +distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +license you chose for the specific language governing permissions and +limitations under that license. +---- + +Code that has been incorporated into Testscenarios from other projects will +naturally be under its own license, and will retain that license. + +A known list of such code is maintained here: +* No entries. diff --git a/test/3rdparty/testscenarios-0.2/GOALS b/test/3rdparty/testscenarios-0.2/GOALS new file mode 100644 index 00000000000..68be00129b2 --- /dev/null +++ b/test/3rdparty/testscenarios-0.2/GOALS @@ -0,0 +1,25 @@ + +testscenarios goals +=================== + + * nice, declarative interface for multiplying tests by scenarios. + + * plays nice with testresources - when a scenario uses a resource, the + resource ordering logic should be able to group them together. + + * (at user discretion) plays nice with $random test discovery + + * arbitrary post-load multiplication. + + * cross-productable scenarios (for X and for Y) + + * extenable scenarios (for X using Y) + + * scenarios and the tests that use them are loosely coupled + + * tests that use scenarios should be easy to debug + + * fast + + * usable in trial, bzr, Zope testrunner, nose and the default unittest + TestRunner diff --git a/test/3rdparty/testscenarios-0.2/HACKING b/test/3rdparty/testscenarios-0.2/HACKING new file mode 100644 index 00000000000..0c68ee7da90 --- /dev/null +++ b/test/3rdparty/testscenarios-0.2/HACKING @@ -0,0 +1,38 @@ +Contributing to testscenarios +============================= + +Code access ++++++++++++ + +Branch from the trunk (all patches should be for trunk unless there are +exceptional circumstances):: + + bzr branch lp:testscenarios path-to-new-local-branch + +Publish your branches whereever you like, I encourage launchpad hosting though, +as it can notify me of new testscenarios branches:: + + bzr push lp:~YOURUSERNAME/testscearios/YOURBRANCHNAME + +Copyright ++++++++++ + +Testscenarios is Copyright (C) 2009 Robert Collins. I'd like to be able to +offer it up for stdlib inclusion once it has proved itself, so am asking for +copyright assignment to me - or for your contributions to be under either the +BSD or Apache-2.0 licences that Testscenarios are with (which permit inclusion +in Python). + +Coding standards +++++++++++++++++ + +PEP-8 coding style please, though I'm not nitpicky. Make sure that 'make check' +passes before sending in a patch. + +Code arrangement +++++++++++++++++ + +The ``testscenarios`` module should simply import classes and functions from +more specific modules, rather than becoming large and bloated itself. For +instance, TestWithScenarios lives in testscenarios.testcase, and is imported in +the testscenarios __init__.py. diff --git a/test/3rdparty/testscenarios-0.2/MANIFEST.in b/test/3rdparty/testscenarios-0.2/MANIFEST.in new file mode 100644 index 00000000000..0edefa19588 --- /dev/null +++ b/test/3rdparty/testscenarios-0.2/MANIFEST.in @@ -0,0 +1,10 @@ +include .bzrignore +include Apache-2.0 +include BSD +include COPYING +include GOALS +include HACKING +include MANIFEST.in +include Makefile +include NEWS +include doc/*.py diff --git a/test/3rdparty/testscenarios-0.2/Makefile b/test/3rdparty/testscenarios-0.2/Makefile new file mode 100644 index 00000000000..c38edf6bfe7 --- /dev/null +++ b/test/3rdparty/testscenarios-0.2/Makefile @@ -0,0 +1,19 @@ +PYTHONPATH:=$(shell pwd)/lib:${PYTHONPATH} +PYTHON ?= python + +all: check + +check: + PYTHONPATH=$(PYTHONPATH) $(PYTHON) -m testtools.run \ + testscenarios.test_suite + +clean: + find . -name '*.pyc' -print0 | xargs -0 rm -f + +TAGS: lib/testscenarios/*.py lib/testscenarios/tests/*.py + ctags -e -R lib/testscenarios/ + +tags: lib/testscenarios/*.py lib/testscenarios/tests/*.py + ctags -R lib/testscenarios/ + +.PHONY: all check clean diff --git a/test/3rdparty/testscenarios-0.2/NEWS b/test/3rdparty/testscenarios-0.2/NEWS new file mode 100644 index 00000000000..311d57664b2 --- /dev/null +++ b/test/3rdparty/testscenarios-0.2/NEWS @@ -0,0 +1,37 @@ +--------------------------- +testscenarios release notes +--------------------------- + + +IN DEVELOPMENT +~~~~~~~~~~~~~~ + +0.2 +~~~ + +CHANGES: + +* Adjust the cloned tests ``shortDescription`` if one is present. (Ben Finney) + +0.1 +~~~ + +CHANGES: + +* Created project. The primary interfaces are + ``testscenarios.TestWithScenarios`` and + ``testscenarios.generate_scenarios``. Documentation is primarily in README. + (Robert Collins) + +* Make the README documentation doctest compatible, to be sure it works. + Also various presentation and language touchups. (Martin Pool) + (Adjusted to use doctest directly, and to not print the demo runners + output to stderror during make check - Robert Collins) + +IMPROVEMENTS: + +BUG FIXES: + +API CHANGES: + +INTERNALS: diff --git a/test/3rdparty/testscenarios-0.2/PKG-INFO b/test/3rdparty/testscenarios-0.2/PKG-INFO new file mode 100644 index 00000000000..4408c965685 --- /dev/null +++ b/test/3rdparty/testscenarios-0.2/PKG-INFO @@ -0,0 +1,274 @@ +Metadata-Version: 1.0 +Name: testscenarios +Version: 0.2 +Summary: Testscenarios, a pyunit extension for dependency injection +Home-page: https://launchpad.net/testscenarios +Author: Robert Collins +Author-email: robertc@robertcollins.net +License: UNKNOWN +Description: ***************************************************************** + testscenarios: extensions to python unittest to support scenarios + ***************************************************************** + + Copyright (c) 2009, Robert Collins <robertc@robertcollins.net> + + Licensed under either the Apache License, Version 2.0 or the BSD 3-clause + license at the users choice. A copy of both licenses are available in the + project source as Apache-2.0 and BSD. You may not use this file except in + compliance with one of these two licences. + + Unless required by applicable law or agreed to in writing, software + distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + license you chose for the specific language governing permissions and + limitations under that license. + + + testscenarios provides clean dependency injection for python unittest style + tests. This can be used for interface testing (testing many implementations via + a single test suite) or for classic dependency injection (provide tests with + dependencies externally to the test code itself, allowing easy testing in + different situations). + + Dependencies + ============ + + * Python 2.4+ + * testtools <https://launchpad.net/testtools> + + + Why TestScenarios + ================= + + Standard Python unittest.py provides on obvious method for running a single + test_foo method with two (or more) scenarios: by creating a mix-in that + provides the functions, objects or settings that make up the scenario. This is + however limited and unsatisfying. Firstly, when two projects are cooperating + on a test suite (for instance, a plugin to a larger project may want to run + the standard tests for a given interface on its implementation), then it is + easy for them to get out of sync with each other: when the list of TestCase + classes to mix-in with changes, the plugin will either fail to run some tests + or error trying to run deleted tests. Secondly, its not as easy to work with + runtime-created-subclasses (a way of dealing with the aforementioned skew) + because they require more indirection to locate the source of the test, and will + often be ignored by e.g. pyflakes pylint etc. + + It is the intent of testscenarios to make dynamically running a single test + in multiple scenarios clear, easy to debug and work with even when the list + of scenarios is dynamically generated. + + + Defining Scenarios + ================== + + A **scenario** is a tuple of a string name for the scenario, and a dict of + parameters describing the scenario. The name is appended to the test name, and + the parameters are made available to the test instance when it's run. + + Scenarios are presented in **scenario lists** which are typically Python lists + but may be any iterable. + + + Getting Scenarios applied + ========================= + + At its heart the concept is simple. For a given test object with a list of + scenarios we prepare a new test object for each scenario. This involves: + + * Clone the test to a new test with a new id uniquely distinguishing it. + * Apply the scenario to the test by setting each key, value in the scenario + as attributes on the test object. + + There are some complicating factors around making this happen seamlessly. These + factors are in two areas: + + * Choosing what scenarios to use. (See Setting Scenarios For A Test). + * Getting the multiplication to happen. + + Subclasssing + ++++++++++++ + + If you can subclass TestWithScenarios, then the ``run()`` method in + TestWithScenarios will take care of test multiplication. It will at test + execution act as a generator causing multiple tests to execute. For this to + work reliably TestWithScenarios must be first in the MRO and you cannot + override run() or __call__. This is the most robust method, in the sense + that any test runner or test loader that obeys the python unittest protocol + will run all your scenarios. + + Manual generation + +++++++++++++++++ + + If you cannot subclass TestWithScenarios (e.g. because you are using + TwistedTestCase, or TestCaseWithResources, or any one of a number of other + useful test base classes, or need to override run() or __call__ yourself) then + you can cause scenario application to happen later by calling + ``testscenarios.generate_scenarios()``. For instance:: + + >>> import unittest + >>> import StringIO + >>> from testscenarios.scenarios import generate_scenarios + + This can work with loaders and runners from the standard library, or possibly other + implementations:: + + >>> loader = unittest.TestLoader() + >>> test_suite = unittest.TestSuite() + >>> runner = unittest.TextTestRunner(stream=StringIO.StringIO()) + + >>> mytests = loader.loadTestsFromNames(['doc.test_sample']) + >>> test_suite.addTests(generate_scenarios(mytests)) + >>> runner.run(test_suite) + <unittest._TextTestResult run=1 errors=0 failures=0> + + Testloaders + +++++++++++ + + Some test loaders support hooks like ``load_tests`` and ``test_suite``. + Ensuring your tests have had scenario application done through these hooks can + be a good idea - it means that external test runners (which support these hooks + like ``nose``, ``trial``, ``tribunal``) will still run your scenarios. (Of + course, if you are using the subclassing approach this is already a surety). + With ``load_tests``:: + + >>> def load_tests(standard_tests, module, loader): + ... result = loader.suiteClass() + ... result.addTests(generate_scenarios(standard_tests)) + ... return result + + With ``test_suite``:: + + >>> def test_suite(): + ... loader = TestLoader() + ... tests = loader.loadTestsFromName(__name__) + ... result = loader.suiteClass() + ... result.addTests(generate_scenarios(tests)) + ... return result + + + Setting Scenarios for a test + ============================ + + A sample test using scenarios can be found in the doc/ folder. + + See `pydoc testscenarios` for details. + + On the TestCase + +++++++++++++++ + + You can set a scenarios attribute on the test case:: + + >>> class MyTest(unittest.TestCase): + ... + ... scenarios = [ + ... ('scenario1', dict(param=1)), + ... ('scenario2', dict(param=2)),] + + This provides the main interface by which scenarios are found for a given test. + Subclasses will inherit the scenarios (unless they override the attribute). + + After loading + +++++++++++++ + + Test scenarios can also be generated arbitrarily later, as long as the test has + not yet run. Simply replace (or alter, but be aware that many tests may share a + single scenarios attribute) the scenarios attribute. For instance in this + example some third party tests are extended to run with a custom scenario. :: + + >>> import testtools + >>> class TestTransport: + ... """Hypothetical test case for bzrlib transport tests""" + ... pass + ... + >>> stock_library_tests = unittest.TestLoader().loadTestsFromNames( + ... ['doc.test_sample']) + ... + >>> for test in testtools.iterate_tests(stock_library_tests): + ... if isinstance(test, TestTransport): + ... test.scenarios = test.scenarios + [my_vfs_scenario] + ... + >>> suite = unittest.TestSuite() + >>> suite.addTests(generate_scenarios(stock_library_tests)) + + Generated tests don't have a ``scenarios`` list, because they don't normally + require any more expansion. However, you can add a ``scenarios`` list back on + to them, and then run them through ``generate_scenarios`` again to generate the + cross product of tests. :: + + >>> class CrossProductDemo(unittest.TestCase): + ... scenarios = [('scenario_0_0', {}), + ... ('scenario_0_1', {})] + ... def test_foo(self): + ... return + ... + >>> suite = unittest.TestSuite() + >>> suite.addTests(generate_scenarios(CrossProductDemo("test_foo"))) + >>> for test in testtools.iterate_tests(suite): + ... test.scenarios = [ + ... ('scenario_1_0', {}), + ... ('scenario_1_1', {})] + ... + >>> suite2 = unittest.TestSuite() + >>> suite2.addTests(generate_scenarios(suite)) + >>> print suite2.countTestCases() + 4 + + Dynamic Scenarios + +++++++++++++++++ + + A common use case is to have the list of scenarios be dynamic based on plugins + and available libraries. An easy way to do this is to provide a global scope + scenarios somewhere relevant to the tests that will use it, and then that can + be customised, or dynamically populate your scenarios from a registry etc. + For instance:: + + >>> hash_scenarios = [] + >>> try: + ... from hashlib import md5 + ... except ImportError: + ... pass + ... else: + ... hash_scenarios.append(("md5", dict(hash=md5))) + >>> try: + ... from hashlib import sha1 + ... except ImportError: + ... pass + ... else: + ... hash_scenarios.append(("sha1", dict(hash=sha1))) + ... + >>> class TestHashContract(unittest.TestCase): + ... + ... scenarios = hash_scenarios + ... + >>> class TestHashPerformance(unittest.TestCase): + ... + ... scenarios = hash_scenarios + + + Forcing Scenarios + +++++++++++++++++ + + The ``apply_scenarios`` function can be useful to apply scenarios to a test + that has none applied. ``apply_scenarios`` is the workhorse for + ``generate_scenarios``, except it takes the scenarios passed in rather than + introspecting the test object to determine the scenarios. The + ``apply_scenarios`` function does not reset the test scenarios attribute, + allowing it to be used to layer scenarios without affecting existing scenario + selection. + + + Advice on Writing Scenarios + =========================== + + If a parameterised test is because of a bug run without being parameterized, + it should fail rather than running with defaults, because this can hide bugs. + +Platform: UNKNOWN +Classifier: Development Status :: 6 - Mature +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Topic :: Software Development :: Quality Assurance +Classifier: Topic :: Software Development :: Testing diff --git a/test/3rdparty/testscenarios-0.2/README b/test/3rdparty/testscenarios-0.2/README new file mode 100644 index 00000000000..b827cb67a82 --- /dev/null +++ b/test/3rdparty/testscenarios-0.2/README @@ -0,0 +1,256 @@ +***************************************************************** +testscenarios: extensions to python unittest to support scenarios +***************************************************************** + + Copyright (c) 2009, Robert Collins <robertc@robertcollins.net> + + Licensed under either the Apache License, Version 2.0 or the BSD 3-clause + license at the users choice. A copy of both licenses are available in the + project source as Apache-2.0 and BSD. You may not use this file except in + compliance with one of these two licences. + + Unless required by applicable law or agreed to in writing, software + distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + license you chose for the specific language governing permissions and + limitations under that license. + + +testscenarios provides clean dependency injection for python unittest style +tests. This can be used for interface testing (testing many implementations via +a single test suite) or for classic dependency injection (provide tests with +dependencies externally to the test code itself, allowing easy testing in +different situations). + +Dependencies +============ + +* Python 2.4+ +* testtools <https://launchpad.net/testtools> + + +Why TestScenarios +================= + +Standard Python unittest.py provides on obvious method for running a single +test_foo method with two (or more) scenarios: by creating a mix-in that +provides the functions, objects or settings that make up the scenario. This is +however limited and unsatisfying. Firstly, when two projects are cooperating +on a test suite (for instance, a plugin to a larger project may want to run +the standard tests for a given interface on its implementation), then it is +easy for them to get out of sync with each other: when the list of TestCase +classes to mix-in with changes, the plugin will either fail to run some tests +or error trying to run deleted tests. Secondly, its not as easy to work with +runtime-created-subclasses (a way of dealing with the aforementioned skew) +because they require more indirection to locate the source of the test, and will +often be ignored by e.g. pyflakes pylint etc. + +It is the intent of testscenarios to make dynamically running a single test +in multiple scenarios clear, easy to debug and work with even when the list +of scenarios is dynamically generated. + + +Defining Scenarios +================== + +A **scenario** is a tuple of a string name for the scenario, and a dict of +parameters describing the scenario. The name is appended to the test name, and +the parameters are made available to the test instance when it's run. + +Scenarios are presented in **scenario lists** which are typically Python lists +but may be any iterable. + + +Getting Scenarios applied +========================= + +At its heart the concept is simple. For a given test object with a list of +scenarios we prepare a new test object for each scenario. This involves: + +* Clone the test to a new test with a new id uniquely distinguishing it. +* Apply the scenario to the test by setting each key, value in the scenario + as attributes on the test object. + +There are some complicating factors around making this happen seamlessly. These +factors are in two areas: + +* Choosing what scenarios to use. (See Setting Scenarios For A Test). +* Getting the multiplication to happen. + +Subclasssing +++++++++++++ + +If you can subclass TestWithScenarios, then the ``run()`` method in +TestWithScenarios will take care of test multiplication. It will at test +execution act as a generator causing multiple tests to execute. For this to +work reliably TestWithScenarios must be first in the MRO and you cannot +override run() or __call__. This is the most robust method, in the sense +that any test runner or test loader that obeys the python unittest protocol +will run all your scenarios. + +Manual generation ++++++++++++++++++ + +If you cannot subclass TestWithScenarios (e.g. because you are using +TwistedTestCase, or TestCaseWithResources, or any one of a number of other +useful test base classes, or need to override run() or __call__ yourself) then +you can cause scenario application to happen later by calling +``testscenarios.generate_scenarios()``. For instance:: + + >>> import unittest + >>> import StringIO + >>> from testscenarios.scenarios import generate_scenarios + +This can work with loaders and runners from the standard library, or possibly other +implementations:: + + >>> loader = unittest.TestLoader() + >>> test_suite = unittest.TestSuite() + >>> runner = unittest.TextTestRunner(stream=StringIO.StringIO()) + + >>> mytests = loader.loadTestsFromNames(['doc.test_sample']) + >>> test_suite.addTests(generate_scenarios(mytests)) + >>> runner.run(test_suite) + <unittest._TextTestResult run=1 errors=0 failures=0> + +Testloaders ++++++++++++ + +Some test loaders support hooks like ``load_tests`` and ``test_suite``. +Ensuring your tests have had scenario application done through these hooks can +be a good idea - it means that external test runners (which support these hooks +like ``nose``, ``trial``, ``tribunal``) will still run your scenarios. (Of +course, if you are using the subclassing approach this is already a surety). +With ``load_tests``:: + + >>> def load_tests(standard_tests, module, loader): + ... result = loader.suiteClass() + ... result.addTests(generate_scenarios(standard_tests)) + ... return result + +With ``test_suite``:: + + >>> def test_suite(): + ... loader = TestLoader() + ... tests = loader.loadTestsFromName(__name__) + ... result = loader.suiteClass() + ... result.addTests(generate_scenarios(tests)) + ... return result + + +Setting Scenarios for a test +============================ + +A sample test using scenarios can be found in the doc/ folder. + +See `pydoc testscenarios` for details. + +On the TestCase ++++++++++++++++ + +You can set a scenarios attribute on the test case:: + + >>> class MyTest(unittest.TestCase): + ... + ... scenarios = [ + ... ('scenario1', dict(param=1)), + ... ('scenario2', dict(param=2)),] + +This provides the main interface by which scenarios are found for a given test. +Subclasses will inherit the scenarios (unless they override the attribute). + +After loading ++++++++++++++ + +Test scenarios can also be generated arbitrarily later, as long as the test has +not yet run. Simply replace (or alter, but be aware that many tests may share a +single scenarios attribute) the scenarios attribute. For instance in this +example some third party tests are extended to run with a custom scenario. :: + + >>> import testtools + >>> class TestTransport: + ... """Hypothetical test case for bzrlib transport tests""" + ... pass + ... + >>> stock_library_tests = unittest.TestLoader().loadTestsFromNames( + ... ['doc.test_sample']) + ... + >>> for test in testtools.iterate_tests(stock_library_tests): + ... if isinstance(test, TestTransport): + ... test.scenarios = test.scenarios + [my_vfs_scenario] + ... + >>> suite = unittest.TestSuite() + >>> suite.addTests(generate_scenarios(stock_library_tests)) + +Generated tests don't have a ``scenarios`` list, because they don't normally +require any more expansion. However, you can add a ``scenarios`` list back on +to them, and then run them through ``generate_scenarios`` again to generate the +cross product of tests. :: + + >>> class CrossProductDemo(unittest.TestCase): + ... scenarios = [('scenario_0_0', {}), + ... ('scenario_0_1', {})] + ... def test_foo(self): + ... return + ... + >>> suite = unittest.TestSuite() + >>> suite.addTests(generate_scenarios(CrossProductDemo("test_foo"))) + >>> for test in testtools.iterate_tests(suite): + ... test.scenarios = [ + ... ('scenario_1_0', {}), + ... ('scenario_1_1', {})] + ... + >>> suite2 = unittest.TestSuite() + >>> suite2.addTests(generate_scenarios(suite)) + >>> print suite2.countTestCases() + 4 + +Dynamic Scenarios ++++++++++++++++++ + +A common use case is to have the list of scenarios be dynamic based on plugins +and available libraries. An easy way to do this is to provide a global scope +scenarios somewhere relevant to the tests that will use it, and then that can +be customised, or dynamically populate your scenarios from a registry etc. +For instance:: + + >>> hash_scenarios = [] + >>> try: + ... from hashlib import md5 + ... except ImportError: + ... pass + ... else: + ... hash_scenarios.append(("md5", dict(hash=md5))) + >>> try: + ... from hashlib import sha1 + ... except ImportError: + ... pass + ... else: + ... hash_scenarios.append(("sha1", dict(hash=sha1))) + ... + >>> class TestHashContract(unittest.TestCase): + ... + ... scenarios = hash_scenarios + ... + >>> class TestHashPerformance(unittest.TestCase): + ... + ... scenarios = hash_scenarios + + +Forcing Scenarios ++++++++++++++++++ + +The ``apply_scenarios`` function can be useful to apply scenarios to a test +that has none applied. ``apply_scenarios`` is the workhorse for +``generate_scenarios``, except it takes the scenarios passed in rather than +introspecting the test object to determine the scenarios. The +``apply_scenarios`` function does not reset the test scenarios attribute, +allowing it to be used to layer scenarios without affecting existing scenario +selection. + + +Advice on Writing Scenarios +=========================== + +If a parameterised test is because of a bug run without being parameterized, +it should fail rather than running with defaults, because this can hide bugs. diff --git a/test/3rdparty/testscenarios-0.2/doc/__init__.py b/test/3rdparty/testscenarios-0.2/doc/__init__.py new file mode 100644 index 00000000000..4dbad55dcbb --- /dev/null +++ b/test/3rdparty/testscenarios-0.2/doc/__init__.py @@ -0,0 +1,16 @@ +# testscenarios: extensions to python unittest to allow declarative +# dependency injection ('scenarios') by tests. +# +# Copyright (c) 2009, Robert Collins <robertc@robertcollins.net> +# +# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause +# license at the users choice. A copy of both licenses are available in the +# project source as Apache-2.0 and BSD. You may not use this file except in +# compliance with one of these two licences. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# license you chose for the specific language governing permissions and +# limitations under that license. + diff --git a/test/3rdparty/testscenarios-0.2/doc/example.py b/test/3rdparty/testscenarios-0.2/doc/example.py new file mode 100644 index 00000000000..a8d195fade2 --- /dev/null +++ b/test/3rdparty/testscenarios-0.2/doc/example.py @@ -0,0 +1,30 @@ +# testscenarios: extensions to python unittest to allow declarative +# dependency injection ('scenarios') by tests. +# Copyright (c) 2009, Robert Collins <robertc@robertcollins.net> +# +# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause +# license at the users choice. A copy of both licenses are available in the +# project source as Apache-2.0 and BSD. You may not use this file except in +# compliance with one of these two licences. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# license you chose for the specific language governing permissions and +# limitations under that license. + +"""Example TestScenario.""" + +from testscenarios import TestWithScenarios + + +scenario1 = ('basic', {'attribute': 'value'}) +scenario2 = ('advanced', {'attribute': 'value2'}) + + +class SampleWithScenarios(TestWithScenarios): + + scenarios = [scenario1, scenario2] + + def test_demo(self): + self.assertIsInstance(self.attribute, str) diff --git a/test/3rdparty/testscenarios-0.2/doc/test_sample.py b/test/3rdparty/testscenarios-0.2/doc/test_sample.py new file mode 100644 index 00000000000..a0b00a5ef54 --- /dev/null +++ b/test/3rdparty/testscenarios-0.2/doc/test_sample.py @@ -0,0 +1,22 @@ +# testscenarios: extensions to python unittest to allow declarative +# dependency injection ('scenarios') by tests. +# +# Copyright (c) 2009, Robert Collins <robertc@robertcollins.net> +# +# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause +# license at the users choice. A copy of both licenses are available in the +# project source as Apache-2.0 and BSD. You may not use this file except in +# compliance with one of these two licences. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# license you chose for the specific language governing permissions and +# limitations under that license. + +import unittest + +class TestSample(unittest.TestCase): + + def test_so_easy(self): + pass diff --git a/test/3rdparty/testscenarios-0.2/lib/testscenarios/__init__.py b/test/3rdparty/testscenarios-0.2/lib/testscenarios/__init__.py new file mode 100644 index 00000000000..d608f13e842 --- /dev/null +++ b/test/3rdparty/testscenarios-0.2/lib/testscenarios/__init__.py @@ -0,0 +1,64 @@ +# testscenarios: extensions to python unittest to allow declarative +# dependency injection ('scenarios') by tests. +# +# Copyright (c) 2009, Robert Collins <robertc@robertcollins.net> +# +# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause +# license at the users choice. A copy of both licenses are available in the +# project source as Apache-2.0 and BSD. You may not use this file except in +# compliance with one of these two licences. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# license you chose for the specific language governing permissions and +# limitations under that license. + + +"""Support for running tests with different scenarios declaratively + +Testscenarios provides clean dependency injection for python unittest style +tests. This can be used for interface testing (testing many implementations via +a single test suite) or for classic dependency injection (provide tests with +dependencies externally to the test code itself, allowing easy testing in +different situations). + +See the README for a manual, and the docstrings on individual functions and +methods for details. +""" + +# same format as sys.version_info: "A tuple containing the five components of +# the version number: major, minor, micro, releaselevel, and serial. All +# values except releaselevel are integers; the release level is 'alpha', +# 'beta', 'candidate', or 'final'. The version_info value corresponding to the +# Python version 2.0 is (2, 0, 0, 'final', 0)." Additionally we use a +# releaselevel of 'dev' for unreleased under-development code. +# +# If the releaselevel is 'alpha' then the major/minor/micro components are not +# established at this point, and setup.py will use a version of next-$(revno). +# If the releaselevel is 'final', then the tarball will be major.minor.micro. +# Otherwise it is major.minor.micro~$(revno). +__version__ = (0, 2, 0, 'final', 0) + +__all__ = [ + 'TestWithScenarios', + 'apply_scenario', + 'apply_scenarios', + 'generate_scenarios', + ] + + +import unittest + +from testscenarios.scenarios import apply_scenario, generate_scenarios +from testscenarios.testcase import TestWithScenarios + + +def test_suite(): + import testscenarios.tests + return testscenarios.tests.test_suite() + + +def load_tests(standard_tests, module, loader): + standard_tests.addTests(loader.loadTestsFromNames(["testscenarios.tests"])) + return standard_tests diff --git a/test/3rdparty/testscenarios-0.2/lib/testscenarios/scenarios.py b/test/3rdparty/testscenarios-0.2/lib/testscenarios/scenarios.py new file mode 100644 index 00000000000..e531b2e0da1 --- /dev/null +++ b/test/3rdparty/testscenarios-0.2/lib/testscenarios/scenarios.py @@ -0,0 +1,78 @@ +# testscenarios: extensions to python unittest to allow declarative +# dependency injection ('scenarios') by tests. +# +# Copyright (c) 2009, Robert Collins <robertc@robertcollins.net> +# +# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause +# license at the users choice. A copy of both licenses are available in the +# project source as Apache-2.0 and BSD. You may not use this file except in +# compliance with one of these two licences. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# license you chose for the specific language governing permissions and +# limitations under that license. + +__all__ = [ + 'apply_scenario', + 'apply_scenarios', + 'generate_scenarios', + ] + +import unittest + +from testtools.testcase import clone_test_with_new_id +from testtools import iterate_tests + + +def apply_scenario((name, parameters), test): + """Apply scenario to test. + + :param scenario: A tuple (name, parameters) to apply to the test. The test + is cloned, its id adjusted to have (name) after it, and the parameters + dict is used to update the new test. + :param test: The test to apply the scenario to. This test is unaltered. + :return: A new test cloned from test, with the scenario applied. + """ + scenario_suffix = '(' + name + ')' + newtest = clone_test_with_new_id(test, + test.id() + scenario_suffix) + test_desc = test.shortDescription() + if test_desc is not None: + newtest_desc = "%(test_desc)s %(scenario_suffix)s" % vars() + newtest.shortDescription = (lambda: newtest_desc) + for key, value in parameters.iteritems(): + setattr(newtest, key, value) + return newtest + + +def apply_scenarios(scenarios, test): + """Apply many scenarios to a test. + + :param scenarios: An iterable of scenarios. + :param test: A test to apply the scenarios to. + :return: A generator of tests. + """ + for scenario in scenarios: + yield apply_scenario(scenario, test) + + +def generate_scenarios(test_or_suite): + """Yield the tests in test_or_suite with scenario multiplication done. + + TestCase objects with no scenarios specified are yielded unaltered. Tests + with scenarios are not yielded at all, instead the results of multiplying + them by the scenarios they specified gets yielded. + + :param test_or_suite: A TestCase or TestSuite. + :return: A generator of tests - objects satisfying the TestCase protocol. + """ + for test in iterate_tests(test_or_suite): + scenarios = getattr(test, 'scenarios', None) + if scenarios: + for newtest in apply_scenarios(scenarios, test): + newtest.scenarios = None + yield newtest + else: + yield test diff --git a/test/3rdparty/testscenarios-0.2/lib/testscenarios/testcase.py b/test/3rdparty/testscenarios-0.2/lib/testscenarios/testcase.py new file mode 100644 index 00000000000..5ec3a94a1d3 --- /dev/null +++ b/test/3rdparty/testscenarios-0.2/lib/testscenarios/testcase.py @@ -0,0 +1,62 @@ +# testscenarios: extensions to python unittest to allow declarative +# dependency injection ('scenarios') by tests. +# +# Copyright (c) 2009, Robert Collins <robertc@robertcollins.net> +# +# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause +# license at the users choice. A copy of both licenses are available in the +# project source as Apache-2.0 and BSD. You may not use this file except in +# compliance with one of these two licences. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# license you chose for the specific language governing permissions and +# limitations under that license. + +__all__ = [ + 'TestWithScenarios', + ] + +import unittest + +from testtools.testcase import clone_test_with_new_id + +from testscenarios.scenarios import generate_scenarios + +class TestWithScenarios(unittest.TestCase): + """A TestCase with support for scenarios via a scenarios attribute. + + When a test object which is an instance of TestWithScenarios is run, + and there is a non-empty scenarios attribute on the object, the test is + multiplied by the run method into one test per scenario. For this to work + reliably the TestWithScenarios.run method must not be overriden in a + subclass (or overridden compatibly with TestWithScenarios). + """ + + def _get_scenarios(self): + return getattr(self, 'scenarios', None) + + def countTestCases(self): + scenarios = self._get_scenarios() + if not scenarios: + return 1 + else: + return len(scenarios) + + def debug(self): + scenarios = self._get_scenarios() + if scenarios: + for test in generate_scenarios(self): + test.debug() + else: + return super(TestWithScenarios, self).debug() + + def run(self, result=None): + scenarios = self._get_scenarios() + if scenarios: + for test in generate_scenarios(self): + test.run(result) + return + else: + return super(TestWithScenarios, self).run(result) diff --git a/test/3rdparty/testscenarios-0.2/lib/testscenarios/tests/__init__.py b/test/3rdparty/testscenarios-0.2/lib/testscenarios/tests/__init__.py new file mode 100644 index 00000000000..e5e2bbeaa84 --- /dev/null +++ b/test/3rdparty/testscenarios-0.2/lib/testscenarios/tests/__init__.py @@ -0,0 +1,42 @@ +# testscenarios: extensions to python unittest to allow declarative +# dependency injection ('scenarios') by tests. +# +# Copyright (c) 2009, Robert Collins <robertc@robertcollins.net> +# +# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause +# license at the users choice. A copy of both licenses are available in the +# project source as Apache-2.0 and BSD. You may not use this file except in +# compliance with one of these two licences. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# license you chose for the specific language governing permissions and +# limitations under that license. + +import doctest +import sys +import unittest + +import testscenarios + + +def test_suite(): + result = unittest.TestSuite() + standard_tests = unittest.TestSuite() + module = sys.modules['testscenarios.tests'] + loader = unittest.TestLoader() + return load_tests(standard_tests, module, loader) + + +def load_tests(standard_tests, module, loader): + test_modules = [ + 'testcase', + 'scenarios', + ] + prefix = "testscenarios.tests.test_" + test_mod_names = [prefix + test_module for test_module in test_modules] + standard_tests.addTests(loader.loadTestsFromNames(test_mod_names)) + doctest.set_unittest_reportflags(doctest.REPORT_ONLY_FIRST_FAILURE) + standard_tests.addTest(doctest.DocFileSuite("../../../README")) + return standard_tests diff --git a/test/3rdparty/testscenarios-0.2/lib/testscenarios/tests/test_scenarios.py b/test/3rdparty/testscenarios-0.2/lib/testscenarios/tests/test_scenarios.py new file mode 100644 index 00000000000..4c801503d2c --- /dev/null +++ b/test/3rdparty/testscenarios-0.2/lib/testscenarios/tests/test_scenarios.py @@ -0,0 +1,173 @@ +# testscenarios: extensions to python unittest to allow declarative +# dependency injection ('scenarios') by tests. +# +# Copyright (c) 2009, Robert Collins <robertc@robertcollins.net> +# +# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause +# license at the users choice. A copy of both licenses are available in the +# project source as Apache-2.0 and BSD. You may not use this file except in +# compliance with one of these two licences. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# license you chose for the specific language governing permissions and +# limitations under that license. + +import unittest + +import testscenarios +from testscenarios.scenarios import ( + apply_scenario, + apply_scenarios, + generate_scenarios, + ) +import testtools +from testtools.tests.helpers import LoggingResult + + +class TestGenerateScenarios(testtools.TestCase): + + def hook_apply_scenarios(self): + self.addCleanup(setattr, testscenarios.scenarios, 'apply_scenarios', + apply_scenarios) + log = [] + def capture(scenarios, test): + log.append((scenarios, test)) + return apply_scenarios(scenarios, test) + testscenarios.scenarios.apply_scenarios = capture + return log + + def test_generate_scenarios_preserves_normal_test(self): + class ReferenceTest(unittest.TestCase): + def test_pass(self): + pass + test = ReferenceTest("test_pass") + log = self.hook_apply_scenarios() + self.assertEqual([test], list(generate_scenarios(test))) + self.assertEqual([], log) + + def test_tests_with_scenarios_calls_apply_scenarios(self): + class ReferenceTest(unittest.TestCase): + scenarios = [('demo', {})] + def test_pass(self): + pass + test = ReferenceTest("test_pass") + log = self.hook_apply_scenarios() + tests = list(generate_scenarios(test)) + self.assertEqual( + 'testscenarios.tests.test_scenarios.ReferenceTest.test_pass(demo)', + tests[0].id()) + self.assertEqual([([('demo', {})], test)], log) + + def test_all_scenarios_yielded(self): + class ReferenceTest(unittest.TestCase): + scenarios = [('1', {}), ('2', {})] + def test_pass(self): + pass + test = ReferenceTest("test_pass") + tests = list(generate_scenarios(test)) + self.assertEqual( + 'testscenarios.tests.test_scenarios.ReferenceTest.test_pass(1)', + tests[0].id()) + self.assertEqual( + 'testscenarios.tests.test_scenarios.ReferenceTest.test_pass(2)', + tests[1].id()) + + def test_scenarios_attribute_cleared(self): + class ReferenceTest(unittest.TestCase): + scenarios = [ + ('1', {'foo': 1, 'bar': 2}), + ('2', {'foo': 2, 'bar': 4})] + def test_check_foo(self): + pass + test = ReferenceTest("test_check_foo") + tests = list(generate_scenarios(test)) + for adapted in tests: + self.assertEqual(None, adapted.scenarios) + + def test_multiple_tests(self): + class Reference1(unittest.TestCase): + scenarios = [('1', {}), ('2', {})] + def test_something(self): + pass + class Reference2(unittest.TestCase): + scenarios = [('3', {}), ('4', {})] + def test_something(self): + pass + suite = unittest.TestSuite() + suite.addTest(Reference1("test_something")) + suite.addTest(Reference2("test_something")) + tests = list(generate_scenarios(suite)) + self.assertEqual(4, len(tests)) + + +class TestApplyScenario(testtools.TestCase): + + def setUp(self): + super(TestApplyScenario, self).setUp() + + self.scenario_name = 'demo' + self.scenario_attrs = {'foo': 'bar'} + self.scenario = (self.scenario_name, self.scenario_attrs) + + class ReferenceTest(unittest.TestCase): + def test_pass(self): + pass + def test_pass_with_docstring(self): + """ The test that always passes. + + This test case has a PEP 257 conformant docstring, + with its first line being a brief synopsis and the + rest of the docstring explaining that this test + does nothing but pass unconditionally. + + """ + pass + + self.ReferenceTest = ReferenceTest + + def test_sets_specified_id(self): + raw_test = self.ReferenceTest('test_pass') + raw_id = "testscenarios.tests.test_scenarios.ReferenceTest.test_pass" + scenario_name = self.scenario_name + expect_id = "%(raw_id)s(%(scenario_name)s)" % vars() + modified_test = apply_scenario(self.scenario, raw_test) + self.assertEqual(expect_id, modified_test.id()) + + def test_sets_specified_attributes(self): + raw_test = self.ReferenceTest('test_pass') + modified_test = apply_scenario(self.scenario, raw_test) + self.assertEqual('bar', modified_test.foo) + + def test_appends_scenario_name_to_short_description(self): + raw_test = self.ReferenceTest('test_pass_with_docstring') + modified_test = apply_scenario(self.scenario, raw_test) + raw_doc = self.ReferenceTest.test_pass_with_docstring.__doc__ + raw_desc = raw_doc.split("\n")[0].strip() + scenario_name = self.scenario_name + expect_desc = "%(raw_desc)s (%(scenario_name)s)" % vars() + self.assertEqual(expect_desc, modified_test.shortDescription()) + +class TestApplyScenarios(testtools.TestCase): + + def test_calls_apply_scenario(self): + self.addCleanup(setattr, testscenarios.scenarios, 'apply_scenario', + apply_scenario) + log = [] + def capture(scenario, test): + log.append((scenario, test)) + testscenarios.scenarios.apply_scenario = capture + scenarios = ["foo", "bar"] + result = list(apply_scenarios(scenarios, "test")) + self.assertEqual([('foo', 'test'), ('bar', 'test')], log) + + def test_preserves_scenarios_attribute(self): + class ReferenceTest(unittest.TestCase): + scenarios = [('demo', {})] + def test_pass(self): + pass + test = ReferenceTest("test_pass") + tests = list(apply_scenarios(ReferenceTest.scenarios, test)) + self.assertEqual([('demo', {})], ReferenceTest.scenarios) + self.assertEqual(ReferenceTest.scenarios, tests[0].scenarios) diff --git a/test/3rdparty/testscenarios-0.2/lib/testscenarios/tests/test_testcase.py b/test/3rdparty/testscenarios-0.2/lib/testscenarios/tests/test_testcase.py new file mode 100644 index 00000000000..6a9bbf997e2 --- /dev/null +++ b/test/3rdparty/testscenarios-0.2/lib/testscenarios/tests/test_testcase.py @@ -0,0 +1,145 @@ +# testscenarios: extensions to python unittest to allow declarative +# dependency injection ('scenarios') by tests. +# +# Copyright (c) 2009, Robert Collins <robertc@robertcollins.net> +# +# Licensed under either the Apache License, Version 2.0 or the BSD 3-clause +# license at the users choice. A copy of both licenses are available in the +# project source as Apache-2.0 and BSD. You may not use this file except in +# compliance with one of these two licences. +# +# Unless required by applicable law or agreed to in writing, software +# distributed under these licenses is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# license you chose for the specific language governing permissions and +# limitations under that license. + +import unittest + +import testscenarios +from testtools.tests.helpers import LoggingResult + + +class TestTestWithScenarios(unittest.TestCase): + + def test_no_scenarios_no_error(self): + class ReferenceTest(testscenarios.TestWithScenarios): + def test_pass(self): + pass + test = ReferenceTest("test_pass") + result = unittest.TestResult() + test.run(result) + self.assertTrue(result.wasSuccessful()) + self.assertEqual(1, result.testsRun) + + def test_with_one_scenario_one_run(self): + class ReferenceTest(testscenarios.TestWithScenarios): + scenarios = [('demo', {})] + def test_pass(self): + pass + test = ReferenceTest("test_pass") + log = [] + result = LoggingResult(log) + test.run(result) + self.assertTrue(result.wasSuccessful()) + self.assertEqual(1, result.testsRun) + self.assertEqual( + 'testscenarios.tests.test_testcase.ReferenceTest.test_pass(demo)', + log[0][1].id()) + + def test_with_two_scenarios_two_run(self): + class ReferenceTest(testscenarios.TestWithScenarios): + scenarios = [('1', {}), ('2', {})] + def test_pass(self): + pass + test = ReferenceTest("test_pass") + log = [] + result = LoggingResult(log) + test.run(result) + self.assertTrue(result.wasSuccessful()) + self.assertEqual(2, result.testsRun) + self.assertEqual( + 'testscenarios.tests.test_testcase.ReferenceTest.test_pass(1)', + log[0][1].id()) + self.assertEqual( + 'testscenarios.tests.test_testcase.ReferenceTest.test_pass(2)', + log[4][1].id()) + + def test_attributes_set(self): + class ReferenceTest(testscenarios.TestWithScenarios): + scenarios = [ + ('1', {'foo': 1, 'bar': 2}), + ('2', {'foo': 2, 'bar': 4})] + def test_check_foo(self): + self.assertEqual(self.foo * 2, self.bar) + test = ReferenceTest("test_check_foo") + log = [] + result = LoggingResult(log) + test.run(result) + self.assertTrue(result.wasSuccessful()) + self.assertEqual(2, result.testsRun) + + def test_scenarios_attribute_cleared(self): + class ReferenceTest(testscenarios.TestWithScenarios): + scenarios = [ + ('1', {'foo': 1, 'bar': 2}), + ('2', {'foo': 2, 'bar': 4})] + def test_check_foo(self): + self.assertEqual(self.foo * 2, self.bar) + test = ReferenceTest("test_check_foo") + log = [] + result = LoggingResult(log) + test.run(result) + self.assertTrue(result.wasSuccessful()) + self.assertEqual(2, result.testsRun) + self.assertNotEqual(None, test.scenarios) + self.assertEqual(None, log[0][1].scenarios) + self.assertEqual(None, log[4][1].scenarios) + + def test_countTestCases_no_scenarios(self): + class ReferenceTest(testscenarios.TestWithScenarios): + def test_check_foo(self): + pass + test = ReferenceTest("test_check_foo") + self.assertEqual(1, test.countTestCases()) + + def test_countTestCases_empty_scenarios(self): + class ReferenceTest(testscenarios.TestWithScenarios): + scenarios = [] + def test_check_foo(self): + pass + test = ReferenceTest("test_check_foo") + self.assertEqual(1, test.countTestCases()) + + def test_countTestCases_1_scenarios(self): + class ReferenceTest(testscenarios.TestWithScenarios): + scenarios = [('1', {'foo': 1, 'bar': 2})] + def test_check_foo(self): + pass + test = ReferenceTest("test_check_foo") + self.assertEqual(1, test.countTestCases()) + + def test_countTestCases_2_scenarios(self): + class ReferenceTest(testscenarios.TestWithScenarios): + scenarios = [ + ('1', {'foo': 1, 'bar': 2}), + ('2', {'foo': 2, 'bar': 4})] + def test_check_foo(self): + pass + test = ReferenceTest("test_check_foo") + self.assertEqual(2, test.countTestCases()) + + def test_debug_2_scenarios(self): + log = [] + class ReferenceTest(testscenarios.TestWithScenarios): + scenarios = [ + ('1', {'foo': 1, 'bar': 2}), + ('2', {'foo': 2, 'bar': 4})] + def test_check_foo(self): + log.append(self) + test = ReferenceTest("test_check_foo") + test.debug() + self.assertEqual(2, len(log)) + self.assertEqual(None, log[0].scenarios) + self.assertEqual(None, log[1].scenarios) + self.assertNotEqual(log[0].id(), log[1].id()) diff --git a/test/3rdparty/testscenarios-0.2/setup.py b/test/3rdparty/testscenarios-0.2/setup.py new file mode 100755 index 00000000000..ace9f08c882 --- /dev/null +++ b/test/3rdparty/testscenarios-0.2/setup.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python + +from distutils.core import setup +import os.path + +description = file(os.path.join(os.path.dirname(__file__), 'README'), 'rb').read() + +setup(name="testscenarios", + version="0.2", + description="Testscenarios, a pyunit extension for dependency injection", + long_description=description, + maintainer="Robert Collins", + maintainer_email="robertc@robertcollins.net", + url="https://launchpad.net/testscenarios", + packages=['testscenarios', 'testscenarios.tests'], + package_dir = {'':'lib'}, + classifiers = [ + 'Development Status :: 6 - Mature', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Topic :: Software Development :: Quality Assurance', + 'Topic :: Software Development :: Testing', + ], + ) diff --git a/test/3rdparty/testtools-0.9.12/.bzrignore b/test/3rdparty/testtools-0.9.12/.bzrignore new file mode 100644 index 00000000000..d6aac0da189 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/.bzrignore @@ -0,0 +1,10 @@ +__pycache__ +./build +MANIFEST +dist +tags +TAGS +apidocs +_trial_temp +doc/_build +./.testrepository diff --git a/test/3rdparty/testtools-0.9.12/LICENSE b/test/3rdparty/testtools-0.9.12/LICENSE new file mode 100644 index 00000000000..42421b0b2d9 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/LICENSE @@ -0,0 +1,57 @@ +Copyright (c) 2008-2011 Jonathan M. Lange <jml@mumak.net> and the testtools +authors. + +The testtools authors are: + * Canonical Ltd + * Twisted Matrix Labs + * Jonathan Lange + * Robert Collins + * Andrew Bennetts + * Benjamin Peterson + * Jamu Kakar + * James Westby + * Martin [gz] + * Michael Hudson-Doyle + * Aaron Bentley + * Christian Kampka + * Gavin Panella + * Martin Pool + +and are collectively referred to as "testtools developers". + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Some code in testtools/run.py taken from Python's unittest module: +Copyright (c) 1999-2003 Steve Purcell +Copyright (c) 2003-2010 Python Software Foundation + +This module is free software, and you may redistribute it and/or modify +it under the same terms as Python itself, so long as this copyright message +and disclaimer are retained in their original form. + +IN NO EVENT SHALL THE AUTHOR BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, +SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OF +THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGE. + +THE AUTHOR SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, +AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, +SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. diff --git a/test/3rdparty/testtools-0.9.12/MANIFEST.in b/test/3rdparty/testtools-0.9.12/MANIFEST.in new file mode 100644 index 00000000000..92a623b2a12 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/MANIFEST.in @@ -0,0 +1,12 @@ +include LICENSE +include HACKING +include Makefile +include MANIFEST.in +include MANUAL +include NEWS +include README +include .bzrignore +graft doc +graft doc/_static +graft doc/_templates +prune doc/_build diff --git a/test/3rdparty/testtools-0.9.12/Makefile b/test/3rdparty/testtools-0.9.12/Makefile new file mode 100644 index 00000000000..b3e40ecddfb --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/Makefile @@ -0,0 +1,56 @@ +# See README for copyright and licensing details. + +PYTHON=python +SOURCES=$(shell find testtools -name "*.py") + +check: + PYTHONPATH=$(PWD) $(PYTHON) -m testtools.run testtools.tests.test_suite + +TAGS: ${SOURCES} + ctags -e -R testtools/ + +tags: ${SOURCES} + ctags -R testtools/ + +clean: clean-sphinx + rm -f TAGS tags + find testtools -name "*.pyc" -exec rm '{}' \; + +prerelease: + # An existing MANIFEST breaks distutils sometimes. Avoid that. + -rm MANIFEST + +release: + ./setup.py sdist upload --sign + $(PYTHON) scripts/_lp_release.py + +snapshot: prerelease + ./setup.py sdist + +### Documentation ### + +apidocs: + # pydoctor emits deprecation warnings under Ubuntu 10.10 LTS + PYTHONWARNINGS='ignore::DeprecationWarning' \ + pydoctor --make-html --add-package testtools \ + --docformat=restructuredtext --project-name=testtools \ + --project-url=https://launchpad.net/testtools + +doc/news.rst: + ln -s ../NEWS doc/news.rst + +docs: doc/news.rst docs-sphinx + rm doc/news.rst + +docs-sphinx: html-sphinx + +# Clean out generated documentation +clean-sphinx: + cd doc && make clean + +# Build the html docs using Sphinx. +html-sphinx: + cd doc && make html + +.PHONY: apidocs docs-sphinx clean-sphinx html-sphinx docs +.PHONY: check clean prerelease release diff --git a/test/3rdparty/testtools-0.9.12/NEWS b/test/3rdparty/testtools-0.9.12/NEWS new file mode 100644 index 00000000000..9ff3c05ce22 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/NEWS @@ -0,0 +1,688 @@ +testtools NEWS +++++++++++++++ + +Changes and improvements to testtools_, grouped by release. + +0.9.12 +~~~~~~ + +This is a very big release. We've made huge improvements on three fronts: + 1. Test failures are way nicer and easier to read + 2. Matchers and ``assertThat`` are much more convenient to use + 3. Correct handling of extended unicode characters + +We've trimmed off the fat from the stack trace you get when tests fail, we've +cut out the bits of error messages that just didn't help, we've made it easier +to annotate mismatch failures, to compare complex objects and to match raised +exceptions. + +Testing code was never this fun. + +Changes +------- + +* ``AfterPreproccessing`` renamed to ``AfterPreprocessing``, which is a more + correct spelling. Old name preserved for backwards compatibility, but is + now deprecated. Please stop using it. + (Jonathan Lange, #813460) + +* ``assertThat`` raises ``MismatchError`` instead of + ``TestCase.failureException``. ``MismatchError`` is a subclass of + ``AssertionError``, so in most cases this change will not matter. However, + if ``self.failureException`` has been set to a non-default value, then + mismatches will become test errors rather than test failures. + +* ``gather_details`` takes two dicts, rather than two detailed objects. + (Jonathan Lange, #801027) + +* ``MatchesRegex`` mismatch now says "<value> does not match /<regex>/" rather + than "<regex> did not match <value>". The regular expression contains fewer + backslashes too. (Jonathan Lange, #818079) + +* Tests that run with ``AsynchronousDeferredRunTest`` now have the ``reactor`` + attribute set to the running reactor. (Jonathan Lange, #720749) + +Improvements +------------ + +* All public matchers are now in ``testtools.matchers.__all__``. + (Jonathan Lange, #784859) + +* ``assertThat`` can actually display mismatches and matchers that contain + extended unicode characters. (Jonathan Lange, Martin [gz], #804127) + +* ``assertThat`` output is much less verbose, displaying only what the mismatch + tells us to display. Old-style verbose output can be had by passing + ``verbose=True`` to assertThat. (Jonathan Lange, #675323, #593190) + +* ``assertThat`` accepts a message which will be used to annotate the matcher. + This can be given as a third parameter or as a keyword parameter. + (Robert Collins) + +* Automated the Launchpad part of the release process. + (Jonathan Lange, #623486) + +* Correctly display non-ASCII unicode output on terminals that claim to have a + unicode encoding. (Martin [gz], #804122) + +* ``DocTestMatches`` correctly handles unicode output from examples, rather + than raising an error. (Martin [gz], #764170) + +* ``ErrorHolder`` and ``PlaceHolder`` added to docs. (Jonathan Lange, #816597) + +* ``ExpectedException`` now matches any exception of the given type by + default, and also allows specifying a ``Matcher`` rather than a mere regular + expression. (Jonathan Lange, #791889) + +* ``FixtureSuite`` added, allows test suites to run with a given fixture. + (Jonathan Lange) + +* Hide testtools's own stack frames when displaying tracebacks, making it + easier for test authors to focus on their errors. + (Jonathan Lange, Martin [gz], #788974) + +* Less boilerplate displayed in test failures and errors. + (Jonathan Lange, #660852) + +* ``MatchesException`` now allows you to match exceptions against any matcher, + rather than just regular expressions. (Jonathan Lange, #791889) + +* ``MatchesException`` now permits a tuple of types rather than a single type + (when using the type matching mode). (Robert Collins) + +* ``MatchesStructure.byEquality`` added to make the common case of matching + many attributes by equality much easier. ``MatchesStructure.byMatcher`` + added in case folk want to match by things other than equality. + (Jonathan Lange) + +* New convenience assertions, ``assertIsNone`` and ``assertIsNotNone``. + (Christian Kampka) + +* New matchers: + + * ``AllMatch`` matches many values against a single matcher. + (Jonathan Lange, #615108) + + * ``Contains``. (Robert Collins) + + * ``GreaterThan``. (Christian Kampka) + +* New helper, ``safe_hasattr`` added. (Jonathan Lange) + +* ``reraise`` added to ``testtools.compat``. (Jonathan Lange) + + +0.9.11 +~~~~~~ + +This release brings consistent use of super for better compatibility with +multiple inheritance, fixed Python3 support, improvements in fixture and mather +outputs and a compat helper for testing libraries that deal with bytestrings. + +Changes +------- + +* ``TestCase`` now uses super to call base ``unittest.TestCase`` constructor, + ``setUp`` and ``tearDown``. (Tim Cole, #771508) + +* If, when calling ``useFixture`` an error occurs during fixture set up, we + still attempt to gather details from the fixture. (Gavin Panella) + + +Improvements +------------ + +* Additional compat helper for ``BytesIO`` for libraries that build on + testtools and are working on Python 3 porting. (Robert Collins) + +* Corrected documentation for ``MatchesStructure`` in the test authors + document. (Jonathan Lange) + +* ``LessThan`` error message now says something that is logically correct. + (Gavin Panella, #762008) + +* Multiple details from a single fixture are now kept separate, rather than + being mooshed together. (Gavin Panella, #788182) + +* Python 3 support now back in action. (Martin [gz], #688729) + +* ``try_import`` and ``try_imports`` have a callback that is called whenever + they fail to import a module. (Martin Pool) + + +0.9.10 +~~~~~~ + +The last release of testtools could not be easy_installed. This is considered +severe enough for a re-release. + +Improvements +------------ + +* Include ``doc/`` in the source distribution, making testtools installable + from PyPI again (Tres Seaver, #757439) + + +0.9.9 +~~~~~ + +Many, many new matchers, vastly expanded documentation, stacks of bug fixes, +better unittest2 integration. If you've ever wanted to try out testtools but +been afraid to do so, this is the release to try. + + +Changes +------- + +* The timestamps generated by ``TestResult`` objects when no timing data has + been received are now datetime-with-timezone, which allows them to be + sensibly serialised and transported. (Robert Collins, #692297) + +Improvements +------------ + +* ``AnnotatedMismatch`` now correctly returns details. + (Jonathan Lange, #724691) + +* distutils integration for the testtools test runner. Can now use it for + 'python setup.py test'. (Christian Kampka, #693773) + +* ``EndsWith`` and ``KeysEqual`` now in testtools.matchers.__all__. + (Jonathan Lange, #692158) + +* ``MatchesException`` extended to support a regular expression check against + the str() of a raised exception. (Jonathan Lange) + +* ``MultiTestResult`` now forwards the ``time`` API. (Robert Collins, #692294) + +* ``MultiTestResult`` now documented in the manual. (Jonathan Lange, #661116) + +* New content helpers ``content_from_file``, ``content_from_stream`` and + ``attach_file`` make it easier to attach file-like objects to a + test. (Jonathan Lange, Robert Collins, #694126) + +* New ``ExpectedException`` context manager to help write tests against things + that are expected to raise exceptions. (Aaron Bentley) + +* New matchers: + + * ``MatchesListwise`` matches an iterable of matchers against an iterable + of values. (Michael Hudson-Doyle) + + * ``MatchesRegex`` matches a string against a regular expression. + (Michael Hudson-Doyle) + + * ``MatchesStructure`` matches attributes of an object against given + matchers. (Michael Hudson-Doyle) + + * ``AfterPreproccessing`` matches values against a matcher after passing them + through a callable. (Michael Hudson-Doyle) + + * ``MatchesSetwise`` matches an iterable of matchers against an iterable of + values, without regard to order. (Michael Hudson-Doyle) + +* ``setup.py`` can now build a snapshot when Bazaar is installed but the tree + is not a Bazaar tree. (Jelmer Vernooij) + +* Support for running tests using distutils (Christian Kampka, #726539) + +* Vastly improved and extended documentation. (Jonathan Lange) + +* Use unittest2 exception classes if available. (Jelmer Vernooij) + + +0.9.8 +~~~~~ + +In this release we bring some very interesting improvements: + +* new matchers for exceptions, sets, lists, dicts and more. + +* experimental (works but the contract isn't supported) twisted reactor + support. + +* The built in runner can now list tests and filter tests (the -l and + --load-list options). + +Changes +------- + +* addUnexpectedSuccess is translated to addFailure for test results that don't + know about addUnexpectedSuccess. Further, it fails the entire result for + all testtools TestResults (i.e. wasSuccessful() returns False after + addUnexpectedSuccess has been called). Note that when using a delegating + result such as ThreadsafeForwardingResult, MultiTestResult or + ExtendedToOriginalDecorator then the behaviour of addUnexpectedSuccess is + determined by the delegated to result(s). + (Jonathan Lange, Robert Collins, #654474, #683332) + +* startTestRun will reset any errors on the result. That is, wasSuccessful() + will always return True immediately after startTestRun() is called. This + only applies to delegated test results (ThreadsafeForwardingResult, + MultiTestResult and ExtendedToOriginalDecorator) if the delegated to result + is a testtools test result - we cannot reliably reset the state of unknown + test result class instances. (Jonathan Lange, Robert Collins, #683332) + +* Responsibility for running test cleanups has been moved to ``RunTest``. + This change does not affect public APIs and can be safely ignored by test + authors. (Jonathan Lange, #662647) + +Improvements +------------ + +* New matchers: + + * ``EndsWith`` which complements the existing ``StartsWith`` matcher. + (Jonathan Lange, #669165) + + * ``MatchesException`` matches an exception class and parameters. (Robert + Collins) + + * ``KeysEqual`` matches a dictionary with particular keys. (Jonathan Lange) + +* ``assertIsInstance`` supports a custom error message to be supplied, which + is necessary when using ``assertDictEqual`` on Python 2.7 with a + ``testtools.TestCase`` base class. (Jelmer Vernooij) + +* Experimental support for running tests that return Deferreds. + (Jonathan Lange, Martin [gz]) + +* Provide a per-test decorator, run_test_with, to specify which RunTest + object to use for a given test. (Jonathan Lange, #657780) + +* Fix the runTest parameter of TestCase to actually work, rather than raising + a TypeError. (Jonathan Lange, #657760) + +* Non-release snapshots of testtools will now work with buildout. + (Jonathan Lange, #613734) + +* Malformed SyntaxErrors no longer blow up the test suite. (Martin [gz]) + +* ``MismatchesAll.describe`` no longer appends a trailing newline. + (Michael Hudson-Doyle, #686790) + +* New helpers for conditionally importing modules, ``try_import`` and + ``try_imports``. (Jonathan Lange) + +* ``Raises`` added to the ``testtools.matchers`` module - matches if the + supplied callable raises, and delegates to an optional matcher for validation + of the exception. (Robert Collins) + +* ``raises`` added to the ``testtools.matchers`` module - matches if the + supplied callable raises and delegates to ``MatchesException`` to validate + the exception. (Jonathan Lange) + +* Tests will now pass on Python 2.6.4 : an ``Exception`` change made only in + 2.6.4 and reverted in Python 2.6.5 was causing test failures on that version. + (Martin [gz], #689858). + +* ``testtools.TestCase.useFixture`` has been added to glue with fixtures nicely. + (Robert Collins) + +* ``testtools.run`` now supports ``-l`` to list tests rather than executing + them. This is useful for integration with external test analysis/processing + tools like subunit and testrepository. (Robert Collins) + +* ``testtools.run`` now supports ``--load-list``, which takes a file containing + test ids, one per line, and intersects those ids with the tests found. This + allows fine grained control of what tests are run even when the tests cannot + be named as objects to import (e.g. due to test parameterisation via + testscenarios). (Robert Collins) + +* Update documentation to say how to use testtools.run() on Python 2.4. + (Jonathan Lange, #501174) + +* ``text_content`` conveniently converts a Python string to a Content object. + (Jonathan Lange, James Westby) + + + +0.9.7 +~~~~~ + +Lots of little cleanups in this release; many small improvements to make your +testing life more pleasant. + +Improvements +------------ + +* Cleanups can raise ``testtools.MultipleExceptions`` if they have multiple + exceptions to report. For instance, a cleanup which is itself responsible for + running several different internal cleanup routines might use this. + +* Code duplication between assertEqual and the matcher Equals has been removed. + +* In normal circumstances, a TestCase will no longer share details with clones + of itself. (Andrew Bennetts, bug #637725) + +* Less exception object cycles are generated (reduces peak memory use between + garbage collection). (Martin [gz]) + +* New matchers 'DoesNotStartWith' and 'StartsWith' contributed by Canonical + from the Launchpad project. Written by James Westby. + +* Timestamps as produced by subunit protocol clients are now forwarded in the + ThreadsafeForwardingResult so correct test durations can be reported. + (Martin [gz], Robert Collins, #625594) + +* With unittest from Python 2.7 skipped tests will now show only the reason + rather than a serialisation of all details. (Martin [gz], #625583) + +* The testtools release process is now a little better documented and a little + smoother. (Jonathan Lange, #623483, #623487) + + +0.9.6 +~~~~~ + +Nothing major in this release, just enough small bits and pieces to make it +useful enough to upgrade to. + +In particular, a serious bug in assertThat() has been fixed, it's easier to +write Matchers, there's a TestCase.patch() method for those inevitable monkey +patches and TestCase.assertEqual gives slightly nicer errors. + +Improvements +------------ + +* 'TestCase.assertEqual' now formats errors a little more nicely, in the + style of bzrlib. + +* Added `PlaceHolder` and `ErrorHolder`, TestCase-like objects that can be + used to add results to a `TestResult`. + +* 'Mismatch' now takes optional description and details parameters, so + custom Matchers aren't compelled to make their own subclass. + +* jml added a built-in UTF8_TEXT ContentType to make it slightly easier to + add details to test results. See bug #520044. + +* Fix a bug in our built-in matchers where assertThat would blow up if any + of them failed. All built-in mismatch objects now provide get_details(). + +* New 'Is' matcher, which lets you assert that a thing is identical to + another thing. + +* New 'LessThan' matcher which lets you assert that a thing is less than + another thing. + +* TestCase now has a 'patch()' method to make it easier to monkey-patching + objects in tests. See the manual for more information. Fixes bug #310770. + +* MultiTestResult methods now pass back return values from the results it + forwards to. + +0.9.5 +~~~~~ + +This release fixes some obscure traceback formatting issues that probably +weren't affecting you but were certainly breaking our own test suite. + +Changes +------- + +* Jamu Kakar has updated classes in testtools.matchers and testtools.runtest + to be new-style classes, fixing bug #611273. + +Improvements +------------ + +* Martin[gz] fixed traceback handling to handle cases where extract_tb returns + a source line of None. Fixes bug #611307. + +* Martin[gz] fixed an unicode issue that was causing the tests to fail, + closing bug #604187. + +* testtools now handles string exceptions (although why would you want to use + them?) and formats their tracebacks correctly. Thanks to Martin[gz] for + fixing bug #592262. + +0.9.4 +~~~~~ + +This release overhauls the traceback formatting layer to deal with Python 2 +line numbers and traceback objects often being local user encoded strings +rather than unicode objects. Test discovery has also been added and Python 3.1 +is also supported. Finally, the Mismatch protocol has been extended to let +Matchers collaborate with tests in supplying detailed data about failures. + +Changes +------- + +* testtools.utils has been renamed to testtools.compat. Importing + testtools.utils will now generate a deprecation warning. + +Improvements +------------ + +* Add machinery for Python 2 to create unicode tracebacks like those used by + Python 3. This means testtools no longer throws on encountering non-ascii + filenames, source lines, or exception strings when displaying test results. + Largely contributed by Martin[gz] with some tweaks from Robert Collins. + +* James Westby has supplied test discovery support using the Python 2.7 + TestRunner in testtools.run. This requires the 'discover' module. This + closes bug #250764. + +* Python 3.1 is now supported, thanks to Martin[gz] for a partial patch. + This fixes bug #592375. + +* TestCase.addCleanup has had its docstring corrected about when cleanups run. + +* TestCase.skip is now deprecated in favour of TestCase.skipTest, which is the + Python2.7 spelling for skip. This closes bug #560436. + +* Tests work on IronPython patch from Martin[gz] applied. + +* Thanks to a patch from James Westby testtools.matchers.Mismatch can now + supply a get_details method, which assertThat will query to provide + additional attachments. This can be used to provide additional detail + about the mismatch that doesn't suite being included in describe(). For + instance, if the match process was complex, a log of the process could be + included, permitting debugging. + +* testtools.testresults.real._StringException will now answer __str__ if its + value is unicode by encoding with UTF8, and vice versa to answer __unicode__. + This permits subunit decoded exceptions to contain unicode and still format + correctly. + +0.9.3 +~~~~~ + +More matchers, Python 2.4 support, faster test cloning by switching to copy +rather than deepcopy and better output when exceptions occur in cleanups are +the defining characteristics of this release. + +Improvements +------------ + +* New matcher "Annotate" that adds a simple string message to another matcher, + much like the option 'message' parameter to standard library assertFoo + methods. + +* New matchers "Not" and "MatchesAll". "Not" will invert another matcher, and + "MatchesAll" that needs a successful match for all of its arguments. + +* On Python 2.4, where types.FunctionType cannot be deepcopied, testtools will + now monkeypatch copy._deepcopy_dispatch using the same trivial patch that + added such support to Python 2.5. The monkey patch is triggered by the + absence of FunctionType from the dispatch dict rather than a version check. + Bug #498030. + +* On windows the test 'test_now_datetime_now' should now work reliably. + +* TestCase.getUniqueInteger and TestCase.getUniqueString now have docstrings. + +* TestCase.getUniqueString now takes an optional prefix parameter, so you can + now use it in circumstances that forbid strings with '.'s, and such like. + +* testtools.testcase.clone_test_with_new_id now uses copy.copy, rather than + copy.deepcopy. Tests that need a deeper copy should use the copy protocol to + control how they are copied. Bug #498869. + +* The backtrace test result output tests should now pass on windows and other + systems where os.sep is not '/'. + +* When a cleanUp or tearDown exception occurs, it is now accumulated as a new + traceback in the test details, rather than as a separate call to addError / + addException. This makes testtools work better with most TestResult objects + and fixes bug #335816. + + +0.9.2 +~~~~~ + +Python 3 support, more matchers and better consistency with Python 2.7 -- +you'd think that would be enough for a point release. Well, we here on the +testtools project think that you deserve more. + +We've added a hook so that user code can be called just-in-time whenever there +is an exception, and we've also factored out the "run" logic of test cases so +that new outcomes can be added without fiddling with the actual flow of logic. + +It might sound like small potatoes, but it's changes like these that will +bring about the end of test frameworks. + + +Improvements +------------ + +* A failure in setUp and tearDown now report as failures not as errors. + +* Cleanups now run after tearDown to be consistent with Python 2.7's cleanup + feature. + +* ExtendedToOriginalDecorator now passes unrecognised attributes through + to the decorated result object, permitting other extensions to the + TestCase -> TestResult protocol to work. + +* It is now possible to trigger code just-in-time after an exception causes + a test outcome such as failure or skip. See the testtools MANUAL or + ``pydoc testtools.TestCase.addOnException``. (bug #469092) + +* New matcher Equals which performs a simple equality test. + +* New matcher MatchesAny which looks for a match of any of its arguments. + +* TestCase no longer breaks if a TestSkipped exception is raised with no + parameters. + +* TestCase.run now clones test cases before they are run and runs the clone. + This reduces memory footprint in large test runs - state accumulated on + test objects during their setup and execution gets freed when test case + has finished running unless the TestResult object keeps a reference. + NOTE: As test cloning uses deepcopy, this can potentially interfere if + a test suite has shared state (such as the testscenarios or testresources + projects use). Use the __deepcopy__ hook to control the copying of such + objects so that the shared references stay shared. + +* Testtools now accepts contributions without copyright assignment under some + circumstances. See HACKING for details. + +* Testtools now provides a convenient way to run a test suite using the + testtools result object: python -m testtools.run testspec [testspec...]. + +* Testtools now works on Python 3, thanks to Benjamin Peterson. + +* Test execution now uses a separate class, testtools.RunTest to run single + tests. This can be customised and extended in a more consistent fashion than + the previous run method idiom. See pydoc for more information. + +* The test doubles that testtools itself uses are now available as part of + the testtools API in testtols.testresult.doubles. + +* TracebackContent now sets utf8 as the charset encoding, rather than not + setting one and encoding with the default encoder. + +* With python2.7 testtools.TestSkipped will be the unittest.case.SkipTest + exception class making skips compatible with code that manually raises the + standard library exception. (bug #490109) + +Changes +------- + +* TestCase.getUniqueInteger is now implemented using itertools.count. Thanks + to Benjamin Peterson for the patch. (bug #490111) + + +0.9.1 +~~~~~ + +The new matcher API introduced in 0.9.0 had a small flaw where the matchee +would be evaluated twice to get a description of the mismatch. This could lead +to bugs if the act of matching caused side effects to occur in the matchee. +Since having such side effects isn't desirable, we have changed the API now +before it has become widespread. + +Changes +------- + +* Matcher API changed to avoid evaluating matchee twice. Please consult + the API documentation. + +* TestCase.getUniqueString now uses the test id, not the test method name, + which works nicer with parameterised tests. + +Improvements +------------ + +* Python2.4 is now supported again. + + +0.9.0 +~~~~~ + +This release of testtools is perhaps the most interesting and exciting one +it's ever had. We've continued in bringing together the best practices of unit +testing from across a raft of different Python projects, but we've also +extended our mission to incorporating unit testing concepts from other +languages and from our own research, led by Robert Collins. + +We now support skipping and expected failures. We'll make sure that you +up-call setUp and tearDown, avoiding unexpected testing weirdnesses. We're +now compatible with Python 2.5, 2.6 and 2.7 unittest library. + +All in all, if you are serious about unit testing and want to get the best +thinking from the whole Python community, you should get this release. + +Improvements +------------ + +* A new TestResult API has been added for attaching details to test outcomes. + This API is currently experimental, but is being prepared with the intent + of becoming an upstream Python API. For more details see pydoc + testtools.TestResult and the TestCase addDetail / getDetails methods. + +* assertThat has been added to TestCase. This new assertion supports + a hamcrest-inspired matching protocol. See pydoc testtools.Matcher for + details about writing matchers, and testtools.matchers for the included + matchers. See http://code.google.com/p/hamcrest/. + +* Compatible with Python 2.6 and Python 2.7 + +* Failing to upcall in setUp or tearDown will now cause a test failure. + While the base methods do nothing, failing to upcall is usually a problem + in deeper hierarchies, and checking that the root method is called is a + simple way to catch this common bug. + +* New TestResult decorator ExtendedToOriginalDecorator which handles + downgrading extended API calls like addSkip to older result objects that + do not support them. This is used internally to make testtools simpler but + can also be used to simplify other code built on or for use with testtools. + +* New TextTestResult supporting the extended APIs that testtools provides. + +* Nose will no longer find 'runTest' tests in classes derived from + testtools.testcase.TestCase (bug #312257). + +* Supports the Python 2.7/3.1 addUnexpectedSuccess and addExpectedFailure + TestResult methods, with a support function 'knownFailure' to let tests + trigger these outcomes. + +* When using the skip feature with TestResult objects that do not support it + a test success will now be reported. Previously an error was reported but + production experience has shown that this is too disruptive for projects that + are using skips: they cannot get a clean run on down-level result objects. + + +.. _testtools: http://pypi.python.org/pypi/testtools diff --git a/test/3rdparty/testtools-0.9.12/PKG-INFO b/test/3rdparty/testtools-0.9.12/PKG-INFO new file mode 100644 index 00000000000..7dbace779b0 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/PKG-INFO @@ -0,0 +1,107 @@ +Metadata-Version: 1.0 +Name: testtools +Version: 0.9.12 +Summary: Extensions to the Python standard library unit testing framework +Home-page: https://launchpad.net/testtools +Author: Jonathan M. Lange +Author-email: jml+testtools@mumak.net +License: UNKNOWN +Description: ====================================== + testtools: tasteful testing for Python + ====================================== + + testtools is a set of extensions to the Python standard library's unit testing + framework. These extensions have been derived from many years of experience + with unit testing in Python and come from many different sources. testtools + also ports recent unittest changes all the way back to Python 2.4. + + What better way to start than with a contrived code snippet?:: + + from testtools import TestCase + from testtools.content import Content + from testtools.content_type import UTF8_TEXT + from testtools.matchers import Equals + + from myproject import SillySquareServer + + class TestSillySquareServer(TestCase): + + def setUp(self): + super(TestSillySquare, self).setUp() + self.server = self.useFixture(SillySquareServer()) + self.addCleanup(self.attach_log_file) + + def attach_log_file(self): + self.addDetail( + 'log-file', + Content(UTF8_TEXT + lambda: open(self.server.logfile, 'r').readlines())) + + def test_server_is_cool(self): + self.assertThat(self.server.temperature, Equals("cool")) + + def test_square(self): + self.assertThat(self.server.silly_square_of(7), Equals(49)) + + + Why use testtools? + ================== + + Better assertion methods + ------------------------ + + The standard assertion methods that come with unittest aren't as helpful as + they could be, and there aren't quite enough of them. testtools adds + ``assertIn``, ``assertIs``, ``assertIsInstance`` and their negatives. + + + Matchers: better than assertion methods + --------------------------------------- + + Of course, in any serious project you want to be able to have assertions that + are specific to that project and the particular problem that it is addressing. + Rather than forcing you to define your own assertion methods and maintain your + own inheritance hierarchy of ``TestCase`` classes, testtools lets you write + your own "matchers", custom predicates that can be plugged into a unit test:: + + def test_response_has_bold(self): + # The response has bold text. + response = self.server.getResponse() + self.assertThat(response, HTMLContains(Tag('bold', 'b'))) + + + More debugging info, when you need it + -------------------------------------- + + testtools makes it easy to add arbitrary data to your test result. If you + want to know what's in a log file when a test fails, or what the load was on + the computer when a test started, or what files were open, you can add that + information with ``TestCase.addDetail``, and it will appear in the test + results if that test fails. + + + Extend unittest, but stay compatible and re-usable + -------------------------------------------------- + + testtools goes to great lengths to allow serious test authors and test + *framework* authors to do whatever they like with their tests and their + extensions while staying compatible with the standard library's unittest. + + testtools has completely parametrized how exceptions raised in tests are + mapped to ``TestResult`` methods and how tests are actually executed (ever + wanted ``tearDown`` to be called regardless of whether ``setUp`` succeeds?) + + It also provides many simple but handy utilities, like the ability to clone a + test, a ``MultiTestResult`` object that lets many result objects get the + results from one test suite, adapters to bring legacy ``TestResult`` objects + into our new golden age. + + + Cross-Python compatibility + -------------------------- + + testtools gives you the very latest in unit testing technology in a way that + will work with Python 2.4, 2.5, 2.6, 2.7 and 3.1. + +Platform: UNKNOWN +Classifier: License :: OSI Approved :: MIT License diff --git a/test/3rdparty/testtools-0.9.12/README b/test/3rdparty/testtools-0.9.12/README new file mode 100644 index 00000000000..78397de85b6 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/README @@ -0,0 +1,86 @@ +========= +testtools +========= + +testtools is a set of extensions to the Python standard library's unit testing +framework. + +These extensions have been derived from years of experience with unit testing +in Python and come from many different sources. + + +Documentation +------------- + +If you would like to learn more about testtools, consult our documentation in +the 'doc/' directory. You might like to start at 'doc/overview.rst' or +'doc/for-test-authors.rst'. + + +Licensing +--------- + +This project is distributed under the MIT license and copyright is owned by +Jonathan M. Lange and the testtools authors. See LICENSE for details. + +Some code in 'testtools/run.py' is taken from Python's unittest module, and is +copyright Steve Purcell and the Python Software Foundation, it is distributed +under the same license as Python, see LICENSE for details. + + +Required Dependencies +--------------------- + + * Python 2.4+ or 3.0+ + + +Optional Dependencies +--------------------- + +If you would like to use our undocumented, unsupported Twisted support, then +you will need Twisted. + +If you want to use ``fixtures`` then you can either install fixtures (e.g. from +https://launchpad.net/python-fixtures or http://pypi.python.org/pypi/fixtures) +or alternatively just make sure your fixture objects obey the same protocol. + + +Bug reports and patches +----------------------- + +Please report bugs using Launchpad at <https://bugs.launchpad.net/testtools>. +Patches can also be submitted via Launchpad, or mailed to the author. You can +mail the author directly at jml@mumak.net. + +There's no mailing list for this project yet, however the testing-in-python +mailing list may be a useful resource: + + * Address: testing-in-python@lists.idyll.org + * Subscription link: http://lists.idyll.org/listinfo/testing-in-python + + +History +------- + +testtools used to be called 'pyunit3k'. The name was changed to avoid +conflating the library with the Python 3.0 release (commonly referred to as +'py3k'). + + +Thanks +------ + + * Canonical Ltd + * Bazaar + * Twisted Matrix Labs + * Robert Collins + * Andrew Bennetts + * Benjamin Peterson + * Jamu Kakar + * James Westby + * Martin [gz] + * Michael Hudson-Doyle + * Aaron Bentley + * Christian Kampka + * Gavin Panella + * Martin Pool diff --git a/test/3rdparty/testtools-0.9.12/doc/Makefile b/test/3rdparty/testtools-0.9.12/doc/Makefile new file mode 100644 index 00000000000..b5d07af57f2 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/doc/Makefile @@ -0,0 +1,89 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest + +help: + @echo "Please use \`make <target>' where <target> is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/testtools.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/testtools.qhc" + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/test/3rdparty/testtools-0.9.12/doc/_static/placeholder.txt b/test/3rdparty/testtools-0.9.12/doc/_static/placeholder.txt new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/doc/_static/placeholder.txt diff --git a/test/3rdparty/testtools-0.9.12/doc/_templates/placeholder.txt b/test/3rdparty/testtools-0.9.12/doc/_templates/placeholder.txt new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/doc/_templates/placeholder.txt diff --git a/test/3rdparty/testtools-0.9.12/doc/conf.py b/test/3rdparty/testtools-0.9.12/doc/conf.py new file mode 100644 index 00000000000..de5fdd4224e --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/doc/conf.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- +# +# testtools documentation build configuration file, created by +# sphinx-quickstart on Sun Nov 28 13:45:40 2010. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.append(os.path.abspath('.')) + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'testtools' +copyright = u'2010, The testtools authors' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = 'VERSION' +# The full version, including alpha/beta/rc tags. +release = 'VERSION' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_use_modindex = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'testtoolsdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'testtools.tex', u'testtools Documentation', + u'The testtools authors', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True diff --git a/test/3rdparty/testtools-0.9.12/doc/for-framework-folk.rst b/test/3rdparty/testtools-0.9.12/doc/for-framework-folk.rst new file mode 100644 index 00000000000..a4b20f64cac --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/doc/for-framework-folk.rst @@ -0,0 +1,219 @@ +============================ +testtools for framework folk +============================ + +Introduction +============ + +In addition to having many features :doc:`for test authors +<for-test-authors>`, testtools also has many bits and pieces that are useful +for folk who write testing frameworks. + +If you are the author of a test runner, are working on a very large +unit-tested project, are trying to get one testing framework to play nicely +with another or are hacking away at getting your test suite to run in parallel +over a heterogenous cluster of machines, this guide is for you. + +This manual is a summary. You can get details by consulting the `testtools +API docs`_. + + +Extensions to TestCase +====================== + +Custom exception handling +------------------------- + +testtools provides a way to control how test exceptions are handled. To do +this, add a new exception to ``self.exception_handlers`` on a +``testtools.TestCase``. For example:: + + >>> self.exception_handlers.insert(-1, (ExceptionClass, handler)). + +Having done this, if any of ``setUp``, ``tearDown``, or the test method raise +``ExceptionClass``, ``handler`` will be called with the test case, test result +and the raised exception. + +Use this if you want to add a new kind of test result, that is, if you think +that ``addError``, ``addFailure`` and so forth are not enough for your needs. + + +Controlling test execution +-------------------------- + +If you want to control more than just how exceptions are raised, you can +provide a custom ``RunTest`` to a ``TestCase``. The ``RunTest`` object can +change everything about how the test executes. + +To work with ``testtools.TestCase``, a ``RunTest`` must have a factory that +takes a test and an optional list of exception handlers. Instances returned +by the factory must have a ``run()`` method that takes an optional ``TestResult`` +object. + +The default is ``testtools.runtest.RunTest``, which calls ``setUp``, the test +method, ``tearDown`` and clean ups (see :ref:`addCleanup`) in the normal, vanilla +way that Python's standard unittest_ does. + +To specify a ``RunTest`` for all the tests in a ``TestCase`` class, do something +like this:: + + class SomeTests(TestCase): + run_tests_with = CustomRunTestFactory + +To specify a ``RunTest`` for a specific test in a ``TestCase`` class, do:: + + class SomeTests(TestCase): + @run_test_with(CustomRunTestFactory, extra_arg=42, foo='whatever') + def test_something(self): + pass + +In addition, either of these can be overridden by passing a factory in to the +``TestCase`` constructor with the optional ``runTest`` argument. + + +Test renaming +------------- + +``testtools.clone_test_with_new_id`` is a function to copy a test case +instance to one with a new name. This is helpful for implementing test +parameterization. + + +Test placeholders +================= + +Sometimes, it's useful to be able to add things to a test suite that are not +actually tests. For example, you might wish to represents import failures +that occur during test discovery as tests, so that your test result object +doesn't have to do special work to handle them nicely. + +testtools provides two such objects, called "placeholders": ``PlaceHolder`` +and ``ErrorHolder``. ``PlaceHolder`` takes a test id and an optional +description. When it's run, it succeeds. ``ErrorHolder`` takes a test id, +and error and an optional short description. When it's run, it reports that +error. + +These placeholders are best used to log events that occur outside the test +suite proper, but are still very relevant to its results. + +e.g.:: + + >>> suite = TestSuite() + >>> suite.add(PlaceHolder('I record an event')) + >>> suite.run(TextTestResult(verbose=True)) + I record an event [OK] + + +Extensions to TestResult +======================== + +TestResult.addSkip +------------------ + +This method is called on result objects when a test skips. The +``testtools.TestResult`` class records skips in its ``skip_reasons`` instance +dict. The can be reported on in much the same way as succesful tests. + + +TestResult.time +--------------- + +This method controls the time used by a ``TestResult``, permitting accurate +timing of test results gathered on different machines or in different threads. +See pydoc testtools.TestResult.time for more details. + + +ThreadsafeForwardingResult +-------------------------- + +A ``TestResult`` which forwards activity to another test result, but synchronises +on a semaphore to ensure that all the activity for a single test arrives in a +batch. This allows simple TestResults which do not expect concurrent test +reporting to be fed the activity from multiple test threads, or processes. + +Note that when you provide multiple errors for a single test, the target sees +each error as a distinct complete test. + + +MultiTestResult +--------------- + +A test result that dispatches its events to many test results. Use this +to combine multiple different test result objects into one test result object +that can be passed to ``TestCase.run()`` or similar. For example:: + + a = TestResult() + b = TestResult() + combined = MultiTestResult(a, b) + combined.startTestRun() # Calls a.startTestRun() and b.startTestRun() + +Each of the methods on ``MultiTestResult`` will return a tuple of whatever the +component test results return. + + +TextTestResult +-------------- + +A ``TestResult`` that provides a text UI very similar to the Python standard +library UI. Key differences are that its supports the extended outcomes and +details API, and is completely encapsulated into the result object, permitting +it to be used without a 'TestRunner' object. Not all the Python 2.7 outcomes +are displayed (yet). It is also a 'quiet' result with no dots or verbose mode. +These limitations will be corrected soon. + + +ExtendedToOriginalDecorator +--------------------------- + +Adapts legacy ``TestResult`` objects, such as those found in older Pythons, to +meet the testtools ``TestResult`` API. + + +Test Doubles +------------ + +In testtools.testresult.doubles there are three test doubles that testtools +uses for its own testing: ``Python26TestResult``, ``Python27TestResult``, +``ExtendedTestResult``. These TestResult objects implement a single variation of +the TestResult API each, and log activity to a list ``self._events``. These are +made available for the convenience of people writing their own extensions. + + +startTestRun and stopTestRun +---------------------------- + +Python 2.7 added hooks ``startTestRun`` and ``stopTestRun`` which are called +before and after the entire test run. 'stopTestRun' is particularly useful for +test results that wish to produce summary output. + +``testtools.TestResult`` provides default ``startTestRun`` and ``stopTestRun`` +methods, and he default testtools runner will call these methods +appropriately. + +The ``startTestRun`` method will reset any errors, failures and so forth on +the result, making the result object look as if no tests have been run. + + +Extensions to TestSuite +======================= + +ConcurrentTestSuite +------------------- + +A TestSuite for parallel testing. This is used in conjuction with a helper that +runs a single suite in some parallel fashion (for instance, forking, handing +off to a subprocess, to a compute cloud, or simple threads). +ConcurrentTestSuite uses the helper to get a number of separate runnable +objects with a run(result), runs them all in threads using the +ThreadsafeForwardingResult to coalesce their activity. + +FixtureSuite +------------ + +A test suite that sets up a fixture_ before running any tests, and then tears +it down after all of the tests are run. The fixture is *not* made available to +any of the tests. + +.. _`testtools API docs`: http://mumak.net/testtools/apidocs/ +.. _unittest: http://docs.python.org/library/unittest.html +.. _fixture: http://pypi.python.org/pypi/fixtures diff --git a/test/3rdparty/testtools-0.9.12/doc/for-test-authors.rst b/test/3rdparty/testtools-0.9.12/doc/for-test-authors.rst new file mode 100644 index 00000000000..04c4be6b0db --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/doc/for-test-authors.rst @@ -0,0 +1,1204 @@ +========================== +testtools for test authors +========================== + +If you are writing tests for a Python project and you (rather wisely) want to +use testtools to do so, this is the manual for you. + +We assume that you already know Python and that you know something about +automated testing already. + +If you are a test author of an unusually large or unusually unusual test +suite, you might be interested in :doc:`for-framework-folk`. + +You might also be interested in the `testtools API docs`_. + + +Introduction +============ + +testtools is a set of extensions to Python's standard unittest module. +Writing tests with testtools is very much like writing tests with standard +Python, or with Twisted's "trial_", or nose_, except a little bit easier and +more enjoyable. + +Below, we'll try to give some examples of how to use testtools in its most +basic way, as well as a sort of feature-by-feature breakdown of the cool bits +that you could easily miss. + + +The basics +========== + +Here's what a basic testtools unit tests look like:: + + from testtools import TestCase + from myproject import silly + + class TestSillySquare(TestCase): + """Tests for silly square function.""" + + def test_square(self): + # 'square' takes a number and multiplies it by itself. + result = silly.square(7) + self.assertEqual(result, 49) + + def test_square_bad_input(self): + # 'square' raises a TypeError if it's given bad input, say a + # string. + self.assertRaises(TypeError, silly.square, "orange") + + +Here you have a class that inherits from ``testtools.TestCase`` and bundles +together a bunch of related tests. The tests themselves are methods on that +class that begin with ``test_``. + +Running your tests +------------------ + +You can run these tests in many ways. testtools provides a very basic +mechanism for doing so:: + + $ python -m testtools.run exampletest + Tests running... + Ran 2 tests in 0.000s + + OK + +where 'exampletest' is a module that contains unit tests. By default, +``testtools.run`` will *not* recursively search the module or package for unit +tests. To do this, you will need to either have the discover_ module +installed or have Python 2.7 or later, and then run:: + + $ python -m testtools.run discover packagecontainingtests + +For more information see the Python 2.7 unittest documentation, or:: + + python -m testtools.run --help + +As your testing needs grow and evolve, you will probably want to use a more +sophisticated test runner. There are many of these for Python, and almost all +of them will happily run testtools tests. In particular: + +* testrepository_ +* Trial_ +* nose_ +* unittest2_ +* `zope.testrunner`_ (aka zope.testing) + +From now on, we'll assume that you know how to run your tests. + +Running test with Distutils +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you are using Distutils_ to build your Python project, you can use the testtools +Distutils_ command to integrate testtools into your Distutils_ workflow:: + + from distutils.core import setup + from testtools import TestCommand + setup(name='foo', + version='1.0', + py_modules=['foo'], + cmdclass={'test': TestCommand} + ) + +You can then run:: + + $ python setup.py test -m exampletest + Tests running... + Ran 2 tests in 0.000s + + OK + +For more information about the capabilities of the `TestCommand` command see:: + + $ python setup.py test --help + +You can use the `setup configuration`_ to specify the default behavior of the +`TestCommand` command. + +Assertions +========== + +The core of automated testing is making assertions about the way things are, +and getting a nice, helpful, informative error message when things are not as +they ought to be. + +All of the assertions that you can find in Python standard unittest_ can be +found in testtools (remember, testtools extends unittest). testtools changes +the behaviour of some of those assertions slightly and adds some new +assertions that you will almost certainly find useful. + + +Improved assertRaises +--------------------- + +``TestCase.assertRaises`` returns the caught exception. This is useful for +asserting more things about the exception than just the type:: + + def test_square_bad_input(self): + # 'square' raises a TypeError if it's given bad input, say a + # string. + e = self.assertRaises(TypeError, silly.square, "orange") + self.assertEqual("orange", e.bad_value) + self.assertEqual("Cannot square 'orange', not a number.", str(e)) + +Note that this is incompatible with the ``assertRaises`` in unittest2 and +Python2.7. + + +ExpectedException +----------------- + +If you are using a version of Python that supports the ``with`` context +manager syntax, you might prefer to use that syntax to ensure that code raises +particular errors. ``ExpectedException`` does just that. For example:: + + def test_square_root_bad_input_2(self): + # 'square' raises a TypeError if it's given bad input. + with ExpectedException(TypeError, "Cannot square.*"): + silly.square('orange') + +The first argument to ``ExpectedException`` is the type of exception you +expect to see raised. The second argument is optional, and can be either a +regular expression or a matcher. If it is a regular expression, the ``str()`` +of the raised exception must match the regular expression. If it is a matcher, +then the raised exception object must match it. + + +assertIn, assertNotIn +--------------------- + +These two assertions check whether a value is in a sequence and whether a +value is not in a sequence. They are "assert" versions of the ``in`` and +``not in`` operators. For example:: + + def test_assert_in_example(self): + self.assertIn('a', 'cat') + self.assertNotIn('o', 'cat') + self.assertIn(5, list_of_primes_under_ten) + self.assertNotIn(12, list_of_primes_under_ten) + + +assertIs, assertIsNot +--------------------- + +These two assertions check whether values are identical to one another. This +is sometimes useful when you want to test something more strict than mere +equality. For example:: + + def test_assert_is_example(self): + foo = [None] + foo_alias = foo + bar = [None] + self.assertIs(foo, foo_alias) + self.assertIsNot(foo, bar) + self.assertEqual(foo, bar) # They are equal, but not identical + + +assertIsInstance +---------------- + +As much as we love duck-typing and polymorphism, sometimes you need to check +whether or not a value is of a given type. This method does that. For +example:: + + def test_assert_is_instance_example(self): + now = datetime.now() + self.assertIsInstance(now, datetime) + +Note that there is no ``assertIsNotInstance`` in testtools currently. + + +expectFailure +------------- + +Sometimes it's useful to write tests that fail. For example, you might want +to turn a bug report into a unit test, but you don't know how to fix the bug +yet. Or perhaps you want to document a known, temporary deficiency in a +dependency. + +testtools gives you the ``TestCase.expectFailure`` to help with this. You use +it to say that you expect this assertion to fail. When the test runs and the +assertion fails, testtools will report it as an "expected failure". + +Here's an example:: + + def test_expect_failure_example(self): + self.expectFailure( + "cats should be dogs", self.assertEqual, 'cats', 'dogs') + +As long as 'cats' is not equal to 'dogs', the test will be reported as an +expected failure. + +If ever by some miracle 'cats' becomes 'dogs', then testtools will report an +"unexpected success". Unlike standard unittest, testtools treats this as +something that fails the test suite, like an error or a failure. + + +Matchers +======== + +The built-in assertion methods are very useful, they are the bread and butter +of writing tests. However, soon enough you will probably want to write your +own assertions. Perhaps there are domain specific things that you want to +check (e.g. assert that two widgets are aligned parallel to the flux grid), or +perhaps you want to check something that could almost but not quite be found +in some other standard library (e.g. assert that two paths point to the same +file). + +When you are in such situations, you could either make a base class for your +project that inherits from ``testtools.TestCase`` and make sure that all of +your tests derive from that, *or* you could use the testtools ``Matcher`` +system. + + +Using Matchers +-------------- + +Here's a really basic example using stock matchers found in testtools:: + + import testtools + from testtools.matchers import Equals + + class TestSquare(TestCase): + def test_square(self): + result = square(7) + self.assertThat(result, Equals(49)) + +The line ``self.assertThat(result, Equals(49))`` is equivalent to +``self.assertEqual(result, 49)`` and means "assert that ``result`` equals 49". +The difference is that ``assertThat`` is a more general method that takes some +kind of observed value (in this case, ``result``) and any matcher object +(here, ``Equals(49)``). + +The matcher object could be absolutely anything that implements the Matcher +protocol. This means that you can make more complex matchers by combining +existing ones:: + + def test_square_silly(self): + result = square(7) + self.assertThat(result, Not(Equals(50))) + +Which is roughly equivalent to:: + + def test_square_silly(self): + result = square(7) + self.assertNotEqual(result, 50) + + +Stock matchers +-------------- + +testtools comes with many matchers built in. They can all be found in and +imported from the ``testtools.matchers`` module. + +Equals +~~~~~~ + +Matches if two items are equal. For example:: + + def test_equals_example(self): + self.assertThat([42], Equals([42])) + + +Is +~~~ + +Matches if two items are identical. For example:: + + def test_is_example(self): + foo = object() + self.assertThat(foo, Is(foo)) + + +IsInstance +~~~~~~~~~~ + +Adapts isinstance() to use as a matcher. For example:: + + def test_isinstance_example(self): + class MyClass:pass + self.assertThat(MyClass(), IsInstance(MyClass)) + self.assertThat(MyClass(), IsInstance(MyClass, str)) + + +The raises helper +~~~~~~~~~~~~~~~~~ + +Matches if a callable raises a particular type of exception. For example:: + + def test_raises_example(self): + self.assertThat(lambda: 1/0, raises(ZeroDivisionError)) + +This is actually a convenience function that combines two other matchers: +Raises_ and MatchesException_. + + +DocTestMatches +~~~~~~~~~~~~~~ + +Matches a string as if it were the output of a doctest_ example. Very useful +for making assertions about large chunks of text. For example:: + + import doctest + + def test_doctest_example(self): + output = "Colorless green ideas" + self.assertThat( + output, + DocTestMatches("Colorless ... ideas", doctest.ELLIPSIS)) + +We highly recommend using the following flags:: + + doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF + + +GreaterThan +~~~~~~~~~~~ + +Matches if the given thing is greater than the thing in the matcher. For +example:: + + def test_greater_than_example(self): + self.assertThat(3, GreaterThan(2)) + + +LessThan +~~~~~~~~ + +Matches if the given thing is less than the thing in the matcher. For +example:: + + def test_less_than_example(self): + self.assertThat(2, LessThan(3)) + + +StartsWith, EndsWith +~~~~~~~~~~~~~~~~~~~~ + +These matchers check to see if a string starts with or ends with a particular +substring. For example:: + + def test_starts_and_ends_with_example(self): + self.assertThat('underground', StartsWith('und')) + self.assertThat('underground', EndsWith('und')) + + +Contains +~~~~~~~~ + +This matcher checks to see if the given thing contains the thing in the +matcher. For example:: + + def test_contains_example(self): + self.assertThat('abc', Contains('b')) + + +MatchesException +~~~~~~~~~~~~~~~~ + +Matches an exc_info tuple if the exception is of the correct type. For +example:: + + def test_matches_exception_example(self): + try: + raise RuntimeError('foo') + except RuntimeError: + exc_info = sys.exc_info() + self.assertThat(exc_info, MatchesException(RuntimeError)) + self.assertThat(exc_info, MatchesException(RuntimeError('bar')) + +Most of the time, you will want to uses `The raises helper`_ instead. + + +NotEquals +~~~~~~~~~ + +Matches if something is not equal to something else. Note that this is subtly +different to ``Not(Equals(x))``. ``NotEquals(x)`` will match if ``y != x``, +``Not(Equals(x))`` will match if ``not y == x``. + +You only need to worry about this distinction if you are testing code that +relies on badly written overloaded equality operators. + + +KeysEqual +~~~~~~~~~ + +Matches if the keys of one dict are equal to the keys of another dict. For +example:: + + def test_keys_equal(self): + x = {'a': 1, 'b': 2} + y = {'a': 2, 'b': 3} + self.assertThat(a, KeysEqual(b)) + + +MatchesRegex +~~~~~~~~~~~~ + +Matches a string against a regular expression, which is a wonderful thing to +be able to do, if you think about it:: + + def test_matches_regex_example(self): + self.assertThat('foo', MatchesRegex('fo+')) + + +Combining matchers +------------------ + +One great thing about matchers is that you can readily combine existing +matchers to get variations on their behaviour or to quickly build more complex +assertions. + +Below are a few of the combining matchers that come with testtools. + + +Not +~~~ + +Negates another matcher. For example:: + + def test_not_example(self): + self.assertThat([42], Not(Equals("potato"))) + self.assertThat([42], Not(Is([42]))) + +If you find yourself using ``Not`` frequently, you may wish to create a custom +matcher for it. For example:: + + IsNot = lambda x: Not(Is(x)) + + def test_not_example_2(self): + self.assertThat([42], IsNot([42])) + + +Annotate +~~~~~~~~ + +Used to add custom notes to a matcher. For example:: + + def test_annotate_example(self): + result = 43 + self.assertThat( + result, Annotate("Not the answer to the Question!", Equals(42)) + +Since the annotation is only ever displayed when there is a mismatch +(e.g. when ``result`` does not equal 42), it's a good idea to phrase the note +negatively, so that it describes what a mismatch actually means. + +As with Not_, you may wish to create a custom matcher that describes a +common operation. For example:: + + PoliticallyEquals = lambda x: Annotate("Death to the aristos!", Equals(x)) + + def test_annotate_example_2(self): + self.assertThat("orange", PoliticallyEquals("yellow")) + +You can have assertThat perform the annotation for you as a convenience:: + + def test_annotate_example_3(self): + self.assertThat("orange", Equals("yellow"), "Death to the aristos!") + + +AfterPreprocessing +~~~~~~~~~~~~~~~~~~ + +Used to make a matcher that applies a function to the matched object before +matching. This can be used to aid in creating trivial matchers as functions, for +example:: + + def test_after_preprocessing_example(self): + def HasFileContent(content): + def _read(path): + return open(path).read() + return AfterPreprocessing(_read, Equals(content)) + self.assertThat('/tmp/foo.txt', PathHasFileContent("Hello world!")) + + +MatchesAll +~~~~~~~~~~ + +Combines many matchers to make a new matcher. The new matcher will only match +things that match every single one of the component matchers. + +It's much easier to understand in Python than in English:: + + def test_matches_all_example(self): + has_und_at_both_ends = MatchesAll(StartsWith("und"), EndsWith("und")) + # This will succeed. + self.assertThat("underground", has_und_at_both_ends) + # This will fail. + self.assertThat("found", has_und_at_both_ends) + # So will this. + self.assertThat("undead", has_und_at_both_ends) + +At this point some people ask themselves, "why bother doing this at all? why +not just have two separate assertions?". It's a good question. + +The first reason is that when a ``MatchesAll`` gets a mismatch, the error will +include information about all of the bits that mismatched. When you have two +separate assertions, as below:: + + def test_two_separate_assertions(self): + self.assertThat("foo", StartsWith("und")) + self.assertThat("foo", EndsWith("und")) + +Then you get absolutely no information from the second assertion if the first +assertion fails. Tests are largely there to help you debug code, so having +more information in error messages is a big help. + +The second reason is that it is sometimes useful to give a name to a set of +matchers. ``has_und_at_both_ends`` is a bit contrived, of course, but it is +clear. + + +MatchesAny +~~~~~~~~~~ + +Like MatchesAll_, ``MatchesAny`` combines many matchers to make a new +matcher. The difference is that the new matchers will match a thing if it +matches *any* of the component matchers. + +For example:: + + def test_matches_any_example(self): + self.assertThat(42, MatchesAny(Equals(5), Not(Equals(6)))) + + +AllMatch +~~~~~~~~ + +Matches many values against a single matcher. Can be used to make sure that +many things all meet the same condition:: + + def test_all_match_example(self): + self.assertThat([2, 3, 5, 7], AllMatch(LessThan(10))) + +If the match fails, then all of the values that fail to match will be included +in the error message. + +In some ways, this is the converse of MatchesAll_. + + +MatchesListwise +~~~~~~~~~~~~~~~ + +Where ``MatchesAny`` and ``MatchesAll`` combine many matchers to match a +single value, ``MatchesListwise`` combines many matches to match many values. + +For example:: + + def test_matches_listwise_example(self): + self.assertThat( + [1, 2, 3], MatchesListwise(map(Equals, [1, 2, 3]))) + +This is useful for writing custom, domain-specific matchers. + + +MatchesSetwise +~~~~~~~~~~~~~~ + +Combines many matchers to match many values, without regard to their order. + +Here's an example:: + + def test_matches_setwise_example(self): + self.assertThat( + [1, 2, 3], MatchesSetwise(Equals(2), Equals(3), Equals(1))) + +Much like ``MatchesListwise``, best used for writing custom, domain-specific +matchers. + + +MatchesStructure +~~~~~~~~~~~~~~~~ + +Creates a matcher that matches certain attributes of an object against a +pre-defined set of matchers. + +It's much easier to understand in Python than in English:: + + def test_matches_structure_example(self): + foo = Foo() + foo.a = 1 + foo.b = 2 + matcher = MatchesStructure(a=Equals(1), b=Equals(2)) + self.assertThat(foo, matcher) + +Since all of the matchers used were ``Equals``, we could also write this using +the ``byEquality`` helper:: + + def test_matches_structure_example(self): + foo = Foo() + foo.a = 1 + foo.b = 2 + matcher = MatchesStructure.byEquality(a=1, b=2) + self.assertThat(foo, matcher) + +``MatchesStructure.fromExample`` takes an object and a list of attributes and +creates a ``MatchesStructure`` matcher where each attribute of the matched +object must equal each attribute of the example object. For example:: + + matcher = MatchesStructure.fromExample(foo, 'a', 'b') + +is exactly equivalent to ``matcher`` in the previous example. + + +Raises +~~~~~~ + +Takes whatever the callable raises as an exc_info tuple and matches it against +whatever matcher it was given. For example, if you want to assert that a +callable raises an exception of a given type:: + + def test_raises_example(self): + self.assertThat( + lambda: 1/0, Raises(MatchesException(ZeroDivisionError))) + +Although note that this could also be written as:: + + def test_raises_example_convenient(self): + self.assertThat(lambda: 1/0, raises(ZeroDivisionError)) + +See also MatchesException_ and `the raises helper`_ + + +Writing your own matchers +------------------------- + +Combining matchers is fun and can get you a very long way indeed, but +sometimes you will have to write your own. Here's how. + +You need to make two closely-linked objects: a ``Matcher`` and a +``Mismatch``. The ``Matcher`` knows how to actually make the comparison, and +the ``Mismatch`` knows how to describe a failure to match. + +Here's an example matcher:: + + class IsDivisibleBy(object): + """Match if a number is divisible by another number.""" + def __init__(self, divider): + self.divider = divider + def __str__(self): + return 'IsDivisibleBy(%s)' % (self.divider,) + def match(self, actual): + remainder = actual % self.divider + if remainder != 0: + return IsDivisibleByMismatch(actual, self.divider, remainder) + else: + return None + +The matcher has a constructor that takes parameters that describe what you +actually *expect*, in this case a number that other numbers ought to be +divisible by. It has a ``__str__`` method, the result of which is displayed +on failure by ``assertThat`` and a ``match`` method that does the actual +matching. + +``match`` takes something to match against, here ``actual``, and decides +whether or not it matches. If it does match, then ``match`` must return +``None``. If it does *not* match, then ``match`` must return a ``Mismatch`` +object. ``assertThat`` will call ``match`` and then fail the test if it +returns a non-None value. For example:: + + def test_is_divisible_by_example(self): + # This succeeds, since IsDivisibleBy(5).match(10) returns None. + self.assertThat(10, IsDivisbleBy(5)) + # This fails, since IsDivisibleBy(7).match(10) returns a mismatch. + self.assertThat(10, IsDivisbleBy(7)) + +The mismatch is responsible for what sort of error message the failing test +generates. Here's an example mismatch:: + + class IsDivisibleByMismatch(object): + def __init__(self, number, divider, remainder): + self.number = number + self.divider = divider + self.remainder = remainder + + def describe(self): + return "%r is not divisible by %r, %r remains" % ( + self.number, self.divider, self.remainder) + + def get_details(self): + return {} + +The mismatch takes information about the mismatch, and provides a ``describe`` +method that assembles all of that into a nice error message for end users. +You can use the ``get_details`` method to provide extra, arbitrary data with +the mismatch (e.g. the contents of a log file). Most of the time it's fine to +just return an empty dict. You can read more about Details_ elsewhere in this +document. + +Sometimes you don't need to create a custom mismatch class. In particular, if +you don't care *when* the description is calculated, then you can just do that +in the Matcher itself like this:: + + def match(self, actual): + remainder = actual % self.divider + if remainder != 0: + return Mismatch( + "%r is not divisible by %r, %r remains" % ( + actual, self.divider, remainder)) + else: + return None + +When writing a ``describe`` method or constructing a ``Mismatch`` object the +code should ensure it only emits printable unicode. As this output must be +combined with other text and forwarded for presentation, letting through +non-ascii bytes of ambiguous encoding or control characters could throw an +exception or mangle the display. In most cases simply avoiding the ``%s`` +format specifier and using ``%r`` instead will be enough. For examples of +more complex formatting see the ``testtools.matchers`` implementatons. + + +Details +======= + +As we may have mentioned once or twice already, one of the great benefits of +automated tests is that they help find, isolate and debug errors in your +system. + +Frequently however, the information provided by a mere assertion failure is +not enough. It's often useful to have other information: the contents of log +files; what queries were run; benchmark timing information; what state certain +subsystem components are in and so forth. + +testtools calls all of these things "details" and provides a single, powerful +mechanism for including this information in your test run. + +Here's an example of how to add them:: + + from testtools import TestCase + from testtools.content import text_content + + class TestSomething(TestCase): + + def test_thingy(self): + self.addDetail('arbitrary-color-name', text_content("blue")) + 1 / 0 # Gratuitous error! + +A detail an arbitrary piece of content given a name that's unique within the +test. Here the name is ``arbitrary-color-name`` and the content is +``text_content("blue")``. The name can be any text string, and the content +can be any ``testtools.content.Content`` object. + +When the test runs, testtools will show you something like this:: + + ====================================================================== + ERROR: exampletest.TestSomething.test_thingy + ---------------------------------------------------------------------- + arbitrary-color-name: {{{blue}}} + + Traceback (most recent call last): + File "exampletest.py", line 8, in test_thingy + 1 / 0 # Gratuitous error! + ZeroDivisionError: integer division or modulo by zero + ------------ + Ran 1 test in 0.030s + +As you can see, the detail is included as an attachment, here saying +that our arbitrary-color-name is "blue". + + +Content +------- + +For the actual content of details, testtools uses its own MIME-based Content +object. This allows you to attach any information that you could possibly +conceive of to a test, and allows testtools to use or serialize that +information. + +The basic ``testtools.content.Content`` object is constructed from a +``testtools.content.ContentType`` and a nullary callable that must return an +iterator of chunks of bytes that the content is made from. + +So, to make a Content object that is just a simple string of text, you can +do:: + + from testtools.content import Content + from testtools.content_type import ContentType + + text = Content(ContentType('text', 'plain'), lambda: ["some text"]) + +Because adding small bits of text content is very common, there's also a +convenience method:: + + text = text_content("some text") + +To make content out of an image stored on disk, you could do something like:: + + image = Content(ContentType('image', 'png'), lambda: open('foo.png').read()) + +Or you could use the convenience function:: + + image = content_from_file('foo.png', ContentType('image', 'png')) + +The ``lambda`` helps make sure that the file is opened and the actual bytes +read only when they are needed – by default, when the test is finished. This +means that tests can construct and add Content objects freely without worrying +too much about how they affect run time. + + +A realistic example +------------------- + +A very common use of details is to add a log file to failing tests. Say your +project has a server represented by a class ``SomeServer`` that you can start +up and shut down in tests, but runs in another process. You want to test +interaction with that server, and whenever the interaction fails, you want to +see the client-side error *and* the logs from the server-side. Here's how you +might do it:: + + from testtools import TestCase + from testtools.content import attach_file, Content + from testtools.content_type import UTF8_TEXT + + from myproject import SomeServer + + class SomeTestCase(TestCase): + + def setUp(self): + super(SomeTestCase, self).setUp() + self.server = SomeServer() + self.server.start_up() + self.addCleanup(self.server.shut_down) + self.addCleanup(attach_file, self.server.logfile, self) + + def attach_log_file(self): + self.addDetail( + 'log-file', + Content(UTF8_TEXT, + lambda: open(self.server.logfile, 'r').readlines())) + + def test_a_thing(self): + self.assertEqual("cool", self.server.temperature) + +This test will attach the log file of ``SomeServer`` to each test that is +run. testtools will only display the log file for failing tests, so it's not +such a big deal. + +If the act of adding at detail is expensive, you might want to use +addOnException_ so that you only do it when a test actually raises an +exception. + + +Controlling test execution +========================== + +.. _addCleanup: + +addCleanup +---------- + +``TestCase.addCleanup`` is a robust way to arrange for a clean up function to +be called before ``tearDown``. This is a powerful and simple alternative to +putting clean up logic in a try/finally block or ``tearDown`` method. For +example:: + + def test_foo(self): + foo.lock() + self.addCleanup(foo.unlock) + ... + +This is particularly useful if you have some sort of factory in your test:: + + def make_locked_foo(self): + foo = Foo() + foo.lock() + self.addCleanup(foo.unlock) + return foo + + def test_frotz_a_foo(self): + foo = self.make_locked_foo() + foo.frotz() + self.assertEqual(foo.frotz_count, 1) + +Any extra arguments or keyword arguments passed to ``addCleanup`` are passed +to the callable at cleanup time. + +Cleanups can also report multiple errors, if appropriate by wrapping them in +a ``testtools.MultipleExceptions`` object:: + + raise MultipleExceptions(exc_info1, exc_info2) + + +Fixtures +-------- + +Tests often depend on a system being set up in a certain way, or having +certain resources available to them. Perhaps a test needs a connection to the +database or access to a running external server. + +One common way of doing this is to do:: + + class SomeTest(TestCase): + def setUp(self): + super(SomeTest, self).setUp() + self.server = Server() + self.server.setUp() + self.addCleanup(self.server.tearDown) + +testtools provides a more convenient, declarative way to do the same thing:: + + class SomeTest(TestCase): + def setUp(self): + super(SomeTest, self).setUp() + self.server = self.useFixture(Server()) + +``useFixture(fixture)`` calls ``setUp`` on the fixture, schedules a clean up +to clean it up, and schedules a clean up to attach all details_ held by the +fixture to the test case. The fixture object must meet the +``fixtures.Fixture`` protocol (version 0.3.4 or newer, see fixtures_). + +If you have anything beyond the most simple test set up, we recommend that +you put this set up into a ``Fixture`` class. Once there, the fixture can be +easily re-used by other tests and can be combined with other fixtures to make +more complex resources. + + +Skipping tests +-------------- + +Many reasons exist to skip a test: a dependency might be missing; a test might +be too expensive and thus should not berun while on battery power; or perhaps +the test is testing an incomplete feature. + +``TestCase.skipTest`` is a simple way to have a test stop running and be +reported as a skipped test, rather than a success, error or failure. For +example:: + + def test_make_symlink(self): + symlink = getattr(os, 'symlink', None) + if symlink is None: + self.skipTest("No symlink support") + symlink(whatever, something_else) + +Using ``skipTest`` means that you can make decisions about what tests to run +as late as possible, and close to the actual tests. Without it, you might be +forced to use convoluted logic during test loading, which is a bit of a mess. + + +Legacy skip support +~~~~~~~~~~~~~~~~~~~ + +If you are using this feature when running your test suite with a legacy +``TestResult`` object that is missing the ``addSkip`` method, then the +``addError`` method will be invoked instead. If you are using a test result +from testtools, you do not have to worry about this. + +In older versions of testtools, ``skipTest`` was known as ``skip``. Since +Python 2.7 added ``skipTest`` support, the ``skip`` name is now deprecated. +No warning is emitted yet – some time in the future we may do so. + + +addOnException +-------------- + +Sometimes, you might wish to do something only when a test fails. Perhaps you +need to run expensive diagnostic routines or some such. +``TestCase.addOnException`` allows you to easily do just this. For example:: + + class SomeTest(TestCase): + def setUp(self): + super(SomeTest, self).setUp() + self.server = self.useFixture(SomeServer()) + self.addOnException(self.attach_server_diagnostics) + + def attach_server_diagnostics(self, exc_info): + self.server.prep_for_diagnostics() # Expensive! + self.addDetail('server-diagnostics', self.server.get_diagnostics) + + def test_a_thing(self): + self.assertEqual('cheese', 'chalk') + +In this example, ``attach_server_diagnostics`` will only be called when a test +fails. It is given the exc_info tuple of the error raised by the test, just +in case it is needed. + + +Twisted support +--------------- + +testtools provides *highly experimental* support for running Twisted tests – +tests that return a Deferred_ and rely on the Twisted reactor. You should not +use this feature right now. We reserve the right to change the API and +behaviour without telling you first. + +However, if you are going to, here's how you do it:: + + from testtools import TestCase + from testtools.deferredruntest import AsynchronousDeferredRunTest + + class MyTwistedTests(TestCase): + + run_tests_with = AsynchronousDeferredRunTest + + def test_foo(self): + # ... + return d + +In particular, note that you do *not* have to use a special base ``TestCase`` +in order to run Twisted tests. + +You can also run individual tests within a test case class using the Twisted +test runner:: + + class MyTestsSomeOfWhichAreTwisted(TestCase): + + def test_normal(self): + pass + + @run_test_with(AsynchronousDeferredRunTest) + def test_twisted(self): + # ... + return d + +Here are some tips for converting your Trial tests into testtools tests. + +* Use the ``AsynchronousDeferredRunTest`` runner +* Make sure to upcall to ``setUp`` and ``tearDown`` +* Don't use ``setUpClass`` or ``tearDownClass`` +* Don't expect setting .todo, .timeout or .skip attributes to do anything +* ``flushLoggedErrors`` is ``testtools.deferredruntest.flush_logged_errors`` +* ``assertFailure`` is ``testtools.deferredruntest.assert_fails_with`` +* Trial spins the reactor a couple of times before cleaning it up, + ``AsynchronousDeferredRunTest`` does not. If you rely on this behavior, use + ``AsynchronousDeferredRunTestForBrokenTwisted``. + + +Test helpers +============ + +testtools comes with a few little things that make it a little bit easier to +write tests. + + +TestCase.patch +-------------- + +``patch`` is a convenient way to monkey-patch a Python object for the duration +of your test. It's especially useful for testing legacy code. e.g.:: + + def test_foo(self): + my_stream = StringIO() + self.patch(sys, 'stderr', my_stream) + run_some_code_that_prints_to_stderr() + self.assertEqual('', my_stream.getvalue()) + +The call to ``patch`` above masks ``sys.stderr`` with ``my_stream`` so that +anything printed to stderr will be captured in a StringIO variable that can be +actually tested. Once the test is done, the real ``sys.stderr`` is restored to +its rightful place. + + +Creation methods +---------------- + +Often when writing unit tests, you want to create an object that is a +completely normal instance of its type. You don't want there to be anything +special about its properties, because you are testing generic behaviour rather +than specific conditions. + +A lot of the time, test authors do this by making up silly strings and numbers +and passing them to constructors (e.g. 42, 'foo', "bar" etc), and that's +fine. However, sometimes it's useful to be able to create arbitrary objects +at will, without having to make up silly sample data. + +To help with this, ``testtools.TestCase`` implements creation methods called +``getUniqueString`` and ``getUniqueInteger``. They return strings and +integers that are unique within the context of the test that can be used to +assemble more complex objects. Here's a basic example where +``getUniqueString`` is used instead of saying "foo" or "bar" or whatever:: + + class SomeTest(TestCase): + + def test_full_name(self): + first_name = self.getUniqueString() + last_name = self.getUniqueString() + p = Person(first_name, last_name) + self.assertEqual(p.full_name, "%s %s" % (first_name, last_name)) + + +And here's how it could be used to make a complicated test:: + + class TestCoupleLogic(TestCase): + + def make_arbitrary_person(self): + return Person(self.getUniqueString(), self.getUniqueString()) + + def test_get_invitation(self): + a = self.make_arbitrary_person() + b = self.make_arbitrary_person() + couple = Couple(a, b) + event_name = self.getUniqueString() + invitation = couple.get_invitation(event_name) + self.assertEqual( + invitation, + "We invite %s and %s to %s" % ( + a.full_name, b.full_name, event_name)) + +Essentially, creation methods like these are a way of reducing the number of +assumptions in your tests and communicating to test readers that the exact +details of certain variables don't actually matter. + +See pages 419-423 of `xUnit Test Patterns`_ by Gerard Meszaros for a detailed +discussion of creation methods. + + +General helpers +=============== + +Conditional imports +------------------- + +Lots of the time we would like to conditionally import modules. testtools +needs to do this itself, and graciously extends the ability to its users. + +Instead of:: + + try: + from twisted.internet import defer + except ImportError: + defer = None + +You can do:: + + defer = try_import('twisted.internet.defer') + + +Instead of:: + + try: + from StringIO import StringIO + except ImportError: + from io import StringIO + +You can do:: + + StringIO = try_imports(['StringIO.StringIO', 'io.StringIO']) + + +Safe attribute testing +---------------------- + +``hasattr`` is broken_ on many versions of Python. testtools provides +``safe_hasattr``, which can be used to safely test whether an object has a +particular attribute. + + +.. _testrepository: https://launchpad.net/testrepository +.. _Trial: http://twistedmatrix.com/documents/current/core/howto/testing.html +.. _nose: http://somethingaboutorange.com/mrl/projects/nose/ +.. _unittest2: http://pypi.python.org/pypi/unittest2 +.. _zope.testrunner: http://pypi.python.org/pypi/zope.testrunner/ +.. _xUnit test patterns: http://xunitpatterns.com/ +.. _fixtures: http://pypi.python.org/pypi/fixtures +.. _unittest: http://docs.python.org/library/unittest.html +.. _doctest: http://docs.python.org/library/doctest.html +.. _Deferred: http://twistedmatrix.com/documents/current/core/howto/defer.html +.. _discover: http://pypi.python.org/pypi/discover +.. _`testtools API docs`: http://mumak.net/testtools/apidocs/ +.. _Distutils: http://docs.python.org/library/distutils.html +.. _`setup configuration`: http://docs.python.org/distutils/configfile.html +.. _broken: http://chipaca.com/post/3210673069/hasattr-17-less-harmful diff --git a/test/3rdparty/testtools-0.9.12/doc/hacking.rst b/test/3rdparty/testtools-0.9.12/doc/hacking.rst new file mode 100644 index 00000000000..b9f5ff22c6c --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/doc/hacking.rst @@ -0,0 +1,154 @@ +========================= +Contributing to testtools +========================= + +Coding style +------------ + +In general, follow `PEP 8`_ except where consistency with the standard +library's unittest_ module would suggest otherwise. + +testtools supports Python 2.4 and later, including Python 3, so avoid any +2.5-only features like the ``with`` statement. + + +Copyright assignment +-------------------- + +Part of testtools raison d'etre is to provide Python with improvements to the +testing code it ships. For that reason we require all contributions (that are +non-trivial) to meet one of the following rules: + +* be inapplicable for inclusion in Python. +* be able to be included in Python without further contact with the contributor. +* be copyright assigned to Jonathan M. Lange. + +Please pick one of these and specify it when contributing code to testtools. + + +Licensing +--------- + +All code that is not copyright assigned to Jonathan M. Lange (see Copyright +Assignment above) needs to be licensed under the `MIT license`_ that testtools +uses, so that testtools can ship it. + + +Testing +------- + +Please write tests for every feature. This project ought to be a model +example of well-tested Python code! + +Take particular care to make sure the *intent* of each test is clear. + +You can run tests with ``make check``. + +By default, testtools hides many levels of its own stack when running tests. +This is for the convenience of users, who do not care about how, say, assert +methods are implemented. However, when writing tests for testtools itself, it +is often useful to see all levels of the stack. To do this, add +``run_tests_with = FullStackRunTest`` to the top of a test's class definition. + + +Documentation +------------- + +Documents are written using the Sphinx_ variant of reStructuredText_. All +public methods, functions, classes and modules must have API documentation. +When changing code, be sure to check the API documentation to see if it could +be improved. Before submitting changes to trunk, look over them and see if +the manuals ought to be updated. + + +Source layout +------------- + +The top-level directory contains the ``testtools/`` package directory, and +miscellaneous files like ``README`` and ``setup.py``. + +The ``testtools/`` directory is the Python package itself. It is separated +into submodules for internal clarity, but all public APIs should be “promoted” +into the top-level package by importing them in ``testtools/__init__.py``. +Users of testtools should never import a submodule in order to use a stable +API. Unstable APIs like ``testtools.matchers`` and +``testtools.deferredruntest`` should be exported as submodules. + +Tests belong in ``testtools/tests/``. + + +Committing to trunk +------------------- + +Testtools is maintained using bzr, with its trunk at lp:testtools. This gives +every contributor the ability to commit their work to their own branches. +However permission must be granted to allow contributors to commit to the trunk +branch. + +Commit access to trunk is obtained by joining the testtools-committers +Launchpad team. Membership in this team is contingent on obeying the testtools +contribution policy, see `Copyright Assignment`_ above. + + +Code Review +----------- + +All code must be reviewed before landing on trunk. The process is to create a +branch in launchpad, and submit it for merging to lp:testtools. It will then +be reviewed before it can be merged to trunk. It will be reviewed by someone: + +* not the author +* a committer (member of the `~testtools-committers`_ team) + +As a special exception, while the testtools committers team is small and prone +to blocking, a merge request from a committer that has not been reviewed after +24 hours may be merged by that committer. When the team is larger this policy +will be revisited. + +Code reviewers should look for the quality of what is being submitted, +including conformance with this HACKING file. + +Changes which all users should be made aware of should be documented in NEWS. + + +NEWS management +--------------- + +The file NEWS is structured as a sorted list of releases. Each release can have +a free form description and more or more sections with bullet point items. +Sections in use today are 'Improvements' and 'Changes'. To ease merging between +branches, the bullet points are kept alphabetically sorted. The release NEXT is +permanently present at the top of the list. + + +Release tasks +------------- + +#. Choose a version number, say X.Y.Z +#. Branch from trunk to testtools-X.Y.Z +#. In testtools-X.Y.Z, ensure __init__ has version ``(X, Y, Z, 'final', 0)`` +#. Replace NEXT in NEWS with the version number X.Y.Z, adjusting the reST. +#. Possibly write a blurb into NEWS. +#. Replace any additional references to NEXT with the version being + released. (There should be none other than the ones in these release tasks + which should not be replaced). +#. Commit the changes. +#. Tag the release, bzr tag testtools-X.Y.Z +#. Run 'make release', this: + #. Creates a source distribution and uploads to PyPI + #. Ensures all Fix Committed bugs are in the release milestone + #. Makes a release on Launchpad and uploads the tarball + #. Marks all the Fix Committed bugs as Fix Released + #. Creates a new milestone +#. Merge the release branch testtools-X.Y.Z into trunk. Before the commit, + add a NEXT heading to the top of NEWS and bump the version in __init__.py. + Push trunk to Launchpad +#. If a new series has been created (e.g. 0.10.0), make the series on Launchpad. + +.. _PEP 8: http://www.python.org/dev/peps/pep-0008/ +.. _unittest: http://docs.python.org/library/unittest.html +.. _~testtools-dev: https://launchpad.net/~testtools-dev +.. _MIT license: http://www.opensource.org/licenses/mit-license.php +.. _Sphinx: http://sphinx.pocoo.org/ +.. _restructuredtext: http://docutils.sourceforge.net/rst.html + diff --git a/test/3rdparty/testtools-0.9.12/doc/index.rst b/test/3rdparty/testtools-0.9.12/doc/index.rst new file mode 100644 index 00000000000..4687cebb62b --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/doc/index.rst @@ -0,0 +1,33 @@ +.. testtools documentation master file, created by + sphinx-quickstart on Sun Nov 28 13:45:40 2010. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +testtools: tasteful testing for Python +====================================== + +testtools is a set of extensions to the Python standard library's unit testing +framework. These extensions have been derived from many years of experience +with unit testing in Python and come from many different sources. testtools +also ports recent unittest changes all the way back to Python 2.4. + + +Contents: + +.. toctree:: + :maxdepth: 1 + + overview + for-test-authors + for-framework-folk + hacking + Changes to testtools <news> + API reference documentation <http://mumak.net/testtools/apidocs/> + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/test/3rdparty/testtools-0.9.12/doc/make.bat b/test/3rdparty/testtools-0.9.12/doc/make.bat new file mode 100644 index 00000000000..f8c1fd520ab --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/doc/make.bat @@ -0,0 +1,113 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +set SPHINXBUILD=sphinx-build +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^<target^>` where ^<target^> is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. changes to make an overview over all changed/added/deprecated items + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\testtools.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\testtools.ghc + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +:end diff --git a/test/3rdparty/testtools-0.9.12/doc/overview.rst b/test/3rdparty/testtools-0.9.12/doc/overview.rst new file mode 100644 index 00000000000..e43265fd1e0 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/doc/overview.rst @@ -0,0 +1,96 @@ +====================================== +testtools: tasteful testing for Python +====================================== + +testtools is a set of extensions to the Python standard library's unit testing +framework. These extensions have been derived from many years of experience +with unit testing in Python and come from many different sources. testtools +also ports recent unittest changes all the way back to Python 2.4. + +What better way to start than with a contrived code snippet?:: + + from testtools import TestCase + from testtools.content import Content + from testtools.content_type import UTF8_TEXT + from testtools.matchers import Equals + + from myproject import SillySquareServer + + class TestSillySquareServer(TestCase): + + def setUp(self): + super(TestSillySquare, self).setUp() + self.server = self.useFixture(SillySquareServer()) + self.addCleanup(self.attach_log_file) + + def attach_log_file(self): + self.addDetail( + 'log-file', + Content(UTF8_TEXT + lambda: open(self.server.logfile, 'r').readlines())) + + def test_server_is_cool(self): + self.assertThat(self.server.temperature, Equals("cool")) + + def test_square(self): + self.assertThat(self.server.silly_square_of(7), Equals(49)) + + +Why use testtools? +================== + +Better assertion methods +------------------------ + +The standard assertion methods that come with unittest aren't as helpful as +they could be, and there aren't quite enough of them. testtools adds +``assertIn``, ``assertIs``, ``assertIsInstance`` and their negatives. + + +Matchers: better than assertion methods +--------------------------------------- + +Of course, in any serious project you want to be able to have assertions that +are specific to that project and the particular problem that it is addressing. +Rather than forcing you to define your own assertion methods and maintain your +own inheritance hierarchy of ``TestCase`` classes, testtools lets you write +your own "matchers", custom predicates that can be plugged into a unit test:: + + def test_response_has_bold(self): + # The response has bold text. + response = self.server.getResponse() + self.assertThat(response, HTMLContains(Tag('bold', 'b'))) + + +More debugging info, when you need it +-------------------------------------- + +testtools makes it easy to add arbitrary data to your test result. If you +want to know what's in a log file when a test fails, or what the load was on +the computer when a test started, or what files were open, you can add that +information with ``TestCase.addDetail``, and it will appear in the test +results if that test fails. + + +Extend unittest, but stay compatible and re-usable +-------------------------------------------------- + +testtools goes to great lengths to allow serious test authors and test +*framework* authors to do whatever they like with their tests and their +extensions while staying compatible with the standard library's unittest. + +testtools has completely parametrized how exceptions raised in tests are +mapped to ``TestResult`` methods and how tests are actually executed (ever +wanted ``tearDown`` to be called regardless of whether ``setUp`` succeeds?) + +It also provides many simple but handy utilities, like the ability to clone a +test, a ``MultiTestResult`` object that lets many result objects get the +results from one test suite, adapters to bring legacy ``TestResult`` objects +into our new golden age. + + +Cross-Python compatibility +-------------------------- + +testtools gives you the very latest in unit testing technology in a way that +will work with Python 2.4, 2.5, 2.6, 2.7 and 3.1. diff --git a/test/3rdparty/testtools-0.9.12/setup.cfg b/test/3rdparty/testtools-0.9.12/setup.cfg new file mode 100644 index 00000000000..9f95adde2b4 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/setup.cfg @@ -0,0 +1,4 @@ +[test] +test_module = testtools.tests +buffer=1 +catch=1 diff --git a/test/3rdparty/testtools-0.9.12/setup.py b/test/3rdparty/testtools-0.9.12/setup.py new file mode 100755 index 00000000000..d07c8f29359 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/setup.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +"""Distutils installer for testtools.""" + +from distutils.core import setup +import email +import os + +import testtools + + +def get_revno(): + import bzrlib.errors + import bzrlib.workingtree + try: + t = bzrlib.workingtree.WorkingTree.open_containing(__file__)[0] + except (bzrlib.errors.NotBranchError, bzrlib.errors.NoWorkingTree): + return None + else: + return t.branch.revno() + + +def get_version_from_pkg_info(): + """Get the version from PKG-INFO file if we can.""" + pkg_info_path = os.path.join(os.path.dirname(__file__), 'PKG-INFO') + try: + pkg_info_file = open(pkg_info_path, 'r') + except (IOError, OSError): + return None + try: + pkg_info = email.message_from_file(pkg_info_file) + except email.MessageError: + return None + return pkg_info.get('Version', None) + + +def get_version(): + """Return the version of testtools that we are building.""" + version = '.'.join( + str(component) for component in testtools.__version__[0:3]) + phase = testtools.__version__[3] + if phase == 'final': + return version + pkg_info_version = get_version_from_pkg_info() + if pkg_info_version: + return pkg_info_version + revno = get_revno() + if revno is None: + return "snapshot" + if phase == 'alpha': + # No idea what the next version will be + return 'next-r%s' % revno + else: + # Preserve the version number but give it a revno prefix + return version + '-r%s' % revno + + +def get_long_description(): + manual_path = os.path.join( + os.path.dirname(__file__), 'doc/overview.rst') + return open(manual_path).read() + + +setup(name='testtools', + author='Jonathan M. Lange', + author_email='jml+testtools@mumak.net', + url='https://launchpad.net/testtools', + description=('Extensions to the Python standard library unit testing ' + 'framework'), + long_description=get_long_description(), + version=get_version(), + classifiers=["License :: OSI Approved :: MIT License"], + packages=['testtools', 'testtools.testresult', 'testtools.tests'], + cmdclass={'test': testtools.TestCommand}) diff --git a/test/3rdparty/testtools-0.9.12/testtools/__init__.py b/test/3rdparty/testtools-0.9.12/testtools/__init__.py new file mode 100644 index 00000000000..11b5662b146 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/__init__.py @@ -0,0 +1,83 @@ +# Copyright (c) 2008-2011 testtools developers. See LICENSE for details. + +"""Extensions to the standard Python unittest library.""" + +__all__ = [ + 'clone_test_with_new_id', + 'ConcurrentTestSuite', + 'ErrorHolder', + 'ExpectedException', + 'ExtendedToOriginalDecorator', + 'FixtureSuite', + 'iterate_tests', + 'MultipleExceptions', + 'MultiTestResult', + 'PlaceHolder', + 'run_test_with', + 'TestCase', + 'TestCommand', + 'TestResult', + 'TextTestResult', + 'RunTest', + 'skip', + 'skipIf', + 'skipUnless', + 'ThreadsafeForwardingResult', + 'try_import', + 'try_imports', + ] + +from testtools.helpers import ( + try_import, + try_imports, + ) +from testtools.matchers import ( + Matcher, + ) +# Shut up, pyflakes. We are importing for documentation, not for namespacing. +Matcher + +from testtools.runtest import ( + MultipleExceptions, + RunTest, + ) +from testtools.testcase import ( + ErrorHolder, + ExpectedException, + PlaceHolder, + TestCase, + clone_test_with_new_id, + run_test_with, + skip, + skipIf, + skipUnless, + ) +from testtools.testresult import ( + ExtendedToOriginalDecorator, + MultiTestResult, + TestResult, + TextTestResult, + ThreadsafeForwardingResult, + ) +from testtools.testsuite import ( + ConcurrentTestSuite, + FixtureSuite, + iterate_tests, + ) +from testtools.distutilscmd import ( + TestCommand, +) + +# same format as sys.version_info: "A tuple containing the five components of +# the version number: major, minor, micro, releaselevel, and serial. All +# values except releaselevel are integers; the release level is 'alpha', +# 'beta', 'candidate', or 'final'. The version_info value corresponding to the +# Python version 2.0 is (2, 0, 0, 'final', 0)." Additionally we use a +# releaselevel of 'dev' for unreleased under-development code. +# +# If the releaselevel is 'alpha' then the major/minor/micro components are not +# established at this point, and setup.py will use a version of next-$(revno). +# If the releaselevel is 'final', then the tarball will be major.minor.micro. +# Otherwise it is major.minor.micro~$(revno). + +__version__ = (0, 9, 12, 'final', 0) diff --git a/test/3rdparty/testtools-0.9.12/testtools/_compat2x.py b/test/3rdparty/testtools-0.9.12/testtools/_compat2x.py new file mode 100644 index 00000000000..2b25c13e081 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/_compat2x.py @@ -0,0 +1,17 @@ +# Copyright (c) 2011 testtools developers. See LICENSE for details. + +"""Compatibility helpers that are valid syntax in Python 2.x. + +Only add things here if they *only* work in Python 2.x or are Python 2 +alternatives to things that *only* work in Python 3.x. +""" + +__all__ = [ + 'reraise', + ] + + +def reraise(exc_class, exc_obj, exc_tb, _marker=object()): + """Re-raise an exception received from sys.exc_info() or similar.""" + raise exc_class, exc_obj, exc_tb + diff --git a/test/3rdparty/testtools-0.9.12/testtools/_compat3x.py b/test/3rdparty/testtools-0.9.12/testtools/_compat3x.py new file mode 100644 index 00000000000..f3d569662da --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/_compat3x.py @@ -0,0 +1,17 @@ +# Copyright (c) 2011 testtools developers. See LICENSE for details. + +"""Compatibility helpers that are valid syntax in Python 3.x. + +Only add things here if they *only* work in Python 3.x or are Python 3 +alternatives to things that *only* work in Python 2.x. +""" + +__all__ = [ + 'reraise', + ] + + +def reraise(exc_class, exc_obj, exc_tb, _marker=object()): + """Re-raise an exception received from sys.exc_info() or similar.""" + raise exc_class(*exc_obj.args).with_traceback(exc_tb) + diff --git a/test/3rdparty/testtools-0.9.12/testtools/_spinner.py b/test/3rdparty/testtools-0.9.12/testtools/_spinner.py new file mode 100644 index 00000000000..baf455a5f94 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/_spinner.py @@ -0,0 +1,316 @@ +# Copyright (c) 2010 testtools developers. See LICENSE for details. + +"""Evil reactor-spinning logic for running Twisted tests. + +This code is highly experimental, liable to change and not to be trusted. If +you couldn't write this yourself, you should not be using it. +""" + +__all__ = [ + 'DeferredNotFired', + 'extract_result', + 'NoResultError', + 'not_reentrant', + 'ReentryError', + 'Spinner', + 'StaleJunkError', + 'TimeoutError', + 'trap_unhandled_errors', + ] + +import signal + +from testtools.monkey import MonkeyPatcher + +from twisted.internet import defer +from twisted.internet.base import DelayedCall +from twisted.internet.interfaces import IReactorThreads +from twisted.python.failure import Failure +from twisted.python.util import mergeFunctionMetadata + + +class ReentryError(Exception): + """Raised when we try to re-enter a function that forbids it.""" + + def __init__(self, function): + Exception.__init__(self, + "%r in not re-entrant but was called within a call to itself." + % (function,)) + + +def not_reentrant(function, _calls={}): + """Decorates a function as not being re-entrant. + + The decorated function will raise an error if called from within itself. + """ + def decorated(*args, **kwargs): + if _calls.get(function, False): + raise ReentryError(function) + _calls[function] = True + try: + return function(*args, **kwargs) + finally: + _calls[function] = False + return mergeFunctionMetadata(function, decorated) + + +class DeferredNotFired(Exception): + """Raised when we extract a result from a Deferred that's not fired yet.""" + + +def extract_result(deferred): + """Extract the result from a fired deferred. + + It can happen that you have an API that returns Deferreds for + compatibility with Twisted code, but is in fact synchronous, i.e. the + Deferreds it returns have always fired by the time it returns. In this + case, you can use this function to convert the result back into the usual + form for a synchronous API, i.e. the result itself or a raised exception. + + It would be very bad form to use this as some way of checking if a + Deferred has fired. + """ + failures = [] + successes = [] + deferred.addCallbacks(successes.append, failures.append) + if len(failures) == 1: + failures[0].raiseException() + elif len(successes) == 1: + return successes[0] + else: + raise DeferredNotFired("%r has not fired yet." % (deferred,)) + + +def trap_unhandled_errors(function, *args, **kwargs): + """Run a function, trapping any unhandled errors in Deferreds. + + Assumes that 'function' will have handled any errors in Deferreds by the + time it is complete. This is almost never true of any Twisted code, since + you can never tell when someone has added an errback to a Deferred. + + If 'function' raises, then don't bother doing any unhandled error + jiggery-pokery, since something horrible has probably happened anyway. + + :return: A tuple of '(result, error)', where 'result' is the value + returned by 'function' and 'error' is a list of 'defer.DebugInfo' + objects that have unhandled errors in Deferreds. + """ + real_DebugInfo = defer.DebugInfo + debug_infos = [] + def DebugInfo(): + info = real_DebugInfo() + debug_infos.append(info) + return info + defer.DebugInfo = DebugInfo + try: + result = function(*args, **kwargs) + finally: + defer.DebugInfo = real_DebugInfo + errors = [] + for info in debug_infos: + if info.failResult is not None: + errors.append(info) + # Disable the destructor that logs to error. We are already + # catching the error here. + info.__del__ = lambda: None + return result, errors + + +class TimeoutError(Exception): + """Raised when run_in_reactor takes too long to run a function.""" + + def __init__(self, function, timeout): + Exception.__init__(self, + "%r took longer than %s seconds" % (function, timeout)) + + +class NoResultError(Exception): + """Raised when the reactor has stopped but we don't have any result.""" + + def __init__(self): + Exception.__init__(self, + "Tried to get test's result from Deferred when no result is " + "available. Probably means we received SIGINT or similar.") + + +class StaleJunkError(Exception): + """Raised when there's junk in the spinner from a previous run.""" + + def __init__(self, junk): + Exception.__init__(self, + "There was junk in the spinner from a previous run. " + "Use clear_junk() to clear it out: %r" % (junk,)) + + +class Spinner(object): + """Spin the reactor until a function is done. + + This class emulates the behaviour of twisted.trial in that it grotesquely + and horribly spins the Twisted reactor while a function is running, and + then kills the reactor when that function is complete and all the + callbacks in its chains are done. + """ + + _UNSET = object() + + # Signals that we save and restore for each spin. + _PRESERVED_SIGNALS = [ + 'SIGINT', + 'SIGTERM', + 'SIGCHLD', + ] + + # There are many APIs within Twisted itself where a Deferred fires but + # leaves cleanup work scheduled for the reactor to do. Arguably, many of + # these are bugs. As such, we provide a facility to iterate the reactor + # event loop a number of times after every call, in order to shake out + # these buggy-but-commonplace events. The default is 0, because that is + # the ideal, and it actually works for many cases. + _OBLIGATORY_REACTOR_ITERATIONS = 0 + + def __init__(self, reactor, debug=False): + """Construct a Spinner. + + :param reactor: A Twisted reactor. + :param debug: Whether or not to enable Twisted's debugging. Defaults + to False. + """ + self._reactor = reactor + self._timeout_call = None + self._success = self._UNSET + self._failure = self._UNSET + self._saved_signals = [] + self._junk = [] + self._debug = debug + + def _cancel_timeout(self): + if self._timeout_call: + self._timeout_call.cancel() + + def _get_result(self): + if self._failure is not self._UNSET: + self._failure.raiseException() + if self._success is not self._UNSET: + return self._success + raise NoResultError() + + def _got_failure(self, result): + self._cancel_timeout() + self._failure = result + + def _got_success(self, result): + self._cancel_timeout() + self._success = result + + def _stop_reactor(self, ignored=None): + """Stop the reactor!""" + self._reactor.crash() + + def _timed_out(self, function, timeout): + e = TimeoutError(function, timeout) + self._failure = Failure(e) + self._stop_reactor() + + def _clean(self): + """Clean up any junk in the reactor. + + Will always iterate the reactor a number of times equal to + ``Spinner._OBLIGATORY_REACTOR_ITERATIONS``. This is to work around + bugs in various Twisted APIs where a Deferred fires but still leaves + work (e.g. cancelling a call, actually closing a connection) for the + reactor to do. + """ + for i in range(self._OBLIGATORY_REACTOR_ITERATIONS): + self._reactor.iterate(0) + junk = [] + for delayed_call in self._reactor.getDelayedCalls(): + delayed_call.cancel() + junk.append(delayed_call) + for selectable in self._reactor.removeAll(): + # Twisted sends a 'KILL' signal to selectables that provide + # IProcessTransport. Since only _dumbwin32proc processes do this, + # we aren't going to bother. + junk.append(selectable) + if IReactorThreads.providedBy(self._reactor): + if self._reactor.threadpool is not None: + self._reactor._stopThreadPool() + self._junk.extend(junk) + return junk + + def clear_junk(self): + """Clear out our recorded junk. + + :return: Whatever junk was there before. + """ + junk = self._junk + self._junk = [] + return junk + + def get_junk(self): + """Return any junk that has been found on the reactor.""" + return self._junk + + def _save_signals(self): + available_signals = [ + getattr(signal, name, None) for name in self._PRESERVED_SIGNALS] + self._saved_signals = [ + (sig, signal.getsignal(sig)) for sig in available_signals if sig] + + def _restore_signals(self): + for sig, hdlr in self._saved_signals: + signal.signal(sig, hdlr) + self._saved_signals = [] + + @not_reentrant + def run(self, timeout, function, *args, **kwargs): + """Run 'function' in a reactor. + + If 'function' returns a Deferred, the reactor will keep spinning until + the Deferred fires and its chain completes or until the timeout is + reached -- whichever comes first. + + :raise TimeoutError: If 'timeout' is reached before the Deferred + returned by 'function' has completed its callback chain. + :raise NoResultError: If the reactor is somehow interrupted before + the Deferred returned by 'function' has completed its callback + chain. + :raise StaleJunkError: If there's junk in the spinner from a previous + run. + :return: Whatever is at the end of the function's callback chain. If + it's an error, then raise that. + """ + debug = MonkeyPatcher() + if self._debug: + debug.add_patch(defer.Deferred, 'debug', True) + debug.add_patch(DelayedCall, 'debug', True) + debug.patch() + try: + junk = self.get_junk() + if junk: + raise StaleJunkError(junk) + self._save_signals() + self._timeout_call = self._reactor.callLater( + timeout, self._timed_out, function, timeout) + # Calling 'stop' on the reactor will make it impossible to + # re-start the reactor. Since the default signal handlers for + # TERM, BREAK and INT all call reactor.stop(), we'll patch it over + # with crash. XXX: It might be a better idea to either install + # custom signal handlers or to override the methods that are + # Twisted's signal handlers. + stop, self._reactor.stop = self._reactor.stop, self._reactor.crash + def run_function(): + d = defer.maybeDeferred(function, *args, **kwargs) + d.addCallbacks(self._got_success, self._got_failure) + d.addBoth(self._stop_reactor) + try: + self._reactor.callWhenRunning(run_function) + self._reactor.run() + finally: + self._reactor.stop = stop + self._restore_signals() + try: + return self._get_result() + finally: + self._clean() + finally: + debug.restore() diff --git a/test/3rdparty/testtools-0.9.12/testtools/compat.py b/test/3rdparty/testtools-0.9.12/testtools/compat.py new file mode 100644 index 00000000000..b7e23c8fec6 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/compat.py @@ -0,0 +1,393 @@ +# Copyright (c) 2008-2011 testtools developers. See LICENSE for details. + +"""Compatibility support for python 2 and 3.""" + +__metaclass__ = type +__all__ = [ + '_b', + '_u', + 'advance_iterator', + 'all', + 'BytesIO', + 'classtypes', + 'isbaseexception', + 'istext', + 'str_is_unicode', + 'StringIO', + 'reraise', + 'unicode_output_stream', + ] + +import codecs +import linecache +import locale +import os +import re +import sys +import traceback +import unicodedata + +from testtools.helpers import try_imports + +BytesIO = try_imports(['StringIO.StringIO', 'io.BytesIO']) +StringIO = try_imports(['StringIO.StringIO', 'io.StringIO']) + +try: + from testtools import _compat2x as _compat + _compat +except SyntaxError: + from testtools import _compat3x as _compat + +reraise = _compat.reraise + + +__u_doc = """A function version of the 'u' prefix. + +This is needed becayse the u prefix is not usable in Python 3 but is required +in Python 2 to get a unicode object. + +To migrate code that was written as u'\u1234' in Python 2 to 2+3 change +it to be _u('\u1234'). The Python 3 interpreter will decode it +appropriately and the no-op _u for Python 3 lets it through, in Python +2 we then call unicode-escape in the _u function. +""" + +if sys.version_info > (3, 0): + import builtins + def _u(s): + return s + _r = ascii + def _b(s): + """A byte literal.""" + return s.encode("latin-1") + advance_iterator = next + # GZ 2011-08-24: Seems istext() is easy to misuse and makes for bad code. + def istext(x): + return isinstance(x, str) + def classtypes(): + return (type,) + str_is_unicode = True +else: + import __builtin__ as builtins + def _u(s): + # The double replace mangling going on prepares the string for + # unicode-escape - \foo is preserved, \u and \U are decoded. + return (s.replace("\\", "\\\\").replace("\\\\u", "\\u") + .replace("\\\\U", "\\U").decode("unicode-escape")) + _r = repr + def _b(s): + return s + advance_iterator = lambda it: it.next() + def istext(x): + return isinstance(x, basestring) + def classtypes(): + import types + return (type, types.ClassType) + str_is_unicode = sys.platform == "cli" + +_u.__doc__ = __u_doc + + +if sys.version_info > (2, 5): + all = all + _error_repr = BaseException.__repr__ + def isbaseexception(exception): + """Return whether exception inherits from BaseException only""" + return (isinstance(exception, BaseException) + and not isinstance(exception, Exception)) +else: + def all(iterable): + """If contents of iterable all evaluate as boolean True""" + for obj in iterable: + if not obj: + return False + return True + def _error_repr(exception): + """Format an exception instance as Python 2.5 and later do""" + return exception.__class__.__name__ + repr(exception.args) + def isbaseexception(exception): + """Return whether exception would inherit from BaseException only + + This approximates the hierarchy in Python 2.5 and later, compare the + difference between the diagrams at the bottom of the pages: + <http://docs.python.org/release/2.4.4/lib/module-exceptions.html> + <http://docs.python.org/release/2.5.4/lib/module-exceptions.html> + """ + return isinstance(exception, (KeyboardInterrupt, SystemExit)) + + +# GZ 2011-08-24: Using isinstance checks like this encourages bad interfaces, +# there should be better ways to write code needing this. +if not issubclass(getattr(builtins, "bytes", str), str): + def _isbytes(x): + return isinstance(x, bytes) +else: + # Never return True on Pythons that provide the name but not the real type + def _isbytes(x): + return False + + +def _slow_escape(text): + """Escape unicode `text` leaving printable characters unmodified + + The behaviour emulates the Python 3 implementation of repr, see + unicode_repr in unicodeobject.c and isprintable definition. + + Because this iterates over the input a codepoint at a time, it's slow, and + does not handle astral characters correctly on Python builds with 16 bit + rather than 32 bit unicode type. + """ + output = [] + for c in text: + o = ord(c) + if o < 256: + if o < 32 or 126 < o < 161: + output.append(c.encode("unicode-escape")) + elif o == 92: + # Separate due to bug in unicode-escape codec in Python 2.4 + output.append("\\\\") + else: + output.append(c) + else: + # To get correct behaviour would need to pair up surrogates here + if unicodedata.category(c)[0] in "CZ": + output.append(c.encode("unicode-escape")) + else: + output.append(c) + return "".join(output) + + +def text_repr(text, multiline=None): + """Rich repr for `text` returning unicode, triple quoted if `multiline`""" + is_py3k = sys.version_info > (3, 0) + nl = _isbytes(text) and bytes((0xA,)) or "\n" + if multiline is None: + multiline = nl in text + if not multiline and (is_py3k or not str_is_unicode and type(text) is str): + # Use normal repr for single line of unicode on Python 3 or bytes + return repr(text) + prefix = repr(text[:0])[:-2] + if multiline: + # To escape multiline strings, split and process each line in turn, + # making sure that quotes are not escaped. + if is_py3k: + offset = len(prefix) + 1 + lines = [] + for l in text.split(nl): + r = repr(l) + q = r[-1] + lines.append(r[offset:-1].replace("\\" + q, q)) + elif not str_is_unicode and isinstance(text, str): + lines = [l.encode("string-escape").replace("\\'", "'") + for l in text.split("\n")] + else: + lines = [_slow_escape(l) for l in text.split("\n")] + # Combine the escaped lines and append two of the closing quotes, + # then iterate over the result to escape triple quotes correctly. + _semi_done = "\n".join(lines) + "''" + p = 0 + while True: + p = _semi_done.find("'''", p) + if p == -1: + break + _semi_done = "\\".join([_semi_done[:p], _semi_done[p:]]) + p += 2 + return "".join([prefix, "'''\\\n", _semi_done, "'"]) + escaped_text = _slow_escape(text) + # Determine which quote character to use and if one gets prefixed with a + # backslash following the same logic Python uses for repr() on strings + quote = "'" + if "'" in text: + if '"' in text: + escaped_text = escaped_text.replace("'", "\\'") + else: + quote = '"' + return "".join([prefix, quote, escaped_text, quote]) + + +def unicode_output_stream(stream): + """Get wrapper for given stream that writes any unicode without exception + + Characters that can't be coerced to the encoding of the stream, or 'ascii' + if valid encoding is not found, will be replaced. The original stream may + be returned in situations where a wrapper is determined unneeded. + + The wrapper only allows unicode to be written, not non-ascii bytestrings, + which is a good thing to ensure sanity and sanitation. + """ + if sys.platform == "cli": + # Best to never encode before writing in IronPython + return stream + try: + writer = codecs.getwriter(stream.encoding or "") + except (AttributeError, LookupError): + # GZ 2010-06-16: Python 3 StringIO ends up here, but probably needs + # different handling as it doesn't want bytestrings + return codecs.getwriter("ascii")(stream, "replace") + if writer.__module__.rsplit(".", 1)[1].startswith("utf"): + # The current stream has a unicode encoding so no error handler is needed + if sys.version_info > (3, 0): + return stream + return writer(stream) + if sys.version_info > (3, 0): + # Python 3 doesn't seem to make this easy, handle a common case + try: + return stream.__class__(stream.buffer, stream.encoding, "replace", + stream.newlines, stream.line_buffering) + except AttributeError: + pass + return writer(stream, "replace") + + +# The default source encoding is actually "iso-8859-1" until Python 2.5 but +# using non-ascii causes a deprecation warning in 2.4 and it's cleaner to +# treat all versions the same way +_default_source_encoding = "ascii" + +# Pattern specified in <http://www.python.org/dev/peps/pep-0263/> +_cookie_search=re.compile("coding[:=]\s*([-\w.]+)").search + +def _detect_encoding(lines): + """Get the encoding of a Python source file from a list of lines as bytes + + This function does less than tokenize.detect_encoding added in Python 3 as + it does not attempt to raise a SyntaxError when the interpreter would, it + just wants the encoding of a source file Python has already compiled and + determined is valid. + """ + if not lines: + return _default_source_encoding + if lines[0].startswith("\xef\xbb\xbf"): + # Source starting with UTF-8 BOM is either UTF-8 or a SyntaxError + return "utf-8" + # Only the first two lines of the source file are examined + magic = _cookie_search("".join(lines[:2])) + if magic is None: + return _default_source_encoding + encoding = magic.group(1) + try: + codecs.lookup(encoding) + except LookupError: + # Some codecs raise something other than LookupError if they don't + # support the given error handler, but not the text ones that could + # actually be used for Python source code + return _default_source_encoding + return encoding + + +class _EncodingTuple(tuple): + """A tuple type that can have an encoding attribute smuggled on""" + + +def _get_source_encoding(filename): + """Detect, cache and return the encoding of Python source at filename""" + try: + return linecache.cache[filename].encoding + except (AttributeError, KeyError): + encoding = _detect_encoding(linecache.getlines(filename)) + if filename in linecache.cache: + newtuple = _EncodingTuple(linecache.cache[filename]) + newtuple.encoding = encoding + linecache.cache[filename] = newtuple + return encoding + + +def _get_exception_encoding(): + """Return the encoding we expect messages from the OS to be encoded in""" + if os.name == "nt": + # GZ 2010-05-24: Really want the codepage number instead, the error + # handling of standard codecs is more deterministic + return "mbcs" + # GZ 2010-05-23: We need this call to be after initialisation, but there's + # no benefit in asking more than once as it's a global + # setting that can change after the message is formatted. + return locale.getlocale(locale.LC_MESSAGES)[1] or "ascii" + + +def _exception_to_text(evalue): + """Try hard to get a sensible text value out of an exception instance""" + try: + return unicode(evalue) + except KeyboardInterrupt: + raise + except: + # Apparently this is what traceback._some_str does. Sigh - RBC 20100623 + pass + try: + return str(evalue).decode(_get_exception_encoding(), "replace") + except KeyboardInterrupt: + raise + except: + # Apparently this is what traceback._some_str does. Sigh - RBC 20100623 + pass + # Okay, out of ideas, let higher level handle it + return None + + +# GZ 2010-05-23: This function is huge and horrible and I welcome suggestions +# on the best way to break it up +_TB_HEADER = _u('Traceback (most recent call last):\n') +def _format_exc_info(eclass, evalue, tb, limit=None): + """Format a stack trace and the exception information as unicode + + Compatibility function for Python 2 which ensures each component of a + traceback is correctly decoded according to its origins. + + Based on traceback.format_exception and related functions. + """ + fs_enc = sys.getfilesystemencoding() + if tb: + list = [_TB_HEADER] + extracted_list = [] + for filename, lineno, name, line in traceback.extract_tb(tb, limit): + extracted_list.append(( + filename.decode(fs_enc, "replace"), + lineno, + name.decode("ascii", "replace"), + line and line.decode( + _get_source_encoding(filename), "replace"))) + list.extend(traceback.format_list(extracted_list)) + else: + list = [] + if evalue is None: + # Is a (deprecated) string exception + list.append((eclass + "\n").decode("ascii", "replace")) + return list + if isinstance(evalue, SyntaxError): + # Avoid duplicating the special formatting for SyntaxError here, + # instead create a new instance with unicode filename and line + # Potentially gives duff spacing, but that's a pre-existing issue + try: + msg, (filename, lineno, offset, line) = evalue + except (TypeError, ValueError): + pass # Strange exception instance, fall through to generic code + else: + # Errors during parsing give the line from buffer encoded as + # latin-1 or utf-8 or the encoding of the file depending on the + # coding and whether the patch for issue #1031213 is applied, so + # give up on trying to decode it and just read the file again + if line: + bytestr = linecache.getline(filename, lineno) + if bytestr: + if lineno == 1 and bytestr.startswith("\xef\xbb\xbf"): + bytestr = bytestr[3:] + line = bytestr.decode( + _get_source_encoding(filename), "replace") + del linecache.cache[filename] + else: + line = line.decode("ascii", "replace") + if filename: + filename = filename.decode(fs_enc, "replace") + evalue = eclass(msg, (filename, lineno, offset, line)) + list.extend(traceback.format_exception_only(eclass, evalue)) + return list + sclass = eclass.__name__ + svalue = _exception_to_text(evalue) + if svalue: + list.append("%s: %s\n" % (sclass, svalue)) + elif svalue is None: + # GZ 2010-05-24: Not a great fallback message, but keep for the moment + list.append("%s: <unprintable %s object>\n" % (sclass, sclass)) + else: + list.append("%s\n" % sclass) + return list diff --git a/test/3rdparty/testtools-0.9.12/testtools/content.py b/test/3rdparty/testtools-0.9.12/testtools/content.py new file mode 100644 index 00000000000..2c6ed9f5860 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/content.py @@ -0,0 +1,238 @@ +# Copyright (c) 2009-2011 testtools developers. See LICENSE for details. + +"""Content - a MIME-like Content object.""" + +__all__ = [ + 'attach_file', + 'Content', + 'content_from_file', + 'content_from_stream', + 'text_content', + 'TracebackContent', + ] + +import codecs +import os + +from testtools import try_import +from testtools.compat import _b +from testtools.content_type import ContentType, UTF8_TEXT +from testtools.testresult import TestResult + +functools = try_import('functools') + +_join_b = _b("").join + + +DEFAULT_CHUNK_SIZE = 4096 + + +def _iter_chunks(stream, chunk_size): + """Read 'stream' in chunks of 'chunk_size'. + + :param stream: A file-like object to read from. + :param chunk_size: The size of each read from 'stream'. + """ + chunk = stream.read(chunk_size) + while chunk: + yield chunk + chunk = stream.read(chunk_size) + + +class Content(object): + """A MIME-like Content object. + + Content objects can be serialised to bytes using the iter_bytes method. + If the Content-Type is recognised by other code, they are welcome to + look for richer contents that mere byte serialisation - for example in + memory object graphs etc. However, such code MUST be prepared to receive + a generic Content object that has been reconstructed from a byte stream. + + :ivar content_type: The content type of this Content. + """ + + def __init__(self, content_type, get_bytes): + """Create a ContentType.""" + if None in (content_type, get_bytes): + raise ValueError("None not permitted in %r, %r" % ( + content_type, get_bytes)) + self.content_type = content_type + self._get_bytes = get_bytes + + def __eq__(self, other): + return (self.content_type == other.content_type and + _join_b(self.iter_bytes()) == _join_b(other.iter_bytes())) + + def iter_bytes(self): + """Iterate over bytestrings of the serialised content.""" + return self._get_bytes() + + def iter_text(self): + """Iterate over the text of the serialised content. + + This is only valid for text MIME types, and will use ISO-8859-1 if + no charset parameter is present in the MIME type. (This is somewhat + arbitrary, but consistent with RFC2617 3.7.1). + + :raises ValueError: If the content type is not text/\*. + """ + if self.content_type.type != "text": + raise ValueError("Not a text type %r" % self.content_type) + return self._iter_text() + + def _iter_text(self): + """Worker for iter_text - does the decoding.""" + encoding = self.content_type.parameters.get('charset', 'ISO-8859-1') + try: + # 2.5+ + decoder = codecs.getincrementaldecoder(encoding)() + for bytes in self.iter_bytes(): + yield decoder.decode(bytes) + final = decoder.decode(_b(''), True) + if final: + yield final + except AttributeError: + # < 2.5 + bytes = ''.join(self.iter_bytes()) + yield bytes.decode(encoding) + + def __repr__(self): + return "<Content type=%r, value=%r>" % ( + self.content_type, _join_b(self.iter_bytes())) + + +class TracebackContent(Content): + """Content object for tracebacks. + + This adapts an exc_info tuple to the Content interface. + text/x-traceback;language=python is used for the mime type, in order to + provide room for other languages to format their tracebacks differently. + """ + + def __init__(self, err, test): + """Create a TracebackContent for err.""" + if err is None: + raise ValueError("err may not be None") + content_type = ContentType('text', 'x-traceback', + {"language": "python", "charset": "utf8"}) + self._result = TestResult() + value = self._result._exc_info_to_unicode(err, test) + super(TracebackContent, self).__init__( + content_type, lambda: [value.encode("utf8")]) + + +def text_content(text): + """Create a `Content` object from some text. + + This is useful for adding details which are short strings. + """ + return Content(UTF8_TEXT, lambda: [text.encode('utf8')]) + + + +def maybe_wrap(wrapper, func): + """Merge metadata for func into wrapper if functools is present.""" + if functools is not None: + wrapper = functools.update_wrapper(wrapper, func) + return wrapper + + +def content_from_file(path, content_type=None, chunk_size=DEFAULT_CHUNK_SIZE, + buffer_now=False): + """Create a `Content` object from a file on disk. + + Note that unless 'read_now' is explicitly passed in as True, the file + will only be read from when ``iter_bytes`` is called. + + :param path: The path to the file to be used as content. + :param content_type: The type of content. If not specified, defaults + to UTF8-encoded text/plain. + :param chunk_size: The size of chunks to read from the file. + Defaults to `DEFAULT_CHUNK_SIZE`. + :param buffer_now: If True, read the file from disk now and keep it in + memory. Otherwise, only read when the content is serialized. + """ + if content_type is None: + content_type = UTF8_TEXT + def reader(): + # This should be try:finally:, but python2.4 makes that hard. When + # We drop older python support we can make this use a context manager + # for maximum simplicity. + stream = open(path, 'rb') + for chunk in _iter_chunks(stream, chunk_size): + yield chunk + stream.close() + return content_from_reader(reader, content_type, buffer_now) + + +def content_from_stream(stream, content_type=None, + chunk_size=DEFAULT_CHUNK_SIZE, buffer_now=False): + """Create a `Content` object from a file-like stream. + + Note that the stream will only be read from when ``iter_bytes`` is + called. + + :param stream: A file-like object to read the content from. The stream + is not closed by this function or the content object it returns. + :param content_type: The type of content. If not specified, defaults + to UTF8-encoded text/plain. + :param chunk_size: The size of chunks to read from the file. + Defaults to `DEFAULT_CHUNK_SIZE`. + :param buffer_now: If True, reads from the stream right now. Otherwise, + only reads when the content is serialized. Defaults to False. + """ + if content_type is None: + content_type = UTF8_TEXT + reader = lambda: _iter_chunks(stream, chunk_size) + return content_from_reader(reader, content_type, buffer_now) + + +def content_from_reader(reader, content_type, buffer_now): + """Create a Content object that will obtain the content from reader. + + :param reader: A callback to read the content. Should return an iterable of + bytestrings. + :param content_type: The content type to create. + :param buffer_now: If True the reader is evaluated immediately and + buffered. + """ + if content_type is None: + content_type = UTF8_TEXT + if buffer_now: + contents = list(reader()) + reader = lambda: contents + return Content(content_type, reader) + + +def attach_file(detailed, path, name=None, content_type=None, + chunk_size=DEFAULT_CHUNK_SIZE, buffer_now=True): + """Attach a file to this test as a detail. + + This is a convenience method wrapping around `addDetail`. + + Note that unless 'read_now' is explicitly passed in as True, the file + *must* exist when the test result is called with the results of this + test, after the test has been torn down. + + :param detailed: An object with details + :param path: The path to the file to attach. + :param name: The name to give to the detail for the attached file. + :param content_type: The content type of the file. If not provided, + defaults to UTF8-encoded text/plain. + :param chunk_size: The size of chunks to read from the file. Defaults + to something sensible. + :param buffer_now: If False the file content is read when the content + object is evaluated rather than when attach_file is called. + Note that this may be after any cleanups that obj_with_details has, so + if the file is a temporary file disabling buffer_now may cause the file + to be read after it is deleted. To handle those cases, using + attach_file as a cleanup is recommended because it guarantees a + sequence for when the attach_file call is made:: + + detailed.addCleanup(attach_file, 'foo.txt', detailed) + """ + if name is None: + name = os.path.basename(path) + content_object = content_from_file( + path, content_type, chunk_size, buffer_now) + detailed.addDetail(name, content_object) diff --git a/test/3rdparty/testtools-0.9.12/testtools/content_type.py b/test/3rdparty/testtools-0.9.12/testtools/content_type.py new file mode 100644 index 00000000000..82c301b38df --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/content_type.py @@ -0,0 +1,39 @@ +# Copyright (c) 2009-2011 testtools developers. See LICENSE for details. + +"""ContentType - a MIME Content Type.""" + + +class ContentType(object): + """A content type from http://www.iana.org/assignments/media-types/ + + :ivar type: The primary type, e.g. "text" or "application" + :ivar subtype: The subtype, e.g. "plain" or "octet-stream" + :ivar parameters: A dict of additional parameters specific to the + content type. + """ + + def __init__(self, primary_type, sub_type, parameters=None): + """Create a ContentType.""" + if None in (primary_type, sub_type): + raise ValueError("None not permitted in %r, %r" % ( + primary_type, sub_type)) + self.type = primary_type + self.subtype = sub_type + self.parameters = parameters or {} + + def __eq__(self, other): + if type(other) != ContentType: + return False + return self.__dict__ == other.__dict__ + + def __repr__(self): + if self.parameters: + params = '; ' + params += ', '.join( + '%s="%s"' % (k, v) for k, v in self.parameters.items()) + else: + params = '' + return "%s/%s%s" % (self.type, self.subtype, params) + + +UTF8_TEXT = ContentType('text', 'plain', {'charset': 'utf8'}) diff --git a/test/3rdparty/testtools-0.9.12/testtools/deferredruntest.py b/test/3rdparty/testtools-0.9.12/testtools/deferredruntest.py new file mode 100644 index 00000000000..b8bfaaaa39f --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/deferredruntest.py @@ -0,0 +1,336 @@ +# Copyright (c) 2010 testtools developers. See LICENSE for details. + +"""Individual test case execution for tests that return Deferreds. + +This module is highly experimental and is liable to change in ways that cause +subtle failures in tests. Use at your own peril. +""" + +__all__ = [ + 'assert_fails_with', + 'AsynchronousDeferredRunTest', + 'AsynchronousDeferredRunTestForBrokenTwisted', + 'SynchronousDeferredRunTest', + ] + +import sys + +from testtools.compat import StringIO +from testtools.content import ( + Content, + text_content, + ) +from testtools.content_type import UTF8_TEXT +from testtools.runtest import RunTest +from testtools._spinner import ( + extract_result, + NoResultError, + Spinner, + TimeoutError, + trap_unhandled_errors, + ) + +from twisted.internet import defer +from twisted.python import log +from twisted.trial.unittest import _LogObserver + + +class _DeferredRunTest(RunTest): + """Base for tests that return Deferreds.""" + + def _got_user_failure(self, failure, tb_label='traceback'): + """We got a failure from user code.""" + return self._got_user_exception( + (failure.type, failure.value, failure.getTracebackObject()), + tb_label=tb_label) + + +class SynchronousDeferredRunTest(_DeferredRunTest): + """Runner for tests that return synchronous Deferreds.""" + + def _run_user(self, function, *args): + d = defer.maybeDeferred(function, *args) + d.addErrback(self._got_user_failure) + result = extract_result(d) + return result + + +def run_with_log_observers(observers, function, *args, **kwargs): + """Run 'function' with the given Twisted log observers.""" + real_observers = log.theLogPublisher.observers + for observer in real_observers: + log.theLogPublisher.removeObserver(observer) + for observer in observers: + log.theLogPublisher.addObserver(observer) + try: + return function(*args, **kwargs) + finally: + for observer in observers: + log.theLogPublisher.removeObserver(observer) + for observer in real_observers: + log.theLogPublisher.addObserver(observer) + + +# Observer of the Twisted log that we install during tests. +_log_observer = _LogObserver() + + + +class AsynchronousDeferredRunTest(_DeferredRunTest): + """Runner for tests that return Deferreds that fire asynchronously. + + That is, this test runner assumes that the Deferreds will only fire if the + reactor is left to spin for a while. + + Do not rely too heavily on the nuances of the behaviour of this class. + What it does to the reactor is black magic, and if we can find nicer ways + of doing it we will gladly break backwards compatibility. + + This is highly experimental code. Use at your own risk. + """ + + def __init__(self, case, handlers=None, reactor=None, timeout=0.005, + debug=False): + """Construct an `AsynchronousDeferredRunTest`. + + :param case: The `TestCase` to run. + :param handlers: A list of exception handlers (ExceptionType, handler) + where 'handler' is a callable that takes a `TestCase`, a + ``testtools.TestResult`` and the exception raised. + :param reactor: The Twisted reactor to use. If not given, we use the + default reactor. + :param timeout: The maximum time allowed for running a test. The + default is 0.005s. + :param debug: Whether or not to enable Twisted's debugging. Use this + to get information about unhandled Deferreds and left-over + DelayedCalls. Defaults to False. + """ + super(AsynchronousDeferredRunTest, self).__init__(case, handlers) + if reactor is None: + from twisted.internet import reactor + self._reactor = reactor + self._timeout = timeout + self._debug = debug + + @classmethod + def make_factory(cls, reactor=None, timeout=0.005, debug=False): + """Make a factory that conforms to the RunTest factory interface.""" + # This is horrible, but it means that the return value of the method + # will be able to be assigned to a class variable *and* also be + # invoked directly. + class AsynchronousDeferredRunTestFactory: + def __call__(self, case, handlers=None): + return cls(case, handlers, reactor, timeout, debug) + return AsynchronousDeferredRunTestFactory() + + @defer.deferredGenerator + def _run_cleanups(self): + """Run the cleanups on the test case. + + We expect that the cleanups on the test case can also return + asynchronous Deferreds. As such, we take the responsibility for + running the cleanups, rather than letting TestCase do it. + """ + while self.case._cleanups: + f, args, kwargs = self.case._cleanups.pop() + d = defer.maybeDeferred(f, *args, **kwargs) + thing = defer.waitForDeferred(d) + yield thing + try: + thing.getResult() + except Exception: + exc_info = sys.exc_info() + self.case._report_traceback(exc_info) + last_exception = exc_info[1] + yield last_exception + + def _make_spinner(self): + """Make the `Spinner` to be used to run the tests.""" + return Spinner(self._reactor, debug=self._debug) + + def _run_deferred(self): + """Run the test, assuming everything in it is Deferred-returning. + + This should return a Deferred that fires with True if the test was + successful and False if the test was not successful. It should *not* + call addSuccess on the result, because there's reactor clean up that + we needs to be done afterwards. + """ + fails = [] + + def fail_if_exception_caught(exception_caught): + if self.exception_caught == exception_caught: + fails.append(None) + + def clean_up(ignored=None): + """Run the cleanups.""" + d = self._run_cleanups() + def clean_up_done(result): + if result is not None: + self._exceptions.append(result) + fails.append(None) + return d.addCallback(clean_up_done) + + def set_up_done(exception_caught): + """Set up is done, either clean up or run the test.""" + if self.exception_caught == exception_caught: + fails.append(None) + return clean_up() + else: + d = self._run_user(self.case._run_test_method, self.result) + d.addCallback(fail_if_exception_caught) + d.addBoth(tear_down) + return d + + def tear_down(ignored): + d = self._run_user(self.case._run_teardown, self.result) + d.addCallback(fail_if_exception_caught) + d.addBoth(clean_up) + return d + + d = self._run_user(self.case._run_setup, self.result) + d.addCallback(set_up_done) + d.addBoth(lambda ignored: len(fails) == 0) + return d + + def _log_user_exception(self, e): + """Raise 'e' and report it as a user exception.""" + try: + raise e + except e.__class__: + self._got_user_exception(sys.exc_info()) + + def _blocking_run_deferred(self, spinner): + try: + return trap_unhandled_errors( + spinner.run, self._timeout, self._run_deferred) + except NoResultError: + # We didn't get a result at all! This could be for any number of + # reasons, but most likely someone hit Ctrl-C during the test. + raise KeyboardInterrupt + except TimeoutError: + # The function took too long to run. + self._log_user_exception(TimeoutError(self.case, self._timeout)) + return False, [] + + def _run_core(self): + # Add an observer to trap all logged errors. + self.case.reactor = self._reactor + error_observer = _log_observer + full_log = StringIO() + full_observer = log.FileLogObserver(full_log) + spinner = self._make_spinner() + successful, unhandled = run_with_log_observers( + [error_observer.gotEvent, full_observer.emit], + self._blocking_run_deferred, spinner) + + self.case.addDetail( + 'twisted-log', Content(UTF8_TEXT, full_log.readlines)) + + logged_errors = error_observer.flushErrors() + for logged_error in logged_errors: + successful = False + self._got_user_failure(logged_error, tb_label='logged-error') + + if unhandled: + successful = False + for debug_info in unhandled: + f = debug_info.failResult + info = debug_info._getDebugTracebacks() + if info: + self.case.addDetail( + 'unhandled-error-in-deferred-debug', + text_content(info)) + self._got_user_failure(f, 'unhandled-error-in-deferred') + + junk = spinner.clear_junk() + if junk: + successful = False + self._log_user_exception(UncleanReactorError(junk)) + + if successful: + self.result.addSuccess(self.case, details=self.case.getDetails()) + + def _run_user(self, function, *args): + """Run a user-supplied function. + + This just makes sure that it returns a Deferred, regardless of how the + user wrote it. + """ + d = defer.maybeDeferred(function, *args) + return d.addErrback(self._got_user_failure) + + +class AsynchronousDeferredRunTestForBrokenTwisted(AsynchronousDeferredRunTest): + """Test runner that works around Twisted brokenness re reactor junk. + + There are many APIs within Twisted itself where a Deferred fires but + leaves cleanup work scheduled for the reactor to do. Arguably, many of + these are bugs. This runner iterates the reactor event loop a number of + times after every test, in order to shake out these buggy-but-commonplace + events. + """ + + def _make_spinner(self): + spinner = super( + AsynchronousDeferredRunTestForBrokenTwisted, self)._make_spinner() + spinner._OBLIGATORY_REACTOR_ITERATIONS = 2 + return spinner + + +def assert_fails_with(d, *exc_types, **kwargs): + """Assert that 'd' will fail with one of 'exc_types'. + + The normal way to use this is to return the result of 'assert_fails_with' + from your unit test. + + Note that this function is experimental and unstable. Use at your own + peril; expect the API to change. + + :param d: A Deferred that is expected to fail. + :param exc_types: The exception types that the Deferred is expected to + fail with. + :param failureException: An optional keyword argument. If provided, will + raise that exception instead of + ``testtools.TestCase.failureException``. + :return: A Deferred that will fail with an ``AssertionError`` if 'd' does + not fail with one of the exception types. + """ + failureException = kwargs.pop('failureException', None) + if failureException is None: + # Avoid circular imports. + from testtools import TestCase + failureException = TestCase.failureException + expected_names = ", ".join(exc_type.__name__ for exc_type in exc_types) + def got_success(result): + raise failureException( + "%s not raised (%r returned)" % (expected_names, result)) + def got_failure(failure): + if failure.check(*exc_types): + return failure.value + raise failureException("%s raised instead of %s:\n %s" % ( + failure.type.__name__, expected_names, failure.getTraceback())) + return d.addCallbacks(got_success, got_failure) + + +def flush_logged_errors(*error_types): + return _log_observer.flushErrors(*error_types) + + +class UncleanReactorError(Exception): + """Raised when the reactor has junk in it.""" + + def __init__(self, junk): + Exception.__init__(self, + "The reactor still thinks it needs to do things. Close all " + "connections, kill all processes and make sure all delayed " + "calls have either fired or been cancelled:\n%s" + % ''.join(map(self._get_junk_info, junk))) + + def _get_junk_info(self, junk): + from twisted.internet.base import DelayedCall + if isinstance(junk, DelayedCall): + ret = str(junk) + else: + ret = repr(junk) + return ' %s\n' % (ret,) diff --git a/test/3rdparty/testtools-0.9.12/testtools/distutilscmd.py b/test/3rdparty/testtools-0.9.12/testtools/distutilscmd.py new file mode 100644 index 00000000000..91e14ca504f --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/distutilscmd.py @@ -0,0 +1,62 @@ +# Copyright (c) 2010-2011 testtools developers . See LICENSE for details. + +"""Extensions to the standard Python unittest library.""" + +import sys + +from distutils.core import Command +from distutils.errors import DistutilsOptionError + +from testtools.run import TestProgram, TestToolsTestRunner + + +class TestCommand(Command): + """Command to run unit tests with testtools""" + + description = "run unit tests with testtools" + + user_options = [ + ('catch', 'c', "Catch ctrl-C and display results so far"), + ('buffer', 'b', "Buffer stdout and stderr during tests"), + ('failfast', 'f', "Stop on first fail or error"), + ('test-module=','m', "Run 'test_suite' in specified module"), + ('test-suite=','s', + "Test suite to run (e.g. 'some_module.test_suite')") + ] + + def __init__(self, dist): + Command.__init__(self, dist) + self.runner = TestToolsTestRunner(sys.stdout) + + + def initialize_options(self): + self.test_suite = None + self.test_module = None + self.catch = None + self.buffer = None + self.failfast = None + + def finalize_options(self): + if self.test_suite is None: + if self.test_module is None: + raise DistutilsOptionError( + "You must specify a module or a suite to run tests from") + else: + self.test_suite = self.test_module+".test_suite" + elif self.test_module: + raise DistutilsOptionError( + "You may specify a module or a suite, but not both") + self.test_args = [self.test_suite] + if self.verbose: + self.test_args.insert(0, '--verbose') + if self.buffer: + self.test_args.insert(0, '--buffer') + if self.catch: + self.test_args.insert(0, '--catch') + if self.failfast: + self.test_args.insert(0, '--failfast') + + def run(self): + self.program = TestProgram( + argv=self.test_args, testRunner=self.runner, stdout=sys.stdout, + exit=False) diff --git a/test/3rdparty/testtools-0.9.12/testtools/helpers.py b/test/3rdparty/testtools-0.9.12/testtools/helpers.py new file mode 100644 index 00000000000..dbf66719edf --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/helpers.py @@ -0,0 +1,87 @@ +# Copyright (c) 2010-2011 testtools developers. See LICENSE for details. + +__all__ = [ + 'safe_hasattr', + 'try_import', + 'try_imports', + ] + +import sys + + +def try_import(name, alternative=None, error_callback=None): + """Attempt to import ``name``. If it fails, return ``alternative``. + + When supporting multiple versions of Python or optional dependencies, it + is useful to be able to try to import a module. + + :param name: The name of the object to import, e.g. ``os.path`` or + ``os.path.join``. + :param alternative: The value to return if no module can be imported. + Defaults to None. + :param error_callback: If non-None, a callable that is passed the ImportError + when the module cannot be loaded. + """ + module_segments = name.split('.') + last_error = None + while module_segments: + module_name = '.'.join(module_segments) + try: + module = __import__(module_name) + except ImportError: + last_error = sys.exc_info()[1] + module_segments.pop() + continue + else: + break + else: + if last_error is not None and error_callback is not None: + error_callback(last_error) + return alternative + nonexistent = object() + for segment in name.split('.')[1:]: + module = getattr(module, segment, nonexistent) + if module is nonexistent: + if last_error is not None and error_callback is not None: + error_callback(last_error) + return alternative + return module + + +_RAISE_EXCEPTION = object() +def try_imports(module_names, alternative=_RAISE_EXCEPTION, error_callback=None): + """Attempt to import modules. + + Tries to import the first module in ``module_names``. If it can be + imported, we return it. If not, we go on to the second module and try + that. The process continues until we run out of modules to try. If none + of the modules can be imported, either raise an exception or return the + provided ``alternative`` value. + + :param module_names: A sequence of module names to try to import. + :param alternative: The value to return if no module can be imported. + If unspecified, we raise an ImportError. + :param error_callback: If None, called with the ImportError for *each* + module that fails to load. + :raises ImportError: If none of the modules can be imported and no + alternative value was specified. + """ + module_names = list(module_names) + for module_name in module_names: + module = try_import(module_name, error_callback=error_callback) + if module: + return module + if alternative is _RAISE_EXCEPTION: + raise ImportError( + "Could not import any of: %s" % ', '.join(module_names)) + return alternative + + +def safe_hasattr(obj, attr, _marker=object()): + """Does 'obj' have an attribute 'attr'? + + Use this rather than built-in hasattr, as the built-in swallows exceptions + in some versions of Python and behaves unpredictably with respect to + properties. + """ + return getattr(obj, attr, _marker) is not _marker diff --git a/test/3rdparty/testtools-0.9.12/testtools/matchers.py b/test/3rdparty/testtools-0.9.12/testtools/matchers.py new file mode 100644 index 00000000000..693a20befa5 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/matchers.py @@ -0,0 +1,1059 @@ +# Copyright (c) 2009-2011 testtools developers. See LICENSE for details. + +"""Matchers, a way to express complex assertions outside the testcase. + +Inspired by 'hamcrest'. + +Matcher provides the abstract API that all matchers need to implement. + +Bundled matchers are listed in __all__: a list can be obtained by running +$ python -c 'import testtools.matchers; print testtools.matchers.__all__' +""" + +__metaclass__ = type +__all__ = [ + 'AfterPreprocessing', + 'AllMatch', + 'Annotate', + 'Contains', + 'DocTestMatches', + 'EndsWith', + 'Equals', + 'GreaterThan', + 'Is', + 'IsInstance', + 'KeysEqual', + 'LessThan', + 'MatchesAll', + 'MatchesAny', + 'MatchesException', + 'MatchesListwise', + 'MatchesRegex', + 'MatchesSetwise', + 'MatchesStructure', + 'NotEquals', + 'Not', + 'Raises', + 'raises', + 'StartsWith', + ] + +import doctest +import operator +from pprint import pformat +import re +import sys +import types + +from testtools.compat import ( + classtypes, + _error_repr, + isbaseexception, + _isbytes, + istext, + str_is_unicode, + text_repr + ) + + +class Matcher(object): + """A pattern matcher. + + A Matcher must implement match and __str__ to be used by + testtools.TestCase.assertThat. Matcher.match(thing) returns None when + thing is completely matched, and a Mismatch object otherwise. + + Matchers can be useful outside of test cases, as they are simply a + pattern matching language expressed as objects. + + testtools.matchers is inspired by hamcrest, but is pythonic rather than + a Java transcription. + """ + + def match(self, something): + """Return None if this matcher matches something, a Mismatch otherwise. + """ + raise NotImplementedError(self.match) + + def __str__(self): + """Get a sensible human representation of the matcher. + + This should include the parameters given to the matcher and any + state that would affect the matches operation. + """ + raise NotImplementedError(self.__str__) + + +class Mismatch(object): + """An object describing a mismatch detected by a Matcher.""" + + def __init__(self, description=None, details=None): + """Construct a `Mismatch`. + + :param description: A description to use. If not provided, + `Mismatch.describe` must be implemented. + :param details: Extra details about the mismatch. Defaults + to the empty dict. + """ + if description: + self._description = description + if details is None: + details = {} + self._details = details + + def describe(self): + """Describe the mismatch. + + This should be either a human-readable string or castable to a string. + In particular, is should either be plain ascii or unicode on Python 2, + and care should be taken to escape control characters. + """ + try: + return self._description + except AttributeError: + raise NotImplementedError(self.describe) + + def get_details(self): + """Get extra details about the mismatch. + + This allows the mismatch to provide extra information beyond the basic + description, including large text or binary files, or debugging internals + without having to force it to fit in the output of 'describe'. + + The testtools assertion assertThat will query get_details and attach + all its values to the test, permitting them to be reported in whatever + manner the test environment chooses. + + :return: a dict mapping names to Content objects. name is a string to + name the detail, and the Content object is the detail to add + to the result. For more information see the API to which items from + this dict are passed testtools.TestCase.addDetail. + """ + return getattr(self, '_details', {}) + + def __repr__(self): + return "<testtools.matchers.Mismatch object at %x attributes=%r>" % ( + id(self), self.__dict__) + + +class MismatchError(AssertionError): + """Raised when a mismatch occurs.""" + + # This class exists to work around + # <https://bugs.launchpad.net/testtools/+bug/804127>. It provides a + # guaranteed way of getting a readable exception, no matter what crazy + # characters are in the matchee, matcher or mismatch. + + def __init__(self, matchee, matcher, mismatch, verbose=False): + # Have to use old-style upcalling for Python 2.4 and 2.5 + # compatibility. + AssertionError.__init__(self) + self.matchee = matchee + self.matcher = matcher + self.mismatch = mismatch + self.verbose = verbose + + def __str__(self): + difference = self.mismatch.describe() + if self.verbose: + # GZ 2011-08-24: Smelly API? Better to take any object and special + # case text inside? + if istext(self.matchee) or _isbytes(self.matchee): + matchee = text_repr(self.matchee, multiline=False) + else: + matchee = repr(self.matchee) + return ( + 'Match failed. Matchee: %s\nMatcher: %s\nDifference: %s\n' + % (matchee, self.matcher, difference)) + else: + return difference + + if not str_is_unicode: + + __unicode__ = __str__ + + def __str__(self): + return self.__unicode__().encode("ascii", "backslashreplace") + + +class MismatchDecorator(object): + """Decorate a ``Mismatch``. + + Forwards all messages to the original mismatch object. Probably the best + way to use this is inherit from this class and then provide your own + custom decoration logic. + """ + + def __init__(self, original): + """Construct a `MismatchDecorator`. + + :param original: A `Mismatch` object to decorate. + """ + self.original = original + + def __repr__(self): + return '<testtools.matchers.MismatchDecorator(%r)>' % (self.original,) + + def describe(self): + return self.original.describe() + + def get_details(self): + return self.original.get_details() + + +class _NonManglingOutputChecker(doctest.OutputChecker): + """Doctest checker that works with unicode rather than mangling strings + + This is needed because current Python versions have tried to fix string + encoding related problems, but regressed the default behaviour with unicode + inputs in the process. + + In Python 2.6 and 2.7 `OutputChecker.output_difference` is was changed to + return a bytestring encoded as per `sys.stdout.encoding`, or utf-8 if that + can't be determined. Worse, that encoding process happens in the innocent + looking `_indent` global function. Because the `DocTestMismatch.describe` + result may well not be destined for printing to stdout, this is no good + for us. To get a unicode return as before, the method is monkey patched if + `doctest._encoding` exists. + + Python 3 has a different problem. For some reason both inputs are encoded + to ascii with 'backslashreplace', making an escaped string matches its + unescaped form. Overriding the offending `OutputChecker._toAscii` method + is sufficient to revert this. + """ + + def _toAscii(self, s): + """Return `s` unchanged rather than mangling it to ascii""" + return s + + # Only do this overriding hackery if doctest has a broken _input function + if getattr(doctest, "_encoding", None) is not None: + from types import FunctionType as __F + __f = doctest.OutputChecker.output_difference.im_func + __g = dict(__f.func_globals) + def _indent(s, indent=4, _pattern=re.compile("^(?!$)", re.MULTILINE)): + """Prepend non-empty lines in `s` with `indent` number of spaces""" + return _pattern.sub(indent*" ", s) + __g["_indent"] = _indent + output_difference = __F(__f.func_code, __g, "output_difference") + del __F, __f, __g, _indent + + +class DocTestMatches(object): + """See if a string matches a doctest example.""" + + def __init__(self, example, flags=0): + """Create a DocTestMatches to match example. + + :param example: The example to match e.g. 'foo bar baz' + :param flags: doctest comparison flags to match on. e.g. + doctest.ELLIPSIS. + """ + if not example.endswith('\n'): + example += '\n' + self.want = example # required variable name by doctest. + self.flags = flags + self._checker = _NonManglingOutputChecker() + + def __str__(self): + if self.flags: + flagstr = ", flags=%d" % self.flags + else: + flagstr = "" + return 'DocTestMatches(%r%s)' % (self.want, flagstr) + + def _with_nl(self, actual): + result = self.want.__class__(actual) + if not result.endswith('\n'): + result += '\n' + return result + + def match(self, actual): + with_nl = self._with_nl(actual) + if self._checker.check_output(self.want, with_nl, self.flags): + return None + return DocTestMismatch(self, with_nl) + + def _describe_difference(self, with_nl): + return self._checker.output_difference(self, with_nl, self.flags) + + +class DocTestMismatch(Mismatch): + """Mismatch object for DocTestMatches.""" + + def __init__(self, matcher, with_nl): + self.matcher = matcher + self.with_nl = with_nl + + def describe(self): + s = self.matcher._describe_difference(self.with_nl) + if str_is_unicode or isinstance(s, unicode): + return s + # GZ 2011-08-24: This is actually pretty bogus, most C0 codes should + # be escaped, in addition to non-ascii bytes. + return s.decode("latin1").encode("ascii", "backslashreplace") + + +class DoesNotContain(Mismatch): + + def __init__(self, matchee, needle): + """Create a DoesNotContain Mismatch. + + :param matchee: the object that did not contain needle. + :param needle: the needle that 'matchee' was expected to contain. + """ + self.matchee = matchee + self.needle = needle + + def describe(self): + return "%r not in %r" % (self.needle, self.matchee) + + +class DoesNotStartWith(Mismatch): + + def __init__(self, matchee, expected): + """Create a DoesNotStartWith Mismatch. + + :param matchee: the string that did not match. + :param expected: the string that 'matchee' was expected to start with. + """ + self.matchee = matchee + self.expected = expected + + def describe(self): + return "%s does not start with %s." % ( + text_repr(self.matchee), text_repr(self.expected)) + + +class DoesNotEndWith(Mismatch): + + def __init__(self, matchee, expected): + """Create a DoesNotEndWith Mismatch. + + :param matchee: the string that did not match. + :param expected: the string that 'matchee' was expected to end with. + """ + self.matchee = matchee + self.expected = expected + + def describe(self): + return "%s does not end with %s." % ( + text_repr(self.matchee), text_repr(self.expected)) + + +class _BinaryComparison(object): + """Matcher that compares an object to another object.""" + + def __init__(self, expected): + self.expected = expected + + def __str__(self): + return "%s(%r)" % (self.__class__.__name__, self.expected) + + def match(self, other): + if self.comparator(other, self.expected): + return None + return _BinaryMismatch(self.expected, self.mismatch_string, other) + + def comparator(self, expected, other): + raise NotImplementedError(self.comparator) + + +class _BinaryMismatch(Mismatch): + """Two things did not match.""" + + def __init__(self, expected, mismatch_string, other): + self.expected = expected + self._mismatch_string = mismatch_string + self.other = other + + def _format(self, thing): + # Blocks of text with newlines are formatted as triple-quote + # strings. Everything else is pretty-printed. + if istext(thing) or _isbytes(thing): + return text_repr(thing) + return pformat(thing) + + def describe(self): + left = repr(self.expected) + right = repr(self.other) + if len(left) + len(right) > 70: + return "%s:\nreference = %s\nactual = %s\n" % ( + self._mismatch_string, self._format(self.expected), + self._format(self.other)) + else: + return "%s %s %s" % (left, self._mismatch_string, right) + + +class Equals(_BinaryComparison): + """Matches if the items are equal.""" + + comparator = operator.eq + mismatch_string = '!=' + + +class NotEquals(_BinaryComparison): + """Matches if the items are not equal. + + In most cases, this is equivalent to ``Not(Equals(foo))``. The difference + only matters when testing ``__ne__`` implementations. + """ + + comparator = operator.ne + mismatch_string = '==' + + +class Is(_BinaryComparison): + """Matches if the items are identical.""" + + comparator = operator.is_ + mismatch_string = 'is not' + + +class IsInstance(object): + """Matcher that wraps isinstance.""" + + def __init__(self, *types): + self.types = tuple(types) + + def __str__(self): + return "%s(%s)" % (self.__class__.__name__, + ', '.join(type.__name__ for type in self.types)) + + def match(self, other): + if isinstance(other, self.types): + return None + return NotAnInstance(other, self.types) + + +class NotAnInstance(Mismatch): + + def __init__(self, matchee, types): + """Create a NotAnInstance Mismatch. + + :param matchee: the thing which is not an instance of any of types. + :param types: A tuple of the types which were expected. + """ + self.matchee = matchee + self.types = types + + def describe(self): + if len(self.types) == 1: + typestr = self.types[0].__name__ + else: + typestr = 'any of (%s)' % ', '.join(type.__name__ for type in + self.types) + return "'%s' is not an instance of %s" % (self.matchee, typestr) + + +class LessThan(_BinaryComparison): + """Matches if the item is less than the matchers reference object.""" + + comparator = operator.__lt__ + mismatch_string = 'is not >' + + +class GreaterThan(_BinaryComparison): + """Matches if the item is greater than the matchers reference object.""" + + comparator = operator.__gt__ + mismatch_string = 'is not <' + + +class MatchesAny(object): + """Matches if any of the matchers it is created with match.""" + + def __init__(self, *matchers): + self.matchers = matchers + + def match(self, matchee): + results = [] + for matcher in self.matchers: + mismatch = matcher.match(matchee) + if mismatch is None: + return None + results.append(mismatch) + return MismatchesAll(results) + + def __str__(self): + return "MatchesAny(%s)" % ', '.join([ + str(matcher) for matcher in self.matchers]) + + +class MatchesAll(object): + """Matches if all of the matchers it is created with match.""" + + def __init__(self, *matchers): + self.matchers = matchers + + def __str__(self): + return 'MatchesAll(%s)' % ', '.join(map(str, self.matchers)) + + def match(self, matchee): + results = [] + for matcher in self.matchers: + mismatch = matcher.match(matchee) + if mismatch is not None: + results.append(mismatch) + if results: + return MismatchesAll(results) + else: + return None + + +class MismatchesAll(Mismatch): + """A mismatch with many child mismatches.""" + + def __init__(self, mismatches): + self.mismatches = mismatches + + def describe(self): + descriptions = ["Differences: ["] + for mismatch in self.mismatches: + descriptions.append(mismatch.describe()) + descriptions.append("]") + return '\n'.join(descriptions) + + +class Not(object): + """Inverts a matcher.""" + + def __init__(self, matcher): + self.matcher = matcher + + def __str__(self): + return 'Not(%s)' % (self.matcher,) + + def match(self, other): + mismatch = self.matcher.match(other) + if mismatch is None: + return MatchedUnexpectedly(self.matcher, other) + else: + return None + + +class MatchedUnexpectedly(Mismatch): + """A thing matched when it wasn't supposed to.""" + + def __init__(self, matcher, other): + self.matcher = matcher + self.other = other + + def describe(self): + return "%r matches %s" % (self.other, self.matcher) + + +class MatchesException(Matcher): + """Match an exc_info tuple against an exception instance or type.""" + + def __init__(self, exception, value_re=None): + """Create a MatchesException that will match exc_info's for exception. + + :param exception: Either an exception instance or type. + If an instance is given, the type and arguments of the exception + are checked. If a type is given only the type of the exception is + checked. If a tuple is given, then as with isinstance, any of the + types in the tuple matching is sufficient to match. + :param value_re: If 'exception' is a type, and the matchee exception + is of the right type, then match against this. If value_re is a + string, then assume value_re is a regular expression and match + the str() of the exception against it. Otherwise, assume value_re + is a matcher, and match the exception against it. + """ + Matcher.__init__(self) + self.expected = exception + if istext(value_re): + value_re = AfterPreproccessing(str, MatchesRegex(value_re), False) + self.value_re = value_re + self._is_instance = type(self.expected) not in classtypes() + (tuple,) + + def match(self, other): + if type(other) != tuple: + return Mismatch('%r is not an exc_info tuple' % other) + expected_class = self.expected + if self._is_instance: + expected_class = expected_class.__class__ + if not issubclass(other[0], expected_class): + return Mismatch('%r is not a %r' % (other[0], expected_class)) + if self._is_instance: + if other[1].args != self.expected.args: + return Mismatch('%s has different arguments to %s.' % ( + _error_repr(other[1]), _error_repr(self.expected))) + elif self.value_re is not None: + return self.value_re.match(other[1]) + + def __str__(self): + if self._is_instance: + return "MatchesException(%s)" % _error_repr(self.expected) + return "MatchesException(%s)" % repr(self.expected) + + +class Contains(Matcher): + """Checks whether something is container in another thing.""" + + def __init__(self, needle): + """Create a Contains Matcher. + + :param needle: the thing that needs to be contained by matchees. + """ + self.needle = needle + + def __str__(self): + return "Contains(%r)" % (self.needle,) + + def match(self, matchee): + try: + if self.needle not in matchee: + return DoesNotContain(matchee, self.needle) + except TypeError: + # e.g. 1 in 2 will raise TypeError + return DoesNotContain(matchee, self.needle) + return None + + +class StartsWith(Matcher): + """Checks whether one string starts with another.""" + + def __init__(self, expected): + """Create a StartsWith Matcher. + + :param expected: the string that matchees should start with. + """ + self.expected = expected + + def __str__(self): + return "StartsWith(%r)" % (self.expected,) + + def match(self, matchee): + if not matchee.startswith(self.expected): + return DoesNotStartWith(matchee, self.expected) + return None + + +class EndsWith(Matcher): + """Checks whether one string starts with another.""" + + def __init__(self, expected): + """Create a EndsWith Matcher. + + :param expected: the string that matchees should end with. + """ + self.expected = expected + + def __str__(self): + return "EndsWith(%r)" % (self.expected,) + + def match(self, matchee): + if not matchee.endswith(self.expected): + return DoesNotEndWith(matchee, self.expected) + return None + + +class KeysEqual(Matcher): + """Checks whether a dict has particular keys.""" + + def __init__(self, *expected): + """Create a `KeysEqual` Matcher. + + :param expected: The keys the dict is expected to have. If a dict, + then we use the keys of that dict, if a collection, we assume it + is a collection of expected keys. + """ + try: + self.expected = expected.keys() + except AttributeError: + self.expected = list(expected) + + def __str__(self): + return "KeysEqual(%s)" % ', '.join(map(repr, self.expected)) + + def match(self, matchee): + expected = sorted(self.expected) + matched = Equals(expected).match(sorted(matchee.keys())) + if matched: + return AnnotatedMismatch( + 'Keys not equal', + _BinaryMismatch(expected, 'does not match', matchee)) + return None + + +class Annotate(object): + """Annotates a matcher with a descriptive string. + + Mismatches are then described as '<mismatch>: <annotation>'. + """ + + def __init__(self, annotation, matcher): + self.annotation = annotation + self.matcher = matcher + + @classmethod + def if_message(cls, annotation, matcher): + """Annotate ``matcher`` only if ``annotation`` is non-empty.""" + if not annotation: + return matcher + return cls(annotation, matcher) + + def __str__(self): + return 'Annotate(%r, %s)' % (self.annotation, self.matcher) + + def match(self, other): + mismatch = self.matcher.match(other) + if mismatch is not None: + return AnnotatedMismatch(self.annotation, mismatch) + + +class AnnotatedMismatch(MismatchDecorator): + """A mismatch annotated with a descriptive string.""" + + def __init__(self, annotation, mismatch): + super(AnnotatedMismatch, self).__init__(mismatch) + self.annotation = annotation + self.mismatch = mismatch + + def describe(self): + return '%s: %s' % (self.original.describe(), self.annotation) + + +class Raises(Matcher): + """Match if the matchee raises an exception when called. + + Exceptions which are not subclasses of Exception propogate out of the + Raises.match call unless they are explicitly matched. + """ + + def __init__(self, exception_matcher=None): + """Create a Raises matcher. + + :param exception_matcher: Optional validator for the exception raised + by matchee. If supplied the exc_info tuple for the exception raised + is passed into that matcher. If no exception_matcher is supplied + then the simple fact of raising an exception is considered enough + to match on. + """ + self.exception_matcher = exception_matcher + + def match(self, matchee): + try: + result = matchee() + return Mismatch('%r returned %r' % (matchee, result)) + # Catch all exceptions: Raises() should be able to match a + # KeyboardInterrupt or SystemExit. + except: + exc_info = sys.exc_info() + if self.exception_matcher: + mismatch = self.exception_matcher.match(exc_info) + if not mismatch: + del exc_info + return + else: + mismatch = None + # The exception did not match, or no explicit matching logic was + # performed. If the exception is a non-user exception (that is, not + # a subclass of Exception on Python 2.5+) then propogate it. + if isbaseexception(exc_info[1]): + del exc_info + raise + return mismatch + + def __str__(self): + return 'Raises()' + + +def raises(exception): + """Make a matcher that checks that a callable raises an exception. + + This is a convenience function, exactly equivalent to:: + + return Raises(MatchesException(exception)) + + See `Raises` and `MatchesException` for more information. + """ + return Raises(MatchesException(exception)) + + +class MatchesListwise(object): + """Matches if each matcher matches the corresponding value. + + More easily explained by example than in words: + + >>> MatchesListwise([Equals(1)]).match([1]) + >>> MatchesListwise([Equals(1), Equals(2)]).match([1, 2]) + >>> print (MatchesListwise([Equals(1), Equals(2)]).match([2, 1]).describe()) + Differences: [ + 1 != 2 + 2 != 1 + ] + """ + + def __init__(self, matchers): + self.matchers = matchers + + def match(self, values): + mismatches = [] + length_mismatch = Annotate( + "Length mismatch", Equals(len(self.matchers))).match(len(values)) + if length_mismatch: + mismatches.append(length_mismatch) + for matcher, value in zip(self.matchers, values): + mismatch = matcher.match(value) + if mismatch: + mismatches.append(mismatch) + if mismatches: + return MismatchesAll(mismatches) + + +class MatchesStructure(object): + """Matcher that matches an object structurally. + + 'Structurally' here means that attributes of the object being matched are + compared against given matchers. + + `fromExample` allows the creation of a matcher from a prototype object and + then modified versions can be created with `update`. + + `byEquality` creates a matcher in much the same way as the constructor, + except that the matcher for each of the attributes is assumed to be + `Equals`. + + `byMatcher` creates a similar matcher to `byEquality`, but you get to pick + the matcher, rather than just using `Equals`. + """ + + def __init__(self, **kwargs): + """Construct a `MatchesStructure`. + + :param kwargs: A mapping of attributes to matchers. + """ + self.kws = kwargs + + @classmethod + def byEquality(cls, **kwargs): + """Matches an object where the attributes equal the keyword values. + + Similar to the constructor, except that the matcher is assumed to be + Equals. + """ + return cls.byMatcher(Equals, **kwargs) + + @classmethod + def byMatcher(cls, matcher, **kwargs): + """Matches an object where the attributes match the keyword values. + + Similar to the constructor, except that the provided matcher is used + to match all of the values. + """ + return cls( + **dict((name, matcher(value)) for name, value in kwargs.items())) + + @classmethod + def fromExample(cls, example, *attributes): + kwargs = {} + for attr in attributes: + kwargs[attr] = Equals(getattr(example, attr)) + return cls(**kwargs) + + def update(self, **kws): + new_kws = self.kws.copy() + for attr, matcher in kws.items(): + if matcher is None: + new_kws.pop(attr, None) + else: + new_kws[attr] = matcher + return type(self)(**new_kws) + + def __str__(self): + kws = [] + for attr, matcher in sorted(self.kws.items()): + kws.append("%s=%s" % (attr, matcher)) + return "%s(%s)" % (self.__class__.__name__, ', '.join(kws)) + + def match(self, value): + matchers = [] + values = [] + for attr, matcher in sorted(self.kws.items()): + matchers.append(Annotate(attr, matcher)) + values.append(getattr(value, attr)) + return MatchesListwise(matchers).match(values) + + +class MatchesRegex(object): + """Matches if the matchee is matched by a regular expression.""" + + def __init__(self, pattern, flags=0): + self.pattern = pattern + self.flags = flags + + def __str__(self): + args = ['%r' % self.pattern] + flag_arg = [] + # dir() sorts the attributes for us, so we don't need to do it again. + for flag in dir(re): + if len(flag) == 1: + if self.flags & getattr(re, flag): + flag_arg.append('re.%s' % flag) + if flag_arg: + args.append('|'.join(flag_arg)) + return '%s(%s)' % (self.__class__.__name__, ', '.join(args)) + + def match(self, value): + if not re.match(self.pattern, value, self.flags): + pattern = self.pattern + if not isinstance(pattern, str_is_unicode and str or unicode): + pattern = pattern.decode("latin1") + pattern = pattern.encode("unicode_escape").decode("ascii") + return Mismatch("%r does not match /%s/" % ( + value, pattern.replace("\\\\", "\\"))) + + +class MatchesSetwise(object): + """Matches if all the matchers match elements of the value being matched. + + That is, each element in the 'observed' set must match exactly one matcher + from the set of matchers, with no matchers left over. + + The difference compared to `MatchesListwise` is that the order of the + matchings does not matter. + """ + + def __init__(self, *matchers): + self.matchers = matchers + + def match(self, observed): + remaining_matchers = set(self.matchers) + not_matched = [] + for value in observed: + for matcher in remaining_matchers: + if matcher.match(value) is None: + remaining_matchers.remove(matcher) + break + else: + not_matched.append(value) + if not_matched or remaining_matchers: + remaining_matchers = list(remaining_matchers) + # There are various cases that all should be reported somewhat + # differently. + + # There are two trivial cases: + # 1) There are just some matchers left over. + # 2) There are just some values left over. + + # Then there are three more interesting cases: + # 3) There are the same number of matchers and values left over. + # 4) There are more matchers left over than values. + # 5) There are more values left over than matchers. + + if len(not_matched) == 0: + if len(remaining_matchers) > 1: + msg = "There were %s matchers left over: " % ( + len(remaining_matchers),) + else: + msg = "There was 1 matcher left over: " + msg += ', '.join(map(str, remaining_matchers)) + return Mismatch(msg) + elif len(remaining_matchers) == 0: + if len(not_matched) > 1: + return Mismatch( + "There were %s values left over: %s" % ( + len(not_matched), not_matched)) + else: + return Mismatch( + "There was 1 value left over: %s" % ( + not_matched, )) + else: + common_length = min(len(remaining_matchers), len(not_matched)) + if common_length == 0: + raise AssertionError("common_length can't be 0 here") + if common_length > 1: + msg = "There were %s mismatches" % (common_length,) + else: + msg = "There was 1 mismatch" + if len(remaining_matchers) > len(not_matched): + extra_matchers = remaining_matchers[common_length:] + msg += " and %s extra matcher" % (len(extra_matchers), ) + if len(extra_matchers) > 1: + msg += "s" + msg += ': ' + ', '.join(map(str, extra_matchers)) + elif len(not_matched) > len(remaining_matchers): + extra_values = not_matched[common_length:] + msg += " and %s extra value" % (len(extra_values), ) + if len(extra_values) > 1: + msg += "s" + msg += ': ' + str(extra_values) + return Annotate( + msg, MatchesListwise(remaining_matchers[:common_length]) + ).match(not_matched[:common_length]) + + +class AfterPreprocessing(object): + """Matches if the value matches after passing through a function. + + This can be used to aid in creating trivial matchers as functions, for + example:: + + def PathHasFileContent(content): + def _read(path): + return open(path).read() + return AfterPreprocessing(_read, Equals(content)) + """ + + def __init__(self, preprocessor, matcher, annotate=True): + """Create an AfterPreprocessing matcher. + + :param preprocessor: A function called with the matchee before + matching. + :param matcher: What to match the preprocessed matchee against. + :param annotate: Whether or not to annotate the matcher with + something explaining how we transformed the matchee. Defaults + to True. + """ + self.preprocessor = preprocessor + self.matcher = matcher + self.annotate = annotate + + def _str_preprocessor(self): + if isinstance(self.preprocessor, types.FunctionType): + return '<function %s>' % self.preprocessor.__name__ + return str(self.preprocessor) + + def __str__(self): + return "AfterPreprocessing(%s, %s)" % ( + self._str_preprocessor(), self.matcher) + + def match(self, value): + after = self.preprocessor(value) + if self.annotate: + matcher = Annotate( + "after %s on %r" % (self._str_preprocessor(), value), + self.matcher) + else: + matcher = self.matcher + return matcher.match(after) + +# This is the old, deprecated. spelling of the name, kept for backwards +# compatibility. +AfterPreproccessing = AfterPreprocessing + + +class AllMatch(object): + """Matches if all provided values match the given matcher.""" + + def __init__(self, matcher): + self.matcher = matcher + + def __str__(self): + return 'AllMatch(%s)' % (self.matcher,) + + def match(self, values): + mismatches = [] + for value in values: + mismatch = self.matcher.match(value) + if mismatch: + mismatches.append(mismatch) + if mismatches: + return MismatchesAll(mismatches) + + +# Signal that this is part of the testing framework, and that code from this +# should not normally appear in tracebacks. +__unittest = True diff --git a/test/3rdparty/testtools-0.9.12/testtools/monkey.py b/test/3rdparty/testtools-0.9.12/testtools/monkey.py new file mode 100644 index 00000000000..ba0ac8fd8bf --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/monkey.py @@ -0,0 +1,97 @@ +# Copyright (c) 2010 testtools developers. See LICENSE for details. + +"""Helpers for monkey-patching Python code.""" + +__all__ = [ + 'MonkeyPatcher', + 'patch', + ] + + +class MonkeyPatcher(object): + """A set of monkey-patches that can be applied and removed all together. + + Use this to cover up attributes with new objects. Particularly useful for + testing difficult code. + """ + + # Marker used to indicate that the patched attribute did not exist on the + # object before we patched it. + _NO_SUCH_ATTRIBUTE = object() + + def __init__(self, *patches): + """Construct a `MonkeyPatcher`. + + :param patches: The patches to apply, each should be (obj, name, + new_value). Providing patches here is equivalent to calling + `add_patch`. + """ + # List of patches to apply in (obj, name, value). + self._patches_to_apply = [] + # List of the original values for things that have been patched. + # (obj, name, value) format. + self._originals = [] + for patch in patches: + self.add_patch(*patch) + + def add_patch(self, obj, name, value): + """Add a patch to overwrite 'name' on 'obj' with 'value'. + + The attribute C{name} on C{obj} will be assigned to C{value} when + C{patch} is called or during C{run_with_patches}. + + You can restore the original values with a call to restore(). + """ + self._patches_to_apply.append((obj, name, value)) + + def patch(self): + """Apply all of the patches that have been specified with `add_patch`. + + Reverse this operation using L{restore}. + """ + for obj, name, value in self._patches_to_apply: + original_value = getattr(obj, name, self._NO_SUCH_ATTRIBUTE) + self._originals.append((obj, name, original_value)) + setattr(obj, name, value) + + def restore(self): + """Restore all original values to any patched objects. + + If the patched attribute did not exist on an object before it was + patched, `restore` will delete the attribute so as to return the + object to its original state. + """ + while self._originals: + obj, name, value = self._originals.pop() + if value is self._NO_SUCH_ATTRIBUTE: + delattr(obj, name) + else: + setattr(obj, name, value) + + def run_with_patches(self, f, *args, **kw): + """Run 'f' with the given args and kwargs with all patches applied. + + Restores all objects to their original state when finished. + """ + self.patch() + try: + return f(*args, **kw) + finally: + self.restore() + + +def patch(obj, attribute, value): + """Set 'obj.attribute' to 'value' and return a callable to restore 'obj'. + + If 'attribute' is not set on 'obj' already, then the returned callable + will delete the attribute when called. + + :param obj: An object to monkey-patch. + :param attribute: The name of the attribute to patch. + :param value: The value to set 'obj.attribute' to. + :return: A nullary callable that, when run, will restore 'obj' to its + original state. + """ + patcher = MonkeyPatcher((obj, attribute, value)) + patcher.patch() + return patcher.restore diff --git a/test/3rdparty/testtools-0.9.12/testtools/run.py b/test/3rdparty/testtools-0.9.12/testtools/run.py new file mode 100755 index 00000000000..72011c74cab --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/run.py @@ -0,0 +1,332 @@ +# Copyright (c) 2009 testtools developers. See LICENSE for details. + +"""python -m testtools.run testspec [testspec...] + +Run some tests with the testtools extended API. + +For instance, to run the testtools test suite. + $ python -m testtools.run testtools.tests.test_suite +""" + +import os +import unittest +import sys + +from testtools import TextTestResult +from testtools.compat import classtypes, istext, unicode_output_stream +from testtools.testsuite import iterate_tests + + +defaultTestLoader = unittest.defaultTestLoader +defaultTestLoaderCls = unittest.TestLoader + +if getattr(defaultTestLoader, 'discover', None) is None: + try: + import discover + defaultTestLoader = discover.DiscoveringTestLoader() + defaultTestLoaderCls = discover.DiscoveringTestLoader + have_discover = True + except ImportError: + have_discover = False +else: + have_discover = True + + +class TestToolsTestRunner(object): + """ A thunk object to support unittest.TestProgram.""" + + def __init__(self, stdout): + self.stdout = stdout + + def run(self, test): + "Run the given test case or test suite." + result = TextTestResult(unicode_output_stream(self.stdout)) + result.startTestRun() + try: + return test.run(result) + finally: + result.stopTestRun() + + +#################### +# Taken from python 2.7 and slightly modified for compatibility with +# older versions. Delete when 2.7 is the oldest supported version. +# Modifications: +# - Use have_discover to raise an error if the user tries to use +# discovery on an old version and doesn't have discover installed. +# - If --catch is given check that installHandler is available, as +# it won't be on old python versions. +# - print calls have been been made single-source python3 compatibile. +# - exception handling likewise. +# - The default help has been changed to USAGE_AS_MAIN and USAGE_FROM_MODULE +# removed. +# - A tweak has been added to detect 'python -m *.run' and use a +# better progName in that case. +# - self.module is more comprehensively set to None when being invoked from +# the commandline - __name__ is used as a sentinel value. +# - --list has been added which can list tests (should be upstreamed). +# - --load-list has been added which can reduce the tests used (should be +# upstreamed). +# - The limitation of using getopt is declared to the user. + +FAILFAST = " -f, --failfast Stop on first failure\n" +CATCHBREAK = " -c, --catch Catch control-C and display results\n" +BUFFEROUTPUT = " -b, --buffer Buffer stdout and stderr during test runs\n" + +USAGE_AS_MAIN = """\ +Usage: %(progName)s [options] [tests] + +Options: + -h, --help Show this message + -v, --verbose Verbose output + -q, --quiet Minimal output + -l, --list List tests rather than executing them. + --load-list Specifies a file containing test ids, only tests matching + those ids are executed. +%(failfast)s%(catchbreak)s%(buffer)s +Examples: + %(progName)s test_module - run tests from test_module + %(progName)s module.TestClass - run tests from module.TestClass + %(progName)s module.Class.test_method - run specified test method + +All options must come before [tests]. [tests] can be a list of any number of +test modules, classes and test methods. + +Alternative Usage: %(progName)s discover [options] + +Options: + -v, --verbose Verbose output +%(failfast)s%(catchbreak)s%(buffer)s -s directory Directory to start discovery ('.' default) + -p pattern Pattern to match test files ('test*.py' default) + -t directory Top level directory of project (default to + start directory) + -l, --list List tests rather than executing them. + --load-list Specifies a file containing test ids, only tests matching + those ids are executed. + +For test discovery all test modules must be importable from the top +level directory of the project. +""" + + +class TestProgram(object): + """A command-line program that runs a set of tests; this is primarily + for making test modules conveniently executable. + """ + USAGE = USAGE_AS_MAIN + + # defaults for testing + failfast = catchbreak = buffer = progName = None + + def __init__(self, module=__name__, defaultTest=None, argv=None, + testRunner=None, testLoader=defaultTestLoader, + exit=True, verbosity=1, failfast=None, catchbreak=None, + buffer=None, stdout=None): + if module == __name__: + self.module = None + elif istext(module): + self.module = __import__(module) + for part in module.split('.')[1:]: + self.module = getattr(self.module, part) + else: + self.module = module + if argv is None: + argv = sys.argv + if stdout is None: + stdout = sys.stdout + + self.exit = exit + self.failfast = failfast + self.catchbreak = catchbreak + self.verbosity = verbosity + self.buffer = buffer + self.defaultTest = defaultTest + self.listtests = False + self.load_list = None + self.testRunner = testRunner + self.testLoader = testLoader + progName = argv[0] + if progName.endswith('%srun.py' % os.path.sep): + elements = progName.split(os.path.sep) + progName = '%s.run' % elements[-2] + else: + progName = os.path.basename(argv[0]) + self.progName = progName + self.parseArgs(argv) + if self.load_list: + # TODO: preserve existing suites (like testresources does in + # OptimisingTestSuite.add, but with a standard protocol). + # This is needed because the load_tests hook allows arbitrary + # suites, even if that is rarely used. + source = open(self.load_list, 'rb') + try: + lines = source.readlines() + finally: + source.close() + test_ids = set(line.strip().decode('utf-8') for line in lines) + filtered = unittest.TestSuite() + for test in iterate_tests(self.test): + if test.id() in test_ids: + filtered.addTest(test) + self.test = filtered + if not self.listtests: + self.runTests() + else: + for test in iterate_tests(self.test): + stdout.write('%s\n' % test.id()) + + def usageExit(self, msg=None): + if msg: + print(msg) + usage = {'progName': self.progName, 'catchbreak': '', 'failfast': '', + 'buffer': ''} + if self.failfast != False: + usage['failfast'] = FAILFAST + if self.catchbreak != False: + usage['catchbreak'] = CATCHBREAK + if self.buffer != False: + usage['buffer'] = BUFFEROUTPUT + print(self.USAGE % usage) + sys.exit(2) + + def parseArgs(self, argv): + if len(argv) > 1 and argv[1].lower() == 'discover': + self._do_discovery(argv[2:]) + return + + import getopt + long_opts = ['help', 'verbose', 'quiet', 'failfast', 'catch', 'buffer', + 'list', 'load-list='] + try: + options, args = getopt.getopt(argv[1:], 'hHvqfcbl', long_opts) + for opt, value in options: + if opt in ('-h','-H','--help'): + self.usageExit() + if opt in ('-q','--quiet'): + self.verbosity = 0 + if opt in ('-v','--verbose'): + self.verbosity = 2 + if opt in ('-f','--failfast'): + if self.failfast is None: + self.failfast = True + # Should this raise an exception if -f is not valid? + if opt in ('-c','--catch'): + if self.catchbreak is None: + self.catchbreak = True + # Should this raise an exception if -c is not valid? + if opt in ('-b','--buffer'): + if self.buffer is None: + self.buffer = True + # Should this raise an exception if -b is not valid? + if opt in ('-l', '--list'): + self.listtests = True + if opt == '--load-list': + self.load_list = value + if len(args) == 0 and self.defaultTest is None: + # createTests will load tests from self.module + self.testNames = None + elif len(args) > 0: + self.testNames = args + else: + self.testNames = (self.defaultTest,) + self.createTests() + except getopt.error: + self.usageExit(sys.exc_info()[1]) + + def createTests(self): + if self.testNames is None: + self.test = self.testLoader.loadTestsFromModule(self.module) + else: + self.test = self.testLoader.loadTestsFromNames(self.testNames, + self.module) + + def _do_discovery(self, argv, Loader=defaultTestLoaderCls): + # handle command line args for test discovery + if not have_discover: + raise AssertionError("Unable to use discovery, must use python 2.7 " + "or greater, or install the discover package.") + self.progName = '%s discover' % self.progName + import optparse + parser = optparse.OptionParser() + parser.prog = self.progName + parser.add_option('-v', '--verbose', dest='verbose', default=False, + help='Verbose output', action='store_true') + if self.failfast != False: + parser.add_option('-f', '--failfast', dest='failfast', default=False, + help='Stop on first fail or error', + action='store_true') + if self.catchbreak != False: + parser.add_option('-c', '--catch', dest='catchbreak', default=False, + help='Catch ctrl-C and display results so far', + action='store_true') + if self.buffer != False: + parser.add_option('-b', '--buffer', dest='buffer', default=False, + help='Buffer stdout and stderr during tests', + action='store_true') + parser.add_option('-s', '--start-directory', dest='start', default='.', + help="Directory to start discovery ('.' default)") + parser.add_option('-p', '--pattern', dest='pattern', default='test*.py', + help="Pattern to match tests ('test*.py' default)") + parser.add_option('-t', '--top-level-directory', dest='top', default=None, + help='Top level directory of project (defaults to start directory)') + parser.add_option('-l', '--list', dest='listtests', default=False, + help='List tests rather than running them.') + parser.add_option('--load-list', dest='load_list', default=None, + help='Specify a filename containing the test ids to use.') + + options, args = parser.parse_args(argv) + if len(args) > 3: + self.usageExit() + + for name, value in zip(('start', 'pattern', 'top'), args): + setattr(options, name, value) + + # only set options from the parsing here + # if they weren't set explicitly in the constructor + if self.failfast is None: + self.failfast = options.failfast + if self.catchbreak is None: + self.catchbreak = options.catchbreak + if self.buffer is None: + self.buffer = options.buffer + self.listtests = options.listtests + self.load_list = options.load_list + + if options.verbose: + self.verbosity = 2 + + start_dir = options.start + pattern = options.pattern + top_level_dir = options.top + + loader = Loader() + self.test = loader.discover(start_dir, pattern, top_level_dir) + + def runTests(self): + if (self.catchbreak + and getattr(unittest, 'installHandler', None) is not None): + unittest.installHandler() + if self.testRunner is None: + self.testRunner = runner.TextTestRunner + if isinstance(self.testRunner, classtypes()): + try: + testRunner = self.testRunner(verbosity=self.verbosity, + failfast=self.failfast, + buffer=self.buffer) + except TypeError: + # didn't accept the verbosity, buffer or failfast arguments + testRunner = self.testRunner() + else: + # it is assumed to be a TestRunner instance + testRunner = self.testRunner + self.result = testRunner.run(self.test) + if self.exit: + sys.exit(not self.result.wasSuccessful()) +################ + +def main(argv, stdout): + runner = TestToolsTestRunner(stdout) + program = TestProgram(argv=argv, testRunner=runner, stdout=stdout) + +if __name__ == '__main__': + main(sys.argv, sys.stdout) diff --git a/test/3rdparty/testtools-0.9.12/testtools/runtest.py b/test/3rdparty/testtools-0.9.12/testtools/runtest.py new file mode 100644 index 00000000000..507ad87c276 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/runtest.py @@ -0,0 +1,205 @@ +# Copyright (c) 2009-2010 testtools developers. See LICENSE for details. + +"""Individual test case execution.""" + +__all__ = [ + 'MultipleExceptions', + 'RunTest', + ] + +import sys + +from testtools.testresult import ExtendedToOriginalDecorator + + +class MultipleExceptions(Exception): + """Represents many exceptions raised from some operation. + + :ivar args: The sys.exc_info() tuples for each exception. + """ + + +class RunTest(object): + """An object to run a test. + + RunTest objects are used to implement the internal logic involved in + running a test. TestCase.__init__ stores _RunTest as the class of RunTest + to execute. Passing the runTest= parameter to TestCase.__init__ allows a + different RunTest class to be used to execute the test. + + Subclassing or replacing RunTest can be useful to add functionality to the + way that tests are run in a given project. + + :ivar case: The test case that is to be run. + :ivar result: The result object a case is reporting to. + :ivar handlers: A list of (ExceptionClass, handler_function) for + exceptions that should be caught if raised from the user + code. Exceptions that are caught are checked against this list in + first to last order. There is a catch-all of 'Exception' at the end + of the list, so to add a new exception to the list, insert it at the + front (which ensures that it will be checked before any existing base + classes in the list. If you add multiple exceptions some of which are + subclasses of each other, add the most specific exceptions last (so + they come before their parent classes in the list). + :ivar exception_caught: An object returned when _run_user catches an + exception. + :ivar _exceptions: A list of caught exceptions, used to do the single + reporting of error/failure/skip etc. + """ + + def __init__(self, case, handlers=None): + """Create a RunTest to run a case. + + :param case: A testtools.TestCase test case object. + :param handlers: Exception handlers for this RunTest. These are stored + in self.handlers and can be modified later if needed. + """ + self.case = case + self.handlers = handlers or [] + self.exception_caught = object() + self._exceptions = [] + + def run(self, result=None): + """Run self.case reporting activity to result. + + :param result: Optional testtools.TestResult to report activity to. + :return: The result object the test was run against. + """ + if result is None: + actual_result = self.case.defaultTestResult() + actual_result.startTestRun() + else: + actual_result = result + try: + return self._run_one(actual_result) + finally: + if result is None: + actual_result.stopTestRun() + + def _run_one(self, result): + """Run one test reporting to result. + + :param result: A testtools.TestResult to report activity to. + This result object is decorated with an ExtendedToOriginalDecorator + to ensure that the latest TestResult API can be used with + confidence by client code. + :return: The result object the test was run against. + """ + return self._run_prepared_result(ExtendedToOriginalDecorator(result)) + + def _run_prepared_result(self, result): + """Run one test reporting to result. + + :param result: A testtools.TestResult to report activity to. + :return: The result object the test was run against. + """ + result.startTest(self.case) + self.result = result + try: + self._exceptions = [] + self._run_core() + if self._exceptions: + # One or more caught exceptions, now trigger the test's + # reporting method for just one. + e = self._exceptions.pop() + for exc_class, handler in self.handlers: + if isinstance(e, exc_class): + handler(self.case, self.result, e) + break + finally: + result.stopTest(self.case) + return result + + def _run_core(self): + """Run the user supplied test code.""" + if self.exception_caught == self._run_user(self.case._run_setup, + self.result): + # Don't run the test method if we failed getting here. + self._run_cleanups(self.result) + return + # Run everything from here on in. If any of the methods raise an + # exception we'll have failed. + failed = False + try: + if self.exception_caught == self._run_user( + self.case._run_test_method, self.result): + failed = True + finally: + try: + if self.exception_caught == self._run_user( + self.case._run_teardown, self.result): + failed = True + finally: + try: + if self.exception_caught == self._run_user( + self._run_cleanups, self.result): + failed = True + finally: + if not failed: + self.result.addSuccess(self.case, + details=self.case.getDetails()) + + def _run_cleanups(self, result): + """Run the cleanups that have been added with addCleanup. + + See the docstring for addCleanup for more information. + + :return: None if all cleanups ran without error, + ``exception_caught`` if there was an error. + """ + failing = False + while self.case._cleanups: + function, arguments, keywordArguments = self.case._cleanups.pop() + got_exception = self._run_user( + function, *arguments, **keywordArguments) + if got_exception == self.exception_caught: + failing = True + if failing: + return self.exception_caught + + def _run_user(self, fn, *args, **kwargs): + """Run a user supplied function. + + Exceptions are processed by `_got_user_exception`. + + :return: Either whatever 'fn' returns or ``exception_caught`` if + 'fn' raised an exception. + """ + try: + return fn(*args, **kwargs) + except KeyboardInterrupt: + raise + except: + return self._got_user_exception(sys.exc_info()) + + def _got_user_exception(self, exc_info, tb_label='traceback'): + """Called when user code raises an exception. + + If 'exc_info' is a `MultipleExceptions`, then we recurse into it + unpacking the errors that it's made up from. + + :param exc_info: A sys.exc_info() tuple for the user error. + :param tb_label: An optional string label for the error. If + not specified, will default to 'traceback'. + :return: 'exception_caught' if we catch one of the exceptions that + have handlers in 'handlers', otherwise raise the error. + """ + if exc_info[0] is MultipleExceptions: + for sub_exc_info in exc_info[1].args: + self._got_user_exception(sub_exc_info, tb_label) + return self.exception_caught + try: + e = exc_info[1] + self.case.onException(exc_info, tb_label=tb_label) + finally: + del exc_info + for exc_class, handler in self.handlers: + if isinstance(e, exc_class): + self._exceptions.append(e) + return self.exception_caught + raise e + + +# Signal that this is part of the testing framework, and that code from this +# should not normally appear in tracebacks. +__unittest = True diff --git a/test/3rdparty/testtools-0.9.12/testtools/testcase.py b/test/3rdparty/testtools-0.9.12/testtools/testcase.py new file mode 100644 index 00000000000..ee5e296cd46 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/testcase.py @@ -0,0 +1,782 @@ +# Copyright (c) 2008-2011 testtools developers. See LICENSE for details. + +"""Test case related stuff.""" + +__metaclass__ = type +__all__ = [ + 'clone_test_with_new_id', + 'ExpectedException', + 'gather_details', + 'run_test_with', + 'skip', + 'skipIf', + 'skipUnless', + 'TestCase', + ] + +import copy +import itertools +import sys +import types +import unittest + +from testtools import ( + content, + try_import, + ) +from testtools.compat import ( + advance_iterator, + reraise, + ) +from testtools.matchers import ( + Annotate, + Contains, + Equals, + MatchesAll, + MatchesException, + MismatchError, + Is, + IsInstance, + Not, + Raises, + ) +from testtools.monkey import patch +from testtools.runtest import RunTest +from testtools.testresult import TestResult + +wraps = try_import('functools.wraps') + +class TestSkipped(Exception): + """Raised within TestCase.run() when a test is skipped.""" +testSkipped = try_import('unittest2.case.SkipTest', TestSkipped) +TestSkipped = try_import('unittest.case.SkipTest', TestSkipped) + + +class _UnexpectedSuccess(Exception): + """An unexpected success was raised. + + Note that this exception is private plumbing in testtools' testcase + module. + """ +_UnexpectedSuccess = try_import( + 'unittest2.case._UnexpectedSuccess', _UnexpectedSuccess) +_UnexpectedSuccess = try_import( + 'unittest.case._UnexpectedSuccess', _UnexpectedSuccess) + +class _ExpectedFailure(Exception): + """An expected failure occured. + + Note that this exception is private plumbing in testtools' testcase + module. + """ +_ExpectedFailure = try_import( + 'unittest2.case._ExpectedFailure', _ExpectedFailure) +_ExpectedFailure = try_import( + 'unittest.case._ExpectedFailure', _ExpectedFailure) + + +def run_test_with(test_runner, **kwargs): + """Decorate a test as using a specific ``RunTest``. + + e.g.:: + + @run_test_with(CustomRunner, timeout=42) + def test_foo(self): + self.assertTrue(True) + + The returned decorator works by setting an attribute on the decorated + function. `TestCase.__init__` looks for this attribute when deciding on a + ``RunTest`` factory. If you wish to use multiple decorators on a test + method, then you must either make this one the top-most decorator, or you + must write your decorators so that they update the wrapping function with + the attributes of the wrapped function. The latter is recommended style + anyway. ``functools.wraps``, ``functools.wrapper`` and + ``twisted.python.util.mergeFunctionMetadata`` can help you do this. + + :param test_runner: A ``RunTest`` factory that takes a test case and an + optional list of exception handlers. See ``RunTest``. + :param kwargs: Keyword arguments to pass on as extra arguments to + 'test_runner'. + :return: A decorator to be used for marking a test as needing a special + runner. + """ + def decorator(function): + # Set an attribute on 'function' which will inform TestCase how to + # make the runner. + function._run_test_with = ( + lambda case, handlers=None: + test_runner(case, handlers=handlers, **kwargs)) + return function + return decorator + + +def _copy_content(content_object): + """Make a copy of the given content object. + + The content within `content_object` is iterated and saved. This is useful + when the source of the content is volatile, a log file in a temporary + directory for example. + + :param content_object: A `content.Content` instance. + :return: A `content.Content` instance with the same mime-type as + `content_object` and a non-volatile copy of its content. + """ + content_bytes = list(content_object.iter_bytes()) + content_callback = lambda: content_bytes + return content.Content(content_object.content_type, content_callback) + + +def gather_details(source_dict, target_dict): + """Merge the details from `source_dict` into `target_dict`. + + :param source_dict: A dictionary of details will be gathered. + :param target_dict: A dictionary into which details will be gathered. + """ + for name, content_object in source_dict.items(): + new_name = name + disambiguator = itertools.count(1) + while new_name in target_dict: + new_name = '%s-%d' % (name, advance_iterator(disambiguator)) + name = new_name + target_dict[name] = _copy_content(content_object) + + +class TestCase(unittest.TestCase): + """Extensions to the basic TestCase. + + :ivar exception_handlers: Exceptions to catch from setUp, runTest and + tearDown. This list is able to be modified at any time and consists of + (exception_class, handler(case, result, exception_value)) pairs. + :cvar run_tests_with: A factory to make the ``RunTest`` to run tests with. + Defaults to ``RunTest``. The factory is expected to take a test case + and an optional list of exception handlers. + """ + + skipException = TestSkipped + + run_tests_with = RunTest + + def __init__(self, *args, **kwargs): + """Construct a TestCase. + + :param testMethod: The name of the method to run. + :keyword runTest: Optional class to use to execute the test. If not + supplied ``RunTest`` is used. The instance to be used is created + when run() is invoked, so will be fresh each time. Overrides + ``TestCase.run_tests_with`` if given. + """ + runTest = kwargs.pop('runTest', None) + super(TestCase, self).__init__(*args, **kwargs) + self._cleanups = [] + self._unique_id_gen = itertools.count(1) + # Generators to ensure unique traceback ids. Maps traceback label to + # iterators. + self._traceback_id_gens = {} + self.__setup_called = False + self.__teardown_called = False + # __details is lazy-initialized so that a constructed-but-not-run + # TestCase is safe to use with clone_test_with_new_id. + self.__details = None + test_method = self._get_test_method() + if runTest is None: + runTest = getattr( + test_method, '_run_test_with', self.run_tests_with) + self.__RunTest = runTest + self.__exception_handlers = [] + self.exception_handlers = [ + (self.skipException, self._report_skip), + (self.failureException, self._report_failure), + (_ExpectedFailure, self._report_expected_failure), + (_UnexpectedSuccess, self._report_unexpected_success), + (Exception, self._report_error), + ] + if sys.version_info < (2, 6): + # Catch old-style string exceptions with None as the instance + self.exception_handlers.append((type(None), self._report_error)) + + def __eq__(self, other): + eq = getattr(unittest.TestCase, '__eq__', None) + if eq is not None and not unittest.TestCase.__eq__(self, other): + return False + return self.__dict__ == other.__dict__ + + def __repr__(self): + # We add id to the repr because it makes testing testtools easier. + return "<%s id=0x%0x>" % (self.id(), id(self)) + + def addDetail(self, name, content_object): + """Add a detail to be reported with this test's outcome. + + For more details see pydoc testtools.TestResult. + + :param name: The name to give this detail. + :param content_object: The content object for this detail. See + testtools.content for more detail. + """ + if self.__details is None: + self.__details = {} + self.__details[name] = content_object + + def getDetails(self): + """Get the details dict that will be reported with this test's outcome. + + For more details see pydoc testtools.TestResult. + """ + if self.__details is None: + self.__details = {} + return self.__details + + def patch(self, obj, attribute, value): + """Monkey-patch 'obj.attribute' to 'value' while the test is running. + + If 'obj' has no attribute, then the monkey-patch will still go ahead, + and the attribute will be deleted instead of restored to its original + value. + + :param obj: The object to patch. Can be anything. + :param attribute: The attribute on 'obj' to patch. + :param value: The value to set 'obj.attribute' to. + """ + self.addCleanup(patch(obj, attribute, value)) + + def shortDescription(self): + return self.id() + + def skipTest(self, reason): + """Cause this test to be skipped. + + This raises self.skipException(reason). skipException is raised + to permit a skip to be triggered at any point (during setUp or the + testMethod itself). The run() method catches skipException and + translates that into a call to the result objects addSkip method. + + :param reason: The reason why the test is being skipped. This must + support being cast into a unicode string for reporting. + """ + raise self.skipException(reason) + + # skipTest is how python2.7 spells this. Sometime in the future + # This should be given a deprecation decorator - RBC 20100611. + skip = skipTest + + def _formatTypes(self, classOrIterable): + """Format a class or a bunch of classes for display in an error.""" + className = getattr(classOrIterable, '__name__', None) + if className is None: + className = ', '.join(klass.__name__ for klass in classOrIterable) + return className + + def addCleanup(self, function, *arguments, **keywordArguments): + """Add a cleanup function to be called after tearDown. + + Functions added with addCleanup will be called in reverse order of + adding after tearDown, or after setUp if setUp raises an exception. + + If a function added with addCleanup raises an exception, the error + will be recorded as a test error, and the next cleanup will then be + run. + + Cleanup functions are always called before a test finishes running, + even if setUp is aborted by an exception. + """ + self._cleanups.append((function, arguments, keywordArguments)) + + def addOnException(self, handler): + """Add a handler to be called when an exception occurs in test code. + + This handler cannot affect what result methods are called, and is + called before any outcome is called on the result object. An example + use for it is to add some diagnostic state to the test details dict + which is expensive to calculate and not interesting for reporting in + the success case. + + Handlers are called before the outcome (such as addFailure) that + the exception has caused. + + Handlers are called in first-added, first-called order, and if they + raise an exception, that will propogate out of the test running + machinery, halting test processing. As a result, do not call code that + may unreasonably fail. + """ + self.__exception_handlers.append(handler) + + def _add_reason(self, reason): + self.addDetail('reason', content.Content( + content.ContentType('text', 'plain'), + lambda: [reason.encode('utf8')])) + + def assertEqual(self, expected, observed, message=''): + """Assert that 'expected' is equal to 'observed'. + + :param expected: The expected value. + :param observed: The observed value. + :param message: An optional message to include in the error. + """ + matcher = Equals(expected) + self.assertThat(observed, matcher, message) + + failUnlessEqual = assertEquals = assertEqual + + def assertIn(self, needle, haystack): + """Assert that needle is in haystack.""" + self.assertThat(haystack, Contains(needle)) + + def assertIsNone(self, observed, message=''): + """Assert that 'observed' is equal to None. + + :param observed: The observed value. + :param message: An optional message describing the error. + """ + matcher = Is(None) + self.assertThat(observed, matcher, message) + + def assertIsNotNone(self, observed, message=''): + """Assert that 'observed' is not equal to None. + + :param observed: The observed value. + :param message: An optional message describing the error. + """ + matcher = Not(Is(None)) + self.assertThat(observed, matcher, message) + + def assertIs(self, expected, observed, message=''): + """Assert that 'expected' is 'observed'. + + :param expected: The expected value. + :param observed: The observed value. + :param message: An optional message describing the error. + """ + matcher = Is(expected) + self.assertThat(observed, matcher, message) + + def assertIsNot(self, expected, observed, message=''): + """Assert that 'expected' is not 'observed'.""" + matcher = Not(Is(expected)) + self.assertThat(observed, matcher, message) + + def assertNotIn(self, needle, haystack): + """Assert that needle is not in haystack.""" + matcher = Not(Contains(needle)) + self.assertThat(haystack, matcher) + + def assertIsInstance(self, obj, klass, msg=None): + if isinstance(klass, tuple): + matcher = IsInstance(*klass) + else: + matcher = IsInstance(klass) + self.assertThat(obj, matcher, msg) + + def assertRaises(self, excClass, callableObj, *args, **kwargs): + """Fail unless an exception of class excClass is thrown + by callableObj when invoked with arguments args and keyword + arguments kwargs. If a different type of exception is + thrown, it will not be caught, and the test case will be + deemed to have suffered an error, exactly as for an + unexpected exception. + """ + class ReRaiseOtherTypes(object): + def match(self, matchee): + if not issubclass(matchee[0], excClass): + reraise(*matchee) + class CaptureMatchee(object): + def match(self, matchee): + self.matchee = matchee[1] + capture = CaptureMatchee() + matcher = Raises(MatchesAll(ReRaiseOtherTypes(), + MatchesException(excClass), capture)) + + self.assertThat(lambda: callableObj(*args, **kwargs), matcher) + return capture.matchee + failUnlessRaises = assertRaises + + def assertThat(self, matchee, matcher, message='', verbose=False): + """Assert that matchee is matched by matcher. + + :param matchee: An object to match with matcher. + :param matcher: An object meeting the testtools.Matcher protocol. + :raises MismatchError: When matcher does not match thing. + """ + matcher = Annotate.if_message(message, matcher) + mismatch = matcher.match(matchee) + if not mismatch: + return + existing_details = self.getDetails() + for (name, content) in mismatch.get_details().items(): + full_name = name + suffix = 1 + while full_name in existing_details: + full_name = "%s-%d" % (name, suffix) + suffix += 1 + self.addDetail(full_name, content) + raise MismatchError(matchee, matcher, mismatch, verbose) + + def defaultTestResult(self): + return TestResult() + + def expectFailure(self, reason, predicate, *args, **kwargs): + """Check that a test fails in a particular way. + + If the test fails in the expected way, a KnownFailure is caused. If it + succeeds an UnexpectedSuccess is caused. + + The expected use of expectFailure is as a barrier at the point in a + test where the test would fail. For example: + >>> def test_foo(self): + >>> self.expectFailure("1 should be 0", self.assertNotEqual, 1, 0) + >>> self.assertEqual(1, 0) + + If in the future 1 were to equal 0, the expectFailure call can simply + be removed. This separation preserves the original intent of the test + while it is in the expectFailure mode. + """ + # TODO: implement with matchers. + self._add_reason(reason) + try: + predicate(*args, **kwargs) + except self.failureException: + # GZ 2010-08-12: Don't know how to avoid exc_info cycle as the new + # unittest _ExpectedFailure wants old traceback + exc_info = sys.exc_info() + try: + self._report_traceback(exc_info) + raise _ExpectedFailure(exc_info) + finally: + del exc_info + else: + raise _UnexpectedSuccess(reason) + + def getUniqueInteger(self): + """Get an integer unique to this test. + + Returns an integer that is guaranteed to be unique to this instance. + Use this when you need an arbitrary integer in your test, or as a + helper for custom anonymous factory methods. + """ + return advance_iterator(self._unique_id_gen) + + def getUniqueString(self, prefix=None): + """Get a string unique to this test. + + Returns a string that is guaranteed to be unique to this instance. Use + this when you need an arbitrary string in your test, or as a helper + for custom anonymous factory methods. + + :param prefix: The prefix of the string. If not provided, defaults + to the id of the tests. + :return: A bytestring of '<prefix>-<unique_int>'. + """ + if prefix is None: + prefix = self.id() + return '%s-%d' % (prefix, self.getUniqueInteger()) + + def onException(self, exc_info, tb_label='traceback'): + """Called when an exception propogates from test code. + + :seealso addOnException: + """ + if exc_info[0] not in [ + TestSkipped, _UnexpectedSuccess, _ExpectedFailure]: + self._report_traceback(exc_info, tb_label=tb_label) + for handler in self.__exception_handlers: + handler(exc_info) + + @staticmethod + def _report_error(self, result, err): + result.addError(self, details=self.getDetails()) + + @staticmethod + def _report_expected_failure(self, result, err): + result.addExpectedFailure(self, details=self.getDetails()) + + @staticmethod + def _report_failure(self, result, err): + result.addFailure(self, details=self.getDetails()) + + @staticmethod + def _report_skip(self, result, err): + if err.args: + reason = err.args[0] + else: + reason = "no reason given." + self._add_reason(reason) + result.addSkip(self, details=self.getDetails()) + + def _report_traceback(self, exc_info, tb_label='traceback'): + id_gen = self._traceback_id_gens.setdefault( + tb_label, itertools.count(0)) + tb_id = advance_iterator(id_gen) + if tb_id: + tb_label = '%s-%d' % (tb_label, tb_id) + self.addDetail(tb_label, content.TracebackContent(exc_info, self)) + + @staticmethod + def _report_unexpected_success(self, result, err): + result.addUnexpectedSuccess(self, details=self.getDetails()) + + def run(self, result=None): + return self.__RunTest(self, self.exception_handlers).run(result) + + def _run_setup(self, result): + """Run the setUp function for this test. + + :param result: A testtools.TestResult to report activity to. + :raises ValueError: If the base class setUp is not called, a + ValueError is raised. + """ + ret = self.setUp() + if not self.__setup_called: + raise ValueError( + "TestCase.setUp was not called. Have you upcalled all the " + "way up the hierarchy from your setUp? e.g. Call " + "super(%s, self).setUp() from your setUp()." + % self.__class__.__name__) + return ret + + def _run_teardown(self, result): + """Run the tearDown function for this test. + + :param result: A testtools.TestResult to report activity to. + :raises ValueError: If the base class tearDown is not called, a + ValueError is raised. + """ + ret = self.tearDown() + if not self.__teardown_called: + raise ValueError( + "TestCase.tearDown was not called. Have you upcalled all the " + "way up the hierarchy from your tearDown? e.g. Call " + "super(%s, self).tearDown() from your tearDown()." + % self.__class__.__name__) + return ret + + def _get_test_method(self): + absent_attr = object() + # Python 2.5+ + method_name = getattr(self, '_testMethodName', absent_attr) + if method_name is absent_attr: + # Python 2.4 + method_name = getattr(self, '_TestCase__testMethodName') + return getattr(self, method_name) + + def _run_test_method(self, result): + """Run the test method for this test. + + :param result: A testtools.TestResult to report activity to. + :return: None. + """ + return self._get_test_method()() + + def useFixture(self, fixture): + """Use fixture in a test case. + + The fixture will be setUp, and self.addCleanup(fixture.cleanUp) called. + + :param fixture: The fixture to use. + :return: The fixture, after setting it up and scheduling a cleanup for + it. + """ + try: + fixture.setUp() + except: + gather_details(fixture.getDetails(), self.getDetails()) + raise + else: + self.addCleanup(fixture.cleanUp) + self.addCleanup( + gather_details, fixture.getDetails(), self.getDetails()) + return fixture + + def setUp(self): + super(TestCase, self).setUp() + self.__setup_called = True + + def tearDown(self): + super(TestCase, self).tearDown() + unittest.TestCase.tearDown(self) + self.__teardown_called = True + + +class PlaceHolder(object): + """A placeholder test. + + `PlaceHolder` implements much of the same interface as TestCase and is + particularly suitable for being added to TestResults. + """ + + def __init__(self, test_id, short_description=None): + """Construct a `PlaceHolder`. + + :param test_id: The id of the placeholder test. + :param short_description: The short description of the place holder + test. If not provided, the id will be used instead. + """ + self._test_id = test_id + self._short_description = short_description + + def __call__(self, result=None): + return self.run(result=result) + + def __repr__(self): + internal = [self._test_id] + if self._short_description is not None: + internal.append(self._short_description) + return "<%s.%s(%s)>" % ( + self.__class__.__module__, + self.__class__.__name__, + ", ".join(map(repr, internal))) + + def __str__(self): + return self.id() + + def countTestCases(self): + return 1 + + def debug(self): + pass + + def id(self): + return self._test_id + + def run(self, result=None): + if result is None: + result = TestResult() + result.startTest(self) + result.addSuccess(self) + result.stopTest(self) + + def shortDescription(self): + if self._short_description is None: + return self.id() + else: + return self._short_description + + +class ErrorHolder(PlaceHolder): + """A placeholder test that will error out when run.""" + + failureException = None + + def __init__(self, test_id, error, short_description=None): + """Construct an `ErrorHolder`. + + :param test_id: The id of the test. + :param error: The exc info tuple that will be used as the test's error. + :param short_description: An optional short description of the test. + """ + super(ErrorHolder, self).__init__( + test_id, short_description=short_description) + self._error = error + + def __repr__(self): + internal = [self._test_id, self._error] + if self._short_description is not None: + internal.append(self._short_description) + return "<%s.%s(%s)>" % ( + self.__class__.__module__, + self.__class__.__name__, + ", ".join(map(repr, internal))) + + def run(self, result=None): + if result is None: + result = TestResult() + result.startTest(self) + result.addError(self, self._error) + result.stopTest(self) + + +# Python 2.4 did not know how to copy functions. +if types.FunctionType not in copy._copy_dispatch: + copy._copy_dispatch[types.FunctionType] = copy._copy_immutable + + +def clone_test_with_new_id(test, new_id): + """Copy a `TestCase`, and give the copied test a new id. + + This is only expected to be used on tests that have been constructed but + not executed. + """ + newTest = copy.copy(test) + newTest.id = lambda: new_id + return newTest + + +def skip(reason): + """A decorator to skip unit tests. + + This is just syntactic sugar so users don't have to change any of their + unit tests in order to migrate to python 2.7, which provides the + @unittest.skip decorator. + """ + def decorator(test_item): + if wraps is not None: + @wraps(test_item) + def skip_wrapper(*args, **kwargs): + raise TestCase.skipException(reason) + else: + def skip_wrapper(test_item): + test_item.skip(reason) + return skip_wrapper + return decorator + + +def skipIf(condition, reason): + """Skip a test if the condition is true.""" + if condition: + return skip(reason) + def _id(obj): + return obj + return _id + + +def skipUnless(condition, reason): + """Skip a test unless the condition is true.""" + if not condition: + return skip(reason) + def _id(obj): + return obj + return _id + + +class ExpectedException: + """A context manager to handle expected exceptions. + + In Python 2.5 or later:: + + def test_foo(self): + with ExpectedException(ValueError, 'fo.*'): + raise ValueError('foo') + + will pass. If the raised exception has a type other than the specified + type, it will be re-raised. If it has a 'str()' that does not match the + given regular expression, an AssertionError will be raised. If no + exception is raised, an AssertionError will be raised. + """ + + def __init__(self, exc_type, value_re=None): + """Construct an `ExpectedException`. + + :param exc_type: The type of exception to expect. + :param value_re: A regular expression to match against the + 'str()' of the raised exception. + """ + self.exc_type = exc_type + self.value_re = value_re + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_value, traceback): + if exc_type is None: + raise AssertionError('%s not raised.' % self.exc_type.__name__) + if exc_type != self.exc_type: + return False + if self.value_re: + matcher = MatchesException(self.exc_type, self.value_re) + mismatch = matcher.match((exc_type, exc_value, traceback)) + if mismatch: + raise AssertionError(mismatch.describe()) + return True + + +# Signal that this is part of the testing framework, and that code from this +# should not normally appear in tracebacks. +__unittest = True diff --git a/test/3rdparty/testtools-0.9.12/testtools/testresult/__init__.py b/test/3rdparty/testtools-0.9.12/testtools/testresult/__init__.py new file mode 100644 index 00000000000..19f88bc8a34 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/testresult/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2009 testtools developers. See LICENSE for details. + +"""Test result objects.""" + +__all__ = [ + 'ExtendedToOriginalDecorator', + 'MultiTestResult', + 'TestResult', + 'TextTestResult', + 'ThreadsafeForwardingResult', + ] + +from testtools.testresult.real import ( + ExtendedToOriginalDecorator, + MultiTestResult, + TestResult, + TextTestResult, + ThreadsafeForwardingResult, + ) diff --git a/test/3rdparty/testtools-0.9.12/testtools/testresult/doubles.py b/test/3rdparty/testtools-0.9.12/testtools/testresult/doubles.py new file mode 100644 index 00000000000..9af5b364ffb --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/testresult/doubles.py @@ -0,0 +1,111 @@ +# Copyright (c) 2009-2010 testtools developers. See LICENSE for details. + +"""Doubles of test result objects, useful for testing unittest code.""" + +__all__ = [ + 'Python26TestResult', + 'Python27TestResult', + 'ExtendedTestResult', + ] + + +class LoggingBase(object): + """Basic support for logging of results.""" + + def __init__(self): + self._events = [] + self.shouldStop = False + self._was_successful = True + + +class Python26TestResult(LoggingBase): + """A precisely python 2.6 like test result, that logs.""" + + def addError(self, test, err): + self._was_successful = False + self._events.append(('addError', test, err)) + + def addFailure(self, test, err): + self._was_successful = False + self._events.append(('addFailure', test, err)) + + def addSuccess(self, test): + self._events.append(('addSuccess', test)) + + def startTest(self, test): + self._events.append(('startTest', test)) + + def stop(self): + self.shouldStop = True + + def stopTest(self, test): + self._events.append(('stopTest', test)) + + def wasSuccessful(self): + return self._was_successful + + +class Python27TestResult(Python26TestResult): + """A precisely python 2.7 like test result, that logs.""" + + def addExpectedFailure(self, test, err): + self._events.append(('addExpectedFailure', test, err)) + + def addSkip(self, test, reason): + self._events.append(('addSkip', test, reason)) + + def addUnexpectedSuccess(self, test): + self._events.append(('addUnexpectedSuccess', test)) + + def startTestRun(self): + self._events.append(('startTestRun',)) + + def stopTestRun(self): + self._events.append(('stopTestRun',)) + + +class ExtendedTestResult(Python27TestResult): + """A test result like the proposed extended unittest result API.""" + + def addError(self, test, err=None, details=None): + self._was_successful = False + self._events.append(('addError', test, err or details)) + + def addFailure(self, test, err=None, details=None): + self._was_successful = False + self._events.append(('addFailure', test, err or details)) + + def addExpectedFailure(self, test, err=None, details=None): + self._events.append(('addExpectedFailure', test, err or details)) + + def addSkip(self, test, reason=None, details=None): + self._events.append(('addSkip', test, reason or details)) + + def addSuccess(self, test, details=None): + if details: + self._events.append(('addSuccess', test, details)) + else: + self._events.append(('addSuccess', test)) + + def addUnexpectedSuccess(self, test, details=None): + self._was_successful = False + if details is not None: + self._events.append(('addUnexpectedSuccess', test, details)) + else: + self._events.append(('addUnexpectedSuccess', test)) + + def progress(self, offset, whence): + self._events.append(('progress', offset, whence)) + + def startTestRun(self): + super(ExtendedTestResult, self).startTestRun() + self._was_successful = True + + def tags(self, new_tags, gone_tags): + self._events.append(('tags', new_tags, gone_tags)) + + def time(self, time): + self._events.append(('time', time)) + + def wasSuccessful(self): + return self._was_successful diff --git a/test/3rdparty/testtools-0.9.12/testtools/testresult/real.py b/test/3rdparty/testtools-0.9.12/testtools/testresult/real.py new file mode 100644 index 00000000000..eb548dfa2c8 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/testresult/real.py @@ -0,0 +1,658 @@ +# Copyright (c) 2008 testtools developers. See LICENSE for details. + +"""Test results and related things.""" + +__metaclass__ = type +__all__ = [ + 'ExtendedToOriginalDecorator', + 'MultiTestResult', + 'TestResult', + 'ThreadsafeForwardingResult', + ] + +import datetime +import sys +import unittest + +from testtools.compat import all, _format_exc_info, str_is_unicode, _u + +# From http://docs.python.org/library/datetime.html +_ZERO = datetime.timedelta(0) + +# A UTC class. + +class UTC(datetime.tzinfo): + """UTC""" + + def utcoffset(self, dt): + return _ZERO + + def tzname(self, dt): + return "UTC" + + def dst(self, dt): + return _ZERO + +utc = UTC() + + +class TestResult(unittest.TestResult): + """Subclass of unittest.TestResult extending the protocol for flexability. + + This test result supports an experimental protocol for providing additional + data to in test outcomes. All the outcome methods take an optional dict + 'details'. If supplied any other detail parameters like 'err' or 'reason' + should not be provided. The details dict is a mapping from names to + MIME content objects (see testtools.content). This permits attaching + tracebacks, log files, or even large objects like databases that were + part of the test fixture. Until this API is accepted into upstream + Python it is considered experimental: it may be replaced at any point + by a newer version more in line with upstream Python. Compatibility would + be aimed for in this case, but may not be possible. + + :ivar skip_reasons: A dict of skip-reasons -> list of tests. See addSkip. + """ + + def __init__(self): + # startTestRun resets all attributes, and older clients don't know to + # call startTestRun, so it is called once here. + # Because subclasses may reasonably not expect this, we call the + # specific version we want to run. + TestResult.startTestRun(self) + + def addExpectedFailure(self, test, err=None, details=None): + """Called when a test has failed in an expected manner. + + Like with addSuccess and addError, testStopped should still be called. + + :param test: The test that has been skipped. + :param err: The exc_info of the error that was raised. + :return: None + """ + # This is the python 2.7 implementation + self.expectedFailures.append( + (test, self._err_details_to_string(test, err, details))) + + def addError(self, test, err=None, details=None): + """Called when an error has occurred. 'err' is a tuple of values as + returned by sys.exc_info(). + + :param details: Alternative way to supply details about the outcome. + see the class docstring for more information. + """ + self.errors.append((test, + self._err_details_to_string(test, err, details))) + + def addFailure(self, test, err=None, details=None): + """Called when an error has occurred. 'err' is a tuple of values as + returned by sys.exc_info(). + + :param details: Alternative way to supply details about the outcome. + see the class docstring for more information. + """ + self.failures.append((test, + self._err_details_to_string(test, err, details))) + + def addSkip(self, test, reason=None, details=None): + """Called when a test has been skipped rather than running. + + Like with addSuccess and addError, testStopped should still be called. + + This must be called by the TestCase. 'addError' and 'addFailure' will + not call addSkip, since they have no assumptions about the kind of + errors that a test can raise. + + :param test: The test that has been skipped. + :param reason: The reason for the test being skipped. For instance, + u"pyGL is not available". + :param details: Alternative way to supply details about the outcome. + see the class docstring for more information. + :return: None + """ + if reason is None: + reason = details.get('reason') + if reason is None: + reason = 'No reason given' + else: + reason = ''.join(reason.iter_text()) + skip_list = self.skip_reasons.setdefault(reason, []) + skip_list.append(test) + + def addSuccess(self, test, details=None): + """Called when a test succeeded.""" + + def addUnexpectedSuccess(self, test, details=None): + """Called when a test was expected to fail, but succeed.""" + self.unexpectedSuccesses.append(test) + + def wasSuccessful(self): + """Has this result been successful so far? + + If there have been any errors, failures or unexpected successes, + return False. Otherwise, return True. + + Note: This differs from standard unittest in that we consider + unexpected successes to be equivalent to failures, rather than + successes. + """ + return not (self.errors or self.failures or self.unexpectedSuccesses) + + if str_is_unicode: + # Python 3 and IronPython strings are unicode, use parent class method + _exc_info_to_unicode = unittest.TestResult._exc_info_to_string + else: + # For Python 2, need to decode components of traceback according to + # their source, so can't use traceback.format_exception + # Here follows a little deep magic to copy the existing method and + # replace the formatter with one that returns unicode instead + from types import FunctionType as __F, ModuleType as __M + __f = unittest.TestResult._exc_info_to_string.im_func + __g = dict(__f.func_globals) + __m = __M("__fake_traceback") + __m.format_exception = _format_exc_info + __g["traceback"] = __m + _exc_info_to_unicode = __F(__f.func_code, __g, "_exc_info_to_unicode") + del __F, __M, __f, __g, __m + + def _err_details_to_string(self, test, err=None, details=None): + """Convert an error in exc_info form or a contents dict to a string.""" + if err is not None: + return self._exc_info_to_unicode(err, test) + return _details_to_str(details, special='traceback') + + def _now(self): + """Return the current 'test time'. + + If the time() method has not been called, this is equivalent to + datetime.now(), otherwise its the last supplied datestamp given to the + time() method. + """ + if self.__now is None: + return datetime.datetime.now(utc) + else: + return self.__now + + def startTestRun(self): + """Called before a test run starts. + + New in Python 2.7. The testtools version resets the result to a + pristine condition ready for use in another test run. Note that this + is different from Python 2.7's startTestRun, which does nothing. + """ + super(TestResult, self).__init__() + self.skip_reasons = {} + self.__now = None + # -- Start: As per python 2.7 -- + self.expectedFailures = [] + self.unexpectedSuccesses = [] + # -- End: As per python 2.7 -- + + def stopTestRun(self): + """Called after a test run completes + + New in python 2.7 + """ + + def time(self, a_datetime): + """Provide a timestamp to represent the current time. + + This is useful when test activity is time delayed, or happening + concurrently and getting the system time between API calls will not + accurately represent the duration of tests (or the whole run). + + Calling time() sets the datetime used by the TestResult object. + Time is permitted to go backwards when using this call. + + :param a_datetime: A datetime.datetime object with TZ information or + None to reset the TestResult to gathering time from the system. + """ + self.__now = a_datetime + + def done(self): + """Called when the test runner is done. + + deprecated in favour of stopTestRun. + """ + + +class MultiTestResult(TestResult): + """A test result that dispatches to many test results.""" + + def __init__(self, *results): + TestResult.__init__(self) + self._results = list(map(ExtendedToOriginalDecorator, results)) + + def _dispatch(self, message, *args, **kwargs): + return tuple( + getattr(result, message)(*args, **kwargs) + for result in self._results) + + def startTest(self, test): + return self._dispatch('startTest', test) + + def stopTest(self, test): + return self._dispatch('stopTest', test) + + def addError(self, test, error=None, details=None): + return self._dispatch('addError', test, error, details=details) + + def addExpectedFailure(self, test, err=None, details=None): + return self._dispatch( + 'addExpectedFailure', test, err, details=details) + + def addFailure(self, test, err=None, details=None): + return self._dispatch('addFailure', test, err, details=details) + + def addSkip(self, test, reason=None, details=None): + return self._dispatch('addSkip', test, reason, details=details) + + def addSuccess(self, test, details=None): + return self._dispatch('addSuccess', test, details=details) + + def addUnexpectedSuccess(self, test, details=None): + return self._dispatch('addUnexpectedSuccess', test, details=details) + + def startTestRun(self): + return self._dispatch('startTestRun') + + def stopTestRun(self): + return self._dispatch('stopTestRun') + + def time(self, a_datetime): + return self._dispatch('time', a_datetime) + + def done(self): + return self._dispatch('done') + + def wasSuccessful(self): + """Was this result successful? + + Only returns True if every constituent result was successful. + """ + return all(self._dispatch('wasSuccessful')) + + +class TextTestResult(TestResult): + """A TestResult which outputs activity to a text stream.""" + + def __init__(self, stream): + """Construct a TextTestResult writing to stream.""" + super(TextTestResult, self).__init__() + self.stream = stream + self.sep1 = '=' * 70 + '\n' + self.sep2 = '-' * 70 + '\n' + + def _delta_to_float(self, a_timedelta): + return (a_timedelta.days * 86400.0 + a_timedelta.seconds + + a_timedelta.microseconds / 1000000.0) + + def _show_list(self, label, error_list): + for test, output in error_list: + self.stream.write(self.sep1) + self.stream.write("%s: %s\n" % (label, test.id())) + self.stream.write(self.sep2) + self.stream.write(output) + + def startTestRun(self): + super(TextTestResult, self).startTestRun() + self.__start = self._now() + self.stream.write("Tests running...\n") + + def stopTestRun(self): + if self.testsRun != 1: + plural = 's' + else: + plural = '' + stop = self._now() + self._show_list('ERROR', self.errors) + self._show_list('FAIL', self.failures) + for test in self.unexpectedSuccesses: + self.stream.write( + "%sUNEXPECTED SUCCESS: %s\n%s" % ( + self.sep1, test.id(), self.sep2)) + self.stream.write("\nRan %d test%s in %.3fs\n" % + (self.testsRun, plural, + self._delta_to_float(stop - self.__start))) + if self.wasSuccessful(): + self.stream.write("OK\n") + else: + self.stream.write("FAILED (") + details = [] + details.append("failures=%d" % ( + sum(map(len, ( + self.failures, self.errors, self.unexpectedSuccesses))))) + self.stream.write(", ".join(details)) + self.stream.write(")\n") + super(TextTestResult, self).stopTestRun() + + +class ThreadsafeForwardingResult(TestResult): + """A TestResult which ensures the target does not receive mixed up calls. + + This is used when receiving test results from multiple sources, and batches + up all the activity for a single test into a thread-safe batch where all + other ThreadsafeForwardingResult objects sharing the same semaphore will be + locked out. + + Typical use of ThreadsafeForwardingResult involves creating one + ThreadsafeForwardingResult per thread in a ConcurrentTestSuite. These + forward to the TestResult that the ConcurrentTestSuite run method was + called with. + + target.done() is called once for each ThreadsafeForwardingResult that + forwards to the same target. If the target's done() takes special action, + care should be taken to accommodate this. + """ + + def __init__(self, target, semaphore): + """Create a ThreadsafeForwardingResult forwarding to target. + + :param target: A TestResult. + :param semaphore: A threading.Semaphore with limit 1. + """ + TestResult.__init__(self) + self.result = ExtendedToOriginalDecorator(target) + self.semaphore = semaphore + + def _add_result_with_semaphore(self, method, test, *args, **kwargs): + self.semaphore.acquire() + try: + self.result.time(self._test_start) + self.result.startTest(test) + self.result.time(self._now()) + try: + method(test, *args, **kwargs) + finally: + self.result.stopTest(test) + finally: + self.semaphore.release() + + def addError(self, test, err=None, details=None): + self._add_result_with_semaphore(self.result.addError, + test, err, details=details) + + def addExpectedFailure(self, test, err=None, details=None): + self._add_result_with_semaphore(self.result.addExpectedFailure, + test, err, details=details) + + def addFailure(self, test, err=None, details=None): + self._add_result_with_semaphore(self.result.addFailure, + test, err, details=details) + + def addSkip(self, test, reason=None, details=None): + self._add_result_with_semaphore(self.result.addSkip, + test, reason, details=details) + + def addSuccess(self, test, details=None): + self._add_result_with_semaphore(self.result.addSuccess, + test, details=details) + + def addUnexpectedSuccess(self, test, details=None): + self._add_result_with_semaphore(self.result.addUnexpectedSuccess, + test, details=details) + + def startTestRun(self): + self.semaphore.acquire() + try: + self.result.startTestRun() + finally: + self.semaphore.release() + + def stopTestRun(self): + self.semaphore.acquire() + try: + self.result.stopTestRun() + finally: + self.semaphore.release() + + def done(self): + self.semaphore.acquire() + try: + self.result.done() + finally: + self.semaphore.release() + + def startTest(self, test): + self._test_start = self._now() + super(ThreadsafeForwardingResult, self).startTest(test) + + def wasSuccessful(self): + return self.result.wasSuccessful() + + +class ExtendedToOriginalDecorator(object): + """Permit new TestResult API code to degrade gracefully with old results. + + This decorates an existing TestResult and converts missing outcomes + such as addSkip to older outcomes such as addSuccess. It also supports + the extended details protocol. In all cases the most recent protocol + is attempted first, and fallbacks only occur when the decorated result + does not support the newer style of calling. + """ + + def __init__(self, decorated): + self.decorated = decorated + + def __getattr__(self, name): + return getattr(self.decorated, name) + + def addError(self, test, err=None, details=None): + self._check_args(err, details) + if details is not None: + try: + return self.decorated.addError(test, details=details) + except TypeError: + # have to convert + err = self._details_to_exc_info(details) + return self.decorated.addError(test, err) + + def addExpectedFailure(self, test, err=None, details=None): + self._check_args(err, details) + addExpectedFailure = getattr( + self.decorated, 'addExpectedFailure', None) + if addExpectedFailure is None: + return self.addSuccess(test) + if details is not None: + try: + return addExpectedFailure(test, details=details) + except TypeError: + # have to convert + err = self._details_to_exc_info(details) + return addExpectedFailure(test, err) + + def addFailure(self, test, err=None, details=None): + self._check_args(err, details) + if details is not None: + try: + return self.decorated.addFailure(test, details=details) + except TypeError: + # have to convert + err = self._details_to_exc_info(details) + return self.decorated.addFailure(test, err) + + def addSkip(self, test, reason=None, details=None): + self._check_args(reason, details) + addSkip = getattr(self.decorated, 'addSkip', None) + if addSkip is None: + return self.decorated.addSuccess(test) + if details is not None: + try: + return addSkip(test, details=details) + except TypeError: + # extract the reason if it's available + try: + reason = ''.join(details['reason'].iter_text()) + except KeyError: + reason = _details_to_str(details) + return addSkip(test, reason) + + def addUnexpectedSuccess(self, test, details=None): + outcome = getattr(self.decorated, 'addUnexpectedSuccess', None) + if outcome is None: + try: + test.fail("") + except test.failureException: + return self.addFailure(test, sys.exc_info()) + if details is not None: + try: + return outcome(test, details=details) + except TypeError: + pass + return outcome(test) + + def addSuccess(self, test, details=None): + if details is not None: + try: + return self.decorated.addSuccess(test, details=details) + except TypeError: + pass + return self.decorated.addSuccess(test) + + def _check_args(self, err, details): + param_count = 0 + if err is not None: + param_count += 1 + if details is not None: + param_count += 1 + if param_count != 1: + raise ValueError("Must pass only one of err '%s' and details '%s" + % (err, details)) + + def _details_to_exc_info(self, details): + """Convert a details dict to an exc_info tuple.""" + return ( + _StringException, + _StringException(_details_to_str(details, special='traceback')), + None) + + def done(self): + try: + return self.decorated.done() + except AttributeError: + return + + def progress(self, offset, whence): + method = getattr(self.decorated, 'progress', None) + if method is None: + return + return method(offset, whence) + + @property + def shouldStop(self): + return self.decorated.shouldStop + + def startTest(self, test): + return self.decorated.startTest(test) + + def startTestRun(self): + try: + return self.decorated.startTestRun() + except AttributeError: + return + + def stop(self): + return self.decorated.stop() + + def stopTest(self, test): + return self.decorated.stopTest(test) + + def stopTestRun(self): + try: + return self.decorated.stopTestRun() + except AttributeError: + return + + def tags(self, new_tags, gone_tags): + method = getattr(self.decorated, 'tags', None) + if method is None: + return + return method(new_tags, gone_tags) + + def time(self, a_datetime): + method = getattr(self.decorated, 'time', None) + if method is None: + return + return method(a_datetime) + + def wasSuccessful(self): + return self.decorated.wasSuccessful() + + +class _StringException(Exception): + """An exception made from an arbitrary string.""" + + if not str_is_unicode: + def __init__(self, string): + if type(string) is not unicode: + raise TypeError("_StringException expects unicode, got %r" % + (string,)) + Exception.__init__(self, string) + + def __str__(self): + return self.args[0].encode("utf-8") + + def __unicode__(self): + return self.args[0] + # For 3.0 and above the default __str__ is fine, so we don't define one. + + def __hash__(self): + return id(self) + + def __eq__(self, other): + try: + return self.args == other.args + except AttributeError: + return False + + +def _format_text_attachment(name, text): + if '\n' in text: + return "%s: {{{\n%s\n}}}\n" % (name, text) + return "%s: {{{%s}}}" % (name, text) + + +def _details_to_str(details, special=None): + """Convert a details dict to a string. + + :param details: A dictionary mapping short names to ``Content`` objects. + :param special: If specified, an attachment that should have special + attention drawn to it. The primary attachment. Normally it's the + traceback that caused the test to fail. + :return: A formatted string that can be included in text test results. + """ + empty_attachments = [] + binary_attachments = [] + text_attachments = [] + special_content = None + # sorted is for testing, may want to remove that and use a dict + # subclass with defined order for items instead. + for key, content in sorted(details.items()): + if content.content_type.type != 'text': + binary_attachments.append((key, content.content_type)) + continue + text = _u('').join(content.iter_text()).strip() + if not text: + empty_attachments.append(key) + continue + # We want the 'special' attachment to be at the bottom. + if key == special: + special_content = '%s\n' % (text,) + continue + text_attachments.append(_format_text_attachment(key, text)) + if text_attachments and not text_attachments[-1].endswith('\n'): + text_attachments.append('') + if special_content: + text_attachments.append(special_content) + lines = [] + if binary_attachments: + lines.append('Binary content:\n') + for name, content_type in binary_attachments: + lines.append(' %s (%s)\n' % (name, content_type)) + if empty_attachments: + lines.append('Empty attachments:\n') + for name in empty_attachments: + lines.append(' %s\n' % (name,)) + if (binary_attachments or empty_attachments) and text_attachments: + lines.append('\n') + lines.append('\n'.join(text_attachments)) + return _u('').join(lines) diff --git a/test/3rdparty/testtools-0.9.12/testtools/tests/__init__.py b/test/3rdparty/testtools-0.9.12/testtools/tests/__init__.py new file mode 100644 index 00000000000..1b1aa38a1f9 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/tests/__init__.py @@ -0,0 +1,44 @@ +"""Tests for testtools itself.""" + +# See README for copyright and licensing details. + +from unittest import TestSuite + + +def test_suite(): + from testtools.tests import ( + test_compat, + test_content, + test_content_type, + test_deferredruntest, + test_distutilscmd, + test_fixturesupport, + test_helpers, + test_matchers, + test_monkey, + test_run, + test_runtest, + test_spinner, + test_testcase, + test_testresult, + test_testsuite, + ) + modules = [ + test_compat, + test_content, + test_content_type, + test_deferredruntest, + test_distutilscmd, + test_fixturesupport, + test_helpers, + test_matchers, + test_monkey, + test_run, + test_runtest, + test_spinner, + test_testcase, + test_testresult, + test_testsuite, + ] + suites = map(lambda x: x.test_suite(), modules) + return TestSuite(suites) diff --git a/test/3rdparty/testtools-0.9.12/testtools/tests/helpers.py b/test/3rdparty/testtools-0.9.12/testtools/tests/helpers.py new file mode 100644 index 00000000000..660cfecb72d --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/tests/helpers.py @@ -0,0 +1,111 @@ +# Copyright (c) 2008-2011 testtools developers. See LICENSE for details. + +"""Helpers for tests.""" + +__all__ = [ + 'LoggingResult', + ] + +import sys + +from testtools import TestResult +from testtools.helpers import ( + safe_hasattr, + try_import, + ) +from testtools import runtest + + +# GZ 2010-08-12: Don't do this, pointlessly creates an exc_info cycle +try: + raise Exception +except Exception: + an_exc_info = sys.exc_info() + +# Deprecated: This classes attributes are somewhat non deterministic which +# leads to hard to predict tests (because Python upstream are changing things. +class LoggingResult(TestResult): + """TestResult that logs its event to a list.""" + + def __init__(self, log): + self._events = log + super(LoggingResult, self).__init__() + + def startTest(self, test): + self._events.append(('startTest', test)) + super(LoggingResult, self).startTest(test) + + def stopTest(self, test): + self._events.append(('stopTest', test)) + super(LoggingResult, self).stopTest(test) + + def addFailure(self, test, error): + self._events.append(('addFailure', test, error)) + super(LoggingResult, self).addFailure(test, error) + + def addError(self, test, error): + self._events.append(('addError', test, error)) + super(LoggingResult, self).addError(test, error) + + def addSkip(self, test, reason): + self._events.append(('addSkip', test, reason)) + super(LoggingResult, self).addSkip(test, reason) + + def addSuccess(self, test): + self._events.append(('addSuccess', test)) + super(LoggingResult, self).addSuccess(test) + + def startTestRun(self): + self._events.append('startTestRun') + super(LoggingResult, self).startTestRun() + + def stopTestRun(self): + self._events.append('stopTestRun') + super(LoggingResult, self).stopTestRun() + + def done(self): + self._events.append('done') + super(LoggingResult, self).done() + + def time(self, a_datetime): + self._events.append(('time', a_datetime)) + super(LoggingResult, self).time(a_datetime) + + +def is_stack_hidden(): + return safe_hasattr(runtest, '__unittest') + + +def hide_testtools_stack(should_hide=True): + modules = [ + 'testtools.matchers', + 'testtools.runtest', + 'testtools.testcase', + ] + result = is_stack_hidden() + for module_name in modules: + module = try_import(module_name) + if should_hide: + setattr(module, '__unittest', True) + else: + try: + delattr(module, '__unittest') + except AttributeError: + # Attribute already doesn't exist. Our work here is done. + pass + return result + + +def run_with_stack_hidden(should_hide, f, *args, **kwargs): + old_should_hide = hide_testtools_stack(should_hide) + try: + return f(*args, **kwargs) + finally: + hide_testtools_stack(old_should_hide) + + + +class FullStackRunTest(runtest.RunTest): + + def _run_user(self, fn, *args, **kwargs): + return run_with_stack_hidden(False, fn, *args, **kwargs) diff --git a/test/3rdparty/testtools-0.9.12/testtools/tests/test_compat.py b/test/3rdparty/testtools-0.9.12/testtools/tests/test_compat.py new file mode 100644 index 00000000000..5e385bf48ce --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/tests/test_compat.py @@ -0,0 +1,394 @@ +# Copyright (c) 2010 testtools developers. See LICENSE for details. + +"""Tests for miscellaneous compatibility functions""" + +import linecache +import os +import sys +import tempfile +import traceback + +import testtools + +from testtools.compat import ( + _b, + _detect_encoding, + _get_source_encoding, + _u, + str_is_unicode, + text_repr, + unicode_output_stream, + ) +from testtools.matchers import ( + MatchesException, + Not, + Raises, + ) + + +class TestDetectEncoding(testtools.TestCase): + """Test detection of Python source encodings""" + + def _check_encoding(self, expected, lines, possibly_invalid=False): + """Check lines are valid Python and encoding is as expected""" + if not possibly_invalid: + compile(_b("".join(lines)), "<str>", "exec") + encoding = _detect_encoding(lines) + self.assertEqual(expected, encoding, + "Encoding %r expected but got %r from lines %r" % + (expected, encoding, lines)) + + def test_examples_from_pep(self): + """Check the examples given in PEP 263 all work as specified + + See 'Examples' section of <http://www.python.org/dev/peps/pep-0263/> + """ + # With interpreter binary and using Emacs style file encoding comment: + self._check_encoding("latin-1", ( + "#!/usr/bin/python\n", + "# -*- coding: latin-1 -*-\n", + "import os, sys\n")) + self._check_encoding("iso-8859-15", ( + "#!/usr/bin/python\n", + "# -*- coding: iso-8859-15 -*-\n", + "import os, sys\n")) + self._check_encoding("ascii", ( + "#!/usr/bin/python\n", + "# -*- coding: ascii -*-\n", + "import os, sys\n")) + # Without interpreter line, using plain text: + self._check_encoding("utf-8", ( + "# This Python file uses the following encoding: utf-8\n", + "import os, sys\n")) + # Text editors might have different ways of defining the file's + # encoding, e.g. + self._check_encoding("latin-1", ( + "#!/usr/local/bin/python\n", + "# coding: latin-1\n", + "import os, sys\n")) + # Without encoding comment, Python's parser will assume ASCII text: + self._check_encoding("ascii", ( + "#!/usr/local/bin/python\n", + "import os, sys\n")) + # Encoding comments which don't work: + # Missing "coding:" prefix: + self._check_encoding("ascii", ( + "#!/usr/local/bin/python\n", + "# latin-1\n", + "import os, sys\n")) + # Encoding comment not on line 1 or 2: + self._check_encoding("ascii", ( + "#!/usr/local/bin/python\n", + "#\n", + "# -*- coding: latin-1 -*-\n", + "import os, sys\n")) + # Unsupported encoding: + self._check_encoding("ascii", ( + "#!/usr/local/bin/python\n", + "# -*- coding: utf-42 -*-\n", + "import os, sys\n"), + possibly_invalid=True) + + def test_bom(self): + """Test the UTF-8 BOM counts as an encoding declaration""" + self._check_encoding("utf-8", ( + "\xef\xbb\xbfimport sys\n", + )) + self._check_encoding("utf-8", ( + "\xef\xbb\xbf# File encoding: UTF-8\n", + )) + self._check_encoding("utf-8", ( + '\xef\xbb\xbf"""Module docstring\n', + '\xef\xbb\xbfThat should just be a ZWNB"""\n')) + self._check_encoding("latin-1", ( + '"""Is this coding: latin-1 or coding: utf-8 instead?\n', + '\xef\xbb\xbfThose should be latin-1 bytes"""\n')) + self._check_encoding("utf-8", ( + "\xef\xbb\xbf# Is the coding: utf-8 or coding: euc-jp instead?\n", + '"""Module docstring say \xe2\x98\x86"""\n')) + + def test_multiple_coding_comments(self): + """Test only the first of multiple coding declarations counts""" + self._check_encoding("iso-8859-1", ( + "# Is the coding: iso-8859-1\n", + "# Or is it coding: iso-8859-2\n"), + possibly_invalid=True) + self._check_encoding("iso-8859-1", ( + "#!/usr/bin/python\n", + "# Is the coding: iso-8859-1\n", + "# Or is it coding: iso-8859-2\n")) + self._check_encoding("iso-8859-1", ( + "# Is the coding: iso-8859-1 or coding: iso-8859-2\n", + "# Or coding: iso-8859-3 or coding: iso-8859-4\n"), + possibly_invalid=True) + self._check_encoding("iso-8859-2", ( + "# Is the coding iso-8859-1 or coding: iso-8859-2\n", + "# Spot the missing colon above\n")) + + +class TestGetSourceEncoding(testtools.TestCase): + """Test reading and caching the encodings of source files""" + + def setUp(self): + testtools.TestCase.setUp(self) + dir = tempfile.mkdtemp() + self.addCleanup(os.rmdir, dir) + self.filename = os.path.join(dir, self.id().rsplit(".", 1)[1] + ".py") + self._written = False + + def put_source(self, text): + f = open(self.filename, "w") + try: + f.write(text) + finally: + f.close() + if not self._written: + self._written = True + self.addCleanup(os.remove, self.filename) + self.addCleanup(linecache.cache.pop, self.filename, None) + + def test_nonexistant_file_as_ascii(self): + """When file can't be found, the encoding should default to ascii""" + self.assertEquals("ascii", _get_source_encoding(self.filename)) + + def test_encoding_is_cached(self): + """The encoding should stay the same if the cache isn't invalidated""" + self.put_source( + "# coding: iso-8859-13\n" + "import os\n") + self.assertEquals("iso-8859-13", _get_source_encoding(self.filename)) + self.put_source( + "# coding: rot-13\n" + "vzcbeg bf\n") + self.assertEquals("iso-8859-13", _get_source_encoding(self.filename)) + + def test_traceback_rechecks_encoding(self): + """A traceback function checks the cache and resets the encoding""" + self.put_source( + "# coding: iso-8859-8\n" + "import os\n") + self.assertEquals("iso-8859-8", _get_source_encoding(self.filename)) + self.put_source( + "# coding: utf-8\n" + "import os\n") + try: + exec (compile("raise RuntimeError\n", self.filename, "exec")) + except RuntimeError: + traceback.extract_tb(sys.exc_info()[2]) + else: + self.fail("RuntimeError not raised") + self.assertEquals("utf-8", _get_source_encoding(self.filename)) + + +class _FakeOutputStream(object): + """A simple file-like object for testing""" + + def __init__(self): + self.writelog = [] + + def write(self, obj): + self.writelog.append(obj) + + +class TestUnicodeOutputStream(testtools.TestCase): + """Test wrapping output streams so they work with arbitrary unicode""" + + uni = _u("pa\u026a\u03b8\u0259n") + + def setUp(self): + super(TestUnicodeOutputStream, self).setUp() + if sys.platform == "cli": + self.skip("IronPython shouldn't wrap streams to do encoding") + + def test_no_encoding_becomes_ascii(self): + """A stream with no encoding attribute gets ascii/replace strings""" + sout = _FakeOutputStream() + unicode_output_stream(sout).write(self.uni) + self.assertEqual([_b("pa???n")], sout.writelog) + + def test_encoding_as_none_becomes_ascii(self): + """A stream with encoding value of None gets ascii/replace strings""" + sout = _FakeOutputStream() + sout.encoding = None + unicode_output_stream(sout).write(self.uni) + self.assertEqual([_b("pa???n")], sout.writelog) + + def test_bogus_encoding_becomes_ascii(self): + """A stream with a bogus encoding gets ascii/replace strings""" + sout = _FakeOutputStream() + sout.encoding = "bogus" + unicode_output_stream(sout).write(self.uni) + self.assertEqual([_b("pa???n")], sout.writelog) + + def test_partial_encoding_replace(self): + """A string which can be partly encoded correctly should be""" + sout = _FakeOutputStream() + sout.encoding = "iso-8859-7" + unicode_output_stream(sout).write(self.uni) + self.assertEqual([_b("pa?\xe8?n")], sout.writelog) + + @testtools.skipIf(str_is_unicode, "Tests behaviour when str is not unicode") + def test_unicode_encodings_wrapped_when_str_is_not_unicode(self): + """A unicode encoding is wrapped but needs no error handler""" + sout = _FakeOutputStream() + sout.encoding = "utf-8" + uout = unicode_output_stream(sout) + self.assertEqual(uout.errors, "strict") + uout.write(self.uni) + self.assertEqual([_b("pa\xc9\xaa\xce\xb8\xc9\x99n")], sout.writelog) + + @testtools.skipIf(not str_is_unicode, "Tests behaviour when str is unicode") + def test_unicode_encodings_not_wrapped_when_str_is_unicode(self): + # No wrapping needed if native str type is unicode + sout = _FakeOutputStream() + sout.encoding = "utf-8" + uout = unicode_output_stream(sout) + self.assertIs(uout, sout) + + def test_stringio(self): + """A StringIO object should maybe get an ascii native str type""" + try: + from cStringIO import StringIO + newio = False + except ImportError: + from io import StringIO + newio = True + sout = StringIO() + soutwrapper = unicode_output_stream(sout) + if newio: + self.expectFailure("Python 3 StringIO expects text not bytes", + self.assertThat, lambda: soutwrapper.write(self.uni), + Not(Raises(MatchesException(TypeError)))) + soutwrapper.write(self.uni) + self.assertEqual("pa???n", sout.getvalue()) + + +class TestTextRepr(testtools.TestCase): + """Ensure in extending repr, basic behaviours are not being broken""" + + ascii_examples = ( + # Single character examples + # C0 control codes should be escaped except multiline \n + ("\x00", "'\\x00'", "'''\\\n\\x00'''"), + ("\b", "'\\x08'", "'''\\\n\\x08'''"), + ("\t", "'\\t'", "'''\\\n\\t'''"), + ("\n", "'\\n'", "'''\\\n\n'''"), + ("\r", "'\\r'", "'''\\\n\\r'''"), + # Quotes and backslash should match normal repr behaviour + ('"', "'\"'", "'''\\\n\"'''"), + ("'", "\"'\"", "'''\\\n\\''''"), + ("\\", "'\\\\'", "'''\\\n\\\\'''"), + # DEL is also unprintable and should be escaped + ("\x7F", "'\\x7f'", "'''\\\n\\x7f'''"), + + # Character combinations that need double checking + ("\r\n", "'\\r\\n'", "'''\\\n\\r\n'''"), + ("\"'", "'\"\\''", "'''\\\n\"\\''''"), + ("'\"", "'\\'\"'", "'''\\\n'\"'''"), + ("\\n", "'\\\\n'", "'''\\\n\\\\n'''"), + ("\\\n", "'\\\\\\n'", "'''\\\n\\\\\n'''"), + ("\\' ", "\"\\\\' \"", "'''\\\n\\\\' '''"), + ("\\'\n", "\"\\\\'\\n\"", "'''\\\n\\\\'\n'''"), + ("\\'\"", "'\\\\\\'\"'", "'''\\\n\\\\'\"'''"), + ("\\'''", "\"\\\\'''\"", "'''\\\n\\\\\\'\\'\\''''"), + ) + + # Bytes with the high bit set should always be escaped + bytes_examples = ( + (_b("\x80"), "'\\x80'", "'''\\\n\\x80'''"), + (_b("\xA0"), "'\\xa0'", "'''\\\n\\xa0'''"), + (_b("\xC0"), "'\\xc0'", "'''\\\n\\xc0'''"), + (_b("\xFF"), "'\\xff'", "'''\\\n\\xff'''"), + (_b("\xC2\xA7"), "'\\xc2\\xa7'", "'''\\\n\\xc2\\xa7'''"), + ) + + # Unicode doesn't escape printable characters as per the Python 3 model + unicode_examples = ( + # C1 codes are unprintable + (_u("\x80"), "'\\x80'", "'''\\\n\\x80'''"), + (_u("\x9F"), "'\\x9f'", "'''\\\n\\x9f'''"), + # No-break space is unprintable + (_u("\xA0"), "'\\xa0'", "'''\\\n\\xa0'''"), + # Letters latin alphabets are printable + (_u("\xA1"), _u("'\xa1'"), _u("'''\\\n\xa1'''")), + (_u("\xFF"), _u("'\xff'"), _u("'''\\\n\xff'''")), + (_u("\u0100"), _u("'\u0100'"), _u("'''\\\n\u0100'''")), + # Line and paragraph seperators are unprintable + (_u("\u2028"), "'\\u2028'", "'''\\\n\\u2028'''"), + (_u("\u2029"), "'\\u2029'", "'''\\\n\\u2029'''"), + # Unpaired surrogates are unprintable + (_u("\uD800"), "'\\ud800'", "'''\\\n\\ud800'''"), + (_u("\uDFFF"), "'\\udfff'", "'''\\\n\\udfff'''"), + # Unprintable general categories not fully tested: Cc, Cf, Co, Cn, Zs + ) + + b_prefix = repr(_b(""))[:-2] + u_prefix = repr(_u(""))[:-2] + + def test_ascii_examples_oneline_bytes(self): + for s, expected, _ in self.ascii_examples: + b = _b(s) + actual = text_repr(b, multiline=False) + # Add self.assertIsInstance check? + self.assertEqual(actual, self.b_prefix + expected) + self.assertEqual(eval(actual), b) + + def test_ascii_examples_oneline_unicode(self): + for s, expected, _ in self.ascii_examples: + u = _u(s) + actual = text_repr(u, multiline=False) + self.assertEqual(actual, self.u_prefix + expected) + self.assertEqual(eval(actual), u) + + def test_ascii_examples_multiline_bytes(self): + for s, _, expected in self.ascii_examples: + b = _b(s) + actual = text_repr(b, multiline=True) + self.assertEqual(actual, self.b_prefix + expected) + self.assertEqual(eval(actual), b) + + def test_ascii_examples_multiline_unicode(self): + for s, _, expected in self.ascii_examples: + u = _u(s) + actual = text_repr(u, multiline=True) + self.assertEqual(actual, self.u_prefix + expected) + self.assertEqual(eval(actual), u) + + def test_ascii_examples_defaultline_bytes(self): + for s, one, multi in self.ascii_examples: + expected = "\n" in s and multi or one + self.assertEqual(text_repr(_b(s)), self.b_prefix + expected) + + def test_ascii_examples_defaultline_unicode(self): + for s, one, multi in self.ascii_examples: + expected = "\n" in s and multi or one + self.assertEqual(text_repr(_u(s)), self.u_prefix + expected) + + def test_bytes_examples_oneline(self): + for b, expected, _ in self.bytes_examples: + actual = text_repr(b, multiline=False) + self.assertEqual(actual, self.b_prefix + expected) + self.assertEqual(eval(actual), b) + + def test_bytes_examples_multiline(self): + for b, _, expected in self.bytes_examples: + actual = text_repr(b, multiline=True) + self.assertEqual(actual, self.b_prefix + expected) + self.assertEqual(eval(actual), b) + + def test_unicode_examples_oneline(self): + for u, expected, _ in self.unicode_examples: + actual = text_repr(u, multiline=False) + self.assertEqual(actual, self.u_prefix + expected) + self.assertEqual(eval(actual), u) + + def test_unicode_examples_multiline(self): + for u, _, expected in self.unicode_examples: + actual = text_repr(u, multiline=True) + self.assertEqual(actual, self.u_prefix + expected) + self.assertEqual(eval(actual), u) + + +def test_suite(): + from unittest import TestLoader + return TestLoader().loadTestsFromName(__name__) diff --git a/test/3rdparty/testtools-0.9.12/testtools/tests/test_content.py b/test/3rdparty/testtools-0.9.12/testtools/tests/test_content.py new file mode 100644 index 00000000000..14f400f04ec --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/tests/test_content.py @@ -0,0 +1,227 @@ +# Copyright (c) 2008-2011 testtools developers. See LICENSE for details. + +import os +import tempfile +import unittest + +from testtools import TestCase +from testtools.compat import ( + _b, + _u, + StringIO, + ) +from testtools.content import ( + attach_file, + Content, + content_from_file, + content_from_stream, + TracebackContent, + text_content, + ) +from testtools.content_type import ( + ContentType, + UTF8_TEXT, + ) +from testtools.matchers import ( + Equals, + MatchesException, + Raises, + raises, + ) +from testtools.tests.helpers import an_exc_info + + +raises_value_error = Raises(MatchesException(ValueError)) + + +class TestContent(TestCase): + + def test___init___None_errors(self): + self.assertThat(lambda: Content(None, None), raises_value_error) + self.assertThat( + lambda: Content(None, lambda: ["traceback"]), raises_value_error) + self.assertThat( + lambda: Content(ContentType("text", "traceback"), None), + raises_value_error) + + def test___init___sets_ivars(self): + content_type = ContentType("foo", "bar") + content = Content(content_type, lambda: ["bytes"]) + self.assertEqual(content_type, content.content_type) + self.assertEqual(["bytes"], list(content.iter_bytes())) + + def test___eq__(self): + content_type = ContentType("foo", "bar") + one_chunk = lambda: [_b("bytes")] + two_chunk = lambda: [_b("by"), _b("tes")] + content1 = Content(content_type, one_chunk) + content2 = Content(content_type, one_chunk) + content3 = Content(content_type, two_chunk) + content4 = Content(content_type, lambda: [_b("by"), _b("te")]) + content5 = Content(ContentType("f", "b"), two_chunk) + self.assertEqual(content1, content2) + self.assertEqual(content1, content3) + self.assertNotEqual(content1, content4) + self.assertNotEqual(content1, content5) + + def test___repr__(self): + content = Content(ContentType("application", "octet-stream"), + lambda: [_b("\x00bin"), _b("ary\xff")]) + self.assertIn("\\x00binary\\xff", repr(content)) + + def test_iter_text_not_text_errors(self): + content_type = ContentType("foo", "bar") + content = Content(content_type, lambda: ["bytes"]) + self.assertThat(content.iter_text, raises_value_error) + + def test_iter_text_decodes(self): + content_type = ContentType("text", "strange", {"charset": "utf8"}) + content = Content( + content_type, lambda: [_u("bytes\xea").encode("utf8")]) + self.assertEqual([_u("bytes\xea")], list(content.iter_text())) + + def test_iter_text_default_charset_iso_8859_1(self): + content_type = ContentType("text", "strange") + text = _u("bytes\xea") + iso_version = text.encode("ISO-8859-1") + content = Content(content_type, lambda: [iso_version]) + self.assertEqual([text], list(content.iter_text())) + + def test_from_file(self): + fd, path = tempfile.mkstemp() + self.addCleanup(os.remove, path) + os.write(fd, _b('some data')) + os.close(fd) + content = content_from_file(path, UTF8_TEXT, chunk_size=2) + self.assertThat( + list(content.iter_bytes()), + Equals([_b('so'), _b('me'), _b(' d'), _b('at'), _b('a')])) + + def test_from_nonexistent_file(self): + directory = tempfile.mkdtemp() + nonexistent = os.path.join(directory, 'nonexistent-file') + content = content_from_file(nonexistent) + self.assertThat(content.iter_bytes, raises(IOError)) + + def test_from_file_default_type(self): + content = content_from_file('/nonexistent/path') + self.assertThat(content.content_type, Equals(UTF8_TEXT)) + + def test_from_file_eager_loading(self): + fd, path = tempfile.mkstemp() + os.write(fd, _b('some data')) + os.close(fd) + content = content_from_file(path, UTF8_TEXT, buffer_now=True) + os.remove(path) + self.assertThat( + ''.join(content.iter_text()), Equals('some data')) + + def test_from_stream(self): + data = StringIO('some data') + content = content_from_stream(data, UTF8_TEXT, chunk_size=2) + self.assertThat( + list(content.iter_bytes()), Equals(['so', 'me', ' d', 'at', 'a'])) + + def test_from_stream_default_type(self): + data = StringIO('some data') + content = content_from_stream(data) + self.assertThat(content.content_type, Equals(UTF8_TEXT)) + + def test_from_stream_eager_loading(self): + fd, path = tempfile.mkstemp() + self.addCleanup(os.remove, path) + os.write(fd, _b('some data')) + stream = open(path, 'rb') + content = content_from_stream(stream, UTF8_TEXT, buffer_now=True) + os.write(fd, _b('more data')) + os.close(fd) + self.assertThat( + ''.join(content.iter_text()), Equals('some data')) + + def test_from_text(self): + data = _u("some data") + expected = Content(UTF8_TEXT, lambda: [data.encode('utf8')]) + self.assertEqual(expected, text_content(data)) + + +class TestTracebackContent(TestCase): + + def test___init___None_errors(self): + self.assertThat( + lambda: TracebackContent(None, None), raises_value_error) + + def test___init___sets_ivars(self): + content = TracebackContent(an_exc_info, self) + content_type = ContentType("text", "x-traceback", + {"language": "python", "charset": "utf8"}) + self.assertEqual(content_type, content.content_type) + result = unittest.TestResult() + expected = result._exc_info_to_string(an_exc_info, self) + self.assertEqual(expected, ''.join(list(content.iter_text()))) + + +class TestAttachFile(TestCase): + + def make_file(self, data): + # GZ 2011-04-21: This helper could be useful for methods above trying + # to use mkstemp, but should handle write failures and + # always close the fd. There must be a better way. + fd, path = tempfile.mkstemp() + self.addCleanup(os.remove, path) + os.write(fd, _b(data)) + os.close(fd) + return path + + def test_simple(self): + class SomeTest(TestCase): + def test_foo(self): + pass + test = SomeTest('test_foo') + data = 'some data' + path = self.make_file(data) + my_content = text_content(data) + attach_file(test, path, name='foo') + self.assertEqual({'foo': my_content}, test.getDetails()) + + def test_optional_name(self): + # If no name is provided, attach_file just uses the base name of the + # file. + class SomeTest(TestCase): + def test_foo(self): + pass + test = SomeTest('test_foo') + path = self.make_file('some data') + base_path = os.path.basename(path) + attach_file(test, path) + self.assertEqual([base_path], list(test.getDetails())) + + def test_lazy_read(self): + class SomeTest(TestCase): + def test_foo(self): + pass + test = SomeTest('test_foo') + path = self.make_file('some data') + attach_file(test, path, name='foo', buffer_now=False) + content = test.getDetails()['foo'] + content_file = open(path, 'w') + content_file.write('new data') + content_file.close() + self.assertEqual(''.join(content.iter_text()), 'new data') + + def test_eager_read_by_default(self): + class SomeTest(TestCase): + def test_foo(self): + pass + test = SomeTest('test_foo') + path = self.make_file('some data') + attach_file(test, path, name='foo') + content = test.getDetails()['foo'] + content_file = open(path, 'w') + content_file.write('new data') + content_file.close() + self.assertEqual(''.join(content.iter_text()), 'some data') + + +def test_suite(): + from unittest import TestLoader + return TestLoader().loadTestsFromName(__name__) diff --git a/test/3rdparty/testtools-0.9.12/testtools/tests/test_content_type.py b/test/3rdparty/testtools-0.9.12/testtools/tests/test_content_type.py new file mode 100644 index 00000000000..9d8c0f6f7af --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/tests/test_content_type.py @@ -0,0 +1,56 @@ +# Copyright (c) 2008 testtools developers. See LICENSE for details. + +from testtools import TestCase +from testtools.matchers import Equals, MatchesException, Raises +from testtools.content_type import ContentType, UTF8_TEXT + + +class TestContentType(TestCase): + + def test___init___None_errors(self): + raises_value_error = Raises(MatchesException(ValueError)) + self.assertThat(lambda:ContentType(None, None), raises_value_error) + self.assertThat(lambda:ContentType(None, "traceback"), + raises_value_error) + self.assertThat(lambda:ContentType("text", None), raises_value_error) + + def test___init___sets_ivars(self): + content_type = ContentType("foo", "bar") + self.assertEqual("foo", content_type.type) + self.assertEqual("bar", content_type.subtype) + self.assertEqual({}, content_type.parameters) + + def test___init___with_parameters(self): + content_type = ContentType("foo", "bar", {"quux": "thing"}) + self.assertEqual({"quux": "thing"}, content_type.parameters) + + def test___eq__(self): + content_type1 = ContentType("foo", "bar", {"quux": "thing"}) + content_type2 = ContentType("foo", "bar", {"quux": "thing"}) + content_type3 = ContentType("foo", "bar", {"quux": "thing2"}) + self.assertTrue(content_type1.__eq__(content_type2)) + self.assertFalse(content_type1.__eq__(content_type3)) + + def test_basic_repr(self): + content_type = ContentType('text', 'plain') + self.assertThat(repr(content_type), Equals('text/plain')) + + def test_extended_repr(self): + content_type = ContentType( + 'text', 'plain', {'foo': 'bar', 'baz': 'qux'}) + self.assertThat( + repr(content_type), Equals('text/plain; foo="bar", baz="qux"')) + + +class TestBuiltinContentTypes(TestCase): + + def test_plain_text(self): + # The UTF8_TEXT content type represents UTF-8 encoded text/plain. + self.assertThat(UTF8_TEXT.type, Equals('text')) + self.assertThat(UTF8_TEXT.subtype, Equals('plain')) + self.assertThat(UTF8_TEXT.parameters, Equals({'charset': 'utf8'})) + + +def test_suite(): + from unittest import TestLoader + return TestLoader().loadTestsFromName(__name__) diff --git a/test/3rdparty/testtools-0.9.12/testtools/tests/test_deferredruntest.py b/test/3rdparty/testtools-0.9.12/testtools/tests/test_deferredruntest.py new file mode 100644 index 00000000000..ab0fd87890a --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/tests/test_deferredruntest.py @@ -0,0 +1,753 @@ +# Copyright (c) 2010-2011 testtools developers. See LICENSE for details. + +"""Tests for the DeferredRunTest single test execution logic.""" + +import os +import signal + +from testtools import ( + skipIf, + TestCase, + TestResult, + ) +from testtools.content import ( + text_content, + ) +from testtools.helpers import try_import +from testtools.matchers import ( + Equals, + KeysEqual, + MatchesException, + Raises, + ) +from testtools.runtest import RunTest +from testtools.testresult.doubles import ExtendedTestResult +from testtools.tests.test_spinner import NeedsTwistedTestCase + +assert_fails_with = try_import('testtools.deferredruntest.assert_fails_with') +AsynchronousDeferredRunTest = try_import( + 'testtools.deferredruntest.AsynchronousDeferredRunTest') +flush_logged_errors = try_import( + 'testtools.deferredruntest.flush_logged_errors') +SynchronousDeferredRunTest = try_import( + 'testtools.deferredruntest.SynchronousDeferredRunTest') + +defer = try_import('twisted.internet.defer') +failure = try_import('twisted.python.failure') +log = try_import('twisted.python.log') +DelayedCall = try_import('twisted.internet.base.DelayedCall') + + +class X(object): + """Tests that we run as part of our tests, nested to avoid discovery.""" + + class Base(TestCase): + def setUp(self): + super(X.Base, self).setUp() + self.calls = ['setUp'] + self.addCleanup(self.calls.append, 'clean-up') + def test_something(self): + self.calls.append('test') + def tearDown(self): + self.calls.append('tearDown') + super(X.Base, self).tearDown() + + class ErrorInSetup(Base): + expected_calls = ['setUp', 'clean-up'] + expected_results = [('addError', RuntimeError)] + def setUp(self): + super(X.ErrorInSetup, self).setUp() + raise RuntimeError("Error in setUp") + + class ErrorInTest(Base): + expected_calls = ['setUp', 'tearDown', 'clean-up'] + expected_results = [('addError', RuntimeError)] + def test_something(self): + raise RuntimeError("Error in test") + + class FailureInTest(Base): + expected_calls = ['setUp', 'tearDown', 'clean-up'] + expected_results = [('addFailure', AssertionError)] + def test_something(self): + self.fail("test failed") + + class ErrorInTearDown(Base): + expected_calls = ['setUp', 'test', 'clean-up'] + expected_results = [('addError', RuntimeError)] + def tearDown(self): + raise RuntimeError("Error in tearDown") + + class ErrorInCleanup(Base): + expected_calls = ['setUp', 'test', 'tearDown', 'clean-up'] + expected_results = [('addError', ZeroDivisionError)] + def test_something(self): + self.calls.append('test') + self.addCleanup(lambda: 1/0) + + class TestIntegration(NeedsTwistedTestCase): + + def assertResultsMatch(self, test, result): + events = list(result._events) + self.assertEqual(('startTest', test), events.pop(0)) + for expected_result in test.expected_results: + result = events.pop(0) + if len(expected_result) == 1: + self.assertEqual((expected_result[0], test), result) + else: + self.assertEqual((expected_result[0], test), result[:2]) + error_type = expected_result[1] + self.assertIn(error_type.__name__, str(result[2])) + self.assertEqual([('stopTest', test)], events) + + def test_runner(self): + result = ExtendedTestResult() + test = self.test_factory('test_something', runTest=self.runner) + test.run(result) + self.assertEqual(test.calls, self.test_factory.expected_calls) + self.assertResultsMatch(test, result) + + +def make_integration_tests(): + from unittest import TestSuite + from testtools import clone_test_with_new_id + runners = [ + ('RunTest', RunTest), + ('SynchronousDeferredRunTest', SynchronousDeferredRunTest), + ('AsynchronousDeferredRunTest', AsynchronousDeferredRunTest), + ] + + tests = [ + X.ErrorInSetup, + X.ErrorInTest, + X.ErrorInTearDown, + X.FailureInTest, + X.ErrorInCleanup, + ] + base_test = X.TestIntegration('test_runner') + integration_tests = [] + for runner_name, runner in runners: + for test in tests: + new_test = clone_test_with_new_id( + base_test, '%s(%s, %s)' % ( + base_test.id(), + runner_name, + test.__name__)) + new_test.test_factory = test + new_test.runner = runner + integration_tests.append(new_test) + return TestSuite(integration_tests) + + +class TestSynchronousDeferredRunTest(NeedsTwistedTestCase): + + def make_result(self): + return ExtendedTestResult() + + def make_runner(self, test): + return SynchronousDeferredRunTest(test, test.exception_handlers) + + def test_success(self): + class SomeCase(TestCase): + def test_success(self): + return defer.succeed(None) + test = SomeCase('test_success') + runner = self.make_runner(test) + result = self.make_result() + runner.run(result) + self.assertThat( + result._events, Equals([ + ('startTest', test), + ('addSuccess', test), + ('stopTest', test)])) + + def test_failure(self): + class SomeCase(TestCase): + def test_failure(self): + return defer.maybeDeferred(self.fail, "Egads!") + test = SomeCase('test_failure') + runner = self.make_runner(test) + result = self.make_result() + runner.run(result) + self.assertThat( + [event[:2] for event in result._events], Equals([ + ('startTest', test), + ('addFailure', test), + ('stopTest', test)])) + + def test_setUp_followed_by_test(self): + class SomeCase(TestCase): + def setUp(self): + super(SomeCase, self).setUp() + return defer.succeed(None) + def test_failure(self): + return defer.maybeDeferred(self.fail, "Egads!") + test = SomeCase('test_failure') + runner = self.make_runner(test) + result = self.make_result() + runner.run(result) + self.assertThat( + [event[:2] for event in result._events], Equals([ + ('startTest', test), + ('addFailure', test), + ('stopTest', test)])) + + +class TestAsynchronousDeferredRunTest(NeedsTwistedTestCase): + + def make_reactor(self): + from twisted.internet import reactor + return reactor + + def make_result(self): + return ExtendedTestResult() + + def make_runner(self, test, timeout=None): + if timeout is None: + timeout = self.make_timeout() + return AsynchronousDeferredRunTest( + test, test.exception_handlers, timeout=timeout) + + def make_timeout(self): + return 0.005 + + def test_setUp_returns_deferred_that_fires_later(self): + # setUp can return a Deferred that might fire at any time. + # AsynchronousDeferredRunTest will not go on to running the test until + # the Deferred returned by setUp actually fires. + call_log = [] + marker = object() + d = defer.Deferred().addCallback(call_log.append) + class SomeCase(TestCase): + def setUp(self): + super(SomeCase, self).setUp() + call_log.append('setUp') + return d + def test_something(self): + call_log.append('test') + def fire_deferred(): + self.assertThat(call_log, Equals(['setUp'])) + d.callback(marker) + test = SomeCase('test_something') + timeout = self.make_timeout() + runner = self.make_runner(test, timeout=timeout) + result = self.make_result() + reactor = self.make_reactor() + reactor.callLater(timeout, fire_deferred) + runner.run(result) + self.assertThat(call_log, Equals(['setUp', marker, 'test'])) + + def test_calls_setUp_test_tearDown_in_sequence(self): + # setUp, the test method and tearDown can all return + # Deferreds. AsynchronousDeferredRunTest will make sure that each of + # these are run in turn, only going on to the next stage once the + # Deferred from the previous stage has fired. + call_log = [] + a = defer.Deferred() + a.addCallback(lambda x: call_log.append('a')) + b = defer.Deferred() + b.addCallback(lambda x: call_log.append('b')) + c = defer.Deferred() + c.addCallback(lambda x: call_log.append('c')) + class SomeCase(TestCase): + def setUp(self): + super(SomeCase, self).setUp() + call_log.append('setUp') + return a + def test_success(self): + call_log.append('test') + return b + def tearDown(self): + super(SomeCase, self).tearDown() + call_log.append('tearDown') + return c + test = SomeCase('test_success') + timeout = self.make_timeout() + runner = self.make_runner(test, timeout) + result = self.make_result() + reactor = self.make_reactor() + def fire_a(): + self.assertThat(call_log, Equals(['setUp'])) + a.callback(None) + def fire_b(): + self.assertThat(call_log, Equals(['setUp', 'a', 'test'])) + b.callback(None) + def fire_c(): + self.assertThat( + call_log, Equals(['setUp', 'a', 'test', 'b', 'tearDown'])) + c.callback(None) + reactor.callLater(timeout * 0.25, fire_a) + reactor.callLater(timeout * 0.5, fire_b) + reactor.callLater(timeout * 0.75, fire_c) + runner.run(result) + self.assertThat( + call_log, Equals(['setUp', 'a', 'test', 'b', 'tearDown', 'c'])) + + def test_async_cleanups(self): + # Cleanups added with addCleanup can return + # Deferreds. AsynchronousDeferredRunTest will run each of them in + # turn. + class SomeCase(TestCase): + def test_whatever(self): + pass + test = SomeCase('test_whatever') + call_log = [] + a = defer.Deferred().addCallback(lambda x: call_log.append('a')) + b = defer.Deferred().addCallback(lambda x: call_log.append('b')) + c = defer.Deferred().addCallback(lambda x: call_log.append('c')) + test.addCleanup(lambda: a) + test.addCleanup(lambda: b) + test.addCleanup(lambda: c) + def fire_a(): + self.assertThat(call_log, Equals([])) + a.callback(None) + def fire_b(): + self.assertThat(call_log, Equals(['a'])) + b.callback(None) + def fire_c(): + self.assertThat(call_log, Equals(['a', 'b'])) + c.callback(None) + timeout = self.make_timeout() + reactor = self.make_reactor() + reactor.callLater(timeout * 0.25, fire_a) + reactor.callLater(timeout * 0.5, fire_b) + reactor.callLater(timeout * 0.75, fire_c) + runner = self.make_runner(test, timeout) + result = self.make_result() + runner.run(result) + self.assertThat(call_log, Equals(['a', 'b', 'c'])) + + def test_clean_reactor(self): + # If there's cruft left over in the reactor, the test fails. + reactor = self.make_reactor() + timeout = self.make_timeout() + class SomeCase(TestCase): + def test_cruft(self): + reactor.callLater(timeout * 10.0, lambda: None) + test = SomeCase('test_cruft') + runner = self.make_runner(test, timeout) + result = self.make_result() + runner.run(result) + self.assertThat( + [event[:2] for event in result._events], + Equals( + [('startTest', test), + ('addError', test), + ('stopTest', test)])) + error = result._events[1][2] + self.assertThat(error, KeysEqual('traceback', 'twisted-log')) + + def test_exports_reactor(self): + # The reactor is set as an attribute on the test case. + reactor = self.make_reactor() + timeout = self.make_timeout() + class SomeCase(TestCase): + def test_cruft(self): + self.assertIs(reactor, self.reactor) + test = SomeCase('test_cruft') + runner = self.make_runner(test, timeout) + result = TestResult() + runner.run(result) + self.assertEqual([], result.errors) + self.assertEqual([], result.failures) + + def test_unhandled_error_from_deferred(self): + # If there's a Deferred with an unhandled error, the test fails. Each + # unhandled error is reported with a separate traceback. + class SomeCase(TestCase): + def test_cruft(self): + # Note we aren't returning the Deferred so that the error will + # be unhandled. + defer.maybeDeferred(lambda: 1/0) + defer.maybeDeferred(lambda: 2/0) + test = SomeCase('test_cruft') + runner = self.make_runner(test) + result = self.make_result() + runner.run(result) + error = result._events[1][2] + result._events[1] = ('addError', test, None) + self.assertThat(result._events, Equals( + [('startTest', test), + ('addError', test, None), + ('stopTest', test)])) + self.assertThat( + error, KeysEqual( + 'twisted-log', + 'unhandled-error-in-deferred', + 'unhandled-error-in-deferred-1', + )) + + def test_unhandled_error_from_deferred_combined_with_error(self): + # If there's a Deferred with an unhandled error, the test fails. Each + # unhandled error is reported with a separate traceback, and the error + # is still reported. + class SomeCase(TestCase): + def test_cruft(self): + # Note we aren't returning the Deferred so that the error will + # be unhandled. + defer.maybeDeferred(lambda: 1/0) + 2 / 0 + test = SomeCase('test_cruft') + runner = self.make_runner(test) + result = self.make_result() + runner.run(result) + error = result._events[1][2] + result._events[1] = ('addError', test, None) + self.assertThat(result._events, Equals( + [('startTest', test), + ('addError', test, None), + ('stopTest', test)])) + self.assertThat( + error, KeysEqual( + 'traceback', + 'twisted-log', + 'unhandled-error-in-deferred', + )) + + @skipIf(os.name != "posix", "Sending SIGINT with os.kill is posix only") + def test_keyboard_interrupt_stops_test_run(self): + # If we get a SIGINT during a test run, the test stops and no more + # tests run. + SIGINT = getattr(signal, 'SIGINT', None) + if not SIGINT: + raise self.skipTest("SIGINT unavailable") + class SomeCase(TestCase): + def test_pause(self): + return defer.Deferred() + test = SomeCase('test_pause') + reactor = self.make_reactor() + timeout = self.make_timeout() + runner = self.make_runner(test, timeout * 5) + result = self.make_result() + reactor.callLater(timeout, os.kill, os.getpid(), SIGINT) + self.assertThat(lambda:runner.run(result), + Raises(MatchesException(KeyboardInterrupt))) + + @skipIf(os.name != "posix", "Sending SIGINT with os.kill is posix only") + def test_fast_keyboard_interrupt_stops_test_run(self): + # If we get a SIGINT during a test run, the test stops and no more + # tests run. + SIGINT = getattr(signal, 'SIGINT', None) + if not SIGINT: + raise self.skipTest("SIGINT unavailable") + class SomeCase(TestCase): + def test_pause(self): + return defer.Deferred() + test = SomeCase('test_pause') + reactor = self.make_reactor() + timeout = self.make_timeout() + runner = self.make_runner(test, timeout * 5) + result = self.make_result() + reactor.callWhenRunning(os.kill, os.getpid(), SIGINT) + self.assertThat(lambda:runner.run(result), + Raises(MatchesException(KeyboardInterrupt))) + + def test_timeout_causes_test_error(self): + # If a test times out, it reports itself as having failed with a + # TimeoutError. + class SomeCase(TestCase): + def test_pause(self): + return defer.Deferred() + test = SomeCase('test_pause') + runner = self.make_runner(test) + result = self.make_result() + runner.run(result) + error = result._events[1][2] + self.assertThat( + [event[:2] for event in result._events], Equals( + [('startTest', test), + ('addError', test), + ('stopTest', test)])) + self.assertIn('TimeoutError', str(error['traceback'])) + + def test_convenient_construction(self): + # As a convenience method, AsynchronousDeferredRunTest has a + # classmethod that returns an AsynchronousDeferredRunTest + # factory. This factory has the same API as the RunTest constructor. + reactor = object() + timeout = object() + handler = object() + factory = AsynchronousDeferredRunTest.make_factory(reactor, timeout) + runner = factory(self, [handler]) + self.assertIs(reactor, runner._reactor) + self.assertIs(timeout, runner._timeout) + self.assertIs(self, runner.case) + self.assertEqual([handler], runner.handlers) + + def test_use_convenient_factory(self): + # Make sure that the factory can actually be used. + factory = AsynchronousDeferredRunTest.make_factory() + class SomeCase(TestCase): + run_tests_with = factory + def test_something(self): + pass + case = SomeCase('test_something') + case.run() + + def test_convenient_construction_default_reactor(self): + # As a convenience method, AsynchronousDeferredRunTest has a + # classmethod that returns an AsynchronousDeferredRunTest + # factory. This factory has the same API as the RunTest constructor. + reactor = object() + handler = object() + factory = AsynchronousDeferredRunTest.make_factory(reactor=reactor) + runner = factory(self, [handler]) + self.assertIs(reactor, runner._reactor) + self.assertIs(self, runner.case) + self.assertEqual([handler], runner.handlers) + + def test_convenient_construction_default_timeout(self): + # As a convenience method, AsynchronousDeferredRunTest has a + # classmethod that returns an AsynchronousDeferredRunTest + # factory. This factory has the same API as the RunTest constructor. + timeout = object() + handler = object() + factory = AsynchronousDeferredRunTest.make_factory(timeout=timeout) + runner = factory(self, [handler]) + self.assertIs(timeout, runner._timeout) + self.assertIs(self, runner.case) + self.assertEqual([handler], runner.handlers) + + def test_convenient_construction_default_debugging(self): + # As a convenience method, AsynchronousDeferredRunTest has a + # classmethod that returns an AsynchronousDeferredRunTest + # factory. This factory has the same API as the RunTest constructor. + handler = object() + factory = AsynchronousDeferredRunTest.make_factory(debug=True) + runner = factory(self, [handler]) + self.assertIs(self, runner.case) + self.assertEqual([handler], runner.handlers) + self.assertEqual(True, runner._debug) + + def test_deferred_error(self): + class SomeTest(TestCase): + def test_something(self): + return defer.maybeDeferred(lambda: 1/0) + test = SomeTest('test_something') + runner = self.make_runner(test) + result = self.make_result() + runner.run(result) + self.assertThat( + [event[:2] for event in result._events], + Equals([ + ('startTest', test), + ('addError', test), + ('stopTest', test)])) + error = result._events[1][2] + self.assertThat(error, KeysEqual('traceback', 'twisted-log')) + + def test_only_addError_once(self): + # Even if the reactor is unclean and the test raises an error and the + # cleanups raise errors, we only called addError once per test. + reactor = self.make_reactor() + class WhenItRains(TestCase): + def it_pours(self): + # Add a dirty cleanup. + self.addCleanup(lambda: 3 / 0) + # Dirty the reactor. + from twisted.internet.protocol import ServerFactory + reactor.listenTCP(0, ServerFactory()) + # Unhandled error. + defer.maybeDeferred(lambda: 2 / 0) + # Actual error. + raise RuntimeError("Excess precipitation") + test = WhenItRains('it_pours') + runner = self.make_runner(test) + result = self.make_result() + runner.run(result) + self.assertThat( + [event[:2] for event in result._events], + Equals([ + ('startTest', test), + ('addError', test), + ('stopTest', test)])) + error = result._events[1][2] + self.assertThat( + error, KeysEqual( + 'traceback', + 'traceback-1', + 'traceback-2', + 'twisted-log', + 'unhandled-error-in-deferred', + )) + + def test_log_err_is_error(self): + # An error logged during the test run is recorded as an error in the + # tests. + class LogAnError(TestCase): + def test_something(self): + try: + 1/0 + except ZeroDivisionError: + f = failure.Failure() + log.err(f) + test = LogAnError('test_something') + runner = self.make_runner(test) + result = self.make_result() + runner.run(result) + self.assertThat( + [event[:2] for event in result._events], + Equals([ + ('startTest', test), + ('addError', test), + ('stopTest', test)])) + error = result._events[1][2] + self.assertThat(error, KeysEqual('logged-error', 'twisted-log')) + + def test_log_err_flushed_is_success(self): + # An error logged during the test run is recorded as an error in the + # tests. + class LogAnError(TestCase): + def test_something(self): + try: + 1/0 + except ZeroDivisionError: + f = failure.Failure() + log.err(f) + flush_logged_errors(ZeroDivisionError) + test = LogAnError('test_something') + runner = self.make_runner(test) + result = self.make_result() + runner.run(result) + self.assertThat( + result._events, + Equals([ + ('startTest', test), + ('addSuccess', test, {'twisted-log': text_content('')}), + ('stopTest', test)])) + + def test_log_in_details(self): + class LogAnError(TestCase): + def test_something(self): + log.msg("foo") + 1/0 + test = LogAnError('test_something') + runner = self.make_runner(test) + result = self.make_result() + runner.run(result) + self.assertThat( + [event[:2] for event in result._events], + Equals([ + ('startTest', test), + ('addError', test), + ('stopTest', test)])) + error = result._events[1][2] + self.assertThat(error, KeysEqual('traceback', 'twisted-log')) + + def test_debugging_unchanged_during_test_by_default(self): + debugging = [(defer.Deferred.debug, DelayedCall.debug)] + class SomeCase(TestCase): + def test_debugging_enabled(self): + debugging.append((defer.Deferred.debug, DelayedCall.debug)) + test = SomeCase('test_debugging_enabled') + runner = AsynchronousDeferredRunTest( + test, handlers=test.exception_handlers, + reactor=self.make_reactor(), timeout=self.make_timeout()) + runner.run(self.make_result()) + self.assertEqual(debugging[0], debugging[1]) + + def test_debugging_enabled_during_test_with_debug_flag(self): + self.patch(defer.Deferred, 'debug', False) + self.patch(DelayedCall, 'debug', False) + debugging = [] + class SomeCase(TestCase): + def test_debugging_enabled(self): + debugging.append((defer.Deferred.debug, DelayedCall.debug)) + test = SomeCase('test_debugging_enabled') + runner = AsynchronousDeferredRunTest( + test, handlers=test.exception_handlers, + reactor=self.make_reactor(), timeout=self.make_timeout(), + debug=True) + runner.run(self.make_result()) + self.assertEqual([(True, True)], debugging) + self.assertEqual(False, defer.Deferred.debug) + self.assertEqual(False, defer.Deferred.debug) + + +class TestAssertFailsWith(NeedsTwistedTestCase): + """Tests for `assert_fails_with`.""" + + if SynchronousDeferredRunTest is not None: + run_tests_with = SynchronousDeferredRunTest + + def test_assert_fails_with_success(self): + # assert_fails_with fails the test if it's given a Deferred that + # succeeds. + marker = object() + d = assert_fails_with(defer.succeed(marker), RuntimeError) + def check_result(failure): + failure.trap(self.failureException) + self.assertThat( + str(failure.value), + Equals("RuntimeError not raised (%r returned)" % (marker,))) + d.addCallbacks( + lambda x: self.fail("Should not have succeeded"), check_result) + return d + + def test_assert_fails_with_success_multiple_types(self): + # assert_fails_with fails the test if it's given a Deferred that + # succeeds. + marker = object() + d = assert_fails_with( + defer.succeed(marker), RuntimeError, ZeroDivisionError) + def check_result(failure): + failure.trap(self.failureException) + self.assertThat( + str(failure.value), + Equals("RuntimeError, ZeroDivisionError not raised " + "(%r returned)" % (marker,))) + d.addCallbacks( + lambda x: self.fail("Should not have succeeded"), check_result) + return d + + def test_assert_fails_with_wrong_exception(self): + # assert_fails_with fails the test if it's given a Deferred that + # succeeds. + d = assert_fails_with( + defer.maybeDeferred(lambda: 1/0), RuntimeError, KeyboardInterrupt) + def check_result(failure): + failure.trap(self.failureException) + lines = str(failure.value).splitlines() + self.assertThat( + lines[:2], + Equals([ + ("ZeroDivisionError raised instead of RuntimeError, " + "KeyboardInterrupt:"), + " Traceback (most recent call last):", + ])) + d.addCallbacks( + lambda x: self.fail("Should not have succeeded"), check_result) + return d + + def test_assert_fails_with_expected_exception(self): + # assert_fails_with calls back with the value of the failure if it's + # one of the expected types of failures. + try: + 1/0 + except ZeroDivisionError: + f = failure.Failure() + d = assert_fails_with(defer.fail(f), ZeroDivisionError) + return d.addCallback(self.assertThat, Equals(f.value)) + + def test_custom_failure_exception(self): + # If assert_fails_with is passed a 'failureException' keyword + # argument, then it will raise that instead of `AssertionError`. + class CustomException(Exception): + pass + marker = object() + d = assert_fails_with( + defer.succeed(marker), RuntimeError, + failureException=CustomException) + def check_result(failure): + failure.trap(CustomException) + self.assertThat( + str(failure.value), + Equals("RuntimeError not raised (%r returned)" % (marker,))) + return d.addCallbacks( + lambda x: self.fail("Should not have succeeded"), check_result) + + +def test_suite(): + from unittest import TestLoader, TestSuite + return TestSuite( + [TestLoader().loadTestsFromName(__name__), + make_integration_tests()]) diff --git a/test/3rdparty/testtools-0.9.12/testtools/tests/test_distutilscmd.py b/test/3rdparty/testtools-0.9.12/testtools/tests/test_distutilscmd.py new file mode 100644 index 00000000000..c485a473d39 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/tests/test_distutilscmd.py @@ -0,0 +1,98 @@ +# Copyright (c) 2010-2011 Testtools authors. See LICENSE for details. + +"""Tests for the distutils test command logic.""" + +from distutils.dist import Distribution + +from testtools.compat import ( + _b, + BytesIO, + ) +from testtools.helpers import try_import +fixtures = try_import('fixtures') + +import testtools +from testtools import TestCase +from testtools.distutilscmd import TestCommand +from testtools.matchers import MatchesRegex + + +if fixtures: + class SampleTestFixture(fixtures.Fixture): + """Creates testtools.runexample temporarily.""" + + def __init__(self): + self.package = fixtures.PythonPackage( + 'runexample', [('__init__.py', _b(""" +from testtools import TestCase + +class TestFoo(TestCase): + def test_bar(self): + pass + def test_quux(self): + pass +def test_suite(): + from unittest import TestLoader + return TestLoader().loadTestsFromName(__name__) +"""))]) + + def setUp(self): + super(SampleTestFixture, self).setUp() + self.useFixture(self.package) + testtools.__path__.append(self.package.base) + self.addCleanup(testtools.__path__.remove, self.package.base) + + +class TestCommandTest(TestCase): + + def setUp(self): + super(TestCommandTest, self).setUp() + if fixtures is None: + self.skipTest("Need fixtures") + + def test_test_module(self): + self.useFixture(SampleTestFixture()) + stream = BytesIO() + dist = Distribution() + dist.script_name = 'setup.py' + dist.script_args = ['test'] + dist.cmdclass = {'test': TestCommand} + dist.command_options = { + 'test': {'test_module': ('command line', 'testtools.runexample')}} + cmd = dist.reinitialize_command('test') + cmd.runner.stdout = stream + dist.run_command('test') + self.assertThat( + stream.getvalue(), + MatchesRegex(_b("""Tests running... + +Ran 2 tests in \\d.\\d\\d\\ds +OK +"""))) + + def test_test_suite(self): + self.useFixture(SampleTestFixture()) + stream = BytesIO() + dist = Distribution() + dist.script_name = 'setup.py' + dist.script_args = ['test'] + dist.cmdclass = {'test': TestCommand} + dist.command_options = { + 'test': { + 'test_suite': ( + 'command line', 'testtools.runexample.test_suite')}} + cmd = dist.reinitialize_command('test') + cmd.runner.stdout = stream + dist.run_command('test') + self.assertThat( + stream.getvalue(), + MatchesRegex(_b("""Tests running... + +Ran 2 tests in \\d.\\d\\d\\ds +OK +"""))) + + +def test_suite(): + from unittest import TestLoader + return TestLoader().loadTestsFromName(__name__) diff --git a/test/3rdparty/testtools-0.9.12/testtools/tests/test_fixturesupport.py b/test/3rdparty/testtools-0.9.12/testtools/tests/test_fixturesupport.py new file mode 100644 index 00000000000..ae6f2ec86e3 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/tests/test_fixturesupport.py @@ -0,0 +1,117 @@ +# Copyright (c) 2010-2011 testtools developers. See LICENSE for details. + +import unittest + +from testtools import ( + TestCase, + content, + content_type, + ) +from testtools.compat import _b, _u +from testtools.helpers import try_import +from testtools.testresult.doubles import ( + ExtendedTestResult, + ) + +fixtures = try_import('fixtures') +LoggingFixture = try_import('fixtures.tests.helpers.LoggingFixture') + + +class TestFixtureSupport(TestCase): + + def setUp(self): + super(TestFixtureSupport, self).setUp() + if fixtures is None or LoggingFixture is None: + self.skipTest("Need fixtures") + + def test_useFixture(self): + fixture = LoggingFixture() + class SimpleTest(TestCase): + def test_foo(self): + self.useFixture(fixture) + result = unittest.TestResult() + SimpleTest('test_foo').run(result) + self.assertTrue(result.wasSuccessful()) + self.assertEqual(['setUp', 'cleanUp'], fixture.calls) + + def test_useFixture_cleanups_raise_caught(self): + calls = [] + def raiser(ignored): + calls.append('called') + raise Exception('foo') + fixture = fixtures.FunctionFixture(lambda:None, raiser) + class SimpleTest(TestCase): + def test_foo(self): + self.useFixture(fixture) + result = unittest.TestResult() + SimpleTest('test_foo').run(result) + self.assertFalse(result.wasSuccessful()) + self.assertEqual(['called'], calls) + + def test_useFixture_details_captured(self): + class DetailsFixture(fixtures.Fixture): + def setUp(self): + fixtures.Fixture.setUp(self) + self.addCleanup(delattr, self, 'content') + self.content = [_b('content available until cleanUp')] + self.addDetail('content', + content.Content(content_type.UTF8_TEXT, self.get_content)) + def get_content(self): + return self.content + fixture = DetailsFixture() + class SimpleTest(TestCase): + def test_foo(self): + self.useFixture(fixture) + # Add a colliding detail (both should show up) + self.addDetail('content', + content.Content(content_type.UTF8_TEXT, lambda:[_b('foo')])) + result = ExtendedTestResult() + SimpleTest('test_foo').run(result) + self.assertEqual('addSuccess', result._events[-2][0]) + details = result._events[-2][2] + self.assertEqual(['content', 'content-1'], sorted(details.keys())) + self.assertEqual('foo', _u('').join(details['content'].iter_text())) + self.assertEqual('content available until cleanUp', + ''.join(details['content-1'].iter_text())) + + def test_useFixture_multiple_details_captured(self): + class DetailsFixture(fixtures.Fixture): + def setUp(self): + fixtures.Fixture.setUp(self) + self.addDetail('aaa', content.text_content("foo")) + self.addDetail('bbb', content.text_content("bar")) + fixture = DetailsFixture() + class SimpleTest(TestCase): + def test_foo(self): + self.useFixture(fixture) + result = ExtendedTestResult() + SimpleTest('test_foo').run(result) + self.assertEqual('addSuccess', result._events[-2][0]) + details = result._events[-2][2] + self.assertEqual(['aaa', 'bbb'], sorted(details)) + self.assertEqual('foo', ''.join(details['aaa'].iter_text())) + self.assertEqual('bar', ''.join(details['bbb'].iter_text())) + + def test_useFixture_details_captured_from_setUp(self): + # Details added during fixture set-up are gathered even if setUp() + # fails with an exception. + class BrokenFixture(fixtures.Fixture): + def setUp(self): + fixtures.Fixture.setUp(self) + self.addDetail('content', content.text_content("foobar")) + raise Exception() + fixture = BrokenFixture() + class SimpleTest(TestCase): + def test_foo(self): + self.useFixture(fixture) + result = ExtendedTestResult() + SimpleTest('test_foo').run(result) + self.assertEqual('addError', result._events[-2][0]) + details = result._events[-2][2] + self.assertEqual(['content', 'traceback'], sorted(details)) + self.assertEqual('foobar', ''.join(details['content'].iter_text())) + + +def test_suite(): + from unittest import TestLoader + return TestLoader().loadTestsFromName(__name__) diff --git a/test/3rdparty/testtools-0.9.12/testtools/tests/test_helpers.py b/test/3rdparty/testtools-0.9.12/testtools/tests/test_helpers.py new file mode 100644 index 00000000000..55de34b7e72 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/tests/test_helpers.py @@ -0,0 +1,240 @@ +# Copyright (c) 2010-2011 testtools developers. See LICENSE for details. + +from testtools import TestCase +from testtools.helpers import ( + try_import, + try_imports, + ) +from testtools.matchers import ( + AllMatch, + AfterPreprocessing, + Equals, + Is, + Not, + ) +from testtools.tests.helpers import ( + FullStackRunTest, + hide_testtools_stack, + is_stack_hidden, + safe_hasattr, + ) + + +def check_error_callback(test, function, arg, expected_error_count, + expect_result): + """General test template for error_callback argument. + + :param test: Test case instance. + :param function: Either try_import or try_imports. + :param arg: Name or names to import. + :param expected_error_count: Expected number of calls to the callback. + :param expect_result: Boolean for whether a module should + ultimately be returned or not. + """ + cb_calls = [] + def cb(e): + test.assertIsInstance(e, ImportError) + cb_calls.append(e) + try: + result = function(arg, error_callback=cb) + except ImportError: + test.assertFalse(expect_result) + else: + if expect_result: + test.assertThat(result, Not(Is(None))) + else: + test.assertThat(result, Is(None)) + test.assertEquals(len(cb_calls), expected_error_count) + + +class TestSafeHasattr(TestCase): + + def test_attribute_not_there(self): + class Foo(object): + pass + self.assertEqual(False, safe_hasattr(Foo(), 'anything')) + + def test_attribute_there(self): + class Foo(object): + pass + foo = Foo() + foo.attribute = None + self.assertEqual(True, safe_hasattr(foo, 'attribute')) + + def test_property_there(self): + class Foo(object): + @property + def attribute(self): + return None + foo = Foo() + self.assertEqual(True, safe_hasattr(foo, 'attribute')) + + def test_property_raises(self): + class Foo(object): + @property + def attribute(self): + 1/0 + foo = Foo() + self.assertRaises(ZeroDivisionError, safe_hasattr, foo, 'attribute') + + +class TestTryImport(TestCase): + + def test_doesnt_exist(self): + # try_import('thing', foo) returns foo if 'thing' doesn't exist. + marker = object() + result = try_import('doesntexist', marker) + self.assertThat(result, Is(marker)) + + def test_None_is_default_alternative(self): + # try_import('thing') returns None if 'thing' doesn't exist. + result = try_import('doesntexist') + self.assertThat(result, Is(None)) + + def test_existing_module(self): + # try_import('thing', foo) imports 'thing' and returns it if it's a + # module that exists. + result = try_import('os', object()) + import os + self.assertThat(result, Is(os)) + + def test_existing_submodule(self): + # try_import('thing.another', foo) imports 'thing' and returns it if + # it's a module that exists. + result = try_import('os.path', object()) + import os + self.assertThat(result, Is(os.path)) + + def test_nonexistent_submodule(self): + # try_import('thing.another', foo) imports 'thing' and returns foo if + # 'another' doesn't exist. + marker = object() + result = try_import('os.doesntexist', marker) + self.assertThat(result, Is(marker)) + + def test_object_from_module(self): + # try_import('thing.object') imports 'thing' and returns + # 'thing.object' if 'thing' is a module and 'object' is not. + result = try_import('os.path.join') + import os + self.assertThat(result, Is(os.path.join)) + + def test_error_callback(self): + # the error callback is called on failures. + check_error_callback(self, try_import, 'doesntexist', 1, False) + + def test_error_callback_missing_module_member(self): + # the error callback is called on failures to find an object + # inside an existing module. + check_error_callback(self, try_import, 'os.nonexistent', 1, False) + + def test_error_callback_not_on_success(self): + # the error callback is not called on success. + check_error_callback(self, try_import, 'os.path', 0, True) + + +class TestTryImports(TestCase): + + def test_doesnt_exist(self): + # try_imports('thing', foo) returns foo if 'thing' doesn't exist. + marker = object() + result = try_imports(['doesntexist'], marker) + self.assertThat(result, Is(marker)) + + def test_fallback(self): + result = try_imports(['doesntexist', 'os']) + import os + self.assertThat(result, Is(os)) + + def test_None_is_default_alternative(self): + # try_imports('thing') returns None if 'thing' doesn't exist. + e = self.assertRaises( + ImportError, try_imports, ['doesntexist', 'noreally']) + self.assertThat( + str(e), + Equals("Could not import any of: doesntexist, noreally")) + + def test_existing_module(self): + # try_imports('thing', foo) imports 'thing' and returns it if it's a + # module that exists. + result = try_imports(['os'], object()) + import os + self.assertThat(result, Is(os)) + + def test_existing_submodule(self): + # try_imports('thing.another', foo) imports 'thing' and returns it if + # it's a module that exists. + result = try_imports(['os.path'], object()) + import os + self.assertThat(result, Is(os.path)) + + def test_nonexistent_submodule(self): + # try_imports('thing.another', foo) imports 'thing' and returns foo if + # 'another' doesn't exist. + marker = object() + result = try_imports(['os.doesntexist'], marker) + self.assertThat(result, Is(marker)) + + def test_fallback_submodule(self): + result = try_imports(['os.doesntexist', 'os.path']) + import os + self.assertThat(result, Is(os.path)) + + def test_error_callback(self): + # One error for every class that doesn't exist. + check_error_callback(self, try_imports, + ['os.doesntexist', 'os.notthiseither'], + 2, False) + check_error_callback(self, try_imports, + ['os.doesntexist', 'os.notthiseither', 'os'], + 2, True) + check_error_callback(self, try_imports, + ['os.path'], + 0, True) + + +import testtools.matchers +import testtools.runtest +import testtools.testcase + + +def StackHidden(is_hidden): + return AllMatch( + AfterPreprocessing( + lambda module: safe_hasattr(module, '__unittest'), + Equals(is_hidden))) + + +class TestStackHiding(TestCase): + + modules = [ + testtools.matchers, + testtools.runtest, + testtools.testcase, + ] + + run_tests_with = FullStackRunTest + + def setUp(self): + super(TestStackHiding, self).setUp() + self.addCleanup(hide_testtools_stack, is_stack_hidden()) + + def test_shown_during_testtools_testsuite(self): + self.assertThat(self.modules, StackHidden(False)) + + def test_is_stack_hidden_consistent_true(self): + hide_testtools_stack(True) + self.assertEqual(True, is_stack_hidden()) + + def test_is_stack_hidden_consistent_false(self): + hide_testtools_stack(False) + self.assertEqual(False, is_stack_hidden()) + + def test_show_stack(self): + hide_testtools_stack(False) + self.assertThat(self.modules, StackHidden(False)) + + +def test_suite(): + from unittest import TestLoader + return TestLoader().loadTestsFromName(__name__) diff --git a/test/3rdparty/testtools-0.9.12/testtools/tests/test_matchers.py b/test/3rdparty/testtools-0.9.12/testtools/tests/test_matchers.py new file mode 100644 index 00000000000..ebdd4a95102 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/tests/test_matchers.py @@ -0,0 +1,1071 @@ +# Copyright (c) 2008-2011 testtools developers. See LICENSE for details. + +"""Tests for matchers.""" + +import doctest +import re +import sys + +from testtools import ( + Matcher, # check that Matcher is exposed at the top level for docs. + TestCase, + ) +from testtools.compat import ( + StringIO, + str_is_unicode, + text_repr, + _b, + _u, + ) +from testtools.matchers import ( + AfterPreprocessing, + AllMatch, + Annotate, + AnnotatedMismatch, + _BinaryMismatch, + Contains, + Equals, + DocTestMatches, + DoesNotEndWith, + DoesNotStartWith, + EndsWith, + KeysEqual, + Is, + IsInstance, + LessThan, + GreaterThan, + MatchesAny, + MatchesAll, + MatchesException, + MatchesListwise, + MatchesRegex, + MatchesSetwise, + MatchesStructure, + Mismatch, + MismatchDecorator, + MismatchError, + Not, + NotEquals, + Raises, + raises, + StartsWith, + ) +from testtools.tests.helpers import FullStackRunTest + +# Silence pyflakes. +Matcher + + +class TestMismatch(TestCase): + + run_tests_with = FullStackRunTest + + def test_constructor_arguments(self): + mismatch = Mismatch("some description", {'detail': "things"}) + self.assertEqual("some description", mismatch.describe()) + self.assertEqual({'detail': "things"}, mismatch.get_details()) + + def test_constructor_no_arguments(self): + mismatch = Mismatch() + self.assertThat(mismatch.describe, + Raises(MatchesException(NotImplementedError))) + self.assertEqual({}, mismatch.get_details()) + + +class TestMismatchError(TestCase): + + def test_is_assertion_error(self): + # MismatchError is an AssertionError, so that most of the time, it + # looks like a test failure, rather than an error. + def raise_mismatch_error(): + raise MismatchError(2, Equals(3), Equals(3).match(2)) + self.assertRaises(AssertionError, raise_mismatch_error) + + def test_default_description_is_mismatch(self): + mismatch = Equals(3).match(2) + e = MismatchError(2, Equals(3), mismatch) + self.assertEqual(mismatch.describe(), str(e)) + + def test_default_description_unicode(self): + matchee = _u('\xa7') + matcher = Equals(_u('a')) + mismatch = matcher.match(matchee) + e = MismatchError(matchee, matcher, mismatch) + self.assertEqual(mismatch.describe(), str(e)) + + def test_verbose_description(self): + matchee = 2 + matcher = Equals(3) + mismatch = matcher.match(2) + e = MismatchError(matchee, matcher, mismatch, True) + expected = ( + 'Match failed. Matchee: %r\n' + 'Matcher: %s\n' + 'Difference: %s\n' % ( + matchee, + matcher, + matcher.match(matchee).describe(), + )) + self.assertEqual(expected, str(e)) + + def test_verbose_unicode(self): + # When assertThat is given matchees or matchers that contain non-ASCII + # unicode strings, we can still provide a meaningful error. + matchee = _u('\xa7') + matcher = Equals(_u('a')) + mismatch = matcher.match(matchee) + expected = ( + 'Match failed. Matchee: %s\n' + 'Matcher: %s\n' + 'Difference: %s\n' % ( + text_repr(matchee), + matcher, + mismatch.describe(), + )) + e = MismatchError(matchee, matcher, mismatch, True) + if str_is_unicode: + actual = str(e) + else: + actual = unicode(e) + # Using str() should still work, and return ascii only + self.assertEqual( + expected.replace(matchee, matchee.encode("unicode-escape")), + str(e).decode("ascii")) + self.assertEqual(expected, actual) + + +class Test_BinaryMismatch(TestCase): + """Mismatches from binary comparisons need useful describe output""" + + _long_string = "This is a longish multiline non-ascii string\n\xa7" + _long_b = _b(_long_string) + _long_u = _u(_long_string) + + def test_short_objects(self): + o1, o2 = object(), object() + mismatch = _BinaryMismatch(o1, "!~", o2) + self.assertEqual(mismatch.describe(), "%r !~ %r" % (o1, o2)) + + def test_short_mixed_strings(self): + b, u = _b("\xa7"), _u("\xa7") + mismatch = _BinaryMismatch(b, "!~", u) + self.assertEqual(mismatch.describe(), "%r !~ %r" % (b, u)) + + def test_long_bytes(self): + one_line_b = self._long_b.replace(_b("\n"), _b(" ")) + mismatch = _BinaryMismatch(one_line_b, "!~", self._long_b) + self.assertEqual(mismatch.describe(), + "%s:\nreference = %s\nactual = %s\n" % ("!~", + text_repr(one_line_b), + text_repr(self._long_b, multiline=True))) + + def test_long_unicode(self): + one_line_u = self._long_u.replace("\n", " ") + mismatch = _BinaryMismatch(one_line_u, "!~", self._long_u) + self.assertEqual(mismatch.describe(), + "%s:\nreference = %s\nactual = %s\n" % ("!~", + text_repr(one_line_u), + text_repr(self._long_u, multiline=True))) + + def test_long_mixed_strings(self): + mismatch = _BinaryMismatch(self._long_b, "!~", self._long_u) + self.assertEqual(mismatch.describe(), + "%s:\nreference = %s\nactual = %s\n" % ("!~", + text_repr(self._long_b, multiline=True), + text_repr(self._long_u, multiline=True))) + + def test_long_bytes_and_object(self): + obj = object() + mismatch = _BinaryMismatch(self._long_b, "!~", obj) + self.assertEqual(mismatch.describe(), + "%s:\nreference = %s\nactual = %s\n" % ("!~", + text_repr(self._long_b, multiline=True), + repr(obj))) + + def test_long_unicode_and_object(self): + obj = object() + mismatch = _BinaryMismatch(self._long_u, "!~", obj) + self.assertEqual(mismatch.describe(), + "%s:\nreference = %s\nactual = %s\n" % ("!~", + text_repr(self._long_u, multiline=True), + repr(obj))) + + +class TestMatchersInterface(object): + + run_tests_with = FullStackRunTest + + def test_matches_match(self): + matcher = self.matches_matcher + matches = self.matches_matches + mismatches = self.matches_mismatches + for candidate in matches: + self.assertEqual(None, matcher.match(candidate)) + for candidate in mismatches: + mismatch = matcher.match(candidate) + self.assertNotEqual(None, mismatch) + self.assertNotEqual(None, getattr(mismatch, 'describe', None)) + + def test__str__(self): + # [(expected, object to __str__)]. + examples = self.str_examples + for expected, matcher in examples: + self.assertThat(matcher, DocTestMatches(expected)) + + def test_describe_difference(self): + # [(expected, matchee, matcher), ...] + examples = self.describe_examples + for difference, matchee, matcher in examples: + mismatch = matcher.match(matchee) + self.assertEqual(difference, mismatch.describe()) + + def test_mismatch_details(self): + # The mismatch object must provide get_details, which must return a + # dictionary mapping names to Content objects. + examples = self.describe_examples + for difference, matchee, matcher in examples: + mismatch = matcher.match(matchee) + details = mismatch.get_details() + self.assertEqual(dict(details), details) + + +class TestDocTestMatchesInterface(TestCase, TestMatchersInterface): + + matches_matcher = DocTestMatches("Ran 1 test in ...s", doctest.ELLIPSIS) + matches_matches = ["Ran 1 test in 0.000s", "Ran 1 test in 1.234s"] + matches_mismatches = ["Ran 1 tests in 0.000s", "Ran 2 test in 0.000s"] + + str_examples = [("DocTestMatches('Ran 1 test in ...s\\n')", + DocTestMatches("Ran 1 test in ...s")), + ("DocTestMatches('foo\\n', flags=8)", DocTestMatches("foo", flags=8)), + ] + + describe_examples = [('Expected:\n Ran 1 tests in ...s\nGot:\n' + ' Ran 1 test in 0.123s\n', "Ran 1 test in 0.123s", + DocTestMatches("Ran 1 tests in ...s", doctest.ELLIPSIS))] + + +class TestDocTestMatchesInterfaceUnicode(TestCase, TestMatchersInterface): + + matches_matcher = DocTestMatches(_u("\xa7..."), doctest.ELLIPSIS) + matches_matches = [_u("\xa7"), _u("\xa7 more\n")] + matches_mismatches = ["\\xa7", _u("more \xa7"), _u("\n\xa7")] + + str_examples = [("DocTestMatches(%r)" % (_u("\xa7\n"),), + DocTestMatches(_u("\xa7"))), + ] + + describe_examples = [( + _u("Expected:\n \xa7\nGot:\n a\n"), + "a", + DocTestMatches(_u("\xa7"), doctest.ELLIPSIS))] + + +class TestDocTestMatchesSpecific(TestCase): + + run_tests_with = FullStackRunTest + + def test___init__simple(self): + matcher = DocTestMatches("foo") + self.assertEqual("foo\n", matcher.want) + + def test___init__flags(self): + matcher = DocTestMatches("bar\n", doctest.ELLIPSIS) + self.assertEqual("bar\n", matcher.want) + self.assertEqual(doctest.ELLIPSIS, matcher.flags) + + def test_describe_non_ascii_bytes(self): + """Even with bytestrings, the mismatch should be coercible to unicode + + DocTestMatches is intended for text, but the Python 2 str type also + permits arbitrary binary inputs. This is a slightly bogus thing to do, + and under Python 3 using bytes objects will reasonably raise an error. + """ + header = _b("\x89PNG\r\n\x1a\n...") + if str_is_unicode: + self.assertRaises(TypeError, + DocTestMatches, header, doctest.ELLIPSIS) + return + matcher = DocTestMatches(header, doctest.ELLIPSIS) + mismatch = matcher.match(_b("GIF89a\1\0\1\0\0\0\0;")) + # Must be treatable as unicode text, the exact output matters less + self.assertTrue(unicode(mismatch.describe())) + + +class TestEqualsInterface(TestCase, TestMatchersInterface): + + matches_matcher = Equals(1) + matches_matches = [1] + matches_mismatches = [2] + + str_examples = [("Equals(1)", Equals(1)), ("Equals('1')", Equals('1'))] + + describe_examples = [("1 != 2", 2, Equals(1))] + + +class TestNotEqualsInterface(TestCase, TestMatchersInterface): + + matches_matcher = NotEquals(1) + matches_matches = [2] + matches_mismatches = [1] + + str_examples = [ + ("NotEquals(1)", NotEquals(1)), ("NotEquals('1')", NotEquals('1'))] + + describe_examples = [("1 == 1", 1, NotEquals(1))] + + +class TestIsInterface(TestCase, TestMatchersInterface): + + foo = object() + bar = object() + + matches_matcher = Is(foo) + matches_matches = [foo] + matches_mismatches = [bar, 1] + + str_examples = [("Is(2)", Is(2))] + + describe_examples = [("1 is not 2", 2, Is(1))] + + +class TestIsInstanceInterface(TestCase, TestMatchersInterface): + + class Foo:pass + + matches_matcher = IsInstance(Foo) + matches_matches = [Foo()] + matches_mismatches = [object(), 1, Foo] + + str_examples = [ + ("IsInstance(str)", IsInstance(str)), + ("IsInstance(str, int)", IsInstance(str, int)), + ] + + describe_examples = [ + ("'foo' is not an instance of int", 'foo', IsInstance(int)), + ("'foo' is not an instance of any of (int, type)", 'foo', + IsInstance(int, type)), + ] + + +class TestLessThanInterface(TestCase, TestMatchersInterface): + + matches_matcher = LessThan(4) + matches_matches = [-5, 3] + matches_mismatches = [4, 5, 5000] + + str_examples = [ + ("LessThan(12)", LessThan(12)), + ] + + describe_examples = [ + ('4 is not > 5', 5, LessThan(4)), + ('4 is not > 4', 4, LessThan(4)), + ] + + +class TestGreaterThanInterface(TestCase, TestMatchersInterface): + + matches_matcher = GreaterThan(4) + matches_matches = [5, 8] + matches_mismatches = [-2, 0, 4] + + str_examples = [ + ("GreaterThan(12)", GreaterThan(12)), + ] + + describe_examples = [ + ('5 is not < 4', 4, GreaterThan(5)), + ('4 is not < 4', 4, GreaterThan(4)), + ] + + +class TestContainsInterface(TestCase, TestMatchersInterface): + + matches_matcher = Contains('foo') + matches_matches = ['foo', 'afoo', 'fooa'] + matches_mismatches = ['f', 'fo', 'oo', 'faoo', 'foao'] + + str_examples = [ + ("Contains(1)", Contains(1)), + ("Contains('foo')", Contains('foo')), + ] + + describe_examples = [("1 not in 2", 2, Contains(1))] + + +def make_error(type, *args, **kwargs): + try: + raise type(*args, **kwargs) + except type: + return sys.exc_info() + + +class TestMatchesExceptionInstanceInterface(TestCase, TestMatchersInterface): + + matches_matcher = MatchesException(ValueError("foo")) + error_foo = make_error(ValueError, 'foo') + error_bar = make_error(ValueError, 'bar') + error_base_foo = make_error(Exception, 'foo') + matches_matches = [error_foo] + matches_mismatches = [error_bar, error_base_foo] + + str_examples = [ + ("MatchesException(Exception('foo',))", + MatchesException(Exception('foo'))) + ] + describe_examples = [ + ("%r is not a %r" % (Exception, ValueError), + error_base_foo, + MatchesException(ValueError("foo"))), + ("ValueError('bar',) has different arguments to ValueError('foo',).", + error_bar, + MatchesException(ValueError("foo"))), + ] + + +class TestMatchesExceptionTypeInterface(TestCase, TestMatchersInterface): + + matches_matcher = MatchesException(ValueError) + error_foo = make_error(ValueError, 'foo') + error_sub = make_error(UnicodeError, 'bar') + error_base_foo = make_error(Exception, 'foo') + matches_matches = [error_foo, error_sub] + matches_mismatches = [error_base_foo] + + str_examples = [ + ("MatchesException(%r)" % Exception, + MatchesException(Exception)) + ] + describe_examples = [ + ("%r is not a %r" % (Exception, ValueError), + error_base_foo, + MatchesException(ValueError)), + ] + + +class TestMatchesExceptionTypeReInterface(TestCase, TestMatchersInterface): + + matches_matcher = MatchesException(ValueError, 'fo.') + error_foo = make_error(ValueError, 'foo') + error_sub = make_error(UnicodeError, 'foo') + error_bar = make_error(ValueError, 'bar') + matches_matches = [error_foo, error_sub] + matches_mismatches = [error_bar] + + str_examples = [ + ("MatchesException(%r)" % Exception, + MatchesException(Exception, 'fo.')) + ] + describe_examples = [ + ("'bar' does not match /fo./", + error_bar, MatchesException(ValueError, "fo.")), + ] + + +class TestMatchesExceptionTypeMatcherInterface(TestCase, TestMatchersInterface): + + matches_matcher = MatchesException( + ValueError, AfterPreprocessing(str, Equals('foo'))) + error_foo = make_error(ValueError, 'foo') + error_sub = make_error(UnicodeError, 'foo') + error_bar = make_error(ValueError, 'bar') + matches_matches = [error_foo, error_sub] + matches_mismatches = [error_bar] + + str_examples = [ + ("MatchesException(%r)" % Exception, + MatchesException(Exception, Equals('foo'))) + ] + describe_examples = [ + ("5 != %r" % (error_bar[1],), + error_bar, MatchesException(ValueError, Equals(5))), + ] + + +class TestNotInterface(TestCase, TestMatchersInterface): + + matches_matcher = Not(Equals(1)) + matches_matches = [2] + matches_mismatches = [1] + + str_examples = [ + ("Not(Equals(1))", Not(Equals(1))), + ("Not(Equals('1'))", Not(Equals('1')))] + + describe_examples = [('1 matches Equals(1)', 1, Not(Equals(1)))] + + +class TestMatchersAnyInterface(TestCase, TestMatchersInterface): + + matches_matcher = MatchesAny(DocTestMatches("1"), DocTestMatches("2")) + matches_matches = ["1", "2"] + matches_mismatches = ["3"] + + str_examples = [( + "MatchesAny(DocTestMatches('1\\n'), DocTestMatches('2\\n'))", + MatchesAny(DocTestMatches("1"), DocTestMatches("2"))), + ] + + describe_examples = [("""Differences: [ +Expected: + 1 +Got: + 3 + +Expected: + 2 +Got: + 3 + +]""", + "3", MatchesAny(DocTestMatches("1"), DocTestMatches("2")))] + + +class TestMatchesAllInterface(TestCase, TestMatchersInterface): + + matches_matcher = MatchesAll(NotEquals(1), NotEquals(2)) + matches_matches = [3, 4] + matches_mismatches = [1, 2] + + str_examples = [ + ("MatchesAll(NotEquals(1), NotEquals(2))", + MatchesAll(NotEquals(1), NotEquals(2)))] + + describe_examples = [("""Differences: [ +1 == 1 +]""", + 1, MatchesAll(NotEquals(1), NotEquals(2)))] + + +class TestKeysEqual(TestCase, TestMatchersInterface): + + matches_matcher = KeysEqual('foo', 'bar') + matches_matches = [ + {'foo': 0, 'bar': 1}, + ] + matches_mismatches = [ + {}, + {'foo': 0}, + {'bar': 1}, + {'foo': 0, 'bar': 1, 'baz': 2}, + {'a': None, 'b': None, 'c': None}, + ] + + str_examples = [ + ("KeysEqual('foo', 'bar')", KeysEqual('foo', 'bar')), + ] + + describe_examples = [ + ("['bar', 'foo'] does not match {'baz': 2, 'foo': 0, 'bar': 1}: " + "Keys not equal", + {'foo': 0, 'bar': 1, 'baz': 2}, KeysEqual('foo', 'bar')), + ] + + +class TestAnnotate(TestCase, TestMatchersInterface): + + matches_matcher = Annotate("foo", Equals(1)) + matches_matches = [1] + matches_mismatches = [2] + + str_examples = [ + ("Annotate('foo', Equals(1))", Annotate("foo", Equals(1)))] + + describe_examples = [("1 != 2: foo", 2, Annotate('foo', Equals(1)))] + + def test_if_message_no_message(self): + # Annotate.if_message returns the given matcher if there is no + # message. + matcher = Equals(1) + not_annotated = Annotate.if_message('', matcher) + self.assertIs(matcher, not_annotated) + + def test_if_message_given_message(self): + # Annotate.if_message returns an annotated version of the matcher if a + # message is provided. + matcher = Equals(1) + expected = Annotate('foo', matcher) + annotated = Annotate.if_message('foo', matcher) + self.assertThat( + annotated, + MatchesStructure.fromExample(expected, 'annotation', 'matcher')) + + +class TestAnnotatedMismatch(TestCase): + + run_tests_with = FullStackRunTest + + def test_forwards_details(self): + x = Mismatch('description', {'foo': 'bar'}) + annotated = AnnotatedMismatch("annotation", x) + self.assertEqual(x.get_details(), annotated.get_details()) + + +class TestRaisesInterface(TestCase, TestMatchersInterface): + + matches_matcher = Raises() + def boom(): + raise Exception('foo') + matches_matches = [boom] + matches_mismatches = [lambda:None] + + # Tricky to get function objects to render constantly, and the interfaces + # helper uses assertEqual rather than (for instance) DocTestMatches. + str_examples = [] + + describe_examples = [] + + +class TestRaisesExceptionMatcherInterface(TestCase, TestMatchersInterface): + + matches_matcher = Raises( + exception_matcher=MatchesException(Exception('foo'))) + def boom_bar(): + raise Exception('bar') + def boom_foo(): + raise Exception('foo') + matches_matches = [boom_foo] + matches_mismatches = [lambda:None, boom_bar] + + # Tricky to get function objects to render constantly, and the interfaces + # helper uses assertEqual rather than (for instance) DocTestMatches. + str_examples = [] + + describe_examples = [] + + +class TestRaisesBaseTypes(TestCase): + + run_tests_with = FullStackRunTest + + def raiser(self): + raise KeyboardInterrupt('foo') + + def test_KeyboardInterrupt_matched(self): + # When KeyboardInterrupt is matched, it is swallowed. + matcher = Raises(MatchesException(KeyboardInterrupt)) + self.assertThat(self.raiser, matcher) + + def test_KeyboardInterrupt_propogates(self): + # The default 'it raised' propogates KeyboardInterrupt. + match_keyb = Raises(MatchesException(KeyboardInterrupt)) + def raise_keyb_from_match(): + matcher = Raises() + matcher.match(self.raiser) + self.assertThat(raise_keyb_from_match, match_keyb) + + def test_KeyboardInterrupt_match_Exception_propogates(self): + # If the raised exception isn't matched, and it is not a subclass of + # Exception, it is propogated. + match_keyb = Raises(MatchesException(KeyboardInterrupt)) + def raise_keyb_from_match(): + if sys.version_info > (2, 5): + matcher = Raises(MatchesException(Exception)) + else: + # On Python 2.4 KeyboardInterrupt is a StandardError subclass + # but should propogate from less generic exception matchers + matcher = Raises(MatchesException(EnvironmentError)) + matcher.match(self.raiser) + self.assertThat(raise_keyb_from_match, match_keyb) + + +class TestRaisesConvenience(TestCase): + + run_tests_with = FullStackRunTest + + def test_exc_type(self): + self.assertThat(lambda: 1/0, raises(ZeroDivisionError)) + + def test_exc_value(self): + e = RuntimeError("You lose!") + def raiser(): + raise e + self.assertThat(raiser, raises(e)) + + +class DoesNotStartWithTests(TestCase): + + run_tests_with = FullStackRunTest + + def test_describe(self): + mismatch = DoesNotStartWith("fo", "bo") + self.assertEqual("'fo' does not start with 'bo'.", mismatch.describe()) + + def test_describe_non_ascii_unicode(self): + string = _u("A\xA7") + suffix = _u("B\xA7") + mismatch = DoesNotStartWith(string, suffix) + self.assertEqual("%s does not start with %s." % ( + text_repr(string), text_repr(suffix)), + mismatch.describe()) + + def test_describe_non_ascii_bytes(self): + string = _b("A\xA7") + suffix = _b("B\xA7") + mismatch = DoesNotStartWith(string, suffix) + self.assertEqual("%r does not start with %r." % (string, suffix), + mismatch.describe()) + + +class StartsWithTests(TestCase): + + run_tests_with = FullStackRunTest + + def test_str(self): + matcher = StartsWith("bar") + self.assertEqual("StartsWith('bar')", str(matcher)) + + def test_str_with_bytes(self): + b = _b("\xA7") + matcher = StartsWith(b) + self.assertEqual("StartsWith(%r)" % (b,), str(matcher)) + + def test_str_with_unicode(self): + u = _u("\xA7") + matcher = StartsWith(u) + self.assertEqual("StartsWith(%r)" % (u,), str(matcher)) + + def test_match(self): + matcher = StartsWith("bar") + self.assertIs(None, matcher.match("barf")) + + def test_mismatch_returns_does_not_start_with(self): + matcher = StartsWith("bar") + self.assertIsInstance(matcher.match("foo"), DoesNotStartWith) + + def test_mismatch_sets_matchee(self): + matcher = StartsWith("bar") + mismatch = matcher.match("foo") + self.assertEqual("foo", mismatch.matchee) + + def test_mismatch_sets_expected(self): + matcher = StartsWith("bar") + mismatch = matcher.match("foo") + self.assertEqual("bar", mismatch.expected) + + +class DoesNotEndWithTests(TestCase): + + run_tests_with = FullStackRunTest + + def test_describe(self): + mismatch = DoesNotEndWith("fo", "bo") + self.assertEqual("'fo' does not end with 'bo'.", mismatch.describe()) + + def test_describe_non_ascii_unicode(self): + string = _u("A\xA7") + suffix = _u("B\xA7") + mismatch = DoesNotEndWith(string, suffix) + self.assertEqual("%s does not end with %s." % ( + text_repr(string), text_repr(suffix)), + mismatch.describe()) + + def test_describe_non_ascii_bytes(self): + string = _b("A\xA7") + suffix = _b("B\xA7") + mismatch = DoesNotEndWith(string, suffix) + self.assertEqual("%r does not end with %r." % (string, suffix), + mismatch.describe()) + + +class EndsWithTests(TestCase): + + run_tests_with = FullStackRunTest + + def test_str(self): + matcher = EndsWith("bar") + self.assertEqual("EndsWith('bar')", str(matcher)) + + def test_str_with_bytes(self): + b = _b("\xA7") + matcher = EndsWith(b) + self.assertEqual("EndsWith(%r)" % (b,), str(matcher)) + + def test_str_with_unicode(self): + u = _u("\xA7") + matcher = EndsWith(u) + self.assertEqual("EndsWith(%r)" % (u,), str(matcher)) + + def test_match(self): + matcher = EndsWith("arf") + self.assertIs(None, matcher.match("barf")) + + def test_mismatch_returns_does_not_end_with(self): + matcher = EndsWith("bar") + self.assertIsInstance(matcher.match("foo"), DoesNotEndWith) + + def test_mismatch_sets_matchee(self): + matcher = EndsWith("bar") + mismatch = matcher.match("foo") + self.assertEqual("foo", mismatch.matchee) + + def test_mismatch_sets_expected(self): + matcher = EndsWith("bar") + mismatch = matcher.match("foo") + self.assertEqual("bar", mismatch.expected) + + +def run_doctest(obj, name): + p = doctest.DocTestParser() + t = p.get_doctest( + obj.__doc__, sys.modules[obj.__module__].__dict__, name, '', 0) + r = doctest.DocTestRunner() + output = StringIO() + r.run(t, out=output.write) + return r.failures, output.getvalue() + + +class TestMatchesListwise(TestCase): + + run_tests_with = FullStackRunTest + + def test_docstring(self): + failure_count, output = run_doctest( + MatchesListwise, "MatchesListwise") + if failure_count: + self.fail("Doctest failed with %s" % output) + + +class TestMatchesStructure(TestCase, TestMatchersInterface): + + class SimpleClass: + def __init__(self, x, y): + self.x = x + self.y = y + + matches_matcher = MatchesStructure(x=Equals(1), y=Equals(2)) + matches_matches = [SimpleClass(1, 2)] + matches_mismatches = [ + SimpleClass(2, 2), + SimpleClass(1, 1), + SimpleClass(3, 3), + ] + + str_examples = [ + ("MatchesStructure(x=Equals(1))", MatchesStructure(x=Equals(1))), + ("MatchesStructure(y=Equals(2))", MatchesStructure(y=Equals(2))), + ("MatchesStructure(x=Equals(1), y=Equals(2))", + MatchesStructure(x=Equals(1), y=Equals(2))), + ] + + describe_examples = [ + ("""\ +Differences: [ +3 != 1: x +]""", SimpleClass(1, 2), MatchesStructure(x=Equals(3), y=Equals(2))), + ("""\ +Differences: [ +3 != 2: y +]""", SimpleClass(1, 2), MatchesStructure(x=Equals(1), y=Equals(3))), + ("""\ +Differences: [ +0 != 1: x +0 != 2: y +]""", SimpleClass(1, 2), MatchesStructure(x=Equals(0), y=Equals(0))), + ] + + def test_fromExample(self): + self.assertThat( + self.SimpleClass(1, 2), + MatchesStructure.fromExample(self.SimpleClass(1, 3), 'x')) + + def test_byEquality(self): + self.assertThat( + self.SimpleClass(1, 2), + MatchesStructure.byEquality(x=1)) + + def test_withStructure(self): + self.assertThat( + self.SimpleClass(1, 2), + MatchesStructure.byMatcher(LessThan, x=2)) + + def test_update(self): + self.assertThat( + self.SimpleClass(1, 2), + MatchesStructure(x=NotEquals(1)).update(x=Equals(1))) + + def test_update_none(self): + self.assertThat( + self.SimpleClass(1, 2), + MatchesStructure(x=Equals(1), z=NotEquals(42)).update( + z=None)) + + +class TestMatchesRegex(TestCase, TestMatchersInterface): + + matches_matcher = MatchesRegex('a|b') + matches_matches = ['a', 'b'] + matches_mismatches = ['c'] + + str_examples = [ + ("MatchesRegex('a|b')", MatchesRegex('a|b')), + ("MatchesRegex('a|b', re.M)", MatchesRegex('a|b', re.M)), + ("MatchesRegex('a|b', re.I|re.M)", MatchesRegex('a|b', re.I|re.M)), + ("MatchesRegex(%r)" % (_b("\xA7"),), MatchesRegex(_b("\xA7"))), + ("MatchesRegex(%r)" % (_u("\xA7"),), MatchesRegex(_u("\xA7"))), + ] + + describe_examples = [ + ("'c' does not match /a|b/", 'c', MatchesRegex('a|b')), + ("'c' does not match /a\d/", 'c', MatchesRegex(r'a\d')), + ("%r does not match /\\s+\\xa7/" % (_b('c'),), + _b('c'), MatchesRegex(_b("\\s+\xA7"))), + ("%r does not match /\\s+\\xa7/" % (_u('c'),), + _u('c'), MatchesRegex(_u("\\s+\xA7"))), + ] + + +class TestMatchesSetwise(TestCase): + + run_tests_with = FullStackRunTest + + def assertMismatchWithDescriptionMatching(self, value, matcher, + description_matcher): + mismatch = matcher.match(value) + if mismatch is None: + self.fail("%s matched %s" % (matcher, value)) + actual_description = mismatch.describe() + self.assertThat( + actual_description, + Annotate( + "%s matching %s" % (matcher, value), + description_matcher)) + + def test_matches(self): + self.assertIs( + None, MatchesSetwise(Equals(1), Equals(2)).match([2, 1])) + + def test_mismatches(self): + self.assertMismatchWithDescriptionMatching( + [2, 3], MatchesSetwise(Equals(1), Equals(2)), + MatchesRegex('.*There was 1 mismatch$', re.S)) + + def test_too_many_matchers(self): + self.assertMismatchWithDescriptionMatching( + [2, 3], MatchesSetwise(Equals(1), Equals(2), Equals(3)), + Equals('There was 1 matcher left over: Equals(1)')) + + def test_too_many_values(self): + self.assertMismatchWithDescriptionMatching( + [1, 2, 3], MatchesSetwise(Equals(1), Equals(2)), + Equals('There was 1 value left over: [3]')) + + def test_two_too_many_matchers(self): + self.assertMismatchWithDescriptionMatching( + [3], MatchesSetwise(Equals(1), Equals(2), Equals(3)), + MatchesRegex( + 'There were 2 matchers left over: Equals\([12]\), ' + 'Equals\([12]\)')) + + def test_two_too_many_values(self): + self.assertMismatchWithDescriptionMatching( + [1, 2, 3, 4], MatchesSetwise(Equals(1), Equals(2)), + MatchesRegex( + 'There were 2 values left over: \[[34], [34]\]')) + + def test_mismatch_and_too_many_matchers(self): + self.assertMismatchWithDescriptionMatching( + [2, 3], MatchesSetwise(Equals(0), Equals(1), Equals(2)), + MatchesRegex( + '.*There was 1 mismatch and 1 extra matcher: Equals\([01]\)', + re.S)) + + def test_mismatch_and_too_many_values(self): + self.assertMismatchWithDescriptionMatching( + [2, 3, 4], MatchesSetwise(Equals(1), Equals(2)), + MatchesRegex( + '.*There was 1 mismatch and 1 extra value: \[[34]\]', + re.S)) + + def test_mismatch_and_two_too_many_matchers(self): + self.assertMismatchWithDescriptionMatching( + [3, 4], MatchesSetwise( + Equals(0), Equals(1), Equals(2), Equals(3)), + MatchesRegex( + '.*There was 1 mismatch and 2 extra matchers: ' + 'Equals\([012]\), Equals\([012]\)', re.S)) + + def test_mismatch_and_two_too_many_values(self): + self.assertMismatchWithDescriptionMatching( + [2, 3, 4, 5], MatchesSetwise(Equals(1), Equals(2)), + MatchesRegex( + '.*There was 1 mismatch and 2 extra values: \[[145], [145]\]', + re.S)) + + +class TestAfterPreprocessing(TestCase, TestMatchersInterface): + + def parity(x): + return x % 2 + + matches_matcher = AfterPreprocessing(parity, Equals(1)) + matches_matches = [3, 5] + matches_mismatches = [2] + + str_examples = [ + ("AfterPreprocessing(<function parity>, Equals(1))", + AfterPreprocessing(parity, Equals(1))), + ] + + describe_examples = [ + ("1 != 0: after <function parity> on 2", 2, + AfterPreprocessing(parity, Equals(1))), + ("1 != 0", 2, + AfterPreprocessing(parity, Equals(1), annotate=False)), + ] + + +class TestMismatchDecorator(TestCase): + + run_tests_with = FullStackRunTest + + def test_forwards_description(self): + x = Mismatch("description", {'foo': 'bar'}) + decorated = MismatchDecorator(x) + self.assertEqual(x.describe(), decorated.describe()) + + def test_forwards_details(self): + x = Mismatch("description", {'foo': 'bar'}) + decorated = MismatchDecorator(x) + self.assertEqual(x.get_details(), decorated.get_details()) + + def test_repr(self): + x = Mismatch("description", {'foo': 'bar'}) + decorated = MismatchDecorator(x) + self.assertEqual( + '<testtools.matchers.MismatchDecorator(%r)>' % (x,), + repr(decorated)) + + +class TestAllMatch(TestCase, TestMatchersInterface): + + matches_matcher = AllMatch(LessThan(10)) + matches_matches = [ + [9, 9, 9], + (9, 9), + iter([9, 9, 9, 9, 9]), + ] + matches_mismatches = [ + [11, 9, 9], + iter([9, 12, 9, 11]), + ] + + str_examples = [ + ("AllMatch(LessThan(12))", AllMatch(LessThan(12))), + ] + + describe_examples = [ + ('Differences: [\n' + '10 is not > 11\n' + '10 is not > 10\n' + ']', + [11, 9, 10], + AllMatch(LessThan(10))), + ] + + +def test_suite(): + from unittest import TestLoader + return TestLoader().loadTestsFromName(__name__) diff --git a/test/3rdparty/testtools-0.9.12/testtools/tests/test_monkey.py b/test/3rdparty/testtools-0.9.12/testtools/tests/test_monkey.py new file mode 100644 index 00000000000..540a2ee909f --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/tests/test_monkey.py @@ -0,0 +1,167 @@ +# Copyright (c) 2010 Twisted Matrix Laboratories. +# See LICENSE for details. + +"""Tests for testtools.monkey.""" + +from testtools import TestCase +from testtools.matchers import MatchesException, Raises +from testtools.monkey import MonkeyPatcher, patch + + +class TestObj: + + def __init__(self): + self.foo = 'foo value' + self.bar = 'bar value' + self.baz = 'baz value' + + +class MonkeyPatcherTest(TestCase): + """ + Tests for 'MonkeyPatcher' monkey-patching class. + """ + + def setUp(self): + super(MonkeyPatcherTest, self).setUp() + self.test_object = TestObj() + self.original_object = TestObj() + self.monkey_patcher = MonkeyPatcher() + + def test_empty(self): + # A monkey patcher without patches doesn't change a thing. + self.monkey_patcher.patch() + + # We can't assert that all state is unchanged, but at least we can + # check our test object. + self.assertEquals(self.original_object.foo, self.test_object.foo) + self.assertEquals(self.original_object.bar, self.test_object.bar) + self.assertEquals(self.original_object.baz, self.test_object.baz) + + def test_construct_with_patches(self): + # Constructing a 'MonkeyPatcher' with patches adds all of the given + # patches to the patch list. + patcher = MonkeyPatcher((self.test_object, 'foo', 'haha'), + (self.test_object, 'bar', 'hehe')) + patcher.patch() + self.assertEquals('haha', self.test_object.foo) + self.assertEquals('hehe', self.test_object.bar) + self.assertEquals(self.original_object.baz, self.test_object.baz) + + def test_patch_existing(self): + # Patching an attribute that exists sets it to the value defined in the + # patch. + self.monkey_patcher.add_patch(self.test_object, 'foo', 'haha') + self.monkey_patcher.patch() + self.assertEquals(self.test_object.foo, 'haha') + + def test_patch_non_existing(self): + # Patching a non-existing attribute sets it to the value defined in + # the patch. + self.monkey_patcher.add_patch(self.test_object, 'doesntexist', 'value') + self.monkey_patcher.patch() + self.assertEquals(self.test_object.doesntexist, 'value') + + def test_restore_non_existing(self): + # Restoring a value that didn't exist before the patch deletes the + # value. + self.monkey_patcher.add_patch(self.test_object, 'doesntexist', 'value') + self.monkey_patcher.patch() + self.monkey_patcher.restore() + marker = object() + self.assertIs(marker, getattr(self.test_object, 'doesntexist', marker)) + + def test_patch_already_patched(self): + # Adding a patch for an object and attribute that already have a patch + # overrides the existing patch. + self.monkey_patcher.add_patch(self.test_object, 'foo', 'blah') + self.monkey_patcher.add_patch(self.test_object, 'foo', 'BLAH') + self.monkey_patcher.patch() + self.assertEquals(self.test_object.foo, 'BLAH') + self.monkey_patcher.restore() + self.assertEquals(self.test_object.foo, self.original_object.foo) + + def test_restore_twice_is_a_no_op(self): + # Restoring an already-restored monkey patch is a no-op. + self.monkey_patcher.add_patch(self.test_object, 'foo', 'blah') + self.monkey_patcher.patch() + self.monkey_patcher.restore() + self.assertEquals(self.test_object.foo, self.original_object.foo) + self.monkey_patcher.restore() + self.assertEquals(self.test_object.foo, self.original_object.foo) + + def test_run_with_patches_decoration(self): + # run_with_patches runs the given callable, passing in all arguments + # and keyword arguments, and returns the return value of the callable. + log = [] + + def f(a, b, c=None): + log.append((a, b, c)) + return 'foo' + + result = self.monkey_patcher.run_with_patches(f, 1, 2, c=10) + self.assertEquals('foo', result) + self.assertEquals([(1, 2, 10)], log) + + def test_repeated_run_with_patches(self): + # We can call the same function with run_with_patches more than + # once. All patches apply for each call. + def f(): + return (self.test_object.foo, self.test_object.bar, + self.test_object.baz) + + self.monkey_patcher.add_patch(self.test_object, 'foo', 'haha') + result = self.monkey_patcher.run_with_patches(f) + self.assertEquals( + ('haha', self.original_object.bar, self.original_object.baz), + result) + result = self.monkey_patcher.run_with_patches(f) + self.assertEquals( + ('haha', self.original_object.bar, self.original_object.baz), + result) + + def test_run_with_patches_restores(self): + # run_with_patches restores the original values after the function has + # executed. + self.monkey_patcher.add_patch(self.test_object, 'foo', 'haha') + self.assertEquals(self.original_object.foo, self.test_object.foo) + self.monkey_patcher.run_with_patches(lambda: None) + self.assertEquals(self.original_object.foo, self.test_object.foo) + + def test_run_with_patches_restores_on_exception(self): + # run_with_patches restores the original values even when the function + # raises an exception. + def _(): + self.assertEquals(self.test_object.foo, 'haha') + self.assertEquals(self.test_object.bar, 'blahblah') + raise RuntimeError("Something went wrong!") + + self.monkey_patcher.add_patch(self.test_object, 'foo', 'haha') + self.monkey_patcher.add_patch(self.test_object, 'bar', 'blahblah') + + self.assertThat(lambda:self.monkey_patcher.run_with_patches(_), + Raises(MatchesException(RuntimeError("Something went wrong!")))) + self.assertEquals(self.test_object.foo, self.original_object.foo) + self.assertEquals(self.test_object.bar, self.original_object.bar) + + +class TestPatchHelper(TestCase): + + def test_patch_patches(self): + # patch(obj, name, value) sets obj.name to value. + test_object = TestObj() + patch(test_object, 'foo', 42) + self.assertEqual(42, test_object.foo) + + def test_patch_returns_cleanup(self): + # patch(obj, name, value) returns a nullary callable that restores obj + # to its original state when run. + test_object = TestObj() + original = test_object.foo + cleanup = patch(test_object, 'foo', 42) + cleanup() + self.assertEqual(original, test_object.foo) + + +def test_suite(): + from unittest import TestLoader + return TestLoader().loadTestsFromName(__name__) diff --git a/test/3rdparty/testtools-0.9.12/testtools/tests/test_run.py b/test/3rdparty/testtools-0.9.12/testtools/tests/test_run.py new file mode 100644 index 00000000000..d2974f63731 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/tests/test_run.py @@ -0,0 +1,80 @@ +# Copyright (c) 2010 testtools developers. See LICENSE for details. + +"""Tests for the test runner logic.""" + +from testtools.compat import ( + _b, + StringIO, + ) +from testtools.helpers import try_import +fixtures = try_import('fixtures') + +import testtools +from testtools import TestCase, run + + +if fixtures: + class SampleTestFixture(fixtures.Fixture): + """Creates testtools.runexample temporarily.""" + + def __init__(self): + self.package = fixtures.PythonPackage( + 'runexample', [('__init__.py', _b(""" +from testtools import TestCase + +class TestFoo(TestCase): + def test_bar(self): + pass + def test_quux(self): + pass +def test_suite(): + from unittest import TestLoader + return TestLoader().loadTestsFromName(__name__) +"""))]) + + def setUp(self): + super(SampleTestFixture, self).setUp() + self.useFixture(self.package) + testtools.__path__.append(self.package.base) + self.addCleanup(testtools.__path__.remove, self.package.base) + + +class TestRun(TestCase): + + def test_run_list(self): + if fixtures is None: + self.skipTest("Need fixtures") + self.useFixture(SampleTestFixture()) + out = StringIO() + run.main(['prog', '-l', 'testtools.runexample.test_suite'], out) + self.assertEqual("""testtools.runexample.TestFoo.test_bar +testtools.runexample.TestFoo.test_quux +""", out.getvalue()) + + def test_run_load_list(self): + if fixtures is None: + self.skipTest("Need fixtures") + self.useFixture(SampleTestFixture()) + out = StringIO() + # We load two tests - one that exists and one that doesn't, and we + # should get the one that exists and neither the one that doesn't nor + # the unmentioned one that does. + tempdir = self.useFixture(fixtures.TempDir()) + tempname = tempdir.path + '/tests.list' + f = open(tempname, 'wb') + try: + f.write(_b(""" +testtools.runexample.TestFoo.test_bar +testtools.runexample.missingtest +""")) + finally: + f.close() + run.main(['prog', '-l', '--load-list', tempname, + 'testtools.runexample.test_suite'], out) + self.assertEqual("""testtools.runexample.TestFoo.test_bar +""", out.getvalue()) + + +def test_suite(): + from unittest import TestLoader + return TestLoader().loadTestsFromName(__name__) diff --git a/test/3rdparty/testtools-0.9.12/testtools/tests/test_runtest.py b/test/3rdparty/testtools-0.9.12/testtools/tests/test_runtest.py new file mode 100644 index 00000000000..afbb8baf395 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/tests/test_runtest.py @@ -0,0 +1,303 @@ +# Copyright (c) 2009-2011 testtools developers. See LICENSE for details. + +"""Tests for the RunTest single test execution logic.""" + +from testtools import ( + ExtendedToOriginalDecorator, + run_test_with, + RunTest, + TestCase, + TestResult, + ) +from testtools.matchers import MatchesException, Is, Raises +from testtools.testresult.doubles import ExtendedTestResult +from testtools.tests.helpers import FullStackRunTest + + +class TestRunTest(TestCase): + + run_tests_with = FullStackRunTest + + def make_case(self): + class Case(TestCase): + def test(self): + pass + return Case('test') + + def test___init___short(self): + run = RunTest("bar") + self.assertEqual("bar", run.case) + self.assertEqual([], run.handlers) + + def test__init____handlers(self): + handlers = [("quux", "baz")] + run = RunTest("bar", handlers) + self.assertEqual(handlers, run.handlers) + + def test_run_with_result(self): + # test.run passes result down to _run_test_method. + log = [] + class Case(TestCase): + def _run_test_method(self, result): + log.append(result) + case = Case('_run_test_method') + run = RunTest(case, lambda x: log.append(x)) + result = TestResult() + run.run(result) + self.assertEqual(1, len(log)) + self.assertEqual(result, log[0].decorated) + + def test_run_no_result_manages_new_result(self): + log = [] + run = RunTest(self.make_case(), lambda x: log.append(x) or x) + result = run.run() + self.assertIsInstance(result.decorated, TestResult) + + def test__run_core_called(self): + case = self.make_case() + log = [] + run = RunTest(case, lambda x: x) + run._run_core = lambda: log.append('foo') + run.run() + self.assertEqual(['foo'], log) + + def test__run_user_does_not_catch_keyboard(self): + case = self.make_case() + def raises(): + raise KeyboardInterrupt("yo") + run = RunTest(case, None) + run.result = ExtendedTestResult() + self.assertThat(lambda: run._run_user(raises), + Raises(MatchesException(KeyboardInterrupt))) + self.assertEqual([], run.result._events) + + def test__run_user_calls_onException(self): + case = self.make_case() + log = [] + def handler(exc_info): + log.append("got it") + self.assertEqual(3, len(exc_info)) + self.assertIsInstance(exc_info[1], KeyError) + self.assertIs(KeyError, exc_info[0]) + case.addOnException(handler) + e = KeyError('Yo') + def raises(): + raise e + run = RunTest(case, [(KeyError, None)]) + run.result = ExtendedTestResult() + status = run._run_user(raises) + self.assertEqual(run.exception_caught, status) + self.assertEqual([], run.result._events) + self.assertEqual(["got it"], log) + + def test__run_user_can_catch_Exception(self): + case = self.make_case() + e = Exception('Yo') + def raises(): + raise e + log = [] + run = RunTest(case, [(Exception, None)]) + run.result = ExtendedTestResult() + status = run._run_user(raises) + self.assertEqual(run.exception_caught, status) + self.assertEqual([], run.result._events) + self.assertEqual([], log) + + def test__run_user_uncaught_Exception_raised(self): + case = self.make_case() + e = KeyError('Yo') + def raises(): + raise e + log = [] + def log_exc(self, result, err): + log.append((result, err)) + run = RunTest(case, [(ValueError, log_exc)]) + run.result = ExtendedTestResult() + self.assertThat(lambda: run._run_user(raises), + Raises(MatchesException(KeyError))) + self.assertEqual([], run.result._events) + self.assertEqual([], log) + + def test__run_user_uncaught_Exception_from_exception_handler_raised(self): + case = self.make_case() + def broken_handler(exc_info): + # ValueError because thats what we know how to catch - and must + # not. + raise ValueError('boo') + case.addOnException(broken_handler) + e = KeyError('Yo') + def raises(): + raise e + log = [] + def log_exc(self, result, err): + log.append((result, err)) + run = RunTest(case, [(ValueError, log_exc)]) + run.result = ExtendedTestResult() + self.assertThat(lambda: run._run_user(raises), + Raises(MatchesException(ValueError))) + self.assertEqual([], run.result._events) + self.assertEqual([], log) + + def test__run_user_returns_result(self): + case = self.make_case() + def returns(): + return 1 + run = RunTest(case) + run.result = ExtendedTestResult() + self.assertEqual(1, run._run_user(returns)) + self.assertEqual([], run.result._events) + + def test__run_one_decorates_result(self): + log = [] + class Run(RunTest): + def _run_prepared_result(self, result): + log.append(result) + return result + run = Run(self.make_case(), lambda x: x) + result = run._run_one('foo') + self.assertEqual([result], log) + self.assertIsInstance(log[0], ExtendedToOriginalDecorator) + self.assertEqual('foo', result.decorated) + + def test__run_prepared_result_calls_start_and_stop_test(self): + result = ExtendedTestResult() + case = self.make_case() + run = RunTest(case, lambda x: x) + run.run(result) + self.assertEqual([ + ('startTest', case), + ('addSuccess', case), + ('stopTest', case), + ], result._events) + + def test__run_prepared_result_calls_stop_test_always(self): + result = ExtendedTestResult() + case = self.make_case() + def inner(): + raise Exception("foo") + run = RunTest(case, lambda x: x) + run._run_core = inner + self.assertThat(lambda: run.run(result), + Raises(MatchesException(Exception("foo")))) + self.assertEqual([ + ('startTest', case), + ('stopTest', case), + ], result._events) + + +class CustomRunTest(RunTest): + + marker = object() + + def run(self, result=None): + return self.marker + + +class TestTestCaseSupportForRunTest(TestCase): + + def test_pass_custom_run_test(self): + class SomeCase(TestCase): + def test_foo(self): + pass + result = TestResult() + case = SomeCase('test_foo', runTest=CustomRunTest) + from_run_test = case.run(result) + self.assertThat(from_run_test, Is(CustomRunTest.marker)) + + def test_default_is_runTest_class_variable(self): + class SomeCase(TestCase): + run_tests_with = CustomRunTest + def test_foo(self): + pass + result = TestResult() + case = SomeCase('test_foo') + from_run_test = case.run(result) + self.assertThat(from_run_test, Is(CustomRunTest.marker)) + + def test_constructor_argument_overrides_class_variable(self): + # If a 'runTest' argument is passed to the test's constructor, that + # overrides the class variable. + marker = object() + class DifferentRunTest(RunTest): + def run(self, result=None): + return marker + class SomeCase(TestCase): + run_tests_with = CustomRunTest + def test_foo(self): + pass + result = TestResult() + case = SomeCase('test_foo', runTest=DifferentRunTest) + from_run_test = case.run(result) + self.assertThat(from_run_test, Is(marker)) + + def test_decorator_for_run_test(self): + # Individual test methods can be marked as needing a special runner. + class SomeCase(TestCase): + @run_test_with(CustomRunTest) + def test_foo(self): + pass + result = TestResult() + case = SomeCase('test_foo') + from_run_test = case.run(result) + self.assertThat(from_run_test, Is(CustomRunTest.marker)) + + def test_extended_decorator_for_run_test(self): + # Individual test methods can be marked as needing a special runner. + # Extra arguments can be passed to the decorator which will then be + # passed on to the RunTest object. + marker = object() + class FooRunTest(RunTest): + def __init__(self, case, handlers=None, bar=None): + super(FooRunTest, self).__init__(case, handlers) + self.bar = bar + def run(self, result=None): + return self.bar + class SomeCase(TestCase): + @run_test_with(FooRunTest, bar=marker) + def test_foo(self): + pass + result = TestResult() + case = SomeCase('test_foo') + from_run_test = case.run(result) + self.assertThat(from_run_test, Is(marker)) + + def test_works_as_inner_decorator(self): + # Even if run_test_with is the innermost decorator, it will be + # respected. + def wrapped(function): + """Silly, trivial decorator.""" + def decorated(*args, **kwargs): + return function(*args, **kwargs) + decorated.__name__ = function.__name__ + decorated.__dict__.update(function.__dict__) + return decorated + class SomeCase(TestCase): + @wrapped + @run_test_with(CustomRunTest) + def test_foo(self): + pass + result = TestResult() + case = SomeCase('test_foo') + from_run_test = case.run(result) + self.assertThat(from_run_test, Is(CustomRunTest.marker)) + + def test_constructor_overrides_decorator(self): + # If a 'runTest' argument is passed to the test's constructor, that + # overrides the decorator. + marker = object() + class DifferentRunTest(RunTest): + def run(self, result=None): + return marker + class SomeCase(TestCase): + @run_test_with(CustomRunTest) + def test_foo(self): + pass + result = TestResult() + case = SomeCase('test_foo', runTest=DifferentRunTest) + from_run_test = case.run(result) + self.assertThat(from_run_test, Is(marker)) + + +def test_suite(): + from unittest import TestLoader + return TestLoader().loadTestsFromName(__name__) diff --git a/test/3rdparty/testtools-0.9.12/testtools/tests/test_spinner.py b/test/3rdparty/testtools-0.9.12/testtools/tests/test_spinner.py new file mode 100644 index 00000000000..3d677bd7545 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/tests/test_spinner.py @@ -0,0 +1,332 @@ +# Copyright (c) 2010 testtools developers. See LICENSE for details. + +"""Tests for the evil Twisted reactor-spinning we do.""" + +import os +import signal + +from testtools import ( + skipIf, + TestCase, + ) +from testtools.helpers import try_import +from testtools.matchers import ( + Equals, + Is, + MatchesException, + Raises, + ) + +_spinner = try_import('testtools._spinner') + +defer = try_import('twisted.internet.defer') +Failure = try_import('twisted.python.failure.Failure') + + +class NeedsTwistedTestCase(TestCase): + + def setUp(self): + super(NeedsTwistedTestCase, self).setUp() + if defer is None or Failure is None: + self.skipTest("Need Twisted to run") + + +class TestNotReentrant(NeedsTwistedTestCase): + + def test_not_reentrant(self): + # A function decorated as not being re-entrant will raise a + # _spinner.ReentryError if it is called while it is running. + calls = [] + @_spinner.not_reentrant + def log_something(): + calls.append(None) + if len(calls) < 5: + log_something() + self.assertThat( + log_something, Raises(MatchesException(_spinner.ReentryError))) + self.assertEqual(1, len(calls)) + + def test_deeper_stack(self): + calls = [] + @_spinner.not_reentrant + def g(): + calls.append(None) + if len(calls) < 5: + f() + @_spinner.not_reentrant + def f(): + calls.append(None) + if len(calls) < 5: + g() + self.assertThat(f, Raises(MatchesException(_spinner.ReentryError))) + self.assertEqual(2, len(calls)) + + +class TestExtractResult(NeedsTwistedTestCase): + + def test_not_fired(self): + # _spinner.extract_result raises _spinner.DeferredNotFired if it's + # given a Deferred that has not fired. + self.assertThat(lambda:_spinner.extract_result(defer.Deferred()), + Raises(MatchesException(_spinner.DeferredNotFired))) + + def test_success(self): + # _spinner.extract_result returns the value of the Deferred if it has + # fired successfully. + marker = object() + d = defer.succeed(marker) + self.assertThat(_spinner.extract_result(d), Equals(marker)) + + def test_failure(self): + # _spinner.extract_result raises the failure's exception if it's given + # a Deferred that is failing. + try: + 1/0 + except ZeroDivisionError: + f = Failure() + d = defer.fail(f) + self.assertThat(lambda:_spinner.extract_result(d), + Raises(MatchesException(ZeroDivisionError))) + + +class TestTrapUnhandledErrors(NeedsTwistedTestCase): + + def test_no_deferreds(self): + marker = object() + result, errors = _spinner.trap_unhandled_errors(lambda: marker) + self.assertEqual([], errors) + self.assertIs(marker, result) + + def test_unhandled_error(self): + failures = [] + def make_deferred_but_dont_handle(): + try: + 1/0 + except ZeroDivisionError: + f = Failure() + failures.append(f) + defer.fail(f) + result, errors = _spinner.trap_unhandled_errors( + make_deferred_but_dont_handle) + self.assertIs(None, result) + self.assertEqual(failures, [error.failResult for error in errors]) + + +class TestRunInReactor(NeedsTwistedTestCase): + + def make_reactor(self): + from twisted.internet import reactor + return reactor + + def make_spinner(self, reactor=None): + if reactor is None: + reactor = self.make_reactor() + return _spinner.Spinner(reactor) + + def make_timeout(self): + return 0.01 + + def test_function_called(self): + # run_in_reactor actually calls the function given to it. + calls = [] + marker = object() + self.make_spinner().run(self.make_timeout(), calls.append, marker) + self.assertThat(calls, Equals([marker])) + + def test_return_value_returned(self): + # run_in_reactor returns the value returned by the function given to + # it. + marker = object() + result = self.make_spinner().run(self.make_timeout(), lambda: marker) + self.assertThat(result, Is(marker)) + + def test_exception_reraised(self): + # If the given function raises an error, run_in_reactor re-raises that + # error. + self.assertThat( + lambda:self.make_spinner().run(self.make_timeout(), lambda: 1/0), + Raises(MatchesException(ZeroDivisionError))) + + def test_keyword_arguments(self): + # run_in_reactor passes keyword arguments on. + calls = [] + function = lambda *a, **kw: calls.extend([a, kw]) + self.make_spinner().run(self.make_timeout(), function, foo=42) + self.assertThat(calls, Equals([(), {'foo': 42}])) + + def test_not_reentrant(self): + # run_in_reactor raises an error if it is called inside another call + # to run_in_reactor. + spinner = self.make_spinner() + self.assertThat(lambda: spinner.run( + self.make_timeout(), spinner.run, self.make_timeout(), + lambda: None), Raises(MatchesException(_spinner.ReentryError))) + + def test_deferred_value_returned(self): + # If the given function returns a Deferred, run_in_reactor returns the + # value in the Deferred at the end of the callback chain. + marker = object() + result = self.make_spinner().run( + self.make_timeout(), lambda: defer.succeed(marker)) + self.assertThat(result, Is(marker)) + + def test_preserve_signal_handler(self): + signals = ['SIGINT', 'SIGTERM', 'SIGCHLD'] + signals = filter( + None, (getattr(signal, name, None) for name in signals)) + for sig in signals: + self.addCleanup(signal.signal, sig, signal.getsignal(sig)) + new_hdlrs = list(lambda *a: None for _ in signals) + for sig, hdlr in zip(signals, new_hdlrs): + signal.signal(sig, hdlr) + spinner = self.make_spinner() + spinner.run(self.make_timeout(), lambda: None) + self.assertEqual(new_hdlrs, map(signal.getsignal, signals)) + + def test_timeout(self): + # If the function takes too long to run, we raise a + # _spinner.TimeoutError. + timeout = self.make_timeout() + self.assertThat( + lambda:self.make_spinner().run(timeout, lambda: defer.Deferred()), + Raises(MatchesException(_spinner.TimeoutError))) + + def test_no_junk_by_default(self): + # If the reactor hasn't spun yet, then there cannot be any junk. + spinner = self.make_spinner() + self.assertThat(spinner.get_junk(), Equals([])) + + def test_clean_do_nothing(self): + # If there's nothing going on in the reactor, then clean does nothing + # and returns an empty list. + spinner = self.make_spinner() + result = spinner._clean() + self.assertThat(result, Equals([])) + + def test_clean_delayed_call(self): + # If there's a delayed call in the reactor, then clean cancels it and + # returns an empty list. + reactor = self.make_reactor() + spinner = self.make_spinner(reactor) + call = reactor.callLater(10, lambda: None) + results = spinner._clean() + self.assertThat(results, Equals([call])) + self.assertThat(call.active(), Equals(False)) + + def test_clean_delayed_call_cancelled(self): + # If there's a delayed call that's just been cancelled, then it's no + # longer there. + reactor = self.make_reactor() + spinner = self.make_spinner(reactor) + call = reactor.callLater(10, lambda: None) + call.cancel() + results = spinner._clean() + self.assertThat(results, Equals([])) + + def test_clean_selectables(self): + # If there's still a selectable (e.g. a listening socket), then + # clean() removes it from the reactor's registry. + # + # Note that the socket is left open. This emulates a bug in trial. + from twisted.internet.protocol import ServerFactory + reactor = self.make_reactor() + spinner = self.make_spinner(reactor) + port = reactor.listenTCP(0, ServerFactory()) + spinner.run(self.make_timeout(), lambda: None) + results = spinner.get_junk() + self.assertThat(results, Equals([port])) + + def test_clean_running_threads(self): + import threading + import time + current_threads = list(threading.enumerate()) + reactor = self.make_reactor() + timeout = self.make_timeout() + spinner = self.make_spinner(reactor) + spinner.run(timeout, reactor.callInThread, time.sleep, timeout / 2.0) + # Python before 2.5 has a race condition with thread handling where + # join() does not remove threads from enumerate before returning - the + # thread being joined does the removal. This was fixed in Python 2.5 + # but we still support 2.4, so we have to workaround the issue. + # http://bugs.python.org/issue1703448. + self.assertThat( + [thread for thread in threading.enumerate() if thread.isAlive()], + Equals(current_threads)) + + def test_leftover_junk_available(self): + # If 'run' is given a function that leaves the reactor dirty in some + # way, 'run' will clean up the reactor and then store information + # about the junk. This information can be got using get_junk. + from twisted.internet.protocol import ServerFactory + reactor = self.make_reactor() + spinner = self.make_spinner(reactor) + port = spinner.run( + self.make_timeout(), reactor.listenTCP, 0, ServerFactory()) + self.assertThat(spinner.get_junk(), Equals([port])) + + def test_will_not_run_with_previous_junk(self): + # If 'run' is called and there's still junk in the spinner's junk + # list, then the spinner will refuse to run. + from twisted.internet.protocol import ServerFactory + reactor = self.make_reactor() + spinner = self.make_spinner(reactor) + timeout = self.make_timeout() + spinner.run(timeout, reactor.listenTCP, 0, ServerFactory()) + self.assertThat(lambda: spinner.run(timeout, lambda: None), + Raises(MatchesException(_spinner.StaleJunkError))) + + def test_clear_junk_clears_previous_junk(self): + # If 'run' is called and there's still junk in the spinner's junk + # list, then the spinner will refuse to run. + from twisted.internet.protocol import ServerFactory + reactor = self.make_reactor() + spinner = self.make_spinner(reactor) + timeout = self.make_timeout() + port = spinner.run(timeout, reactor.listenTCP, 0, ServerFactory()) + junk = spinner.clear_junk() + self.assertThat(junk, Equals([port])) + self.assertThat(spinner.get_junk(), Equals([])) + + @skipIf(os.name != "posix", "Sending SIGINT with os.kill is posix only") + def test_sigint_raises_no_result_error(self): + # If we get a SIGINT during a run, we raise _spinner.NoResultError. + SIGINT = getattr(signal, 'SIGINT', None) + if not SIGINT: + self.skipTest("SIGINT not available") + reactor = self.make_reactor() + spinner = self.make_spinner(reactor) + timeout = self.make_timeout() + reactor.callLater(timeout, os.kill, os.getpid(), SIGINT) + self.assertThat(lambda:spinner.run(timeout * 5, defer.Deferred), + Raises(MatchesException(_spinner.NoResultError))) + self.assertEqual([], spinner._clean()) + + @skipIf(os.name != "posix", "Sending SIGINT with os.kill is posix only") + def test_sigint_raises_no_result_error_second_time(self): + # If we get a SIGINT during a run, we raise _spinner.NoResultError. + # This test is exactly the same as test_sigint_raises_no_result_error, + # and exists to make sure we haven't futzed with state. + self.test_sigint_raises_no_result_error() + + @skipIf(os.name != "posix", "Sending SIGINT with os.kill is posix only") + def test_fast_sigint_raises_no_result_error(self): + # If we get a SIGINT during a run, we raise _spinner.NoResultError. + SIGINT = getattr(signal, 'SIGINT', None) + if not SIGINT: + self.skipTest("SIGINT not available") + reactor = self.make_reactor() + spinner = self.make_spinner(reactor) + timeout = self.make_timeout() + reactor.callWhenRunning(os.kill, os.getpid(), SIGINT) + self.assertThat(lambda:spinner.run(timeout * 5, defer.Deferred), + Raises(MatchesException(_spinner.NoResultError))) + self.assertEqual([], spinner._clean()) + + @skipIf(os.name != "posix", "Sending SIGINT with os.kill is posix only") + def test_fast_sigint_raises_no_result_error_second_time(self): + self.test_fast_sigint_raises_no_result_error() + + +def test_suite(): + from unittest import TestLoader + return TestLoader().loadTestsFromName(__name__) diff --git a/test/3rdparty/testtools-0.9.12/testtools/tests/test_testcase.py b/test/3rdparty/testtools-0.9.12/testtools/tests/test_testcase.py new file mode 100644 index 00000000000..52f93c3c526 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/tests/test_testcase.py @@ -0,0 +1,1288 @@ +# Copyright (c) 2008-2011 testtools developers. See LICENSE for details. + +"""Tests for extensions to the base test library.""" + +from doctest import ELLIPSIS +from pprint import pformat +import sys +import unittest + +from testtools import ( + ErrorHolder, + MultipleExceptions, + PlaceHolder, + TestCase, + clone_test_with_new_id, + content, + skip, + skipIf, + skipUnless, + testcase, + ) +from testtools.compat import ( + _b, + _u, + ) +from testtools.matchers import ( + Annotate, + DocTestMatches, + Equals, + MatchesException, + Raises, + ) +from testtools.testresult.doubles import ( + Python26TestResult, + Python27TestResult, + ExtendedTestResult, + ) +from testtools.testresult.real import TestResult +from testtools.tests.helpers import ( + an_exc_info, + FullStackRunTest, + LoggingResult, + ) +try: + exec('from __future__ import with_statement') +except SyntaxError: + pass +else: + from testtools.tests.test_with_with import * + + +class TestPlaceHolder(TestCase): + + run_test_with = FullStackRunTest + + def makePlaceHolder(self, test_id="foo", short_description=None): + return PlaceHolder(test_id, short_description) + + def test_id_comes_from_constructor(self): + # The id() of a PlaceHolder is whatever you pass into the constructor. + test = PlaceHolder("test id") + self.assertEqual("test id", test.id()) + + def test_shortDescription_is_id(self): + # The shortDescription() of a PlaceHolder is the id, by default. + test = PlaceHolder("test id") + self.assertEqual(test.id(), test.shortDescription()) + + def test_shortDescription_specified(self): + # If a shortDescription is provided to the constructor, then + # shortDescription() returns that instead. + test = PlaceHolder("test id", "description") + self.assertEqual("description", test.shortDescription()) + + def test_repr_just_id(self): + # repr(placeholder) shows you how the object was constructed. + test = PlaceHolder("test id") + self.assertEqual( + "<testtools.testcase.PlaceHolder(%s)>" % repr(test.id()), + repr(test)) + + def test_repr_with_description(self): + # repr(placeholder) shows you how the object was constructed. + test = PlaceHolder("test id", "description") + self.assertEqual( + "<testtools.testcase.PlaceHolder(%r, %r)>" % ( + test.id(), test.shortDescription()), + repr(test)) + + def test_counts_as_one_test(self): + # A placeholder test counts as one test. + test = self.makePlaceHolder() + self.assertEqual(1, test.countTestCases()) + + def test_str_is_id(self): + # str(placeholder) is always the id(). We are not barbarians. + test = self.makePlaceHolder() + self.assertEqual(test.id(), str(test)) + + def test_runs_as_success(self): + # When run, a PlaceHolder test records a success. + test = self.makePlaceHolder() + log = [] + test.run(LoggingResult(log)) + self.assertEqual( + [('startTest', test), ('addSuccess', test), ('stopTest', test)], + log) + + def test_call_is_run(self): + # A PlaceHolder can be called, in which case it behaves like run. + test = self.makePlaceHolder() + run_log = [] + test.run(LoggingResult(run_log)) + call_log = [] + test(LoggingResult(call_log)) + self.assertEqual(run_log, call_log) + + def test_runs_without_result(self): + # A PlaceHolder can be run without a result, in which case there's no + # way to actually get at the result. + self.makePlaceHolder().run() + + def test_debug(self): + # A PlaceHolder can be debugged. + self.makePlaceHolder().debug() + + +class TestErrorHolder(TestCase): + + run_test_with = FullStackRunTest + + def makeException(self): + try: + raise RuntimeError("danger danger") + except: + return sys.exc_info() + + def makePlaceHolder(self, test_id="foo", error=None, + short_description=None): + if error is None: + error = self.makeException() + return ErrorHolder(test_id, error, short_description) + + def test_id_comes_from_constructor(self): + # The id() of a PlaceHolder is whatever you pass into the constructor. + test = ErrorHolder("test id", self.makeException()) + self.assertEqual("test id", test.id()) + + def test_shortDescription_is_id(self): + # The shortDescription() of a PlaceHolder is the id, by default. + test = ErrorHolder("test id", self.makeException()) + self.assertEqual(test.id(), test.shortDescription()) + + def test_shortDescription_specified(self): + # If a shortDescription is provided to the constructor, then + # shortDescription() returns that instead. + test = ErrorHolder("test id", self.makeException(), "description") + self.assertEqual("description", test.shortDescription()) + + def test_repr_just_id(self): + # repr(placeholder) shows you how the object was constructed. + error = self.makeException() + test = ErrorHolder("test id", error) + self.assertEqual( + "<testtools.testcase.ErrorHolder(%r, %r)>" % (test.id(), error), + repr(test)) + + def test_repr_with_description(self): + # repr(placeholder) shows you how the object was constructed. + error = self.makeException() + test = ErrorHolder("test id", error, "description") + self.assertEqual( + "<testtools.testcase.ErrorHolder(%r, %r, %r)>" % ( + test.id(), error, test.shortDescription()), + repr(test)) + + def test_counts_as_one_test(self): + # A placeholder test counts as one test. + test = self.makePlaceHolder() + self.assertEqual(1, test.countTestCases()) + + def test_str_is_id(self): + # str(placeholder) is always the id(). We are not barbarians. + test = self.makePlaceHolder() + self.assertEqual(test.id(), str(test)) + + def test_runs_as_error(self): + # When run, a PlaceHolder test records a success. + error = self.makeException() + test = self.makePlaceHolder(error=error) + log = [] + test.run(LoggingResult(log)) + self.assertEqual( + [('startTest', test), + ('addError', test, error), + ('stopTest', test)], log) + + def test_call_is_run(self): + # A PlaceHolder can be called, in which case it behaves like run. + test = self.makePlaceHolder() + run_log = [] + test.run(LoggingResult(run_log)) + call_log = [] + test(LoggingResult(call_log)) + self.assertEqual(run_log, call_log) + + def test_runs_without_result(self): + # A PlaceHolder can be run without a result, in which case there's no + # way to actually get at the result. + self.makePlaceHolder().run() + + def test_debug(self): + # A PlaceHolder can be debugged. + self.makePlaceHolder().debug() + + +class TestEquality(TestCase): + """Test ``TestCase``'s equality implementation.""" + + run_test_with = FullStackRunTest + + def test_identicalIsEqual(self): + # TestCase's are equal if they are identical. + self.assertEqual(self, self) + + def test_nonIdenticalInUnequal(self): + # TestCase's are not equal if they are not identical. + self.assertNotEqual(TestCase(methodName='run'), + TestCase(methodName='skip')) + + +class TestAssertions(TestCase): + """Test assertions in TestCase.""" + + run_test_with = FullStackRunTest + + def raiseError(self, exceptionFactory, *args, **kwargs): + raise exceptionFactory(*args, **kwargs) + + def test_formatTypes_single(self): + # Given a single class, _formatTypes returns the name. + class Foo(object): + pass + self.assertEqual('Foo', self._formatTypes(Foo)) + + def test_formatTypes_multiple(self): + # Given multiple types, _formatTypes returns the names joined by + # commas. + class Foo(object): + pass + class Bar(object): + pass + self.assertEqual('Foo, Bar', self._formatTypes([Foo, Bar])) + + def test_assertRaises(self): + # assertRaises asserts that a callable raises a particular exception. + self.assertRaises(RuntimeError, self.raiseError, RuntimeError) + + def test_assertRaises_fails_when_no_error_raised(self): + # assertRaises raises self.failureException when it's passed a + # callable that raises no error. + ret = ('orange', 42) + self.assertFails("<function <lambda> at ...> returned ('orange', 42)", + self.assertRaises, RuntimeError, lambda: ret) + + def test_assertRaises_fails_when_different_error_raised(self): + # assertRaises re-raises an exception that it didn't expect. + self.assertThat(lambda: self.assertRaises(RuntimeError, + self.raiseError, ZeroDivisionError), + Raises(MatchesException(ZeroDivisionError))) + + def test_assertRaises_returns_the_raised_exception(self): + # assertRaises returns the exception object that was raised. This is + # useful for testing that exceptions have the right message. + + # This contraption stores the raised exception, so we can compare it + # to the return value of assertRaises. + raisedExceptions = [] + def raiseError(): + try: + raise RuntimeError('Deliberate error') + except RuntimeError: + raisedExceptions.append(sys.exc_info()[1]) + raise + + exception = self.assertRaises(RuntimeError, raiseError) + self.assertEqual(1, len(raisedExceptions)) + self.assertTrue( + exception is raisedExceptions[0], + "%r is not %r" % (exception, raisedExceptions[0])) + + def test_assertRaises_with_multiple_exceptions(self): + # assertRaises((ExceptionOne, ExceptionTwo), function) asserts that + # function raises one of ExceptionTwo or ExceptionOne. + expectedExceptions = (RuntimeError, ZeroDivisionError) + self.assertRaises( + expectedExceptions, self.raiseError, expectedExceptions[0]) + self.assertRaises( + expectedExceptions, self.raiseError, expectedExceptions[1]) + + def test_assertRaises_with_multiple_exceptions_failure_mode(self): + # If assertRaises is called expecting one of a group of exceptions and + # a callable that doesn't raise an exception, then fail with an + # appropriate error message. + expectedExceptions = (RuntimeError, ZeroDivisionError) + failure = self.assertRaises( + self.failureException, + self.assertRaises, expectedExceptions, lambda: None) + self.assertFails('<function <lambda> at ...> returned None', + self.assertRaises, expectedExceptions, lambda: None) + + def assertFails(self, message, function, *args, **kwargs): + """Assert that function raises a failure with the given message.""" + failure = self.assertRaises( + self.failureException, function, *args, **kwargs) + self.assertThat(failure, DocTestMatches(message, ELLIPSIS)) + + def test_assertIn_success(self): + # assertIn(needle, haystack) asserts that 'needle' is in 'haystack'. + self.assertIn(3, range(10)) + self.assertIn('foo', 'foo bar baz') + self.assertIn('foo', 'foo bar baz'.split()) + + def test_assertIn_failure(self): + # assertIn(needle, haystack) fails the test when 'needle' is not in + # 'haystack'. + self.assertFails('3 not in [0, 1, 2]', self.assertIn, 3, [0, 1, 2]) + self.assertFails( + '%r not in %r' % ('qux', 'foo bar baz'), + self.assertIn, 'qux', 'foo bar baz') + + def test_assertNotIn_success(self): + # assertNotIn(needle, haystack) asserts that 'needle' is not in + # 'haystack'. + self.assertNotIn(3, [0, 1, 2]) + self.assertNotIn('qux', 'foo bar baz') + + def test_assertNotIn_failure(self): + # assertNotIn(needle, haystack) fails the test when 'needle' is in + # 'haystack'. + self.assertFails('[1, 2, 3] matches Contains(3)', self.assertNotIn, + 3, [1, 2, 3]) + self.assertFails( + "'foo bar baz' matches Contains('foo')", + self.assertNotIn, 'foo', 'foo bar baz') + + def test_assertIsInstance(self): + # assertIsInstance asserts that an object is an instance of a class. + + class Foo(object): + """Simple class for testing assertIsInstance.""" + + foo = Foo() + self.assertIsInstance(foo, Foo) + + def test_assertIsInstance_multiple_classes(self): + # assertIsInstance asserts that an object is an instance of one of a + # group of classes. + + class Foo(object): + """Simple class for testing assertIsInstance.""" + + class Bar(object): + """Another simple class for testing assertIsInstance.""" + + foo = Foo() + self.assertIsInstance(foo, (Foo, Bar)) + self.assertIsInstance(Bar(), (Foo, Bar)) + + def test_assertIsInstance_failure(self): + # assertIsInstance(obj, klass) fails the test when obj is not an + # instance of klass. + + class Foo(object): + """Simple class for testing assertIsInstance.""" + + self.assertFails( + "'42' is not an instance of %s" % self._formatTypes(Foo), + self.assertIsInstance, 42, Foo) + + def test_assertIsInstance_failure_multiple_classes(self): + # assertIsInstance(obj, (klass1, klass2)) fails the test when obj is + # not an instance of klass1 or klass2. + + class Foo(object): + """Simple class for testing assertIsInstance.""" + + class Bar(object): + """Another simple class for testing assertIsInstance.""" + + self.assertFails( + "'42' is not an instance of any of (%s)" % self._formatTypes([Foo, Bar]), + self.assertIsInstance, 42, (Foo, Bar)) + + def test_assertIsInstance_overridden_message(self): + # assertIsInstance(obj, klass, msg) permits a custom message. + self.assertFails("'42' is not an instance of str: foo", + self.assertIsInstance, 42, str, "foo") + + def test_assertIs(self): + # assertIs asserts that an object is identical to another object. + self.assertIs(None, None) + some_list = [42] + self.assertIs(some_list, some_list) + some_object = object() + self.assertIs(some_object, some_object) + + def test_assertIs_fails(self): + # assertIs raises assertion errors if one object is not identical to + # another. + self.assertFails('None is not 42', self.assertIs, None, 42) + self.assertFails('[42] is not [42]', self.assertIs, [42], [42]) + + def test_assertIs_fails_with_message(self): + # assertIs raises assertion errors if one object is not identical to + # another, and includes a user-supplied message, if it's provided. + self.assertFails( + 'None is not 42: foo bar', self.assertIs, None, 42, 'foo bar') + + def test_assertIsNot(self): + # assertIsNot asserts that an object is not identical to another + # object. + self.assertIsNot(None, 42) + self.assertIsNot([42], [42]) + self.assertIsNot(object(), object()) + + def test_assertIsNot_fails(self): + # assertIsNot raises assertion errors if one object is identical to + # another. + self.assertFails('None matches Is(None)', self.assertIsNot, None, None) + some_list = [42] + self.assertFails( + '[42] matches Is([42])', self.assertIsNot, some_list, some_list) + + def test_assertIsNot_fails_with_message(self): + # assertIsNot raises assertion errors if one object is identical to + # another, and includes a user-supplied message if it's provided. + self.assertFails( + 'None matches Is(None): foo bar', self.assertIsNot, None, None, + "foo bar") + + def test_assertThat_matches_clean(self): + class Matcher(object): + def match(self, foo): + return None + self.assertThat("foo", Matcher()) + + def test_assertThat_mismatch_raises_description(self): + calls = [] + class Mismatch(object): + def __init__(self, thing): + self.thing = thing + def describe(self): + calls.append(('describe_diff', self.thing)) + return "object is not a thing" + def get_details(self): + return {} + class Matcher(object): + def match(self, thing): + calls.append(('match', thing)) + return Mismatch(thing) + def __str__(self): + calls.append(('__str__',)) + return "a description" + class Test(TestCase): + def test(self): + self.assertThat("foo", Matcher()) + result = Test("test").run() + self.assertEqual([ + ('match', "foo"), + ('describe_diff', "foo"), + ], calls) + self.assertFalse(result.wasSuccessful()) + + def test_assertThat_output(self): + matchee = 'foo' + matcher = Equals('bar') + expected = matcher.match(matchee).describe() + self.assertFails(expected, self.assertThat, matchee, matcher) + + def test_assertThat_message_is_annotated(self): + matchee = 'foo' + matcher = Equals('bar') + expected = Annotate('woo', matcher).match(matchee).describe() + self.assertFails(expected, self.assertThat, matchee, matcher, 'woo') + + def test_assertThat_verbose_output(self): + matchee = 'foo' + matcher = Equals('bar') + expected = ( + 'Match failed. Matchee: %r\n' + 'Matcher: %s\n' + 'Difference: %s\n' % ( + matchee, + matcher, + matcher.match(matchee).describe(), + )) + self.assertFails( + expected, self.assertThat, matchee, matcher, verbose=True) + + def get_error_string(self, e): + """Get the string showing how 'e' would be formatted in test output. + + This is a little bit hacky, since it's designed to give consistent + output regardless of Python version. + + In testtools, TestResult._exc_info_to_unicode is the point of dispatch + between various different implementations of methods that format + exceptions, so that's what we have to call. However, that method cares + about stack traces and formats the exception class. We don't care + about either of these, so we take its output and parse it a little. + """ + error = TestResult()._exc_info_to_unicode((e.__class__, e, None), self) + # We aren't at all interested in the traceback. + if error.startswith('Traceback (most recent call last):\n'): + lines = error.splitlines(True)[1:] + for i, line in enumerate(lines): + if not line.startswith(' '): + break + error = ''.join(lines[i:]) + # We aren't interested in how the exception type is formatted. + exc_class, error = error.split(': ', 1) + return error + + def test_assertThat_verbose_unicode(self): + # When assertThat is given matchees or matchers that contain non-ASCII + # unicode strings, we can still provide a meaningful error. + matchee = _u('\xa7') + matcher = Equals(_u('a')) + expected = ( + 'Match failed. Matchee: %s\n' + 'Matcher: %s\n' + 'Difference: %s\n\n' % ( + repr(matchee).replace("\\xa7", matchee), + matcher, + matcher.match(matchee).describe(), + )) + e = self.assertRaises( + self.failureException, self.assertThat, matchee, matcher, + verbose=True) + self.assertEqual(expected, self.get_error_string(e)) + + def test_assertEqual_nice_formatting(self): + message = "These things ought not be equal." + a = ['apple', 'banana', 'cherry'] + b = {'Thatcher': 'One who mends roofs of straw', + 'Major': 'A military officer, ranked below colonel', + 'Blair': 'To shout loudly', + 'Brown': 'The colour of healthy human faeces'} + expected_error = '\n'.join([ + '!=:', + 'reference = %s' % pformat(a), + 'actual = %s' % pformat(b), + ': ' + message, + ]) + self.assertFails(expected_error, self.assertEqual, a, b, message) + self.assertFails(expected_error, self.assertEquals, a, b, message) + self.assertFails(expected_error, self.failUnlessEqual, a, b, message) + + def test_assertEqual_formatting_no_message(self): + a = "cat" + b = "dog" + expected_error = "'cat' != 'dog'" + self.assertFails(expected_error, self.assertEqual, a, b) + self.assertFails(expected_error, self.assertEquals, a, b) + self.assertFails(expected_error, self.failUnlessEqual, a, b) + + def test_assertEqual_non_ascii_str_with_newlines(self): + message = _u("Be careful mixing unicode and bytes") + a = "a\n\xa7\n" + b = "Just a longish string so the more verbose output form is used." + expected_error = '\n'.join([ + '!=:', + "reference = '''\\", + 'a', + repr('\xa7')[1:-1], + "'''", + 'actual = %r' % (b,), + ': ' + message, + ]) + self.assertFails(expected_error, self.assertEqual, a, b, message) + + def test_assertIsNone(self): + self.assertIsNone(None) + + expected_error = 'None is not 0' + self.assertFails(expected_error, self.assertIsNone, 0) + + def test_assertIsNotNone(self): + self.assertIsNotNone(0) + self.assertIsNotNone("0") + + expected_error = 'None matches Is(None)' + self.assertFails(expected_error, self.assertIsNotNone, None) + + +class TestAddCleanup(TestCase): + """Tests for TestCase.addCleanup.""" + + run_test_with = FullStackRunTest + + class LoggingTest(TestCase): + """A test that logs calls to setUp, runTest and tearDown.""" + + def setUp(self): + TestCase.setUp(self) + self._calls = ['setUp'] + + def brokenSetUp(self): + # A tearDown that deliberately fails. + self._calls = ['brokenSetUp'] + raise RuntimeError('Deliberate Failure') + + def runTest(self): + self._calls.append('runTest') + + def brokenTest(self): + raise RuntimeError('Deliberate broken test') + + def tearDown(self): + self._calls.append('tearDown') + TestCase.tearDown(self) + + def setUp(self): + TestCase.setUp(self) + self._result_calls = [] + self.test = TestAddCleanup.LoggingTest('runTest') + self.logging_result = LoggingResult(self._result_calls) + + def assertErrorLogEqual(self, messages): + self.assertEqual(messages, [call[0] for call in self._result_calls]) + + def assertTestLogEqual(self, messages): + """Assert that the call log equals 'messages'.""" + case = self._result_calls[0][1] + self.assertEqual(messages, case._calls) + + def logAppender(self, message): + """A cleanup that appends 'message' to the tests log. + + Cleanups are callables that are added to a test by addCleanup. To + verify that our cleanups run in the right order, we add strings to a + list that acts as a log. This method returns a cleanup that will add + the given message to that log when run. + """ + self.test._calls.append(message) + + def test_fixture(self): + # A normal run of self.test logs 'setUp', 'runTest' and 'tearDown'. + # This test doesn't test addCleanup itself, it just sanity checks the + # fixture. + self.test.run(self.logging_result) + self.assertTestLogEqual(['setUp', 'runTest', 'tearDown']) + + def test_cleanup_run_before_tearDown(self): + # Cleanup functions added with 'addCleanup' are called before tearDown + # runs. + self.test.addCleanup(self.logAppender, 'cleanup') + self.test.run(self.logging_result) + self.assertTestLogEqual(['setUp', 'runTest', 'tearDown', 'cleanup']) + + def test_add_cleanup_called_if_setUp_fails(self): + # Cleanup functions added with 'addCleanup' are called even if setUp + # fails. Note that tearDown has a different behavior: it is only + # called when setUp succeeds. + self.test.setUp = self.test.brokenSetUp + self.test.addCleanup(self.logAppender, 'cleanup') + self.test.run(self.logging_result) + self.assertTestLogEqual(['brokenSetUp', 'cleanup']) + + def test_addCleanup_called_in_reverse_order(self): + # Cleanup functions added with 'addCleanup' are called in reverse + # order. + # + # One of the main uses of addCleanup is to dynamically create + # resources that need some sort of explicit tearDown. Often one + # resource will be created in terms of another, e.g., + # self.first = self.makeFirst() + # self.second = self.makeSecond(self.first) + # + # When this happens, we generally want to clean up the second resource + # before the first one, since the second depends on the first. + self.test.addCleanup(self.logAppender, 'first') + self.test.addCleanup(self.logAppender, 'second') + self.test.run(self.logging_result) + self.assertTestLogEqual( + ['setUp', 'runTest', 'tearDown', 'second', 'first']) + + def test_tearDown_runs_after_cleanup_failure(self): + # tearDown runs even if a cleanup function fails. + self.test.addCleanup(lambda: 1/0) + self.test.run(self.logging_result) + self.assertTestLogEqual(['setUp', 'runTest', 'tearDown']) + + def test_cleanups_continue_running_after_error(self): + # All cleanups are always run, even if one or two of them fail. + self.test.addCleanup(self.logAppender, 'first') + self.test.addCleanup(lambda: 1/0) + self.test.addCleanup(self.logAppender, 'second') + self.test.run(self.logging_result) + self.assertTestLogEqual( + ['setUp', 'runTest', 'tearDown', 'second', 'first']) + + def test_error_in_cleanups_are_captured(self): + # If a cleanup raises an error, we want to record it and fail the the + # test, even though we go on to run other cleanups. + self.test.addCleanup(lambda: 1/0) + self.test.run(self.logging_result) + self.assertErrorLogEqual(['startTest', 'addError', 'stopTest']) + + def test_keyboard_interrupt_not_caught(self): + # If a cleanup raises KeyboardInterrupt, it gets reraised. + def raiseKeyboardInterrupt(): + raise KeyboardInterrupt() + self.test.addCleanup(raiseKeyboardInterrupt) + self.assertThat(lambda:self.test.run(self.logging_result), + Raises(MatchesException(KeyboardInterrupt))) + + def test_all_errors_from_MultipleExceptions_reported(self): + # When a MultipleExceptions exception is caught, all the errors are + # reported. + def raiseMany(): + try: + 1/0 + except Exception: + exc_info1 = sys.exc_info() + try: + 1/0 + except Exception: + exc_info2 = sys.exc_info() + raise MultipleExceptions(exc_info1, exc_info2) + self.test.addCleanup(raiseMany) + self.logging_result = ExtendedTestResult() + self.test.run(self.logging_result) + self.assertEqual(['startTest', 'addError', 'stopTest'], + [event[0] for event in self.logging_result._events]) + self.assertEqual(set(['traceback', 'traceback-1']), + set(self.logging_result._events[1][2].keys())) + + def test_multipleCleanupErrorsReported(self): + # Errors from all failing cleanups are reported as separate backtraces. + self.test.addCleanup(lambda: 1/0) + self.test.addCleanup(lambda: 1/0) + self.logging_result = ExtendedTestResult() + self.test.run(self.logging_result) + self.assertEqual(['startTest', 'addError', 'stopTest'], + [event[0] for event in self.logging_result._events]) + self.assertEqual(set(['traceback', 'traceback-1']), + set(self.logging_result._events[1][2].keys())) + + def test_multipleErrorsCoreAndCleanupReported(self): + # Errors from all failing cleanups are reported, with stopTest, + # startTest inserted. + self.test = TestAddCleanup.LoggingTest('brokenTest') + self.test.addCleanup(lambda: 1/0) + self.test.addCleanup(lambda: 1/0) + self.logging_result = ExtendedTestResult() + self.test.run(self.logging_result) + self.assertEqual(['startTest', 'addError', 'stopTest'], + [event[0] for event in self.logging_result._events]) + self.assertEqual(set(['traceback', 'traceback-1', 'traceback-2']), + set(self.logging_result._events[1][2].keys())) + + +class TestWithDetails(TestCase): + + run_test_with = FullStackRunTest + + def assertDetailsProvided(self, case, expected_outcome, expected_keys): + """Assert that when case is run, details are provided to the result. + + :param case: A TestCase to run. + :param expected_outcome: The call that should be made. + :param expected_keys: The keys to look for. + """ + result = ExtendedTestResult() + case.run(result) + case = result._events[0][1] + expected = [ + ('startTest', case), + (expected_outcome, case), + ('stopTest', case), + ] + self.assertEqual(3, len(result._events)) + self.assertEqual(expected[0], result._events[0]) + self.assertEqual(expected[1], result._events[1][0:2]) + # Checking the TB is right is rather tricky. doctest line matching + # would help, but 'meh'. + self.assertEqual(sorted(expected_keys), + sorted(result._events[1][2].keys())) + self.assertEqual(expected[-1], result._events[-1]) + + def get_content(self): + return content.Content( + content.ContentType("text", "foo"), lambda: [_b('foo')]) + + +class TestExpectedFailure(TestWithDetails): + """Tests for expected failures and unexpected successess.""" + + run_test_with = FullStackRunTest + + def make_unexpected_case(self): + class Case(TestCase): + def test(self): + raise testcase._UnexpectedSuccess + case = Case('test') + return case + + def test_raising__UnexpectedSuccess_py27(self): + case = self.make_unexpected_case() + result = Python27TestResult() + case.run(result) + case = result._events[0][1] + self.assertEqual([ + ('startTest', case), + ('addUnexpectedSuccess', case), + ('stopTest', case), + ], result._events) + + def test_raising__UnexpectedSuccess_extended(self): + case = self.make_unexpected_case() + result = ExtendedTestResult() + case.run(result) + case = result._events[0][1] + self.assertEqual([ + ('startTest', case), + ('addUnexpectedSuccess', case, {}), + ('stopTest', case), + ], result._events) + + def make_xfail_case_xfails(self): + content = self.get_content() + class Case(TestCase): + def test(self): + self.addDetail("foo", content) + self.expectFailure("we are sad", self.assertEqual, + 1, 0) + case = Case('test') + return case + + def make_xfail_case_succeeds(self): + content = self.get_content() + class Case(TestCase): + def test(self): + self.addDetail("foo", content) + self.expectFailure("we are sad", self.assertEqual, + 1, 1) + case = Case('test') + return case + + def test_expectFailure_KnownFailure_extended(self): + case = self.make_xfail_case_xfails() + self.assertDetailsProvided(case, "addExpectedFailure", + ["foo", "traceback", "reason"]) + + def test_expectFailure_KnownFailure_unexpected_success(self): + case = self.make_xfail_case_succeeds() + self.assertDetailsProvided(case, "addUnexpectedSuccess", + ["foo", "reason"]) + + +class TestUniqueFactories(TestCase): + """Tests for getUniqueString and getUniqueInteger.""" + + run_test_with = FullStackRunTest + + def test_getUniqueInteger(self): + # getUniqueInteger returns an integer that increments each time you + # call it. + one = self.getUniqueInteger() + self.assertEqual(1, one) + two = self.getUniqueInteger() + self.assertEqual(2, two) + + def test_getUniqueString(self): + # getUniqueString returns the current test id followed by a unique + # integer. + name_one = self.getUniqueString() + self.assertEqual('%s-%d' % (self.id(), 1), name_one) + name_two = self.getUniqueString() + self.assertEqual('%s-%d' % (self.id(), 2), name_two) + + def test_getUniqueString_prefix(self): + # If getUniqueString is given an argument, it uses that argument as + # the prefix of the unique string, rather than the test id. + name_one = self.getUniqueString('foo') + self.assertThat(name_one, Equals('foo-1')) + name_two = self.getUniqueString('bar') + self.assertThat(name_two, Equals('bar-2')) + + +class TestCloneTestWithNewId(TestCase): + """Tests for clone_test_with_new_id.""" + + run_test_with = FullStackRunTest + + def test_clone_test_with_new_id(self): + class FooTestCase(TestCase): + def test_foo(self): + pass + test = FooTestCase('test_foo') + oldName = test.id() + newName = self.getUniqueString() + newTest = clone_test_with_new_id(test, newName) + self.assertEqual(newName, newTest.id()) + self.assertEqual(oldName, test.id(), + "the original test instance should be unchanged.") + + def test_cloned_testcase_does_not_share_details(self): + """A cloned TestCase does not share the details dict.""" + class Test(TestCase): + def test_foo(self): + self.addDetail( + 'foo', content.Content('text/plain', lambda: 'foo')) + orig_test = Test('test_foo') + cloned_test = clone_test_with_new_id(orig_test, self.getUniqueString()) + orig_test.run(unittest.TestResult()) + self.assertEqual('foo', orig_test.getDetails()['foo'].iter_bytes()) + self.assertEqual(None, cloned_test.getDetails().get('foo')) + + +class TestDetailsProvided(TestWithDetails): + + run_test_with = FullStackRunTest + + def test_addDetail(self): + mycontent = self.get_content() + self.addDetail("foo", mycontent) + details = self.getDetails() + self.assertEqual({"foo": mycontent}, details) + + def test_addError(self): + class Case(TestCase): + def test(this): + this.addDetail("foo", self.get_content()) + 1/0 + self.assertDetailsProvided(Case("test"), "addError", + ["foo", "traceback"]) + + def test_addFailure(self): + class Case(TestCase): + def test(this): + this.addDetail("foo", self.get_content()) + self.fail('yo') + self.assertDetailsProvided(Case("test"), "addFailure", + ["foo", "traceback"]) + + def test_addSkip(self): + class Case(TestCase): + def test(this): + this.addDetail("foo", self.get_content()) + self.skip('yo') + self.assertDetailsProvided(Case("test"), "addSkip", + ["foo", "reason"]) + + def test_addSucccess(self): + class Case(TestCase): + def test(this): + this.addDetail("foo", self.get_content()) + self.assertDetailsProvided(Case("test"), "addSuccess", + ["foo"]) + + def test_addUnexpectedSuccess(self): + class Case(TestCase): + def test(this): + this.addDetail("foo", self.get_content()) + raise testcase._UnexpectedSuccess() + self.assertDetailsProvided(Case("test"), "addUnexpectedSuccess", + ["foo"]) + + def test_addDetails_from_Mismatch(self): + content = self.get_content() + class Mismatch(object): + def describe(self): + return "Mismatch" + def get_details(self): + return {"foo": content} + class Matcher(object): + def match(self, thing): + return Mismatch() + def __str__(self): + return "a description" + class Case(TestCase): + def test(self): + self.assertThat("foo", Matcher()) + self.assertDetailsProvided(Case("test"), "addFailure", + ["foo", "traceback"]) + + def test_multiple_addDetails_from_Mismatch(self): + content = self.get_content() + class Mismatch(object): + def describe(self): + return "Mismatch" + def get_details(self): + return {"foo": content, "bar": content} + class Matcher(object): + def match(self, thing): + return Mismatch() + def __str__(self): + return "a description" + class Case(TestCase): + def test(self): + self.assertThat("foo", Matcher()) + self.assertDetailsProvided(Case("test"), "addFailure", + ["bar", "foo", "traceback"]) + + def test_addDetails_with_same_name_as_key_from_get_details(self): + content = self.get_content() + class Mismatch(object): + def describe(self): + return "Mismatch" + def get_details(self): + return {"foo": content} + class Matcher(object): + def match(self, thing): + return Mismatch() + def __str__(self): + return "a description" + class Case(TestCase): + def test(self): + self.addDetail("foo", content) + self.assertThat("foo", Matcher()) + self.assertDetailsProvided(Case("test"), "addFailure", + ["foo", "foo-1", "traceback"]) + + +class TestSetupTearDown(TestCase): + + run_test_with = FullStackRunTest + + def test_setUpNotCalled(self): + class DoesnotcallsetUp(TestCase): + def setUp(self): + pass + def test_method(self): + pass + result = unittest.TestResult() + DoesnotcallsetUp('test_method').run(result) + self.assertEqual(1, len(result.errors)) + + def test_tearDownNotCalled(self): + class DoesnotcalltearDown(TestCase): + def test_method(self): + pass + def tearDown(self): + pass + result = unittest.TestResult() + DoesnotcalltearDown('test_method').run(result) + self.assertEqual(1, len(result.errors)) + + +class TestSkipping(TestCase): + """Tests for skipping of tests functionality.""" + + run_test_with = FullStackRunTest + + def test_skip_causes_skipException(self): + self.assertThat(lambda:self.skip("Skip this test"), + Raises(MatchesException(self.skipException))) + + def test_can_use_skipTest(self): + self.assertThat(lambda:self.skipTest("Skip this test"), + Raises(MatchesException(self.skipException))) + + def test_skip_without_reason_works(self): + class Test(TestCase): + def test(self): + raise self.skipException() + case = Test("test") + result = ExtendedTestResult() + case.run(result) + self.assertEqual('addSkip', result._events[1][0]) + self.assertEqual('no reason given.', + ''.join(result._events[1][2]['reason'].iter_text())) + + def test_skipException_in_setup_calls_result_addSkip(self): + class TestThatRaisesInSetUp(TestCase): + def setUp(self): + TestCase.setUp(self) + self.skip("skipping this test") + def test_that_passes(self): + pass + calls = [] + result = LoggingResult(calls) + test = TestThatRaisesInSetUp("test_that_passes") + test.run(result) + case = result._events[0][1] + self.assertEqual([('startTest', case), + ('addSkip', case, "skipping this test"), ('stopTest', case)], + calls) + + def test_skipException_in_test_method_calls_result_addSkip(self): + class SkippingTest(TestCase): + def test_that_raises_skipException(self): + self.skip("skipping this test") + result = Python27TestResult() + test = SkippingTest("test_that_raises_skipException") + test.run(result) + case = result._events[0][1] + self.assertEqual([('startTest', case), + ('addSkip', case, "skipping this test"), ('stopTest', case)], + result._events) + + def test_skip__in_setup_with_old_result_object_calls_addSuccess(self): + class SkippingTest(TestCase): + def setUp(self): + TestCase.setUp(self) + raise self.skipException("skipping this test") + def test_that_raises_skipException(self): + pass + result = Python26TestResult() + test = SkippingTest("test_that_raises_skipException") + test.run(result) + self.assertEqual('addSuccess', result._events[1][0]) + + def test_skip_with_old_result_object_calls_addError(self): + class SkippingTest(TestCase): + def test_that_raises_skipException(self): + raise self.skipException("skipping this test") + result = Python26TestResult() + test = SkippingTest("test_that_raises_skipException") + test.run(result) + self.assertEqual('addSuccess', result._events[1][0]) + + def test_skip_decorator(self): + class SkippingTest(TestCase): + @skip("skipping this test") + def test_that_is_decorated_with_skip(self): + self.fail() + result = Python26TestResult() + test = SkippingTest("test_that_is_decorated_with_skip") + test.run(result) + self.assertEqual('addSuccess', result._events[1][0]) + + def test_skipIf_decorator(self): + class SkippingTest(TestCase): + @skipIf(True, "skipping this test") + def test_that_is_decorated_with_skipIf(self): + self.fail() + result = Python26TestResult() + test = SkippingTest("test_that_is_decorated_with_skipIf") + test.run(result) + self.assertEqual('addSuccess', result._events[1][0]) + + def test_skipUnless_decorator(self): + class SkippingTest(TestCase): + @skipUnless(False, "skipping this test") + def test_that_is_decorated_with_skipUnless(self): + self.fail() + result = Python26TestResult() + test = SkippingTest("test_that_is_decorated_with_skipUnless") + test.run(result) + self.assertEqual('addSuccess', result._events[1][0]) + + +class TestOnException(TestCase): + + run_test_with = FullStackRunTest + + def test_default_works(self): + events = [] + class Case(TestCase): + def method(self): + self.onException(an_exc_info) + events.append(True) + case = Case("method") + case.run() + self.assertThat(events, Equals([True])) + + def test_added_handler_works(self): + events = [] + class Case(TestCase): + def method(self): + self.addOnException(events.append) + self.onException(an_exc_info) + case = Case("method") + case.run() + self.assertThat(events, Equals([an_exc_info])) + + def test_handler_that_raises_is_not_caught(self): + events = [] + class Case(TestCase): + def method(self): + self.addOnException(events.index) + self.assertThat(lambda: self.onException(an_exc_info), + Raises(MatchesException(ValueError))) + case = Case("method") + case.run() + self.assertThat(events, Equals([])) + + +class TestPatchSupport(TestCase): + + run_test_with = FullStackRunTest + + class Case(TestCase): + def test(self): + pass + + def test_patch(self): + # TestCase.patch masks obj.attribute with the new value. + self.foo = 'original' + test = self.Case('test') + test.patch(self, 'foo', 'patched') + self.assertEqual('patched', self.foo) + + def test_patch_restored_after_run(self): + # TestCase.patch masks obj.attribute with the new value, but restores + # the original value after the test is finished. + self.foo = 'original' + test = self.Case('test') + test.patch(self, 'foo', 'patched') + test.run() + self.assertEqual('original', self.foo) + + def test_successive_patches_apply(self): + # TestCase.patch can be called multiple times per test. Each time you + # call it, it overrides the original value. + self.foo = 'original' + test = self.Case('test') + test.patch(self, 'foo', 'patched') + test.patch(self, 'foo', 'second') + self.assertEqual('second', self.foo) + + def test_successive_patches_restored_after_run(self): + # TestCase.patch restores the original value, no matter how many times + # it was called. + self.foo = 'original' + test = self.Case('test') + test.patch(self, 'foo', 'patched') + test.patch(self, 'foo', 'second') + test.run() + self.assertEqual('original', self.foo) + + def test_patch_nonexistent_attribute(self): + # TestCase.patch can be used to patch a non-existent attribute. + test = self.Case('test') + test.patch(self, 'doesntexist', 'patched') + self.assertEqual('patched', self.doesntexist) + + def test_restore_nonexistent_attribute(self): + # TestCase.patch can be used to patch a non-existent attribute, after + # the test run, the attribute is then removed from the object. + test = self.Case('test') + test.patch(self, 'doesntexist', 'patched') + test.run() + marker = object() + value = getattr(self, 'doesntexist', marker) + self.assertIs(marker, value) + + +class TestTestCaseSuper(TestCase): + + run_test_with = FullStackRunTest + + def test_setup_uses_super(self): + class OtherBaseCase(unittest.TestCase): + setup_called = False + def setUp(self): + self.setup_called = True + super(OtherBaseCase, self).setUp() + class OurCase(TestCase, OtherBaseCase): + def runTest(self): + pass + test = OurCase() + test.setUp() + test.tearDown() + self.assertTrue(test.setup_called) + + def test_teardown_uses_super(self): + class OtherBaseCase(unittest.TestCase): + teardown_called = False + def tearDown(self): + self.teardown_called = True + super(OtherBaseCase, self).tearDown() + class OurCase(TestCase, OtherBaseCase): + def runTest(self): + pass + test = OurCase() + test.setUp() + test.tearDown() + self.assertTrue(test.teardown_called) + + +def test_suite(): + from unittest import TestLoader + return TestLoader().loadTestsFromName(__name__) diff --git a/test/3rdparty/testtools-0.9.12/testtools/tests/test_testresult.py b/test/3rdparty/testtools-0.9.12/testtools/tests/test_testresult.py new file mode 100644 index 00000000000..a241788bea8 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/tests/test_testresult.py @@ -0,0 +1,1507 @@ +# Copyright (c) 2008-2011 testtools developers. See LICENSE for details. + +"""Test TestResults and related things.""" + +__metaclass__ = type + +import codecs +import datetime +import doctest +import os +import shutil +import sys +import tempfile +import threading +import warnings + +from testtools import ( + ExtendedToOriginalDecorator, + MultiTestResult, + TestCase, + TestResult, + TextTestResult, + ThreadsafeForwardingResult, + testresult, + ) +from testtools.compat import ( + _b, + _get_exception_encoding, + _r, + _u, + str_is_unicode, + StringIO, + ) +from testtools.content import ( + Content, + content_from_stream, + text_content, + ) +from testtools.content_type import ContentType, UTF8_TEXT +from testtools.matchers import ( + DocTestMatches, + Equals, + MatchesException, + Raises, + ) +from testtools.tests.helpers import ( + an_exc_info, + FullStackRunTest, + LoggingResult, + run_with_stack_hidden, + ) +from testtools.testresult.doubles import ( + Python26TestResult, + Python27TestResult, + ExtendedTestResult, + ) +from testtools.testresult.real import ( + _details_to_str, + utc, + ) + + +def make_erroring_test(): + class Test(TestCase): + def error(self): + 1/0 + return Test("error") + + +def make_failing_test(): + class Test(TestCase): + def failed(self): + self.fail("yo!") + return Test("failed") + + +def make_unexpectedly_successful_test(): + class Test(TestCase): + def succeeded(self): + self.expectFailure("yo!", lambda: None) + return Test("succeeded") + + +def make_test(): + class Test(TestCase): + def test(self): + pass + return Test("test") + + +def make_exception_info(exceptionFactory, *args, **kwargs): + try: + raise exceptionFactory(*args, **kwargs) + except: + return sys.exc_info() + + +class Python26Contract(object): + + def test_fresh_result_is_successful(self): + # A result is considered successful before any tests are run. + result = self.makeResult() + self.assertTrue(result.wasSuccessful()) + + def test_addError_is_failure(self): + # addError fails the test run. + result = self.makeResult() + result.startTest(self) + result.addError(self, an_exc_info) + result.stopTest(self) + self.assertFalse(result.wasSuccessful()) + + def test_addFailure_is_failure(self): + # addFailure fails the test run. + result = self.makeResult() + result.startTest(self) + result.addFailure(self, an_exc_info) + result.stopTest(self) + self.assertFalse(result.wasSuccessful()) + + def test_addSuccess_is_success(self): + # addSuccess does not fail the test run. + result = self.makeResult() + result.startTest(self) + result.addSuccess(self) + result.stopTest(self) + self.assertTrue(result.wasSuccessful()) + + +class Python27Contract(Python26Contract): + + def test_addExpectedFailure(self): + # Calling addExpectedFailure(test, exc_info) completes ok. + result = self.makeResult() + result.startTest(self) + result.addExpectedFailure(self, an_exc_info) + + def test_addExpectedFailure_is_success(self): + # addExpectedFailure does not fail the test run. + result = self.makeResult() + result.startTest(self) + result.addExpectedFailure(self, an_exc_info) + result.stopTest(self) + self.assertTrue(result.wasSuccessful()) + + def test_addSkipped(self): + # Calling addSkip(test, reason) completes ok. + result = self.makeResult() + result.startTest(self) + result.addSkip(self, _u("Skipped for some reason")) + + def test_addSkip_is_success(self): + # addSkip does not fail the test run. + result = self.makeResult() + result.startTest(self) + result.addSkip(self, _u("Skipped for some reason")) + result.stopTest(self) + self.assertTrue(result.wasSuccessful()) + + def test_addUnexpectedSuccess(self): + # Calling addUnexpectedSuccess(test) completes ok. + result = self.makeResult() + result.startTest(self) + result.addUnexpectedSuccess(self) + + def test_addUnexpectedSuccess_was_successful(self): + # addUnexpectedSuccess does not fail the test run in Python 2.7. + result = self.makeResult() + result.startTest(self) + result.addUnexpectedSuccess(self) + result.stopTest(self) + self.assertTrue(result.wasSuccessful()) + + def test_startStopTestRun(self): + # Calling startTestRun completes ok. + result = self.makeResult() + result.startTestRun() + result.stopTestRun() + + +class DetailsContract(Python27Contract): + """Tests for the contract of TestResults.""" + + def test_addExpectedFailure_details(self): + # Calling addExpectedFailure(test, details=xxx) completes ok. + result = self.makeResult() + result.startTest(self) + result.addExpectedFailure(self, details={}) + + def test_addError_details(self): + # Calling addError(test, details=xxx) completes ok. + result = self.makeResult() + result.startTest(self) + result.addError(self, details={}) + + def test_addFailure_details(self): + # Calling addFailure(test, details=xxx) completes ok. + result = self.makeResult() + result.startTest(self) + result.addFailure(self, details={}) + + def test_addSkipped_details(self): + # Calling addSkip(test, reason) completes ok. + result = self.makeResult() + result.startTest(self) + result.addSkip(self, details={}) + + def test_addUnexpectedSuccess_details(self): + # Calling addUnexpectedSuccess(test) completes ok. + result = self.makeResult() + result.startTest(self) + result.addUnexpectedSuccess(self, details={}) + + def test_addSuccess_details(self): + # Calling addSuccess(test) completes ok. + result = self.makeResult() + result.startTest(self) + result.addSuccess(self, details={}) + + +class FallbackContract(DetailsContract): + """When we fallback we take our policy choice to map calls. + + For instance, we map unexpectedSuccess to an error code, not to success. + """ + + def test_addUnexpectedSuccess_was_successful(self): + # addUnexpectedSuccess fails test run in testtools. + result = self.makeResult() + result.startTest(self) + result.addUnexpectedSuccess(self) + result.stopTest(self) + self.assertFalse(result.wasSuccessful()) + + +class StartTestRunContract(FallbackContract): + """Defines the contract for testtools policy choices. + + That is things which are not simply extensions to unittest but choices we + have made differently. + """ + + def test_startTestRun_resets_unexpected_success(self): + result = self.makeResult() + result.startTest(self) + result.addUnexpectedSuccess(self) + result.stopTest(self) + result.startTestRun() + self.assertTrue(result.wasSuccessful()) + + def test_startTestRun_resets_failure(self): + result = self.makeResult() + result.startTest(self) + result.addFailure(self, an_exc_info) + result.stopTest(self) + result.startTestRun() + self.assertTrue(result.wasSuccessful()) + + def test_startTestRun_resets_errors(self): + result = self.makeResult() + result.startTest(self) + result.addError(self, an_exc_info) + result.stopTest(self) + result.startTestRun() + self.assertTrue(result.wasSuccessful()) + + +class TestTestResultContract(TestCase, StartTestRunContract): + + run_test_with = FullStackRunTest + + def makeResult(self): + return TestResult() + + +class TestMultiTestResultContract(TestCase, StartTestRunContract): + + run_test_with = FullStackRunTest + + def makeResult(self): + return MultiTestResult(TestResult(), TestResult()) + + +class TestTextTestResultContract(TestCase, StartTestRunContract): + + run_test_with = FullStackRunTest + + def makeResult(self): + return TextTestResult(StringIO()) + + +class TestThreadSafeForwardingResultContract(TestCase, StartTestRunContract): + + run_test_with = FullStackRunTest + + def makeResult(self): + result_semaphore = threading.Semaphore(1) + target = TestResult() + return ThreadsafeForwardingResult(target, result_semaphore) + + +class TestExtendedTestResultContract(TestCase, StartTestRunContract): + + def makeResult(self): + return ExtendedTestResult() + + +class TestPython26TestResultContract(TestCase, Python26Contract): + + def makeResult(self): + return Python26TestResult() + + +class TestAdaptedPython26TestResultContract(TestCase, FallbackContract): + + def makeResult(self): + return ExtendedToOriginalDecorator(Python26TestResult()) + + +class TestPython27TestResultContract(TestCase, Python27Contract): + + def makeResult(self): + return Python27TestResult() + + +class TestAdaptedPython27TestResultContract(TestCase, DetailsContract): + + def makeResult(self): + return ExtendedToOriginalDecorator(Python27TestResult()) + + +class TestTestResult(TestCase): + """Tests for 'TestResult'.""" + + run_tests_with = FullStackRunTest + + def makeResult(self): + """Make an arbitrary result for testing.""" + return TestResult() + + def test_addSkipped(self): + # Calling addSkip on a TestResult records the test that was skipped in + # its skip_reasons dict. + result = self.makeResult() + result.addSkip(self, _u("Skipped for some reason")) + self.assertEqual({_u("Skipped for some reason"):[self]}, + result.skip_reasons) + result.addSkip(self, _u("Skipped for some reason")) + self.assertEqual({_u("Skipped for some reason"):[self, self]}, + result.skip_reasons) + result.addSkip(self, _u("Skipped for another reason")) + self.assertEqual({_u("Skipped for some reason"):[self, self], + _u("Skipped for another reason"):[self]}, + result.skip_reasons) + + def test_now_datetime_now(self): + result = self.makeResult() + olddatetime = testresult.real.datetime + def restore(): + testresult.real.datetime = olddatetime + self.addCleanup(restore) + class Module: + pass + now = datetime.datetime.now(utc) + stubdatetime = Module() + stubdatetime.datetime = Module() + stubdatetime.datetime.now = lambda tz: now + testresult.real.datetime = stubdatetime + # Calling _now() looks up the time. + self.assertEqual(now, result._now()) + then = now + datetime.timedelta(0, 1) + # Set an explicit datetime, which gets returned from then on. + result.time(then) + self.assertNotEqual(now, result._now()) + self.assertEqual(then, result._now()) + # go back to looking it up. + result.time(None) + self.assertEqual(now, result._now()) + + def test_now_datetime_time(self): + result = self.makeResult() + now = datetime.datetime.now(utc) + result.time(now) + self.assertEqual(now, result._now()) + + def test_traceback_formatting_without_stack_hidden(self): + # During the testtools test run, we show our levels of the stack, + # because we want to be able to use our test suite to debug our own + # code. + result = self.makeResult() + test = make_erroring_test() + test.run(result) + self.assertThat( + result.errors[0][1], + DocTestMatches( + 'Traceback (most recent call last):\n' + ' File "...testtools...runtest.py", line ..., in _run_user\n' + ' return fn(*args, **kwargs)\n' + ' File "...testtools...testcase.py", line ..., in _run_test_method\n' + ' return self._get_test_method()()\n' + ' File "...testtools...tests...test_testresult.py", line ..., in error\n' + ' 1/0\n' + 'ZeroDivisionError: ...\n', + doctest.ELLIPSIS | doctest.REPORT_UDIFF)) + + def test_traceback_formatting_with_stack_hidden(self): + result = self.makeResult() + test = make_erroring_test() + run_with_stack_hidden(True, test.run, result) + self.assertThat( + result.errors[0][1], + DocTestMatches( + 'Traceback (most recent call last):\n' + ' File "...testtools...tests...test_testresult.py", line ..., in error\n' + ' 1/0\n' + 'ZeroDivisionError: ...\n', + doctest.ELLIPSIS)) + + +class TestMultiTestResult(TestCase): + """Tests for 'MultiTestResult'.""" + + def setUp(self): + super(TestMultiTestResult, self).setUp() + self.result1 = LoggingResult([]) + self.result2 = LoggingResult([]) + self.multiResult = MultiTestResult(self.result1, self.result2) + + def assertResultLogsEqual(self, expectedEvents): + """Assert that our test results have received the expected events.""" + self.assertEqual(expectedEvents, self.result1._events) + self.assertEqual(expectedEvents, self.result2._events) + + def test_empty(self): + # Initializing a `MultiTestResult` doesn't do anything to its + # `TestResult`s. + self.assertResultLogsEqual([]) + + def test_startTest(self): + # Calling `startTest` on a `MultiTestResult` calls `startTest` on all + # its `TestResult`s. + self.multiResult.startTest(self) + self.assertResultLogsEqual([('startTest', self)]) + + def test_stopTest(self): + # Calling `stopTest` on a `MultiTestResult` calls `stopTest` on all + # its `TestResult`s. + self.multiResult.stopTest(self) + self.assertResultLogsEqual([('stopTest', self)]) + + def test_addSkipped(self): + # Calling `addSkip` on a `MultiTestResult` calls addSkip on its + # results. + reason = _u("Skipped for some reason") + self.multiResult.addSkip(self, reason) + self.assertResultLogsEqual([('addSkip', self, reason)]) + + def test_addSuccess(self): + # Calling `addSuccess` on a `MultiTestResult` calls `addSuccess` on + # all its `TestResult`s. + self.multiResult.addSuccess(self) + self.assertResultLogsEqual([('addSuccess', self)]) + + def test_done(self): + # Calling `done` on a `MultiTestResult` calls `done` on all its + # `TestResult`s. + self.multiResult.done() + self.assertResultLogsEqual([('done')]) + + def test_addFailure(self): + # Calling `addFailure` on a `MultiTestResult` calls `addFailure` on + # all its `TestResult`s. + exc_info = make_exception_info(AssertionError, 'failure') + self.multiResult.addFailure(self, exc_info) + self.assertResultLogsEqual([('addFailure', self, exc_info)]) + + def test_addError(self): + # Calling `addError` on a `MultiTestResult` calls `addError` on all + # its `TestResult`s. + exc_info = make_exception_info(RuntimeError, 'error') + self.multiResult.addError(self, exc_info) + self.assertResultLogsEqual([('addError', self, exc_info)]) + + def test_startTestRun(self): + # Calling `startTestRun` on a `MultiTestResult` forwards to all its + # `TestResult`s. + self.multiResult.startTestRun() + self.assertResultLogsEqual([('startTestRun')]) + + def test_stopTestRun(self): + # Calling `stopTestRun` on a `MultiTestResult` forwards to all its + # `TestResult`s. + self.multiResult.stopTestRun() + self.assertResultLogsEqual([('stopTestRun')]) + + def test_stopTestRun_returns_results(self): + # `MultiTestResult.stopTestRun` returns a tuple of all of the return + # values the `stopTestRun`s that it forwards to. + class Result(LoggingResult): + def stopTestRun(self): + super(Result, self).stopTestRun() + return 'foo' + multi_result = MultiTestResult(Result([]), Result([])) + result = multi_result.stopTestRun() + self.assertEqual(('foo', 'foo'), result) + + def test_time(self): + # the time call is dispatched, not eaten by the base class + self.multiResult.time('foo') + self.assertResultLogsEqual([('time', 'foo')]) + + +class TestTextTestResult(TestCase): + """Tests for 'TextTestResult'.""" + + def setUp(self): + super(TestTextTestResult, self).setUp() + self.result = TextTestResult(StringIO()) + + def getvalue(self): + return self.result.stream.getvalue() + + def test__init_sets_stream(self): + result = TextTestResult("fp") + self.assertEqual("fp", result.stream) + + def reset_output(self): + self.result.stream = StringIO() + + def test_startTestRun(self): + self.result.startTestRun() + self.assertEqual("Tests running...\n", self.getvalue()) + + def test_stopTestRun_count_many(self): + test = make_test() + self.result.startTestRun() + self.result.startTest(test) + self.result.stopTest(test) + self.result.startTest(test) + self.result.stopTest(test) + self.result.stream = StringIO() + self.result.stopTestRun() + self.assertThat(self.getvalue(), + DocTestMatches("\nRan 2 tests in ...s\n...", doctest.ELLIPSIS)) + + def test_stopTestRun_count_single(self): + test = make_test() + self.result.startTestRun() + self.result.startTest(test) + self.result.stopTest(test) + self.reset_output() + self.result.stopTestRun() + self.assertThat(self.getvalue(), + DocTestMatches("\nRan 1 test in ...s\nOK\n", doctest.ELLIPSIS)) + + def test_stopTestRun_count_zero(self): + self.result.startTestRun() + self.reset_output() + self.result.stopTestRun() + self.assertThat(self.getvalue(), + DocTestMatches("\nRan 0 tests in ...s\nOK\n", doctest.ELLIPSIS)) + + def test_stopTestRun_current_time(self): + test = make_test() + now = datetime.datetime.now(utc) + self.result.time(now) + self.result.startTestRun() + self.result.startTest(test) + now = now + datetime.timedelta(0, 0, 0, 1) + self.result.time(now) + self.result.stopTest(test) + self.reset_output() + self.result.stopTestRun() + self.assertThat(self.getvalue(), + DocTestMatches("... in 0.001s\n...", doctest.ELLIPSIS)) + + def test_stopTestRun_successful(self): + self.result.startTestRun() + self.result.stopTestRun() + self.assertThat(self.getvalue(), + DocTestMatches("...\nOK\n", doctest.ELLIPSIS)) + + def test_stopTestRun_not_successful_failure(self): + test = make_failing_test() + self.result.startTestRun() + test.run(self.result) + self.result.stopTestRun() + self.assertThat(self.getvalue(), + DocTestMatches("...\nFAILED (failures=1)\n", doctest.ELLIPSIS)) + + def test_stopTestRun_not_successful_error(self): + test = make_erroring_test() + self.result.startTestRun() + test.run(self.result) + self.result.stopTestRun() + self.assertThat(self.getvalue(), + DocTestMatches("...\nFAILED (failures=1)\n", doctest.ELLIPSIS)) + + def test_stopTestRun_not_successful_unexpected_success(self): + test = make_unexpectedly_successful_test() + self.result.startTestRun() + test.run(self.result) + self.result.stopTestRun() + self.assertThat(self.getvalue(), + DocTestMatches("...\nFAILED (failures=1)\n", doctest.ELLIPSIS)) + + def test_stopTestRun_shows_details(self): + def run_tests(): + self.result.startTestRun() + make_erroring_test().run(self.result) + make_unexpectedly_successful_test().run(self.result) + make_failing_test().run(self.result) + self.reset_output() + self.result.stopTestRun() + run_with_stack_hidden(True, run_tests) + self.assertThat(self.getvalue(), + DocTestMatches("""...====================================================================== +ERROR: testtools.tests.test_testresult.Test.error +---------------------------------------------------------------------- +Traceback (most recent call last): + File "...testtools...tests...test_testresult.py", line ..., in error + 1/0 +ZeroDivisionError:... divi... by zero... +====================================================================== +FAIL: testtools.tests.test_testresult.Test.failed +---------------------------------------------------------------------- +Traceback (most recent call last): + File "...testtools...tests...test_testresult.py", line ..., in failed + self.fail("yo!") +AssertionError: yo! +====================================================================== +UNEXPECTED SUCCESS: testtools.tests.test_testresult.Test.succeeded +---------------------------------------------------------------------- +...""", doctest.ELLIPSIS | doctest.REPORT_NDIFF)) + + +class TestThreadSafeForwardingResult(TestCase): + """Tests for `TestThreadSafeForwardingResult`.""" + + def setUp(self): + super(TestThreadSafeForwardingResult, self).setUp() + self.result_semaphore = threading.Semaphore(1) + self.target = LoggingResult([]) + self.result1 = ThreadsafeForwardingResult(self.target, + self.result_semaphore) + + def test_nonforwarding_methods(self): + # startTest and stopTest are not forwarded because they need to be + # batched. + self.result1.startTest(self) + self.result1.stopTest(self) + self.assertEqual([], self.target._events) + + def test_startTestRun(self): + self.result1.startTestRun() + self.result2 = ThreadsafeForwardingResult(self.target, + self.result_semaphore) + self.result2.startTestRun() + self.assertEqual(["startTestRun", "startTestRun"], self.target._events) + + def test_stopTestRun(self): + self.result1.stopTestRun() + self.result2 = ThreadsafeForwardingResult(self.target, + self.result_semaphore) + self.result2.stopTestRun() + self.assertEqual(["stopTestRun", "stopTestRun"], self.target._events) + + def test_forwarding_methods(self): + # error, failure, skip and success are forwarded in batches. + exc_info1 = make_exception_info(RuntimeError, 'error') + starttime1 = datetime.datetime.utcfromtimestamp(1.489) + endtime1 = datetime.datetime.utcfromtimestamp(51.476) + self.result1.time(starttime1) + self.result1.startTest(self) + self.result1.time(endtime1) + self.result1.addError(self, exc_info1) + exc_info2 = make_exception_info(AssertionError, 'failure') + starttime2 = datetime.datetime.utcfromtimestamp(2.489) + endtime2 = datetime.datetime.utcfromtimestamp(3.476) + self.result1.time(starttime2) + self.result1.startTest(self) + self.result1.time(endtime2) + self.result1.addFailure(self, exc_info2) + reason = _u("Skipped for some reason") + starttime3 = datetime.datetime.utcfromtimestamp(4.489) + endtime3 = datetime.datetime.utcfromtimestamp(5.476) + self.result1.time(starttime3) + self.result1.startTest(self) + self.result1.time(endtime3) + self.result1.addSkip(self, reason) + starttime4 = datetime.datetime.utcfromtimestamp(6.489) + endtime4 = datetime.datetime.utcfromtimestamp(7.476) + self.result1.time(starttime4) + self.result1.startTest(self) + self.result1.time(endtime4) + self.result1.addSuccess(self) + self.assertEqual([ + ('time', starttime1), + ('startTest', self), + ('time', endtime1), + ('addError', self, exc_info1), + ('stopTest', self), + ('time', starttime2), + ('startTest', self), + ('time', endtime2), + ('addFailure', self, exc_info2), + ('stopTest', self), + ('time', starttime3), + ('startTest', self), + ('time', endtime3), + ('addSkip', self, reason), + ('stopTest', self), + ('time', starttime4), + ('startTest', self), + ('time', endtime4), + ('addSuccess', self), + ('stopTest', self), + ], self.target._events) + + +class TestExtendedToOriginalResultDecoratorBase(TestCase): + + def make_26_result(self): + self.result = Python26TestResult() + self.make_converter() + + def make_27_result(self): + self.result = Python27TestResult() + self.make_converter() + + def make_converter(self): + self.converter = ExtendedToOriginalDecorator(self.result) + + def make_extended_result(self): + self.result = ExtendedTestResult() + self.make_converter() + + def check_outcome_details(self, outcome): + """Call an outcome with a details dict to be passed through.""" + # This dict is /not/ convertible - thats deliberate, as it should + # not hit the conversion code path. + details = {'foo': 'bar'} + getattr(self.converter, outcome)(self, details=details) + self.assertEqual([(outcome, self, details)], self.result._events) + + def get_details_and_string(self): + """Get a details dict and expected string.""" + text1 = lambda: [_b("1\n2\n")] + text2 = lambda: [_b("3\n4\n")] + bin1 = lambda: [_b("5\n")] + details = {'text 1': Content(ContentType('text', 'plain'), text1), + 'text 2': Content(ContentType('text', 'strange'), text2), + 'bin 1': Content(ContentType('application', 'binary'), bin1)} + return (details, + ("Binary content:\n" + " bin 1 (application/binary)\n" + "\n" + "text 1: {{{\n" + "1\n" + "2\n" + "}}}\n" + "\n" + "text 2: {{{\n" + "3\n" + "4\n" + "}}}\n")) + + def check_outcome_details_to_exec_info(self, outcome, expected=None): + """Call an outcome with a details dict to be made into exc_info.""" + # The conversion is a done using RemoteError and the string contents + # of the text types in the details dict. + if not expected: + expected = outcome + details, err_str = self.get_details_and_string() + getattr(self.converter, outcome)(self, details=details) + err = self.converter._details_to_exc_info(details) + self.assertEqual([(expected, self, err)], self.result._events) + + def check_outcome_details_to_nothing(self, outcome, expected=None): + """Call an outcome with a details dict to be swallowed.""" + if not expected: + expected = outcome + details = {'foo': 'bar'} + getattr(self.converter, outcome)(self, details=details) + self.assertEqual([(expected, self)], self.result._events) + + def check_outcome_details_to_string(self, outcome): + """Call an outcome with a details dict to be stringified.""" + details, err_str = self.get_details_and_string() + getattr(self.converter, outcome)(self, details=details) + self.assertEqual([(outcome, self, err_str)], self.result._events) + + def check_outcome_details_to_arg(self, outcome, arg, extra_detail=None): + """Call an outcome with a details dict to have an arg extracted.""" + details, _ = self.get_details_and_string() + if extra_detail: + details.update(extra_detail) + getattr(self.converter, outcome)(self, details=details) + self.assertEqual([(outcome, self, arg)], self.result._events) + + def check_outcome_exc_info(self, outcome, expected=None): + """Check that calling a legacy outcome still works.""" + # calling some outcome with the legacy exc_info style api (no keyword + # parameters) gets passed through. + if not expected: + expected = outcome + err = sys.exc_info() + getattr(self.converter, outcome)(self, err) + self.assertEqual([(expected, self, err)], self.result._events) + + def check_outcome_exc_info_to_nothing(self, outcome, expected=None): + """Check that calling a legacy outcome on a fallback works.""" + # calling some outcome with the legacy exc_info style api (no keyword + # parameters) gets passed through. + if not expected: + expected = outcome + err = sys.exc_info() + getattr(self.converter, outcome)(self, err) + self.assertEqual([(expected, self)], self.result._events) + + def check_outcome_nothing(self, outcome, expected=None): + """Check that calling a legacy outcome still works.""" + if not expected: + expected = outcome + getattr(self.converter, outcome)(self) + self.assertEqual([(expected, self)], self.result._events) + + def check_outcome_string_nothing(self, outcome, expected): + """Check that calling outcome with a string calls expected.""" + getattr(self.converter, outcome)(self, "foo") + self.assertEqual([(expected, self)], self.result._events) + + def check_outcome_string(self, outcome): + """Check that calling outcome with a string works.""" + getattr(self.converter, outcome)(self, "foo") + self.assertEqual([(outcome, self, "foo")], self.result._events) + + +class TestExtendedToOriginalResultDecorator( + TestExtendedToOriginalResultDecoratorBase): + + def test_progress_py26(self): + self.make_26_result() + self.converter.progress(1, 2) + + def test_progress_py27(self): + self.make_27_result() + self.converter.progress(1, 2) + + def test_progress_pyextended(self): + self.make_extended_result() + self.converter.progress(1, 2) + self.assertEqual([('progress', 1, 2)], self.result._events) + + def test_shouldStop(self): + self.make_26_result() + self.assertEqual(False, self.converter.shouldStop) + self.converter.decorated.stop() + self.assertEqual(True, self.converter.shouldStop) + + def test_startTest_py26(self): + self.make_26_result() + self.converter.startTest(self) + self.assertEqual([('startTest', self)], self.result._events) + + def test_startTest_py27(self): + self.make_27_result() + self.converter.startTest(self) + self.assertEqual([('startTest', self)], self.result._events) + + def test_startTest_pyextended(self): + self.make_extended_result() + self.converter.startTest(self) + self.assertEqual([('startTest', self)], self.result._events) + + def test_startTestRun_py26(self): + self.make_26_result() + self.converter.startTestRun() + self.assertEqual([], self.result._events) + + def test_startTestRun_py27(self): + self.make_27_result() + self.converter.startTestRun() + self.assertEqual([('startTestRun',)], self.result._events) + + def test_startTestRun_pyextended(self): + self.make_extended_result() + self.converter.startTestRun() + self.assertEqual([('startTestRun',)], self.result._events) + + def test_stopTest_py26(self): + self.make_26_result() + self.converter.stopTest(self) + self.assertEqual([('stopTest', self)], self.result._events) + + def test_stopTest_py27(self): + self.make_27_result() + self.converter.stopTest(self) + self.assertEqual([('stopTest', self)], self.result._events) + + def test_stopTest_pyextended(self): + self.make_extended_result() + self.converter.stopTest(self) + self.assertEqual([('stopTest', self)], self.result._events) + + def test_stopTestRun_py26(self): + self.make_26_result() + self.converter.stopTestRun() + self.assertEqual([], self.result._events) + + def test_stopTestRun_py27(self): + self.make_27_result() + self.converter.stopTestRun() + self.assertEqual([('stopTestRun',)], self.result._events) + + def test_stopTestRun_pyextended(self): + self.make_extended_result() + self.converter.stopTestRun() + self.assertEqual([('stopTestRun',)], self.result._events) + + def test_tags_py26(self): + self.make_26_result() + self.converter.tags(1, 2) + + def test_tags_py27(self): + self.make_27_result() + self.converter.tags(1, 2) + + def test_tags_pyextended(self): + self.make_extended_result() + self.converter.tags(1, 2) + self.assertEqual([('tags', 1, 2)], self.result._events) + + def test_time_py26(self): + self.make_26_result() + self.converter.time(1) + + def test_time_py27(self): + self.make_27_result() + self.converter.time(1) + + def test_time_pyextended(self): + self.make_extended_result() + self.converter.time(1) + self.assertEqual([('time', 1)], self.result._events) + + +class TestExtendedToOriginalAddError(TestExtendedToOriginalResultDecoratorBase): + + outcome = 'addError' + + def test_outcome_Original_py26(self): + self.make_26_result() + self.check_outcome_exc_info(self.outcome) + + def test_outcome_Original_py27(self): + self.make_27_result() + self.check_outcome_exc_info(self.outcome) + + def test_outcome_Original_pyextended(self): + self.make_extended_result() + self.check_outcome_exc_info(self.outcome) + + def test_outcome_Extended_py26(self): + self.make_26_result() + self.check_outcome_details_to_exec_info(self.outcome) + + def test_outcome_Extended_py27(self): + self.make_27_result() + self.check_outcome_details_to_exec_info(self.outcome) + + def test_outcome_Extended_pyextended(self): + self.make_extended_result() + self.check_outcome_details(self.outcome) + + def test_outcome__no_details(self): + self.make_extended_result() + self.assertThat( + lambda: getattr(self.converter, self.outcome)(self), + Raises(MatchesException(ValueError))) + + +class TestExtendedToOriginalAddFailure( + TestExtendedToOriginalAddError): + + outcome = 'addFailure' + + +class TestExtendedToOriginalAddExpectedFailure( + TestExtendedToOriginalAddError): + + outcome = 'addExpectedFailure' + + def test_outcome_Original_py26(self): + self.make_26_result() + self.check_outcome_exc_info_to_nothing(self.outcome, 'addSuccess') + + def test_outcome_Extended_py26(self): + self.make_26_result() + self.check_outcome_details_to_nothing(self.outcome, 'addSuccess') + + + +class TestExtendedToOriginalAddSkip( + TestExtendedToOriginalResultDecoratorBase): + + outcome = 'addSkip' + + def test_outcome_Original_py26(self): + self.make_26_result() + self.check_outcome_string_nothing(self.outcome, 'addSuccess') + + def test_outcome_Original_py27(self): + self.make_27_result() + self.check_outcome_string(self.outcome) + + def test_outcome_Original_pyextended(self): + self.make_extended_result() + self.check_outcome_string(self.outcome) + + def test_outcome_Extended_py26(self): + self.make_26_result() + self.check_outcome_string_nothing(self.outcome, 'addSuccess') + + def test_outcome_Extended_py27_no_reason(self): + self.make_27_result() + self.check_outcome_details_to_string(self.outcome) + + def test_outcome_Extended_py27_reason(self): + self.make_27_result() + self.check_outcome_details_to_arg(self.outcome, 'foo', + {'reason': Content(UTF8_TEXT, lambda:[_b('foo')])}) + + def test_outcome_Extended_pyextended(self): + self.make_extended_result() + self.check_outcome_details(self.outcome) + + def test_outcome__no_details(self): + self.make_extended_result() + self.assertThat( + lambda: getattr(self.converter, self.outcome)(self), + Raises(MatchesException(ValueError))) + + +class TestExtendedToOriginalAddSuccess( + TestExtendedToOriginalResultDecoratorBase): + + outcome = 'addSuccess' + expected = 'addSuccess' + + def test_outcome_Original_py26(self): + self.make_26_result() + self.check_outcome_nothing(self.outcome, self.expected) + + def test_outcome_Original_py27(self): + self.make_27_result() + self.check_outcome_nothing(self.outcome) + + def test_outcome_Original_pyextended(self): + self.make_extended_result() + self.check_outcome_nothing(self.outcome) + + def test_outcome_Extended_py26(self): + self.make_26_result() + self.check_outcome_details_to_nothing(self.outcome, self.expected) + + def test_outcome_Extended_py27(self): + self.make_27_result() + self.check_outcome_details_to_nothing(self.outcome) + + def test_outcome_Extended_pyextended(self): + self.make_extended_result() + self.check_outcome_details(self.outcome) + + +class TestExtendedToOriginalAddUnexpectedSuccess( + TestExtendedToOriginalResultDecoratorBase): + + outcome = 'addUnexpectedSuccess' + expected = 'addFailure' + + def test_outcome_Original_py26(self): + self.make_26_result() + getattr(self.converter, self.outcome)(self) + [event] = self.result._events + self.assertEqual((self.expected, self), event[:2]) + + def test_outcome_Original_py27(self): + self.make_27_result() + self.check_outcome_nothing(self.outcome) + + def test_outcome_Original_pyextended(self): + self.make_extended_result() + self.check_outcome_nothing(self.outcome) + + def test_outcome_Extended_py26(self): + self.make_26_result() + getattr(self.converter, self.outcome)(self) + [event] = self.result._events + self.assertEqual((self.expected, self), event[:2]) + + def test_outcome_Extended_py27(self): + self.make_27_result() + self.check_outcome_details_to_nothing(self.outcome) + + def test_outcome_Extended_pyextended(self): + self.make_extended_result() + self.check_outcome_details(self.outcome) + + +class TestExtendedToOriginalResultOtherAttributes( + TestExtendedToOriginalResultDecoratorBase): + + def test_other_attribute(self): + class OtherExtendedResult: + def foo(self): + return 2 + bar = 1 + self.result = OtherExtendedResult() + self.make_converter() + self.assertEqual(1, self.converter.bar) + self.assertEqual(2, self.converter.foo()) + + +class TestNonAsciiResults(TestCase): + """Test all kinds of tracebacks are cleanly interpreted as unicode + + Currently only uses weak "contains" assertions, would be good to be much + stricter about the expected output. This would add a few failures for the + current release of IronPython for instance, which gets some traceback + lines muddled. + """ + + _sample_texts = ( + _u("pa\u026a\u03b8\u0259n"), # Unicode encodings only + _u("\u5357\u7121"), # In ISO 2022 encodings + _u("\xa7\xa7\xa7"), # In ISO 8859 encodings + ) + # Everything but Jython shows syntax errors on the current character + _error_on_character = os.name != "java" + + def _run(self, stream, test): + """Run the test, the same as in testtools.run but not to stdout""" + result = TextTestResult(stream) + result.startTestRun() + try: + return test.run(result) + finally: + result.stopTestRun() + + def _write_module(self, name, encoding, contents): + """Create Python module on disk with contents in given encoding""" + try: + # Need to pre-check that the coding is valid or codecs.open drops + # the file without closing it which breaks non-refcounted pythons + codecs.lookup(encoding) + except LookupError: + self.skip("Encoding unsupported by implementation: %r" % encoding) + f = codecs.open(os.path.join(self.dir, name + ".py"), "w", encoding) + try: + f.write(contents) + finally: + f.close() + + def _test_external_case(self, testline, coding="ascii", modulelevel="", + suffix=""): + """Create and run a test case in a seperate module""" + self._setup_external_case(testline, coding, modulelevel, suffix) + return self._run_external_case() + + def _setup_external_case(self, testline, coding="ascii", modulelevel="", + suffix=""): + """Create a test case in a seperate module""" + _, prefix, self.modname = self.id().rsplit(".", 2) + self.dir = tempfile.mkdtemp(prefix=prefix, suffix=suffix) + self.addCleanup(shutil.rmtree, self.dir) + self._write_module(self.modname, coding, + # Older Python 2 versions don't see a coding declaration in a + # docstring so it has to be in a comment, but then we can't + # workaround bug: <http://ironpython.codeplex.com/workitem/26940> + "# coding: %s\n" + "import testtools\n" + "%s\n" + "class Test(testtools.TestCase):\n" + " def runTest(self):\n" + " %s\n" % (coding, modulelevel, testline)) + + def _run_external_case(self): + """Run the prepared test case in a seperate module""" + sys.path.insert(0, self.dir) + self.addCleanup(sys.path.remove, self.dir) + module = __import__(self.modname) + self.addCleanup(sys.modules.pop, self.modname) + stream = StringIO() + self._run(stream, module.Test()) + return stream.getvalue() + + def _silence_deprecation_warnings(self): + """Shut up DeprecationWarning for this test only""" + warnings.simplefilter("ignore", DeprecationWarning) + self.addCleanup(warnings.filters.remove, warnings.filters[0]) + + def _get_sample_text(self, encoding="unicode_internal"): + if encoding is None and str_is_unicode: + encoding = "unicode_internal" + for u in self._sample_texts: + try: + b = u.encode(encoding) + if u == b.decode(encoding): + if str_is_unicode: + return u, u + return u, b + except (LookupError, UnicodeError): + pass + self.skip("Could not find a sample text for encoding: %r" % encoding) + + def _as_output(self, text): + return text + + def test_non_ascii_failure_string(self): + """Assertion contents can be non-ascii and should get decoded""" + text, raw = self._get_sample_text(_get_exception_encoding()) + textoutput = self._test_external_case("self.fail(%s)" % _r(raw)) + self.assertIn(self._as_output(text), textoutput) + + def test_non_ascii_failure_string_via_exec(self): + """Assertion via exec can be non-ascii and still gets decoded""" + text, raw = self._get_sample_text(_get_exception_encoding()) + textoutput = self._test_external_case( + testline='exec ("self.fail(%s)")' % _r(raw)) + self.assertIn(self._as_output(text), textoutput) + + def test_control_characters_in_failure_string(self): + """Control characters in assertions should be escaped""" + textoutput = self._test_external_case("self.fail('\\a\\a\\a')") + self.expectFailure("Defense against the beeping horror unimplemented", + self.assertNotIn, self._as_output("\a\a\a"), textoutput) + self.assertIn(self._as_output(_u("\uFFFD\uFFFD\uFFFD")), textoutput) + + def test_os_error(self): + """Locale error messages from the OS shouldn't break anything""" + textoutput = self._test_external_case( + modulelevel="import os", + testline="os.mkdir('/')") + if os.name != "nt" or sys.version_info < (2, 5): + self.assertIn(self._as_output("OSError: "), textoutput) + else: + self.assertIn(self._as_output("WindowsError: "), textoutput) + + def test_assertion_text_shift_jis(self): + """A terminal raw backslash in an encoded string is weird but fine""" + example_text = _u("\u5341") + textoutput = self._test_external_case( + coding="shift_jis", + testline="self.fail('%s')" % example_text) + if str_is_unicode: + output_text = example_text + else: + output_text = example_text.encode("shift_jis").decode( + _get_exception_encoding(), "replace") + self.assertIn(self._as_output("AssertionError: %s" % output_text), + textoutput) + + def test_file_comment_iso2022_jp(self): + """Control character escapes must be preserved if valid encoding""" + example_text, _ = self._get_sample_text("iso2022_jp") + textoutput = self._test_external_case( + coding="iso2022_jp", + testline="self.fail('Simple') # %s" % example_text) + self.assertIn(self._as_output(example_text), textoutput) + + def test_unicode_exception(self): + """Exceptions that can be formated losslessly as unicode should be""" + example_text, _ = self._get_sample_text() + exception_class = ( + "class FancyError(Exception):\n" + # A __unicode__ method does nothing on py3k but the default works + " def __unicode__(self):\n" + " return self.args[0]\n") + textoutput = self._test_external_case( + modulelevel=exception_class, + testline="raise FancyError(%s)" % _r(example_text)) + self.assertIn(self._as_output(example_text), textoutput) + + def test_unprintable_exception(self): + """A totally useless exception instance still prints something""" + exception_class = ( + "class UnprintableError(Exception):\n" + " def __str__(self):\n" + " raise RuntimeError\n" + " def __unicode__(self):\n" + " raise RuntimeError\n" + " def __repr__(self):\n" + " raise RuntimeError\n") + textoutput = self._test_external_case( + modulelevel=exception_class, + testline="raise UnprintableError") + self.assertIn(self._as_output( + "UnprintableError: <unprintable UnprintableError object>\n"), + textoutput) + + def test_string_exception(self): + """Raise a string rather than an exception instance if supported""" + if sys.version_info > (2, 6): + self.skip("No string exceptions in Python 2.6 or later") + elif sys.version_info > (2, 5): + self._silence_deprecation_warnings() + textoutput = self._test_external_case(testline="raise 'plain str'") + self.assertIn(self._as_output("\nplain str\n"), textoutput) + + def test_non_ascii_dirname(self): + """Script paths in the traceback can be non-ascii""" + text, raw = self._get_sample_text(sys.getfilesystemencoding()) + textoutput = self._test_external_case( + # Avoid bug in Python 3 by giving a unicode source encoding rather + # than just ascii which raises a SyntaxError with no other details + coding="utf-8", + testline="self.fail('Simple')", + suffix=raw) + self.assertIn(self._as_output(text), textoutput) + + def test_syntax_error(self): + """Syntax errors should still have fancy special-case formatting""" + textoutput = self._test_external_case("exec ('f(a, b c)')") + self.assertIn(self._as_output( + ' File "<string>", line 1\n' + ' f(a, b c)\n' + + ' ' * self._error_on_character + + ' ^\n' + 'SyntaxError: ' + ), textoutput) + + def test_syntax_error_malformed(self): + """Syntax errors with bogus parameters should break anything""" + textoutput = self._test_external_case("raise SyntaxError(3, 2, 1)") + self.assertIn(self._as_output("\nSyntaxError: "), textoutput) + + def test_syntax_error_import_binary(self): + """Importing a binary file shouldn't break SyntaxError formatting""" + if sys.version_info < (2, 5): + # Python 2.4 assumes the file is latin-1 and tells you off + self._silence_deprecation_warnings() + self._setup_external_case("import bad") + f = open(os.path.join(self.dir, "bad.py"), "wb") + try: + f.write(_b("x\x9c\xcb*\xcd\xcb\x06\x00\x04R\x01\xb9")) + finally: + f.close() + textoutput = self._run_external_case() + self.assertIn(self._as_output("\nSyntaxError: "), textoutput) + + def test_syntax_error_line_iso_8859_1(self): + """Syntax error on a latin-1 line shows the line decoded""" + text, raw = self._get_sample_text("iso-8859-1") + textoutput = self._setup_external_case("import bad") + self._write_module("bad", "iso-8859-1", + "# coding: iso-8859-1\n! = 0 # %s\n" % text) + textoutput = self._run_external_case() + self.assertIn(self._as_output(_u( + #'bad.py", line 2\n' + ' ! = 0 # %s\n' + ' ^\n' + 'SyntaxError: ') % + (text,)), textoutput) + + def test_syntax_error_line_iso_8859_5(self): + """Syntax error on a iso-8859-5 line shows the line decoded""" + text, raw = self._get_sample_text("iso-8859-5") + textoutput = self._setup_external_case("import bad") + self._write_module("bad", "iso-8859-5", + "# coding: iso-8859-5\n%% = 0 # %s\n" % text) + textoutput = self._run_external_case() + self.assertIn(self._as_output(_u( + #'bad.py", line 2\n' + ' %% = 0 # %s\n' + + ' ' * self._error_on_character + + ' ^\n' + 'SyntaxError: ') % + (text,)), textoutput) + + def test_syntax_error_line_euc_jp(self): + """Syntax error on a euc_jp line shows the line decoded""" + text, raw = self._get_sample_text("euc_jp") + textoutput = self._setup_external_case("import bad") + self._write_module("bad", "euc_jp", + "# coding: euc_jp\n$ = 0 # %s\n" % text) + textoutput = self._run_external_case() + self.assertIn(self._as_output(_u( + #'bad.py", line 2\n' + ' $ = 0 # %s\n' + + ' ' * self._error_on_character + + ' ^\n' + 'SyntaxError: ') % + (text,)), textoutput) + + def test_syntax_error_line_utf_8(self): + """Syntax error on a utf-8 line shows the line decoded""" + text, raw = self._get_sample_text("utf-8") + textoutput = self._setup_external_case("import bad") + self._write_module("bad", "utf-8", _u("\ufeff^ = 0 # %s\n") % text) + textoutput = self._run_external_case() + self.assertIn(self._as_output(_u( + 'bad.py", line 1\n' + ' ^ = 0 # %s\n' + + ' ' * self._error_on_character + + ' ^\n' + 'SyntaxError: ') % + text), textoutput) + + +class TestNonAsciiResultsWithUnittest(TestNonAsciiResults): + """Test that running under unittest produces clean ascii strings""" + + def _run(self, stream, test): + from unittest import TextTestRunner as _Runner + return _Runner(stream).run(test) + + def _as_output(self, text): + if str_is_unicode: + return text + return text.encode("utf-8") + + +class TestDetailsToStr(TestCase): + + def test_no_details(self): + string = _details_to_str({}) + self.assertThat(string, Equals('')) + + def test_binary_content(self): + content = content_from_stream( + StringIO('foo'), content_type=ContentType('image', 'jpeg')) + string = _details_to_str({'attachment': content}) + self.assertThat( + string, Equals("""\ +Binary content: + attachment (image/jpeg) +""")) + + def test_single_line_content(self): + content = text_content('foo') + string = _details_to_str({'attachment': content}) + self.assertThat(string, Equals('attachment: {{{foo}}}\n')) + + def test_multi_line_text_content(self): + content = text_content('foo\nbar\nbaz') + string = _details_to_str({'attachment': content}) + self.assertThat(string, Equals('attachment: {{{\nfoo\nbar\nbaz\n}}}\n')) + + def test_special_text_content(self): + content = text_content('foo') + string = _details_to_str({'attachment': content}, special='attachment') + self.assertThat(string, Equals('foo\n')) + + def test_multiple_text_content(self): + string = _details_to_str( + {'attachment': text_content('foo\nfoo'), + 'attachment-1': text_content('bar\nbar')}) + self.assertThat( + string, Equals('attachment: {{{\n' + 'foo\n' + 'foo\n' + '}}}\n' + '\n' + 'attachment-1: {{{\n' + 'bar\n' + 'bar\n' + '}}}\n')) + + def test_empty_attachment(self): + string = _details_to_str({'attachment': text_content('')}) + self.assertThat( + string, Equals("""\ +Empty attachments: + attachment +""")) + + def test_lots_of_different_attachments(self): + jpg = lambda x: content_from_stream( + StringIO(x), ContentType('image', 'jpeg')) + attachments = { + 'attachment': text_content('foo'), + 'attachment-1': text_content('traceback'), + 'attachment-2': jpg('pic1'), + 'attachment-3': text_content('bar'), + 'attachment-4': text_content(''), + 'attachment-5': jpg('pic2'), + } + string = _details_to_str(attachments, special='attachment-1') + self.assertThat( + string, Equals("""\ +Binary content: + attachment-2 (image/jpeg) + attachment-5 (image/jpeg) +Empty attachments: + attachment-4 + +attachment: {{{foo}}} +attachment-3: {{{bar}}} + +traceback +""")) + + +def test_suite(): + from unittest import TestLoader + return TestLoader().loadTestsFromName(__name__) diff --git a/test/3rdparty/testtools-0.9.12/testtools/tests/test_testsuite.py b/test/3rdparty/testtools-0.9.12/testtools/tests/test_testsuite.py new file mode 100644 index 00000000000..05647577cdb --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/tests/test_testsuite.py @@ -0,0 +1,75 @@ +# Copyright (c) 2009-2011 testtools developers. See LICENSE for details. + +"""Test ConcurrentTestSuite and related things.""" + +__metaclass__ = type + +import unittest + +from testtools import ( + ConcurrentTestSuite, + iterate_tests, + TestCase, + ) +from testtools.helpers import try_import +from testtools.testsuite import FixtureSuite +from testtools.tests.helpers import LoggingResult + +FunctionFixture = try_import('fixtures.FunctionFixture') + + +class TestConcurrentTestSuiteRun(TestCase): + + def test_trivial(self): + log = [] + result = LoggingResult(log) + class Sample(TestCase): + def __hash__(self): + return id(self) + + def test_method1(self): + pass + def test_method2(self): + pass + test1 = Sample('test_method1') + test2 = Sample('test_method2') + original_suite = unittest.TestSuite([test1, test2]) + suite = ConcurrentTestSuite(original_suite, self.split_suite) + suite.run(result) + # 0 is the timestamp for the first test starting. + test1 = log[1][1] + test2 = log[-1][1] + self.assertIsInstance(test1, Sample) + self.assertIsInstance(test2, Sample) + self.assertNotEqual(test1.id(), test2.id()) + + def split_suite(self, suite): + tests = list(iterate_tests(suite)) + return tests[0], tests[1] + + +class TestFixtureSuite(TestCase): + + def setUp(self): + super(TestFixtureSuite, self).setUp() + if FunctionFixture is None: + self.skip("Need fixtures") + + def test_fixture_suite(self): + log = [] + class Sample(TestCase): + def test_one(self): + log.append(1) + def test_two(self): + log.append(2) + fixture = FunctionFixture( + lambda: log.append('setUp'), + lambda fixture: log.append('tearDown')) + suite = FixtureSuite(fixture, [Sample('test_one'), Sample('test_two')]) + suite.run(LoggingResult([])) + self.assertEqual(['setUp', 1, 2, 'tearDown'], log) + + +def test_suite(): + from unittest import TestLoader + return TestLoader().loadTestsFromName(__name__) diff --git a/test/3rdparty/testtools-0.9.12/testtools/tests/test_with_with.py b/test/3rdparty/testtools-0.9.12/testtools/tests/test_with_with.py new file mode 100644 index 00000000000..e06adeb1816 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/tests/test_with_with.py @@ -0,0 +1,73 @@ +# Copyright (c) 2011 testtools developers. See LICENSE for details. + +from __future__ import with_statement + +import sys + +from testtools import ( + ExpectedException, + TestCase, + ) +from testtools.matchers import ( + AfterPreprocessing, + Equals, + ) + + +class TestExpectedException(TestCase): + """Test the ExpectedException context manager.""" + + def test_pass_on_raise(self): + with ExpectedException(ValueError, 'tes.'): + raise ValueError('test') + + def test_pass_on_raise_matcher(self): + with ExpectedException( + ValueError, AfterPreprocessing(str, Equals('test'))): + raise ValueError('test') + + def test_raise_on_text_mismatch(self): + try: + with ExpectedException(ValueError, 'tes.'): + raise ValueError('mismatch') + except AssertionError: + e = sys.exc_info()[1] + self.assertEqual("'mismatch' does not match /tes./", str(e)) + else: + self.fail('AssertionError not raised.') + + def test_raise_on_general_mismatch(self): + matcher = AfterPreprocessing(str, Equals('test')) + value_error = ValueError('mismatch') + try: + with ExpectedException(ValueError, matcher): + raise value_error + except AssertionError: + e = sys.exc_info()[1] + self.assertEqual(matcher.match(value_error).describe(), str(e)) + else: + self.fail('AssertionError not raised.') + + def test_raise_on_error_mismatch(self): + try: + with ExpectedException(TypeError, 'tes.'): + raise ValueError('mismatch') + except ValueError: + e = sys.exc_info()[1] + self.assertEqual('mismatch', str(e)) + else: + self.fail('ValueError not raised.') + + def test_raise_if_no_exception(self): + try: + with ExpectedException(TypeError, 'tes.'): + pass + except AssertionError: + e = sys.exc_info()[1] + self.assertEqual('TypeError not raised.', str(e)) + else: + self.fail('AssertionError not raised.') + + def test_pass_on_raise_any_message(self): + with ExpectedException(ValueError): + raise ValueError('whatever') diff --git a/test/3rdparty/testtools-0.9.12/testtools/testsuite.py b/test/3rdparty/testtools-0.9.12/testtools/testsuite.py new file mode 100644 index 00000000000..18de8b89e12 --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/testsuite.py @@ -0,0 +1,101 @@ +# Copyright (c) 2009-2011 testtools developers. See LICENSE for details. + +"""Test suites and related things.""" + +__metaclass__ = type +__all__ = [ + 'ConcurrentTestSuite', + 'iterate_tests', + ] + +from testtools.helpers import try_imports + +Queue = try_imports(['Queue.Queue', 'queue.Queue']) + +import threading +import unittest + +import testtools + + +def iterate_tests(test_suite_or_case): + """Iterate through all of the test cases in 'test_suite_or_case'.""" + try: + suite = iter(test_suite_or_case) + except TypeError: + yield test_suite_or_case + else: + for test in suite: + for subtest in iterate_tests(test): + yield subtest + + +class ConcurrentTestSuite(unittest.TestSuite): + """A TestSuite whose run() calls out to a concurrency strategy.""" + + def __init__(self, suite, make_tests): + """Create a ConcurrentTestSuite to execute suite. + + :param suite: A suite to run concurrently. + :param make_tests: A helper function to split the tests in the + ConcurrentTestSuite into some number of concurrently executing + sub-suites. make_tests must take a suite, and return an iterable + of TestCase-like object, each of which must have a run(result) + method. + """ + super(ConcurrentTestSuite, self).__init__([suite]) + self.make_tests = make_tests + + def run(self, result): + """Run the tests concurrently. + + This calls out to the provided make_tests helper, and then serialises + the results so that result only sees activity from one TestCase at + a time. + + ConcurrentTestSuite provides no special mechanism to stop the tests + returned by make_tests, it is up to the make_tests to honour the + shouldStop attribute on the result object they are run with, which will + be set if an exception is raised in the thread which + ConcurrentTestSuite.run is called in. + """ + tests = self.make_tests(self) + try: + threads = {} + queue = Queue() + result_semaphore = threading.Semaphore(1) + for test in tests: + process_result = testtools.ThreadsafeForwardingResult(result, + result_semaphore) + reader_thread = threading.Thread( + target=self._run_test, args=(test, process_result, queue)) + threads[test] = reader_thread, process_result + reader_thread.start() + while threads: + finished_test = queue.get() + threads[finished_test][0].join() + del threads[finished_test] + except: + for thread, process_result in threads.values(): + process_result.stop() + raise + + def _run_test(self, test, process_result, queue): + try: + test.run(process_result) + finally: + queue.put(test) + + +class FixtureSuite(unittest.TestSuite): + + def __init__(self, fixture, tests): + super(FixtureSuite, self).__init__(tests) + self._fixture = fixture + + def run(self, result): + self._fixture.setUp() + try: + super(FixtureSuite, self).run(result) + finally: + self._fixture.cleanUp() diff --git a/test/3rdparty/testtools-0.9.12/testtools/utils.py b/test/3rdparty/testtools-0.9.12/testtools/utils.py new file mode 100644 index 00000000000..0f39d8f5b6e --- /dev/null +++ b/test/3rdparty/testtools-0.9.12/testtools/utils.py @@ -0,0 +1,13 @@ +# Copyright (c) 2008-2010 testtools developers. See LICENSE for details. + +"""Utilities for dealing with stuff in unittest. + +Legacy - deprecated - use testtools.testsuite.iterate_tests +""" + +import warnings +warnings.warn("Please import iterate_tests from testtools.testsuite - " + "testtools.utils is deprecated.", DeprecationWarning, stacklevel=2) + +from testtools.testsuite import iterate_tests + diff --git a/test/config.i b/test/config.i new file mode 100644 index 00000000000..f7dfd6234b5 --- /dev/null +++ b/test/config.i @@ -0,0 +1,37 @@ +%module config + +%{ +#include <wt_internal.h> +%} + +typedef struct WT_CONFIG WT_CONFIG; +typedef struct WT_CONFIG_ITEM WT_CONFIG_ITEM; + +struct WT_CONFIG +{ + char *orig; + char *end; + char *cur; + + int depth, top; + void **go; +}; + +struct WT_CONFIG_ITEM +{ + char *str; + size_t len; + uint64_t val; + enum { ITEM_STRING, ITEM_ID, ITEM_NUM, ITEM_STRUCT } type; +}; + +/* + * XXX: leak, but the string needs to last beyond the call to config_init. + * It should be freed when the WT_CONFIG object is deleted. + */ +%typemap(in) char *confstr { + $1 = strdup(PyString_AsString($input)); +} + +int config_init(WT_CONFIG *conf, char *confstr, int len); +int config_next(WT_CONFIG *conf, WT_CONFIG_ITEM *key, WT_CONFIG_ITEM *value); diff --git a/test/config_test.c b/test/config_test.c new file mode 100644 index 00000000000..13702342153 --- /dev/null +++ b/test/config_test.c @@ -0,0 +1,21 @@ +#include <wt_int.h> + +#include <string.h> + +int main(int argc, char *argv[]) +{ + int ret; + WT_CONFIG c; + WT_CONFIG_ITEM k, v; + const char *cstr = "create,cachesize=10MB"; + + config_init(&c, cstr, strlen(cstr)); + while ((ret = config_next(&c, &k, &v)) == 0) { + printf("Got key '%.*s', value '%.*s'\n", + (int)k.len, k.str, (int)v.len, v.str); + } + + printf("Last call to config_next failed with %d\n", ret); + + return (0); +} diff --git a/test/config_test.py b/test/config_test.py new file mode 100644 index 00000000000..b5199452f7c --- /dev/null +++ b/test/config_test.py @@ -0,0 +1,39 @@ +import config + +def parse(s): + print "Parsing '%s':" % s + + c = config.WT_CONFIG() + k = config.WT_CONFIG_ITEM() + v = config.WT_CONFIG_ITEM() + + config.config_init(c, s, len(s)) + ret = config.config_next(c, k, v) + while ret == 0: + print " => '%s' = '%s'" % (k.str[:k.len], v.str[:v.len]) + ret = config.config_next(c, k, v) + + # XXX hard-coding WT_NOTFOUND until we fix this + if ret != -31801: + print "Last call to config_next failed with %d" % ret + +if __name__ == '__main__': + parse("create") + parse("create,cachesize=10MB") + parse('create,cachesize=10MB,path="/foo/bar"') + parse('columns=(first,second, third)') + parse('key_format="S", value_format="5sq", columns=(first,second, third)') + parse('key_columns=(first=S),value_columns=(second="5s", third=q)') + parse(',,columns=(first=S,second="5s", third=q),,') + parse('index.country_year=(country,year),key_format=r,colgroup.population=(population),columns=(id,country,year,population),value_format=5sHQ') + + import json + parse(json.dumps({'hello' : 'world', 'columns' : ('one', 'two', 'three')})) + + parse(json.dumps({ + "key_format" : "r", + "value_format" : "5sHQ", + "columns" : ("id", "country", "year", "population"), + "colgroup.population" : ("population",), + "index.country_year" : ("country","year") + })) diff --git a/test/format/CONFIG.example b/test/format/CONFIG.example new file mode 100644 index 00000000000..fa759cd9a7f --- /dev/null +++ b/test/format/CONFIG.example @@ -0,0 +1,60 @@ +#bitcnt +# number of bits for fixed-length column-store files + +#bzip +# if blocks are BZIP2 encoded + +#cache +# size of the cache in MB + +#delete_pct +# percent operations that are deletes + +#file_type +# type of file to create (fix | var | row) + +#huffman_key +# if keys are huffman encoded + +#huffman_value +# if values are huffman encoded + +#insert_pct +# percent operations that are inserts + +#internal_page_max +# maximum size of Btree internal nodes + +#key_max +# maximum size of keys + +#key_min +# minimum size of keys + +#leaf_page_max +# maximum size of Btree leaf nodes + +#ops +# the number of modification operations done per run + +#repeat_data_pct +# percent duplicate values in variable-len column-store files + +#reverse +# collate in reverse order + +#rows +# the number of rows to create + +#runs +# the number of runs + +#value_max +# maximum size of values + +#value_min +# minimum size of values + +#write_pct +# percent operations that are writes + diff --git a/test/format/Makefile.am b/test/format/Makefile.am new file mode 100644 index 00000000000..eece395a5f4 --- /dev/null +++ b/test/format/Makefile.am @@ -0,0 +1,17 @@ +BDB = $(top_builddir)/db +INCLUDES = -I$(top_builddir) -I$(BDB) + +noinst_PROGRAMS = t +noinst_DATA = s_dumpcmp +t_SOURCES = config.h format.h bdb.c config.c t.c util.c wts.c +t_LDADD = $(top_builddir)/libwiredtiger.la -L$(BDB)/build_unix -ldb +t_LDFLAGS = -static + +s_dumpcmp: $(srcdir)/s_dumpcmp.in + cp $(srcdir)/s_dumpcmp.in $@ + +backup: + rm -rf BACKUP && mkdir BACKUP && cp -p -r WiredTiger* __* BACKUP/ + +clean-local: + rm -rf WiredTiger WiredTiger.* __* diff --git a/test/format/README b/test/format/README new file mode 100644 index 00000000000..5f276ad48ac --- /dev/null +++ b/test/format/README @@ -0,0 +1,9 @@ +The test program format randomly generates WiredTiger files with +different size objects and then does single-threaded operations +on those files. The goal is to test the WiredTiger file formats. + +format should be compiled with a version of Berkeley DB (which it +uses to verify format's results). Create a link "db" in the +build_posix directory that links to the top-level of a Berkeley DB +distribution directory which contains a configured and compiled +build_unix subdirectory. diff --git a/test/format/bdb.c b/test/format/bdb.c new file mode 100644 index 00000000000..70633817661 --- /dev/null +++ b/test/format/bdb.c @@ -0,0 +1,200 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#define BDB 1 /* Berkeley DB header files */ +#include "format.h" + +static int +bdb_compare_reverse(DB *dbp, const DBT *k1, const DBT *k2) +{ + int cmp; + size_t len; + + UNUSED(dbp); + + len = (k1->size < k2->size) ? k1->size : k2->size; + if ((cmp = memcmp(k2->data, k1->data, len)) == 0) + cmp = ((int)k1->size - (int)k2->size); + return (cmp); +} + +void +bdb_startup(void) +{ + DB *db; + DBC *dbc; + DB_ENV *dbenv; + + assert(db_env_create(&dbenv, 0) == 0); + dbenv->set_errpfx(dbenv, "bdb"); + dbenv->set_errfile(dbenv, stderr); + assert(dbenv->mutex_set_max(dbenv, 10000) == 0); + assert(dbenv->set_cachesize(dbenv, 0, 50 * 1024 * 1024, 1) == 0); + assert(dbenv->open(dbenv, NULL, + DB_CREATE | + (g.c_delete_pct == 0 && g.c_insert_pct == 0 && g.c_write_pct == 0 ? + 0 : DB_INIT_LOCK) | + DB_INIT_MPOOL | DB_PRIVATE, 0) == 0); + assert(db_create(&db, dbenv, 0) == 0); + + if (g.c_file_type == ROW && g.c_reverse) + assert(db->set_bt_compare(db, bdb_compare_reverse) == 0); + + assert(db->open(db, NULL, "__bdb", NULL, DB_BTREE, DB_CREATE, 0) == 0); + g.bdb = db; + assert(db->cursor(db, NULL, &dbc, 0) == 0); + g.dbc = dbc; +} + +void +bdb_teardown(void) +{ + DB *db; + DBC *dbc; + DB_ENV *dbenv; + + dbc = g.dbc; + db = g.bdb; + dbenv = db->dbenv; + assert(dbc->close(dbc) == 0); + assert(db->close(db, 0) == 0); + assert(dbenv->close(dbenv, 0) == 0); +} + +void +bdb_insert( + const void *key_data, uint32_t key_size, + const void *value_data, uint32_t value_size) +{ + static DBT key, value; + DBC *dbc; + + key.data = (void *)key_data; + key.size = key_size; + value.data = (void *)value_data; + value.size = value_size; + + dbc = g.dbc; + + assert(dbc->put(dbc, &key, &value, DB_KEYFIRST) == 0); +} + +int +bdb_np(int next, + void *keyp, uint32_t *keysizep, + void *valuep, uint32_t *valuesizep, int *notfoundp) +{ + static DBT key, value; + DB *db = g.bdb; + DBC *dbc = g.dbc; + int ret; + + *notfoundp = 0; + + if ((ret = + dbc->get(dbc, &key, &value, next ? DB_NEXT : DB_PREV)) != 0) { + if (ret == DB_NOTFOUND) { + *notfoundp = 1; + return (0); + } + db->err(db, ret, + "dbc->get: %s: {%.*s}", + next ? "DB_NEXT" : "DB_PREV", + (int)key.size, (char *)key.data); + return (1); + } + *(void **)keyp = key.data; + *keysizep = key.size; + *(void **)valuep = value.data; + *valuesizep = value.size; + return (0); +} + +int +bdb_read(uint64_t keyno, void *valuep, uint32_t *valuesizep, int *notfoundp) +{ + static DBT key, value; + DB *db = g.bdb; + DBC *dbc = g.dbc; + int ret; + + *notfoundp = 0; + + key_gen(&key.data, &key.size, keyno, 0); + + if ((ret = dbc->get(dbc, &key, &value, DB_SET)) != 0) { + if (ret == DB_NOTFOUND) { + *notfoundp = 1; + return (0); + } + db->err(db, ret, + "dbc->get: DB_SET: {%.*s}", + (int)key.size, (char *)key.data); + return (1); + } + *(void **)valuep = value.data; + *valuesizep = value.size; + return (0); +} + +int +bdb_put(const void *arg_key, uint32_t arg_key_size, + const void *arg_value, uint32_t arg_value_size, int *notfoundp) +{ + static DBT key, value; + DB *db = g.bdb; + DBC *dbc = g.dbc; + int ret; + + *notfoundp = 0; + + key.data = (void *)arg_key; + key.size = arg_key_size; + value.data = (void *)arg_value; + value.size = arg_value_size; + + if ((ret = dbc->put(dbc, &key, &value, DB_KEYFIRST)) != 0) { + if (ret == DB_NOTFOUND) { + *notfoundp = 1; + return (0); + } + db->err(db, ret, "dbc->put: DB_KEYFIRST: {%.*s}{%.*s}", + (int)key.size, (char *)key.data, + (int)value.size, (char *)value.data); + return (1); + } + return (0); +} + +int +bdb_del(uint64_t keyno, int *notfoundp) +{ + static DBT value; + static DBT key; + DB *db = g.bdb; + DBC *dbc = g.dbc; + int ret; + + *notfoundp = 0; + + key_gen(&key.data, &key.size, keyno, 0); + + if ((ret = bdb_read(keyno, &value.data, &value.size, notfoundp)) != 0) + return (1); + if (*notfoundp) + return (0); + if ((ret = dbc->del(dbc, 0)) != 0) { + if (ret == DB_NOTFOUND) { + *notfoundp = 1; + return (0); + } + db->err(db, ret, + "dbc->del: {%.*s}", (int)key.size, (char *)key.data); + return (1); + } + return (0); +} diff --git a/test/format/config.c b/test/format/config.c new file mode 100644 index 00000000000..1c9c2bc920e --- /dev/null +++ b/test/format/config.c @@ -0,0 +1,267 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "format.h" +#include "config.h" + +static void config_clear(void); +static CONFIG *config_find(const char *, size_t); +static uint32_t config_translate(const char *); + +/* + * config_setup -- + * Initialize configuration for a run. + */ +void +config_setup(void) +{ + CONFIG *cp; + + /* Clear any temporary values. */ + config_clear(); + + /* Pick a file type next, other items depend on it. */ + cp = config_find("file_type", strlen("file_type")); + if (!(cp->flags & C_PERM)) { + switch (MMRAND(0, 2)) { + case 0: + config_single("file_type=fix", 0); + break; + case 1: + config_single("file_type=var", 0); + break; + case 2: + config_single("file_type=row", 0); + break; + } + } + + /* Reset the key count. */ + g.key_cnt = 0; + + /* Fill in random values for the rest of the run. */ + for (cp = c; cp->name != NULL; ++cp) { + if (cp->flags & (C_IGNORE | C_PERM | C_TEMP)) + continue; + + /* + * Boolean flags are 0 or 1, but only set N in 100 where the + * variable's min value is N. Set the flag if we rolled >= + * the min, 0 otherwise. + */ + if (cp->flags & C_BOOL) + *cp->v = MMRAND(1, 100) <= cp->min ? 1 : 0; + else + *cp->v = CONF_RAND(cp); + } + + /* Periodically, set the delete percentage to 0 so salvage gets run. */ + if (!g.replay && g.run_cnt % 10 == 0) + g.c_delete_pct = 0; +} + +/* + * config_error -- + * Display configuration information on error. + */ +void +config_error(void) +{ + CONFIG *cp; + FILE *fp; + + /* Display configuration names. */ + fprintf(stderr, "Configuration names:\n"); + for (cp = c; cp->name != NULL; ++cp) + fprintf(stderr, "%16s : %s\n", cp->name, cp->desc); + + fprintf(stderr, "\n"); + if ((fp = fopen("CONFIG.example", "w")) != NULL) { + fprintf(stderr, "Re-creating CONFIG.example file... "); + for (cp = c; cp->name != NULL; ++cp) + fprintf(fp, "#%s\n#\t%s\n\n", cp->name, cp->desc); + (void)fclose(fp); + fprintf(stderr, "done\n"); + } +} + +/* + * config_print -- + * Print configuration information. + */ +void +config_print(int error_display) +{ + CONFIG *cp; + FILE *fp; + + if (error_display) + fp = stdout; + else + if ((fp = fopen("__run", "w")) == NULL) + die("__run", errno); + + fprintf(fp, "############################################\n"); + fprintf(fp, "# RUN PARAMETERS\n"); + fprintf(fp, "############################################\n"); + + /* Display configuration values. */ + for (cp = c; cp->name != NULL; ++cp) + if (cp->type_mask != 0 && + ((g.c_file_type == FIX && !(cp->type_mask & C_FIX)) || + (g.c_file_type == ROW && !(cp->type_mask & C_ROW)) || + (g.c_file_type == VAR && !(cp->type_mask & C_VAR)))) + fprintf(fp, + "# %s not applicable to this run\n", cp->name); + else if (strcmp(cp->name, "file_type")) + fprintf(fp, "%s=%" PRIu32 "\n", cp->name, *cp->v); + else + fprintf(fp, "%s=%s\n", cp->name, config_dtype()); + + fprintf(fp, "############################################\n"); + if (fp != stdout) + (void)fclose(fp); +} + +/* + * config_file -- + * Read configuration values from a file. + */ +void +config_file(const char *name) +{ + FILE *fp; + char *p, buf[256]; + + if ((fp = fopen(name, "r")) == NULL) + die(name, errno); + while (fgets(buf, sizeof(buf), fp) != NULL) { + for (p = buf; *p != '\0' && *p != '\n'; ++p) + ; + *p = '\0'; + if (buf[0] == '\0' || buf[0] == '#') + continue; + config_single(buf, 1); + } + (void)fclose(fp); +} + +/* + * config_clear -- + * Clear per-run values. + */ +static void +config_clear(void) +{ + CONFIG *cp; + + /* Display configuration names. */ + for (cp = c; cp->name != NULL; ++cp) + cp->flags &= ~(uint32_t)C_TEMP; +} + + +/* + * config_single -- + * Set a single configuration structure value. + */ +void +config_single(const char *s, int perm) +{ + CONFIG *cp; + const char *ep; + + if ((ep = strchr(s, '=')) == NULL) { + fprintf(stderr, + "%s: %s: illegal configuration value\n", g.progname, s); + exit(EXIT_FAILURE); + } + + cp = config_find(s, (size_t)(ep - s)); + cp->flags |= perm ? C_PERM : C_TEMP; + + *cp->v = config_translate(ep + 1); + if (cp->flags & C_BOOL) { + if (*cp->v != 0 && *cp->v != 1) { + fprintf(stderr, "%s: %s: value of boolean not 0 or 1\n", + g.progname, s); + exit(EXIT_FAILURE); + } + } else if (*cp->v < cp->min || *cp->v > cp->max) { + fprintf(stderr, "%s: %s: value of %" PRIu32 + " outside min/max values of %" PRIu32 "-%" PRIu32 "\n", + g.progname, s, *cp->v, cp->min, cp->max); + exit(EXIT_FAILURE); + } +} + +/* + * config_translate -- + * Return an integer value representing the argument. + */ +static uint32_t +config_translate(const char *s) +{ + /* If it's already a integer value, we're done. */ + if (isdigit(s[0])) + return (uint32_t)atoi(s); + + /* Currently, all we translate are the file type names. */ + if (strcmp(s, "fix") == 0 || + strcmp(s, "flcs") == 0 || /* Deprecated */ + strcmp(s, "fixed-length column-store") == 0) + return ((uint32_t)FIX); + if (strcmp(s, "var") == 0 || + strcmp(s, "vlcs") == 0 || /* Deprecated */ + strcmp(s, "variable-length column-store") == 0) + return ((uint32_t)VAR); + if (strcmp(s, "row") == 0 || + strcmp(s, "row-store") == 0) + return ((uint32_t)ROW); + + fprintf(stderr, "%s: %s: unknown configuration value\n", g.progname, s); + exit(EXIT_FAILURE); +} + +/* + * config_find + * Find a specific configuration entry. + */ +static CONFIG * +config_find(const char *s, size_t len) +{ + CONFIG *cp; + + for (cp = c; cp->name != NULL; ++cp) + if (strncmp(s, cp->name, len) == 0) + return (cp); + + fprintf(stderr, + "%s: %s: unknown configuration keyword\n", g.progname, s); + config_error(); + exit(EXIT_FAILURE); +} + +/* + * config_dtype -- + * Return the file type as a string. + */ +const char * +config_dtype(void) +{ + switch (g.c_file_type) { + case FIX: + return ("fixed-length column-store"); + case VAR: + return ("variable-length column-store"); + case ROW: + return ("row-store"); + default: + break; + } + return ("error: UNKNOWN FILE TYPE"); +} diff --git a/test/format/config.h b/test/format/config.h new file mode 100644 index 00000000000..90d641d00e7 --- /dev/null +++ b/test/format/config.h @@ -0,0 +1,127 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +/* + * Configuration for the wts program is an array of string-based paramters. + * This is the structure used to declare them. + */ +typedef struct { + const char *name; /* Configuration item */ + const char *desc; /* Configuration description */ + +#define C_FIX 0x01 /* File types */ +#define C_VAR 0x02 +#define C_ROW 0x04 + uint8_t type_mask; /* File type mask */ + + /* Value is a boolean, yes if roll of 1-to-100 is <= CONFIG->min. */ +#define C_BOOL 0x01 + + /* Not a simple randomization, handle outside the main loop. */ +#define C_IGNORE 0x02 + + /* Value was set from command-line or file, ignore for all runs. */ +#define C_PERM 0x04 + + /* Value isn't random for this run, ignore just for this run. */ +#define C_TEMP 0x08 + uint32_t flags; + + uint32_t min; /* Minimum value */ + uint32_t max; /* Maximum value */ + uint32_t *v; /* Value for this run */ +} CONFIG; + +/* + * Get a random value between a config min/max pair (inclusive for both min + * and max). + */ +#define CONF_RAND(cp) MMRAND((cp)->min, (cp)->max) + +static CONFIG c[] = { + { "bitcnt", + "number of bits for fixed-length column-store files", + C_FIX, 0, 1, 8, &g.c_bitcnt }, + + { "bzip", + "if blocks are BZIP2 encoded", /* 80% */ + 0, C_BOOL, 80, 0, &g.c_bzip }, + + { "cache", + "size of the cache in MB", + 0, 0, 1, 100, &g.c_cache }, + + { "delete_pct", + "percent operations that are deletes", + 0, 0, 0, 45, &g.c_delete_pct }, + + { "file_type", + "type of file to create (fix | var | row)", + 0, C_IGNORE, 1, 3, &g.c_file_type }, + + { "huffman_key", + "if keys are huffman encoded", /* 40% */ + C_ROW, C_BOOL, 40, 0, &g.c_huffman_key }, + + { "huffman_value", + "if values are huffman encoded", /* 40% */ + C_ROW|C_VAR, C_BOOL, 40, 0, &g.c_huffman_value }, + + { "insert_pct", + "percent operations that are inserts", + 0, 0, 0, 45, &g.c_insert_pct }, + + { "internal_page_max", + "maximum size of Btree internal nodes", + 0, 0, 9, 17, &g.c_intl_page_max }, + + { "key_max", + "maximum size of keys", + C_ROW, 0, 64, 128, &g.c_key_max }, + + { "key_min", + "minimum size of keys", + C_ROW, 0, 10, 32, &g.c_key_min }, + + { "leaf_page_max", + "maximum size of Btree leaf nodes", + 0, 0, 9, 24, &g.c_leaf_page_max }, + + { "ops", + "the number of modification operations done per run", + 0, 0, 0, M(2), &g.c_ops }, + + { "repeat_data_pct", + "percent duplicate values in variable-len column-store files", + C_VAR, 0, 0, 90, &g.c_repeat_data_pct }, + + { "reverse", + "collate in reverse order", /* 10% */ + 0, C_BOOL, 10, 0, &g.c_reverse }, + + { "rows", + "the number of rows to create", + 0, 0, 10, M(1), &g.c_rows }, + + { "runs", + "the number of runs", + 0, C_IGNORE, 0, UINT_MAX, &g.c_runs }, + + { "value_max", + "maximum size of values", + C_ROW|C_VAR, 0, 32, 4096, &g.c_value_max }, + + { "value_min", + "minimum size of values", + C_ROW|C_VAR, 0, 1, 20, &g.c_value_min }, + + { "write_pct", + "percent operations that are writes", + 0, 0, 0, 90, &g.c_write_pct }, + + { NULL, NULL, 0, 0, 0, 0, NULL } +}; diff --git a/test/format/format.h b/test/format/format.h new file mode 100644 index 00000000000..2f3bac9dd3b --- /dev/null +++ b/test/format/format.h @@ -0,0 +1,123 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include <sys/types.h> + +#include <assert.h> +#include <ctype.h> +#include <errno.h> +#include <inttypes.h> +#include <limits.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <unistd.h> + +#ifdef BDB +#include "build_unix/db.h" +#else +#include <wiredtiger.h> +#endif + +#define M(v) ((v) * 1000000) /* Million */ +#define UNUSED(var) (void)(var) /* Quiet unused var warnings */ + +#define FIX 1 /* File types */ +#define ROW 2 +#define VAR 3 + +/* Get a random value between a min/max pair. */ +#define MMRAND(min, max) (wts_rand() % (((max) + 1) - (min)) + (min)) + +#define WT_TABLENAME "file:__wt" + +typedef struct { + char *progname; /* Program name */ + + void *bdb; /* BDB comparison handle */ + void *dbc; /* BDB cursor handle */ + + void *wts_conn; /* WT_CONNECTION handle */ + void *wts_cursor; /* WT_CURSOR handle */ + void *wts_cursor_insert; /* WT_CURSOR insert handle */ + void *wts_session; /* WT_SESSION handle */ + + FILE *rand_log; /* Random number log */ + + uint32_t run_cnt; /* Run counter */ + + enum { + LOG_FILE=1, /* Use a log file */ + LOG_OPS=2 /* Log all operations */ + } logging; + FILE *logfp; /* Log file. */ + + int replay; /* Replaying a run. */ + int track; /* Track progress */ + + char *config_open; /* Command-line configuration */ + + char *key_gen_buf; + + uint32_t c_bitcnt; /* Config values */ + uint32_t c_bzip; + uint32_t c_cache; + uint32_t c_delete_pct; + uint32_t c_file_type; + uint32_t c_huffman_key; + uint32_t c_huffman_value; + uint32_t c_insert_pct; + uint32_t c_intl_page_max; + uint32_t c_key_max; + uint32_t c_key_min; + uint32_t c_leaf_page_max; + uint32_t c_ops; + uint32_t c_repeat_data_pct; + uint32_t c_reverse; + uint32_t c_rows; + uint32_t c_runs; + uint32_t c_value_max; + uint32_t c_value_min; + uint32_t c_write_pct; + + uint32_t key_cnt; /* Keys loaded so far */ + uint32_t rows; /* Total rows */ + uint16_t key_rand_len[1031]; /* Key lengths */ +} GLOBAL; +extern GLOBAL g; + +int bdb_del(uint64_t, int *); +void bdb_insert(const void *, uint32_t, const void *, uint32_t); +int bdb_np(int, void *, uint32_t *, void *, uint32_t *, int *); +int bdb_put(const void *, uint32_t, const void *, uint32_t, int *); +int bdb_read(uint64_t, void *, uint32_t *, int *); +void bdb_startup(void); +void bdb_teardown(void); +const char * + config_dtype(void); +void config_error(void); +void config_file(const char *); +void config_print(int); +void config_setup(void); +void config_single(const char *, int); +void die(const char *, int); +void key_gen(void *, uint32_t *, uint64_t, int); +void key_gen_setup(void); +void track(const char *, uint64_t); +void value_gen(void *, uint32_t *, uint64_t); +int wts_bulk_load(void); +int wts_dump(const char *, int); +int wts_ops(void); +uint32_t wts_rand(void); +int wts_read_scan(void); +int wts_salvage(void); +int wts_startup(int); +int wts_stats(void); +int wts_teardown(void); +int wts_verify(const char *); diff --git a/test/format/s_dumpcmp.in b/test/format/s_dumpcmp.in new file mode 100644 index 00000000000..c523fcd507c --- /dev/null +++ b/test/format/s_dumpcmp.in @@ -0,0 +1,60 @@ +#! /bin/sh + +trap 'rm -f __*; exit 1' 1 2 + +build_top=../.. +bdb=$build_top/db/build_unix + +colflag=0 +dump_bdb=0 +while : + do case "$1" in + # -b means we need to dump the BDB database + -b) + dump_bdb=1; + shift ;; + # -c means it was a column-store. + -c) + colflag=1 + shift ;; + *) + break ;; + esac +done + +if test $# -ne 0; then + echo 'usage: s_dumpcmp [-bc]' >&2 + exit 1 +fi + +if test $dump_bdb -eq 1; then + if test $colflag -eq 0; then + $bdb/db_dump -p __bdb | + sed -e '1,/HEADER=END/d' \ + -e '/DATA=END/d' \ + -e 's/^ //' > __bdb_dump + else + # Format stores record numbers in Berkeley DB as string keys, + # it's simpler that way. Convert record numbers from strings + # to numbers. + $bdb/db_dump -p __bdb | + sed -e '1,/HEADER=END/d' \ + -e '/DATA=END/d' \ + -e 's/^ //' | + sed -e 's/^0*//' \ + -e 's/\.00$//' \ + -e N > __bdb_dump + fi +fi + +ext='"../../ext/collators/reverse/.libs/reverse_collator.so"' +bzext="../../ext/compressors/bzip2_compress/.libs/bzip2_compress.so" +if test -e $bzext ; then + ext="$ext,\"$bzext\"" +fi +config='extensions=['$ext']' + +$build_top/wt -C "$config" dump file:__wt | + sed -e '1,/^Data$/d' > __wt_dump +cmp __wt_dump __bdb_dump > /dev/null +exit $? diff --git a/test/format/t.c b/test/format/t.c new file mode 100644 index 00000000000..32d05f4e56d --- /dev/null +++ b/test/format/t.c @@ -0,0 +1,255 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "format.h" + +GLOBAL g; + +static void onint(int); +static void startup(void); +static void usage(void); + +int +main(int argc, char *argv[]) +{ + int ch, reps, ret; + + ret = 0; + + if ((g.progname = strrchr(argv[0], '/')) == NULL) + g.progname = argv[0]; + else + ++g.progname; + + /* Configure the FreeBSD malloc for debugging. */ + (void)setenv("MALLOC_OPTIONS", "AJZ", 1); + + /* Set values from the "CONFIG" file, if it exists. */ + if (access("CONFIG", R_OK) == 0) { + printf("... reading CONFIG file\n"); + config_file("CONFIG"); + } + + /* Track progress unless we're re-directing output to a file. */ + g.track = isatty(STDOUT_FILENO) ? 1 : 0; + + /* Set values from the command line. */ + while ((ch = getopt(argc, argv, "1C:c:Llqr")) != EOF) + switch (ch) { + case '1': /* One run */ + g.c_runs = 1; + break; + case 'C': /* wiredtiger_open config */ + g.config_open = optarg; + break; + case 'c': /* Configuration from a file */ + config_file(optarg); + break; + case 'L': /* Re-direct output to a log */ + /* + * The -l option is a superset of -L, ignore -L if we + * have already configured logging for operations. + */ + if (g.logging == 0) + g.logging = LOG_FILE; + break; + case 'l': /* Turn on operation logging */ + g.logging = LOG_OPS; + break; + case 'q': /* Quiet */ + g.track = 0; + break; + case 'r': /* Replay a run */ + g.replay = 1; + g.c_runs = 1; + break; + default: + usage(); + } + + argc -= optind; + argv += optind; + for (; *argv != NULL; ++argv) + config_single(*argv, 1); + + /* Clean up on signal. */ + (void)signal(SIGINT, onint); + + printf("%s: process %" PRIdMAX "\n", g.progname, (intmax_t)getpid()); + while (++g.run_cnt <= g.c_runs || g.c_runs == 0 ) { + startup(); /* Start a run */ + + config_setup(); /* Run configuration */ + config_print(0); /* Dump run configuration */ + + bdb_startup(); /* Initial file config */ + if (wts_startup(0)) + return (EXIT_FAILURE); + + key_gen_setup(); /* Setup keys */ + if (wts_bulk_load()) /* Load initial records */ + goto err; + /* Close, verify */ + if (wts_teardown() || wts_verify("post-bulk verify")) + goto err; + /* Loop reading & operations */ + for (reps = 0; reps < 3; ++reps) { + if (wts_startup(1)) + goto err; + + if (wts_read_scan()) /* Read scan */ + goto err; + + /* Random operations */ + if (g.c_ops != 0 && wts_ops()) + goto err; + + /* Statistics */ + if ((g.c_ops == 0 || reps == 2) && wts_stats()) + goto err; + + /* Close, verify */ + if (wts_teardown() || wts_verify("post-ops verify")) + goto err; + + /* + * If no operations scheduled, quit after a single + * read pass. + */ + if (g.c_ops == 0) + break; + } + + track("shutting down BDB", 0ULL); + bdb_teardown(); + + if (wts_dump("standard", 1)) /* Dump the file */ + goto err; + + /* + * If we don't delete any records, we can salvage the file. The + * problem with deleting records is that salvage will restore + * deleted records if a page fragments leaving a deleted record + * on one side of the split. + * + * Save a copy, salvage, verify, dump. + */ + if (g.c_delete_pct == 0) { + /* + * Save a copy of the interesting files so we can replay + * the salvage step as necessary. + */ + if (system( + "rm -rf __slvg.copy && " + "mkdir __slvg.copy && " + "cp WiredTiger* __wt __slvg.copy/") != 0) + goto err; + + if (wts_salvage() || + wts_verify("post-salvage verify") || + wts_dump("salvage", 0)) + goto err; + } + + printf("%4d: %-40s\n", g.run_cnt, config_dtype()); + } + + if (0) { +err: ret = 1; + } + + if (g.logfp != NULL) + (void)fclose(g.logfp); + if (g.rand_log != NULL) + (void)fclose(g.rand_log); + + config_print(ret); + return (ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE); +} + +/* + * startup -- + * Initialize for a run. + */ +static void +startup(void) +{ + /* Close the logging file. */ + if (g.logfp != NULL) { + (void)fclose(g.logfp); + g.logfp = NULL; + } + + /* Close the random number file. */ + if (g.rand_log != NULL) { + (void)fclose(g.rand_log); + g.rand_log = NULL; + } + + /* Remove the run's files except for __rand. */ + (void)system("rm -rf WiredTiger WiredTiger.* __[a-qs-z]* __run"); + + /* Open/truncate the logging file. */ + if (g.logging != 0) { + if ((g.logfp = fopen("__log", "w")) == NULL) + die("__log", errno); + (void)setvbuf(g.logfp, NULL, _IOLBF, 0); + } +} + +/* + * onint -- + * Interrupt signal handler. + */ +static void +onint(int signo) +{ + UNUSED(signo); + + /* Remove the run's files except for __rand. */ + (void)system("rm -rf WiredTiger WiredTiger.* __[a-qs-z]* __run"); + + fprintf(stderr, "\n"); + exit (EXIT_FAILURE); +} + +/* + * die -- + * Report an error and quit. + */ +void +die(const char *m, int e) +{ + fprintf(stderr, "%s: %s: %s\n", g.progname, m, wiredtiger_strerror(e)); + exit (EXIT_FAILURE); +} + +/* + * usage -- + * Display usage statement and exit failure. + */ +static void +usage(void) +{ + fprintf(stderr, + "usage: %s [-1Llqr] [-C wiredtiger-config] [-c config-file] " + "[name=value ...]\n", + g.progname); + fprintf(stderr, "%s", + "\t-1 run once\n" + "\t-C specify wiredtiger_open configuration arguments\n" + "\t-c read test program configuration from a file\n" + "\t-L output to a log file\n" + "\t-l log operations (implies -L)\n" + "\t-q run quietly\n" + "\t-r replay the last run\n"); + + fprintf(stderr, "\n"); + + config_error(); + exit(EXIT_FAILURE); +} diff --git a/test/format/util.c b/test/format/util.c new file mode 100644 index 00000000000..16b6377601b --- /dev/null +++ b/test/format/util.c @@ -0,0 +1,215 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "format.h" + +void +key_gen(void *keyp, uint32_t *sizep, uint64_t keyno, int insert) +{ + int len; + + /* + * The key always starts with a 10-digit string (the specified cnt) + * followed by two digits, a random number between 1 and 15 if it's + * an insert, otherwise 00. + */ + len = insert ? + sprintf(g.key_gen_buf, "%010" PRIu64 ".%02d", keyno, + (int)MMRAND(1, 15)) : + sprintf(g.key_gen_buf, "%010" PRIu64 ".00", keyno); + + /* + * In a column-store, the key is only used for BDB, and so it doesn't + * need a random length. + */ + if (g.c_file_type == ROW) { + g.key_gen_buf[len] = '/'; + len = g.key_rand_len[keyno % + (sizeof(g.key_rand_len) / sizeof(g.key_rand_len[0]))]; + } + *(void **)keyp = g.key_gen_buf; + *sizep = (uint32_t)len; +} + +void +key_gen_setup(void) +{ + size_t i; + + /* + * The key is a variable length item with a leading 10-digit value. + * Since we have to be able re-construct it from the record number + * (when doing row lookups), we pre-load a set of random lengths in + * a lookup table, and then use the record number to choose one of + * the pre-loaded lengths. + * + * Fill in the random key lengths. + */ + if (g.key_gen_buf != NULL) { + free(g.key_gen_buf); + g.key_gen_buf = NULL; + } + for (i = 0; i < sizeof(g.key_rand_len) / sizeof(g.key_rand_len[0]); ++i) + g.key_rand_len[i] = (uint16_t)MMRAND(g.c_key_min, g.c_key_max); + + if ((g.key_gen_buf = malloc(g.c_key_max)) == NULL) + die("malloc", errno); + for (i = 0; i < g.c_key_max; ++i) + g.key_gen_buf[i] = "abcdefghijklmnopqrstuvwxyz"[i % 26]; +} + +void +value_gen(void *valuep, uint32_t *sizep, uint64_t keyno) +{ + static size_t blen = 0; + static const char *dup_data = "duplicate data item"; + static u_char *buf = NULL; + size_t i; + + /* + * Set initial buffer contents to reconizable text. + * + * Add a few extra bytes in order to guarantee we can always offset + * into the buffer by a few extra bytes, used to generate different + * data for column-store run-length encoded files. + */ + if (blen < g.c_value_max + 10) { + if (buf != NULL) { + free(buf); + buf = NULL; + } + blen = g.c_value_max + 10; + if ((buf = malloc(blen)) == NULL) + die("malloc", errno); + for (i = 0; i < blen; ++i) + buf[i] = (u_char)"ABCDEFGHIJKLMNOPQRSTUVWXYZ"[i % 26]; + } + + /* + * Fixed-length records: take the low N bits from the last digit of + * the record number. + */ + if (g.c_file_type == FIX) { + switch (g.c_bitcnt) { + case 8: buf[0] = MMRAND(1, 0xff); break; + case 7: buf[0] = MMRAND(1, 0x7f); break; + case 6: buf[0] = MMRAND(1, 0x3f); break; + case 5: buf[0] = MMRAND(1, 0x1f); break; + case 4: buf[0] = MMRAND(1, 0x0f); break; + case 3: buf[0] = MMRAND(1, 0x07); break; + case 2: buf[0] = MMRAND(1, 0x03); break; + case 1: buf[0] = 1; break; + } + *(void **)valuep = buf; + *sizep = 1; + return; + } + + /* + * WiredTiger doesn't store zero-length data items in row-store files, + * test that by inserting a zero-length data item every so often. + */ + if (++keyno % 63 == 0) { + *(void **)valuep = buf; + *sizep = 0; + return; + } + + /* + * Start the data with a 10-digit number. + * + * For row and non-repeated variable-length column-stores, change the + * leading number to ensure every data item is unique. For repeated + * variable-length column-stores (that is, to test run-length encoding), + * use the same data value all the time. + */ + if (g.c_file_type == VAR && + g.c_repeat_data_pct != 0 && + (u_int)wts_rand() % 100 > g.c_repeat_data_pct) { + *(void **)valuep = (void *)dup_data; + *sizep = (uint32_t)strlen(dup_data); + return; + } + + snprintf((char *)buf, blen, "%010" PRIu64, keyno); + buf[10] = '/'; + *(void **)valuep = buf; + *sizep = MMRAND(g.c_value_min, g.c_value_max); +} + +void +track(const char *s, uint64_t i) +{ + static int lastlen = 0; + int len; + char msg[128]; + + if (!g.track || s == NULL) + return; + + if (i == 0) + len = snprintf(msg, sizeof(msg), "%4d: %s", + g.run_cnt, s); + else + len = snprintf(msg, sizeof(msg), "%4d: %s %" PRIu64, + g.run_cnt, s, i); + + if (lastlen > len) { + memset(msg + len, ' ', (size_t)(lastlen - len)); + msg[lastlen] = '\0'; + } + lastlen = len; + + (void)printf("%s\r", msg); + (void)fflush(stdout); +} + +/* + * wts_rand -- + * Return a random number. + */ +uint32_t +wts_rand(void) +{ + char buf[64]; + uint32_t r; + + /* + * We can entirely reproduce a run based on the random numbers used + * in the initial run, plus the configuration files. It would be + * nice to just log the initial RNG seed, rather than logging every + * random number generated, but we can't -- Berkeley DB calls rand() + * internally, and so that messes up the pattern of random numbers + * (and WT might call rand() in the future, who knows?) + */ + if (g.rand_log == NULL) { + if ((g.rand_log = + fopen("__rand", g.replay ? "r" : "w")) == NULL) + die("__rand", errno); + if (!g.replay) { + srand((u_int)(0xdeadbeef ^ (u_int)time(NULL))); + (void)setvbuf(g.rand_log, NULL, _IOLBF, 0); + } + } + if (g.replay) { + if (fgets(buf, sizeof(buf), g.rand_log) == NULL) { + if (feof(g.rand_log)) { + fprintf(stderr, + "end of random number log reached, " + "exiting\n"); + exit(EXIT_SUCCESS); + } + die("random number log", errno); + } + + r = (uint32_t)strtoul(buf, NULL, 10); + } else { + r = (uint32_t)rand(); + fprintf(g.rand_log, "%" PRIu32 "\n", r); + } + return (r); +} diff --git a/test/format/vt b/test/format/vt new file mode 100644 index 00000000000..00ec92df1ec --- /dev/null +++ b/test/format/vt @@ -0,0 +1,21 @@ +#! /bin/sh + +rm -f vgout.* + +# Command line argument is the number of iterations. +r=1 +test $# -eq 0 || r=$1 + +# Add +# --db-attach=yes +# to wait on a debugger attach. +while test $r -gt 0; do + r=`expr $r - 1` + valgrind \ + --leak-check=yes \ + --log-file=vgout.%p \ + --read-var-info=yes \ + --suppressions=vt.suppress \ + --track-fds=yes \ + ./t -1 +done diff --git a/test/format/vt.suppress b/test/format/vt.suppress new file mode 100644 index 00000000000..562af6810c0 --- /dev/null +++ b/test/format/vt.suppress @@ -0,0 +1,46 @@ +{ + <FreeBSD 8.2 dlopen #1> + Memcheck:Addr8 + obj:/libexec/ld-elf.so.1 + obj:/libexec/ld-elf.so.1 + obj:/libexec/ld-elf.so.1 + fun:dlopen + fun:__wt_dlopen + fun:__conn_load_extension + fun:wiredtiger_open + fun:wts_open + fun:wts_startup + fun:main +} +{ + <FreeBSD 8.2 dlopen #2> + Memcheck:Addr8 + obj:/libexec/ld-elf.so.1 + obj:/libexec/ld-elf.so.1 + obj:/libexec/ld-elf.so.1 + obj:/libexec/ld-elf.so.1 + fun:dlopen + fun:__wt_dlopen + fun:__conn_load_extension + fun:wiredtiger_open + fun:wts_open + fun:wts_startup + fun:main +} +{ + <Berkeley DB uninitialized write> + Memcheck:Param + write(buf) + obj:/lib/libc.so.7 + fun:__os_io + fun:__memp_pgwrite + fun:__memp_bhwrite + fun:__memp_sync_int + fun:__memp_fsync + fun:__db_sync + fun:__db_refresh + fun:__db_close + fun:__db_close_pp + fun:bdb_teardown + fun:main +} diff --git a/test/format/wts.c b/test/format/wts.c new file mode 100644 index 00000000000..ecb2a517c8e --- /dev/null +++ b/test/format/wts.c @@ -0,0 +1,1126 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "format.h" + +static int bulk(WT_ITEM **, WT_ITEM **); +static int wts_close(WT_CONNECTION *); +static int wts_col_del(uint64_t, int *); +static int wts_col_insert(uint64_t *); +static int wts_col_put(uint64_t); +static int wts_notfound_chk(const char *, int, int, uint64_t); +static int wts_np(int, int, int *); +static int wts_open(WT_CONNECTION **, WT_SESSION **session); +static int wts_read(uint64_t); +static int wts_row_del(uint64_t, int *); +static int wts_row_put(uint64_t, int); +static void wts_stream_item(const char *, WT_ITEM *); +static int wts_sync(void); + +static int +handle_message(WT_EVENT_HANDLER *handler, const char *message) +{ + UNUSED(handler); + + if (g.logfp != NULL) + fprintf(g.logfp, "%s\n", message); + else + printf("%s\n", message); + return (0); +} + +/* + * __handle_progress_default -- + * Default WT_EVENT_HANDLER->handle_progress implementation: ignore. + */ +static int +handle_progress(WT_EVENT_HANDLER *handler, + const char *operation, uint64_t progress) +{ + UNUSED(handler); + + track(operation, progress); + return (0); +} + +static WT_EVENT_HANDLER event_handler = { + NULL, + handle_message, + handle_progress +}; + +static int +wts_open(WT_CONNECTION **connp, WT_SESSION **sessionp) +{ + WT_CONNECTION *conn; + WT_SESSION *session; + int ret; + const char *ext1, *ext2; + char config[256]; + + /* If the bzip2 compression module has been built, use it. */ + ext1 = "../../ext/compressors/bzip2_compress/.libs/bzip2_compress.so"; + if (access(ext1, R_OK) != 0) { + ext1 = ""; + g.c_bzip = 0; + } + ext2 = "../../ext/collators/reverse/.libs/reverse_collator.so"; + + /* + * Open configuration -- put command line configuration options at the + * end so they can override "standard" configuration. + */ + snprintf(config, sizeof(config), + "create,error_prefix=\"%s\",cache_size=%" PRIu32 "MB," + "extensions=[\"%s\",\"%s\"],%s", + g.progname, g.c_cache, ext1, ext2, + g.config_open == NULL ? "" : g.config_open); + + if ((ret = wiredtiger_open(NULL, &event_handler, config, &conn)) != 0) { + fprintf(stderr, "%s: wiredtiger_open: %s\n", + g.progname, wiredtiger_strerror(ret)); + return (1); + } + + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) { + fprintf(stderr, "%s: conn.session: %s\n", + g.progname, wiredtiger_strerror(ret)); + (void)conn->close(conn, NULL); + return (1); + } + + *sessionp = session; + *connp = conn; + return (0); +} + +static int +wts_close(WT_CONNECTION *conn) +{ + int ret; + if ((ret = conn->close(conn, NULL)) != 0) { + fprintf(stderr, "%s: conn.close: %s\n", + g.progname, wiredtiger_strerror(ret)); + return (1); + } + + return (0); +} + +int +wts_startup(int open_cursors) +{ + time_t now; + WT_CONNECTION *conn; + WT_CURSOR *cursor, *cursor_insert; + WT_SESSION *session; + uint32_t maxintlpage, maxintlitem, maxleafpage, maxleafitem; + int ret; + char config[512], *end, *p; + + if (wts_open(&conn, &session)) + return (1); + + maxintlpage = 1U << g.c_intl_page_max; + maxintlitem = MMRAND(maxintlpage / 50, maxintlpage / 40); + if (maxintlitem < 40) + maxintlitem = 40; + maxleafpage = 1U << g.c_leaf_page_max; + /* Make sure at least 3 leaf pages can fix in cache. */ + while (3 * maxleafpage > g.c_cache << 20) + maxleafpage >>= 1; + maxleafitem = MMRAND(maxleafpage / 50, maxleafpage / 40); + if (maxleafitem < 40) + maxleafitem = 40; + + p = config; + end = config + sizeof(config); + p += snprintf(p, (size_t)(end - p), + "key_format=%s," + "internal_page_max=%d,internal_item_max=%d," + "leaf_page_max=%d,leaf_item_max=%d", + (g.c_file_type == ROW) ? "u" : "r", + maxintlpage, maxintlitem, maxleafpage, maxleafitem); + + if (g.c_bzip) + p += snprintf(p, (size_t)(end - p), + ",block_compressor=\"bzip2_compress\""); + + switch (g.c_file_type) { + case FIX: + p += snprintf(p, (size_t)(end - p), + ",value_format=%dt", g.c_bitcnt); + break; + case ROW: + if (g.c_huffman_key) + p += snprintf(p, (size_t)(end - p), + ",huffman_key=english"); + if (g.c_reverse) + p += snprintf(p, (size_t)(end - p), + ",collator=reverse"); + /* FALLTHROUGH */ + case VAR: + if (g.c_huffman_value) + p += snprintf(p, (size_t)(end - p), + ",huffman_value=english"); + break; + } + + if ((ret = session->create(session, WT_TABLENAME, config)) != 0) { + fprintf(stderr, "%s: create table: %s\n", + g.progname, wiredtiger_strerror(ret)); + return (1); + } + + cursor = cursor_insert = NULL; + if (open_cursors) { + /* + * We open two cursors: one configured for overwriting and one + * configured for append if we're dealing with a column-store. + * + * The reason is when testing with existing records, we don't + * track if a record was deleted or not, which means we need to + * use cursor->insert with overwriting configured. But, in + * column-store files where we're testing with new, appended + * records, we don't want to have to specify the record number, + * which requires an append configuration. + */ + if ((ret = session->open_cursor( + session, WT_TABLENAME, NULL, "overwrite", &cursor)) != 0) { + fprintf(stderr, "%s: open_cursor: %s\n", + g.progname, wiredtiger_strerror(ret)); + return (1); + } + if ((g.c_file_type == FIX || g.c_file_type == VAR) && + (ret = session->open_cursor(session, + WT_TABLENAME, NULL, "append", &cursor_insert)) != 0) { + fprintf(stderr, "%s: open_cursor: %s\n", + g.progname, wiredtiger_strerror(ret)); + return (1); + } + } + + if (g.logging == LOG_OPS) { + (void)time(&now); + (void)session->msg_printf(session, + "===============\nWT start: %s===============", + ctime(&now)); + } + + g.wts_conn = conn; + g.wts_cursor = cursor; + g.wts_cursor_insert = cursor_insert; + g.wts_session = session; + + return (0); +} + +int +wts_teardown(void) +{ + WT_CONNECTION *conn; + WT_CURSOR *cursor, *cursor_insert; + WT_SESSION *session; + time_t now; + int ret; + + conn = g.wts_conn; + cursor = g.wts_cursor; + cursor_insert = g.wts_cursor_insert; + session = g.wts_session; + + if (g.logging == LOG_OPS) { + (void)time(&now); + (void)session->msg_printf(session, + "===============\nWT stop: %s===============", + ctime(&now)); + } + + /* + * Close the open cursors -- they will block sync. + */ + if ((cursor_insert != NULL && + (ret = cursor_insert->close(cursor_insert)) != 0) || + (cursor != NULL && (ret = cursor->close(cursor)) != 0)) + die("cursor.close", ret); + + ret = wts_sync(); + return (wts_close(conn) ? 1 : ret); +} + +int +wts_bulk_load(void) +{ + WT_CURSOR *cursor; + WT_SESSION *session; + WT_ITEM *key, *value; + uint64_t insert_count; + int ret; + + session = g.wts_session; + key = NULL; /* -Wuninitialized */ + + /* + * Avoid bulk load with a custom collator, because the order of + * insertion will not match the collation order. + */ + if ((ret = session->open_cursor(session, WT_TABLENAME, NULL, + (g.c_file_type == ROW && g.c_reverse) ? NULL : "bulk", + &cursor)) != 0) { + fprintf(stderr, "%s: cursor open failed: %s\n", + g.progname, wiredtiger_strerror(ret)); + return (1); + } + + insert_count = 0; + while (bulk(&key, &value) == 0) { + /* Report on progress every 100 inserts. */ + if (++insert_count % 100 == 0) + track("bulk load", insert_count); + + if (key != NULL) + cursor->set_key(cursor, key); + if (g.c_file_type == FIX) + cursor->set_value(cursor, *(uint8_t *)value->data); + else + cursor->set_value(cursor, value); + if ((ret = cursor->insert(cursor)) != 0) { + fprintf(stderr, "%s: cursor insert failed: %s\n", + g.progname, wiredtiger_strerror(ret)); + ret = 1; + goto err; + } + } + +err: (void)cursor->close(cursor); + return (ret); +} + +int +wts_dump(const char *tag, int dump_bdb) +{ + char cmd[128]; + + track("dump files and compare", 0ULL); + switch (g.c_file_type) { + case FIX: + case VAR: + snprintf(cmd, sizeof(cmd), + "sh ./s_dumpcmp%s -c", dump_bdb ? " -b" : ""); + break; + case ROW: + snprintf(cmd, sizeof(cmd), + "sh ./s_dumpcmp%s", dump_bdb ? " -b" : ""); + break; + default: + return (1); + } + if (system(cmd) != 0) { + fprintf(stderr, + "%s: %s dump comparison failed\n", g.progname, tag); + return (1); + } + + return (0); +} + +int +wts_salvage(void) +{ + WT_CONNECTION *conn; + WT_SESSION *session; + int ret; + + track("salvage", 0ULL); + + if (wts_open(&conn, &session)) + return (1); + + if ((ret = session->salvage(session, WT_TABLENAME, NULL)) != 0) { + fprintf(stderr, "%s: salvage: %s\n", + g.progname, wiredtiger_strerror(ret)); + return (1); + } + + return (wts_close(conn)); +} + +static int +wts_sync(void) +{ + WT_SESSION *session; + int ret; + + session = g.wts_session; + + track("sync", 0ULL); + + if ((ret = session->sync( + session, WT_TABLENAME, NULL)) != 0 && ret != EBUSY) { + fprintf(stderr, "%s: sync: %s\n", + g.progname, wiredtiger_strerror(ret)); + return (1); + } + return (0); +} + +int +wts_verify(const char *tag) +{ + WT_CONNECTION *conn; + WT_SESSION *session; + int ret; + + track("verify", 0ULL); + + if (wts_open(&conn, &session)) + return (1); + + if ((ret = session->verify(session, WT_TABLENAME, NULL)) != 0) + fprintf(stderr, "%s: %s verify: %s\n", + g.progname, tag, wiredtiger_strerror(ret)); + + return (wts_close(conn) ? 1 : ret); +} + +/* + * wts_stats -- + * Dump the run's statistics. + */ +int +wts_stats(void) +{ + WT_CURSOR *cursor; + WT_SESSION *session; + FILE *fp; + const char *pval, *desc; + uint64_t v; + int ret; + + session = g.wts_session; + + track("stat", 0ULL); + + if ((fp = fopen("__stats", "w")) == NULL) + die("__stats", errno); + + /* Connection statistics. */ + if ((ret = session->open_cursor(session, + "statistics:", NULL, NULL, &cursor)) != 0) { + fprintf(stderr, "%s: stat cursor open failed: %s\n", + g.progname, wiredtiger_strerror(ret)); + return (1); + } + while ((ret = cursor->next(cursor)) == 0 && + (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) + if (fprintf(fp, "%s=%s\n", desc, pval) < 0) { + ret = errno; + break; + } + + if (ret != WT_NOTFOUND) + die("cursor.next", ret); + if ((ret = cursor->close(cursor)) != 0) + die("cursor.close", ret); + + /* File statistics. */ + if ((ret = session->open_cursor(session, + "statistics:" WT_TABLENAME, NULL, NULL, &cursor)) != 0) { + fprintf(stderr, "%s: stat cursor open failed: %s\n", + g.progname, wiredtiger_strerror(ret)); + return (1); + } + while ((ret = cursor->next(cursor)) == 0 && + (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) + if (fprintf(fp, "%s=%s\n", desc, pval) < 0) { + ret = errno; + break; + } + + if (ret != WT_NOTFOUND) + die("cursor.next", ret); + if ((ret = cursor->close(cursor)) != 0) + die("cursor.close", ret); + + (void)fclose(fp); + + return (0); +} + +/* + * bulk -- + * WiredTiger bulk load routine. + */ +static int +bulk(WT_ITEM **keyp, WT_ITEM **valuep) +{ + static WT_ITEM key, value; + WT_SESSION *session; + + session = g.wts_session; + + if (++g.key_cnt > g.c_rows) { + g.key_cnt = g.rows = g.c_rows; + return (1); + } + + key_gen(&key.data, &key.size, (uint64_t)g.key_cnt, 0); + value_gen(&value.data, &value.size, (uint64_t)g.key_cnt); + + switch (g.c_file_type) { + case FIX: + *keyp = NULL; + *valuep = &value; + if (g.logging == LOG_OPS) + (void)session->msg_printf(session, + "%-10s %" PRIu32 " {0x%02" PRIx8 "}", + "bulk V", + g.key_cnt, ((uint8_t *)value.data)[0]); + break; + case VAR: + *keyp = NULL; + *valuep = &value; + if (g.logging == LOG_OPS) + (void)session->msg_printf(session, + "%-10s %" PRIu32 " {%.*s}", "bulk V", + g.key_cnt, (int)value.size, (char *)value.data); + break; + case ROW: + *keyp = &key; + if (g.logging == LOG_OPS) + (void)session->msg_printf(session, + "%-10s %" PRIu32 " {%.*s}", "bulk K", + g.key_cnt, (int)key.size, (char *)key.data); + *valuep = &value; + if (g.logging == LOG_OPS) + (void)session->msg_printf(session, + "%-10s %" PRIu32 " {%.*s}", "bulk V", + g.key_cnt, (int)value.size, (char *)value.data); + break; + } + + /* Insert the item into BDB. */ + bdb_insert(key.data, key.size, value.data, value.size); + + return (0); +} + +/* + * wts_ops -- + * Perform a number of operations. + */ +int +wts_ops(void) +{ + uint64_t cnt, keyno; + uint32_t op; + u_int np; + int dir, insert, notfound; + + for (cnt = 0; cnt < g.c_ops; ++cnt) { + if (cnt % 10 == 0) + track("read/write ops", cnt); + + insert = notfound = 0; + keyno = MMRAND(1, g.rows); + + /* + * Perform some number of operations: the percentage of deletes, + * inserts and writes are specified, reads are the rest. The + * percentages don't have to add up to 100, a high percentage + * of deletes will mean fewer inserts and writes. A read + * operation always follows a modification to confirm it worked. + */ + op = (uint32_t)(wts_rand() % 100); + if (op < g.c_delete_pct) { + switch (g.c_file_type) { + case ROW: + /* + * If deleting a non-existent record, the cursor + * won't be positioned, and so can't do a next. + */ + if (wts_row_del(keyno, ¬found)) + return (1); + break; + case FIX: + case VAR: + if (wts_col_del(keyno, ¬found)) + return (1); + break; + } + } else if (op < g.c_delete_pct + g.c_insert_pct) { + switch (g.c_file_type) { + case ROW: + if (wts_row_put(keyno, 1)) + return (1); + break; + case FIX: + case VAR: + if (wts_col_insert(&keyno)) + return (1); + insert = 1; + break; + } + } else if ( + op < g.c_delete_pct + g.c_insert_pct + g.c_write_pct) { + switch (g.c_file_type) { + case ROW: + if (wts_row_put(keyno, 0)) + return (1); + break; + case FIX: + case VAR: + if (wts_col_put(keyno)) + return (1); + break; + } + } else { + if (wts_read(keyno)) + return (1); + continue; + } + + /* + * If we did any operation, we've set the cursor, do a small + * number of next/prev cursor operations in a random direction. + */ + dir = MMRAND(0, 1); + for (np = 0; np < MMRAND(1, 8); ++np) { + if (notfound) + break; + if (wts_np(dir, insert, ¬found)) + return (1); + } + + /* Then read the value we modified to confirm it worked. */ + if (wts_read(keyno)) + return (1); + } + return (0); +} + +/* + * wts_read_scan -- + * Read and verify all elements in a file. + */ +int +wts_read_scan(void) +{ + uint64_t cnt, last_cnt; + + /* Check a random subset of the records using the key. */ + for (last_cnt = cnt = 0; cnt < g.key_cnt;) { + cnt += wts_rand() % 17 + 1; + if (cnt > g.rows) + cnt = g.rows; + if (cnt - last_cnt > 1000) { + track("read row scan", cnt); + last_cnt = cnt; + } + + if (wts_read(cnt)) + return (1); + } + return (0); +} + +#define NTF_CHK(a) do { \ + switch (a) { \ + case 0: \ + break; \ + case 1: \ + return (1); \ + case 2: \ + return (0); \ + } \ +} while (0) + +/* + * wts_read -- + * Read and verify a single element in a row- or column-store file. + */ +static int +wts_read(uint64_t keyno) +{ + static WT_ITEM key, value, bdb_value; + WT_CURSOR *cursor; + WT_SESSION *session; + int notfound, ret; + uint8_t bitfield; + + cursor = g.wts_cursor; + session = g.wts_session; + + /* Log the operation */ + if (g.logging == LOG_OPS) + (void)session->msg_printf( + session, "%-10s%" PRIu64, "read", keyno); + + /* Retrieve the BDB value. */ + if (bdb_read(keyno, &bdb_value.data, &bdb_value.size, ¬found)) + return (1); + + /* Retrieve the key/value pair by key. */ + switch (g.c_file_type) { + case FIX: + case VAR: + cursor->set_key(cursor, keyno); + break; + case ROW: + key_gen(&key.data, &key.size, keyno, 0); + cursor->set_key(cursor, &key); + break; + } + + if ((ret = cursor->search(cursor)) == 0) { + if (g.c_file_type == FIX) { + ret = cursor->get_value(cursor, &bitfield); + value.data = &bitfield; + value.size = 1; + } else + ret = cursor->get_value(cursor, &value); + } + if (ret != 0 && ret != WT_NOTFOUND) { + fprintf(stderr, "%s: wts_read: read row %" PRIu64 ": %s\n", + g.progname, keyno, wiredtiger_strerror(ret)); + return (1); + } + + /* + * Check for not-found status. + * + * In fixed length stores, zero values at the end of the key space + * are treated as not found. Treat this the same as a zero value + * in the key space, to match BDB's behavior. + */ + if (g.c_file_type == FIX && ret == WT_NOTFOUND) { + bitfield = 0; + ret = 0; + } + + NTF_CHK(wts_notfound_chk("wts_read", ret, notfound, keyno)); + + /* Compare the two. */ + if (value.size != bdb_value.size || + memcmp(value.data, bdb_value.data, value.size) != 0) { + fprintf(stderr, + "wts_read: read row value mismatch %" PRIu64 ":\n", keyno); + wts_stream_item("bdb", &bdb_value); + wts_stream_item(" wt", &value); + return (1); + } + return (0); +} + +/* + * wts_np -- + * Read and verify the next/prev element in a row- or column-store file. + */ +static int +wts_np(int next, int insert, int *notfoundp) +{ + static WT_ITEM key, value, bdb_key, bdb_value; + WT_CURSOR *cursor; + WT_SESSION *session; + uint64_t keyno; + int notfound, ret; + uint8_t bitfield; + const char *which; + char *p; + + cursor = insert ? g.wts_cursor_insert : g.wts_cursor; + session = g.wts_session; + which = next ? "next" : "prev"; + + /* Retrieve the BDB value. */ + if (bdb_np(next, &bdb_key.data, &bdb_key.size, + &bdb_value.data, &bdb_value.size, ¬found)) + return (1); + *notfoundp = notfound; + + keyno = 0; + ret = next ? cursor->next(cursor) : cursor->prev(cursor); + if (ret == 0) + switch (g.c_file_type) { + case FIX: + if ((ret = cursor->get_key(cursor, &keyno)) == 0 && + (ret = cursor->get_value(cursor, &bitfield)) == 0) { + value.data = &bitfield; + value.size = 1; + } + break; + case ROW: + if ((ret = cursor->get_key(cursor, &key)) == 0) + ret = cursor->get_value(cursor, &value); + break; + case VAR: + if ((ret = cursor->get_key(cursor, &keyno)) == 0) + ret = cursor->get_value(cursor, &value); + break; + } + if (ret != 0 && ret != WT_NOTFOUND) { + fprintf(stderr, + "%s: wts_%s: %s\n", + g.progname, which, wiredtiger_strerror(ret)); + return (1); + } + + NTF_CHK(wts_notfound_chk( + next ? "wts_np(next)" : "wts_np(prev)", ret, notfound, keyno)); + + /* Compare the two. */ + if (g.c_file_type == ROW) { + if (key.size != bdb_key.size || + memcmp(key.data, bdb_key.data, key.size) != 0) { + fprintf(stderr, "wts_np: %s key mismatch:\n", which); + wts_stream_item("bdb-key", &bdb_key); + wts_stream_item(" wt-key", &key); + return (1); + } + } else { + if (keyno != (uint64_t)atoll(bdb_key.data)) { + if ((p = strchr((char *)bdb_key.data, '.')) != NULL) + *p = '\0'; + fprintf(stderr, + "wts_np: %s key mismatch: %.*s != %" PRIu64 "\n", + which, + (int)bdb_key.size, (char *)bdb_key.data, keyno); + return (1); + } + } + if (value.size != bdb_value.size || + memcmp(value.data, bdb_value.data, value.size) != 0) { + fprintf(stderr, "wts_np: %s value mismatch:\n", which); + wts_stream_item("bdb-value", &bdb_value); + wts_stream_item(" wt-value", &value); + return (1); + } + + if (g.logging == LOG_OPS) + switch (g.c_file_type) { + case FIX: + (void)session->msg_printf( + session, "%-10s%" PRIu64 " {0x%02x}", which, + keyno, ((char *)value.data)[0]); + break; + case ROW: + (void)session->msg_printf(session, "%-10s{%.*s/%.*s}", + which, + (int)key.size, (char *)key.data, + (int)value.size, (char *)value.data); + break; + case VAR: + (void)session->msg_printf( + session, "%-10s%" PRIu64 " {%.*s}", + which, keyno, (int)value.size, (char *)value.data); + break; + } + + return (0); +} + +/* + * wts_row_put -- + * Update an element in a row-store file. + */ +static int +wts_row_put(uint64_t keyno, int insert) +{ + static WT_ITEM key, value; + WT_CURSOR *cursor; + WT_SESSION *session; + int notfound, ret; + + cursor = g.wts_cursor; + session = g.wts_session; + + key_gen(&key.data, &key.size, keyno, insert); + value_gen(&value.data, &value.size, keyno); + + /* Log the operation */ + if (g.logging == LOG_OPS) + (void)session->msg_printf(session, "%-10s{%.*s}\n%-10s{%.*s}", + insert ? "insertK" : "putK", + (int)key.size, (char *)key.data, + insert ? "insertV" : "putV", + (int)value.size, (char *)value.data); + + if (bdb_put(key.data, key.size, value.data, value.size, ¬found)) + return (1); + + cursor->set_key(cursor, &key); + cursor->set_value(cursor, &value); + ret = cursor->insert(cursor); + if (ret != 0 && ret != WT_NOTFOUND) { + fprintf(stderr, + "%s: wts_row_put: %s row %" PRIu64 " by key: %s\n", + g.progname, insert ? "insert" : "update", + keyno, wiredtiger_strerror(ret)); + return (1); + } + + NTF_CHK(wts_notfound_chk("wts_row_put", ret, notfound, keyno)); + return (0); +} + +/* + * wts_col_put -- + * Update an element in a column-store file. + */ +static int +wts_col_put(uint64_t keyno) +{ + static WT_ITEM key, value; + WT_CURSOR *cursor; + WT_SESSION *session; + int notfound, ret; + + cursor = g.wts_cursor; + session = g.wts_session; + + key_gen(&key.data, &key.size, keyno, 0); + value_gen(&value.data, &value.size, keyno); + + /* Log the operation */ + if (g.logging == LOG_OPS) { + if (g.c_file_type == FIX) + (void)session->msg_printf(session, + "%-10s%" PRIu64 " {0x%02" PRIx8 "}", + "update", keyno, + ((uint8_t *)value.data)[0]); + else + (void)session->msg_printf(session, + "%-10s%" PRIu64 " {%.*s}", + "update", keyno, + (int)value.size, (char *)value.data); + } + + cursor->set_key(cursor, keyno); + if (g.c_file_type == FIX) + cursor->set_value(cursor, *(uint8_t *)value.data); + else + cursor->set_value(cursor, &value); + ret = cursor->insert(cursor); + if (ret != 0 && ret != WT_NOTFOUND) { + fprintf(stderr, + "%s: wts_col_put: %" PRIu64 " : %s\n", + g.progname, keyno, wiredtiger_strerror(ret)); + return (1); + } + + if (bdb_put(key.data, key.size, value.data, value.size, ¬found)) + return (1); + + NTF_CHK(wts_notfound_chk("wts_col_put", ret, notfound, keyno)); + return (0); +} + +/* + * wts_col_insert -- + * Insert an element in a column-store file. + */ +static int +wts_col_insert(uint64_t *keynop) +{ + static WT_ITEM key, value; + WT_CURSOR *cursor; + WT_SESSION *session; + uint64_t keyno; + int notfound, ret; + + cursor = g.wts_cursor_insert; + session = g.wts_session; + + value_gen(&value.data, &value.size, g.rows + 1); + + if (g.c_file_type == FIX) + cursor->set_value(cursor, *(uint8_t *)value.data); + else + cursor->set_value(cursor, &value); + ret = cursor->insert(cursor); + if (ret != 0) { + fprintf(stderr, "%s: wts_col_insert: %s\n", + g.progname, wiredtiger_strerror(ret)); + return (1); + } + if ((ret = cursor->get_key(cursor, &keyno)) != 0) { + fprintf(stderr, "%s: cursor->get_key: %s\n", + g.progname, wiredtiger_strerror(ret)); + return (1); + } + if (keyno <= g.rows) { + fprintf(stderr, + "%s: inserted key did not create new row\n", g.progname); + return (1); + } + *keynop = g.rows = (uint32_t)keyno; + + if (g.logging == LOG_OPS) { + if (g.c_file_type == FIX) + (void)session->msg_printf(session, + "%-10s%" PRIu64 " {0x%02" PRIx8 "}", + "insert", keyno, + ((uint8_t *)value.data)[0]); + else + (void)session->msg_printf(session, + "%-10s%" PRIu64 " {%.*s}", + "insert", keyno, + (int)value.size, (char *)value.data); + } + + key_gen(&key.data, &key.size, keyno, 0); + return (bdb_put( + key.data, key.size, value.data, value.size, ¬found) ? 1 : 0); +} + +/* + * wts_row_del -- + * Delete an element from a row-store file. + */ +static int +wts_row_del(uint64_t keyno, int *notfoundp) +{ + static WT_ITEM key; + WT_CURSOR *cursor; + WT_SESSION *session; + int notfound, ret; + + *notfoundp = 0; + cursor = g.wts_cursor; + session = g.wts_session; + + key_gen(&key.data, &key.size, keyno, 0); + + /* Log the operation */ + if (g.logging == LOG_OPS) + (void)session->msg_printf( + session, "%-10s%" PRIu64, "remove", keyno); + + if (bdb_del(keyno, ¬found)) + return (1); + *notfoundp = notfound; + + cursor->set_key(cursor, &key); + ret = cursor->remove(cursor); + if (ret != 0 && ret != WT_NOTFOUND) { + fprintf(stderr, + "%s: wts_row_del: remove %" PRIu64 " by key: %s\n", + g.progname, keyno, wiredtiger_strerror(ret)); + return (1); + } + + NTF_CHK(wts_notfound_chk("wts_row_del", ret, notfound, keyno)); + return (0); +} + +/* + * wts_col_del -- + * Delete an element from a column-store file. + */ +static int +wts_col_del(uint64_t keyno, int *notfoundp) +{ + static WT_ITEM key; + WT_CURSOR *cursor; + WT_SESSION *session; + int notfound, ret; + + cursor = g.wts_cursor; + session = g.wts_session; + + /* Log the operation */ + if (g.logging == LOG_OPS) + (void)session->msg_printf( + session, "%-10s%" PRIu64, "remove", keyno); + + /* + * Deleting a fixed-length item is the same as setting the bits to 0; + * do the same thing for the BDB store. + */ + if (g.c_file_type == FIX) { + key_gen(&key.data, &key.size, keyno, 0); + if (bdb_put(key.data, key.size, "\0", 1, ¬found)) + return (1); + } else { + if (bdb_del(keyno, ¬found)) + return (1); + *notfoundp = notfound; + } + + cursor->set_key(cursor, keyno); + ret = cursor->remove(cursor); + if (ret != 0 && ret != WT_NOTFOUND) { + fprintf(stderr, + "%s: wts_col_del: remove %" PRIu64 " by key: %s\n", + g.progname, keyno, wiredtiger_strerror(ret)); + return (1); + } + + NTF_CHK(wts_notfound_chk("wts_col_del", ret, notfound, keyno)); + return (0); +} + +/* + * wts_notfound_chk -- + * Compare notfound returns for consistency. + */ +static int +wts_notfound_chk(const char *f, int wt_ret, int bdb_notfound, uint64_t keyno) +{ + /* Check for not found status. */ + if (bdb_notfound) { + if (wt_ret == WT_NOTFOUND) + return (2); + + fprintf(stderr, "%s: %s:", g.progname, f); + if (keyno != 0) + fprintf(stderr, " row %" PRIu64 ":", keyno); + fprintf(stderr, + " not found in Berkeley DB, found in WiredTiger\n"); + return (1); + } + if (wt_ret == WT_NOTFOUND) { + fprintf(stderr, "%s: %s:", g.progname, f); + if (keyno != 0) + fprintf(stderr, " row %" PRIu64 ":", keyno); + fprintf(stderr, + " found in Berkeley DB, not found in WiredTiger\n"); + return (1); + } + return (0); +} + +/* + * wts_stream_item -- + * Dump a single data/size pair, with a tag. + */ +static void +wts_stream_item(const char *tag, WT_ITEM *item) +{ + static const char hex[] = "0123456789abcdef"; + const uint8_t *data; + uint32_t size; + int ch; + + data = item->data; + size = item->size; + + fprintf(stderr, "\t%s {", tag); + if (g.c_file_type == FIX) + fprintf(stderr, "0x%02x", data[0]); + else + for (; size > 0; --size, ++data) { + ch = data[0]; + if (isprint(ch)) + fprintf(stderr, "%c", ch); + else + fprintf(stderr, "%x%x", + hex[(data[0] & 0xf0) >> 4], + hex[data[0] & 0x0f]); + } + fprintf(stderr, "}\n"); +} diff --git a/test/packing/intpack-test.c b/test/packing/intpack-test.c new file mode 100644 index 00000000000..bf3fa13a2c8 --- /dev/null +++ b/test/packing/intpack-test.c @@ -0,0 +1,46 @@ +#include <assert.h> +#include <stdlib.h> +#include <time.h> + +#include <wt_internal.h> +#include "intpack.i" + +int main() { + uint8_t buf[10], *p; + uint64_t r, r2, ncalls; + int i, s; + + ncalls = 0; + + for (i = 0; i < 10000000; i++) { + for (s = 0; s < 50; s += 5) { + ++ncalls; + r = 1 << s; + +#if 1 + p = buf; + __wt_vpack_uint(NULL, &p, sizeof buf, r); + p = buf; + __wt_vunpack_uint(NULL, &p, sizeof buf, &r2); +#else + /* + * Note: use memmove for comparison because GCC does + * aggressive optimization of memcpy and it's difficult + * to measure anything. + */ + p = buf; + memmove(p, &r, sizeof r); + p = buf; + memmove(&r2, p, sizeof r2); +#endif + if (r != r2) { + fprintf(stderr, "mismatch!\n"); + break; + } + } + } + + printf("Number of calls: %llu\n", (unsigned long long)ncalls); + + return (0); +} diff --git a/test/packing/intpack-test2.c b/test/packing/intpack-test2.c new file mode 100644 index 00000000000..9a0d4a584ec --- /dev/null +++ b/test/packing/intpack-test2.c @@ -0,0 +1,29 @@ +#include <assert.h> +#include <stdlib.h> +#include <time.h> + +#include <wt_internal.h> +#include "intpack.i" + +int main() { + uint8_t buf[10], *p, *end; + int64_t i; + + for (i = 1; i < 1LL << 60; i <<= 1) { + end = buf; + __wt_vpack_uint(NULL, &end, sizeof buf, i); + printf("%lld ", i); + for (p = buf; p < end; p++) + printf("%02x", *p); + printf("\n"); + + end = buf; + __wt_vpack_int(NULL, &end, sizeof buf, -i); + printf("%lld ", -i); + for (p = buf; p < end; p++) + printf("%02x", *p); + printf("\n"); + } + + return (0); +} diff --git a/test/packing/packing-test.c b/test/packing/packing-test.c new file mode 100644 index 00000000000..ed4390d27f5 --- /dev/null +++ b/test/packing/packing-test.c @@ -0,0 +1,40 @@ +#include <assert.h> +#include <stdlib.h> +#include <time.h> + +#include <wiredtiger.h> +#include <stdarg.h> + +void check(const char *fmt, ...) +{ + char buf[200], *end, *p; + va_list ap; + size_t len; + + va_start(ap, fmt); + len = wiredtiger_struct_sizev(fmt, ap); + va_end(ap); + + assert(len < sizeof buf); + + va_start(ap, fmt); + assert(wiredtiger_struct_packv(buf, sizeof buf, fmt, ap) == 0); + va_end(ap); + + printf("%s ", fmt); + for (p = buf, end = p + len; p < end; p++) + printf("%02x", *p & 0xff); + printf("\n"); +} + +int main() { + check("iii", 0, 101, -99); + check("3i", 0, 101, -99); + check("iS", 42, "forty two"); +#if 0 + /* TODO: need a WT_ITEM */ + check("u", r"\x42" * 20) + check("uu", r"\x42" * 10, r"\x42" * 10) +#endif + return (0); +} diff --git a/test/salvage/Makefile.am b/test/salvage/Makefile.am new file mode 100644 index 00000000000..ca0ef890e72 --- /dev/null +++ b/test/salvage/Makefile.am @@ -0,0 +1,9 @@ +INCLUDES = -I$(top_builddir) -I$(top_srcdir)/src/include + +noinst_PROGRAMS = t +t_SOURCES = salvage.c +t_LDADD = $(top_builddir)/libwiredtiger.la +t_LDFLAGS = -static + +clean-local: + rm -rf WiredTiger WiredTiger.* __slvg.* diff --git a/test/salvage/salvage.c b/test/salvage/salvage.c new file mode 100644 index 00000000000..daaa3bf4acc --- /dev/null +++ b/test/salvage/salvage.c @@ -0,0 +1,675 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +#include <assert.h> + +#define DUMP "__slvg.dump" /* Dump file */ +#define LOAD "__slvg.load" /* Build file */ +#define RSLT "__slvg.result" /* Result file */ +#define SLVG "__slvg.slvg" /* Salvage file */ + +#define PSIZE (2 * 1024) +#define OSIZE (PSIZE / 20) + +void build(int, int, int); +void copy(u_int, u_int); +void empty(int); +void print_res(int, int, int); +void process(void); +void run(int); +void t(int, u_int, int); +int usage(void); + +FILE *res_fp; /* Results file */ +u_int page_type; /* File types */ +int value_unique; /* Values are unique */ +int verbose; /* -v flag */ + +const char *progname; /* Program name */ + +int +main(int argc, char *argv[]) +{ + u_int ptype; + int ch, r; + + if ((progname = strrchr(argv[0], '/')) == NULL) + progname = argv[0]; + else + ++progname; + + r = 0; + ptype = 0; + while ((ch = getopt(argc, argv, "r:t:v")) != EOF) + switch (ch) { + case 'r': + r = atoi(optarg); + if (r == 0) + return (usage()); + break; + case 't': + if (strcmp(optarg, "fix") == 0) + ptype = WT_PAGE_COL_FIX; + else if (strcmp(optarg, "var") == 0) + ptype = WT_PAGE_COL_VAR; + else if (strcmp(optarg, "row") == 0) + ptype = WT_PAGE_ROW_LEAF; + else + return (usage()); + break; + case 'v': + verbose = 1; + break; + case '?': + default: + return (usage()); + } + argc -= optind; + argv += optind; + if (argc != 0) + return (usage()); + + printf("salvage test run started\n"); + + t(r, ptype, 1); + t(r, ptype, 0); + + printf("salvage test run completed\n"); + return (EXIT_SUCCESS); +} + +void +t(int r, u_int ptype, int unique) +{ + printf("%sunique values\n", unique ? "" : "non-"); + value_unique = unique; + +#define NTESTS 24 + if (r == 0) { + if (ptype == 0) { + page_type = WT_PAGE_COL_FIX; + for (r = 1; r <= NTESTS; ++r) + run(r); + + page_type = WT_PAGE_COL_VAR; + for (r = 1; r <= NTESTS; ++r) + run(r); + + page_type = WT_PAGE_ROW_LEAF; + for (r = 1; r <= NTESTS; ++r) + run(r); + } else { + page_type = ptype; + for (r = 1; r <= NTESTS; ++r) + run(r); + } + } else if (ptype == 0) { + page_type = WT_PAGE_COL_FIX; + run(r); + page_type = WT_PAGE_COL_VAR; + run(r); + page_type = WT_PAGE_ROW_LEAF; + run(r); + } else { + page_type = ptype; + run(r); + } +} + +int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s [-v] [-r run] [-t fix|rle|var|row]\n", progname); + return (EXIT_FAILURE); +} + +void +run(int r) +{ + char buf[128]; + + printf("\t%s: run %d\n", __wt_page_type_string(page_type), r); + + (void)system("rm -f WiredTiger WiredTiger.* __slvg.* __schema.*"); + assert((res_fp = fopen(RSLT, "w")) != NULL); + + /* + * Each run builds the LOAD file, and then appends the first page of + * the LOAD file into the SLVG file. The SLVG file is then salvaged, + * verified, and dumped into the DUMP file, which is compared to the + * results file, which are the expected results. + */ + switch (r) { + case 1: + /* + * Smoke test: empty files. + */ + build(0, 0, 0); copy(0, 0); + break; + case 2: + /* + * Smoke test: + * Sequential pages, all pages should be kept. + */ + build(100, 100, 20); copy(6, 1); + build(200, 200, 20); copy(7, 21); + build(300, 300, 20); copy(8, 41); + print_res(100, 100, 20); + print_res(200, 200, 20); + print_res(300, 300, 20); + break; + case 3: + /* + * Smoke test: + * Sequential pages, all pages should be kept. + */ + build(100, 100, 20); copy(8, 1); + build(200, 200, 20); copy(7, 21); + build(300, 300, 20); copy(6, 41); + print_res(100, 100, 20); + print_res(200, 200, 20); + print_res(300, 300, 20); + break; + case 4: + /* + * Case #1: + * 3 pages, each with 20 records starting with the same record + * and sequential LSNs; salvage should leave the page with the + * largest LSN. + */ + build(100, 100, 20); copy(6, 1); + build(100, 200, 20); copy(7, 1); + build(100, 300, 20); copy(8, 1); + print_res(100, 300, 20); + break; + case 5: + /* + * Case #1: + * 3 pages, each with 20 records starting with the same record + * and sequential LSNs; salvage should leave the page with the + * largest LSN. + */ + build(100, 100, 20); copy(6, 1); + build(100, 200, 20); copy(8, 1); + build(100, 300, 20); copy(7, 1); + print_res(100, 200, 20); + break; + case 6: + /* + * Case #1: + * 3 pages, each with 20 records starting with the same record + * and sequential LSNs; salvage should leave the page with the + * largest LSN. + */ + build(100, 100, 20); copy(8, 1); + build(100, 200, 20); copy(7, 1); + build(100, 300, 20); copy(6, 1); + print_res(100, 100, 20); + break; + case 7: + /* + * Case #2: + * The second page overlaps the beginning of the first page, and + * the first page has a higher LSN. + */ + build(110, 100, 20); copy(7, 11); + build(100, 200, 20); copy(6, 1); + print_res(100, 200, 10); + print_res(110, 100, 20); + break; + case 8: + /* + * Case #2: + * The second page overlaps the beginning of the first page, and + * the second page has a higher LSN. + */ + build(110, 100, 20); copy(6, 11); + build(100, 200, 20); copy(7, 1); + print_res(100, 200, 20); + print_res(120, 110, 10); + break; + case 9: + /* + * Case #3: + * The second page overlaps with the end of the first page, and + * the first page has a higher LSN. + */ + build(100, 100, 20); copy(7, 1); + build(110, 200, 20); copy(6, 11); + print_res(100, 100, 20); + print_res(120, 210, 10); + break; + case 10: + /* + * Case #3: + * The second page overlaps with the end of the first page, and + * the second page has a higher LSN. + */ + build(100, 100, 20); copy(6, 1); + build(110, 200, 20); copy(7, 11); + print_res(100, 100, 10); + print_res(110, 200, 20); + break; + case 11: + /* + * Case #4: + * The second page is a prefix of the first page, and the first + * page has a higher LSN. + */ + build(100, 100, 20); copy(7, 1); + build(100, 200, 5); copy(6, 1); + print_res(100, 100, 20); + break; + case 12: + /* + * Case #4: + * The second page is a prefix of the first page, and the second + * page has a higher LSN. + */ + build(100, 100, 20); copy(6, 1); + build(100, 200, 5); copy(7, 1); + print_res(100, 200, 5); + print_res(105, 105, 15); + break; + case 13: + /* + * Case #5: + * The second page is in the middle of the first page, and the + * first page has a higher LSN. + */ + build(100, 100, 40); copy(7, 1); + build(110, 200, 10); copy(6, 11); + print_res(100, 100, 40); + break; + case 14: + /* + * Case #5: + * The second page is in the middle of the first page, and the + * second page has a higher LSN. + */ + build(100, 100, 40); copy(6, 1); + build(110, 200, 10); copy(7, 11); + print_res(100, 100, 10); + print_res(110, 200, 10); + print_res(120, 120, 20); + break; + case 15: + /* + * Case #6: + * The second page is a suffix of the first page, and the first + * page has a higher LSN. + */ + build(100, 100, 40); copy(7, 1); + build(130, 200, 10); copy(6, 31); + print_res(100, 100, 40); + break; + case 16: + /* + * Case #6: + * The second page is a suffix of the first page, and the second + * page has a higher LSN. + */ + build(100, 100, 40); copy(6, 1); + build(130, 200, 10); copy(7, 31); + print_res(100, 100, 30); + print_res(130, 200, 10); + break; + case 17: + /* + * Case #9: + * The first page is a prefix of the second page, and the first + * page has a higher LSN. + */ + build(100, 100, 20); copy(7, 1); + build(100, 200, 40); copy(6, 1); + print_res(100, 100, 20); + print_res(120, 220, 20); + break; + case 18: + /* + * Case #9: + * The first page is a prefix of the second page, and the second + * page has a higher LSN. + */ + build(100, 100, 20); copy(6, 1); + build(100, 200, 40); copy(7, 1); + print_res(100, 200, 40); + break; + case 19: + /* + * Case #10: + * The first page is a suffix of the second page, and the first + * page has a higher LSN. + */ + build(130, 100, 10); copy(7, 31); + build(100, 200, 40); copy(6, 1); + print_res(100, 200, 30); + print_res(130, 100, 10); + break; + case 20: + /* + * Case #10: + * The first page is a suffix of the second page, and the second + * page has a higher LSN. + */ + build(130, 100, 10); copy(6, 31); + build(100, 200, 40); copy(7, 1); + print_res(100, 200, 40); + break; + case 21: + /* + * Case #11: + * The first page is in the middle of the second page, and the + * first page has a higher LSN. + */ + build(110, 100, 10); copy(7, 11); + build(100, 200, 40); copy(6, 1); + print_res(100, 200, 10); + print_res(110, 100, 10); + print_res(120, 220, 20); + break; + case 22: + /* + * Case #11: + * The first page is in the middle of the second page, and the + * second page has a higher LSN. + */ + build(110, 100, 10); copy(6, 11); + build(100, 200, 40); copy(7, 1); + print_res(100, 200, 40); + break; + case 23: + /* + * Column-store only: missing an initial key range of 99 + * records. + */ + build(100, 100, 10); copy(1, 100); + empty(99); + print_res(100, 100, 10); + break; + case 24: + /* + * Column-store only: missing a middle key range of 37 + * records. + */ + build(100, 100, 10); copy(1, 1); + build(138, 138, 10); copy(1, 48); + print_res(100, 100, 10); + empty(37); + print_res(138, 138, 10); + break; + default: + fprintf(stderr, "salvage: %d: no such test\n", r); + exit (EXIT_FAILURE); + } + + assert(fclose(res_fp) == 0); + + process(); + + snprintf(buf, sizeof(buf), "cmp %s %s > /dev/null", DUMP, RSLT); + if (system(buf)) { + fprintf(stderr, + "check failed, salvage results were incorrect\n"); + exit (EXIT_FAILURE); + } +} + +/* + * build -- + * Build a row- or column-store page in a file. + */ +void +build(int ikey, int ivalue, int cnt) +{ + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_ITEM key, value; + WT_SESSION *session; + char config[256], kbuf[64], vbuf[64]; + int new_slvg; + + assert(wiredtiger_open(NULL, NULL, "create", &conn) == 0); + assert(conn->open_session(conn, NULL, NULL, &session) == 0); + assert(session->drop(session, "file:" LOAD, "force") == 0); + + switch (page_type) { + case WT_PAGE_COL_FIX: + (void)snprintf(config, sizeof(config), + "key_format=r,value_format=7t," + "allocation_size=%d," + "internal_page_max=%d,internal_item_max=%d," + "leaf_page_max=%d,leaf_item_max=%d", + PSIZE, PSIZE, OSIZE, PSIZE, OSIZE); + break; + case WT_PAGE_COL_VAR: + (void)snprintf(config, sizeof(config), + "key_format=r," + "allocation_size=%d," + "internal_page_max=%d,internal_item_max=%d," + "leaf_page_max=%d,leaf_item_max=%d", + PSIZE, PSIZE, OSIZE, PSIZE, OSIZE); + break; + case WT_PAGE_ROW_LEAF: + (void)snprintf(config, sizeof(config), + "key_format=u," + "allocation_size=%d," + "internal_page_max=%d,internal_item_max=%d," + "leaf_page_max=%d,leaf_item_max=%d", + PSIZE, PSIZE, OSIZE, PSIZE, OSIZE); + break; + default: + assert(0); + } + assert(session->create(session, "file:" LOAD, config) == 0); + assert(session->open_cursor( + session, "file:" LOAD, NULL, "bulk", &cursor) == 0); + for (; cnt > 0; --cnt, ++ikey, ++ivalue) { + switch (page_type) { /* Build the key. */ + case WT_PAGE_COL_FIX: + case WT_PAGE_COL_VAR: + break; + case WT_PAGE_ROW_LEAF: + snprintf(kbuf, sizeof(kbuf), "%010d KEY------", ikey); + key.data = kbuf; + key.size = 20; + cursor->set_key(cursor, &key); + break; + } + + switch (page_type) { /* Build the value. */ + case WT_PAGE_COL_FIX: + cursor->set_value(cursor, ivalue & 0x7f); + break; + case WT_PAGE_COL_VAR: + case WT_PAGE_ROW_LEAF: + snprintf(vbuf, sizeof(vbuf), + "%010d VALUE----", value_unique ? ivalue : 37); + value.data = vbuf; + value.size = 20; + cursor->set_value(cursor, &value); + } + assert(cursor->insert(cursor) == 0); + } + + /* + * The first time through this routine we put a matching configuration + * in for the salvage file. + */ + new_slvg = (access(SLVG, F_OK) != 0); + if (new_slvg) { + assert(session->drop(session, "file:" SLVG, "force") == 0); + assert(session->create(session, "file:" SLVG, config) == 0); + } + + assert(conn->close(conn, 0) == 0); + + /* + * We created the salvage file above, but all we want is the schema, + * we're creating the salvage file by hand. + */ + if (new_slvg) + (void)remove(SLVG); +} + +/* + * copy -- + * Copy the created page to the end of the salvage file. + */ +void +copy(u_int gen, u_int recno) +{ + FILE *ifp, *ofp; + WT_PAGE_HEADER *dsk; + WT_BLOCK_HEADER *blk; + char buf[PSIZE]; + + assert((ifp = fopen(LOAD, "r")) != NULL); + + /* + * If the salvage file doesn't exist, then we're creating it: + * copy the first sector (the file description). + * Otherwise, we are appending to an existing file. + */ + if (access(SLVG, F_OK)) { + assert((ofp = fopen(SLVG, "w")) != NULL); + assert(fread(buf, 1, 512, ifp) == 512); + assert(fwrite(buf, 1, 512, ofp) == 512); + } else + assert((ofp = fopen(SLVG, "a")) != NULL); + + /* + * If there's data, copy/update the first formatted page. + */ + if (gen != 0) { + assert(fseek(ifp, (long)512, SEEK_SET) == 0); + assert(fread(buf, 1, PSIZE, ifp) == PSIZE); + dsk = (WT_PAGE_HEADER *)buf; + if (page_type != WT_PAGE_ROW_LEAF) + dsk->recno = recno; + blk = WT_BLOCK_HEADER_REF(buf); + blk->write_gen = gen; + blk->cksum = 0; + blk->cksum = __wt_cksum(dsk, PSIZE); + assert(fwrite(buf, 1, PSIZE, ofp) == PSIZE); + } + + assert(fclose(ifp) == 0); + assert(fclose(ofp) == 0); +} + +/* + * process -- + * Salvage, verify and dump the created file. + */ +void +process(void) +{ + FILE *fp; + WT_CONNECTION *conn; + WT_CURSOR *cursor; + const char *key, *value; + WT_SESSION *session; + char config[100]; + + /* Salvage. */ + config[0] = '\0'; + if (verbose) + snprintf(config, sizeof(config), + "error_prefix=\"%s\",verbose=[salvage,verify]", + progname); + assert(wiredtiger_open(NULL, NULL, config, &conn) == 0); + assert(conn->open_session(conn, NULL, NULL, &session) == 0); + assert(session->salvage(session, "file:" SLVG, 0) == 0); + assert(conn->close(conn, 0) == 0); + + /* Verify. */ + assert(wiredtiger_open(NULL, NULL, "", &conn) == 0); + assert(conn->open_session(conn, NULL, NULL, &session) == 0); + assert(session->verify(session, "file:" SLVG, 0) == 0); + assert(conn->close(conn, 0) == 0); + + /* Dump. */ + assert((fp = fopen(DUMP, "w")) != NULL); + assert(wiredtiger_open(NULL, NULL, "", &conn) == 0); + assert(conn->open_session(conn, NULL, NULL, &session) == 0); + assert(session->create(session, "file:" SLVG, NULL) == 0); + assert(session->open_cursor( + session, "file:" SLVG, NULL, "dump=print", &cursor) == 0); + while (cursor->next(cursor) == 0) { + if (page_type == WT_PAGE_ROW_LEAF) { + assert(cursor->get_key(cursor, &key) == 0); + assert(fputs(key, fp) >= 0); + assert(fputc('\n', fp) >= 0); + } + assert(cursor->get_value(cursor, &value) == 0); + assert(fputs(value, fp) >= 0); + assert(fputc('\n', fp) >= 0); + } + assert(conn->close(conn, 0) == 0); + assert(fclose(fp) == 0); +} + +/* + * empty -- + * Print empty print_res, for fixed-length column-store files. + */ +void +empty(int cnt) +{ + int i; + + if (page_type == WT_PAGE_COL_FIX) + for (i = 0; i < cnt; ++i) + fputs("\\00\n", res_fp); +} + +/* + * print_res -- + * Write results file. + */ +void +print_res(int key, int value, int cnt) +{ + static const char hex[] = "0123456789abcdef"; + int ch; + + for (; cnt > 0; ++key, ++value, --cnt) { + switch (page_type) { /* Print key */ + case WT_PAGE_COL_FIX: + case WT_PAGE_COL_VAR: + break; + case WT_PAGE_ROW_LEAF: + fprintf(res_fp, "%010d KEY------\n", key); + break; + } + + switch (page_type) { /* Print value */ + case WT_PAGE_COL_FIX: + ch = value & 0x7f; + if (isprint(ch)) { + if (ch == '\\') + fputc('\\', res_fp); + fputc(ch, res_fp); + } else { + fputc('\\', res_fp); + fputc(hex[(ch & 0xf0) >> 4], res_fp); + fputc(hex[ch & 0x0f], res_fp); + } + fputc('\n', res_fp); + break; + case WT_PAGE_COL_VAR: + case WT_PAGE_ROW_LEAF: + fprintf(res_fp, + "%010d VALUE----\n", value_unique ? value : 37); + break; + } + } +} diff --git a/test/suite/run.py b/test/suite/run.py new file mode 100644 index 00000000000..577d83c7ce5 --- /dev/null +++ b/test/suite/run.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# WiredTigerTestCase +# parent class for all test cases +# + +import os +import sys +import unittest +import glob +import re +import json + +# Set paths +suitedir = sys.path[0] +wt_disttop = os.path.dirname(os.path.dirname(suitedir)) +wt_builddir = os.path.join(wt_disttop, 'build_posix') +wt_3rdpartydir = os.path.join(wt_disttop, 'test', '3rdparty') + +# Cannot import wiredtiger and supporting utils until we set up paths +sys.path.append(os.path.join(wt_builddir, 'lang', 'python')) +sys.path.append(os.path.join(wt_disttop, 'lang', 'python')) +sys.path.append(os.path.join(wt_3rdpartydir, 'discover-0.4.0')) +sys.path.append(os.path.join(wt_3rdpartydir, 'testtools-0.9.12')) +sys.path.append(os.path.join(wt_3rdpartydir, 'testscenarios-0.2', 'lib')) + +import wttest +from testscenarios.scenarios import generate_scenarios + +def usage(): + print 'Usage:\n\ + $ cd build_posix\n\ + $ python ../test/suite/run.py [ options ] [ tests ]\n\ +\n\ +Options:\n\ + -C file | --configcreate file create a config file for controlling tests\n\ + -c file | --config file use a config file for controlling tests\n\ + -d | --debug run with \'pdb\', the python debugger\n\ + -g | --gdb all subprocesses (like calls to wt) use gdb\n\ + -h | --help show this message\n\ + -p | --preserve preserve output files in WT_TEST/<testname>\n\ + -t | --timestamp name WT_TEST according to timestamp\n\ + -v N | --verbose N set verboseness to N (0<=N<=3, default=1)\n\ +\n\ +Tests:\n\ + may be a file name in test/suite: (e.g. test_base01.py)\n\ + may be a subsuite name (e.g. \'base\' runs test_base*.py)\n\ +\n\ + When -C or -c are present, there may not be any tests named.\n\ +' + +# capture the category (AKA 'subsuite') part of a test name, +# e.g. test_util03 -> util +reCatname = re.compile(r"test_([^0-9]+)[0-9]*") + +def addScenarioTests(tests, loader, testname): + loaded = loader.loadTestsFromName(testname) + tests.addTests(generate_scenarios(loaded)) + +def configRecord(cmap, tup): + """ + Records this tuple in the config. It is marked as None + (appearing as null in json), so it can be easily adjusted + in the output file. + """ + tuplen = len(tup) + pos = 0 + for name in tup: + last = (pos == tuplen - 1) + pos += 1 + if not name in cmap: + if last: + cmap[name] = {"run":None} + else: + cmap[name] = {"run":None, "sub":{}} + if not last: + cmap = cmap[name]["sub"] + +def configGet(cmap, tup): + """ + Answers the question, should we do this test, given this config file? + Following the values of the tuple through the map, + returning the first non-null value. If all values are null, + return True (handles tests that may have been added after the + config was generated). + """ + for name in tup: + if not name in cmap: + return True + run = cmap[name]["run"] if "run" in cmap[name] else None + if run != None: + return run + cmap = cmap[name]["sub"] if "sub" in cmap[name] else {} + return True + +def configApplyInner(suites, configmap, configwrite): + newsuite = unittest.TestSuite() + for s in suites: + if type(s) is unittest.TestSuite: + newsuite.addTest(configApplyInner(s, configmap, configwrite)) + else: + modname = s.__module__ + catname = re.sub(reCatname, r"\1", modname) + classname = s.__class__.__name__ + methname = s._testMethodName + + tup = (catname, modname, classname, methname) + add = True + if configwrite: + configRecord(configmap, tup) + else: + add = configGet(configmap, tup) + if add: + newsuite.addTest(s) + return newsuite + +def configApply(suites, configfilename, configwrite): + configmap = None + if not configwrite: + with open(configfilename, 'r') as f: + line = f.readline() + while line != '\n' and line != '': + line = f.readline() + configmap = json.load(f) + else: + configmap = {} + newsuite = configApplyInner(suites, configmap, configwrite) + if configwrite: + with open(configfilename, 'w') as f: + f.write("""# Configuration file for wiredtiger test/suite/run.py, +# generated with '-C filename' and consumed with '-c filename'. +# This shows the hierarchy of tests, and can be used to rerun with +# a specific subset of tests. The value of "run" controls whether +# a test or subtests will be run: +# +# true turn on a test and all subtests (overriding values beneath) +# false turn on a test and all subtests (overriding values beneath) +# null do not effect subtests +# +# If a test does not appear, or is marked as '"run": null' all the way down, +# then the test is run. +# +# The remainder of the file is in JSON format. +# !!! There must be a single blank line following this line!!! + +""") + json.dump(configmap, f, sort_keys=True, indent=4) + return newsuite + +def testsFromArg(tests, loader, arg): + # If a group of test is mentioned, do all tests in that group + # e.g. 'run.py base' + groupedfiles = glob.glob(suitedir + os.sep + 'test_' + arg + '*.py') + if len(groupedfiles) > 0: + for file in groupedfiles: + testsFromArg(tests, loader, os.path.basename(file)) + return + + # Explicit test class names + if not arg[0].isdigit(): + if arg.endswith('.py'): + arg = arg[:-3] + addScenarioTests(tests, loader, arg) + return + + # Deal with ranges + if '-' in arg: + start, end = (int(a) for a in arg.split('-')) + else: + start, end = int(arg), int(arg) + for t in xrange(start, end+1): + addScenarioTests(tests, loader, 'test%03d' % t) + +if __name__ == '__main__': + tests = unittest.TestSuite() + + # Turn numbers and ranges into test module names + preserve = timestamp = debug = gdbSub = False + configfile = None + configwrite = False + verbose = 1 + args = sys.argv[1:] + testargs = [] + while len(args) > 0: + arg = args.pop(0) + from unittest import defaultTestLoader as loader + + # Command line options + if arg[0] == '-': + option = arg[1:] + if option == '-debug' or option == 'd': + debug = True + continue + if option == '-preserve' or option == 'p': + preserve = True + continue + if option == '-timestamp' or option == 't': + timestamp = True + continue + if option == '-gdb' or option == 'g': + gdbSub = True + continue + if option == '-help' or option == 'h': + usage() + sys.exit(True) + if option == '-verbose' or option == 'v': + if len(args) == 0: + usage() + sys.exit(False) + verbose = int(args.pop(0)) + continue + if option == '-config' or option == 'c': + if configfile != None or len(args) == 0: + usage() + sys.exit(False) + configfile = args.pop(0) + continue + if option == '-configcreate' or option == 'C': + if configfile != None or len(args) == 0: + usage() + sys.exit(False) + configfile = args.pop(0) + configwrite = True + continue + print 'unknown arg: ' + arg + usage() + sys.exit(False) + testargs.append(arg) + + # All global variables should be set before any test classes are loaded. + # That way, verbose printing can be done at the class definition level. + wttest.WiredTigerTestCase.globalSetup(preserve, timestamp, gdbSub, verbose) + + # Without any tests listed as arguments, do discovery + if len(testargs) == 0: + from discover import defaultTestLoader as loader + suites = loader.discover(suitedir) + suites = sorted(suites, key=lambda c: str(list(c)[0])) + if configfile != None: + suites = configApply(suites, configfile, configwrite) + tests.addTests(generate_scenarios(suites)) + else: + for arg in testargs: + testsFromArg(tests, loader, arg) + + if debug: + import pdb + pdb.set_trace() + + result = wttest.runsuite(tests) + sys.exit(not result.wasSuccessful()) diff --git a/test/suite/suite_random.py b/test/suite/suite_random.py new file mode 100644 index 00000000000..c0e0a2a9d85 --- /dev/null +++ b/test/suite/suite_random.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# suite_random.py +# A quick and predictable pseudo random number generator. +# + +class suite_random: + """ + Generate random 32 bit integers that are predictable, + and use no global state. We use the Multiply-with-carry + method invented by George Marsaglia, because it is quick + and easy to implement. + """ + def __init__(self, *args): + arglen = len(args) + if arglen == 1: + self.seedw = int(args[0]) & 0xffffffff + self.seedz = int(args[0]) & 0Xffffffff + elif arglen == 2: + self.seedw = int(args[0]) & 0xffffffff + self.seedz = int(args[1]) & 0Xffffffff + else: + self.seedw = 0 + self.seedz = 0 + + # The seeds cannot be 0 + if self.seedw == 0: + self.seedw += 22233 + if self.seedz == 0: + self.seedz += 11133 + + def rand32(self): + """ + returns a random 32 bit integer + """ + w = self.seedw + z = self.seedz + self.seedw = (18000 * (w & 0xffff) + (w >> 16)) & 0xffffffff + self.seedz = (36969 * (z & 0xffff) + (z >> 16)) & 0xffffffff + return ((z << 16) + w) & 0xffffffff + + def rand_range(self, n, m): + """ + returns a random integer in the range [N,M). + """ + if m > 0xffffffff or n < 0: + raise ValueError("rand32_range expects args between 0 , 2^32") + if n >= m: + raise ValueError("rand32_range(n,m) expects n < m") + r = self.rand32() + return (r % (m - n)) + n + + def rand_float(self): + """ + returns a random floating point value between 0 and 1.0 + The number returned does not encompass all possibilities, + only 2^32 values within the range. + """ + return (self.rand32() + 0.0)/0x100000000 diff --git a/test/suite/suite_subprocess.py b/test/suite/suite_subprocess.py new file mode 100644 index 00000000000..065f4c06e09 --- /dev/null +++ b/test/suite/suite_subprocess.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# suite_subprocess.py +# Run a subprocess within the test suite +# + +import unittest +from wiredtiger import WiredTigerError +import wttest +from run import wt_builddir +import subprocess +import os + +# Used as a 'mixin' class along with a WiredTigerTestCase class +class suite_subprocess: + subproc = None + + def has_error_in_file(self, filename): + """ + Return whether the file contains 'ERROR'. + WT utilities issue a 'WT_ERROR' output string upon error. + """ + with open(filename, 'r') as f: + for line in f: + if 'ERROR' in line: + return True + return False + + def check_no_error_in_file(self, filename, match='ERROR'): + """ + Raise an error and show output context if the file contains 'ERROR'. + WT utilities issue a 'WT_ERROR' output string upon error. + """ + lines = [] + hasError = False + hasPrevious = False # do we need to prefix an ellipsis? + hasNext = False # do we need to suffix an ellipsis? + with open(filename, 'r') as f: + for line in f: + lines.append(line) + hasError = hasError or match in line + if hasError: + if len(lines) > 10: + hasNext = True + break + else: + if len(lines) > 5: + lines.pop(0) + hasPrevious = True + if hasError: + print '**************** ' + match + ' in output file: ' + filename + ' ****************' + if hasPrevious: + print '...' + for line in lines: + print line, + if hasNext: + print '...' + print '********************************' + self.fail('ERROR found in output file: ' + filename) + + def check_file_content(self, filename, expect): + with open(filename, 'r') as f: + got = f.read(len(expect) + 100) + self.assertEqual(got, expect, filename + ': does not contain expected:\n\'' + expect + '\', but contains:\n\'' + got + '\'.') + + def check_file_contains(self, filename, expect): + """ + Check that the file contains the expected string in the first 100K bytes + """ + maxbytes = 1024*100 + with open(filename, 'r') as f: + got = f.read(maxbytes) + if not (expect in got): + if len(got) >= maxbytes: + self.fail(filename + ': does not contain expected \'' + expect + '\', or output is too large') + else: + self.fail(filename + ': does not contain expected \'' + expect + '\'') + + def check_empty_file(self, filename): + """ + Raise an error if the file is not empty + """ + filesize = os.path.getsize(filename) + if filesize > 0: + with open(filename, 'r') as f: + contents = f.read(1000) + print 'ERROR: ' + filename + ' expected to be empty, but contains:\n' + print contents + '...\n' + self.assertEqual(filesize, 0, filename + ': expected to be empty') + + def check_non_empty_file(self, filename): + """ + Raise an error if the file is empty + """ + filesize = os.path.getsize(filename) + if filesize == 0: + print 'ERROR: ' + filename + ' should not be empty (this command expected error output)' + self.assertNotEqual(filesize, 0, filename + ': expected to not be empty') + + def runWt(self, args, infilename=None, outfilename=None, errfilename=None, reopensession=True): + """ + Run the 'wt' process + """ + + # we close the connection to guarantee everything is + # flushed, and that we can open it from another process + self.close_conn() + + wtoutname = outfilename or "wt.out" + wterrname = errfilename or "wt.err" + with open(wterrname, "w") as wterr: + with open(wtoutname, "w") as wtout: + procargs = [os.path.join(wt_builddir, "wt")] + if self._gdbSubprocess: + procargs = [os.path.join(wt_builddir, "libtool"), + "--mode=execute", "gdb", "--args"] + procargs + procargs.extend(args) + if self._gdbSubprocess: + infilepart = "" + if infilename != None: + infilepart = "<" + infilename + " " + print str(procargs) + print "*********************************************" + print "**** Run 'wt' via: run " + " ".join(procargs[3:]) + infilepart + ">" + wtoutname + " 2>" + wterrname + print "*********************************************" + proc = subprocess.Popen(procargs) + else: + if infilename != None: + with open(infilename, "r") as wtin: + proc = subprocess.Popen(procargs, stdin=wtin, stdout=wtout, stderr=wterr) + else: + proc = subprocess.Popen(procargs, stdout=wtout, stderr=wterr) + proc.wait() + if errfilename == None: + self.check_empty_file(wterrname) + if outfilename == None: + self.check_empty_file(wtoutname) + + # Reestablish the connection if needed + if reopensession: + self.open_conn() diff --git a/test/suite/test_base01.py b/test/suite/test_base01.py new file mode 100644 index 00000000000..3dbbc8c5527 --- /dev/null +++ b/test/suite/test_base01.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_base01.py +# Basic operations +# + +import unittest +import wiredtiger +import wttest + +class test_base01(wttest.WiredTigerTestCase): + """ + Test basic operations + """ + table_name1 = 'test_base01a.wt' + table_name2 = 'test_base01b.wt' + + def create_table(self, tablename): + extra_params = ',allocation_size=512,internal_page_max=16384,leaf_page_max=131072' + self.pr('create_table') + self.session.create('table:' + tablename, 'key_format=S,value_format=S' + extra_params) + + def cursor_s(self, tablename, key): + cursor = self.session.open_cursor('table:' + tablename, None, None) + cursor.set_key(key) + return cursor + + def cursor_ss(self, tablename, key, val): + cursor = self.cursor_s(tablename, key) + cursor.set_value(val) + return cursor + + def test_error(self): + gotException = False + try: + self.pr('expect an error message...') + self.session.create('table:' + self.table_name1, 'expect_this_error,okay?') + except wiredtiger.WiredTigerError as e: + gotException = True + self.pr('got expected exception: ' + str(e)) + self.assertTrue(str(e).find('nvalid argument') >= 0) + self.assertTrue(gotException, 'expected exception') + + def test_empty(self): + """ + Create a table, look for a nonexistent key + """ + self.create_table(self.table_name1) + self.pr('creating cursor') + cursor = self.cursor_s(self.table_name1, 'somekey') + self.pr('search') + ret = cursor.search() + self.assertTrue(ret == wiredtiger.WT_NOTFOUND) + self.pr('closing cursor') + cursor.close() + + def test_insert(self): + """ + Create a table, add a key, get it back + """ + self.create_table(self.table_name2) + + self.pr('insert') + inscursor = self.cursor_ss(self.table_name2, 'key1', 'value1') + inscursor.insert() + inscursor.close + + self.pr('search') + getcursor = self.cursor_s(self.table_name2, 'key1') + ret = getcursor.search() + self.assertTrue(ret == 0) + self.assertTrue(getcursor.get_value(), 'value1') + self.pr('closing cursor') + getcursor.close() + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_base02.py b/test/suite/test_base02.py new file mode 100644 index 00000000000..1f064da0b84 --- /dev/null +++ b/test/suite/test_base02.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_base02.py +# Configuration +# + +#### This test has workarounds to allow it to complete, marked with '####' comments + +import unittest +import wiredtiger +import wttest +import json + +class test_base02(wttest.WiredTigerTestCase): + """ + Test configuration strings + """ + table_name1 = 'test_base02a' + + def create_and_drop_table(self, tablename, confstr): + self.pr('create_table with config:\n ' + confstr) + self.session.create('table:' + tablename, confstr) + self.session.drop('table:' + tablename, None) + + def test_config_combinations(self): + """ + Spot check various combinations of configuration options. + """ + conf_confsize = [ + None, + 'allocation_size=1024', + 'internal_page_max=64k,internal_item_max=1k', + 'leaf_page_max=128k,leaf_item_max=512', + 'leaf_page_max=256k,leaf_item_max=256,internal_page_max=8k,internal_item_max=128', + ] + conf_col = [ + 'columns=(first,second)', + 'columns=(first, second,,,)', + 'key_format="5S", value_format="Su", columns=(first,second, third)', + ',,columns=(first=S,second="4u"),,', + ] + conf_encoding = [ + None, + 'huffman_key=,huffman_value=english', + ] + for size in conf_confsize: + for col in conf_col: + for enc in conf_encoding: + conflist = [size, col, enc] + confstr = ",".join([c for c in conflist if c != None]) + self.create_and_drop_table(self.table_name1, confstr) + + def test_config_json(self): + """ + Spot check various combinations of configuration options, using JSON format. + """ + conf_jsonstr = [ + json.dumps({'columns' : ('key', 'value')}), + json.dumps({ + "key_format" : "r", + "value_format" : "5sHQ", + "columns" : ("id", "country", "year", "population"), + "colgroups" : ("cyear", "population"), + })] + for confstr in conf_jsonstr: + self.create_and_drop_table(self.table_name1, confstr) + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_base03.py b/test/suite/test_base03.py new file mode 100644 index 00000000000..6744e65f4db --- /dev/null +++ b/test/suite/test_base03.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_base03.py +# Cursor operations +# + +import unittest +import wiredtiger +import wttest +import wtscenario + +class test_base03(wttest.WiredTigerTestCase): + """ + Test basic operations + """ + table_name1 = 'test_base03a' + table_name2 = 'test_base03b' + table_name3 = 'test_base03c' + table_name4 = 'test_base03d' + nentries = 10 + + def config_string(self): + """ + Return any additional configuration. + This method may be overridden. + """ + return '' + + def session_create(self, name, args): + """ + session.create, but report errors more completely + """ + try: + self.session.create(name, args) + except: + print('**** ERROR in session.create("' + name + '","' + args + '") ***** ') + raise + + def test_table_ss(self): + """ + Create entries, and read back in a cursor: key=string, value=string + """ + create_args = 'key_format=S,value_format=S' + self.config_string() + self.session_create("table:" + self.table_name1, create_args) + self.pr('creating cursor') + cursor = self.session.open_cursor('table:' + self.table_name1, None, None) + for i in range(0, self.nentries): + cursor.set_key('key' + str(i)) + cursor.set_value('value' + str(i)) + cursor.insert() + + i = 0 + cursor.reset() + for key, value in cursor: + self.assertEqual(key, ('key' + str(i))) + self.assertEqual(value, ('value' + str(i))) + i += 1 + + self.assertEqual(i, self.nentries) + cursor.close() + + def test_table_si(self): + """ + Create entries, and read back in a cursor: key=string, value=int + """ + create_args = 'key_format=S,value_format=i' + self.config_string() + self.session_create("table:" + self.table_name2, create_args) + self.pr('creating cursor') + cursor = self.session.open_cursor('table:' + self.table_name2, None, None) + for i in range(0, self.nentries): + cursor.set_key('key' + str(i)) + cursor.set_value(i) + cursor.insert() + + i = 0 + cursor.reset() + for key, value in cursor: + self.pr('got: ' + str(key) + ': ' + str(value)) + self.assertEqual(key, 'key' + str(i)) + self.assertEqual(value, i) + i += 1 + + self.pr("i = " + str(i)) + self.pr("self.... = " + str(self.nentries)) + self.assertEqual(i, self.nentries) + cursor.close() + + def test_table_is(self): + """ + Create entries, and read back in a cursor: key=int, value=string + """ + create_args = 'key_format=i,value_format=S' + self.config_string() + self.session_create("table:" + self.table_name3, create_args) + self.pr('creating cursor') + cursor = self.session.open_cursor('table:' + self.table_name3, None, None) + for i in range(0, self.nentries): + cursor.set_key(i) + cursor.set_value('value' + str(i)) + cursor.insert() + + i = 0 + cursor.reset() + for key, value in cursor: + self.pr('got: ' + str(key) + ': ' + str(value)) + self.assertEqual(key, i) + self.assertEqual(value, 'value' + str(i)) + i += 1 + + self.assertEqual(i, self.nentries) + cursor.close() + + def test_table_ii(self): + """ + Create entries, and read back in a cursor: key=int, value=int + """ + create_args = 'key_format=i,value_format=i' + self.config_string() + self.session_create("table:" + self.table_name4, create_args) + self.pr('creating cursor') + cursor = self.session.open_cursor('table:' + self.table_name4, None, None) + self.pr('stepping') + for i in range(0, self.nentries): + self.pr('put %d -> %d' % (i, i)) + cursor.set_key(i) + cursor.set_value(i) + cursor.insert() + + i = 0 + cursor.reset() + for key, value in cursor: + self.pr('got %d -> %d' % (key, value)) + self.assertEqual(key, i) + self.assertEqual(value, i) + i += 1 + + self.assertEqual(i, self.nentries) + cursor.close() + + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_base04.py b/test/suite/test_base04.py new file mode 100644 index 00000000000..b737d32842c --- /dev/null +++ b/test/suite/test_base04.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_base04.py +# Test that tables are reconciled correctly when they are empty. +# + +import unittest +import wiredtiger +from wiredtiger import WT_NOTFOUND +import wttest + +class test_base04(wttest.WiredTigerTestCase): + '''Test various tree types becoming empty''' + + tablename = 'table:test_base04' + + def __init__(self, *args, **kwargs): + wttest.WiredTigerTestCase.__init__(self, *args, **kwargs) + self.reconcile = False + + def reopen(self): + self.conn.close() + self.conn = wiredtiger.wiredtiger_open('.', None) + self.session = self.conn.open_session() + + def create_table(self): + self.pr('create table') + self.session.create(self.tablename, 'key_format=S,value_format=S') + + def drop_table(self): + self.pr('drop table') + self.session.drop(self.tablename, None) + + def cursor(self): + self.pr('open cursor') + return self.session.open_cursor(self.tablename, None, None) + + def check_exists(self, key, expected): + cursor = self.cursor() + cursor.set_key(key) + self.pr('search') + self.assertEqual(cursor.search(), expected) + self.pr('closing cursor') + cursor.close() + + def insert(self, key, value): + self.pr('insert') + cursor = self.cursor() + cursor.set_key(key); + cursor.set_value(value) + cursor.insert() + cursor.close() + if self.reconcile: + self.reopen() + + def remove(self, key): + self.pr('remove') + cursor = self.cursor() + cursor.set_key(key); + cursor.remove() + cursor.close() + if self.reconcile: + self.reopen() + + def test_empty(self): + '''Create a table, look for a nonexistent key''' + self.create_table() + self.check_exists('somekey', WT_NOTFOUND) + self.drop_table() + + def test_insert(self): + '''Create a table, add a key, get it back''' + for self.reconcile in (False, True): + self.create_table() + self.insert('key1', 'value1') + self.check_exists('key1', 0) + self.drop_table() + + def test_insert_delete(self): + '''Create a table, add a key, get it back''' + for reconcile in (False, True): + self.create_table() + self.insert('key1', 'value1') + self.check_exists('key1', 0) + self.remove('key1') + self.check_exists('key1', WT_NOTFOUND) + self.drop_table() + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_base05.py b/test/suite/test_base05.py new file mode 100644 index 00000000000..2fd8ccd4135 --- /dev/null +++ b/test/suite/test_base05.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_base05.py +# Cursor operations +# + +import unittest +import wiredtiger +import wttest + +class test_base05(wttest.WiredTigerTestCase): + """ + Test that various types of content can be stored + Test the 'english' huffman encoding with English and non-English strings. + """ + + table_name1 = 'test_base05a' + table_name2 = 'test_base05b' + nentries = 1000 + scenarios = [ + ('no_huffman', dict(extraconfig='')), + ('huffman_key', dict(extraconfig='huffman_key="english"')), + ('huffman_val', dict(extraconfig='huffman_value="english"')), + ('huffman_keyval', dict(extraconfig='huffman_key="english",huffman_value="english"')) + ] + + def config_string(self): + """ + Return any additional configuration. + This method may be overridden. + """ + return self.extraconfig + + def session_create(self, name, args): + """ + session.create, but report errors more completely + """ + try: + self.session.create(name, args) + except: + print('**** ERROR in session.create("' + name + '","' + args + '") ***** ') + raise + + # Moby Dick by Herman Melville, Chapter 1 (excerpt) + english_strings = [ + 'Call me Ishmael.', + 'Some years ago-never mind how long precisely-having little or no money in my purse, and nothing particular to interest me on shore, I thought I would sail about a little and see the watery part of the world.', + 'It is a way I have of driving off the spleen and regulating the circulation.', + 'Whenever I find myself growing grim about the mouth; whenever it is a damp, drizzly November in my soul; whenever I find myself involuntarily pausing before coffin warehouses, and bringing up the rear of every funeral I meet; and especially whenever my hypos get such an upper hand of me, that it requires a strong moral principle to prevent me from deliberately stepping into the street, and methodically knocking people\'s hats off-then, I account it high time to get to sea as soon as I can.', + 'This is my substitute for pistol and ball.', + 'With a philosophical flourish Cato throws himself upon his sword; I quietly take to the ship.', + 'There is nothing surprising in this.', + 'If they but knew it, almost all men in their degree, some time or other, cherish very nearly the same feelings towards the ocean with me.', + 'There now is your insular city of the Manhattoes, belted round by wharves as Indian isles by coral reefs-commerce surrounds it with her surf.', + 'Right and left, the streets take you waterward.', + 'Its extreme downtown is the battery, where that noble mole is washed by waves, and cooled by breezes, which a few hours previous were out of sight of land.', + 'Look at the crowds of water-gazers there.', + 'Circumambulate the city of a dreamy Sabbath afternoon.', + 'Go from Corlears Hook to Coenties Slip, and from thence, by Whitehall, northward.', + 'What do you see?-Posted like silent sentinels all around the town, stand thousands upon thousands of mortal men fixed in ocean reveries.', + 'Some leaning against the spiles; some seated upon the pier-heads; some looking over the bulwarks glasses! of ships from China; some high aloft in the rigging, as if striving to get a still better seaward peep.', + 'But these are all landsmen; of week days pent up in lath and plaster- tied to counters, nailed to benches, clinched to desks.', + 'How then is this? Are the green fields gone? What do they here? But look! here come more crowds, pacing straight for the water, and seemingly bound for a dive.', + 'Strange! Nothing will content them but the extremest limit of the land; loitering under the shady lee of yonder warehouses will not suffice.', + 'No.', + 'They must get just as nigh the water as they possibly can without falling in.', + 'And there they stand-miles of them-leagues.', + 'Inlanders all, they come from lanes and alleys, streets and avenues,- north, east, south, and west.', + 'Yet here they all unite.', + 'Tell me, does the magnetic virtue of the needles of the compasses of all those ships attract them thither? Once more.', + 'Say you are in the country; in some high land of lakes.', + 'Take almost any path you please, and ten to one it carries you down in a dale, and leaves you there by a pool in the stream.', + 'There is magic in it.', + 'Let the most absent-minded of men be plunged in his deepest reveries-stand that man on his legs, set his feet a-going, and he will infallibly lead you to water, if water there be in all that region.', + 'Should you ever be athirst in the great American desert, try this experiment, if your caravan happen to be supplied with a metaphysical professor.', + 'Yes, as every one knows, meditation and water are wedded for ever.', + 'But here is an artist.', + 'He desires to paint you the dreamiest, shadiest, quietest, most enchanting bit of romantic landscape in all the valley of the Saco.', + 'What is the chief element he employs? There stand his trees, each with a hollow trunk, as if a hermit and a crucifix were within; and here sleeps his meadow, and there sleep his cattle; and up from yonder cottage goes a sleepy smoke.', + 'Deep into distant woodlands winds a mazy way, reaching to overlapping spurs of mountains bathed in their hill-side blue.', + 'But though the picture lies thus tranced, and though this pine-tree shakes down its sighs like leaves upon this shepherd\'s head, yet all were vain, unless the shepherd\'s eye were fixed upon the magic stream before him.', + 'Go visit the Prairies in June, when for scores on scores of miles you wade knee-deep among Tiger-lilies-what is the one charm wanting?- Water - there is not a drop of water there! Were Niagara but a cataract of sand, would you travel your thousand miles to see it? Why did the poor poet of Tennessee, upon suddenly receiving two handfuls of silver, deliberate whether to buy him a coat, which he sadly needed, or invest his money in a pedestrian trip to Rockaway Beach? Why is almost every robust healthy boy with a robust healthy soul in him, at some time or other crazy to go to sea? Why upon your first voyage as a passenger, did you yourself feel such a mystical vibration, when first told that you and your ship were now out of sight of land? Why did the old Persians hold the sea holy? Why did the Greeks give it a separate deity, and own brother of Jove? Surely all this is not without meaning.', + 'And still deeper the meaning of that story of Narcissus, who because he could not grasp the tormenting, mild image he saw in the fountain, plunged into it and was drowned.', + 'But that same image, we ourselves see in all rivers and oceans.', + 'It is the image of the ungraspable phantom of life; and this is the key to it all.', + 'Now, when I say that I am in the habit of going to sea whenever I begin to grow hazy about the eyes, and begin to be over conscious of my lungs, I do not mean to have it inferred that I ever go to sea as a passenger.', + 'For to go as a passenger you must needs have a purse, and a purse is but a rag unless you have something in it.', + 'Besides, passengers get sea-sick- grow quarrelsome-don\'t sleep of nights-do not enjoy themselves much, as a general thing;-no, I never go as a passenger; nor, though I am something of a salt, do I ever go to sea as a Commodore, or a Captain, or a Cook.', + 'I abandon the glory and distinction of such offices to those who like them.', + 'For my part, I abominate all honorable respectable toils, trials, and tribulations of every kind whatsoever.', + 'It is quite as much as I can do to take care of myself, without taking care of ships, barques, brigs, schooners, and what not.', + 'And as for going as cook,-though I confess there is considerable glory in that, a cook being a sort of officer on ship-board-yet, somehow, I never fancied broiling fowls;-though once broiled, judiciously buttered, and judgmatically salted and peppered, there is no one who will speak more respectfully, not to say reverentially, of a broiled fowl than I will.', + 'It is out of the idolatrous dotings of the old Egyptians upon broiled ibis and roasted river horse, that you see the mummies of those creatures in their huge bakehouses the pyramids.', + 'No, when I go to sea, I go as a simple sailor, right before the mast, plumb down into the fore-castle, aloft there to the royal mast-head.', + 'True, they rather order me about some, and make me jump from spar to spar, like a grasshopper in a May meadow.', + 'And at first, this sort of thing is unpleasant enough.', + 'It touches one\'s sense of honor, particularly if you come of an old established family in the land, the Van Rensselaers, or Randolphs, or Hardicanutes.', + 'And more than all, if just previous to putting your hand into the tar-pot, you have been lording it as a country schoolmaster, making the tallest boys stand in awe of you.', + 'The transition is a keen one, I assure you, from a schoolmaster to a sailor, and requires a strong decoction of Seneca and the Stoics to enable you to grin and bear it.', + 'But even this wears off in time.' + ] + + # 'Hello' in several languages that use the non-latin part of unicode + non_english_strings = [ + '\u20320\u22909', + '\u1571\u1604\u1587\u1617\u1604\u1575\u1605\u32\u1593\u1604\u1610\u1603\u1605', + '\u1513\u1500\u1493\u1501', + '\u20170\u26085\u12399', + '\u50504\u45397\u54616\u49464\u50836', + '\u1047\u1076\u1088\u1072\u1074\u1089\u1090\u1074\u1091\u1081\u1090\u1077', + "\u4306\u4304\u4315\u4304\u4320\u4335\u4317\u4305\u4304" + ] + + def mixed_string(self, n): + """ + Build a string composed of some wide ranging number of substrings, + chosen mostly from the english_strings list with occasional + entries from non_english_strings list. + The returned value should be a somewhat random looking + mix, but must be repeatable for any given N. + """ + nstrings = 2 << (n % 10) + result = '' + for i in range(nstrings): + if (n + i) % 20 == 0: + reflist = self.non_english_strings + else: + reflist = self.english_strings + choice = (n + i) % len(reflist) + result += reflist[choice] + return result + ':' + str(n) + + def test_table_ss(self): + """ + Create entries, and read back in a cursor: key=string, value=string + """ + create_args = 'key_format=S,value_format=S,' + self.config_string() + self.session_create("table:" + self.table_name1, create_args) + self.pr('creating cursor') + cursor = self.session.open_cursor('table:' + self.table_name1, None, None) + numbers = {} + for i in range(0, self.nentries): + numbers[i] = i + key = self.mixed_string(i) + value = self.mixed_string(i+1) + cursor.set_key(key) + cursor.set_value(value) + cursor.insert() + + # quick spot check to make sure searches work + for divisor in [3, 5, 7]: + i = self.nentries / divisor + key = self.mixed_string(i) + value = self.mixed_string(i+1) + cursor.set_key(key) + self.assertEqual(0, cursor.search()) + self.assertEqual(key, cursor.get_key()) + self.assertEqual(value, cursor.get_value()) + + total = 0 + cursor.reset() + for key, value in cursor: + colonpos = key.rfind(':') + i = int(key[(colonpos+1):]) + del numbers[i] + self.assertEqual(key, self.mixed_string(i)) + self.assertEqual(value, self.mixed_string(i+1)) + total += 1 + + self.assertEqual(total, self.nentries) + self.assertEqual(0, len(numbers)) + cursor.close() + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_base06.py b/test/suite/test_base06.py new file mode 100644 index 00000000000..8affaf65541 --- /dev/null +++ b/test/suite/test_base06.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_base06.py +# session level operations on tables +# + +import unittest +import wiredtiger +from wiredtiger import WiredTigerError +import wttest +import time +import os + +class test_base06(wttest.WiredTigerTestCase): + """ + Test various session level operations on tables, including + rename, drop, sync, truncate + + """ + + table_name1 = 'test_base06a' + table_name2 = 'test_base06b' + #nentries = 1000 + nentries = 30 + + def populate(self, tablename): + create_args = 'key_format=i,value_format=S' + self.session.create("table:" + tablename, create_args) + cursor = self.session.open_cursor('table:' + tablename, None, None) + for i in range(0, self.nentries): + cursor.set_key(i) + cursor.set_value(str(i)) + cursor.insert() + self.pr('populate: ' + tablename + ': added ' + str(self.nentries)) + cursor.close() + + def checkContents(self, tablename): + cursor = self.session.open_cursor('table:' + tablename, None, None) + want = 0 + for key,val in cursor: + self.assertEqual(key, want) + self.assertEqual(val, str(want)) + want += 1 + self.assertEqual(want, self.nentries) + cursor.close() + + def checkDoesNotExist(self, t): + self.assertFalse(os.path.exists(t + ".wt")) + self.assertRaises(WiredTigerError, lambda: + self.session.open_cursor('table:' + t, + None, None)) + + def setupCursor(self, tablename, value): + if value == None: + return None + cursor = self.session.open_cursor('table:' + tablename, None, None) + if value >= 0 and value < self.nentries: + cursor.set_key(value) + self.assertEqual(cursor.search(), 0) + return cursor + + def truncateRangeAndCheck(self, tablename, begin, end): + self.pr('truncateRangeAndCheck: ' + str(begin) + ',' + str(end)) + self.populate(tablename) + cur1 = self.setupCursor(tablename, begin) + cur2 = self.setupCursor(tablename, end) + beginerror = (begin != None and begin < 0) + if not cur1 and not cur2: + self.session.truncate('table:' + tablename, None, None, None) + elif beginerror: + self.assertRaises(WiredTigerError, lambda: + self.session.truncate(None, cur1, cur2, None)) + else: + self.session.truncate(None, cur1, cur2, None) + if cur1: + cur1.close() + if cur2: + cur2.close() + if begin is None: + begin = 0 + if end is None: + end = self.nentries - 1 + + cursor = self.session.open_cursor('table:' + tablename, None, None) + self.pr('truncate(' + str(begin) + ' through ' + str(end) + ') => ' + \ + str([i for i in cursor])) + cursor.close() + + cursor = self.session.open_cursor('table:' + tablename, None, None) + want = 0 + # skip over numbers truncated + if (not beginerror) and want >= begin and want <= end: + want = end + 1 + for key,val in cursor: + self.assertEqual(key, want) + self.assertEqual(val, str(want)) + want += 1 + # skip over numbers truncated + if want >= begin and want <= end: + want = end + 1 + self.assertEqual(want, self.nentries) + cursor.close() + + def checkEmpty(self, t): + cursor = self.session.open_cursor('table:' + t, None, None) + got = 0 + for key,val in cursor: + got += 1 + self.assertEqual(got, 0) + cursor.close() + + def test_nop(self): + """ Make sure our test functions work """ + self.populate(self.table_name1) + self.checkContents(self.table_name1) + self.checkDoesNotExist(self.table_name2) + + def test_rename(self): + self.populate(self.table_name1) + self.session.rename('table:' + self.table_name1, + 'table:' + self.table_name2, None) + self.checkContents(self.table_name2) + self.checkDoesNotExist(self.table_name1) + self.session.rename('table:' + self.table_name2, + 'table:' + self.table_name1, None) + self.checkContents(self.table_name1) + self.checkDoesNotExist(self.table_name2) + self.assertRaises(WiredTigerError, lambda: + self.session.rename('table:' + self.table_name2, + 'table:' + self.table_name1, None)) + + def test_drop(self): + self.populate(self.table_name1) + self.session.drop('table:' + self.table_name1, None) + self.checkDoesNotExist(self.table_name1) + self.session.drop('table:' + self.table_name1, 'force') + self.assertRaises(WiredTigerError, lambda: + self.session.drop('table:' + self.table_name1, None)) + + def test_truncate(self): + self.populate(self.table_name1) + + self.session.truncate('table:' + self.table_name1, None, None, None) + self.checkEmpty(self.table_name1) + self.session.drop('table:' + self.table_name1, None) + + def test_truncate_cursor(self): + # Test using cursors for the begin and end of truncate + # For begin and end, the following applies: + # None means pass None for the cursor arg to truncate + # An integer N, with 0 <= N < self.nentries, passes + # a cursor positioned at that element. 0 positions + # a cursor before the first element, and self.nentries + # positions a cursor after the last element. + # + # We assume that the begin cursor must be positioned over + # a value. If not, we assume it raises an error. + for begin in [None, 0, 10]: + for end in [None, self.nentries - 1, self.nentries - 10]: + self.truncateRangeAndCheck(self.table_name1, begin, end) + self.session.drop('table:' + self.table_name1, None) + self.truncateRangeAndCheck(self.table_name1, 10, 10) + self.session.drop('table:' + self.table_name1, None) + + def test_sync(self): + self.populate(self.table_name1) + origmtime = os.path.getmtime(self.table_name1 + ".wt") + time.sleep(2) + self.session.sync('table:' + self.table_name1, None) + newmtime = os.path.getmtime(self.table_name1 + ".wt") + self.assertGreater(newmtime, origmtime) + self.checkContents(self.table_name1) + + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_compress01.py b/test/suite/test_compress01.py new file mode 100644 index 00000000000..34e5e0dff76 --- /dev/null +++ b/test/suite/test_compress01.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test001.py +# Basic operations +# + +import unittest +import wiredtiger +import wttest +import os +import sys + +class test_compress01_base(wttest.WiredTigerTestCase): + """ + Test basic compression + """ + nrecords = 10000 + + # running tests from the base class uses no compressor, + # a reasonable way to test the test case. + def __init__(self, testname, compressor_name=None, abbrev='none'): + wttest.WiredTigerTestCase.__init__(self, testname) + self.compressor_name = compressor_name + self.table_name1 = 'test_compress01' + abbrev + '.wt' + # bigvalue = '1234567891011121314....' + self.bigvalue = ''.join([`num` for num in xrange(1,10000)]) # about 38K chars long + + def create_table(self, tablename): + extra_params = ',internal_page_max=16384,leaf_page_max=131072' + comp_params = '' + if self.compressor_name != None: + comp_params = ',block_compressor=' + self.compressor_name + params = 'key_format=S,value_format=S' + extra_params + comp_params + self.pr('create_table: ' + tablename + ', params: ' + params) + self.session.create('file:' + tablename, params) + + def cursor_s(self, tablename, key): + cursor = self.session.open_cursor('file:' + tablename, None) + cursor.set_key(key) + return cursor + + def cursor_ss(self, tablename, key, val): + cursor = self.cursor_s(tablename, key) + cursor.set_value(val) + return cursor + + def record_count(self): + return self.nrecords + + def do_insert(self): + """ + Create a table, add keys with big values, get them back + """ + self.create_table(self.table_name1) + + self.pr("inserting `len(self.bigvalue)` byte values") + for idx in xrange(1,self.record_count()): + val = `idx` + self.bigvalue + `idx` + inscursor = self.cursor_ss(self.table_name1, 'key' + `idx`, val) + inscursor.insert() + inscursor.close() + + def do_verify(self): + self.pr('search') + for idx in xrange(1,self.record_count()): + val = `idx` + self.bigvalue + `idx` + getcursor = self.cursor_s(self.table_name1, 'key' + `idx`) + ret = getcursor.search() + self.assertTrue(ret == 0) + self.assertEquals(getcursor.get_value(), val) + getcursor.close() + + def do_fresh_cache(self): + # Since we are running WT in-process, we just need + # to shut down the connection and start again. + self.conn.close() + self.conn = self.setUpConnectionOpen(".") + self.session = self.setUpSessionOpen(self.conn) + + def test_insert_and_verify(self): + self.do_insert() + # We want a fresh cache so that compressed pages + # are really read from disk. + self.do_fresh_cache() + self.do_verify() + + def extensionArg(self, name): + if name != None: + testdir = os.path.dirname(__file__) + import run + extdir = os.path.join(run.wt_builddir, 'ext/compressors') + extfile = os.path.join(extdir, name, '.libs', name + '.so') + if not os.path.exists(extfile): + self.skipTest('Extension "' + extfile + '" not built') + return 'extensions=["' + extfile + '"]' + else: + return '' + + # override WiredTigerTestCase + def setUpConnectionOpen(self, dir): + return self.setUpConnectionWithExtension(dir, self.compressor_name) + + def setUpConnectionWithExtension(self, dir, name): + conn = wiredtiger.wiredtiger_open(dir, 'create,' + self.extensionArg(name)) + self.pr(`conn`) + return conn + + +class test_compress01_1_nop(test_compress01_base): + def __init__(self, testname): + test_compress01_base.__init__(self, testname, 'nop_compress', 'nop') + + +class test_compress01_2_bz(test_compress01_base): + def __init__(self, testname): + test_compress01_base.__init__(self, testname, 'bzip2_compress', 'bz') + +class test_compress01_3_sn(test_compress01_base): + def __init__(self, testname): + test_compress01_base.__init__(self, testname, 'snappy_compress', 'sn') + + +if __name__ == '__main__': + wttest.run(test_compress01_base) + wttest.run(test_compress01_1_nop) + wttest.run(test_compress01_2_bz) diff --git a/test/suite/test_config01.py b/test/suite/test_config01.py new file mode 100644 index 00000000000..4c68635bfad --- /dev/null +++ b/test/suite/test_config01.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_config01.py +# Configuration strings for wiredtiger_open +# + +import unittest +import wiredtiger +import wttest +import test_base03 +import wtscenario + +class test_config01(test_base03.test_base03): + scenarios = wtscenario.wtscenario.session_create_scenario() + + def config_string(self): + return self.session_create_scenario.configString() + + def setUpConnectionOpen(self, dir): + wtopen_args = 'create' + if hasattr(self, 'cache_size'): + wtopen_args += ',cache_size=' + str(self.cache_size) + conn = wiredtiger.wiredtiger_open(dir, wtopen_args) + self.pr(`conn`) + return conn + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_config02.py b/test/suite/test_config02.py new file mode 100644 index 00000000000..728adb9c85a --- /dev/null +++ b/test/suite/test_config02.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_config01.py +# The home directory for wiredtiger_open +# + +import unittest +import wiredtiger +from wiredtiger import WiredTigerError +import wttest +import os + +class test_config02(wttest.WiredTigerTestCase): + table_name1 = 'test_config02' + nentries = 100 + + # Each test needs to set up its connection in its own way, + # so override these methods to do nothing + def setUpConnectionOpen(self, dir): + return None + + def setUpSessionOpen(self, conn): + return None + + def populate_and_check(self): + """ + Create entries, and read back in a cursor: key=string, value=string + """ + create_args = 'key_format=S,value_format=S' + self.session.create("table:" + self.table_name1, create_args) + cursor = self.session.open_cursor('table:' + self.table_name1, None, None) + for i in range(0, self.nentries): + cursor.set_key(str(1000000 + i)) + cursor.set_value('value' + str(i)) + cursor.insert() + i = 0 + cursor.reset() + for key, value in cursor: + self.assertEqual(key, str(1000000 + i)) + self.assertEqual(value, ('value' + str(i))) + i += 1 + self.assertEqual(i, self.nentries) + cursor.close() + + def checkfiles(self, dirname): + self.assertTrue(os.path.exists(dirname + os.sep + self.table_name1 + ".wt")) + + def checknofiles(self, dirname): + self.assertEqual(len(os.listdir(dirname)), 0) + + def common_test(self, homearg, homeenv, configextra): + """ + Call wiredtiger_open and run a simple test. + homearg is the first arg to wiredtiger_open, it may be null. + WIREDTIGER_HOME is set to homeenv, if it is not null. + configextra are any extra configuration strings needed on the open. + """ + configarg = 'create' + if configextra != None: + configarg += ',' + configextra + if homeenv == None: + os.unsetenv('WIREDTIGER_HOME') + else: + os.putenv('WIREDTIGER_HOME', homeenv) + self.conn = wiredtiger.wiredtiger_open(homearg, configarg) + self.session = self.conn.open_session(None) + self.populate_and_check() + + def test_home_nohome(self): + self.common_test(None, None, None) + self.checkfiles(".") + + def test_home_rel(self): + dir = 'subdir' + os.mkdir(dir) + self.common_test(dir, None, None) + self.checkfiles(dir) + + def test_home_abs(self): + dir = os.path.realpath('.') + os.sep + 'subdir' + os.mkdir(dir) + self.common_test(dir, None, None) + self.checkfiles(dir) + + def test_home_and_env(self): + hdir = 'homedir' + edir = 'envdir' + os.mkdir(hdir) + os.mkdir(edir) + self.common_test(hdir, edir, None) + self.checkfiles(hdir) + self.checknofiles(edir) + + def test_home_and_env_conf(self): + # If homedir is set, the environment is ignored + hdir = 'homedir' + edir = 'envdir' + os.mkdir(hdir) + os.mkdir(edir) + self.common_test(hdir, edir, 'home_environment=true') + self.checkfiles(hdir) + self.checknofiles(edir) + + def test_home_and_missing_env(self): + # If homedir is set, it is used no matter what + hdir = 'homedir' + os.mkdir(hdir) + self.common_test(hdir, None, 'home_environment=true') + self.checkfiles(hdir) + + def test_env_conf(self): + edir = 'envdir' + os.mkdir(edir) + self.common_test(None, edir, 'home_environment=true') + self.checkfiles(edir) + + def test_env_conf_without_env_var(self): + # no env var set, so should use current directory + self.common_test(None, None, 'home_environment=true') + self.checkfiles(".") + + def test_env_no_conf(self): + # env var, but no open configuration string, should fail + edir = 'envdir' + os.mkdir(edir) + self.assertRaises(WiredTigerError, + lambda: self.common_test(None, edir, None)) + + def test_home_does_not_exist(self): + dir = 'nondir' + self.assertRaises(WiredTigerError, + lambda: wiredtiger.wiredtiger_open(dir, 'create')) + + def test_home_not_writeable(self): + dir = 'subdir' + os.mkdir(dir) + os.chmod(dir, 0555) + self.assertRaises(WiredTigerError, + lambda: wiredtiger.wiredtiger_open(dir, 'create')) + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_config03.py b/test/suite/test_config03.py new file mode 100644 index 00000000000..91fced07edd --- /dev/null +++ b/test/suite/test_config03.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_config03.py +# More configuration strings for wiredtiger_open, +# combined probabilistically. +# + +import unittest +from wiredtiger import WiredTigerError +import wiredtiger +import wttest +import test_base03 +import wtscenario + +class test_config03(test_base03.test_base03): + K = 1024 + M = 1024 * K + G = 1024 * M + + cache_size_scenarios = wtscenario.quick_scenarios('s_cache_size', + [1*M,20*M,100*M,1*G,None], [0.6,0.6,0.6,0.6,0.6]) + create_scenarios = wtscenario.quick_scenarios('s_create', + [True,False,None], [1.0,0.2,0.3]) + error_prefix_scenarios = wtscenario.quick_scenarios('s_error_prefix', + [None,"errpfx:"], [1.0,0.2]) + # eviction_target < eviction_trigger -- checked later + eviction_target_scenarios = wtscenario.quick_scenarios('s_eviction_target', + [10, 40, 85, 98], None) + eviction_trigger_scenarios = wtscenario.quick_scenarios( + 's_eviction_trigger', + [50, 90, 95, 99], None) + hazard_max_scenarios = wtscenario.quick_scenarios('s_hazard_max', + [15, 50, 500], [0.4, 0.8, 0.8]) + logging_scenarios = wtscenario.quick_scenarios('s_logging', + [True,False], [1.0,1.0]) + multiprocess_scenarios = wtscenario.quick_scenarios('s_multiprocess', + [True,False], [1.0,1.0]) + session_max_scenarios = wtscenario.quick_scenarios('s_session_max', + [3, 30, 300], None) + transactional_scenarios = wtscenario.quick_scenarios('s_transactional', + [True,False], [0.2,1.0]) + + # Note: we are not using any truly verbose scenarios until we have + # a way to redirect verbose output to a file in Python. + # + #verbose_scenarios = wtscenario.quick_scenarios('s_verbose', + # ['block', 'evict,evictserver', 'fileops,hazard,mutex', + # 'read,readserver,reconcile,salvage','verify,write',''], None) + verbose_scenarios = wtscenario.quick_scenarios('s_verbose', [None], None) + + config_vars = [ 'cache_size', 'create', 'error_prefix', 'eviction_target', + 'eviction_trigger', 'hazard_max', 'logging', + 'multiprocess', 'session_max', 'transactional', 'verbose' ] + + all_scenarios = wtscenario.multiply_scenarios('_', + cache_size_scenarios, create_scenarios, error_prefix_scenarios, + eviction_target_scenarios, eviction_trigger_scenarios, + hazard_max_scenarios, logging_scenarios, + multiprocess_scenarios, session_max_scenarios, + transactional_scenarios, verbose_scenarios) + + scenarios = wtscenario.prune_scenarios(all_scenarios, 1000) + scenarios = wtscenario.number_scenarios(scenarios) + + wttest.WiredTigerTestCase.printVerbose(2, 'test_config03: running ' + \ + str(len(scenarios)) + ' of ' + \ + str(len(all_scenarios)) + ' possible scenarios') + + def setUpConnectionOpen(self, dir): + args = '' + # add names to args, e.g. args += ',session_max=30' + for var in self.config_vars: + value = getattr(self, 's_' + var) + if value != None: + if var == 'verbose': + value = '[' + str(value) + ']' + if value == True: + value = 'true' + if value == False: + value = 'false' + args += ',' + var + '=' + str(value) + args += ',' + self.pr('wiredtiger_open with args: ' + args) + + expect_fail = False + successargs = args + if self.s_create == False: + successargs = successargs.replace(',create=false,',',create,') + expect_fail = True + elif self.s_create == None: + successargs = successargs + 'create=true,' + expect_fail = True + if self.s_eviction_target >= self.s_eviction_trigger: + # construct args that guarantee that target < trigger + # we know that trigger >= 1 + repfrom = ',eviction_target=' + str(self.s_eviction_target) + repto = ',eviction_target=' + str(self.s_eviction_trigger - 1) + successargs = successargs.replace(repfrom, repto); + expect_fail = True + + if expect_fail: + self.verbose(3, 'wiredtiger_open (should fail) with args: ' + args) + self.assertRaises(WiredTigerError, lambda: + wiredtiger.wiredtiger_open(dir, args)) + args = successargs + + self.verbose(3, 'wiredtiger_open with args: ' + args) + conn = wiredtiger.wiredtiger_open(dir, args) + self.pr(`conn`) + return conn + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_config04.py b/test/suite/test_config04.py new file mode 100644 index 00000000000..d22d7cd75cf --- /dev/null +++ b/test/suite/test_config04.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_config04.py +# Individually test config options +# + +import unittest +import wiredtiger +from wiredtiger import WiredTigerError +import wttest +import os + +class test_config04(wttest.WiredTigerTestCase): + table_name1 = 'test_config04' + nentries = 100 + + K = 1024 + M = K * K + G = K * M + T = K * G + + # Each test needs to set up its connection in its own way, + # so override these methods to do nothing + def setUpConnectionOpen(self, dir): + return None + + def setUpSessionOpen(self, conn): + return None + + def populate_and_check(self): + """ + Create entries, and read back in a cursor: key=string, value=string + """ + create_args = 'key_format=S,value_format=S' + self.session.create("table:" + self.table_name1, create_args) + cursor = self.session.open_cursor('table:' + self.table_name1, None, None) + for i in range(0, self.nentries): + cursor.set_key(str(1000000 + i)) + cursor.set_value('value' + str(i)) + cursor.insert() + i = 0 + cursor.reset() + for key, value in cursor: + self.assertEqual(key, str(1000000 + i)) + self.assertEqual(value, ('value' + str(i))) + i += 1 + self.assertEqual(i, self.nentries) + cursor.close() + + def common_test(self, configextra): + """ + Call wiredtiger_open and run a simple test. + configextra are any extra configuration strings needed on the open. + """ + configarg = 'create' + if configextra != None: + configarg += ',' + configextra + self.conn = wiredtiger.wiredtiger_open('.', configarg) + self.session = self.conn.open_session(None) + self.populate_and_check() + + def common_cache_size_test(self, sizestr, size): + self.common_test('cache_size=' + sizestr) + cursor = self.session.open_cursor('statistics:', None, None) + cursor.set_key(wiredtiger.stat.cache_bytes_max) + self.assertEqual(cursor.search(), 0) + got_cache = cursor.get_values()[2] + self.assertEqual(got_cache, size) + + def test_bad_config(self): + self.assertRaises(WiredTigerError, lambda: + wiredtiger.wiredtiger_open('.', 'not_valid,another_bad=10')) + + def test_cache_size_number(self): + # Use a number without multipliers + # 1M is the minimum, we'll ask for 1025 * 1024 + cache_size_str = str(1025 * 1024) + self.common_cache_size_test(cache_size_str, 1025*self.K) + + def test_cache_size_K(self): + # Kilobyte sizing test + # 1M is the minimum, so ask for that using K notation. + self.common_cache_size_test('1024K', 1024*self.K) + + def test_cache_size_M(self): + # Megabyte sizing test + self.common_cache_size_test('30M', 30*self.M) + + def test_cache_size_G(self): + # Gigabyte sizing test + # We are specifying the maximum the cache can grow, + # not the initial cache amount, so small tests like + # this can still run on smaller machines. + self.common_cache_size_test('7G', 7*self.G) + + def test_cache_size_T(self): + # Terabyte sizing test + # We are specifying the maximum the cache can grow, + # not the initial cache amount, so small tests like + # this can still run on smaller machines. + self.common_cache_size_test('2T', 2*self.T) + + def test_cache_too_small(self): + self.assertRaises(WiredTigerError, lambda: + wiredtiger.wiredtiger_open('.', 'create,cache_size=900000')) + + def test_cache_too_large(self): + T11 = 11 * self.T # 11 Terabytes + self.assertRaises(WiredTigerError, lambda: + wiredtiger.wiredtiger_open('.', 'create,cache_size=' + str(T11))) + + def test_eviction(self): + self.common_test('eviction_target=84,eviction_trigger=94') + # Note + + def test_eviction_bad(self): + self.assertRaises(WiredTigerError, lambda: + wiredtiger.wiredtiger_open('.', 'create,eviction_target=91,' + + 'eviction_trigger=81')) + + def test_eviction_bad2(self): + self.assertRaises(WiredTigerError, lambda: + wiredtiger.wiredtiger_open('.', 'create,eviction_target=86,' + + 'eviction_trigger=86')) + + def test_hazard_max(self): + # Note: There isn't any direct way to know that this was set. + self.common_test('hazard_max=50') + + def test_session_max(self): + # Note: There isn't any direct way to know that this was set, + # but we'll have a separate functionality test to test for + # this indirectly. + self.common_test('session_max=99') + + def test_multiprocess(self): + self.common_test('multiprocess') + # TODO: how do we verify that it was set? + + def test_error_prefix(self): + self.common_test('error_prefix="MyOwnPrefix"') + # TODO: how do we verify that it was set? + + def test_logging(self): + # Note: this will have functional tests in the future. + self.common_test('logging') + + def test_transactional(self): + # Note: this will have functional tests in the future. + self.common_test('transactional') + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_config05.py b/test/suite/test_config05.py new file mode 100644 index 00000000000..a34fcb605eb --- /dev/null +++ b/test/suite/test_config05.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_config05.py +# Test multiple connection opens +# + +import unittest +import wiredtiger +from wiredtiger import WiredTigerError +import wttest +import os + +class test_config05(wttest.WiredTigerTestCase): + table_name1 = 'test_config05' + nentries = 100 + + # Each test needs to set up its connection in its own way, + # so override these methods to do nothing + def setUpConnectionOpen(self, dir): + return None + + def setUpSessionOpen(self, conn): + return None + + def close_conn(self): + if self.conn != None: + self.conn.close() + self.conn = None + if hasattr(self, 'conn2') and self.conn2 != None: + self.conn2.close() + self.conn2 = None + + def populate(self, session): + """ + Create entries using key=string, value=string + """ + create_args = 'key_format=S,value_format=S' + session.create("table:" + self.table_name1, create_args) + cursor = session.open_cursor('table:' + self.table_name1, None, None) + for i in range(0, self.nentries): + cursor.set_key(str(1000000 + i)) + cursor.set_value('value' + str(i)) + cursor.insert() + cursor.close() + + def verify_entries(self, session): + """ + Verify all entries created in populate() + """ + cursor = session.open_cursor('table:' + self.table_name1, None, None) + i = 0 + for key, value in cursor: + self.assertEqual(key, str(1000000 + i)) + self.assertEqual(value, ('value' + str(i))) + i += 1 + self.assertEqual(i, self.nentries) + cursor.close() + + def test_one(self): + self.conn = wiredtiger.wiredtiger_open('.', 'create') + self.session = self.conn.open_session(None) + self.populate(self.session) + self.verify_entries(self.session) + + def test_multi_create(self): + self.conn = wiredtiger.wiredtiger_open('.', 'create') + self.session = self.conn.open_session(None) + self.assertRaises(WiredTigerError, lambda: wiredtiger.wiredtiger_open + ('.', 'create')) + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_cursor01.py b/test/suite/test_cursor01.py new file mode 100644 index 00000000000..ccb62750e25 --- /dev/null +++ b/test/suite/test_cursor01.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_cursor01.py +# Cursor operations +# + +import unittest +import wiredtiger +from wiredtiger import WiredTigerError +import wttest + +class test_cursor01(wttest.WiredTigerTestCase): + """ + Test basic operations + """ + table_name1 = 'test_cursor01' + nentries = 10 + + scenarios = [ + ('row', dict(tablekind='row')), + ('col', dict(tablekind='col')), + ('fix', dict(tablekind='fix')) + ] + + def config_string(self): + """ + Return any additional configuration. + This method may be overridden. + """ + return '' + + def session_create(self, name, args): + """ + session.create, but report errors more completely + """ + try: + self.session.create(name, args) + except: + print('**** ERROR in session.create("' + name + '","' + args + '") ***** ') + raise + + def create_session_and_cursor(self): + tablearg = "table:" + self.table_name1 + if self.tablekind == 'row': + keyformat = 'key_format=S' + else: + keyformat = 'key_format=r' # record format + if self.tablekind == 'fix': + valformat = 'value_format=8t' + else: + valformat = 'value_format=S' + create_args = keyformat + ',' + valformat + self.config_string() + print('creating session: ' + create_args) + self.session_create(tablearg, create_args) + self.pr('creating cursor') + return self.session.open_cursor(tablearg, None, None) + + def genkey(self, i): + if self.tablekind == 'row': + return 'key' + str(i) + else: + return long(i+1) + + def genvalue(self, i): + if self.tablekind == 'fix': + return int(i & 0xff) + else: + return 'value' + str(i) + + def assertCursorHasNoKeyValue(self, cursor): + print('Expect to see messages: \'requires key/value to be set\'') + self.assertRaises(WiredTigerError, cursor.get_key) + self.assertRaises(WiredTigerError, cursor.get_value) + + def test_forward_iter(self): + """ + Create entries, and read back in a cursor: key=string, value=string + """ + cursor = self.create_session_and_cursor() + self.assertCursorHasNoKeyValue(cursor) + + for i in range(0, self.nentries): + cursor.set_key(self.genkey(i)) + cursor.set_value(self.genvalue(i)) + cursor.insert() + + # Don't use the builtin 'for ... in cursor', + # iterate using the basic API. + + # 1. Start with the first k/v pair. + cursor.reset() + i = 0 + while True: + nextret = cursor.next() + if nextret != 0: + break + key = cursor.get_key() + value = cursor.get_value() + #print('want: ' + str(self.genkey(i)) + ' got: ' + str(key)) + self.assertEqual(key, self.genkey(i)) + self.assertEqual(value, self.genvalue(i)) + i += 1 + + self.assertEqual(nextret, wiredtiger.WT_NOTFOUND) + self.assertEqual(i, self.nentries) + + # After an error, we can no longer access the key or value + self.assertCursorHasNoKeyValue(cursor) + + # 2. Setting reset() should place us just before first pair. + cursor.reset() + self.assertCursorHasNoKeyValue(cursor) + + nextret = cursor.next() + i = 0 + while nextret == 0: + key = cursor.get_key() + value = cursor.get_value() + self.assertEqual(key, self.genkey(i)) + self.assertEqual(value, self.genvalue(i)) + i += 1 + nextret = cursor.next() + + self.assertEqual(nextret, wiredtiger.WT_NOTFOUND) + self.assertEqual(i, self.nentries) + cursor.close() + + def test_backward_iter(self): + """ + Create entries, and read back in a cursor: key=string, value=string + """ + cursor = self.create_session_and_cursor() + self.assertCursorHasNoKeyValue(cursor) + + for i in range(0, self.nentries): + cursor.set_key(self.genkey(i)) + cursor.set_value(self.genvalue(i)) + cursor.insert() + + # Don't use the builtin 'for ... in cursor', + # iterate using the basic API. + + # 1. Start with the last k/v pair. + cursor.reset() + i = self.nentries - 1 + while True: + prevret = cursor.prev() + if prevret != 0: + break + key = cursor.get_key() + value = cursor.get_value() + self.assertEqual(key, self.genkey(i)) + self.assertEqual(value, self.genvalue(i)) + i -= 1 + + self.assertEqual(prevret, wiredtiger.WT_NOTFOUND) + self.assertEqual(i, -1) + + # After an error, we can no longer access the key or value + self.assertCursorHasNoKeyValue(cursor) + + # 2. Setting reset() should place us just after last pair. + cursor.reset() + self.assertCursorHasNoKeyValue(cursor) + + prevret = cursor.prev() + i = self.nentries - 1 + while prevret == 0: + key = cursor.get_key() + value = cursor.get_value() + self.assertEqual(key, self.genkey(i)) + self.assertEqual(value, self.genvalue(i)) + i -= 1 + prevret = cursor.prev() + + self.assertEqual(prevret, wiredtiger.WT_NOTFOUND) + self.assertEqual(i, -1) + cursor.close() + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_cursor02.py b/test/suite/test_cursor02.py new file mode 100644 index 00000000000..faeda169690 --- /dev/null +++ b/test/suite/test_cursor02.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_cursor02.py +# Cursor operations on small tables. +# + +import unittest +import wiredtiger +from test_cursor_tracker import TestCursorTracker + +class test_cursor02(TestCursorTracker): + """ + Cursor operations on small tables of each access method. + We use the TestCursorTracker base class to generate + key/value content and to track/verify content + after inserts and removes. + """ + scenarios = [ + ('row', dict(tablekind='row')), + ('col', dict(tablekind='col')), + #('fix', dict(tablekind='fix')) + ] + + def create_session_and_cursor(self, ninitialentries): + tablearg = "table:" + self.table_name1 + if self.tablekind == 'row': + keyformat = 'key_format=S' + else: + keyformat = 'key_format=r' # record format + if self.tablekind == 'fix': + valformat = 'value_format=8t' + else: + valformat = 'value_format=S' + create_args = keyformat + ',' + valformat + self.config_string() + self.session_create(tablearg, create_args) + self.pr('creating cursor') + self.cur_initial_conditions(self.table_name1, ninitialentries, self.tablekind, None, None) + return self.session.open_cursor(tablearg, None, 'append') + + def test_multiple_remove(self): + """ + Test multiple deletes at the same place + """ + cursor = self.create_session_and_cursor(10) + self.cur_first(cursor) + self.cur_check_forward(cursor, 5) + + self.cur_remove_here(cursor) + self.cur_next(cursor) + self.cur_check_here(cursor) + + self.cur_remove_here(cursor) + self.cur_next(cursor) + self.cur_check_here(cursor) + self.cur_check_forward(cursor, 2) + + #self.cur_dump_here(cursor, 'after second next: ') + cursor.close() + + def test_insert_and_remove(self): + """ + Test a variety of insert, deletes and iteration + """ + cursor = self.create_session_and_cursor(10) + #self.table_dump(self.table_name1) + self.cur_insert(cursor, 2, 5) + self.cur_insert(cursor, 2, 6) + self.cur_insert(cursor, 2, 4) + self.cur_insert(cursor, 2, 7) + self.cur_first(cursor) + self.cur_check_forward(cursor, 5) + self.cur_remove_here(cursor) + self.cur_check_forward(cursor, 5) + self.cur_remove_here(cursor) + self.cur_check_forward(cursor, 1) + self.cur_remove_here(cursor) + self.cur_check_forward(cursor, 2) + self.cur_check_backward(cursor, 4) + self.cur_remove_here(cursor) + self.cur_check_backward(cursor, 2) + self.cur_check_forward(cursor, 5) + self.cur_check_backward(cursor, -1) + self.cur_check_forward(cursor, -1) + self.cur_last(cursor) + self.cur_check_backward(cursor, -1) + cursor.close() + + def test_iterate_empty(self): + """ + Test iterating an empty table + """ + cursor = self.create_session_and_cursor(0) + self.cur_first(cursor, wiredtiger.WT_NOTFOUND) + self.cur_check_forward(cursor, -1) + self.cur_check_backward(cursor, -1) + self.cur_last(cursor, wiredtiger.WT_NOTFOUND) + self.cur_check_backward(cursor, -1) + self.cur_check_forward(cursor, -1) + cursor.close() + + def test_iterate_one_preexisting(self): + """ + Test iterating a table that contains one preexisting entry + """ + cursor = self.create_session_and_cursor(1) + self.cur_first(cursor) + self.cur_check_forward(cursor, -1) + self.cur_check_backward(cursor, -1) + self.cur_last(cursor) + self.cur_check_backward(cursor, -1) + self.cur_check_forward(cursor, -1) + cursor.close() + + def test_iterate_one_added(self): + """ + Test iterating a table that contains one newly added entry + """ + cursor = self.create_session_and_cursor(0) + self.cur_insert(cursor, 1, 0) + self.cur_check_forward(cursor, -1) + self.cur_first(cursor) + self.cur_check_forward(cursor, -1) + self.cur_check_backward(cursor, -1) + self.cur_last(cursor) + self.cur_check_backward(cursor, -1) + self.cur_check_forward(cursor, -1) + cursor.close() + + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_cursor03.py b/test/suite/test_cursor03.py new file mode 100644 index 00000000000..c956a26f8d9 --- /dev/null +++ b/test/suite/test_cursor03.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_cursor02.py +# Cursor operations on tables of various sizes, +# with key/values of various sizes. +# + +import unittest +import wiredtiger +from test_cursor_tracker import TestCursorTracker +from wtscenario import multiply_scenarios + +class test_cursor03(TestCursorTracker): + """ + Cursor operations on small tables of each access method. + We use the TestCursorTracker base class to generate + key/value content and to track/verify content + after inserts and removes. + """ + scenarios = multiply_scenarios('.', [ + ('row', dict(tablekind='row', keysize=None, valsize=None)), + ('col', dict(tablekind='col', keysize=None, valsize=None)), + #('fix', dict(tablekind='fix', keysize=None, valsize=None)) + ('row.val10k', dict(tablekind='row', keysize=None, valsize=[10, 10000])), + ('col.val10k', dict(tablekind='col', keysize=None, valsize=[10, 10000])), + ('row.keyval10k', dict(tablekind='row', keysize=[10,10000], valsize=[10, 10000])), + ], [ + ('count1000', dict(tablecount=1000,cache_size=20*1024*1024)), + ('count10000', dict(tablecount=10000, cache_size=64*1024*1024)) + ]) + + def create_session_and_cursor(self): + tablearg = "table:" + self.table_name1 + if self.tablekind == 'row': + keyformat = 'key_format=S' + else: + keyformat = 'key_format=r' # record format + if self.tablekind == 'fix': + valformat = 'value_format=8t' + else: + valformat = 'value_format=S' + create_args = keyformat + ',' + valformat + self.config_string() + self.session_create(tablearg, create_args) + self.pr('creating cursor') + self.cur_initial_conditions(self.table_name1, self.tablecount, self.tablekind, self.keysize, self.valsize) + return self.session.open_cursor(tablearg, None, 'append') + + def setUpConnectionOpen(self, dir): + wtopen_args = 'create,cache_size=' + str(self.cache_size); + conn = wiredtiger.wiredtiger_open(dir, wtopen_args) + self.pr(`conn`) + return conn + + def test_multiple_remove(self): + """ + Test multiple deletes at the same place + """ + cursor = self.create_session_and_cursor() + self.cur_first(cursor) + self.cur_check_forward(cursor, 5) + + self.cur_remove_here(cursor) + self.cur_next(cursor) + self.cur_check_here(cursor) + + self.cur_remove_here(cursor) + self.cur_next(cursor) + self.cur_check_here(cursor) + self.cur_check_forward(cursor, 2) + + cursor.close() + + def test_insert_and_remove(self): + """ + Test a variety of insert, deletes and iteration + """ + cursor = self.create_session_and_cursor() + #self.table_dump(self.table_name1) + self.cur_insert(cursor, 2, 5) + self.cur_insert(cursor, 2, 6) + self.cur_insert(cursor, 2, 4) + self.cur_insert(cursor, 2, 7) + self.cur_first(cursor) + self.cur_check_forward(cursor, 5) + self.cur_remove_here(cursor) + self.cur_check_forward(cursor, 5) + self.cur_remove_here(cursor) + self.cur_check_forward(cursor, 1) + self.cur_remove_here(cursor) + self.cur_check_forward(cursor, 2) + self.cur_check_backward(cursor, 4) + self.cur_remove_here(cursor) + self.cur_check_backward(cursor, 2) + self.cur_check_forward(cursor, 5) + self.cur_check_backward(cursor, -1) + self.cur_check_forward(cursor, -1) + self.cur_last(cursor) + self.cur_check_backward(cursor, -1) + cursor.close() + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_cursor04.py b/test/suite/test_cursor04.py new file mode 100644 index 00000000000..4b82fa076c8 --- /dev/null +++ b/test/suite/test_cursor04.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_base03.py +# Cursor operations +# + +import unittest +import wiredtiger +from wiredtiger import WiredTigerError +import wttest + +class test_cursor04(wttest.WiredTigerTestCase): + """ + Test cursor search and search_near + """ + table_name1 = 'test_cursor04' + nentries = 20 + + scenarios = [ + ('row', dict(tablekind='row')), + ('col', dict(tablekind='col')), + ('fix', dict(tablekind='fix')) + ] + + def config_string(self): + """ + Return any additional configuration. + This method may be overridden. + """ + return '' + + def session_create(self, name, args): + """ + session.create, but report errors more completely + """ + try: + self.session.create(name, args) + except: + print('**** ERROR in session.create("' + name + '","' + args + '") ***** ') + raise + + def create_session_and_cursor(self): + tablearg = "table:" + self.table_name1 + if self.tablekind == 'row': + keyformat = 'key_format=S' + else: + keyformat = 'key_format=r' # record format + if self.tablekind == 'fix': + valformat = 'value_format=8t' + else: + valformat = 'value_format=S' + create_args = keyformat + ',' + valformat + self.config_string() + print('creating session: ' + create_args) + self.session_create(tablearg, create_args) + self.pr('creating cursor') + return self.session.open_cursor(tablearg, None, None) + + def genkey(self, i): + if self.tablekind == 'row': + return 'key' + str(i).zfill(5) # return key00001, key00002, etc. + else: + return long(i+1) + + def genvalue(self, i): + if self.tablekind == 'fix': + return int(i & 0xff) + else: + return 'value' + str(i) + + def expect_either(self, cursor, lt, gt): + origkey = cursor.get_key() + near_ret = cursor.search_near() + self.assertEqual(near_ret[0], 0) + direction = near_ret[1] + + # Deletions for 'fix' clear the value, they + # do not remove the key, so we expect '0' direction + # (that is key found) for fix. + if self.tablekind != 'fix': + self.assertTrue(direction == 1 or direction == -1) + else: + self.assertEqual(direction, 0) + + if direction == 1: + self.assertEqual(cursor.get_key(), self.genkey(gt)) + self.assertEqual(cursor.get_value(), self.genvalue(gt)) + elif direction == -1: + self.assertEqual(cursor.get_key(), self.genkey(lt)) + self.assertEqual(cursor.get_value(), self.genvalue(lt)) + else: + self.assertEqual(direction, 0) + self.assertEqual(cursor.get_key(), origkey) + self.assertEqual(cursor.get_value(), 0) + + def test_searches(self): + """ + Create entries, and read back in a cursor: key=string, value=string + """ + cursor = self.create_session_and_cursor() + + # Some tests below expect keys between 0-10 to be available + self.assertTrue(self.nentries > 10) + + # 0. Populate the key space + for i in range(0, self.nentries): + cursor.set_key(self.genkey(i)) + cursor.set_value(self.genvalue(i)) + cursor.insert() + + # 1. Calling search for a value that exists + cursor.set_key(self.genkey(5)) + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_key(), self.genkey(5)) + self.assertEqual(cursor.get_value(), self.genvalue(5)) + + # 2. Calling search for a value that does not exist + cursor.set_key(self.genkey(self.nentries)) + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + + # The key/value should be cleared on NOTFOUND + self.assertRaises(WiredTigerError, cursor.get_key) + self.assertRaises(WiredTigerError, cursor.get_value) + + # 2. Calling search_near for a value beyond the end + cursor.set_key(self.genkey(self.nentries)) + near_ret = cursor.search_near() + self.assertEqual(near_ret[0], 0) + self.assertEqual(near_ret[1], -1) + self.assertEqual(cursor.get_key(), self.genkey(self.nentries-1)) + self.assertEqual(cursor.get_value(), self.genvalue(self.nentries-1)) + + # 2.a calling search_near for an existing value + cursor.set_key(self.genkey(7)) + near_ret = cursor.search_near() + self.assertEqual(near_ret[0], 0) + self.assertEqual(near_ret[1], 0) + self.assertEqual(cursor.get_key(), self.genkey(7)) + self.assertEqual(cursor.get_value(), self.genvalue(7)) + + # 3. Delete some keys + # Deletions for 'fix' clear the value, they + # do not remove the key + cursor.set_key(self.genkey(0)) + cursor.remove() + cursor.set_key(self.genkey(5)) + cursor.remove() + cursor.set_key(self.genkey(9)) + cursor.remove() + cursor.set_key(self.genkey(10)) + cursor.remove() + + #cursor.reset() + #for key, value in cursor: + # print('key: ' + str(key)) + # print('value: ' + str(value)) + + cursor.set_key(self.genkey(0)) + near_ret = cursor.search_near() + self.assertEqual(near_ret[0], 0) + if self.tablekind != 'fix': + self.assertEqual(near_ret[1], 1) + self.assertEqual(cursor.get_key(), self.genkey(1)) + self.assertEqual(cursor.get_value(), self.genvalue(1)) + else: + self.assertEqual(near_ret[1], 0) + self.assertEqual(cursor.get_key(), self.genkey(0)) + self.assertEqual(cursor.get_value(), 0) + + cursor.set_key(self.genkey(5)) + self.expect_either(cursor, 4, 6) + + cursor.set_key(self.genkey(9)) + self.expect_either(cursor, 8, 11) + + cursor.set_key(self.genkey(10)) + self.expect_either(cursor, 8, 11) + + cursor.close() + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_cursor05.py b/test/suite/test_cursor05.py new file mode 100644 index 00000000000..2034e6ccd50 --- /dev/null +++ b/test/suite/test_cursor05.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_cursor05.py +# Test cursors at the point where a cursor is first +# initialized, and when it hits an endpoint. +# Mix that in with column groups. +# + +import unittest +import wiredtiger +from wiredtiger import WiredTigerError +import wttest + +class test_cursor05(wttest.WiredTigerTestCase): + """ + Test basic operations + """ + nentries = 2 + + def populate(self, count): + """ Populate the given number of entries. """ + cursor = self.session.open_cursor('table:main', None, None) + for i in range(0, count): + cursor.set_key(i, 'key' + str(i)) + cursor.set_value('val' + str(i), i, 'val' + str(i), i) + cursor.insert() + cursor.close() + + def check_iterate_forward(self, cursor, expectcount): + """ Use the cursor to iterate and check for the expected entries. """ + i = 0 + for ikey, skey, s1, i2, s3, i4 in cursor: + print 'forward: ' + str([ikey, skey, s1, i2, s3, i4]) + self.assertEqual(ikey, i) + self.assertEqual(skey, 'key' + str(i)) + self.assertEqual(s1, 'val' + str(i)) + self.assertEqual(i2, i) + self.assertEqual(s3, 'val' + str(i)) + self.assertEqual(i4, i) + i += 1 + self.assertEqual(i, expectcount) + + def check_iterate_backward(self, cursor, expectcount): + """ Iterate backwards and check for the expected entries. """ + i = expectcount + while cursor.prev() == 0: + i -= 1 + (ikey, skey) = cursor.get_keys() + (s1, i2, s3, i4) = cursor.get_values() + print 'backward: ' + str([ikey, skey, s1, i2, s3, i4]) + self.assertEqual(ikey, i) + self.assertEqual(skey, 'key' + str(i)) + self.assertEqual(s1, 'val' + str(i)) + self.assertEqual(i2, i) + self.assertEqual(s3, 'val' + str(i)) + self.assertEqual(i4, i) + self.assertEqual(i, 0) + + def check_iterate(self, cursor, expectcount, isforward): + """ + Use the cursor to iterate (forwards or backwards) + and check for the expected entries. + """ + if isforward: + self.check_iterate_forward(cursor, expectcount) + else: + self.check_iterate_backward(cursor, expectcount) + + def check_entries(self, testmode, expectcount, isforward): + """ + Use various modes to get the cursor to the 'uninitialized' state, + and verify that is correct by iterating and checking each element. + """ + cursor = self.session.open_cursor('table:main', None, None) + + # The cursor is uninitialized. Any of these sequences should + # leave the cursor uninitialized again - ready to iterate. + if testmode == 0: + pass + elif testmode == 1: + cursor.next() + cursor.prev() + elif testmode == 2: + cursor.prev() + cursor.next() + + # Verify that by iterating + self.check_iterate(cursor, expectcount, isforward) + + # Do something that leaves the cursor in an uninitialized spot + if expectcount > 0: + n = expectcount - 1 + cursor.set_key(n, 'key' + str(n)) + cursor.search() + (s1, i2, s3, i4) = cursor.get_values() + self.assertEqual(s1, 'val' + str(n)) + self.assertEqual(i2, n) + self.assertEqual(s3, 'val' + str(n)) + self.assertEqual(i4, n) + + # Any of these should leave the cursor again positioned at + # an uninitialized spot - ready to iterate + if testmode == 0: + cursor.reset() + elif testmode == 1: + cursor.reset() + cursor.next() + cursor.prev() + elif testmode == 2: + cursor.reset() + cursor.prev() + cursor.next() + + # Verify that by iterating + self.check_iterate(cursor, expectcount, isforward) + + # After an iteration is complete, the cursor should be in + # the same state as after reset(), or when first created. + if testmode == 0: + pass + elif testmode == 1: + cursor.next() + cursor.prev() + elif testmode == 2: + cursor.prev() + cursor.next() + + # Verify that by iterating + self.check_iterate(cursor, expectcount, isforward) + + cursor.close() + + def common_test(self, nentries, hascolgroups): + cgstr = ',colgroups=(c1,c2)' if hascolgroups else '' + self.session.create('table:main', 'key_format=iS,value_format=SiSi,' + 'columns=(ikey,Skey,S1,i2,S3,i4)' + cgstr) + if hascolgroups: + self.session.create("colgroup:main:c1", "columns=(S1,i2)") + self.session.create("colgroup:main:c2", "columns=(S3,i4)") + self.populate(nentries) + self.check_entries(0, nentries, True) + self.check_entries(1, nentries, True) + self.check_entries(2, nentries, True) + self.check_entries(0, nentries, False) + self.check_entries(1, nentries, False) + self.check_entries(2, nentries, False) + + def test_without_colgroups(self): + self.common_test(3, False) + + def test_with_colgroups(self): + self.common_test(3, True) + + def test_empty_without_colgroups(self): + self.common_test(0, False) + + def test_empty_with_colgroups(self): + self.common_test(0, True) + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_cursor_tracker.py b/test/suite/test_cursor_tracker.py new file mode 100644 index 00000000000..9f77bc76a93 --- /dev/null +++ b/test/suite/test_cursor_tracker.py @@ -0,0 +1,493 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_cursor_tracker.py +# Tracker for testing cursor operations. Keys and values +# are generated automatically based somewhat on position, +# and are stored simultaneously in the WT table and +# in a compact representation in python data structures +# (self.bitlist, self.vers). A set of convenience functions +# allows us to insert/remove/update keys on a cursor, +# navigate forward, back, etc. and verify K/V pairs in +# the table. Comprehensive tests can then be built up. +# +# All keys and values are generated, based on a triple: +# [major, minor, version]. The key generator is a pure function +# K that returns a string and takes inputs (major, minor). +# K is implemented as encode_key() below, its inverse is decode_key(). +# The value generator is a pure function V that returns a string +# based on inputs (major, minor, version). It is implemented +# as encode_value(), its inverse being decode_value(). +# +# When a test starts, calling cur_initial_conditions, a set +# of N K/V populates the table. These correspond to major +# numbers. For example, with N==3, major/minor numbers of +# [0,0], [0,1], [0,2] are inserted into the table. +# The table is then closed (and session closed) to guarantee +# that the associated data is written to disk before continuing. +# The theory is that since changes in WT are stored in skip lists, +# we want the test to be aware of preexisting values (those having +# minor number == 0) so we can try all combinations of adding +# new skip list entries, and removing/updating both skip list +# and original values. +# +# After initial conditions are set up, the test calls functions +# such as cur_insert to insert values. Since minor numbers +# sort secondarily to major, they will take logical positions +# in specific places relative to major (preexisting) K/V pairs. +# +# Updating an existing value is detected in the python data +# structures, and result in incrementing the version number. +# Thus, the key remains unchanged, but the associated value +# changes (the size of the value may be altered as well). +# +# TODO: we need to separate the cursor tracking information +# (the current position where we believe we are) from +# the database information (what we think is in the data storage). +# Once that's done, we can have multiple cursor tests +# (though simulating transactions would probably be beyond what +# we want to do here). + +import unittest +import hashlib +import wiredtiger +from wiredtiger import WiredTigerError +import wttest + +class TestCursorTracker(wttest.WiredTigerTestCase): + table_name1 = 'test_cursor' + DELETED = 0xffffffffffffffff + TRACE_API = False # a print output for each WT API call + + def config_string(self): + """ + Return any additional configuration. + This method may be overridden. + """ + return '' + + def session_create(self, name, args): + """ + session.create, but report errors more completely + """ + try: + self.session.create(name, args) + except: + print('**** ERROR in session.create("' + name + '","' + args + '") ***** ') + raise + + def table_dump(self, name): + cursor = self.session.open_cursor('table:' + name, None, None) + self._dumpcursor(cursor) + cursor.close() + + def __init__(self, testname): + wttest.WiredTigerTestCase.__init__(self, testname) + self.cur_initial_conditions(None, 0, None, None, None) + + # traceapi and friends are used internally in this module + def traceapi(self, s): + if self.TRACE_API: + print('> ' + s) + + def traceapi_before(self, s): + if self.TRACE_API: + print('> ' + s + '...') + + def traceapi_after(self, s): + if self.TRACE_API: + print(' ==> ' + s) + + def setup_encoders_decoders(self): + if self.tablekind == 'row': + self.encode_key = self.encode_key_row + self.decode_key = self.decode_key_row + self.encode_value = self.encode_value_row_or_col + self.decode_value = self.decode_value_row_or_col + elif self.tablekind == 'col': + self.encode_key = self.encode_key_col_or_fix + self.decode_key = self.decode_key_col_or_fix + self.encode_value = self.encode_value_row_or_col + self.decode_value = self.decode_value_row_or_col + else: + self.encode_key = self.encode_key_col_or_fix + self.decode_key = self.decode_key_col_or_fix + self.encode_value = self.encode_value_fix + self.decode_value = self.decode_value_fix + + def cur_initial_conditions(self, tablename, npairs, tablekind, keysizes, valuesizes): + if npairs >= 0xffffffff: + raise Exception('cur_initial_conditions: npairs too big') + self.tablekind = tablekind + self.isrow = (tablekind == 'row') + self.setup_encoders_decoders() + self.bitlist = [(x << 32) for x in range(npairs)] + self.vers = dict((x << 32, 0) for x in range(npairs)) + self.nopos = True # not positioned on a valid element + self.curpos = -1 + self.curbits = 0xffffffffffff + self.curremoved = False # K/V data in cursor does not correspond to active data + self.keysizes = keysizes + self.valuesizes = valuesizes + if tablekind != None: + cursor = self.session.open_cursor('table:' + tablename, None, 'append') + for i in range(npairs): + wtkey = self.encode_key(i << 32) + wtval = self.encode_value(i << 32) + self.traceapi('cursor.set_key(' + str(wtkey) + ')') + cursor.set_key(wtkey) + self.traceapi('cursor.set_value(' + str(wtval) + ')') + cursor.set_value(wtval) + self.traceapi('cursor.insert()') + cursor.insert() + cursor.close() + self.pr('reopening the connection') + self.conn.close() + self.conn = self.setUpConnectionOpen(".") + self.session = self.setUpSessionOpen(self.conn) + + def bits_to_triple(self, bits): + major = (bits >> 32) & 0xffffffff + minor = (bits >> 16) & 0xffff + version = (bits) & 0xffff + return [major, minor, version] + + def triple_to_bits(self, major, minor, version): + return ((major & 0xffffffff) << 32) | ((minor & 0xffff) << 16) | (version & 0xffff) + + def key_to_bits(self, key): + pass + + def stretch_content(self, s, sizes): + result = s + if sizes != None: + sha224 = hashlib.sha224(s) + md5 = sha224.digest() + low = sizes[0] - len(s) + if low < 0: + low = 0 + high = sizes[1] - len(s) + if high < 0: + high = 0 + diff = high - low + nextra = (ord(md5[4]) % (diff + 1)) + low + extra = sha224.hexdigest() + while len(extra) < nextra: + extra = extra + extra + result = s + extra[0:nextra] + return result + + def check_content(self, s, sizes): + if sizes != None: + stretched = self.stretch_content(s[0:20], sizes) + self.assertEquals(s, stretched) + + # There are variants of {encode,decode}_{key,value} to be + # used with each table kind: 'row', 'col', 'fix' + + def encode_key_row(self, bits): + # Prepend 0's to make the string exactly len 20 + # 64 bits fits in 20 decimal digits + # Then, if we're configured to have a longer key + # size, we'll append some additional length + # that can be regenerated based on the first part of the key + result = str(bits) + result = '00000000000000000000'[len(result):] + result + if self.keysizes != None: + result = self.stretch_content(result, self.keysizes) + return result + + def decode_key_row(self, s): + self.check_content(s, self.keysizes) + return int(s[0:20]) + + def encode_value_row_or_col(self, bits): + # We use the same scheme as key. So the 20 digits will + # be the same, but the last part may well be different + # if the size configuration is different. + result = str(bits) + result = '00000000000000000000'[len(result):] + result + if self.valuesizes != None: + result = self.stretch_content(result, self.valuesizes) + return result + + def decode_value_row_or_col(self, s): + self.check_content(s, self.valuesizes) + return int(s[0:20]) + + def encode_key_col_or_fix(self, bits): + # 64 bit key + maj = ((bits >> 32) & 0xffffffff) + 1 + min = (bits >> 16) & 0xffff + return long((maj << 16) | min) + + def decode_key_col_or_fix(self, bits): + maj = ((bits << 16) & 0xffffffff) - 1 + min = bits & 0xffff + return ((maj << 32) | (min << 16)) + + def encode_value_fix(self, bits): + # can only encode only 8 bits + maj = ((bits >> 32) & 0xff) + min = (bits >> 16) & 0xff + return (maj ^ min) + + def decode_value_fix(self, s): + return int(s) + + def setpos(self, newpos, isforward): + length = len(self.bitlist) + while newpos >= 0 and newpos < length: + if not self.isrow and self.bitlist[newpos] == self.DELETED: + if isforward: + newpos = newpos + 1 + else: + newpos = newpos - 1 + else: + self.curpos = newpos + self.nopos = False + self.curremoved = False + self.curbits = self.bitlist[newpos] + return True + if newpos < 0: + self.curpos = -1 + else: + self.curpos = length - 1 + self.nopos = True + self.curremoved = False + self.curbits = 0xffffffffffff + return False + + def cur_first(self, cursor, expect=0): + self.setpos(0, True) + self.traceapi('cursor.first()') + self.assertEquals(0, cursor.reset()) + self.assertEquals(expect, cursor.next()) + self.curremoved = False + + def cur_last(self, cursor, expect=0): + self.setpos(len(self.bitlist) - 1, False) + self.traceapi('cursor.last()') + self.assertEquals(0, cursor.reset()) + self.assertEquals(expect, cursor.prev()) + self.curremoved = False + + def cur_update(self, cursor, key): + # TODO: + pass + + def bitspos(self, bits): + list = self.bitlist + return next(i for i in xrange(len(list)) if list[i] == bits) + + def cur_insert(self, cursor, major, minor): + bits = self.triple_to_bits(major, minor, 0) + if bits not in self.vers: + self.bitlist.append(bits) + if self.isrow: + #TODO: why doesn't self.bitlist.sort() work? + self.bitlist = sorted(self.bitlist) + self.vers[bits] = 0 + else: + raise Exception('cur_insert: key already exists: ' + str(major) + ',' + str(minor)) + pos = self.bitspos(bits) + self.setpos(pos, True) + wtkey = self.encode_key(bits) + wtval = self.encode_value(bits) + self.traceapi('cursor.set_key(' + str(wtkey) + ')') + cursor.set_key(wtkey) + self.traceapi('cursor.set_value(' + str(wtval) + ')') + cursor.set_value(wtval) + self.traceapi('cursor.insert()') + cursor.insert() + + def cur_remove_here(self, cursor): + # TODO: handle the exception case + if self.nopos: + expectException = True + else: + expectException = False + del self.vers[self.curbits & 0xffffffff0000] + if self.isrow: + self.bitlist.pop(self.curpos) + self.setpos(self.curpos - 1, True) + self.nopos = True + else: + self.bitlist[self.curpos] = self.DELETED + self.curremoved = True + self.traceapi('cursor.remove()') + cursor.remove() + + def cur_recno_search(self, cursor, recno): + wtkey = long(recno) + self.traceapi('cursor.set_key(' + str(wtkey) + ')') + cursor.set_key(wtkey) + if recno > 0 and recno <= len(self.bitlist): + want = 0 + else: + want = wiredtiger.WT_NOTFOUND + self.traceapi('cursor.search()') + self.check_cursor_ret(cursor.search(), want) + + def cur_search(self, cursor, major, minor): + bits = self.triple_to_bits(major, minor, 0) + wtkey = self.encode_key(bits) + self.traceapi('cursor.set_key(' + str(wtkey) + ')') + cursor.set_key(wtkey) + if bits in self.vers: + want = 0 + else: + want = wiredtiger.WT_NOTFOUND + self.traceapi('cursor.search()') + self.check_cursor_ret(cursor.search(), want) + + def check_cursor_ret(self, ret, want): + if ret != want: + if ret == 0: + self.fail('cursor did not return NOTFOUND') + elif ret == wiredtiger.WT_NOTFOUND: + self.fail('cursor returns NOTFOUND unexpectedly') + else: + self.fail('unexpected return from cursor: ' + ret) + + def cur_check_forward(self, cursor, n): + if n < 0: + n = len(self.bitlist) + for i in range(n): + self.cur_next(cursor) + self.cur_check_here(cursor) + if self.nopos: + break + + def cur_next(self, cursor): + # Note: asymmetric with cur_previous, nopos corresponds to 'half' + if self.setpos(self.curpos + 1, True): + wantret = 0 + else: + wantret = wiredtiger.WT_NOTFOUND + self.traceapi('cursor.next()') + self.check_cursor_ret(cursor.next(), wantret) + + def cur_check_backward(self, cursor, n): + if n < 0: + n = len(self.bitlist) + for i in range(n): + self.cur_previous(cursor) + self.cur_check_here(cursor) + if self.nopos: + break + + def cur_previous(self, cursor): + if self.nopos: + pos = self.curpos + else: + pos = self.curpos - 1 + if self.setpos(pos, False): + wantret = 0 + else: + wantret = wiredtiger.WT_NOTFOUND + self.traceapi('cursor.prev()') + self.check_cursor_ret(cursor.prev(), wantret) + + def cur_check_here(self, cursor): + # Cannot check immediately after a remove, since the K/V in the cursor + # does not correspond to anything + if self.curremoved: + raise Exception('cur_check_here: cursor.get_key, get_value are not valid') + elif self.nopos: + self.traceapi_before('cursor.get_key()') + self.assertRaises(WiredTigerError, cursor.get_key) + self.traceapi_after('<unknown>') + self.traceapi_before('cursor.get_value()') + self.assertRaises(WiredTigerError, cursor.get_value) + self.traceapi_after('<unknown>') + else: + bits = self.curbits + self.traceapi_before('cursor.get_key()') + wtkey = cursor.get_key() + self.traceapi_after(str(wtkey)) + if self.isrow: + self.cur_check(cursor, wtkey, self.encode_key(bits), True) + else: + self.cur_check(cursor, wtkey, self.bitspos(bits) + 1, True) + self.traceapi_before('cursor.get_value()') + wtval = cursor.get_value() + self.traceapi_after(str(wtval)) + self.cur_check(cursor, wtval, self.encode_value(bits), False) + + def dumpbitlist(self): + print('bits array:') + for bits in self.bitlist: + print(' ' + str(self.bits_to_triple(bits)) + ' = ' + str(bits)) + + def _cursor_key_to_string(self, k): + return str(self.bits_to_triple(self.decode_key(k))) + ' = ' + str(k) + + def _cursor_value_to_string(self, v): + return str(self.bits_to_triple(self.decode_value(v))) + ' = ' + v + + def _dumpcursor(self, cursor): + print('cursor') + cursor.reset() + for k,v in cursor: + print(' ' + self._cursor_key_to_string(k) + ' ' + + self._cursor_value_to_string(v)) + + def cur_dump_here(self, cursor, prefix): + try: + k = self._cursor_key_to_string(cursor.get_key()) + except: + k = '[invalid]' + try: + v = self._cursor_value_to_string(cursor.get_value()) + except: + v = '[invalid]' + print(prefix + k + ' ' + v) + + def cur_check(self, cursor, got, want, iskey): + if got != want: + if iskey: + goti = self.decode_key(got) + wanti = self.decode_key(want) + else: + goti = self.decode_value(got) + wanti = self.decode_value(want) + gotstr = str(self.bits_to_triple(goti)) + wantstr = str(self.bits_to_triple(wanti)) + self.dumpbitlist() + # Note: dumpcursor() resets the cursor position, + # but we're about to issue a fatal error, so it's okay + self._dumpcursor(cursor) + if iskey: + kind = 'key' + else: + kind = 'value' + self.fail('mismatched ' + kind + ', want: ' + wantstr + ', got: ' + gotstr) + + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_drop_create.py b/test/suite/test_drop_create.py new file mode 100644 index 00000000000..61f53380192 --- /dev/null +++ b/test/suite/test_drop_create.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_base03.py +# Cursor operations +# + +import unittest +import wiredtiger +from wiredtiger import WiredTigerError +import wttest + +class test_drop_create(wttest.WiredTigerTestCase): + def test_drop_create(self): + s, self.session = self.session, None + self.assertEqual(s.close(), 0) + + for config in [None, 'key_format=S,value_format=S', None]: + s = self.conn.open_session() + self.assertEqual(s.drop("table:test", "force"), 0) + self.assertEqual(s.create("table:test", config), 0) + self.assertEqual(s.drop("table:test"), 0) + self.assertEqual(s.close(), 0) + s = self.conn.open_session() + self.assertNotEqual(s, None) + self.assertEqual(s.create("table:test", config), 0) + self.assertEqual(s.close(), 0) + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_index01.py b/test/suite/test_index01.py new file mode 100644 index 00000000000..ff2530c213e --- /dev/null +++ b/test/suite/test_index01.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_index01.py +# basic tests for indices +# + +import unittest +import wiredtiger +from wiredtiger import WT_NOTFOUND +import wttest + +class test_index01(wttest.WiredTigerTestCase): + '''Test basic operations for indices''' + + basename = 'test_index01' + tablename = 'table:' + basename + indexbase = 'index:' + basename + NUM_INDICES = 6 + index = ['%s:index%d' % (indexbase, i) for i in xrange(NUM_INDICES)] + + def reopen(self): + self.conn.close() + self.conn = wiredtiger.wiredtiger_open('.', None) + self.session = self.conn.open_session() + + def create_table(self): + self.pr('create table') + self.session.create(self.tablename, 'key_format=Si,value_format=SSii,columns=(name,ID,dept,job,salary,year)') + self.session.create(self.index[0], 'columns=(dept)') + self.session.create(self.index[1], 'columns=(name,year)') + self.session.create(self.index[2], 'columns=(salary)') + self.session.create(self.index[3], 'columns=(dept,job,name)') + self.session.create(self.index[4], 'columns=(name,ID)') + self.session.create(self.index[5], 'columns=(ID,name)') + + def drop_table(self): + self.pr('drop table') + self.session.drop(self.tablename, None) + + def cursor(self, config=None): + self.pr('open cursor') + c = self.session.open_cursor(self.tablename, None, config) + self.assertNotEqual(c, None) + return c + + def index_cursor(self, i): + self.pr('open index cursor(%d)' % i) + c = self.session.open_cursor(self.index[i], None, None) + self.assertNotEqual(c, None) + return c + + def index_iter(self, i): + cursor = self.index_cursor(i) + for cols in cursor: + yield cols + cursor.close() + + def check_exists(self, name, ID, expected): + cursor = self.cursor() + cursor.set_key(name, ID) + self.pr('search') + self.assertEqual(cursor.search(), expected) + self.pr('closing cursor') + cursor.close() + + def insert(self, *cols): + self.pr('insert') + cursor = self.cursor() + cursor.set_key(*cols[:2]); + cursor.set_value(*cols[2:]) + self.assertEqual(cursor.insert(), 0) + cursor.close() + + def insert_overwrite(self, *cols): + self.pr('insert') + cursor = self.cursor(config='overwrite') + cursor.set_key(*cols[:2]); + cursor.set_value(*cols[2:]) + self.assertEqual(cursor.insert(), 0) + cursor.close() + + def update(self, *cols): + self.pr('update') + cursor = self.cursor() + cursor.set_key(*cols[:2]); + cursor.set_value(*cols[2:]) + self.assertEqual(cursor.update(), 0) + cursor.close() + + def update_nonexistent(self, *cols): + self.pr('update') + cursor = self.cursor() + cursor.set_key(*cols[:2]); + cursor.set_value(*cols[2:]) + self.assertEqual(cursor.update(), WT_NOTFOUND) + cursor.close() + + def remove(self, name, ID): + self.pr('remove') + cursor = self.cursor() + cursor.set_key(name, ID); + self.assertEqual(cursor.remove(), 0) + cursor.close() + + def test_empty(self): + '''Create a table, look for a nonexistent key''' + self.create_table() + self.check_exists('jones', 10, WT_NOTFOUND) + for i in xrange(self.NUM_INDICES): + self.assertEqual(list(self.index_iter(i)), []) + self.drop_table() + + def test_insert(self): + '''Create a table, add a key, get it back''' + self.create_table() + self.insert('smith', 1, 'HR', 'manager', 100000, 1970) + self.check_exists('smith', 1, 0) + result = '' + for i in xrange(self.NUM_INDICES): + result += '\n'.join(repr(cols) + for cols in self.index_iter(i)) + result += '\n\n' + self.assertEqual(result, \ + "['HR', 'HR', 'manager', 100000, 1970]\n\n" + \ + "['smith', 1970, 'HR', 'manager', 100000, 1970]\n\n" + \ + "[100000, 'HR', 'manager', 100000, 1970]\n\n" + \ + "['HR', 'manager', 'smith', 'HR', 'manager', 100000, 1970]\n\n" + \ + "['smith', 1, 'HR', 'manager', 100000, 1970]\n\n" + \ + "[1, 'smith', 'HR', 'manager', 100000, 1970]\n\n") + self.drop_table() + + def test_update(self): + '''Create a table, add a key, update it, get it back''' + self.create_table() + self.insert('smith', 1, 'HR', 'manager', 100000, 1970) + self.update('smith', 1, 'HR', 'janitor', 10000, 1970) + self.update_nonexistent('smith', 2, 'HR', 'janitor', 1000, 1970) + self.update_nonexistent('Smith', 1, 'HR', 'janitor', 1000, 1970) + self.check_exists('smith', 1, 0) + result = '' + for i in xrange(self.NUM_INDICES): + result += '\n'.join(repr(cols) + for cols in self.index_iter(i)) + result += '\n\n' + self.assertEqual(result, \ + "['HR', 'HR', 'janitor', 10000, 1970]\n\n" + \ + "['smith', 1970, 'HR', 'janitor', 10000, 1970]\n\n" + \ + "[10000, 'HR', 'janitor', 10000, 1970]\n\n" + \ + "['HR', 'janitor', 'smith', 'HR', 'janitor', 10000, 1970]\n\n" + \ + "['smith', 1, 'HR', 'janitor', 10000, 1970]\n\n" + \ + "[1, 'smith', 'HR', 'janitor', 10000, 1970]\n\n") + self.drop_table() + + def test_insert_overwrite(self): + '''Create a table, add a key, insert-overwrite it, + insert-overwrite a nonexistent record, get them both back''' + self.create_table() + self.insert('smith', 1, 'HR', 'manager', 100000, 1970) + self.insert_overwrite('smith', 1, 'HR', 'janitor', 10000, 1970) + self.insert_overwrite('jones', 2, 'IT', 'sysadmin', 50000, 1980) + self.check_exists('smith', 1, 0) + self.check_exists('jones', 2, 0) + result = '' + for i in xrange(self.NUM_INDICES): + result += '\n'.join(repr(cols) + for cols in self.index_iter(i)) + result += '\n\n' + self.assertEqual(result, \ + "['HR', 'HR', 'janitor', 10000, 1970]\n" + \ + "['IT', 'IT', 'sysadmin', 50000, 1980]\n\n" + \ + "['jones', 1980, 'IT', 'sysadmin', 50000, 1980]\n" + \ + "['smith', 1970, 'HR', 'janitor', 10000, 1970]\n\n" + \ + "[10000, 'HR', 'janitor', 10000, 1970]\n" + \ + "[50000, 'IT', 'sysadmin', 50000, 1980]\n\n" + \ + "['HR', 'janitor', 'smith', 'HR', 'janitor', 10000, 1970]\n" + \ + "['IT', 'sysadmin', 'jones', 'IT', 'sysadmin', 50000, 1980]\n\n" + \ + "['jones', 2, 'IT', 'sysadmin', 50000, 1980]\n" + \ + "['smith', 1, 'HR', 'janitor', 10000, 1970]\n\n" + \ + "[1, 'smith', 'HR', 'janitor', 10000, 1970]\n" + \ + "[2, 'jones', 'IT', 'sysadmin', 50000, 1980]\n\n") + + self.drop_table() + + def test_insert_delete(self): + '''Create a table, add a key, remove it''' + self.create_table() + self.insert('smith', 1, 'HR', 'manager', 100000, 1970) + self.check_exists('smith', 1, 0) + self.remove('smith', 1) + self.check_exists('smith', 1, WT_NOTFOUND) + for i in xrange(self.NUM_INDICES): + self.assertEqual(list(self.index_iter(i)), []) + self.drop_table() + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_priv01.py b/test/suite/test_priv01.py new file mode 100644 index 00000000000..68e6fa0b0c2 --- /dev/null +++ b/test/suite/test_priv01.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_priv01.py +# Test privileged operations. +# This is a variant of test_config02.py. +# This test should be run as both normal and privileged +# (e.g. root) user, and should pass in both cases. +# + +import unittest +import wiredtiger +from wiredtiger import WiredTigerError +import wttest +import os + +class test_priv01(wttest.WiredTigerTestCase): + """ + This tests privileged operations. + The only part of WiredTiger that currently cares about privilege + levels is the 'home_environment_priv' configuration option for + wiredtiger_open, which by design, behaves differently depending on + whether the current process is running as a privileged user or + not. This test should be run as both normal and privileged + (e.g. root) user to fully test both cases. + """ + table_name1 = 'test_priv01' + nentries = 100 + + # Each test needs to set up its connection in its own way, + # so override these methods to do nothing + def setUpConnectionOpen(self, dir): + return None + + def setUpSessionOpen(self, conn): + return None + + def populate_and_check(self): + """ + Create entries, and read back in a cursor: key=string, value=string + """ + create_args = 'key_format=S,value_format=S' + self.session.create("table:" + self.table_name1, create_args) + cursor = self.session.open_cursor('table:' + self.table_name1, None, None) + for i in range(0, self.nentries): + cursor.set_key(str(1000000 + i)) + cursor.set_value('value' + str(i)) + cursor.insert() + i = 0 + cursor.reset() + for key, value in cursor: + self.assertEqual(key, str(1000000 + i)) + self.assertEqual(value, ('value' + str(i))) + i += 1 + self.assertEqual(i, self.nentries) + cursor.close() + + def checkfiles(self, dirname): + self.assertTrue(os.path.exists(dirname + os.sep + self.table_name1 + ".wt")) + + def checknofiles(self, dirname): + nfiles = len([nm for nm in os.listdir(dirname) if os.path.isfile(nm)]) + self.assertEqual(nfiles, 0) + + def common_test(self, homearg, homeenv, configextra): + """ + Call wiredtiger_open and run a simple test. + homearg is the first arg to wiredtiger_open, it may be null. + WIREDTIGER_HOME is set to homeenv, if it is not null. + configextra are any extra configuration strings needed on the open. + """ + configarg = 'create' + if configextra != None: + configarg += ',' + configextra + if homeenv == None: + os.unsetenv('WIREDTIGER_HOME') + else: + os.putenv('WIREDTIGER_HOME', homeenv) + self.conn = wiredtiger.wiredtiger_open(homearg, configarg) + self.session = self.conn.open_session(None) + self.populate_and_check() + + def test_home_and_env(self): + hdir = 'homedir' + edir = 'envdir' + os.mkdir(hdir) + os.mkdir(edir) + self.common_test(hdir, edir, None) + self.checkfiles(hdir) + self.checknofiles(edir) + + def test_home_and_env_conf(self): + # If homedir is set, the environment is ignored + hdir = 'homedir' + edir = 'envdir' + os.mkdir(hdir) + os.mkdir(edir) + self.common_test(hdir, edir, 'home_environment=true') + self.checkfiles(hdir) + self.checknofiles(edir) + + def test_home_and_missing_env(self): + # If homedir is set, it is used no matter what + hdir = 'homedir' + os.mkdir(hdir) + self.common_test(hdir, None, 'home_environment=true') + self.checkfiles(hdir) + + def test_env_conf(self): + edir = 'envdir' + os.mkdir(edir) + self.common_test(None, edir, 'home_environment=true') + self.checkfiles(edir) + + def test_env_conf_without_env_var(self): + # no env var set, so should use current directory + self.common_test(None, None, 'home_environment=true') + self.checkfiles(".") + + def test_home_and_env_conf_priv(self): + # If homedir is set, the environment is ignored + hdir = 'homedir' + edir = 'envdir' + os.mkdir(hdir) + os.mkdir(edir) + self.common_test(hdir, edir, 'home_environment_priv=true') + self.checkfiles(hdir) + self.checknofiles(edir) + + def test_home_and_missing_env_priv(self): + # If homedir is set, it is used no matter what + hdir = 'homedir' + os.mkdir(hdir) + self.common_test(hdir, None, 'home_environment_priv=true') + self.checkfiles(hdir) + + def test_env_conf_priv(self): + euid = os.geteuid() + edir = 'envdir' + os.mkdir(edir) + privarg = 'home_environment_priv=true' + if euid == 0: + print 'Running ' + str(self) + ' as privileged user' + self.common_test(None, edir, privarg) + self.checkfiles(edir) + self.checknofiles(".") + else: + self.assertRaises(WiredTigerError, + lambda: self.common_test(None, edir, privarg)) + + def test_env_conf_without_env_var_priv(self): + # no env var set, so should use current directory + self.common_test(None, None, 'home_environment_priv=true') + self.checkfiles(".") + + + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_schema01.py b/test/suite/test_schema01.py new file mode 100644 index 00000000000..4e7a22dc5d1 --- /dev/null +++ b/test/suite/test_schema01.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_schema01.py +# Test that tables are reconciled correctly when they are empty. +# + +import unittest +import wiredtiger +from wiredtiger import WT_NOTFOUND +import wttest + +pop_data = [ + ( 'USA', 1980, 226542250 ), + ( 'USA', 2009, 307006550 ), + ( 'UK', 2008, 61414062 ), + ( 'CAN', 2008, 33311400 ), + ( 'AU', 2008, 21431800 ) +] + +class test_schema01(wttest.WiredTigerTestCase): + '''Test various tree types becoming empty''' + + basename = 'test_schema01' + tablename = 'table:' + basename + cgname = 'colgroup:' + basename + + def __init__(self, *args, **kwargs): + wttest.WiredTigerTestCase.__init__(self, *args, **kwargs) + self.reconcile = False + + def reopen(self): + self.conn.close() + self.conn = wiredtiger.wiredtiger_open('.', None) + self.session = self.conn.open_session() + + def create_table(self): + self.pr('create table') + self.session.create(self.tablename, 'key_format=5s,value_format=HQ,' + + 'columns=(country,year,population),' + + 'colgroups=(year,population)') + self.session.create(self.cgname + ':year', 'columns=(year)') + self.session.create(self.cgname + ':population', 'columns=(population)') + + def drop_table(self): + self.pr('drop table') + self.session.drop(self.tablename) + + def cursor(self, config=None): + self.pr('open cursor') + return self.session.open_cursor(self.tablename, None, config) + + def test_populate(self): + '''Populate a table''' + for reopen in (False, True): + self.create_table() + c = self.cursor('overwrite') + try: + for record in pop_data: + c.set_key(record[0]) + c.set_value(*record[1:]) + c.insert() + finally: + c.close() + + if reopen: + self.reopen() + + c = self.cursor() + for record in c: + print record + c.close() + self.drop_table() + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_schema02.py b/test/suite/test_schema02.py new file mode 100644 index 00000000000..259c4d5a9f1 --- /dev/null +++ b/test/suite/test_schema02.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_schema02.py +# Columns, column groups, indexes +# + +import unittest +import wiredtiger +from wiredtiger import WiredTigerError +import wttest + +class test_schema02(wttest.WiredTigerTestCase): + """ + Test basic operations + """ + nentries = 1000 + + def expect_failure_primary(self, configstr): + self.assertRaises(WiredTigerError, + lambda:self.session.create("table:main", configstr)) + + def expect_failure_colgroup(self, name, configstr): + self.assertRaises(WiredTigerError, + lambda:self.session.create("colgroup:" + name, configstr)) + + def test_colgroup_after_failure(self): + # bogus formats + self.expect_failure_primary("key_format=Z,value_format=Y") + + # These should succeed + self.session.create("table:main", "key_format=iS,value_format=SiSi," + "columns=(ikey,Skey,S1,i2,S3,i4),colgroups=(c1,c2)") + self.session.create("colgroup:main:c1", "columns=(S1,i2)") + + def test_colgroup_failures(self): + # too many columns + #### TEST FAILURE HERE #### + self.expect_failure_primary("key_format=S,value_format=,columns=(a,b)") + # Note: too few columns is allowed + + # expect this to work + self.session.create("table:main", "key_format=iS,value_format=SiSi," + "columns=(ikey,Skey,S1,i2,S3,i4),colgroups=(c1,c2)") + + # bad table name + #### TEST FAILURE HERE #### + self.expect_failure_colgroup("nomatch:c", "columns=(S1,i2)") + # colgroup not declared in initial create + self.expect_failure_colgroup("main:nomatch", "columns=(S1,i2)") + # bad column + self.expect_failure_colgroup("main:c1", "columns=(S1,i2,bad)") + # no columns + #### TEST FAILURE HERE #### + #self.expect_failure_colgroup("main:c1", "columns=()") + # key in a column group + self.expect_failure_colgroup("main:c1", "columns=(ikey,S1,i2)") + + # expect this to work + self.session.create("colgroup:main:c1", "columns=(S1,i2)") + + # colgroup not declared in initial create + self.expect_failure_colgroup("main:c3", "columns=(S3,i4)") + + # missing columns + self.expect_failure_colgroup("main:c2", "columns=(S1,i4)") + + # expect this to work + #### TEST FAILURE HERE #### + self.session.create("colgroup:main:c2", "columns=(S3,i4)") + + # expect these to work - each table name is a separate namespace + self.session.create("table:main2", "key_format=iS,value_format=SiSi," + "columns=(ikey,Skey,S1,i2,S3,i4),colgroups=(c1,c2)") + self.session.create("colgroup:main2:c1", "columns=(S1,i2)") + self.session.create("colgroup:main2:c2", "columns=(S3,i4)") + + def test_index(self): + self.session.create("table:main", "key_format=iS,value_format=SiSi," + "columns=(ikey,Skey,S1,i2,S3,i4),colgroups=(c1,c2)") + + # should be able to create colgroups before indices + self.session.create("colgroup:main:c2", "columns=(S3,i4)") + + # should be able to create indices on all key combinations + self.session.create("index:main:ikey", "columns=(ikey)") + self.session.create("index:main:Skey", "columns=(Skey)") + self.session.create("index:main:ikeySkey", "columns=(ikey,Skey)") + self.session.create("index:main:Skeyikey", "columns=(Skey,ikey)") + + # should be able to create indices on all value combinations + self.session.create("index:main:S1", "columns=(S1)") + self.session.create("index:main:i2", "columns=(i2)") + self.session.create("index:main:i2S1", "columns=(i2,S1)") + self.session.create("index:main:S1i4", "columns=(S1,i4)") + + # somewhat nonsensical to repeat columns within an index, but allowed + self.session.create("index:main:i4S3i4S1", "columns=(i4,S3,i4,S1)") + + # should be able to create colgroups after indices + self.session.create("colgroup:main:c1", "columns=(S1,i2)") + + self.populate() + + # should be able to create indices after populating + self.session.create("index:main:i2S1i4", "columns=(i2,S1,i4)") + + self.check_entries() + + def populate(self): + cursor = self.session.open_cursor('table:main', None, None) + for i in range(0, self.nentries): + cursor.set_key(i, 'key' + str(i)) + square = i * i + cube = square * i + cursor.set_value('val' + str(square), square, 'val' + str(cube), cube) + cursor.insert() + cursor.close() + + def check_entries(self): + cursor = self.session.open_cursor('table:main', None, None) + # spot check via search + n = self.nentries + for i in (n / 5, 0, n - 1, n - 2, 1): + cursor.set_key(i, 'key' + str(i)) + square = i * i + cube = square * i + cursor.search() + (s1, i2, s3, i4) = cursor.get_values() + self.assertEqual(s1, 'val' + str(square)) + self.assertEqual(i2, square) + self.assertEqual(s3, 'val' + str(cube)) + self.assertEqual(i4, cube) + + i = 0 + # then check all via cursor + cursor.reset() + for ikey, skey, s1, i2, s3, i4 in cursor: + square = i * i + cube = square * i + self.assertEqual(ikey, i) + self.assertEqual(skey, 'key' + str(i)) + self.assertEqual(s1, 'val' + str(square)) + self.assertEqual(i2, square) + self.assertEqual(s3, 'val' + str(cube)) + self.assertEqual(i4, cube) + i += 1 + cursor.close() + self.assertEqual(i, n) + + def test_colgroups(self): + self.session.create("table:main", "key_format=iS,value_format=SiSi," + "columns=(ikey,Skey,S1,i2,S3,i4),colgroups=(c1,c2)") + self.session.create("colgroup:main:c1", "columns=(S1,i2)") + self.session.create("colgroup:main:c2", "columns=(S3,i4)") + self.populate() + self.check_entries() + + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_schema03.py b/test/suite/test_schema03.py new file mode 100644 index 00000000000..cf864a2a347 --- /dev/null +++ b/test/suite/test_schema03.py @@ -0,0 +1,561 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_schema03.py +# Bigger, more 'randomly generated' schemas and data. +# This test is complex. If it fails, rerun with +# modified values for SHOW_PYTHON* variables. +# + +import unittest +import wiredtiger +from wiredtiger import WiredTigerError +import wttest +import wtscenario +import suite_random +import resource + +def extract_random_from_list(rand, list): + pos = rand.rand_range(0, len(list)) + result = list[pos] + list = list[:pos] + list[pos+1:] + return (result, list) + +class tabconfig: + """ + Configuration for a table used in the test + """ + def __init__(self): + self.tableidx = -1 + self.tablename = '' + self.cglist = [] # list of related cgconfig + self.idxlist = [] # list of related idxconfig + self.nkeys = 0 # how many key columns + self.nvalues = 0 # how many value columns + self.nentries = 0 + self.keyformats = '' + self.valueformats = '' + + # we don't want to insert the keys in order, + # so generate them with backwards digits e.g. + # 235 => 532. However, 100 backwards is 001, + # so we append a positive integer to the end + # before reversing. + def gen_keys(self, i): + addmod = i * 10 + (i % 7) + 1 + rev = int((str(addmod))[::-1]) + keys = [] + # ASSUME: each format is 1 char + for format in self.keyformats: + if format == 'S': + keys.append(str(rev)) + elif format == 'i': + keys.append(rev) + elif format == 'r': + keys.append(long(i+1)) + return keys + + def gen_values(self, i): + vals = [] + # ASSUME: each format is 1 char + for format in self.valueformats: + if format == 'S': + vals.append(str(i)) + elif format == 'i': + vals.append(i) + return vals + + def columns_for_groups(self, collist): + totalgroups = len(self.cglist) + ncolumns = len(collist) + rand = suite_random.suite_random(ncolumns, totalgroups) + + # Each columngroup must have at least one column, so + # the only choice about distribution is with the + # excess columns. + excess = ncolumns - totalgroups + if excess < 0: + raise ValueError('columns_for_groups expects a column list (len=' + str(ncolumns) + ') larger than column group list (len=' + str(totalgroups) + ')') + + # Initially, all groups get column from the collist + for cg in self.cglist: + (colno, collist) = extract_random_from_list(rand, collist) + cg.columns.append(colno) + + # Then divy up remainder in the collist + for i in range(0, excess): + pos = rand.rand_range(0, totalgroups) + cg = self.cglist[pos] + (colno, collist) = extract_random_from_list(rand, collist) + cg.columns.append(colno) + + # collist should be emptied + if len(collist) != 0: + raise AssertionError('column list did not get emptied') + + def columns_for_indices(self, collist): + totalindices = len(self.idxlist) + ncolumns = len(collist) + startcol = 0 + + # KNOWN LIMITATION: Indices should not include primary keys + # Remove this statement when the limitation is fixed. + #startcol = self.nkeys + # END KNOWN LIMITATION. + + rand = suite_random.suite_random(ncolumns, totalindices) + + # Initially, all indices get one column from the collist. + # Overlaps are allowed. Then probalistically, add some + # more columns. + for idx in self.idxlist: + prob = 1.0 + for i in range(0, ncolumns - startcol): + if rand.rand_float() > prob: + break + colno = collist[rand.rand_range(startcol, ncolumns)] + if not any(x == colno for x in idx.columns): + idx.columns.append(colno) + if colno < self.nkeys: + # ASSUME: each format is 1 char + idx.formats += self.keyformats[colno] + else: + # ASSUME: each format is 1 char + idx.formats += self.valueformats[colno - self.nkeys] + prob *= 0.5 + + +class cgconfig: + """ + Configuration for a column group used in the test. + Each tabconfig contains a list of these. + """ + def __init__(self): + self.cgname = '' + self.columns = [] + self.createset = 0 # 0 or 1 depending on which set to create them. + +class idxconfig: + """ + Configuration for an index used in the test. + Each tabconfig contains a list of these. + """ + def __init__(self): + self.idxname = '' + self.columns = [] + self.createset = 0 # 0 or 1 depending on which set to create them. + self.formats = '' # piece + self.tab = None # references the tabconfig + + def gen_keys(self, i): + keys = [] + colpos = 0 + addmod = i * 10 + (i % 7) + 1 + rev = int((str(addmod))[::-1]) + for format in self.formats: + if self.columns[colpos] >= self.tab.nkeys: + # The column is a value in the primary table + key = i + else: + # The column is a key in the primary table + key = rev + if format == 'S': + key = str(key) + keys.append(key) + colpos += 1 + return keys + +class test_schema03(wttest.WiredTigerTestCase): + """ + Test schemas - a 'predictably random' assortment of columns, + column groups and indices are created within tables, and are + created in various orders as much as the API allows. On some runs + the connection will be closed and reopened at a particular point + to test that the schemas (and data) are saved and read correctly. + + The test is run multiple times, using scenarios. + The test always follows these steps: + - table: create tables + - colgroup0: create (some) colgroups + - index0: create (some) indices + - colgroup1: create (more) colgroups + - index1: create (more) indices + - populate0: populate 1st time + - index2: create (more) indices + - populate1: populate 2nd time (more key/values) + - check: check key/values + + The variations represented by scenarios are: + - how many tables to create + - how many colgroups to create at each step (may be 0) + - how many indices to create at each step (may be 0) + - between each step, whether to close/reopen the connection + """ + + ################################################################ + # These three variables can be altered to help generate + # and pare down failing test cases. + + # Set to true to get python test program fragment on stdout, + # used by show_python() below. + SHOW_PYTHON = False + + # When SHOW_PYTHON is set, we print an enormous amount of output. + # To only print for a given scenario, set this + SHOW_PYTHON_ONLY_SCEN = None # could be e.g. [2] or [0,1] + + # To print verbosely for only a given table, set this + SHOW_PYTHON_ONLY_TABLE = None # could be e.g. [2] or [0,1] + + ################################################################ + + # Set whenever we are working with a table + current_table = None + + nentries = 50 + + # We need to have a large number of open files available + # to run this test. We probably don't need quite this many, + # but boost it up to this limit anyway. + OPEN_FILE_LIMIT = 8192 + + restart_scenarios = [('table', dict(s_restart=['table'],P=0.3)), + ('colgroup0', dict(s_restart=['colgroup0'],P=0.3)), + ('index0', dict(s_restart=['index0'],P=0.3)), + ('colgroup1', dict(s_restart=['colgroup1'],P=0.3)), + ('index1', dict(s_restart=['index1'],P=0.3)), + ('populate0', dict(s_restart=['populate0'],P=0.3)), + ('index2', dict(s_restart=['index2'],P=0.3)), + ('populate1', dict(s_restart=['populate1'],P=0.3)), + ('ipop', dict(s_restart=['index0','populate0'],P=0.3)), + ('all', dict(s_restart=['table','colgroup0','index0','colgroup1','index1','populate0','index2','populate1'],P=1.0))] + + ntable_scenarios = wtscenario.quick_scenarios('s_ntable', + [1,2,7,43], [1.0,0.4,0.5,0.5]) + ncolgroup_scenarios = wtscenario.quick_scenarios('s_colgroup', + [[1,0],[0,1],[2,4],[18,5]], [1.0,0.2,0.3,1.0]) + nindex_scenarios = wtscenario.quick_scenarios('s_index', + [[1,1,1],[3,2,4],[15,7,3]], [1.0,0.5,1.0]) + + all_scenarios = wtscenario.multiply_scenarios('_', restart_scenarios, ntable_scenarios, ncolgroup_scenarios, nindex_scenarios) + + # Prune the scenarios according to the probabilities given above. + scenarios = wtscenario.prune_scenarios(all_scenarios, 30) + scenarios = wtscenario.number_scenarios(scenarios) + + # Note: the set can be reduced here for debugging, e.g. + # scenarios = scenarios[40:44] + # or + # scenarios = [ scenarios[0], scenarios[30], scenarios[40] ] + + print 'test_schema03: running ' + str(len(scenarios)) + ' scenarios' + + # This test requires a large number of open files. + # Increase our resource limits before we start + def setUp(self): + self.origFileLimit = resource.getrlimit(resource.RLIMIT_NOFILE) + newlimit = (self.OPEN_FILE_LIMIT, self.origFileLimit[1]) + if newlimit[0] > newlimit[1]: + self.skipTest('Require %d open files, only %d available' % newlimit) + resource.setrlimit(resource.RLIMIT_NOFILE, newlimit) + super(test_schema03, self).setUp() + + def setUpConnectionOpen(self, dir): + cs = 100 * 1024 * 1024 + conn = wiredtiger.wiredtiger_open(dir, 'create,cache_size=' + + str(cs) + ',hazard_max=300') + self.pr(`conn`) + return conn + + def tearDown(self): + super(test_schema03, self).tearDown() + resource.setrlimit(resource.RLIMIT_NOFILE, self.origFileLimit) + + def gen_formats(self, rand, n, iskey): + if iskey and n == 1: + if rand.rand_range(0, 2) == 0: + return 'r' # record number + result = '' + for i in range(0, n): + if rand.rand_range(0, 2) == 0: + result += 'S' + else: + result += 'i' + return result + + def show_python(self, s): + if self.SHOW_PYTHON: + if self.SHOW_PYTHON_ONLY_TABLE == None or self.current_table in self.SHOW_PYTHON_ONLY_TABLE: + if self.SHOW_PYTHON_ONLY_SCEN == None or self.scenario_number in self.SHOW_PYTHON_ONLY_SCEN: + print ' ' + s + + def join_names(self, sep, prefix, list): + return sep.join([prefix + str(val) for val in list]) + + def create(self, what, tablename, whatname, columnlist): + createarg = what + ":" + tablename + ":" + whatname + colarg = self.join_names(',', 'c', columnlist) + self.show_python("self.session.create('" + createarg + "', 'columns=(" + colarg + ")')") + result = self.session.create(createarg, "columns=(" + colarg + ")") + self.assertEqual(result, 0) + + def finished_step(self, name): + if self.s_restart == name: + print " # Reopening connection at step: " + name + self.reopen_conn() + + def test_schema(self): + rand = suite_random.suite_random() + if self.SHOW_PYTHON: + print ' ################################################' + print ' # Running scenario ' + str(self.scenario_number) + + ntables = self.s_ntable + + # Report known limitations in the test, + # we'll work around these later, in a loop where we don't want to print. + self.KNOWN_LIMITATION('Indices created after data population will have no entries') + self.KNOWN_LIMITATION('Column groups created after indices confuses things') + + # Column groups are created in two different times. + # We call these two batches 'createsets'. + # So we don't have the exactly the same number of column groups + # for each table, for tests that indicate >1 colgroup, we + # increase the number of column groups for each table + tabconfigs = [] + for i in range(0, ntables): + self.current_table = i + tc = tabconfig() + tc.tablename = 't' + str(i) + tc.tableidx = i + tabconfigs.append(tc) + + for createset in range(0, 2): + ncg = self.s_colgroup[createset] + if ncg > 1: + ncg += i + for k in range(0, ncg): + thiscg = cgconfig() + thiscg.createset = createset + + # KNOWN LIMITATION: Column groups created after + # indices confuses things. So for now, put all + # column group creation in the first set. + # Remove this statement when the limitation is fixed. + thiscg.createset = 0 + # END KNOWN LIMITATION + + thiscg.cgname = 'g' + str(len(tc.cglist)) + tc.cglist.append(thiscg) + + # The same idea for indices, except that we create them in + # three sets + for createset in range(0, 3): + nindex = self.s_index[createset] + if nindex > 1: + nindex += i + for k in range(0, nindex): + thisidx = idxconfig() + thisidx.createset = createset + thisidx.idxname = 'i' + str(len(tc.idxlist)) + thisidx.tab = tc + tc.idxlist.append(thisidx) + + # We'll base the number of key/value columns + # loosely on the number of column groups and indices. + + colgroups = len(tc.cglist) + indices = len(tc.idxlist) + nall = colgroups * 2 + indices + k = rand.rand_range(1, nall) + v = rand.rand_range(0, nall) + # we need at least one value per column group + if v < colgroups: + v = colgroups + tc.nkeys = k + tc.nvalues = v + tc.keyformats = self.gen_formats(rand, tc.nkeys, True) + tc.valueformats = self.gen_formats(rand, tc.nvalues, False) + + # Simple naming (we'll test odd naming elsewhere): + # tables named 't0' --> 't<N>' + # within each table: + # columns named 'c0' --> 'c<N>' + # colgroups named 'g0' --> 'g<N>' + # indices named 'i0' --> 'i<N>' + + config = ""; + config += "key_format=" + tc.keyformats + config += ",value_format=" + tc.valueformats + config += ",columns=(" + for j in range(0, tc.nkeys + tc.nvalues): + if j != 0: + config += "," + config += "c" + str(j) + config += "),colgroups=(" + for j in range(0, len(tc.cglist)): + if j != 0: + config += "," + config += "g" + str(j) + config += ")" + # indices are not declared here + self.show_python("self.session.create('table:" + tc.tablename + "', '" + config + "')") + self.session.create("table:" + tc.tablename, config) + + tc.columns_for_groups(range(tc.nkeys, tc.nkeys + tc.nvalues)) + tc.columns_for_indices(range(0, tc.nkeys + tc.nvalues)) + + self.finished_step('table') + + for createset in (0, 1): + # Create column groups in this set + # e.g. self.session.create("colgroup:t0:g1", "columns=(c3,c4)") + for tc in tabconfigs: + self.current_table = tc.tableidx + for cg in tc.cglist: + if cg.createset == createset: + self.create('colgroup', tc.tablename, cg.cgname, cg.columns) + + self.finished_step('colgroup' + str(createset)) + + # Create indices in this set + # e.g. self.session.create("index:t0:i1", "columns=(c3,c4)") + for tc in tabconfigs: + self.current_table = tc.tableidx + for idx in tc.idxlist: + if idx.createset == createset: + self.create('index', tc.tablename, idx.idxname, idx.columns) + + self.finished_step('index' + str(createset)) + + # populate first batch + for tc in tabconfigs: + self.current_table = tc.tableidx + max = rand.rand_range(0, self.nentries) + self.populate(tc, xrange(0, max)) + + self.finished_step('populate0') + +#TODO + # Create indices in third set +# for tc in tabconfigs: +# for idx in tc.idxlist: +# if idx.createset == 2: +# self.create('index', tc.tablename, idx.idxname, idx.columns) + + self.finished_step('index2') + + # populate second batch + for tc in tabconfigs: + self.current_table = tc.tableidx + self.populate(tc, xrange(tc.nentries, self.nentries)) + + self.finished_step('populate1') + + for tc in tabconfigs: + self.current_table = tc.tableidx + self.check_entries(tc) + + def populate(self, tc, insertrange): + self.show_python("cursor = self.session.open_cursor('table:" + tc.tablename + "', None, None)") + cursor = self.session.open_cursor('table:' + tc.tablename, None, None) + for i in insertrange: + key = tc.gen_keys(i) + val = tc.gen_values(i) + self.show_python("cursor.set_key(*" + str(key) + ")") + cursor.set_key(*key) + self.show_python("cursor.set_value(*" + str(val) + ")") + cursor.set_value(*val) + self.show_python("cursor.insert()") + cursor.insert() + tc.nentries += 1 + self.show_python("cursor.close()") + cursor.close() + + def check_one(self, name, cursor, key, val): + keystr = str(key) + valstr = str(val) + self.show_python('# search[' + name + '](' + keystr + ')') + self.show_python("cursor.set_key(*" + keystr + ")") + cursor.set_key(*key) + self.show_python("ok = cursor.search()") + ok = cursor.search() + self.show_python("self.assertEqual(ok, 0)") + self.assertEqual(ok, 0) + self.show_python("self.assertEqual(" + keystr + ", cursor.get_keys())") + self.assertEqual(key, cursor.get_keys()) + self.show_python("self.assertEqual(" + valstr + ", cursor.get_values())") + self.assertEqual(val, cursor.get_values()) + + def check_entries(self, tc): + """ + Verify entries in the primary and index table + related to the tabconfig. + """ + self.show_python('# check_entries: ' + tc.tablename) + self.show_python("cursor = self.session.open_cursor('table:" + tc.tablename + "', None, None)") + cursor = self.session.open_cursor('table:' + tc.tablename, None, None) + count = 0 + for x in cursor: + count += 1 + self.assertEqual(count, tc.nentries) + for i in range(0, tc.nentries): + key = tc.gen_keys(i) + val = tc.gen_values(i) + self.check_one(tc.tablename, cursor, key, val) + cursor.close() + self.show_python("cursor.close()") + + # for each index, check each entry + for idx in tc.idxlist: + # KNOWN LIMITATION: Indices created after data population + # will have no entries, so don't bother with them here + # Remove these statements when the limitation is fixed. + if idx.createset == 2: + continue + # END KNOWN LIMITATION + + # Although it's possible to open an index on some partial + # list of columns, we'll keep it simple here, and always + # use all columns. + full_idxname = 'index:' + tc.tablename + ':' + idx.idxname + cols = '(' + ','.join([('c' + str(x)) for x in range(tc.nkeys, tc.nvalues + tc.nkeys)]) + ')' + self.show_python('# check_entries: ' + full_idxname + cols) + self.show_python("cursor = self.session.open_cursor('" + full_idxname + cols + "', None, None)") + cursor = self.session.open_cursor(full_idxname + cols, None, None) + count = 0 + for x in cursor: + count += 1 + self.assertEqual(count, tc.nentries) + for i in range(0, tc.nentries): + key = idx.gen_keys(i) + val = tc.gen_values(i) + self.check_one(full_idxname, cursor, key, val) + cursor.close() + self.show_python("cursor.close()") +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_stat01.py b/test/suite/test_stat01.py new file mode 100644 index 00000000000..008cfb2d4ac --- /dev/null +++ b/test/suite/test_stat01.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_stat01.py +# Statistics operations +# + +import unittest +import wiredtiger +import wttest +import test_base03 + +class test_stat01(wttest.WiredTigerTestCase): + """ + Test statistics + """ + + tablename = 'test_stat01.wt' + nentries = 25 + + def statstr_to_int(self, str): + """ + Convert a statistics value string, which may be in either form: + '12345' or '33M (33604836)' + """ + parts = str.rpartition('(') + return int(parts[2].rstrip(')')) + + def check_stats(self, statcursor, mincount, lookfor): + """ + Do a quick check of the entries in the the stats cursor, + There should be at least 'mincount' entries, + and the 'lookfor' string should appear + """ + stringclass = ''.__class__ + intclass = (0).__class__ + # make sure statistics basically look right + count = 0 + found = False + for id, desc, valstr, val in statcursor: + self.assertEqual(type(desc), stringclass) + self.assertEqual(type(valstr), stringclass) + self.assertEqual(type(val), intclass) + self.assertEqual(val, self.statstr_to_int(valstr)) + self.printVerbose(2, ' stat: \'' + desc + '\', \'' + + valstr + '\', ' + str(val)) + count += 1 + if desc == lookfor: + found = True + self.assertTrue(count > mincount) + self.assertTrue(found, 'in stats, did not see: ' + lookfor) + + def test_statistics(self): + extra_params = ',allocation_size=512,internal_page_max=16384,leaf_page_max=131072' + self.session.create('table:' + self.tablename, 'key_format=S,value_format=S' + extra_params) + cursor = self.session.open_cursor('table:' + self.tablename, None, None) + value = "" + for i in range(0, self.nentries): + key = str(i) + value = value + key + value # size grows exponentially + cursor.set_key(key) + cursor.set_value(value) + cursor.insert() + cursor.close() + + self.printVerbose(2, 'overall database stats:') + allstat_cursor = self.session.open_cursor('statistics:', None, None) + self.check_stats(allstat_cursor, 10, 'blocks written to a file') + + # See that we can get a specific stat value by its key, + # and verify that its entry is self-consistent + allstat_cursor.set_key(wiredtiger.stat.block_write) + self.assertEqual(allstat_cursor.search(), 0) + values = allstat_cursor.get_values() + self.assertEqual(values[0], 'blocks written to a file') + val = self.statstr_to_int(values[1]) + self.assertEqual(val, values[2]) + allstat_cursor.close() + + self.printVerbose(2, 'file specific stats:') + filestat_cursor = self.session.open_cursor('statistics:file:' + self.tablename + ".wt", None, None) + self.check_stats(filestat_cursor, 10, 'overflow pages') + + # See that we can get a specific stat value by its key, + # and verify that its entry is self-consistent + filestat_cursor.set_key(wiredtiger.filestat.overflow) + self.assertEqual(filestat_cursor.search(), 0) + values = filestat_cursor.get_values() + self.assertEqual(values[0], 'overflow pages') + val = self.statstr_to_int(values[1]) + self.assertEqual(val, values[2]) + filestat_cursor.close() + + no_cursor = self.session.open_cursor('statistics:file:DoesNotExist', None, None) + self.assertEqual(no_cursor, None) + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_util01.py b/test/suite/test_util01.py new file mode 100644 index 00000000000..6c1886def71 --- /dev/null +++ b/test/suite/test_util01.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_util01.py +# Utilities: wt dump, as well as the dump cursor +# + +import unittest +import wiredtiger +import wttest +from suite_subprocess import suite_subprocess +import os +import string + +class test_util01(wttest.WiredTigerTestCase, suite_subprocess): + """ + Test wt dump. We check for specific output. + Note that we don't test dumping {key,value}_format that are integer + here. That's because the integer values are encoded and we don't + want to duplicate the encoding/decoding algorithms. Integer dump + is tested implicity by test_util02 (which loads dumps created + in various ways). + """ + + tablename = 'test_util01.a' + nentries = 1000 + stringclass = ''.__class__ + + # python has a filecmp.cmp function, but different versions + # of python approach file comparison differently. To make + # sure we really get byte for byte comparison, we define it here. + + def compare_files(self, filename1, filename2): + bufsize = 4096 + if os.path.getsize(filename1) != os.path.getsize(filename2): + print filename1 + ' size = ' + str(os.path.getsize(filename1)) + print filename2 + ' size = ' + str(os.path.getsize(filename2)) + return False + with open(filename1, "rb") as fp1: + with open(filename2, "rb") as fp2: + while True: + b1 = fp1.read(bufsize) + b2 = fp2.read(bufsize) + if b1 != b2: + return False + # files are identical size + if not b1: + return True + + def get_string(self, i, len): + """ + Return a pseudo-random, but predictable string that uses + all characters. As a special case, key 0 returns all characters + 1-255 repeated + """ + ret = '' + if i == 0: + for j in range (0, len): + # we ensure that there are no internal nulls, that would + # truncate the string when we're using the 'S' encoding + # The last char in a string is null anyway, so that's tested. + ret += chr(j%255 + 1) + else: + for j in range(0, len / 3): + k = i + j + # no internal nulls... + ret += chr(k%255 + 1) + chr((k*3)%255 + 1) + chr((k*7)%255 + 1) + return ret + + def get_key(self, i): + return ("%0.6d" % i) + ':' + self.get_string(i, 20) + + def get_value(self, i): + return self.get_string(i, 1000) + + def dumpstr(self, s, hexoutput): + """ + Return a key or value string formatted just as 'wt dump' would. + Most printable characters (except tab, newline,...) are printed + as is, otherwise, backslash hex is used. + """ + result = '' + for c in s: + if hexoutput: + result += "%0.2x" % ord(c) + elif c == '\\': + result += '\\\\' + elif c == ' ' or (c in string.printable and not c in string.whitespace): + result += c + else: + result += '\\' + "%0.2x" % ord(c) + if hexoutput: + result += '00\n' + else: + result += '\\00\n' + return result + + def table_config(self): + return 'key_format=S,value_format=S' + + def dump(self, usingapi, hexoutput): + params = self.table_config() + self.session.create('table:' + self.tablename, params) + cursor = self.session.open_cursor('table:' + self.tablename, None, None) + ver = wiredtiger.wiredtiger_version() + verstring = str(ver[1]) + '.' + str(ver[2]) + '.' + str(ver[3]) + with open("expect.out", "w") as expectout: + if not usingapi: + # Note: this output is sensitive to the precise output format + # generated by wt dump. If this is likely to change, we should + # make this test more accommodating. + expectout.write('WiredTiger Dump (WiredTiger Version ' + verstring + ')\n') + if hexoutput: + expectout.write('Format=hex\n') + else: + expectout.write('Format=print\n') + expectout.write('Header\n') + expectout.write('table:' + self.tablename + '\n') + expectout.write('colgroups=(),columns=(),' + params + '\n') + expectout.write('Data\n') + for i in range(0, self.nentries): + key = self.get_key(i) + value = self.get_value(i) + cursor.set_key(key) + cursor.set_value(value) + cursor.insert() + expectout.write(self.dumpstr(key, hexoutput)) + expectout.write(self.dumpstr(value, hexoutput)) + cursor.close() + + self.pr('calling dump') + with open("dump.out", "w") as dumpout: + if usingapi: + if hexoutput: + dumpopt = "dump=hex" + else: + dumpopt = "dump=print" + dumpcurs = self.session.open_cursor('table:' + self.tablename, + None, dumpopt) + for key, val in dumpcurs: + dumpout.write(str(key) + "\n" + str(val) + "\n") + dumpcurs.close() + else: + dumpargs = ["dump"] + if hexoutput: + dumpargs.append("-x") + dumpargs.append(self.tablename) + self.runWt(dumpargs, outfilename="dump.out") + + self.assertTrue(self.compare_files("expect.out", "dump.out")) + + def test_dump_process(self): + self.dump(False, False) + + def test_dump_process_hex(self): + self.dump(False, True) + + def test_dump_api(self): + self.dump(True, False) + + def test_dump_api_hex(self): + self.dump(True, True) + + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_util02.py b/test/suite/test_util02.py new file mode 100644 index 00000000000..685c5d75179 --- /dev/null +++ b/test/suite/test_util02.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_util02.py +# Utilities: wt load +# + +import unittest +import wiredtiger +import wttest +from suite_subprocess import suite_subprocess +import os +import string + +class test_util02(wttest.WiredTigerTestCase, suite_subprocess): + """ + Test wt load + """ + + tablename = 'test_util02.a' + tablename2 = 'test_util02.b' + nentries = 1000 + stringclass = ''.__class__ + + scenarios = [ + ('SS', dict(key_format='S',value_format='S')), + ('rS', dict(key_format='r',value_format='S')), + ('ri', dict(key_format='r',value_format='i')), + ('ii', dict(key_format='i',value_format='i')), + ] + + # python has a filecmp.cmp function, but different versions + # of python approach file comparison differently. To make + # sure we really get byte for byte comparison, we define it here. + + def compare_files(self, filename1, filename2): + bufsize = 4096 + if os.path.getsize(filename1) != os.path.getsize(filename2): + print filename1 + ' size = ' + str(os.path.getsize(filename1)) + print filename2 + ' size = ' + str(os.path.getsize(filename2)) + return False + with open(filename1, "rb") as fp1: + with open(filename2, "rb") as fp2: + while True: + b1 = fp1.read(bufsize) + b2 = fp2.read(bufsize) + if b1 != b2: + return False + # files are identical size + if not b1: + return True + + def get_string(self, i, len): + """ + Return a pseudo-random, but predictable string that uses + all characters. As a special case, key 0 returns all characters + 1-255 repeated + """ + ret = '' + if i == 0: + for j in range (0, len): + # we ensure that there are no internal nulls, that would + # truncate the string when we're using the 'S' encoding + # The last char in a string is null anyway, so that's tested. + ret += chr(j%255 + 1) + else: + for j in range(0, len / 3): + k = i + j + # no internal nulls... + ret += chr(k%255 + 1) + chr((k*3)%255 + 1) + chr((k*7)%255 + 1) + return ret + + def get_key(self, i): + if self.key_format == 'S': + return ("%0.6d" % i) + ':' + self.get_string(i, 20) + elif self.key_format == 'r': + return long(i + 1) + else: + return i + 1 + + def get_value(self, i): + if self.value_format == 'S': + return self.get_string(i, 1000) + else: + # format is 'i' for this test + # return numbers throughout the 2^64 range + if i == 0: + return 0 + else: + if i < 64: + mask = (1 << i) - 1 + else: + mask = 0xffffffffffffffff + # multiply by a large prime get pseudo random bits + n = (i * 48112959837082048697) & mask + if n & 0x8000000000000000 != 0: + n = n - 0x10000000000000000 + return n + + def dumpstr(self, obj, hexoutput): + """ + Return a key or value string formatted just as 'wt dump' would. + Most printable characters (except tab, newline,...) are printed + as is, otherwise, backslash hex is used. + """ + result = '' + if type(obj) == self.stringclass: + for c in s: + if hexoutput: + result += "%0.2x" % ord(c) + elif c == '\\': + result += '\\\\' + elif c == ' ' or (c in string.printable and not c in string.whitespace): + result += c + else: + result += '\\' + "%0.2x" % ord(c) + if hexoutput: + result += '00\n' + else: + result += '\\00\n' + return result + + def table_config(self): + return 'key_format=' + self.key_format + ',value_format=' + self.value_format + + def load_process(self, hexoutput): + params = self.table_config() + self.session.create('table:' + self.tablename, params) + cursor = self.session.open_cursor('table:' + self.tablename, None, None) + for i in range(0, self.nentries): + key = self.get_key(i) + value = self.get_value(i) + cursor.set_key(key) + cursor.set_value(value) + cursor.insert() + cursor.close() + + dumpargs = ["dump"] + if hexoutput: + dumpargs.append("-x") + dumpargs.append(self.tablename) + self.runWt(dumpargs, outfilename="dump.out") + + # Create a placeholder for the new table. + self.session.create('table:' + self.tablename2, params) + + self.runWt(["load", "-f", "dump.out", "-r", self.tablename2]) + + cursor = self.session.open_cursor('table:' + self.tablename2, None, None) + self.assertEqual(cursor.key_format, self.key_format) + self.assertEqual(cursor.value_format, self.value_format) + i = 0 + for key, val in cursor: + self.assertEqual(key, self.get_key(i)) + self.assertEqual(val, self.get_value(i)) + i += 1 + cursor.close() + + def test_load_process(self): + self.load_process(False) + + def test_load_process_hex(self): + self.load_process(True) + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_util03.py b/test/suite/test_util03.py new file mode 100644 index 00000000000..1667ca89d62 --- /dev/null +++ b/test/suite/test_util03.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_util03.py +# Utilities: wt create +# + +import unittest +from wiredtiger import WiredTigerError +import wttest +from suite_subprocess import suite_subprocess + +class test_util03(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_util03.a' + nentries = 1000 + + scenarios = [ + ('none', dict(key_format=None,value_format=None)), + ('SS', dict(key_format='S',value_format='S')), + ('rS', dict(key_format='r',value_format='S')), + ('ri', dict(key_format='r',value_format='i')), + ] + + def test_create_process(self): + """ + Test create in a 'wt' process + """ + + args = ["create"] + if self.key_format != None or self.value_format != None: + args.append('-c') + config = '' + if self.key_format != None: + config += 'key_format=' + self.key_format + ',' + if self.value_format != None: + config += 'value_format=' + self.value_format + args.append(config) + args.append('table:' + self.tablename) + self.runWt(args) + + cursor = self.session.open_cursor('table:' + self.tablename, None, None) + if self.key_format != None: + self.assertEqual(cursor.key_format, self.key_format) + if self.value_format != None: + self.assertEqual(cursor.value_format, self.value_format) + for key,val in cursor: + self.fail('table should be empty') + cursor.close() + + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_util04.py b/test/suite/test_util04.py new file mode 100644 index 00000000000..196955e8d59 --- /dev/null +++ b/test/suite/test_util04.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_util04.py +# Utilities: wt drop +# + +import unittest +from wiredtiger import WiredTigerError +import wttest +from suite_subprocess import suite_subprocess +import os + +class test_util04(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_util04.a' + nentries = 1000 + + def test_drop_process(self): + """ + Test drop in a 'wt' process + """ + params = 'key_format=S,value_format=S' + self.session.create('table:' + self.tablename, params) + + self.assertTrue(os.path.exists(self.tablename + ".wt")) + self.runWt(["drop", "table:" + self.tablename]) + + self.assertFalse(os.path.exists(self.tablename + ".wt")) + self.assertRaises(WiredTigerError, lambda: self.session.open_cursor('table:' + self.tablename, None, None)) + + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_util05.py b/test/suite/test_util05.py new file mode 100644 index 00000000000..336c54d18b0 --- /dev/null +++ b/test/suite/test_util05.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_util05.py +# Utilities: wt verify +# + +import unittest +from wiredtiger import WiredTigerError +import wttest +from suite_subprocess import suite_subprocess +import os +import struct + +class test_util05(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_util05.a' + nentries = 1000 + + def populate(self, tablename): + """ + Insert some simple entries into the table + """ + cursor = self.session.open_cursor('table:' + tablename, None, None) + key = '' + for i in range(0, self.nentries): + key += str(i) + val = key + key + cursor.set_key(key) + cursor.set_value(val) + cursor.insert() + cursor.close() + + def check_populate(self, tablename): + """ + Verify that items added by populate are still there + """ + cursor = self.session.open_cursor('table:' + tablename, None, None) + wantkey = '' + i = 0 + for gotkey, gotval in cursor: + wantkey += str(i) + wantval = wantkey + wantkey + self.assertEqual(gotkey, wantkey) + self.assertEqual(gotval, wantval) + i += 1 + self.assertEqual(i, self.nentries) + cursor.close() + + def open_and_position(self, tablename, pct): + """ + Open the file for the table, position it at a 4K page + at roughly the given percentage into the file. + As a side effect, the connection is closed. + """ + # we close the connection to guarantee everything is + # flushed and closed from the WT point of view. + if self.conn != None: + self.conn.close() + self.conn = None + filename = tablename + ".wt" + + filesize = os.path.getsize(filename) + position = (filesize * pct) / 100 + + self.pr('damaging file at: ' + str(position)) + fp = open(filename, "r+b") + fp.seek(position) + return fp + + def test_verify_process_empty(self): + """ + Test verify in a 'wt' process, using an empty table + """ + params = 'key_format=S,value_format=S' + self.session.create('table:' + self.tablename, params) + # Run verify with an empty table + self.runWt(["verify", "table:" + self.tablename]) + + def test_verify_process(self): + """ + Test verify in a 'wt' process, using a populated table. + """ + params = 'key_format=S,value_format=S' + self.session.create('table:' + self.tablename, params) + self.populate(self.tablename) + self.runWt(["verify", "table:" + self.tablename]) + + def test_verify_api_empty(self): + """ + Test verify via API, using an empty table + """ + params = 'key_format=S,value_format=S' + self.session.create('table:' + self.tablename, params) + self.session.verify('table:' + self.tablename, None) + + def test_verify_api(self): + """ + Test verify via API, using a populated table. + """ + params = 'key_format=S,value_format=S' + self.session.create('table:' + self.tablename, params) + self.populate(self.tablename) + self.session.verify('table:' + self.tablename, None) + self.check_populate(self.tablename) + + def test_verify_api_75pct_null(self): + """ + Test verify via API, on a damaged table. + This is our only 'negative' test for verify using the API, + it's uncertain that we can have reliable tests for this. + """ + params = 'key_format=S,value_format=S' + self.session.create('table:' + self.tablename, params) + self.populate(self.tablename) + with self.open_and_position(self.tablename, 75) as f: + for i in range(0, 4096): + f.write(struct.pack('B', 0)) + + # open_and_position closed the session/connection, reopen them now. + self.conn = self.setUpConnectionOpen(".") + self.session = self.setUpSessionOpen(self.conn) + self.assertRaises(WiredTigerError, lambda: self.session.verify('table:' + self.tablename, None)) + + def test_verify_process_75pct_null(self): + """ + Test verify in a 'wt' process on a table that is purposely damaged, + with nulls at a position about 75% through. + """ + params = 'key_format=S,value_format=S' + self.session.create('table:' + self.tablename, params) + self.populate(self.tablename) + with self.open_and_position(self.tablename, 75) as f: + for i in range(0, 4096): + f.write(struct.pack('B', 0)) + self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out") + self.check_non_empty_file("verifyerr.out") + + def test_verify_process_25pct_junk(self): + """ + Test verify in a 'wt' process on a table that is purposely damaged, + with junk at a position about 25% through. + """ + params = 'key_format=S,value_format=S' + self.session.create('table:' + self.tablename, params) + self.populate(self.tablename) + with self.open_and_position(self.tablename, 25) as f: + for i in range(0, 100): + f.write('\x01\xff\x80') + self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out") + self.check_non_empty_file("verifyerr.out") + + def test_verify_process_appended_null(self): + """ + Test verify in a 'wt' process on a table that is purposely damaged, + with some null bytes at the end of the file. + """ + params = 'key_format=S,value_format=S' + self.session.create('table:' + self.tablename, params) + self.populate(self.tablename) + with self.open_and_position(self.tablename, 100) as f: + for i in range(0, 6): + f.write(struct.pack('B', 0)) + self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out") + self.check_non_empty_file("verifyerr.out") + + def test_verify_process_appended_null_block(self): + """ + Test verify in a 'wt' process on a table that is purposely damaged, + with some null bytes at the end of the file. + """ + params = 'key_format=S,value_format=S' + self.session.create('table:' + self.tablename, params) + self.populate(self.tablename) + with self.open_and_position(self.tablename, 100) as f: + for i in range(0, 4096): + f.write(struct.pack('B', 0)) + self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out") + self.check_non_empty_file("verifyerr.out") + + def test_verify_process_appended_junk(self): + """ + Test verify in a 'wt' process on a table that is purposely damaged, + with some junk bytes at the end of the file. + """ + params = 'key_format=S,value_format=S' + self.session.create('table:' + self.tablename, params) + self.populate(self.tablename) + with self.open_and_position(self.tablename, 100) as f: + for i in range(0, 1024): + f.write('\x01\0x02\x03\x04') + self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out") + self.check_non_empty_file("verifyerr.out") + + def test_verify_process_truncated(self): + """ + Test verify in a 'wt' process on a table that is purposely damaged, + truncated about 75% through. + """ + params = 'key_format=S,value_format=S' + self.session.create('table:' + self.tablename, params) + self.populate(self.tablename) + with self.open_and_position(self.tablename, 75) as f: + f.truncate(0) + self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out") + self.check_non_empty_file("verifyerr.out") + + def test_verify_process_zero_length(self): + """ + Test verify in a 'wt' process on a table that has junk added + """ + params = 'key_format=S,value_format=S' + self.session.create('table:' + self.tablename, params) + self.populate(self.tablename) + with self.open_and_position(self.tablename, 0) as f: + f.truncate(0) + self.runWt(["verify", "table:" + self.tablename], errfilename="verifyerr.out") + self.check_non_empty_file("verifyerr.out") + + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_util06.py b/test/suite/test_util06.py new file mode 100644 index 00000000000..511e436f7e6 --- /dev/null +++ b/test/suite/test_util06.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_util06.py +# Utilities: wt salvage +# + +import unittest +from wiredtiger import WiredTigerError +import wttest +from suite_subprocess import suite_subprocess +import os +import struct + +class test_util06(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_util06.a' + nentries = 1000 + session_params = 'key_format=S,value_format=S' + unique = 'SomeUniqueString' + + def populate(self, tablename): + """ + Insert some simple entries into the table + """ + cursor = self.session.open_cursor('table:' + tablename, None, None) + key = '' + for i in range(0, self.nentries): + key += str(i) + if i == self.nentries / 2: + val = self.unique + '0' + else: + val = key + key + cursor.set_key(key) + cursor.set_value(val) + cursor.insert() + cursor.close() + + def check_populate(self, tablename): + """ + Verify that items added by populate are still there + """ + cursor = self.session.open_cursor('table:' + tablename, None, None) + wantkey = '' + i = 0 + for gotkey, gotval in cursor: + wantkey += str(i) + if i == self.nentries / 2: + wantval = self.unique + '0' + else: + wantval = wantkey + wantkey + self.assertEqual(gotkey, wantkey) + self.assertTrue(gotval, wantval) + i += 1 + self.assertEqual(i, self.nentries) + cursor.close() + + def check_damaged(self, tablename): + """ + Check a damaged table with a lower standard than check_populate. + We don't require that all entries are here, + just that the ones that are here are correct. + """ + cursor = self.session.open_cursor('table:' + tablename, None, None) + wantkey = '' + i = -1 + correct = 0 + for gotkey, gotval in cursor: + i += 1 + wantkey += str(i) + if gotkey != wantkey: + continue + if i == self.nentries / 2: + wantval = self.unique + '0' + else: + wantval = wantkey + wantkey + self.assertEqual(gotkey, wantkey) + self.assertTrue(gotval, wantval) + correct += 1 + self.assertTrue(correct > 0) + self.printVerbose(2, 'after salvaging, file has ' + str(correct) + '/' + + str(self.nentries) + ' entries') + cursor.close() + + def check_empty_table(self, tablename): + cursor = self.session.open_cursor('table:' + tablename, None, None) + for gotkey, gotval in cursor: + self.fail(tablename + ': has unexpected entries') + cursor.close() + + def damage(self, tablename): + """ + Open the file for the table, find the unique string + and modify it. + """ + self.close_conn() + # we close the connection to guarantee everything is + # flushed and closed from the WT point of view. + filename = tablename + ".wt" + + fp = open(filename, "r+b") + found = matchpos = 0 + match = self.unique + matchlen = len(match) + c = fp.read(1) + while c: + if match[matchpos] == c: + matchpos += 1 + if matchpos == matchlen: + # We're already positioned, so alter it + fp.seek(-1, 1) + fp.write('G') + matchpos = 0 + found = 1 + else: + matchpos = 0 + c = fp.read(1) + # Make sure we found the embedded string + self.assertEqual(found, 1) + fp.close() + + def test_salvage_process_empty(self): + """ + Test salvage in a 'wt' process, using an empty table + """ + self.session.create('table:' + self.tablename, self.session_params) + errfile = "salvageerr.out" + self.runWt(["salvage", self.tablename + ".wt"], errfilename=errfile) + self.check_empty_file(errfile) + self.check_empty_table(self.tablename) + + def test_salvage_process(self): + """ + Test salvage in a 'wt' process, using a populated table. + """ + self.session.create('table:' + self.tablename, self.session_params) + self.populate(self.tablename) + errfile = "salvageerr.out" + self.runWt(["salvage", self.tablename + ".wt"], errfilename=errfile) + self.check_empty_file(errfile) + self.check_populate(self.tablename) + + def test_salvage_api_empty(self): + """ + Test salvage via API, using an empty table + """ + self.session.create('table:' + self.tablename, self.session_params) + self.session.salvage('table:' + self.tablename, None) + self.check_empty_table(self.tablename) + + def test_salvage_api(self): + """ + Test salvage via API, using a populated table. + """ + self.session.create('table:' + self.tablename, self.session_params) + self.populate(self.tablename) + self.session.salvage('file:' + self.tablename + ".wt", None) + self.check_populate(self.tablename) + + def test_salvage_api_open_handle(self): + """ + Test salvage via API, with an open connection/session. + It should raise an exception. + """ + self.session.create('table:' + self.tablename, self.session_params) + self.populate(self.tablename) + self.damage(self.tablename) + + # damage() closed the session/connection, reopen them now. + self.open_conn() + self.session.salvage('file:' + self.tablename + ".wt", None) + self.check_damaged(self.tablename) + + def test_salvage_api_damaged(self): + """ + Test salvage via API, on a damaged table. + """ + self.session.create('table:' + self.tablename, self.session_params) + self.populate(self.tablename) + self.damage(self.tablename) + + # damage() closed the session/connection, reopen them now. + self.open_conn() + self.assertRaises(WiredTigerError, lambda: self.session.verify('table:' + self.tablename, None)) + + self.session.salvage('file:' + self.tablename + ".wt", None) + self.check_damaged(self.tablename) + + def test_salvage_process_damaged(self): + """ + Test salvage in a 'wt' process on a table that is purposely damaged. + """ + self.session.create('table:' + self.tablename, self.session_params) + self.populate(self.tablename) + self.damage(self.tablename) + errfile = "salvageerr.out" + self.runWt(["salvage", self.tablename + ".wt"], errfilename=errfile) + self.check_empty_file(errfile) # expect no output + self.check_no_error_in_file(errfile) + self.check_damaged(self.tablename) + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_util07.py b/test/suite/test_util07.py new file mode 100644 index 00000000000..5d5467ba1cc --- /dev/null +++ b/test/suite/test_util07.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_util07.py +# Utilities: wt read +# + +import unittest +from wiredtiger import WiredTigerError +import wttest +from suite_subprocess import suite_subprocess +import os +import struct + +class test_util07(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_util07.a' + nentries = 1000 + session_params = 'key_format=S,value_format=S' + + def populate(self, tablename): + """ + Insert some simple entries into the table + """ + cursor = self.session.open_cursor('table:' + tablename, None, None) + for i in range(0, self.nentries): + key = 'KEY' + str(i) + val = 'VAL' + str(i) + cursor.set_key(key) + cursor.set_value(val) + cursor.insert() + cursor.close() + + def close_conn(self): + """ + Close the connection if already open. + """ + if self.conn != None: + self.conn.close() + self.conn = None + + def open_conn(self): + """ + Open the connection if already closed. + """ + if self.conn == None: + self.conn = self.setUpConnectionOpen(".") + self.session = self.setUpSessionOpen(self.conn) + + def test_read_empty(self): + """ + Test read in a 'wt' process, using an empty table + """ + self.session.create('table:' + self.tablename, self.session_params) + outfile = "readout.txt" + errfile = "readerr.txt" + self.runWt(["read", 'table:' + self.tablename, 'NoMatch'], outfilename=outfile, errfilename=errfile) + self.check_empty_file(outfile) + self.check_file_content(errfile, 'wt: NoMatch: not found\n') + + def test_read_populated(self): + """ + Test read in a 'wt' process, using an empty table + """ + self.session.create('table:' + self.tablename, self.session_params) + self.populate(self.tablename) + outfile = "readout.txt" + errfile = "readerr.txt" + self.runWt(["read", 'table:' + self.tablename, 'KEY49'], outfilename=outfile, errfilename=errfile) + self.check_file_content(outfile, 'VAL49\n') + self.check_empty_file(errfile) + self.runWt(["read", 'table:' + self.tablename, 'key49'], outfilename=outfile, errfilename=errfile) + self.check_empty_file(outfile) + self.check_file_content(errfile, 'wt: key49: not found\n') + + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_util08.py b/test/suite/test_util08.py new file mode 100644 index 00000000000..4c1ebc4eeb2 --- /dev/null +++ b/test/suite/test_util08.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_util08.py +# Utilities: wt copyright +# + +import unittest +from wiredtiger import WiredTigerError +import wttest +from suite_subprocess import suite_subprocess + +class test_util08(wttest.WiredTigerTestCase, suite_subprocess): + def test_copyright(self): + """ + Test copyright in a 'wt' process + """ + outfile = "copyrightout.txt" + self.runWt(["copyright"], outfilename=outfile) + with open(outfile, 'r') as f: + text = f.read(1000) + self.assertTrue('Copyright' in text) + + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_util09.py b/test/suite/test_util09.py new file mode 100644 index 00000000000..f0d4aadc893 --- /dev/null +++ b/test/suite/test_util09.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_util09.py +# Utilities: wt loadtext +# + +import unittest +from wiredtiger import WiredTigerError +import wttest +from suite_subprocess import suite_subprocess +import os +import struct + +class test_util09(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_util09.a' + nentries = 1000 + session_params = 'key_format=S,value_format=S' + + def populate_file(self, filename, low, high): + """ + Insert some simple key / value lines into the file + """ + keys = {} + with open("loadtext.in", "w") as f: + for i in range(low, high): + key = str(i) + str(i) + val = key + key + key + f.write(key + '\n') + f.write(val + '\n') + keys[key] = val + #print 'Populated ' + str(len(keys)) + return keys + + def check_keys(self, tablename, keys): + """ + Check that all the values in the table match the saved dictionary. + Values in the dictionary are removed as a side effect. + """ + cursor = self.session.open_cursor('table:' + tablename, None, None) + for key, val in cursor: + self.assertEqual(keys[key], val) + del keys[key] + cursor.close() + self.assertEqual(len(keys), 0) + + def test_loadtext_empty(self): + """ + Test loadtext in a 'wt' process, using an empty table + """ + self.session.create('table:' + self.tablename, self.session_params) + keys = self.populate_file("loadtext.in", 0, 0) + self.runWt(["loadtext", "-f", "loadtext.in", "table:" + self.tablename]) + self.check_keys(self.tablename, keys) + + def test_loadtext_empty_stdin(self): + """ + Test loadtext in a 'wt' process using stdin, using an empty table + """ + self.session.create('table:' + self.tablename, self.session_params) + keys = self.populate_file("loadtext.in", 0, 0) + self.runWt(["loadtext", "table:" + self.tablename], infilename="loadtext.in") + self.check_keys(self.tablename, keys) + + def test_loadtext_populated(self): + """ + Test loadtext in a 'wt' process, creating entries in a table + """ + self.session.create('table:' + self.tablename, self.session_params) + keys = self.populate_file("loadtext.in", 1010, 1220) + self.runWt(["loadtext", "-f", "loadtext.in", "table:" + self.tablename]) + self.check_keys(self.tablename, keys) + + def test_loadtext_populated_stdin(self): + """ + Test loadtext in a 'wt' process using stding, creating entries in a table + """ + self.session.create('table:' + self.tablename, self.session_params) + keys = self.populate_file("loadtext.in", 200, 300) + self.runWt(["loadtext", "table:" + self.tablename], infilename="loadtext.in") + self.check_keys(self.tablename, keys) + + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_util10.py b/test/suite/test_util10.py new file mode 100644 index 00000000000..e79ab09476a --- /dev/null +++ b/test/suite/test_util10.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_util10.py +# Utilities: wt dumpfile +# + +import unittest +from wiredtiger import WiredTigerError +import wttest +from suite_subprocess import suite_subprocess +import os +import struct + +class test_util10(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_util10.a' + nentries = 1000 + session_params = 'key_format=S,value_format=S' + + def populate(self, tablename): + """ + Insert some simple entries into the table + """ + cursor = self.session.open_cursor('table:' + tablename, None, None) + for i in range(0, self.nentries): + key = 'KEY' + str(i) + val = 'VAL' + str(i) + cursor.set_key(key) + cursor.set_value(val) + cursor.insert() + cursor.set_key('SOMEKEY') + cursor.set_value('SOMEVALUE') + cursor.insert() + cursor.set_key('ANOTHERKEY') + cursor.set_value('ANOTHERVALUE') + cursor.insert() + cursor.close() + + def test_dumpfile_empty(self): + """ + Test read in a 'wt' process, using an empty table + """ + self.session.create('table:' + self.tablename, self.session_params) + outfile = "dumpfileout.txt" + self.runWt(["dumpfile", self.tablename + ".wt"], outfilename=outfile) + self.check_empty_file(outfile) + + def test_dumpfile_populated(self): + """ + Test read in a 'wt' process, using an empty table + """ + self.session.create('table:' + self.tablename, self.session_params) + self.populate(self.tablename) + outfile = "dumpfileout.txt" + self.runWt(["dumpfile", self.tablename + ".wt"], outfilename=outfile) + + # Expected output is roughly K/V pairs in this format: + # K {xxxxxx#00} + # V {xxxxxx#00} + # except that by default keys use prefix compression. + # 'KEY340' would not be found in the output, but rather K {0#00} + # because it appears immediately after 'KEY34' so uses the five + # bytes of that key. We've chosen keys to find that will not be + # compressed. + self.check_file_contains(outfile, 'V {VAL22#00}') + self.check_file_contains(outfile, 'K {KEY0#00}') + self.check_file_contains(outfile, 'K {SOMEKEY#00}') + self.check_file_contains(outfile, 'V {SOMEVALUE#00}') + self.check_file_contains(outfile, 'K {SOMEKEY#00}') + self.check_file_contains(outfile, 'V {ANOTHERVALUE#00}') + + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_util11.py b/test/suite/test_util11.py new file mode 100644 index 00000000000..d5b9fb3f9c2 --- /dev/null +++ b/test/suite/test_util11.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_util11.py +# Utilities: wt list +# + +import unittest +from wiredtiger import WiredTigerError +import wttest +from suite_subprocess import suite_subprocess +import os +import struct + +class test_util11(wttest.WiredTigerTestCase, suite_subprocess): + tablenamepfx = 'test_util11.' + session_params = 'key_format=S,value_format=S' + + def populate(self, tablename): + """ + Insert some simple entries into the table + """ + cursor = self.session.open_cursor('table:' + tablename, None, None) + cursor.set_key('SOMEKEY') + cursor.set_value('SOMEVALUE') + cursor.close() + + def test_list_none(self): + """ + Test list in a 'wt' process, with no tables + """ + + # Construct what we think we'll find + filelist = '' + outfile = "listout.txt" + self.runWt(["list"], outfilename=outfile) + self.check_file_content(outfile, filelist) + + def test_list(self): + """ + Test list in a 'wt' process, with a mix of populated and empty tables + """ + pfx = self.tablenamepfx + params = self.session_params + self.session.create('table:' + pfx + '5', params) + self.session.create('table:' + pfx + '3', params) + self.session.create('table:' + pfx + '1', params) + self.session.create('table:' + pfx + '2', params) + self.session.create('table:' + pfx + '4', params) + self.populate(pfx + '2') + self.populate(pfx + '3') + + # Construct what we think we'll find + filelist = 'file:WiredTiger.wt\n' + tablelist = '' + for i in range(1, 6): + filelist += 'file:' + pfx + str(i) + '.wt\n' + tablelist += 'table:' + pfx + str(i) + '\n' + + outfile = "listout.txt" + self.runWt(["list"], outfilename=outfile) + self.check_file_content(outfile, filelist + tablelist) + + def test_list_drop(self): + """ + Test list in a 'wt' process, with a mix of populated and empty tables, + after some tables have been dropped. + """ + pfx = self.tablenamepfx + params = self.session_params + self.session.create('table:' + pfx + '5', params) + self.session.create('table:' + pfx + '3', params) + self.session.create('table:' + pfx + '1', params) + self.session.create('table:' + pfx + '2', params) + self.session.create('table:' + pfx + '4', params) + self.populate(pfx + '2') + self.populate(pfx + '3') + self.session.drop('table:' + pfx + '2', None) + self.session.drop('table:' + pfx + '4', None) + + # Construct what we think we'll find + filelist = 'file:WiredTiger.wt\n' + tablelist = '' + filelist += 'file:' + pfx + '1.wt\n' + tablelist += 'table:' + pfx + '1\n' + filelist += 'file:' + pfx + '3.wt\n' + tablelist += 'table:' + pfx + '3\n' + filelist += 'file:' + pfx + '5.wt\n' + tablelist += 'table:' + pfx + '5\n' + + outfile = "listout.txt" + self.runWt(["list"], outfilename=outfile) + self.check_file_content(outfile, filelist + tablelist) + + def test_list_drop_all(self): + """ + Test list in a 'wt' process, with a mix of populated and empty tables, + after all tables have been dropped. + """ + pfx = self.tablenamepfx + params = self.session_params + self.session.create('table:' + pfx + '5', params) + self.session.create('table:' + pfx + '3', params) + self.session.create('table:' + pfx + '1', params) + self.session.create('table:' + pfx + '2', params) + self.session.create('table:' + pfx + '4', params) + self.populate(pfx + '2') + self.populate(pfx + '3') + self.session.drop('table:' + pfx + '5', None) + self.session.drop('table:' + pfx + '4', None) + self.session.drop('table:' + pfx + '3', None) + self.session.drop('table:' + pfx + '2', None) + self.session.drop('table:' + pfx + '1', None) + + # Construct what we think we'll find + filelist = 'file:WiredTiger.wt\n' + outfile = "listout.txt" + self.runWt(["list"], outfilename=outfile) + self.check_file_content(outfile, filelist) + + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/test_util12.py b/test/suite/test_util12.py new file mode 100644 index 00000000000..20f8270ffe9 --- /dev/null +++ b/test/suite/test_util12.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_util12.py +# Utilities: wt write +# + +import unittest +import wiredtiger +from wiredtiger import WiredTigerError +import wttest +from suite_subprocess import suite_subprocess +import os +import struct + +class test_util12(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_util12.a' + nentries = 1000 + session_params = 'key_format=S,value_format=S' + + def populate(self, tablename): + """ + Insert some simple entries into the table + """ + cursor = self.session.open_cursor('table:' + tablename, None, None) + cursor.set_key('SOMEKEY') + cursor.set_value('SOMEVALUE') + cursor.close() + + def test_write(self): + self.session.create('table:' + self.tablename, self.session_params) + self.runWt(['write', 'table:' + self.tablename, + 'def', '456', 'abc', '123']) + cursor = self.session.open_cursor('table:' + self.tablename, None, None) + self.assertEqual(cursor.next(), 0) + self.assertEqual(cursor.get_key(), 'abc') + self.assertEqual(cursor.get_value(), '123') + self.assertEqual(cursor.next(), 0) + self.assertEqual(cursor.get_key(), 'def') + self.assertEqual(cursor.get_value(), '456') + self.assertEqual(cursor.next(), wiredtiger.WT_NOTFOUND) + cursor.close() + + def test_write_no_keys(self): + """ + Test write in a 'wt' process, with no args + """ + self.session.create('table:' + self.tablename, self.session_params) + + errfile = 'writeerr.txt' + self.runWt(['write', 'table:' + self.tablename], errfilename=errfile) + self.check_file_contains(errfile, 'usage:') + + def test_write_overwrite(self): + self.session.create('table:' + self.tablename, self.session_params) + cursor = self.session.open_cursor('table:' + self.tablename, None, None) + cursor.set_key('def') + cursor.set_value('789') + cursor.close() + self.runWt(['write', 'table:' + self.tablename, + 'def', '456', 'abc', '123']) + cursor = self.session.open_cursor('table:' + self.tablename, None, None) + self.assertEqual(cursor.next(), 0) + self.assertEqual(cursor.get_key(), 'abc') + self.assertEqual(cursor.get_value(), '123') + self.assertEqual(cursor.next(), 0) + self.assertEqual(cursor.get_key(), 'def') + self.assertEqual(cursor.get_value(), '456') + self.assertEqual(cursor.next(), wiredtiger.WT_NOTFOUND) + cursor.close() + + def test_write_bad_args(self): + self.session.create('table:' + self.tablename, self.session_params) + errfile = 'writeerr.txt' + self.runWt(['write', 'table:' + self.tablename, + 'def', '456', 'abc'], errfilename=errfile) + self.check_file_contains(errfile, 'usage:') + + +if __name__ == '__main__': + wttest.run() diff --git a/test/suite/valgrind-python.supp b/test/suite/valgrind-python.supp new file mode 100644 index 00000000000..43c99c4a3af --- /dev/null +++ b/test/suite/valgrind-python.supp @@ -0,0 +1,396 @@ +# +# This is a valgrind suppression file that should be used when using valgrind. +# +# Here's an example of running valgrind: +# +# cd python/dist/src +# valgrind --tool=memcheck --suppressions=Misc/valgrind-python.supp \ +# ./python -E -tt ./Lib/test/regrtest.py -u bsddb,network +# +# You must edit Objects/obmalloc.c and uncomment Py_USING_MEMORY_DEBUGGER +# to use the preferred suppressions with Py_ADDRESS_IN_RANGE. +# +# If you do not want to recompile Python, you can uncomment +# suppressions for PyObject_Free and PyObject_Realloc. +# +# See Misc/README.valgrind for more information. + +# all tool names: Addrcheck,Memcheck,cachegrind,helgrind,massif +{ + ADDRESS_IN_RANGE/Invalid read of size 4 + Memcheck:Addr4 + fun:Py_ADDRESS_IN_RANGE +} + +{ + ADDRESS_IN_RANGE/Invalid read of size 4 + Memcheck:Value4 + fun:Py_ADDRESS_IN_RANGE +} + +{ + ADDRESS_IN_RANGE/Invalid read of size 8 (x86_64 aka amd64) + Memcheck:Value8 + fun:Py_ADDRESS_IN_RANGE +} + +{ + ADDRESS_IN_RANGE/Conditional jump or move depends on uninitialised value + Memcheck:Cond + fun:Py_ADDRESS_IN_RANGE +} + +# +# Leaks (including possible leaks) +# Hmmm, I wonder if this masks some real leaks. I think it does. +# Will need to fix that. +# + +{ + Suppress leaking the GIL. Happens once per process, see comment in ceval.c. + Memcheck:Leak + fun:malloc + fun:PyThread_allocate_lock + fun:PyEval_InitThreads +} + +{ + Suppress leaking the GIL after a fork. + Memcheck:Leak + fun:malloc + fun:PyThread_allocate_lock + fun:PyEval_ReInitThreads +} + +{ + Suppress leaking the autoTLSkey. This looks like it shouldn't leak though. + Memcheck:Leak + fun:malloc + fun:PyThread_create_key + fun:_PyGILState_Init + fun:Py_InitializeEx + fun:Py_Main +} + +{ + Hmmm, is this a real leak or like the GIL? + Memcheck:Leak + fun:malloc + fun:PyThread_ReInitTLS +} + +{ + Handle PyMalloc confusing valgrind (possibly leaked) + Memcheck:Leak + fun:realloc + fun:_PyObject_GC_Resize + fun:COMMENT_THIS_LINE_TO_DISABLE_LEAK_WARNING +} + +{ + Handle PyMalloc confusing valgrind (possibly leaked) + Memcheck:Leak + fun:malloc + fun:_PyObject_GC_New + fun:COMMENT_THIS_LINE_TO_DISABLE_LEAK_WARNING +} + +{ + Handle PyMalloc confusing valgrind (possibly leaked) + Memcheck:Leak + fun:malloc + fun:_PyObject_GC_NewVar + fun:COMMENT_THIS_LINE_TO_DISABLE_LEAK_WARNING +} + +# +# Non-python specific leaks +# + +{ + Handle pthread issue (possibly leaked) + Memcheck:Leak + fun:calloc + fun:allocate_dtv + fun:_dl_allocate_tls_storage + fun:_dl_allocate_tls +} + +{ + Handle pthread issue (possibly leaked) + Memcheck:Leak + fun:memalign + fun:_dl_allocate_tls_storage + fun:_dl_allocate_tls +} + +# +# Python's malloc / realloc / free functions +# + +{ + ADDRESS_IN_RANGE/Invalid read of size 4 + Memcheck:Addr4 + fun:PyObject_Free +} + +{ + ADDRESS_IN_RANGE/Invalid read of size 8 + Memcheck:Addr8 + fun:PyObject_Free +} + +{ + ADDRESS_IN_RANGE/Invalid read of size 4 + Memcheck:Value4 + fun:PyObject_Free +} + +{ + ADDRESS_IN_RANGE/Invalid read of size 8 + Memcheck:Value8 + fun:PyObject_Free +} + +{ + ADDRESS_IN_RANGE/Conditional jump or move depends on uninitialised value + Memcheck:Cond + fun:PyObject_Free +} + +{ + ADDRESS_IN_RANGE/Invalid read of size 4 + Memcheck:Addr4 + fun:PyObject_Realloc +} + +{ + ADDRESS_IN_RANGE/Invalid read of size 8 + Memcheck:Addr8 + fun:PyObject_Realloc +} + +{ + ADDRESS_IN_RANGE/Invalid read of size 4 + Memcheck:Value4 + fun:PyObject_Realloc +} + +{ + ADDRESS_IN_RANGE/Invalid read of size 8 + Memcheck:Value8 + fun:PyObject_Realloc +} + +{ + ADDRESS_IN_RANGE/Conditional jump or move depends on uninitialised value + Memcheck:Cond + fun:PyObject_Realloc +} + +{ + ADDRESS_IN_RANGE/Use of uninitialised value of size 8 + Memcheck:Cond + fun:PyObject_Realloc +} + +### +### All the suppressions below are for errors that occur within libraries +### that Python uses. The problems to not appear to be related to Python's +### use of the libraries. +### + +{ + Generic ubuntu ld problems + Memcheck:Addr8 + obj:/lib/ld-2.4.so + obj:/lib/ld-2.4.so + obj:/lib/ld-2.4.so + obj:/lib/ld-2.4.so +} + +{ + Generic gentoo ld problems + Memcheck:Cond + obj:/lib/ld-2.3.4.so + obj:/lib/ld-2.3.4.so + obj:/lib/ld-2.3.4.so + obj:/lib/ld-2.3.4.so +} + +{ + DBM problems, see test_dbm + Memcheck:Param + write(buf) + fun:write + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + fun:dbm_close +} + +{ + DBM problems, see test_dbm + Memcheck:Value8 + fun:memmove + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + fun:dbm_store + fun:dbm_ass_sub +} + +{ + DBM problems, see test_dbm + Memcheck:Cond + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + fun:dbm_store + fun:dbm_ass_sub +} + +{ + DBM problems, see test_dbm + Memcheck:Cond + fun:memmove + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + fun:dbm_store + fun:dbm_ass_sub +} + +{ + GDBM problems, see test_gdbm + Memcheck:Param + write(buf) + fun:write + fun:gdbm_open + +} + +{ + ZLIB problems, see test_gzip + Memcheck:Cond + obj:/lib/libz.so.1.2.3 + obj:/lib/libz.so.1.2.3 + fun:deflate +} + +{ + Avoid problems w/readline doing a putenv and leaking on exit + Memcheck:Leak + fun:malloc + fun:xmalloc + fun:sh_set_lines_and_columns + fun:_rl_get_screen_size + fun:_rl_init_terminal_io + obj:/lib/libreadline.so.4.3 + fun:rl_initialize +} + +# +# All of these problems come from using test_socket_ssl +# +{ + from test_socket_ssl + Memcheck:Cond + fun:BN_bin2bn +} + +{ + from test_socket_ssl + Memcheck:Cond + fun:BN_num_bits_word +} + +{ + from test_socket_ssl + Memcheck:Value4 + fun:BN_num_bits_word +} + +{ + from test_socket_ssl + Memcheck:Cond + fun:BN_mod_exp_mont_word +} + +{ + from test_socket_ssl + Memcheck:Cond + fun:BN_mod_exp_mont +} + +{ + from test_socket_ssl + Memcheck:Param + write(buf) + fun:write + obj:/usr/lib/libcrypto.so.0.9.7 +} + +{ + from test_socket_ssl + Memcheck:Cond + fun:RSA_verify +} + +{ + from test_socket_ssl + Memcheck:Value4 + fun:RSA_verify +} + +{ + from test_socket_ssl + Memcheck:Value4 + fun:DES_set_key_unchecked +} + +{ + from test_socket_ssl + Memcheck:Value4 + fun:DES_encrypt2 +} + +{ + from test_socket_ssl + Memcheck:Cond + obj:/usr/lib/libssl.so.0.9.7 +} + +{ + from test_socket_ssl + Memcheck:Value4 + obj:/usr/lib/libssl.so.0.9.7 +} + +{ + from test_socket_ssl + Memcheck:Cond + fun:BUF_MEM_grow_clean +} + +{ + from test_socket_ssl + Memcheck:Cond + fun:memcpy + fun:ssl3_read_bytes +} + +{ + from test_socket_ssl + Memcheck:Cond + fun:SHA1_Update +} + +{ + from test_socket_ssl + Memcheck:Value4 + fun:SHA1_Update +} diff --git a/test/suite/wtscenario.py b/test/suite/wtscenario.py new file mode 100644 index 00000000000..82322d65cb2 --- /dev/null +++ b/test/suite/wtscenario.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# wtscenarios.py +# Support scenarios based testing +# + +import testscenarios +import suite_random + +def powerrange(start, stop, mult): + """ + Like xrange, generates a range from start to stop. + Unlike xrange, the range is inclusive of stop, + each step is multiplicative, and as a special case, + the stop value is returned as the last item. + """ + val = start + while val <= stop: + yield val + newval = val * mult + if val < stop and newval > stop: + val = stop + else: + val = newval + +def log2chr(val): + """ + For the log-base 2 of val, return the numeral or letter + corresponding to val (which is < 36). Hence, 1 return '0', + 2 return '1', 2*15 returns 'f', 2*16 returns 'g', etc. + """ + p = 0 + while val >= 2: + p += 1 + val /= 2 + if p < 10: + return chr(ord('0') + p) + else: + return chr(ord('a') + p - 10) + +megabyte = 1024 * 1024 + +def multiply_scenarios(sep, *args): + """ + Create the cross product of two lists of scenarios + """ + result = None + for scenes in args: + if result == None: + result = scenes + else: + total = [] + for scena in scenes: + for scenb in result: + # Create a merged scenario with a concatenated name + name = scena[0] + sep + scenb[0] + tdict = {} + tdict.update(scena[1]) + tdict.update(scenb[1]) + + # If there is a 'P' value, it represents the + # probability that we want to use this scenario + # If both scenarios list a probability, multiply them. + if 'P' in scena[1] and 'P' in scenb[1]: + P = scena[1]['P'] * scenb[1]['P'] + tdict['P'] = P + total.append((name, tdict)) + result = total + return result + +def prune_sorter_key(scene): + """ + Used by prune_scenerios to extract key for sorting. + The key is the saved random value multiplied by + the probability of choosing. + """ + p = 1.0 + if 'P' in scene[1]: + p = scene[1]['P'] + return p * scene[1]['_rand'] + +def prune_scenarios(scenes, count = -1): + """ + Use listed probabilities for pruning the list of scenarios. + That is, the highest probability (value of P in the scendario) + are chosen more often. With a second argument, only the + given number of scenarios are returned. With no second argument, + only scenarios with P > .5 are returned half the time, etc. + """ + r = suite_random.suite_random() + result = [] + if count == -1: + # Missing second arg - return those with P == .3 at + # 30% probability, for example. + for scene in scenes: + if 'P' in scene[1]: + p = scene[1]['P'] + if p < r.rand_float(): + continue + result.append(scene) + return result + else: + # With second arg, we want exactly 'count' items + # returned. So we'll sort them all and choose + # the top number. Not the most efficient solution, + # but it's easy. + for scene in scenes: + scene[1]['_rand'] = r.rand_float() + scenes = sorted(scenes, key=prune_sorter_key) + for scene in scenes: + del scene[1]['_rand'] + l = len(scenes) + return scenes[l-count:l] + +def number_scenarios(scenes): + """ + Add a 'scenario_number' variable to each scenario. + The hash table for each scenario is altered! + """ + count = 0 + for scene in scenes: + scene[1]['scenario_number'] = count + count += 1 + return scenes + +def quick_scenarios(fieldname, values, probabilities): + """ + Quickly build common scenarios, like: + [('foo', dict(somefieldname='foo')), + ('bar', dict(somefieldname='bar')), + ('boo', dict(somefieldname='boo'))] + via a call to: + quick_scenario('somefieldname', ['foo', 'bar', 'boo']) + """ + result = [] + if probabilities == None: + plen = 0 + else: + plen = len(probabilities) + ppos = 0 + for value in values: + if ppos >= plen: + d = dict([[fieldname, value]]) + else: + p = probabilities[ppos] + ppos += 1 + d = dict([[fieldname, value],['P', p]]) + result.append((str(value), d)) + return result + +class wtscenario: + """ + A set of generators for different test scenarios + """ + + @staticmethod + def session_create_scenario(): + """ + Return a set of scenarios with the name of this method + 'session_create_scenario' as the name of instance + variable containing a wtscenario object. The wtscenario + object can be queried to get a config string. + Each scenario is named according to the shortName() method. + """ + s = [ + ('default', dict(session_create_scenario=wtscenario())) ] + for imin in powerrange(512, 512*megabyte, 1024): + for imax in powerrange(imin, 512*megabyte, 1024): + for lmin in powerrange(512, 512*megabyte, 1024): + for lmax in powerrange(lmin, 512*megabyte, 1024): + for cache in [megabyte, 32*megabyte, 1000*megabyte]: + scen = wtscenario() + scen.ioverflow = max(imin / 40, 40) + scen.imax = imax + scen.loverflow = max(lmin / 40, 40) + scen.lmax = lmax + scen.cache_size = cache + s.append((scen.shortName(), dict(session_create_scenario=scen))) + return s + + def shortName(self): + """ + Return a name of a scenario, based on the 'log2chr-ed numerals' + representing the four values for {internal,leaf} {minimum, maximum} + page size. + """ + return 'scen_' + log2chr(self.ioverflow) + log2chr(self.imax) + log2chr(self.loverflow) + log2chr(self.lmax) + log2chr(self.cache_size) + + def configString(self): + """ + Return the associated configuration string + """ + res = '' + if hasattr(self, 'ioverflow'): + res += ',internal_item_max=' + str(self.ioverflow) + if hasattr(self, 'imax'): + res += ',internal_page_max=' + str(self.imax) + if hasattr(self, 'loverflow'): + res += ',leaf_item_max=' + str(self.loverflow) + if hasattr(self, 'lmax'): + res += ',leaf_page_max=' + str(self.lmax) + return res diff --git a/test/suite/wttest.py b/test/suite/wttest.py new file mode 100644 index 00000000000..1c5f0dabc11 --- /dev/null +++ b/test/suite/wttest.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python +# +# Copyright (c) 2008-2012 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# WiredTigerTestCase +# parent class for all test cases +# + +import unittest +import sys +import os +import wiredtiger +import traceback +import time + +def removeAll(top): + if not os.path.isdir(top): + return + for root, dirs, files in os.walk(top, topdown=False): + for name in files: + os.remove(os.path.join(root, name)) + for name in dirs: + os.rmdir(os.path.join(root, name)) + os.rmdir(top) + +class WiredTigerTestCase(unittest.TestCase): + _globalSetup = False + _printOnceSeen = {} + + @staticmethod + def globalSetup(preserveFiles = False, useTimestamp = False, + gdbSub = False, verbose = 1): + WiredTigerTestCase._preserveFiles = preserveFiles + if useTimestamp: + d = 'WT_TEST.' + time.strftime('%Y%m%d-%H%M%S', time.localtime()) + else: + d = 'WT_TEST' + removeAll(d) + os.makedirs(d) + WiredTigerTestCase._parentTestdir = d + WiredTigerTestCase._resultfile = open(os.path.join(d, 'results.txt'), "w", 0) # unbuffered + WiredTigerTestCase._gdbSubprocess = gdbSub + WiredTigerTestCase._verbose = verbose + WiredTigerTestCase._globalSetup = True + + def __init__(self, *args, **kwargs): + unittest.TestCase.__init__(self, *args, **kwargs) + if not self._globalSetup: + WiredTigerTestCase.globalSetup() + + def __str__(self): + # when running with scenarios, if the number_scenarios() method + # is used, then each scenario is given a number, which can + # help distinguish tests. + scen = '' + if hasattr(self, 'scenario_number'): + scen = '(scenario ' + str(self.scenario_number) + ')' + return self.simpleName() + scen + + def simpleName(self): + return "%s.%s.%s" % (self.__module__, + self.className(), self._testMethodName) + + # Can be overridden + def setUpConnectionOpen(self, dir): + conn = wiredtiger.wiredtiger_open(dir, 'create,error_prefix="' + + self.shortid() + ': ' + '"') + self.pr(`conn`) + return conn + + # Can be overridden + def setUpSessionOpen(self, conn): + return conn.open_session(None) + + # Can be overridden + def close_conn(self): + """ + Close the connection if already open. + """ + if self.conn != None: + self.conn.close() + self.conn = None + + def open_conn(self): + """ + Open the connection if already closed. + """ + if self.conn == None: + self.conn = self.setUpConnectionOpen(".") + self.session = self.setUpSessionOpen(self.conn) + + def reopen_conn(self): + """ + Reopen the connection. + """ + self.close_conn() + self.open_conn() + + def setUp(self): + if not hasattr(self.__class__, 'wt_ntests'): + self.__class__.wt_ntests = 0 + self.__class__.wt_ntests += 1 + self.testdir = os.path.join(WiredTigerTestCase._parentTestdir, self.className() + '.' + str(self.__class__.wt_ntests)) + if WiredTigerTestCase._verbose > 2: + self.prhead('started in ' + self.testdir, True) + self.origcwd = os.getcwd() + removeAll(self.testdir) + if os.path.exists(self.testdir): + raise Exception(self.testdir + ": cannot remove directory"); + os.makedirs(self.testdir) + try: + os.chdir(self.testdir) + self.conn = self.setUpConnectionOpen(".") + self.session = self.setUpSessionOpen(self.conn) + except: + os.chdir(self.origcwd) + raise + + def tearDown(self): + self.pr('finishing') + self.close_conn() + os.chdir(self.origcwd) + # Clean up unless there's a failure + excinfo = sys.exc_info() + if excinfo == (None, None, None): + if not WiredTigerTestCase._preserveFiles: + removeAll(self.testdir) + else: + self.pr('preserving directory ' + self.testdir) + else: + self.pr('FAIL') + self.prexception(excinfo) + self.pr('preserving directory ' + self.testdir) + if WiredTigerTestCase._verbose > 2: + self.prhead('TEST COMPLETED') + + @staticmethod + def printOnce(msg): + # There's a race condition with multiple threads, + # but we won't worry about it. We err on the side + # of printing the message too many times. + if not msg in WiredTigerTestCase._printOnceSeen: + WiredTigerTestCase._printOnceSeen[msg] = msg + print msg + + def KNOWN_FAILURE(self, name): + myname = self.simpleName() + msg = '**** ' + myname + ' HAS A KNOWN FAILURE: ' + name + ' ****' + self.printOnce(msg) + self.skipTest('KNOWN FAILURE: ' + name) + + def KNOWN_LIMITATION(self, name): + myname = self.simpleName() + msg = '**** ' + myname + ' HAS A KNOWN LIMITATION: ' + name + ' ****' + self.printOnce(msg) + + @staticmethod + def printVerbose(level, message): + if level <= WiredTigerTestCase._verbose: + print message + + def verbose(self, level, message): + WiredTigerTestCase.printVerbose(level, message) + + def pr(self, s): + """ + print a progress line for testing + """ + msg = ' ' + self.shortid() + ': ' + s + WiredTigerTestCase._resultfile.write(msg + '\n') + + def prhead(self, s, *beginning): + """ + print a header line for testing, something important + """ + msg = '' + if len(beginning) > 0: + msg += '\n' + msg += ' ' + self.shortid() + ': ' + s + print(msg) + WiredTigerTestCase._resultfile.write(msg + '\n') + + def prexception(self, excinfo): + WiredTigerTestCase._resultfile.write('\n') + traceback.print_exception(excinfo[0], excinfo[1], excinfo[2], None, WiredTigerTestCase._resultfile) + WiredTigerTestCase._resultfile.write('\n') + + def shortid(self): + return self.id().replace("__main__.","") + + def className(self): + return self.__class__.__name__ + + +def runsuite(suite): + try: + return unittest.TextTestRunner( + verbosity=WiredTigerTestCase._verbose).run(suite) + except BaseException as e: + # This should not happen for regular test errors, unittest should catch everything + print('ERROR: running test: ', e) + raise e + +def run(name='__main__'): + result = runsuite(unittest.TestLoader().loadTestsFromName(name)) + sys.exit(not result.wasSuccessful()) diff --git a/test/thread/Makefile.am b/test/thread/Makefile.am new file mode 100644 index 00000000000..5ffa1a8aada --- /dev/null +++ b/test/thread/Makefile.am @@ -0,0 +1,9 @@ +INCLUDES = -I$(top_builddir) + +noinst_PROGRAMS = t +t_LDADD = $(top_builddir)/libwiredtiger.la +t_SOURCES = thread.h t.c load.c run.c stats.c +t_LDFLAGS = -static + +clean-local: + rm -rf WiredTiger __* diff --git a/test/thread/load.c b/test/thread/load.c new file mode 100644 index 00000000000..71450ac973a --- /dev/null +++ b/test/thread/load.c @@ -0,0 +1,65 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "thread.h" + +void +load(void) +{ + WT_CURSOR *cursor; + WT_ITEM *key, _key, *value, _value; + WT_SESSION *session; + char *p, *end, keybuf[64], valuebuf[64], config[128]; + u_int keyno; + int ret; + + key = &_key; + value = &_value; + + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + die("conn.session", ret); + + p = config; + end = config + sizeof(config); + p += snprintf(p, (size_t)(end - p), + "key_format=%s," + "internal_page_max=%d," + "leaf_page_max=%d,", + ftype == ROW ? "u" : "r", 16 * 1024, 128 * 1024); + if (ftype == FIX) + (void)snprintf(p, (size_t)(end - p), ",value_format=3t"); + + if ((ret = session->create(session, FNAME, config)) != 0) + die("session.create", ret); + + if ((ret = + session->open_cursor(session, FNAME, NULL, "bulk", &cursor)) != 0) + die("cursor.open", ret); + + for (keyno = 0; keyno < nkeys; ++keyno) { + if (ftype == ROW) { + key->data = keybuf; + key->size = (uint32_t) + snprintf(keybuf, sizeof(keybuf), "%017u", keyno); + cursor->set_key(cursor, key); + } else + cursor->set_key(cursor, (uint32_t)keyno); + value->data = valuebuf; + if (ftype == FIX) + cursor->set_value(cursor, 0x01); + else { + value->size = (uint32_t) + snprintf(valuebuf, sizeof(valuebuf), "%37u", keyno); + cursor->set_value(cursor, value); + } + if ((ret = cursor->insert(cursor)) != 0) + die("cursor.insert", ret); + } + + if ((ret = session->close(session, NULL)) != 0) + die("session.close", ret); +} diff --git a/test/thread/run.c b/test/thread/run.c new file mode 100644 index 00000000000..ee9f7b3d8b9 --- /dev/null +++ b/test/thread/run.c @@ -0,0 +1,224 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "thread.h" + +static void print_stats(int); +static void *reader(void *); +static void *writer(void *); + +typedef struct { + int remove; /* cursor.remove */ + int update; /* cursor.update */ + int reads; /* cursor.search */ +} STATS; + +static STATS *run_stats; + +/* + * r -- + * Return a 32-bit pseudo-random number. + * + * This is an implementation of George Marsaglia's multiply-with-carry pseudo- + * random number generator. Computationally fast, with reasonable randomness + * properties. + */ +static inline uint32_t +r(void) +{ + static uint32_t m_w = 0, m_z = 0; + + if (m_w == 0) { + struct timeval t; + (void)gettimeofday(&t, NULL); + m_w = (uint32_t)t.tv_sec; + m_z = (uint32_t)t.tv_usec; + } + + m_z = 36969 * (m_z & 65535) + (m_z >> 16); + m_w = 18000 * (m_w & 65535) + (m_w >> 16); + return (m_z << 16) + (m_w & 65535); +} + +int +run(int readers, int writers) +{ + clock_t start, stop; + double seconds; + pthread_t *tids; + int i, ret; + void *thread_ret; + + /* Create statistics and thread structures. */ + if ((run_stats = calloc( + (size_t)(readers + writers), sizeof(*run_stats))) == NULL || + (tids = calloc((size_t)(readers + writers), sizeof(*tids))) == NULL) + die("calloc", errno); + + start = clock(); + + /* Create threads. */ + for (i = 0; i < readers; ++i) + if ((ret = pthread_create( + &tids[i], NULL, reader, (void *)(uintptr_t)i)) != 0) + die("pthread_create", ret); + for (; i < readers + writers; ++i) { + if ((ret = pthread_create( + &tids[i], NULL, writer, (void *)(uintptr_t)i)) != 0) + die("pthread_create", ret); + } + + /* Wait for the threads. */ + for (i = 0; i < readers + writers; ++i) + (void)pthread_join(tids[i], &thread_ret); + + stop = clock(); + seconds = (stop - start) / (double)CLOCKS_PER_SEC; + fprintf(stderr, "timer: %.2lf seconds (%d ops/second)\n", + seconds, (int)(((readers + writers) * nops) / seconds)); + + print_stats(readers + writers); + + free(run_stats); + free(tids); + + return (0); +} + +/* + * reader -- + * Reader thread start function. + */ +static void * +reader(void *arg) +{ + STATS *s; + WT_CURSOR *cursor; + WT_ITEM *key, _key; + WT_SESSION *session; + pthread_t tid; + u_int i, keyno; + int id, ret; + char keybuf[64]; + + id = (int)(uintptr_t)arg; + tid = pthread_self(); + printf(" read thread %2d starting: tid: %p\n", id, (void *)tid); + sched_yield(); /* Get all the threads created. */ + + key = &_key; + s = &run_stats[id]; + + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + die("conn.open_session", ret); + + if ((ret = + session->open_cursor(session, FNAME, NULL, NULL, &cursor)) != 0) + die("session.open_cursor", ret); + + for (i = 0; i < nops; ++i, ++s->reads, sched_yield()) { + keyno = r() % nkeys; + if (ftype == ROW) { + key->data = keybuf; + key->size = (uint32_t) + snprintf(keybuf, sizeof(keybuf), "%017u", keyno); + cursor->set_key(cursor, key); + } else + cursor->set_key(cursor, (uint32_t)keyno); + if ((ret = cursor->search(cursor)) != 0 && ret != WT_NOTFOUND) + die("cursor.search", ret); + } + + if ((ret = session->close(session, NULL)) != 0) + die("session.close", ret); + + return (NULL); +} + +/* + * writer -- + * Writer thread start function. + */ +static void * +writer(void *arg) +{ + STATS *s; + WT_CURSOR *cursor; + WT_ITEM *key, _key, *value, _value; + WT_SESSION *session; + pthread_t tid; + u_int i, keyno; + int id, ret; + char keybuf[64], valuebuf[64]; + + id = (int)(uintptr_t)arg; + tid = pthread_self(); + printf("write thread %2d starting: tid: %p\n", id, (void *)tid); + sched_yield(); /* Get all the threads created. */ + + key = &_key; + value = &_value; + s = &run_stats[id]; + + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + die("conn.open_session", ret); + + if ((ret = + session->open_cursor(session, FNAME, NULL, NULL, &cursor)) != 0) + die("session.open_cursor", ret); + + for (i = 0; i < nops; ++i, sched_yield()) { + keyno = r() % nkeys; + if (ftype == ROW) { + key->data = keybuf; + key->size = (uint32_t) + snprintf(keybuf, sizeof(keybuf), "%017u", keyno); + cursor->set_key(cursor, key); + } else + cursor->set_key(cursor, (uint32_t)keyno); + if (keyno % 5 == 0) { + ++s->remove; + if ((ret = + cursor->remove(cursor)) != 0 && ret != WT_NOTFOUND) + die("cursor.remove", ret); + } else { + ++s->update; + value->data = valuebuf; + if (ftype == FIX) + cursor->set_value(cursor, 0x10); + else { + value->size = (uint32_t) + snprintf(valuebuf, + sizeof(valuebuf), "XXX %37u", keyno); + cursor->set_value(cursor, value); + } + if ((ret = cursor->update(cursor)) != 0) + die("cursor.update", ret); + } + } + + if ((ret = session->close(session, NULL)) != 0) + die("session.close", ret); + + return (NULL); +} + +/* + * print_stats -- + * Display reader/writer thread stats. + */ +static void +print_stats(int nthreads) +{ + STATS *s; + int id; + + s = run_stats; + for (id = 0; id < nthreads; ++id, ++s) + printf("%2d: read: %6d, remove: %6d, update: %6d\n", + id, s->reads, s->remove, s->update); +} diff --git a/test/thread/stats.c b/test/thread/stats.c new file mode 100644 index 00000000000..21b4ee1f698 --- /dev/null +++ b/test/thread/stats.c @@ -0,0 +1,62 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "thread.h" + +/* + * stats + * Dump the database/file statistics. + */ +void +stats(void) +{ + WT_CURSOR *cursor; + uint64_t v; + const char *pval, *desc; + WT_SESSION *session; + FILE *fp; + int ret; + + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + die("conn.session", ret); + + if ((fp = fopen(FNAME_STAT, "w")) == NULL) + die("fopen " FNAME_STAT , errno); + + /* Connection statistics. */ + if ((ret = session->open_cursor(session, + "statistics:", NULL, NULL, &cursor)) != 0) + die("session.open_cursor", ret); + + while ((ret = cursor->next(cursor)) == 0 && + (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) + (void)fprintf(fp, "%s=%s\n", desc, pval); + + if (ret != WT_NOTFOUND) + die("cursor.next", ret); + if ((ret = cursor->close(cursor)) != 0) + die("cursor.close", ret); + + /* File statistics. */ + if ((ret = session->open_cursor(session, + "statistics:" FNAME, NULL, NULL, &cursor)) != 0) + die("session.open_cursor", ret); + + while ((ret = cursor->next(cursor)) == 0 && + (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) + (void)fprintf(fp, "%s=%s\n", desc, pval); + + if (ret != WT_NOTFOUND) + die("cursor.next", ret); + if ((ret = cursor->close(cursor)) != 0) + die("cursor.close", ret); + + if ((ret = session->close(session, NULL)) != 0) + die("session.close", ret); + + (void)fclose(fp); +} diff --git a/test/thread/t.c b/test/thread/t.c new file mode 100644 index 00000000000..f35012f382b --- /dev/null +++ b/test/thread/t.c @@ -0,0 +1,240 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "thread.h" + +WT_CONNECTION *conn; /* WiredTiger connection */ +__ftype ftype; /* File type */ +u_int nkeys, nops; /* Keys, Operations */ + +static char *progname; /* Program name */ +static FILE *logfp; /* Log file */ + +static int handle_message(WT_EVENT_HANDLER *, const char *); +static void onint(int); +static void shutdown(void); +static int usage(void); +static void wt_connect(char *); +static void wt_shutdown(void); + +int +main(int argc, char *argv[]) +{ + int ch, cnt, readers, runs, writers; + char *config_open; + + if ((progname = strrchr(argv[0], '/')) == NULL) + progname = argv[0]; + else + ++progname; + + config_open = NULL; + ftype = ROW; + nkeys = 1000; + nops = 10000; + readers = 10; + runs = 0; + writers = 10; + + while ((ch = getopt(argc, argv, "1C:k:l:n:R:r:t:W:")) != EOF) + switch (ch) { + case '1': /* One run */ + runs = 1; + break; + case 'C': /* wiredtiger_open config */ + config_open = optarg; + break; + case 'k': /* rows */ + nkeys = (u_int)atoi(optarg); + break; + case 'l': /* log */ + if ((logfp = fopen(optarg, "w")) == NULL) { + fprintf(stderr, + "%s: %s\n", optarg, strerror(errno)); + return (EXIT_FAILURE); + } + break; + case 'n': /* operations */ + nops = (u_int)atoi(optarg); + break; + case 'R': + readers = atoi(optarg); + break; + case 'r': /* runs */ + runs = atoi(optarg); + break; + case 't': + switch (optarg[0]) { + case 'f': + ftype = FIX; + break; + case 'r': + ftype = ROW; + break; + case 'v': + ftype = VAR; + break; + default: + return (usage()); + } + break; + case 'W': + writers = atoi(optarg); + break; + default: + return (usage()); + } + + argc -= optind; + argv += optind; + if (argc != 0) + return (usage()); + + /* Clean up on signal. */ + (void)signal(SIGINT, onint); + + printf("%s: process %" PRIu64 "\n", progname, (uint64_t)getpid()); + for (cnt = 1; runs == 0 || cnt <= runs; ++cnt) { + printf( + " %d: %d readers, %d writers\n", cnt, readers, writers); + + shutdown(); /* Clean up previous runs */ + + wt_connect(config_open); /* WiredTiger connection */ + + load(); /* Load initial records */ + /* Loop operations */ + if (run(readers, writers)) + return (EXIT_FAILURE); + + stats(); /* Statistics */ + + wt_shutdown(); /* WiredTiger shut down */ + } + return (0); +} + +/* + * wt_connect -- + * Configure the WiredTiger connection. + */ +static void +wt_connect(char *config_open) +{ + static WT_EVENT_HANDLER event_handler = { + NULL, + handle_message, + NULL + }; + int ret; + char config[128]; + + snprintf(config, sizeof(config), + "create,error_prefix=\"%s\",cache_size=5MB%s%s", + progname, + config_open == NULL ? "" : ",", + config_open == NULL ? "" : config_open); + + if ((ret = wiredtiger_open(NULL, &event_handler, config, &conn)) != 0) + die("wiredtiger_open", ret); +} + +/* + * wt_shutdown -- + * Flush the file to disk and shut down the WiredTiger connection. + */ +static void +wt_shutdown(void) +{ + WT_SESSION *session; + int ret; + + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + die("conn.session", ret); + + if ((ret = session->verify(session, FNAME, NULL)) != 0) + die("session.sync", ret); + + if ((ret = session->sync(session, FNAME, NULL)) != 0) + die("session.sync", ret); + + if ((ret = conn->close(conn, NULL)) != 0) + die("conn.close", ret); +} + +/* + * shutdown -- + * Clean up from previous runs. + */ +static void +shutdown(void) +{ + (void)system("rm -f WildTiger WiredTiger.* __wt*"); +} + +static int +handle_message(WT_EVENT_HANDLER *handler, const char *message) +{ + UNUSED(handler); + + if (logfp == NULL) + printf("%s\n", message); + else + fprintf(logfp, "%s\n", message); + return (0); +} + +/* + * onint -- + * Interrupt signal handler. + */ +static void +onint(int signo) +{ + UNUSED(signo); + + shutdown(); + + fprintf(stderr, "\n"); + exit (EXIT_FAILURE); +} + +/* + * die -- + * Report an error and quit. + */ +void +die(const char *m, int e) +{ + fprintf(stderr, "%s: %s: %s\n", progname, m, wiredtiger_strerror(e)); + exit (EXIT_FAILURE); +} + +/* + * usage -- + * Display usage statement and exit failure. + */ +static int +usage(void) +{ + fprintf(stderr, + "usage: %s " + "[-1] [-C wiredtiger-config]\n " + "[-l log] [-R readers] [-r runs] [-t f|r|v] [-W writers]\n", + progname); + fprintf(stderr, "%s", + "\t-1 run once\n" + "\t-C specify wiredtiger_open configuration arguments\n" + "\t-k set number of keys to load\n" + "\t-l specify a log file\n" + "\t-n set number of operations each thread does\n" + "\t-R set number of reading threads\n" + "\t-r set number of runs\n" + "\t-t set a file type (fix | row | var)\n" + "\t-W set number of writing threads\n"); + return (EXIT_FAILURE); +} diff --git a/test/thread/thread.h b/test/thread/thread.h new file mode 100644 index 00000000000..d3c4715be29 --- /dev/null +++ b/test/thread/thread.h @@ -0,0 +1,42 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include <sys/types.h> +#include <sys/time.h> + +#include <errno.h> +#include <inttypes.h> +#include <pthread.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#include <wiredtiger.h> + +#define FNAME "file:__wt" /* File name */ +#define FNAME_STAT "__stats" /* File name for statistics */ + +#define UNUSED(v) (void)(v) /* Quiet unused var warnings */ + +extern WT_CONNECTION *conn; /* WiredTiger connection */ + +typedef enum { FIX, ROW, VAR } __ftype; /* File type */ +extern __ftype ftype; + +extern u_int nkeys; /* Keys to load */ +extern u_int nops; /* Operations per thread */ + +#if defined (__GNUC__) +void die(const char *, int) __attribute__((noreturn)); +#else +void die(const char *, int); +#endif +void load(void); +int run(int, int); +void stats(void); |
