From ff15b814aaed8f1c58e95b6574e9c7e97a43f5c9 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Fri, 4 Dec 2015 18:02:56 +1100 Subject: WT-2264 Don't have write leaves chase inserts indefinitely: cut off inserts that are more recent than when write leaves starts. --- src/btree/bt_sync.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/btree/bt_sync.c b/src/btree/bt_sync.c index 07bb2eb3a01..31e16d5b49e 100644 --- a/src/btree/bt_sync.c +++ b/src/btree/bt_sync.c @@ -23,12 +23,11 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) WT_REF *walk; WT_TXN *txn; uint64_t internal_bytes, internal_pages, leaf_bytes, leaf_pages; - uint64_t saved_snap_min; + uint64_t oldest_id, saved_snap_min; uint32_t flags; bool evict_reset; btree = S2BT(session); - walk = NULL; txn = &session->txn; saved_snap_min = WT_SESSION_TXN_STATE(session)->snap_min; @@ -56,6 +55,15 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) return (0); } + /* + * Save the oldest transaction ID we need to keep around. + * Otherwise, in a busy system, we could be updating pages so + * fast that write leaves never catches up. We deliberately + * have no transaction running at this point that would keep + * the oldest ID from moving forwards as we walk the tree. + */ + oldest_id = __wt_txn_oldest_id(session); + flags |= WT_READ_NO_WAIT | WT_READ_SKIP_INTL; for (walk = NULL;;) { WT_ERR(__wt_tree_walk(session, &walk, NULL, flags)); @@ -64,13 +72,13 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) /* * Write dirty pages if nobody beat us to it. Don't - * try to write the hottest pages: checkpoint will have - * to visit them anyway. + * try to write hot pages (defined as pages that have + * been updated since the write phase leaves started): + * checkpoint will have to visit them anyway. */ page = walk->page; if (__wt_page_is_modified(page) && - __wt_txn_visible_all( - session, page->modify->update_txn)) { + WT_TXNID_LT(page->modify->update_txn, oldest_id)) { if (txn->isolation == WT_ISO_READ_COMMITTED) __wt_txn_get_snapshot(session); leaf_bytes += page->memory_footprint; -- cgit v1.2.1 From a10a19363307e28c512c3df9d577e0ea85bec65a Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Fri, 22 Jan 2016 14:35:43 +1100 Subject: WT-2346 Don't hold the schema lock during checkpoint I/O. This required removing the special case code for named checkpoints of trees that are in the process of being bulk loaded. --- dist/flags.py | 1 + dist/s_string.ok | 1 + src/conn/conn_dhandle.c | 115 ++++----------------------------------------- src/conn/conn_handle.c | 2 + src/docs/checkpoint.dox | 34 ++++++++------ src/include/connection.h | 1 + src/include/extern.h | 2 - src/include/flags.h | 27 ++++++----- src/include/schema.h | 9 ++++ src/include/session.h | 5 +- src/meta/meta_apply.c | 20 ++++---- src/meta/meta_track.c | 11 +++-- src/schema/schema_worker.c | 17 ++----- src/txn/txn_ckpt.c | 81 ++++++++++--------------------- test/suite/test_bulk02.py | 10 ++-- 15 files changed, 106 insertions(+), 230 deletions(-) diff --git a/dist/flags.py b/dist/flags.py index 8133028857c..9d16ec82e8c 100644 --- a/dist/flags.py +++ b/dist/flags.py @@ -113,6 +113,7 @@ flags = { 'SESSION_INTERNAL', 'SESSION_LOCKED_CHECKPOINT', 'SESSION_LOCKED_HANDLE_LIST', + 'SESSION_LOCKED_METADATA', 'SESSION_LOCKED_SCHEMA', 'SESSION_LOCKED_SLOT', 'SESSION_LOCKED_TABLE', diff --git a/dist/s_string.ok b/dist/s_string.ok index d4af6b8eb00..235caff3c9f 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -189,6 +189,7 @@ MALLOC MEM MEMALIGN MERCHANTABILITY +METADATA MONGODB MSVC MULTIBLOCK diff --git a/src/conn/conn_dhandle.c b/src/conn/conn_dhandle.c index 1b9809c37fb..53e81ba2fe7 100644 --- a/src/conn/conn_dhandle.c +++ b/src/conn/conn_dhandle.c @@ -368,24 +368,21 @@ __conn_btree_apply_internal(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle, * sure it's referenced to stop other internal code dropping the handle * (e.g in LSM when cleaning up obsolete chunks). */ - ret = __wt_session_get_btree(session, - dhandle->name, dhandle->checkpoint, NULL, 0); - if (ret == 0) { - WT_SAVE_DHANDLE(session, - ret = func(session, cfg)); - if (WT_META_TRACKING(session)) - WT_TRET(__wt_meta_track_handle_lock(session, false)); - else - WT_TRET(__wt_session_release_btree(session)); - } else if (ret == EBUSY) - ret = __wt_conn_btree_apply_single(session, dhandle->name, - dhandle->checkpoint, func, cfg); + if ((ret = __wt_session_get_btree(session, + dhandle->name, dhandle->checkpoint, NULL, 0)) != 0) + return (ret == EBUSY ? 0 : ret); + + WT_SAVE_DHANDLE(session, ret = func(session, cfg)); + if (WT_META_TRACKING(session)) + WT_TRET(__wt_meta_track_handle_lock(session, false)); + else + WT_TRET(__wt_session_release_btree(session)); return (ret); } /* * __wt_conn_btree_apply -- - * Apply a function to all open btree handles apart from the metadata. + * Apply a function to all open btree handles with the given URI. */ int __wt_conn_btree_apply(WT_SESSION_IMPL *session, @@ -429,98 +426,6 @@ __wt_conn_btree_apply(WT_SESSION_IMPL *session, return (0); } -/* - * __wt_conn_btree_apply_single_ckpt -- - * Decode any checkpoint information from the configuration string then - * call btree apply single. - */ -int -__wt_conn_btree_apply_single_ckpt(WT_SESSION_IMPL *session, - const char *uri, - int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]) -{ - WT_CONFIG_ITEM cval; - WT_DECL_RET; - const char *checkpoint; - - checkpoint = NULL; - - /* - * This function exists to handle checkpoint configuration. Callers - * that never open a checkpoint call the underlying function directly. - */ - WT_RET_NOTFOUND_OK( - __wt_config_gets_def(session, cfg, "checkpoint", 0, &cval)); - if (cval.len != 0) { - /* - * The internal checkpoint name is special, find the last - * unnamed checkpoint of the object. - */ - if (WT_STRING_MATCH(WT_CHECKPOINT, cval.str, cval.len)) { - WT_RET(__wt_meta_checkpoint_last_name( - session, uri, &checkpoint)); - } else - WT_RET(__wt_strndup( - session, cval.str, cval.len, &checkpoint)); - } - - ret = __wt_conn_btree_apply_single(session, uri, checkpoint, func, cfg); - - __wt_free(session, checkpoint); - - return (ret); -} - -/* - * __wt_conn_btree_apply_single -- - * Apply a function to a single btree handle that couldn't be locked - * (attempting to get the handle returned EBUSY). - */ -int -__wt_conn_btree_apply_single(WT_SESSION_IMPL *session, - const char *uri, const char *checkpoint, - int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]) -{ - WT_CONNECTION_IMPL *conn; - WT_DATA_HANDLE *dhandle; - WT_DECL_RET; - uint64_t bucket, hash; - - conn = S2C(session); - - WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)); - - hash = __wt_hash_city64(uri, strlen(uri)); - bucket = hash % WT_HASH_ARRAY_SIZE; - TAILQ_FOREACH(dhandle, &conn->dhhash[bucket], hashq) - if (F_ISSET(dhandle, WT_DHANDLE_OPEN) && - !F_ISSET(dhandle, WT_DHANDLE_DEAD) && - (hash == dhandle->name_hash && - strcmp(uri, dhandle->name) == 0) && - ((dhandle->checkpoint == NULL && checkpoint == NULL) || - (dhandle->checkpoint != NULL && checkpoint != NULL && - strcmp(dhandle->checkpoint, checkpoint) == 0))) { - /* - * We're holding the handle list lock which locks out - * handle open (which might change the state of the - * underlying object). However, closing a handle - * doesn't require the handle list lock, lock out - * closing the handle and then confirm the handle is - * still open. - */ - __wt_spin_lock(session, &dhandle->close_lock); - if (F_ISSET(dhandle, WT_DHANDLE_OPEN) && - !F_ISSET(dhandle, WT_DHANDLE_DEAD)) { - WT_WITH_DHANDLE(session, dhandle, - ret = func(session, cfg)); - } - __wt_spin_unlock(session, &dhandle->close_lock); - WT_RET(ret); - } - - return (0); -} - /* * __wt_conn_dhandle_close_all -- * Close all data handles handles with matching name (including all diff --git a/src/conn/conn_handle.c b/src/conn/conn_handle.c index 12b4e87e921..b33ec18dfca 100644 --- a/src/conn/conn_handle.c +++ b/src/conn/conn_handle.c @@ -56,6 +56,7 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn) WT_RET(__wt_rwlock_alloc(session, &conn->hot_backup_lock, "hot backup")); WT_RET(__wt_spin_init(session, &conn->las_lock, "lookaside table")); + WT_RET(__wt_spin_init(session, &conn->metadata_lock, "metadata")); WT_RET(__wt_spin_init(session, &conn->reconfig_lock, "reconfigure")); WT_RET(__wt_spin_init(session, &conn->schema_lock, "schema")); WT_RET(__wt_spin_init(session, &conn->table_lock, "table creation")); @@ -143,6 +144,7 @@ __wt_connection_destroy(WT_CONNECTION_IMPL *conn) __wt_spin_destroy(session, &conn->fh_lock); WT_TRET(__wt_rwlock_destroy(session, &conn->hot_backup_lock)); __wt_spin_destroy(session, &conn->las_lock); + __wt_spin_destroy(session, &conn->metadata_lock); __wt_spin_destroy(session, &conn->reconfig_lock); __wt_spin_destroy(session, &conn->schema_lock); __wt_spin_destroy(session, &conn->table_lock); diff --git a/src/docs/checkpoint.dox b/src/docs/checkpoint.dox index 523c0887859..ec28fea13c3 100644 --- a/src/docs/checkpoint.dox +++ b/src/docs/checkpoint.dox @@ -23,11 +23,16 @@ All transactional updates committed before a checkpoint are made durable by the checkpoint, therefore the frequency of checkpoints limits the volume of data that may be lost due to application or system failure. -When WiredTiger data sources are first opened, they are opened in the -state of the most recent checkpoint taken on the file, in other words, -updates after the most recent checkpoint will not appear in the data -source. If no checkpoint is found when the data source is opened, the -data source will appear empty. +Data sources that are involved in an exclusive operation when the +checkpoint starts, including bulk load, verify or salvage, will be skipped +by the checkpoint. Operations requiring exclusive access may fail with +an \c EBUSY error if attempted during a checkpoint. + +When data sources are first opened, they are opened in the state of the +most recent checkpoint taken on the file, in other words, updates after the +most recent checkpoint will not appear in the data source. If no +checkpoint is found when the data source is opened, the data source will +appear empty. @section checkpoint_server Automatic checkpoints @@ -54,15 +59,16 @@ checkpoint cursor is closed. @section checkpoint_naming Checkpoint naming -Additionally, checkpoints that do not include LSM trees may optionally -be given names by the application. Checkpoints named by the application -persist until explicitly discarded or the application creates a new -checkpoint with the same name (which replaces the previous checkpoint -of that name). If the previous checkpoint cannot be replaced, either -because a cursor is reading from the previous checkpoint, or backups are -in progress, the checkpoint will fail. Because named checkpoints -persist until discarded or replaced, they can be used to periodically -snapshot data for later use. +Additionally, checkpoints that do not include LSM trees may optionally be +given names by the application. Because named checkpoints persist until +discarded or replaced, they can be used to periodically snapshot data for +later use. + +Checkpoints named by the application persist until explicitly discarded or +the application creates a new checkpoint with the same name (which replaces +the previous checkpoint of that name). If the previous checkpoint cannot be +replaced, either because a cursor is reading from the previous checkpoint, +or backups are in progress, the checkpoint will fail. Internal checkpoints (that is, checkpoints not named by the application) use the reserved name "WiredTigerCheckpoint". Applications can open the diff --git a/src/include/connection.h b/src/include/connection.h index 5d61f9456b3..49dc17beef8 100644 --- a/src/include/connection.h +++ b/src/include/connection.h @@ -175,6 +175,7 @@ struct __wt_connection_impl { WT_SPINLOCK checkpoint_lock; /* Checkpoint spinlock */ WT_SPINLOCK dhandle_lock; /* Data handle list spinlock */ WT_SPINLOCK fh_lock; /* File handle queue spinlock */ + WT_SPINLOCK metadata_lock; /* Metadata update spinlock */ WT_SPINLOCK reconfig_lock; /* Single thread reconfigure */ WT_SPINLOCK schema_lock; /* Schema operation spinlock */ WT_SPINLOCK table_lock; /* Table creation spinlock */ diff --git a/src/include/extern.h b/src/include/extern.h index 0a2d1645b2f..338a9ac930a 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -254,8 +254,6 @@ extern int __wt_conn_dhandle_find( WT_SESSION_IMPL *session, const char *uri, co extern int __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force); extern int __wt_conn_btree_open( WT_SESSION_IMPL *session, const char *cfg[], uint32_t flags); extern int __wt_conn_btree_apply(WT_SESSION_IMPL *session, bool apply_checkpoints, const char *uri, int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]); -extern int __wt_conn_btree_apply_single_ckpt(WT_SESSION_IMPL *session, const char *uri, int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]); -extern int __wt_conn_btree_apply_single(WT_SESSION_IMPL *session, const char *uri, const char *checkpoint, int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]); extern int __wt_conn_dhandle_close_all( WT_SESSION_IMPL *session, const char *uri, bool force); extern int __wt_conn_dhandle_discard_single( WT_SESSION_IMPL *session, bool final, bool force); extern int __wt_conn_dhandle_discard(WT_SESSION_IMPL *session); diff --git a/src/include/flags.h b/src/include/flags.h index 2180db7b2b6..ee850e64a09 100644 --- a/src/include/flags.h +++ b/src/include/flags.h @@ -55,19 +55,20 @@ #define WT_SESSION_INTERNAL 0x00000004 #define WT_SESSION_LOCKED_CHECKPOINT 0x00000008 #define WT_SESSION_LOCKED_HANDLE_LIST 0x00000010 -#define WT_SESSION_LOCKED_SCHEMA 0x00000020 -#define WT_SESSION_LOCKED_SLOT 0x00000040 -#define WT_SESSION_LOCKED_TABLE 0x00000080 -#define WT_SESSION_LOCKED_TURTLE 0x00000100 -#define WT_SESSION_LOGGING_INMEM 0x00000200 -#define WT_SESSION_LOOKASIDE_CURSOR 0x00000400 -#define WT_SESSION_NO_CACHE 0x00000800 -#define WT_SESSION_NO_DATA_HANDLES 0x00001000 -#define WT_SESSION_NO_EVICTION 0x00002000 -#define WT_SESSION_NO_LOGGING 0x00004000 -#define WT_SESSION_NO_SCHEMA_LOCK 0x00008000 -#define WT_SESSION_QUIET_CORRUPT_FILE 0x00010000 -#define WT_SESSION_SERVER_ASYNC 0x00020000 +#define WT_SESSION_LOCKED_METADATA 0x00000020 +#define WT_SESSION_LOCKED_SCHEMA 0x00000040 +#define WT_SESSION_LOCKED_SLOT 0x00000080 +#define WT_SESSION_LOCKED_TABLE 0x00000100 +#define WT_SESSION_LOCKED_TURTLE 0x00000200 +#define WT_SESSION_LOGGING_INMEM 0x00000400 +#define WT_SESSION_LOOKASIDE_CURSOR 0x00000800 +#define WT_SESSION_NO_CACHE 0x00001000 +#define WT_SESSION_NO_DATA_HANDLES 0x00002000 +#define WT_SESSION_NO_EVICTION 0x00004000 +#define WT_SESSION_NO_LOGGING 0x00008000 +#define WT_SESSION_NO_SCHEMA_LOCK 0x00010000 +#define WT_SESSION_QUIET_CORRUPT_FILE 0x00020000 +#define WT_SESSION_SERVER_ASYNC 0x00040000 #define WT_TXN_LOG_CKPT_CLEANUP 0x00000001 #define WT_TXN_LOG_CKPT_PREPARE 0x00000002 #define WT_TXN_LOG_CKPT_START 0x00000004 diff --git a/src/include/schema.h b/src/include/schema.h index 9cca4794b6b..428dbb91ad0 100644 --- a/src/include/schema.h +++ b/src/include/schema.h @@ -109,6 +109,15 @@ struct __wt_table { #define WT_WITH_HANDLE_LIST_LOCK(session, op) \ WT_WITH_LOCK(session, \ &S2C(session)->dhandle_lock, WT_SESSION_LOCKED_HANDLE_LIST, op) + +/* + * WT_WITH_METADATA_LOCK -- + * Acquire the metadata lock, perform an operation, drop the lock. + */ +#define WT_WITH_METADATA_LOCK(session, op) \ + WT_WITH_LOCK(session, \ + &S2C(session)->metadata_lock, WT_SESSION_LOCKED_METADATA, op) + /* * WT_WITH_SCHEMA_LOCK -- * Acquire the schema lock, perform an operation, drop the lock. diff --git a/src/include/session.h b/src/include/session.h index 5c3291230b4..b3c475805a4 100644 --- a/src/include/session.h +++ b/src/include/session.h @@ -127,10 +127,7 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { int (*block_manager_cleanup)(WT_SESSION_IMPL *); /* Checkpoint support */ - struct { - WT_DATA_HANDLE *dhandle; - const char *name; - } *ckpt_handle; /* Handle list */ + WT_DATA_HANDLE **ckpt_handle; /* Handle list */ u_int ckpt_handle_next; /* Next empty slot */ size_t ckpt_handle_allocated; /* Bytes allocated */ diff --git a/src/meta/meta_apply.c b/src/meta/meta_apply.c index 92766213b33..7722cd55fbd 100644 --- a/src/meta/meta_apply.c +++ b/src/meta/meta_apply.c @@ -37,17 +37,15 @@ __meta_btree_apply(WT_SESSION_IMPL *session, WT_CURSOR *cursor, * dropping the handle (e.g in LSM when cleaning up obsolete * chunks). Holding the metadata lock isn't enough. */ - ret = __wt_session_get_btree(session, uri, NULL, NULL, 0); - if (ret == 0) { - WT_SAVE_DHANDLE(session, ret = func(session, cfg)); - if (WT_META_TRACKING(session)) - WT_TRET(__wt_meta_track_handle_lock( - session, false)); - else - WT_TRET(__wt_session_release_btree(session)); - } else if (ret == EBUSY) - ret = __wt_conn_btree_apply_single( - session, uri, NULL, func, cfg); + if ((ret = __wt_session_get_btree( + session, uri, NULL, NULL, 0)) != 0) + return (ret == EBUSY ? 0 : ret); + WT_SAVE_DHANDLE(session, ret = func(session, cfg)); + if (WT_META_TRACKING(session)) + WT_TRET(__wt_meta_track_handle_lock( + session, false)); + else + WT_TRET(__wt_session_release_btree(session)); WT_RET(ret); } WT_RET_NOTFOUND_OK(ret); diff --git a/src/meta/meta_track.c b/src/meta/meta_track.c index 1baab2deae1..90f2e327406 100644 --- a/src/meta/meta_track.c +++ b/src/meta/meta_track.c @@ -284,11 +284,12 @@ __wt_meta_track_off(WT_SESSION_IMPL *session, bool need_sync, bool unroll) * should be included in the checkpoint. */ ckpt_session->txn.id = session->txn.id; - F_SET(ckpt_session, WT_SESSION_LOCKED_SCHEMA); - WT_WITH_DHANDLE(ckpt_session, - WT_SESSION_META_DHANDLE(session), - ret = __wt_checkpoint(ckpt_session, NULL)); - F_CLR(ckpt_session, WT_SESSION_LOCKED_SCHEMA); + F_SET(ckpt_session, WT_SESSION_LOCKED_METADATA); + WT_WITH_METADATA_LOCK(session, + WT_WITH_DHANDLE(ckpt_session, + WT_SESSION_META_DHANDLE(session), + ret = __wt_checkpoint(ckpt_session, NULL))); + F_CLR(ckpt_session, WT_SESSION_LOCKED_METADATA); ckpt_session->txn.id = WT_TXN_NONE; WT_RET(ret); WT_WITH_DHANDLE(session, diff --git a/src/schema/schema_worker.c b/src/schema/schema_worker.c index b5ee3bb7f7d..e60a7107786 100644 --- a/src/schema/schema_worker.c +++ b/src/schema/schema_worker.c @@ -55,18 +55,11 @@ __wt_schema_worker(WT_SESSION_IMPL *session, WT_ERR(ret); } - if ((ret = __wt_session_get_btree_ckpt( - session, uri, cfg, open_flags)) == 0) { - WT_SAVE_DHANDLE(session, - ret = file_func(session, cfg)); - WT_TRET(__wt_session_release_btree(session)); - } else if (ret == EBUSY) { - WT_ASSERT(session, !FLD_ISSET( - open_flags, WT_DHANDLE_EXCLUSIVE)); - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __wt_conn_btree_apply_single_ckpt( - session, uri, file_func, cfg)); - } + WT_ERR(__wt_session_get_btree_ckpt( + session, uri, cfg, open_flags)); + WT_SAVE_DHANDLE(session, + ret = file_func(session, cfg)); + WT_TRET(__wt_session_release_btree(session)); WT_ERR(ret); } } else if (WT_PREFIX_MATCH(uri, "colgroup:")) { diff --git a/src/txn/txn_ckpt.c b/src/txn/txn_ckpt.c index eac7d4bef75..ce3c76933d7 100644 --- a/src/txn/txn_ckpt.c +++ b/src/txn/txn_ckpt.c @@ -179,14 +179,8 @@ __checkpoint_apply(WT_SESSION_IMPL *session, const char *cfg[], /* If we have already locked the handles, apply the operation. */ for (i = 0; i < session->ckpt_handle_next; ++i) { - if (session->ckpt_handle[i].dhandle != NULL) - WT_WITH_DHANDLE(session, - session->ckpt_handle[i].dhandle, - ret = (*op)(session, cfg)); - else - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __wt_conn_btree_apply_single(session, - session->ckpt_handle[i].name, NULL, op, cfg)); + WT_WITH_DHANDLE(session, session->ckpt_handle[i], + ret = (*op)(session, cfg)); WT_RET(ret); } @@ -249,23 +243,18 @@ __wt_checkpoint_list(WT_SESSION_IMPL *session, const char *cfg[]) if (F_ISSET(S2BT(session), WT_BTREE_NO_CHECKPOINT)) return (0); - /* Make sure there is space for the next entry. */ - WT_RET(__wt_realloc_def(session, &session->ckpt_handle_allocated, - session->ckpt_handle_next + 1, &session->ckpt_handle)); - /* Not strictly necessary, but cleaner to clear the current handle. */ name = session->dhandle->name; session->dhandle = NULL; - /* Record busy file names, we'll deal with them in the checkpoint. */ - if ((ret = __wt_session_get_btree(session, name, NULL, NULL, 0)) == 0) - session->ckpt_handle[session->ckpt_handle_next++].dhandle = - session->dhandle; - else if (ret == EBUSY) - ret = __wt_strdup(session, name, - &session->ckpt_handle[session->ckpt_handle_next++].name); + if ((ret = __wt_session_get_btree(session, name, NULL, NULL, 0)) != 0) + return (ret == EBUSY ? 0 : ret); - return (ret); + /* Make sure there is space for the next entry. */ + WT_RET(__wt_realloc_def(session, &session->ckpt_handle_allocated, + session->ckpt_handle_next + 1, &session->ckpt_handle)); + session->ckpt_handle[session->ckpt_handle_next++] = session->dhandle; + return (0); } /* @@ -411,20 +400,7 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) session->isolation = txn->isolation = WT_ISO_READ_COMMITTED; WT_ERR(__checkpoint_apply(session, cfg, __checkpoint_write_leaves)); - /* - * The underlying flush routine scheduled an asynchronous flush - * after writing the leaf pages, but in order to minimize I/O - * while holding the schema lock, do a flush and wait for the - * completion. Do it after flushing the pages to give the - * asynchronous flush as much time as possible before we wait. - */ - if (F_ISSET(conn, WT_CONN_CKPT_SYNC)) - WT_ERR(__checkpoint_apply(session, cfg, __wt_checkpoint_sync)); - - /* Acquire the schema lock. */ - F_SET(session, WT_SESSION_LOCKED_SCHEMA); - __wt_spin_lock(session, &conn->schema_lock); - + /* Start the checkpoint for real. */ WT_ERR(__wt_meta_track_on(session)); tracking = true; @@ -543,16 +519,24 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) * Recovery relies on the checkpoint LSN in the metadata only being * updated by full checkpoints so only checkpoint the metadata for * full or non-logged checkpoints. + * + * XXX lots of overlap with __wt_meta_track_off. */ if (full || !logging) { session->isolation = txn->isolation = WT_ISO_READ_UNCOMMITTED; /* Disable metadata tracking during the metadata checkpoint. */ saved_meta_next = session->meta_track_next; session->meta_track_next = NULL; + WT_WITH_METADATA_LOCK(session, + WT_WITH_DHANDLE(session, + WT_SESSION_META_DHANDLE(session), + ret = __wt_checkpoint(session, cfg))); + session->meta_track_next = saved_meta_next; + WT_ERR(ret); + WT_WITH_DHANDLE(session, WT_SESSION_META_DHANDLE(session), - ret = __wt_checkpoint(session, cfg)); - session->meta_track_next = saved_meta_next; + ret = __wt_checkpoint_sync(session, NULL)); WT_ERR(ret); WT_ERR(__checkpoint_verbose_track(session, @@ -610,23 +594,13 @@ err: /* WT_TXN_LOG_CKPT_STOP : WT_TXN_LOG_CKPT_CLEANUP, NULL)); } - for (i = 0; i < session->ckpt_handle_next; ++i) { - if (session->ckpt_handle[i].dhandle == NULL) { - __wt_free(session, session->ckpt_handle[i].name); - continue; - } - WT_WITH_DHANDLE(session, session->ckpt_handle[i].dhandle, + for (i = 0; i < session->ckpt_handle_next; ++i) + WT_WITH_DHANDLE(session, session->ckpt_handle[i], WT_TRET(__wt_session_release_btree(session))); - } __wt_free(session, session->ckpt_handle); session->ckpt_handle_allocated = session->ckpt_handle_next = 0; - if (F_ISSET(session, WT_SESSION_LOCKED_SCHEMA)) { - F_CLR(session, WT_SESSION_LOCKED_SCHEMA); - __wt_spin_unlock(session, &conn->schema_lock); - } - session->isolation = txn->isolation = saved_isolation; return (ret); } @@ -1188,7 +1162,8 @@ __wt_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_ASSERT(session, session->dhandle->checkpoint == NULL); /* Should be holding the schema lock. */ - WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_SCHEMA)); + WT_ASSERT(session, !WT_IS_METADATA(session->dhandle) || + F_ISSET(session, WT_SESSION_LOCKED_METADATA)); return (__checkpoint_worker(session, cfg, true, true)); } @@ -1251,14 +1226,6 @@ __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) __wt_cache_op(session, NULL, WT_SYNC_DISCARD) : EBUSY); } - /* - * We should already have the schema lock unless we're finishing a bulk - * load -- the only other paths to closing files (sweep and LSM) have - * already checked for read-only trees. - */ - WT_ASSERT(session, - final || bulk || F_ISSET(session, WT_SESSION_LOCKED_SCHEMA)); - /* * Turn on metadata tracking if: * - The session is not already doing metadata tracking. diff --git a/test/suite/test_bulk02.py b/test/suite/test_bulk02.py index eeca6a56967..fe8118209f2 100644 --- a/test/suite/test_bulk02.py +++ b/test/suite/test_bulk02.py @@ -49,8 +49,7 @@ class test_bulkload_checkpoint(wttest.WiredTigerTestCase, suite_subprocess): scenarios = number_scenarios(multiply_scenarios('.', types, ckpt_type)) - # Bulk-load handles return EBUSY to the checkpoint code, causing the - # checkpoint call to find a handle anyway, and create fake checkpoint. + # Bulk-load handles are skipped by checkpoints. # Named and unnamed checkpoint versions. def test_bulkload_checkpoint(self): # Open a bulk cursor and insert a few records. @@ -72,11 +71,8 @@ class test_bulkload_checkpoint(wttest.WiredTigerTestCase, suite_subprocess): # In the case of named checkpoints, verify they're still there, # reflecting an empty file. if self.ckpt_type == 'named': - cursor = self.session.open_cursor( - self.uri, None, 'checkpoint=myckpt') - self.assertEquals(cursor.next(), wiredtiger.WT_NOTFOUND) - cursor.close() - + self.assertRaises(wiredtiger.WiredTigerError, + lambda: self.session.open_cursor(self.uri, None, 'checkpoint=myckpt')) # test_bulkload_backup # Test bulk-load with hot-backup. -- cgit v1.2.1 From 7b479236ea199693f8f0905f8b91cd97b1f01b9d Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Mon, 25 Jan 2016 16:24:51 +1100 Subject: WT-2346 Protect LSM flushes from racing with checkpoints. Internal flush operations used to rely on the schema lock to avoid racing with checkpoints. That is no longer sufficient, use the checkpoint lock instead but also switch on "no wait" mode to avoid blocking for long periods while checkpoints are running. --- src/lsm/lsm_work_unit.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/lsm/lsm_work_unit.c b/src/lsm/lsm_work_unit.c index 4faa25967ad..dbaf8602846 100644 --- a/src/lsm/lsm_work_unit.c +++ b/src/lsm/lsm_work_unit.c @@ -334,14 +334,27 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, /* * Turn on metadata tracking to ensure the checkpoint gets the * necessary handle locks. + * + * Ensure that we don't race with a running checkpoint: the checkpoint + * lock protects against us racing with an application checkpoint in + * this chunk. Don't wait for it, though: checkpoints can take a long + * time, and our checkpoint operation should be very quick. */ WT_ERR(__wt_meta_track_on(session)); - WT_WITH_SCHEMA_LOCK(session, ret, - ret = __wt_schema_worker( - session, chunk->uri, __wt_checkpoint, NULL, NULL, 0)); + F_SET(session, WT_SESSION_LOCK_NO_WAIT); + WT_WITH_CHECKPOINT_LOCK(session, ret, + WT_WITH_SCHEMA_LOCK(session, ret, + ret = __wt_schema_worker( + session, chunk->uri, __wt_checkpoint, NULL, NULL, 0))); WT_TRET(__wt_meta_track_off(session, false, ret != 0)); - if (ret != 0) + F_CLR(session, WT_SESSION_LOCK_NO_WAIT); + if (ret != 0) { + if (ret == EBUSY) { + ret = 0; + goto err; + } WT_ERR_MSG(session, ret, "LSM checkpoint"); + } /* Now the file is written, get the chunk size. */ WT_ERR(__wt_lsm_tree_set_chunk_size(session, chunk)); -- cgit v1.2.1 From 583c55ccfcaa9e4f11e7122187676adc2e3096bf Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 29 Jan 2016 15:04:22 -0500 Subject: WT-2123: Don't clear allocated memory if not required. Create the __wt_malloc() function. Change __wt_strndup() to call malloc instead of calloc. --- src/include/extern.h | 1 + src/os_posix/os_alloc.c | 59 ++++++++++++++++++++++++++++++++++--------------- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/src/include/extern.h b/src/include/extern.h index b71f4b12486..8ead7d1bf3c 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -480,6 +480,7 @@ extern int __wt_turtle_read(WT_SESSION_IMPL *session, const char *key, char **va extern int __wt_turtle_update(WT_SESSION_IMPL *session, const char *key, const char *value); extern void __wt_abort(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); extern int __wt_calloc(WT_SESSION_IMPL *session, size_t number, size_t size, void *retp); +extern int __wt_malloc(WT_SESSION_IMPL *session, size_t bytes_to_allocate, void *retp); extern int __wt_realloc(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp); extern int __wt_realloc_aligned(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp); extern int __wt_strndup(WT_SESSION_IMPL *session, const void *str, size_t len, void *retp); diff --git a/src/os_posix/os_alloc.c b/src/os_posix/os_alloc.c index 3876f9a1afe..35d46ece464 100644 --- a/src/os_posix/os_alloc.c +++ b/src/os_posix/os_alloc.c @@ -23,16 +23,6 @@ #define free tc_free #endif -/* - * There's no malloc interface, WiredTiger never calls malloc. - * - * The problem is an application might allocate memory, write secret stuff in - * it, free the memory, then WiredTiger allocates the memory and uses it for a - * file page or log record, then writes it to disk, without having overwritten - * it fully. That results in the secret stuff being protected by WiredTiger's - * permission mechanisms, potentially inappropriate for the secret stuff. - */ - /* * __wt_calloc -- * ANSI calloc function. @@ -66,6 +56,39 @@ __wt_calloc(WT_SESSION_IMPL *session, size_t number, size_t size, void *retp) return (0); } +/* + * __wt_malloc -- + * ANSI malloc function. + */ +int +__wt_malloc(WT_SESSION_IMPL *session, size_t bytes_to_allocate, void *retp) +{ + void *p; + + /* + * Defensive: if our caller doesn't handle errors correctly, ensure a + * free won't fail. + */ + *(void **)retp = NULL; + + /* + * !!! + * This function MUST handle a NULL WT_SESSION_IMPL handle. + */ + WT_ASSERT(session, bytes_to_allocate != 0); + + if (session != NULL) + WT_STAT_FAST_CONN_INCR(session, memory_allocation); + + if ((p = malloc(bytes_to_allocate)) == NULL) + WT_RET_MSG(session, __wt_errno(), + "memory allocation of %" WT_SIZET_FMT " bytes failed", + bytes_to_allocate); + + *(void **)retp = p; + return (0); +} + /* * __wt_realloc -- * ANSI realloc function. @@ -107,12 +130,8 @@ __wt_realloc(WT_SESSION_IMPL *session, bytes_to_allocate); /* - * Clear the allocated memory -- an application might: allocate memory, - * write secret stuff into it, free the memory, then we re-allocate the - * memory and use it for a file page or log record, and then write it to - * disk. That would result in the secret stuff being protected by the - * WiredTiger permission mechanisms, potentially inappropriate for the - * secret stuff. + * Clear the allocated memory, parts of WiredTiger depend on allocated + * memory being cleared. */ memset((uint8_t *) p + bytes_allocated, 0, bytes_to_allocate - bytes_allocated); @@ -184,7 +203,10 @@ __wt_realloc_aligned(WT_SESSION_IMPL *session, __wt_free(session, p); p = newp; - /* Clear the allocated memory (see above). */ + /* + * Clear the allocated memory, parts of WiredTiger depend on + * allocated memory being cleared. + */ memset((uint8_t *)p + bytes_allocated, 0, bytes_to_allocate - bytes_allocated); @@ -221,13 +243,14 @@ __wt_strndup(WT_SESSION_IMPL *session, const void *str, size_t len, void *retp) return (0); } - WT_RET(__wt_calloc(session, len + 1, 1, &p)); + WT_RET(__wt_malloc(session, len + 1, &p)); /* * Don't change this to strncpy, we rely on this function to duplicate * "strings" that contain nul bytes. */ memcpy(p, str, len); + ((uint8_t *)p)[len] = '\0'; *(void **)retp = p; return (0); -- cgit v1.2.1 From 530dadf9018d623956758f4cbba498f346ff33a4 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 29 Jan 2016 17:00:42 -0500 Subject: WT-2123: Don't clear allocated memory if not required The buffers we don't want to waste time clearing are read/write WT_ITEMs. Change the standard realloc code to take an argument if memory needs to be cleared. Add a new standard realloc API that clears memory, and replace __wt_realloc_aligned with a WT_ITEM-specific function that never clears memory (whether is uses the underlying realloc or posix-memalign functions). --- src/include/extern.h | 2 +- src/os_posix/os_alloc.c | 63 +++++++++++++++++++++++++++++-------------------- src/support/scratch.c | 10 ++------ 3 files changed, 41 insertions(+), 34 deletions(-) diff --git a/src/include/extern.h b/src/include/extern.h index 8ead7d1bf3c..eb17aa602fe 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -482,7 +482,7 @@ extern void __wt_abort(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((nor extern int __wt_calloc(WT_SESSION_IMPL *session, size_t number, size_t size, void *retp); extern int __wt_malloc(WT_SESSION_IMPL *session, size_t bytes_to_allocate, void *retp); extern int __wt_realloc(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp); -extern int __wt_realloc_aligned(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp); +extern int __wt_realloc_item( WT_SESSION_IMPL *session, WT_ITEM *buf, size_t bytes_to_allocate); extern int __wt_strndup(WT_SESSION_IMPL *session, const void *str, size_t len, void *retp); extern void __wt_free_int(WT_SESSION_IMPL *session, const void *p_arg); extern int __wt_dirlist(WT_SESSION_IMPL *session, const char *dir, const char *prefix, uint32_t flags, char ***dirlist, u_int *countp); diff --git a/src/os_posix/os_alloc.c b/src/os_posix/os_alloc.c index 35d46ece464..dcc16fde666 100644 --- a/src/os_posix/os_alloc.c +++ b/src/os_posix/os_alloc.c @@ -90,12 +90,13 @@ __wt_malloc(WT_SESSION_IMPL *session, size_t bytes_to_allocate, void *retp) } /* - * __wt_realloc -- + * __realloc_func -- * ANSI realloc function. */ -int -__wt_realloc(WT_SESSION_IMPL *session, - size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp) +static int +__realloc_func(WT_SESSION_IMPL *session, + size_t *bytes_allocated_ret, size_t bytes_to_allocate, bool clear_memory, + void *retp) { void *p; size_t bytes_allocated; @@ -133,8 +134,9 @@ __wt_realloc(WT_SESSION_IMPL *session, * Clear the allocated memory, parts of WiredTiger depend on allocated * memory being cleared. */ - memset((uint8_t *) - p + bytes_allocated, 0, bytes_to_allocate - bytes_allocated); + if (clear_memory) + memset((uint8_t *)p + bytes_allocated, + 0, bytes_to_allocate - bytes_allocated); /* Update caller's bytes allocated value. */ if (bytes_allocated_ret != NULL) @@ -145,24 +147,42 @@ __wt_realloc(WT_SESSION_IMPL *session, } /* - * __wt_realloc_aligned -- - * ANSI realloc function that aligns to buffer boundaries, configured with - * the "buffer_alignment" key to wiredtiger_open. + * __wt_realloc -- + * WiredTiger's realloc API. */ int -__wt_realloc_aligned(WT_SESSION_IMPL *session, +__wt_realloc(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp) { -#if defined(HAVE_POSIX_MEMALIGN) - WT_DECL_RET; + return (__realloc_func( + session, bytes_allocated_ret, bytes_to_allocate, true, retp)); +} + +/* + * __wt_realloc_item -- + * Allocate or reallocate memory for a WT_ITEM, optionally aligning to + * buffer boundaries (configured with "buffer_alignment" to wiredtiger_open). + */ +int +__wt_realloc_item( + WT_SESSION_IMPL *session, WT_ITEM *buf, size_t bytes_to_allocate) +{ + size_t *bytes_allocated_ret; + void *retp; + bytes_allocated_ret = &buf->memsize; + retp = &buf->mem; + +#if defined(HAVE_POSIX_MEMALIGN) /* * !!! * This function MUST handle a NULL WT_SESSION_IMPL handle. */ - if (session != NULL && S2C(session)->buffer_alignment > 0) { - void *p, *newp; + if (F_ISSET(buf, WT_ITEM_ALIGNED) && + session != NULL && S2C(session)->buffer_alignment > 0) { + WT_DECL_RET; size_t bytes_allocated; + void *p, *newp; /* * Sometimes we're allocating memory and we don't care about the @@ -203,13 +223,6 @@ __wt_realloc_aligned(WT_SESSION_IMPL *session, __wt_free(session, p); p = newp; - /* - * Clear the allocated memory, parts of WiredTiger depend on - * allocated memory being cleared. - */ - memset((uint8_t *)p + bytes_allocated, 0, - bytes_to_allocate - bytes_allocated); - /* Update caller's bytes allocated value. */ if (bytes_allocated_ret != NULL) *bytes_allocated_ret = bytes_to_allocate; @@ -222,11 +235,11 @@ __wt_realloc_aligned(WT_SESSION_IMPL *session, * If there is no posix_memalign function, or no alignment configured, * fall back to realloc. * - * Windows note: Visual C CRT memalign does not match Posix behavior - * and would also double each allocation so it is bad for memory use + * Windows note: Visual C CRT memalign does not match POSIX behavior + * and would also double each allocation so it is bad for memory use. */ - return (__wt_realloc( - session, bytes_allocated_ret, bytes_to_allocate, retp)); + return (__realloc_func( + session, bytes_allocated_ret, bytes_to_allocate, false, retp)); } /* diff --git a/src/support/scratch.c b/src/support/scratch.c index 94020ba2621..0e23d0dd695 100644 --- a/src/support/scratch.c +++ b/src/support/scratch.c @@ -40,14 +40,8 @@ __wt_buf_grow_worker(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size) * This function is also used to ensure data is local to the buffer, * check to see if we actually need to grow anything. */ - if (size > buf->memsize) { - if (F_ISSET(buf, WT_ITEM_ALIGNED)) - WT_RET(__wt_realloc_aligned( - session, &buf->memsize, size, &buf->mem)); - else - WT_RET(__wt_realloc( - session, &buf->memsize, size, &buf->mem)); - } + if (size > buf->memsize) + WT_RET(__wt_realloc_item(session, buf, size)); if (buf->data == NULL) { buf->data = buf->mem; -- cgit v1.2.1 From 5c561344025122cd7c7e3857e65e5f98096200d7 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 1 Feb 2016 11:35:33 -0500 Subject: WT-2361: column-store starting record number error Fix a comment. --- src/reconcile/rec_write.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/reconcile/rec_write.c b/src/reconcile/rec_write.c index 332449027a9..24999ec82cc 100644 --- a/src/reconcile/rec_write.c +++ b/src/reconcile/rec_write.c @@ -3963,7 +3963,7 @@ __rec_col_fix(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) * record 100 moves to another page. When we reconcile * the original page, we write record 98, then we don't * see record 99 for whatever reason. If we've moved - * record 1000, we don't know to write a deleted record + * record 100, we don't know to write a deleted record * 99 on the page.) * * The record number recorded during the split is the @@ -4522,7 +4522,7 @@ compare: /* * record 100 moves to another page. When we reconcile * the original page, we write record 98, then we don't * see record 99 for whatever reason. If we've moved - * record 1000, we don't know to write a deleted record + * record 100, we don't know to write a deleted record * 99 on the page.) * * The record number recorded during the split is the -- cgit v1.2.1 From 9b87a2f1211d36231a765a8f363a1a90eda26f7c Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Mon, 1 Feb 2016 22:34:40 -0500 Subject: WT-2375 Added tests using revint, a reverse integer collator for testing. Revint demonstrates how collators are expected to decode non-trivial key formats and work in situations with duplicate keys. The new test uses both a collator and an extractor. --- build_posix/Make.subdirs | 1 + ext/collators/revint/Makefile.am | 10 ++ ext/collators/revint/revint_collator.c | 157 ++++++++++++++++++++++++++++++++ test/suite/test_collator.py | 161 +++++++++++++++++++++++++++++++++ 4 files changed, 329 insertions(+) create mode 100644 ext/collators/revint/Makefile.am create mode 100644 ext/collators/revint/revint_collator.c create mode 100644 test/suite/test_collator.py diff --git a/build_posix/Make.subdirs b/build_posix/Make.subdirs index e1f8a05c613..9a028827171 100644 --- a/build_posix/Make.subdirs +++ b/build_posix/Make.subdirs @@ -6,6 +6,7 @@ # If the directory exists, it is added to AUTO_SUBDIRS. # If a condition is included, the subdir is made conditional via AM_CONDITIONAL ext/collators/reverse +ext/collators/revint ext/compressors/lz4 LZ4 ext/compressors/nop ext/compressors/snappy SNAPPY diff --git a/ext/collators/revint/Makefile.am b/ext/collators/revint/Makefile.am new file mode 100644 index 00000000000..8c85c6a4701 --- /dev/null +++ b/ext/collators/revint/Makefile.am @@ -0,0 +1,10 @@ +AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include + +noinst_LTLIBRARIES = libwiredtiger_revint_collator.la +libwiredtiger_revint_collator_la_SOURCES = revint_collator.c + +# libtool hack: noinst_LTLIBRARIES turns off building shared libraries as well +# as installation, it will only build static libraries. As far as I can tell, +# the "approved" libtool way to turn them back on is by adding -rpath. +libwiredtiger_revint_collator_la_LDFLAGS = \ + -avoid-version -module -rpath /nowhere diff --git a/ext/collators/revint/revint_collator.c b/ext/collators/revint/revint_collator.c new file mode 100644 index 00000000000..ac4a51970bb --- /dev/null +++ b/ext/collators/revint/revint_collator.c @@ -0,0 +1,157 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include + +#include +#include +#include + +/* + * Set up a collator used with indices having a single integer key, + * where the ordering is descending (reversed). + */ + +/* + * collate_error -- + * Handle errors in the collator in a standard way. + */ +static void +collate_error(int ret, const char *description) +{ + fprintf(stderr, "revint_collator error: %s: %s\n", + wiredtiger_strerror(ret), description); + /* + * For errors, we call abort. Since this collator is used in testing + * WiredTiger, we want it to immediately fail hard. A product-ready + * version of this function would not abort. + */ + abort(); +} + +/* + * collate_revint -- + * WiredTiger reverse integer collation, used for tests. + */ +static int +collate_revint(WT_COLLATOR *collator, + WT_SESSION *session, const WT_ITEM *k1, const WT_ITEM *k2, int *cmp) +{ + WT_PACK_STREAM *s1, *s2; + int ret; + int64_t i1, i2, p1, p2; + + (void)collator; /* Unused */ + (void)session; + + s1 = NULL; + s2 = NULL; + + /* + * All indices using this collator have an integer key, and the + * primary key is also an integer. A collator is passed the + * concatenation of index key and primary key (when available), + * hence we unpack using "ii". + */ + if ((ret = wiredtiger_unpack_start(session, "ii", + k1->data, k1->size, &s1)) != 0 || + (ret = wiredtiger_unpack_start(session, "ii", + k2->data, k2->size, &s2)) != 0) + collate_error(ret, "unpack start"); + /* does not return */ + + if ((ret = wiredtiger_unpack_int(s1, &i1)) != 0) + collate_error(ret, "unpack index key 1"); + if ((ret = wiredtiger_unpack_int(s2, &i2)) != 0) + collate_error(ret, "unpack index key 2"); + + /* sorting is reversed */ + if (i1 < i2) + *cmp = 1; + else if (i1 > i2) + *cmp = -1; + else { + /* + * Compare primary keys next. + * + * Allow failures here. A collator may be called with an + * item that includes a index key and no primary key. Among + * items having the same index key, an item with no primary + * key should sort before an item with a primary key. The + * reason is that if the application calls WT_CURSOR::search + * on a index key for which there are more than one value, + * the search key will not yet have a primary key. We want + * to position the cursor at the 'first' matching index key + * so that repeated calls to WT_CURSOR::next will see them + * all. + * + * To keep this code simple, we do keep secondary sort of + * primary keys in forward (not reversed) order. + */ + if ((ret = wiredtiger_unpack_int(s1, &p1)) != 0) { + if (ret == ENOMEM) + p1 = INT64_MIN; /* sort this first */ + else + collate_error(ret, "unpack primary key 1"); + } + if ((ret = wiredtiger_unpack_int(s2, &p2)) != 0) { + if (ret == ENOMEM) + p2 = INT64_MIN; /* sort this first */ + else + collate_error(ret, "unpack primary key 2"); + } + + /* sorting is not reversed here */ + if (p1 < p2) + *cmp = -1; + else if (p1 > p2) + *cmp = 1; + else + *cmp = 0; /* index key and primary key are same */ + } + if ((ret = wiredtiger_pack_close(s1, NULL)) != 0 || + (ret = wiredtiger_pack_close(s2, NULL)) != 0) + collate_error(ret, "unpack close"); + + return (0); +} + +static WT_COLLATOR revint_collator = { collate_revint, NULL, NULL }; + +/* + * wiredtiger_extension_init -- + * WiredTiger revint collation extension. + */ +int +wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) +{ + (void)config; /* Unused parameters */ + + return (connection->add_collator( + connection, "revint", &revint_collator, NULL)); +} diff --git a/test/suite/test_collator.py b/test/suite/test_collator.py new file mode 100644 index 00000000000..7c13cdc4736 --- /dev/null +++ b/test/suite/test_collator.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os +import wiredtiger, wttest, run +from wtscenario import check_scenarios, number_scenarios + +# test_collator.py +# Test indices using a custom extractor and collator. +class test_collator(wttest.WiredTigerTestCase): + """ + Test indices with a custom extractor to create an index, + with our own collator. + Our set of rows looks like a multiplication table: + row '0': '0,0,0,0' + row '1': '0,1,2,3' + row '2': '0,2,4,6' + with the twist that entries are mod 100. So, looking further: + row '40': '0,40,80,20' + + Each column is placed into its own index. Our collator reverses + the values. + """ + nentries = 100 + nindices = 4 + + # Return the wiredtiger_open extension argument for a shared library. + def extensionArg(self, exts): + extfiles = [] + for ext in exts: + (dirname, name, libname) = ext + if name != None and name != 'none': + testdir = os.path.dirname(__file__) + extdir = os.path.join(run.wt_builddir, 'ext', dirname) + extfile = os.path.join( + extdir, name, '.libs', 'libwiredtiger_' + libname + '.so') + if not os.path.exists(extfile): + self.skipTest('extension "' + extfile + '" not built') + if not extfile in extfiles: + extfiles.append(extfile) + if len(extfiles) == 0: + return '' + else: + return ',extensions=["' + '","'.join(extfiles) + '"]' + + # Override WiredTigerTestCase, we have extensions. + def setUpConnectionOpen(self, dir): + extarg = self.extensionArg([('extractors', 'csv', 'csv_extractor'), + ('collators', 'revint', 'revint_collator')]) + connarg = 'create,error_prefix="{0}: ",{1}'.format( + self.shortid(), extarg) + conn = self.wiredtiger_open(dir, connarg) + self.pr(`conn`) + return conn + + def create_indices(self): + # Create self.nindices index files, each with a column from the CSV + for i in range(0, self.nindices): + si = str(i) + self.session.create('index:collator:x' + si, + 'key_format=i,columns=(key),' + + 'collator=revint,' + + 'extractor=csv,app_metadata={"format" : "i",' + + '"field" : "' + si + '"}') + + def drop_indices(self): + for i in range(0, self.nindices): + self.session.drop("index:collator:x" + str(i)) + + def csv(self, s, i): + return s.split(',')[i] + + def expected_main_value(self, i): + return ','.join([str((i*j)%100) for j in range(0, self.nindices)]) + + # We split the population into two phases + # (in anticipation of future tests that create + # indices between the two population steps). + def populate(self): + cursor = self.session.open_cursor('table:collator', None, None) + for i in range(0, self.nentries): + cursor[i] = self.expected_main_value(i) + cursor.close() + + def check_entries(self): + cursor = self.session.open_cursor('table:collator', None, None) + icursor = [] + for i in range(0, self.nindices): + icursor.append(self.session.open_cursor('index:collator:x' + str(i), + None, None)) + i = 0 + for primkey, value in cursor: + # Check main table + expect = self.expected_main_value(i) + self.assertEqual(i, primkey) + self.assertEqual(value, expect) + for idx in range(0, self.nindices): + c = icursor[idx] + indexkey = (i*idx)%100 + c.set_key(indexkey) + self.assertEqual(c.search(), 0) + value = c.get_value() + key = c.get_key() + while value != expect and key == indexkey and \ + self.csv(value, idx) == self.csv(expect, idx): + self.assertEqual(0, c.next()) + value = c.get_value() + key = c.get_key() + self.assertEqual(value, expect) + i += 1 + self.assertEqual(self.nentries, i) + for i in range(0, self.nindices): + c = icursor[i] + c.reset() + expected = set(range(0, self.nentries)) + for key, val in c: + primkey = int(val.split(',')[1]) + expected.remove(primkey) + self.assertEquals(0, len(expected)) + c.close() + + def test_index(self): + self.session.create("table:collator", "key_format=i,value_format=S," + "columns=(primarykey,value)") + self.create_indices() + self.populate() + self.check_entries() + + # Drop and recreate all indices, everything should be there. + self.drop_indices() + self.create_indices() + self.check_entries() + + +if __name__ == '__main__': + wttest.run() -- cgit v1.2.1 From faf504846ba81e991e5784166571c6fdcee8b493 Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Mon, 1 Feb 2016 22:55:46 -0500 Subject: WT-2375 Whitespace and spelling. --- dist/s_string.ok | 1 + ext/collators/revint/revint_collator.c | 4 ++-- test/suite/test_collator.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dist/s_string.ok b/dist/s_string.ok index 19fa27cd719..92ad1f23347 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -891,6 +891,7 @@ resize resizing ret retp +revint rf rle rmw diff --git a/ext/collators/revint/revint_collator.c b/ext/collators/revint/revint_collator.c index ac4a51970bb..3cc251c6c4a 100644 --- a/ext/collators/revint/revint_collator.c +++ b/ext/collators/revint/revint_collator.c @@ -79,7 +79,7 @@ collate_revint(WT_COLLATOR *collator, * hence we unpack using "ii". */ if ((ret = wiredtiger_unpack_start(session, "ii", - k1->data, k1->size, &s1)) != 0 || + k1->data, k1->size, &s1)) != 0 || (ret = wiredtiger_unpack_start(session, "ii", k2->data, k2->size, &s2)) != 0) collate_error(ret, "unpack start"); @@ -125,7 +125,7 @@ collate_revint(WT_COLLATOR *collator, else collate_error(ret, "unpack primary key 2"); } - + /* sorting is not reversed here */ if (p1 < p2) *cmp = -1; diff --git a/test/suite/test_collator.py b/test/suite/test_collator.py index 7c13cdc4736..34b5c20247f 100644 --- a/test/suite/test_collator.py +++ b/test/suite/test_collator.py @@ -97,7 +97,7 @@ class test_collator(wttest.WiredTigerTestCase): def expected_main_value(self, i): return ','.join([str((i*j)%100) for j in range(0, self.nindices)]) - + # We split the population into two phases # (in anticipation of future tests that create # indices between the two population steps). -- cgit v1.2.1 From 41a08af67329467555385b22f9d7dbc3d9b27a5d Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 2 Feb 2016 07:23:08 -0500 Subject: WT-2361: column-store starting record number error Add a "quiet" configuration command, so it's possible to keep format quiet from the CONFIG file. --- test/format/config.h | 4 ++++ test/format/format.h | 2 +- test/format/t.c | 10 +++++----- test/format/util.c | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/test/format/config.h b/test/format/config.h index d8b11b005d4..a17614bc044 100644 --- a/test/format/config.h +++ b/test/format/config.h @@ -246,6 +246,10 @@ static CONFIG c[] = { "minimum gain before prefix compression is used", 0x0, 0, 8, 256, &g.c_prefix_compression_min, NULL }, + { "quiet", + "quiet run (same as -q)", + C_IGNORE|C_BOOL, 0, 0, 0, &g.c_quiet, NULL }, + { "repeat_data_pct", "percent duplicate values in row- or var-length column-stores", 0x0, 0, 90, 90, &g.c_repeat_data_pct, NULL }, diff --git a/test/format/format.h b/test/format/format.h index 41c9de3dd30..03da1a84c9c 100644 --- a/test/format/format.h +++ b/test/format/format.h @@ -142,7 +142,6 @@ typedef struct { FILE *logfp; /* Log file */ int replay; /* Replaying a run. */ - int track; /* Track progress */ int workers_finished; /* Operations completed */ pthread_rwlock_t backup_lock; /* Hot backup running */ @@ -210,6 +209,7 @@ typedef struct { uint32_t c_merge_max; uint32_t c_mmap; uint32_t c_ops; + uint32_t c_quiet; uint32_t c_prefix_compression; uint32_t c_prefix_compression_min; uint32_t c_repeat_data_pct; diff --git a/test/format/t.c b/test/format/t.c index ccbc0442e4a..0c0485c8bfe 100644 --- a/test/format/t.c +++ b/test/format/t.c @@ -64,7 +64,7 @@ main(int argc, char *argv[]) #endif /* Track progress unless we're re-directing output to a file. */ - g.track = isatty(1) ? 1 : 0; + g.c_quiet = isatty(1) ? 0 : 1; /* Set values from the command line. */ home = NULL; @@ -99,7 +99,7 @@ main(int argc, char *argv[]) g.logging = LOG_OPS; break; case 'q': /* Quiet */ - g.track = 0; + g.c_quiet = 1; break; case 'r': /* Replay a run */ g.replay = 1; @@ -259,7 +259,7 @@ main(int argc, char *argv[]) wts_salvage(); /* Overwrite the progress line with a completion line. */ - if (g.track) + if (!g.c_quiet) printf("\r%78s\r", " "); printf("%4d: %s, %s (%.0f seconds)\n", g.run_cnt, g.c_data_source, @@ -322,8 +322,8 @@ die(int e, const char *fmt, ...) (void)pthread_rwlock_wrlock(&g.death_lock); /* Try and turn off tracking so it doesn't obscure the error message. */ - if (g.track) { - g.track = 0; + if (!g.c_quiet) { + g.c_quiet = 1; fprintf(stderr, "\n"); } if (fmt != NULL) { /* Death message. */ diff --git a/test/format/util.c b/test/format/util.c index 2b6b9d67fc3..82a6de97ab6 100644 --- a/test/format/util.c +++ b/test/format/util.c @@ -236,7 +236,7 @@ track(const char *tag, uint64_t cnt, TINFO *tinfo) int len; char msg[128]; - if (!g.track || tag == NULL) + if (g.c_quiet || tag == NULL) return; if (tinfo == NULL && cnt == 0) -- cgit v1.2.1 From af53c1f29e041e1b8ec7b889a1f328190203dcbe Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 2 Feb 2016 11:54:39 -0500 Subject: WT-2361: column-store starting record number error The page-modify field is in a union, don't clear it for row-store type splits (not a bug at the moment, but it could easily become one in the future, shoudl the row-store page-modify structure change). --- src/btree/bt_split.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/btree/bt_split.c b/src/btree/bt_split.c index bd38451d5d1..faa40602edf 100644 --- a/src/btree/bt_split.c +++ b/src/btree/bt_split.c @@ -1941,7 +1941,8 @@ __split_insert(WT_SESSION_IMPL *session, WT_REF *ref) * * Reset the split column-store page record. */ - page->modify->mod_split_recno = WT_RECNO_OOB; + if (type != WT_PAGE_ROW_LEAF) + page->modify->mod_split_recno = WT_RECNO_OOB; /* * Clear the allocated page's reference to the moved insert list element -- cgit v1.2.1 From 3544728eb8be319cf93931251195ca77dd18c4ad Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Tue, 2 Feb 2016 14:43:35 -0500 Subject: WT-2375 Use WT_EXTENSION API to unpack, for portability. In addition, various naming cleanups and error path simplification. --- ext/collators/revint/revint_collator.c | 161 ++++++++++++++++----------------- 1 file changed, 76 insertions(+), 85 deletions(-) diff --git a/ext/collators/revint/revint_collator.c b/ext/collators/revint/revint_collator.c index 3cc251c6c4a..1e432842caf 100644 --- a/ext/collators/revint/revint_collator.c +++ b/ext/collators/revint/revint_collator.c @@ -27,121 +27,103 @@ */ #include - -#include #include #include +#include /* - * Set up a collator used with indices having a single integer key, - * where the ordering is descending (reversed). + * A simple WiredTiger collator for indices having a single integer key, + * where the ordering is descending (reversed). This collator also + * requires that primary key be an integer. */ -/* - * collate_error -- - * Handle errors in the collator in a standard way. - */ -static void -collate_error(int ret, const char *description) -{ - fprintf(stderr, "revint_collator error: %s: %s\n", - wiredtiger_strerror(ret), description); - /* - * For errors, we call abort. Since this collator is used in testing - * WiredTiger, we want it to immediately fail hard. A product-ready - * version of this function would not abort. - */ - abort(); -} +/* Local collator structure. */ +typedef struct { + WT_COLLATOR collator; /* Must come first */ + WT_EXTENSION_API *wt_api; /* Extension API */ +} REVINT_COLLATOR; /* - * collate_revint -- + * revint_compare -- * WiredTiger reverse integer collation, used for tests. */ static int -collate_revint(WT_COLLATOR *collator, +revint_compare(WT_COLLATOR *collator, WT_SESSION *session, const WT_ITEM *k1, const WT_ITEM *k2, int *cmp) { - WT_PACK_STREAM *s1, *s2; + const REVINT_COLLATOR *revint_collator; + WT_EXTENSION_API *wtapi; int ret; int64_t i1, i2, p1, p2; - (void)collator; /* Unused */ - (void)session; - - s1 = NULL; - s2 = NULL; + i1 = i2 = p1 = p2 = 0; + revint_collator = (const REVINT_COLLATOR *)collator; + wtapi = revint_collator->wt_api; /* * All indices using this collator have an integer key, and the - * primary key is also an integer. A collator is passed the + * primary key is also an integer. A collator is usually passed the * concatenation of index key and primary key (when available), - * hence we unpack using "ii". + * hence we initially unpack using "ii". + * + * A collator may also be called with an item that includes a index + * key and no primary key. Among items having the same index key, + * an item with no primary key should sort before an item with a + * primary key. The reason is that if the application calls + * WT_CURSOR::search on a index key for which there are more than + * one value, the search key will not yet have a primary key. We + * want to position the cursor at the 'first' matching index key so + * that repeated calls to WT_CURSOR::next will see them all. + * + * To keep this code simple, we do not reverse the ordering + * when comparing primary keys. */ - if ((ret = wiredtiger_unpack_start(session, "ii", - k1->data, k1->size, &s1)) != 0 || - (ret = wiredtiger_unpack_start(session, "ii", - k2->data, k2->size, &s2)) != 0) - collate_error(ret, "unpack start"); - /* does not return */ - - if ((ret = wiredtiger_unpack_int(s1, &i1)) != 0) - collate_error(ret, "unpack index key 1"); - if ((ret = wiredtiger_unpack_int(s2, &i2)) != 0) - collate_error(ret, "unpack index key 2"); + if ((ret = wtapi->struct_unpack(wtapi, session, + k1->data, k1->size, "ii", &i1, &p1)) != 0) { + /* could be missing primary key, try again without */ + if ((ret = wtapi->struct_unpack(wtapi, session, + k1->data, k1->size, "i", &i1)) != 0) + return (ret); + p1 = INT64_MIN; /* sort this first */ + } + if ((ret = wtapi->struct_unpack(wtapi, session, + k2->data, k2->size, "ii", &i2, &p2)) != 0) { + /* could be missing primary key, try again without */ + if ((ret = wtapi->struct_unpack(wtapi, session, + k2->data, k2->size, "i", &i2)) != 0) + return (ret); + p2 = INT64_MIN; /* sort this first */ + } /* sorting is reversed */ if (i1 < i2) *cmp = 1; else if (i1 > i2) *cmp = -1; - else { - /* - * Compare primary keys next. - * - * Allow failures here. A collator may be called with an - * item that includes a index key and no primary key. Among - * items having the same index key, an item with no primary - * key should sort before an item with a primary key. The - * reason is that if the application calls WT_CURSOR::search - * on a index key for which there are more than one value, - * the search key will not yet have a primary key. We want - * to position the cursor at the 'first' matching index key - * so that repeated calls to WT_CURSOR::next will see them - * all. - * - * To keep this code simple, we do keep secondary sort of - * primary keys in forward (not reversed) order. - */ - if ((ret = wiredtiger_unpack_int(s1, &p1)) != 0) { - if (ret == ENOMEM) - p1 = INT64_MIN; /* sort this first */ - else - collate_error(ret, "unpack primary key 1"); - } - if ((ret = wiredtiger_unpack_int(s2, &p2)) != 0) { - if (ret == ENOMEM) - p2 = INT64_MIN; /* sort this first */ - else - collate_error(ret, "unpack primary key 2"); - } - - /* sorting is not reversed here */ - if (p1 < p2) - *cmp = -1; - else if (p1 > p2) - *cmp = 1; - else - *cmp = 0; /* index key and primary key are same */ - } - if ((ret = wiredtiger_pack_close(s1, NULL)) != 0 || - (ret = wiredtiger_pack_close(s2, NULL)) != 0) - collate_error(ret, "unpack close"); + /* compare primary keys next, not reversed */ + else if (p1 < p2) + *cmp = -1; + else if (p1 > p2) + *cmp = 1; + else + *cmp = 0; /* index key and primary key are same */ return (0); } -static WT_COLLATOR revint_collator = { collate_revint, NULL, NULL }; +/* + * revint_terminate -- + * Terminate is called to free the collator and any associated memory. + */ +static int +revint_terminate(WT_COLLATOR *collator, WT_SESSION *session) +{ + (void)session; /* Unused parameters */ + + /* Free the allocated memory. */ + free(collator); + return (0); +} /* * wiredtiger_extension_init -- @@ -150,8 +132,17 @@ static WT_COLLATOR revint_collator = { collate_revint, NULL, NULL }; int wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) { + REVINT_COLLATOR *revint_collator; + (void)config; /* Unused parameters */ + if ((revint_collator = calloc(1, sizeof(REVINT_COLLATOR))) == NULL) + return (errno); + + revint_collator->collator.compare = revint_compare; + revint_collator->collator.terminate = revint_terminate; + revint_collator->wt_api = connection->get_extension_api(connection); + return (connection->add_collator( - connection, "revint", &revint_collator, NULL)); + connection, "revint", &revint_collator->collator, NULL)); } -- cgit v1.2.1 From a8ea2664db11e181e391f1642d31e368ee4de279 Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Tue, 2 Feb 2016 14:46:01 -0500 Subject: WT-2375 Added a buffer overflow check during unpack for reliable termination. --- src/include/packing.i | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/include/packing.i b/src/include/packing.i index 784a55ef2ae..374dbd2fba3 100644 --- a/src/include/packing.i +++ b/src/include/packing.i @@ -677,6 +677,8 @@ __wt_struct_unpackv(WT_SESSION_IMPL *session, if (fmt[0] != '\0' && fmt[1] == '\0') { pv.type = fmt[0]; + if (p >= end) + return (ENOMEM); if ((ret = __unpack_read(session, &pv, &p, size)) == 0) WT_UNPACK_PUT(session, pv, ap); return (0); @@ -684,6 +686,8 @@ __wt_struct_unpackv(WT_SESSION_IMPL *session, WT_RET(__pack_init(session, &pack, fmt)); while ((ret = __pack_next(&pack, &pv)) == 0) { + if (p >= end) + return (ENOMEM); WT_RET(__unpack_read(session, &pv, &p, (size_t)(end - p))); WT_UNPACK_PUT(session, pv, ap); } -- cgit v1.2.1 From ff1bf6bba3a15c1c8ce60bde6a7b6b1db9ab6fcc Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Tue, 2 Feb 2016 15:02:39 -0500 Subject: WT-2375 Buffer overflow check needs special handling for 'u' format. --- src/include/packing.i | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/include/packing.i b/src/include/packing.i index 374dbd2fba3..7f277596345 100644 --- a/src/include/packing.i +++ b/src/include/packing.i @@ -677,7 +677,7 @@ __wt_struct_unpackv(WT_SESSION_IMPL *session, if (fmt[0] != '\0' && fmt[1] == '\0') { pv.type = fmt[0]; - if (p >= end) + if (p >= end && pv.type != 'u') return (ENOMEM); if ((ret = __unpack_read(session, &pv, &p, size)) == 0) WT_UNPACK_PUT(session, pv, ap); @@ -686,7 +686,7 @@ __wt_struct_unpackv(WT_SESSION_IMPL *session, WT_RET(__pack_init(session, &pack, fmt)); while ((ret = __pack_next(&pack, &pv)) == 0) { - if (p >= end) + if (p >= end && pv.type != 'u') return (ENOMEM); WT_RET(__unpack_read(session, &pv, &p, (size_t)(end - p))); WT_UNPACK_PUT(session, pv, ap); -- cgit v1.2.1 From 92131cafef1dafc28c6da6a393af4db72be22d37 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 2 Feb 2016 16:37:38 -0500 Subject: WT-2374 If a backup file exists, remove/recreate metadata and turtle. --- src/include/meta.h | 1 + src/meta/meta_turtle.c | 23 +++++++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/include/meta.h b/src/include/meta.h index d61022c0c44..ac0f5fedac4 100644 --- a/src/include/meta.h +++ b/src/include/meta.h @@ -21,6 +21,7 @@ #define WT_METADATA_TURTLE_SET "WiredTiger.turtle.set" /* Turtle temp file */ #define WT_METADATA_URI "metadata:" /* Metadata alias */ +#define WT_METAFILE "WiredTiger.wt" /* Metadata table */ #define WT_METAFILE_URI "file:WiredTiger.wt" /* Metadata table URI */ #define WT_LAS_URI "file:WiredTigerLAS.wt" /* Lookaside table URI*/ diff --git a/src/meta/meta_turtle.c b/src/meta/meta_turtle.c index 7182bb0fe5f..299c820029f 100644 --- a/src/meta/meta_turtle.c +++ b/src/meta/meta_turtle.c @@ -153,10 +153,11 @@ int __wt_turtle_init(WT_SESSION_IMPL *session) { WT_DECL_RET; - bool exist, exist_incr; + bool exist_backup, exist_incr, exist_turtle, load; char *metaconf; metaconf = NULL; + load = false; /* * Discard any turtle setup file left-over from previous runs. This @@ -179,13 +180,27 @@ __wt_turtle_init(WT_SESSION_IMPL *session) * done. */ WT_RET(__wt_exist(session, WT_INCREMENTAL_BACKUP, &exist_incr)); - WT_RET(__wt_exist(session, WT_METADATA_TURTLE, &exist)); - if (exist) { + WT_RET(__wt_exist(session, WT_METADATA_BACKUP, &exist_backup)); + WT_RET(__wt_exist(session, WT_METADATA_TURTLE, &exist_turtle)); + if (exist_turtle) { if (exist_incr) WT_RET_MSG(session, EINVAL, "Incremental backup after running recovery " "is not allowed."); - } else { + /* + * If we have a backup file and metadata and turtle files, + * we want to recreate the metadata from the backup. + */ + if (exist_backup) { + WT_RET(__wt_remove_if_exists( + session, WT_METAFILE)); + WT_RET(__wt_remove_if_exists( + session, WT_METADATA_TURTLE)); + load = true; + } + } else + load = true; + if (load) { if (exist_incr) F_SET(S2C(session), WT_CONN_WAS_BACKUP); -- cgit v1.2.1 From 152839497426d1e894cc69e7cae2123a711364e8 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 3 Feb 2016 12:11:21 -0500 Subject: WT-2361: column-store starting record number error Add a write barrier to the insert split code before splitting the parent. It's vanishingly unlikely we could split into the parent without writing the page's modified skip lists, but I don't see anything that prevents it. --- src/btree/bt_split.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/btree/bt_split.c b/src/btree/bt_split.c index faa40602edf..58b3ac024f5 100644 --- a/src/btree/bt_split.c +++ b/src/btree/bt_split.c @@ -1927,6 +1927,12 @@ __split_insert(WT_SESSION_IMPL *session, WT_REF *ref) __wt_cache_page_inmem_decr(session, page, page_decr); __wt_cache_page_inmem_incr(session, right, right_incr); + /* + * The act of splitting into the parent releases the pages for eviction; + * ensure the page contents are correct. + */ + WT_WRITE_BARRIER(); + /* * Split into the parent. On successful return, the original page is no * longer locked, so we cannot safely look at it. -- cgit v1.2.1 From 8dcc189ad6aede870542517d5228d12edba795c5 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Wed, 3 Feb 2016 21:42:47 -0500 Subject: WT-2374 Fix test and add warning message (turned off for now). --- src/meta/meta_turtle.c | 6 ++++++ test/suite/test_txn04.py | 8 ++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/meta/meta_turtle.c b/src/meta/meta_turtle.c index 299c820029f..0d8f61aa7cf 100644 --- a/src/meta/meta_turtle.c +++ b/src/meta/meta_turtle.c @@ -197,6 +197,12 @@ __wt_turtle_init(WT_SESSION_IMPL *session) WT_RET(__wt_remove_if_exists( session, WT_METADATA_TURTLE)); load = true; +#if 0 + __wt_msg(session, "Both %s and %s exist. " + "Remove %s and recreate metadata from %s.", + WT_METADATA_TURTLE, WT_METADATA_BACKUP, + WT_METADATA_TURTLE, WT_METADATA_BACKUP); +#endif } } else load = true; diff --git a/test/suite/test_txn04.py b/test/suite/test_txn04.py index de49c5fe235..d05204e0616 100644 --- a/test/suite/test_txn04.py +++ b/test/suite/test_txn04.py @@ -192,14 +192,10 @@ class test_txn04(wttest.WiredTigerTestCase, suite_subprocess): # Check the state after each commit/rollback. self.check_all(current, committed) - # Backup the target we modified. We expect that running - # recovery now will generate an exception if we committed. + # Backup the target we modified and verify the data. # print 'Call hot_backup with ' + self.uri self.hot_backup(self.uri, committed) - if txn == 'commit': - self.assertEqual(True, self.exception == 'true') - else: - self.assertEqual(True, self.exception == 'false') + self.assertEqual(True, self.exception == 'false') if __name__ == '__main__': wttest.run() -- cgit v1.2.1 From 5c2297305ddf5aab3ded6def9c2cc9087f876f06 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 4 Feb 2016 09:42:32 -0500 Subject: WT-2361: column-store starting record number error Don't continue searches from the "current" page if a cihld page we're waiting on splits. The problem is a thread holding a hazard pointer on a parent page X, and waiting on a child page A. Then, A is insert-split, moving a chunk of A's name-space to a new page B, that follows A in the parent page's page-index. If the parent page X then does a split into its parent, B could move to the parent of X, and the thread has to search from the root of the tree, not just from page X, to find the name-space now in page B. I've never seen this bug fire (as far as I know), it's improbable, but I think it's possible. --- src/btree/col_srch.c | 18 ++++++++++++------ src/btree/row_srch.c | 18 ++++++++++++------ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/btree/col_srch.c b/src/btree/col_srch.c index cb5a227495f..84b1e24aa77 100644 --- a/src/btree/col_srch.c +++ b/src/btree/col_srch.c @@ -116,12 +116,12 @@ __wt_col_search(WT_SESSION_IMPL *session, goto leaf_only; } -restart_root: +restart: /* Search the internal pages of the tree. */ current = &btree->root; for (depth = 2, pindex = NULL;; ++depth) { parent_pindex = pindex; -restart_page: page = current->page; + page = current->page; if (page->type != WT_PAGE_COL_INT) break; @@ -141,7 +141,7 @@ restart_page: page = current->page; __wt_split_intl_race( session, current->home, parent_pindex)) { WT_RET(__wt_page_release(session, current, 0)); - goto restart_root; + goto restart; } goto descend; } @@ -178,8 +178,14 @@ descend: /* /* * Swap the current page for the child page. If the page splits - * while we're retrieving it, restart the search in the current - * page; otherwise return on error, the swap call ensures we're + * while we're retrieving it, restart the search at the root. + * We cannot restart in the "current" page; for example, if a + * thread is appending to the tree, the page it's waiting for + * did an insert-split into the parent, then the parent split + * into its parent, the name space we are searching for may have + * moved above the current page in the tree. + * + * On other error, simply return, the swap call ensures we're * holding nothing on failure. */ if ((ret = __wt_page_swap( @@ -188,7 +194,7 @@ descend: /* continue; } if (ret == WT_RESTART) - goto restart_page; + goto restart; return (ret); } diff --git a/src/btree/row_srch.c b/src/btree/row_srch.c index 71564a7b3c5..0e7846bebe5 100644 --- a/src/btree/row_srch.c +++ b/src/btree/row_srch.c @@ -276,12 +276,12 @@ __wt_row_search(WT_SESSION_IMPL *session, goto leaf_only; } +restart: /* Search the internal pages of the tree. */ -restart_root: current = &btree->root; for (depth = 2, pindex = NULL;; ++depth) { parent_pindex = pindex; -restart_page: page = current->page; + page = current->page; if (page->type != WT_PAGE_ROW_INT) break; @@ -426,14 +426,20 @@ append: if (parent_pindex != NULL && return (ret); skiplow = skiphigh = 0; - goto restart_root; + goto restart; } } descend: /* * Swap the current page for the child page. If the page splits - * while we're retrieving it, restart the search in the current - * page; otherwise return on error, the swap call ensures we're + * while we're retrieving it, restart the search at the root. + * We cannot restart in the "current" page; for example, if a + * thread is appending to the tree, the page it's waiting for + * did an insert-split into the parent, then the parent split + * into its parent, the name space we are searching for may have + * moved above the current page in the tree. + * + * On other error, simply return, the swap call ensures we're * holding nothing on failure. */ if ((ret = __wt_page_swap( @@ -443,7 +449,7 @@ descend: /* } if (ret == WT_RESTART) { skiphigh = skiplow = 0; - goto restart_page; + goto restart; } return (ret); } -- cgit v1.2.1 From ab34a66f7f84aac2749ce8ef05019d67819f892c Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 4 Feb 2016 11:26:43 -0500 Subject: WT-2361: column-store starting record number error If we don't restart in the current page, we have to release it, we're holding a hazard references. This means all of the jumps to the restart label release the current page, simplify the world by doing the work at the lable. Fix the cursor-random code as well, make it all look the same. --- src/btree/col_srch.c | 15 +++++++++++---- src/btree/row_srch.c | 51 ++++++++++++++++++++++++++++----------------------- 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/src/btree/col_srch.c b/src/btree/col_srch.c index 84b1e24aa77..fa3618c7077 100644 --- a/src/btree/col_srch.c +++ b/src/btree/col_srch.c @@ -77,6 +77,7 @@ __wt_col_search(WT_SESSION_IMPL *session, int depth; btree = S2BT(session); + current = NULL; __cursor_pos_clear(cbt); @@ -116,7 +117,14 @@ __wt_col_search(WT_SESSION_IMPL *session, goto leaf_only; } -restart: + if (0) { +restart: /* + * Discard the currently held page and retart the search from + * the root. + */ + WT_RET(__wt_page_release(session, current, 0)); + } + /* Search the internal pages of the tree. */ current = &btree->root; for (depth = 2, pindex = NULL;; ++depth) { @@ -139,10 +147,9 @@ restart: */ if (parent_pindex != NULL && __wt_split_intl_race( - session, current->home, parent_pindex)) { - WT_RET(__wt_page_release(session, current, 0)); + session, current->home, parent_pindex)) goto restart; - } + goto descend; } diff --git a/src/btree/row_srch.c b/src/btree/row_srch.c index 0e7846bebe5..c9dae41b589 100644 --- a/src/btree/row_srch.c +++ b/src/btree/row_srch.c @@ -276,7 +276,15 @@ __wt_row_search(WT_SESSION_IMPL *session, goto leaf_only; } -restart: + if (0) { +restart: /* + * Discard the currently held page and retart the search from + * the root. + */ + WT_RET(__wt_page_release(session, current, 0)); + skiphigh = skiplow = 0; + } + /* Search the internal pages of the tree. */ current = &btree->root; for (depth = 2, pindex = NULL;; ++depth) { @@ -420,14 +428,8 @@ restart: if (pindex->entries == base) { append: if (parent_pindex != NULL && __wt_split_intl_race( - session, current->home, parent_pindex)) { - if ((ret = __wt_page_release( - session, current, 0)) != 0) - return (ret); - - skiplow = skiphigh = 0; + session, current->home, parent_pindex)) goto restart; - } } descend: /* @@ -447,10 +449,8 @@ descend: /* current = descent; continue; } - if (ret == WT_RESTART) { - skiphigh = skiplow = 0; + if (ret == WT_RESTART) goto restart; - } return (ret); } @@ -793,11 +793,19 @@ __wt_row_random_descent(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_REF *current, *descent; btree = S2BT(session); + current = NULL; __cursor_pos_clear(cbt); -restart_root: - /* Walk the internal pages of the tree. */ + if (0) { +restart: /* + * Discard the currently held page and retart the search from + * the root. + */ + WT_RET(__wt_page_release(session, current, 0)); + } + + /* Search the internal pages of the tree. */ current = &btree->root; for (;;) { page = current->page; @@ -809,22 +817,19 @@ restart_root: __wt_random(&session->rnd) % pindex->entries]; /* - * Swap the parent page for the child page; return on error, - * the swap function ensures we're holding nothing on failure. + * Swap the current page for the child page. If the page splits + * while we're retrieving it, restart the search at the root. + * + * On other error, simply return, the swap call ensures we're + * holding nothing on failure. */ if ((ret = __wt_page_swap( session, current, descent, WT_READ_RESTART_OK)) == 0) { current = descent; continue; } - /* - * Restart is returned if we find a page that's been split; the - * held page isn't discarded when restart is returned, discard - * it and restart the search from the top of the tree. - */ - if (ret == WT_RESTART && - (ret = __wt_page_release(session, current, 0)) == 0) - goto restart_root; + if (ret == WT_RESTART) + goto restart; return (ret); } -- cgit v1.2.1 From 8b8614acbc88e14c19da178391cfe141261f9486 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 4 Feb 2016 11:29:39 -0500 Subject: WT-2361: column-store starting record number error Fix a typo. --- src/btree/col_srch.c | 2 +- src/btree/row_srch.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/btree/col_srch.c b/src/btree/col_srch.c index fa3618c7077..862327699c0 100644 --- a/src/btree/col_srch.c +++ b/src/btree/col_srch.c @@ -119,7 +119,7 @@ __wt_col_search(WT_SESSION_IMPL *session, if (0) { restart: /* - * Discard the currently held page and retart the search from + * Discard the currently held page and restart the search from * the root. */ WT_RET(__wt_page_release(session, current, 0)); diff --git a/src/btree/row_srch.c b/src/btree/row_srch.c index c9dae41b589..097b0b7a520 100644 --- a/src/btree/row_srch.c +++ b/src/btree/row_srch.c @@ -278,7 +278,7 @@ __wt_row_search(WT_SESSION_IMPL *session, if (0) { restart: /* - * Discard the currently held page and retart the search from + * Discard the currently held page and restart the search from * the root. */ WT_RET(__wt_page_release(session, current, 0)); @@ -799,7 +799,7 @@ __wt_row_random_descent(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) if (0) { restart: /* - * Discard the currently held page and retart the search from + * Discard the currently held page and restart the search from * the root. */ WT_RET(__wt_page_release(session, current, 0)); -- cgit v1.2.1 From 3bc60a9b2eff6f8e1af1cfc704fbf994d1bd471f Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 4 Feb 2016 13:36:58 -0500 Subject: WT-2349 Initial addition of readonly configuration option. --- dist/api_data.py | 4 ++ dist/flags.py | 1 + dist/s_stat | 25 ++++++++++++ src/config/config_def.c | 25 +++++++----- src/conn/conn_api.c | 97 ++++++++++++++++++++++++++++++++++++++++++--- src/conn/conn_handle.c | 3 +- src/conn/conn_log.c | 7 +++- src/conn/conn_stat.c | 8 ++++ src/cursor/cur_std.c | 2 +- src/include/extern.h | 2 + src/include/flags.h | 15 +++---- src/include/wiredtiger.in | 6 +++ src/lsm/lsm_manager.c | 2 + src/os_posix/os_fallocate.c | 1 + src/os_posix/os_fsync.c | 4 ++ src/os_posix/os_ftruncate.c | 1 + src/os_posix/os_open.c | 3 ++ src/os_posix/os_remove.c | 1 + src/os_posix/os_rename.c | 1 + src/os_posix/os_rw.c | 1 + src/os_win/os_fallocate.c | 1 + src/os_win/os_fsync.c | 4 ++ src/os_win/os_ftruncate.c | 1 + src/os_win/os_open.c | 3 ++ src/os_win/os_remove.c | 1 + src/os_win/os_rename.c | 1 + src/os_win/os_rw.c | 1 + src/session/session_api.c | 64 +++++++++++++++++++++++++++++- src/txn/txn_recover.c | 8 +++- test/suite/test_txn02.py | 8 ++++ 30 files changed, 273 insertions(+), 28 deletions(-) diff --git a/dist/api_data.py b/dist/api_data.py index c386c0b345d..5a07440912f 100644 --- a/dist/api_data.py +++ b/dist/api_data.py @@ -478,6 +478,10 @@ connection_runtime_config = [ vary depending on the current eviction load''', min=1, max=20), ]), + Config('readonly', 'false', r''' + open connection in read-only mode. The database must exist. All + methods that may modify a database are disabled.''', + type='boolean'), Config('shared_cache', '', r''' shared cache configuration options. A database should configure either a cache_size or a shared_cache not both. Enabling a diff --git a/dist/flags.py b/dist/flags.py index b97235b965a..188e1fdecd0 100644 --- a/dist/flags.py +++ b/dist/flags.py @@ -99,6 +99,7 @@ flags = { 'CONN_LOG_SERVER_RUN', 'CONN_LSM_MERGE', 'CONN_PANIC', + 'CONN_READONLY', 'CONN_SERVER_ASYNC', 'CONN_SERVER_CHECKPOINT', 'CONN_SERVER_LSM', diff --git a/dist/s_stat b/dist/s_stat index 44c22ab56bb..6f9bc1a9e91 100755 --- a/dist/s_stat +++ b/dist/s_stat @@ -2,6 +2,15 @@ # Complain about unused statistics fields. t=__wt.$$ +t1=xx.search +t2=xx.files +t3=xx.output +t4=xx.bad1 +t5=xx.bad2 +t6=xx.search2 +t7=xx.bad3 +t8=xx.search3 +t9=xx.bad4 trap 'rm -f $t; exit 0' 0 1 2 3 13 15 # List of files to search: skip stat.c, it lists all of the fields by @@ -13,6 +22,22 @@ l=`sed \ -e 's,^,../,' filelist` l="$l `echo ../src/include/*.i`" +echo "$l" > $t2 + +fsearch=`sed \ + -e 's/^ int64_t \([a-z_*]*\);$/\1/p' \ + -e d ../src/include/stat.h | + sort` +echo "$fsearch" > $t1 +#fgrep -who "$fsearch" $l > $t3 +xx=`cat $t1 $t3 | sort | uniq -u` +cat $t1 $t3| sort | uniq -u > $t4 +fgrep -who "$xx" $l > $t6 +cat $t4 $t6 | sort | uniq -u > $t5 +xx=`cat $t4 $t6 | sort | uniq -u` +fgrep -who "$xx" $l > $t8 +cat $t5 $t8 | sort | uniq -u > $t7 + ( # Get the list of statistics fields. search=`sed \ diff --git a/src/config/config_def.c b/src/config/config_def.c index 879de670695..9f1110cd3d7 100644 --- a/src/config/config_def.c +++ b/src/config/config_def.c @@ -138,6 +138,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2 }, { "lsm_merge", "boolean", NULL, NULL, NULL, 0 }, + { "readonly", "boolean", NULL, NULL, NULL, 0 }, { "shared_cache", "category", NULL, NULL, confchk_wiredtiger_open_shared_cache_subconfigs, 5 }, @@ -544,6 +545,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { { "lsm_merge", "boolean", NULL, NULL, NULL, 0 }, { "mmap", "boolean", NULL, NULL, NULL, 0 }, { "multiprocess", "boolean", NULL, NULL, NULL, 0 }, + { "readonly", "boolean", NULL, NULL, NULL, 0 }, { "session_max", "int", NULL, "min=1", NULL, 0 }, { "session_scratch_max", "int", NULL, NULL, NULL, 0 }, { "shared_cache", "category", @@ -624,6 +626,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { { "lsm_merge", "boolean", NULL, NULL, NULL, 0 }, { "mmap", "boolean", NULL, NULL, NULL, 0 }, { "multiprocess", "boolean", NULL, NULL, NULL, 0 }, + { "readonly", "boolean", NULL, NULL, NULL, 0 }, { "session_max", "int", NULL, "min=1", NULL, 0 }, { "session_scratch_max", "int", NULL, NULL, NULL, 0 }, { "shared_cache", "category", @@ -701,6 +704,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { { "lsm_merge", "boolean", NULL, NULL, NULL, 0 }, { "mmap", "boolean", NULL, NULL, NULL, 0 }, { "multiprocess", "boolean", NULL, NULL, NULL, 0 }, + { "readonly", "boolean", NULL, NULL, NULL, 0 }, { "session_max", "int", NULL, "min=1", NULL, 0 }, { "session_scratch_max", "int", NULL, NULL, NULL, 0 }, { "shared_cache", "category", @@ -776,6 +780,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { { "lsm_merge", "boolean", NULL, NULL, NULL, 0 }, { "mmap", "boolean", NULL, NULL, NULL, 0 }, { "multiprocess", "boolean", NULL, NULL, NULL, 0 }, + { "readonly", "boolean", NULL, NULL, NULL, 0 }, { "session_max", "int", NULL, "min=1", NULL, 0 }, { "session_scratch_max", "int", NULL, NULL, NULL, 0 }, { "shared_cache", "category", @@ -851,12 +856,12 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",file_manager=(close_handle_minimum=250,close_idle_time=30," "close_scan_interval=10),log=(archive=,compressor=,enabled=0," "file_max=100MB,path=,prealloc=,recover=on,zero_fill=0)," - "lsm_manager=(merge=,worker_thread_max=4),lsm_merge=," + "lsm_manager=(merge=,worker_thread_max=4),lsm_merge=,readonly=0," "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," "statistics=none,statistics_log=(on_close=0," "path=\"WiredTigerStat.%d.%H\",sources=," "timestamp=\"%b %d %H:%M:%S\",wait=0),verbose=", - confchk_WT_CONNECTION_reconfigure, 18 + confchk_WT_CONNECTION_reconfigure, 19 }, { "WT_CURSOR.close", "", @@ -1017,14 +1022,14 @@ static const WT_CONFIG_ENTRY config_entries[] = { "close_idle_time=30,close_scan_interval=10),hazard_max=1000," "in_memory=0,log=(archive=,compressor=,enabled=0,file_max=100MB," "path=,prealloc=,recover=on,zero_fill=0),lsm_manager=(merge=," - "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0," + "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,readonly=0," "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" ",name=,quota=0,reserve=0,size=500MB),statistics=none," "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "transaction_sync=(enabled=0,method=fsync),use_environment=," "use_environment_priv=0,verbose=,write_through=", - confchk_wiredtiger_open, 37 + confchk_wiredtiger_open, 38 }, { "wiredtiger_open_all", "async=(enabled=0,ops_max=1024,threads=2),buffer_alignment=-1," @@ -1038,7 +1043,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { "close_idle_time=30,close_scan_interval=10),hazard_max=1000," "in_memory=0,log=(archive=,compressor=,enabled=0,file_max=100MB," "path=,prealloc=,recover=on,zero_fill=0),lsm_manager=(merge=," - "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0," + "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,readonly=0," "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" ",name=,quota=0,reserve=0,size=500MB),statistics=none," "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," @@ -1046,7 +1051,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { "transaction_sync=(enabled=0,method=fsync),use_environment=," "use_environment_priv=0,verbose=,version=(major=0,minor=0)," "write_through=", - confchk_wiredtiger_open_all, 38 + confchk_wiredtiger_open_all, 39 }, { "wiredtiger_open_basecfg", "async=(enabled=0,ops_max=1024,threads=2),buffer_alignment=-1," @@ -1059,14 +1064,14 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," "log=(archive=,compressor=,enabled=0,file_max=100MB,path=," "prealloc=,recover=on,zero_fill=0),lsm_manager=(merge=," - "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0," + "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,readonly=0," "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" ",name=,quota=0,reserve=0,size=500MB),statistics=none," "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "transaction_sync=(enabled=0,method=fsync),verbose=," "version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_basecfg, 32 + confchk_wiredtiger_open_basecfg, 33 }, { "wiredtiger_open_usercfg", "async=(enabled=0,ops_max=1024,threads=2),buffer_alignment=-1," @@ -1079,14 +1084,14 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," "log=(archive=,compressor=,enabled=0,file_max=100MB,path=," "prealloc=,recover=on,zero_fill=0),lsm_manager=(merge=," - "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0," + "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,readonly=0," "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" ",name=,quota=0,reserve=0,size=500MB),statistics=none," "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "transaction_sync=(enabled=0,method=fsync),verbose=," "write_through=", - confchk_wiredtiger_open_usercfg, 31 + confchk_wiredtiger_open_usercfg, 32 }, { NULL, NULL, NULL, 0 } }; diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index a53f7d96e64..4f1e4b8e23b 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -1107,6 +1107,29 @@ __conn_config_append(const char *cfg[], const char *config) *cfg = config; } + +/* + * __conn_config_readonly -- + * Append an entry to a config stack that overrides some settings + * when read-only is configured. + */ +static void +__conn_config_readonly(const char *cfg[]) +{ + const char *readonly; + + /* + * Override certain settings. Other settings at odds will return + * an error and will be checked when those settings are processed. + */ + readonly="checkpoint=(wait=0)," + "config_base=false," + "create=false," + "log=(archive=false,prealloc=false)," + "lsm_manager=(merge=false),"; + __conn_config_append(cfg, readonly); +} + /* * __conn_config_check_version -- * Check if a configuration version isn't compatible. @@ -1390,6 +1413,9 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET(__wt_config_gets(session, cfg, "create", &cval)); is_create = cval.val != 0; + if (F_ISSET(conn, WT_CONN_READONLY)) + is_create = false; + __wt_spin_lock(session, &__wt_process.spinlock); /* @@ -1447,8 +1473,21 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) exist = false; if (!is_create) WT_ERR(__wt_exist(session, WT_WIREDTIGER, &exist)); - WT_ERR(__wt_open(session, - WT_SINGLETHREAD, is_create || exist, false, 0, &conn->lock_fh)); + ret = __wt_open(session, + WT_SINGLETHREAD, is_create || exist, false, 0, &conn->lock_fh); + + /* + * If this is a read-only connection and we cannot grab the lock + * file, check if it is because there is not write permission. + * If the failure is due to permission then ignore the error. + * XXX Ignoring a permission error does allow multiple read-only + * connections to exist at the same time on a read-only directory. + */ + if (F_ISSET(conn, WT_CONN_READONLY) && ret == EPERM) { + ret = 0; + goto open_wt; + } + WT_ERR(ret); /* * Lock a byte of the file: if we don't get the lock, some other process @@ -1475,6 +1514,7 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR(__wt_write(session, conn->lock_fh, (wt_off_t)0, strlen(WT_SINGLETHREAD_STRING), WT_SINGLETHREAD_STRING)); +open_wt: /* We own the lock file, optionally create the WiredTiger file. */ WT_ERR(__wt_open(session, WT_WIREDTIGER, is_create, false, 0, &fh)); @@ -1506,6 +1546,13 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR(__wt_write(session, fh, (wt_off_t)0, len, buf)); WT_ERR(__wt_fsync(session, fh)); } else { + /* + * Although exclusive and the read-only configuration settings + * are at odds, we do not have to check against read-only here + * because it falls out from earlier code in this function + * preventing creation and confirming the database + * already exists. + */ WT_ERR(__wt_config_gets(session, cfg, "exclusive", &cval)); if (cval.val != 0) WT_ERR_MSG(session, EEXIST, @@ -1735,6 +1782,7 @@ __conn_write_base_config(WT_SESSION_IMPL *session, const char *cfg[]) "exclusive=," "in_memory=," "log=(recover=)," + "readonly=," "use_environment_priv=," "verbose=,", &base_config)); WT_ERR(__wt_config_init(session, &parser, base_config)); @@ -1807,7 +1855,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, const WT_NAME_FLAG *ft; WT_SESSION_IMPL *session; bool config_base_set; - const char *enc_cfg[] = { NULL, NULL }; + const char *enc_cfg[] = { NULL, NULL }, *merge_cfg; char version[64]; /* Leave lots of space for optional additional configuration. */ @@ -1858,6 +1906,16 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, WT_ERR(__wt_strndup( session, cval.str, cval.len, &conn->error_prefix)); + /* + * We need to look for read-only early so that we can use it + * in __conn_single and whether to use the base config file. + * XXX that means we can only make the choice in __conn_single if the + * user passes it in via the config string to wiredtiger_open. + */ + WT_ERR(__wt_config_gets(session, cfg, "readonly", &cval)); + if (cval.val) + F_SET(conn, WT_CONN_READONLY); + /* * XXX ideally, we would check "in_memory" here, so we could completely * avoid having a database directory. However, it can be convenient to @@ -1882,6 +1940,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, * 4. the config passed in by the application * 5. user configuration file (optional) * 6. environment variable settings (optional) + * 7. overrides for a read-only connection * * Clear the entries we added to the stack, we're going to build it in * order. @@ -1897,8 +1956,8 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, (int)sizeof(version), ENOMEM); __conn_config_append(cfg, version); - /* Ignore the base_config file if we config_base set to false. */ - if (config_base_set) + /* Ignore the base_config file if config_base_set is false. */ + if (config_base_set || F_ISSET(conn, WT_CONN_READONLY)) WT_ERR( __conn_config_file(session, WT_BASECONFIG, false, cfg, i1)); __conn_config_append(cfg, config); @@ -1908,7 +1967,33 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, /* * Merge the full configuration stack and save it for reconfiguration. */ - WT_ERR(__wt_config_merge(session, cfg, NULL, &conn->cfg)); + WT_ERR(__wt_config_merge(session, cfg, NULL, &merge_cfg)); + /* + * The read-only setting may have been set in a configuration file. + * Get it again so that we can override other configuration settings + * before they are processed by the subsystems. + */ + WT_ERR(__wt_config_gets(session, cfg, "readonly", &cval)); + if (cval.val) + F_SET(conn, WT_CONN_READONLY); + if (F_ISSET(conn, WT_CONN_READONLY)) { + /* + * Create a new stack with the merged configuration as the + * base. The read-only string will use entry 1 and then + * we'll merge it again. + */ + cfg[0] = merge_cfg; + cfg[1] = NULL; + cfg[2] = NULL; + /* + * We override some configuration settings for read-only. + * Other settings that conflict with and are an error with + * read-only are tested in their individual locations later. + */ + __conn_config_readonly(cfg); + WT_ERR(__wt_config_merge(session, cfg, NULL, &conn->cfg)); + } else + conn->cfg = merge_cfg; /* * Configuration ... diff --git a/src/conn/conn_handle.c b/src/conn/conn_handle.c index 12b4e87e921..eade51a48cf 100644 --- a/src/conn/conn_handle.c +++ b/src/conn/conn_handle.c @@ -123,7 +123,8 @@ __wt_connection_destroy(WT_CONNECTION_IMPL *conn) * underlying file-close code uses the mutex to guard lists of * open files. */ - WT_TRET(__wt_close(session, &conn->lock_fh)); + if (conn->lock_fh) + WT_TRET(__wt_close(session, &conn->lock_fh)); /* Remove from the list of connections. */ __wt_spin_lock(session, &__wt_process.spinlock); diff --git a/src/conn/conn_log.c b/src/conn/conn_log.c index 60f46288072..99cad7e8d00 100644 --- a/src/conn/conn_log.c +++ b/src/conn/conn_log.c @@ -133,8 +133,13 @@ __logmgr_config( FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_ERR); WT_RET(__wt_config_gets(session, cfg, "log.zero_fill", &cval)); - if (cval.val != 0) + if (cval.val != 0) { + if (F_ISSET(conn, WT_CONN_READONLY)) + WT_RET_MSG(session, EINVAL, + "Read-only configuration incompatible with " + "zero-filling log files"); FLD_SET(conn->log_flags, WT_CONN_LOG_ZERO_FILL); + } WT_RET(__logmgr_sync_cfg(session, cfg)); return (0); diff --git a/src/conn/conn_stat.c b/src/conn/conn_stat.c index 9edc6091b10..da94ee2da0a 100644 --- a/src/conn/conn_stat.c +++ b/src/conn/conn_stat.c @@ -97,6 +97,14 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp) if (!*runp && !FLD_ISSET(conn->stat_flags, WT_CONN_STAT_ON_CLOSE)) return (0); + /* + * If any statistics logging is done, this must not be a read-only + * connection. + */ + if (F_ISSET(conn, WT_CONN_READONLY)) + WT_RET_MSG(session, EINVAL, + "Read-only configuration incompatible with statistics " + "logging"); WT_RET(__wt_config_gets(session, cfg, "statistics_log.sources", &cval)); WT_RET(__wt_config_subinit(session, &objectconf, &cval)); for (cnt = 0; (ret = __wt_config_next(&objectconf, &k, &v)) == 0; ++cnt) diff --git a/src/cursor/cur_std.c b/src/cursor/cur_std.c index 051f36c8854..899df8a07cc 100644 --- a/src/cursor/cur_std.c +++ b/src/cursor/cur_std.c @@ -628,7 +628,7 @@ __wt_cursor_init(WT_CURSOR *cursor, } else { WT_RET( __wt_config_gets_def(session, cfg, "readonly", 0, &cval)); - if (cval.val != 0) { + if (cval.val != 0 || F_ISSET(S2C(session), WT_CONN_READONLY)) { cursor->insert = __wt_cursor_notsup; cursor->update = __wt_cursor_notsup; cursor->remove = __wt_cursor_notsup; diff --git a/src/include/extern.h b/src/include/extern.h index 1999ff6b732..9f8bd4e5d9a 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -612,6 +612,8 @@ extern WT_DATA_SOURCE *__wt_schema_get_source(WT_SESSION_IMPL *session, const ch extern int __wt_str_name_check(WT_SESSION_IMPL *session, const char *str); extern int __wt_name_check(WT_SESSION_IMPL *session, const char *str, size_t len); extern int __wt_schema_worker(WT_SESSION_IMPL *session, const char *uri, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[], uint32_t open_flags); +extern int __wt_session_notsup_cfg( WT_SESSION *wt_session, const char *config); +extern int __wt_session_notsup_uri( WT_SESSION *wt_session, const char *uri, const char *config); extern int __wt_session_reset_cursors(WT_SESSION_IMPL *session, bool free_buffers); extern int __wt_session_copy_values(WT_SESSION_IMPL *session); extern int __wt_session_release_resources(WT_SESSION_IMPL *session); diff --git a/src/include/flags.h b/src/include/flags.h index 24fae4abccd..2011eaa63a9 100644 --- a/src/include/flags.h +++ b/src/include/flags.h @@ -12,13 +12,14 @@ #define WT_CONN_LOG_SERVER_RUN 0x00000080 #define WT_CONN_LSM_MERGE 0x00000100 #define WT_CONN_PANIC 0x00000200 -#define WT_CONN_SERVER_ASYNC 0x00000400 -#define WT_CONN_SERVER_CHECKPOINT 0x00000800 -#define WT_CONN_SERVER_LSM 0x00001000 -#define WT_CONN_SERVER_RUN 0x00002000 -#define WT_CONN_SERVER_STATISTICS 0x00004000 -#define WT_CONN_SERVER_SWEEP 0x00008000 -#define WT_CONN_WAS_BACKUP 0x00010000 +#define WT_CONN_READONLY 0x00000400 +#define WT_CONN_SERVER_ASYNC 0x00000800 +#define WT_CONN_SERVER_CHECKPOINT 0x00001000 +#define WT_CONN_SERVER_LSM 0x00002000 +#define WT_CONN_SERVER_RUN 0x00004000 +#define WT_CONN_SERVER_STATISTICS 0x00008000 +#define WT_CONN_SERVER_SWEEP 0x00010000 +#define WT_CONN_WAS_BACKUP 0x00020000 #define WT_EVICTING 0x00000001 #define WT_EVICT_IN_MEMORY 0x00000002 #define WT_EVICT_LOOKASIDE 0x00000004 diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index 767c176b53f..7d529e728ac 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -1853,6 +1853,9 @@ struct __wt_connection { * thread uses a session handle from the configured session_max., an * integer between 3 and 20; default \c 4.} * @config{ ),,} + * @config{readonly, open connection in read-only mode. The database + * must exist. All methods that may modify a database are disabled., a + * boolean flag; default \c false.} * @config{shared_cache = (, shared cache configuration options. A * database should configure either a cache_size or a shared_cache not * both. Enabling a shared cache uses a session from the configured @@ -2326,6 +2329,9 @@ struct __wt_connection { * start an RPC server for primary processes and use RPC for secondary * processes). Not yet supported in WiredTiger., a boolean flag; default * \c false.} + * @config{readonly, open connection in read-only mode. The database must + * exist. All methods that may modify a database are disabled., a boolean flag; + * default \c false.} * @config{session_max, maximum expected number of sessions (including server * threads)., an integer greater than or equal to 1; default \c 100.} * @config{shared_cache = (, shared cache configuration options. A database diff --git a/src/lsm/lsm_manager.c b/src/lsm/lsm_manager.c index dac8d987328..b3f60b25c21 100644 --- a/src/lsm/lsm_manager.c +++ b/src/lsm/lsm_manager.c @@ -212,6 +212,8 @@ __wt_lsm_manager_start(WT_SESSION_IMPL *session) conn = S2C(session); manager = &conn->lsm_manager; + if (F_ISSET(conn, WT_CONN_READONLY)) + return (0); /* * We need at least a manager, a switch thread and a generic * worker. diff --git a/src/os_posix/os_fallocate.c b/src/os_posix/os_fallocate.c index 9d160afd179..bf20a99bdef 100644 --- a/src/os_posix/os_fallocate.c +++ b/src/os_posix/os_fallocate.c @@ -115,6 +115,7 @@ __wt_fallocate( { WT_DECL_RET; + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); switch (fh->fallocate_available) { /* * Check for already configured handles and make the configured call. diff --git a/src/os_posix/os_fsync.c b/src/os_posix/os_fsync.c index f5afddc557b..fa133d5f16d 100644 --- a/src/os_posix/os_fsync.c +++ b/src/os_posix/os_fsync.c @@ -60,6 +60,7 @@ __wt_directory_sync_fh(WT_SESSION_IMPL *session, WT_FH *fh) #ifdef __linux__ WT_DECL_RET; + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); if ((ret = __wt_handle_sync(fh->fd)) == 0) return (0); WT_RET_MSG(session, ret, "%s: fsync", fh->name); @@ -108,6 +109,7 @@ __wt_directory_sync(WT_SESSION_IMPL *session, const char *path) if (ret != 0) WT_RET_MSG(session, ret, "%s: open", path); + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); if ((ret = __wt_handle_sync(fd)) != 0) WT_ERR_MSG(session, ret, "%s: fsync", path); @@ -134,6 +136,7 @@ __wt_fsync(WT_SESSION_IMPL *session, WT_FH *fh) WT_RET(__wt_verbose(session, WT_VERB_FILEOPS, "%s: fsync", fh->name)); + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); if ((ret = __wt_handle_sync(fh->fd)) == 0) return (0); WT_RET_MSG(session, ret, "%s fsync error", fh->name); @@ -146,6 +149,7 @@ __wt_fsync(WT_SESSION_IMPL *session, WT_FH *fh) int __wt_fsync_async(WT_SESSION_IMPL *session, WT_FH *fh) { + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); #ifdef HAVE_SYNC_FILE_RANGE WT_DECL_RET; diff --git a/src/os_posix/os_ftruncate.c b/src/os_posix/os_ftruncate.c index 2af90512f26..94d6cba3bf5 100644 --- a/src/os_posix/os_ftruncate.c +++ b/src/os_posix/os_ftruncate.c @@ -17,6 +17,7 @@ __wt_ftruncate(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t len) { WT_DECL_RET; + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); WT_SYSCALL_RETRY(ftruncate(fh->fd, len), ret); if (ret == 0) return (0); diff --git a/src/os_posix/os_open.c b/src/os_posix/os_open.c index b085676c53b..130c6f16f42 100644 --- a/src/os_posix/os_open.c +++ b/src/os_posix/os_open.c @@ -94,6 +94,9 @@ __wt_open(WT_SESSION_IMPL *session, #endif if (ok_create) { + WT_ASSERT(session, !F_ISSET(conn, WT_CONN_READONLY) || + WT_STRING_MATCH(name, WT_SINGLETHREAD, + strlen(WT_SINGLETHREAD))); f |= O_CREAT; if (exclusive) f |= O_EXCL; diff --git a/src/os_posix/os_remove.c b/src/os_posix/os_remove.c index bc244c12e46..eb2e37fdc38 100644 --- a/src/os_posix/os_remove.c +++ b/src/os_posix/os_remove.c @@ -21,6 +21,7 @@ __remove_file_check(WT_SESSION_IMPL *session, const char *name) uint64_t bucket; conn = S2C(session); + WT_ASSERT(session, !F_ISSET(conn, WT_CONN_READONLY)); fh = NULL; bucket = __wt_hash_city64(name, strlen(name)) % WT_HASH_ARRAY_SIZE; diff --git a/src/os_posix/os_rename.c b/src/os_posix/os_rename.c index 301190305c4..8ec4ee3aa23 100644 --- a/src/os_posix/os_rename.c +++ b/src/os_posix/os_rename.c @@ -21,6 +21,7 @@ __wt_rename(WT_SESSION_IMPL *session, const char *from, const char *to) WT_RET(__wt_verbose( session, WT_VERB_FILEOPS, "rename %s to %s", from, to)); + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); from_path = to_path = NULL; WT_RET(__wt_filename(session, from, &from_path)); diff --git a/src/os_posix/os_rw.c b/src/os_posix/os_rw.c index 8733bfe0f53..f9303fc75be 100644 --- a/src/os_posix/os_rw.c +++ b/src/os_posix/os_rw.c @@ -65,6 +65,7 @@ __wt_write(WT_SESSION_IMPL *session, "%s: write %" WT_SIZET_FMT " bytes at offset %" PRIuMAX, fh->name, len, (uintmax_t)offset)); + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); /* Assert direct I/O is aligned and a multiple of the alignment. */ WT_ASSERT(session, !fh->direct_io || diff --git a/src/os_win/os_fallocate.c b/src/os_win/os_fallocate.c index cdc7a1c46ee..a324687ca73 100644 --- a/src/os_win/os_fallocate.c +++ b/src/os_win/os_fallocate.c @@ -35,6 +35,7 @@ int __wt_fallocate( WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t offset, wt_off_t len) { + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); WT_UNUSED(session); WT_UNUSED(fh); WT_UNUSED(offset); diff --git a/src/os_win/os_fsync.c b/src/os_win/os_fsync.c index 913b7ca5a4e..1671ade5d94 100644 --- a/src/os_win/os_fsync.c +++ b/src/os_win/os_fsync.c @@ -15,6 +15,7 @@ int __wt_directory_sync_fh(WT_SESSION_IMPL *session, WT_FH *fh) { + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); WT_UNUSED(session); WT_UNUSED(fh); return (0); @@ -27,6 +28,7 @@ __wt_directory_sync_fh(WT_SESSION_IMPL *session, WT_FH *fh) int __wt_directory_sync(WT_SESSION_IMPL *session, const char *path) { + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); WT_UNUSED(session); WT_UNUSED(path); return (0); @@ -44,6 +46,7 @@ __wt_fsync(WT_SESSION_IMPL *session, WT_FH *fh) WT_RET(__wt_verbose(session, WT_VERB_FILEOPS, "%s: FlushFileBuffers", fh->name)); + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); if ((ret = FlushFileBuffers(fh->filehandle)) == FALSE) WT_RET_MSG(session, __wt_errno(), "%s FlushFileBuffers error", fh->name); @@ -58,6 +61,7 @@ __wt_fsync(WT_SESSION_IMPL *session, WT_FH *fh) int __wt_fsync_async(WT_SESSION_IMPL *session, WT_FH *fh) { + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); WT_UNUSED(session); WT_UNUSED(fh); diff --git a/src/os_win/os_ftruncate.c b/src/os_win/os_ftruncate.c index 0c11b5509b7..88fcf9542c1 100644 --- a/src/os_win/os_ftruncate.c +++ b/src/os_win/os_ftruncate.c @@ -18,6 +18,7 @@ __wt_ftruncate(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t len) WT_DECL_RET; LARGE_INTEGER largeint; + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); largeint.QuadPart = len; if ((ret = SetFilePointerEx( diff --git a/src/os_win/os_open.c b/src/os_win/os_open.c index 3ec53daf001..10db95be6ce 100644 --- a/src/os_win/os_open.c +++ b/src/os_win/os_open.c @@ -72,6 +72,9 @@ __wt_open(WT_SESSION_IMPL *session, dwCreationDisposition = 0; if (ok_create) { + WT_ASSERT(session, !F_ISSET(conn, WT_CONN_READONLY) || + WT_STRING_MATCH(name, WT_SINGLETHREAD, + strlen(WT_SINGLETHREAD))); dwCreationDisposition = CREATE_NEW; if (exclusive) dwCreationDisposition = CREATE_ALWAYS; diff --git a/src/os_win/os_remove.c b/src/os_win/os_remove.c index 5682a25d7f2..84f1dd86674 100644 --- a/src/os_win/os_remove.c +++ b/src/os_win/os_remove.c @@ -21,6 +21,7 @@ __remove_file_check(WT_SESSION_IMPL *session, const char *name) uint64_t bucket; conn = S2C(session); + WT_ASSERT(session, !F_ISSET(conn, WT_CONN_READONLY)); fh = NULL; bucket = __wt_hash_city64(name, strlen(name)) % WT_HASH_ARRAY_SIZE; diff --git a/src/os_win/os_rename.c b/src/os_win/os_rename.c index 829ab1d16e9..b4be2dba24c 100644 --- a/src/os_win/os_rename.c +++ b/src/os_win/os_rename.c @@ -22,6 +22,7 @@ __wt_rename(WT_SESSION_IMPL *session, const char *from, const char *to) WT_RET(__wt_verbose( session, WT_VERB_FILEOPS, "rename %s to %s", from, to)); + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); from_path = to_path = NULL; WT_RET(__wt_filename(session, from, &from_path)); diff --git a/src/os_win/os_rw.c b/src/os_win/os_rw.c index 49f011001a4..222d98ad7cf 100644 --- a/src/os_win/os_rw.c +++ b/src/os_win/os_rw.c @@ -74,6 +74,7 @@ __wt_write(WT_SESSION_IMPL *session, "%s: write %" WT_SIZET_FMT " bytes at offset %" PRIuMAX, fh->name, len, (uintmax_t)offset)); + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); /* Assert direct I/O is aligned and a multiple of the alignment. */ WT_ASSERT(session, !fh->direct_io || diff --git a/src/session/session_api.c b/src/session/session_api.c index c03b5fdc044..ea524b12ada 100644 --- a/src/session/session_api.c +++ b/src/session/session_api.c @@ -12,6 +12,36 @@ static int __session_checkpoint(WT_SESSION *, const char *); static int __session_snapshot(WT_SESSION *, const char *); static int __session_rollback_transaction(WT_SESSION *, const char *); +/* + * __wt_session_notsup_cfg -- + * Unsupported session actions that have a signature of a config string. + */ +int +__wt_session_notsup_cfg( + WT_SESSION *wt_session, const char *config) +{ + WT_UNUSED(wt_session); + WT_UNUSED(config); + + return (ENOTSUP); +} + +/* + * __wt_session_notsup_uri -- + * Unsupported session actions that have a signature of a URI and + * a config string. + */ +int +__wt_session_notsup_uri( + WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_UNUSED(wt_session); + WT_UNUSED(uri); + WT_UNUSED(config); + + return (ENOTSUP); +} + /* * __wt_session_reset_cursors -- * Reset all open cursors. @@ -546,6 +576,9 @@ __session_log_printf(WT_SESSION *wt_session, const char *fmt, ...) session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL_NOCONF(session, log_printf); + if (F_ISSET(S2C(session), WT_CONN_READONLY)) + WT_ERR(ENOTSUP); + va_start(ap, fmt); ret = __wt_log_vprintf(session, fmt, ap); va_end(ap); @@ -567,7 +600,7 @@ __session_rebalance(WT_SESSION *wt_session, const char *uri, const char *config) SESSION_API_CALL(session, rebalance, config, cfg); - if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) + if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY | WT_CONN_READONLY)) WT_ERR(ENOTSUP); /* Block out checkpoints to avoid spurious EBUSY errors. */ @@ -593,6 +626,9 @@ __session_rename(WT_SESSION *wt_session, session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, rename, config, cfg); + if (F_ISSET(S2C(session), WT_CONN_READONLY)) + WT_ERR(ENOTSUP); + /* Disallow objects in the WiredTiger name space. */ WT_ERR(__wt_str_name_check(session, uri)); WT_ERR(__wt_str_name_check(session, newuri)); @@ -953,6 +989,9 @@ __session_truncate(WT_SESSION *wt_session, SESSION_TXN_API_CALL(session, truncate, config, cfg); WT_STAT_FAST_CONN_INCR(session, cursor_truncate); + if (F_ISSET(S2C(session), WT_CONN_READONLY)) + WT_ERR(ENOTSUP); + /* * If the URI is specified, we don't need a start/stop, if start/stop * is specified, we don't need a URI. One exception is the log URI @@ -1382,6 +1421,7 @@ __open_session(WT_CONNECTION_IMPL *conn, __session_transaction_sync }; WT_DECL_RET; + WT_SESSION *wt_session; WT_SESSION_IMPL *session, *session_ret; uint32_t i; @@ -1421,6 +1461,28 @@ __open_session(WT_CONNECTION_IMPL *conn, session_ret->id = i; session_ret->iface = stds; session_ret->iface.connection = &conn->iface; + /* + * Disable some methods if this is a read-only connection. Group + * them by call signature. + */ + if (F_ISSET(conn, WT_CONN_READONLY)) { + wt_session = &session_ret->iface; + wt_session->checkpoint = __wt_session_notsup_cfg; + wt_session->log_flush = __wt_session_notsup_cfg; + wt_session->transaction_sync = __wt_session_notsup_cfg; + + wt_session->compact = __wt_session_notsup_uri; + wt_session->create = __wt_session_notsup_uri; + wt_session->drop = __wt_session_notsup_uri; + wt_session->salvage = __wt_session_notsup_uri; + wt_session->upgrade = __wt_session_notsup_uri; + + /* + * The other methods that are not supported but are + * checked individually in the function are: + * log_printf, rebalance, rename, truncate + */ + } WT_ERR(__wt_cond_alloc(session, "session", false, &session_ret->cond)); diff --git a/src/txn/txn_recover.c b/src/txn/txn_recover.c index e6bd8a8d755..94df0586ced 100644 --- a/src/txn/txn_recover.c +++ b/src/txn/txn_recover.c @@ -492,8 +492,14 @@ __wt_txn_recover(WT_SESSION_IMPL *session) * return an error if the user does not want automatic * recovery. */ - if (needs_rec && FLD_ISSET(conn->log_flags, WT_CONN_LOG_RECOVER_ERR)) + if (needs_rec && + (FLD_ISSET(conn->log_flags, WT_CONN_LOG_RECOVER_ERR) || + F_ISSET(conn, WT_CONN_READONLY))) { + if (F_ISSET(conn, WT_CONN_READONLY)) + WT_ERR_MSG(session, WT_RUN_RECOVERY, + "Read-only database needs recovery"); WT_ERR(WT_RUN_RECOVERY); + } /* * Recovery can touch more data than fits in cache, so it relies on diff --git a/test/suite/test_txn02.py b/test/suite/test_txn02.py index fccf123d3bc..2ccd48109e6 100644 --- a/test/suite/test_txn02.py +++ b/test/suite/test_txn02.py @@ -152,6 +152,14 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): # Opening a clone of the database home directory should run # recovery and see the committed results. self.backup(self.backup_dir) + backup_conn_params = 'read_only=true' + backup_conn = self.wiredtiger_open(self.backup_dir, backup_conn_params) + try: + self.check(backup_conn.open_session(), None, committed) + finally: + backup_conn.close() + + # Now open the clone of the database home directory with backup_conn_params = 'log=(enabled,file_max=%s)' % self.logmax backup_conn = self.wiredtiger_open(self.backup_dir, backup_conn_params) try: -- cgit v1.2.1 From f9e6eda6cf97f46ff0dffbb9163e95406e48e6d7 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 4 Feb 2016 14:24:40 -0500 Subject: WT-2349 Whitespace --- src/conn/conn_api.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 8e4b2ad3679..0d6269daa54 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -1108,7 +1108,6 @@ __conn_config_append(const char *cfg[], const char *config) *cfg = config; } - /* * __conn_config_readonly -- * Append an entry to a config stack that overrides some settings -- cgit v1.2.1 From 4cdb0023a1c50eb6e87fef14449007a13d392a39 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 4 Feb 2016 14:30:10 -0500 Subject: WT-2349 Move assert. --- src/os_posix/os_fsync.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os_posix/os_fsync.c b/src/os_posix/os_fsync.c index fa133d5f16d..da36d45c0fe 100644 --- a/src/os_posix/os_fsync.c +++ b/src/os_posix/os_fsync.c @@ -149,10 +149,10 @@ __wt_fsync(WT_SESSION_IMPL *session, WT_FH *fh) int __wt_fsync_async(WT_SESSION_IMPL *session, WT_FH *fh) { - WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); #ifdef HAVE_SYNC_FILE_RANGE WT_DECL_RET; + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); WT_RET(__wt_verbose( session, WT_VERB_FILEOPS, "%s: sync_file_range", fh->name)); -- cgit v1.2.1 From 1ffdd724b180bc6eae550723521f46946a514227 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 4 Feb 2016 15:34:41 -0500 Subject: WT-2349 Fix assertion in write. Remove not-yet-ready change to test. --- src/os_posix/os_rw.c | 4 +++- test/suite/test_txn02.py | 8 -------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/os_posix/os_rw.c b/src/os_posix/os_rw.c index f9303fc75be..e9f3aa52453 100644 --- a/src/os_posix/os_rw.c +++ b/src/os_posix/os_rw.c @@ -65,7 +65,9 @@ __wt_write(WT_SESSION_IMPL *session, "%s: write %" WT_SIZET_FMT " bytes at offset %" PRIuMAX, fh->name, len, (uintmax_t)offset)); - WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY || + WT_STRING_MATCH(fh->name, WT_SINGLETHREAD, + strlen(WT_SINGLETHREAD)))); /* Assert direct I/O is aligned and a multiple of the alignment. */ WT_ASSERT(session, !fh->direct_io || diff --git a/test/suite/test_txn02.py b/test/suite/test_txn02.py index 2ccd48109e6..fccf123d3bc 100644 --- a/test/suite/test_txn02.py +++ b/test/suite/test_txn02.py @@ -152,14 +152,6 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): # Opening a clone of the database home directory should run # recovery and see the committed results. self.backup(self.backup_dir) - backup_conn_params = 'read_only=true' - backup_conn = self.wiredtiger_open(self.backup_dir, backup_conn_params) - try: - self.check(backup_conn.open_session(), None, committed) - finally: - backup_conn.close() - - # Now open the clone of the database home directory with backup_conn_params = 'log=(enabled,file_max=%s)' % self.logmax backup_conn = self.wiredtiger_open(self.backup_dir, backup_conn_params) try: -- cgit v1.2.1 From 34e4f89a8ba0f3b98ce19363738a2cf6faa730f6 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Fri, 5 Feb 2016 17:03:52 +1100 Subject: WT-2374 Enable message when overriding metadata from a backup. (And update the test to expect it). --- src/meta/meta_turtle.c | 9 +++------ test/suite/test_txn04.py | 11 +++++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/meta/meta_turtle.c b/src/meta/meta_turtle.c index 0d8f61aa7cf..957a67d250f 100644 --- a/src/meta/meta_turtle.c +++ b/src/meta/meta_turtle.c @@ -192,17 +192,14 @@ __wt_turtle_init(WT_SESSION_IMPL *session) * we want to recreate the metadata from the backup. */ if (exist_backup) { + __wt_msg(session, "Both %s and %s exist. " + "Recreating metadata from backup.", + WT_METADATA_TURTLE, WT_METADATA_BACKUP); WT_RET(__wt_remove_if_exists( session, WT_METAFILE)); WT_RET(__wt_remove_if_exists( session, WT_METADATA_TURTLE)); load = true; -#if 0 - __wt_msg(session, "Both %s and %s exist. " - "Remove %s and recreate metadata from %s.", - WT_METADATA_TURTLE, WT_METADATA_BACKUP, - WT_METADATA_TURTLE, WT_METADATA_BACKUP); -#endif } } else load = true; diff --git a/test/suite/test_txn04.py b/test/suite/test_txn04.py index d05204e0616..bbd6ce8c4e2 100644 --- a/test/suite/test_txn04.py +++ b/test/suite/test_txn04.py @@ -121,17 +121,14 @@ class test_txn04(wttest.WiredTigerTestCase, suite_subprocess): cmd += self.backup_dir self.runWt(cmd.split()) - self.exception='false' backup_conn_params = 'log=(enabled,file_max=%s)' % self.logmax backup_conn = self.wiredtiger_open(self.backup_dir, backup_conn_params) try: self.check(backup_conn.open_session(), None, committed) - except: - self.exception='true' finally: backup_conn.close() - def test_ops(self): + def ops(self): self.session.create(self.uri, self.create_params) c = self.session.open_cursor(self.uri, None, 'overwrite') # Set up the table with entries for 1-5. @@ -149,7 +146,6 @@ class test_txn04(wttest.WiredTigerTestCase, suite_subprocess): # The runWt command closes our connection and sessions so # we need to reopen them here. self.hot_backup(None, committed) - self.assertEqual(True, self.exception == 'false') c = self.session.open_cursor(self.uri, None, 'overwrite') c.set_value(1) # Then do the given modification. @@ -195,7 +191,10 @@ class test_txn04(wttest.WiredTigerTestCase, suite_subprocess): # Backup the target we modified and verify the data. # print 'Call hot_backup with ' + self.uri self.hot_backup(self.uri, committed) - self.assertEqual(True, self.exception == 'false') + + def test_ops(self): + with self.expectedStdoutPattern('Recreating metadata'): + self.ops() if __name__ == '__main__': wttest.run() -- cgit v1.2.1 From 5e86baa19ad1c67b05ccb05b04b14efeb3088753 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Fri, 5 Feb 2016 21:45:41 +1100 Subject: WT-2374 Fix another test that triggers the new warning. --- test/suite/test_backup05.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/suite/test_backup05.py b/test/suite/test_backup05.py index 8b176d0f7d7..8ffeb6752df 100644 --- a/test/suite/test_backup05.py +++ b/test/suite/test_backup05.py @@ -71,7 +71,7 @@ class test_backup05(wttest.WiredTigerTestCase, suite_subprocess): session.verify(self.uri) conn.close() - def test_backup(self): + def backup(self): '''Check manual fsyncLock backup strategy''' # Here's the strategy: @@ -95,5 +95,9 @@ class test_backup05(wttest.WiredTigerTestCase, suite_subprocess): else: self.session.verify(self.uri) + def test_backup(self): + with self.expectedStdoutPattern('Recreating metadata'): + self.backup() + if __name__ == '__main__': wttest.run() -- cgit v1.2.1 From 92d903909ae1a811a9175c24b21ef68f8acafcbc Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 5 Feb 2016 12:36:17 -0500 Subject: WT-2361: column-store starting record number error Diagnostic check for insert split and checkpoint races. The test/format program stresses insert splits vs checkpoints, and we can spot a race when format is appending a new record, the library is allocating the record number, and we're on the wrong page, which we know because the page has already done an insert-split. --- src/btree/col_modify.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/btree/col_modify.c b/src/btree/col_modify.c index 645d98d9c9b..fd60b12538a 100644 --- a/src/btree/col_modify.c +++ b/src/btree/col_modify.c @@ -25,6 +25,7 @@ __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_INSERT_HEAD *ins_head, **ins_headp; WT_ITEM _value; WT_PAGE *page; + WT_PAGE_MODIFY *mod; WT_UPDATE *old_upd, *upd; size_t ins_size, upd_size; u_int i, skipdepth; @@ -60,6 +61,7 @@ __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, /* If we don't yet have a modify structure, we'll need one. */ WT_RET(__wt_page_modify_init(session, page)); + mod = page->modify; /* * Delete, insert or update a column-store entry. @@ -105,17 +107,17 @@ __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, /* Allocate the append/update list reference as necessary. */ if (append) { WT_PAGE_ALLOC_AND_SWAP(session, - page, page->modify->mod_append, ins_headp, 1); - ins_headp = &page->modify->mod_append[0]; + page, mod->mod_append, ins_headp, 1); + ins_headp = &mod->mod_append[0]; } else if (page->type == WT_PAGE_COL_FIX) { WT_PAGE_ALLOC_AND_SWAP(session, - page, page->modify->mod_update, ins_headp, 1); - ins_headp = &page->modify->mod_update[0]; + page, mod->mod_update, ins_headp, 1); + ins_headp = &mod->mod_update[0]; } else { WT_PAGE_ALLOC_AND_SWAP(session, - page, page->modify->mod_update, ins_headp, + page, mod->mod_update, ins_headp, page->pg_var_entries); - ins_headp = &page->modify->mod_update[cbt->slot]; + ins_headp = &mod->mod_update[cbt->slot]; } /* Allocate the WT_INSERT_HEAD structure as necessary. */ @@ -135,6 +137,14 @@ __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, cbt->ins_head = ins_head; cbt->ins = ins; + /* + * Check for insert split and checkpoint races in column-store: + * it's easy (as opposed to in row-store) and a difficult bug to + * otherwise diagnose. + */ + WT_ASSERT(session, mod->mod_split_recno == WT_RECNO_OOB || + (recno != WT_RECNO_OOB && mod->mod_split_recno > recno)); + if (upd_arg == NULL) { WT_ERR( __wt_update_alloc(session, value, &upd, &upd_size)); -- cgit v1.2.1 From d7c0eb2a2cae47b7dc10a4c6da58a1b951d21fc9 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 5 Feb 2016 12:38:56 -0500 Subject: WT-2361: column-store starting record number error btree->checkpointing is an enum, use an explicit test instead of checking for a non-zero value. --- src/include/btree.i | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/btree.i b/src/include/btree.i index 94111397abd..16906aa7cfa 100644 --- a/src/include/btree.i +++ b/src/include/btree.i @@ -1149,7 +1149,7 @@ __wt_page_can_evict(WT_SESSION_IMPL *session, WT_REF *ref, bool *inmem_splitp) * parent frees the backing blocks for any no-longer-used overflow keys, * which will corrupt the checkpoint's block management. */ - if (btree->checkpointing && + if (btree->checkpointing != WT_CKPT_OFF && F_ISSET_ATOMIC(ref->home, WT_PAGE_OVERFLOW_KEYS)) return (false); -- cgit v1.2.1 From fc1c98312f83c963cc9144b8187c1afb056e7f83 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 5 Feb 2016 12:42:44 -0500 Subject: WT-2361: column-store starting record number error Clean up comments around WT_CHILD_IGNORE: we now use it for insert-split pages being ignored within a checkpoint, as well as deleted child pages. Changes to __rec_child_modify(): I don't think eviction should ever see a WT_REF in a WT_REF_LOCKED or WT_REF_MEM state, since we no longer pass subtrees to reconciliation for eviction. Additionally, when evicting in non-diagnostic mode, return EBUSY if we encounter a WT_REF in an illegal state. That will cause eviction to gracefully fail, and the previous code isn't doing anybody any favors by just continuing with the eviction as if nothing is wrong. --- src/reconcile/rec_write.c | 106 ++++++++++++++++++++++++++-------------------- 1 file changed, 59 insertions(+), 47 deletions(-) diff --git a/src/reconcile/rec_write.c b/src/reconcile/rec_write.c index 24999ec82cc..60482cb25b4 100644 --- a/src/reconcile/rec_write.c +++ b/src/reconcile/rec_write.c @@ -1313,7 +1313,7 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, } while (0) typedef enum { - WT_CHILD_IGNORE, /* Deleted child: ignore */ + WT_CHILD_IGNORE, /* Ignored child */ WT_CHILD_MODIFIED, /* Modified child */ WT_CHILD_ORIGINAL, /* Original child */ WT_CHILD_PROXY /* Deleted child: proxy */ @@ -1450,16 +1450,15 @@ __rec_child_modify(WT_SESSION_IMPL *session, /* * This function is called when walking an internal page to decide how - * to handle child pages referenced by the internal page, specifically - * if the child page is to be merged into its parent. + * to handle child pages referenced by the internal page. * * Internal pages are reconciled for two reasons: first, when evicting * an internal page, second by the checkpoint code when writing internal - * pages. During eviction, the subtree is locked down so all pages - * should be in the WT_REF_DISK or WT_REF_LOCKED state. During - * checkpoint, any eviction that might affect our review of an internal - * page is prohibited, however, as the subtree is not reserved for our - * exclusive use, there are other page states that must be considered. + * pages. During eviction, all pages should be in the WT_REF_DISK or + * WT_REF_DELETED state. During checkpoint, eviction that might affect + * review of an internal page is prohibited, however, as the subtree is + * not reserved for our exclusive use, there are other page states that + * must be considered. */ for (;; __wt_yield()) switch (r->tested_ref_state = ref->state) { @@ -1488,15 +1487,14 @@ __rec_child_modify(WT_SESSION_IMPL *session, /* * Locked. * - * If evicting, the evicted page's subtree, including - * this child, was selected for eviction by us and the - * state is stable until we reset it, it's an in-memory - * state. This is the expected state for a child being - * merged into a page (where the page was selected by - * the eviction server for eviction). + * We should never be here during eviction, active child + * pages in an evicted page's subtree fails the eviction + * attempt. */ - if (F_ISSET(r, WT_EVICTING)) - goto in_memory; + if (F_ISSET(r, WT_EVICTING)) { + WT_ASSERT(session, !F_ISSET(r, WT_EVICTING)); + return (EBUSY); + } /* * If called during checkpoint, the child is being @@ -1514,24 +1512,21 @@ __rec_child_modify(WT_SESSION_IMPL *session, /* * In memory. * - * If evicting, the evicted page's subtree, including - * this child, was selected for eviction by us and the - * state is stable until we reset it, it's an in-memory - * state. This is the expected state for a child being - * merged into a page (where the page belongs to a file - * being discarded from the cache during close). + * We should never be here during eviction, active child + * pages in an evicted page's subtree fails the eviction + * attempt. */ - if (F_ISSET(r, WT_EVICTING)) - goto in_memory; + if (F_ISSET(r, WT_EVICTING)) { + WT_ASSERT(session, !F_ISSET(r, WT_EVICTING)); + return (EBUSY); + } /* * If called during checkpoint, acquire a hazard pointer * so the child isn't evicted, it's an in-memory case. * - * This call cannot return split/restart, eviction of - * pages that split into their parent is shutout during - * checkpoint, all splits in process will have completed - * before we walk any pages for checkpoint. + * This call cannot return split/restart, we have a lock + * on the parent which prevents a child page split. */ ret = __wt_page_in(session, ref, WT_READ_CACHE | WT_READ_NO_EVICT | @@ -1548,29 +1543,31 @@ __rec_child_modify(WT_SESSION_IMPL *session, /* * Being read, not modified by definition. * - * We should never be here during eviction, a child page - * in this state within an evicted page's subtree would - * have caused normally eviction to fail, and exclusive - * eviction shouldn't ever see pages being read. + * We should never be here during eviction, active child + * pages in an evicted page's subtree fails the eviction + * attempt. */ - WT_ASSERT(session, !F_ISSET(r, WT_EVICTING)); + if (F_ISSET(r, WT_EVICTING)) { + WT_ASSERT(session, !F_ISSET(r, WT_EVICTING)); + return (EBUSY); + } goto done; case WT_REF_SPLIT: /* * The page was split out from under us. * - * We should never be here during eviction, a child page - * in this state within an evicted page's subtree would - * have caused eviction to fail. + * We should never be here during eviction, active child + * pages in an evicted page's subtree fails the eviction + * attempt. * * We should never be here during checkpoint, dirty page * eviction is shutout during checkpoint, all splits in * process will have completed before we walk any pages * for checkpoint. */ - WT_ASSERT(session, ref->state != WT_REF_SPLIT); - /* FALLTHROUGH */ + WT_ASSERT(session, WT_REF_SPLIT != WT_REF_SPLIT); + return (EBUSY); WT_ILLEGAL_VALUE(session); } @@ -1581,11 +1578,21 @@ in_memory: * modify structure has been instantiated. If the modify structure * exists and the page has actually been modified, set that state. * If that's not the case, we would normally use the original cell's - * disk address as our reference, but, if we're forced to instantiate - * a deleted child page and it's never modified, we end up here with - * a page that has a modify structure, no modifications, and no disk - * address. Ignore those pages, they're not modified and there is no - * reason to write the cell. + * disk address as our reference, however there are two special cases, + * both flagged by a missing block address. + * + * First, if forced to instantiate a deleted child page and it's never + * modified, we end up here with a page that has a modify structure, no + * modifications, and no disk address. Ignore those pages, they're not + * modified and there is no reason to write the cell. + * + * Second, insert splits are permitted during checkpoint. When doing the + * final checkpoint pass, we first walk the internal page's page-index + * and write out any dirty pages we find, then we write out the internal + * page in post-order traversal. If we found the split page in the first + * step, it will have an address; if we didn't find the split page in + * the first step, it won't have an address and we ignore it, it's not + * part of the checkpoint. */ mod = ref->page->modify; if (mod != NULL && mod->rec_result != 0) @@ -3794,7 +3801,7 @@ __rec_col_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) switch (state) { case WT_CHILD_IGNORE: - /* Deleted child we don't have to write. */ + /* Ignored child. */ WT_CHILD_RELEASE_ERR(session, hazard, ref); continue; @@ -4525,13 +4532,18 @@ compare: /* * record 100, we don't know to write a deleted record * 99 on the page.) * + * Assert the recorded record number is past the end of + * the page. + * * The record number recorded during the split is the * first key on the split page, that is, one larger than * the last key on this page, we have to decrement it. */ if ((n = page->modify->mod_split_recno) == WT_RECNO_OOB) break; + WT_ASSERT(session, n >= src_recno); n -= 1; + upd = NULL; } else { WT_ERR( @@ -4720,10 +4732,10 @@ __rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) switch (state) { case WT_CHILD_IGNORE: /* - * Deleted child we don't have to write. + * Ignored child. * - * Overflow keys referencing discarded pages are no - * longer useful, schedule them for discard. Don't + * Overflow keys referencing pages we're not writing are + * no longer useful, schedule them for discard. Don't * worry about instantiation, internal page keys are * always instantiated. Don't worry about reuse, * reusing this key in this reconciliation is unlikely. -- cgit v1.2.1 From 4f54a7330e2cfeafd884a0a2f46e1b40568446b0 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 5 Feb 2016 12:49:37 -0500 Subject: WT-2361: column-store starting record number error Don't push splits up the tree when checkpoint is doing its final pass. Splits up the tree move the tree's namespace around, which can corrupt the checkpoint. --- src/btree/bt_split.c | 44 ++++++++++++++++++++++++++++++++++++++++++-- src/btree/bt_sync.c | 16 +++++++++------- src/include/btree.h | 10 ++++++---- src/include/extern.h | 1 + 4 files changed, 58 insertions(+), 13 deletions(-) diff --git a/src/btree/bt_split.c b/src/btree/bt_split.c index 58b3ac024f5..b8d6ffae252 100644 --- a/src/btree/bt_split.c +++ b/src/btree/bt_split.c @@ -1331,6 +1331,21 @@ __split_internal_unlock(WT_SESSION_IMPL *session, WT_PAGE *parent, bool hazard) return (ret); } +/* + * __wt_split_drain -- + * Wait for ongoing splits to drain. + */ +void +__wt_split_drain(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + + btree = S2BT(session); + + while (btree->split_busy > 0) + __wt_yield(); +} + /* * __split_internal_should_split -- * Return if we should split an internal page. @@ -1383,11 +1398,36 @@ __split_internal_should_split(WT_SESSION_IMPL *session, WT_REF *ref) static int __split_parent_climb(WT_SESSION_IMPL *session, WT_PAGE *page, bool page_hazard) { + WT_BTREE *btree; WT_DECL_RET; WT_PAGE *parent; WT_REF *ref; bool parent_hazard; + btree = S2BT(session); + parent = NULL; + parent_hazard = false; + + /* + * Disallow internal splits during the final pass of a checkpoint. Most + * splits are already disallowed during checkpoints, but an important + * exception is insert splits. The danger is an insert split creates a + * new chunk of the namespace, and then the internal split will move it + * to a different part of the tree where it will be written; in other + * words, in one part of the tree we'll skip the newly created insert + * split chunk, but we'll write it upon finding it in a different part + * of the tree. + * + * Checkpoints wait until the count of internal splits goes to zero; + * check before doing the increment because it's easy, and afterward + * to avoid any races. + */ + if (btree->checkpointing != WT_CKPT_OFF) + return (__split_internal_unlock(session, page, page_hazard)); + (void)__wt_atomic_addv32(&btree->split_busy, 1); + if (btree->checkpointing != WT_CKPT_OFF) + WT_ERR(EBUSY); + /* * Page splits trickle up the tree, that is, as leaf pages grow large * enough and are evicted, they'll split into their parent. And, as @@ -1407,8 +1447,6 @@ __split_parent_climb(WT_SESSION_IMPL *session, WT_PAGE *page, bool page_hazard) * Split up the tree. */ for (;;) { - parent = NULL; - parent_hazard = false; ref = page->pg_intl_parent_ref; /* If we don't need to split the page, we're done. */ @@ -1445,6 +1483,8 @@ err: if (parent != NULL) __split_internal_unlock(session, parent, parent_hazard)); WT_TRET(__split_internal_unlock(session, page, page_hazard)); + (void)__wt_atomic_subv32(&btree->split_busy, 1); + /* A page may have been busy, in which case return without error. */ WT_RET_BUSY_OK(ret); return (0); diff --git a/src/btree/bt_sync.c b/src/btree/bt_sync.c index 5cbd8d1e996..1b0a1056d9a 100644 --- a/src/btree/bt_sync.c +++ b/src/btree/bt_sync.c @@ -105,13 +105,13 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) __wt_spin_lock(session, &btree->flush_lock); /* - * When internal pages are being reconciled by checkpoint their - * child pages cannot disappear from underneath them or be split - * into them, nor can underlying blocks be freed until the block - * lists for the checkpoint are stable. Set the checkpointing - * flag to block eviction of dirty pages until the checkpoint's - * internal page pass is complete, then wait for any existing - * eviction to complete. + * In the final checkpoint pass, child pages cannot be evicted + * from underneath internal pages nor can underlying blocks be + * freed until the checkpoint's block lists are stable. Also, + * we cannot split child pages into parents unless we know the + * final pass will write a consistent view of that namespace. + * Set the checkpointing flag to block such actions and wait for + * any problematic eviction or page splits to complete. */ WT_PUBLISH(btree->checkpointing, WT_CKPT_PREPARE); @@ -119,6 +119,8 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) if (evict_reset) __wt_evict_file_exclusive_off(session); + __wt_split_drain(session); + WT_PUBLISH(btree->checkpointing, WT_CKPT_RUNNING); /* Write all dirty in-cache pages. */ diff --git a/src/include/btree.h b/src/include/btree.h index 703de0f2fc6..202f8893af3 100644 --- a/src/include/btree.h +++ b/src/include/btree.h @@ -125,16 +125,18 @@ struct __wt_btree { WT_BM *bm; /* Block manager reference */ u_int block_header; /* WT_PAGE_HEADER_BYTE_SIZE */ - uint64_t checkpoint_gen; /* Checkpoint generation */ - uint64_t rec_max_txn; /* Maximum txn seen (clean trees) */ - uint64_t write_gen; /* Write generation */ - WT_REF *evict_ref; /* Eviction thread's location */ uint64_t evict_priority; /* Relative priority of cached pages */ u_int evict_walk_period; /* Skip this many LRU walks */ u_int evict_walk_skips; /* Number of walks skipped */ volatile uint32_t evict_busy; /* Count of threads in eviction */ + volatile uint32_t split_busy; /* Count of threads in internal split */ + + uint64_t checkpoint_gen; /* Checkpoint generation */ + uint64_t rec_max_txn; /* Maximum txn seen (clean trees) */ + uint64_t write_gen; /* Write generation */ + enum { WT_CKPT_OFF, WT_CKPT_PREPARE, WT_CKPT_RUNNING } checkpointing; /* Checkpoint in progress */ diff --git a/src/include/extern.h b/src/include/extern.h index 1999ff6b732..a4fd22a290a 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -162,6 +162,7 @@ extern int __wt_kv_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPD extern int __wt_bt_salvage(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, const char *cfg[]); extern void __wt_split_stash_discard(WT_SESSION_IMPL *session); extern void __wt_split_stash_discard_all( WT_SESSION_IMPL *session_safe, WT_SESSION_IMPL *session); +extern void __wt_split_drain(WT_SESSION_IMPL *session); extern int __wt_multi_to_ref(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi, WT_REF **refp, size_t *incrp); extern int __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref); extern int __wt_split_multi(WT_SESSION_IMPL *session, WT_REF *ref, int closing); -- cgit v1.2.1 From 05ac79db3376cc77f9b4d62adce8c32791d84844 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 5 Feb 2016 15:37:08 -0500 Subject: WT-2361: column-store starting record number error Remove unused variable, __evict_walk_file() doesn't care if the page can split or not. --- src/evict/evict_lru.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index 0536a06bc22..8ef7164dbc6 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -1209,7 +1209,7 @@ __evict_walk_file(WT_SESSION_IMPL *session, u_int *slotp) uint64_t pages_walked; uint32_t walk_flags; int internal_pages, restarts; - bool enough, modified, would_split; + bool enough, modified; conn = S2C(session); btree = S2BT(session); @@ -1298,7 +1298,7 @@ __evict_walk_file(WT_SESSION_IMPL *session, u_int *slotp) page->read_gen = __wt_cache_read_gen_new(session); fast: /* If the page can't be evicted, give up. */ - if (!__wt_page_can_evict(session, ref, &would_split)) + if (!__wt_page_can_evict(session, ref, NULL)) continue; /* -- cgit v1.2.1 From b96f95cc8b351635f487e5413eb2796b858e11c5 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 5 Feb 2016 15:50:28 -0500 Subject: WT-2361: column-store starting record number error This will drop core if split ever fails; the change in bd8889a should have removed the "page = NULL" assignment when reversing the sense of the test of the parent-split return. --- src/btree/bt_split.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/btree/bt_split.c b/src/btree/bt_split.c index b8d6ffae252..57c32ce903f 100644 --- a/src/btree/bt_split.c +++ b/src/btree/bt_split.c @@ -1969,15 +1969,13 @@ __split_insert(WT_SESSION_IMPL *session, WT_REF *ref) /* * The act of splitting into the parent releases the pages for eviction; - * ensure the page contents are correct. + * ensure the page contents are consistent. */ WT_WRITE_BARRIER(); /* - * Split into the parent. On successful return, the original page is no - * longer locked, so we cannot safely look at it. + * Split into the parent. */ - page = NULL; if ((ret = __split_parent( session, ref, split_ref, 2, parent_incr, false, true)) == 0) return (0); -- cgit v1.2.1 From 9bf9311365e9b213d5e67a965c8dd21185c9fbd3 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sun, 7 Feb 2016 09:33:58 -0500 Subject: WT-2361: column-store starting record number error Column-store cannot skip past aborted or not-yet-visible entries in the page. If they are the only entries on the page, the page will be marked empty and reverse-split into its parent, discarding chunks of the name space, corrupting the tree. --- src/reconcile/rec_write.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/reconcile/rec_write.c b/src/reconcile/rec_write.c index 60482cb25b4..f60da40a0d2 100644 --- a/src/reconcile/rec_write.c +++ b/src/reconcile/rec_write.c @@ -3992,8 +3992,6 @@ __rec_col_fix(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) } else { WT_RET( __rec_txn_read(session, r, ins, NULL, NULL, &upd)); - if (upd == NULL) - continue; recno = WT_INSERT_RECNO(ins); } for (;;) { @@ -4548,8 +4546,6 @@ compare: /* } else { WT_ERR( __rec_txn_read(session, r, ins, NULL, NULL, &upd)); - if (upd == NULL) - continue; n = WT_INSERT_RECNO(ins); } while (src_recno <= n) { -- cgit v1.2.1 From 8765d9de63ebca86d5bb4ec564657ad89d8f3382 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sun, 7 Feb 2016 14:55:32 -0500 Subject: WT-2367: WT_CURSOR.next out-of-order returns failure Don't leave WT_CURSOR.ins_head set if there weren't any entries on the list to search, the iteration setup functions use the non-NULL state of that field to decide where to start the iteration. Rework row-store search's approach to setting WT_CURSOR.ins_head to match column-store, don't set WT_CURSOR.ins_head at all unless we're about to return a record in the insert list. Minor change, don't use "__wt" as a prefix for and inlined function. --- src/btree/bt_curprev.c | 3 ++- src/btree/row_srch.c | 61 +++++++++++++++++++++----------------------------- src/include/extern.h | 2 +- 3 files changed, 29 insertions(+), 37 deletions(-) diff --git a/src/btree/bt_curprev.c b/src/btree/bt_curprev.c index a083ec4016e..7475c0f1312 100644 --- a/src/btree/bt_curprev.c +++ b/src/btree/bt_curprev.c @@ -51,7 +51,8 @@ restart: if (cbt->btree->type == BTREE_ROW) { key.data = WT_INSERT_KEY(current); key.size = WT_INSERT_KEY_SIZE(current); - WT_RET(__wt_search_insert(session, cbt, &key)); + WT_RET(__wt_search_insert( + session, cbt, cbt->ins_head, &key)); } else cbt->ins = __col_insert_search(cbt->ins_head, cbt->ins_stack, cbt->next_stack, diff --git a/src/btree/row_srch.c b/src/btree/row_srch.c index 71564a7b3c5..17498a3c681 100644 --- a/src/btree/row_srch.c +++ b/src/btree/row_srch.c @@ -9,18 +9,17 @@ #include "wt_internal.h" /* - * __wt_search_insert_append -- + * __search_insert_append -- * Fast append search of a row-store insert list, creating a skiplist stack * as we go. */ static inline int -__wt_search_insert_append(WT_SESSION_IMPL *session, - WT_CURSOR_BTREE *cbt, WT_ITEM *srch_key, bool *donep) +__search_insert_append(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, + WT_INSERT_HEAD *inshead, WT_ITEM *srch_key, bool *donep) { WT_BTREE *btree; WT_COLLATOR *collator; WT_INSERT *ins; - WT_INSERT_HEAD *inshead; WT_ITEM key; int cmp, i; @@ -28,7 +27,6 @@ __wt_search_insert_append(WT_SESSION_IMPL *session, collator = btree->collator; *donep = 0; - inshead = cbt->ins_head; if ((ins = WT_SKIP_LAST(inshead)) == NULL) return (0); key.data = WT_INSERT_KEY(ins); @@ -54,6 +52,7 @@ __wt_search_insert_append(WT_SESSION_IMPL *session, } cbt->compare = -cmp; cbt->ins = ins; + cbt->ins_head = inshead; *donep = 1; } return (0); @@ -64,20 +63,18 @@ __wt_search_insert_append(WT_SESSION_IMPL *session, * Search a row-store insert list, creating a skiplist stack as we go. */ int -__wt_search_insert( - WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_ITEM *srch_key) +__wt_search_insert(WT_SESSION_IMPL *session, + WT_CURSOR_BTREE *cbt, WT_INSERT_HEAD *inshead, WT_ITEM *srch_key) { WT_BTREE *btree; WT_COLLATOR *collator; WT_INSERT *ins, **insp, *last_ins; - WT_INSERT_HEAD *inshead; WT_ITEM key; size_t match, skiphigh, skiplow; int cmp, i; btree = S2BT(session); collator = btree->collator; - inshead = cbt->ins_head; cmp = 0; /* -Wuninitialized */ /* @@ -128,6 +125,7 @@ __wt_search_insert( */ cbt->compare = -cmp; cbt->ins = (ins != NULL) ? ins : last_ins; + cbt->ins_head = inshead; return (0); } @@ -212,6 +210,7 @@ __wt_row_search(WT_SESSION_IMPL *session, WT_BTREE *btree; WT_COLLATOR *collator; WT_DECL_RET; + WT_INSERT_HEAD *inshead; WT_ITEM *item; WT_PAGE *page; WT_PAGE_INDEX *pindex, *parent_pindex; @@ -480,24 +479,18 @@ leaf_only: cbt->slot = WT_ROW_SLOT(page, page->pg_row_d); F_SET(cbt, WT_CBT_SEARCH_SMALLEST); - cbt->ins_head = WT_ROW_INSERT_SMALLEST(page); + inshead = WT_ROW_INSERT_SMALLEST(page); } else { cbt->slot = WT_ROW_SLOT(page, page->pg_row_d + (page->pg_row_entries - 1)); - cbt->ins_head = WT_ROW_INSERT_SLOT(page, cbt->slot); + inshead = WT_ROW_INSERT_SLOT(page, cbt->slot); } - WT_ERR( - __wt_search_insert_append(session, cbt, srch_key, &done)); + WT_ERR(__search_insert_append( + session, cbt, inshead, srch_key, &done)); if (done) return (0); - - /* - * Don't leave the insert list head set, code external to the - * search uses it. - */ - cbt->ins_head = NULL; } /* @@ -590,16 +583,16 @@ leaf_match: cbt->compare = 0; cbt->slot = WT_ROW_SLOT(page, page->pg_row_d); F_SET(cbt, WT_CBT_SEARCH_SMALLEST); - cbt->ins_head = WT_ROW_INSERT_SMALLEST(page); + inshead = WT_ROW_INSERT_SMALLEST(page); } else { cbt->compare = -1; cbt->slot = WT_ROW_SLOT(page, page->pg_row_d + (base - 1)); - cbt->ins_head = WT_ROW_INSERT_SLOT(page, cbt->slot); + inshead = WT_ROW_INSERT_SLOT(page, cbt->slot); } /* If there's no insert list, we're done. */ - if (WT_SKIP_FIRST(cbt->ins_head) == NULL) + if (WT_SKIP_FIRST(inshead) == NULL) return (0); /* @@ -607,12 +600,12 @@ leaf_match: cbt->compare = 0; * catch cursors repeatedly inserting at a single point. */ if (insert) { - WT_ERR( - __wt_search_insert_append(session, cbt, srch_key, &done)); + WT_ERR(__search_insert_append( + session, cbt, inshead, srch_key, &done)); if (done) return (0); } - WT_ERR(__wt_search_insert(session, cbt, srch_key)); + WT_ERR(__wt_search_insert(session, cbt, inshead, srch_key)); return (0); @@ -635,7 +628,7 @@ int __wt_row_random_leaf(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) { WT_INSERT *ins, **start, **stop; - WT_INSERT_HEAD *ins_head; + WT_INSERT_HEAD *inshead; WT_PAGE *page; uint32_t choice, entries, i; int level; @@ -661,20 +654,17 @@ __wt_row_random_leaf(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) /* * If the tree is new (and not empty), it might have a large insert * list. - */ - F_SET(cbt, WT_CBT_SEARCH_SMALLEST); - if ((cbt->ins_head = WT_ROW_INSERT_SMALLEST(page)) == NULL) - return (WT_NOTFOUND); - - /* + * * Walk down the list until we find a level with at least 50 entries, * that's where we'll start rolling random numbers. The value 50 is * used to ignore levels with only a few entries, that is, levels which * are potentially badly skewed. */ - for (ins_head = cbt->ins_head, - level = WT_SKIP_MAXDEPTH - 1; level >= 0; --level) { - start = &ins_head->head[level]; + F_SET(cbt, WT_CBT_SEARCH_SMALLEST); + if ((inshead = WT_ROW_INSERT_SMALLEST(page)) == NULL) + return (WT_NOTFOUND); + for (level = WT_SKIP_MAXDEPTH - 1; level >= 0; --level) { + start = &inshead->head[level]; for (entries = 0, stop = start; *stop != NULL; stop = &(*stop)->next[level]) ++entries; @@ -768,6 +758,7 @@ __wt_row_random_leaf(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) ins = ins->next[0]; cbt->ins = ins; + cbt->ins_head = inshead; cbt->compare = 0; return (0); diff --git a/src/include/extern.h b/src/include/extern.h index 1999ff6b732..9bfe367c0fd 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -190,7 +190,7 @@ extern int __wt_row_insert_alloc(WT_SESSION_IMPL *session, WT_ITEM *key, u_int s extern int __wt_update_alloc( WT_SESSION_IMPL *session, WT_ITEM *value, WT_UPDATE **updp, size_t *sizep); extern WT_UPDATE *__wt_update_obsolete_check( WT_SESSION_IMPL *session, WT_PAGE *page, WT_UPDATE *upd); extern void __wt_update_obsolete_free( WT_SESSION_IMPL *session, WT_PAGE *page, WT_UPDATE *upd); -extern int __wt_search_insert( WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_ITEM *srch_key); +extern int __wt_search_insert(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_INSERT_HEAD *inshead, WT_ITEM *srch_key); extern int __wt_row_search(WT_SESSION_IMPL *session, WT_ITEM *srch_key, WT_REF *leaf, WT_CURSOR_BTREE *cbt, bool insert); extern int __wt_row_random_leaf(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt); extern int __wt_row_random_descent(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt); -- cgit v1.2.1 From 37ce8b68a345f33cc77ce2d0fdf15de80d26ee51 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sun, 7 Feb 2016 15:05:22 -0500 Subject: WT-2367: WT_CURSOR.next out-of-order returns failure Consistently use "ins_head" instead of "inshead" for WT_INSERT_HEAD variables. --- src/btree/row_srch.c | 42 +++++++++++++++++++++--------------------- src/include/column.i | 30 +++++++++++++++--------------- src/include/extern.h | 2 +- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/btree/row_srch.c b/src/btree/row_srch.c index 17498a3c681..f3463b9924d 100644 --- a/src/btree/row_srch.c +++ b/src/btree/row_srch.c @@ -15,7 +15,7 @@ */ static inline int __search_insert_append(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, - WT_INSERT_HEAD *inshead, WT_ITEM *srch_key, bool *donep) + WT_INSERT_HEAD *ins_head, WT_ITEM *srch_key, bool *donep) { WT_BTREE *btree; WT_COLLATOR *collator; @@ -27,7 +27,7 @@ __search_insert_append(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, collator = btree->collator; *donep = 0; - if ((ins = WT_SKIP_LAST(inshead)) == NULL) + if ((ins = WT_SKIP_LAST(ins_head)) == NULL) return (0); key.data = WT_INSERT_KEY(ins); key.size = WT_INSERT_KEY_SIZE(ins); @@ -46,13 +46,13 @@ __search_insert_append(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, */ for (i = WT_SKIP_MAXDEPTH - 1; i >= 0; i--) { cbt->ins_stack[i] = (i == 0) ? &ins->next[0] : - (inshead->tail[i] != NULL) ? - &inshead->tail[i]->next[i] : &inshead->head[i]; + (ins_head->tail[i] != NULL) ? + &ins_head->tail[i]->next[i] : &ins_head->head[i]; cbt->next_stack[i] = NULL; } cbt->compare = -cmp; cbt->ins = ins; - cbt->ins_head = inshead; + cbt->ins_head = ins_head; *donep = 1; } return (0); @@ -64,7 +64,7 @@ __search_insert_append(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, */ int __wt_search_insert(WT_SESSION_IMPL *session, - WT_CURSOR_BTREE *cbt, WT_INSERT_HEAD *inshead, WT_ITEM *srch_key) + WT_CURSOR_BTREE *cbt, WT_INSERT_HEAD *ins_head, WT_ITEM *srch_key) { WT_BTREE *btree; WT_COLLATOR *collator; @@ -83,7 +83,7 @@ __wt_search_insert(WT_SESSION_IMPL *session, */ match = skiphigh = skiplow = 0; ins = last_ins = NULL; - for (i = WT_SKIP_MAXDEPTH - 1, insp = &inshead->head[i]; i >= 0;) { + for (i = WT_SKIP_MAXDEPTH - 1, insp = &ins_head->head[i]; i >= 0;) { if ((ins = *insp) == NULL) { cbt->next_stack[i] = NULL; cbt->ins_stack[i--] = insp--; @@ -125,7 +125,7 @@ __wt_search_insert(WT_SESSION_IMPL *session, */ cbt->compare = -cmp; cbt->ins = (ins != NULL) ? ins : last_ins; - cbt->ins_head = inshead; + cbt->ins_head = ins_head; return (0); } @@ -210,7 +210,7 @@ __wt_row_search(WT_SESSION_IMPL *session, WT_BTREE *btree; WT_COLLATOR *collator; WT_DECL_RET; - WT_INSERT_HEAD *inshead; + WT_INSERT_HEAD *ins_head; WT_ITEM *item; WT_PAGE *page; WT_PAGE_INDEX *pindex, *parent_pindex; @@ -479,16 +479,16 @@ leaf_only: cbt->slot = WT_ROW_SLOT(page, page->pg_row_d); F_SET(cbt, WT_CBT_SEARCH_SMALLEST); - inshead = WT_ROW_INSERT_SMALLEST(page); + ins_head = WT_ROW_INSERT_SMALLEST(page); } else { cbt->slot = WT_ROW_SLOT(page, page->pg_row_d + (page->pg_row_entries - 1)); - inshead = WT_ROW_INSERT_SLOT(page, cbt->slot); + ins_head = WT_ROW_INSERT_SLOT(page, cbt->slot); } WT_ERR(__search_insert_append( - session, cbt, inshead, srch_key, &done)); + session, cbt, ins_head, srch_key, &done)); if (done) return (0); } @@ -583,16 +583,16 @@ leaf_match: cbt->compare = 0; cbt->slot = WT_ROW_SLOT(page, page->pg_row_d); F_SET(cbt, WT_CBT_SEARCH_SMALLEST); - inshead = WT_ROW_INSERT_SMALLEST(page); + ins_head = WT_ROW_INSERT_SMALLEST(page); } else { cbt->compare = -1; cbt->slot = WT_ROW_SLOT(page, page->pg_row_d + (base - 1)); - inshead = WT_ROW_INSERT_SLOT(page, cbt->slot); + ins_head = WT_ROW_INSERT_SLOT(page, cbt->slot); } /* If there's no insert list, we're done. */ - if (WT_SKIP_FIRST(inshead) == NULL) + if (WT_SKIP_FIRST(ins_head) == NULL) return (0); /* @@ -601,11 +601,11 @@ leaf_match: cbt->compare = 0; */ if (insert) { WT_ERR(__search_insert_append( - session, cbt, inshead, srch_key, &done)); + session, cbt, ins_head, srch_key, &done)); if (done) return (0); } - WT_ERR(__wt_search_insert(session, cbt, inshead, srch_key)); + WT_ERR(__wt_search_insert(session, cbt, ins_head, srch_key)); return (0); @@ -628,7 +628,7 @@ int __wt_row_random_leaf(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) { WT_INSERT *ins, **start, **stop; - WT_INSERT_HEAD *inshead; + WT_INSERT_HEAD *ins_head; WT_PAGE *page; uint32_t choice, entries, i; int level; @@ -661,10 +661,10 @@ __wt_row_random_leaf(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) * are potentially badly skewed. */ F_SET(cbt, WT_CBT_SEARCH_SMALLEST); - if ((inshead = WT_ROW_INSERT_SMALLEST(page)) == NULL) + if ((ins_head = WT_ROW_INSERT_SMALLEST(page)) == NULL) return (WT_NOTFOUND); for (level = WT_SKIP_MAXDEPTH - 1; level >= 0; --level) { - start = &inshead->head[level]; + start = &ins_head->head[level]; for (entries = 0, stop = start; *stop != NULL; stop = &(*stop)->next[level]) ++entries; @@ -758,7 +758,7 @@ __wt_row_random_leaf(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) ins = ins->next[0]; cbt->ins = ins; - cbt->ins_head = inshead; + cbt->ins_head = ins_head; cbt->compare = 0; return (0); diff --git a/src/include/column.i b/src/include/column.i index 9f3e2101f6f..d64e68420a5 100644 --- a/src/include/column.i +++ b/src/include/column.i @@ -11,13 +11,13 @@ * Search a column-store insert list for the next larger record. */ static inline WT_INSERT * -__col_insert_search_gt(WT_INSERT_HEAD *inshead, uint64_t recno) +__col_insert_search_gt(WT_INSERT_HEAD *ins_head, uint64_t recno) { WT_INSERT *ins, **insp; int i; /* If there's no insert chain to search, we're done. */ - if ((ins = WT_SKIP_LAST(inshead)) == NULL) + if ((ins = WT_SKIP_LAST(ins_head)) == NULL) return (NULL); /* Fast path check for targets past the end of the skiplist. */ @@ -29,7 +29,7 @@ __col_insert_search_gt(WT_INSERT_HEAD *inshead, uint64_t recno) * go as far as possible at each level before stepping down to the next. */ ins = NULL; - for (i = WT_SKIP_MAXDEPTH - 1, insp = &inshead->head[i]; i >= 0;) + for (i = WT_SKIP_MAXDEPTH - 1, insp = &ins_head->head[i]; i >= 0;) if (*insp != NULL && recno >= WT_INSERT_RECNO(*insp)) { ins = *insp; /* GTE: keep going at this level */ insp = &(*insp)->next[i]; @@ -50,7 +50,7 @@ __col_insert_search_gt(WT_INSERT_HEAD *inshead, uint64_t recno) * such a record exists before searching. */ if (ins == NULL) - ins = WT_SKIP_FIRST(inshead); + ins = WT_SKIP_FIRST(ins_head); while (recno >= WT_INSERT_RECNO(ins)) ins = WT_SKIP_NEXT(ins); return (ins); @@ -61,13 +61,13 @@ __col_insert_search_gt(WT_INSERT_HEAD *inshead, uint64_t recno) * Search a column-store insert list for the next smaller record. */ static inline WT_INSERT * -__col_insert_search_lt(WT_INSERT_HEAD *inshead, uint64_t recno) +__col_insert_search_lt(WT_INSERT_HEAD *ins_head, uint64_t recno) { WT_INSERT *ins, **insp; int i; /* If there's no insert chain to search, we're done. */ - if ((ins = WT_SKIP_FIRST(inshead)) == NULL) + if ((ins = WT_SKIP_FIRST(ins_head)) == NULL) return (NULL); /* Fast path check for targets before the skiplist. */ @@ -78,7 +78,7 @@ __col_insert_search_lt(WT_INSERT_HEAD *inshead, uint64_t recno) * The insert list is a skip list: start at the highest skip level, then * go as far as possible at each level before stepping down to the next. */ - for (i = WT_SKIP_MAXDEPTH - 1, insp = &inshead->head[i]; i >= 0;) + for (i = WT_SKIP_MAXDEPTH - 1, insp = &ins_head->head[i]; i >= 0;) if (*insp != NULL && recno > WT_INSERT_RECNO(*insp)) { ins = *insp; /* GT: keep going at this level */ insp = &(*insp)->next[i]; @@ -95,14 +95,14 @@ __col_insert_search_lt(WT_INSERT_HEAD *inshead, uint64_t recno) * Search a column-store insert list for an exact match. */ static inline WT_INSERT * -__col_insert_search_match(WT_INSERT_HEAD *inshead, uint64_t recno) +__col_insert_search_match(WT_INSERT_HEAD *ins_head, uint64_t recno) { WT_INSERT **insp, *ret_ins; uint64_t ins_recno; int cmp, i; /* If there's no insert chain to search, we're done. */ - if ((ret_ins = WT_SKIP_LAST(inshead)) == NULL) + if ((ret_ins = WT_SKIP_LAST(ins_head)) == NULL) return (NULL); /* Fast path the check for values at the end of the skiplist. */ @@ -115,7 +115,7 @@ __col_insert_search_match(WT_INSERT_HEAD *inshead, uint64_t recno) * The insert list is a skip list: start at the highest skip level, then * go as far as possible at each level before stepping down to the next. */ - for (i = WT_SKIP_MAXDEPTH - 1, insp = &inshead->head[i]; i >= 0; ) { + for (i = WT_SKIP_MAXDEPTH - 1, insp = &ins_head->head[i]; i >= 0; ) { if (*insp == NULL) { --i; --insp; @@ -143,7 +143,7 @@ __col_insert_search_match(WT_INSERT_HEAD *inshead, uint64_t recno) * Search a column-store insert list, creating a skiplist stack as we go. */ static inline WT_INSERT * -__col_insert_search(WT_INSERT_HEAD *inshead, +__col_insert_search(WT_INSERT_HEAD *ins_head, WT_INSERT ***ins_stack, WT_INSERT **next_stack, uint64_t recno) { WT_INSERT **insp, *ret_ins; @@ -151,15 +151,15 @@ __col_insert_search(WT_INSERT_HEAD *inshead, int cmp, i; /* If there's no insert chain to search, we're done. */ - if ((ret_ins = WT_SKIP_LAST(inshead)) == NULL) + if ((ret_ins = WT_SKIP_LAST(ins_head)) == NULL) return (NULL); /* Fast path appends. */ if (recno >= WT_INSERT_RECNO(ret_ins)) { for (i = 0; i < WT_SKIP_MAXDEPTH; i++) { ins_stack[i] = (i == 0) ? &ret_ins->next[0] : - (inshead->tail[i] != NULL) ? - &inshead->tail[i]->next[i] : &inshead->head[i]; + (ins_head->tail[i] != NULL) ? + &ins_head->tail[i]->next[i] : &ins_head->head[i]; next_stack[i] = NULL; } return (ret_ins); @@ -169,7 +169,7 @@ __col_insert_search(WT_INSERT_HEAD *inshead, * The insert list is a skip list: start at the highest skip level, then * go as far as possible at each level before stepping down to the next. */ - for (i = WT_SKIP_MAXDEPTH - 1, insp = &inshead->head[i]; i >= 0; ) { + for (i = WT_SKIP_MAXDEPTH - 1, insp = &ins_head->head[i]; i >= 0; ) { if ((ret_ins = *insp) == NULL) { next_stack[i] = NULL; ins_stack[i--] = insp--; diff --git a/src/include/extern.h b/src/include/extern.h index 9bfe367c0fd..64d46a5a254 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -190,7 +190,7 @@ extern int __wt_row_insert_alloc(WT_SESSION_IMPL *session, WT_ITEM *key, u_int s extern int __wt_update_alloc( WT_SESSION_IMPL *session, WT_ITEM *value, WT_UPDATE **updp, size_t *sizep); extern WT_UPDATE *__wt_update_obsolete_check( WT_SESSION_IMPL *session, WT_PAGE *page, WT_UPDATE *upd); extern void __wt_update_obsolete_free( WT_SESSION_IMPL *session, WT_PAGE *page, WT_UPDATE *upd); -extern int __wt_search_insert(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_INSERT_HEAD *inshead, WT_ITEM *srch_key); +extern int __wt_search_insert(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_INSERT_HEAD *ins_head, WT_ITEM *srch_key); extern int __wt_row_search(WT_SESSION_IMPL *session, WT_ITEM *srch_key, WT_REF *leaf, WT_CURSOR_BTREE *cbt, bool insert); extern int __wt_row_random_leaf(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt); extern int __wt_row_random_descent(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt); -- cgit v1.2.1 From 0ef9946c93ab5272acab59eba8158f19ff68e1cf Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Mon, 8 Feb 2016 10:25:54 -0500 Subject: WT-2374 Lint --- src/meta/meta_turtle.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/meta/meta_turtle.c b/src/meta/meta_turtle.c index 957a67d250f..ec93f526f89 100644 --- a/src/meta/meta_turtle.c +++ b/src/meta/meta_turtle.c @@ -192,9 +192,9 @@ __wt_turtle_init(WT_SESSION_IMPL *session) * we want to recreate the metadata from the backup. */ if (exist_backup) { - __wt_msg(session, "Both %s and %s exist. " + WT_RET(__wt_msg(session, "Both %s and %s exist. " "Recreating metadata from backup.", - WT_METADATA_TURTLE, WT_METADATA_BACKUP); + WT_METADATA_TURTLE, WT_METADATA_BACKUP)); WT_RET(__wt_remove_if_exists( session, WT_METAFILE)); WT_RET(__wt_remove_if_exists( -- cgit v1.2.1 From 5c9b3f6de26f254063a378775ba9949f442d1607 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Mon, 8 Feb 2016 10:40:48 -0500 Subject: WT-2349 Restore s_stat. Accidentally committed with debugging. --- dist/s_stat | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/dist/s_stat b/dist/s_stat index 6f9bc1a9e91..44c22ab56bb 100755 --- a/dist/s_stat +++ b/dist/s_stat @@ -2,15 +2,6 @@ # Complain about unused statistics fields. t=__wt.$$ -t1=xx.search -t2=xx.files -t3=xx.output -t4=xx.bad1 -t5=xx.bad2 -t6=xx.search2 -t7=xx.bad3 -t8=xx.search3 -t9=xx.bad4 trap 'rm -f $t; exit 0' 0 1 2 3 13 15 # List of files to search: skip stat.c, it lists all of the fields by @@ -22,22 +13,6 @@ l=`sed \ -e 's,^,../,' filelist` l="$l `echo ../src/include/*.i`" -echo "$l" > $t2 - -fsearch=`sed \ - -e 's/^ int64_t \([a-z_*]*\);$/\1/p' \ - -e d ../src/include/stat.h | - sort` -echo "$fsearch" > $t1 -#fgrep -who "$fsearch" $l > $t3 -xx=`cat $t1 $t3 | sort | uniq -u` -cat $t1 $t3| sort | uniq -u > $t4 -fgrep -who "$xx" $l > $t6 -cat $t4 $t6 | sort | uniq -u > $t5 -xx=`cat $t4 $t6 | sort | uniq -u` -fgrep -who "$xx" $l > $t8 -cat $t5 $t8 | sort | uniq -u > $t7 - ( # Get the list of statistics fields. search=`sed \ -- cgit v1.2.1 From 4f43fb358627a5eb69b9aee48af9681e1f0a27f8 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Mon, 8 Feb 2016 16:39:20 -0500 Subject: WT-2349 Fixes for readonly and new python test. --- src/btree/bt_handle.c | 3 +- src/cache/cache_las.c | 2 + src/conn/conn_api.c | 3 + src/lsm/lsm_tree.c | 46 +++++++------- src/os_posix/os_fsync.c | 4 +- src/os_win/os_fsync.c | 4 +- src/os_win/os_rw.c | 4 +- test/suite/test_readonly01.py | 135 ++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 177 insertions(+), 24 deletions(-) create mode 100644 test/suite/test_readonly01.py diff --git a/src/btree/bt_handle.c b/src/btree/bt_handle.c index 2db3ca7d984..52152a2fcac 100644 --- a/src/btree/bt_handle.c +++ b/src/btree/bt_handle.c @@ -36,7 +36,8 @@ __wt_btree_open(WT_SESSION_IMPL *session, const char *op_cfg[]) btree = S2BT(session); /* Checkpoint files are readonly. */ - readonly = dhandle->checkpoint != NULL; + readonly = (dhandle->checkpoint != NULL || + F_ISSET(S2C(session), WT_CONN_READONLY)); /* Get the checkpoint information for this name/checkpoint pair. */ WT_CLEAR(ckpt); diff --git a/src/cache/cache_las.c b/src/cache/cache_las.c index 1ef8dd32bb4..3549e41e80d 100644 --- a/src/cache/cache_las.c +++ b/src/cache/cache_las.c @@ -58,6 +58,8 @@ __wt_las_create(WT_SESSION_IMPL *session) conn = S2C(session); + if (F_ISSET(conn, WT_CONN_READONLY)) + return (0); /* * Done at startup: we cannot do it on demand because we require the * schema lock to create and drop the table, and it may not always be diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 0d6269daa54..0ce5d4285e5 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -1541,6 +1541,9 @@ open_wt: conn->is_new = exist ? 0 : 1; if (conn->is_new) { + if (F_ISSET(conn, WT_CONN_READONLY)) + WT_ERR_MSG(session, EINVAL, "Creating a new database is" + " incompatible with read-only configuration."); len = (size_t)snprintf(buf, sizeof(buf), "%s\n%s\n", WT_WIREDTIGER, WIREDTIGER_VERSION_STRING); WT_ERR(__wt_write(session, fh, (wt_off_t)0, len, buf)); diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index ab18e41a2f5..cdf2bc0595c 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -420,26 +420,32 @@ __wt_lsm_tree_create(WT_SESSION_IMPL *session, WT_ERR_MSG(session, EINVAL, "LSM merge_min must be less than or equal to merge_max"); - /* - * Set up the config for each chunk. - * - * Make the memory_page_max double the chunk size, so application - * threads don't immediately try to force evict the chunk when the - * worker thread clears the NO_EVICTION flag. - */ - WT_ERR(__wt_scr_alloc(session, 0, &buf)); - WT_ERR(__wt_buf_fmt(session, buf, - "%s,key_format=u,value_format=u,memory_page_max=%" PRIu64, - config, 2 * lsm_tree->chunk_max)); - WT_ERR(__wt_strndup( - session, buf->data, buf->size, &lsm_tree->file_config)); - - /* Create the first chunk and flush the metadata. */ - WT_ERR(__wt_lsm_meta_write(session, lsm_tree)); - - /* Discard our partially populated handle. */ - ret = __lsm_tree_discard(session, lsm_tree, false); - lsm_tree = NULL; + if (!F_ISSET(S2C(session), WT_CONN_READONLY)) { + /* + * Set up the config for each chunk. + * + * Make the memory_page_max double the chunk size, so application + * threads don't immediately try to force evict the chunk when the + * worker thread clears the NO_EVICTION flag. + */ + WT_ERR(__wt_scr_alloc(session, 0, &buf)); + WT_ERR(__wt_buf_fmt(session, buf, + "%s,key_format=u,value_format=u,memory_page_max=%" PRIu64, + config, 2 * lsm_tree->chunk_max)); + WT_ERR(__wt_strndup( + session, buf->data, buf->size, &lsm_tree->file_config)); + + /* Create the first chunk and flush the metadata. */ + WT_ERR(__wt_lsm_meta_write(session, lsm_tree)); + + /* Discard our partially populated handle. */ + ret = __lsm_tree_discard(session, lsm_tree, false); + lsm_tree = NULL; + } else { + F_CLR(lsm_tree, WT_LSM_TREE_MERGES); + FLD_SET(lsm_tree->bloom, WT_LSM_BLOOM_OFF); + FLD_CLR(lsm_tree->bloom, WT_LSM_BLOOM_OLDEST); + } /* * Open our new tree and add it to the handle cache. Don't discard on diff --git a/src/os_posix/os_fsync.c b/src/os_posix/os_fsync.c index da36d45c0fe..3c091f10a54 100644 --- a/src/os_posix/os_fsync.c +++ b/src/os_posix/os_fsync.c @@ -136,7 +136,9 @@ __wt_fsync(WT_SESSION_IMPL *session, WT_FH *fh) WT_RET(__wt_verbose(session, WT_VERB_FILEOPS, "%s: fsync", fh->name)); - WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY || + WT_STRING_MATCH(fh->name, WT_SINGLETHREAD, + strlen(WT_SINGLETHREAD)))); if ((ret = __wt_handle_sync(fh->fd)) == 0) return (0); WT_RET_MSG(session, ret, "%s fsync error", fh->name); diff --git a/src/os_win/os_fsync.c b/src/os_win/os_fsync.c index 1671ade5d94..4586a0c1bb7 100644 --- a/src/os_win/os_fsync.c +++ b/src/os_win/os_fsync.c @@ -46,7 +46,9 @@ __wt_fsync(WT_SESSION_IMPL *session, WT_FH *fh) WT_RET(__wt_verbose(session, WT_VERB_FILEOPS, "%s: FlushFileBuffers", fh->name)); - WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY || + WT_STRING_MATCH(fh->name, WT_SINGLETHREAD, + strlen(WT_SINGLETHREAD)))); if ((ret = FlushFileBuffers(fh->filehandle)) == FALSE) WT_RET_MSG(session, __wt_errno(), "%s FlushFileBuffers error", fh->name); diff --git a/src/os_win/os_rw.c b/src/os_win/os_rw.c index 222d98ad7cf..a1b409e2984 100644 --- a/src/os_win/os_rw.c +++ b/src/os_win/os_rw.c @@ -74,7 +74,9 @@ __wt_write(WT_SESSION_IMPL *session, "%s: write %" WT_SIZET_FMT " bytes at offset %" PRIuMAX, fh->name, len, (uintmax_t)offset)); - WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY || + WT_STRING_MATCH(fh->name, WT_SINGLETHREAD, + strlen(WT_SINGLETHREAD)))); /* Assert direct I/O is aligned and a multiple of the alignment. */ WT_ASSERT(session, !fh->direct_io || diff --git a/test/suite/test_readonly01.py b/test/suite/test_readonly01.py new file mode 100644 index 00000000000..b5314686206 --- /dev/null +++ b/test/suite/test_readonly01.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python +# +# Public Domain 2016-2016 MongoDB, Inc. +# Public Domain 2008-2016 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_readonly01.py +# Transactions: Test readonly mode +# + +import fnmatch, os, shutil, time +from suite_subprocess import suite_subprocess +from wtscenario import multiply_scenarios, number_scenarios, prune_scenarios +import wttest + +class test_readonly01(wttest.WiredTigerTestCase, suite_subprocess): + t1 = 'table:test_readonly01_1' + create = True + entries = 10000 + extra_entries = 5 + + # + # We want a list of directory writable or readonly. + # + basecfg_list = [ + ('basecfg', dict(basecfg='config_base=true,')), + ('no_basecfg', dict(basecfg='config_base=false,')), + ] + dir_list = [ + ('write', dict(dirchmod=False)), + ('readonly', dict(dirchmod=True)), + ] + log_list = [ + ('logging', dict(logcfg='log=(archive=false,enabled,file_max=100K),')), + ('no_logging', dict(logcfg='log=(enabled=false),')), + ] + log_list = [ + ('no_logging', dict(logcfg='log=(enabled=false),')), + ] + + types = [ + ('lsm', dict(tabletype='lsm', + create_params = 'key_format=i,value_format=i')), + ('row', dict(tabletype='row', + create_params = 'key_format=i,value_format=i')), + ('var', dict(tabletype='var', + create_params = 'key_format=r,value_format=i')), + ('fix', dict(tabletype='fix', + create_params = 'key_format=r,value_format=8t')), + ] + + scenarios = multiply_scenarios('.', + basecfg_list, dir_list, log_list, types) + + def conn_config(self, dir): + self.home = dir + params = \ + 'error_prefix="%s",' % self.shortid() + \ + '%s' % self.logcfg + \ + '%s' % self.basecfg + if self.create: + conn_params = 'create,' + params + else: + conn_params = 'readonly=true,' + params + return conn_params + + def close_reopen(self): + ''' Close the connection and reopen readonly''' + # + # close the original connection. If needed, chmod the + # database directory to readonly mode. Then reopen the + # connection with readonly. + # + self.close_conn() + if self.dirchmod: + os.chmod(self.home, 0555) + self.conn = self.setUpConnectionOpen(self.home) + self.session = self.setUpSessionOpen(self.conn) + + def test_readonly(self): + # Here's the strategy: + # - Create a table. + # - Insert data into table. + # - Close connection. + # - Possibly chmod to readonly + # - Open connection readonly + # - Confirm we can read the data. + # + self.session.create(self.t1, self.create_params) + c = self.session.open_cursor(self.t1, None, None) + for i in range(self.entries): + c[i+1] = i % 255 + # Close the connection. Reopen readonly + self.create = False + self.close_reopen() + c = self.session.open_cursor(self.t1, None, None) + i = 0 + for key, value in c: + self.assertEqual(i+1, key) + self.assertEqual(i % 255, value) + i += 1 + self.assertEqual(i, self.entries) + self.pr('Read %d entries' % i) + c.close() + self.create = True + + # If we changed the directory to readonly, change it back so that + # the cleanup functions can remove it and set up for the next test. + if self.dirchmod: + os.chmod(self.home, 0777) + +if __name__ == '__main__': + wttest.run() -- cgit v1.2.1 From 64fd83193f254aaea3149b34d796b57a80c31fa7 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Mon, 8 Feb 2016 17:05:13 -0500 Subject: WT-2349 Fix misplaced parens on assert. --- src/os_posix/os_fsync.c | 4 ++-- src/os_posix/os_rw.c | 4 ++-- src/os_win/os_fsync.c | 4 ++-- src/os_win/os_rw.c | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/os_posix/os_fsync.c b/src/os_posix/os_fsync.c index 3c091f10a54..0bd0359338b 100644 --- a/src/os_posix/os_fsync.c +++ b/src/os_posix/os_fsync.c @@ -136,9 +136,9 @@ __wt_fsync(WT_SESSION_IMPL *session, WT_FH *fh) WT_RET(__wt_verbose(session, WT_VERB_FILEOPS, "%s: fsync", fh->name)); - WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY || + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY) || WT_STRING_MATCH(fh->name, WT_SINGLETHREAD, - strlen(WT_SINGLETHREAD)))); + strlen(WT_SINGLETHREAD))); if ((ret = __wt_handle_sync(fh->fd)) == 0) return (0); WT_RET_MSG(session, ret, "%s fsync error", fh->name); diff --git a/src/os_posix/os_rw.c b/src/os_posix/os_rw.c index e9f3aa52453..3d49fa7e712 100644 --- a/src/os_posix/os_rw.c +++ b/src/os_posix/os_rw.c @@ -65,9 +65,9 @@ __wt_write(WT_SESSION_IMPL *session, "%s: write %" WT_SIZET_FMT " bytes at offset %" PRIuMAX, fh->name, len, (uintmax_t)offset)); - WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY || + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY) || WT_STRING_MATCH(fh->name, WT_SINGLETHREAD, - strlen(WT_SINGLETHREAD)))); + strlen(WT_SINGLETHREAD))); /* Assert direct I/O is aligned and a multiple of the alignment. */ WT_ASSERT(session, !fh->direct_io || diff --git a/src/os_win/os_fsync.c b/src/os_win/os_fsync.c index 4586a0c1bb7..c196fc6c06a 100644 --- a/src/os_win/os_fsync.c +++ b/src/os_win/os_fsync.c @@ -46,9 +46,9 @@ __wt_fsync(WT_SESSION_IMPL *session, WT_FH *fh) WT_RET(__wt_verbose(session, WT_VERB_FILEOPS, "%s: FlushFileBuffers", fh->name)); - WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY || + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY) || WT_STRING_MATCH(fh->name, WT_SINGLETHREAD, - strlen(WT_SINGLETHREAD)))); + strlen(WT_SINGLETHREAD))); if ((ret = FlushFileBuffers(fh->filehandle)) == FALSE) WT_RET_MSG(session, __wt_errno(), "%s FlushFileBuffers error", fh->name); diff --git a/src/os_win/os_rw.c b/src/os_win/os_rw.c index a1b409e2984..a9537a648f9 100644 --- a/src/os_win/os_rw.c +++ b/src/os_win/os_rw.c @@ -74,9 +74,9 @@ __wt_write(WT_SESSION_IMPL *session, "%s: write %" WT_SIZET_FMT " bytes at offset %" PRIuMAX, fh->name, len, (uintmax_t)offset)); - WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY || + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY) || WT_STRING_MATCH(fh->name, WT_SINGLETHREAD, - strlen(WT_SINGLETHREAD)))); + strlen(WT_SINGLETHREAD))); /* Assert direct I/O is aligned and a multiple of the alignment. */ WT_ASSERT(session, !fh->direct_io || -- cgit v1.2.1 From 3511f7656b94d3cb69d3fc6b9c07c972886b30b4 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 8 Feb 2016 19:23:22 -0500 Subject: WT-2361: column-store starting record number error Review comment: revert most of 4f54a73: eviction waits for scheduled eviction work to drain, all we have to do before splitting up the tree is check if the last phase of a checkpoint is in process. --- src/btree/bt_split.c | 30 +++--------------------------- src/btree/bt_sync.c | 2 -- src/include/btree.h | 10 ++++------ src/include/extern.h | 1 - 4 files changed, 7 insertions(+), 36 deletions(-) diff --git a/src/btree/bt_split.c b/src/btree/bt_split.c index 57c32ce903f..ae083a41ba9 100644 --- a/src/btree/bt_split.c +++ b/src/btree/bt_split.c @@ -1331,21 +1331,6 @@ __split_internal_unlock(WT_SESSION_IMPL *session, WT_PAGE *parent, bool hazard) return (ret); } -/* - * __wt_split_drain -- - * Wait for ongoing splits to drain. - */ -void -__wt_split_drain(WT_SESSION_IMPL *session) -{ - WT_BTREE *btree; - - btree = S2BT(session); - - while (btree->split_busy > 0) - __wt_yield(); -} - /* * __split_internal_should_split -- * Return if we should split an internal page. @@ -1405,8 +1390,6 @@ __split_parent_climb(WT_SESSION_IMPL *session, WT_PAGE *page, bool page_hazard) bool parent_hazard; btree = S2BT(session); - parent = NULL; - parent_hazard = false; /* * Disallow internal splits during the final pass of a checkpoint. Most @@ -1417,16 +1400,9 @@ __split_parent_climb(WT_SESSION_IMPL *session, WT_PAGE *page, bool page_hazard) * words, in one part of the tree we'll skip the newly created insert * split chunk, but we'll write it upon finding it in a different part * of the tree. - * - * Checkpoints wait until the count of internal splits goes to zero; - * check before doing the increment because it's easy, and afterward - * to avoid any races. */ - if (btree->checkpointing != WT_CKPT_OFF) + if (btree->checkpointing == WT_CKPT_RUNNING) return (__split_internal_unlock(session, page, page_hazard)); - (void)__wt_atomic_addv32(&btree->split_busy, 1); - if (btree->checkpointing != WT_CKPT_OFF) - WT_ERR(EBUSY); /* * Page splits trickle up the tree, that is, as leaf pages grow large @@ -1447,6 +1423,8 @@ __split_parent_climb(WT_SESSION_IMPL *session, WT_PAGE *page, bool page_hazard) * Split up the tree. */ for (;;) { + parent = NULL; + parent_hazard = false; ref = page->pg_intl_parent_ref; /* If we don't need to split the page, we're done. */ @@ -1483,8 +1461,6 @@ err: if (parent != NULL) __split_internal_unlock(session, parent, parent_hazard)); WT_TRET(__split_internal_unlock(session, page, page_hazard)); - (void)__wt_atomic_subv32(&btree->split_busy, 1); - /* A page may have been busy, in which case return without error. */ WT_RET_BUSY_OK(ret); return (0); diff --git a/src/btree/bt_sync.c b/src/btree/bt_sync.c index 1b0a1056d9a..bbfb06c636f 100644 --- a/src/btree/bt_sync.c +++ b/src/btree/bt_sync.c @@ -119,8 +119,6 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) if (evict_reset) __wt_evict_file_exclusive_off(session); - __wt_split_drain(session); - WT_PUBLISH(btree->checkpointing, WT_CKPT_RUNNING); /* Write all dirty in-cache pages. */ diff --git a/src/include/btree.h b/src/include/btree.h index 202f8893af3..703de0f2fc6 100644 --- a/src/include/btree.h +++ b/src/include/btree.h @@ -125,18 +125,16 @@ struct __wt_btree { WT_BM *bm; /* Block manager reference */ u_int block_header; /* WT_PAGE_HEADER_BYTE_SIZE */ + uint64_t checkpoint_gen; /* Checkpoint generation */ + uint64_t rec_max_txn; /* Maximum txn seen (clean trees) */ + uint64_t write_gen; /* Write generation */ + WT_REF *evict_ref; /* Eviction thread's location */ uint64_t evict_priority; /* Relative priority of cached pages */ u_int evict_walk_period; /* Skip this many LRU walks */ u_int evict_walk_skips; /* Number of walks skipped */ volatile uint32_t evict_busy; /* Count of threads in eviction */ - volatile uint32_t split_busy; /* Count of threads in internal split */ - - uint64_t checkpoint_gen; /* Checkpoint generation */ - uint64_t rec_max_txn; /* Maximum txn seen (clean trees) */ - uint64_t write_gen; /* Write generation */ - enum { WT_CKPT_OFF, WT_CKPT_PREPARE, WT_CKPT_RUNNING } checkpointing; /* Checkpoint in progress */ diff --git a/src/include/extern.h b/src/include/extern.h index 3a3d1d9ea92..64d46a5a254 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -162,7 +162,6 @@ extern int __wt_kv_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPD extern int __wt_bt_salvage(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, const char *cfg[]); extern void __wt_split_stash_discard(WT_SESSION_IMPL *session); extern void __wt_split_stash_discard_all( WT_SESSION_IMPL *session_safe, WT_SESSION_IMPL *session); -extern void __wt_split_drain(WT_SESSION_IMPL *session); extern int __wt_multi_to_ref(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi, WT_REF **refp, size_t *incrp); extern int __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref); extern int __wt_split_multi(WT_SESSION_IMPL *session, WT_REF *ref, int closing); -- cgit v1.2.1 From 86b3d2f681862117289c0e8944d5166c84ffa54a Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 8 Feb 2016 19:30:17 -0500 Subject: WT-2361: column-store starting record number error Test against WT_CKPT_OFF, not WT_CKPT_RUNNING, the prepare phase is when checkpoint is waiting for eviction to drain, testing against running would allow us to race. --- src/btree/bt_split.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/btree/bt_split.c b/src/btree/bt_split.c index ae083a41ba9..68d3cc91eb3 100644 --- a/src/btree/bt_split.c +++ b/src/btree/bt_split.c @@ -1401,7 +1401,7 @@ __split_parent_climb(WT_SESSION_IMPL *session, WT_PAGE *page, bool page_hazard) * split chunk, but we'll write it upon finding it in a different part * of the tree. */ - if (btree->checkpointing == WT_CKPT_RUNNING) + if (btree->checkpointing != WT_CKPT_OFF) return (__split_internal_unlock(session, page, page_hazard)); /* -- cgit v1.2.1 From a03e0958273788d2c2a8a44ba7abffe10fc55897 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Tue, 9 Feb 2016 12:46:22 +1100 Subject: WT-2387 Fix the error string we test for next_random on column stores. --- test/suite/test_cursor_random.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/suite/test_cursor_random.py b/test/suite/test_cursor_random.py index 2cef62b218a..1fd30d93c11 100644 --- a/test/suite/test_cursor_random.py +++ b/test/suite/test_cursor_random.py @@ -137,7 +137,7 @@ class test_cursor_random_column(wttest.WiredTigerTestCase): def test_cursor_random_column(self): self.session.create(self.uri, 'key_format=r,value_format=S') - msg = '/Operation not supported/' + msg = '/next_random .* not supported/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.open_cursor(self.uri, None, "next_random=true"), msg) -- cgit v1.2.1 From 3afbc3749499312f95c85a8e9e1b3ed44bf4bf93 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 9 Feb 2016 09:40:06 -0500 Subject: WT-2361: column-store starting record number error Insert splits happen concurrently with checkpoints, and checkpoints can skip the page being split based on its first-dirty transaction value, but not skip the newly created split page, which corrupts the checkpoint. Don't let that happen, set the first-dirty transaction value before splitting. --- src/btree/bt_split.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/btree/bt_split.c b/src/btree/bt_split.c index 68d3cc91eb3..e036d6e3334 100644 --- a/src/btree/bt_split.c +++ b/src/btree/bt_split.c @@ -1943,6 +1943,15 @@ __split_insert(WT_SESSION_IMPL *session, WT_REF *ref) __wt_cache_page_inmem_decr(session, page, page_decr); __wt_cache_page_inmem_incr(session, right, right_incr); + /* + * We perform insert splits concurrently with checkpoints, where the + * requirement is a checkpoint must include either the original page + * or both new pages. The page we're splitting is dirty, but that's + * insufficient: set the first dirty transaction to an impossibly old + * value so this page is not skipped by a checkpoint. + */ + page->modify->first_dirty_txn = WT_TXN_FIRST; + /* * The act of splitting into the parent releases the pages for eviction; * ensure the page contents are consistent. -- cgit v1.2.1 From 0f2420492e7ce145114d70af66ae7268d027ae27 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 9 Feb 2016 09:44:22 -0500 Subject: WT-2361: column-store starting record number error Acquire page lock before reading or updating any page transactional information. I've never seen a bug here, but insert splits are concurrent with checkpoints and I'm worried this is a problem waiting to happen. --- src/reconcile/rec_write.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/reconcile/rec_write.c b/src/reconcile/rec_write.c index 032b3fa31fa..f245ff5d921 100644 --- a/src/reconcile/rec_write.c +++ b/src/reconcile/rec_write.c @@ -362,6 +362,17 @@ __wt_reconcile(WT_SESSION_IMPL *session, /* We shouldn't get called with a clean page, that's an error. */ WT_ASSERT(session, __wt_page_is_modified(page)); + /* + * Reconciliation locks the page for three reasons: + * Reconciliation reads the lists of page updates, obsolete updates + * cannot be discarded while reconciliation is in progress; + * The compaction process reads page modification information, which + * reconciliation modifies; + * In-memory splits: reconciliation of an internal page cannot handle + * a child page splitting during the reconciliation. + */ + WT_RET(__wt_fair_lock(session, &page->page_lock)); + /* * Check that transaction time always moves forward for a given page. * If this check fails, reconciliation can free something that a future @@ -376,17 +387,6 @@ __wt_reconcile(WT_SESSION_IMPL *session, session, ref, flags, salvage, &session->reconcile)); r = session->reconcile; - /* - * Reconciliation locks the page for three reasons: - * Reconciliation reads the lists of page updates, obsolete updates - * cannot be discarded while reconciliation is in progress; - * The compaction process reads page modification information, which - * reconciliation modifies; - * In-memory splits: reconciliation of an internal page cannot handle - * a child page splitting during the reconciliation. - */ - WT_RET(__wt_fair_lock(session, &page->page_lock)); - /* Reconcile the page. */ switch (page->type) { case WT_PAGE_COL_FIX: -- cgit v1.2.1 From 6cc6b78a2071de8600caf28682f84230c8beb34a Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 9 Feb 2016 09:55:15 -0500 Subject: WT-2361: column-store starting record number error Move the comment/code for setting the first dirty transaction in the page we're splitting, and the page we're creating, next to each other in the function for clarity. --- src/btree/bt_split.c | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/btree/bt_split.c b/src/btree/bt_split.c index e036d6e3334..3dea03316ce 100644 --- a/src/btree/bt_split.c +++ b/src/btree/bt_split.c @@ -1787,8 +1787,8 @@ __split_insert(WT_SESSION_IMPL *session, WT_REF *ref) type, WT_INSERT_RECNO(moved_ins), 0, false, &right)); /* - * The new page is dirty by definition, column-store splits update the - * page-modify structure, so create it now. + * The new page is dirty by definition, plus column-store splits update + * the page-modify structure, so create it now. */ WT_ERR(__wt_page_modify_init(session, right)); __wt_page_modify_set(session, right); @@ -1828,15 +1828,6 @@ __split_insert(WT_SESSION_IMPL *session, WT_REF *ref) page->modify->mod_split_recno = child->key.recno; } - /* - * We modified the page above, which will have set the first dirty - * transaction to the last transaction current running. However, the - * updates we installed may be older than that. Set the first dirty - * transaction to an impossibly old value so this page is never skipped - * in a checkpoint. - */ - right->modify->first_dirty_txn = WT_TXN_FIRST; - /* * Calculate how much memory we're moving: figure out how deep the skip * list stack is for the element we are moving, and the memory used by @@ -1934,15 +1925,6 @@ __split_insert(WT_SESSION_IMPL *session, WT_REF *ref) WT_ASSERT(session, ins != moved_ins); #endif - /* - * Update the page accounting. - * - * XXX - * If we fail to split the parent, the page's accounting will be wrong. - */ - __wt_cache_page_inmem_decr(session, page, page_decr); - __wt_cache_page_inmem_incr(session, right, right_incr); - /* * We perform insert splits concurrently with checkpoints, where the * requirement is a checkpoint must include either the original page @@ -1952,6 +1934,24 @@ __split_insert(WT_SESSION_IMPL *session, WT_REF *ref) */ page->modify->first_dirty_txn = WT_TXN_FIRST; + /* + * We modified the page above, which will have set the first dirty + * transaction to the last transaction current running. However, the + * updates we installed may be older than that. Set the first dirty + * transaction to an impossibly old value so this page is never skipped + * in a checkpoint. + */ + right->modify->first_dirty_txn = WT_TXN_FIRST; + + /* + * Update the page accounting. + * + * XXX + * If we fail to split the parent, the page's accounting will be wrong. + */ + __wt_cache_page_inmem_decr(session, page, page_decr); + __wt_cache_page_inmem_incr(session, right, right_incr); + /* * The act of splitting into the parent releases the pages for eviction; * ensure the page contents are consistent. -- cgit v1.2.1 From e5e2ad13875a6d1f6015bcedbf1ccc012e15c725 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 9 Feb 2016 11:31:42 -0500 Subject: WT-2374 Fix inconsistent comment. --- src/txn/txn_ckpt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/txn/txn_ckpt.c b/src/txn/txn_ckpt.c index 6a2c1eef826..c4ed92fba25 100644 --- a/src/txn/txn_ckpt.c +++ b/src/txn/txn_ckpt.c @@ -1263,7 +1263,7 @@ __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) /* * Turn on metadata tracking if: * - The session is not already doing metadata tracking. - * - The file was bulk loaded. + * - The file was not bulk loaded. * - The close is not during connection close. */ need_tracking = !WT_META_TRACKING(session) && !bulk && !final; -- cgit v1.2.1 From 55a9a0f8d53ca2af2fec6287c6605bbaaaefd2aa Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 9 Feb 2016 11:40:00 -0500 Subject: WT-2349 Fix readonly issues with logging and recovery. --- src/log/log.c | 58 +++++++++++++++++++++++++------------------ src/txn/txn_recover.c | 6 ++++- test/suite/test_readonly01.py | 3 --- 3 files changed, 39 insertions(+), 28 deletions(-) diff --git a/src/log/log.c b/src/log/log.c index ce2d7191491..26c6aed95ae 100644 --- a/src/log/log.c +++ b/src/log/log.c @@ -1088,28 +1088,36 @@ __wt_log_open(WT_SESSION_IMPL *session) WT_RET(__wt_open(session, conn->log_path, false, false, WT_FILE_TYPE_DIRECTORY, &log->log_dir_fh)); } - /* - * Clean up any old interim pre-allocated files. - * We clean up these files because settings have changed upon reboot - * and we want those settings to take effect right away. - */ - WT_ERR(__log_get_files(session, - WT_LOG_TMPNAME, &logfiles, &logcount)); - for (i = 0; i < logcount; i++) { - WT_ERR(__wt_log_extract_lognum(session, logfiles[i], &lognum)); - WT_ERR(__wt_log_remove(session, WT_LOG_TMPNAME, lognum)); - } - __wt_log_files_free(session, logfiles, logcount); - logfiles = NULL; - logcount = 0; - WT_ERR(__log_get_files(session, - WT_LOG_PREPNAME, &logfiles, &logcount)); - for (i = 0; i < logcount; i++) { - WT_ERR(__wt_log_extract_lognum(session, logfiles[i], &lognum)); - WT_ERR(__wt_log_remove(session, WT_LOG_PREPNAME, lognum)); + + if (!F_ISSET(conn, WT_CONN_READONLY)) { + /* + * Clean up any old interim pre-allocated files. We clean + * up these files because settings have changed upon reboot + * and we want those settings to take effect right away. + */ + WT_ERR(__log_get_files(session, + WT_LOG_TMPNAME, &logfiles, &logcount)); + for (i = 0; i < logcount; i++) { + WT_ERR(__wt_log_extract_lognum( + session, logfiles[i], &lognum)); + WT_ERR(__wt_log_remove( + session, WT_LOG_TMPNAME, lognum)); + } + __wt_log_files_free(session, logfiles, logcount); + logfiles = NULL; + logcount = 0; + WT_ERR(__log_get_files(session, + WT_LOG_PREPNAME, &logfiles, &logcount)); + for (i = 0; i < logcount; i++) { + WT_ERR(__wt_log_extract_lognum( + session, logfiles[i], &lognum)); + WT_ERR(__wt_log_remove( + session, WT_LOG_PREPNAME, lognum)); + } + __wt_log_files_free(session, logfiles, logcount); + logfiles = NULL; } - __wt_log_files_free(session, logfiles, logcount); - logfiles = NULL; + /* * Now look at the log files and set our LSNs. */ @@ -1132,9 +1140,11 @@ __wt_log_open(WT_SESSION_IMPL *session) * Start logging at the beginning of the next log file, no matter * where the previous log file ends. */ - WT_WITH_SLOT_LOCK(session, log, ret, - ret = __log_newfile(session, true, NULL)); - WT_ERR(ret); + if (!F_ISSET(conn, WT_CONN_READONLY)) { + WT_WITH_SLOT_LOCK(session, log, ret, + ret = __log_newfile(session, true, NULL)); + WT_ERR(ret); + } /* If we found log files, save the new state. */ if (logcount > 0) { diff --git a/src/txn/txn_recover.c b/src/txn/txn_recover.c index 94df0586ced..029c0d7522c 100644 --- a/src/txn/txn_recover.c +++ b/src/txn/txn_recover.c @@ -501,6 +501,9 @@ __wt_txn_recover(WT_SESSION_IMPL *session) WT_ERR(WT_RUN_RECOVERY); } + if (F_ISSET(conn, WT_CONN_READONLY)) + goto done; + /* * Recovery can touch more data than fits in cache, so it relies on * regular eviction to manage paging. Start eviction threads for @@ -510,7 +513,8 @@ __wt_txn_recover(WT_SESSION_IMPL *session) eviction_started = true; /* - * Always run recovery even if it was a clean shutdown. + * Always run recovery even if it was a clean shutdown only if + * this is not a read-only connection. * We can consider skipping it in the future. */ if (WT_IS_INIT_LSN(&r.ckpt_lsn)) diff --git a/test/suite/test_readonly01.py b/test/suite/test_readonly01.py index b5314686206..62352b55584 100644 --- a/test/suite/test_readonly01.py +++ b/test/suite/test_readonly01.py @@ -56,9 +56,6 @@ class test_readonly01(wttest.WiredTigerTestCase, suite_subprocess): ('logging', dict(logcfg='log=(archive=false,enabled,file_max=100K),')), ('no_logging', dict(logcfg='log=(enabled=false),')), ] - log_list = [ - ('no_logging', dict(logcfg='log=(enabled=false),')), - ] types = [ ('lsm', dict(tabletype='lsm', -- cgit v1.2.1 From 5c269eec00656eb0053a22ea97f886f6cac23a7c Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 9 Feb 2016 12:37:13 -0500 Subject: WT-2361: column-store starting record number error Whitespace fix. --- src/meta/meta_turtle.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/meta/meta_turtle.c b/src/meta/meta_turtle.c index ec93f526f89..3d27f0b5845 100644 --- a/src/meta/meta_turtle.c +++ b/src/meta/meta_turtle.c @@ -195,8 +195,7 @@ __wt_turtle_init(WT_SESSION_IMPL *session) WT_RET(__wt_msg(session, "Both %s and %s exist. " "Recreating metadata from backup.", WT_METADATA_TURTLE, WT_METADATA_BACKUP)); - WT_RET(__wt_remove_if_exists( - session, WT_METAFILE)); + WT_RET(__wt_remove_if_exists(session, WT_METAFILE)); WT_RET(__wt_remove_if_exists( session, WT_METADATA_TURTLE)); load = true; -- cgit v1.2.1 From 3d269cddf9d1140151e0ef1ce0f619636e3299dc Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 9 Feb 2016 12:44:48 -0500 Subject: WT-2349 Add URI prefixes to test. Fix some LSM manager issues. --- src/lsm/lsm_manager.c | 6 +++++- test/suite/test_readonly01.py | 23 +++++++++++++++-------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/lsm/lsm_manager.c b/src/lsm/lsm_manager.c index b3f60b25c21..6fd96aa2af3 100644 --- a/src/lsm/lsm_manager.c +++ b/src/lsm/lsm_manager.c @@ -212,8 +212,10 @@ __wt_lsm_manager_start(WT_SESSION_IMPL *session) conn = S2C(session); manager = &conn->lsm_manager; - if (F_ISSET(conn, WT_CONN_READONLY)) + if (F_ISSET(conn, WT_CONN_READONLY)) { + manager->lsm_workers = 0; return (0); + } /* * We need at least a manager, a switch thread and a generic * worker. @@ -286,6 +288,8 @@ __wt_lsm_manager_destroy(WT_SESSION_IMPL *session) manager = &conn->lsm_manager; removed = 0; + WT_ASSERT(session, !F_ISSET(conn, WT_CONN_READONLY) || + manager->lsm_workers == 0); if (manager->lsm_workers > 0) { /* * Stop the main LSM manager thread first. diff --git a/test/suite/test_readonly01.py b/test/suite/test_readonly01.py index 62352b55584..8d8f6a6a293 100644 --- a/test/suite/test_readonly01.py +++ b/test/suite/test_readonly01.py @@ -36,7 +36,7 @@ from wtscenario import multiply_scenarios, number_scenarios, prune_scenarios import wttest class test_readonly01(wttest.WiredTigerTestCase, suite_subprocess): - t1 = 'table:test_readonly01_1' + tablename = 'test_readonly01' create = True entries = 10000 extra_entries = 5 @@ -58,13 +58,19 @@ class test_readonly01(wttest.WiredTigerTestCase, suite_subprocess): ] types = [ - ('lsm', dict(tabletype='lsm', + ('lsm', dict(tabletype='lsm', uri='lsm', create_params = 'key_format=i,value_format=i')), - ('row', dict(tabletype='row', + ('file-row', dict(tabletype='row', uri='file', create_params = 'key_format=i,value_format=i')), - ('var', dict(tabletype='var', + ('file-var', dict(tabletype='var', uri='file', create_params = 'key_format=r,value_format=i')), - ('fix', dict(tabletype='fix', + ('file-fix', dict(tabletype='fix', uri='file', + create_params = 'key_format=r,value_format=8t')), + ('table-row', dict(tabletype='row', uri='table', + create_params = 'key_format=i,value_format=i')), + ('table-var', dict(tabletype='var', uri='table', + create_params = 'key_format=r,value_format=i')), + ('table-fix', dict(tabletype='fix', uri='table', create_params = 'key_format=r,value_format=8t')), ] @@ -105,14 +111,15 @@ class test_readonly01(wttest.WiredTigerTestCase, suite_subprocess): # - Open connection readonly # - Confirm we can read the data. # - self.session.create(self.t1, self.create_params) - c = self.session.open_cursor(self.t1, None, None) + tablearg = self.uri + ':' + self.tablename + self.session.create(tablearg, self.create_params) + c = self.session.open_cursor(tablearg, None, None) for i in range(self.entries): c[i+1] = i % 255 # Close the connection. Reopen readonly self.create = False self.close_reopen() - c = self.session.open_cursor(self.t1, None, None) + c = self.session.open_cursor(tablearg, None, None) i = 0 for key, value in c: self.assertEqual(i+1, key) -- cgit v1.2.1 From 9fe1439f18538ffcb7f7d60438944e9d6a6df117 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 9 Feb 2016 13:33:45 -0500 Subject: WT-2349 For readonly don't sync log on close. --- src/log/log.c | 10 +++++++--- src/lsm/lsm_tree.c | 6 +++--- test/suite/test_readonly01.py | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/log/log.c b/src/log/log.c index 26c6aed95ae..03145d8408c 100644 --- a/src/log/log.c +++ b/src/log/log.c @@ -1173,20 +1173,24 @@ __wt_log_close(WT_SESSION_IMPL *session) if (log->log_close_fh != NULL && log->log_close_fh != log->log_fh) { WT_RET(__wt_verbose(session, WT_VERB_LOG, "closing old log %s", log->log_close_fh->name)); - WT_RET(__wt_fsync(session, log->log_close_fh)); + if (!F_ISSET(conn, WT_CONN_READONLY)) + WT_RET(__wt_fsync(session, log->log_close_fh)); WT_RET(__wt_close(session, &log->log_close_fh)); } if (log->log_fh != NULL) { WT_RET(__wt_verbose(session, WT_VERB_LOG, "closing log %s", log->log_fh->name)); - WT_RET(__wt_fsync(session, log->log_fh)); + if (!F_ISSET(conn, WT_CONN_READONLY)) + WT_RET(__wt_fsync(session, log->log_fh)); WT_RET(__wt_close(session, &log->log_fh)); log->log_fh = NULL; } if (log->log_dir_fh != NULL) { WT_RET(__wt_verbose(session, WT_VERB_LOG, "closing log directory %s", log->log_dir_fh->name)); - WT_RET(__wt_directory_sync_fh(session, log->log_dir_fh)); + if (!F_ISSET(conn, WT_CONN_READONLY)) + WT_RET( + __wt_directory_sync_fh(session, log->log_dir_fh)); WT_RET(__wt_close(session, &log->log_dir_fh)); log->log_dir_fh = NULL; } diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index cdf2bc0595c..33cfb242a1e 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -424,9 +424,9 @@ __wt_lsm_tree_create(WT_SESSION_IMPL *session, /* * Set up the config for each chunk. * - * Make the memory_page_max double the chunk size, so application - * threads don't immediately try to force evict the chunk when the - * worker thread clears the NO_EVICTION flag. + * Make the memory_page_max double the chunk size, so + * application threads don't immediately try to force evict + * the chunk when the worker thread clears the NO_EVICTION flag. */ WT_ERR(__wt_scr_alloc(session, 0, &buf)); WT_ERR(__wt_buf_fmt(session, buf, diff --git a/test/suite/test_readonly01.py b/test/suite/test_readonly01.py index 8d8f6a6a293..4cc65405301 100644 --- a/test/suite/test_readonly01.py +++ b/test/suite/test_readonly01.py @@ -27,7 +27,7 @@ # OTHER DEALINGS IN THE SOFTWARE. # # test_readonly01.py -# Transactions: Test readonly mode +# Readonly: Test readonly mode. # import fnmatch, os, shutil, time -- cgit v1.2.1 From a6c183b0ce4c02a6bdb49f1a4a8ed9c61c482ced Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Tue, 9 Feb 2016 14:26:27 -0500 Subject: WT-2382 For cursor joins, treat all trailing 'u' index keys as 'U'. - add "no-op" padding ("0x") to format when extracting the logical index key. - use a more descriptive name for 'positioned' flag. - added a macro to extract just the range flags from the endpoint. - fixed some logic around when repacking needs to be done. - using "no-op" padding allows __wt_struct_repack to be simplified, now it never allocates memory. - fixed error path in __curjoin_extract_insert. - simplified code, eliminated unneeded vars. - when the first index is just 'eq', finish iteration when it becomes non-equal. --- src/cursor/cur_join.c | 130 ++++++++++++++++++++++++++++-------------------- src/include/cursor.h | 8 ++- src/include/extern.h | 2 +- src/packing/pack_impl.c | 43 +++------------- 4 files changed, 91 insertions(+), 92 deletions(-) diff --git a/src/cursor/cur_join.c b/src/cursor/cur_join.c index 2cbefa68c5e..e9ffea813ac 100644 --- a/src/cursor/cur_join.c +++ b/src/cursor/cur_join.c @@ -53,7 +53,9 @@ __curjoin_entry_iter_init(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, iter->session = session; iter->entry = entry; iter->cursor = newcur; - iter->advance = false; + iter->positioned = false; + iter->isequal = (entry->ends_next == 1 && + WT_CURJOIN_END_RANGE(&entry->ends[0]) == WT_CURJOIN_END_EQ); *iterp = iter; if (0) { @@ -101,10 +103,10 @@ __curjoin_entry_iter_next(WT_CURSOR_JOIN_ITER *iter, WT_ITEM *primkey, WT_SESSION_IMPL *session; uint64_t r; - if (iter->advance) + if (iter->positioned) WT_ERR(iter->cursor->next(iter->cursor)); else - iter->advance = true; + iter->positioned = true; session = iter->session; cjoin = iter->cjoin; @@ -143,11 +145,11 @@ __curjoin_entry_iter_reset(WT_CURSOR_JOIN_ITER *iter) { WT_DECL_RET; - if (iter->advance) { + if (iter->positioned) { WT_ERR(iter->cursor->reset(iter->cursor)); WT_ERR(__wt_cursor_dup_position( iter->cjoin->entries[0].ends[0].cursor, iter->cursor)); - iter->advance = false; + iter->positioned = false; iter->entry->stats.actual_count = 0; } @@ -162,7 +164,7 @@ err: return (ret); static bool __curjoin_entry_iter_ready(WT_CURSOR_JOIN_ITER *iter) { - return (iter->advance); + return (iter->positioned); } /* @@ -255,18 +257,16 @@ __curjoin_init_bloom(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, WT_CURSOR_JOIN_ENDPOINT *end, *endmax; WT_DECL_RET; WT_DECL_ITEM(uribuf); - WT_ITEM curkey, curvalue, *k; + WT_ITEM curkey, curvalue; WT_TABLE *maintable; const char *raw_cfg[] = { WT_CONFIG_BASE( session, WT_SESSION_open_cursor), "raw", NULL }; const char *mainkey_str, *p; - void *allocbuf; size_t mainkey_len, size; u_int i; int cmp, skip; c = NULL; - allocbuf = NULL; skip = 0; if (entry->index != NULL) { @@ -305,7 +305,7 @@ __curjoin_init_bloom(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, if ((end = &entry->ends[0]) < endmax && F_ISSET(end, WT_CURJOIN_END_GE)) { WT_ERR(__wt_cursor_dup_position(end->cursor, c)); - if (end->flags == WT_CURJOIN_END_GE) + if (WT_CURJOIN_END_RANGE(end) == WT_CURJOIN_END_GE) skip = 1; } collator = (entry->index == NULL) ? NULL : entry->index->collator; @@ -313,18 +313,15 @@ __curjoin_init_bloom(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, WT_ERR(c->get_key(c, &curkey)); if (entry->index != NULL) { cindex = (WT_CURSOR_INDEX *)c; - if (cindex->index->extractor == NULL) { - /* - * Repack so it's comparable to the - * reference endpoints. - */ - k = &cindex->child->key; - WT_ERR(__wt_struct_repack(session, - cindex->child->key_format, - entry->main->value_format, k, &curkey, - &allocbuf)); - } else - curkey = cindex->child->key; + /* + * Repack so it's comparable to the + * reference endpoints. + */ + WT_ERR(__wt_struct_repack(session, + cindex->child->key_format, + (entry->repack_format ? entry->repack_format : + cindex->iface.key_format), + &cindex->child->key, &curkey)); } for (end = &entry->ends[skip]; end < endmax; end++) { WT_ERR(__wt_compare(session, collator, &curkey, @@ -361,7 +358,6 @@ done: err: if (c != NULL) WT_TRET(c->close(c)); __wt_scr_free(session, &uribuf); - __wt_free(session, allocbuf); return (ret); } @@ -378,19 +374,16 @@ __curjoin_endpoint_init_key(WT_SESSION_IMPL *session, WT_DECL_RET; WT_ITEM *k; uint64_t r; - void *allocbuf; - allocbuf = NULL; if ((cursor = endpoint->cursor) != NULL) { if (entry->index != NULL) { /* Extract and save the index's logical key. */ cindex = (WT_CURSOR_INDEX *)endpoint->cursor; WT_ERR(__wt_struct_repack(session, cindex->child->key_format, - cindex->iface.key_format, - &cindex->child->key, &endpoint->key, &allocbuf)); - if (allocbuf != NULL) - F_SET(endpoint, WT_CURJOIN_END_OWN_KEY); + (entry->repack_format ? entry->repack_format : + cindex->iface.key_format), + &cindex->child->key, &endpoint->key)); } else { k = &((WT_CURSOR_TABLE *)cursor)->cg_cursors[0]->key; if (WT_CURSOR_RECNO(cursor)) { @@ -404,10 +397,8 @@ __curjoin_endpoint_init_key(WT_SESSION_IMPL *session, endpoint->key = *k; } } - if (0) { -err: __wt_free(session, allocbuf); - } - return (ret); + +err: return (ret); } /* @@ -548,7 +539,7 @@ err: return (ret); typedef struct { WT_CURSOR iface; WT_CURSOR_JOIN_ENTRY *entry; - int ismember; + bool ismember; } WT_CURJOIN_EXTRACTOR; /* @@ -584,8 +575,8 @@ __curjoin_extract_insert(WT_CURSOR *cursor) { ret = __curjoin_entry_in_range(session, cextract->entry, &ikey, false); if (ret == WT_NOTFOUND) ret = 0; - else - cextract->ismember = 1; + else if (ret == 0) + cextract->ismember = true; return (ret); } @@ -659,10 +650,11 @@ __curjoin_entry_member(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, v = *key; if ((idx = entry->index) != NULL && idx->extractor != NULL) { + WT_CLEAR(extract_cursor); extract_cursor.iface = iface; extract_cursor.iface.session = &session->iface; extract_cursor.iface.key_format = idx->exkey_format; - extract_cursor.ismember = 0; + extract_cursor.ismember = false; extract_cursor.entry = entry; WT_ERR(idx->extractor->extract(idx->extractor, &session->iface, key, &v, &extract_cursor.iface)); @@ -715,8 +707,15 @@ nextkey: for (i = 0; i < cjoin->entries_next; i++) { ret = __curjoin_entry_member(session, cjoin, &cjoin->entries[i], skip_left); - if (ret == WT_NOTFOUND) + if (ret == WT_NOTFOUND) { + /* + * If this is compare=eq on our outer iterator, + * and we've moved past it, we're done. + */ + if (cjoin->iter->isequal && i == 0) + break; goto nextkey; + } skip_left = false; WT_ERR(ret); } @@ -783,12 +782,10 @@ __curjoin_close(WT_CURSOR *cursor) if (F_ISSET(entry, WT_CURJOIN_ENTRY_OWN_BLOOM)) WT_TRET(__wt_bloom_close(entry->bloom)); for (end = &entry->ends[0]; - end < &entry->ends[entry->ends_next]; end++) { + end < &entry->ends[entry->ends_next]; end++) F_CLR(end->cursor, WT_CURSTD_JOINED); - if (F_ISSET(end, WT_CURJOIN_END_OWN_KEY)) - __wt_free(session, end->key.data); - } __wt_free(session, entry->ends); + __wt_free(session, entry->repack_format); } if (cjoin->iter != NULL) @@ -891,22 +888,22 @@ __wt_curjoin_join(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, WT_INDEX *idx, WT_CURSOR *ref_cursor, uint8_t flags, uint8_t range, uint64_t count, uint32_t bloom_bit_count, uint32_t bloom_hash_count) { + WT_CURSOR_INDEX *cindex; + WT_CURSOR_JOIN_ENDPOINT *end, *newend; WT_CURSOR_JOIN_ENTRY *entry; WT_DECL_RET; - WT_CURSOR_JOIN_ENDPOINT *end, *newend; bool hasins, needbloom, range_eq; - u_int i, ins, nonbloom; + char *main_uri, *newformat; const char *raw_cfg[] = { WT_CONFIG_BASE( session, WT_SESSION_open_cursor), "raw", NULL }; - char *main_uri; - size_t namesize, newsize; + size_t len, newsize; + u_int i, ins, nonbloom; entry = NULL; hasins = needbloom = false; ins = 0; /* -Wuninitialized */ main_uri = NULL; nonbloom = 0; /* -Wuninitialized */ - namesize = strlen(cjoin->table->name); for (i = 0; i < cjoin->entries_next; i++) { if (cjoin->entries[i].index == idx) { @@ -982,13 +979,13 @@ __wt_curjoin_join(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, ((range & WT_CURJOIN_END_GT) != 0 || range_eq)) || (F_ISSET(end, WT_CURJOIN_END_LT) && ((range & WT_CURJOIN_END_LT) != 0 || range_eq)) || - (end->flags == WT_CURJOIN_END_EQ && + (WT_CURJOIN_END_RANGE(end) == WT_CURJOIN_END_EQ && (range & (WT_CURJOIN_END_LT | WT_CURJOIN_END_GT)) != 0)) WT_ERR_MSG(session, EINVAL, "join has overlapping ranges"); if (range == WT_CURJOIN_END_EQ && - end->flags == WT_CURJOIN_END_EQ && + WT_CURJOIN_END_RANGE(end) == WT_CURJOIN_END_EQ && !F_ISSET(entry, WT_CURJOIN_ENTRY_DISJUNCTION)) WT_ERR_MSG(session, EINVAL, "compare=eq can only be combined " @@ -1026,15 +1023,40 @@ __wt_curjoin_join(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, F_SET(newend, range); /* Open the main file with a projection of the indexed columns. */ - if (entry->main == NULL && entry->index != NULL) { - namesize = strlen(cjoin->table->name); - newsize = namesize + entry->index->colconf.len + 1; + if (entry->main == NULL && idx != NULL) { + newsize = strlen(cjoin->table->name) + idx->colconf.len + 1; WT_ERR(__wt_calloc(session, 1, newsize, &main_uri)); snprintf(main_uri, newsize, "%s%.*s", - cjoin->table->name, (int)entry->index->colconf.len, - entry->index->colconf.str); + cjoin->table->name, (int)idx->colconf.len, + idx->colconf.str); WT_ERR(__wt_open_cursor(session, main_uri, (WT_CURSOR *)cjoin, raw_cfg, &entry->main)); + if (idx->extractor == NULL) { + /* + * Add no-op padding so trailing 'u' formats are not + * transformed to 'U'. This matches what happens in + * the index. We don't do this when we have an + * extractor, extractors already use the padding + * byte trick. + */ + len = strlen(entry->main->value_format) + 3; + WT_ERR(__wt_calloc(session, len, 1, &newformat)); + snprintf(newformat, len, "%s0x", + entry->main->value_format); + __wt_free(session, entry->main->value_format); + entry->main->value_format = newformat; + } + + /* + * When we are repacking index keys to remove the primary + * key, we never want to transform trailing 'u'. Use no-op + * padding to force this. + */ + cindex = (WT_CURSOR_INDEX *)ref_cursor; + len = strlen(cindex->iface.key_format) + 3; + WT_ERR(__wt_calloc(session, len, 1, &entry->repack_format)); + snprintf(entry->repack_format, len, "%s0x", + cindex->iface.key_format); } err: if (main_uri != NULL) diff --git a/src/include/cursor.h b/src/include/cursor.h index 7f7b5dceb79..f9bd20c8ba1 100644 --- a/src/include/cursor.h +++ b/src/include/cursor.h @@ -289,7 +289,8 @@ struct __wt_cursor_join_iter { WT_CURSOR_JOIN_ENTRY *entry; WT_CURSOR *cursor; WT_ITEM *curkey; - bool advance; + bool positioned; + bool isequal; /* advancing means we're done */ }; struct __wt_cursor_join_endpoint { @@ -302,14 +303,17 @@ struct __wt_cursor_join_endpoint { #define WT_CURJOIN_END_GT 0x04 /* include values > cursor */ #define WT_CURJOIN_END_GE (WT_CURJOIN_END_GT | WT_CURJOIN_END_EQ) #define WT_CURJOIN_END_LE (WT_CURJOIN_END_LT | WT_CURJOIN_END_EQ) -#define WT_CURJOIN_END_OWN_KEY 0x08 /* must free key's data */ uint8_t flags; /* range for this endpoint */ }; +#define WT_CURJOIN_END_RANGE(endp) \ + ((endp)->flags & \ + (WT_CURJOIN_END_GT | WT_CURJOIN_END_EQ | WT_CURJOIN_END_LT)) struct __wt_cursor_join_entry { WT_INDEX *index; WT_CURSOR *main; /* raw main table cursor */ WT_BLOOM *bloom; /* Bloom filter handle */ + char *repack_format; /* target format for repack */ uint32_t bloom_bit_count; /* bits per item in bloom */ uint32_t bloom_hash_count; /* hash functions in bloom */ uint64_t count; /* approx number of matches */ diff --git a/src/include/extern.h b/src/include/extern.h index 64d46a5a254..92ae968affd 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -553,7 +553,7 @@ extern int __wt_struct_size(WT_SESSION_IMPL *session, size_t *sizep, const char extern int __wt_struct_pack(WT_SESSION_IMPL *session, void *buffer, size_t size, const char *fmt, ...); extern int __wt_struct_unpack(WT_SESSION_IMPL *session, const void *buffer, size_t size, const char *fmt, ...); extern int __wt_struct_unpack_size(WT_SESSION_IMPL *session, const void *buffer, size_t size, const char *fmt, size_t *resultp); -extern int __wt_struct_repack(WT_SESSION_IMPL *session, const char *infmt, const char *outfmt, const WT_ITEM *inbuf, WT_ITEM *outbuf, void **reallocp); +extern int __wt_struct_repack(WT_SESSION_IMPL *session, const char *infmt, const char *outfmt, const WT_ITEM *inbuf, WT_ITEM *outbuf); extern int __wt_ovfl_discard_add(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL *cell); extern void __wt_ovfl_discard_free(WT_SESSION_IMPL *session, WT_PAGE *page); extern int __wt_ovfl_reuse_search(WT_SESSION_IMPL *session, WT_PAGE *page, uint8_t **addrp, size_t *addr_sizep, const void *value, size_t value_size); diff --git a/src/packing/pack_impl.c b/src/packing/pack_impl.c index 0e3ed44ba6a..bd1c90525a6 100644 --- a/src/packing/pack_impl.c +++ b/src/packing/pack_impl.c @@ -144,33 +144,19 @@ __wt_struct_unpack_size(WT_SESSION_IMPL *session, */ int __wt_struct_repack(WT_SESSION_IMPL *session, const char *infmt, - const char *outfmt, const WT_ITEM *inbuf, WT_ITEM *outbuf, void **reallocp) + const char *outfmt, const WT_ITEM *inbuf, WT_ITEM *outbuf) { WT_DECL_PACK_VALUE(pvin); WT_DECL_PACK_VALUE(pvout); WT_DECL_RET; WT_PACK packin, packout; const uint8_t *before, *end, *p; - uint8_t *pout; - size_t len; const void *start; start = NULL; p = inbuf->data; end = p + inbuf->size; - /* - * Handle this non-contiguous case: 'U' -> 'u' at the end of the buf. - * The former case has the size embedded before the item, the latter - * does not. - */ - if ((len = strlen(outfmt)) > 1 && outfmt[len - 1] == 'u' && - strlen(infmt) > len && infmt[len - 1] == 'U') { - WT_ERR(__wt_realloc(session, NULL, inbuf->size, reallocp)); - pout = *reallocp; - } else - pout = NULL; - WT_ERR(__pack_init(session, &packout, outfmt)); WT_ERR(__pack_init(session, &packin, infmt)); @@ -178,22 +164,14 @@ __wt_struct_repack(WT_SESSION_IMPL *session, const char *infmt, while ((ret = __pack_next(&packout, &pvout)) == 0) { if (p >= end) WT_ERR(EINVAL); + if (pvout.type == 'x' && pvout.size == 0 && pvout.havesize) + continue; WT_ERR(__pack_next(&packin, &pvin)); before = p; WT_ERR(__unpack_read(session, &pvin, &p, (size_t)(end - p))); - if (pvout.type != pvin.type) { - if (pvout.type == 'u' && pvin.type == 'U') { - /* Skip the prefixed size, we don't need it */ - WT_ERR(__wt_struct_unpack_size(session, before, - (size_t)(end - before), "I", &len)); - before += len; - } else - WT_ERR(ENOTSUP); - } - if (pout != NULL) { - memcpy(pout, before, WT_PTRDIFF(p, before)); - pout += p - before; - } else if (start == NULL) + if (pvout.type != pvin.type) + WT_ERR(ENOTSUP); + if (start == NULL) start = before; } WT_ERR_NOTFOUND_OK(ret); @@ -201,13 +179,8 @@ __wt_struct_repack(WT_SESSION_IMPL *session, const char *infmt, /* Be paranoid - __pack_write should never overflow. */ WT_ASSERT(session, p <= end); - if (pout != NULL) { - outbuf->data = *reallocp; - outbuf->size = WT_PTRDIFF(pout, *reallocp); - } else { - outbuf->data = start; - outbuf->size = WT_PTRDIFF(p, start); - } + outbuf->data = start; + outbuf->size = WT_PTRDIFF(p, start); err: return (ret); } -- cgit v1.2.1 From 5b005b336d8d23cba166c2f5ae2ee12b2fa151fe Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Tue, 9 Feb 2016 16:16:27 -0500 Subject: WT-2382. Modified test to use collators with joins in the future. The scenario to run the collator test is currently disabled, it needs the revint collator (see pull request #2473 to solve WT-2375). --- test/suite/test_join03.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/test/suite/test_join03.py b/test/suite/test_join03.py index 613d2396b07..d7b5925f922 100644 --- a/test/suite/test_join03.py +++ b/test/suite/test_join03.py @@ -37,6 +37,11 @@ class test_join03(wttest.WiredTigerTestCase): table_name1 = 'test_join03' nentries = 100 + scenarios = [ + ('no-collator', dict(collate=False)), +# ('collator', dict(collate=True)) + ] + # Return the wiredtiger_open extension argument for a shared library. def extensionArg(self, exts): extfiles = [] @@ -58,7 +63,9 @@ class test_join03(wttest.WiredTigerTestCase): # Override WiredTigerTestCase, we have extensions. def setUpConnectionOpen(self, dir): - extarg = self.extensionArg([('extractors', 'csv', 'csv_extractor')]) + extarg = self.extensionArg([ + ('extractors', 'csv', 'csv_extractor'), + ('collators', 'revint', 'revint_collator')]) connarg = 'create,error_prefix="{0}: ",{1}'.format( self.shortid(), extarg) conn = self.wiredtiger_open(dir, connarg) @@ -90,14 +97,18 @@ class test_join03(wttest.WiredTigerTestCase): # Common function for testing the most basic functionality # of joins def join(self, csvformat, args0, args1): - self.session.create('table:join03', 'key_format=r' + + collator = 'collator=revint,' if self.collate else '' + + self.session.create('table:join03', 'key_format=i' + ',value_format=S,columns=(k,v)') fmt = csvformat[0] self.session.create('index:join03:index0','key_format=' + fmt + ',' + + (collator if fmt == 'i' else '') + 'extractor=csv,app_metadata={"format" : "' + fmt + '","field" : "0"}') fmt = csvformat[1] self.session.create('index:join03:index1','key_format=' + fmt + ',' + + (collator if fmt == 'i' else '') + 'extractor=csv,app_metadata={"format" : "' + fmt + '","field" : "1"}') -- cgit v1.2.1 From a5e3fa9c24987840863cf693837f9174ee7e00e4 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 9 Feb 2016 16:22:22 -0500 Subject: WT-2349 Add mis-matched configuration test. Fix a problem in recovery it found. --- src/txn/txn_recover.c | 14 +++++ test/suite/test_readonly01.py | 1 - test/suite/test_readonly02.py | 118 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 test/suite/test_readonly02.py diff --git a/src/txn/txn_recover.c b/src/txn/txn_recover.c index 029c0d7522c..8c34f3d9393 100644 --- a/src/txn/txn_recover.c +++ b/src/txn/txn_recover.c @@ -449,6 +449,20 @@ __wt_txn_recover(WT_SESSION_IMPL *session) */ if (!was_backup) { r.metadata_only = true; + /* + * If this is a read-only connection, check if the checkpoint + * LSN in the metadata file is up to date, indicating a clean + * shutdown. + */ + if (F_ISSET(conn, WT_CONN_READONLY)) { + WT_ERR(__wt_log_needs_recovery( + session, &metafile->ckpt_lsn, &needs_rec)); + if (needs_rec) { + WT_ERR_MSG(session, WT_RUN_RECOVERY, + "Read-only database needs recovery"); + WT_ERR(WT_RUN_RECOVERY); + } + } if (WT_IS_INIT_LSN(&metafile->ckpt_lsn)) WT_ERR(__wt_log_scan(session, NULL, WT_LOGSCAN_FIRST, __txn_log_recover, &r)); diff --git a/test/suite/test_readonly01.py b/test/suite/test_readonly01.py index 4cc65405301..f37c90876a0 100644 --- a/test/suite/test_readonly01.py +++ b/test/suite/test_readonly01.py @@ -39,7 +39,6 @@ class test_readonly01(wttest.WiredTigerTestCase, suite_subprocess): tablename = 'test_readonly01' create = True entries = 10000 - extra_entries = 5 # # We want a list of directory writable or readonly. diff --git a/test/suite/test_readonly02.py b/test/suite/test_readonly02.py new file mode 100644 index 00000000000..b195ab60cf9 --- /dev/null +++ b/test/suite/test_readonly02.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +# +# Public Domain 2026-2026 MongoDB, Inc. +# Public Domain 2008-2026 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_readonly02.py +# Readonly: Test readonly mode with illegal config combinations +# and error checking during updates. +# + +from helper import copy_wiredtiger_home +from suite_subprocess import suite_subprocess +import os, wiredtiger, wttest + +class test_readonly02(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'table:test_readonly02' + create = True + create_params = 'key_format=i,value_format=i' + entries = 10 + + conn_params = \ + 'create,statistics=(fast),log=(enabled,file_max=100K,zero_fill=true),' + conn_params_rd = \ + 'create,readonly=true,statistics=(fast),log=(enabled,zero_fill=false),' + conn_params_rdcfg = \ + 'create,readonly=true,statistics=(fast),log=(enabled),' + + # + # Run to make sure incompatible configuration options return an error. + # The situations that cause failures (instead of silent overrides) are: + # 1. setting readonly on a new database directory + # 2. an unclean shutdown and reopening readonly + # 3. logging with zero-fill enabled and readonly + # 4. readonly and statistics logging + # + badcfg1 = 'log=(enabled,zero_fill=true)' + badcfg2 = 'statistics_log=(wait=3)' + + def setUpConnectionOpen(self, dir): + self.home = dir + rdonlydir = dir + '.rdonly' + # + # First time through check readonly on a non-existent database. + # + if self.create: + # 1. setting readonly on a new database directory + # Setting readonly prevents creation so we should see an + # ENOENT error because the lock file does not exist. + msg = '/No such file/' + os.mkdir(rdonlydir) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.wiredtiger_open( + rdonlydir, self.conn_params_rd), msg) + + self.create = False + conn = self.wiredtiger_open(dir, self.conn_params) + return conn + + def check_unclean(self): + backup = "WT_COPYDIR" + copy_wiredtiger_home(self.home, backup, True) + msg = '/needs recovery/' + # 2. an unclean shutdown and reopening readonly + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.wiredtiger_open(backup, self.conn_params_rd), msg) + + def close_checkerror(self, cfg): + ''' Close the connection and reopen readonly''' + # + # Close the original connection. Reopen readonly and also with + # the given + # + self.close_conn() + conn_params = self.conn_params_rd + cfg + msg = '/Invalid argument/' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.wiredtiger_open(self.home, conn_params), msg) + + def test_readonly(self): + tablearg = self.tablename + self.session.create(tablearg, self.create_params) + c = self.session.open_cursor(tablearg, None, None) + for i in range(self.entries): + c[i+1] = i % 255 + # Check for an error on an unclean recovery/restart. + self.check_unclean() + + # Close the connection. Reopen readonly with other bad settings. + # 3. logging with zero-fill enabled and readonly + self.close_checkerror(self.badcfg1) + # 4. readonly and statistics logging + self.close_checkerror(self.badcfg2) + +if __name__ == '__main__': + wttest.run() -- cgit v1.2.1 From 5c872815b1f6b42f73056ee9d837b0ceafcd74d7 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 9 Feb 2016 16:31:22 -0500 Subject: WT-2349 Fix comment in test. --- test/suite/test_readonly02.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/suite/test_readonly02.py b/test/suite/test_readonly02.py index b195ab60cf9..63504d35ac2 100644 --- a/test/suite/test_readonly02.py +++ b/test/suite/test_readonly02.py @@ -91,7 +91,7 @@ class test_readonly02(wttest.WiredTigerTestCase, suite_subprocess): ''' Close the connection and reopen readonly''' # # Close the original connection. Reopen readonly and also with - # the given + # the given configuration string. # self.close_conn() conn_params = self.conn_params_rd + cfg -- cgit v1.2.1 From cdcea07d2b98ec7c33cde3b0f60eff68e0231470 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Wed, 10 Feb 2016 21:56:36 +1100 Subject: WT-2390 Fix OS X warning. src/conn/conn_log.c:636:16: error: implicit conversion loses integer precision: 'wt_off_t' (aka 'long long') to 'uint32_t' (aka 'unsigned int') [-Werror,-Wshorten-64-to-32] slot->slot_last_offset; --- src/conn/conn_log.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/conn/conn_log.c b/src/conn/conn_log.c index 60f46288072..fa3928a8539 100644 --- a/src/conn/conn_log.c +++ b/src/conn/conn_log.c @@ -633,7 +633,7 @@ restart: if (slot->slot_start_lsn.l.offset != slot->slot_last_offset) slot->slot_start_lsn.l.offset = - slot->slot_last_offset; + (uint32_t)slot->slot_last_offset; log->write_start_lsn = slot->slot_start_lsn; log->write_lsn = slot->slot_end_lsn; WT_ERR(__wt_cond_signal( -- cgit v1.2.1 From c09b6acc7ad97f202d57bf966e12aac10a59f04d Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Wed, 10 Feb 2016 13:15:44 -0500 Subject: WT-2349 Add test testing methods not supported. Added error message to unsupported options and fixed several tests to check for that message. --- src/cursor/cur_std.c | 5 +- src/include/txn.i | 2 + src/session/session_api.c | 20 ++++--- test/suite/test_checkpoint01.py | 10 +++- test/suite/test_cursor06.py | 5 +- test/suite/test_cursor_random.py | 24 +++++--- test/suite/test_join01.py | 9 +-- test/suite/test_readonly02.py | 4 +- test/suite/test_readonly03.py | 124 +++++++++++++++++++++++++++++++++++++++ 9 files changed, 173 insertions(+), 30 deletions(-) create mode 100644 test/suite/test_readonly03.py diff --git a/src/cursor/cur_std.c b/src/cursor/cur_std.c index 899df8a07cc..ec5a2e88331 100644 --- a/src/cursor/cur_std.c +++ b/src/cursor/cur_std.c @@ -15,9 +15,10 @@ int __wt_cursor_notsup(WT_CURSOR *cursor) { - WT_UNUSED(cursor); + WT_SESSION_IMPL *session; - return (ENOTSUP); + session = (WT_SESSION_IMPL *)cursor->session; + WT_RET_MSG(session, ENOTSUP, "Unsupported cursor operation"); } /* diff --git a/src/include/txn.i b/src/include/txn.i index 46f2ff3e5f1..40e2a6175d6 100644 --- a/src/include/txn.i +++ b/src/include/txn.i @@ -266,6 +266,8 @@ __wt_txn_begin(WT_SESSION_IMPL *session, const char *cfg[]) } F_SET(txn, WT_TXN_RUNNING); + if (F_ISSET(S2C(session), WT_CONN_READONLY)) + F_SET(txn, WT_TXN_READONLY); return (false); } diff --git a/src/session/session_api.c b/src/session/session_api.c index ea524b12ada..e2723c660fe 100644 --- a/src/session/session_api.c +++ b/src/session/session_api.c @@ -20,10 +20,12 @@ int __wt_session_notsup_cfg( WT_SESSION *wt_session, const char *config) { - WT_UNUSED(wt_session); + WT_SESSION_IMPL *session; + + session = (WT_SESSION_IMPL *)wt_session; WT_UNUSED(config); - return (ENOTSUP); + WT_RET_MSG(session, ENOTSUP, "Unsupported session method"); } /* @@ -35,11 +37,13 @@ int __wt_session_notsup_uri( WT_SESSION *wt_session, const char *uri, const char *config) { - WT_UNUSED(wt_session); + WT_SESSION_IMPL *session; + + session = (WT_SESSION_IMPL *)wt_session; WT_UNUSED(uri); WT_UNUSED(config); - return (ENOTSUP); + WT_RET_MSG(session, ENOTSUP, "Unsupported session method"); } /* @@ -577,7 +581,7 @@ __session_log_printf(WT_SESSION *wt_session, const char *fmt, ...) SESSION_API_CALL_NOCONF(session, log_printf); if (F_ISSET(S2C(session), WT_CONN_READONLY)) - WT_ERR(ENOTSUP); + WT_ERR_MSG(session, ENOTSUP, "Unsupported session method"); va_start(ap, fmt); ret = __wt_log_vprintf(session, fmt, ap); @@ -601,7 +605,7 @@ __session_rebalance(WT_SESSION *wt_session, const char *uri, const char *config) SESSION_API_CALL(session, rebalance, config, cfg); if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY | WT_CONN_READONLY)) - WT_ERR(ENOTSUP); + WT_ERR_MSG(session, ENOTSUP, "Unsupported session method"); /* Block out checkpoints to avoid spurious EBUSY errors. */ WT_WITH_CHECKPOINT_LOCK(session, ret, @@ -627,7 +631,7 @@ __session_rename(WT_SESSION *wt_session, SESSION_API_CALL(session, rename, config, cfg); if (F_ISSET(S2C(session), WT_CONN_READONLY)) - WT_ERR(ENOTSUP); + WT_ERR_MSG(session, ENOTSUP, "Unsupported session method"); /* Disallow objects in the WiredTiger name space. */ WT_ERR(__wt_str_name_check(session, uri)); @@ -990,7 +994,7 @@ __session_truncate(WT_SESSION *wt_session, WT_STAT_FAST_CONN_INCR(session, cursor_truncate); if (F_ISSET(S2C(session), WT_CONN_READONLY)) - WT_ERR(ENOTSUP); + WT_ERR_MSG(session, ENOTSUP, "Unsupported session method"); /* * If the URI is specified, we don't need a start/stop, if start/stop diff --git a/test/suite/test_checkpoint01.py b/test/suite/test_checkpoint01.py index 7d4503b84b7..36f1ef733a4 100644 --- a/test/suite/test_checkpoint01.py +++ b/test/suite/test_checkpoint01.py @@ -265,9 +265,13 @@ class test_checkpoint_cursor_update(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt") cursor.set_key(key_populate(cursor, 10)) cursor.set_value("XXX") - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.insert()) - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.remove()) - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.update()) + msg = "/not supported/" + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: cursor.insert(), msg) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: cursor.remove(), msg) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: cursor.update(), msg) cursor.close() diff --git a/test/suite/test_cursor06.py b/test/suite/test_cursor06.py index ff7c1144344..d702f97c5dd 100644 --- a/test/suite/test_cursor06.py +++ b/test/suite/test_cursor06.py @@ -89,10 +89,11 @@ class test_cursor06(wttest.WiredTigerTestCase): self.session.drop(uri, "force") self.populate(uri) cursor = self.session.open_cursor(uri, None, open_config) + msg = '/not supported/' if open_config == "readonly=1": self.set_kv(cursor) - self.assertRaises(wiredtiger.WiredTigerError, - lambda: cursor.update()) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: cursor.update(), msg) else: self.set_kv(cursor) cursor.update() diff --git a/test/suite/test_cursor_random.py b/test/suite/test_cursor_random.py index 1fd30d93c11..cd91a925b0c 100644 --- a/test/suite/test_cursor_random.py +++ b/test/suite/test_cursor_random.py @@ -51,15 +51,21 @@ class test_cursor_random(wttest.WiredTigerTestCase): uri = self.type self.session.create(uri, 'key_format=S,value_format=S') cursor = self.session.open_cursor(uri, None, self.config) - self.assertRaises( - wiredtiger.WiredTigerError, lambda: cursor.compare(cursor)) - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.insert()) - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.prev()) - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.remove()) - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.search()) - self.assertRaises( - wiredtiger.WiredTigerError, lambda: cursor.search_near()) - self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.update()) + msg = "/not supported/" + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda: cursor.compare(cursor), msg) + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda: cursor.insert(), msg) + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda: cursor.prev(), msg) + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda: cursor.remove(), msg) + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda: cursor.search(), msg) + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda: cursor.search_near(), msg) + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, lambda: cursor.update(), msg) self.assertTrue(cursor.next(), wiredtiger.WT_NOTFOUND) self.assertEquals(cursor.reconfigure(), 0) diff --git a/test/suite/test_join01.py b/test/suite/test_join01.py index f03c7c6f06c..ee4d6e22870 100644 --- a/test/suite/test_join01.py +++ b/test/suite/test_join01.py @@ -342,11 +342,12 @@ class test_join01(wttest.WiredTigerTestCase): '/index cursor is being used in a join/') # Only a small number of operations allowed on a join cursor - self.assertRaises(wiredtiger.WiredTigerError, - lambda: jc.search()) + msg = "/not supported/" + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: jc.search(), msg) - self.assertRaises(wiredtiger.WiredTigerError, - lambda: jc.prev()) + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: jc.prev(), msg) self.assertEquals(jc.next(), 0) self.assertEquals(jc.next(), wiredtiger.WT_NOTFOUND) diff --git a/test/suite/test_readonly02.py b/test/suite/test_readonly02.py index 63504d35ac2..e94dd85857d 100644 --- a/test/suite/test_readonly02.py +++ b/test/suite/test_readonly02.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -# Public Domain 2026-2026 MongoDB, Inc. -# Public Domain 2008-2026 WiredTiger, Inc. +# Public Domain 2016-2016 MongoDB, Inc. +# Public Domain 2008-2016 WiredTiger, Inc. # # This is free and unencumbered software released into the public domain. # diff --git a/test/suite/test_readonly03.py b/test/suite/test_readonly03.py new file mode 100644 index 00000000000..e7932839231 --- /dev/null +++ b/test/suite/test_readonly03.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# +# Public Domain 2016-2016 MongoDB, Inc. +# Public Domain 2008-2016 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_readonly03.py +# Readonly: Test connection readonly mode with modifying methods. Confirm +# all return ENOTSUP. +# + +from helper import simple_populate +from suite_subprocess import suite_subprocess +import os, sys, wiredtiger, wttest + +class test_readonly03(wttest.WiredTigerTestCase, suite_subprocess): + uri = 'table:test_readonly03' + uri2 = 'table:test_readonly03_2' + create = True + + conn_params = 'create,log=(enabled),' + conn_params_rd = 'readonly=true' + + session_ops = [ 'create', 'compact', 'drop', 'log_flush', 'log_printf', + 'rebalance', 'rename', 'salvage', 'truncate', 'upgrade', ] + cursor_ops = [ 'insert', 'remove', 'update', ] + + def setUpConnectionOpen(self, dir): + self.home = dir + if self.create: + conn_cfg = self.conn_params + else: + conn_cfg = self.conn_params_rd + conn = self.wiredtiger_open(dir, conn_cfg) + self.create = False + return conn + + + def test_readonly(self): + create_params = 'key_format=i,value_format=i' + entries = 10 + # Create a database and a table. + simple_populate(self, self.uri, create_params, entries) + + # + # Now close and reopen. Note that the connection function + # above will reopen it readonly. + self.reopen_conn() + msg = '/not supported/' + c = self.session.open_cursor(self.uri, None, None) + for op in self.cursor_ops: + c.set_key(1) + c.set_value(1) + if op == 'insert': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: c.insert(), msg) + elif op == 'remove': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: c.remove(), msg) + elif op == 'update': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: c.update(), msg) + else: + self.fail('Unknown cursor operation: ' + op) + c.close() + for op in self.session_ops: + if op == 'create': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.create(self.uri2, create_params), + msg) + elif op == 'compact': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.compact(self.uri, None), msg) + elif op == 'drop': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.drop(self.uri, None), msg) + elif op == 'log_flush': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.log_flush(None), msg) + elif op == 'log_printf': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.log_printf("test"), msg) + elif op == 'rebalance': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.rebalance(self.uri, None), msg) + elif op == 'rename': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.rename(self.uri, self.uri2, None), msg) + elif op == 'salvage': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.salvage(self.uri, None), msg) + elif op == 'truncate': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.truncate(self.uri, None, None, None), msg) + elif op == 'upgrade': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.upgrade(self.uri, None), msg) + else: + self.fail('Unknown session method: ' + op) + +if __name__ == '__main__': + wttest.run() -- cgit v1.2.1 From 77cf87708750ad86cf16ae5eb88899039979b1be Mon Sep 17 00:00:00 2001 From: Mark Benvenuto Date: Sat, 6 Feb 2016 12:25:15 -0500 Subject: WT-2280 CRC32 for POWER8 --- src/docs/license.dox | 13 +- src/support/power8/LICENSE.TXT | 476 ++++++++++++++++++ src/support/power8/README.md | 208 ++++++++ src/support/power8/crc32.S | 739 ++++++++++++++++++++++++++++ src/support/power8/crc32_constants.h | 901 +++++++++++++++++++++++++++++++++++ src/support/power8/crc32_wrapper.c | 64 +++ src/support/power8/ppc-opcode.h | 23 + 7 files changed, 2422 insertions(+), 2 deletions(-) create mode 100644 src/support/power8/LICENSE.TXT create mode 100644 src/support/power8/README.md create mode 100644 src/support/power8/crc32.S create mode 100644 src/support/power8/crc32_constants.h create mode 100644 src/support/power8/crc32_wrapper.c create mode 100644 src/support/power8/ppc-opcode.h diff --git a/src/docs/license.dox b/src/docs/license.dox index febced2c6af..fa78fa1e02c 100644 --- a/src/docs/license.dox +++ b/src/docs/license.dox @@ -6,12 +6,12 @@ are welcome to modify and redistribute it under the terms of version 2 or version 3 of the -GNU General Public License +GNU General Public License as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for details. +GNU General Public License for details. Additionally, portions of the WiredTiger distribution are distributed under the terms of the @@ -46,6 +46,15 @@ of the WiredTiger library should comply with these copyrights. @row{\c src/support/hash_fnv.c, Authors, Public Domain} +@section license_crc32-vpmsum 3rd party software included in the WiredTiger library on PPC64: crc32-vpmsum + +The PPC64 and PPC64LE builds of WiredTiger include the 3rd party software component "crc32-vpmsum" to +use POWER8 hardware acceleration for CRC32. This library is dual-licensed +under GNU General Public License + version 2 or later +and +Apache License 2.0 . See src/support/power8/LICENSE.TXT. + @section license_leveldb 3rd party software optionally included in the WiredTiger library: LevelDB If the \c --enable-leveldb configuration option is specified when diff --git a/src/support/power8/LICENSE.TXT b/src/support/power8/LICENSE.TXT new file mode 100644 index 00000000000..2f4bb91f574 --- /dev/null +++ b/src/support/power8/LICENSE.TXT @@ -0,0 +1,476 @@ +Copyright (C) 2015 Anton Blanchard , IBM + +crc32-vpmsum is free software; you can redistribute it and/or +modify it under the terms of either: + + a) the GNU General Public License as published by the Free Software Foundation; either + version 2 of the License, or (at your option) any later version., or + b) the Apache License, Version 2.0 + + + + + + + + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + + + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/src/support/power8/README.md b/src/support/power8/README.md new file mode 100644 index 00000000000..3e2976650cd --- /dev/null +++ b/src/support/power8/README.md @@ -0,0 +1,208 @@ +crc32-vpmsum +============ + +A set of examples for accelerating CRC32 calculations using the vector +polynomial multiply sum (vpmsum) instructions introduced in POWER8. These +instructions implement byte, halfword, word and doubleword carryless +multiply/add. + +Performance +----------- + +An implementation of slice-by-8, one of the fastest lookup table methods +is included so we can compare performance against it. Testing 5000000 +iterations of a CRC of 32 kB of data (to keep it L1 cache contained): + +``` +# time slice_by_8_bench 32768 5000000 +122.220 seconds + +# time crc32_bench 32768 5000000 +2.937 seconds +``` + +The vpmsum accelerated CRC is just over 41x faster. + +This test was run on a 4.1 GHz POWER8, so the algorithm sustains about +52 GiB/sec or 13.6 bytes/cycle. The theoretical limit is 16 bytes/cycle +since we can execute a maximum of one vpmsum instruction per cycle. + +In another test, a version was added to the kernel and btrfs write +performance was shown to be 3.8x faster. The test was done to a ramdisk +to mitigate any I/O induced variability. + +Quick start +----------- + +- Modify CRC and OPTIONS in the Makefile. There are examples for the two most + common crc32s. + +- Type make to create the constants (crc32_constants.h) + +- Import the code into your application (crc32.S crc32_wrapper.c + crc32_constants.h ppc-opcode.h) and call the CRC: + +``` +unsigned int crc32_vpmsum(unsigned int crc, unsigned char *p, unsigned long len); +``` + +CRC background +-------------- + +For a good background on CRCs, check out: + +http://www.ross.net/crc/download/crc_v3.txt + +A few key points: + +- A CRC is the remainder after dividing a message by the CRC polynomial, + ie M mod CRC_POLY +- multiply/divide is carryless +- add/subtract is an xor +- n (where n is the order of the CRC) bits of zeroes are appended to the + end of the message. + +One more important piece of information - a CRC is a linear function, so: + +``` + CRC(A xor B) = CRC(A) xor CRC(B) + + CRC(A . B) = CRC(A) . CRC(B) (remember this is carryless multiply) +``` + +If we take 64bits of data, represented by two 32 bit chunks (AAAAAAAA +and BBBBBBBB): + +``` +CRC(AAAAAAAABBBBBBBB) + = CRC(AAAAAAAA00000000 xor BBBBBBBB) + = CRC(AAAAAAAA00000000) xor CRC(BBBBBBBB) +``` + +If we operate on AAAAAAAA: + +``` +CRC(AAAAAAAA00000000) + = CRC(AAAAAAAA . 100000000) + = CRC(AAAAAAAA) . CRC(100000000) +``` + +And CRC(100000000) is a constant which we can pre-calculate: + +``` +CRC(100000000) + = 100000000 mod CRC_POLY + = 2^32 mod CRC_POLY +``` + +Finally we can add our modified AAAAAAAA to BBBBBBBB: + +``` +CRC(AAAAAAAABBBBBBBB) + = ((2^32 mod CRC_POLY) . CRC(AAAAAAAA)) xor CRC(BBBBBBBB) +``` + +In other words, with the right constants pre-calculated we can shift the +input data around and we can also calculate the CRC in as many parallel +chunks as we want. + +No matter how much shifting we do, the final result will be be 64 bits of +data (63 actually, because there is no carry into the top bit). To reduce +it further we need a another trick, and that is Barrett reduction: + +http://en.wikipedia.org/wiki/Barrett_reduction + +Barrett reduction is a method of calculating a mod n. The idea is to +calculate q, the multiple of our polynomial that we need to subtract. By +doing the computation 2x bits higher (ie 64 bits) and shifting the +result back down 2x bits, we round down to the nearest multiple. + +``` + k = 32 + m = floor((4^k)/n) = floor((4^32))/n) + n = 64 bits of data + a = 32 bit CRC + + q = floor(ma/(2^64)) + result = a - qn +``` + +An example in the floating point domain makes it clearer how this works: + +``` +a mod n = a - floor(am) * n +``` + +Let's use it to calculate 22 mod 10: + +``` + a = 22 + n = 10 + m = 1/n = 1/10 = 0.1 + +22 mod 10 + = 22 - floor(22*0.1) * 10 + = 22 - 2 * 10 + = 22 - 20 + = 2 +``` + +There is one more issue left - bit reflection. Some CRCs are defined to +operate on the least significant bit first (eg CRC32c). Lets look at +how this would get laid out in a register, and lets simplify it to just +two bytes (vs a 16 byte VMX register): + + [ 8..15 ] [ 0..7 ] + +Notice how the bits and bytes are out of order. Since we are doing +multi word multiplication on these values we need them to both be +in order. + +The simplest way to fix this is to reflect the bits in each byte: + + [ 15..8 ] [ 7..0 ] + +However shuffling bits in a byte is expensive on most CPUs. It is +however relatively cheap to shuffle bytes around. What if we load +the bytes in reversed: + + [ 0..7 ] [ 8..15 ] + +Now the bits and bytes are in order, except the least significant bit +of the register is now on the left and the most significant bit is on the +right. We operate as if the register is reflected, which normally we +cannot do. The reason we get away with this is our multiplies are carryless +and our addition and subtraction is xor, so our operations never create +carries. + +The only trick is we have to shift the result of multiplies left one +because the high bit of the multiply is always 0, and we want that high bit +on the right not the left. + +Implementation +-------------- + +The vpmsum instructions on POWER8 have a 6 cycle latency and we can +execute one every cycle. In light of this the main loop has 8 parallel +streams which consume 8 x 16 B each iteration. At the completion of this +loop we have taken 32 kB of data and reduced it to 8 x 16 B (128 B). + +The next step is to take this 128 B and reduce it to 8 B. At this stage +we also add 32 bits of 0 to the end. + +We then apply Barrett reduction to get our CRC. + +Examples +-------- +- barrett_reduction: An example of Barrett reduction + +- final_fold: Starting with 128 bits, add 32 bits of zeros and reduce it to + 64 bits, then apply Barrett reduction + +- final_fold2: A second method of reduction + +Acknowledgements +---------------- + +Thanks to Michael Gschwind, Jeff Derby, Lorena Pesantez and Stewart Smith +for their ideas and assistance. diff --git a/src/support/power8/crc32.S b/src/support/power8/crc32.S new file mode 100644 index 00000000000..621125b67c6 --- /dev/null +++ b/src/support/power8/crc32.S @@ -0,0 +1,739 @@ +/* + * Calculate the checksum of data that is 16 byte aligned and a multiple of + * 16 bytes. + * + * The first step is to reduce it to 1024 bits. We do this in 8 parallel + * chunks in order to mask the latency of the vpmsum instructions. If we + * have more than 32 kB of data to checksum we repeat this step multiple + * times, passing in the previous 1024 bits. + * + * The next step is to reduce the 1024 bits to 64 bits. This step adds + * 32 bits of 0s to the end - this matches what a CRC does. We just + * calculate constants that land the data in this 32 bits. + * + * We then use fixed point Barrett reduction to compute a mod n over GF(2) + * for n = CRC using POWER8 instructions. We use x = 32. + * + * http://en.wikipedia.org/wiki/Barrett_reduction + * + * Copyright (C) 2015 Anton Blanchard , IBM + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + */ +#include +#include "ppc-opcode.h" + +#undef toc + +#ifndef r1 +#define r1 1 +#endif + +#ifndef r2 +#define r2 2 +#endif + + .section .rodata +.balign 16 + +.byteswap_constant: + /* byte reverse permute constant */ + .octa 0x0F0E0D0C0B0A09080706050403020100 + +#define __ASSEMBLY__ +#include "crc32_constants.h" + + .text + +#if defined(__BIG_ENDIAN__) && defined(REFLECT) +#define BYTESWAP_DATA +#elif defined(__LITTLE_ENDIAN__) && !defined(REFLECT) +#define BYTESWAP_DATA +#else +#undef BYTESWAP_DATA +#endif + +#define off16 r25 +#define off32 r26 +#define off48 r27 +#define off64 r28 +#define off80 r29 +#define off96 r30 +#define off112 r31 + +#define const1 v25 +#define const2 v26 + +#define byteswap v27 +#define mask_32bit v28 +#define mask_64bit v29 +#define zeroes v30 +#define ones v31 + +#ifdef BYTESWAP_DATA +#define VPERM(A, B, C, D) vperm A, B, C, D +#else +#define VPERM(A, B, C, D) +#endif + +/* unsigned int __crc32_vpmsum(unsigned int crc, void *p, unsigned long len) */ +FUNC_START(__crc32_vpmsum) + std r31,-8(r1) + std r30,-16(r1) + std r29,-24(r1) + std r28,-32(r1) + std r27,-40(r1) + std r26,-48(r1) + std r25,-56(r1) + + li off16,16 + li off32,32 + li off48,48 + li off64,64 + li off80,80 + li off96,96 + li off112,112 + li r0,0 + + mr r10,r3 + + vxor zeroes,zeroes,zeroes + vspltisw ones,-1 + + vsldoi mask_32bit,zeroes,ones,4 + vsldoi mask_64bit,zeroes,ones,8 + + /* Get the initial value into v8 */ + vxor v8,v8,v8 + MTVRD(v8, r3) +#ifdef REFLECT + vsldoi v8,zeroes,v8,8 /* shift into bottom 32 bits */ +#else + vsldoi v8,v8,zeroes,4 /* shift into top 32 bits */ +#endif + +#ifdef BYTESWAP_DATA + addis r3,r2,.byteswap_constant@toc@ha + addi r3,r3,.byteswap_constant@toc@l + + lvx byteswap,0,r3 + addi r3,r3,16 +#endif + + cmpdi r5,256 + blt .Lshort + + rldicr r6,r5,0,56 + + /* Checksum in blocks of MAX_SIZE */ +1: lis r7,MAX_SIZE@h + ori r7,r7,MAX_SIZE@l + mr r9,r7 + cmpd r6,r7 + bgt 2f + mr r7,r6 +2: subf r6,r7,r6 + + /* our main loop does 128 bytes at a time */ + srdi r7,r7,7 + + /* + * Work out the offset into the constants table to start at. Each + * constant is 16 bytes, and it is used against 128 bytes of input + * data - 128 / 16 = 8 + */ + sldi r8,r7,4 + srdi r9,r9,3 + subf r8,r8,r9 + + /* We reduce our final 128 bytes in a separate step */ + addi r7,r7,-1 + mtctr r7 + + addis r3,r2,.constants@toc@ha + addi r3,r3,.constants@toc@l + + /* Find the start of our constants */ + add r3,r3,r8 + + /* zero v0-v7 which will contain our checksums */ + vxor v0,v0,v0 + vxor v1,v1,v1 + vxor v2,v2,v2 + vxor v3,v3,v3 + vxor v4,v4,v4 + vxor v5,v5,v5 + vxor v6,v6,v6 + vxor v7,v7,v7 + + lvx const1,0,r3 + + /* + * If we are looping back to consume more data we use the values + * already in v16-v23. + */ + cmpdi r0,1 + beq 2f + + /* First warm up pass */ + lvx v16,0,r4 + lvx v17,off16,r4 + VPERM(v16,v16,v16,byteswap) + VPERM(v17,v17,v17,byteswap) + lvx v18,off32,r4 + lvx v19,off48,r4 + VPERM(v18,v18,v18,byteswap) + VPERM(v19,v19,v19,byteswap) + lvx v20,off64,r4 + lvx v21,off80,r4 + VPERM(v20,v20,v20,byteswap) + VPERM(v21,v21,v21,byteswap) + lvx v22,off96,r4 + lvx v23,off112,r4 + VPERM(v22,v22,v22,byteswap) + VPERM(v23,v23,v23,byteswap) + addi r4,r4,8*16 + + /* xor in initial value */ + vxor v16,v16,v8 + +2: bdz .Lfirst_warm_up_done + + addi r3,r3,16 + lvx const2,0,r3 + + /* Second warm up pass */ + VPMSUMD(v8,v16,const1) + lvx v16,0,r4 + VPERM(v16,v16,v16,byteswap) + ori r2,r2,0 + + VPMSUMD(v9,v17,const1) + lvx v17,off16,r4 + VPERM(v17,v17,v17,byteswap) + ori r2,r2,0 + + VPMSUMD(v10,v18,const1) + lvx v18,off32,r4 + VPERM(v18,v18,v18,byteswap) + ori r2,r2,0 + + VPMSUMD(v11,v19,const1) + lvx v19,off48,r4 + VPERM(v19,v19,v19,byteswap) + ori r2,r2,0 + + VPMSUMD(v12,v20,const1) + lvx v20,off64,r4 + VPERM(v20,v20,v20,byteswap) + ori r2,r2,0 + + VPMSUMD(v13,v21,const1) + lvx v21,off80,r4 + VPERM(v21,v21,v21,byteswap) + ori r2,r2,0 + + VPMSUMD(v14,v22,const1) + lvx v22,off96,r4 + VPERM(v22,v22,v22,byteswap) + ori r2,r2,0 + + VPMSUMD(v15,v23,const1) + lvx v23,off112,r4 + VPERM(v23,v23,v23,byteswap) + + addi r4,r4,8*16 + + bdz .Lfirst_cool_down + + /* + * main loop. We modulo schedule it such that it takes three iterations + * to complete - first iteration load, second iteration vpmsum, third + * iteration xor. + */ + .balign 16 +4: lvx const1,0,r3 + addi r3,r3,16 + ori r2,r2,0 + + vxor v0,v0,v8 + VPMSUMD(v8,v16,const2) + lvx v16,0,r4 + VPERM(v16,v16,v16,byteswap) + ori r2,r2,0 + + vxor v1,v1,v9 + VPMSUMD(v9,v17,const2) + lvx v17,off16,r4 + VPERM(v17,v17,v17,byteswap) + ori r2,r2,0 + + vxor v2,v2,v10 + VPMSUMD(v10,v18,const2) + lvx v18,off32,r4 + VPERM(v18,v18,v18,byteswap) + ori r2,r2,0 + + vxor v3,v3,v11 + VPMSUMD(v11,v19,const2) + lvx v19,off48,r4 + VPERM(v19,v19,v19,byteswap) + lvx const2,0,r3 + ori r2,r2,0 + + vxor v4,v4,v12 + VPMSUMD(v12,v20,const1) + lvx v20,off64,r4 + VPERM(v20,v20,v20,byteswap) + ori r2,r2,0 + + vxor v5,v5,v13 + VPMSUMD(v13,v21,const1) + lvx v21,off80,r4 + VPERM(v21,v21,v21,byteswap) + ori r2,r2,0 + + vxor v6,v6,v14 + VPMSUMD(v14,v22,const1) + lvx v22,off96,r4 + VPERM(v22,v22,v22,byteswap) + ori r2,r2,0 + + vxor v7,v7,v15 + VPMSUMD(v15,v23,const1) + lvx v23,off112,r4 + VPERM(v23,v23,v23,byteswap) + + addi r4,r4,8*16 + + bdnz 4b + +.Lfirst_cool_down: + /* First cool down pass */ + lvx const1,0,r3 + addi r3,r3,16 + + vxor v0,v0,v8 + VPMSUMD(v8,v16,const1) + ori r2,r2,0 + + vxor v1,v1,v9 + VPMSUMD(v9,v17,const1) + ori r2,r2,0 + + vxor v2,v2,v10 + VPMSUMD(v10,v18,const1) + ori r2,r2,0 + + vxor v3,v3,v11 + VPMSUMD(v11,v19,const1) + ori r2,r2,0 + + vxor v4,v4,v12 + VPMSUMD(v12,v20,const1) + ori r2,r2,0 + + vxor v5,v5,v13 + VPMSUMD(v13,v21,const1) + ori r2,r2,0 + + vxor v6,v6,v14 + VPMSUMD(v14,v22,const1) + ori r2,r2,0 + + vxor v7,v7,v15 + VPMSUMD(v15,v23,const1) + ori r2,r2,0 + +.Lsecond_cool_down: + /* Second cool down pass */ + vxor v0,v0,v8 + vxor v1,v1,v9 + vxor v2,v2,v10 + vxor v3,v3,v11 + vxor v4,v4,v12 + vxor v5,v5,v13 + vxor v6,v6,v14 + vxor v7,v7,v15 + +#ifdef REFLECT + /* + * vpmsumd produces a 96 bit result in the least significant bits + * of the register. Since we are bit reflected we have to shift it + * left 32 bits so it occupies the least significant bits in the + * bit reflected domain. + */ + vsldoi v0,v0,zeroes,4 + vsldoi v1,v1,zeroes,4 + vsldoi v2,v2,zeroes,4 + vsldoi v3,v3,zeroes,4 + vsldoi v4,v4,zeroes,4 + vsldoi v5,v5,zeroes,4 + vsldoi v6,v6,zeroes,4 + vsldoi v7,v7,zeroes,4 +#endif + + /* xor with last 1024 bits */ + lvx v8,0,r4 + lvx v9,off16,r4 + VPERM(v8,v8,v8,byteswap) + VPERM(v9,v9,v9,byteswap) + lvx v10,off32,r4 + lvx v11,off48,r4 + VPERM(v10,v10,v10,byteswap) + VPERM(v11,v11,v11,byteswap) + lvx v12,off64,r4 + lvx v13,off80,r4 + VPERM(v12,v12,v12,byteswap) + VPERM(v13,v13,v13,byteswap) + lvx v14,off96,r4 + lvx v15,off112,r4 + VPERM(v14,v14,v14,byteswap) + VPERM(v15,v15,v15,byteswap) + + addi r4,r4,8*16 + + vxor v16,v0,v8 + vxor v17,v1,v9 + vxor v18,v2,v10 + vxor v19,v3,v11 + vxor v20,v4,v12 + vxor v21,v5,v13 + vxor v22,v6,v14 + vxor v23,v7,v15 + + li r0,1 + cmpdi r6,0 + addi r6,r6,128 + bne 1b + + /* Work out how many bytes we have left */ + andi. r5,r5,127 + + /* Calculate where in the constant table we need to start */ + subfic r6,r5,128 + add r3,r3,r6 + + /* How many 16 byte chunks are in the tail */ + srdi r7,r5,4 + mtctr r7 + + /* + * Reduce the previously calculated 1024 bits to 64 bits, shifting + * 32 bits to include the trailing 32 bits of zeros + */ + lvx v0,0,r3 + lvx v1,off16,r3 + lvx v2,off32,r3 + lvx v3,off48,r3 + lvx v4,off64,r3 + lvx v5,off80,r3 + lvx v6,off96,r3 + lvx v7,off112,r3 + addi r3,r3,8*16 + + VPMSUMW(v0,v16,v0) + VPMSUMW(v1,v17,v1) + VPMSUMW(v2,v18,v2) + VPMSUMW(v3,v19,v3) + VPMSUMW(v4,v20,v4) + VPMSUMW(v5,v21,v5) + VPMSUMW(v6,v22,v6) + VPMSUMW(v7,v23,v7) + + /* Now reduce the tail (0 - 112 bytes) */ + cmpdi r7,0 + beq 1f + + lvx v16,0,r4 + lvx v17,0,r3 + VPERM(v16,v16,v16,byteswap) + VPMSUMW(v16,v16,v17) + vxor v0,v0,v16 + bdz 1f + + lvx v16,off16,r4 + lvx v17,off16,r3 + VPERM(v16,v16,v16,byteswap) + VPMSUMW(v16,v16,v17) + vxor v0,v0,v16 + bdz 1f + + lvx v16,off32,r4 + lvx v17,off32,r3 + VPERM(v16,v16,v16,byteswap) + VPMSUMW(v16,v16,v17) + vxor v0,v0,v16 + bdz 1f + + lvx v16,off48,r4 + lvx v17,off48,r3 + VPERM(v16,v16,v16,byteswap) + VPMSUMW(v16,v16,v17) + vxor v0,v0,v16 + bdz 1f + + lvx v16,off64,r4 + lvx v17,off64,r3 + VPERM(v16,v16,v16,byteswap) + VPMSUMW(v16,v16,v17) + vxor v0,v0,v16 + bdz 1f + + lvx v16,off80,r4 + lvx v17,off80,r3 + VPERM(v16,v16,v16,byteswap) + VPMSUMW(v16,v16,v17) + vxor v0,v0,v16 + bdz 1f + + lvx v16,off96,r4 + lvx v17,off96,r3 + VPERM(v16,v16,v16,byteswap) + VPMSUMW(v16,v16,v17) + vxor v0,v0,v16 + + /* Now xor all the parallel chunks together */ +1: vxor v0,v0,v1 + vxor v2,v2,v3 + vxor v4,v4,v5 + vxor v6,v6,v7 + + vxor v0,v0,v2 + vxor v4,v4,v6 + + vxor v0,v0,v4 + +.Lbarrett_reduction: + /* Barrett constants */ + addis r3,r2,.barrett_constants@toc@ha + addi r3,r3,.barrett_constants@toc@l + + lvx const1,0,r3 + lvx const2,off16,r3 + + vsldoi v1,v0,v0,8 + vxor v0,v0,v1 /* xor two 64 bit results together */ + +#ifdef REFLECT + /* shift left one bit */ + vspltisb v1,1 + vsl v0,v0,v1 +#endif + + vand v0,v0,mask_64bit + +#ifndef REFLECT + /* + * Now for the Barrett reduction algorithm. The idea is to calculate q, + * the multiple of our polynomial that we need to subtract. By + * doing the computation 2x bits higher (ie 64 bits) and shifting the + * result back down 2x bits, we round down to the nearest multiple. + */ + VPMSUMD(v1,v0,const1) /* ma */ + vsldoi v1,zeroes,v1,8 /* q = floor(ma/(2^64)) */ + VPMSUMD(v1,v1,const2) /* qn */ + vxor v0,v0,v1 /* a - qn, subtraction is xor in GF(2) */ + + /* + * Get the result into r3. We need to shift it left 8 bytes: + * V0 [ 0 1 2 X ] + * V0 [ 0 X 2 3 ] + */ + vsldoi v0,v0,zeroes,8 /* shift result into top 64 bits */ +#else + /* + * The reflected version of Barrett reduction. Instead of bit + * reflecting our data (which is expensive to do), we bit reflect our + * constants and our algorithm, which means the intermediate data in + * our vector registers goes from 0-63 instead of 63-0. We can reflect + * the algorithm because we don't carry in mod 2 arithmetic. + */ + vand v1,v0,mask_32bit /* bottom 32 bits of a */ + VPMSUMD(v1,v1,const1) /* ma */ + vand v1,v1,mask_32bit /* bottom 32bits of ma */ + VPMSUMD(v1,v1,const2) /* qn */ + vxor v0,v0,v1 /* a - qn, subtraction is xor in GF(2) */ + + /* + * Since we are bit reflected, the result (ie the low 32 bits) is in + * the high 32 bits. We just need to shift it left 4 bytes + * V0 [ 0 1 X 3 ] + * V0 [ 0 X 2 3 ] + */ + vsldoi v0,v0,zeroes,4 /* shift result into top 64 bits of */ +#endif + + /* Get it into r3 */ + MFVRD(r3, v0) + + ld r31,-8(r1) + ld r30,-16(r1) + ld r29,-24(r1) + ld r28,-32(r1) + ld r27,-40(r1) + ld r26,-48(r1) + ld r25,-56(r1) + + blr + +.Lfirst_warm_up_done: + lvx const1,0,r3 + addi r3,r3,16 + + VPMSUMD(v8,v16,const1) + VPMSUMD(v9,v17,const1) + VPMSUMD(v10,v18,const1) + VPMSUMD(v11,v19,const1) + VPMSUMD(v12,v20,const1) + VPMSUMD(v13,v21,const1) + VPMSUMD(v14,v22,const1) + VPMSUMD(v15,v23,const1) + + b .Lsecond_cool_down + +.Lshort: + cmpdi r5,0 + beq .Lzero + + addis r3,r2,.short_constants@toc@ha + addi r3,r3,.short_constants@toc@l + + /* Calculate where in the constant table we need to start */ + subfic r6,r5,256 + add r3,r3,r6 + + /* How many 16 byte chunks? */ + srdi r7,r5,4 + mtctr r7 + + vxor v19,v19,v19 + vxor v20,v20,v20 + + lvx v0,0,r4 + lvx v16,0,r3 + VPERM(v0,v0,v16,byteswap) + vxor v0,v0,v8 /* xor in initial value */ + VPMSUMW(v0,v0,v16) + bdz .Lv0 + + lvx v1,off16,r4 + lvx v17,off16,r3 + VPERM(v1,v1,v17,byteswap) + VPMSUMW(v1,v1,v17) + bdz .Lv1 + + lvx v2,off32,r4 + lvx v16,off32,r3 + VPERM(v2,v2,v16,byteswap) + VPMSUMW(v2,v2,v16) + bdz .Lv2 + + lvx v3,off48,r4 + lvx v17,off48,r3 + VPERM(v3,v3,v17,byteswap) + VPMSUMW(v3,v3,v17) + bdz .Lv3 + + lvx v4,off64,r4 + lvx v16,off64,r3 + VPERM(v4,v4,v16,byteswap) + VPMSUMW(v4,v4,v16) + bdz .Lv4 + + lvx v5,off80,r4 + lvx v17,off80,r3 + VPERM(v5,v5,v17,byteswap) + VPMSUMW(v5,v5,v17) + bdz .Lv5 + + lvx v6,off96,r4 + lvx v16,off96,r3 + VPERM(v6,v6,v16,byteswap) + VPMSUMW(v6,v6,v16) + bdz .Lv6 + + lvx v7,off112,r4 + lvx v17,off112,r3 + VPERM(v7,v7,v17,byteswap) + VPMSUMW(v7,v7,v17) + bdz .Lv7 + + addi r3,r3,128 + addi r4,r4,128 + + lvx v8,0,r4 + lvx v16,0,r3 + VPERM(v8,v8,v16,byteswap) + VPMSUMW(v8,v8,v16) + bdz .Lv8 + + lvx v9,off16,r4 + lvx v17,off16,r3 + VPERM(v9,v9,v17,byteswap) + VPMSUMW(v9,v9,v17) + bdz .Lv9 + + lvx v10,off32,r4 + lvx v16,off32,r3 + VPERM(v10,v10,v16,byteswap) + VPMSUMW(v10,v10,v16) + bdz .Lv10 + + lvx v11,off48,r4 + lvx v17,off48,r3 + VPERM(v11,v11,v17,byteswap) + VPMSUMW(v11,v11,v17) + bdz .Lv11 + + lvx v12,off64,r4 + lvx v16,off64,r3 + VPERM(v12,v12,v16,byteswap) + VPMSUMW(v12,v12,v16) + bdz .Lv12 + + lvx v13,off80,r4 + lvx v17,off80,r3 + VPERM(v13,v13,v17,byteswap) + VPMSUMW(v13,v13,v17) + bdz .Lv13 + + lvx v14,off96,r4 + lvx v16,off96,r3 + VPERM(v14,v14,v16,byteswap) + VPMSUMW(v14,v14,v16) + bdz .Lv14 + + lvx v15,off112,r4 + lvx v17,off112,r3 + VPERM(v15,v15,v17,byteswap) + VPMSUMW(v15,v15,v17) + +.Lv15: vxor v19,v19,v15 +.Lv14: vxor v20,v20,v14 +.Lv13: vxor v19,v19,v13 +.Lv12: vxor v20,v20,v12 +.Lv11: vxor v19,v19,v11 +.Lv10: vxor v20,v20,v10 +.Lv9: vxor v19,v19,v9 +.Lv8: vxor v20,v20,v8 +.Lv7: vxor v19,v19,v7 +.Lv6: vxor v20,v20,v6 +.Lv5: vxor v19,v19,v5 +.Lv4: vxor v20,v20,v4 +.Lv3: vxor v19,v19,v3 +.Lv2: vxor v20,v20,v2 +.Lv1: vxor v19,v19,v1 +.Lv0: vxor v20,v20,v0 + + vxor v0,v19,v20 + + b .Lbarrett_reduction + +.Lzero: + mr r3,r10 + blr +FUNC_END(__crc32_vpmsum) diff --git a/src/support/power8/crc32_constants.h b/src/support/power8/crc32_constants.h new file mode 100644 index 00000000000..02c471d1c56 --- /dev/null +++ b/src/support/power8/crc32_constants.h @@ -0,0 +1,901 @@ +#define CRC 0x1edc6f41 +#define CRC_XOR +#define REFLECT + +#ifndef __ASSEMBLY__ +#ifdef CRC_TABLE +static const unsigned int crc_table[] = { + 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, + 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb, + 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, + 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, + 0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b, + 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, + 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, + 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, + 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, + 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, + 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, + 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, + 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, + 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a, + 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, + 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, + 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, + 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, + 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, + 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, + 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, + 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, + 0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, + 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, + 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, + 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, + 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, + 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, + 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, + 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, + 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, + 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829, + 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, + 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, + 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, + 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, + 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, + 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, + 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, + 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, + 0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652, + 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, + 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, + 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, + 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, + 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, + 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, + 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, + 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, + 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, + 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, + 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, + 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, + 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, + 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, + 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, + 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, + 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, + 0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321, + 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, + 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, + 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, + 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, + 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351,}; + +#endif +#else +#define MAX_SIZE 32768 +.constants: + + /* Reduce 262144 kbits to 1024 bits */ + /* x^261120 mod p(x)` << 1, x^261184 mod p(x)` << 1 */ + .octa 0x00000000b6ca9e20000000009c37c408 + + /* x^260096 mod p(x)` << 1, x^260160 mod p(x)` << 1 */ + .octa 0x00000000350249a800000001b51df26c + + /* x^259072 mod p(x)` << 1, x^259136 mod p(x)` << 1 */ + .octa 0x00000001862dac54000000000724b9d0 + + /* x^258048 mod p(x)` << 1, x^258112 mod p(x)` << 1 */ + .octa 0x00000001d87fb48c00000001c00532fe + + /* x^257024 mod p(x)` << 1, x^257088 mod p(x)` << 1 */ + .octa 0x00000001f39b699e00000000f05a9362 + + /* x^256000 mod p(x)` << 1, x^256064 mod p(x)` << 1 */ + .octa 0x0000000101da11b400000001e1007970 + + /* x^254976 mod p(x)` << 1, x^255040 mod p(x)` << 1 */ + .octa 0x00000001cab571e000000000a57366ee + + /* x^253952 mod p(x)` << 1, x^254016 mod p(x)` << 1 */ + .octa 0x00000000c7020cfe0000000192011284 + + /* x^252928 mod p(x)` << 1, x^252992 mod p(x)` << 1 */ + .octa 0x00000000cdaed1ae0000000162716d9a + + /* x^251904 mod p(x)` << 1, x^251968 mod p(x)` << 1 */ + .octa 0x00000001e804effc00000000cd97ecde + + /* x^250880 mod p(x)` << 1, x^250944 mod p(x)` << 1 */ + .octa 0x0000000077c3ea3a0000000058812bc0 + + /* x^249856 mod p(x)` << 1, x^249920 mod p(x)` << 1 */ + .octa 0x0000000068df31b40000000088b8c12e + + /* x^248832 mod p(x)` << 1, x^248896 mod p(x)` << 1 */ + .octa 0x00000000b059b6c200000001230b234c + + /* x^247808 mod p(x)` << 1, x^247872 mod p(x)` << 1 */ + .octa 0x0000000145fb8ed800000001120b416e + + /* x^246784 mod p(x)` << 1, x^246848 mod p(x)` << 1 */ + .octa 0x00000000cbc0916800000001974aecb0 + + /* x^245760 mod p(x)` << 1, x^245824 mod p(x)` << 1 */ + .octa 0x000000005ceeedc2000000008ee3f226 + + /* x^244736 mod p(x)` << 1, x^244800 mod p(x)` << 1 */ + .octa 0x0000000047d74e8600000001089aba9a + + /* x^243712 mod p(x)` << 1, x^243776 mod p(x)` << 1 */ + .octa 0x00000001407e9e220000000065113872 + + /* x^242688 mod p(x)` << 1, x^242752 mod p(x)` << 1 */ + .octa 0x00000001da967bda000000005c07ec10 + + /* x^241664 mod p(x)` << 1, x^241728 mod p(x)` << 1 */ + .octa 0x000000006c8983680000000187590924 + + /* x^240640 mod p(x)` << 1, x^240704 mod p(x)` << 1 */ + .octa 0x00000000f2d14c9800000000e35da7c6 + + /* x^239616 mod p(x)` << 1, x^239680 mod p(x)` << 1 */ + .octa 0x00000001993c6ad4000000000415855a + + /* x^238592 mod p(x)` << 1, x^238656 mod p(x)` << 1 */ + .octa 0x000000014683d1ac0000000073617758 + + /* x^237568 mod p(x)` << 1, x^237632 mod p(x)` << 1 */ + .octa 0x00000001a7c93e6c0000000176021d28 + + /* x^236544 mod p(x)` << 1, x^236608 mod p(x)` << 1 */ + .octa 0x000000010211e90a00000001c358fd0a + + /* x^235520 mod p(x)` << 1, x^235584 mod p(x)` << 1 */ + .octa 0x000000001119403e00000001ff7a2c18 + + /* x^234496 mod p(x)` << 1, x^234560 mod p(x)` << 1 */ + .octa 0x000000001c3261aa00000000f2d9f7e4 + + /* x^233472 mod p(x)` << 1, x^233536 mod p(x)` << 1 */ + .octa 0x000000014e37a634000000016cf1f9c8 + + /* x^232448 mod p(x)` << 1, x^232512 mod p(x)` << 1 */ + .octa 0x0000000073786c0c000000010af9279a + + /* x^231424 mod p(x)` << 1, x^231488 mod p(x)` << 1 */ + .octa 0x000000011dc037f80000000004f101e8 + + /* x^230400 mod p(x)` << 1, x^230464 mod p(x)` << 1 */ + .octa 0x0000000031433dfc0000000070bcf184 + + /* x^229376 mod p(x)` << 1, x^229440 mod p(x)` << 1 */ + .octa 0x000000009cde8348000000000a8de642 + + /* x^228352 mod p(x)` << 1, x^228416 mod p(x)` << 1 */ + .octa 0x0000000038d3c2a60000000062ea130c + + /* x^227328 mod p(x)` << 1, x^227392 mod p(x)` << 1 */ + .octa 0x000000011b25f26000000001eb31cbb2 + + /* x^226304 mod p(x)` << 1, x^226368 mod p(x)` << 1 */ + .octa 0x000000001629e6f00000000170783448 + + /* x^225280 mod p(x)` << 1, x^225344 mod p(x)` << 1 */ + .octa 0x0000000160838b4c00000001a684b4c6 + + /* x^224256 mod p(x)` << 1, x^224320 mod p(x)` << 1 */ + .octa 0x000000007a44011c00000000253ca5b4 + + /* x^223232 mod p(x)` << 1, x^223296 mod p(x)` << 1 */ + .octa 0x00000000226f417a0000000057b4b1e2 + + /* x^222208 mod p(x)` << 1, x^222272 mod p(x)` << 1 */ + .octa 0x0000000045eb2eb400000000b6bd084c + + /* x^221184 mod p(x)` << 1, x^221248 mod p(x)` << 1 */ + .octa 0x000000014459d70c0000000123c2d592 + + /* x^220160 mod p(x)` << 1, x^220224 mod p(x)` << 1 */ + .octa 0x00000001d406ed8200000000159dafce + + /* x^219136 mod p(x)` << 1, x^219200 mod p(x)` << 1 */ + .octa 0x0000000160c8e1a80000000127e1a64e + + /* x^218112 mod p(x)` << 1, x^218176 mod p(x)` << 1 */ + .octa 0x0000000027ba80980000000056860754 + + /* x^217088 mod p(x)` << 1, x^217152 mod p(x)` << 1 */ + .octa 0x000000006d92d01800000001e661aae8 + + /* x^216064 mod p(x)` << 1, x^216128 mod p(x)` << 1 */ + .octa 0x000000012ed7e3f200000000f82c6166 + + /* x^215040 mod p(x)` << 1, x^215104 mod p(x)` << 1 */ + .octa 0x000000002dc8778800000000c4f9c7ae + + /* x^214016 mod p(x)` << 1, x^214080 mod p(x)` << 1 */ + .octa 0x0000000018240bb80000000074203d20 + + /* x^212992 mod p(x)` << 1, x^213056 mod p(x)` << 1 */ + .octa 0x000000001ad381580000000198173052 + + /* x^211968 mod p(x)` << 1, x^212032 mod p(x)` << 1 */ + .octa 0x00000001396b78f200000001ce8aba54 + + /* x^210944 mod p(x)` << 1, x^211008 mod p(x)` << 1 */ + .octa 0x000000011a68133400000001850d5d94 + + /* x^209920 mod p(x)` << 1, x^209984 mod p(x)` << 1 */ + .octa 0x000000012104732e00000001d609239c + + /* x^208896 mod p(x)` << 1, x^208960 mod p(x)` << 1 */ + .octa 0x00000000a140d90c000000001595f048 + + /* x^207872 mod p(x)` << 1, x^207936 mod p(x)` << 1 */ + .octa 0x00000001b7215eda0000000042ccee08 + + /* x^206848 mod p(x)` << 1, x^206912 mod p(x)` << 1 */ + .octa 0x00000001aaf1df3c000000010a389d74 + + /* x^205824 mod p(x)` << 1, x^205888 mod p(x)` << 1 */ + .octa 0x0000000029d15b8a000000012a840da6 + + /* x^204800 mod p(x)` << 1, x^204864 mod p(x)` << 1 */ + .octa 0x00000000f1a96922000000001d181c0c + + /* x^203776 mod p(x)` << 1, x^203840 mod p(x)` << 1 */ + .octa 0x00000001ac80d03c0000000068b7d1f6 + + /* x^202752 mod p(x)` << 1, x^202816 mod p(x)` << 1 */ + .octa 0x000000000f11d56a000000005b0f14fc + + /* x^201728 mod p(x)` << 1, x^201792 mod p(x)` << 1 */ + .octa 0x00000001f1c022a20000000179e9e730 + + /* x^200704 mod p(x)` << 1, x^200768 mod p(x)` << 1 */ + .octa 0x0000000173d00ae200000001ce1368d6 + + /* x^199680 mod p(x)` << 1, x^199744 mod p(x)` << 1 */ + .octa 0x00000001d4ffe4ac0000000112c3a84c + + /* x^198656 mod p(x)` << 1, x^198720 mod p(x)` << 1 */ + .octa 0x000000016edc5ae400000000de940fee + + /* x^197632 mod p(x)` << 1, x^197696 mod p(x)` << 1 */ + .octa 0x00000001f1a0214000000000fe896b7e + + /* x^196608 mod p(x)` << 1, x^196672 mod p(x)` << 1 */ + .octa 0x00000000ca0b28a000000001f797431c + + /* x^195584 mod p(x)` << 1, x^195648 mod p(x)` << 1 */ + .octa 0x00000001928e30a20000000053e989ba + + /* x^194560 mod p(x)` << 1, x^194624 mod p(x)` << 1 */ + .octa 0x0000000097b1b002000000003920cd16 + + /* x^193536 mod p(x)` << 1, x^193600 mod p(x)` << 1 */ + .octa 0x00000000b15bf90600000001e6f579b8 + + /* x^192512 mod p(x)` << 1, x^192576 mod p(x)` << 1 */ + .octa 0x00000000411c5d52000000007493cb0a + + /* x^191488 mod p(x)` << 1, x^191552 mod p(x)` << 1 */ + .octa 0x00000001c36f330000000001bdd376d8 + + /* x^190464 mod p(x)` << 1, x^190528 mod p(x)` << 1 */ + .octa 0x00000001119227e0000000016badfee6 + + /* x^189440 mod p(x)` << 1, x^189504 mod p(x)` << 1 */ + .octa 0x00000000114d47020000000071de5c58 + + /* x^188416 mod p(x)` << 1, x^188480 mod p(x)` << 1 */ + .octa 0x00000000458b5b9800000000453f317c + + /* x^187392 mod p(x)` << 1, x^187456 mod p(x)` << 1 */ + .octa 0x000000012e31fb8e0000000121675cce + + /* x^186368 mod p(x)` << 1, x^186432 mod p(x)` << 1 */ + .octa 0x000000005cf619d800000001f409ee92 + + /* x^185344 mod p(x)` << 1, x^185408 mod p(x)` << 1 */ + .octa 0x0000000063f4d8b200000000f36b9c88 + + /* x^184320 mod p(x)` << 1, x^184384 mod p(x)` << 1 */ + .octa 0x000000004138dc8a0000000036b398f4 + + /* x^183296 mod p(x)` << 1, x^183360 mod p(x)` << 1 */ + .octa 0x00000001d29ee8e000000001748f9adc + + /* x^182272 mod p(x)` << 1, x^182336 mod p(x)` << 1 */ + .octa 0x000000006a08ace800000001be94ec00 + + /* x^181248 mod p(x)` << 1, x^181312 mod p(x)` << 1 */ + .octa 0x0000000127d4201000000000b74370d6 + + /* x^180224 mod p(x)` << 1, x^180288 mod p(x)` << 1 */ + .octa 0x0000000019d76b6200000001174d0b98 + + /* x^179200 mod p(x)` << 1, x^179264 mod p(x)` << 1 */ + .octa 0x00000001b1471f6e00000000befc06a4 + + /* x^178176 mod p(x)` << 1, x^178240 mod p(x)` << 1 */ + .octa 0x00000001f64c19cc00000001ae125288 + + /* x^177152 mod p(x)` << 1, x^177216 mod p(x)` << 1 */ + .octa 0x00000000003c0ea00000000095c19b34 + + /* x^176128 mod p(x)` << 1, x^176192 mod p(x)` << 1 */ + .octa 0x000000014d73abf600000001a78496f2 + + /* x^175104 mod p(x)` << 1, x^175168 mod p(x)` << 1 */ + .octa 0x00000001620eb84400000001ac5390a0 + + /* x^174080 mod p(x)` << 1, x^174144 mod p(x)` << 1 */ + .octa 0x0000000147655048000000002a80ed6e + + /* x^173056 mod p(x)` << 1, x^173120 mod p(x)` << 1 */ + .octa 0x0000000067b5077e00000001fa9b0128 + + /* x^172032 mod p(x)` << 1, x^172096 mod p(x)` << 1 */ + .octa 0x0000000010ffe20600000001ea94929e + + /* x^171008 mod p(x)` << 1, x^171072 mod p(x)` << 1 */ + .octa 0x000000000fee8f1e0000000125f4305c + + /* x^169984 mod p(x)` << 1, x^170048 mod p(x)` << 1 */ + .octa 0x00000001da26fbae00000001471e2002 + + /* x^168960 mod p(x)` << 1, x^169024 mod p(x)` << 1 */ + .octa 0x00000001b3a8bd880000000132d2253a + + /* x^167936 mod p(x)` << 1, x^168000 mod p(x)` << 1 */ + .octa 0x00000000e8f3898e00000000f26b3592 + + /* x^166912 mod p(x)` << 1, x^166976 mod p(x)` << 1 */ + .octa 0x00000000b0d0d28c00000000bc8b67b0 + + /* x^165888 mod p(x)` << 1, x^165952 mod p(x)` << 1 */ + .octa 0x0000000030f2a798000000013a826ef2 + + /* x^164864 mod p(x)` << 1, x^164928 mod p(x)` << 1 */ + .octa 0x000000000fba10020000000081482c84 + + /* x^163840 mod p(x)` << 1, x^163904 mod p(x)` << 1 */ + .octa 0x00000000bdb9bd7200000000e77307c2 + + /* x^162816 mod p(x)` << 1, x^162880 mod p(x)` << 1 */ + .octa 0x0000000075d3bf5a00000000d4a07ec8 + + /* x^161792 mod p(x)` << 1, x^161856 mod p(x)` << 1 */ + .octa 0x00000000ef1f98a00000000017102100 + + /* x^160768 mod p(x)` << 1, x^160832 mod p(x)` << 1 */ + .octa 0x00000000689c760200000000db406486 + + /* x^159744 mod p(x)` << 1, x^159808 mod p(x)` << 1 */ + .octa 0x000000016d5fa5fe0000000192db7f88 + + /* x^158720 mod p(x)` << 1, x^158784 mod p(x)` << 1 */ + .octa 0x00000001d0d2b9ca000000018bf67b1e + + /* x^157696 mod p(x)` << 1, x^157760 mod p(x)` << 1 */ + .octa 0x0000000041e7b470000000007c09163e + + /* x^156672 mod p(x)` << 1, x^156736 mod p(x)` << 1 */ + .octa 0x00000001cbb6495e000000000adac060 + + /* x^155648 mod p(x)` << 1, x^155712 mod p(x)` << 1 */ + .octa 0x000000010052a0b000000000bd8316ae + + /* x^154624 mod p(x)` << 1, x^154688 mod p(x)` << 1 */ + .octa 0x00000001d8effb5c000000019f09ab54 + + /* x^153600 mod p(x)` << 1, x^153664 mod p(x)` << 1 */ + .octa 0x00000001d969853c0000000125155542 + + /* x^152576 mod p(x)` << 1, x^152640 mod p(x)` << 1 */ + .octa 0x00000000523ccce2000000018fdb5882 + + /* x^151552 mod p(x)` << 1, x^151616 mod p(x)` << 1 */ + .octa 0x000000001e2436bc00000000e794b3f4 + + /* x^150528 mod p(x)` << 1, x^150592 mod p(x)` << 1 */ + .octa 0x00000000ddd1c3a2000000016f9bb022 + + /* x^149504 mod p(x)` << 1, x^149568 mod p(x)` << 1 */ + .octa 0x0000000019fcfe3800000000290c9978 + + /* x^148480 mod p(x)` << 1, x^148544 mod p(x)` << 1 */ + .octa 0x00000001ce95db640000000083c0f350 + + /* x^147456 mod p(x)` << 1, x^147520 mod p(x)` << 1 */ + .octa 0x00000000af5828060000000173ea6628 + + /* x^146432 mod p(x)` << 1, x^146496 mod p(x)` << 1 */ + .octa 0x00000001006388f600000001c8b4e00a + + /* x^145408 mod p(x)` << 1, x^145472 mod p(x)` << 1 */ + .octa 0x0000000179eca00a00000000de95d6aa + + /* x^144384 mod p(x)` << 1, x^144448 mod p(x)` << 1 */ + .octa 0x0000000122410a6a000000010b7f7248 + + /* x^143360 mod p(x)` << 1, x^143424 mod p(x)` << 1 */ + .octa 0x000000004288e87c00000001326e3a06 + + /* x^142336 mod p(x)` << 1, x^142400 mod p(x)` << 1 */ + .octa 0x000000016c5490da00000000bb62c2e6 + + /* x^141312 mod p(x)` << 1, x^141376 mod p(x)` << 1 */ + .octa 0x00000000d1c71f6e0000000156a4b2c2 + + /* x^140288 mod p(x)` << 1, x^140352 mod p(x)` << 1 */ + .octa 0x00000001b4ce08a6000000011dfe763a + + /* x^139264 mod p(x)` << 1, x^139328 mod p(x)` << 1 */ + .octa 0x00000001466ba60c000000007bcca8e2 + + /* x^138240 mod p(x)` << 1, x^138304 mod p(x)` << 1 */ + .octa 0x00000001f6c488a40000000186118faa + + /* x^137216 mod p(x)` << 1, x^137280 mod p(x)` << 1 */ + .octa 0x000000013bfb06820000000111a65a88 + + /* x^136192 mod p(x)` << 1, x^136256 mod p(x)` << 1 */ + .octa 0x00000000690e9e54000000003565e1c4 + + /* x^135168 mod p(x)` << 1, x^135232 mod p(x)` << 1 */ + .octa 0x00000000281346b6000000012ed02a82 + + /* x^134144 mod p(x)` << 1, x^134208 mod p(x)` << 1 */ + .octa 0x000000015646402400000000c486ecfc + + /* x^133120 mod p(x)` << 1, x^133184 mod p(x)` << 1 */ + .octa 0x000000016063a8dc0000000001b951b2 + + /* x^132096 mod p(x)` << 1, x^132160 mod p(x)` << 1 */ + .octa 0x0000000116a663620000000048143916 + + /* x^131072 mod p(x)` << 1, x^131136 mod p(x)` << 1 */ + .octa 0x000000017e8aa4d200000001dc2ae124 + + /* x^130048 mod p(x)` << 1, x^130112 mod p(x)` << 1 */ + .octa 0x00000001728eb10c00000001416c58d6 + + /* x^129024 mod p(x)` << 1, x^129088 mod p(x)` << 1 */ + .octa 0x00000001b08fd7fa00000000a479744a + + /* x^128000 mod p(x)` << 1, x^128064 mod p(x)` << 1 */ + .octa 0x00000001092a16e80000000096ca3a26 + + /* x^126976 mod p(x)` << 1, x^127040 mod p(x)` << 1 */ + .octa 0x00000000a505637c00000000ff223d4e + + /* x^125952 mod p(x)` << 1, x^126016 mod p(x)` << 1 */ + .octa 0x00000000d94869b2000000010e84da42 + + /* x^124928 mod p(x)` << 1, x^124992 mod p(x)` << 1 */ + .octa 0x00000001c8b203ae00000001b61ba3d0 + + /* x^123904 mod p(x)` << 1, x^123968 mod p(x)` << 1 */ + .octa 0x000000005704aea000000000680f2de8 + + /* x^122880 mod p(x)` << 1, x^122944 mod p(x)` << 1 */ + .octa 0x000000012e295fa2000000008772a9a8 + + /* x^121856 mod p(x)` << 1, x^121920 mod p(x)` << 1 */ + .octa 0x000000011d0908bc0000000155f295bc + + /* x^120832 mod p(x)` << 1, x^120896 mod p(x)` << 1 */ + .octa 0x0000000193ed97ea00000000595f9282 + + /* x^119808 mod p(x)` << 1, x^119872 mod p(x)` << 1 */ + .octa 0x000000013a0f1c520000000164b1c25a + + /* x^118784 mod p(x)` << 1, x^118848 mod p(x)` << 1 */ + .octa 0x000000010c2c40c000000000fbd67c50 + + /* x^117760 mod p(x)` << 1, x^117824 mod p(x)` << 1 */ + .octa 0x00000000ff6fac3e0000000096076268 + + /* x^116736 mod p(x)` << 1, x^116800 mod p(x)` << 1 */ + .octa 0x000000017b3609c000000001d288e4cc + + /* x^115712 mod p(x)` << 1, x^115776 mod p(x)` << 1 */ + .octa 0x0000000088c8c92200000001eaac1bdc + + /* x^114688 mod p(x)` << 1, x^114752 mod p(x)` << 1 */ + .octa 0x00000001751baae600000001f1ea39e2 + + /* x^113664 mod p(x)` << 1, x^113728 mod p(x)` << 1 */ + .octa 0x000000010795297200000001eb6506fc + + /* x^112640 mod p(x)` << 1, x^112704 mod p(x)` << 1 */ + .octa 0x0000000162b00abe000000010f806ffe + + /* x^111616 mod p(x)` << 1, x^111680 mod p(x)` << 1 */ + .octa 0x000000000d7b404c000000010408481e + + /* x^110592 mod p(x)` << 1, x^110656 mod p(x)` << 1 */ + .octa 0x00000000763b13d40000000188260534 + + /* x^109568 mod p(x)` << 1, x^109632 mod p(x)` << 1 */ + .octa 0x00000000f6dc22d80000000058fc73e0 + + /* x^108544 mod p(x)` << 1, x^108608 mod p(x)` << 1 */ + .octa 0x000000007daae06000000000391c59b8 + + /* x^107520 mod p(x)` << 1, x^107584 mod p(x)` << 1 */ + .octa 0x000000013359ab7c000000018b638400 + + /* x^106496 mod p(x)` << 1, x^106560 mod p(x)` << 1 */ + .octa 0x000000008add438a000000011738f5c4 + + /* x^105472 mod p(x)` << 1, x^105536 mod p(x)` << 1 */ + .octa 0x00000001edbefdea000000008cf7c6da + + /* x^104448 mod p(x)` << 1, x^104512 mod p(x)` << 1 */ + .octa 0x000000004104e0f800000001ef97fb16 + + /* x^103424 mod p(x)` << 1, x^103488 mod p(x)` << 1 */ + .octa 0x00000000b48a82220000000102130e20 + + /* x^102400 mod p(x)` << 1, x^102464 mod p(x)` << 1 */ + .octa 0x00000001bcb4684400000000db968898 + + /* x^101376 mod p(x)` << 1, x^101440 mod p(x)` << 1 */ + .octa 0x000000013293ce0a00000000b5047b5e + + /* x^100352 mod p(x)` << 1, x^100416 mod p(x)` << 1 */ + .octa 0x00000001710d0844000000010b90fdb2 + + /* x^99328 mod p(x)` << 1, x^99392 mod p(x)` << 1 */ + .octa 0x0000000117907f6e000000004834a32e + + /* x^98304 mod p(x)` << 1, x^98368 mod p(x)` << 1 */ + .octa 0x0000000087ddf93e0000000059c8f2b0 + + /* x^97280 mod p(x)` << 1, x^97344 mod p(x)` << 1 */ + .octa 0x000000005970e9b00000000122cec508 + + /* x^96256 mod p(x)` << 1, x^96320 mod p(x)` << 1 */ + .octa 0x0000000185b2b7d0000000000a330cda + + /* x^95232 mod p(x)` << 1, x^95296 mod p(x)` << 1 */ + .octa 0x00000001dcee0efc000000014a47148c + + /* x^94208 mod p(x)` << 1, x^94272 mod p(x)` << 1 */ + .octa 0x0000000030da27220000000042c61cb8 + + /* x^93184 mod p(x)` << 1, x^93248 mod p(x)` << 1 */ + .octa 0x000000012f925a180000000012fe6960 + + /* x^92160 mod p(x)` << 1, x^92224 mod p(x)` << 1 */ + .octa 0x00000000dd2e357c00000000dbda2c20 + + /* x^91136 mod p(x)` << 1, x^91200 mod p(x)` << 1 */ + .octa 0x00000000071c80de000000011122410c + + /* x^90112 mod p(x)` << 1, x^90176 mod p(x)` << 1 */ + .octa 0x000000011513140a00000000977b2070 + + /* x^89088 mod p(x)` << 1, x^89152 mod p(x)` << 1 */ + .octa 0x00000001df876e8e000000014050438e + + /* x^88064 mod p(x)` << 1, x^88128 mod p(x)` << 1 */ + .octa 0x000000015f81d6ce0000000147c840e8 + + /* x^87040 mod p(x)` << 1, x^87104 mod p(x)` << 1 */ + .octa 0x000000019dd94dbe00000001cc7c88ce + + /* x^86016 mod p(x)` << 1, x^86080 mod p(x)` << 1 */ + .octa 0x00000001373d206e00000001476b35a4 + + /* x^84992 mod p(x)` << 1, x^85056 mod p(x)` << 1 */ + .octa 0x00000000668ccade000000013d52d508 + + /* x^83968 mod p(x)` << 1, x^84032 mod p(x)` << 1 */ + .octa 0x00000001b192d268000000008e4be32e + + /* x^82944 mod p(x)` << 1, x^83008 mod p(x)` << 1 */ + .octa 0x00000000e30f3a7800000000024120fe + + /* x^81920 mod p(x)` << 1, x^81984 mod p(x)` << 1 */ + .octa 0x000000010ef1f7bc00000000ddecddb4 + + /* x^80896 mod p(x)` << 1, x^80960 mod p(x)` << 1 */ + .octa 0x00000001f5ac738000000000d4d403bc + + /* x^79872 mod p(x)` << 1, x^79936 mod p(x)` << 1 */ + .octa 0x000000011822ea7000000001734b89aa + + /* x^78848 mod p(x)` << 1, x^78912 mod p(x)` << 1 */ + .octa 0x00000000c3a33848000000010e7a58d6 + + /* x^77824 mod p(x)` << 1, x^77888 mod p(x)` << 1 */ + .octa 0x00000001bd151c2400000001f9f04e9c + + /* x^76800 mod p(x)` << 1, x^76864 mod p(x)` << 1 */ + .octa 0x0000000056002d7600000000b692225e + + /* x^75776 mod p(x)` << 1, x^75840 mod p(x)` << 1 */ + .octa 0x000000014657c4f4000000019b8d3f3e + + /* x^74752 mod p(x)` << 1, x^74816 mod p(x)` << 1 */ + .octa 0x0000000113742d7c00000001a874f11e + + /* x^73728 mod p(x)` << 1, x^73792 mod p(x)` << 1 */ + .octa 0x000000019c5920ba000000010d5a4254 + + /* x^72704 mod p(x)` << 1, x^72768 mod p(x)` << 1 */ + .octa 0x000000005216d2d600000000bbb2f5d6 + + /* x^71680 mod p(x)` << 1, x^71744 mod p(x)` << 1 */ + .octa 0x0000000136f5ad8a0000000179cc0e36 + + /* x^70656 mod p(x)` << 1, x^70720 mod p(x)` << 1 */ + .octa 0x000000018b07beb600000001dca1da4a + + /* x^69632 mod p(x)` << 1, x^69696 mod p(x)` << 1 */ + .octa 0x00000000db1e93b000000000feb1a192 + + /* x^68608 mod p(x)` << 1, x^68672 mod p(x)` << 1 */ + .octa 0x000000000b96fa3a00000000d1eeedd6 + + /* x^67584 mod p(x)` << 1, x^67648 mod p(x)` << 1 */ + .octa 0x00000001d9968af0000000008fad9bb4 + + /* x^66560 mod p(x)` << 1, x^66624 mod p(x)` << 1 */ + .octa 0x000000000e4a77a200000001884938e4 + + /* x^65536 mod p(x)` << 1, x^65600 mod p(x)` << 1 */ + .octa 0x00000000508c2ac800000001bc2e9bc0 + + /* x^64512 mod p(x)` << 1, x^64576 mod p(x)` << 1 */ + .octa 0x0000000021572a8000000001f9658a68 + + /* x^63488 mod p(x)` << 1, x^63552 mod p(x)` << 1 */ + .octa 0x00000001b859daf2000000001b9224fc + + /* x^62464 mod p(x)` << 1, x^62528 mod p(x)` << 1 */ + .octa 0x000000016f7884740000000055b2fb84 + + /* x^61440 mod p(x)` << 1, x^61504 mod p(x)` << 1 */ + .octa 0x00000001b438810e000000018b090348 + + /* x^60416 mod p(x)` << 1, x^60480 mod p(x)` << 1 */ + .octa 0x0000000095ddc6f2000000011ccbd5ea + + /* x^59392 mod p(x)` << 1, x^59456 mod p(x)` << 1 */ + .octa 0x00000001d977c20c0000000007ae47f8 + + /* x^58368 mod p(x)` << 1, x^58432 mod p(x)` << 1 */ + .octa 0x00000000ebedb99a0000000172acbec0 + + /* x^57344 mod p(x)` << 1, x^57408 mod p(x)` << 1 */ + .octa 0x00000001df9e9e9200000001c6e3ff20 + + /* x^56320 mod p(x)` << 1, x^56384 mod p(x)` << 1 */ + .octa 0x00000001a4a3f95200000000e1b38744 + + /* x^55296 mod p(x)` << 1, x^55360 mod p(x)` << 1 */ + .octa 0x00000000e2f5122000000000791585b2 + + /* x^54272 mod p(x)` << 1, x^54336 mod p(x)` << 1 */ + .octa 0x000000004aa01f3e00000000ac53b894 + + /* x^53248 mod p(x)` << 1, x^53312 mod p(x)` << 1 */ + .octa 0x00000000b3e90a5800000001ed5f2cf4 + + /* x^52224 mod p(x)` << 1, x^52288 mod p(x)` << 1 */ + .octa 0x000000000c9ca2aa00000001df48b2e0 + + /* x^51200 mod p(x)` << 1, x^51264 mod p(x)` << 1 */ + .octa 0x000000015168231600000000049c1c62 + + /* x^50176 mod p(x)` << 1, x^50240 mod p(x)` << 1 */ + .octa 0x0000000036fce78c000000017c460c12 + + /* x^49152 mod p(x)` << 1, x^49216 mod p(x)` << 1 */ + .octa 0x000000009037dc10000000015be4da7e + + /* x^48128 mod p(x)` << 1, x^48192 mod p(x)` << 1 */ + .octa 0x00000000d3298582000000010f38f668 + + /* x^47104 mod p(x)` << 1, x^47168 mod p(x)` << 1 */ + .octa 0x00000001b42e8ad60000000039f40a00 + + /* x^46080 mod p(x)` << 1, x^46144 mod p(x)` << 1 */ + .octa 0x00000000142a983800000000bd4c10c4 + + /* x^45056 mod p(x)` << 1, x^45120 mod p(x)` << 1 */ + .octa 0x0000000109c7f1900000000042db1d98 + + /* x^44032 mod p(x)` << 1, x^44096 mod p(x)` << 1 */ + .octa 0x0000000056ff931000000001c905bae6 + + /* x^43008 mod p(x)` << 1, x^43072 mod p(x)` << 1 */ + .octa 0x00000001594513aa00000000069d40ea + + /* x^41984 mod p(x)` << 1, x^42048 mod p(x)` << 1 */ + .octa 0x00000001e3b5b1e8000000008e4fbad0 + + /* x^40960 mod p(x)` << 1, x^41024 mod p(x)` << 1 */ + .octa 0x000000011dd5fc080000000047bedd46 + + /* x^39936 mod p(x)` << 1, x^40000 mod p(x)` << 1 */ + .octa 0x00000001675f0cc20000000026396bf8 + + /* x^38912 mod p(x)` << 1, x^38976 mod p(x)` << 1 */ + .octa 0x00000000d1c8dd4400000000379beb92 + + /* x^37888 mod p(x)` << 1, x^37952 mod p(x)` << 1 */ + .octa 0x0000000115ebd3d8000000000abae54a + + /* x^36864 mod p(x)` << 1, x^36928 mod p(x)` << 1 */ + .octa 0x00000001ecbd0dac0000000007e6a128 + + /* x^35840 mod p(x)` << 1, x^35904 mod p(x)` << 1 */ + .octa 0x00000000cdf67af2000000000ade29d2 + + /* x^34816 mod p(x)` << 1, x^34880 mod p(x)` << 1 */ + .octa 0x000000004c01ff4c00000000f974c45c + + /* x^33792 mod p(x)` << 1, x^33856 mod p(x)` << 1 */ + .octa 0x00000000f2d8657e00000000e77ac60a + + /* x^32768 mod p(x)` << 1, x^32832 mod p(x)` << 1 */ + .octa 0x000000006bae74c40000000145895816 + + /* x^31744 mod p(x)` << 1, x^31808 mod p(x)` << 1 */ + .octa 0x0000000152af8aa00000000038e362be + + /* x^30720 mod p(x)` << 1, x^30784 mod p(x)` << 1 */ + .octa 0x0000000004663802000000007f991a64 + + /* x^29696 mod p(x)` << 1, x^29760 mod p(x)` << 1 */ + .octa 0x00000001ab2f5afc00000000fa366d3a + + /* x^28672 mod p(x)` << 1, x^28736 mod p(x)` << 1 */ + .octa 0x0000000074a4ebd400000001a2bb34f0 + + /* x^27648 mod p(x)` << 1, x^27712 mod p(x)` << 1 */ + .octa 0x00000001d7ab3a4c0000000028a9981e + + /* x^26624 mod p(x)` << 1, x^26688 mod p(x)` << 1 */ + .octa 0x00000001a8da60c600000001dbc672be + + /* x^25600 mod p(x)` << 1, x^25664 mod p(x)` << 1 */ + .octa 0x000000013cf6382000000000b04d77f6 + + /* x^24576 mod p(x)` << 1, x^24640 mod p(x)` << 1 */ + .octa 0x00000000bec12e1e0000000124400d96 + + /* x^23552 mod p(x)` << 1, x^23616 mod p(x)` << 1 */ + .octa 0x00000001c6368010000000014ca4b414 + + /* x^22528 mod p(x)` << 1, x^22592 mod p(x)` << 1 */ + .octa 0x00000001e6e78758000000012fe2c938 + + /* x^21504 mod p(x)` << 1, x^21568 mod p(x)` << 1 */ + .octa 0x000000008d7f2b3c00000001faed01e6 + + /* x^20480 mod p(x)` << 1, x^20544 mod p(x)` << 1 */ + .octa 0x000000016b4a156e000000007e80ecfe + + /* x^19456 mod p(x)` << 1, x^19520 mod p(x)` << 1 */ + .octa 0x00000001c63cfeb60000000098daee94 + + /* x^18432 mod p(x)` << 1, x^18496 mod p(x)` << 1 */ + .octa 0x000000015f902670000000010a04edea + + /* x^17408 mod p(x)` << 1, x^17472 mod p(x)` << 1 */ + .octa 0x00000001cd5de11e00000001c00b4524 + + /* x^16384 mod p(x)` << 1, x^16448 mod p(x)` << 1 */ + .octa 0x000000001acaec540000000170296550 + + /* x^15360 mod p(x)` << 1, x^15424 mod p(x)` << 1 */ + .octa 0x000000002bd0ca780000000181afaa48 + + /* x^14336 mod p(x)` << 1, x^14400 mod p(x)` << 1 */ + .octa 0x0000000032d63d5c0000000185a31ffa + + /* x^13312 mod p(x)` << 1, x^13376 mod p(x)` << 1 */ + .octa 0x000000001c6d4e4c000000002469f608 + + /* x^12288 mod p(x)` << 1, x^12352 mod p(x)` << 1 */ + .octa 0x0000000106a60b92000000006980102a + + /* x^11264 mod p(x)` << 1, x^11328 mod p(x)` << 1 */ + .octa 0x00000000d3855e120000000111ea9ca8 + + /* x^10240 mod p(x)` << 1, x^10304 mod p(x)` << 1 */ + .octa 0x00000000e312563600000001bd1d29ce + + /* x^9216 mod p(x)` << 1, x^9280 mod p(x)` << 1 */ + .octa 0x000000009e8f7ea400000001b34b9580 + + /* x^8192 mod p(x)` << 1, x^8256 mod p(x)` << 1 */ + .octa 0x00000001c82e562c000000003076054e + + /* x^7168 mod p(x)` << 1, x^7232 mod p(x)` << 1 */ + .octa 0x00000000ca9f09ce000000012a608ea4 + + /* x^6144 mod p(x)` << 1, x^6208 mod p(x)` << 1 */ + .octa 0x00000000c63764e600000000784d05fe + + /* x^5120 mod p(x)` << 1, x^5184 mod p(x)` << 1 */ + .octa 0x0000000168d2e49e000000016ef0d82a + + /* x^4096 mod p(x)` << 1, x^4160 mod p(x)` << 1 */ + .octa 0x00000000e986c1480000000075bda454 + + /* x^3072 mod p(x)` << 1, x^3136 mod p(x)` << 1 */ + .octa 0x00000000cfb65894000000003dc0a1c4 + + /* x^2048 mod p(x)` << 1, x^2112 mod p(x)` << 1 */ + .octa 0x0000000111cadee400000000e9a5d8be + + /* x^1024 mod p(x)` << 1, x^1088 mod p(x)` << 1 */ + .octa 0x0000000171fb63ce00000001609bc4b4 + +.short_constants: + + /* Reduce final 1024-2048 bits to 64 bits, shifting 32 bits to include the trailing 32 bits of zeros */ + /* x^1952 mod p(x)`, x^1984 mod p(x)`, x^2016 mod p(x)`, x^2048 mod p(x)` */ + .octa 0x7fec2963e5bf80485cf015c388e56f72 + + /* x^1824 mod p(x)`, x^1856 mod p(x)`, x^1888 mod p(x)`, x^1920 mod p(x)` */ + .octa 0x38e888d4844752a9963a18920246e2e6 + + /* x^1696 mod p(x)`, x^1728 mod p(x)`, x^1760 mod p(x)`, x^1792 mod p(x)` */ + .octa 0x42316c00730206ad419a441956993a31 + + /* x^1568 mod p(x)`, x^1600 mod p(x)`, x^1632 mod p(x)`, x^1664 mod p(x)` */ + .octa 0x543d5c543e65ddf9924752ba2b830011 + + /* x^1440 mod p(x)`, x^1472 mod p(x)`, x^1504 mod p(x)`, x^1536 mod p(x)` */ + .octa 0x78e87aaf56767c9255bd7f9518e4a304 + + /* x^1312 mod p(x)`, x^1344 mod p(x)`, x^1376 mod p(x)`, x^1408 mod p(x)` */ + .octa 0x8f68fcec1903da7f6d76739fe0553f1e + + /* x^1184 mod p(x)`, x^1216 mod p(x)`, x^1248 mod p(x)`, x^1280 mod p(x)` */ + .octa 0x3f4840246791d588c133722b1fe0b5c3 + + /* x^1056 mod p(x)`, x^1088 mod p(x)`, x^1120 mod p(x)`, x^1152 mod p(x)` */ + .octa 0x34c96751b04de25a64b67ee0e55ef1f3 + + /* x^928 mod p(x)`, x^960 mod p(x)`, x^992 mod p(x)`, x^1024 mod p(x)` */ + .octa 0x156c8e180b4a395b069db049b8fdb1e7 + + /* x^800 mod p(x)`, x^832 mod p(x)`, x^864 mod p(x)`, x^896 mod p(x)` */ + .octa 0xe0b99ccbe661f7bea11bfaf3c9e90b9e + + /* x^672 mod p(x)`, x^704 mod p(x)`, x^736 mod p(x)`, x^768 mod p(x)` */ + .octa 0x041d37768cd75659817cdc5119b29a35 + + /* x^544 mod p(x)`, x^576 mod p(x)`, x^608 mod p(x)`, x^640 mod p(x)` */ + .octa 0x3a0777818cfaa9651ce9d94b36c41f1c + + /* x^416 mod p(x)`, x^448 mod p(x)`, x^480 mod p(x)`, x^512 mod p(x)` */ + .octa 0x0e148e8252377a554f256efcb82be955 + + /* x^288 mod p(x)`, x^320 mod p(x)`, x^352 mod p(x)`, x^384 mod p(x)` */ + .octa 0x9c25531d19e65ddeec1631edb2dea967 + + /* x^160 mod p(x)`, x^192 mod p(x)`, x^224 mod p(x)`, x^256 mod p(x)` */ + .octa 0x790606ff9957c0a65d27e147510ac59a + + /* x^32 mod p(x)`, x^64 mod p(x)`, x^96 mod p(x)`, x^128 mod p(x)` */ + .octa 0x82f63b786ea2d55ca66805eb18b8ea18 + + +.barrett_constants: + /* 33 bit reflected Barrett constant m - (4^32)/n */ + .octa 0x000000000000000000000000dea713f1 /* x^64 div p(x)` */ + /* 33 bit reflected Barrett constant n */ + .octa 0x00000000000000000000000105ec76f1 +#endif diff --git a/src/support/power8/crc32_wrapper.c b/src/support/power8/crc32_wrapper.c new file mode 100644 index 00000000000..3d466cac013 --- /dev/null +++ b/src/support/power8/crc32_wrapper.c @@ -0,0 +1,64 @@ +#define CRC_TABLE +#include "crc32_constants.h" + +#define VMX_ALIGN 16 +#define VMX_ALIGN_MASK (VMX_ALIGN-1) + +#ifdef REFLECT +static unsigned int crc32_align(unsigned int crc, unsigned char *p, + unsigned long len) +{ + while (len--) + crc = crc_table[(crc ^ *p++) & 0xff] ^ (crc >> 8); + return crc; +} +#else +static unsigned int crc32_align(unsigned int crc, unsigned char *p, + unsigned long len) +{ + while (len--) + crc = crc_table[((crc >> 24) ^ *p++) & 0xff] ^ (crc << 8); + return crc; +} +#endif + +unsigned int __crc32_vpmsum(unsigned int crc, unsigned char *p, + unsigned long len); + +unsigned int crc32_vpmsum(unsigned int crc, unsigned char *p, + unsigned long len) +{ + unsigned int prealign; + unsigned int tail; + +#ifdef CRC_XOR + crc ^= 0xffffffff; +#endif + + if (len < VMX_ALIGN + VMX_ALIGN_MASK) { + crc = crc32_align(crc, p, len); + goto out; + } + + if ((unsigned long)p & VMX_ALIGN_MASK) { + prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK); + crc = crc32_align(crc, p, prealign); + len -= prealign; + p += prealign; + } + + crc = __crc32_vpmsum(crc, p, len & ~VMX_ALIGN_MASK); + + tail = len & VMX_ALIGN_MASK; + if (tail) { + p += len & ~VMX_ALIGN_MASK; + crc = crc32_align(crc, p, tail); + } + +out: +#ifdef CRC_XOR + crc ^= 0xffffffff; +#endif + + return crc; +} diff --git a/src/support/power8/ppc-opcode.h b/src/support/power8/ppc-opcode.h new file mode 100644 index 00000000000..b63feea60a0 --- /dev/null +++ b/src/support/power8/ppc-opcode.h @@ -0,0 +1,23 @@ +#ifndef __OPCODES_H +#define __OPCODES_H + +#define __PPC_RA(a) (((a) & 0x1f) << 16) +#define __PPC_RB(b) (((b) & 0x1f) << 11) +#define __PPC_XA(a) ((((a) & 0x1f) << 16) | (((a) & 0x20) >> 3)) +#define __PPC_XB(b) ((((b) & 0x1f) << 11) | (((b) & 0x20) >> 4)) +#define __PPC_XS(s) ((((s) & 0x1f) << 21) | (((s) & 0x20) >> 5)) +#define __PPC_XT(s) __PPC_XS(s) +#define VSX_XX3(t, a, b) (__PPC_XT(t) | __PPC_XA(a) | __PPC_XB(b)) +#define VSX_XX1(s, a, b) (__PPC_XS(s) | __PPC_RA(a) | __PPC_RB(b)) + +#define PPC_INST_VPMSUMW 0x10000488 +#define PPC_INST_VPMSUMD 0x100004c8 +#define PPC_INST_MFVSRD 0x7c000066 +#define PPC_INST_MTVSRD 0x7c000166 + +#define VPMSUMW(t, a, b) .long PPC_INST_VPMSUMW | VSX_XX3((t), a, b) +#define VPMSUMD(t, a, b) .long PPC_INST_VPMSUMD | VSX_XX3((t), a, b) +#define MFVRD(a, t) .long PPC_INST_MFVSRD | VSX_XX1((t)+32, a, 0) +#define MTVRD(t, a) .long PPC_INST_MTVSRD | VSX_XX1((t)+32, a, 0) + +#endif -- cgit v1.2.1 From 1cd41bee33812da0bd8656527f90ced91ddb2a02 Mon Sep 17 00:00:00 2001 From: Mark Benvenuto Date: Tue, 9 Feb 2016 18:32:11 -0500 Subject: WT-2280 Build Integration --- build_posix/configure.ac.in | 1 + dist/filelist | 2 ++ dist/s_string.ok | 2 ++ dist/s_win | 2 +- src/docs/spell.ok | 4 +++ src/support/cksum.c | 19 ++++++++++++ src/support/power8/crc32.S | 19 ++++++++++++ src/support/power8/crc32_constants.h | 56 ++++++++++++++++++++++++++---------- src/support/power8/crc32_wrapper.c | 32 +++++++++++++++++---- src/support/power8/ppc-opcode.h | 51 +++++++++++++++++++++----------- 10 files changed, 149 insertions(+), 39 deletions(-) diff --git a/build_posix/configure.ac.in b/build_posix/configure.ac.in index 875c8b436a8..06d73e2fe12 100644 --- a/build_posix/configure.ac.in +++ b/build_posix/configure.ac.in @@ -32,6 +32,7 @@ AC_SUBST([LIBTOOL_DEPS]) AC_PROG_CC(cc gcc) AC_PROG_CXX(c++ g++) +AM_PROG_AS(as gas) if test "$GCC" = "yes"; then # The Solaris gcc compiler gets the additional -pthreads flag. diff --git a/dist/filelist b/dist/filelist index edd59435841..71843f909ff 100644 --- a/dist/filelist +++ b/dist/filelist @@ -166,6 +166,8 @@ src/support/pow.c src/support/rand.c src/support/scratch.c src/support/stat.c +src/support/power8/crc32.S +src/support/power8/crc32_wrapper.c src/txn/txn.c src/txn/txn_ckpt.c src/txn/txn_ext.c diff --git a/dist/s_string.ok b/dist/s_string.ok index 19fa27cd719..63b15af1370 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -686,6 +686,7 @@ jnr jrx json kb +kbits keycmp keyid keyv @@ -838,6 +839,7 @@ pos posint posix postsize +powerpc pragmas pre prealloc diff --git a/dist/s_win b/dist/s_win index 1eb4702d517..40b605acaff 100755 --- a/dist/s_win +++ b/dist/s_win @@ -73,7 +73,7 @@ win_filelist() -e 's;os_posix/os_time.c;os_win/os_time.c;' \ -e 's;os_posix/os_yield.c;os_win/os_yield.c;' echo 'src/os_win/os_snprintf.c' - echo 'src/os_win/os_vsnprintf.c') < filelist | sort > $t + echo 'src/os_win/os_vsnprintf.c') < filelist | grep -v power8 | sort > $t cmp $t $f > /dev/null 2>&1 || (echo "Building $f" && rm -f $f && cp $t $f) } diff --git a/src/docs/spell.ok b/src/docs/spell.ok index 80597302cbb..ac71214f8b1 100644 --- a/src/docs/spell.ok +++ b/src/docs/spell.ok @@ -7,11 +7,13 @@ Atomicity BLOBs CFLAGS CPPFLAGS +CRC Cheng Christoph Collet's Coverity Coverity's +crc DB's DBTs Datastore @@ -64,6 +66,7 @@ NOTFOUND NUMA NoSQL OPTYPE +PPC PRELOAD README Rebalance @@ -78,6 +81,7 @@ Seward's SiH TXT URIs +vpmsum WiredTiger WiredTiger's WiredTigerCheckpoint diff --git a/src/support/cksum.c b/src/support/cksum.c index c2982c40015..0b086753406 100644 --- a/src/support/cksum.c +++ b/src/support/cksum.c @@ -1260,6 +1260,23 @@ __wt_cksum_hw(const void *chunk, size_t len) } #endif +#if defined(__powerpc64__) + +unsigned int crc32_vpmsum(unsigned int crc, const unsigned char *p, + unsigned long len); + +/* + * __wt_cksum_hw -- + * Return a checksum for a chunk of memory, computed in hardware + * using 8 byte steps. + */ +static uint32_t +__wt_cksum_hw(const void *chunk, size_t len) +{ + return crc32_vpmsum(0, chunk, len); +} +#endif + /* * __wt_cksum -- * Return a checksum for a chunk of memory using the fastest method @@ -1302,6 +1319,8 @@ __wt_cksum_init(void) __wt_cksum_func = __wt_cksum_hw; else __wt_cksum_func = __wt_cksum_sw; +#elif defined(__powerpc64__) + __wt_cksum_func = __wt_cksum_hw; #else __wt_cksum_func = __wt_cksum_sw; #endif diff --git a/src/support/power8/crc32.S b/src/support/power8/crc32.S index 621125b67c6..16990cc6dbc 100644 --- a/src/support/power8/crc32.S +++ b/src/support/power8/crc32.S @@ -1,3 +1,21 @@ +/*- + * Copyright 2016 MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Local modifications to add __powerpc64__ conditional defines. + */ +#if defined(__powerpc64__) /* * Calculate the checksum of data that is 16 byte aligned and a multiple of * 16 bytes. @@ -737,3 +755,4 @@ FUNC_START(__crc32_vpmsum) mr r3,r10 blr FUNC_END(__crc32_vpmsum) +#endif diff --git a/src/support/power8/crc32_constants.h b/src/support/power8/crc32_constants.h index 02c471d1c56..c407c08974b 100644 --- a/src/support/power8/crc32_constants.h +++ b/src/support/power8/crc32_constants.h @@ -1,6 +1,23 @@ -#define CRC 0x1edc6f41 -#define CRC_XOR -#define REFLECT +/*- + * Copyright 2016 MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Local modifications to format long lines. + */ +#define CRC 0x1edc6f41 +#define CRC_XOR +#define REFLECT #ifndef __ASSEMBLY__ #ifdef CRC_TABLE @@ -72,7 +89,7 @@ static const unsigned int crc_table[] = { #endif #else -#define MAX_SIZE 32768 +#define MAX_SIZE 32768 .constants: /* Reduce 262144 kbits to 1024 bits */ @@ -843,32 +860,42 @@ static const unsigned int crc_table[] = { .short_constants: - /* Reduce final 1024-2048 bits to 64 bits, shifting 32 bits to include the trailing 32 bits of zeros */ - /* x^1952 mod p(x)`, x^1984 mod p(x)`, x^2016 mod p(x)`, x^2048 mod p(x)` */ + /* Reduce final 1024-2048 bits to 64 bits, shifting 32 bits to include + the trailing 32 bits of zeros */ + /* x^1952 mod p(x)`, x^1984 mod p(x)`, x^2016 mod p(x)`, + x^2048 mod p()`x */ .octa 0x7fec2963e5bf80485cf015c388e56f72 - /* x^1824 mod p(x)`, x^1856 mod p(x)`, x^1888 mod p(x)`, x^1920 mod p(x)` */ + /* x^1824 mod p(x)`, x^1856 mod p(x)`, x^1888 mod p(x)`, + x^1920 mod p(x)` */ .octa 0x38e888d4844752a9963a18920246e2e6 - /* x^1696 mod p(x)`, x^1728 mod p(x)`, x^1760 mod p(x)`, x^1792 mod p(x)` */ + /* x^1696 mod p(x)`, x^1728 mod p(x)`, x^1760 mod p(x)`, + x^1792 mod p(x)` */ .octa 0x42316c00730206ad419a441956993a31 - /* x^1568 mod p(x)`, x^1600 mod p(x)`, x^1632 mod p(x)`, x^1664 mod p(x)` */ + /* x^1568 mod p(x)`, x^1600 mod p(x)`, x^1632 mod p(x)`, + x^1664 mod p(x)` */ .octa 0x543d5c543e65ddf9924752ba2b830011 - /* x^1440 mod p(x)`, x^1472 mod p(x)`, x^1504 mod p(x)`, x^1536 mod p(x)` */ + /* x^1440 mod p(x)`, x^1472 mod p(x)`, x^1504 mod p(x)`, + x^1536 mod p(x)` */ .octa 0x78e87aaf56767c9255bd7f9518e4a304 - /* x^1312 mod p(x)`, x^1344 mod p(x)`, x^1376 mod p(x)`, x^1408 mod p(x)` */ + /* x^1312 mod p(x)`, x^1344 mod p(x)`, x^1376 mod p(x)`, + x^1408 mod p(x)` */ .octa 0x8f68fcec1903da7f6d76739fe0553f1e - /* x^1184 mod p(x)`, x^1216 mod p(x)`, x^1248 mod p(x)`, x^1280 mod p(x)` */ + /* x^1184 mod p(x)`, x^1216 mod p(x)`, x^1248 mod p(x)`, + x^1280 mod p(x)` */ .octa 0x3f4840246791d588c133722b1fe0b5c3 - /* x^1056 mod p(x)`, x^1088 mod p(x)`, x^1120 mod p(x)`, x^1152 mod p(x)` */ + /* x^1056 mod p(x)`, x^1088 mod p(x)`, x^1120 mod p(x)`, + x^1152 mod p(x)` */ .octa 0x34c96751b04de25a64b67ee0e55ef1f3 - /* x^928 mod p(x)`, x^960 mod p(x)`, x^992 mod p(x)`, x^1024 mod p(x)` */ + /* x^928 mod p(x)`, x^960 mod p(x)`, x^992 mod p(x)`, + x^1024 mod p(x)` */ .octa 0x156c8e180b4a395b069db049b8fdb1e7 /* x^800 mod p(x)`, x^832 mod p(x)`, x^864 mod p(x)`, x^896 mod p(x)` */ @@ -892,7 +919,6 @@ static const unsigned int crc_table[] = { /* x^32 mod p(x)`, x^64 mod p(x)`, x^96 mod p(x)`, x^128 mod p(x)` */ .octa 0x82f63b786ea2d55ca66805eb18b8ea18 - .barrett_constants: /* 33 bit reflected Barrett constant m - (4^32)/n */ .octa 0x000000000000000000000000dea713f1 /* x^64 div p(x)` */ diff --git a/src/support/power8/crc32_wrapper.c b/src/support/power8/crc32_wrapper.c index 3d466cac013..bb2dd0c70df 100644 --- a/src/support/power8/crc32_wrapper.c +++ b/src/support/power8/crc32_wrapper.c @@ -1,8 +1,27 @@ -#define CRC_TABLE +/*- + * Copyright 2016 MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Local modifications to add __powerpc64__ conditional defines, and + * code formatting fixes. + */ +#if defined(__powerpc64__) +#define CRC_TABLE #include "crc32_constants.h" -#define VMX_ALIGN 16 -#define VMX_ALIGN_MASK (VMX_ALIGN-1) +#define VMX_ALIGN 16 +#define VMX_ALIGN_MASK (VMX_ALIGN-1) #ifdef REFLECT static unsigned int crc32_align(unsigned int crc, unsigned char *p, @@ -10,7 +29,7 @@ static unsigned int crc32_align(unsigned int crc, unsigned char *p, { while (len--) crc = crc_table[(crc ^ *p++) & 0xff] ^ (crc >> 8); - return crc; + return (crc); } #else static unsigned int crc32_align(unsigned int crc, unsigned char *p, @@ -18,7 +37,7 @@ static unsigned int crc32_align(unsigned int crc, unsigned char *p, { while (len--) crc = crc_table[((crc >> 24) ^ *p++) & 0xff] ^ (crc << 8); - return crc; + return (crc); } #endif @@ -60,5 +79,6 @@ out: crc ^= 0xffffffff; #endif - return crc; + return (crc); } +#endif diff --git a/src/support/power8/ppc-opcode.h b/src/support/power8/ppc-opcode.h index b63feea60a0..40058b0d50d 100644 --- a/src/support/power8/ppc-opcode.h +++ b/src/support/power8/ppc-opcode.h @@ -1,23 +1,40 @@ +/*- + * Copyright 2016 MongoDB, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Local modifications to format for style. + */ #ifndef __OPCODES_H -#define __OPCODES_H +#define __OPCODES_H -#define __PPC_RA(a) (((a) & 0x1f) << 16) -#define __PPC_RB(b) (((b) & 0x1f) << 11) -#define __PPC_XA(a) ((((a) & 0x1f) << 16) | (((a) & 0x20) >> 3)) -#define __PPC_XB(b) ((((b) & 0x1f) << 11) | (((b) & 0x20) >> 4)) -#define __PPC_XS(s) ((((s) & 0x1f) << 21) | (((s) & 0x20) >> 5)) -#define __PPC_XT(s) __PPC_XS(s) -#define VSX_XX3(t, a, b) (__PPC_XT(t) | __PPC_XA(a) | __PPC_XB(b)) -#define VSX_XX1(s, a, b) (__PPC_XS(s) | __PPC_RA(a) | __PPC_RB(b)) +#define __PPC_RA(a) (((a) & 0x1f) << 16) +#define __PPC_RB(b) (((b) & 0x1f) << 11) +#define __PPC_XA(a) ((((a) & 0x1f) << 16) | (((a) & 0x20) >> 3)) +#define __PPC_XB(b) ((((b) & 0x1f) << 11) | (((b) & 0x20) >> 4)) +#define __PPC_XS(s) ((((s) & 0x1f) << 21) | (((s) & 0x20) >> 5)) +#define __PPC_XT(s) __PPC_XS(s) +#define VSX_XX3(t, a, b) (__PPC_XT(t) | __PPC_XA(a) | __PPC_XB(b)) +#define VSX_XX1(s, a, b) (__PPC_XS(s) | __PPC_RA(a) | __PPC_RB(b)) -#define PPC_INST_VPMSUMW 0x10000488 -#define PPC_INST_VPMSUMD 0x100004c8 -#define PPC_INST_MFVSRD 0x7c000066 -#define PPC_INST_MTVSRD 0x7c000166 +#define PPC_INST_VPMSUMW 0x10000488 +#define PPC_INST_VPMSUMD 0x100004c8 +#define PPC_INST_MFVSRD 0x7c000066 +#define PPC_INST_MTVSRD 0x7c000166 -#define VPMSUMW(t, a, b) .long PPC_INST_VPMSUMW | VSX_XX3((t), a, b) -#define VPMSUMD(t, a, b) .long PPC_INST_VPMSUMD | VSX_XX3((t), a, b) -#define MFVRD(a, t) .long PPC_INST_MFVSRD | VSX_XX1((t)+32, a, 0) -#define MTVRD(t, a) .long PPC_INST_MTVSRD | VSX_XX1((t)+32, a, 0) +#define VPMSUMW(t, a, b) .long PPC_INST_VPMSUMW | VSX_XX3((t), a, b) +#define VPMSUMD(t, a, b) .long PPC_INST_VPMSUMD | VSX_XX3((t), a, b) +#define MFVRD(a, t) .long PPC_INST_MFVSRD | VSX_XX1((t)+32, a, 0) +#define MTVRD(t, a) .long PPC_INST_MTVSRD | VSX_XX1((t)+32, a, 0) #endif -- cgit v1.2.1 From 7962cd7e7e25310b38825de33e9c0f45b98b1ea5 Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Wed, 10 Feb 2016 20:44:41 -0500 Subject: WT-2382. Style fix for ternary operators. --- src/cursor/cur_join.c | 8 ++++---- test/suite/test_join03.py | 15 ++------------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/src/cursor/cur_join.c b/src/cursor/cur_join.c index e9ffea813ac..556b6f7ea2a 100644 --- a/src/cursor/cur_join.c +++ b/src/cursor/cur_join.c @@ -319,8 +319,8 @@ __curjoin_init_bloom(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, */ WT_ERR(__wt_struct_repack(session, cindex->child->key_format, - (entry->repack_format ? entry->repack_format : - cindex->iface.key_format), + (entry->repack_format != NULL ? + entry->repack_format : cindex->iface.key_format), &cindex->child->key, &curkey)); } for (end = &entry->ends[skip]; end < endmax; end++) { @@ -381,8 +381,8 @@ __curjoin_endpoint_init_key(WT_SESSION_IMPL *session, cindex = (WT_CURSOR_INDEX *)endpoint->cursor; WT_ERR(__wt_struct_repack(session, cindex->child->key_format, - (entry->repack_format ? entry->repack_format : - cindex->iface.key_format), + (entry->repack_format != NULL ? + entry->repack_format : cindex->iface.key_format), &cindex->child->key, &endpoint->key)); } else { k = &((WT_CURSOR_TABLE *)cursor)->cg_cursors[0]->key; diff --git a/test/suite/test_join03.py b/test/suite/test_join03.py index d7b5925f922..613d2396b07 100644 --- a/test/suite/test_join03.py +++ b/test/suite/test_join03.py @@ -37,11 +37,6 @@ class test_join03(wttest.WiredTigerTestCase): table_name1 = 'test_join03' nentries = 100 - scenarios = [ - ('no-collator', dict(collate=False)), -# ('collator', dict(collate=True)) - ] - # Return the wiredtiger_open extension argument for a shared library. def extensionArg(self, exts): extfiles = [] @@ -63,9 +58,7 @@ class test_join03(wttest.WiredTigerTestCase): # Override WiredTigerTestCase, we have extensions. def setUpConnectionOpen(self, dir): - extarg = self.extensionArg([ - ('extractors', 'csv', 'csv_extractor'), - ('collators', 'revint', 'revint_collator')]) + extarg = self.extensionArg([('extractors', 'csv', 'csv_extractor')]) connarg = 'create,error_prefix="{0}: ",{1}'.format( self.shortid(), extarg) conn = self.wiredtiger_open(dir, connarg) @@ -97,18 +90,14 @@ class test_join03(wttest.WiredTigerTestCase): # Common function for testing the most basic functionality # of joins def join(self, csvformat, args0, args1): - collator = 'collator=revint,' if self.collate else '' - - self.session.create('table:join03', 'key_format=i' + + self.session.create('table:join03', 'key_format=r' + ',value_format=S,columns=(k,v)') fmt = csvformat[0] self.session.create('index:join03:index0','key_format=' + fmt + ',' + - (collator if fmt == 'i' else '') + 'extractor=csv,app_metadata={"format" : "' + fmt + '","field" : "0"}') fmt = csvformat[1] self.session.create('index:join03:index1','key_format=' + fmt + ',' + - (collator if fmt == 'i' else '') + 'extractor=csv,app_metadata={"format" : "' + fmt + '","field" : "1"}') -- cgit v1.2.1 From df2fc002e51251043ca4c4c358cbd47353965586 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Thu, 11 Feb 2016 16:32:14 +1100 Subject: WT-2346 Review feedback: avoid locked handle on OOM. Allocate memory before getting a handle so that if the memory allocation fails, the handle isn't left refrenced. --- src/txn/txn_ckpt.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/txn/txn_ckpt.c b/src/txn/txn_ckpt.c index fb4be465b82..99c47663970 100644 --- a/src/txn/txn_ckpt.c +++ b/src/txn/txn_ckpt.c @@ -243,6 +243,10 @@ __wt_checkpoint_list(WT_SESSION_IMPL *session, const char *cfg[]) if (F_ISSET(S2BT(session), WT_BTREE_NO_CHECKPOINT)) return (0); + /* Make sure there is space for the next entry. */ + WT_RET(__wt_realloc_def(session, &session->ckpt_handle_allocated, + session->ckpt_handle_next + 1, &session->ckpt_handle)); + /* Not strictly necessary, but cleaner to clear the current handle. */ name = session->dhandle->name; session->dhandle = NULL; @@ -250,9 +254,6 @@ __wt_checkpoint_list(WT_SESSION_IMPL *session, const char *cfg[]) if ((ret = __wt_session_get_btree(session, name, NULL, NULL, 0)) != 0) return (ret == EBUSY ? 0 : ret); - /* Make sure there is space for the next entry. */ - WT_RET(__wt_realloc_def(session, &session->ckpt_handle_allocated, - session->ckpt_handle_next + 1, &session->ckpt_handle)); session->ckpt_handle[session->ckpt_handle_next++] = session->dhandle; return (0); } -- cgit v1.2.1 From 8120bd2c39bbdf5a6bcf6e39f70c34cb8784587a Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Thu, 11 Feb 2016 16:51:00 +1100 Subject: WT-2346 More review feedback. Add upgrading notes, clarify a comment, revert a change to remove a (performance-related) sync during checkpoints. --- src/docs/upgrading.dox | 35 +++++++++++++++++++++-------------- src/txn/txn_ckpt.c | 13 ++++++++++++- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/src/docs/upgrading.dox b/src/docs/upgrading.dox index e0239919f0b..f5d669e681f 100644 --- a/src/docs/upgrading.dox +++ b/src/docs/upgrading.dox @@ -4,29 +4,36 @@
Column-store bulk-load cursors
-Historically, bulk-load of a column-store object ignored any key set in -the cursor and automatically assigned each inserted row the next -sequential record number for its key. In the 2.7.1 release, column-store -objects match row-store behavior and require the cursor key be set -before an insert. (This also allows allows sparse tables to be created -in column-store objects, any skipped records are created as -already-deleted rows.) To match the previous behavior, specify the -\c append configuration string when opening the column-store bulk-load -cursor; this causes the cursor's key to be ignored and each inserted row -will be assigned the next record number. +Historically, bulk-load of a column-store object ignored any key set in the +cursor and automatically assigned each inserted row the next sequential +record number for its key. In the 2.7.1 release, column-store objects match +row-store behavior and require the cursor key be set before an insert. +(This allows sparse tables to be created in column-store objects, any +skipped records are created as already-deleted rows.) To match the previous +behavior, specify the \c append configuration string when opening the +column-store bulk-load cursor; this causes the cursor's key to be ignored +and each inserted row will be assigned the next record number.
Change to WT_SESSION::truncate with URI
If using the WT_SESSION::truncate API with a file: URI for a full table -truncate, underlying algorithmic changes result in some visible differences. -This call can now return WT_ROLLBACK. Applications should be prepared to -handle this error. This method no longer requires exclusive access to the -table. Also the underlying disk space may not be immediately +truncate, underlying algorithmic changes result in some visible +differences. This call can now return WT_ROLLBACK. Applications should be +prepared to handle this error. This method no longer requires exclusive +access to the table. Also the underlying disk space may not be immediately reclaimed when the call returns. The performance of this API may differ from earlier releases.
+
Change to named checkpoints with bulk loads
+
+Previous versions of WiredTiger created empty named checkpoints in files +being bulk-loaded. In this release, checkpoint skips files being +bulk-loaded, so they do not get named checkpoints that complete during the +bulk load. +
+
@section version_270 Upgrading to Version 2.7.0 diff --git a/src/txn/txn_ckpt.c b/src/txn/txn_ckpt.c index 99c47663970..10ca033e881 100644 --- a/src/txn/txn_ckpt.c +++ b/src/txn/txn_ckpt.c @@ -401,6 +401,16 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) session->isolation = txn->isolation = WT_ISO_READ_COMMITTED; WT_ERR(__checkpoint_apply(session, cfg, __checkpoint_write_leaves)); + /* + * The underlying flush routine scheduled an asynchronous flush + * after writing the leaf pages, but in order to minimize I/O + * while holding the schema lock, do a flush and wait for the + * completion. Do it after flushing the pages to give the + * asynchronous flush as much time as possible before we wait. + */ + if (F_ISSET(conn, WT_CONN_CKPT_SYNC)) + WT_ERR(__checkpoint_apply(session, cfg, __wt_checkpoint_sync)); + /* Start the checkpoint for real. */ WT_ERR(__wt_meta_track_on(session)); tracking = true; @@ -521,7 +531,8 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) * updated by full checkpoints so only checkpoint the metadata for * full or non-logged checkpoints. * - * XXX lots of overlap with __wt_meta_track_off. + * This is very similar to __wt_meta_track_off, ideally they would be + * merged. */ if (full || !logging) { session->isolation = txn->isolation = WT_ISO_READ_UNCOMMITTED; -- cgit v1.2.1 From db58fef15da999a34764b5da4ae6b016f1b8529c Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 11 Feb 2016 10:29:27 -0500 Subject: WT-2173: test/format cache stuck full If the eviction server has no helping threads, try and evict all of the candidate pages. --- src/evict/evict_lru.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index 8ef7164dbc6..fe503c0777c 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -1369,10 +1369,12 @@ __evict_get_ref( WT_SESSION_IMPL *session, bool is_server, WT_BTREE **btreep, WT_REF **refp) { WT_CACHE *cache; + WT_CONNECTION_IMPL *conn; WT_EVICT_ENTRY *evict; uint32_t candidates; - cache = S2C(session)->cache; + conn = S2C(session); + cache = conn->cache; *btreep = NULL; *refp = NULL; @@ -1392,11 +1394,14 @@ __evict_get_ref( } /* - * The eviction server only tries to evict half of the pages before - * looking for more. + * The eviction server normally only tries to evict half of the pages + * before looking for more, unless there isn't anybody else to do the + * work, we're stuck, or there's only a single candidate (that happens + * in very small environments). */ candidates = cache->evict_candidates; - if (is_server && candidates > 1) + if (is_server && conn->evict_workers_max != 0 && + !F_ISSET(cache, WT_CACHE_STUCK) && candidates > 1) candidates /= 2; /* Get the next page queued for eviction. */ -- cgit v1.2.1 From 98d083dab3059ded91c186575f1e84018ca17cc4 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 11 Feb 2016 10:30:33 -0500 Subject: WT-2173: test/format cache stuck full Revert part of 5eb3df7, allow appplication threads to write into the lookaside file once the cache is stuck. --- src/evict/evict_page.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/evict/evict_page.c b/src/evict/evict_page.c index 72c07eaa05d..f0d4752cc83 100644 --- a/src/evict/evict_page.c +++ b/src/evict/evict_page.c @@ -471,8 +471,7 @@ __evict_review( LF_SET(WT_EVICT_IN_MEMORY | WT_EVICT_UPDATE_RESTORE); else if (page->read_gen == WT_READGEN_OLDEST) LF_SET(WT_EVICT_UPDATE_RESTORE); - else if (F_ISSET(session, WT_SESSION_INTERNAL) && - F_ISSET(S2C(session)->cache, WT_CACHE_STUCK)) + else if (F_ISSET(S2C(session)->cache, WT_CACHE_STUCK)) LF_SET(WT_EVICT_LOOKASIDE); } -- cgit v1.2.1 From b3b51ccd136588515a4dde04258a7c83d79891a3 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 11 Feb 2016 11:09:48 -0500 Subject: WT-2382: Problem with custom collator for 'u' format with join cursor Commit a6c183b removed the only caller of __wt_struct_unpack_size. --- src/include/extern.h | 1 - src/packing/pack_impl.c | 30 ------------------------------ 2 files changed, 31 deletions(-) diff --git a/src/include/extern.h b/src/include/extern.h index 92ae968affd..a8f20176ad6 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -552,7 +552,6 @@ extern int __wt_struct_confchk(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *v); extern int __wt_struct_size(WT_SESSION_IMPL *session, size_t *sizep, const char *fmt, ...); extern int __wt_struct_pack(WT_SESSION_IMPL *session, void *buffer, size_t size, const char *fmt, ...); extern int __wt_struct_unpack(WT_SESSION_IMPL *session, const void *buffer, size_t size, const char *fmt, ...); -extern int __wt_struct_unpack_size(WT_SESSION_IMPL *session, const void *buffer, size_t size, const char *fmt, size_t *resultp); extern int __wt_struct_repack(WT_SESSION_IMPL *session, const char *infmt, const char *outfmt, const WT_ITEM *inbuf, WT_ITEM *outbuf); extern int __wt_ovfl_discard_add(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL *cell); extern void __wt_ovfl_discard_free(WT_SESSION_IMPL *session, WT_PAGE *page); diff --git a/src/packing/pack_impl.c b/src/packing/pack_impl.c index bd1c90525a6..7042fa1d857 100644 --- a/src/packing/pack_impl.c +++ b/src/packing/pack_impl.c @@ -106,36 +106,6 @@ __wt_struct_unpack(WT_SESSION_IMPL *session, return (ret); } -/* - * __wt_struct_unpack_size -- - * Determine the packed size of a buffer matching the format. - */ -int -__wt_struct_unpack_size(WT_SESSION_IMPL *session, - const void *buffer, size_t size, const char *fmt, size_t *resultp) -{ - WT_DECL_PACK_VALUE(pv); - WT_DECL_RET; - WT_PACK pack; - const uint8_t *p, *end; - - p = buffer; - end = p + size; - - WT_RET(__pack_init(session, &pack, fmt)); - while ((ret = __pack_next(&pack, &pv)) == 0) - WT_RET(__unpack_read(session, &pv, &p, (size_t)(end - p))); - - /* Be paranoid - __pack_write should never overflow. */ - WT_ASSERT(session, p <= end); - - if (ret != WT_NOTFOUND) - return (ret); - - *resultp = WT_PTRDIFF(p, buffer); - return (0); -} - /* * __wt_struct_repack -- * Return the subset of the packed buffer that represents part of -- cgit v1.2.1 From 57a2850705431dd4b6cbde841e912a92fcb0bbe8 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 11 Feb 2016 11:33:32 -0500 Subject: WT-2393: Unnecessary error handling labels. Remove a few unnecessary error handling labels. --- src/cursor/cur_join.c | 42 ++++++++++++++++-------------------------- src/cursor/cur_stat.c | 6 ++---- src/packing/pack_impl.c | 16 ++++++++-------- 3 files changed, 26 insertions(+), 38 deletions(-) diff --git a/src/cursor/cur_join.c b/src/cursor/cur_join.c index 556b6f7ea2a..797e6e5879a 100644 --- a/src/cursor/cur_join.c +++ b/src/cursor/cur_join.c @@ -74,18 +74,16 @@ static int __curjoin_pack_recno(WT_SESSION_IMPL *session, uint64_t r, uint8_t *buf, size_t bufsize, WT_ITEM *item) { - WT_DECL_RET; WT_SESSION *wtsession; size_t sz; wtsession = (WT_SESSION *)session; - WT_ERR(wiredtiger_struct_size(wtsession, &sz, "r", r)); + WT_RET(wiredtiger_struct_size(wtsession, &sz, "r", r)); WT_ASSERT(session, sz < bufsize); - WT_ERR(wiredtiger_struct_pack(wtsession, buf, bufsize, "r", r)); + WT_RET(wiredtiger_struct_pack(wtsession, buf, bufsize, "r", r)); item->size = sz; item->data = buf; - -err: return (ret); + return (0); } /* @@ -99,12 +97,11 @@ __curjoin_entry_iter_next(WT_CURSOR_JOIN_ITER *iter, WT_ITEM *primkey, { WT_CURSOR *firstcg_cur; WT_CURSOR_JOIN *cjoin; - WT_DECL_RET; WT_SESSION_IMPL *session; uint64_t r; if (iter->positioned) - WT_ERR(iter->cursor->next(iter->cursor)); + WT_RET(iter->cursor->next(iter->cursor)); else iter->positioned = true; @@ -121,7 +118,7 @@ __curjoin_entry_iter_next(WT_CURSOR_JOIN_ITER *iter, WT_ITEM *primkey, firstcg_cur = ((WT_CURSOR_TABLE *)iter->cursor)->cg_cursors[0]; if (WT_CURSOR_RECNO(&cjoin->iface)) { r = *(uint64_t *)firstcg_cur->key.data; - WT_ERR(__curjoin_pack_recno(session, r, cjoin->recno_buf, + WT_RET(__curjoin_pack_recno(session, r, cjoin->recno_buf, sizeof(cjoin->recno_buf), primkey)); *rp = r; } else { @@ -131,8 +128,7 @@ __curjoin_entry_iter_next(WT_CURSOR_JOIN_ITER *iter, WT_ITEM *primkey, iter->curkey = primkey; iter->entry->stats.actual_count++; iter->entry->stats.accesses++; - -err: return (ret); + return (0); } /* @@ -143,17 +139,14 @@ err: return (ret); static int __curjoin_entry_iter_reset(WT_CURSOR_JOIN_ITER *iter) { - WT_DECL_RET; - if (iter->positioned) { - WT_ERR(iter->cursor->reset(iter->cursor)); - WT_ERR(__wt_cursor_dup_position( + WT_RET(iter->cursor->reset(iter->cursor)); + WT_RET(__wt_cursor_dup_position( iter->cjoin->entries[0].ends[0].cursor, iter->cursor)); iter->positioned = false; iter->entry->stats.actual_count = 0; } - -err: return (ret); + return (0); } /* @@ -371,7 +364,6 @@ __curjoin_endpoint_init_key(WT_SESSION_IMPL *session, { WT_CURSOR *cursor; WT_CURSOR_INDEX *cindex; - WT_DECL_RET; WT_ITEM *k; uint64_t r; @@ -379,7 +371,7 @@ __curjoin_endpoint_init_key(WT_SESSION_IMPL *session, if (entry->index != NULL) { /* Extract and save the index's logical key. */ cindex = (WT_CURSOR_INDEX *)endpoint->cursor; - WT_ERR(__wt_struct_repack(session, + WT_RET(__wt_struct_repack(session, cindex->child->key_format, (entry->repack_format != NULL ? entry->repack_format : cindex->iface.key_format), @@ -388,7 +380,7 @@ __curjoin_endpoint_init_key(WT_SESSION_IMPL *session, k = &((WT_CURSOR_TABLE *)cursor)->cg_cursors[0]->key; if (WT_CURSOR_RECNO(cursor)) { r = *(uint64_t *)k->data; - WT_ERR(__curjoin_pack_recno(session, r, + WT_RET(__curjoin_pack_recno(session, r, endpoint->recno_buf, sizeof(endpoint->recno_buf), &endpoint->key)); @@ -397,8 +389,7 @@ __curjoin_endpoint_init_key(WT_SESSION_IMPL *session, endpoint->key = *k; } } - -err: return (ret); + return (0); } /* @@ -511,29 +502,28 @@ __curjoin_entry_in_range(WT_SESSION_IMPL *session, WT_CURSOR_JOIN_ENTRY *entry, { WT_COLLATOR *collator; WT_CURSOR_JOIN_ENDPOINT *end, *endmax; - WT_DECL_RET; int cmp; collator = (entry->index != NULL) ? entry->index->collator : NULL; endmax = &entry->ends[entry->ends_next]; for (end = &entry->ends[skip_left ? 1 : 0]; end < endmax; end++) { - WT_ERR(__wt_compare(session, collator, curkey, &end->key, + WT_RET(__wt_compare(session, collator, curkey, &end->key, &cmp)); if (!F_ISSET(end, WT_CURJOIN_END_LT)) { if (cmp < 0 || (cmp == 0 && !F_ISSET(end, WT_CURJOIN_END_EQ)) || (cmp > 0 && !F_ISSET(end, WT_CURJOIN_END_GT))) - WT_ERR(WT_NOTFOUND); + WT_RET(WT_NOTFOUND); } else { if (cmp > 0 || (cmp == 0 && !F_ISSET(end, WT_CURJOIN_END_EQ)) || (cmp < 0 && !F_ISSET(end, WT_CURJOIN_END_LT))) - WT_ERR(WT_NOTFOUND); + WT_RET(WT_NOTFOUND); } } -err: return (ret); + return (0); } typedef struct { diff --git a/src/cursor/cur_stat.c b/src/cursor/cur_stat.c index 00a6ade21c6..bb492c66ace 100644 --- a/src/cursor/cur_stat.c +++ b/src/cursor/cur_stat.c @@ -504,14 +504,13 @@ __curstat_join_init(WT_SESSION_IMPL *session, WT_CURSOR *curjoin, const char *cfg[], WT_CURSOR_STAT *cst) { WT_CURSOR_JOIN *cjoin; - WT_DECL_RET; WT_UNUSED(cfg); if (curjoin == NULL && cst->u.join_stats_group.join_cursor != NULL) curjoin = &cst->u.join_stats_group.join_cursor->iface; if (curjoin == NULL || !WT_PREFIX_MATCH(curjoin->uri, "join:")) - WT_ERR_MSG(session, EINVAL, + WT_RET_MSG(session, EINVAL, "join cursor must be used with statistics:join"); cjoin = (WT_CURSOR_JOIN *)curjoin; memset(&cst->u.join_stats_group, 0, sizeof(WT_JOIN_STATS_GROUP)); @@ -522,8 +521,7 @@ __curstat_join_init(WT_SESSION_IMPL *session, cst->stats_count = sizeof(WT_JOIN_STATS) / sizeof(int64_t); cst->stats_desc = __curstat_join_desc; cst->next_set = __curstat_join_next_set; - -err: return (ret); + return (0); } /* diff --git a/src/packing/pack_impl.c b/src/packing/pack_impl.c index bd1c90525a6..2bd850bfc9a 100644 --- a/src/packing/pack_impl.c +++ b/src/packing/pack_impl.c @@ -157,24 +157,24 @@ __wt_struct_repack(WT_SESSION_IMPL *session, const char *infmt, p = inbuf->data; end = p + inbuf->size; - WT_ERR(__pack_init(session, &packout, outfmt)); - WT_ERR(__pack_init(session, &packin, infmt)); + WT_RET(__pack_init(session, &packout, outfmt)); + WT_RET(__pack_init(session, &packin, infmt)); /* Outfmt should complete before infmt */ while ((ret = __pack_next(&packout, &pvout)) == 0) { if (p >= end) - WT_ERR(EINVAL); + WT_RET(EINVAL); if (pvout.type == 'x' && pvout.size == 0 && pvout.havesize) continue; - WT_ERR(__pack_next(&packin, &pvin)); + WT_RET(__pack_next(&packin, &pvin)); before = p; - WT_ERR(__unpack_read(session, &pvin, &p, (size_t)(end - p))); + WT_RET(__unpack_read(session, &pvin, &p, (size_t)(end - p))); if (pvout.type != pvin.type) - WT_ERR(ENOTSUP); + WT_RET(ENOTSUP); if (start == NULL) start = before; } - WT_ERR_NOTFOUND_OK(ret); + WT_RET_NOTFOUND_OK(ret); /* Be paranoid - __pack_write should never overflow. */ WT_ASSERT(session, p <= end); @@ -182,5 +182,5 @@ __wt_struct_repack(WT_SESSION_IMPL *session, const char *infmt, outbuf->data = start; outbuf->size = WT_PTRDIFF(p, start); -err: return (ret); + return (0); } -- cgit v1.2.1 From 6b5b22e5ddaf259c5c1b5d2308b7688cbb29a447 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 11 Feb 2016 12:24:22 -0500 Subject: WT-2280: Add CRC32 Optimized code for PPC64LE Consistency pass over the license file, make all of the 'optional' sections appear the same. --- src/docs/license.dox | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/src/docs/license.dox b/src/docs/license.dox index fa78fa1e02c..d7814d04fd6 100644 --- a/src/docs/license.dox +++ b/src/docs/license.dox @@ -2,10 +2,10 @@ The complete WiredTiger software package is Open Source software: you are welcome to modify and redistribute it under the terms of - -version 2 or - -version 3 of the +version 2 +or +version 3 +of the GNU General Public License as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY @@ -31,10 +31,10 @@ those described above, or for technical support for this software, please contact MongoDB, Inc. at info@wiredtiger.com. -@section license_library 3rd party software included in the WiredTiger library +@section license_library 3rd party software always included in the WiredTiger library Every build of the WiredTiger library binary includes the following 3rd -party software, distributed under their license terms. Redistribution +party software, distributed under separate license terms. Redistribution of the WiredTiger library should comply with these copyrights. @@ -46,23 +46,26 @@ of the WiredTiger library should comply with these copyrights. @row{\c src/support/hash_fnv.c, Authors, Public Domain}
-@section license_crc32-vpmsum 3rd party software included in the WiredTiger library on PPC64: crc32-vpmsum +@section license_crc32-vpmsum 3rd party software optionally included in the WiredTiger library: PPC64 -The PPC64 and PPC64LE builds of WiredTiger include the 3rd party software component "crc32-vpmsum" to -use POWER8 hardware acceleration for CRC32. This library is dual-licensed -under GNU General Public License - version 2 or later -and -Apache License 2.0 . See src/support/power8/LICENSE.TXT. +PPC64 and PPC64LE builds of the WiredTiger library binary include additional +3rd party software, distributed under separate license terms. Redistribution +of the WiredTiger library PPC64 and PPC64LE builds should comply with these +copyrights. + + +@hrow{Distribution Files, Copyright Holder, License} +@row{\c src/support/power8/*, Anton Blanchard, Apache License\, Version 2.0 or the GNU General Public License\, version 2 or later} +
@section license_leveldb 3rd party software optionally included in the WiredTiger library: LevelDB If the \c --enable-leveldb configuration option is specified when configuring the WiredTiger build, additional 3rd party software is -included in the WiredTiger LevelDB library binary, distributed under -their license terms. Redistribution of the WiredTiger library built -with the \c --enable-leveldb configuration option should comply with -these copyrights. +included in the WiredTiger library binary, distributed under separate +license terms. Redistribution of the WiredTiger library built with the +\c --enable-leveldb configuration option should comply with these +copyrights. @hrow{Distribution Files, Copyright Holder, License} -- cgit v1.2.1 From bc72e3436c3bffaf7ee89a20d4450c0a8c7e5274 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 11 Feb 2016 12:35:04 -0500 Subject: WT-2280: Add CRC32 Optimized code for PPC64LE The PPC files are POSIX-only, switch to a model where we discard POSIX-only files from the Windows build by name. --- dist/s_win | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dist/s_win b/dist/s_win index 40b605acaff..0b7d5184037 100755 --- a/dist/s_win +++ b/dist/s_win @@ -44,7 +44,7 @@ win_filelist() f='../build_win/filelist.win' # Process the files for which there's a Windows-specific version, then - # append Windows-only files. (There aren't yet any POSIX-only files.) + # append Windows-only files and discard POSIX-only files. (sed \ -e 's;os_posix/os_dir.c;os_win/os_dir.c;' \ -e 's;os_posix/os_dlopen.c;os_win/os_dlopen.c;' \ @@ -71,9 +71,11 @@ win_filelist() -e 's;os_posix/os_sleep.c;os_win/os_sleep.c;' \ -e 's;os_posix/os_thread.c;os_win/os_thread.c;' \ -e 's;os_posix/os_time.c;os_win/os_time.c;' \ - -e 's;os_posix/os_yield.c;os_win/os_yield.c;' + -e 's;os_posix/os_yield.c;os_win/os_yield.c;' \ + -e '/src\/support\/power8\/crc32.S/d' \ + -e '/src\/support\/power8\/crc32_wrapper.c/d' echo 'src/os_win/os_snprintf.c' - echo 'src/os_win/os_vsnprintf.c') < filelist | grep -v power8 | sort > $t + echo 'src/os_win/os_vsnprintf.c') < filelist | sort > $t cmp $t $f > /dev/null 2>&1 || (echo "Building $f" && rm -f $f && cp $t $f) } -- cgit v1.2.1 From d63086df80d1941dd3717abbc72025299aa08fd4 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 11 Feb 2016 12:36:03 -0500 Subject: WT-2280: Add CRC32 Optimized code for PPC64LE Update. --- dist/s_funcs.list | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dist/s_funcs.list b/dist/s_funcs.list index ed6cf43bb2f..8d32eecdfb7 100644 --- a/dist/s_funcs.list +++ b/dist/s_funcs.list @@ -1,4 +1,6 @@ # List of functions that aren't found by s_funcs, but that's OK. +FUNC_END +FUNC_START WT_CURDUMP_PASS __bit_ffs __bit_nclr -- cgit v1.2.1 From 673b678e4896fabb21b0f924824082334f5d2988 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 11 Feb 2016 12:38:40 -0500 Subject: WT-2280: Add CRC32 Optimized code for PPC64LE Sort the list of files. --- dist/filelist | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dist/filelist b/dist/filelist index 71843f909ff..4ed7d7e3beb 100644 --- a/dist/filelist +++ b/dist/filelist @@ -163,11 +163,11 @@ src/support/hazard.c src/support/hex.c src/support/huffman.c src/support/pow.c +src/support/power8/crc32.S +src/support/power8/crc32_wrapper.c src/support/rand.c src/support/scratch.c src/support/stat.c -src/support/power8/crc32.S -src/support/power8/crc32_wrapper.c src/txn/txn.c src/txn/txn_ckpt.c src/txn/txn_ext.c -- cgit v1.2.1 From 1d7958c316cde6aa184b5bc4b9874d7891c1540f Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 11 Feb 2016 14:35:40 -0500 Subject: WT-2349 Add multi-connection test and code fixes for it. --- build_posix/Make.subdirs | 1 + dist/s_string.ok | 5 + src/conn/conn_api.c | 44 +- src/os_posix/os_open.c | 11 +- src/os_win/os_open.c | 12 +- test/readonly/Makefile.am | 13 + test/readonly/Makefile.in | 1040 +++++++++++++++++++++++++++++++++++++++++++++ test/readonly/readonly.c | 285 +++++++++++++ 8 files changed, 1396 insertions(+), 15 deletions(-) create mode 100644 test/readonly/Makefile.am create mode 100644 test/readonly/Makefile.in create mode 100644 test/readonly/readonly.c diff --git a/build_posix/Make.subdirs b/build_posix/Make.subdirs index e1f8a05c613..c21d74e5179 100644 --- a/build_posix/Make.subdirs +++ b/build_posix/Make.subdirs @@ -30,6 +30,7 @@ test/fops test/format test/huge test/packing +test/readonly test/recovery test/salvage test/thread diff --git a/dist/s_string.ok b/dist/s_string.ok index 19fa27cd719..83d8f417c17 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -210,6 +210,7 @@ MySecret NEEDKEY NEEDVALUE NOLL +NOLOCK NONINFRINGEMENT NOTFOUND NOTREACHED @@ -255,6 +256,7 @@ RNG RPC RUNDIR Radu +Readonly Rebalance RedHat Redistributions @@ -437,6 +439,7 @@ checksum checksums children's chk +chmod chongo cip cjoin @@ -501,6 +504,7 @@ datasets datasource datastore dbc +dbs dcalloc decile deciles @@ -898,6 +902,7 @@ rng rocksdb rotN rotn +rp rpc run's runtime diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 0ce5d4285e5..79cba904008 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -1478,12 +1478,17 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) /* * If this is a read-only connection and we cannot grab the lock - * file, check if it is because there is not write permission. - * If the failure is due to permission then ignore the error. - * XXX Ignoring a permission error does allow multiple read-only + * file, check if it is because there is not write permission or + * if the file does not exist. If so, then ignore the error. + * XXX Ignoring the error does allow multiple read-only * connections to exist at the same time on a read-only directory. */ - if (F_ISSET(conn, WT_CONN_READONLY) && ret == EPERM) { + if (F_ISSET(conn, WT_CONN_READONLY) && + (ret == EACCES || ret == ENOENT)) { + /* + * If we got an expected permission or non-existence error + * then skip the byte lock. + */ ret = 0; goto open_wt; } @@ -1516,18 +1521,31 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) open_wt: /* We own the lock file, optionally create the WiredTiger file. */ - WT_ERR(__wt_open(session, WT_WIREDTIGER, is_create, false, 0, &fh)); - + ret = __wt_open(session, WT_WIREDTIGER, is_create, false, 0, &fh); /* - * Lock the WiredTiger file (for backward compatibility reasons as - * described above). Immediately release the lock, it's just a test. + * If we're read-only, check for success as well as handled errors. + * Even if we're able to open the WiredTiger file successfully, we + * do not try to lock it. The lock file test for read-only is the + * only one we do. */ - if (__wt_bytelock(fh, (wt_off_t)0, true) != 0) { - WT_ERR_MSG(session, EBUSY, - "WiredTiger database is already being managed by another " - "process"); + if (F_ISSET(conn, WT_CONN_READONLY) && + (ret == 0 || ret == EACCES || ret == ENOENT)) + ret = 0; + else { + WT_ERR(ret); + + /* + * Lock the WiredTiger file (for backward compatibility reasons + * as described above). Immediately release the lock, it's + * just a test. + */ + if (__wt_bytelock(fh, (wt_off_t)0, true) != 0) { + WT_ERR_MSG(session, EBUSY, + "WiredTiger database is already being managed by " + "another process"); + } + WT_ERR(__wt_bytelock(fh, (wt_off_t)0, false)); } - WT_ERR(__wt_bytelock(fh, (wt_off_t)0, false)); /* * We own the database home, figure out if we're creating it. There are diff --git a/src/os_posix/os_open.c b/src/os_posix/os_open.c index 130c6f16f42..219b26c2fa1 100644 --- a/src/os_posix/os_open.c +++ b/src/os_posix/os_open.c @@ -73,7 +73,16 @@ __wt_open(WT_SESSION_IMPL *session, goto setupfh; } - f = O_RDWR; + /* + * If this is a read-only connection, open all files read-only + * except the lock file. + */ + if (F_ISSET(conn, WT_CONN_READONLY) && + !WT_STRING_MATCH(name, WT_SINGLETHREAD, + strlen(WT_SINGLETHREAD))) + f = O_RDONLY; + else + f = O_RDWR; #ifdef O_BINARY /* Windows clones: we always want to treat the file as a binary. */ f |= O_BINARY; diff --git a/src/os_win/os_open.c b/src/os_win/os_open.c index 10db95be6ce..40f46b001db 100644 --- a/src/os_win/os_open.c +++ b/src/os_win/os_open.c @@ -58,7 +58,17 @@ __wt_open(WT_SESSION_IMPL *session, WT_RET(__wt_filename(session, name, &path)); - share_mode = FILE_SHARE_READ | FILE_SHARE_WRITE; + /* + * If this is a read-only connection, open all files read-only + * except the lock file. + */ + if (F_ISSET(conn, WT_CONN_READONLY) && + !WT_STRING_MATCH(name, WT_SINGLETHREAD, + strlen(WT_SINGLETHREAD))); + share_mode = FILE_SHARE_READ; + else + share_mode = FILE_SHARE_READ | FILE_SHARE_WRITE; + /* * Security: * The application may spawn a new process, and we don't want another diff --git a/test/readonly/Makefile.am b/test/readonly/Makefile.am new file mode 100644 index 00000000000..faab0a1b224 --- /dev/null +++ b/test/readonly/Makefile.am @@ -0,0 +1,13 @@ +AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \ + -I$(top_srcdir)/test/utility + +noinst_PROGRAMS = t +t_SOURCES = readonly.c +t_LDADD = $(top_builddir)/libwiredtiger.la +t_LDFLAGS = -static + +# Run this during a "make check" smoke test. +TESTS = $(noinst_PROGRAMS) + +clean-local: + rm -rf WiredTiger* *.core __* diff --git a/test/readonly/Makefile.in b/test/readonly/Makefile.in new file mode 100644 index 00000000000..16964a05d0d --- /dev/null +++ b/test/readonly/Makefile.in @@ -0,0 +1,1040 @@ +# Makefile.in generated by automake 1.15 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2014 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +noinst_PROGRAMS = t$(EXEEXT) +subdir = test/readonly +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = \ + $(top_srcdir)/build_posix/aclocal/ax_check_class.m4 \ + $(top_srcdir)/build_posix/aclocal/ax_check_junit.m4 \ + $(top_srcdir)/build_posix/aclocal/ax_func_posix_memalign.m4 \ + $(top_srcdir)/build_posix/aclocal/ax_jni_include_dir.m4 \ + $(top_srcdir)/build_posix/aclocal/ax_pkg_swig.m4 \ + $(top_srcdir)/build_posix/aclocal/ax_prog_jar.m4 \ + $(top_srcdir)/build_posix/aclocal/ax_prog_java.m4 \ + $(top_srcdir)/build_posix/aclocal/ax_prog_java_works.m4 \ + $(top_srcdir)/build_posix/aclocal/ax_prog_javac.m4 \ + $(top_srcdir)/build_posix/aclocal/ax_prog_javac_works.m4 \ + $(top_srcdir)/build_posix/aclocal/ax_try_compile_java.m4 \ + $(top_srcdir)/build_posix/aclocal/cond-if.m4 \ + $(top_srcdir)/build_posix/aclocal/libtool.m4 \ + $(top_srcdir)/build_posix/aclocal/ltoptions.m4 \ + $(top_srcdir)/build_posix/aclocal/ltsugar.m4 \ + $(top_srcdir)/build_posix/aclocal/ltversion.m4 \ + $(top_srcdir)/build_posix/aclocal/lt~obsolete.m4 \ + $(top_srcdir)/build_posix/aclocal/options.m4 \ + $(top_srcdir)/build_posix/aclocal/types.m4 \ + $(top_srcdir)/build_posix/aclocal/version.m4 \ + $(top_srcdir)/build_posix/aclocal/version-set.m4 \ + $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/wiredtiger_config.h \ + $(top_builddir)/api/leveldb/leveldb_wt_config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +PROGRAMS = $(noinst_PROGRAMS) +am_t_OBJECTS = readonly.$(OBJEXT) +t_OBJECTS = $(am_t_OBJECTS) +t_DEPENDENCIES = $(top_builddir)/libwiredtiger.la +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +t_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(t_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) -I$(top_builddir)/api/leveldb +depcomp = $(SHELL) $(top_srcdir)/build_posix/gnu-support/depcomp +am__depfiles_maybe = depfiles +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(t_SOURCES) +DIST_SOURCES = $(t_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__tty_colors_dummy = \ + mgn= red= grn= lgn= blu= brg= std=; \ + am__color_tests=no +am__tty_colors = { \ + $(am__tty_colors_dummy); \ + if test "X$(AM_COLOR_TESTS)" = Xno; then \ + am__color_tests=no; \ + elif test "X$(AM_COLOR_TESTS)" = Xalways; then \ + am__color_tests=yes; \ + elif test "X$$TERM" != Xdumb && { test -t 1; } 2>/dev/null; then \ + am__color_tests=yes; \ + fi; \ + if test $$am__color_tests = yes; then \ + red=''; \ + grn=''; \ + lgn=''; \ + blu=''; \ + mgn=''; \ + brg=''; \ + std=''; \ + fi; \ +} +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__recheck_rx = ^[ ]*:recheck:[ ]* +am__global_test_result_rx = ^[ ]*:global-test-result:[ ]* +am__copy_in_global_log_rx = ^[ ]*:copy-in-global-log:[ ]* +# A command that, given a newline-separated list of test names on the +# standard input, print the name of the tests that are to be re-run +# upon "make recheck". +am__list_recheck_tests = $(AWK) '{ \ + recheck = 1; \ + while ((rc = (getline line < ($$0 ".trs"))) != 0) \ + { \ + if (rc < 0) \ + { \ + if ((getline line2 < ($$0 ".log")) < 0) \ + recheck = 0; \ + break; \ + } \ + else if (line ~ /$(am__recheck_rx)[nN][Oo]/) \ + { \ + recheck = 0; \ + break; \ + } \ + else if (line ~ /$(am__recheck_rx)[yY][eE][sS]/) \ + { \ + break; \ + } \ + }; \ + if (recheck) \ + print $$0; \ + close ($$0 ".trs"); \ + close ($$0 ".log"); \ +}' +# A command that, given a newline-separated list of test names on the +# standard input, create the global log from their .trs and .log files. +am__create_global_log = $(AWK) ' \ +function fatal(msg) \ +{ \ + print "fatal: making $@: " msg | "cat >&2"; \ + exit 1; \ +} \ +function rst_section(header) \ +{ \ + print header; \ + len = length(header); \ + for (i = 1; i <= len; i = i + 1) \ + printf "="; \ + printf "\n\n"; \ +} \ +{ \ + copy_in_global_log = 1; \ + global_test_result = "RUN"; \ + while ((rc = (getline line < ($$0 ".trs"))) != 0) \ + { \ + if (rc < 0) \ + fatal("failed to read from " $$0 ".trs"); \ + if (line ~ /$(am__global_test_result_rx)/) \ + { \ + sub("$(am__global_test_result_rx)", "", line); \ + sub("[ ]*$$", "", line); \ + global_test_result = line; \ + } \ + else if (line ~ /$(am__copy_in_global_log_rx)[nN][oO]/) \ + copy_in_global_log = 0; \ + }; \ + if (copy_in_global_log) \ + { \ + rst_section(global_test_result ": " $$0); \ + while ((rc = (getline line < ($$0 ".log"))) != 0) \ + { \ + if (rc < 0) \ + fatal("failed to read from " $$0 ".log"); \ + print line; \ + }; \ + printf "\n"; \ + }; \ + close ($$0 ".trs"); \ + close ($$0 ".log"); \ +}' +# Restructured Text title. +am__rst_title = { sed 's/.*/ & /;h;s/./=/g;p;x;s/ *$$//;p;g' && echo; } +# Solaris 10 'make', and several other traditional 'make' implementations, +# pass "-e" to $(SHELL), and POSIX 2008 even requires this. Work around it +# by disabling -e (using the XSI extension "set +e") if it's set. +am__sh_e_setup = case $$- in *e*) set +e;; esac +# Default flags passed to test drivers. +am__common_driver_flags = \ + --color-tests "$$am__color_tests" \ + --enable-hard-errors "$$am__enable_hard_errors" \ + --expect-failure "$$am__expect_failure" +# To be inserted before the command running the test. Creates the +# directory for the log if needed. Stores in $dir the directory +# containing $f, in $tst the test, in $log the log. Executes the +# developer- defined test setup AM_TESTS_ENVIRONMENT (if any), and +# passes TESTS_ENVIRONMENT. Set up options for the wrapper that +# will run the test scripts (or their associated LOG_COMPILER, if +# thy have one). +am__check_pre = \ +$(am__sh_e_setup); \ +$(am__vpath_adj_setup) $(am__vpath_adj) \ +$(am__tty_colors); \ +srcdir=$(srcdir); export srcdir; \ +case "$@" in \ + */*) am__odir=`echo "./$@" | sed 's|/[^/]*$$||'`;; \ + *) am__odir=.;; \ +esac; \ +test "x$$am__odir" = x"." || test -d "$$am__odir" \ + || $(MKDIR_P) "$$am__odir" || exit $$?; \ +if test -f "./$$f"; then dir=./; \ +elif test -f "$$f"; then dir=; \ +else dir="$(srcdir)/"; fi; \ +tst=$$dir$$f; log='$@'; \ +if test -n '$(DISABLE_HARD_ERRORS)'; then \ + am__enable_hard_errors=no; \ +else \ + am__enable_hard_errors=yes; \ +fi; \ +case " $(XFAIL_TESTS) " in \ + *[\ \ ]$$f[\ \ ]* | *[\ \ ]$$dir$$f[\ \ ]*) \ + am__expect_failure=yes;; \ + *) \ + am__expect_failure=no;; \ +esac; \ +$(AM_TESTS_ENVIRONMENT) $(TESTS_ENVIRONMENT) +# A shell command to get the names of the tests scripts with any registered +# extension removed (i.e., equivalently, the names of the test logs, with +# the '.log' extension removed). The result is saved in the shell variable +# '$bases'. This honors runtime overriding of TESTS and TEST_LOGS. Sadly, +# we cannot use something simpler, involving e.g., "$(TEST_LOGS:.log=)", +# since that might cause problem with VPATH rewrites for suffix-less tests. +# See also 'test-harness-vpath-rewrite.sh' and 'test-trs-basic.sh'. +am__set_TESTS_bases = \ + bases='$(TEST_LOGS)'; \ + bases=`for i in $$bases; do echo $$i; done | sed 's/\.log$$//'`; \ + bases=`echo $$bases` +RECHECK_LOGS = $(TEST_LOGS) +AM_RECURSIVE_TARGETS = check recheck +TEST_SUITE_LOG = test-suite.log +TEST_EXTENSIONS = @EXEEXT@ .test +LOG_DRIVER = $(SHELL) \ + $(top_srcdir)/build_posix/gnu-support/test-driver +LOG_COMPILE = $(LOG_COMPILER) $(AM_LOG_FLAGS) $(LOG_FLAGS) +am__set_b = \ + case '$@' in \ + */*) \ + case '$*' in \ + */*) b='$*';; \ + *) b=`echo '$@' | sed 's/\.log$$//'`; \ + esac;; \ + *) \ + b='$*';; \ + esac +am__test_logs1 = $(TESTS:=.log) +am__test_logs2 = $(am__test_logs1:@EXEEXT@.log=.log) +TEST_LOGS = $(am__test_logs2:.test.log=.log) +TEST_LOG_DRIVER = $(SHELL) \ + $(top_srcdir)/build_posix/gnu-support/test-driver +TEST_LOG_COMPILE = $(TEST_LOG_COMPILER) $(AM_TEST_LOG_FLAGS) \ + $(TEST_LOG_FLAGS) +am__DIST_COMMON = $(srcdir)/Makefile.in \ + $(top_srcdir)/build_posix/gnu-support/depcomp \ + $(top_srcdir)/build_posix/gnu-support/test-driver +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +AMTAR = @AMTAR@ +AM_CFLAGS = @AM_CFLAGS@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +AR = @AR@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BERKELEY_DB_PATH = @BERKELEY_DB_PATH@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CFLAGS = @CFLAGS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DLLTOOL = @DLLTOOL@ +DOXYGEN = @DOXYGEN@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FILE_t_decl = @FILE_t_decl@ +GREP = @GREP@ +HELIUM_PATH = @HELIUM_PATH@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +JAR = @JAR@ +JAVA = @JAVA@ +JAVAC = @JAVAC@ +JAVA_JUNIT = @JAVA_JUNIT@ +JNI_CPPFLAGS = @JNI_CPPFLAGS@ +JUNIT = @JUNIT@ +LD = @LD@ +LDFLAGS = @LDFLAGS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBTOOL = @LIBTOOL@ +LIBTOOL_DEPS = @LIBTOOL_DEPS@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +MAKEINFO = @MAKEINFO@ +MAN1_PAGES = @MAN1_PAGES@ +MAN3_PAGES = @MAN3_PAGES@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +NM = @NM@ +NMEDIT = @NMEDIT@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PYTHON = @PYTHON@ +PYTHON_EXEC_PREFIX = @PYTHON_EXEC_PREFIX@ +PYTHON_INSTALL_ARG = @PYTHON_INSTALL_ARG@ +PYTHON_PLATFORM = @PYTHON_PLATFORM@ +PYTHON_PREFIX = @PYTHON_PREFIX@ +PYTHON_VERSION = @PYTHON_VERSION@ +RANLIB = @RANLIB@ +SED = @SED@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +STRIP = @STRIP@ +SWIG = @SWIG@ +SWIG_LIB = @SWIG_LIB@ +TESTS_JUNIT = @TESTS_JUNIT@ +UUDECODE = @UUDECODE@ +VERSION = @VERSION@ +VERSION_MAJOR = @VERSION_MAJOR@ +VERSION_MINOR = @VERSION_MINOR@ +VERSION_NOPATCH = @VERSION_NOPATCH@ +VERSION_PATCH = @VERSION_PATCH@ +VERSION_STRING = @VERSION_STRING@ +_ACJNI_JAVAC = @_ACJNI_JAVAC@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +off_t_decl = @off_t_decl@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +pid_t_decl = @pid_t_decl@ +pkgpyexecdir = @pkgpyexecdir@ +pkgpythondir = @pkgpythondir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +pyexecdir = @pyexecdir@ +pythondir = @pythondir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +size_t_decl = @size_t_decl@ +srcdir = @srcdir@ +ssize_t_decl = @ssize_t_decl@ +sysconfdir = @sysconfdir@ +target_alias = @target_alias@ +time_t_decl = @time_t_decl@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +uintmax_t_decl = @uintmax_t_decl@ +uintptr_t_decl = @uintptr_t_decl@ +wiredtiger_includes_decl = @wiredtiger_includes_decl@ +AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \ + -I$(top_srcdir)/test/utility + +t_SOURCES = readonly.c +t_LDADD = $(top_builddir)/libwiredtiger.la +t_LDFLAGS = -static + +# Run this during a "make check" smoke test. +TESTS = $(noinst_PROGRAMS) +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .log .o .obj .test .test$(EXEEXT) .trs +$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign test/readonly/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign test/readonly/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstPROGRAMS: + @list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list + +t$(EXEEXT): $(t_OBJECTS) $(t_DEPENDENCIES) $(EXTRA_t_DEPENDENCIES) + @rm -f t$(EXEEXT) + $(AM_V_CCLD)$(t_LINK) $(t_OBJECTS) $(t_LDADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/readonly.Po@am__quote@ + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.o$$||'`;\ +@am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ $< &&\ +@am__fastdepCC_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.obj$$||'`;\ +@am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ `$(CYGPATH_W) '$<'` &&\ +@am__fastdepCC_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.lo$$||'`;\ +@am__fastdepCC_TRUE@ $(LTCOMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ $< &&\ +@am__fastdepCC_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +# Recover from deleted '.trs' file; this should ensure that +# "rm -f foo.log; make foo.trs" re-run 'foo.test', and re-create +# both 'foo.log' and 'foo.trs'. Break the recipe in two subshells +# to avoid problems with "make -n". +.log.trs: + rm -f $< $@ + $(MAKE) $(AM_MAKEFLAGS) $< + +# Leading 'am--fnord' is there to ensure the list of targets does not +# expand to empty, as could happen e.g. with make check TESTS=''. +am--fnord $(TEST_LOGS) $(TEST_LOGS:.log=.trs): $(am__force_recheck) +am--force-recheck: + @: + +$(TEST_SUITE_LOG): $(TEST_LOGS) + @$(am__set_TESTS_bases); \ + am__f_ok () { test -f "$$1" && test -r "$$1"; }; \ + redo_bases=`for i in $$bases; do \ + am__f_ok $$i.trs && am__f_ok $$i.log || echo $$i; \ + done`; \ + if test -n "$$redo_bases"; then \ + redo_logs=`for i in $$redo_bases; do echo $$i.log; done`; \ + redo_results=`for i in $$redo_bases; do echo $$i.trs; done`; \ + if $(am__make_dryrun); then :; else \ + rm -f $$redo_logs && rm -f $$redo_results || exit 1; \ + fi; \ + fi; \ + if test -n "$$am__remaking_logs"; then \ + echo "fatal: making $(TEST_SUITE_LOG): possible infinite" \ + "recursion detected" >&2; \ + elif test -n "$$redo_logs"; then \ + am__remaking_logs=yes $(MAKE) $(AM_MAKEFLAGS) $$redo_logs; \ + fi; \ + if $(am__make_dryrun); then :; else \ + st=0; \ + errmsg="fatal: making $(TEST_SUITE_LOG): failed to create"; \ + for i in $$redo_bases; do \ + test -f $$i.trs && test -r $$i.trs \ + || { echo "$$errmsg $$i.trs" >&2; st=1; }; \ + test -f $$i.log && test -r $$i.log \ + || { echo "$$errmsg $$i.log" >&2; st=1; }; \ + done; \ + test $$st -eq 0 || exit 1; \ + fi + @$(am__sh_e_setup); $(am__tty_colors); $(am__set_TESTS_bases); \ + ws='[ ]'; \ + results=`for b in $$bases; do echo $$b.trs; done`; \ + test -n "$$results" || results=/dev/null; \ + all=` grep "^$$ws*:test-result:" $$results | wc -l`; \ + pass=` grep "^$$ws*:test-result:$$ws*PASS" $$results | wc -l`; \ + fail=` grep "^$$ws*:test-result:$$ws*FAIL" $$results | wc -l`; \ + skip=` grep "^$$ws*:test-result:$$ws*SKIP" $$results | wc -l`; \ + xfail=`grep "^$$ws*:test-result:$$ws*XFAIL" $$results | wc -l`; \ + xpass=`grep "^$$ws*:test-result:$$ws*XPASS" $$results | wc -l`; \ + error=`grep "^$$ws*:test-result:$$ws*ERROR" $$results | wc -l`; \ + if test `expr $$fail + $$xpass + $$error` -eq 0; then \ + success=true; \ + else \ + success=false; \ + fi; \ + br='==================='; br=$$br$$br$$br$$br; \ + result_count () \ + { \ + if test x"$$1" = x"--maybe-color"; then \ + maybe_colorize=yes; \ + elif test x"$$1" = x"--no-color"; then \ + maybe_colorize=no; \ + else \ + echo "$@: invalid 'result_count' usage" >&2; exit 4; \ + fi; \ + shift; \ + desc=$$1 count=$$2; \ + if test $$maybe_colorize = yes && test $$count -gt 0; then \ + color_start=$$3 color_end=$$std; \ + else \ + color_start= color_end=; \ + fi; \ + echo "$${color_start}# $$desc $$count$${color_end}"; \ + }; \ + create_testsuite_report () \ + { \ + result_count $$1 "TOTAL:" $$all "$$brg"; \ + result_count $$1 "PASS: " $$pass "$$grn"; \ + result_count $$1 "SKIP: " $$skip "$$blu"; \ + result_count $$1 "XFAIL:" $$xfail "$$lgn"; \ + result_count $$1 "FAIL: " $$fail "$$red"; \ + result_count $$1 "XPASS:" $$xpass "$$red"; \ + result_count $$1 "ERROR:" $$error "$$mgn"; \ + }; \ + { \ + echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" | \ + $(am__rst_title); \ + create_testsuite_report --no-color; \ + echo; \ + echo ".. contents:: :depth: 2"; \ + echo; \ + for b in $$bases; do echo $$b; done \ + | $(am__create_global_log); \ + } >$(TEST_SUITE_LOG).tmp || exit 1; \ + mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \ + if $$success; then \ + col="$$grn"; \ + else \ + col="$$red"; \ + test x"$$VERBOSE" = x || cat $(TEST_SUITE_LOG); \ + fi; \ + echo "$${col}$$br$${std}"; \ + echo "$${col}Testsuite summary for $(PACKAGE_STRING)$${std}"; \ + echo "$${col}$$br$${std}"; \ + create_testsuite_report --maybe-color; \ + echo "$$col$$br$$std"; \ + if $$success; then :; else \ + echo "$${col}See $(subdir)/$(TEST_SUITE_LOG)$${std}"; \ + if test -n "$(PACKAGE_BUGREPORT)"; then \ + echo "$${col}Please report to $(PACKAGE_BUGREPORT)$${std}"; \ + fi; \ + echo "$$col$$br$$std"; \ + fi; \ + $$success || exit 1 + +check-TESTS: + @list='$(RECHECK_LOGS)'; test -z "$$list" || rm -f $$list + @list='$(RECHECK_LOGS:.log=.trs)'; test -z "$$list" || rm -f $$list + @test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG) + @set +e; $(am__set_TESTS_bases); \ + log_list=`for i in $$bases; do echo $$i.log; done`; \ + trs_list=`for i in $$bases; do echo $$i.trs; done`; \ + log_list=`echo $$log_list`; trs_list=`echo $$trs_list`; \ + $(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_LOG) TEST_LOGS="$$log_list"; \ + exit $$?; +recheck: all + @test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG) + @set +e; $(am__set_TESTS_bases); \ + bases=`for i in $$bases; do echo $$i; done \ + | $(am__list_recheck_tests)` || exit 1; \ + log_list=`for i in $$bases; do echo $$i.log; done`; \ + log_list=`echo $$log_list`; \ + $(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_LOG) \ + am__force_recheck=am--force-recheck \ + TEST_LOGS="$$log_list"; \ + exit $$? +t.log: t$(EXEEXT) + @p='t$(EXEEXT)'; \ + b='t'; \ + $(am__check_pre) $(LOG_DRIVER) --test-name "$$f" \ + --log-file $$b.log --trs-file $$b.trs \ + $(am__common_driver_flags) $(AM_LOG_DRIVER_FLAGS) $(LOG_DRIVER_FLAGS) -- $(LOG_COMPILE) \ + "$$tst" $(AM_TESTS_FD_REDIRECT) +.test.log: + @p='$<'; \ + $(am__set_b); \ + $(am__check_pre) $(TEST_LOG_DRIVER) --test-name "$$f" \ + --log-file $$b.log --trs-file $$b.trs \ + $(am__common_driver_flags) $(AM_TEST_LOG_DRIVER_FLAGS) $(TEST_LOG_DRIVER_FLAGS) -- $(TEST_LOG_COMPILE) \ + "$$tst" $(AM_TESTS_FD_REDIRECT) +@am__EXEEXT_TRUE@.test$(EXEEXT).log: +@am__EXEEXT_TRUE@ @p='$<'; \ +@am__EXEEXT_TRUE@ $(am__set_b); \ +@am__EXEEXT_TRUE@ $(am__check_pre) $(TEST_LOG_DRIVER) --test-name "$$f" \ +@am__EXEEXT_TRUE@ --log-file $$b.log --trs-file $$b.trs \ +@am__EXEEXT_TRUE@ $(am__common_driver_flags) $(AM_TEST_LOG_DRIVER_FLAGS) $(TEST_LOG_DRIVER_FLAGS) -- $(TEST_LOG_COMPILE) \ +@am__EXEEXT_TRUE@ "$$tst" $(AM_TESTS_FD_REDIRECT) + +distdir: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am + $(MAKE) $(AM_MAKEFLAGS) check-TESTS +check: check-am +all-am: Makefile $(PROGRAMS) +installdirs: +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + -test -z "$(TEST_LOGS)" || rm -f $(TEST_LOGS) + -test -z "$(TEST_LOGS:.log=.trs)" || rm -f $(TEST_LOGS:.log=.trs) + -test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG) + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-local clean-noinstPROGRAMS \ + mostlyclean-am + +distclean: distclean-am + -rm -rf ./$(DEPDIR) + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -rf ./$(DEPDIR) + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: + +.MAKE: check-am install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am check check-TESTS check-am clean \ + clean-generic clean-libtool clean-local clean-noinstPROGRAMS \ + cscopelist-am ctags ctags-am distclean distclean-compile \ + distclean-generic distclean-libtool distclean-tags distdir dvi \ + dvi-am html html-am info info-am install install-am \ + install-data install-data-am install-dvi install-dvi-am \ + install-exec install-exec-am install-html install-html-am \ + install-info install-info-am install-man install-pdf \ + install-pdf-am install-ps install-ps-am install-strip \ + installcheck installcheck-am installdirs maintainer-clean \ + maintainer-clean-generic mostlyclean mostlyclean-compile \ + mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \ + recheck tags tags-am uninstall uninstall-am + +.PRECIOUS: Makefile + + +clean-local: + rm -rf WiredTiger* *.core __* + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/test/readonly/readonly.c b/test/readonly/readonly.c new file mode 100644 index 00000000000..36416ea9a57 --- /dev/null +++ b/test/readonly/readonly.c @@ -0,0 +1,285 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#ifndef _WIN32 +#include +#endif + +#include + +#include "test_util.i" + +#define HOME_SIZE 512 +static char home[HOME_SIZE]; /* Program working dir */ +static char home_rd[HOME_SIZE]; /* Read-only dir */ +static char home_rd2[HOME_SIZE]; /* Read-only dir */ +static const char *progname; /* Program name */ +static const char *uri = "table:main"; + +#define ENV_CONFIG \ + "create,log=(file_max=10M,archive=false,enabled)," \ + "transaction_sync=(enabled,method=none)" +#define ENV_CONFIG_RD "readonly=true" +#define MAX_VAL 4096 +#define MAX_KV 10000 + +static void +usage(void) +{ + fprintf(stderr, "usage: %s [-h dir]\n", progname); + exit(EXIT_FAILURE); +} + +static int +run_child(const char *home) +{ + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_SESSION *session; + int i, ret; + + /* + * We expect the read-only database will allow the second read-only + * handle to succeed because no one can create or set the lock file. + */ + if ((ret = wiredtiger_open(home, NULL, ENV_CONFIG_RD, &conn)) != 0) + testutil_die(ret, "wiredtiger_open readonly"); + + /* + * Make sure we can read the data. + */ + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + testutil_die(ret, "WT_CONNECTION:open_session"); + + if ((ret = + session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0) + testutil_die(ret, "WT_SESSION.open_cursor: %s", uri); + + i = 0; + while ((ret = cursor->next(cursor)) == 0) + ++i; + if (i != MAX_KV) + testutil_die(EPERM, "cursor walk"); + if ((ret = conn->close(conn, NULL)) != 0) + testutil_die(ret, "conn_close"); + return (0); +} + +/* + * Child process opens both databases readonly. + */ +static void +open_dbs(const char *home, const char *home_rd, const char *home_rd2) +{ + WT_CONNECTION *conn; + int ret; + + /* + * The parent has an open connection to all directories. + * We expect opening the writeable home to return an error. + * It is a failure if the child successfully opens that. + */ + if ((ret = wiredtiger_open(home, NULL, ENV_CONFIG_RD, &conn)) == 0) + testutil_die(ret, "wiredtiger_open readonly allowed"); + + if ((ret = run_child(home_rd)) != 0) + testutil_die(ret, "run child 1"); + if ((ret = run_child(home_rd2)) != 0) + testutil_die(ret, "run child 2"); + exit(EXIT_SUCCESS); +} + +extern int __wt_optind; +extern char *__wt_optarg; + +int +main(int argc, char *argv[]) +{ + WT_CONNECTION *conn, *conn2, *conn3; + WT_CURSOR *cursor; + WT_ITEM data; + WT_SESSION *session; + uint64_t i; + int ch, status, ret; + pid_t pid; + bool child; + const char *working_dir; + char cmd[512]; + uint8_t buf[MAX_VAL]; + + if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) + progname = argv[0]; + else + ++progname; + + working_dir = "WT_RD"; + child = false; + while ((ch = __wt_getopt(progname, argc, argv, "Ch:")) != EOF) + switch (ch) { + case 'C': + child = true; + break; + case 'h': + working_dir = __wt_optarg; + break; + default: + usage(); + } + argc -= __wt_optind; + argv += __wt_optind; + if (argc != 0) + usage(); + + memset(buf, 0, sizeof(buf)); + /* + * Set up all the directory names. + */ + testutil_work_dir_from_path(home, 512, working_dir); + strncpy(home_rd, home, HOME_SIZE); + strcat(home_rd, ".RD"); + strncpy(home_rd2, home, HOME_SIZE); + strcat(home_rd2, ".NOLOCK"); + if (!child) { + testutil_make_work_dir(home); + testutil_make_work_dir(home_rd); + testutil_make_work_dir(home_rd2); + } else { + /* + * We are a child process, we just want to call + * the open_dbs with the directories we have. + * The child function will exit. + */ + open_dbs(home, home_rd, home_rd2); + } + + /* + * Parent creates a database and table. Then cleanly shuts down. + * Then copy database to read-only directory and chmod. + * Also copy database to read-only directory and remove the lock + * file. One read-only database will have a lock file in the + * file system and the other will not. + * Parent opens all databases with read-only configuration flag. + * Parent forks off child who tries to also open all databases + * with the read-only flag. It should error on the writeable + * directory, but allow it on the read-only directories. + * The child then confirms it can read all the data. + */ + /* + * Run in the home directory and create the table. + */ + if ((ret = wiredtiger_open(home, NULL, ENV_CONFIG, &conn)) != 0) + testutil_die(ret, "wiredtiger_open"); + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + testutil_die(ret, "WT_CONNECTION:open_session"); + if ((ret = session->create(session, + uri, "key_format=Q,value_format=u")) != 0) + testutil_die(ret, "WT_SESSION.create: %s", uri); + if ((ret = + session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0) + testutil_die(ret, "WT_SESSION.open_cursor: %s", uri); + + /* + * Write data into the table and then cleanly shut down connection. + */ + data.data = buf; + data.size = MAX_VAL; + for (i = 0; i < MAX_KV; ++i) { + cursor->set_key(cursor, i); + cursor->set_value(cursor, &data); + if ((ret = cursor->insert(cursor)) != 0) + testutil_die(ret, "WT_CURSOR.insert"); + } + if ((ret = conn->close(conn, NULL)) != 0) + testutil_die(ret, "WT_CONNECTION:close"); + + /* + * Copy the database. Remove any lock file from one copy + * and chmod the copies to be read-only permissions. + */ + (void)snprintf(cmd, sizeof(cmd), + "cp -rp %s/* %s; chmod 0555 %s; chmod -R 0444 %s/*", + home, home_rd, home_rd, home_rd); + (void)system(cmd); + (void)snprintf(cmd, sizeof(cmd), + "cp -rp %s/* %s; rm -f %s/WiredTiger.lock; " + "chmod 0555 %s; chmod -R 0444 %s/*", + home, home_rd2, home_rd2, home_rd2, home_rd2); + (void)system(cmd); + + /* + * Open a connection handle to all databases. + */ + fprintf(stderr, " *** Expect several error messages from WT ***\n"); + if ((ret = wiredtiger_open(home, NULL, ENV_CONFIG_RD, &conn)) != 0) + testutil_die(ret, "wiredtiger_open readonly"); + if ((ret = wiredtiger_open(home_rd, NULL, ENV_CONFIG_RD, &conn2)) != 0) + testutil_die(ret, "wiredtiger_open readonly2"); + if ((ret = wiredtiger_open(home_rd2, NULL, ENV_CONFIG_RD, &conn3)) != 0) + testutil_die(ret, "wiredtiger_open readonly3"); + + /* + * Create a child to also open a connection handle to the databases. + * We cannot use fork here because using fork the child inherits the + * same memory image. Therefore the WT process structure is set in + * the child even though it should not be. So use 'system' to spawn + * an entirely new process. + */ + (void)snprintf(cmd, sizeof(cmd), "%s -C", progname); + if ((status = system(cmd)) < 0) + testutil_die(status, "system"); + + /* + * The child will exit with success if its test passes. + */ + if (WEXITSTATUS(status) != 0) + testutil_die(WEXITSTATUS(status), "system"); + + if ((ret = conn->close(conn, NULL)) != 0) + testutil_die(ret, "WT_CONNECTION:close"); + if ((ret = conn2->close(conn2, NULL)) != 0) + testutil_die(ret, "WT_CONNECTION:close"); + if ((ret = conn3->close(conn3, NULL)) != 0) + testutil_die(ret, "WT_CONNECTION:close"); + /* + * We need to chmod the read-only databases back so that they can + * be removed by scripts. + */ + (void)snprintf(cmd, sizeof(cmd), "chmod 0777 %s %s", home_rd, home_rd2); + (void)system(cmd); + (void)snprintf(cmd, sizeof(cmd), "chmod -R 0666 %s/* %s/*", + home_rd, home_rd2); + (void)system(cmd); + printf(" *** Readonly test successful ***\n"); + return (EXIT_SUCCESS); +} -- cgit v1.2.1 From 76ccfbc1419ccb6296c8c0d2d73297fed8bceb7f Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 11 Feb 2016 15:02:01 -0500 Subject: WT-2349 Remove Makefile.in. It is generated. --- test/readonly/Makefile.in | 1040 --------------------------------------------- 1 file changed, 1040 deletions(-) delete mode 100644 test/readonly/Makefile.in diff --git a/test/readonly/Makefile.in b/test/readonly/Makefile.in deleted file mode 100644 index 16964a05d0d..00000000000 --- a/test/readonly/Makefile.in +++ /dev/null @@ -1,1040 +0,0 @@ -# Makefile.in generated by automake 1.15 from Makefile.am. -# @configure_input@ - -# Copyright (C) 1994-2014 Free Software Foundation, Inc. - -# This Makefile.in is free software; the Free Software Foundation -# gives unlimited permission to copy and/or distribute it, -# with or without modifications, as long as this notice is preserved. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY, to the extent permitted by law; without -# even the implied warranty of MERCHANTABILITY or FITNESS FOR A -# PARTICULAR PURPOSE. - -@SET_MAKE@ - -VPATH = @srcdir@ -am__is_gnu_make = { \ - if test -z '$(MAKELEVEL)'; then \ - false; \ - elif test -n '$(MAKE_HOST)'; then \ - true; \ - elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ - true; \ - else \ - false; \ - fi; \ -} -am__make_running_with_option = \ - case $${target_option-} in \ - ?) ;; \ - *) echo "am__make_running_with_option: internal error: invalid" \ - "target option '$${target_option-}' specified" >&2; \ - exit 1;; \ - esac; \ - has_opt=no; \ - sane_makeflags=$$MAKEFLAGS; \ - if $(am__is_gnu_make); then \ - sane_makeflags=$$MFLAGS; \ - else \ - case $$MAKEFLAGS in \ - *\\[\ \ ]*) \ - bs=\\; \ - sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ - | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ - esac; \ - fi; \ - skip_next=no; \ - strip_trailopt () \ - { \ - flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ - }; \ - for flg in $$sane_makeflags; do \ - test $$skip_next = yes && { skip_next=no; continue; }; \ - case $$flg in \ - *=*|--*) continue;; \ - -*I) strip_trailopt 'I'; skip_next=yes;; \ - -*I?*) strip_trailopt 'I';; \ - -*O) strip_trailopt 'O'; skip_next=yes;; \ - -*O?*) strip_trailopt 'O';; \ - -*l) strip_trailopt 'l'; skip_next=yes;; \ - -*l?*) strip_trailopt 'l';; \ - -[dEDm]) skip_next=yes;; \ - -[JT]) skip_next=yes;; \ - esac; \ - case $$flg in \ - *$$target_option*) has_opt=yes; break;; \ - esac; \ - done; \ - test $$has_opt = yes -am__make_dryrun = (target_option=n; $(am__make_running_with_option)) -am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) -pkgdatadir = $(datadir)/@PACKAGE@ -pkgincludedir = $(includedir)/@PACKAGE@ -pkglibdir = $(libdir)/@PACKAGE@ -pkglibexecdir = $(libexecdir)/@PACKAGE@ -am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd -install_sh_DATA = $(install_sh) -c -m 644 -install_sh_PROGRAM = $(install_sh) -c -install_sh_SCRIPT = $(install_sh) -c -INSTALL_HEADER = $(INSTALL_DATA) -transform = $(program_transform_name) -NORMAL_INSTALL = : -PRE_INSTALL = : -POST_INSTALL = : -NORMAL_UNINSTALL = : -PRE_UNINSTALL = : -POST_UNINSTALL = : -build_triplet = @build@ -host_triplet = @host@ -noinst_PROGRAMS = t$(EXEEXT) -subdir = test/readonly -ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 -am__aclocal_m4_deps = \ - $(top_srcdir)/build_posix/aclocal/ax_check_class.m4 \ - $(top_srcdir)/build_posix/aclocal/ax_check_junit.m4 \ - $(top_srcdir)/build_posix/aclocal/ax_func_posix_memalign.m4 \ - $(top_srcdir)/build_posix/aclocal/ax_jni_include_dir.m4 \ - $(top_srcdir)/build_posix/aclocal/ax_pkg_swig.m4 \ - $(top_srcdir)/build_posix/aclocal/ax_prog_jar.m4 \ - $(top_srcdir)/build_posix/aclocal/ax_prog_java.m4 \ - $(top_srcdir)/build_posix/aclocal/ax_prog_java_works.m4 \ - $(top_srcdir)/build_posix/aclocal/ax_prog_javac.m4 \ - $(top_srcdir)/build_posix/aclocal/ax_prog_javac_works.m4 \ - $(top_srcdir)/build_posix/aclocal/ax_try_compile_java.m4 \ - $(top_srcdir)/build_posix/aclocal/cond-if.m4 \ - $(top_srcdir)/build_posix/aclocal/libtool.m4 \ - $(top_srcdir)/build_posix/aclocal/ltoptions.m4 \ - $(top_srcdir)/build_posix/aclocal/ltsugar.m4 \ - $(top_srcdir)/build_posix/aclocal/ltversion.m4 \ - $(top_srcdir)/build_posix/aclocal/lt~obsolete.m4 \ - $(top_srcdir)/build_posix/aclocal/options.m4 \ - $(top_srcdir)/build_posix/aclocal/types.m4 \ - $(top_srcdir)/build_posix/aclocal/version.m4 \ - $(top_srcdir)/build_posix/aclocal/version-set.m4 \ - $(top_srcdir)/configure.ac -am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ - $(ACLOCAL_M4) -DIST_COMMON = $(srcdir)/Makefile.am $(am__DIST_COMMON) -mkinstalldirs = $(install_sh) -d -CONFIG_HEADER = $(top_builddir)/wiredtiger_config.h \ - $(top_builddir)/api/leveldb/leveldb_wt_config.h -CONFIG_CLEAN_FILES = -CONFIG_CLEAN_VPATH_FILES = -PROGRAMS = $(noinst_PROGRAMS) -am_t_OBJECTS = readonly.$(OBJEXT) -t_OBJECTS = $(am_t_OBJECTS) -t_DEPENDENCIES = $(top_builddir)/libwiredtiger.la -AM_V_lt = $(am__v_lt_@AM_V@) -am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) -am__v_lt_0 = --silent -am__v_lt_1 = -t_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ - $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ - $(t_LDFLAGS) $(LDFLAGS) -o $@ -AM_V_P = $(am__v_P_@AM_V@) -am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) -am__v_P_0 = false -am__v_P_1 = : -AM_V_GEN = $(am__v_GEN_@AM_V@) -am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) -am__v_GEN_0 = @echo " GEN " $@; -am__v_GEN_1 = -AM_V_at = $(am__v_at_@AM_V@) -am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) -am__v_at_0 = @ -am__v_at_1 = -DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) -I$(top_builddir)/api/leveldb -depcomp = $(SHELL) $(top_srcdir)/build_posix/gnu-support/depcomp -am__depfiles_maybe = depfiles -am__mv = mv -f -COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ - $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ - $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ - $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ - $(AM_CFLAGS) $(CFLAGS) -AM_V_CC = $(am__v_CC_@AM_V@) -am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) -am__v_CC_0 = @echo " CC " $@; -am__v_CC_1 = -CCLD = $(CC) -LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ - $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ - $(AM_LDFLAGS) $(LDFLAGS) -o $@ -AM_V_CCLD = $(am__v_CCLD_@AM_V@) -am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) -am__v_CCLD_0 = @echo " CCLD " $@; -am__v_CCLD_1 = -SOURCES = $(t_SOURCES) -DIST_SOURCES = $(t_SOURCES) -am__can_run_installinfo = \ - case $$AM_UPDATE_INFO_DIR in \ - n|no|NO) false;; \ - *) (install-info --version) >/dev/null 2>&1;; \ - esac -am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) -# Read a list of newline-separated strings from the standard input, -# and print each of them once, without duplicates. Input order is -# *not* preserved. -am__uniquify_input = $(AWK) '\ - BEGIN { nonempty = 0; } \ - { items[$$0] = 1; nonempty = 1; } \ - END { if (nonempty) { for (i in items) print i; }; } \ -' -# Make sure the list of sources is unique. This is necessary because, -# e.g., the same source file might be shared among _SOURCES variables -# for different programs/libraries. -am__define_uniq_tagged_files = \ - list='$(am__tagged_files)'; \ - unique=`for i in $$list; do \ - if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ - done | $(am__uniquify_input)` -ETAGS = etags -CTAGS = ctags -am__tty_colors_dummy = \ - mgn= red= grn= lgn= blu= brg= std=; \ - am__color_tests=no -am__tty_colors = { \ - $(am__tty_colors_dummy); \ - if test "X$(AM_COLOR_TESTS)" = Xno; then \ - am__color_tests=no; \ - elif test "X$(AM_COLOR_TESTS)" = Xalways; then \ - am__color_tests=yes; \ - elif test "X$$TERM" != Xdumb && { test -t 1; } 2>/dev/null; then \ - am__color_tests=yes; \ - fi; \ - if test $$am__color_tests = yes; then \ - red=''; \ - grn=''; \ - lgn=''; \ - blu=''; \ - mgn=''; \ - brg=''; \ - std=''; \ - fi; \ -} -am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; -am__vpath_adj = case $$p in \ - $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ - *) f=$$p;; \ - esac; -am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; -am__install_max = 40 -am__nobase_strip_setup = \ - srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` -am__nobase_strip = \ - for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" -am__nobase_list = $(am__nobase_strip_setup); \ - for p in $$list; do echo "$$p $$p"; done | \ - sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ - $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ - if (++n[$$2] == $(am__install_max)) \ - { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ - END { for (dir in files) print dir, files[dir] }' -am__base_list = \ - sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ - sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' -am__uninstall_files_from_dir = { \ - test -z "$$files" \ - || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ - || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ - $(am__cd) "$$dir" && rm -f $$files; }; \ - } -am__recheck_rx = ^[ ]*:recheck:[ ]* -am__global_test_result_rx = ^[ ]*:global-test-result:[ ]* -am__copy_in_global_log_rx = ^[ ]*:copy-in-global-log:[ ]* -# A command that, given a newline-separated list of test names on the -# standard input, print the name of the tests that are to be re-run -# upon "make recheck". -am__list_recheck_tests = $(AWK) '{ \ - recheck = 1; \ - while ((rc = (getline line < ($$0 ".trs"))) != 0) \ - { \ - if (rc < 0) \ - { \ - if ((getline line2 < ($$0 ".log")) < 0) \ - recheck = 0; \ - break; \ - } \ - else if (line ~ /$(am__recheck_rx)[nN][Oo]/) \ - { \ - recheck = 0; \ - break; \ - } \ - else if (line ~ /$(am__recheck_rx)[yY][eE][sS]/) \ - { \ - break; \ - } \ - }; \ - if (recheck) \ - print $$0; \ - close ($$0 ".trs"); \ - close ($$0 ".log"); \ -}' -# A command that, given a newline-separated list of test names on the -# standard input, create the global log from their .trs and .log files. -am__create_global_log = $(AWK) ' \ -function fatal(msg) \ -{ \ - print "fatal: making $@: " msg | "cat >&2"; \ - exit 1; \ -} \ -function rst_section(header) \ -{ \ - print header; \ - len = length(header); \ - for (i = 1; i <= len; i = i + 1) \ - printf "="; \ - printf "\n\n"; \ -} \ -{ \ - copy_in_global_log = 1; \ - global_test_result = "RUN"; \ - while ((rc = (getline line < ($$0 ".trs"))) != 0) \ - { \ - if (rc < 0) \ - fatal("failed to read from " $$0 ".trs"); \ - if (line ~ /$(am__global_test_result_rx)/) \ - { \ - sub("$(am__global_test_result_rx)", "", line); \ - sub("[ ]*$$", "", line); \ - global_test_result = line; \ - } \ - else if (line ~ /$(am__copy_in_global_log_rx)[nN][oO]/) \ - copy_in_global_log = 0; \ - }; \ - if (copy_in_global_log) \ - { \ - rst_section(global_test_result ": " $$0); \ - while ((rc = (getline line < ($$0 ".log"))) != 0) \ - { \ - if (rc < 0) \ - fatal("failed to read from " $$0 ".log"); \ - print line; \ - }; \ - printf "\n"; \ - }; \ - close ($$0 ".trs"); \ - close ($$0 ".log"); \ -}' -# Restructured Text title. -am__rst_title = { sed 's/.*/ & /;h;s/./=/g;p;x;s/ *$$//;p;g' && echo; } -# Solaris 10 'make', and several other traditional 'make' implementations, -# pass "-e" to $(SHELL), and POSIX 2008 even requires this. Work around it -# by disabling -e (using the XSI extension "set +e") if it's set. -am__sh_e_setup = case $$- in *e*) set +e;; esac -# Default flags passed to test drivers. -am__common_driver_flags = \ - --color-tests "$$am__color_tests" \ - --enable-hard-errors "$$am__enable_hard_errors" \ - --expect-failure "$$am__expect_failure" -# To be inserted before the command running the test. Creates the -# directory for the log if needed. Stores in $dir the directory -# containing $f, in $tst the test, in $log the log. Executes the -# developer- defined test setup AM_TESTS_ENVIRONMENT (if any), and -# passes TESTS_ENVIRONMENT. Set up options for the wrapper that -# will run the test scripts (or their associated LOG_COMPILER, if -# thy have one). -am__check_pre = \ -$(am__sh_e_setup); \ -$(am__vpath_adj_setup) $(am__vpath_adj) \ -$(am__tty_colors); \ -srcdir=$(srcdir); export srcdir; \ -case "$@" in \ - */*) am__odir=`echo "./$@" | sed 's|/[^/]*$$||'`;; \ - *) am__odir=.;; \ -esac; \ -test "x$$am__odir" = x"." || test -d "$$am__odir" \ - || $(MKDIR_P) "$$am__odir" || exit $$?; \ -if test -f "./$$f"; then dir=./; \ -elif test -f "$$f"; then dir=; \ -else dir="$(srcdir)/"; fi; \ -tst=$$dir$$f; log='$@'; \ -if test -n '$(DISABLE_HARD_ERRORS)'; then \ - am__enable_hard_errors=no; \ -else \ - am__enable_hard_errors=yes; \ -fi; \ -case " $(XFAIL_TESTS) " in \ - *[\ \ ]$$f[\ \ ]* | *[\ \ ]$$dir$$f[\ \ ]*) \ - am__expect_failure=yes;; \ - *) \ - am__expect_failure=no;; \ -esac; \ -$(AM_TESTS_ENVIRONMENT) $(TESTS_ENVIRONMENT) -# A shell command to get the names of the tests scripts with any registered -# extension removed (i.e., equivalently, the names of the test logs, with -# the '.log' extension removed). The result is saved in the shell variable -# '$bases'. This honors runtime overriding of TESTS and TEST_LOGS. Sadly, -# we cannot use something simpler, involving e.g., "$(TEST_LOGS:.log=)", -# since that might cause problem with VPATH rewrites for suffix-less tests. -# See also 'test-harness-vpath-rewrite.sh' and 'test-trs-basic.sh'. -am__set_TESTS_bases = \ - bases='$(TEST_LOGS)'; \ - bases=`for i in $$bases; do echo $$i; done | sed 's/\.log$$//'`; \ - bases=`echo $$bases` -RECHECK_LOGS = $(TEST_LOGS) -AM_RECURSIVE_TARGETS = check recheck -TEST_SUITE_LOG = test-suite.log -TEST_EXTENSIONS = @EXEEXT@ .test -LOG_DRIVER = $(SHELL) \ - $(top_srcdir)/build_posix/gnu-support/test-driver -LOG_COMPILE = $(LOG_COMPILER) $(AM_LOG_FLAGS) $(LOG_FLAGS) -am__set_b = \ - case '$@' in \ - */*) \ - case '$*' in \ - */*) b='$*';; \ - *) b=`echo '$@' | sed 's/\.log$$//'`; \ - esac;; \ - *) \ - b='$*';; \ - esac -am__test_logs1 = $(TESTS:=.log) -am__test_logs2 = $(am__test_logs1:@EXEEXT@.log=.log) -TEST_LOGS = $(am__test_logs2:.test.log=.log) -TEST_LOG_DRIVER = $(SHELL) \ - $(top_srcdir)/build_posix/gnu-support/test-driver -TEST_LOG_COMPILE = $(TEST_LOG_COMPILER) $(AM_TEST_LOG_FLAGS) \ - $(TEST_LOG_FLAGS) -am__DIST_COMMON = $(srcdir)/Makefile.in \ - $(top_srcdir)/build_posix/gnu-support/depcomp \ - $(top_srcdir)/build_posix/gnu-support/test-driver -DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) -ACLOCAL = @ACLOCAL@ -AMTAR = @AMTAR@ -AM_CFLAGS = @AM_CFLAGS@ -AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ -AR = @AR@ -AUTOCONF = @AUTOCONF@ -AUTOHEADER = @AUTOHEADER@ -AUTOMAKE = @AUTOMAKE@ -AWK = @AWK@ -BERKELEY_DB_PATH = @BERKELEY_DB_PATH@ -CC = @CC@ -CCDEPMODE = @CCDEPMODE@ -CFLAGS = @CFLAGS@ -CPP = @CPP@ -CPPFLAGS = @CPPFLAGS@ -CXX = @CXX@ -CXXCPP = @CXXCPP@ -CXXDEPMODE = @CXXDEPMODE@ -CXXFLAGS = @CXXFLAGS@ -CYGPATH_W = @CYGPATH_W@ -DEFS = @DEFS@ -DEPDIR = @DEPDIR@ -DLLTOOL = @DLLTOOL@ -DOXYGEN = @DOXYGEN@ -DSYMUTIL = @DSYMUTIL@ -DUMPBIN = @DUMPBIN@ -ECHO_C = @ECHO_C@ -ECHO_N = @ECHO_N@ -ECHO_T = @ECHO_T@ -EGREP = @EGREP@ -EXEEXT = @EXEEXT@ -FGREP = @FGREP@ -FILE_t_decl = @FILE_t_decl@ -GREP = @GREP@ -HELIUM_PATH = @HELIUM_PATH@ -INSTALL = @INSTALL@ -INSTALL_DATA = @INSTALL_DATA@ -INSTALL_PROGRAM = @INSTALL_PROGRAM@ -INSTALL_SCRIPT = @INSTALL_SCRIPT@ -INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ -JAR = @JAR@ -JAVA = @JAVA@ -JAVAC = @JAVAC@ -JAVA_JUNIT = @JAVA_JUNIT@ -JNI_CPPFLAGS = @JNI_CPPFLAGS@ -JUNIT = @JUNIT@ -LD = @LD@ -LDFLAGS = @LDFLAGS@ -LIBOBJS = @LIBOBJS@ -LIBS = @LIBS@ -LIBTOOL = @LIBTOOL@ -LIBTOOL_DEPS = @LIBTOOL_DEPS@ -LIPO = @LIPO@ -LN_S = @LN_S@ -LTLIBOBJS = @LTLIBOBJS@ -LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ -MAKEINFO = @MAKEINFO@ -MAN1_PAGES = @MAN1_PAGES@ -MAN3_PAGES = @MAN3_PAGES@ -MANIFEST_TOOL = @MANIFEST_TOOL@ -MKDIR_P = @MKDIR_P@ -NM = @NM@ -NMEDIT = @NMEDIT@ -OBJDUMP = @OBJDUMP@ -OBJEXT = @OBJEXT@ -OTOOL = @OTOOL@ -OTOOL64 = @OTOOL64@ -PACKAGE = @PACKAGE@ -PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ -PACKAGE_NAME = @PACKAGE_NAME@ -PACKAGE_STRING = @PACKAGE_STRING@ -PACKAGE_TARNAME = @PACKAGE_TARNAME@ -PACKAGE_URL = @PACKAGE_URL@ -PACKAGE_VERSION = @PACKAGE_VERSION@ -PATH_SEPARATOR = @PATH_SEPARATOR@ -PYTHON = @PYTHON@ -PYTHON_EXEC_PREFIX = @PYTHON_EXEC_PREFIX@ -PYTHON_INSTALL_ARG = @PYTHON_INSTALL_ARG@ -PYTHON_PLATFORM = @PYTHON_PLATFORM@ -PYTHON_PREFIX = @PYTHON_PREFIX@ -PYTHON_VERSION = @PYTHON_VERSION@ -RANLIB = @RANLIB@ -SED = @SED@ -SET_MAKE = @SET_MAKE@ -SHELL = @SHELL@ -STRIP = @STRIP@ -SWIG = @SWIG@ -SWIG_LIB = @SWIG_LIB@ -TESTS_JUNIT = @TESTS_JUNIT@ -UUDECODE = @UUDECODE@ -VERSION = @VERSION@ -VERSION_MAJOR = @VERSION_MAJOR@ -VERSION_MINOR = @VERSION_MINOR@ -VERSION_NOPATCH = @VERSION_NOPATCH@ -VERSION_PATCH = @VERSION_PATCH@ -VERSION_STRING = @VERSION_STRING@ -_ACJNI_JAVAC = @_ACJNI_JAVAC@ -abs_builddir = @abs_builddir@ -abs_srcdir = @abs_srcdir@ -abs_top_builddir = @abs_top_builddir@ -abs_top_srcdir = @abs_top_srcdir@ -ac_ct_AR = @ac_ct_AR@ -ac_ct_CC = @ac_ct_CC@ -ac_ct_CXX = @ac_ct_CXX@ -ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ -am__include = @am__include@ -am__leading_dot = @am__leading_dot@ -am__quote = @am__quote@ -am__tar = @am__tar@ -am__untar = @am__untar@ -bindir = @bindir@ -build = @build@ -build_alias = @build_alias@ -build_cpu = @build_cpu@ -build_os = @build_os@ -build_vendor = @build_vendor@ -builddir = @builddir@ -datadir = @datadir@ -datarootdir = @datarootdir@ -docdir = @docdir@ -dvidir = @dvidir@ -exec_prefix = @exec_prefix@ -host = @host@ -host_alias = @host_alias@ -host_cpu = @host_cpu@ -host_os = @host_os@ -host_vendor = @host_vendor@ -htmldir = @htmldir@ -includedir = @includedir@ -infodir = @infodir@ -install_sh = @install_sh@ -libdir = @libdir@ -libexecdir = @libexecdir@ -localedir = @localedir@ -localstatedir = @localstatedir@ -mandir = @mandir@ -mkdir_p = @mkdir_p@ -off_t_decl = @off_t_decl@ -oldincludedir = @oldincludedir@ -pdfdir = @pdfdir@ -pid_t_decl = @pid_t_decl@ -pkgpyexecdir = @pkgpyexecdir@ -pkgpythondir = @pkgpythondir@ -prefix = @prefix@ -program_transform_name = @program_transform_name@ -psdir = @psdir@ -pyexecdir = @pyexecdir@ -pythondir = @pythondir@ -sbindir = @sbindir@ -sharedstatedir = @sharedstatedir@ -size_t_decl = @size_t_decl@ -srcdir = @srcdir@ -ssize_t_decl = @ssize_t_decl@ -sysconfdir = @sysconfdir@ -target_alias = @target_alias@ -time_t_decl = @time_t_decl@ -top_build_prefix = @top_build_prefix@ -top_builddir = @top_builddir@ -top_srcdir = @top_srcdir@ -uintmax_t_decl = @uintmax_t_decl@ -uintptr_t_decl = @uintptr_t_decl@ -wiredtiger_includes_decl = @wiredtiger_includes_decl@ -AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \ - -I$(top_srcdir)/test/utility - -t_SOURCES = readonly.c -t_LDADD = $(top_builddir)/libwiredtiger.la -t_LDFLAGS = -static - -# Run this during a "make check" smoke test. -TESTS = $(noinst_PROGRAMS) -all: all-am - -.SUFFIXES: -.SUFFIXES: .c .lo .log .o .obj .test .test$(EXEEXT) .trs -$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) - @for dep in $?; do \ - case '$(am__configure_deps)' in \ - *$$dep*) \ - ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ - && { if test -f $@; then exit 0; else break; fi; }; \ - exit 1;; \ - esac; \ - done; \ - echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign test/readonly/Makefile'; \ - $(am__cd) $(top_srcdir) && \ - $(AUTOMAKE) --foreign test/readonly/Makefile -Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status - @case '$?' in \ - *config.status*) \ - cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ - *) \ - echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \ - cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \ - esac; - -$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) - cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh - -$(top_srcdir)/configure: $(am__configure_deps) - cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh -$(ACLOCAL_M4): $(am__aclocal_m4_deps) - cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh -$(am__aclocal_m4_deps): - -clean-noinstPROGRAMS: - @list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \ - echo " rm -f" $$list; \ - rm -f $$list || exit $$?; \ - test -n "$(EXEEXT)" || exit 0; \ - list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ - echo " rm -f" $$list; \ - rm -f $$list - -t$(EXEEXT): $(t_OBJECTS) $(t_DEPENDENCIES) $(EXTRA_t_DEPENDENCIES) - @rm -f t$(EXEEXT) - $(AM_V_CCLD)$(t_LINK) $(t_OBJECTS) $(t_LDADD) $(LIBS) - -mostlyclean-compile: - -rm -f *.$(OBJEXT) - -distclean-compile: - -rm -f *.tab.c - -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/readonly.Po@am__quote@ - -.c.o: -@am__fastdepCC_TRUE@ $(AM_V_CC)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.o$$||'`;\ -@am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ $< &&\ -@am__fastdepCC_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Po -@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ -@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ -@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< - -.c.obj: -@am__fastdepCC_TRUE@ $(AM_V_CC)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.obj$$||'`;\ -@am__fastdepCC_TRUE@ $(COMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ `$(CYGPATH_W) '$<'` &&\ -@am__fastdepCC_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Po -@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ -@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ -@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` - -.c.lo: -@am__fastdepCC_TRUE@ $(AM_V_CC)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.lo$$||'`;\ -@am__fastdepCC_TRUE@ $(LTCOMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ $< &&\ -@am__fastdepCC_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Plo -@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ -@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ -@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< - -mostlyclean-libtool: - -rm -f *.lo - -clean-libtool: - -rm -rf .libs _libs - -ID: $(am__tagged_files) - $(am__define_uniq_tagged_files); mkid -fID $$unique -tags: tags-am -TAGS: tags - -tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) - set x; \ - here=`pwd`; \ - $(am__define_uniq_tagged_files); \ - shift; \ - if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ - test -n "$$unique" || unique=$$empty_fix; \ - if test $$# -gt 0; then \ - $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ - "$$@" $$unique; \ - else \ - $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ - $$unique; \ - fi; \ - fi -ctags: ctags-am - -CTAGS: ctags -ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) - $(am__define_uniq_tagged_files); \ - test -z "$(CTAGS_ARGS)$$unique" \ - || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ - $$unique - -GTAGS: - here=`$(am__cd) $(top_builddir) && pwd` \ - && $(am__cd) $(top_srcdir) \ - && gtags -i $(GTAGS_ARGS) "$$here" -cscopelist: cscopelist-am - -cscopelist-am: $(am__tagged_files) - list='$(am__tagged_files)'; \ - case "$(srcdir)" in \ - [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ - *) sdir=$(subdir)/$(srcdir) ;; \ - esac; \ - for i in $$list; do \ - if test -f "$$i"; then \ - echo "$(subdir)/$$i"; \ - else \ - echo "$$sdir/$$i"; \ - fi; \ - done >> $(top_builddir)/cscope.files - -distclean-tags: - -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags - -# Recover from deleted '.trs' file; this should ensure that -# "rm -f foo.log; make foo.trs" re-run 'foo.test', and re-create -# both 'foo.log' and 'foo.trs'. Break the recipe in two subshells -# to avoid problems with "make -n". -.log.trs: - rm -f $< $@ - $(MAKE) $(AM_MAKEFLAGS) $< - -# Leading 'am--fnord' is there to ensure the list of targets does not -# expand to empty, as could happen e.g. with make check TESTS=''. -am--fnord $(TEST_LOGS) $(TEST_LOGS:.log=.trs): $(am__force_recheck) -am--force-recheck: - @: - -$(TEST_SUITE_LOG): $(TEST_LOGS) - @$(am__set_TESTS_bases); \ - am__f_ok () { test -f "$$1" && test -r "$$1"; }; \ - redo_bases=`for i in $$bases; do \ - am__f_ok $$i.trs && am__f_ok $$i.log || echo $$i; \ - done`; \ - if test -n "$$redo_bases"; then \ - redo_logs=`for i in $$redo_bases; do echo $$i.log; done`; \ - redo_results=`for i in $$redo_bases; do echo $$i.trs; done`; \ - if $(am__make_dryrun); then :; else \ - rm -f $$redo_logs && rm -f $$redo_results || exit 1; \ - fi; \ - fi; \ - if test -n "$$am__remaking_logs"; then \ - echo "fatal: making $(TEST_SUITE_LOG): possible infinite" \ - "recursion detected" >&2; \ - elif test -n "$$redo_logs"; then \ - am__remaking_logs=yes $(MAKE) $(AM_MAKEFLAGS) $$redo_logs; \ - fi; \ - if $(am__make_dryrun); then :; else \ - st=0; \ - errmsg="fatal: making $(TEST_SUITE_LOG): failed to create"; \ - for i in $$redo_bases; do \ - test -f $$i.trs && test -r $$i.trs \ - || { echo "$$errmsg $$i.trs" >&2; st=1; }; \ - test -f $$i.log && test -r $$i.log \ - || { echo "$$errmsg $$i.log" >&2; st=1; }; \ - done; \ - test $$st -eq 0 || exit 1; \ - fi - @$(am__sh_e_setup); $(am__tty_colors); $(am__set_TESTS_bases); \ - ws='[ ]'; \ - results=`for b in $$bases; do echo $$b.trs; done`; \ - test -n "$$results" || results=/dev/null; \ - all=` grep "^$$ws*:test-result:" $$results | wc -l`; \ - pass=` grep "^$$ws*:test-result:$$ws*PASS" $$results | wc -l`; \ - fail=` grep "^$$ws*:test-result:$$ws*FAIL" $$results | wc -l`; \ - skip=` grep "^$$ws*:test-result:$$ws*SKIP" $$results | wc -l`; \ - xfail=`grep "^$$ws*:test-result:$$ws*XFAIL" $$results | wc -l`; \ - xpass=`grep "^$$ws*:test-result:$$ws*XPASS" $$results | wc -l`; \ - error=`grep "^$$ws*:test-result:$$ws*ERROR" $$results | wc -l`; \ - if test `expr $$fail + $$xpass + $$error` -eq 0; then \ - success=true; \ - else \ - success=false; \ - fi; \ - br='==================='; br=$$br$$br$$br$$br; \ - result_count () \ - { \ - if test x"$$1" = x"--maybe-color"; then \ - maybe_colorize=yes; \ - elif test x"$$1" = x"--no-color"; then \ - maybe_colorize=no; \ - else \ - echo "$@: invalid 'result_count' usage" >&2; exit 4; \ - fi; \ - shift; \ - desc=$$1 count=$$2; \ - if test $$maybe_colorize = yes && test $$count -gt 0; then \ - color_start=$$3 color_end=$$std; \ - else \ - color_start= color_end=; \ - fi; \ - echo "$${color_start}# $$desc $$count$${color_end}"; \ - }; \ - create_testsuite_report () \ - { \ - result_count $$1 "TOTAL:" $$all "$$brg"; \ - result_count $$1 "PASS: " $$pass "$$grn"; \ - result_count $$1 "SKIP: " $$skip "$$blu"; \ - result_count $$1 "XFAIL:" $$xfail "$$lgn"; \ - result_count $$1 "FAIL: " $$fail "$$red"; \ - result_count $$1 "XPASS:" $$xpass "$$red"; \ - result_count $$1 "ERROR:" $$error "$$mgn"; \ - }; \ - { \ - echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" | \ - $(am__rst_title); \ - create_testsuite_report --no-color; \ - echo; \ - echo ".. contents:: :depth: 2"; \ - echo; \ - for b in $$bases; do echo $$b; done \ - | $(am__create_global_log); \ - } >$(TEST_SUITE_LOG).tmp || exit 1; \ - mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \ - if $$success; then \ - col="$$grn"; \ - else \ - col="$$red"; \ - test x"$$VERBOSE" = x || cat $(TEST_SUITE_LOG); \ - fi; \ - echo "$${col}$$br$${std}"; \ - echo "$${col}Testsuite summary for $(PACKAGE_STRING)$${std}"; \ - echo "$${col}$$br$${std}"; \ - create_testsuite_report --maybe-color; \ - echo "$$col$$br$$std"; \ - if $$success; then :; else \ - echo "$${col}See $(subdir)/$(TEST_SUITE_LOG)$${std}"; \ - if test -n "$(PACKAGE_BUGREPORT)"; then \ - echo "$${col}Please report to $(PACKAGE_BUGREPORT)$${std}"; \ - fi; \ - echo "$$col$$br$$std"; \ - fi; \ - $$success || exit 1 - -check-TESTS: - @list='$(RECHECK_LOGS)'; test -z "$$list" || rm -f $$list - @list='$(RECHECK_LOGS:.log=.trs)'; test -z "$$list" || rm -f $$list - @test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG) - @set +e; $(am__set_TESTS_bases); \ - log_list=`for i in $$bases; do echo $$i.log; done`; \ - trs_list=`for i in $$bases; do echo $$i.trs; done`; \ - log_list=`echo $$log_list`; trs_list=`echo $$trs_list`; \ - $(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_LOG) TEST_LOGS="$$log_list"; \ - exit $$?; -recheck: all - @test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG) - @set +e; $(am__set_TESTS_bases); \ - bases=`for i in $$bases; do echo $$i; done \ - | $(am__list_recheck_tests)` || exit 1; \ - log_list=`for i in $$bases; do echo $$i.log; done`; \ - log_list=`echo $$log_list`; \ - $(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_LOG) \ - am__force_recheck=am--force-recheck \ - TEST_LOGS="$$log_list"; \ - exit $$? -t.log: t$(EXEEXT) - @p='t$(EXEEXT)'; \ - b='t'; \ - $(am__check_pre) $(LOG_DRIVER) --test-name "$$f" \ - --log-file $$b.log --trs-file $$b.trs \ - $(am__common_driver_flags) $(AM_LOG_DRIVER_FLAGS) $(LOG_DRIVER_FLAGS) -- $(LOG_COMPILE) \ - "$$tst" $(AM_TESTS_FD_REDIRECT) -.test.log: - @p='$<'; \ - $(am__set_b); \ - $(am__check_pre) $(TEST_LOG_DRIVER) --test-name "$$f" \ - --log-file $$b.log --trs-file $$b.trs \ - $(am__common_driver_flags) $(AM_TEST_LOG_DRIVER_FLAGS) $(TEST_LOG_DRIVER_FLAGS) -- $(TEST_LOG_COMPILE) \ - "$$tst" $(AM_TESTS_FD_REDIRECT) -@am__EXEEXT_TRUE@.test$(EXEEXT).log: -@am__EXEEXT_TRUE@ @p='$<'; \ -@am__EXEEXT_TRUE@ $(am__set_b); \ -@am__EXEEXT_TRUE@ $(am__check_pre) $(TEST_LOG_DRIVER) --test-name "$$f" \ -@am__EXEEXT_TRUE@ --log-file $$b.log --trs-file $$b.trs \ -@am__EXEEXT_TRUE@ $(am__common_driver_flags) $(AM_TEST_LOG_DRIVER_FLAGS) $(TEST_LOG_DRIVER_FLAGS) -- $(TEST_LOG_COMPILE) \ -@am__EXEEXT_TRUE@ "$$tst" $(AM_TESTS_FD_REDIRECT) - -distdir: $(DISTFILES) - @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ - topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ - list='$(DISTFILES)'; \ - dist_files=`for file in $$list; do echo $$file; done | \ - sed -e "s|^$$srcdirstrip/||;t" \ - -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ - case $$dist_files in \ - */*) $(MKDIR_P) `echo "$$dist_files" | \ - sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ - sort -u` ;; \ - esac; \ - for file in $$dist_files; do \ - if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ - if test -d $$d/$$file; then \ - dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ - if test -d "$(distdir)/$$file"; then \ - find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ - fi; \ - if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ - cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ - find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ - fi; \ - cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ - else \ - test -f "$(distdir)/$$file" \ - || cp -p $$d/$$file "$(distdir)/$$file" \ - || exit 1; \ - fi; \ - done -check-am: all-am - $(MAKE) $(AM_MAKEFLAGS) check-TESTS -check: check-am -all-am: Makefile $(PROGRAMS) -installdirs: -install: install-am -install-exec: install-exec-am -install-data: install-data-am -uninstall: uninstall-am - -install-am: all-am - @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am - -installcheck: installcheck-am -install-strip: - if test -z '$(STRIP)'; then \ - $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ - install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ - install; \ - else \ - $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ - install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ - "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ - fi -mostlyclean-generic: - -test -z "$(TEST_LOGS)" || rm -f $(TEST_LOGS) - -test -z "$(TEST_LOGS:.log=.trs)" || rm -f $(TEST_LOGS:.log=.trs) - -test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG) - -clean-generic: - -distclean-generic: - -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) - -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) - -maintainer-clean-generic: - @echo "This command is intended for maintainers to use" - @echo "it deletes files that may require special tools to rebuild." -clean: clean-am - -clean-am: clean-generic clean-libtool clean-local clean-noinstPROGRAMS \ - mostlyclean-am - -distclean: distclean-am - -rm -rf ./$(DEPDIR) - -rm -f Makefile -distclean-am: clean-am distclean-compile distclean-generic \ - distclean-tags - -dvi: dvi-am - -dvi-am: - -html: html-am - -html-am: - -info: info-am - -info-am: - -install-data-am: - -install-dvi: install-dvi-am - -install-dvi-am: - -install-exec-am: - -install-html: install-html-am - -install-html-am: - -install-info: install-info-am - -install-info-am: - -install-man: - -install-pdf: install-pdf-am - -install-pdf-am: - -install-ps: install-ps-am - -install-ps-am: - -installcheck-am: - -maintainer-clean: maintainer-clean-am - -rm -rf ./$(DEPDIR) - -rm -f Makefile -maintainer-clean-am: distclean-am maintainer-clean-generic - -mostlyclean: mostlyclean-am - -mostlyclean-am: mostlyclean-compile mostlyclean-generic \ - mostlyclean-libtool - -pdf: pdf-am - -pdf-am: - -ps: ps-am - -ps-am: - -uninstall-am: - -.MAKE: check-am install-am install-strip - -.PHONY: CTAGS GTAGS TAGS all all-am check check-TESTS check-am clean \ - clean-generic clean-libtool clean-local clean-noinstPROGRAMS \ - cscopelist-am ctags ctags-am distclean distclean-compile \ - distclean-generic distclean-libtool distclean-tags distdir dvi \ - dvi-am html html-am info info-am install install-am \ - install-data install-data-am install-dvi install-dvi-am \ - install-exec install-exec-am install-html install-html-am \ - install-info install-info-am install-man install-pdf \ - install-pdf-am install-ps install-ps-am install-strip \ - installcheck installcheck-am installdirs maintainer-clean \ - maintainer-clean-generic mostlyclean mostlyclean-compile \ - mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \ - recheck tags tags-am uninstall uninstall-am - -.PRECIOUS: Makefile - - -clean-local: - rm -rf WiredTiger* *.core __* - -# Tell versions [3.59,3.63) of GNU make to not export all variables. -# Otherwise a system limit (for SysV at least) may be exceeded. -.NOEXPORT: -- cgit v1.2.1 From 256caddff0e3aaee3c3e2a655a039f2c2bb0457f Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 11 Feb 2016 15:12:29 -0500 Subject: WT-2349 Unused variable. --- test/readonly/readonly.c | 1 - 1 file changed, 1 deletion(-) diff --git a/test/readonly/readonly.c b/test/readonly/readonly.c index 36416ea9a57..370e6100035 100644 --- a/test/readonly/readonly.c +++ b/test/readonly/readonly.c @@ -132,7 +132,6 @@ main(int argc, char *argv[]) WT_SESSION *session; uint64_t i; int ch, status, ret; - pid_t pid; bool child; const char *working_dir; char cmd[512]; -- cgit v1.2.1 From aff55747f1a92397fe316819613411e38b1a4b9a Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 11 Feb 2016 15:23:10 -0500 Subject: WT-2349 Naming. --- test/readonly/readonly.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/readonly/readonly.c b/test/readonly/readonly.c index 370e6100035..a289296ea65 100644 --- a/test/readonly/readonly.c +++ b/test/readonly/readonly.c @@ -62,7 +62,7 @@ usage(void) } static int -run_child(const char *home) +run_child(const char *homedir) { WT_CONNECTION *conn; WT_CURSOR *cursor; @@ -73,7 +73,7 @@ run_child(const char *home) * We expect the read-only database will allow the second read-only * handle to succeed because no one can create or set the lock file. */ - if ((ret = wiredtiger_open(home, NULL, ENV_CONFIG_RD, &conn)) != 0) + if ((ret = wiredtiger_open(homedir, NULL, ENV_CONFIG_RD, &conn)) != 0) testutil_die(ret, "wiredtiger_open readonly"); /* @@ -100,7 +100,7 @@ run_child(const char *home) * Child process opens both databases readonly. */ static void -open_dbs(const char *home, const char *home_rd, const char *home_rd2) +open_dbs(const char *dir, const char *dir_rd, const char *dir_rd2) { WT_CONNECTION *conn; int ret; @@ -110,12 +110,12 @@ open_dbs(const char *home, const char *home_rd, const char *home_rd2) * We expect opening the writeable home to return an error. * It is a failure if the child successfully opens that. */ - if ((ret = wiredtiger_open(home, NULL, ENV_CONFIG_RD, &conn)) == 0) + if ((ret = wiredtiger_open(dir, NULL, ENV_CONFIG_RD, &conn)) == 0) testutil_die(ret, "wiredtiger_open readonly allowed"); - if ((ret = run_child(home_rd)) != 0) + if ((ret = run_child(dir_rd)) != 0) testutil_die(ret, "run child 1"); - if ((ret = run_child(home_rd2)) != 0) + if ((ret = run_child(dir_rd2)) != 0) testutil_die(ret, "run child 2"); exit(EXIT_SUCCESS); } -- cgit v1.2.1 From 1d081b431605026d3c8e45f5269ad4ba17b4e86e Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 11 Feb 2016 15:31:42 -0500 Subject: WT-2349 Chmod all database files. Look for expected messages. --- test/suite/test_readonly01.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/test/suite/test_readonly01.py b/test/suite/test_readonly01.py index f37c90876a0..858c30e7457 100644 --- a/test/suite/test_readonly01.py +++ b/test/suite/test_readonly01.py @@ -97,11 +97,14 @@ class test_readonly01(wttest.WiredTigerTestCase, suite_subprocess): # self.close_conn() if self.dirchmod: + for f in os.listdir(self.home): + if os.path.isfile(f): + os.chmod(f, 0444) os.chmod(self.home, 0555) self.conn = self.setUpConnectionOpen(self.home) self.session = self.setUpSessionOpen(self.conn) - def test_readonly(self): + def readonly(self): # Here's the strategy: # - Create a table. # - Insert data into table. @@ -132,7 +135,17 @@ class test_readonly01(wttest.WiredTigerTestCase, suite_subprocess): # If we changed the directory to readonly, change it back so that # the cleanup functions can remove it and set up for the next test. if self.dirchmod: + for f in os.listdir(self.home): + if os.path.isfile(f): + os.chmod(f, 0666) os.chmod(self.home, 0777) + def test_readonly(self): + if self.dirchmod: + with self.expectedStderrPattern('Permission'): + self.readonly() + else: + self.readonly() + if __name__ == '__main__': wttest.run() -- cgit v1.2.1 From 58f9466e02fea96279a919078e3b3fba9a2d8cc8 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 11 Feb 2016 15:48:35 -0500 Subject: WT-2349 Fix argv[0] usage for system command. --- test/readonly/readonly.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/readonly/readonly.c b/test/readonly/readonly.c index a289296ea65..f4ad27991e5 100644 --- a/test/readonly/readonly.c +++ b/test/readonly/readonly.c @@ -45,6 +45,7 @@ static char home[HOME_SIZE]; /* Program working dir */ static char home_rd[HOME_SIZE]; /* Read-only dir */ static char home_rd2[HOME_SIZE]; /* Read-only dir */ static const char *progname; /* Program name */ +static const char *saved_argv0; /* Program name */ static const char *uri = "table:main"; #define ENV_CONFIG \ @@ -141,6 +142,10 @@ main(int argc, char *argv[]) progname = argv[0]; else ++progname; + /* + * Needed unaltered for system command later. + */ + saved_argv0 = argv[0]; working_dir = "WT_RD"; child = false; @@ -254,7 +259,7 @@ main(int argc, char *argv[]) * the child even though it should not be. So use 'system' to spawn * an entirely new process. */ - (void)snprintf(cmd, sizeof(cmd), "%s -C", progname); + (void)snprintf(cmd, sizeof(cmd), "%s -C", saved_argv0); if ((status = system(cmd)) < 0) testutil_die(status, "system"); -- cgit v1.2.1 From 2791d71f2f526fe38fa8bda86cd329c6cd86b8c1 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 11 Feb 2016 17:35:08 -0500 Subject: WT-2280: Add CRC32 Optimized code for PPC64LE Revert to the original code so it's easier to compare with the source in the future. --- src/support/power8/crc32.S | 17 ----------- src/support/power8/crc32_constants.h | 56 ++++++++++-------------------------- src/support/power8/crc32_wrapper.c | 30 ++++--------------- src/support/power8/ppc-opcode.h | 51 +++++++++++--------------------- 4 files changed, 38 insertions(+), 116 deletions(-) diff --git a/src/support/power8/crc32.S b/src/support/power8/crc32.S index 16990cc6dbc..4bc1fad416d 100644 --- a/src/support/power8/crc32.S +++ b/src/support/power8/crc32.S @@ -1,20 +1,3 @@ -/*- - * Copyright 2016 MongoDB, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Local modifications to add __powerpc64__ conditional defines. - */ #if defined(__powerpc64__) /* * Calculate the checksum of data that is 16 byte aligned and a multiple of diff --git a/src/support/power8/crc32_constants.h b/src/support/power8/crc32_constants.h index c407c08974b..02c471d1c56 100644 --- a/src/support/power8/crc32_constants.h +++ b/src/support/power8/crc32_constants.h @@ -1,23 +1,6 @@ -/*- - * Copyright 2016 MongoDB, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Local modifications to format long lines. - */ -#define CRC 0x1edc6f41 -#define CRC_XOR -#define REFLECT +#define CRC 0x1edc6f41 +#define CRC_XOR +#define REFLECT #ifndef __ASSEMBLY__ #ifdef CRC_TABLE @@ -89,7 +72,7 @@ static const unsigned int crc_table[] = { #endif #else -#define MAX_SIZE 32768 +#define MAX_SIZE 32768 .constants: /* Reduce 262144 kbits to 1024 bits */ @@ -860,42 +843,32 @@ static const unsigned int crc_table[] = { .short_constants: - /* Reduce final 1024-2048 bits to 64 bits, shifting 32 bits to include - the trailing 32 bits of zeros */ - /* x^1952 mod p(x)`, x^1984 mod p(x)`, x^2016 mod p(x)`, - x^2048 mod p()`x */ + /* Reduce final 1024-2048 bits to 64 bits, shifting 32 bits to include the trailing 32 bits of zeros */ + /* x^1952 mod p(x)`, x^1984 mod p(x)`, x^2016 mod p(x)`, x^2048 mod p(x)` */ .octa 0x7fec2963e5bf80485cf015c388e56f72 - /* x^1824 mod p(x)`, x^1856 mod p(x)`, x^1888 mod p(x)`, - x^1920 mod p(x)` */ + /* x^1824 mod p(x)`, x^1856 mod p(x)`, x^1888 mod p(x)`, x^1920 mod p(x)` */ .octa 0x38e888d4844752a9963a18920246e2e6 - /* x^1696 mod p(x)`, x^1728 mod p(x)`, x^1760 mod p(x)`, - x^1792 mod p(x)` */ + /* x^1696 mod p(x)`, x^1728 mod p(x)`, x^1760 mod p(x)`, x^1792 mod p(x)` */ .octa 0x42316c00730206ad419a441956993a31 - /* x^1568 mod p(x)`, x^1600 mod p(x)`, x^1632 mod p(x)`, - x^1664 mod p(x)` */ + /* x^1568 mod p(x)`, x^1600 mod p(x)`, x^1632 mod p(x)`, x^1664 mod p(x)` */ .octa 0x543d5c543e65ddf9924752ba2b830011 - /* x^1440 mod p(x)`, x^1472 mod p(x)`, x^1504 mod p(x)`, - x^1536 mod p(x)` */ + /* x^1440 mod p(x)`, x^1472 mod p(x)`, x^1504 mod p(x)`, x^1536 mod p(x)` */ .octa 0x78e87aaf56767c9255bd7f9518e4a304 - /* x^1312 mod p(x)`, x^1344 mod p(x)`, x^1376 mod p(x)`, - x^1408 mod p(x)` */ + /* x^1312 mod p(x)`, x^1344 mod p(x)`, x^1376 mod p(x)`, x^1408 mod p(x)` */ .octa 0x8f68fcec1903da7f6d76739fe0553f1e - /* x^1184 mod p(x)`, x^1216 mod p(x)`, x^1248 mod p(x)`, - x^1280 mod p(x)` */ + /* x^1184 mod p(x)`, x^1216 mod p(x)`, x^1248 mod p(x)`, x^1280 mod p(x)` */ .octa 0x3f4840246791d588c133722b1fe0b5c3 - /* x^1056 mod p(x)`, x^1088 mod p(x)`, x^1120 mod p(x)`, - x^1152 mod p(x)` */ + /* x^1056 mod p(x)`, x^1088 mod p(x)`, x^1120 mod p(x)`, x^1152 mod p(x)` */ .octa 0x34c96751b04de25a64b67ee0e55ef1f3 - /* x^928 mod p(x)`, x^960 mod p(x)`, x^992 mod p(x)`, - x^1024 mod p(x)` */ + /* x^928 mod p(x)`, x^960 mod p(x)`, x^992 mod p(x)`, x^1024 mod p(x)` */ .octa 0x156c8e180b4a395b069db049b8fdb1e7 /* x^800 mod p(x)`, x^832 mod p(x)`, x^864 mod p(x)`, x^896 mod p(x)` */ @@ -919,6 +892,7 @@ static const unsigned int crc_table[] = { /* x^32 mod p(x)`, x^64 mod p(x)`, x^96 mod p(x)`, x^128 mod p(x)` */ .octa 0x82f63b786ea2d55ca66805eb18b8ea18 + .barrett_constants: /* 33 bit reflected Barrett constant m - (4^32)/n */ .octa 0x000000000000000000000000dea713f1 /* x^64 div p(x)` */ diff --git a/src/support/power8/crc32_wrapper.c b/src/support/power8/crc32_wrapper.c index bb2dd0c70df..34ac4150338 100644 --- a/src/support/power8/crc32_wrapper.c +++ b/src/support/power8/crc32_wrapper.c @@ -1,27 +1,9 @@ -/*- - * Copyright 2016 MongoDB, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Local modifications to add __powerpc64__ conditional defines, and - * code formatting fixes. - */ #if defined(__powerpc64__) -#define CRC_TABLE +#define CRC_TABLE #include "crc32_constants.h" -#define VMX_ALIGN 16 -#define VMX_ALIGN_MASK (VMX_ALIGN-1) +#define VMX_ALIGN 16 +#define VMX_ALIGN_MASK (VMX_ALIGN-1) #ifdef REFLECT static unsigned int crc32_align(unsigned int crc, unsigned char *p, @@ -29,7 +11,7 @@ static unsigned int crc32_align(unsigned int crc, unsigned char *p, { while (len--) crc = crc_table[(crc ^ *p++) & 0xff] ^ (crc >> 8); - return (crc); + return crc; } #else static unsigned int crc32_align(unsigned int crc, unsigned char *p, @@ -37,7 +19,7 @@ static unsigned int crc32_align(unsigned int crc, unsigned char *p, { while (len--) crc = crc_table[((crc >> 24) ^ *p++) & 0xff] ^ (crc << 8); - return (crc); + return crc; } #endif @@ -79,6 +61,6 @@ out: crc ^= 0xffffffff; #endif - return (crc); + return crc; } #endif diff --git a/src/support/power8/ppc-opcode.h b/src/support/power8/ppc-opcode.h index 40058b0d50d..b63feea60a0 100644 --- a/src/support/power8/ppc-opcode.h +++ b/src/support/power8/ppc-opcode.h @@ -1,40 +1,23 @@ -/*- - * Copyright 2016 MongoDB, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Local modifications to format for style. - */ #ifndef __OPCODES_H -#define __OPCODES_H +#define __OPCODES_H -#define __PPC_RA(a) (((a) & 0x1f) << 16) -#define __PPC_RB(b) (((b) & 0x1f) << 11) -#define __PPC_XA(a) ((((a) & 0x1f) << 16) | (((a) & 0x20) >> 3)) -#define __PPC_XB(b) ((((b) & 0x1f) << 11) | (((b) & 0x20) >> 4)) -#define __PPC_XS(s) ((((s) & 0x1f) << 21) | (((s) & 0x20) >> 5)) -#define __PPC_XT(s) __PPC_XS(s) -#define VSX_XX3(t, a, b) (__PPC_XT(t) | __PPC_XA(a) | __PPC_XB(b)) -#define VSX_XX1(s, a, b) (__PPC_XS(s) | __PPC_RA(a) | __PPC_RB(b)) +#define __PPC_RA(a) (((a) & 0x1f) << 16) +#define __PPC_RB(b) (((b) & 0x1f) << 11) +#define __PPC_XA(a) ((((a) & 0x1f) << 16) | (((a) & 0x20) >> 3)) +#define __PPC_XB(b) ((((b) & 0x1f) << 11) | (((b) & 0x20) >> 4)) +#define __PPC_XS(s) ((((s) & 0x1f) << 21) | (((s) & 0x20) >> 5)) +#define __PPC_XT(s) __PPC_XS(s) +#define VSX_XX3(t, a, b) (__PPC_XT(t) | __PPC_XA(a) | __PPC_XB(b)) +#define VSX_XX1(s, a, b) (__PPC_XS(s) | __PPC_RA(a) | __PPC_RB(b)) -#define PPC_INST_VPMSUMW 0x10000488 -#define PPC_INST_VPMSUMD 0x100004c8 -#define PPC_INST_MFVSRD 0x7c000066 -#define PPC_INST_MTVSRD 0x7c000166 +#define PPC_INST_VPMSUMW 0x10000488 +#define PPC_INST_VPMSUMD 0x100004c8 +#define PPC_INST_MFVSRD 0x7c000066 +#define PPC_INST_MTVSRD 0x7c000166 -#define VPMSUMW(t, a, b) .long PPC_INST_VPMSUMW | VSX_XX3((t), a, b) -#define VPMSUMD(t, a, b) .long PPC_INST_VPMSUMD | VSX_XX3((t), a, b) -#define MFVRD(a, t) .long PPC_INST_MFVSRD | VSX_XX1((t)+32, a, 0) -#define MTVRD(t, a) .long PPC_INST_MTVSRD | VSX_XX1((t)+32, a, 0) +#define VPMSUMW(t, a, b) .long PPC_INST_VPMSUMW | VSX_XX3((t), a, b) +#define VPMSUMD(t, a, b) .long PPC_INST_VPMSUMD | VSX_XX3((t), a, b) +#define MFVRD(a, t) .long PPC_INST_MFVSRD | VSX_XX1((t)+32, a, 0) +#define MTVRD(t, a) .long PPC_INST_MTVSRD | VSX_XX1((t)+32, a, 0) #endif -- cgit v1.2.1 From 264bd99941370d8a9bfd056d850d1a7997a766c0 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 11 Feb 2016 17:41:42 -0500 Subject: WT-2280: Add CRC32 Optimized code for PPC64LE Update dist scripts to leave the new power8 subdirectory alone. --- dist/s_longlines | 5 +++-- dist/s_style | 4 +++- dist/s_whitespace | 7 +++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/dist/s_longlines b/dist/s_longlines index decedb58f44..000f33d51d5 100755 --- a/dist/s_longlines +++ b/dist/s_longlines @@ -9,8 +9,9 @@ l=`(cd .. && find dist -name '*.py' && find src -name '*.in') | sed -e '/dist\/stat_data\.py/d' \ - -e '/support\/stat\.c/d' \ - -e '/include\/extern\.h/d'` + -e '/include\/extern\.h/d' \ + -e '/support\/power8/d' \ + -e '/support\/stat\.c/d'` for f in $l ; do expand -t8 < ../$f | awk -- \ diff --git a/dist/s_style b/dist/s_style index 44a5bdda741..78fb7a6eb03 100755 --- a/dist/s_style +++ b/dist/s_style @@ -18,7 +18,9 @@ if [ $# -ne 1 ]; then find bench examples ext src test \ -name '*.[chisy]' -o -name '*.in' -o -name '*.dox' | - sed -e '/Makefile.in/d' -e '/build_win\/wiredtiger_config.h/d' | + sed -e '/Makefile.in/d' \ + -e '/build_win\/wiredtiger_config.h/d' \ + -e '/support\/power8/d' | xargs $xp -n 1 -I{} sh ./dist/s_style {} else # General style correction and cleanup for a single file diff --git a/dist/s_whitespace b/dist/s_whitespace index d13de4b5989..74820a4f0e9 100755 --- a/dist/s_whitespace +++ b/dist/s_whitespace @@ -36,10 +36,9 @@ for f in `find bench examples ext src test \ -name '*.[chi]' -o \ -name '*.dox' -o \ -name '*.in' -o \ - -name 'Makefile.am'`; do - if expr "$f" : ".*/Makefile.in" > /dev/null; then - continue - fi + -name 'Makefile.am' | + sed -e '/Makefile.in/d' \ + -e '/support\/power8/d'`; do whitespace_and_empty_line $f done -- cgit v1.2.1 From adb0461954b8cabdec5e745146238bf59c596e33 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Fri, 12 Feb 2016 11:44:45 +1100 Subject: WT-2349 Clean up after tests with read-only directories. --- test/readonly/Makefile.am | 2 +- test/readonly/smoke.sh | 8 ++++++++ test/suite/test_readonly01.py | 8 -------- test/suite/wttest.py | 8 ++++++++ 4 files changed, 17 insertions(+), 9 deletions(-) create mode 100755 test/readonly/smoke.sh diff --git a/test/readonly/Makefile.am b/test/readonly/Makefile.am index faab0a1b224..384e197a1f8 100644 --- a/test/readonly/Makefile.am +++ b/test/readonly/Makefile.am @@ -7,7 +7,7 @@ t_LDADD = $(top_builddir)/libwiredtiger.la t_LDFLAGS = -static # Run this during a "make check" smoke test. -TESTS = $(noinst_PROGRAMS) +TESTS = smoke.sh clean-local: rm -rf WiredTiger* *.core __* diff --git a/test/readonly/smoke.sh b/test/readonly/smoke.sh new file mode 100755 index 00000000000..740deb5743a --- /dev/null +++ b/test/readonly/smoke.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +trap 'chmod -R u+w WT_*; exit 0' 0 1 2 3 13 15 + +set -e + +# Smoke-test format as part of running "make check". +$TEST_WRAPPER ./t diff --git a/test/suite/test_readonly01.py b/test/suite/test_readonly01.py index 858c30e7457..86604eb6bfb 100644 --- a/test/suite/test_readonly01.py +++ b/test/suite/test_readonly01.py @@ -132,14 +132,6 @@ class test_readonly01(wttest.WiredTigerTestCase, suite_subprocess): c.close() self.create = True - # If we changed the directory to readonly, change it back so that - # the cleanup functions can remove it and set up for the next test. - if self.dirchmod: - for f in os.listdir(self.home): - if os.path.isfile(f): - os.chmod(f, 0666) - os.chmod(self.home, 0777) - def test_readonly(self): if self.dirchmod: with self.expectedStderrPattern('Permission'): diff --git a/test/suite/wttest.py b/test/suite/wttest.py index b5a58d1566f..a1945b4325d 100644 --- a/test/suite/wttest.py +++ b/test/suite/wttest.py @@ -335,6 +335,14 @@ class WiredTigerTestCase(unittest.TestCase): # always get back to original directory os.chdir(self.origcwd) + # Make sure no read-only files or directories were left behind + os.chmod(self.testdir, 0777) + for root, dirs, files in os.walk(self.testdir): + for d in dirs: + os.chmod(os.path.join(root, d), 0777) + for f in files: + os.chmod(os.path.join(root, f), 0666) + # Clean up unless there's a failure if (passed or skipped) and not WiredTigerTestCase._preserveFiles: shutil.rmtree(self.testdir, ignore_errors=True) -- cgit v1.2.1 From 048e15c188b73a1eb83048e3b9afcac7b58fe1c3 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 12 Feb 2016 17:22:56 -0500 Subject: WT-2381: dump utility discards table config The loadtext command requires a URI argument. --- src/docs/command-line.dox | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/docs/command-line.dox b/src/docs/command-line.dox index e2b376d5e3f..a9f6a5676c4 100644 --- a/src/docs/command-line.dox +++ b/src/docs/command-line.dox @@ -233,7 +233,7 @@ The \c loadtext command reads the standard input for text and loads it into a table or file. The input data should be printable characters, with newline delimiters for each key or value. -The \c loadtext command does not create the file if it does not yet +The \c loadtext command does not create the object if it does not yet exist. In the case of inserting values into a column-store table or file, each @@ -244,7 +244,7 @@ row-store table or file already exists, data in the table or file will be overwritten by the new data. @subsection util_loadtext_synopsis Synopsis -wt [-RVv] [-C config] [-E secretkey ] [-h directory] loadtext [-f input] +wt [-RVv] [-C config] [-E secretkey ] [-h directory] loadtext [-f input] uri @subsection util_loadtext_options Options The following are command-specific options for the \c loadtext command: -- cgit v1.2.1 From 66f7dc9cd47db0e2440b07c7e787fc14cd60fbca Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 12 Feb 2016 17:27:43 -0500 Subject: WT-2381: dump utility discards table config Rewrite the command-line tool entries that refer to "tables or files" to simply refer to tables, it's simpler and less confusing, and users are unlikely to be using file URIs. --- src/docs/command-line.dox | 90 +++++++++++++++++++++++------------------------ 1 file changed, 44 insertions(+), 46 deletions(-) diff --git a/src/docs/command-line.dox b/src/docs/command-line.dox index a9f6a5676c4..0f5c56d25ce 100644 --- a/src/docs/command-line.dox +++ b/src/docs/command-line.dox @@ -41,7 +41,7 @@ by default and commands that only read data will not run recovery. Perform a backup of a database or set of data sources. The \c backup command performs a backup of the database, copying the -database files to a \c specified directory, which can be subsequently +underlying files to a \c specified directory, which can be subsequently opened as a WiredTiger database. See @ref backup for more information, and @ref file_permissions for specifics on the copied file permissions. @@ -58,10 +58,10 @@ the named data sources.
@section util_compact wt compact -Compact a table or file. +Compact a table. -The \c compact command attempts to rewrite the specified table or file -to consume less disk space. +The \c compact command attempts to rewrite the specified table to +consume less disk space. @subsection util_compact_synopsis Synopsis wt [-RVv] [-C config] [-E secretkey ] [-h directory] compact uri @@ -71,7 +71,7 @@ The \c compact command has no command-specific options.
@section util_create wt create -Create a table or file. +Create a table. The \c create command creates the specified \c uri with the specified configuration. It is equivalent to a call to WT_SESSION::create with @@ -88,7 +88,7 @@ Include a configuration string to be passed to WT_SESSION::create.
@section util_drop wt drop -Drop a table or file. +Drop a table. The \c drop command drops the specified \c uri. It is equivalent to a call to WT_SESSION::drop with the "force" configuration argument. @@ -136,10 +136,10 @@ printable characters unencoded).
@section util_list wt list -List the tables and files in the database. +List the tables in the database. -By default, the \c list command prints out the tables and files stored in -the database. If a URI is specified as an argument, only information about +By default, the \c list command prints out the tables stored in the +database. If a URI is specified as an argument, only information about that data source is printed. @subsection util_list_synopsis Synopsis @@ -158,16 +158,16 @@ value is printed.
@section util_load wt load -Load a table or file from dump output. +Load a table from dump output. The \c load command reads the standard input for data and loads it into -a table or file, creating the table or file if it does not yet exist. -The data should be the format produced by the \c dump command; see -@ref dump_formats for details. +a table, creating the table if it does not yet exist. The data should +be the format produced by the \c dump command; see @ref dump_formats for +details. -By default, if the table or file already exists, data in the file or -table will be overwritten by the new data (use the \c -n option to -make an attempt to overwrite existing data return an error). +By default, if the table already exists, data in the table will be +overwritten by the new data (use the \c -n option to make an attempt to +overwrite existing data return an error). @subsection util_load_synopsis Synopsis wt [-RVv] [-C config] [-E secretkey ] [-h directory] load [-ajn] [-f input] [-r name] [uri configuration ...] @@ -182,8 +182,8 @@ number keys. The \c -a option is only applicable when loading into a column store. @par -f -By default, the \c load command reads from the standard input; the \c --f option reads the input from the specified file. +By default, the \c load command reads from the standard input; the \c -f +option reads the input from the specified file. @par -j Load input in the JSON (JavaScript Object @@ -196,7 +196,7 @@ load command to fail if there's an attempt to overwrite already existing data. @par -r -By default, the \c load command uses the table or file name taken from the +By default, the \c load command uses the table name taken from the input; the \c -r option renames the data source. Additionally, \c uri and \c configuration pairs may be specified to the @@ -227,21 +227,20 @@ table:xxx block_allocation=first table:xxx prefix_compress=false
@section util_loadtext wt loadtext -Load text into a table or file. +Load text into a table. The \c loadtext command reads the standard input for text and loads it -into a table or file. The input data should be printable characters, -with newline delimiters for each key or value. +into a table. The input data should be printable characters, with +newline delimiters for each key or value. The \c loadtext command does not create the object if it does not yet exist. -In the case of inserting values into a column-store table or file, each -value is appended to the table or file; in the case of inserting values -into a row-store table or file, lines are handled in pairs, where the -first line is the key and the second line is the value. If the -row-store table or file already exists, data in the table or file will -be overwritten by the new data. +In the case of inserting values into a column-store table, each value +is appended to the table; in the case of inserting values into a +row-store table, lines are handled in pairs, where the first line is the +key and the second line is the value. If the row-store table already +exists, data in the table will be overwritten by the new data. @subsection util_loadtext_synopsis Synopsis wt [-RVv] [-C config] [-E secretkey ] [-h directory] loadtext [-f input] uri @@ -275,7 +274,7 @@ to the default string format.
@section util_read wt read -Read records from a table or file. +Read records from a table. The \c read command prints out the records associated with the specified keys from the specified data source. The data source must be configured @@ -291,9 +290,9 @@ The \c read command has no command-specific options.
@section util_rename wt rename -Rename a table or file. +Rename a table. -The \c rename command renames the specified table or file. +The \c rename command renames the specified table. @subsection util_rename_synopsis Synopsis wt [-RVv] [-C config] [-E secretkey ] [-h directory] rename uri name @@ -303,11 +302,11 @@ The \c rename command has no command-specific options.
@section util_salvage wt salvage -Recover data from a corrupted file. +Recover data from a corrupted table. The \c salvage command salvages the specified data source, discarding any -data that cannot be recovered. Underlying files are re-written in -place, overwriting the original file contents. +data that cannot be recovered. Underlying files are re-written in place, +overwriting the original file contents. @subsection util_salvage_synopsis Synopsis wt [-RVv] [-C config] [-E secretkey ] [-h directory] salvage [-F force] uri @@ -316,9 +315,9 @@ place, overwriting the original file contents. The following are command-specific options for the \c salvage command: @par -F -By default, salvage will refuse to salvage files that fail basic tests -(for example, files that don't appear to be in a WiredTiger format). -The \c -F option forces the salvage of the file, regardless. +By default, salvage will refuse to salvage tables that fail basic tests +(for example, tables that don't appear to be in a WiredTiger format). +The \c -F option forces the salvage of the table, regardless.
@section util_stat wt stat @@ -339,11 +338,11 @@ Include only "fast" statistics in the output (equivalent to passing
@section util_upgrade wt upgrade -Upgrade a table or file. +Upgrade a table. -The \c upgrade command upgrades the specified table or file, exiting -success if the data source is up-to-date, and failure if the data source -cannot be upgraded. +The \c upgrade command upgrades the specified table, exiting success if +the data source is up-to-date, and failure if the data source cannot be +upgraded. @subsection util_upgrade_synopsis Synopsis wt [-RVv] [-C config] [-E secretkey ] [-h directory] upgrade uri @@ -353,11 +352,10 @@ The \c upgrade command has no command-specific options.
@section util_verify wt verify -Check the structural integrity of a table or file. +Check the structural integrity of a table. -The \c verify command verifies the specified table or file, exiting -success if the data source is correct, and failure if the data source is -corrupted. +The \c verify command verifies the specified table, exiting success if +the data source is correct, and failure if the data source is corrupted. @subsection util_verify_synopsis Synopsis wt [-RVv] [-C config] [-E secretkey ] [-h directory] verify uri @@ -367,7 +365,7 @@ The \c verify command has no command-specific options.
@section util_write wt write -Write records to a table or file. +Write records to a table. The \c write command stores records into the specified data source. The data source must be configured with string or record number keys and -- cgit v1.2.1 From 68ae93fc02ac2e2e8dce69820c6d736dfbe87766 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 12 Feb 2016 17:40:22 -0500 Subject: WT-2381: dump utility discards table config Fix for WiredTiger "simple" table handling. Simple tables have column-group entries, but they aren't listed in the metadata's table entry. Figure out if it's a simple table and in that case, retrieve the column-group entry and use the value from its "source" file. --- dist/s_string.ok | 1 + src/utilities/util_dump.c | 112 ++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 94 insertions(+), 19 deletions(-) diff --git a/dist/s_string.ok b/dist/s_string.ok index 7a8f3a9b0bd..d41acc36ed6 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -758,6 +758,7 @@ memset memsize metaconf metadata +metadata's metafile mfence minorp diff --git a/src/utilities/util_dump.c b/src/utilities/util_dump.c index ca761a52d8a..287f784770c 100644 --- a/src/utilities/util_dump.c +++ b/src/utilities/util_dump.c @@ -352,12 +352,23 @@ match: if ((ret = cursor->get_key(cursor, &key)) != 0) static int dump_json_table_config(WT_SESSION *session, const char *uri) { + WT_CONFIG_ITEM cval; WT_CURSOR *cursor; WT_DECL_RET; + size_t len; int tret; - char *value; + const char *name, *value; + char *p; + + p = NULL; + + /* Get the table name. */ + if ((name = strchr(uri, ':')) == NULL) { + fprintf(stderr, "%s: %s: corrupted uri\n", progname, uri); + return (1); + } + ++name; - /* Dump the config. */ /* Open a metadata cursor. */ if ((ret = session->open_cursor( session, "metadata:create", NULL, NULL, &cursor)) != 0) { @@ -368,12 +379,41 @@ dump_json_table_config(WT_SESSION *session, const char *uri) } /* - * Search for the object itself, to make sure it - * exists, and get its config string. This where we - * find out a table object doesn't exist, use a simple - * error message. + * Search for the object itself, just to make sure it exists, we don't + * want to output a header if the user entered the wrong name. This is + * where we find out a table doesn't exist, use a simple error message. + * + * Workaround for WiredTiger "simple" table handling. Simple tables + * have column-group entries, but they aren't listed in the metadata's + * table entry. Figure out if it's a simple table and in that case, + * retrieve the column-group entry and use the value from its "source" + * file. */ - cursor->set_key(cursor, uri); + if (WT_PREFIX_MATCH(uri, "table:")) { + len = strlen(uri) + strlen("colgroup:"); + if ((p = malloc(len)) == NULL) + return (util_err(session, errno, NULL)); + (void)snprintf(p, len, "colgroup:%s", name); + cursor->set_key(cursor, p); + if ((ret = cursor->search(cursor)) == 0) { + if ((ret = cursor->get_value(cursor, &value)) != 0) + return (util_cerr(cursor, "get_value", ret)); + if ((ret = __wt_config_getones( + (WT_SESSION_IMPL *)session, + value, "source", &cval)) != 0) + return (util_err( + session, ret, "%s: source entry", p)); + free(p); + len = cval.len + 10; + if ((p = malloc(len)) == NULL) + return (util_err(session, errno, NULL)); + (void)snprintf(p, len, "%.*s", (int)cval.len, cval.str); + cursor->set_key(cursor, p); + } else + cursor->set_key(cursor, uri); + } else + cursor->set_key(cursor, uri); + if ((ret = cursor->search(cursor)) == 0) { if ((ret = cursor->get_value(cursor, &value)) != 0) ret = util_cerr(cursor, "get_value", ret); @@ -381,8 +421,7 @@ dump_json_table_config(WT_SESSION *session, const char *uri) session, cursor, uri, value) != 0) ret = 1; } else if (ret == WT_NOTFOUND) - ret = util_err( - session, 0, "%s: No such object exists", uri); + ret = util_err(session, 0, "%s: No such object exists", uri); else ret = util_err(session, ret, "%s", uri); @@ -392,6 +431,7 @@ dump_json_table_config(WT_SESSION *session, const char *uri) ret = tret; } + free(p); return (ret); } @@ -414,10 +454,15 @@ dump_json_table_end(WT_SESSION *session) static int dump_table_config(WT_SESSION *session, WT_CURSOR *cursor, const char *uri) { + WT_CONFIG_ITEM cval; WT_CURSOR *srch; WT_DECL_RET; + size_t len; int tret; - const char *key, *name, *value; + const char *name, *value; + char *p; + + p = NULL; /* Get the table name. */ if ((name = strchr(uri, ':')) == NULL) { @@ -427,17 +472,44 @@ dump_table_config(WT_SESSION *session, WT_CURSOR *cursor, const char *uri) ++name; /* - * Dump out the config information: first, dump the uri entry itself - * (requires a lookup). + * Dump out the config information: first, dump the uri entry itself. + * + * Workaround for WiredTiger "simple" table handling. Simple tables + * have column-group entries, but they aren't listed in the metadata's + * table entry. Figure out if it's a simple table and in that case, + * retrieve the column-group entry and use the value from its "source" + * file. */ - cursor->set_key(cursor, uri); + if (WT_PREFIX_MATCH(uri, "table:")) { + len = strlen(uri) + strlen("colgroup:"); + if ((p = malloc(len)) == NULL) + return (util_err(session, errno, NULL)); + (void)snprintf(p, len, "colgroup:%s", name); + cursor->set_key(cursor, p); + if ((ret = cursor->search(cursor)) == 0) { + if ((ret = cursor->get_value(cursor, &value)) != 0) + return (util_cerr(cursor, "get_value", ret)); + if ((ret =__wt_config_getones( + (WT_SESSION_IMPL *)session, + value, "source", &cval)) != 0) + return (util_err( + session, ret, "%s: source entry", p)); + free(p); + len = cval.len + 10; + if ((p = malloc(len)) == NULL) + return (util_err(session, errno, NULL)); + (void)snprintf(p, len, "%.*s", (int)cval.len, cval.str); + cursor->set_key(cursor, p); + } else + cursor->set_key(cursor, uri); + } else + cursor->set_key(cursor, uri); + if ((ret = cursor->search(cursor)) != 0) return (util_cerr(cursor, "search", ret)); - if ((ret = cursor->get_key(cursor, &key)) != 0) - return (util_cerr(cursor, "get_key", ret)); if ((ret = cursor->get_value(cursor, &value)) != 0) return (util_cerr(cursor, "get_value", ret)); - if (print_config(session, key, value, NULL) != 0) + if (print_config(session, uri, value, NULL) != 0) return (1); /* @@ -459,6 +531,7 @@ dump_table_config(WT_SESSION *session, WT_CURSOR *cursor, const char *uri) ret = tret; } + free(p); return (ret); } @@ -473,6 +546,7 @@ dump_table_config_type(WT_SESSION *session, WT_CONFIG_ITEM cval; WT_DECL_RET; const char *key, *skip, *value, *value_source; + size_t len; int exact; char *p; @@ -514,10 +588,10 @@ match: if ((ret = cursor->get_key(cursor, &key)) != 0) util_err(session, ret, "%s: source entry", key)); /* Nul-terminate the source entry. */ - if ((p = malloc(cval.len + 10)) == NULL) + len = cval.len + 10; + if ((p = malloc(len)) == NULL) return (util_err(session, errno, NULL)); - (void)strncpy(p, cval.str, cval.len); - p[cval.len] = '\0'; + (void)snprintf(p, len, "%.*s", (int)cval.len, cval.str); srch->set_key(srch, p); if ((ret = srch->search(srch)) != 0) ret = util_err(session, ret, "%s: %s", key, p); -- cgit v1.2.1 From f70cd24413cf311305c58a3156849ebf5ecbdf39 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 15 Feb 2016 12:21:50 -0500 Subject: WT-2397: Cursor traversal from end of the tree skips records. Positioning a cursor at the end of the tree can race with page splits. During the initial descent of the tree in the cursor.prev case, do the same tests for races we do for a search past the end of the tree. --- src/btree/bt_walk.c | 116 +++++++++++++++++++++++++++++++++++++++------------ src/btree/col_srch.c | 5 +-- src/btree/row_srch.c | 5 +-- src/include/btree.i | 12 ++++-- 4 files changed, 101 insertions(+), 37 deletions(-) diff --git a/src/btree/bt_walk.c b/src/btree/bt_walk.c index d7785c689d9..90c806c6ab8 100644 --- a/src/btree/bt_walk.c +++ b/src/btree/bt_walk.c @@ -163,12 +163,12 @@ __page_ascend(WT_SESSION_IMPL *session, } /* - * __page_descend -- - * Descend the tree one level. + * __page_descend_prev -- + * Descend the tree one level, during a previous-cursor walk. */ static void -__page_descend(WT_SESSION_IMPL *session, - WT_PAGE *page, WT_PAGE_INDEX **pindexp, uint32_t *slotp, bool prev) +__page_descend_prev(WT_SESSION_IMPL *session, + WT_REF *ref, WT_PAGE_INDEX **pindexp, uint32_t *slotp) { WT_PAGE_INDEX *pindex; @@ -177,9 +177,6 @@ __page_descend(WT_SESSION_IMPL *session, * we have a hazard pointer. */ for (;; __wt_yield()) { - WT_INTL_INDEX_GET(session, page, pindex); - *slotp = prev ? pindex->entries - 1 : 0; - /* * There's a split race when a cursor moving backwards through * the tree descends the tree. If we're splitting an internal @@ -233,20 +230,40 @@ __page_descend(WT_SESSION_IMPL *session, * being split and part of its namespace moved. We have the * correct page and we don't have to move, all we have to do is * wait until the split page's page index is updated. - * - * No test is necessary for a next-cursor movement because we - * do right-hand splits on internal pages and the initial part - * of the page's namespace won't change as part of a split. - * Instead of testing the direction boolean, do the test the - * previous cursor movement requires in all cases, even though - * it will always succeed for a next-cursor movement. */ - if (pindex->index[*slotp]->home == page) + WT_INTL_INDEX_GET(session, ref->page, pindex); + *slotp = pindex->entries - 1; + if (pindex->index[*slotp]->home == ref->page) break; } *pindexp = pindex; } +/* + * __page_initial_descent_prev -- + * Descend the tree one level, when setting up the initial cursor position + * for a previous-cursor walk. + */ +static bool +__page_initial_descent_prev(WT_SESSION_IMPL *session, + WT_REF *ref, WT_PAGE_INDEX **pindexp, uint32_t *slotp) +{ + WT_PAGE_INDEX *parent_pindex, *pindex; + + /* + * We're passed a child page into which we're descending, and on which + * we have a hazard pointer. + */ + parent_pindex = *pindexp; + WT_INTL_INDEX_GET(session, ref->page, pindex); + *slotp = pindex->entries - 1; + if (__wt_split_descent_race(session, ref, parent_pindex)) + return (false); + + *pindexp = pindex; + return (true); +} + /* * __tree_walk_internal -- * Move to the next/previous page in the tree. @@ -259,11 +276,12 @@ __tree_walk_internal(WT_SESSION_IMPL *session, WT_DECL_RET; WT_PAGE_INDEX *pindex; WT_REF *couple, *couple_orig, *ref; - bool empty_internal, prev, skip; + bool empty_internal, initial_descent, prev, skip; uint32_t slot; btree = S2BT(session); - empty_internal = false; + pindex = NULL; + empty_internal = initial_descent = false; /* * Tree walks are special: they look inside page structures that splits @@ -325,9 +343,14 @@ __tree_walk_internal(WT_SESSION_IMPL *session, /* If no page is active, begin a walk from the start of the tree. */ if (ref == NULL) { - ref = &btree->root; +restart: if (couple != &btree->root) + WT_ERR(__wt_page_release(session, couple, flags)); + + couple = couple_orig = ref = &btree->root; if (ref->page == NULL) goto done; + + initial_descent = true; goto descend; } @@ -336,7 +359,7 @@ __tree_walk_internal(WT_SESSION_IMPL *session, * Release any hazard-pointer we're holding. */ if (__wt_ref_is_root(ref)) { - WT_ERR(__wt_page_release(session, couple, flags)); + WT_ERR(__wt_page_release(session, ref, flags)); goto done; } @@ -525,12 +548,8 @@ __tree_walk_internal(WT_SESSION_IMPL *session, * root to a new saved position in the tree, * restart the walk. */ - if (couple == &btree->root) { - ref = &btree->root; - if (ref->page == NULL) - goto done; - goto descend; - } + if (couple == &btree->root) + goto restart; /* * If restarting from some original position, @@ -561,9 +580,52 @@ __tree_walk_internal(WT_SESSION_IMPL *session, descend: couple = ref; empty_internal = true; - __page_descend( - session, ref->page, &pindex, &slot, prev); + /* + * There's a split race when a cursor is setting + * up at the end of the tree or moving backwards + * through the tree and descending a level. When + * splitting an internal page into its parent, + * we move the WT_REF structures and update the + * parent's page index before updating the split + * page's page index, and it's not an atomic + * update. A thread can read the parent page's + * replacement page index, then read the split + * page's original index. + * + * This isn't a problem for a cursor setting up + * at the start of the tree or moving forwards + * through the tree because we do right-hand + * splits on internal pages and the initial part + * of the split page's namespace won't change as + * part of a split. A thread reading the parent + * page's and split page's indexes will move to + * the same slot no matter what combination of + * indexes are read. + * + * Handle a cursor setting up at the end of the + * tree or moving backwards through the tree. + */ + if (!prev) { + WT_INTL_INDEX_GET( + session, ref->page, pindex); + slot = 0; + } else if (initial_descent) { + if (!__page_initial_descent_prev( + session, ref, &pindex, &slot)) + goto restart; + } else + __page_descend_prev( + session, ref, &pindex, &slot); } else { + /* + * At the lowest tree level (considering a leaf + * page), turn off the initial-descent test. + * The descent race tests are different when + * moving through the tree vs. doing the initial + * descent. + */ + initial_descent = false; + /* * Optionally skip leaf pages, the second half. * We didn't have an on-page cell to figure out diff --git a/src/btree/col_srch.c b/src/btree/col_srch.c index 862327699c0..23eae75ec2b 100644 --- a/src/btree/col_srch.c +++ b/src/btree/col_srch.c @@ -145,9 +145,8 @@ restart: /* * If on the last slot (the key is larger than any key * on the page), check for an internal page split race. */ - if (parent_pindex != NULL && - __wt_split_intl_race( - session, current->home, parent_pindex)) + if (__wt_split_descent_race( + session, current, parent_pindex)) goto restart; goto descend; diff --git a/src/btree/row_srch.c b/src/btree/row_srch.c index 955568bcd50..9d68c8e0ce7 100644 --- a/src/btree/row_srch.c +++ b/src/btree/row_srch.c @@ -425,9 +425,8 @@ restart: /* * page), check for an internal page split race. */ if (pindex->entries == base) { -append: if (parent_pindex != NULL && - __wt_split_intl_race( - session, current->home, parent_pindex)) +append: if (__wt_split_descent_race( + session, current, parent_pindex)) goto restart; } diff --git a/src/include/btree.i b/src/include/btree.i index 16906aa7cfa..422e67a1ebe 100644 --- a/src/include/btree.i +++ b/src/include/btree.i @@ -1446,15 +1446,19 @@ __wt_btree_lsm_over_size(WT_SESSION_IMPL *session, uint64_t maxsize) } /* - * __wt_split_intl_race -- + * __wt_split_descent_race -- * Return if we raced with an internal page split when descending the tree. */ static inline bool -__wt_split_intl_race( - WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE_INDEX *saved_pindex) +__wt_split_descent_race( + WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE_INDEX *saved_pindex) { WT_PAGE_INDEX *pindex; + /* No test when starting the descent (there's no home to check). */ + if (__wt_ref_is_root(ref)) + return (false); + /* * A place to hang this comment... * @@ -1509,6 +1513,6 @@ __wt_split_intl_race( * content the split page retains after the split, and we ignore this * race. */ - WT_INTL_INDEX_GET(session, parent, pindex); + WT_INTL_INDEX_GET(session, ref->home, pindex); return (pindex != saved_pindex); } -- cgit v1.2.1 From 1678425f39d2573fed5624bad150f63bb3069a34 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 15 Feb 2016 20:57:30 +0000 Subject: WT-2397: Cursor traversal from end of the tree skips records. Comment/clarify/simplify the hazard reference handling, clean up some other comments. --- src/btree/bt_walk.c | 33 ++++++++++++++++++--------------- src/include/btree.i | 14 +++++++------- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/src/btree/bt_walk.c b/src/btree/bt_walk.c index 90c806c6ab8..aacca540fa8 100644 --- a/src/btree/bt_walk.c +++ b/src/btree/bt_walk.c @@ -341,10 +341,14 @@ __tree_walk_internal(WT_SESSION_IMPL *session, couple = couple_orig = ref = *refp; *refp = NULL; - /* If no page is active, begin a walk from the start of the tree. */ + /* If no page is active, begin a walk from the start/end of the tree. */ if (ref == NULL) { -restart: if (couple != &btree->root) - WT_ERR(__wt_page_release(session, couple, flags)); +restart: /* + * We can reach here with a NULL or root reference; the release + * function handles them internally, don't complicate this code + * by calling them out. + */ + WT_ERR(__wt_page_release(session, couple, flags)); couple = couple_orig = ref = &btree->root; if (ref->page == NULL) @@ -355,13 +359,12 @@ restart: if (couple != &btree->root) } /* - * If the active page was the root, we've reached the walk's end. - * Release any hazard-pointer we're holding. + * If the active page was the root, we've reached the walk's end; we + * only get here if we've returned the root to our caller, so we're + * holding no hazard references. */ - if (__wt_ref_is_root(ref)) { - WT_ERR(__wt_page_release(session, ref, flags)); + if (__wt_ref_is_root(ref)) goto done; - } /* Figure out the current slot in the WT_REF array. */ __ref_index_slot(session, ref, &pindex, &slot); @@ -590,7 +593,8 @@ descend: couple = ref; * page's page index, and it's not an atomic * update. A thread can read the parent page's * replacement page index, then read the split - * page's original index. + * page's original index, or the parent page's + * original and the split page's replacement. * * This isn't a problem for a cursor setting up * at the start of the tree or moving forwards @@ -599,8 +603,8 @@ descend: couple = ref; * of the split page's namespace won't change as * part of a split. A thread reading the parent * page's and split page's indexes will move to - * the same slot no matter what combination of - * indexes are read. + * the same slot no matter what order of indexes + * are read. * * Handle a cursor setting up at the end of the * tree or moving backwards through the tree. @@ -619,10 +623,9 @@ descend: couple = ref; } else { /* * At the lowest tree level (considering a leaf - * page), turn off the initial-descent test. - * The descent race tests are different when - * moving through the tree vs. doing the initial - * descent. + * page), turn off the initial-descent state. + * Descent race tests are different when moving + * through the tree vs. the initial descent. */ initial_descent = false; diff --git a/src/include/btree.i b/src/include/btree.i index 422e67a1ebe..6df7f87073f 100644 --- a/src/include/btree.i +++ b/src/include/btree.i @@ -1293,20 +1293,20 @@ __wt_page_swap_func( WT_DECL_RET; bool acquired; - /* - * In rare cases when walking the tree, we try to swap to the same - * page. Fast-path that to avoid thinking about error handling. - */ - if (held == want) - return (0); - /* * This function is here to simplify the error handling during hazard * pointer coupling so we never leave a hazard pointer dangling. The * assumption is we're holding a hazard pointer on "held", and want to * acquire a hazard pointer on "want", releasing the hazard pointer on * "held" when we're done. + * + * When walking the tree, we sometimes swap to the same page. Fast-path + * that to avoid thinking about error handling. */ + if (held == want) + return (0); + + /* Get the wanted page. */ ret = __wt_page_in_func(session, want, flags #ifdef HAVE_DIAGNOSTIC , file, line -- cgit v1.2.1 From 642b4cdeebdac14c15a27f08084b9e43c3ed18cb Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 15 Feb 2016 17:41:45 -0500 Subject: WT-2397: Cursor traversal from end of the tree skips records. whitespace cleanup. --- src/btree/bt_walk.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/btree/bt_walk.c b/src/btree/bt_walk.c index aacca540fa8..d5a57406a37 100644 --- a/src/btree/bt_walk.c +++ b/src/btree/bt_walk.c @@ -361,7 +361,7 @@ restart: /* /* * If the active page was the root, we've reached the walk's end; we * only get here if we've returned the root to our caller, so we're - * holding no hazard references. + * holding no hazard pointers. */ if (__wt_ref_is_root(ref)) goto done; @@ -670,7 +670,7 @@ __wt_tree_walk(WT_SESSION_IMPL *session, WT_REF **refp, uint32_t flags) /* * __wt_tree_walk_count -- * Move to the next/previous page in the tree, tracking how many - * references were visited to get there. + * references were visited to get there. */ int __wt_tree_walk_count(WT_SESSION_IMPL *session, -- cgit v1.2.1 From d3893dbcbff0b0b03f6d0b83eb6651f1810cc2c6 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 16 Feb 2016 00:37:22 +0000 Subject: WT-2397: Cursor traversal from end of the tree skips records. If we're half-way down the tree, and get a RESTART trying to swap to a new page, and the reason we get the RESTART is that our page split into our parent, and caused our parent to split into its parent, we can't use our parent's page-index, because it's been truncated. Do a full restart. Simplify the __page_descend_prev() and __page_initial_descent_prev() functions, they don't need to set the slot, the caller can do that, and remove an unnecessary local variable. --- src/btree/bt_walk.c | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/src/btree/bt_walk.c b/src/btree/bt_walk.c index d5a57406a37..4956bcbb20e 100644 --- a/src/btree/bt_walk.c +++ b/src/btree/bt_walk.c @@ -167,8 +167,8 @@ __page_ascend(WT_SESSION_IMPL *session, * Descend the tree one level, during a previous-cursor walk. */ static void -__page_descend_prev(WT_SESSION_IMPL *session, - WT_REF *ref, WT_PAGE_INDEX **pindexp, uint32_t *slotp) +__page_descend_prev( + WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE_INDEX **pindexp) { WT_PAGE_INDEX *pindex; @@ -232,8 +232,7 @@ __page_descend_prev(WT_SESSION_IMPL *session, * wait until the split page's page index is updated. */ WT_INTL_INDEX_GET(session, ref->page, pindex); - *slotp = pindex->entries - 1; - if (pindex->index[*slotp]->home == ref->page) + if (pindex->index[pindex->entries - 1]->home == ref->page) break; } *pindexp = pindex; @@ -245,19 +244,20 @@ __page_descend_prev(WT_SESSION_IMPL *session, * for a previous-cursor walk. */ static bool -__page_initial_descent_prev(WT_SESSION_IMPL *session, - WT_REF *ref, WT_PAGE_INDEX **pindexp, uint32_t *slotp) +__page_initial_descent_prev( + WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE_INDEX **pindexp) { - WT_PAGE_INDEX *parent_pindex, *pindex; + WT_PAGE_INDEX *pindex; /* * We're passed a child page into which we're descending, and on which * we have a hazard pointer. + * + * Acquire a page index for the child page and then confirm we haven't + * raced with a parent split. */ - parent_pindex = *pindexp; WT_INTL_INDEX_GET(session, ref->page, pindex); - *slotp = pindex->entries - 1; - if (__wt_split_descent_race(session, ref, parent_pindex)) + if (__wt_split_descent_race(session, ref, *pindexp)) return (false); *pindexp = pindex; @@ -546,6 +546,15 @@ restart: /* if (ret == WT_RESTART) { ret = 0; + /* + * If a cursor is setting up at the end of the + * tree, we can't use our parent page's index, + * because it may have already split; restart + * the walk. + */ + if (prev && initial_descent) + goto restart; + /* * If a new walk that never coupled from the * root to a new saved position in the tree, @@ -615,11 +624,14 @@ descend: couple = ref; slot = 0; } else if (initial_descent) { if (!__page_initial_descent_prev( - session, ref, &pindex, &slot)) + session, ref, &pindex)) goto restart; - } else + slot = pindex->entries - 1; + } else { __page_descend_prev( - session, ref, &pindex, &slot); + session, ref, &pindex); + slot = pindex->entries - 1; + } } else { /* * At the lowest tree level (considering a leaf -- cgit v1.2.1 From 8a95c6eed396d0e837e36fc33a5378be85c8b978 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Tue, 16 Feb 2016 14:37:43 +1100 Subject: WT-2396 Grab the checkpoint lock around drops and renames. Since database-wide checkpoints have references to all handles in the system, drops are generally not possible during a checkpoint regardless of this change. This prevents a deadlock where checkpoint waits on the handle list lock after it has some handles locked exclusive. A better longer-term solution would be to have checkpoint gather all handles (including locking old checkpoints) at the beginning so it doesn't have to acquire the handle list lock after locking handles. --- src/session/session_api.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/session/session_api.c b/src/session/session_api.c index c03b5fdc044..5511674dc5e 100644 --- a/src/session/session_api.c +++ b/src/session/session_api.c @@ -597,9 +597,10 @@ __session_rename(WT_SESSION *wt_session, WT_ERR(__wt_str_name_check(session, uri)); WT_ERR(__wt_str_name_check(session, newuri)); - WT_WITH_SCHEMA_LOCK(session, ret, - WT_WITH_TABLE_LOCK(session, ret, - ret = __wt_schema_rename(session, uri, newuri, cfg))); + WT_WITH_CHECKPOINT_LOCK(session, ret, + WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_TABLE_LOCK(session, ret, + ret = __wt_schema_rename(session, uri, newuri, cfg)))); err: API_END_RET_NOTFOUND_MAP(session, ret); } @@ -646,9 +647,10 @@ __wt_session_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) if (!lock_wait) F_SET(session, WT_SESSION_LOCK_NO_WAIT); - WT_WITH_SCHEMA_LOCK(session, ret, - WT_WITH_TABLE_LOCK(session, ret, - ret = __wt_schema_drop(session, uri, cfg))); + WT_WITH_CHECKPOINT_LOCK(session, ret, + WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_TABLE_LOCK(session, ret, + ret = __wt_schema_drop(session, uri, cfg)))); if (!lock_wait) F_CLR(session, WT_SESSION_LOCK_NO_WAIT); -- cgit v1.2.1 From 9f1a9bc198fdac975bc5bea1fed487b877d37a18 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Tue, 16 Feb 2016 17:44:17 +1100 Subject: WT-2399 Add initial implementation for cursor traversal test application. Based loosely on the test/checkpoint implementation. Hopefully we can refactor, so code is shared between the two via test_util.i. --- build_posix/Make.subdirs | 1 + test/cursor_order/Makefile.am | 13 ++ test/cursor_order/cursor_order.c | 303 ++++++++++++++++++++++++++++ test/cursor_order/cursor_order.h | 63 ++++++ test/cursor_order/cursor_order_file.c | 129 ++++++++++++ test/cursor_order/cursor_order_ops.c | 361 ++++++++++++++++++++++++++++++++++ 6 files changed, 870 insertions(+) create mode 100644 test/cursor_order/Makefile.am create mode 100644 test/cursor_order/cursor_order.c create mode 100644 test/cursor_order/cursor_order.h create mode 100644 test/cursor_order/cursor_order_file.c create mode 100644 test/cursor_order/cursor_order_ops.c diff --git a/build_posix/Make.subdirs b/build_posix/Make.subdirs index e1f8a05c613..bc4283a4876 100644 --- a/build_posix/Make.subdirs +++ b/build_posix/Make.subdirs @@ -26,6 +26,7 @@ lang/python PYTHON # Make the tests test/bloom test/checkpoint +test/cursor_order test/fops test/format test/huge diff --git a/test/cursor_order/Makefile.am b/test/cursor_order/Makefile.am new file mode 100644 index 00000000000..c0c0ed639bf --- /dev/null +++ b/test/cursor_order/Makefile.am @@ -0,0 +1,13 @@ +AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \ + -I$(top_srcdir)/test/utility + +noinst_PROGRAMS = cursor_order +cursor_order_LDADD = $(top_builddir)/libwiredtiger.la + +cursor_order_SOURCES = cursor_order_file.c cursor_order_ops.c cursor_order.c +cursor_order_LDFLAGS = -static + +TESTS = $(noinst_PROGRAMS) + +clean-local: + rm -rf WiredTiger* wt.* *.core __stats diff --git a/test/cursor_order/cursor_order.c b/test/cursor_order/cursor_order.c new file mode 100644 index 00000000000..0155339d71e --- /dev/null +++ b/test/cursor_order/cursor_order.c @@ -0,0 +1,303 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "cursor_order.h" + +static char home[512]; /* Program working dir */ +static char *progname; /* Program name */ +static FILE *logfp; /* Log file */ + +static int handle_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); +static int handle_message(WT_EVENT_HANDLER *, WT_SESSION *, const char *); +static void onint(int); +static void shutdown(void); +static int usage(void); +static void wt_connect(SHARED_CONFIG *, char *); +static void wt_shutdown(SHARED_CONFIG *); + +extern int __wt_optind; +extern char *__wt_optarg; + +int +main(int argc, char *argv[]) +{ + SHARED_CONFIG _cfg, *cfg; + int ch, cnt, runs; + char *config_open, *working_dir; + + if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) + progname = argv[0]; + else + ++progname; + + cfg = &_cfg; + config_open = NULL; + working_dir = NULL; + runs = 1; + + /* + * Explicitly initialize the shared configuration object before + * parsing command line options. + */ + cfg->append_inserters = 1; + cfg->conn = NULL; + cfg->ftype = ROW; + cfg->max_nops = 1000000; + cfg->multiple_files = false; + cfg->nkeys = 1000; + cfg->reverse_scanners = 5; + cfg->reverse_scan_ops = 10; + cfg->thread_finish = false; + cfg->vary_nops = false; + + while ((ch = __wt_getopt( + progname, argc, argv, "C:Fk:h:l:n:R:r:t:vw:W:")) != EOF) + switch (ch) { + case 'C': /* wiredtiger_open config */ + config_open = __wt_optarg; + break; + case 'F': /* multiple files */ + cfg->multiple_files = true; + break; + case 'h': + working_dir = __wt_optarg; + break; + case 'k': /* rows */ + cfg->nkeys = (u_int64_t)atol(__wt_optarg); + break; + case 'l': /* log */ + if ((logfp = fopen(__wt_optarg, "w")) == NULL) { + fprintf(stderr, + "%s: %s\n", __wt_optarg, strerror(errno)); + return (EXIT_FAILURE); + } + break; + case 'n': /* operations */ + cfg->max_nops = (u_int64_t)atol(__wt_optarg); + break; + case 'R': + cfg->reverse_scanners = (u_int64_t)atol(__wt_optarg); + break; + case 'r': /* runs */ + runs = atoi(__wt_optarg); + break; + case 't': + switch (__wt_optarg[0]) { + case 'f': + cfg->ftype = FIX; + break; + case 'r': + cfg->ftype = ROW; + break; + case 'v': + cfg->ftype = VAR; + break; + default: + return (usage()); + } + break; + case 'v': /* vary operation count */ + cfg->vary_nops = true; + break; + case 'w': + cfg->reverse_scan_ops = (u_int64_t)atol(__wt_optarg); + break; + case 'W': + cfg->append_inserters = (u_int64_t)atol(__wt_optarg); + break; + default: + return (usage()); + } + + argc -= __wt_optind; + argv += __wt_optind; + if (argc != 0) + return (usage()); + + testutil_work_dir_from_path(home, 512, working_dir); + + if (cfg->vary_nops && !cfg->multiple_files) { + fprintf(stderr, + "Variable op counts only supported with multiple tables\n"); + return (usage()); + } + + /* Clean up on signal. */ + (void)signal(SIGINT, onint); + + printf("%s: process %" PRIu64 "\n", progname, (uint64_t)getpid()); + for (cnt = 1; runs == 0 || cnt <= runs; ++cnt) { + printf(" %d: %u reverse scanners, %u writers\n", cnt, + (int)cfg->reverse_scanners, (int)cfg->append_inserters); + + shutdown(); /* Clean up previous runs */ + + wt_connect(cfg, config_open); /* WiredTiger connection */ + + if (ops_start(cfg)) + return (EXIT_FAILURE); + + wt_shutdown(cfg); /* WiredTiger shut down */ + } + return (0); +} + +/* + * wt_connect -- + * Configure the WiredTiger connection. + */ +static void +wt_connect(SHARED_CONFIG *cfg, char *config_open) +{ + static WT_EVENT_HANDLER event_handler = { + handle_error, + handle_message, + NULL, + NULL /* Close handler. */ + }; + int ret; + char config[512]; + size_t print_count; + + testutil_clean_work_dir(home); + testutil_make_work_dir(home); + + print_count = (size_t)snprintf(config, sizeof(config), + "create,statistics=(all),error_prefix=\"%s\",%s%s", + progname, + config_open == NULL ? "" : ",", + config_open == NULL ? "" : config_open); + + if (print_count >= sizeof(config)) + testutil_die(EINVAL, "Config string too long"); + + if ((ret = wiredtiger_open( + home, &event_handler, config, &cfg->conn)) != 0) + testutil_die(ret, "wiredtiger_open"); +} + +/* + * wt_shutdown -- + * Flush the file to disk and shut down the WiredTiger connection. + */ +static void +wt_shutdown(SHARED_CONFIG *cfg) +{ + WT_CONNECTION *conn; + WT_SESSION *session; + int ret; + + conn = cfg->conn; + + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + testutil_die(ret, "conn.session"); + + if ((ret = session->checkpoint(session, NULL)) != 0) + testutil_die(ret, "session.checkpoint"); + + if ((ret = conn->close(conn, NULL)) != 0) + testutil_die(ret, "conn.close"); +} + +/* + * shutdown -- + * Clean up from previous runs. + */ +static void +shutdown(void) +{ + testutil_clean_work_dir(home); +} + +static int +handle_error(WT_EVENT_HANDLER *handler, + WT_SESSION *session, int error, const char *errmsg) +{ + (void)(handler); + (void)(session); + (void)(error); + + return (fprintf(stderr, "%s\n", errmsg) < 0 ? -1 : 0); +} + +static int +handle_message(WT_EVENT_HANDLER *handler, + WT_SESSION *session, const char *message) +{ + (void)(handler); + (void)(session); + + if (logfp != NULL) + return (fprintf(logfp, "%s\n", message) < 0 ? -1 : 0); + + return (printf("%s\n", message) < 0 ? -1 : 0); +} + +/* + * onint -- + * Interrupt signal handler. + */ +static void +onint(int signo) +{ + (void)(signo); + + shutdown(); + + fprintf(stderr, "\n"); + exit(EXIT_FAILURE); +} + +/* + * usage -- + * Display usage statement and exit failure. + */ +static int +usage(void) +{ + fprintf(stderr, + "usage: %s " + "[-FLv] [-C wiredtiger-config] [-k keys] [-l log]\n\t" + "[-n ops] [-R reverse_scanners] [-r runs] [-t f|r|v] " + "[-W append_inserters]\n", + progname); + fprintf(stderr, "%s", + "\t-C specify wiredtiger_open configuration arguments\n" + "\t-F create a file per thread\n" + "\t-k set number of keys to load\n" + "\t-L log print per operation\n" + "\t-l specify a log file\n" + "\t-n set number of operations each thread does\n" + "\t-R set number of reverse scanner threads\n" + "\t-r set number of runs (0 for continuous)\n" + "\t-t set a file type (fix | row | var)\n" + "\t-v do a different number of operations on different tables\n" + "\t-w set number of items to walk in a reverse scan\n" + "\t-W set number of threads doing append inserts\n"); + return (EXIT_FAILURE); +} diff --git a/test/cursor_order/cursor_order.h b/test/cursor_order/cursor_order.h new file mode 100644 index 00000000000..9a3ae51ed91 --- /dev/null +++ b/test/cursor_order/cursor_order.h @@ -0,0 +1,63 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test_util.i" + +#define FNAME "file:cursor_order.%03d" /* File name */ + +typedef enum { FIX, ROW, VAR } __ftype; /* File type */ +typedef struct { + uint64_t append_inserters; /* Number of append threads */ + WT_CONNECTION *conn; /* WiredTiger connection */ + __ftype ftype; + uint64_t key_range; /* Current key range */ + uint64_t max_nops; /* Operations per thread */ + bool multiple_files; /* File per thread */ + uint64_t nkeys; /* Keys to load */ + uint64_t reverse_scanners; /* Number of scan threads */ + uint64_t reverse_scan_ops; /* Keys to visit per scan */ + bool thread_finish; /* Signal to finish run. */ + bool vary_nops; /* Operations per thread */ + +} SHARED_CONFIG; + +void load(SHARED_CONFIG *, const char *); +int ops_start(SHARED_CONFIG *); +void verify(SHARED_CONFIG *, const char *); diff --git a/test/cursor_order/cursor_order_file.c b/test/cursor_order/cursor_order_file.c new file mode 100644 index 00000000000..81c76a3a63e --- /dev/null +++ b/test/cursor_order/cursor_order_file.c @@ -0,0 +1,129 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "cursor_order.h" + +static void +file_create(SHARED_CONFIG *cfg, const char *name) +{ + WT_CONNECTION *conn; + WT_SESSION *session; + int ret; + char *p, *end, config[128]; + + conn = cfg->conn; + + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + testutil_die(ret, "conn.session"); + + p = config; + end = config + sizeof(config); + p += snprintf(p, (size_t)(end - p), + "key_format=%s," + "internal_page_max=%d," + "split_deepen_min_child=200," + "leaf_page_max=%d,", + cfg->ftype == ROW ? "S" : "r", 16 * 1024, 128 * 1024); + if (cfg->ftype == FIX) + (void)snprintf(p, (size_t)(end - p), ",value_format=3t"); + + if ((ret = session->create(session, name, config)) != 0) + if (ret != EEXIST) + testutil_die(ret, "session.create"); + + if ((ret = session->close(session, NULL)) != 0) + testutil_die(ret, "session.close"); +} + +void +load(SHARED_CONFIG *cfg, const char *name) +{ + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_ITEM *value, _value; + WT_SESSION *session; + char keybuf[64], valuebuf[64]; + int64_t keyno; + int ret; + + conn = cfg->conn; + + file_create(cfg, name); + + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + testutil_die(ret, "conn.session"); + + if ((ret = + session->open_cursor(session, name, NULL, "bulk", &cursor)) != 0) + testutil_die(ret, "cursor.open"); + + value = &_value; + for (keyno = 1; keyno <= (int64_t)cfg->nkeys; ++keyno) { + if (cfg->ftype == ROW) { + snprintf(keybuf, sizeof(keybuf), "%016u", (u_int)keyno); + cursor->set_key(cursor, &keybuf); + } else + cursor->set_key(cursor, (uint32_t)keyno); + value->data = valuebuf; + if (cfg->ftype == FIX) + cursor->set_value(cursor, 0x01); + else { + value->size = (uint32_t)snprintf( + valuebuf, sizeof(valuebuf), "%37u", (u_int)keyno); + cursor->set_value(cursor, value); + } + if ((ret = cursor->insert(cursor)) != 0) + testutil_die(ret, "cursor.insert"); + } + /* Setup the starting key range for the workload phase. */ + cfg->key_range = cfg->nkeys; + cursor->close(cursor); + session->checkpoint(session, NULL); + + if ((ret = session->close(session, NULL)) != 0) + testutil_die(ret, "session.close"); +} + +void +verify(SHARED_CONFIG *cfg, const char *name) +{ + WT_CONNECTION *conn; + WT_SESSION *session; + int ret; + + conn = cfg->conn; + + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + testutil_die(ret, "conn.session"); + + if ((ret = session->verify(session, name, NULL)) != 0) + testutil_die(ret, "session.create"); + + if ((ret = session->close(session, NULL)) != 0) + testutil_die(ret, "session.close"); +} diff --git a/test/cursor_order/cursor_order_ops.c b/test/cursor_order/cursor_order_ops.c new file mode 100644 index 00000000000..c60923d9cc0 --- /dev/null +++ b/test/cursor_order/cursor_order_ops.c @@ -0,0 +1,361 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "cursor_order.h" + +static void print_stats(SHARED_CONFIG *); +static void *reverse_scan(void *); +static void *append_insert(void *); + +typedef struct { + char *name; /* object name */ + uint64_t nops; /* Thread op count */ + + WT_RAND_STATE rnd; /* RNG */ + + int append_insert; /* cursor.insert */ + int reverse_scans; /* cursor.prev sequences */ + SHARED_CONFIG *cfg; +} INFO; + +static INFO *run_info; + +int +ops_start(SHARED_CONFIG *cfg) +{ + struct timeval start, stop; + double seconds; + pthread_t *tids; + uint64_t i, name_index, offset, total_nops; + int ret; + void *thread_ret; + + tids = NULL; /* Keep GCC 4.1 happy. */ + total_nops = 0; + + /* Create per-thread structures. */ + if ((run_info = calloc( + (size_t)(cfg->reverse_scanners + cfg->append_inserters), + sizeof(*run_info))) == NULL) + testutil_die(errno, "calloc"); + + if ((tids = calloc( + (size_t)(cfg->reverse_scanners + cfg->append_inserters), + sizeof(*tids))) == NULL) + testutil_die(errno, "calloc"); + + /* Create the files and load the initial records. */ + for (i = 0; i < cfg->append_inserters; ++i) { + run_info[i].cfg = cfg; + if (i == 0 || cfg->multiple_files) { + if ((run_info[i].name = malloc(64)) == NULL) + testutil_die(errno, "malloc"); + snprintf(run_info[i].name, 64, FNAME, (int)i); + + /* Vary by orders of magnitude */ + if (cfg->vary_nops) + run_info[i].nops = + WT_MAX(1000, cfg->max_nops >> i); + load(cfg, run_info[i].name); + } else + run_info[i].name = run_info[0].name; + + /* Setup op count if not varying ops. */ + if (run_info[i].nops == 0) + run_info[i].nops = cfg->max_nops; + total_nops += run_info[i].nops; + } + + /* Setup the reverse scanner configurations */ + for (i = 0; i < cfg->reverse_scanners; ++i) { + offset = i + cfg->append_inserters; + run_info[offset].cfg = cfg; + if (cfg->multiple_files) { + if ((run_info[offset].name = malloc(64)) == NULL) + testutil_die(errno, "malloc"); + /* Have reverse scans read from tables with writes. */ + name_index = i % cfg->append_inserters; + snprintf( + run_info[offset].name, 64, FNAME, (int)name_index); + + /* Vary by orders of magnitude */ + if (cfg->vary_nops) + run_info[offset].nops = + WT_MAX(1000, cfg->max_nops >> name_index); + } else + run_info[offset].name = run_info[0].name; + + /* Setup op count if not varying ops. */ + if (run_info[offset].nops == 0) + run_info[offset].nops = cfg->max_nops; + total_nops += run_info[offset].nops; + } + + (void)gettimeofday(&start, NULL); + + /* Create threads. */ + for (i = 0; i < cfg->reverse_scanners; ++i) + if ((ret = pthread_create( + &tids[i], NULL, reverse_scan, (void *)(uintptr_t)i)) != 0) + testutil_die(ret, "pthread_create"); + for (; i < cfg->reverse_scanners + cfg->append_inserters; ++i) { + if ((ret = pthread_create( + &tids[i], NULL, append_insert, (void *)(uintptr_t)i)) != 0) + testutil_die(ret, "pthread_create"); + } + + /* Wait for the threads. */ + for (i = 0; i < cfg->reverse_scanners + cfg->append_inserters; ++i) + (void)pthread_join(tids[i], &thread_ret); + + (void)gettimeofday(&stop, NULL); + seconds = (stop.tv_sec - start.tv_sec) + + (stop.tv_usec - start.tv_usec) * 1e-6; + fprintf(stderr, "timer: %.2lf seconds (%d ops/second)\n", + seconds, (int)(((cfg->reverse_scanners + cfg->append_inserters) * + total_nops) / seconds)); + + /* Verify the files. */ + for (i = 0; i < cfg->reverse_scanners + cfg->append_inserters; ++i) { + verify(cfg, run_info[i].name); + if (!cfg->multiple_files) + break; + } + + /* Output run statistics. */ + print_stats(cfg); + + /* Free allocated memory. */ + for (i = 0; i < cfg->reverse_scanners + cfg->append_inserters; ++i) { + free(run_info[i].name); + if (!cfg->multiple_files) + break; + } + + free(run_info); + free(tids); + + return (0); +} + +/* + * reverse_scan_op -- + * Walk a cursor back from the end of the file. + */ +static inline void +reverse_scan_op( + SHARED_CONFIG *cfg, WT_SESSION *session, WT_CURSOR *cursor, INFO *s) +{ + uint64_t i; + int ret; + char *strkey; + uint64_t initial_key_range; + uint64_t prev_key, this_key; + + WT_UNUSED(session); + WT_UNUSED(s); + + /* Reset the cursor */ + cursor->reset(cursor); + + /* Save the key range. */ + initial_key_range = cfg->key_range - cfg->append_inserters; + + for (i = 0; i < cfg->reverse_scan_ops; i++) { + if ((ret = cursor->prev(cursor)) != 0) { + if (ret == WT_NOTFOUND) + break; + testutil_die(ret, "cursor.prev"); + } + + if (cfg->ftype == ROW) { + cursor->get_key(cursor, &strkey); + this_key = (uint64_t)atol(strkey); + } else + cursor->get_key(cursor, (uint64_t*)&this_key); + + if (i == 0 && this_key < initial_key_range) + testutil_die(ret, + "cursor scan start range wrong first prev %" PRIu64 + " initial range: %" PRIu64, + this_key, initial_key_range); + if (i != 0 && this_key >= prev_key) + testutil_die(ret, + "cursor scan out of order this: %" PRIu64 + " prev: %" PRIu64, + this_key, prev_key); + prev_key = this_key; + } +} + +/* + * reverse_scan -- + * Reader thread start function. + */ +static void * +reverse_scan(void *arg) +{ + INFO *s; + SHARED_CONFIG *cfg; + WT_CURSOR *cursor; + WT_SESSION *session; + uint64_t i; + int id, ret; + char tid[128]; + + id = (int)(uintptr_t)arg; + s = &run_info[id]; + cfg = s->cfg; + __wt_thread_id(tid, sizeof(tid)); + __wt_random_init(&s->rnd); + + printf(" reverse scan thread %2d starting: tid: %s, file: %s\n", + id, tid, s->name); + + __wt_yield(); /* Get all the threads created. */ + + if ((ret = cfg->conn->open_session( + cfg->conn, NULL, "isolation=snapshot", &session)) != 0) + testutil_die(ret, "conn.open_session"); + if ((ret = session->open_cursor( + session, s->name, NULL, NULL, &cursor)) != 0) + testutil_die(ret, "session.open_cursor"); + for (i = 0; i < s->nops && !cfg->thread_finish; + ++i, ++s->reverse_scans, __wt_yield()) + reverse_scan_op(cfg, session, cursor, s); + if ((ret = session->close(session, NULL)) != 0) + testutil_die(ret, "session.close"); + + printf(" reverse scan thread %2d stopping: tid: %s, file: %s\n", + id, tid, s->name); + + /* Notify all other threads to finish once the first thread is done */ + cfg->thread_finish = true; + + return (NULL); +} + +/* + * append_insert_op -- + * Write operation. + */ +static inline void +append_insert_op( + SHARED_CONFIG *cfg, WT_SESSION *session, WT_CURSOR *cursor, INFO *s) +{ + WT_ITEM *value, _value; + int64_t keyno; + int ret; + char keybuf[64], valuebuf[64]; + + WT_UNUSED(session); + + value = &_value; + + keyno = __wt_atomic_add64(&cfg->key_range, 1); + if (cfg->ftype == ROW) { + snprintf(keybuf, sizeof(keybuf), "%016u", (u_int)keyno); + cursor->set_key(cursor, &keybuf); + } else + cursor->set_key(cursor, (uint32_t)keyno); + + ++s->append_insert; + value->data = valuebuf; + if (cfg->ftype == FIX) + cursor->set_value(cursor, 0x10); + else { + value->size = (uint32_t)snprintf( + valuebuf, sizeof(valuebuf), "XXX %37u", (u_int)keyno); + cursor->set_value(cursor, value); + } + if ((ret = cursor->insert(cursor)) != 0) + testutil_die(ret, "cursor.insert"); +} + +/* + * append_insert -- + * Writer thread start function. + */ +static void * +append_insert(void *arg) +{ + INFO *s; + SHARED_CONFIG *cfg; + WT_CURSOR *cursor; + WT_SESSION *session; + uint64_t i; + int id, ret; + char tid[128]; + + id = (int)(uintptr_t)arg; + s = &run_info[id]; + cfg = s->cfg; + __wt_thread_id(tid, sizeof(tid)); + __wt_random_init(&s->rnd); + + printf("write thread %2d starting: tid: %s, file: %s\n", + id, tid, s->name); + + __wt_yield(); /* Get all the threads created. */ + + if ((ret = cfg->conn->open_session( + cfg->conn, NULL, "isolation=snapshot", &session)) != 0) + testutil_die(ret, "conn.open_session"); + if ((ret = session->open_cursor( + session, s->name, NULL, NULL, &cursor)) != 0) + testutil_die(ret, "session.open_cursor"); + for (i = 0; i < s->nops && !cfg->thread_finish; ++i, __wt_yield()) + append_insert_op(cfg, session, cursor, s); + if ((ret = session->close(session, NULL)) != 0) + testutil_die(ret, "session.close"); + + printf("write thread %2d stopping: tid: %s, file: %s\n", + id, tid, s->name); + + /* Notify all other threads to finish once the first thread is done */ + cfg->thread_finish = true; + + return (NULL); +} + +/* + * print_stats -- + * Display reverse scan/writer thread stats. + */ +static void +print_stats(SHARED_CONFIG *cfg) +{ + INFO *s; + uint64_t id, total_threads; + + total_threads = cfg->reverse_scanners + cfg->append_inserters; + s = run_info; + for (id = 0; id < total_threads; ++id, ++s) + printf("%3d: reverse scans %6d, append inserts %6d\n", + (int)id, (int)s->reverse_scans, (int)s->append_insert); +} -- cgit v1.2.1 From 7b2f2f75ab3e3fd20a261b26d8c09d53fc9232ea Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 16 Feb 2016 08:27:26 -0500 Subject: WT-2397: Cursor traversal from end of the tree skips records. Minor function renaming, inline the static walk functions. --- src/btree/bt_walk.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/btree/bt_walk.c b/src/btree/bt_walk.c index 4956bcbb20e..55b11d7b2d1 100644 --- a/src/btree/bt_walk.c +++ b/src/btree/bt_walk.c @@ -89,11 +89,11 @@ __ref_is_leaf(WT_REF *ref) } /* - * __page_ascend -- + * __ref_ascend -- * Ascend the tree one level. */ -static void -__page_ascend(WT_SESSION_IMPL *session, +static inline void +__ref_ascend(WT_SESSION_IMPL *session, WT_REF **refp, WT_PAGE_INDEX **pindexp, uint32_t *slotp) { WT_REF *parent_ref, *ref; @@ -163,11 +163,11 @@ __page_ascend(WT_SESSION_IMPL *session, } /* - * __page_descend_prev -- + * __ref_descend_prev -- * Descend the tree one level, during a previous-cursor walk. */ -static void -__page_descend_prev( +static inline void +__ref_descend_prev( WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE_INDEX **pindexp) { WT_PAGE_INDEX *pindex; @@ -239,12 +239,12 @@ __page_descend_prev( } /* - * __page_initial_descent_prev -- + * __ref_initial_descent_prev -- * Descend the tree one level, when setting up the initial cursor position * for a previous-cursor walk. */ -static bool -__page_initial_descent_prev( +static inline bool +__ref_initial_descent_prev( WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE_INDEX **pindexp) { WT_PAGE_INDEX *pindex; @@ -378,7 +378,7 @@ restart: /* while ((prev && slot == 0) || (!prev && slot == pindex->entries - 1)) { /* Ascend to the parent. */ - __page_ascend(session, &ref, &pindex, &slot); + __ref_ascend(session, &ref, &pindex, &slot); /* * If we got all the way through an internal page and @@ -623,12 +623,12 @@ descend: couple = ref; session, ref->page, pindex); slot = 0; } else if (initial_descent) { - if (!__page_initial_descent_prev( + if (!__ref_initial_descent_prev( session, ref, &pindex)) goto restart; slot = pindex->entries - 1; } else { - __page_descend_prev( + __ref_descend_prev( session, ref, &pindex); slot = pindex->entries - 1; } -- cgit v1.2.1 From 48cbd17938a8016244a863442e7216ed09eca740 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 16 Feb 2016 12:08:59 -0500 Subject: WT-2349 Changes from review comments. Add WT-specific error codes. --- dist/api_err.py | 2 ++ dist/s_string.ok | 1 + src/conn/api_strerror.c | 2 ++ src/conn/conn_api.c | 81 +++++++++++++++++++++++++++-------------------- src/docs/spell.ok | 1 + src/include/extern.h | 3 +- src/include/wiredtiger.in | 4 +++ src/lsm/lsm_manager.c | 1 + src/os_posix/os_errno.c | 16 ++++++++++ src/os_win/os_errno.c | 22 +++++++++++-- src/session/session_api.c | 11 +++---- 11 files changed, 100 insertions(+), 44 deletions(-) diff --git a/dist/api_err.py b/dist/api_err.py index 09332d508a2..a17c68ee196 100644 --- a/dist/api_err.py +++ b/dist/api_err.py @@ -56,6 +56,8 @@ errors = [ This error is generated when wiredtiger_open is configured to run in-memory, and an insert or update operation requires more than the configured cache size to complete.''', undoc=True), + Error('WT_PERM_DENIED', -31808, + 'permission denied (internal)', undoc=True), ] # Update the #defines in the wiredtiger.in file. diff --git a/dist/s_string.ok b/dist/s_string.ok index 83d8f417c17..179815f5277 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -873,6 +873,7 @@ rS rb rbrace rbracket +rdonly rduppo readlock readonly diff --git a/src/conn/api_strerror.c b/src/conn/api_strerror.c index edb11957556..87864f7f4b0 100644 --- a/src/conn/api_strerror.c +++ b/src/conn/api_strerror.c @@ -40,6 +40,8 @@ __wt_wiredtiger_error(int error) return ("WT_RUN_RECOVERY: recovery must be run to continue"); case WT_CACHE_FULL: return ("WT_CACHE_FULL: operation would overflow cache"); + case WT_PERM_DENIED: + return ("WT_PERM_DENIED: permission denied (internal)"); } /* diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 79cba904008..40eb7e2b1a9 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -1404,7 +1404,7 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) WT_FH *fh; size_t len; wt_off_t size; - bool exist, is_create; + bool bytelock, exist, is_create; char buf[256]; conn = S2C(session); @@ -1416,6 +1416,7 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) if (F_ISSET(conn, WT_CONN_READONLY)) is_create = false; + bytelock = true; __wt_spin_lock(session, &__wt_process.spinlock); /* @@ -1483,55 +1484,65 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) * XXX Ignoring the error does allow multiple read-only * connections to exist at the same time on a read-only directory. */ - if (F_ISSET(conn, WT_CONN_READONLY) && - (ret == EACCES || ret == ENOENT)) { + if (F_ISSET(conn, WT_CONN_READONLY)) { /* * If we got an expected permission or non-existence error * then skip the byte lock. */ - ret = 0; - goto open_wt; + ret = __wt_map_error_rdonly(ret); + if (ret == WT_NOTFOUND || ret == WT_PERM_DENIED) { + bytelock = false; + ret = 0; + } } WT_ERR(ret); + if (bytelock) { + /* + * Lock a byte of the file: if we don't get the lock, some other + * process is holding it, we're done. The file may be + * zero-length, and that's OK, the underlying call supports + * locking past the end-of-file. + */ + if (__wt_bytelock(conn->lock_fh, (wt_off_t)0, true) != 0) + WT_ERR_MSG(session, EBUSY, + "WiredTiger database is already being managed by " + "another process"); - /* - * Lock a byte of the file: if we don't get the lock, some other process - * is holding it, we're done. The file may be zero-length, and that's - * OK, the underlying call supports locking past the end-of-file. - */ - if (__wt_bytelock(conn->lock_fh, (wt_off_t)0, true) != 0) - WT_ERR_MSG(session, EBUSY, - "WiredTiger database is already being managed by another " - "process"); - - /* - * If the size of the lock file is non-zero, we created it (or won a - * locking race with the thread that created it, it doesn't matter). - * - * Write something into the file, zero-length files make me nervous. - * - * The test against the expected length is sheer paranoia (the length - * should be 0 or correct), but it shouldn't hurt. - */ + /* + * If the size of the lock file is non-zero, we created it (or + * won a locking race with the thread that created it, it + * doesn't matter). + * + * Write something into the file, zero-length files make me + * nervous. + * + * The test against the expected length is sheer paranoia (the + * length should be 0 or correct), but it shouldn't hurt. + */ #define WT_SINGLETHREAD_STRING "WiredTiger lock file\n" - WT_ERR(__wt_filesize(session, conn->lock_fh, &size)); - if (size != strlen(WT_SINGLETHREAD_STRING)) - WT_ERR(__wt_write(session, conn->lock_fh, (wt_off_t)0, - strlen(WT_SINGLETHREAD_STRING), WT_SINGLETHREAD_STRING)); + WT_ERR(__wt_filesize(session, conn->lock_fh, &size)); + if (size != strlen(WT_SINGLETHREAD_STRING)) + WT_ERR(__wt_write(session, conn->lock_fh, (wt_off_t)0, + strlen(WT_SINGLETHREAD_STRING), + WT_SINGLETHREAD_STRING)); + + } -open_wt: /* We own the lock file, optionally create the WiredTiger file. */ ret = __wt_open(session, WT_WIREDTIGER, is_create, false, 0, &fh); + /* * If we're read-only, check for success as well as handled errors. * Even if we're able to open the WiredTiger file successfully, we - * do not try to lock it. The lock file test for read-only is the - * only one we do. + * do not try to lock it. The lock file test above is the only + * one we do for read-only. */ - if (F_ISSET(conn, WT_CONN_READONLY) && - (ret == 0 || ret == EACCES || ret == ENOENT)) - ret = 0; - else { + if (F_ISSET(conn, WT_CONN_READONLY)) { + ret = __wt_map_error_rdonly(ret); + if (ret == 0 || ret == WT_NOTFOUND || ret == WT_PERM_DENIED) + ret = 0; + WT_ERR(ret); + } else { WT_ERR(ret); /* diff --git a/src/docs/spell.ok b/src/docs/spell.ok index 80597302cbb..39a8c50545d 100644 --- a/src/docs/spell.ok +++ b/src/docs/spell.ok @@ -377,6 +377,7 @@ rVv rdbms rdlock readlock +readonly realclean realloc realloc'd diff --git a/src/include/extern.h b/src/include/extern.h index ed9d8d2fbb2..f4ebda6f42d 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -490,6 +490,7 @@ extern int __wt_dlopen(WT_SESSION_IMPL *session, const char *path, WT_DLH **dlhp extern int __wt_dlsym(WT_SESSION_IMPL *session, WT_DLH *dlh, const char *name, bool fail, void *sym_ret); extern int __wt_dlclose(WT_SESSION_IMPL *session, WT_DLH *dlh); extern int __wt_errno(void); +extern int __wt_map_error_rdonly(int error); extern const char *__wt_strerror(WT_SESSION_IMPL *session, int error, char *errbuf, size_t errlen); extern int __wt_exist(WT_SESSION_IMPL *session, const char *filename, bool *existp); extern void __wt_fallocate_config(WT_SESSION_IMPL *session, WT_FH *fh); @@ -612,7 +613,7 @@ extern WT_DATA_SOURCE *__wt_schema_get_source(WT_SESSION_IMPL *session, const ch extern int __wt_str_name_check(WT_SESSION_IMPL *session, const char *str); extern int __wt_name_check(WT_SESSION_IMPL *session, const char *str, size_t len); extern int __wt_schema_worker(WT_SESSION_IMPL *session, const char *uri, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[], uint32_t open_flags); -extern int __wt_session_notsup_cfg( WT_SESSION *wt_session, const char *config); +extern int __wt_session_notsup_config(WT_SESSION *wt_session, const char *config); extern int __wt_session_notsup_uri( WT_SESSION *wt_session, const char *uri, const char *config); extern int __wt_session_reset_cursors(WT_SESSION_IMPL *session, bool free_buffers); extern int __wt_session_copy_values(WT_SESSION_IMPL *session); diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index 7d529e728ac..93eb2d4be8f 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -3004,6 +3004,10 @@ const char *wiredtiger_version(int *majorp, int *minorp, int *patchp); */ #define WT_CACHE_FULL -31807 /*! @endcond */ +/*! @cond internal */ +/*! Permission denied (internal). */ +#define WT_PERM_DENIED -31808 +/*! @endcond */ /* * Error return section: END * DO NOT EDIT: automatically built by dist/api_err.py. diff --git a/src/lsm/lsm_manager.c b/src/lsm/lsm_manager.c index 6fd96aa2af3..cf581475d2c 100644 --- a/src/lsm/lsm_manager.c +++ b/src/lsm/lsm_manager.c @@ -622,6 +622,7 @@ __wt_lsm_manager_push_entry(WT_SESSION_IMPL *session, manager = &S2C(session)->lsm_manager; + WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_READONLY)); /* * Don't add merges or bloom filter creates if merges * or bloom filters are disabled in the tree. diff --git a/src/os_posix/os_errno.c b/src/os_posix/os_errno.c index a58ae88447e..a0f1202c6ef 100644 --- a/src/os_posix/os_errno.c +++ b/src/os_posix/os_errno.c @@ -22,6 +22,22 @@ __wt_errno(void) return (errno == 0 ? WT_ERROR : errno); } +/* + * __wt_map_error_rdonly -- + * Map an error into a WiredTiger error code specific for + * read-only operation which intercepts based on certain types + * of failures. + */ +int +__wt_map_error_rdonly(int error) +{ + if (error == ENOENT) + return (WT_NOTFOUND); + else if (error == EACCES) + return (WT_PERM_DENIED); + return (error); +} + /* * __wt_strerror -- * POSIX implementation of WT_SESSION.strerror and wiredtiger_strerror. diff --git a/src/os_win/os_errno.c b/src/os_win/os_errno.c index 6a9daf8443f..e87d4c59c1b 100644 --- a/src/os_win/os_errno.c +++ b/src/os_win/os_errno.c @@ -17,7 +17,8 @@ static const int windows_error_offset = -29000; * Windows errors are from 0 - 15999 according to the documentation */ static DWORD -__wt_map_error_to_windows_error(int error) { +__wt_map_error_to_windows_error(int error) +{ /* Ensure we do not exceed the error range Also validate he do not get any COM errors (which are negative integers) @@ -32,10 +33,27 @@ __wt_map_error_to_windows_error(int error) { * Return a positive integer, a decoded Windows error */ static int -__wt_map_windows_error_to_error(DWORD winerr) { +__wt_map_windows_error_to_error(DWORD winerr) +{ return (winerr + windows_error_offset); } +/* + * __wt_map_error_rdonly -- + * Map an error into a WiredTiger error code specific for + * read-only operation which intercepts based on certain types + * of failures. + */ +int +__wt_map_error_rdonly(int winerr) +{ + if (winerr == ERROR_FILE_NOT_FOUND) + return (WT_NOTFOUND); + else if (winerr == ERROR_ACCESS_DENIED) + return (WT_PERM_DENIED); + return (winerr); +} + /* * __wt_errno -- * Return errno, or WT_ERROR if errno not set. diff --git a/src/session/session_api.c b/src/session/session_api.c index e2723c660fe..abaa369b383 100644 --- a/src/session/session_api.c +++ b/src/session/session_api.c @@ -13,12 +13,11 @@ static int __session_snapshot(WT_SESSION *, const char *); static int __session_rollback_transaction(WT_SESSION *, const char *); /* - * __wt_session_notsup_cfg -- + * __wt_session_notsup_config -- * Unsupported session actions that have a signature of a config string. */ int -__wt_session_notsup_cfg( - WT_SESSION *wt_session, const char *config) +__wt_session_notsup_config(WT_SESSION *wt_session, const char *config) { WT_SESSION_IMPL *session; @@ -1471,9 +1470,9 @@ __open_session(WT_CONNECTION_IMPL *conn, */ if (F_ISSET(conn, WT_CONN_READONLY)) { wt_session = &session_ret->iface; - wt_session->checkpoint = __wt_session_notsup_cfg; - wt_session->log_flush = __wt_session_notsup_cfg; - wt_session->transaction_sync = __wt_session_notsup_cfg; + wt_session->checkpoint = __wt_session_notsup_config; + wt_session->log_flush = __wt_session_notsup_config; + wt_session->transaction_sync = __wt_session_notsup_config; wt_session->compact = __wt_session_notsup_uri; wt_session->create = __wt_session_notsup_uri; -- cgit v1.2.1 From 9ba69ab7180c49c6a0f01b002a94e3a3798ec00a Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 16 Feb 2016 15:18:43 -0500 Subject: WT-2349 Read-only documentation --- dist/api_data.py | 3 ++- src/docs/programming.dox | 1 + src/docs/readonly.dox | 62 +++++++++++++++++++++++++++++++++++++++++++++++ src/docs/wtperf.dox | 4 +++ src/include/wiredtiger.in | 9 ++++--- 5 files changed, 74 insertions(+), 5 deletions(-) create mode 100644 src/docs/readonly.dox diff --git a/dist/api_data.py b/dist/api_data.py index 5a07440912f..1623c315710 100644 --- a/dist/api_data.py +++ b/dist/api_data.py @@ -480,7 +480,8 @@ connection_runtime_config = [ ]), Config('readonly', 'false', r''' open connection in read-only mode. The database must exist. All - methods that may modify a database are disabled.''', + methods that may modify a database are disabled. See @ref readonly + for more information''', type='boolean'), Config('shared_cache', '', r''' shared cache configuration options. A database should configure diff --git a/src/docs/programming.dox b/src/docs/programming.dox index 5d79edd660b..f717f4ed1fe 100644 --- a/src/docs/programming.dox +++ b/src/docs/programming.dox @@ -30,6 +30,7 @@ each of which is ordered by one or more columns.

Programming notes

- @subpage threads - @subpage namespace +- @subpage readonly @m_if{c} - @subpage signals @m_endif diff --git a/src/docs/readonly.dox b/src/docs/readonly.dox new file mode 100644 index 00000000000..1da103085bf --- /dev/null +++ b/src/docs/readonly.dox @@ -0,0 +1,62 @@ +/*! @m_page{{c,java},readonly,Database read-only mode} + +WiredTiger supports opening a database in read-only mode. When a database +is opened in read-only mode, all modifications are disabled on the WT_CONNECTION +handle, any sessions opened in that connection and any cursors opened on any +of those sessions. Any session handle method that modifies any part of the +database will return an error. Similarly, any WiredTiger internal threads +that perform modifications will not perform that work. + +When a database is opened in read-only mode, WiredTiger assumes the database +directory and content already exist and that it was a clean shutdown, +particularly if logging and recovery are used. + +@section readonly_config Database readonly configuration considerations + +Use of the \c readonly configuration may affect other configuration settings. +Where the default setting contradicts read-only operation, WiredTiger +overrides those defaults to perform in a non-modifying mode. For example, +merges are turned off if LSM trees are in use, and log file archiving is +disabled if logging is configured. + +Other settings where the user must explicitly turn them on return an error +if that setting contradicts read-only mode. For example, statistics logging +or zero filling log files are not allowed during read-only mode and trying +to set those will result in an error. + +@section readonly_logging Readonly configuration and logging + +If logging is enabled on the database when it is opened with the \c readonly +configuration, log file archiving and log file pre-allocation are disabled. +The log files will not be modified any way. Any pre-allocated or temporary +log files that exist from an earlier run that would normally be removed will +remain. + +@section readonly_recovery Readonly configuration and recovery + +Since recovery only reapplies modifications, clearly no recovery actions can +occur during read-only mode. The system will detect if recovery is necessary +or if it was a clean shutdown. Attempting to open the database via +::wiredtiger_open will only proceed from a clean shutdown and return an error +if any recovery is required. + +@section readonly_lsm Readonly configuration and LSM trees + +If LSM trees are in use, starting up in read-only mode turns off all +modification. Internal LSM operations such as merging, creating new chunks, +creating bloom filters and dropping old chunks are disabled. + +@section readonly_handles Readonly configuration and multiple database handles + +One unusual affect of read-only operations is the potential for multiple +read-only database handles open to the same database at the same time, in +limited circumstances. WiredTiger prevents multiple connection handles +by writing to a lock file. Even in read-only mode, creating and writing +to the lock file is attempted. In the case where the database directory or +the lock file does not exist or cannot be created or written to, a read-only +handle is allowed to proceed. So in the case where the database directory +and its files are read-only, multiple read-only connection handles could +be open at the same time. In the case where the database directory is +writable the normal file locking occurs preventing multiple connections. + +*/ diff --git a/src/docs/wtperf.dox b/src/docs/wtperf.dox index 1f0d1533ac4..6f3d2f87ee0 100644 --- a/src/docs/wtperf.dox +++ b/src/docs/wtperf.dox @@ -212,6 +212,10 @@ insert operations generate random content for the value @par read_range (unsigned int, default=0) scan a range of keys after each search +@par readonly (boolean, default=false) +reopen the connection between populate and workload phases in readonly +mode. Requires reopen_connection turned on (default). Requires that +read be the only workload specified @par reopen_connection (boolean, default=true) close and reopen the connection between populate and workload phases @par report_interval (unsigned int, default=2) diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index 93eb2d4be8f..6c09983fead 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -1854,8 +1854,9 @@ struct __wt_connection { * integer between 3 and 20; default \c 4.} * @config{ ),,} * @config{readonly, open connection in read-only mode. The database - * must exist. All methods that may modify a database are disabled., a - * boolean flag; default \c false.} + * must exist. All methods that may modify a database are disabled. + * See @ref readonly for more information., a boolean flag; default \c + * false.} * @config{shared_cache = (, shared cache configuration options. A * database should configure either a cache_size or a shared_cache not * both. Enabling a shared cache uses a session from the configured @@ -2330,8 +2331,8 @@ struct __wt_connection { * processes). Not yet supported in WiredTiger., a boolean flag; default * \c false.} * @config{readonly, open connection in read-only mode. The database must - * exist. All methods that may modify a database are disabled., a boolean flag; - * default \c false.} + * exist. All methods that may modify a database are disabled. See @ref + * readonly for more information., a boolean flag; default \c false.} * @config{session_max, maximum expected number of sessions (including server * threads)., an integer greater than or equal to 1; default \c 100.} * @config{shared_cache = (, shared cache configuration options. A database -- cgit v1.2.1 From 5e06255e8281468ab6d258d87046c3c444dc8aff Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 16 Feb 2016 15:22:19 -0500 Subject: WT-2349 Remove not-yet-ready wtperf config from wtperf doc. --- src/docs/wtperf.dox | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/docs/wtperf.dox b/src/docs/wtperf.dox index 6f3d2f87ee0..1f0d1533ac4 100644 --- a/src/docs/wtperf.dox +++ b/src/docs/wtperf.dox @@ -212,10 +212,6 @@ insert operations generate random content for the value @par read_range (unsigned int, default=0) scan a range of keys after each search -@par readonly (boolean, default=false) -reopen the connection between populate and workload phases in readonly -mode. Requires reopen_connection turned on (default). Requires that -read be the only workload specified @par reopen_connection (boolean, default=true) close and reopen the connection between populate and workload phases @par report_interval (unsigned int, default=2) -- cgit v1.2.1 From 4d98d8d4de199ebf462b69d581fb3908922a79ea Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 16 Feb 2016 16:36:40 -0500 Subject: WT-2349 Expand test/readonly for more combinations. --- test/readonly/readonly.c | 161 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 136 insertions(+), 25 deletions(-) diff --git a/test/readonly/readonly.c b/test/readonly/readonly.c index f4ad27991e5..6895f53c2d0 100644 --- a/test/readonly/readonly.c +++ b/test/readonly/readonly.c @@ -41,20 +41,28 @@ #include "test_util.i" #define HOME_SIZE 512 -static char home[HOME_SIZE]; /* Program working dir */ +static char home[HOME_SIZE]; /* Program working dir lock file */ +static char home_wr[HOME_SIZE]; /* Writable dir copy no lock file */ static char home_rd[HOME_SIZE]; /* Read-only dir */ -static char home_rd2[HOME_SIZE]; /* Read-only dir */ +static char home_rd2[HOME_SIZE]; /* Read-only dir no lock file */ static const char *progname; /* Program name */ -static const char *saved_argv0; /* Program name */ +static const char *saved_argv0; /* Program command */ static const char *uri = "table:main"; #define ENV_CONFIG \ "create,log=(file_max=10M,archive=false,enabled)," \ "transaction_sync=(enabled,method=none)" #define ENV_CONFIG_RD "readonly=true" +#define ENV_CONFIG_WR "readonly=false" #define MAX_VAL 4096 #define MAX_KV 10000 +#define EXPECT_ERR 1 +#define EXPECT_SUCCESS 0 + +#define OP_READ 0 +#define OP_WRITE 1 + static void usage(void) { @@ -63,19 +71,33 @@ usage(void) } static int -run_child(const char *homedir) +run_child(const char *homedir, int op, int expect) { WT_CONNECTION *conn; WT_CURSOR *cursor; WT_SESSION *session; int i, ret; + const char *cfg; /* * We expect the read-only database will allow the second read-only * handle to succeed because no one can create or set the lock file. */ - if ((ret = wiredtiger_open(homedir, NULL, ENV_CONFIG_RD, &conn)) != 0) - testutil_die(ret, "wiredtiger_open readonly"); + if (op == OP_READ) + cfg = ENV_CONFIG_RD; + else + cfg = ENV_CONFIG_WR; + ret = wiredtiger_open(homedir, NULL, cfg, &conn); + if (expect == EXPECT_SUCCESS && ret != 0) + testutil_die(ret, "wiredtiger_open success err"); + if (expect == EXPECT_ERR) { + if (ret == 0) + testutil_die(ret, "wiredtiger_open expected err succeeded"); + /* + * If we expect an error and got one, we're done. + */ + return (0); + } /* * Make sure we can read the data. @@ -101,22 +123,33 @@ run_child(const char *homedir) * Child process opens both databases readonly. */ static void -open_dbs(const char *dir, const char *dir_rd, const char *dir_rd2) +open_dbs(int op, const char *dir, + const char *dir_wr, const char *dir_rd, const char *dir_rd2) { - WT_CONNECTION *conn; - int ret; + int expect, ret; /* * The parent has an open connection to all directories. - * We expect opening the writeable home to return an error. + * We expect opening the writeable homes to return an error. * It is a failure if the child successfully opens that. */ - if ((ret = wiredtiger_open(dir, NULL, ENV_CONFIG_RD, &conn)) == 0) + expect = EXPECT_ERR; + if ((ret = run_child(dir, op, expect)) != 0) + testutil_die(ret, "wiredtiger_open readonly allowed"); + if ((ret = run_child(dir_wr, op, expect)) != 0) testutil_die(ret, "wiredtiger_open readonly allowed"); - if ((ret = run_child(dir_rd)) != 0) + /* + * The parent must have a read-only connection open to the + * read-only databases. If the child is opening read-only + * too, we expect success. Otherwise an error if the child + * attempts to open read/write (permission error). + */ + if (op == OP_READ) + expect = EXPECT_SUCCESS; + if ((ret = run_child(dir_rd, op, expect)) != 0) testutil_die(ret, "run child 1"); - if ((ret = run_child(dir_rd2)) != 0) + if ((ret = run_child(dir_rd2, op, expect)) != 0) testutil_die(ret, "run child 2"); exit(EXIT_SUCCESS); } @@ -127,12 +160,12 @@ extern char *__wt_optarg; int main(int argc, char *argv[]) { - WT_CONNECTION *conn, *conn2, *conn3; + WT_CONNECTION *conn, *conn2, *conn3, *conn4; WT_CURSOR *cursor; WT_ITEM data; WT_SESSION *session; uint64_t i; - int ch, status, ret; + int ch, status, op, ret; bool child; const char *working_dir; char cmd[512]; @@ -149,10 +182,15 @@ main(int argc, char *argv[]) working_dir = "WT_RD"; child = false; - while ((ch = __wt_getopt(progname, argc, argv, "Ch:")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "Rh:W")) != EOF) switch (ch) { - case 'C': + case 'R': child = true; + op = OP_READ; + break; + case 'W': + child = true; + op = OP_WRITE; break; case 'h': working_dir = __wt_optarg; @@ -170,12 +208,15 @@ main(int argc, char *argv[]) * Set up all the directory names. */ testutil_work_dir_from_path(home, 512, working_dir); + strncpy(home_wr, home, HOME_SIZE); + strcat(home_wr, ".WRNOLOCK"); strncpy(home_rd, home, HOME_SIZE); strcat(home_rd, ".RD"); strncpy(home_rd2, home, HOME_SIZE); - strcat(home_rd2, ".NOLOCK"); + strcat(home_rd2, ".RDNOLOCK"); if (!child) { testutil_make_work_dir(home); + testutil_make_work_dir(home_wr); testutil_make_work_dir(home_rd); testutil_make_work_dir(home_rd2); } else { @@ -184,7 +225,7 @@ main(int argc, char *argv[]) * the open_dbs with the directories we have. * The child function will exit. */ - open_dbs(home, home_rd, home_rd2); + open_dbs(op, home, home_wr, home_rd, home_rd2); } /* @@ -231,26 +272,52 @@ main(int argc, char *argv[]) * Copy the database. Remove any lock file from one copy * and chmod the copies to be read-only permissions. */ + (void)snprintf(cmd, sizeof(cmd), + "cp -rp %s/* %s; rm -f %s/WiredTiger.lock", + home, home_wr, home_wr); + (void)system(cmd); + (void)snprintf(cmd, sizeof(cmd), "cp -rp %s/* %s; chmod 0555 %s; chmod -R 0444 %s/*", home, home_rd, home_rd, home_rd); (void)system(cmd); + (void)snprintf(cmd, sizeof(cmd), "cp -rp %s/* %s; rm -f %s/WiredTiger.lock; " "chmod 0555 %s; chmod -R 0444 %s/*", home, home_rd2, home_rd2, home_rd2, home_rd2); (void)system(cmd); + /* + * Run four scenarios. Sometimes expect errors, sometimes success. + * The writable database directories should always fail to allow the + * child to open due to the lock file. The read-only ones will only + * succeed when the child attempts read-only. + * + * 1. Parent has read-only handle to all databases. Child opens + * read-only also. + * 2. Parent has read-only handle to all databases. Child opens + * read-write. + * 3. Parent has read-write handle to writable databases and + * read-only to read-only databases. Child opens read-only. + * 4. Parent has read-write handle to writable databases and + * read-only to read-only databases. Child opens read-write. + */ /* * Open a connection handle to all databases. */ fprintf(stderr, " *** Expect several error messages from WT ***\n"); + /* + * Scenario 1. + */ if ((ret = wiredtiger_open(home, NULL, ENV_CONFIG_RD, &conn)) != 0) + testutil_die(ret, "wiredtiger_open original home"); + if ((ret = wiredtiger_open(home_wr, NULL, ENV_CONFIG_RD, &conn2)) != 0) + testutil_die(ret, "wiredtiger_open write nolock"); + if ((ret = wiredtiger_open(home_rd, NULL, ENV_CONFIG_RD, &conn3)) != 0) testutil_die(ret, "wiredtiger_open readonly"); - if ((ret = wiredtiger_open(home_rd, NULL, ENV_CONFIG_RD, &conn2)) != 0) - testutil_die(ret, "wiredtiger_open readonly2"); - if ((ret = wiredtiger_open(home_rd2, NULL, ENV_CONFIG_RD, &conn3)) != 0) - testutil_die(ret, "wiredtiger_open readonly3"); + if ((ret = wiredtiger_open(home_rd2, NULL, ENV_CONFIG_RD, &conn4)) != 0) + testutil_die(ret, "wiredtiger_open readonly nolock"); /* * Create a child to also open a connection handle to the databases. @@ -259,22 +326,66 @@ main(int argc, char *argv[]) * the child even though it should not be. So use 'system' to spawn * an entirely new process. */ - (void)snprintf(cmd, sizeof(cmd), "%s -C", saved_argv0); + (void)snprintf(cmd, sizeof(cmd), "%s -R", saved_argv0); if ((status = system(cmd)) < 0) testutil_die(status, "system"); - /* * The child will exit with success if its test passes. */ if (WEXITSTATUS(status) != 0) testutil_die(WEXITSTATUS(status), "system"); + /* + * Scenario 2. Run child with writable config. + */ + (void)snprintf(cmd, sizeof(cmd), "%s -W", saved_argv0); + if ((status = system(cmd)) < 0) + testutil_die(status, "system"); + + if (WEXITSTATUS(status) != 0) + testutil_die(WEXITSTATUS(status), "system"); + + /* + * Reopen the two writable directories and rerun the child. + */ + if ((ret = conn->close(conn, NULL)) != 0) + testutil_die(ret, "WT_CONNECTION:close"); + if ((ret = conn2->close(conn2, NULL)) != 0) + testutil_die(ret, "WT_CONNECTION:close"); + if ((ret = wiredtiger_open(home, NULL, ENV_CONFIG_RD, &conn)) != 0) + testutil_die(ret, "wiredtiger_open original home"); + if ((ret = wiredtiger_open(home_wr, NULL, ENV_CONFIG_RD, &conn2)) != 0) + testutil_die(ret, "wiredtiger_open write nolock"); + /* + * Scenario 3. Child read-only. + */ + (void)snprintf(cmd, sizeof(cmd), "%s -R", saved_argv0); + if ((status = system(cmd)) < 0) + testutil_die(status, "system"); + if (WEXITSTATUS(status) != 0) + testutil_die(WEXITSTATUS(status), "system"); + + /* + * Scenario 4. Run child with writable config. + */ + (void)snprintf(cmd, sizeof(cmd), "%s -W", saved_argv0); + if ((status = system(cmd)) < 0) + testutil_die(status, "system"); + if (WEXITSTATUS(status) != 0) + testutil_die(WEXITSTATUS(status), "system"); + + + /* + * Clean-up. + */ if ((ret = conn->close(conn, NULL)) != 0) testutil_die(ret, "WT_CONNECTION:close"); if ((ret = conn2->close(conn2, NULL)) != 0) testutil_die(ret, "WT_CONNECTION:close"); if ((ret = conn3->close(conn3, NULL)) != 0) testutil_die(ret, "WT_CONNECTION:close"); + if ((ret = conn4->close(conn4, NULL)) != 0) + testutil_die(ret, "WT_CONNECTION:close"); /* * We need to chmod the read-only databases back so that they can * be removed by scripts. -- cgit v1.2.1 From fd888b7ab26a622740c1e640d74360d7cff52c18 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 16 Feb 2016 16:39:23 -0500 Subject: WT-2349 Whitespace. --- test/readonly/readonly.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/readonly/readonly.c b/test/readonly/readonly.c index 6895f53c2d0..66885e135dc 100644 --- a/test/readonly/readonly.c +++ b/test/readonly/readonly.c @@ -298,9 +298,9 @@ main(int argc, char *argv[]) * read-only also. * 2. Parent has read-only handle to all databases. Child opens * read-write. - * 3. Parent has read-write handle to writable databases and + * 3. Parent has read-write handle to writable databases and * read-only to read-only databases. Child opens read-only. - * 4. Parent has read-write handle to writable databases and + * 4. Parent has read-write handle to writable databases and * read-only to read-only databases. Child opens read-write. */ /* @@ -374,7 +374,6 @@ main(int argc, char *argv[]) if (WEXITSTATUS(status) != 0) testutil_die(WEXITSTATUS(status), "system"); - /* * Clean-up. */ -- cgit v1.2.1 From 288b62c853de6856d7b7d071b8e1bbb1af4cf22e Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 16 Feb 2016 16:43:46 -0500 Subject: WT-2349 OK strings. --- dist/s_string.ok | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dist/s_string.ok b/dist/s_string.ok index 47d5e4c643a..9c5eb411ea3 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -245,6 +245,7 @@ Preload Prepend Qsort RCS +RDNOLOCK RECNO REF's REFs @@ -331,6 +332,7 @@ VxWorks WAL WIREDTIGER WRLSN +WRNOLOCK WakeAllConditionVariable Wconditional WeakHashLen @@ -799,6 +801,7 @@ nhex nlpo nocase nocrypto +nolock nonliteral noop nop -- cgit v1.2.1 From 40472bab97aa5deb0e07ffa16d78429d42dda295 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 16 Feb 2016 16:49:31 -0500 Subject: WT-2349 Fix warning. --- test/readonly/readonly.c | 1 + 1 file changed, 1 insertion(+) diff --git a/test/readonly/readonly.c b/test/readonly/readonly.c index 66885e135dc..42e787ab42c 100644 --- a/test/readonly/readonly.c +++ b/test/readonly/readonly.c @@ -182,6 +182,7 @@ main(int argc, char *argv[]) working_dir = "WT_RD"; child = false; + op = OP_READ; while ((ch = __wt_getopt(progname, argc, argv, "Rh:W")) != EOF) switch (ch) { case 'R': -- cgit v1.2.1 From 99f2005a55a5eb6a555b71841d6cf4feae3c5f40 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Wed, 17 Feb 2016 02:08:56 +0000 Subject: Minor cleanup. --- test/cursor_order/cursor_order.h | 11 +---------- test/cursor_order/cursor_order_file.c | 1 + test/cursor_order/cursor_order_ops.c | 4 ++-- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/test/cursor_order/cursor_order.h b/test/cursor_order/cursor_order.h index 9a3ae51ed91..dd49fce124b 100644 --- a/test/cursor_order/cursor_order.h +++ b/test/cursor_order/cursor_order.h @@ -26,23 +26,14 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include -#include - -#include -#include -#include #include -#include -#include -#include -#include #include "test_util.i" #define FNAME "file:cursor_order.%03d" /* File name */ typedef enum { FIX, ROW, VAR } __ftype; /* File type */ + typedef struct { uint64_t append_inserters; /* Number of append threads */ WT_CONNECTION *conn; /* WiredTiger connection */ diff --git a/test/cursor_order/cursor_order_file.c b/test/cursor_order/cursor_order_file.c index 81c76a3a63e..e5dd76fa1a1 100644 --- a/test/cursor_order/cursor_order_file.c +++ b/test/cursor_order/cursor_order_file.c @@ -100,6 +100,7 @@ load(SHARED_CONFIG *cfg, const char *name) if ((ret = cursor->insert(cursor)) != 0) testutil_die(ret, "cursor.insert"); } + /* Setup the starting key range for the workload phase. */ cfg->key_range = cfg->nkeys; cursor->close(cursor); diff --git a/test/cursor_order/cursor_order_ops.c b/test/cursor_order/cursor_order_ops.c index c60923d9cc0..e2578f7544e 100644 --- a/test/cursor_order/cursor_order_ops.c +++ b/test/cursor_order/cursor_order_ops.c @@ -28,9 +28,9 @@ #include "cursor_order.h" +static void *append_insert(void *); static void print_stats(SHARED_CONFIG *); static void *reverse_scan(void *); -static void *append_insert(void *); typedef struct { char *name; /* object name */ @@ -224,9 +224,9 @@ reverse_scan(void *arg) SHARED_CONFIG *cfg; WT_CURSOR *cursor; WT_SESSION *session; - uint64_t i; int id, ret; char tid[128]; + uint64_t i; id = (int)(uintptr_t)arg; s = &run_info[id]; -- cgit v1.2.1 From 3000f62be47ef71ffce6b67c889d02c517a66c9b Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Wed, 17 Feb 2016 04:11:38 +0000 Subject: SERVER-22676 Don't check duplicated create calls for matching configurations. We added the check in WT-2295 as part of another fix. We have never checked before and it makes maintaining backward compatability more complex in the future for limited gain. --- src/schema/schema_create.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/schema/schema_create.c b/src/schema/schema_create.c index 25bbd496798..064c8986318 100644 --- a/src/schema/schema_create.c +++ b/src/schema/schema_create.c @@ -275,15 +275,11 @@ __create_colgroup(WT_SESSION_IMPL *session, WT_ERR(__wt_schema_create(session, source, sourceconf)); WT_ERR(__wt_config_collapse(session, cfg, &cgconf)); - if (exists) { - if (strcmp(cgconf, origconf) != 0) - WT_ERR_MSG(session, EINVAL, - "%s: does not match existing configuration", name); - goto err; - } - WT_ERR(__wt_metadata_insert(session, name, cgconf)); - WT_ERR(__wt_schema_open_colgroups(session, table)); + if (!exists) { + WT_ERR(__wt_metadata_insert(session, name, cgconf)); + WT_ERR(__wt_schema_open_colgroups(session, table)); + } err: __wt_free(session, cgconf); __wt_free(session, sourceconf); -- cgit v1.2.1 From ec28d6ed1c2d9a622af0479c6af4f63e667de03b Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Wed, 17 Feb 2016 04:32:53 +0000 Subject: SERVER-22676 Remove test case for error checking this branch removes. --- test/suite/test_schema02.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/suite/test_schema02.py b/test/suite/test_schema02.py index 6895e947efe..b404261c066 100644 --- a/test/suite/test_schema02.py +++ b/test/suite/test_schema02.py @@ -103,10 +103,6 @@ class test_schema02(wttest.WiredTigerTestCase): self.expect_failure_colgroup("main:c1", "columns=(S1,i2),exclusive", "") - # exists with different config - self.expect_failure_colgroup("main:c1", "columns=(S1,i4)", - "/does not match existing configuration/") - # colgroup not declared in initial create self.expect_failure_colgroup("main:c3", "columns=(S3,i4)", "/Column group 'c3' not found in" -- cgit v1.2.1 From 34a963af9d2fd3b5243adbc335158fe8a121c906 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Wed, 17 Feb 2016 05:00:36 +0000 Subject: SERVER-22676 Remove other create configuration matching checks. --- src/schema/schema_create.c | 51 +++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/src/schema/schema_create.c b/src/schema/schema_create.c index 064c8986318..9b3b76b62de 100644 --- a/src/schema/schema_create.c +++ b/src/schema/schema_create.c @@ -535,20 +535,17 @@ __create_index(WT_SESSION_IMPL *session, cfg[1] = sourceconf; cfg[2] = confbuf.data; WT_ERR(__wt_config_collapse(session, cfg, &idxconf)); - if (exists) { - if (strcmp(idxconf, origconf) != 0) - WT_ERR_MSG(session, EINVAL, - "%s: does not match existing configuration", name); - goto err; - } - WT_ERR(__wt_metadata_insert(session, name, idxconf)); - /* Make sure that the configuration is valid. */ - WT_ERR(__wt_schema_open_index( - session, table, idxname, strlen(idxname), &idx)); + if (!exists) { + WT_ERR(__wt_metadata_insert(session, name, idxconf)); + + /* Make sure that the configuration is valid. */ + WT_ERR(__wt_schema_open_index( + session, table, idxname, strlen(idxname), &idx)); - /* If there is data in the table, fill the index. */ - WT_ERR(__fill_index(session, table, idx)); + /* If there is data in the table, fill the index. */ + WT_ERR(__fill_index(session, table, idx)); + } err: __wt_free(session, idxconf); __wt_free(session, origconf); @@ -608,23 +605,21 @@ __create_table(WT_SESSION_IMPL *session, WT_ERR_NOTFOUND_OK(ret); WT_ERR(__wt_config_collapse(session, cfg, &tableconf)); - if (exists) { - if (strcmp(tableconf, table->config) != 0) - WT_ERR_MSG(session, EINVAL, - "%s: does not match existing configuration", name); - goto err; - } - WT_ERR(__wt_metadata_insert(session, name, tableconf)); - /* Attempt to open the table now to catch any errors. */ - WT_ERR(__wt_schema_get_table( - session, tablename, strlen(tablename), true, &table)); - - if (ncolgroups == 0) { - cgsize = strlen("colgroup:") + strlen(tablename) + 1; - WT_ERR(__wt_calloc_def(session, cgsize, &cgname)); - snprintf(cgname, cgsize, "colgroup:%s", tablename); - WT_ERR(__create_colgroup(session, cgname, exclusive, config)); + if (!exists) { + WT_ERR(__wt_metadata_insert(session, name, tableconf)); + + /* Attempt to open the table now to catch any errors. */ + WT_ERR(__wt_schema_get_table( + session, tablename, strlen(tablename), true, &table)); + + if (ncolgroups == 0) { + cgsize = strlen("colgroup:") + strlen(tablename) + 1; + WT_ERR(__wt_calloc_def(session, cgsize, &cgname)); + snprintf(cgname, cgsize, "colgroup:%s", tablename); + WT_ERR(__create_colgroup( + session, cgname, exclusive, config)); + } } if (0) { -- cgit v1.2.1 From 52a2084358e2cc7303bcee7b8a75ca66aac231d2 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Wed, 17 Feb 2016 05:39:29 +0000 Subject: Remove invalid test code. --- test/suite/test_index01.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/suite/test_index01.py b/test/suite/test_index01.py index bebeb191ef0..5dfa5506277 100644 --- a/test/suite/test_index01.py +++ b/test/suite/test_index01.py @@ -226,10 +226,6 @@ class test_index01(wttest.WiredTigerTestCase): self.assertRaises(wiredtiger.WiredTigerError, lambda: self.session.create(self.index[0], 'columns=(dept),exclusive')) - # non-exclusive create with differing configuration - self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.create(self.index[0], - 'columns=(salary)'), '/does not match existing configuration/') self.drop_table() if __name__ == '__main__': -- cgit v1.2.1 From b9073cd9076923ce6ba3084d7427091f773df956 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 17 Feb 2016 10:06:28 -0500 Subject: WT-2349 Initial addition of readonly configuration option. Whitespace. --- test/readonly/readonly.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/readonly/readonly.c b/test/readonly/readonly.c index 42e787ab42c..17cc78b8f56 100644 --- a/test/readonly/readonly.c +++ b/test/readonly/readonly.c @@ -92,7 +92,8 @@ run_child(const char *homedir, int op, int expect) testutil_die(ret, "wiredtiger_open success err"); if (expect == EXPECT_ERR) { if (ret == 0) - testutil_die(ret, "wiredtiger_open expected err succeeded"); + testutil_die( + ret, "wiredtiger_open expected err succeeded"); /* * If we expect an error and got one, we're done. */ -- cgit v1.2.1 From bac501eacc75d9d514bb4023d70134feaf8ae6a2 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 17 Feb 2016 10:28:03 -0500 Subject: WT-2349 Initial addition of readonly configuration option. Remove unreachable code. --- src/txn/txn_recover.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/txn/txn_recover.c b/src/txn/txn_recover.c index 8c34f3d9393..f41691bbc3b 100644 --- a/src/txn/txn_recover.c +++ b/src/txn/txn_recover.c @@ -457,11 +457,9 @@ __wt_txn_recover(WT_SESSION_IMPL *session) if (F_ISSET(conn, WT_CONN_READONLY)) { WT_ERR(__wt_log_needs_recovery( session, &metafile->ckpt_lsn, &needs_rec)); - if (needs_rec) { + if (needs_rec) WT_ERR_MSG(session, WT_RUN_RECOVERY, "Read-only database needs recovery"); - WT_ERR(WT_RUN_RECOVERY); - } } if (WT_IS_INIT_LSN(&metafile->ckpt_lsn)) WT_ERR(__wt_log_scan(session, -- cgit v1.2.1 From b89f959f289446915935be72f8f3887658470b5d Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 17 Feb 2016 12:17:12 -0500 Subject: WT-2349 Initial addition of readonly configuration option. Edit pass over the initial read-only docs. --- src/docs/readonly.dox | 85 +++++++++++++++++++++++---------------------------- 1 file changed, 39 insertions(+), 46 deletions(-) diff --git a/src/docs/readonly.dox b/src/docs/readonly.dox index 1da103085bf..015f7490c21 100644 --- a/src/docs/readonly.dox +++ b/src/docs/readonly.dox @@ -1,62 +1,55 @@ /*! @m_page{{c,java},readonly,Database read-only mode} -WiredTiger supports opening a database in read-only mode. When a database -is opened in read-only mode, all modifications are disabled on the WT_CONNECTION -handle, any sessions opened in that connection and any cursors opened on any -of those sessions. Any session handle method that modifies any part of the -database will return an error. Similarly, any WiredTiger internal threads -that perform modifications will not perform that work. - -When a database is opened in read-only mode, WiredTiger assumes the database -directory and content already exist and that it was a clean shutdown, -particularly if logging and recovery are used. - -@section readonly_config Database readonly configuration considerations - -Use of the \c readonly configuration may affect other configuration settings. -Where the default setting contradicts read-only operation, WiredTiger -overrides those defaults to perform in a non-modifying mode. For example, -merges are turned off if LSM trees are in use, and log file archiving is -disabled if logging is configured. - -Other settings where the user must explicitly turn them on return an error -if that setting contradicts read-only mode. For example, statistics logging -or zero filling log files are not allowed during read-only mode and trying -to set those will result in an error. +WiredTiger supports read-only mode databases. When a database is opened +in read-only mode, all modifications are disabled on the WT_CONNECTION +handle, any sessions opened in that connection and any cursors opened +in any of those sessions. For example, all cursor or session handle +methods that modify the database will instead return errors. -@section readonly_logging Readonly configuration and logging +When a database is opened in read-only mode, the database directory and +content must already exist and have been shutdown cleanly. + +@section readonly_config Database read-only configuration considerations + +The \c readonly configuration affects other configuration settings. +Where a default setting contradicts read-only operation, WiredTiger +defaults are overridden to perform in a read-only mode. For example, LSM +tree merges are turned off when LSM trees are configured, and log file +archiving is disabled when logging is configured. -If logging is enabled on the database when it is opened with the \c readonly -configuration, log file archiving and log file pre-allocation are disabled. -The log files will not be modified any way. Any pre-allocated or temporary -log files that exist from an earlier run that would normally be removed will -remain. +Where a user configured setting contradicts read-only operation, WiredTiger +will return an error. For example, statistics logging or zero-filling +log files is not allowed in read-only mode, and attempting to configure +them will return an error. @section readonly_recovery Readonly configuration and recovery -Since recovery only reapplies modifications, clearly no recovery actions can -occur during read-only mode. The system will detect if recovery is necessary -or if it was a clean shutdown. Attempting to open the database via -::wiredtiger_open will only proceed from a clean shutdown and return an error -if any recovery is required. +Because recovery modifies the database, recovery cannot be done in +read-only mode. A ::wiredtiger_open call to open a database in read-only +mode will fail if the database was not cleanly shutdown and recovery is +required. + +@section readonly_logging Readonly configuration and logging + +If logging is enabled on the database when opened in read-only mode, log +file archiving and log file pre-allocation are disabled and the log files +will not be modified any way. @section readonly_lsm Readonly configuration and LSM trees -If LSM trees are in use, starting up in read-only mode turns off all -modification. Internal LSM operations such as merging, creating new chunks, -creating bloom filters and dropping old chunks are disabled. +If LSM trees are in use, read-only mode turns off all modification. +Internal LSM operations such as merging, creating new chunks, creating +bloom filters and dropping old chunks are disabled. @section readonly_handles Readonly configuration and multiple database handles One unusual affect of read-only operations is the potential for multiple -read-only database handles open to the same database at the same time, in -limited circumstances. WiredTiger prevents multiple connection handles -by writing to a lock file. Even in read-only mode, creating and writing -to the lock file is attempted. In the case where the database directory or -the lock file does not exist or cannot be created or written to, a read-only -handle is allowed to proceed. So in the case where the database directory -and its files are read-only, multiple read-only connection handles could -be open at the same time. In the case where the database directory is -writable the normal file locking occurs preventing multiple connections. +read-only database handles open on the same database at the same time. +WiredTiger prevents multiple connection handles by writing lock file, +and this locking is done even in read-only mode. However, if the lock +file cannot be written, opening in read-only mode is still allowed to +proceed. For that reason, multiple read-only connection handles could +be open at the same time. Normal locking occurs if the lock file can be +written in read-only mode, preventing multiple database connections. */ -- cgit v1.2.1 From af2bf588eaaa918c5c4051baff9e5daf20819f08 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Wed, 17 Feb 2016 13:51:14 -0500 Subject: WT-2349 Add a 'readonly' option to wtperf. --- bench/wtperf/config.c | 14 ++++++++++++++ bench/wtperf/runners/evict-btree-readonly.wtperf | 12 ++++++++++++ bench/wtperf/runners/evict-lsm-readonly.wtperf | 13 +++++++++++++ bench/wtperf/wtperf.c | 21 +++++++++++++++++++-- bench/wtperf/wtperf.h | 2 ++ bench/wtperf/wtperf_opt.i | 4 ++++ src/docs/wtperf.dox | 4 ++++ 7 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 bench/wtperf/runners/evict-btree-readonly.wtperf create mode 100644 bench/wtperf/runners/evict-lsm-readonly.wtperf diff --git a/bench/wtperf/config.c b/bench/wtperf/config.c index 3cb20ff2b26..89069a170d8 100644 --- a/bench/wtperf/config.c +++ b/bench/wtperf/config.c @@ -634,6 +634,9 @@ config_opt_str(CONFIG *cfg, const char *name, const char *value) int config_sanity(CONFIG *cfg) { + WORKLOAD *workp; + u_int i; + /* Various intervals should be less than the run-time. */ if (cfg->run_time > 0 && ((cfg->checkpoint_threads != 0 && @@ -660,6 +663,17 @@ config_sanity(CONFIG *cfg) "Invalid pareto distribution - should be a percentage\n"); return (EINVAL); } + + if (cfg->readonly && cfg->workload != NULL) + for (i = 0, workp = cfg->workload; + i < cfg->workload_cnt; ++i, ++workp) + if (workp->insert != 0 || workp->update != 0 || + workp->truncate != 0) { + fprintf(stderr, + "Invalid workload: insert, update or " + "truncate specified with readonly\n"); + return (EINVAL); + } return (0); } diff --git a/bench/wtperf/runners/evict-btree-readonly.wtperf b/bench/wtperf/runners/evict-btree-readonly.wtperf new file mode 100644 index 00000000000..d79af2b762b --- /dev/null +++ b/bench/wtperf/runners/evict-btree-readonly.wtperf @@ -0,0 +1,12 @@ +# wtperf options file: evict btree configuration +conn_config="cache_size=50M" +table_config="type=file" +icount=10000000 +report_interval=5 +run_time=120 +populate_threads=1 +readonly=true +threads=((count=16,reads=1)) +# Add throughput/latency monitoring +max_latency=2000 +sample_interval=5 diff --git a/bench/wtperf/runners/evict-lsm-readonly.wtperf b/bench/wtperf/runners/evict-lsm-readonly.wtperf new file mode 100644 index 00000000000..fe45c0e93b6 --- /dev/null +++ b/bench/wtperf/runners/evict-lsm-readonly.wtperf @@ -0,0 +1,13 @@ +# wtperf options file: evict lsm configuration +conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6)" +table_config="type=lsm,lsm=(chunk_size=2M),os_cache_dirty_max=16MB" +compact=true +icount=10000000 +report_interval=5 +run_time=120 +populate_threads=1 +readonly=true +threads=((count=16,reads=1)) +# Add throughput/latency monitoring +max_latency=2000 +sample_interval=5 diff --git a/bench/wtperf/wtperf.c b/bench/wtperf/wtperf.c index b2e68198e9a..c5c2d294708 100644 --- a/bench/wtperf/wtperf.c +++ b/bench/wtperf/wtperf.c @@ -33,6 +33,7 @@ static const CONFIG default_cfg = { "WT_TEST", /* home */ "WT_TEST", /* monitor dir */ NULL, /* partial logging */ + NULL, /* reopen config */ NULL, /* base_uri */ NULL, /* uris */ NULL, /* helium_mount */ @@ -1517,7 +1518,7 @@ close_reopen(CONFIG *cfg) { int ret; - if (!cfg->reopen_connection) + if (!cfg->readonly && !cfg->reopen_connection) return (0); /* * Reopen the connection. We do this so that the workload phase always @@ -1533,7 +1534,7 @@ close_reopen(CONFIG *cfg) return (ret); } if ((ret = wiredtiger_open( - cfg->home, NULL, cfg->conn_config, &cfg->conn)) != 0) { + cfg->home, NULL, cfg->reopen_config, &cfg->conn)) != 0) { lprintf(cfg, ret, 0, "Re-opening the connection failed"); return (ret); } @@ -2300,6 +2301,22 @@ main(int argc, char *argv[]) snprintf((char *)cfg->partial_config, req_len, "%s%s", (char *)cfg->table_config, LOG_PARTIAL_CONFIG); } + /* + * Set the config for reopen. If readonly add in that string. + * If not readonly then just copy the original conn_config. + */ + if (cfg->readonly) + req_len = strlen(cfg->conn_config) + + strlen(READONLY_CONFIG) + 1; + else + req_len = strlen(cfg->conn_config) + 1; + cfg->reopen_config = dcalloc(req_len, 1); + if (cfg->readonly) + snprintf((char *)cfg->reopen_config, req_len, "%s%s", + (char *)cfg->conn_config, READONLY_CONFIG); + else + snprintf((char *)cfg->reopen_config, req_len, "%s", + (char *)cfg->conn_config); /* Sanity-check the configuration. */ if ((ret = config_sanity(cfg)) != 0) diff --git a/bench/wtperf/wtperf.h b/bench/wtperf/wtperf.h index 929880b0aef..0b0fb88a99c 100644 --- a/bench/wtperf/wtperf.h +++ b/bench/wtperf/wtperf.h @@ -138,6 +138,7 @@ typedef struct { } THROTTLE_CONFIG; #define LOG_PARTIAL_CONFIG ",log=(enabled=false)" +#define READONLY_CONFIG ",readonly=true" /* * NOTE: If you add any fields to this structure here, you must also add * an initialization in wtperf.c in the default_cfg. @@ -146,6 +147,7 @@ struct __config { /* Configuration structure */ const char *home; /* WiredTiger home */ const char *monitor_dir; /* Monitor output dir */ const char *partial_config; /* Config string for partial logging */ + const char *reopen_config; /* Config string for conn reopen */ char *base_uri; /* Object URI */ char **uris; /* URIs if multiple tables */ const char *helium_mount; /* Optional Helium mount point */ diff --git a/bench/wtperf/wtperf_opt.i b/bench/wtperf/wtperf_opt.i index 60bbaff56e5..ecc1f216299 100644 --- a/bench/wtperf/wtperf_opt.i +++ b/bench/wtperf/wtperf_opt.i @@ -145,6 +145,10 @@ DEF_OPT_AS_UINT32(random_range, 0, "insert operations") DEF_OPT_AS_BOOL(random_value, 0, "generate random content for the value") DEF_OPT_AS_UINT32(read_range, 0, "scan a range of keys after each search") +DEF_OPT_AS_BOOL(readonly, 0, + "reopen the connection between populate and workload phases in readonly " + "mode. Requires reopen_connection turned on (default). Requires that " + "read be the only workload specified") DEF_OPT_AS_BOOL(reopen_connection, 1, "close and reopen the connection between populate and workload phases") DEF_OPT_AS_UINT32(report_interval, 2, diff --git a/src/docs/wtperf.dox b/src/docs/wtperf.dox index 1f0d1533ac4..6f3d2f87ee0 100644 --- a/src/docs/wtperf.dox +++ b/src/docs/wtperf.dox @@ -212,6 +212,10 @@ insert operations generate random content for the value @par read_range (unsigned int, default=0) scan a range of keys after each search +@par readonly (boolean, default=false) +reopen the connection between populate and workload phases in readonly +mode. Requires reopen_connection turned on (default). Requires that +read be the only workload specified @par reopen_connection (boolean, default=true) close and reopen the connection between populate and workload phases @par report_interval (unsigned int, default=2) -- cgit v1.2.1 From 8dc8d3afe2b238feb149a5ff69c9aaca82a151dd Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Wed, 17 Feb 2016 14:23:09 -0500 Subject: WT-2349 Use a single session notsup function. --- src/include/extern.h | 3 +- src/session/session_api.c | 76 +++++++++++++++---------------------------- test/suite/test_readonly03.py | 3 +- 3 files changed, 30 insertions(+), 52 deletions(-) diff --git a/src/include/extern.h b/src/include/extern.h index afb6b6135ba..8945309c4a7 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -611,8 +611,7 @@ extern WT_DATA_SOURCE *__wt_schema_get_source(WT_SESSION_IMPL *session, const ch extern int __wt_str_name_check(WT_SESSION_IMPL *session, const char *str); extern int __wt_name_check(WT_SESSION_IMPL *session, const char *str, size_t len); extern int __wt_schema_worker(WT_SESSION_IMPL *session, const char *uri, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[], uint32_t open_flags); -extern int __wt_session_notsup_config(WT_SESSION *wt_session, const char *config); -extern int __wt_session_notsup_uri( WT_SESSION *wt_session, const char *uri, const char *config); +extern int __wt_session_notsup(WT_SESSION *wt_session); extern int __wt_session_reset_cursors(WT_SESSION_IMPL *session, bool free_buffers); extern int __wt_session_copy_values(WT_SESSION_IMPL *session); extern int __wt_session_release_resources(WT_SESSION_IMPL *session); diff --git a/src/session/session_api.c b/src/session/session_api.c index cf95bae1444..106a2eb31ca 100644 --- a/src/session/session_api.c +++ b/src/session/session_api.c @@ -13,34 +13,15 @@ static int __session_snapshot(WT_SESSION *, const char *); static int __session_rollback_transaction(WT_SESSION *, const char *); /* - * __wt_session_notsup_config -- - * Unsupported session actions that have a signature of a config string. + * __wt_session_notsup -- + * Unsupported session method. */ int -__wt_session_notsup_config(WT_SESSION *wt_session, const char *config) +__wt_session_notsup(WT_SESSION *wt_session) { WT_SESSION_IMPL *session; session = (WT_SESSION_IMPL *)wt_session; - WT_UNUSED(config); - - WT_RET_MSG(session, ENOTSUP, "Unsupported session method"); -} - -/* - * __wt_session_notsup_uri -- - * Unsupported session actions that have a signature of a URI and - * a config string. - */ -int -__wt_session_notsup_uri( - WT_SESSION *wt_session, const char *uri, const char *config) -{ - WT_SESSION_IMPL *session; - - session = (WT_SESSION_IMPL *)wt_session; - WT_UNUSED(uri); - WT_UNUSED(config); WT_RET_MSG(session, ENOTSUP, "Unsupported session method"); } @@ -579,9 +560,6 @@ __session_log_printf(WT_SESSION *wt_session, const char *fmt, ...) session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL_NOCONF(session, log_printf); - if (F_ISSET(S2C(session), WT_CONN_READONLY)) - WT_ERR_MSG(session, ENOTSUP, "Unsupported session method"); - va_start(ap, fmt); ret = __wt_log_vprintf(session, fmt, ap); va_end(ap); @@ -603,9 +581,6 @@ __session_rebalance(WT_SESSION *wt_session, const char *uri, const char *config) SESSION_API_CALL(session, rebalance, config, cfg); - if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY | WT_CONN_READONLY)) - WT_ERR_MSG(session, ENOTSUP, "Unsupported session method"); - /* Block out checkpoints to avoid spurious EBUSY errors. */ WT_WITH_CHECKPOINT_LOCK(session, ret, WT_WITH_SCHEMA_LOCK(session, ret, @@ -629,9 +604,6 @@ __session_rename(WT_SESSION *wt_session, session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, rename, config, cfg); - if (F_ISSET(S2C(session), WT_CONN_READONLY)) - WT_ERR_MSG(session, ENOTSUP, "Unsupported session method"); - /* Disallow objects in the WiredTiger name space. */ WT_ERR(__wt_str_name_check(session, uri)); WT_ERR(__wt_str_name_check(session, newuri)); @@ -994,9 +966,6 @@ __session_truncate(WT_SESSION *wt_session, SESSION_TXN_API_CALL(session, truncate, config, cfg); WT_STAT_FAST_CONN_INCR(session, cursor_truncate); - if (F_ISSET(S2C(session), WT_CONN_READONLY)) - WT_ERR_MSG(session, ENOTSUP, "Unsupported session method"); - /* * If the URI is specified, we don't need a start/stop, if start/stop * is specified, we don't need a URI. One exception is the log URI @@ -1472,21 +1441,30 @@ __open_session(WT_CONNECTION_IMPL *conn, */ if (F_ISSET(conn, WT_CONN_READONLY)) { wt_session = &session_ret->iface; - wt_session->checkpoint = __wt_session_notsup_config; - wt_session->log_flush = __wt_session_notsup_config; - wt_session->transaction_sync = __wt_session_notsup_config; - - wt_session->compact = __wt_session_notsup_uri; - wt_session->create = __wt_session_notsup_uri; - wt_session->drop = __wt_session_notsup_uri; - wt_session->salvage = __wt_session_notsup_uri; - wt_session->upgrade = __wt_session_notsup_uri; - - /* - * The other methods that are not supported but are - * checked individually in the function are: - * log_printf, rebalance, rename, truncate - */ + wt_session->checkpoint = + (int (*)(WT_SESSION *, const char *))__wt_session_notsup; + wt_session->compact = + (int (*)(WT_SESSION *, const char *, const char *))__wt_session_notsup; + wt_session->create = + (int (*)(WT_SESSION *, const char *, const char *))__wt_session_notsup; + wt_session->drop = + (int (*)(WT_SESSION *, const char *, const char *))__wt_session_notsup; + wt_session->log_flush = + (int (*)(WT_SESSION *, const char *))__wt_session_notsup; + wt_session->log_printf = + (int (*)(WT_SESSION *, const char *, ...))__wt_session_notsup; + wt_session->rebalance = + (int (*)(WT_SESSION *, const char *, const char *))__wt_session_notsup; + wt_session->rename = (int (*)(WT_SESSION *, + const char *, const char *, const char *))__wt_session_notsup; + wt_session->salvage = + (int (*)(WT_SESSION *, const char *, const char *))__wt_session_notsup; + wt_session->transaction_sync = + (int (*)(WT_SESSION *, const char *))__wt_session_notsup; + wt_session->truncate = (int (*)(WT_SESSION *, + const char *, WT_CURSOR *, WT_CURSOR *, const char *))__wt_session_notsup; + wt_session->upgrade = + (int (*)(WT_SESSION *, const char *, const char *))__wt_session_notsup; } WT_ERR(__wt_cond_alloc(session, "session", false, &session_ret->cond)); diff --git a/test/suite/test_readonly03.py b/test/suite/test_readonly03.py index e7932839231..981a21d51ac 100644 --- a/test/suite/test_readonly03.py +++ b/test/suite/test_readonly03.py @@ -113,7 +113,8 @@ class test_readonly03(wttest.WiredTigerTestCase, suite_subprocess): lambda: self.session.salvage(self.uri, None), msg) elif op == 'truncate': self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.truncate(self.uri, None, None, None), msg) + lambda: self.session.truncate(self.uri, None, None, None), + msg) elif op == 'upgrade': self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.upgrade(self.uri, None), msg) -- cgit v1.2.1 From 8f76cb90ab76924a076eeb25b0308d3c3cd2f4d0 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Wed, 17 Feb 2016 14:26:02 -0500 Subject: WT-2349 Minor text change. --- src/docs/readonly.dox | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docs/readonly.dox b/src/docs/readonly.dox index 015f7490c21..9935f5d1b17 100644 --- a/src/docs/readonly.dox +++ b/src/docs/readonly.dox @@ -45,7 +45,7 @@ bloom filters and dropping old chunks are disabled. One unusual affect of read-only operations is the potential for multiple read-only database handles open on the same database at the same time. -WiredTiger prevents multiple connection handles by writing lock file, +WiredTiger prevents multiple connection handles by writing a lock file, and this locking is done even in read-only mode. However, if the lock file cannot be written, opening in read-only mode is still allowed to proceed. For that reason, multiple read-only connection handles could -- cgit v1.2.1 From f312f22dff972056d1d6bd248d48f3c2a05d7cee Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Thu, 18 Feb 2016 14:34:53 +1100 Subject: WT-2399 clean up GCC 4.1 build warning. --- test/cursor_order/cursor_order_ops.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/cursor_order/cursor_order_ops.c b/test/cursor_order/cursor_order_ops.c index e2578f7544e..e5c3eed07f5 100644 --- a/test/cursor_order/cursor_order_ops.c +++ b/test/cursor_order/cursor_order_ops.c @@ -180,6 +180,9 @@ reverse_scan_op( WT_UNUSED(session); WT_UNUSED(s); + /* Make GCC 4.1 happy */ + prev_key = this_key = 0; + /* Reset the cursor */ cursor->reset(cursor); -- cgit v1.2.1 From b82ac0b95e1af1c077c34448b08fbe24e493951d Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Thu, 18 Feb 2016 14:52:23 +1100 Subject: WT-2399 Further compiler warning cleanup --- test/cursor_order/cursor_order.c | 10 +++++----- test/cursor_order/cursor_order_ops.c | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/cursor_order/cursor_order.c b/test/cursor_order/cursor_order.c index 0155339d71e..14709a2e88e 100644 --- a/test/cursor_order/cursor_order.c +++ b/test/cursor_order/cursor_order.c @@ -88,7 +88,7 @@ main(int argc, char *argv[]) working_dir = __wt_optarg; break; case 'k': /* rows */ - cfg->nkeys = (u_int64_t)atol(__wt_optarg); + cfg->nkeys = (uint64_t)atol(__wt_optarg); break; case 'l': /* log */ if ((logfp = fopen(__wt_optarg, "w")) == NULL) { @@ -98,10 +98,10 @@ main(int argc, char *argv[]) } break; case 'n': /* operations */ - cfg->max_nops = (u_int64_t)atol(__wt_optarg); + cfg->max_nops = (uint64_t)atol(__wt_optarg); break; case 'R': - cfg->reverse_scanners = (u_int64_t)atol(__wt_optarg); + cfg->reverse_scanners = (uint64_t)atol(__wt_optarg); break; case 'r': /* runs */ runs = atoi(__wt_optarg); @@ -125,10 +125,10 @@ main(int argc, char *argv[]) cfg->vary_nops = true; break; case 'w': - cfg->reverse_scan_ops = (u_int64_t)atol(__wt_optarg); + cfg->reverse_scan_ops = (uint64_t)atol(__wt_optarg); break; case 'W': - cfg->append_inserters = (u_int64_t)atol(__wt_optarg); + cfg->append_inserters = (uint64_t)atol(__wt_optarg); break; default: return (usage()); diff --git a/test/cursor_order/cursor_order_ops.c b/test/cursor_order/cursor_order_ops.c index e5c3eed07f5..9077f500594 100644 --- a/test/cursor_order/cursor_order_ops.c +++ b/test/cursor_order/cursor_order_ops.c @@ -272,7 +272,7 @@ append_insert_op( SHARED_CONFIG *cfg, WT_SESSION *session, WT_CURSOR *cursor, INFO *s) { WT_ITEM *value, _value; - int64_t keyno; + uint64_t keyno; int ret; char keybuf[64], valuebuf[64]; -- cgit v1.2.1 From 6dbcdde3ea5d23e5f5d9f04c1a541936ce4e7217 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 18 Feb 2016 08:03:40 -0500 Subject: WT-2399 Add initial implementation for cursor traversal test application Update spelling lists. --- dist/s_string.ok | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dist/s_string.ok b/dist/s_string.ok index 7a8f3a9b0bd..26998d13214 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -107,6 +107,7 @@ FALLTHROUGH FH FLD FLSv +FLv FNV FORALL FOREACH @@ -671,6 +672,7 @@ inline inmem insertK insertV +inserters instantiation intl intnum @@ -1057,6 +1059,7 @@ vsize vsnprintf vtype vunpack +vw walk's warmup wb -- cgit v1.2.1 From ca48eaa794f6ff410c578e7cffb253c616441bac Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 18 Feb 2016 08:58:39 -0500 Subject: WT-2405: test utility error handling. Add a check routine to make simple error handling easier. --- test/format/backup.c | 16 ++----- test/format/bulk.c | 18 +++---- test/format/compact.c | 6 +-- test/format/format.h | 10 ++++ test/format/lrt.c | 22 +++------ test/format/ops.c | 121 ++++++++++++++++++------------------------------ test/format/rebalance.c | 6 +-- test/format/salvage.c | 6 +-- test/format/t.c | 17 +++---- test/format/wts.c | 37 +++++---------- 10 files changed, 97 insertions(+), 162 deletions(-) diff --git a/test/format/backup.c b/test/format/backup.c index 748494bf841..f545930d992 100644 --- a/test/format/backup.c +++ b/test/format/backup.c @@ -96,8 +96,7 @@ backup(void *arg) return (NULL); /* Open a session. */ - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + check(conn->open_session(conn, NULL, NULL, &session)); /* * Perform a backup at somewhere under 10 seconds (so we get at @@ -113,8 +112,7 @@ backup(void *arg) break; /* Lock out named checkpoints */ - if ((ret = pthread_rwlock_wrlock(&g.backup_lock)) != 0) - die(ret, "pthread_rwlock_wrlock: backup lock"); + check(pthread_rwlock_wrlock(&g.backup_lock)); /* Re-create the backup directory. */ if ((ret = system(g.home_backup_init)) != 0) @@ -137,17 +135,13 @@ backup(void *arg) copy_file(key); } - if ((ret = backup_cursor->close(backup_cursor)) != 0) - die(ret, "cursor.close"); - - if ((ret = pthread_rwlock_unlock(&g.backup_lock)) != 0) - die(ret, "pthread_rwlock_unlock: backup lock"); + check(backup_cursor->close(backup_cursor)); + check(pthread_rwlock_unlock(&g.backup_lock)); check_copy(); } - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + check(session->close(session, NULL)); return (NULL); } diff --git a/test/format/bulk.c b/test/format/bulk.c index 28189e25b65..b344403f06f 100644 --- a/test/format/bulk.c +++ b/test/format/bulk.c @@ -37,13 +37,11 @@ wts_load(void) WT_SESSION *session; uint8_t *keybuf, *valbuf; bool is_bulk; - int ret; conn = g.wts_conn; keybuf = valbuf = NULL; - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + check(conn->open_session(conn, NULL, NULL, &session)); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, @@ -61,9 +59,8 @@ wts_load(void) if (g.c_reverse) is_bulk = false; - if ((ret = session->open_cursor(session, g.uri, NULL, - is_bulk ? "bulk,append" : NULL, &cursor)) != 0) - die(ret, "session.open_cursor"); + check(session->open_cursor(session, g.uri, NULL, + is_bulk ? "bulk,append" : NULL, &cursor)); /* Set up the key/value buffers. */ key_gen_setup(&keybuf); @@ -120,8 +117,7 @@ wts_load(void) break; } - if ((ret = cursor->insert(cursor)) != 0) - die(ret, "cursor.insert"); + check(cursor->insert(cursor)); #ifdef HAVE_BERKELEY_DB if (SINGLETHREADED) @@ -129,15 +125,13 @@ wts_load(void) #endif } - if ((ret = cursor->close(cursor)) != 0) - die(ret, "cursor.close"); + check(cursor->close(cursor)); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, "=============== bulk load stop ==============="); - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + check(session->close(session, NULL)); free(keybuf); free(valbuf); diff --git a/test/format/compact.c b/test/format/compact.c index fdfa597e07e..a29fb5e3c4b 100644 --- a/test/format/compact.c +++ b/test/format/compact.c @@ -48,8 +48,7 @@ compact(void *arg) /* Open a session. */ conn = g.wts_conn; - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + check(conn->open_session(conn, NULL, NULL, &session)); /* * Perform compaction at somewhere under 15 seconds (so we get at @@ -69,8 +68,7 @@ compact(void *arg) die(ret, "session.compact"); } - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + check(session->close(session, NULL)); return (NULL); } diff --git a/test/format/format.h b/test/format/format.h index 03da1a84c9c..7037da51e57 100644 --- a/test/format/format.h +++ b/test/format/format.h @@ -340,6 +340,16 @@ __attribute__((__noreturn__)) #endif ; +/* + * check -- + * Complain and quit if a function call fails. + */ +#define check(call) do { \ + int __r; \ + if ((__r = (call)) != 0) \ + die(__r, "%s/%d: %s", __func__, __LINE__, #call); \ +} while (0) + /* * mmrand -- * Return a random value between a min/max pair. diff --git a/test/format/lrt.c b/test/format/lrt.c index b7392829d30..319eb0fb467 100644 --- a/test/format/lrt.c +++ b/test/format/lrt.c @@ -60,11 +60,8 @@ lrt(void *arg) /* Open a session and cursor. */ conn = g.wts_conn; - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); - if ((ret = session->open_cursor( - session, g.uri, NULL, NULL, &cursor)) != 0) - die(ret, "session.open_cursor"); + check(conn->open_session(conn, NULL, NULL, &session)); + check(session->open_cursor(session, g.uri, NULL, NULL, &cursor)); for (pinned = 0;;) { if (pinned) { @@ -91,13 +88,10 @@ lrt(void *arg) die(0, "mismatched start/stop values"); /* End the transaction. */ - if ((ret = - session->commit_transaction(session, NULL)) != 0) - die(ret, "session.commit_transaction"); + check(session->commit_transaction(session, NULL)); /* Reset the cursor, releasing our pin. */ - if ((ret = cursor->reset(cursor)) != 0) - die(ret, "cursor.reset"); + check(cursor->reset(cursor)); pinned = 0; } else { /* @@ -106,9 +100,8 @@ lrt(void *arg) * positioned. As soon as the cursor loses its position * a new snapshot will be allocated. */ - if ((ret = session->begin_transaction( - session, "isolation=snapshot")) != 0) - die(ret, "session.begin_transaction"); + check(session->begin_transaction( + session, "isolation=snapshot")); /* Read a record at the end of the table. */ do { @@ -166,8 +159,7 @@ lrt(void *arg) break; } - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + check(session->close(session, NULL)); free(keybuf); free(buf); diff --git a/test/format/ops.c b/test/format/ops.c index 36d56df1505..7fac11a12b7 100644 --- a/test/format/ops.c +++ b/test/format/ops.c @@ -56,7 +56,7 @@ wts_ops(int lastrun) pthread_t backup_tid, compact_tid, lrt_tid; int64_t fourths, thread_ops; uint32_t i; - int ret, running; + int running; conn = g.wts_conn; @@ -97,8 +97,7 @@ wts_ops(int lastrun) /* Open a session. */ if (g.logging != 0) { - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + check(conn->open_session(conn, NULL, NULL, &session)); (void)g.wt_api->msg_printf(g.wt_api, session, "=============== thread ops start ==============="); } @@ -109,24 +108,19 @@ wts_ops(int lastrun) for (i = 0; i < g.c_threads; ++i) { tinfo[i].id = (int)i + 1; tinfo[i].state = TINFO_RUNNING; - if ((ret = - pthread_create(&tinfo[i].tid, NULL, ops, &tinfo[i])) != 0) - die(ret, "pthread_create"); + check(pthread_create(&tinfo[i].tid, NULL, ops, &tinfo[i])); } /* * If a multi-threaded run, start optional backup, compaction and * long-running reader threads. */ - if (g.c_backups && - (ret = pthread_create(&backup_tid, NULL, backup, NULL)) != 0) - die(ret, "pthread_create: backup"); - if (g.c_compact && - (ret = pthread_create(&compact_tid, NULL, compact, NULL)) != 0) - die(ret, "pthread_create: compaction"); - if (!SINGLETHREADED && g.c_long_running_txn && - (ret = pthread_create(&lrt_tid, NULL, lrt, NULL)) != 0) - die(ret, "pthread_create: long-running reader"); + if (g.c_backups) + check(pthread_create(&backup_tid, NULL, backup, NULL)); + if (g.c_compact) + check(pthread_create(&compact_tid, NULL, compact, NULL)); + if (!SINGLETHREADED && g.c_long_running_txn) + check(pthread_create(&lrt_tid, NULL, lrt, NULL)); /* Spin on the threads, calculating the totals. */ for (;;) { @@ -192,8 +186,7 @@ wts_ops(int lastrun) if (g.logging != 0) { (void)g.wt_api->msg_printf(g.wt_api, session, "=============== thread ops stop ==============="); - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + check(session->close(session, NULL)); } } @@ -269,9 +262,7 @@ ops(void *arg) */ if (intxn && (tinfo->ops == ckpt_op || tinfo->ops == session_op)) { - if ((ret = session->commit_transaction( - session, NULL)) != 0) - die(ret, "session.commit_transaction"); + check(session->commit_transaction(session, NULL)); ++tinfo->commit; intxn = 0; } @@ -279,13 +270,11 @@ ops(void *arg) /* Open up a new session and cursors. */ if (tinfo->ops == session_op || session == NULL || cursor == NULL) { - if (session != NULL && - (ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + if (session != NULL) + check(session->close(session, NULL)); - if ((ret = conn->open_session(conn, NULL, - ops_session_config(&tinfo->rnd), &session)) != 0) - die(ret, "connection.open_session"); + check(conn->open_session(conn, NULL, + ops_session_config(&tinfo->rnd), &session)); /* * 10% of the time, perform some read-only operations @@ -300,9 +289,8 @@ ops(void *arg) */ if (!SINGLETHREADED && !DATASOURCE("lsm") && ckpt_available && mmrand(&tinfo->rnd, 1, 10) == 1) { - if ((ret = session->open_cursor(session, - g.uri, NULL, ckpt_name, &cursor)) != 0) - die(ret, "session.open_cursor"); + check(session->open_cursor(session, + g.uri, NULL, ckpt_name, &cursor)); /* Pick the next session/cursor close/open. */ session_op += 250; @@ -323,13 +311,12 @@ ops(void *arg) * want to have to specify the record number, * which requires an append configuration. */ - if ((ret = session->open_cursor(session, g.uri, - NULL, "overwrite", &cursor)) != 0) - die(ret, "session.open_cursor"); - if ((g.type == FIX || g.type == VAR) && - (ret = session->open_cursor(session, g.uri, - NULL, "append", &cursor_insert)) != 0) - die(ret, "session.open_cursor"); + check(session->open_cursor(session, + g.uri, NULL, "overwrite", &cursor)); + if (g.type == FIX || g.type == VAR) + check(session->open_cursor( + session, g.uri, + NULL, "append", &cursor_insert)); /* Pick the next session/cursor close/open. */ session_op += mmrand(&tinfo->rnd, 100, 5000); @@ -358,10 +345,8 @@ ops(void *arg) } /* Named checkpoints lock out backups */ - if (ckpt_config != NULL && - (ret = pthread_rwlock_wrlock(&g.backup_lock)) != 0) - die(ret, - "pthread_rwlock_wrlock: backup lock"); + if (ckpt_config != NULL) + check(pthread_rwlock_wrlock(&g.backup_lock)); if ((ret = session->checkpoint(session, ckpt_config)) != 0) @@ -369,10 +354,8 @@ ops(void *arg) ckpt_config == NULL ? "" : ": ", ckpt_config == NULL ? "" : ckpt_config); - if (ckpt_config != NULL && - (ret = pthread_rwlock_unlock(&g.backup_lock)) != 0) - die(ret, - "pthread_rwlock_wrlock: backup lock"); + if (ckpt_config != NULL) + check(pthread_rwlock_unlock(&g.backup_lock)); /* Rephrase the checkpoint name for cursor open. */ if (ckpt_config == NULL) @@ -393,8 +376,7 @@ ops(void *arg) * have to do the reset outside of a transaction. */ if (tinfo->ops > reset_op && !intxn) { - if ((ret = session->reset(session)) != 0) - die(ret, "session.reset"); + check(session->reset(session)); /* Pick the next reset operation. */ reset_op += mmrand(&tinfo->rnd, 20000, 50000); @@ -406,9 +388,7 @@ ops(void *arg) */ if (!SINGLETHREADED && !intxn && mmrand(&tinfo->rnd, 1, 10) >= 8) { - if ((ret = - session->begin_transaction(session, NULL)) != 0) - die(ret, "session.begin_transaction"); + check(session->begin_transaction(session, NULL)); intxn = 1; } @@ -466,9 +446,7 @@ ops(void *arg) if (col_insert(tinfo, cursor_insert, &key, &value, &keyno)) goto deadlock; - if ((ret = - cursor_insert->reset(cursor_insert)) != 0) - die(ret, "cursor.reset"); + check(cursor_insert->reset(cursor_insert)); insert = 1; break; @@ -518,8 +496,7 @@ skip_insert: if (col_update(tinfo, goto deadlock; /* Reset the cursor: there is no reason to keep pages pinned. */ - if ((ret = cursor->reset(cursor)) != 0) - die(ret, "cursor.reset"); + check(cursor->reset(cursor)); /* * If we're in the transaction, commit 40% of the time and @@ -528,9 +505,8 @@ skip_insert: if (col_update(tinfo, if (intxn) switch (mmrand(&tinfo->rnd, 1, 10)) { case 1: case 2: case 3: case 4: /* 40% */ - if ((ret = session->commit_transaction( - session, NULL)) != 0) - die(ret, "session.commit_transaction"); + check(session->commit_transaction( + session, NULL)); ++tinfo->commit; intxn = 0; break; @@ -538,10 +514,8 @@ skip_insert: if (col_update(tinfo, if (0) { deadlock: ++tinfo->deadlock; } - if ((ret = session->rollback_transaction( - session, NULL)) != 0) - die(ret, - "session.rollback_transaction"); + check(session->rollback_transaction( + session, NULL)); ++tinfo->rollback; intxn = 0; break; @@ -550,8 +524,8 @@ deadlock: ++tinfo->deadlock; } } - if (session != NULL && (ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + if (session != NULL) + check(session->close(session, NULL)); free(keybuf); free(valbuf); @@ -581,12 +555,10 @@ wts_read_scan(void) key_gen_setup(&keybuf); /* Open a session and cursor pair. */ - if ((ret = conn->open_session( - conn, NULL, ops_session_config(NULL), &session)) != 0) - die(ret, "connection.open_session"); - if ((ret = session->open_cursor( - session, g.uri, NULL, NULL, &cursor)) != 0) - die(ret, "session.open_cursor"); + check(conn->open_session( + conn, NULL, ops_session_config(NULL), &session)); + check(session->open_cursor( + session, g.uri, NULL, NULL, &cursor)); /* Check a random subset of the records using the key. */ for (last_cnt = cnt = 0; cnt < g.key_cnt;) { @@ -603,8 +575,7 @@ wts_read_scan(void) die(ret, "read_scan"); } - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + check(session->close(session, NULL)); free(keybuf); } @@ -948,7 +919,7 @@ static void table_append(uint64_t keyno) { uint64_t *p, *ep; - int done, ret; + int done; ep = g.append + g.append_max; @@ -979,8 +950,7 @@ table_append(uint64_t keyno) * and we find a slot. */ for (done = 0;;) { - if ((ret = pthread_rwlock_wrlock(&g.append_lock)) != 0) - die(ret, "pthread_rwlock_wrlock: append_lock"); + check(pthread_rwlock_wrlock(&g.append_lock)); /* * If this is the thread we've been waiting for, and its record @@ -1017,8 +987,7 @@ table_append(uint64_t keyno) break; } - if ((ret = pthread_rwlock_unlock(&g.append_lock)) != 0) - die(ret, "pthread_rwlock_unlock: append_lock"); + check(pthread_rwlock_unlock(&g.append_lock)); if (done) break; diff --git a/test/format/rebalance.c b/test/format/rebalance.c index 8e8fa1a371f..0a89ce5f8e0 100644 --- a/test/format/rebalance.c +++ b/test/format/rebalance.c @@ -51,8 +51,7 @@ wts_rebalance(void) /* Rebalance, then verify the object. */ wts_reopen(); conn = g.wts_conn; - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + check(conn->open_session(conn, NULL, NULL, &session)); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, "=============== rebalance start ==============="); @@ -63,8 +62,7 @@ wts_rebalance(void) if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, "=============== rebalance stop ==============="); - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + check(session->close(session, NULL)); wts_verify("post-rebalance verify"); wts_close(); diff --git a/test/format/salvage.c b/test/format/salvage.c index d0358e998b4..54937d2fed0 100644 --- a/test/format/salvage.c +++ b/test/format/salvage.c @@ -42,12 +42,10 @@ salvage(void) conn = g.wts_conn; track("salvage", 0ULL, NULL); - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + check(conn->open_session(conn, NULL, NULL, &session)); if ((ret = session->salvage(session, g.uri, "force=true")) != 0) die(ret, "session.salvage: %s", g.uri); - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + check(session->close(session, NULL)); } /* diff --git a/test/format/t.c b/test/format/t.c index 0c0485c8bfe..e1ef0c18c68 100644 --- a/test/format/t.c +++ b/test/format/t.c @@ -40,7 +40,7 @@ int main(int argc, char *argv[]) { time_t start; - int ch, i, onerun, reps, ret; + int ch, i, onerun, reps; const char *config, *home; config = NULL; @@ -176,12 +176,9 @@ main(int argc, char *argv[]) * Initialize locks to single-thread named checkpoints and backups, last * last-record updates, and failures. */ - if ((ret = pthread_rwlock_init(&g.append_lock, NULL)) != 0) - die(ret, "pthread_rwlock_init: append lock"); - if ((ret = pthread_rwlock_init(&g.backup_lock, NULL)) != 0) - die(ret, "pthread_rwlock_init: backup lock"); - if ((ret = pthread_rwlock_init(&g.death_lock, NULL)) != 0) - die(ret, "pthread_rwlock_init: death lock"); + check(pthread_rwlock_init(&g.append_lock, NULL)); + check(pthread_rwlock_init(&g.backup_lock, NULL)); + check(pthread_rwlock_init(&g.death_lock, NULL)); printf("%s: process %" PRIdMAX "\n", g.progname, (intmax_t)getpid()); while (++g.run_cnt <= g.c_runs || g.c_runs == 0 ) { @@ -273,10 +270,8 @@ main(int argc, char *argv[]) config_print(0); - if ((ret = pthread_rwlock_destroy(&g.append_lock)) != 0) - die(ret, "pthread_rwlock_destroy: append lock"); - if ((ret = pthread_rwlock_destroy(&g.backup_lock)) != 0) - die(ret, "pthread_rwlock_destroy: backup lock"); + check(pthread_rwlock_destroy(&g.append_lock)); + check(pthread_rwlock_destroy(&g.backup_lock)); config_clear(); diff --git a/test/format/wts.c b/test/format/wts.c index 9d4d3fe5cb8..ee72dfff2d5 100644 --- a/test/format/wts.c +++ b/test/format/wts.c @@ -436,27 +436,23 @@ wts_create(void) /* * Create the underlying store. */ - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + check(conn->open_session(conn, NULL, NULL, &session)); if ((ret = session->create(session, g.uri, config)) != 0) die(ret, "session.create: %s", g.uri); - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + check(session->close(session, NULL)); } void wts_close(void) { WT_CONNECTION *conn; - int ret; const char *config; conn = g.wts_conn; config = g.c_leak_memory ? "leak_memory" : NULL; - if ((ret = conn->close(conn, config)) != 0) - die(ret, "connection.close"); + check(conn->close(conn, config)); g.wts_conn = NULL; g.wt_api = NULL; } @@ -513,8 +509,7 @@ wts_verify(const char *tag) conn = g.wts_conn; track("verify", 0ULL, NULL); - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + check(conn->open_session(conn, NULL, NULL, &session)); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, "=============== verify start ==============="); @@ -527,8 +522,7 @@ wts_verify(const char *tag) if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, "=============== verify stop ==============="); - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + check(session->close(session, NULL)); } /* @@ -558,17 +552,15 @@ wts_stats(void) conn = g.wts_conn; track("stat", 0ULL, NULL); - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session"); + check(conn->open_session(conn, NULL, NULL, &session)); if ((fp = fopen(g.home_stats, "w")) == NULL) die(errno, "fopen: %s", g.home_stats); /* Connection statistics. */ fprintf(fp, "====== Connection statistics:\n"); - if ((ret = session->open_cursor(session, - "statistics:", NULL, NULL, &cursor)) != 0) - die(ret, "session.open_cursor"); + check(session->open_cursor( + session, "statistics:", NULL, NULL, &cursor)); while ((ret = cursor->next(cursor)) == 0 && (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) @@ -577,16 +569,13 @@ wts_stats(void) if (ret != WT_NOTFOUND) die(ret, "cursor.next"); - if ((ret = cursor->close(cursor)) != 0) - die(ret, "cursor.close"); + check(cursor->close(cursor)); /* Data source statistics. */ fprintf(fp, "\n\n====== Data source statistics:\n"); stat_name = dmalloc(strlen("statistics:") + strlen(g.uri) + 1); sprintf(stat_name, "statistics:%s", g.uri); - if ((ret = session->open_cursor( - session, stat_name, NULL, NULL, &cursor)) != 0) - die(ret, "session.open_cursor"); + check(session->open_cursor(session, stat_name, NULL, NULL, &cursor)); free(stat_name); while ((ret = cursor->next(cursor)) == 0 && @@ -596,11 +585,9 @@ wts_stats(void) if (ret != WT_NOTFOUND) die(ret, "cursor.next"); - if ((ret = cursor->close(cursor)) != 0) - die(ret, "cursor.close"); + check(cursor->close(cursor)); fclose_and_clear(&fp); - if ((ret = session->close(session, NULL)) != 0) - die(ret, "session.close"); + check(session->close(session, NULL)); } -- cgit v1.2.1 From eaca8799dee9d645d6795b80bef0fadfcf47085d Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 18 Feb 2016 09:20:35 -0500 Subject: WT-2406: cursor_order lint, minor bug fixes Don't pass the address of the array to cursor.set_key. Check all routines that return an error status. Don't down-cast pointers to ints. --- test/cursor_order/cursor_order_file.c | 8 +++++--- test/cursor_order/cursor_order_ops.c | 38 ++++++++++++++++++++--------------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/test/cursor_order/cursor_order_file.c b/test/cursor_order/cursor_order_file.c index e5dd76fa1a1..5dc7194b5fb 100644 --- a/test/cursor_order/cursor_order_file.c +++ b/test/cursor_order/cursor_order_file.c @@ -86,7 +86,7 @@ load(SHARED_CONFIG *cfg, const char *name) for (keyno = 1; keyno <= (int64_t)cfg->nkeys; ++keyno) { if (cfg->ftype == ROW) { snprintf(keybuf, sizeof(keybuf), "%016u", (u_int)keyno); - cursor->set_key(cursor, &keybuf); + cursor->set_key(cursor, keybuf); } else cursor->set_key(cursor, (uint32_t)keyno); value->data = valuebuf; @@ -103,8 +103,10 @@ load(SHARED_CONFIG *cfg, const char *name) /* Setup the starting key range for the workload phase. */ cfg->key_range = cfg->nkeys; - cursor->close(cursor); - session->checkpoint(session, NULL); + if ((ret = cursor->close(cursor)) != 0) + testutil_die(ret, "cursor.close"); + if ((ret = session->checkpoint(session, NULL)) != 0) + testutil_die(ret, "session.checkpoint"); if ((ret = session->close(session, NULL)) != 0) testutil_die(ret, "session.close"); diff --git a/test/cursor_order/cursor_order_ops.c b/test/cursor_order/cursor_order_ops.c index 9077f500594..d44505ab2f3 100644 --- a/test/cursor_order/cursor_order_ops.c +++ b/test/cursor_order/cursor_order_ops.c @@ -171,11 +171,9 @@ static inline void reverse_scan_op( SHARED_CONFIG *cfg, WT_SESSION *session, WT_CURSOR *cursor, INFO *s) { - uint64_t i; + uint64_t i, initial_key_range, prev_key, this_key; int ret; char *strkey; - uint64_t initial_key_range; - uint64_t prev_key, this_key; WT_UNUSED(session); WT_UNUSED(s); @@ -184,7 +182,8 @@ reverse_scan_op( prev_key = this_key = 0; /* Reset the cursor */ - cursor->reset(cursor); + if ((ret = cursor->reset(cursor)) != 0) + testutil_die(ret, "cursor.reset"); /* Save the key range. */ initial_key_range = cfg->key_range - cfg->append_inserters; @@ -197,10 +196,13 @@ reverse_scan_op( } if (cfg->ftype == ROW) { - cursor->get_key(cursor, &strkey); + if ((ret = cursor->get_key(cursor, &strkey)) != 0) + testutil_die(ret, "cursor.get_key"); this_key = (uint64_t)atol(strkey); } else - cursor->get_key(cursor, (uint64_t*)&this_key); + if ((ret = cursor->get_key( + cursor, (uint64_t *)&this_key)) != 0) + testutil_die(ret, "cursor.get_key"); if (i == 0 && this_key < initial_key_range) testutil_die(ret, @@ -227,17 +229,19 @@ reverse_scan(void *arg) SHARED_CONFIG *cfg; WT_CURSOR *cursor; WT_SESSION *session; - int id, ret; - char tid[128]; + uintmax_t id; uint64_t i; + int ret; + char tid[128]; - id = (int)(uintptr_t)arg; + id = (uintmax_t)arg; s = &run_info[id]; cfg = s->cfg; __wt_thread_id(tid, sizeof(tid)); __wt_random_init(&s->rnd); - printf(" reverse scan thread %2d starting: tid: %s, file: %s\n", + printf(" reverse scan thread %2" PRIuMAX + " starting: tid: %s, file: %s\n", id, tid, s->name); __wt_yield(); /* Get all the threads created. */ @@ -254,7 +258,8 @@ reverse_scan(void *arg) if ((ret = session->close(session, NULL)) != 0) testutil_die(ret, "session.close"); - printf(" reverse scan thread %2d stopping: tid: %s, file: %s\n", + printf(" reverse scan thread %2" PRIuMAX + " stopping: tid: %s, file: %s\n", id, tid, s->name); /* Notify all other threads to finish once the first thread is done */ @@ -283,7 +288,7 @@ append_insert_op( keyno = __wt_atomic_add64(&cfg->key_range, 1); if (cfg->ftype == ROW) { snprintf(keybuf, sizeof(keybuf), "%016u", (u_int)keyno); - cursor->set_key(cursor, &keybuf); + cursor->set_key(cursor, keybuf); } else cursor->set_key(cursor, (uint32_t)keyno); @@ -311,17 +316,18 @@ append_insert(void *arg) SHARED_CONFIG *cfg; WT_CURSOR *cursor; WT_SESSION *session; + uintmax_t id; uint64_t i; - int id, ret; + int ret; char tid[128]; - id = (int)(uintptr_t)arg; + id = (uintmax_t)arg; s = &run_info[id]; cfg = s->cfg; __wt_thread_id(tid, sizeof(tid)); __wt_random_init(&s->rnd); - printf("write thread %2d starting: tid: %s, file: %s\n", + printf("write thread %2" PRIuMAX " starting: tid: %s, file: %s\n", id, tid, s->name); __wt_yield(); /* Get all the threads created. */ @@ -337,7 +343,7 @@ append_insert(void *arg) if ((ret = session->close(session, NULL)) != 0) testutil_die(ret, "session.close"); - printf("write thread %2d stopping: tid: %s, file: %s\n", + printf("write thread %2" PRIuMAX " stopping: tid: %s, file: %s\n", id, tid, s->name); /* Notify all other threads to finish once the first thread is done */ -- cgit v1.2.1 From afc75859956ca72ca51099031dcb424e3e974ad4 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 18 Feb 2016 09:42:24 -0500 Subject: WT-2407: recovery lint Check all functions for error returns. Minor const, #define cleanup. --- dist/s_string.ok | 2 ++ test/recovery/random-abort.c | 16 ++++++++++------ test/recovery/truncated-log.c | 21 +++++++++++++-------- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/dist/s_string.ok b/dist/s_string.ok index 26998d13214..91a9c1cdf46 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -432,6 +432,7 @@ cfg cfkos change's changelog +chdir checkpointed checkpointer checkpointing @@ -1060,6 +1061,7 @@ vsnprintf vtype vunpack vw +waitpid walk's warmup wb diff --git a/test/recovery/random-abort.c b/test/recovery/random-abort.c index ddcafbc80fd..b356ddc6379 100644 --- a/test/recovery/random-abort.c +++ b/test/recovery/random-abort.c @@ -42,7 +42,7 @@ static char home[512]; /* Program working dir */ static const char *progname; /* Program name */ -static const char *uri = "table:main"; +static const char * const uri = "table:main"; #define RECORDS_FILE "records" @@ -88,7 +88,8 @@ fill_db(void) /* * Run in the home directory so that the records file is in there too. */ - chdir(home); + if (chdir(home) != 0) + testutil_die(errno, "chdir: %s", home); if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG, &conn)) != 0) testutil_die(ret, "wiredtiger_open"); if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) @@ -109,7 +110,7 @@ fill_db(void) /* * Set to no buffering. */ - setvbuf(fp, NULL, _IONBF, 0); + (void)setvbuf(fp, NULL, _IONBF, 0); /* * Write data into the table until we are killed by the parent. @@ -201,13 +202,15 @@ main(int argc, char *argv[]) printf("Kill child\n"); if (kill(pid, SIGKILL) != 0) testutil_die(errno, "kill"); - waitpid(pid, &status, 0); + if (waitpid(pid, &status, 0) == -1) + testutil_die(errno, "waitpid"); /* * !!! If we wanted to take a copy of the directory before recovery, * this is the place to do it. */ - chdir(home); + if (chdir(home) != 0) + testutil_die(errno, "chdir: %s", home); printf("Open database, run recovery and verify content\n"); if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn)) != 0) testutil_die(ret, "wiredtiger_open"); @@ -239,7 +242,8 @@ main(int argc, char *argv[]) ++absent; } } - fclose(fp); + if (fclose(fp) != 0) + testutil_die(errno, "fclose"); if ((ret = conn->close(conn, NULL)) != 0) testutil_die(ret, "WT_CONNECTION:close"); if (absent) { diff --git a/test/recovery/truncated-log.c b/test/recovery/truncated-log.c index 4add7a61f66..d1046064c2f 100644 --- a/test/recovery/truncated-log.c +++ b/test/recovery/truncated-log.c @@ -45,7 +45,7 @@ static char home[512]; /* Program working dir */ static const char *progname; /* Program name */ -static const char *uri = "table:main"; +static const char * const uri = "table:main"; #define RECORDS_FILE "records" @@ -54,7 +54,6 @@ static const char *uri = "table:main"; "transaction_sync=(enabled,method=none)" #define ENV_CONFIG_REC "log=(recover=on)" #define LOG_FILE_1 "WiredTigerLog.0000000001" -#define MAX_VAL 4096 #define K_SIZE 16 #define V_SIZE 256 @@ -86,7 +85,8 @@ fill_db(void) /* * Run in the home directory so that the records file is in there too. */ - chdir(home); + if (chdir(home) != 0) + testutil_die(errno, "chdir: %s", home); if ((ret = wiredtiger_open(NULL, NULL, ENV_CONFIG, &conn)) != 0) testutil_die(ret, "wiredtiger_open"); if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) @@ -107,7 +107,7 @@ fill_db(void) /* * Set to no buffering. */ - setvbuf(fp, NULL, _IONBF, 0); + (void)setvbuf(fp, NULL, _IONBF, 0); save_lsn.l.file = 0; /* @@ -156,7 +156,8 @@ fill_db(void) "%" PRIu32 " %" PRIu32 "\n", save_lsn.l.offset, i - 1) == -1) testutil_die(errno, "fprintf"); - fclose(fp); + if (fclose(fp) != 0) + testutil_die(errno, "fclose"); abort(); } } @@ -218,20 +219,24 @@ main(int argc, char *argv[]) /* parent */ /* Wait for child to kill itself. */ - waitpid(pid, &status, 0); + if (waitpid(pid, &status, 0) == -1) + testutil_die(errno, "waitpid"); /* * !!! If we wanted to take a copy of the directory before recovery, * this is the place to do it. */ - chdir(home); + if (chdir(home) != 0) + testutil_die(errno, "chdir: %s", home); + printf("Open database, run recovery and verify content\n"); if ((fp = fopen(RECORDS_FILE, "r")) == NULL) testutil_die(errno, "fopen"); ret = fscanf(fp, "%" SCNu64 " %" SCNu32 "\n", &offset, &max_key); - fclose(fp); if (ret != 2) testutil_die(errno, "fscanf"); + if (fclose(fp) != 0) + testutil_die(errno, "fclose"); /* * The offset is the beginning of the last record. Truncate to * the middle of that last record (i.e. ahead of that offset). -- cgit v1.2.1 From 9f6073d27771473a026c9a683682ade7065977ab Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 18 Feb 2016 10:59:54 -0500 Subject: WT-2349 Make readonly option not supported in reconfigure. --- dist/api_data.py | 10 +++++----- src/config/config_def.c | 5 ++--- src/conn/conn_api.c | 3 ++- src/include/wiredtiger.in | 4 ---- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/dist/api_data.py b/dist/api_data.py index 1623c315710..0eee515e851 100644 --- a/dist/api_data.py +++ b/dist/api_data.py @@ -478,11 +478,6 @@ connection_runtime_config = [ vary depending on the current eviction load''', min=1, max=20), ]), - Config('readonly', 'false', r''' - open connection in read-only mode. The database must exist. All - methods that may modify a database are disabled. See @ref readonly - for more information''', - type='boolean'), Config('shared_cache', '', r''' shared cache configuration options. A database should configure either a cache_size or a shared_cache not both. Enabling a @@ -660,6 +655,11 @@ wiredtiger_open_common = connection_runtime_config + [ RPC server for primary processes and use RPC for secondary processes). Not yet supported in WiredTiger''', type='boolean'), + Config('readonly', 'false', r''' + open connection in read-only mode. The database must exist. All + methods that may modify a database are disabled. See @ref readonly + for more information''', + type='boolean'), Config('session_max', '100', r''' maximum expected number of sessions (including server threads)''', diff --git a/src/config/config_def.c b/src/config/config_def.c index 9f1110cd3d7..f1e48d85894 100644 --- a/src/config/config_def.c +++ b/src/config/config_def.c @@ -138,7 +138,6 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { NULL, NULL, confchk_wiredtiger_open_lsm_manager_subconfigs, 2 }, { "lsm_merge", "boolean", NULL, NULL, NULL, 0 }, - { "readonly", "boolean", NULL, NULL, NULL, 0 }, { "shared_cache", "category", NULL, NULL, confchk_wiredtiger_open_shared_cache_subconfigs, 5 }, @@ -856,12 +855,12 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",file_manager=(close_handle_minimum=250,close_idle_time=30," "close_scan_interval=10),log=(archive=,compressor=,enabled=0," "file_max=100MB,path=,prealloc=,recover=on,zero_fill=0)," - "lsm_manager=(merge=,worker_thread_max=4),lsm_merge=,readonly=0," + "lsm_manager=(merge=,worker_thread_max=4),lsm_merge=," "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," "statistics=none,statistics_log=(on_close=0," "path=\"WiredTigerStat.%d.%H\",sources=," "timestamp=\"%b %d %H:%M:%S\",wait=0),verbose=", - confchk_WT_CONNECTION_reconfigure, 19 + confchk_WT_CONNECTION_reconfigure, 18 }, { "WT_CURSOR.close", "", diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 40eb7e2b1a9..e510e147f88 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -1119,7 +1119,8 @@ __conn_config_readonly(const char *cfg[]) const char *readonly; /* - * Override certain settings. Other settings at odds will return + * Override certain settings. In general we override the options + * whose default conflicts. Other settings at odds will return * an error and will be checked when those settings are processed. */ readonly="checkpoint=(wait=0)," diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index 6c09983fead..91d3ea03617 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -1853,10 +1853,6 @@ struct __wt_connection { * thread uses a session handle from the configured session_max., an * integer between 3 and 20; default \c 4.} * @config{ ),,} - * @config{readonly, open connection in read-only mode. The database - * must exist. All methods that may modify a database are disabled. - * See @ref readonly for more information., a boolean flag; default \c - * false.} * @config{shared_cache = (, shared cache configuration options. A * database should configure either a cache_size or a shared_cache not * both. Enabling a shared cache uses a session from the configured -- cgit v1.2.1 From ca2e5946c95763c5c17fa6acc5f1addd3e98636e Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 18 Feb 2016 11:01:20 -0500 Subject: WT-2349 KNF --- src/os_win/os_errno.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/os_win/os_errno.c b/src/os_win/os_errno.c index e87d4c59c1b..590fcdc9d44 100644 --- a/src/os_win/os_errno.c +++ b/src/os_win/os_errno.c @@ -19,10 +19,11 @@ static const int windows_error_offset = -29000; static DWORD __wt_map_error_to_windows_error(int error) { - /* Ensure we do not exceed the error range - Also validate he do not get any COM errors - (which are negative integers) - */ + /* + * Ensure we do not exceed the error range + * Also validate we do not get any COM errors + * (which are negative integers) + */ WT_ASSERT(NULL, error < 0); return (error + -(windows_error_offset)); -- cgit v1.2.1 From 0c28585fb1ac06b92a936111edb8cdce09e3182f Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 18 Feb 2016 11:18:50 -0500 Subject: WT-2349 KNF --- src/session/session_api.c | 36 ++++++++++++++++++------------------ test/readonly/readonly.c | 5 ++--- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/session/session_api.c b/src/session/session_api.c index 106a2eb31ca..db1feabab45 100644 --- a/src/session/session_api.c +++ b/src/session/session_api.c @@ -1443,28 +1443,28 @@ __open_session(WT_CONNECTION_IMPL *conn, wt_session = &session_ret->iface; wt_session->checkpoint = (int (*)(WT_SESSION *, const char *))__wt_session_notsup; - wt_session->compact = - (int (*)(WT_SESSION *, const char *, const char *))__wt_session_notsup; - wt_session->create = - (int (*)(WT_SESSION *, const char *, const char *))__wt_session_notsup; - wt_session->drop = - (int (*)(WT_SESSION *, const char *, const char *))__wt_session_notsup; + wt_session->compact = (int (*)(WT_SESSION *, + const char *, const char *))__wt_session_notsup; + wt_session->create = (int (*)(WT_SESSION *, + const char *, const char *))__wt_session_notsup; + wt_session->drop = (int (*)(WT_SESSION *, + const char *, const char *))__wt_session_notsup; wt_session->log_flush = (int (*)(WT_SESSION *, const char *))__wt_session_notsup; - wt_session->log_printf = - (int (*)(WT_SESSION *, const char *, ...))__wt_session_notsup; - wt_session->rebalance = - (int (*)(WT_SESSION *, const char *, const char *))__wt_session_notsup; - wt_session->rename = (int (*)(WT_SESSION *, - const char *, const char *, const char *))__wt_session_notsup; - wt_session->salvage = - (int (*)(WT_SESSION *, const char *, const char *))__wt_session_notsup; + wt_session->log_printf = (int (*)( + WT_SESSION *, const char *, ...))__wt_session_notsup; + wt_session->rebalance = (int (*)(WT_SESSION *, + const char *, const char *))__wt_session_notsup; + wt_session->rename = (int (*)(WT_SESSION *, const char *, + const char *, const char *))__wt_session_notsup; + wt_session->salvage = (int (*)(WT_SESSION *, + const char *, const char *))__wt_session_notsup; wt_session->transaction_sync = (int (*)(WT_SESSION *, const char *))__wt_session_notsup; - wt_session->truncate = (int (*)(WT_SESSION *, - const char *, WT_CURSOR *, WT_CURSOR *, const char *))__wt_session_notsup; - wt_session->upgrade = - (int (*)(WT_SESSION *, const char *, const char *))__wt_session_notsup; + wt_session->truncate = (int (*)(WT_SESSION *, const char *, + WT_CURSOR *, WT_CURSOR *, const char *))__wt_session_notsup; + wt_session->upgrade = (int (*)(WT_SESSION *, + const char *, const char *))__wt_session_notsup; } WT_ERR(__wt_cond_alloc(session, "session", false, &session_ret->cond)); diff --git a/test/readonly/readonly.c b/test/readonly/readonly.c index 17cc78b8f56..6f74ee47c5a 100644 --- a/test/readonly/readonly.c +++ b/test/readonly/readonly.c @@ -47,7 +47,7 @@ static char home_rd[HOME_SIZE]; /* Read-only dir */ static char home_rd2[HOME_SIZE]; /* Read-only dir no lock file */ static const char *progname; /* Program name */ static const char *saved_argv0; /* Program command */ -static const char *uri = "table:main"; +static const char * const uri = "table:main"; #define ENV_CONFIG \ "create,log=(file_max=10M,archive=false,enabled)," \ @@ -221,14 +221,13 @@ main(int argc, char *argv[]) testutil_make_work_dir(home_wr); testutil_make_work_dir(home_rd); testutil_make_work_dir(home_rd2); - } else { + } else /* * We are a child process, we just want to call * the open_dbs with the directories we have. * The child function will exit. */ open_dbs(op, home, home_wr, home_rd, home_rd2); - } /* * Parent creates a database and table. Then cleanly shuts down. -- cgit v1.2.1 From 01e40180b00a1f216e3ce8bc55d7ccd07a2dbcd5 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 18 Feb 2016 18:02:07 -0500 Subject: WT-2395 Use allocation_size from metadata. If we find a file that needs to be created, use its metadata config in case it sets its own allocation size. --- src/meta/meta_turtle.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/meta/meta_turtle.c b/src/meta/meta_turtle.c index 3d27f0b5845..471bb65cac0 100644 --- a/src/meta/meta_turtle.c +++ b/src/meta/meta_turtle.c @@ -113,8 +113,9 @@ __metadata_load_bulk(WT_SESSION_IMPL *session) WT_DECL_RET; uint32_t allocsize; bool exist; - const char *filecfg[] = { WT_CONFIG_BASE(session, file_meta), NULL }; - const char *key; + const char *filecfg[] = { + WT_CONFIG_BASE(session, file_meta), NULL, NULL }; + const char *key, *value; /* * If a file was being bulk-loaded during the hot backup, it will appear @@ -135,6 +136,8 @@ __metadata_load_bulk(WT_SESSION_IMPL *session) * If the file doesn't exist, assume it's a bulk-loaded file; * retrieve the allocation size and re-create the file. */ + WT_ERR(cursor->get_value(cursor, &value)); + filecfg[1] = value; WT_ERR(__wt_direct_io_size_check( session, filecfg, "allocation_size", &allocsize)); WT_ERR(__wt_block_manager_create(session, key, allocsize)); -- cgit v1.2.1 From ec9a4510252a85e4d243d5025a4e2d5e5384981e Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Fri, 19 Feb 2016 01:44:58 +0000 Subject: WT-2405 Switch test/format to use shared testutil_die --- test/format/backup.c | 15 ++++++++------- test/format/bdb.c | 8 ++++---- test/format/compact.c | 2 +- test/format/config.c | 29 ++++++++++++++++------------- test/format/format.h | 8 ++------ test/format/lrt.c | 16 +++++++++------- test/format/ops.c | 42 +++++++++++++++++++++++------------------- test/format/rebalance.c | 8 ++++---- test/format/salvage.c | 16 ++++++++-------- test/format/t.c | 34 +++++++++++++--------------------- test/format/util.c | 12 ++++++------ test/format/wts.c | 34 ++++++++++++++++++---------------- test/utility/test_util.i | 17 ++++++++++++++++- 13 files changed, 128 insertions(+), 113 deletions(-) diff --git a/test/format/backup.c b/test/format/backup.c index f545930d992..8e2ad1b4883 100644 --- a/test/format/backup.c +++ b/test/format/backup.c @@ -43,14 +43,15 @@ check_copy(void) if ((ret = conn->open_session( conn, NULL, NULL, &session)) != 0) - die(ret, "connection.open_session: %s", g.home_backup); + testutil_die(ret, "connection.open_session: %s", g.home_backup); ret = session->verify(session, g.uri, NULL); if (ret != 0) - die(ret, "session.verify: %s: %s", g.home_backup, g.uri); + testutil_die(ret, + "session.verify: %s: %s", g.home_backup, g.uri); if ((ret = conn->close(conn, NULL)) != 0) - die(ret, "connection.close: %s", g.home_backup); + testutil_die(ret, "connection.close: %s", g.home_backup); } /* @@ -69,7 +70,7 @@ copy_file(const char *name) (void)snprintf(cmd, len, "cp %s/%s %s/%s", g.home, name, g.home_backup, name); if ((ret = system(cmd)) != 0) - die(ret, "backup copy: %s", cmd); + testutil_die(ret, "backup copy: %s", cmd); free(cmd); } @@ -116,7 +117,7 @@ backup(void *arg) /* Re-create the backup directory. */ if ((ret = system(g.home_backup_init)) != 0) - die(ret, "backup directory creation failed"); + testutil_die(ret, "backup directory creation failed"); /* * open_cursor can return EBUSY if a metadata operation is @@ -126,12 +127,12 @@ backup(void *arg) "backup:", NULL, NULL, &backup_cursor)) == EBUSY) sleep(1); if (ret != 0) - die(ret, "session.open_cursor: backup"); + testutil_die(ret, "session.open_cursor: backup"); while ((ret = backup_cursor->next(backup_cursor)) == 0) { if ((ret = backup_cursor->get_key(backup_cursor, &key)) != 0) - die(ret, "cursor.get_key"); + testutil_die(ret, "cursor.get_key"); copy_file(key); } diff --git a/test/format/bdb.c b/test/format/bdb.c index d7b4bca62f2..823fc8ff888 100644 --- a/test/format/bdb.c +++ b/test/format/bdb.c @@ -128,7 +128,7 @@ bdb_np(int next, if ((ret = dbc->get(dbc, &key, &value, next ? DB_NEXT : DB_PREV)) != 0) { if (ret != DB_NOTFOUND) - die(ret, "dbc.get: %s: {%.*s}", + testutil_die(ret, "dbc.get: %s: {%.*s}", next ? "DB_NEXT" : "DB_PREV", (int)key.size, (char *)key.data); *notfoundp = 1; @@ -154,7 +154,7 @@ bdb_read(uint64_t keyno, void *valuep, size_t *valuesizep, int *notfoundp) *notfoundp = 0; if ((ret = dbc->get(dbc, &key, &value, DB_SET)) != 0) { if (ret != DB_NOTFOUND) - die(ret, "dbc.get: DB_SET: {%.*s}", + testutil_die(ret, "dbc.get: DB_SET: {%.*s}", (int)key.size, (char *)key.data); *notfoundp = 1; } else { @@ -178,7 +178,7 @@ bdb_update(const void *arg_key, size_t arg_key_size, *notfoundp = 0; if ((ret = dbc->put(dbc, &key, &value, DB_KEYFIRST)) != 0) { if (ret != DB_NOTFOUND) { - die(ret, "dbc.put: DB_KEYFIRST: {%.*s}{%.*s}", + testutil_die(ret, "dbc.put: DB_KEYFIRST: {%.*s}{%.*s}", (int)key.size, (char *)key.data, (int)value.size, (char *)value.data); } @@ -204,7 +204,7 @@ bdb_remove(uint64_t keyno, int *notfoundp) if ((ret = dbc->del(dbc, 0)) != 0) { if (ret != DB_NOTFOUND) - die(ret, "dbc.del: {%.*s}", + testutil_die(ret, "dbc.del: {%.*s}", (int)key.size, (char *)key.data); *notfoundp = 1; } diff --git a/test/format/compact.c b/test/format/compact.c index a29fb5e3c4b..a3a67a35464 100644 --- a/test/format/compact.c +++ b/test/format/compact.c @@ -65,7 +65,7 @@ compact(void *arg) if ((ret = session->compact( session, g.uri, NULL)) != 0 && ret != WT_ROLLBACK) - die(ret, "session.compact"); + testutil_die(ret, "session.compact"); } check(session->close(session, NULL)); diff --git a/test/format/config.c b/test/format/config.c index d431546f254..042316d8344 100644 --- a/test/format/config.c +++ b/test/format/config.c @@ -138,9 +138,10 @@ config_setup(void) /* Required shared libraries. */ if (DATASOURCE("helium") && access(HELIUM_PATH, R_OK) != 0) - die(errno, "Levyx/helium shared library: %s", HELIUM_PATH); + testutil_die(errno, + "Levyx/helium shared library: %s", HELIUM_PATH); if (DATASOURCE("kvsbdb") && access(KVS_BDB_PATH, R_OK) != 0) - die(errno, "kvsbdb shared library: %s", KVS_BDB_PATH); + testutil_die(errno, "kvsbdb shared library: %s", KVS_BDB_PATH); /* Some data-sources don't support user-specified collations. */ if (DATASOURCE("helium") || DATASOURCE("kvsbdb")) @@ -199,14 +200,15 @@ config_setup(void) if (!config_is_perm("key_max") && g.c_key_max < g.c_key_min) g.c_key_max = g.c_key_min; if (g.c_key_min > g.c_key_max) - die(EINVAL, "key_min may not be larger than key_max"); + testutil_die(EINVAL, "key_min may not be larger than key_max"); if (!config_is_perm("value_min") && g.c_value_min > g.c_value_max) g.c_value_min = g.c_value_max; if (!config_is_perm("value_max") && g.c_value_max < g.c_value_min) g.c_value_max = g.c_value_min; if (g.c_value_min > g.c_value_max) - die(EINVAL, "value_min may not be larger than value_max"); + testutil_die(EINVAL, + "value_min may not be larger than value_max"); /* Reset the key count. */ g.key_cnt = 0; @@ -412,7 +414,7 @@ config_lrt(void) */ if (g.type == FIX) { if (g.c_long_running_txn && config_is_perm("long_running_txn")) - die(EINVAL, + testutil_die(EINVAL, "long_running_txn not supported with fixed-length " "column store"); g.c_long_running_txn = 0; @@ -453,7 +455,7 @@ config_print(int error_display) fp = stdout; else if ((fp = fopen(g.home_config, "w")) == NULL) - die(errno, "fopen: %s", g.home_config); + testutil_die(errno, "fopen: %s", g.home_config); fprintf(fp, "############################################\n"); fprintf(fp, "# RUN PARAMETERS\n"); @@ -487,7 +489,7 @@ config_file(const char *name) char *p, buf[256]; if ((fp = fopen(name, "r")) == NULL) - die(errno, "fopen: %s", name); + testutil_die(errno, "fopen: %s", name); while (fgets(buf, sizeof(buf), fp) != NULL) { for (p = buf; *p != '\0' && *p != '\n'; ++p) ; @@ -582,7 +584,7 @@ config_single(const char *s, int perm) *cp->vstr = strdup(ep); } if (*cp->vstr == NULL) - die(errno, "malloc"); + testutil_die(errno, "malloc"); return; } @@ -625,7 +627,7 @@ config_map_file_type(const char *s, u_int *vp) strcmp(s, "row-store") == 0) *vp = ROW; else - die(EINVAL, "illegal file type configuration: %s", s); + testutil_die(EINVAL, "illegal file type configuration: %s", s); } /* @@ -642,7 +644,7 @@ config_map_checksum(const char *s, u_int *vp) else if (strcmp(s, "uncompressed") == 0) *vp = CHECKSUM_UNCOMPRESSED; else - die(EINVAL, "illegal checksum configuration: %s", s); + testutil_die(EINVAL, "illegal checksum configuration: %s", s); } /* @@ -667,7 +669,8 @@ config_map_compression(const char *s, u_int *vp) else if (strcmp(s, "zlib-noraw") == 0) *vp = COMPRESS_ZLIB_NO_RAW; else - die(EINVAL, "illegal compression configuration: %s", s); + testutil_die(EINVAL, + "illegal compression configuration: %s", s); } /* @@ -682,7 +685,7 @@ config_map_encryption(const char *s, u_int *vp) else if (strcmp(s, "rotn-7") == 0) *vp = ENCRYPT_ROTN_7; else - die(EINVAL, "illegal encryption configuration: %s", s); + testutil_die(EINVAL, "illegal encryption configuration: %s", s); } /* @@ -701,7 +704,7 @@ config_map_isolation(const char *s, u_int *vp) else if (strcmp(s, "snapshot") == 0) *vp = ISOLATION_SNAPSHOT; else - die(EINVAL, "illegal isolation configuration: %s", s); + testutil_die(EINVAL, "illegal isolation configuration: %s", s); } /* diff --git a/test/format/format.h b/test/format/format.h index 7037da51e57..19b2e71d1ec 100644 --- a/test/format/format.h +++ b/test/format/format.h @@ -334,11 +334,7 @@ void wts_salvage(void); void wts_stats(void); void wts_verify(const char *); -void die(int, const char *, ...) -#if defined(__GNUC__) -__attribute__((__noreturn__)) -#endif -; +void format_die(void); /* * check -- @@ -347,7 +343,7 @@ __attribute__((__noreturn__)) #define check(call) do { \ int __r; \ if ((__r = (call)) != 0) \ - die(__r, "%s/%d: %s", __func__, __LINE__, #call); \ + testutil_die(__r, "%s/%d: %s", __func__, __LINE__, #call);\ } while (0) /* diff --git a/test/format/lrt.c b/test/format/lrt.c index 319eb0fb467..57d4119b62d 100644 --- a/test/format/lrt.c +++ b/test/format/lrt.c @@ -70,7 +70,8 @@ lrt(void *arg) &key, saved_keyno, 1)) == WT_ROLLBACK) ; if (ret != 0) - die(ret, "read_row %" PRIu64, saved_keyno); + testutil_die(ret, + "read_row %" PRIu64, saved_keyno); /* Compare the previous value with the current one. */ if (g.type == FIX) { @@ -80,12 +81,12 @@ lrt(void *arg) } else ret = cursor->get_value(cursor, &value); if (ret != 0) - die(ret, + testutil_die(ret, "cursor.get_value: %" PRIu64, saved_keyno); if (buf_size != value.size || memcmp(buf, value.data, value.size) != 0) - die(0, "mismatched start/stop values"); + testutil_die(0, "mismatched start/stop values"); /* End the transaction. */ check(session->commit_transaction(session, NULL)); @@ -113,7 +114,8 @@ lrt(void *arg) ; } while (ret == WT_NOTFOUND); if (ret != 0) - die(ret, "read_row %" PRIu64, saved_keyno); + testutil_die(ret, + "read_row %" PRIu64, saved_keyno); /* Copy the cursor's value. */ if (g.type == FIX) { @@ -123,11 +125,11 @@ lrt(void *arg) } else ret = cursor->get_value(cursor, &value); if (ret != 0) - die(ret, + testutil_die(ret, "cursor.get_value: %" PRIu64, saved_keyno); if (buf_len < value.size && (buf = realloc(buf, buf_len = value.size)) == NULL) - die(errno, "malloc"); + testutil_die(errno, "malloc"); memcpy(buf, value.data, buf_size = value.size); /* @@ -142,7 +144,7 @@ lrt(void *arg) ; } while (ret == WT_NOTFOUND); if (ret != 0) - die(ret, "read_row %" PRIu64, keyno); + testutil_die(ret, "read_row %" PRIu64, keyno); pinned = 1; } diff --git a/test/format/ops.c b/test/format/ops.c index 7fac11a12b7..bafe20e7afd 100644 --- a/test/format/ops.c +++ b/test/format/ops.c @@ -104,7 +104,7 @@ wts_ops(int lastrun) /* Create thread structure; start the worker threads. */ if ((tinfo = calloc((size_t)g.c_threads, sizeof(*tinfo))) == NULL) - die(errno, "calloc"); + testutil_die(errno, "calloc"); for (i = 0; i < g.c_threads; ++i) { tinfo[i].id = (int)i + 1; tinfo[i].state = TINFO_RUNNING; @@ -350,7 +350,7 @@ ops(void *arg) if ((ret = session->checkpoint(session, ckpt_config)) != 0) - die(ret, "session.checkpoint%s%s", + testutil_die(ret, "session.checkpoint%s%s", ckpt_config == NULL ? "" : ": ", ckpt_config == NULL ? "" : ckpt_config); @@ -572,7 +572,7 @@ wts_read_scan(void) key.data = keybuf; if ((ret = read_row(cursor, &key, cnt, 0)) != 0) - die(ret, "read_scan"); + testutil_die(ret, "read_scan"); } check(session->close(session, NULL)); @@ -637,7 +637,7 @@ read_row(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int notfound_err) return (WT_NOTFOUND); break; default: - die(ret, "read_row: read row %" PRIu64, keyno); + testutil_die(ret, "read_row: read row %" PRIu64, keyno); } #ifdef HAVE_BERKELEY_DB @@ -674,7 +674,7 @@ read_row(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int notfound_err) "read_row: value mismatch %" PRIu64 ":\n", keyno); print_item("bdb", &bdb_value); print_item(" wt", &value); - die(0, NULL); + testutil_die(0, NULL); } } #endif @@ -719,7 +719,7 @@ nextprev(WT_CURSOR *cursor, int next, int *notfoundp) break; } if (ret != 0 && ret != WT_NOTFOUND) - die(ret, "%s", which); + testutil_die(ret, "%s", which); *notfoundp = (ret == WT_NOTFOUND); #ifdef HAVE_BERKELEY_DB @@ -748,7 +748,7 @@ nextprev(WT_CURSOR *cursor, int next, int *notfoundp) fprintf(stderr, "nextprev: %s key mismatch:\n", which); print_item("bdb-key", &bdb_key); print_item(" wt-key", &key); - die(0, NULL); + testutil_die(0, NULL); } } else { if (keyno != (uint64_t)atoll(bdb_key.data)) { @@ -758,7 +758,7 @@ nextprev(WT_CURSOR *cursor, int next, int *notfoundp) "nextprev: %s key mismatch: %.*s != %" PRIu64 "\n", which, (int)bdb_key.size, (char *)bdb_key.data, keyno); - die(0, NULL); + testutil_die(0, NULL); } } if (value.size != bdb_value.size || @@ -766,7 +766,7 @@ nextprev(WT_CURSOR *cursor, int next, int *notfoundp) fprintf(stderr, "nextprev: %s value mismatch:\n", which); print_item("bdb-value", &bdb_value); print_item(" wt-value", &value); - die(0, NULL); + testutil_die(0, NULL); } if (g.logging == LOG_OPS) @@ -822,7 +822,8 @@ row_update(TINFO *tinfo, if (ret == WT_ROLLBACK) return (WT_ROLLBACK); if (ret != 0 && ret != WT_NOTFOUND) - die(ret, "row_update: update row %" PRIu64 " by key", keyno); + testutil_die(ret, + "row_update: update row %" PRIu64 " by key", keyno); #ifdef HAVE_BERKELEY_DB if (!SINGLETHREADED) @@ -876,7 +877,7 @@ col_update(TINFO *tinfo, if (ret == WT_ROLLBACK) return (WT_ROLLBACK); if (ret != 0 && ret != WT_NOTFOUND) - die(ret, "col_update: %" PRIu64, keyno); + testutil_die(ret, "col_update: %" PRIu64, keyno); #ifdef HAVE_BERKELEY_DB if (!SINGLETHREADED) @@ -908,7 +909,7 @@ table_append_init(void) free(g.append); if ((g.append = calloc(g.append_max, sizeof(uint64_t))) == NULL) - die(errno, "calloc"); + testutil_die(errno, "calloc"); } /* @@ -1024,7 +1025,8 @@ row_insert(TINFO *tinfo, if (ret == WT_ROLLBACK) return (WT_ROLLBACK); if (ret != 0 && ret != WT_NOTFOUND) - die(ret, "row_insert: insert row %" PRIu64 " by key", keyno); + testutil_die(ret, + "row_insert: insert row %" PRIu64 " by key", keyno); #ifdef HAVE_BERKELEY_DB if (!SINGLETHREADED) @@ -1063,10 +1065,10 @@ col_insert(TINFO *tinfo, if ((ret = cursor->insert(cursor)) != 0) { if (ret == WT_ROLLBACK) return (WT_ROLLBACK); - die(ret, "cursor.insert"); + testutil_die(ret, "cursor.insert"); } if ((ret = cursor->get_key(cursor, &keyno)) != 0) - die(ret, "cursor.get_key"); + testutil_die(ret, "cursor.get_key"); *keynop = (uint32_t)keyno; table_append(keyno); /* Extend the object. */ @@ -1126,7 +1128,8 @@ row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int *notfoundp) if (ret == WT_ROLLBACK) return (WT_ROLLBACK); if (ret != 0 && ret != WT_NOTFOUND) - die(ret, "row_remove: remove %" PRIu64 " by key", keyno); + testutil_die(ret, + "row_remove: remove %" PRIu64 " by key", keyno); *notfoundp = (ret == WT_NOTFOUND); #ifdef HAVE_BERKELEY_DB @@ -1169,7 +1172,8 @@ col_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int *notfoundp) if (ret == WT_ROLLBACK) return (WT_ROLLBACK); if (ret != 0 && ret != WT_NOTFOUND) - die(ret, "col_remove: remove %" PRIu64 " by key", keyno); + testutil_die(ret, + "col_remove: remove %" PRIu64 " by key", keyno); *notfoundp = (ret == WT_NOTFOUND); #ifdef HAVE_BERKELEY_DB @@ -1214,7 +1218,7 @@ notfound_chk(const char *f, int wt_ret, int bdb_notfound, uint64_t keyno) fprintf(stderr, " row %" PRIu64 ":", keyno); fprintf(stderr, " not found in Berkeley DB, found in WiredTiger\n"); - die(0, NULL); + testutil_die(0, NULL); } if (wt_ret == WT_NOTFOUND) { fprintf(stderr, "%s: %s:", g.progname, f); @@ -1222,7 +1226,7 @@ notfound_chk(const char *f, int wt_ret, int bdb_notfound, uint64_t keyno) fprintf(stderr, " row %" PRIu64 ":", keyno); fprintf(stderr, " found in Berkeley DB, not found in WiredTiger\n"); - die(0, NULL); + testutil_die(0, NULL); } return (0); } diff --git a/test/format/rebalance.c b/test/format/rebalance.c index 0a89ce5f8e0..c448413fd00 100644 --- a/test/format/rebalance.c +++ b/test/format/rebalance.c @@ -46,7 +46,7 @@ wts_rebalance(void) "../../wt -h %s dump -f %s/rebalance.orig %s", g.home, g.home, g.uri); if ((ret = system(cmd)) != 0) - die(ret, "command failed: %s", cmd); + testutil_die(ret, "command failed: %s", cmd); /* Rebalance, then verify the object. */ wts_reopen(); @@ -57,7 +57,7 @@ wts_rebalance(void) "=============== rebalance start ==============="); if ((ret = session->rebalance(session, g.uri, NULL)) != 0) - die(ret, "session.rebalance: %s: %s", g.uri); + testutil_die(ret, "session.rebalance: %s: %s", g.uri); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, @@ -71,12 +71,12 @@ wts_rebalance(void) "../../wt -h %s dump -f %s/rebalance.new %s", g.home, g.home, g.uri); if ((ret = system(cmd)) != 0) - die(ret, "command failed: %s", cmd); + testutil_die(ret, "command failed: %s", cmd); /* Compare the old/new versions of the object. */ (void)snprintf(cmd, sizeof(cmd), "cmp %s/rebalance.orig %s/rebalance.new > /dev/null", g.home, g.home); if ((ret = system(cmd)) != 0) - die(ret, "command failed: %s", cmd); + testutil_die(ret, "command failed: %s", cmd); } diff --git a/test/format/salvage.c b/test/format/salvage.c index 54937d2fed0..6036f25df39 100644 --- a/test/format/salvage.c +++ b/test/format/salvage.c @@ -44,7 +44,7 @@ salvage(void) check(conn->open_session(conn, NULL, NULL, &session)); if ((ret = session->salvage(session, g.uri, "force=true")) != 0) - die(ret, "session.salvage: %s", g.uri); + testutil_die(ret, "session.salvage: %s", g.uri); check(session->close(session, NULL)); } @@ -99,37 +99,37 @@ corrupt(void) return (0); found: if (fstat(fd, &sb) == -1) - die(errno, "salvage-corrupt: fstat"); + testutil_die(errno, "salvage-corrupt: fstat"); offset = mmrand(NULL, 0, (u_int)sb.st_size); len = (size_t)(20 + (sb.st_size / 100) * 2); (void)snprintf(buf, sizeof(buf), "%s/slvg.corrupt", g.home); if ((fp = fopen(buf, "w")) == NULL) - die(errno, "salvage-corrupt: open: %s", buf); + testutil_die(errno, "salvage-corrupt: open: %s", buf); (void)fprintf(fp, "salvage-corrupt: offset %" PRIuMAX ", length " SIZET_FMT "\n", (uintmax_t)offset, len); fclose_and_clear(&fp); if (lseek(fd, offset, SEEK_SET) == -1) - die(errno, "salvage-corrupt: lseek"); + testutil_die(errno, "salvage-corrupt: lseek"); memset(buf, 'z', sizeof(buf)); for (; len > 0; len -= nw) { nw = (size_t)(len > sizeof(buf) ? sizeof(buf) : len); if (write(fd, buf, nw) == -1) - die(errno, "salvage-corrupt: write"); + testutil_die(errno, "salvage-corrupt: write"); } if (close(fd) == -1) - die(errno, "salvage-corrupt: close"); + testutil_die(errno, "salvage-corrupt: close"); /* * Save a copy of the corrupted file so we can replay the salvage step * as necessary. */ if ((ret = system(copycmd)) != 0) - die(ret, "salvage corrupt copy step failed"); + testutil_die(ret, "salvage corrupt copy step failed"); return (1); } @@ -155,7 +155,7 @@ wts_salvage(void) * step as necessary. */ if ((ret = system(g.home_salvage_copy)) != 0) - die(ret, "salvage copy step failed"); + testutil_die(ret, "salvage copy step failed"); /* Salvage, then verify. */ wts_open(g.home, 1, &g.wts_conn); diff --git a/test/format/t.c b/test/format/t.c index e1ef0c18c68..f2409290acb 100644 --- a/test/format/t.c +++ b/test/format/t.c @@ -45,6 +45,8 @@ main(int argc, char *argv[]) config = NULL; + /* Register a fatal cleanup handler */ + custom_die = format_die; #ifdef _WIN32 g.progname = "t_format.exe"; #else @@ -125,9 +127,9 @@ main(int argc, char *argv[]) /* If it's a replay, use the home directory's CONFIG file. */ if (g.replay) { if (config != NULL) - die(EINVAL, "-c incompatible with -r"); + testutil_die(EINVAL, "-c incompatible with -r"); if (access(g.home_config, R_OK) != 0) - die(ENOENT, "%s", g.home_config); + testutil_die(ENOENT, "%s", g.home_config); config = g.home_config; } @@ -293,27 +295,28 @@ startup(void) /* Create or initialize the home and data-source directories. */ if ((ret = system(g.home_init)) != 0) - die(ret, "home directory initialization failed"); + testutil_die(ret, "home directory initialization failed"); /* Open/truncate the logging file. */ if (g.logging != 0 && (g.logfp = fopen(g.home_log, "w")) == NULL) - die(errno, "fopen: %s", g.home_log); + testutil_die(errno, "fopen: %s", g.home_log); /* Open/truncate the random number logging file. */ if ((g.randfp = fopen(g.home_rand, g.replay ? "r" : "w")) == NULL) - die(errno, "%s", g.home_rand); + testutil_die(errno, "%s", g.home_rand); } /* * die -- - * Report an error and quit, dumping the configuration. + * Report an error, dumping the configuration. */ void -die(int e, const char *fmt, ...) +format_die(void) { - va_list ap; - - /* Single-thread error handling. */ + /* + * Single-thread error handling, our caller exits after calling + * us - don't release the lock. + */ (void)pthread_rwlock_wrlock(&g.death_lock); /* Try and turn off tracking so it doesn't obscure the error message. */ @@ -321,15 +324,6 @@ die(int e, const char *fmt, ...) g.c_quiet = 1; fprintf(stderr, "\n"); } - if (fmt != NULL) { /* Death message. */ - fprintf(stderr, "%s: ", g.progname); - va_start(ap, fmt); - vfprintf(stderr, fmt, ap); - va_end(ap); - if (e != 0) - fprintf(stderr, ": %s", wiredtiger_strerror(e)); - fprintf(stderr, "\n"); - } /* Flush/close any logging information. */ fclose_and_clear(&g.logfp); @@ -338,8 +332,6 @@ die(int e, const char *fmt, ...) /* Display the configuration that failed. */ if (g.run_cnt) config_print(1); - - exit(EXIT_FAILURE); } /* diff --git a/test/format/util.c b/test/format/util.c index 82a6de97ab6..347b2ea1db3 100644 --- a/test/format/util.c +++ b/test/format/util.c @@ -42,7 +42,7 @@ dmalloc(size_t len) void *p; if ((p = malloc(len)) == NULL) - die(errno, "malloc"); + testutil_die(errno, "malloc"); return (p); } @@ -56,7 +56,7 @@ dstrdup(const char *str) char *p; if ((p = strdup(str)) == NULL) - die(errno, "strdup"); + testutil_die(errno, "strdup"); return (p); } @@ -268,9 +268,9 @@ track(const char *tag, uint64_t cnt, TINFO *tinfo) lastlen = len; if (printf("%s\r", msg) < 0) - die(EIO, "printf"); + testutil_die(EIO, "printf"); if (fflush(stdout) == EOF) - die(errno, "fflush"); + testutil_die(errno, "fflush"); } /* @@ -407,7 +407,7 @@ rng(WT_RAND_STATE *rnd) "\n" "end of random number log reached\n"); exit(EXIT_SUCCESS); } - die(errno, "random number log"); + testutil_die(errno, "random number log"); } return ((uint32_t)strtoul(buf, NULL, 10)); @@ -435,6 +435,6 @@ fclose_and_clear(FILE **fpp) return; *fpp = NULL; if (fclose(fp) != 0) - die(errno, "fclose"); + testutil_die(errno, "fclose"); return; } diff --git a/test/format/wts.c b/test/format/wts.c index ee72dfff2d5..3c3d03195fa 100644 --- a/test/format/wts.c +++ b/test/format/wts.c @@ -53,7 +53,7 @@ compressor(uint32_t compress_flag) default: break; } - die(EINVAL, "illegal compression flag: 0x%x", compress_flag); + testutil_die(EINVAL, "illegal compression flag: 0x%x", compress_flag); } /* @@ -71,7 +71,7 @@ encryptor(uint32_t encrypt_flag) default: break; } - die(EINVAL, "illegal encryption flag: 0x%x", encrypt_flag); + testutil_die(EINVAL, "illegal encryption flag: 0x%x", encrypt_flag); } static int @@ -222,7 +222,8 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp) p += snprintf(p, REMAIN(p, end), ",%s", g.config_open); if (REMAIN(p, end) == 0) - die(ENOMEM, "wiredtiger_open configuration buffer too small"); + testutil_die(ENOMEM, + "wiredtiger_open configuration buffer too small"); /* * Direct I/O may not work with backups, doing copies through the buffer @@ -234,7 +235,7 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp) g.c_backups = 0; if ((ret = wiredtiger_open(home, &event_handler, config, &conn)) != 0) - die(ret, "wiredtiger_open: %s", home); + testutil_die(ret, "wiredtiger_open: %s", home); if (set_api) g.wt_api = conn->get_extension_api(conn); @@ -247,7 +248,7 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp) */ if (DATASOURCE("helium")) { if (g.helium_mount == NULL) - die(EINVAL, "no Helium mount point specified"); + testutil_die(EINVAL, "no Helium mount point specified"); (void)snprintf(helium_config, sizeof(helium_config), "entry=wiredtiger_extension_init,config=[" "helium_verbose=0," @@ -256,7 +257,7 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp) g.helium_mount); if ((ret = conn->load_extension( conn, HELIUM_PATH, helium_config)) != 0) - die(ret, + testutil_die(ret, "WT_CONNECTION.load_extension: %s:%s", HELIUM_PATH, helium_config); } @@ -274,7 +275,7 @@ wts_reopen(void) if ((ret = wiredtiger_open(g.home, &event_handler, g.wiredtiger_open_config, &g.wts_conn)) != 0) - die(ret, "wiredtiger_open: %s", g.home); + testutil_die(ret, "wiredtiger_open: %s", g.home); } /* @@ -431,14 +432,15 @@ wts_create(void) } if (REMAIN(p, end) == 0) - die(ENOMEM, "WT_SESSION.create configuration buffer too small"); + testutil_die(ENOMEM, + "WT_SESSION.create configuration buffer too small"); /* * Create the underlying store. */ check(conn->open_session(conn, NULL, NULL, &session)); if ((ret = session->create(session, g.uri, config)) != 0) - die(ret, "session.create: %s", g.uri); + testutil_die(ret, "session.create: %s", g.uri); check(session->close(session, NULL)); } @@ -488,7 +490,7 @@ wts_dump(const char *tag, int dump_bdb) g.uri == NULL ? "" : g.uri); if ((ret = system(cmd)) != 0) - die(ret, "%s: dump comparison failed", tag); + testutil_die(ret, "%s: dump comparison failed", tag); free(cmd); #else (void)tag; /* [-Wunused-variable] */ @@ -517,7 +519,7 @@ wts_verify(const char *tag) /* Session operations for LSM can return EBUSY. */ ret = session->verify(session, g.uri, "strict"); if (ret != 0 && !(ret == EBUSY && DATASOURCE("lsm"))) - die(ret, "session.verify: %s: %s", g.uri, tag); + testutil_die(ret, "session.verify: %s: %s", g.uri, tag); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, @@ -555,7 +557,7 @@ wts_stats(void) check(conn->open_session(conn, NULL, NULL, &session)); if ((fp = fopen(g.home_stats, "w")) == NULL) - die(errno, "fopen: %s", g.home_stats); + testutil_die(errno, "fopen: %s", g.home_stats); /* Connection statistics. */ fprintf(fp, "====== Connection statistics:\n"); @@ -565,10 +567,10 @@ wts_stats(void) while ((ret = cursor->next(cursor)) == 0 && (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) if (fprintf(fp, "%s=%s\n", desc, pval) < 0) - die(errno, "fprintf"); + testutil_die(errno, "fprintf"); if (ret != WT_NOTFOUND) - die(ret, "cursor.next"); + testutil_die(ret, "cursor.next"); check(cursor->close(cursor)); /* Data source statistics. */ @@ -581,10 +583,10 @@ wts_stats(void) while ((ret = cursor->next(cursor)) == 0 && (ret = cursor->get_value(cursor, &desc, &pval, &v)) == 0) if (fprintf(fp, "%s=%s\n", desc, pval) < 0) - die(errno, "fprintf"); + testutil_die(errno, "fprintf"); if (ret != WT_NOTFOUND) - die(ret, "cursor.next"); + testutil_die(ret, "cursor.next"); check(cursor->close(cursor)); fclose_and_clear(&fp); diff --git a/test/utility/test_util.i b/test/utility/test_util.i index 3b88d375381..de9424115d0 100644 --- a/test/utility/test_util.i +++ b/test/utility/test_util.i @@ -42,21 +42,36 @@ #define DEFAULT_DIR "WT_TEST" #define MKDIR_COMMAND "mkdir " +/* Setup a function pointer so tests can override the content of die. */ +typedef void (*die_func)(void); +die_func custom_die; + +static void testutil_die(int, const char *, ...) +#if defined(__GNUC__) +__attribute__((__noreturn__)) +#endif +; + /* * die -- * Report an error and quit. */ -static inline void +static void testutil_die(int e, const char *fmt, ...) { va_list ap; + /* Allow test programs to cleanup on fatal error. */ + if (custom_die != NULL) + (*custom_die)(); + va_start(ap, fmt); vfprintf(stderr, fmt, ap); va_end(ap); if (e != 0) fprintf(stderr, ": %s", wiredtiger_strerror(e)); fprintf(stderr, "\n"); + exit(EXIT_FAILURE); } -- cgit v1.2.1 From 2b4cd8de33de080348eaac967609dd6ace44f0d0 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Fri, 19 Feb 2016 01:54:23 +0000 Subject: WT-2405 Switch new testing error handler to be shared in test util --- test/format/backup.c | 10 ++++---- test/format/bulk.c | 10 ++++---- test/format/compact.c | 4 ++-- test/format/format.h | 13 +---------- test/format/lrt.c | 14 +++++++----- test/format/ops.c | 59 +++++++++++++++++++++++++++--------------------- test/format/rebalance.c | 4 ++-- test/format/salvage.c | 4 ++-- test/format/t.c | 10 ++++---- test/format/wts.c | 23 ++++++++++--------- test/utility/test_util.i | 10 ++++++++ 11 files changed, 85 insertions(+), 76 deletions(-) diff --git a/test/format/backup.c b/test/format/backup.c index 8e2ad1b4883..ea6306ccb27 100644 --- a/test/format/backup.c +++ b/test/format/backup.c @@ -97,7 +97,7 @@ backup(void *arg) return (NULL); /* Open a session. */ - check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); /* * Perform a backup at somewhere under 10 seconds (so we get at @@ -113,7 +113,7 @@ backup(void *arg) break; /* Lock out named checkpoints */ - check(pthread_rwlock_wrlock(&g.backup_lock)); + testutil_check(pthread_rwlock_wrlock(&g.backup_lock)); /* Re-create the backup directory. */ if ((ret = system(g.home_backup_init)) != 0) @@ -136,13 +136,13 @@ backup(void *arg) copy_file(key); } - check(backup_cursor->close(backup_cursor)); - check(pthread_rwlock_unlock(&g.backup_lock)); + testutil_check(backup_cursor->close(backup_cursor)); + testutil_check(pthread_rwlock_unlock(&g.backup_lock)); check_copy(); } - check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); return (NULL); } diff --git a/test/format/bulk.c b/test/format/bulk.c index b344403f06f..64b005d294f 100644 --- a/test/format/bulk.c +++ b/test/format/bulk.c @@ -41,7 +41,7 @@ wts_load(void) conn = g.wts_conn; keybuf = valbuf = NULL; - check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, @@ -59,7 +59,7 @@ wts_load(void) if (g.c_reverse) is_bulk = false; - check(session->open_cursor(session, g.uri, NULL, + testutil_check(session->open_cursor(session, g.uri, NULL, is_bulk ? "bulk,append" : NULL, &cursor)); /* Set up the key/value buffers. */ @@ -117,7 +117,7 @@ wts_load(void) break; } - check(cursor->insert(cursor)); + testutil_check(cursor->insert(cursor)); #ifdef HAVE_BERKELEY_DB if (SINGLETHREADED) @@ -125,13 +125,13 @@ wts_load(void) #endif } - check(cursor->close(cursor)); + testutil_check(cursor->close(cursor)); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, "=============== bulk load stop ==============="); - check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); free(keybuf); free(valbuf); diff --git a/test/format/compact.c b/test/format/compact.c index a3a67a35464..a75ee4f2adf 100644 --- a/test/format/compact.c +++ b/test/format/compact.c @@ -48,7 +48,7 @@ compact(void *arg) /* Open a session. */ conn = g.wts_conn; - check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); /* * Perform compaction at somewhere under 15 seconds (so we get at @@ -68,7 +68,7 @@ compact(void *arg) testutil_die(ret, "session.compact"); } - check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); return (NULL); } diff --git a/test/format/format.h b/test/format/format.h index 19b2e71d1ec..b32877636c8 100644 --- a/test/format/format.h +++ b/test/format/format.h @@ -310,6 +310,7 @@ void config_single(const char *, int); void *dmalloc(size_t); char *dstrdup(const char *); void fclose_and_clear(FILE **); +void format_die(void); void key_gen(uint8_t *, size_t *, uint64_t); void key_gen_insert(WT_RAND_STATE *, uint8_t *, size_t *, uint64_t); void key_gen_setup(uint8_t **); @@ -334,18 +335,6 @@ void wts_salvage(void); void wts_stats(void); void wts_verify(const char *); -void format_die(void); - -/* - * check -- - * Complain and quit if a function call fails. - */ -#define check(call) do { \ - int __r; \ - if ((__r = (call)) != 0) \ - testutil_die(__r, "%s/%d: %s", __func__, __LINE__, #call);\ -} while (0) - /* * mmrand -- * Return a random value between a min/max pair. diff --git a/test/format/lrt.c b/test/format/lrt.c index 57d4119b62d..451d2f4fa3c 100644 --- a/test/format/lrt.c +++ b/test/format/lrt.c @@ -60,8 +60,9 @@ lrt(void *arg) /* Open a session and cursor. */ conn = g.wts_conn; - check(conn->open_session(conn, NULL, NULL, &session)); - check(session->open_cursor(session, g.uri, NULL, NULL, &cursor)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->open_cursor( + session, g.uri, NULL, NULL, &cursor)); for (pinned = 0;;) { if (pinned) { @@ -89,10 +90,11 @@ lrt(void *arg) testutil_die(0, "mismatched start/stop values"); /* End the transaction. */ - check(session->commit_transaction(session, NULL)); + testutil_check( + session->commit_transaction(session, NULL)); /* Reset the cursor, releasing our pin. */ - check(cursor->reset(cursor)); + testutil_check(cursor->reset(cursor)); pinned = 0; } else { /* @@ -101,7 +103,7 @@ lrt(void *arg) * positioned. As soon as the cursor loses its position * a new snapshot will be allocated. */ - check(session->begin_transaction( + testutil_check(session->begin_transaction( session, "isolation=snapshot")); /* Read a record at the end of the table. */ @@ -161,7 +163,7 @@ lrt(void *arg) break; } - check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); free(keybuf); free(buf); diff --git a/test/format/ops.c b/test/format/ops.c index bafe20e7afd..d248736bb57 100644 --- a/test/format/ops.c +++ b/test/format/ops.c @@ -97,7 +97,7 @@ wts_ops(int lastrun) /* Open a session. */ if (g.logging != 0) { - check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); (void)g.wt_api->msg_printf(g.wt_api, session, "=============== thread ops start ==============="); } @@ -108,7 +108,8 @@ wts_ops(int lastrun) for (i = 0; i < g.c_threads; ++i) { tinfo[i].id = (int)i + 1; tinfo[i].state = TINFO_RUNNING; - check(pthread_create(&tinfo[i].tid, NULL, ops, &tinfo[i])); + testutil_check( + pthread_create(&tinfo[i].tid, NULL, ops, &tinfo[i])); } /* @@ -116,11 +117,12 @@ wts_ops(int lastrun) * long-running reader threads. */ if (g.c_backups) - check(pthread_create(&backup_tid, NULL, backup, NULL)); + testutil_check(pthread_create(&backup_tid, NULL, backup, NULL)); if (g.c_compact) - check(pthread_create(&compact_tid, NULL, compact, NULL)); + testutil_check( + pthread_create(&compact_tid, NULL, compact, NULL)); if (!SINGLETHREADED && g.c_long_running_txn) - check(pthread_create(&lrt_tid, NULL, lrt, NULL)); + testutil_check(pthread_create(&lrt_tid, NULL, lrt, NULL)); /* Spin on the threads, calculating the totals. */ for (;;) { @@ -186,7 +188,7 @@ wts_ops(int lastrun) if (g.logging != 0) { (void)g.wt_api->msg_printf(g.wt_api, session, "=============== thread ops stop ==============="); - check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); } } @@ -262,7 +264,8 @@ ops(void *arg) */ if (intxn && (tinfo->ops == ckpt_op || tinfo->ops == session_op)) { - check(session->commit_transaction(session, NULL)); + testutil_check( + session->commit_transaction(session, NULL)); ++tinfo->commit; intxn = 0; } @@ -271,9 +274,9 @@ ops(void *arg) if (tinfo->ops == session_op || session == NULL || cursor == NULL) { if (session != NULL) - check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); - check(conn->open_session(conn, NULL, + testutil_check(conn->open_session(conn, NULL, ops_session_config(&tinfo->rnd), &session)); /* @@ -289,7 +292,7 @@ ops(void *arg) */ if (!SINGLETHREADED && !DATASOURCE("lsm") && ckpt_available && mmrand(&tinfo->rnd, 1, 10) == 1) { - check(session->open_cursor(session, + testutil_check(session->open_cursor(session, g.uri, NULL, ckpt_name, &cursor)); /* Pick the next session/cursor close/open. */ @@ -311,10 +314,10 @@ ops(void *arg) * want to have to specify the record number, * which requires an append configuration. */ - check(session->open_cursor(session, + testutil_check(session->open_cursor(session, g.uri, NULL, "overwrite", &cursor)); if (g.type == FIX || g.type == VAR) - check(session->open_cursor( + testutil_check(session->open_cursor( session, g.uri, NULL, "append", &cursor_insert)); @@ -346,7 +349,8 @@ ops(void *arg) /* Named checkpoints lock out backups */ if (ckpt_config != NULL) - check(pthread_rwlock_wrlock(&g.backup_lock)); + testutil_check( + pthread_rwlock_wrlock(&g.backup_lock)); if ((ret = session->checkpoint(session, ckpt_config)) != 0) @@ -355,7 +359,8 @@ ops(void *arg) ckpt_config == NULL ? "" : ckpt_config); if (ckpt_config != NULL) - check(pthread_rwlock_unlock(&g.backup_lock)); + testutil_check( + pthread_rwlock_unlock(&g.backup_lock)); /* Rephrase the checkpoint name for cursor open. */ if (ckpt_config == NULL) @@ -376,7 +381,7 @@ ops(void *arg) * have to do the reset outside of a transaction. */ if (tinfo->ops > reset_op && !intxn) { - check(session->reset(session)); + testutil_check(session->reset(session)); /* Pick the next reset operation. */ reset_op += mmrand(&tinfo->rnd, 20000, 50000); @@ -388,7 +393,8 @@ ops(void *arg) */ if (!SINGLETHREADED && !intxn && mmrand(&tinfo->rnd, 1, 10) >= 8) { - check(session->begin_transaction(session, NULL)); + testutil_check( + session->begin_transaction(session, NULL)); intxn = 1; } @@ -446,7 +452,8 @@ ops(void *arg) if (col_insert(tinfo, cursor_insert, &key, &value, &keyno)) goto deadlock; - check(cursor_insert->reset(cursor_insert)); + testutil_check( + cursor_insert->reset(cursor_insert)); insert = 1; break; @@ -496,7 +503,7 @@ skip_insert: if (col_update(tinfo, goto deadlock; /* Reset the cursor: there is no reason to keep pages pinned. */ - check(cursor->reset(cursor)); + testutil_check(cursor->reset(cursor)); /* * If we're in the transaction, commit 40% of the time and @@ -505,7 +512,7 @@ skip_insert: if (col_update(tinfo, if (intxn) switch (mmrand(&tinfo->rnd, 1, 10)) { case 1: case 2: case 3: case 4: /* 40% */ - check(session->commit_transaction( + testutil_check(session->commit_transaction( session, NULL)); ++tinfo->commit; intxn = 0; @@ -514,7 +521,7 @@ skip_insert: if (col_update(tinfo, if (0) { deadlock: ++tinfo->deadlock; } - check(session->rollback_transaction( + testutil_check(session->rollback_transaction( session, NULL)); ++tinfo->rollback; intxn = 0; @@ -525,7 +532,7 @@ deadlock: ++tinfo->deadlock; } if (session != NULL) - check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); free(keybuf); free(valbuf); @@ -555,9 +562,9 @@ wts_read_scan(void) key_gen_setup(&keybuf); /* Open a session and cursor pair. */ - check(conn->open_session( + testutil_check(conn->open_session( conn, NULL, ops_session_config(NULL), &session)); - check(session->open_cursor( + testutil_check(session->open_cursor( session, g.uri, NULL, NULL, &cursor)); /* Check a random subset of the records using the key. */ @@ -575,7 +582,7 @@ wts_read_scan(void) testutil_die(ret, "read_scan"); } - check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); free(keybuf); } @@ -951,7 +958,7 @@ table_append(uint64_t keyno) * and we find a slot. */ for (done = 0;;) { - check(pthread_rwlock_wrlock(&g.append_lock)); + testutil_check(pthread_rwlock_wrlock(&g.append_lock)); /* * If this is the thread we've been waiting for, and its record @@ -988,7 +995,7 @@ table_append(uint64_t keyno) break; } - check(pthread_rwlock_unlock(&g.append_lock)); + testutil_check(pthread_rwlock_unlock(&g.append_lock)); if (done) break; diff --git a/test/format/rebalance.c b/test/format/rebalance.c index c448413fd00..eea4735a530 100644 --- a/test/format/rebalance.c +++ b/test/format/rebalance.c @@ -51,7 +51,7 @@ wts_rebalance(void) /* Rebalance, then verify the object. */ wts_reopen(); conn = g.wts_conn; - check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, "=============== rebalance start ==============="); @@ -62,7 +62,7 @@ wts_rebalance(void) if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, "=============== rebalance stop ==============="); - check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); wts_verify("post-rebalance verify"); wts_close(); diff --git a/test/format/salvage.c b/test/format/salvage.c index 6036f25df39..526e1563390 100644 --- a/test/format/salvage.c +++ b/test/format/salvage.c @@ -42,10 +42,10 @@ salvage(void) conn = g.wts_conn; track("salvage", 0ULL, NULL); - check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); if ((ret = session->salvage(session, g.uri, "force=true")) != 0) testutil_die(ret, "session.salvage: %s", g.uri); - check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); } /* diff --git a/test/format/t.c b/test/format/t.c index f2409290acb..b9dfdaabb33 100644 --- a/test/format/t.c +++ b/test/format/t.c @@ -178,9 +178,9 @@ main(int argc, char *argv[]) * Initialize locks to single-thread named checkpoints and backups, last * last-record updates, and failures. */ - check(pthread_rwlock_init(&g.append_lock, NULL)); - check(pthread_rwlock_init(&g.backup_lock, NULL)); - check(pthread_rwlock_init(&g.death_lock, NULL)); + testutil_check(pthread_rwlock_init(&g.append_lock, NULL)); + testutil_check(pthread_rwlock_init(&g.backup_lock, NULL)); + testutil_check(pthread_rwlock_init(&g.death_lock, NULL)); printf("%s: process %" PRIdMAX "\n", g.progname, (intmax_t)getpid()); while (++g.run_cnt <= g.c_runs || g.c_runs == 0 ) { @@ -272,8 +272,8 @@ main(int argc, char *argv[]) config_print(0); - check(pthread_rwlock_destroy(&g.append_lock)); - check(pthread_rwlock_destroy(&g.backup_lock)); + testutil_check(pthread_rwlock_destroy(&g.append_lock)); + testutil_check(pthread_rwlock_destroy(&g.backup_lock)); config_clear(); diff --git a/test/format/wts.c b/test/format/wts.c index 3c3d03195fa..f1b8eaff3c3 100644 --- a/test/format/wts.c +++ b/test/format/wts.c @@ -438,10 +438,10 @@ wts_create(void) /* * Create the underlying store. */ - check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); if ((ret = session->create(session, g.uri, config)) != 0) testutil_die(ret, "session.create: %s", g.uri); - check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); } void @@ -454,7 +454,7 @@ wts_close(void) config = g.c_leak_memory ? "leak_memory" : NULL; - check(conn->close(conn, config)); + testutil_check(conn->close(conn, config)); g.wts_conn = NULL; g.wt_api = NULL; } @@ -511,7 +511,7 @@ wts_verify(const char *tag) conn = g.wts_conn; track("verify", 0ULL, NULL); - check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, "=============== verify start ==============="); @@ -524,7 +524,7 @@ wts_verify(const char *tag) if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, "=============== verify stop ==============="); - check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); } /* @@ -554,14 +554,14 @@ wts_stats(void) conn = g.wts_conn; track("stat", 0ULL, NULL); - check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); if ((fp = fopen(g.home_stats, "w")) == NULL) testutil_die(errno, "fopen: %s", g.home_stats); /* Connection statistics. */ fprintf(fp, "====== Connection statistics:\n"); - check(session->open_cursor( + testutil_check(session->open_cursor( session, "statistics:", NULL, NULL, &cursor)); while ((ret = cursor->next(cursor)) == 0 && @@ -571,13 +571,14 @@ wts_stats(void) if (ret != WT_NOTFOUND) testutil_die(ret, "cursor.next"); - check(cursor->close(cursor)); + testutil_check(cursor->close(cursor)); /* Data source statistics. */ fprintf(fp, "\n\n====== Data source statistics:\n"); stat_name = dmalloc(strlen("statistics:") + strlen(g.uri) + 1); sprintf(stat_name, "statistics:%s", g.uri); - check(session->open_cursor(session, stat_name, NULL, NULL, &cursor)); + testutil_check(session->open_cursor( + session, stat_name, NULL, NULL, &cursor)); free(stat_name); while ((ret = cursor->next(cursor)) == 0 && @@ -587,9 +588,9 @@ wts_stats(void) if (ret != WT_NOTFOUND) testutil_die(ret, "cursor.next"); - check(cursor->close(cursor)); + testutil_check(cursor->close(cursor)); fclose_and_clear(&fp); - check(session->close(session, NULL)); + testutil_check(session->close(session, NULL)); } diff --git a/test/utility/test_util.i b/test/utility/test_util.i index de9424115d0..f1f0768dfcf 100644 --- a/test/utility/test_util.i +++ b/test/utility/test_util.i @@ -75,6 +75,16 @@ testutil_die(int e, const char *fmt, ...) exit(EXIT_FAILURE); } +/* + * check -- + * Complain and quit if a function call fails. + */ +#define testutil_check(call) do { \ + int __r; \ + if ((__r = (call)) != 0) \ + testutil_die(__r, "%s/%d: %s", __func__, __LINE__, #call);\ +} while (0) + /* * testutil_work_dir_from_path -- * Takes a buffer, its size and the intended work directory. -- cgit v1.2.1 From 439367837ea2df0e8cee4e74f387a94baf649833 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Fri, 19 Feb 2016 01:59:48 +0000 Subject: WT-2405 Convert bloom test to use new error checking macro. --- test/bloom/test_bloom.c | 41 +++++++++++++++-------------------------- 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/test/bloom/test_bloom.c b/test/bloom/test_bloom.c index 04fc8d1c371..4a3d3372794 100644 --- a/test/bloom/test_bloom.c +++ b/test/bloom/test_bloom.c @@ -129,11 +129,9 @@ setup(void) "create,error_prefix=\"%s\",cache_size=%" PRIu32 "MB,%s", g.progname, g.c_cache, g.config_open == NULL ? "" : g.config_open); - if ((ret = wiredtiger_open(NULL, NULL, config, &conn)) != 0) - testutil_die(ret, "wiredtiger_open"); + testutil_check(wiredtiger_open(NULL, NULL, config, &conn)); - if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) - testutil_die(ret, "connection.open_session"); + testutil_check(conn->open_session(conn, NULL, NULL, &session)); g.wt_conn = conn; g.wt_session = session; @@ -153,9 +151,8 @@ run(void) /* Use the internal session handle to access private APIs. */ sess = (WT_SESSION_IMPL *)g.wt_session; - if ((ret = __wt_bloom_create( - sess, uri, NULL, g.c_ops, g.c_factor, g.c_k, &bloomp)) != 0) - testutil_die(ret, "__wt_bloom_create"); + testutil_check(__wt_bloom_create( + sess, uri, NULL, g.c_ops, g.c_factor, g.c_k, &bloomp)); item.size = g.c_key_max; for (i = 0; i < g.c_ops; i++) { @@ -164,8 +161,7 @@ run(void) testutil_die(ret, "__wt_bloom_insert: %d", i); } - if ((ret = __wt_bloom_finalize(bloomp)) != 0) - testutil_die(ret, "__wt_bloom_finalize"); + testutil_check(__wt_bloom_finalize(bloomp)); for (i = 0; i < g.c_ops; i++) { item.data = g.entries[i]; @@ -174,18 +170,15 @@ run(void) testutil_die(ret, "__wt_bloom_get"); } } - if ((ret = __wt_bloom_close(bloomp)) != 0) - testutil_die(ret, "__wt_bloom_close"); - - if ((ret = g.wt_session->checkpoint(g.wt_session, NULL)) != 0) - testutil_die(ret, "WT_SESSION.checkpoint"); - if ((ret = __wt_bloom_open( - sess, uri, g.c_factor, g.c_k, NULL, &bloomp)) != 0) - testutil_die(ret, "__wt_bloom_open"); + testutil_check(__wt_bloom_close(bloomp)); + + testutil_check(g.wt_session->checkpoint(g.wt_session, NULL)); + testutil_check(__wt_bloom_open( + sess, uri, g.c_factor, g.c_k, NULL, &bloomp)); + for (i = 0; i < g.c_ops; i++) { item.data = g.entries[i]; - if ((ret = __wt_bloom_get(bloomp, &item)) != 0) - testutil_die(ret, "__wt_bloom_get"); + testutil_check(__wt_bloom_get(bloomp, &item)); } /* @@ -204,23 +197,19 @@ run(void) free((void *)item.data); printf("Out of %d ops, got %d false positives, %.4f%%\n", g.c_ops, fp, 100.0 * fp/g.c_ops); - if ((ret = __wt_bloom_drop(bloomp, NULL)) != 0) - testutil_die(ret, "__wt_bloom_drop"); + testutil_check(__wt_bloom_drop(bloomp, NULL)); } void cleanup(void) { uint32_t i; - int ret; for (i = 0; i < g.c_ops; i++) free(g.entries[i]); free(g.entries); - if ((ret = g.wt_session->close(g.wt_session, NULL)) != 0) - testutil_die(ret, "WT_SESSION.close"); - if ((g.wt_conn->close(g.wt_conn, NULL)) != 0) - testutil_die(ret, "WT_CONNECTION.close"); + testutil_check(g.wt_session->close(g.wt_session, NULL)); + testutil_check(g.wt_conn->close(g.wt_conn, NULL)); } /* -- cgit v1.2.1 From 93dac5e896f6747adefad7cf46d0bf847fa3dc75 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Fri, 19 Feb 2016 15:15:58 +1100 Subject: WT-2411 Drop the checkpoint lock when LSM is draining its queue. --- src/include/schema.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/include/schema.h b/src/include/schema.h index 1e4e4fde81f..f93c596e2ca 100644 --- a/src/include/schema.h +++ b/src/include/schema.h @@ -174,6 +174,8 @@ struct __wt_table { */ #define WT_WITHOUT_LOCKS(session, op) do { \ WT_CONNECTION_IMPL *__conn = S2C(session); \ + bool __checkpoint_locked = \ + F_ISSET(session, WT_SESSION_LOCKED_CHECKPOINT); \ bool __handle_locked = \ F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST); \ bool __table_locked = \ @@ -192,7 +194,15 @@ struct __wt_table { F_CLR(session, WT_SESSION_LOCKED_SCHEMA); \ __wt_spin_unlock(session, &__conn->schema_lock); \ } \ + if (__checkpoint_locked) { \ + F_CLR(session, WT_SESSION_LOCKED_CHECKPOINT); \ + __wt_spin_unlock(session, &__conn->checkpoint_lock); \ + } \ op; \ + if (__checkpoint_locked) { \ + __wt_spin_lock(session, &__conn->checkpoint_lock); \ + F_SET(session, WT_SESSION_LOCKED_CHECKPOINT); \ + } \ if (__schema_locked) { \ __wt_spin_lock(session, &__conn->schema_lock); \ F_SET(session, WT_SESSION_LOCKED_SCHEMA); \ -- cgit v1.2.1 From 4f3828797f41cdd2e3118b1cd45c654491ff6c2d Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Fri, 19 Feb 2016 15:33:56 +1100 Subject: Merge pull request #2505 from wiredtiger/wt-2411 WT-2411 Drop the checkpoint lock when LSM is draining its queue. (cherry picked from commit 2b78ad8a6bd446d06d1a453198b68befed57fbe5) --- src/include/schema.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/include/schema.h b/src/include/schema.h index 1e4e4fde81f..f93c596e2ca 100644 --- a/src/include/schema.h +++ b/src/include/schema.h @@ -174,6 +174,8 @@ struct __wt_table { */ #define WT_WITHOUT_LOCKS(session, op) do { \ WT_CONNECTION_IMPL *__conn = S2C(session); \ + bool __checkpoint_locked = \ + F_ISSET(session, WT_SESSION_LOCKED_CHECKPOINT); \ bool __handle_locked = \ F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST); \ bool __table_locked = \ @@ -192,7 +194,15 @@ struct __wt_table { F_CLR(session, WT_SESSION_LOCKED_SCHEMA); \ __wt_spin_unlock(session, &__conn->schema_lock); \ } \ + if (__checkpoint_locked) { \ + F_CLR(session, WT_SESSION_LOCKED_CHECKPOINT); \ + __wt_spin_unlock(session, &__conn->checkpoint_lock); \ + } \ op; \ + if (__checkpoint_locked) { \ + __wt_spin_lock(session, &__conn->checkpoint_lock); \ + F_SET(session, WT_SESSION_LOCKED_CHECKPOINT); \ + } \ if (__schema_locked) { \ __wt_spin_lock(session, &__conn->schema_lock); \ F_SET(session, WT_SESSION_LOCKED_SCHEMA); \ -- cgit v1.2.1 From 6dd0d440414c7d76f6caf7487d48a0237ff2b754 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 19 Feb 2016 09:03:31 -0500 Subject: WT-2410: Casting function pointers to different types Don't cast function pointers to different types, it's undefined behavior. Fixes for the block manager. --- src/block/block_mgr.c | 291 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 220 insertions(+), 71 deletions(-) diff --git a/src/block/block_mgr.c b/src/block/block_mgr.c index dceaae8bb99..0bb75d129e1 100644 --- a/src/block/block_mgr.c +++ b/src/block/block_mgr.c @@ -68,6 +68,21 @@ __bm_checkpoint(WT_BM *bm, session, bm->block, buf, ckptbase, data_cksum)); } +/* + * __bm_checkpoint_readonly -- + * Write a buffer into a block, creating a checkpoint; readonly version. + */ +static int +__bm_checkpoint_readonly(WT_BM *bm, + WT_SESSION_IMPL *session, WT_ITEM *buf, WT_CKPT *ckptbase, bool data_cksum) +{ + WT_UNUSED(buf); + WT_UNUSED(ckptbase); + WT_UNUSED(data_cksum); + + return (__bm_readonly(bm, session)); +} + /* * __bm_checkpoint_load -- * Load a checkpoint. @@ -112,6 +127,16 @@ __bm_checkpoint_resolve(WT_BM *bm, WT_SESSION_IMPL *session) return (__wt_block_checkpoint_resolve(session, bm->block)); } +/* + * __bm_checkpoint_resolve_readonly -- + * Resolve the checkpoint; readonly version. + */ +static int +__bm_checkpoint_resolve_readonly(WT_BM *bm, WT_SESSION_IMPL *session) +{ + return (__bm_readonly(bm, session)); +} + /* * __bm_checkpoint_unload -- * Unload a checkpoint point. @@ -160,6 +185,16 @@ __bm_compact_end(WT_BM *bm, WT_SESSION_IMPL *session) return (__wt_block_compact_end(session, bm->block)); } +/* + * __bm_compact_end_readonly -- + * End a block manager compaction; readonly version. + */ +static int +__bm_compact_end_readonly(WT_BM *bm, WT_SESSION_IMPL *session) +{ + return (__bm_readonly(bm, session)); +} + /* * __bm_compact_page_skip -- * Return if a page is useful for compaction. @@ -172,6 +207,21 @@ __bm_compact_page_skip(WT_BM *bm, WT_SESSION_IMPL *session, session, bm->block, addr, addr_size, skipp)); } +/* + * __bm_compact_page_skip_readonly -- + * Return if a page is useful for compaction; readonly version. + */ +static int +__bm_compact_page_skip_readonly(WT_BM *bm, WT_SESSION_IMPL *session, + const uint8_t *addr, size_t addr_size, bool *skipp) +{ + WT_UNUSED(addr); + WT_UNUSED(addr_size); + WT_UNUSED(skipp); + + return (__bm_readonly(bm, session)); +} + /* * __bm_compact_skip -- * Return if a file can be compacted. @@ -182,6 +232,18 @@ __bm_compact_skip(WT_BM *bm, WT_SESSION_IMPL *session, bool *skipp) return (__wt_block_compact_skip(session, bm->block, skipp)); } +/* + * __bm_compact_skip_readonly -- + * Return if a file can be compacted; readonly version. + */ +static int +__bm_compact_skip_readonly(WT_BM *bm, WT_SESSION_IMPL *session, bool *skipp) +{ + WT_UNUSED(skipp); + + return (__bm_readonly(bm, session)); +} + /* * __bm_compact_start -- * Start a block manager compaction. @@ -192,6 +254,16 @@ __bm_compact_start(WT_BM *bm, WT_SESSION_IMPL *session) return (__wt_block_compact_start(session, bm->block)); } +/* + * __bm_compact_start_readonly -- + * Start a block manager compaction; readonly version. + */ +static int +__bm_compact_start_readonly(WT_BM *bm, WT_SESSION_IMPL *session) +{ + return (__bm_readonly(bm, session)); +} + /* * __bm_free -- * Free a block of space to the underlying file. @@ -203,6 +275,20 @@ __bm_free(WT_BM *bm, return (__wt_block_free(session, bm->block, addr, addr_size)); } +/* + * __bm_free_readonly -- + * Free a block of space to the underlying file; readonly version. + */ +static int +__bm_free_readonly(WT_BM *bm, + WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size) +{ + WT_UNUSED(addr); + WT_UNUSED(addr_size); + + return (__bm_readonly(bm, session)); +} + /* * __bm_is_mapped -- * Return if the file is mapped into memory. @@ -225,6 +311,31 @@ __bm_salvage_end(WT_BM *bm, WT_SESSION_IMPL *session) return (__wt_block_salvage_end(session, bm->block)); } +/* + * __bm_salvage_end_readonly -- + * End a block manager salvage; readonly version. + */ +static int +__bm_salvage_end_readonly(WT_BM *bm, WT_SESSION_IMPL *session) +{ + return (__bm_readonly(bm, session)); +} + +/* + * __bm_salvage_next_readonly -- + * Return the next block from the file; readonly version. + */ +static int +__bm_salvage_next_readonly(WT_BM *bm, + WT_SESSION_IMPL *session, uint8_t *addr, size_t *addr_sizep, bool *eofp) +{ + WT_UNUSED(addr); + WT_UNUSED(addr_sizep); + WT_UNUSED(eofp); + + return (__bm_readonly(bm, session)); +} + /* * __bm_salvage_next -- * Return the next block from the file. @@ -247,6 +358,16 @@ __bm_salvage_start(WT_BM *bm, WT_SESSION_IMPL *session) return (__wt_block_salvage_start(session, bm->block)); } +/* + * __bm_salvage_start_readonly -- + * Start a block manager salvage; readonly version. + */ +static int +__bm_salvage_start_readonly(WT_BM *bm, WT_SESSION_IMPL *session) +{ + return (__bm_readonly(bm, session)); +} + /* * __bm_salvage_valid -- * Inform salvage a block is valid. @@ -259,6 +380,21 @@ __bm_salvage_valid(WT_BM *bm, session, bm->block, addr, addr_size, valid)); } +/* + * __bm_salvage_valid_readonly -- + * Inform salvage a block is valid; readonly version. + */ +static int +__bm_salvage_valid_readonly(WT_BM *bm, + WT_SESSION_IMPL *session, uint8_t *addr, size_t addr_size, bool valid) +{ + WT_UNUSED(addr); + WT_UNUSED(addr_size); + WT_UNUSED(valid); + + return (__bm_readonly(bm, session)); +} + /* * __bm_stat -- * Block-manager statistics. @@ -282,6 +418,18 @@ __bm_sync(WT_BM *bm, WT_SESSION_IMPL *session, bool async) __wt_fsync(session, bm->block->fh)); } +/* + * __bm_sync_readonly -- + * Flush a file to disk; readonly version. + */ +static int +__bm_sync_readonly(WT_BM *bm, WT_SESSION_IMPL *session, bool async) +{ + WT_UNUSED(async); + + return (__bm_readonly(bm, session)); +} + /* * __bm_verify_addr -- * Verify an address. @@ -326,6 +474,23 @@ __bm_write(WT_BM *bm, WT_SESSION_IMPL *session, session, bm->block, buf, addr, addr_sizep, data_cksum)); } +/* + * __bm_write_readonly -- + * Write a buffer into a block, returning the block's address cookie; + * readonly version. + */ +static int +__bm_write_readonly(WT_BM *bm, WT_SESSION_IMPL *session, + WT_ITEM *buf, uint8_t *addr, size_t *addr_sizep, bool data_cksum) +{ + WT_UNUSED(buf); + WT_UNUSED(addr); + WT_UNUSED(addr_sizep); + WT_UNUSED(data_cksum); + + return (__bm_readonly(bm, session)); +} + /* * __bm_write_size -- * Return the buffer size required to write a block. @@ -336,6 +501,18 @@ __bm_write_size(WT_BM *bm, WT_SESSION_IMPL *session, size_t *sizep) return (__wt_block_write_size(session, bm->block, sizep)); } +/* + * __bm_write_size_readonly -- + * Return the buffer size required to write a block; readonly version. + */ +static int +__bm_write_size_readonly(WT_BM *bm, WT_SESSION_IMPL *session, size_t *sizep) +{ + WT_UNUSED(sizep); + + return (__bm_readonly(bm, session)); +} + /* * __bm_method_set -- * Set up the legal methods. @@ -343,78 +520,50 @@ __bm_write_size(WT_BM *bm, WT_SESSION_IMPL *session, size_t *sizep) static void __bm_method_set(WT_BM *bm, bool readonly) { + bm->addr_invalid = __bm_addr_invalid; + bm->addr_string = __bm_addr_string; + bm->block_header = __bm_block_header; + bm->checkpoint = __bm_checkpoint; + bm->checkpoint_load = __bm_checkpoint_load; + bm->checkpoint_resolve = __bm_checkpoint_resolve; + bm->checkpoint_unload = __bm_checkpoint_unload; + bm->close = __bm_close; + bm->compact_end = __bm_compact_end; + bm->compact_page_skip = __bm_compact_page_skip; + bm->compact_skip = __bm_compact_skip; + bm->compact_start = __bm_compact_start; + bm->free = __bm_free; + bm->is_mapped = __bm_is_mapped; + bm->preload = __wt_bm_preload; + bm->read = __wt_bm_read; + bm->salvage_end = __bm_salvage_end; + bm->salvage_next = __bm_salvage_next; + bm->salvage_start = __bm_salvage_start; + bm->salvage_valid = __bm_salvage_valid; + bm->size = __wt_block_manager_size; + bm->stat = __bm_stat; + bm->sync = __bm_sync; + bm->verify_addr = __bm_verify_addr; + bm->verify_end = __bm_verify_end; + bm->verify_start = __bm_verify_start; + bm->write = __bm_write; + bm->write_size = __bm_write_size; + if (readonly) { - bm->addr_invalid = __bm_addr_invalid; - bm->addr_string = __bm_addr_string; - bm->block_header = __bm_block_header; - bm->checkpoint = (int (*)(WT_BM *, WT_SESSION_IMPL *, - WT_ITEM *, WT_CKPT *, bool))__bm_readonly; - bm->checkpoint_load = __bm_checkpoint_load; - bm->checkpoint_resolve = - (int (*)(WT_BM *, WT_SESSION_IMPL *))__bm_readonly; - bm->checkpoint_unload = __bm_checkpoint_unload; - bm->close = __bm_close; - bm->compact_end = - (int (*)(WT_BM *, WT_SESSION_IMPL *))__bm_readonly; - bm->compact_page_skip = (int (*)(WT_BM *, WT_SESSION_IMPL *, - const uint8_t *, size_t, bool *))__bm_readonly; - bm->compact_skip = (int (*) - (WT_BM *, WT_SESSION_IMPL *, bool *))__bm_readonly; - bm->compact_start = - (int (*)(WT_BM *, WT_SESSION_IMPL *))__bm_readonly; - bm->free = (int (*)(WT_BM *, - WT_SESSION_IMPL *, const uint8_t *, size_t))__bm_readonly; - bm->is_mapped = __bm_is_mapped; - bm->preload = __wt_bm_preload; - bm->read = __wt_bm_read; - bm->salvage_end = (int (*) - (WT_BM *, WT_SESSION_IMPL *))__bm_readonly; - bm->salvage_next = (int (*)(WT_BM *, WT_SESSION_IMPL *, - uint8_t *, size_t *, bool *))__bm_readonly; - bm->salvage_start = (int (*) - (WT_BM *, WT_SESSION_IMPL *))__bm_readonly; - bm->salvage_valid = (int (*)(WT_BM *, - WT_SESSION_IMPL *, uint8_t *, size_t, bool))__bm_readonly; - bm->size = __wt_block_manager_size; - bm->stat = __bm_stat; - bm->sync = - (int (*)(WT_BM *, WT_SESSION_IMPL *, bool))__bm_readonly; - bm->verify_addr = __bm_verify_addr; - bm->verify_end = __bm_verify_end; - bm->verify_start = __bm_verify_start; - bm->write = (int (*)(WT_BM *, WT_SESSION_IMPL *, - WT_ITEM *, uint8_t *, size_t *, bool))__bm_readonly; - bm->write_size = (int (*) - (WT_BM *, WT_SESSION_IMPL *, size_t *))__bm_readonly; - } else { - bm->addr_invalid = __bm_addr_invalid; - bm->addr_string = __bm_addr_string; - bm->block_header = __bm_block_header; - bm->checkpoint = __bm_checkpoint; - bm->checkpoint_load = __bm_checkpoint_load; - bm->checkpoint_resolve = __bm_checkpoint_resolve; - bm->checkpoint_unload = __bm_checkpoint_unload; - bm->close = __bm_close; - bm->compact_end = __bm_compact_end; - bm->compact_page_skip = __bm_compact_page_skip; - bm->compact_skip = __bm_compact_skip; - bm->compact_start = __bm_compact_start; - bm->free = __bm_free; - bm->is_mapped = __bm_is_mapped; - bm->preload = __wt_bm_preload; - bm->read = __wt_bm_read; - bm->salvage_end = __bm_salvage_end; - bm->salvage_next = __bm_salvage_next; - bm->salvage_start = __bm_salvage_start; - bm->salvage_valid = __bm_salvage_valid; - bm->size = __wt_block_manager_size; - bm->stat = __bm_stat; - bm->sync = __bm_sync; - bm->verify_addr = __bm_verify_addr; - bm->verify_end = __bm_verify_end; - bm->verify_start = __bm_verify_start; - bm->write = __bm_write; - bm->write_size = __bm_write_size; + bm->checkpoint = __bm_checkpoint_readonly; + bm->checkpoint_resolve = __bm_checkpoint_resolve_readonly; + bm->compact_end = __bm_compact_end_readonly; + bm->compact_page_skip = __bm_compact_page_skip_readonly; + bm->compact_skip = __bm_compact_skip_readonly; + bm->compact_start = __bm_compact_start_readonly; + bm->free = __bm_free_readonly; + bm->salvage_end = __bm_salvage_end_readonly; + bm->salvage_next = __bm_salvage_next_readonly; + bm->salvage_start = __bm_salvage_start_readonly; + bm->salvage_valid = __bm_salvage_valid_readonly; + bm->sync = __bm_sync_readonly; + bm->write = __bm_write_readonly; + bm->write_size = __bm_write_size_readonly; } } -- cgit v1.2.1 From 53af61ebc08b960601e26179e6780db1bef17cd3 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 19 Feb 2016 09:19:35 -0500 Subject: WT-2410: Casting function pointers to different types Don't cast function pointers to different types, it's undefined behavior. Fixes for the session layer. --- src/include/extern.h | 1 + src/session/session_api.c | 211 +++++++++++++++++++++++++++++++++++------- src/session/session_compact.c | 15 +++ 3 files changed, 195 insertions(+), 32 deletions(-) diff --git a/src/include/extern.h b/src/include/extern.h index 8945309c4a7..de66498692e 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -623,6 +623,7 @@ extern int __wt_open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_h extern int __wt_open_internal_session(WT_CONNECTION_IMPL *conn, const char *name, bool open_metadata, uint32_t session_flags, WT_SESSION_IMPL **sessionp); extern int __wt_compact_uri_analyze(WT_SESSION_IMPL *session, const char *uri, bool *skipp); extern int __wt_session_compact( WT_SESSION *wt_session, const char *uri, const char *config); +extern int __wt_session_compact_readonly( WT_SESSION *wt_session, const char *uri, const char *config); extern int __wt_session_lock_dhandle( WT_SESSION_IMPL *session, uint32_t flags, bool *is_deadp); extern int __wt_session_release_btree(WT_SESSION_IMPL *session); extern int __wt_session_get_btree_ckpt(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], uint32_t flags); diff --git a/src/session/session_api.c b/src/session/session_api.c index db1feabab45..38e21069009 100644 --- a/src/session/session_api.c +++ b/src/session/session_api.c @@ -508,6 +508,20 @@ __session_create(WT_SESSION *wt_session, const char *uri, const char *config) err: API_END_RET_NOTFOUND_MAP(session, ret); } +/* + * __session_create_readonly -- + * WT_SESSION->create method; readonly version. + */ +static int +__session_create_readonly( + WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_UNUSED(uri); + WT_UNUSED(config); + + return (__wt_session_notsup(wt_session)); +} + /* * __session_log_flush -- * WT_SESSION->log_flush method. @@ -545,6 +559,18 @@ __session_log_flush(WT_SESSION *wt_session, const char *config) err: API_END_RET(session, ret); } +/* + * __session_log_flush_readonly -- + * WT_SESSION->log_flush method; readonly version. + */ +static int +__session_log_flush_readonly(WT_SESSION *wt_session, const char *config) +{ + WT_UNUSED(config); + + return (__wt_session_notsup(wt_session)); +} + /* * __session_log_printf -- * WT_SESSION->log_printf method. @@ -567,6 +593,19 @@ __session_log_printf(WT_SESSION *wt_session, const char *fmt, ...) err: API_END_RET(session, ret); } +/* + * __session_log_printf_readonly -- + * WT_SESSION->log_printf method; readonly version. + */ +static int +__session_log_printf_readonly(WT_SESSION *wt_session, const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 2, 3))) +{ + WT_UNUSED(fmt); + + return (__wt_session_notsup(wt_session)); +} + /* * __session_rebalance -- * WT_SESSION->rebalance method. @@ -590,6 +629,20 @@ __session_rebalance(WT_SESSION *wt_session, const char *uri, const char *config) err: API_END_RET_NOTFOUND_MAP(session, ret); } +/* + * __session_rebalance_readonly -- + * WT_SESSION->rebalance method; readonly version. + */ +static int +__session_rebalance_readonly( + WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_UNUSED(uri); + WT_UNUSED(config); + + return (__wt_session_notsup(wt_session)); +} + /* * __session_rename -- * WT_SESSION->rename method. @@ -616,6 +669,21 @@ __session_rename(WT_SESSION *wt_session, err: API_END_RET_NOTFOUND_MAP(session, ret); } +/* + * __session_rename_readonly -- + * WT_SESSION->rename method; readonly version. + */ +static int +__session_rename_readonly(WT_SESSION *wt_session, + const char *uri, const char *newuri, const char *config) +{ + WT_UNUSED(uri); + WT_UNUSED(newuri); + WT_UNUSED(config); + + return (__wt_session_notsup(wt_session)); +} + /* * __session_reset -- * WT_SESSION->reset method. @@ -691,6 +759,20 @@ err: /* Note: drop operations cannot be unrolled (yet?). */ API_END_RET_NOTFOUND_MAP(session, ret); } +/* + * __session_drop_readonly -- + * WT_SESSION->drop method; readonly version. + */ +static int +__session_drop_readonly( + WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_UNUSED(uri); + WT_UNUSED(config); + + return (__wt_session_notsup(wt_session)); +} + /* * __session_join -- * WT_SESSION->join method. @@ -835,6 +917,20 @@ __session_salvage(WT_SESSION *wt_session, const char *uri, const char *config) err: API_END_RET_NOTFOUND_MAP(session, ret); } +/* + * __session_salvage_readonly -- + * WT_SESSION->salvage method; readonly version. + */ +static int +__session_salvage_readonly( + WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_UNUSED(uri); + WT_UNUSED(config); + + return (__wt_session_notsup(wt_session)); +} + /* * __wt_session_range_truncate -- * Session handling of a range truncate. @@ -1016,6 +1112,22 @@ err: TXN_API_END_RETRY(session, ret, 0); return (ret == WT_NOTFOUND && uri != NULL ? ENOENT : ret); } +/* + * __session_truncate_readonly -- + * WT_SESSION->truncate method; readonly version. + */ +static int +__session_truncate_readonly(WT_SESSION *wt_session, + const char *uri, WT_CURSOR *start, WT_CURSOR *stop, const char *config) +{ + WT_UNUSED(uri); + WT_UNUSED(start); + WT_UNUSED(stop); + WT_UNUSED(config); + + return (__wt_session_notsup(wt_session)); +} + /* * __session_upgrade -- * WT_SESSION->upgrade method. @@ -1038,6 +1150,20 @@ __session_upgrade(WT_SESSION *wt_session, const char *uri, const char *config) err: API_END_RET_NOTFOUND_MAP(session, ret); } +/* + * __session_upgrade_readonly -- + * WT_SESSION->upgrade method; readonly version. + */ +static int +__session_upgrade_readonly( + WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_UNUSED(uri); + WT_UNUSED(config); + + return (__wt_session_notsup(wt_session)); +} + /* * __session_verify -- * WT_SESSION->verify method. @@ -1259,6 +1385,18 @@ __session_transaction_sync(WT_SESSION *wt_session, const char *config) err: API_END_RET(session, ret); } +/* + * __session_transaction_sync_readonly -- + * WT_SESSION->transaction_sync method; readonly version. + */ +static int +__session_transaction_sync_readonly(WT_SESSION *wt_session, const char *config) +{ + WT_UNUSED(config); + + return (__wt_session_notsup(wt_session)); +} + /* * __session_checkpoint -- * WT_SESSION->checkpoint method. @@ -1307,6 +1445,18 @@ __session_checkpoint(WT_SESSION *wt_session, const char *config) err: API_END_RET_NOTFOUND_MAP(session, ret); } +/* + * __session_checkpoint_readonly -- + * WT_SESSION->checkpoint method; readonly version. + */ +static int +__session_checkpoint_readonly(WT_SESSION *wt_session, const char *config) +{ + WT_UNUSED(config); + + return (__wt_session_notsup(wt_session)); +} + /* * __session_snapshot -- * WT_SESSION->snapshot method. @@ -1393,6 +1543,33 @@ __open_session(WT_CONNECTION_IMPL *conn, __session_snapshot, __session_transaction_pinned_range, __session_transaction_sync + }, stds_readonly = { + NULL, + NULL, + __session_close, + __session_reconfigure, + __session_strerror, + __session_open_cursor, + __session_create_readonly, + __wt_session_compact_readonly, + __session_drop_readonly, + __session_join, + __session_log_flush_readonly, + __session_log_printf_readonly, + __session_rebalance_readonly, + __session_rename_readonly, + __session_reset, + __session_salvage_readonly, + __session_truncate_readonly, + __session_upgrade_readonly, + __session_verify, + __session_begin_transaction, + __session_commit_transaction, + __session_rollback_transaction, + __session_checkpoint_readonly, + __session_snapshot, + __session_transaction_pinned_range, + __session_transaction_sync_readonly }; WT_DECL_RET; WT_SESSION *wt_session; @@ -1433,39 +1610,9 @@ __open_session(WT_CONNECTION_IMPL *conn, conn->session_cnt = i + 1; session_ret->id = i; - session_ret->iface = stds; + session_ret->iface = + F_ISSET(conn, WT_CONN_READONLY) ? stds_readonly : stds; session_ret->iface.connection = &conn->iface; - /* - * Disable some methods if this is a read-only connection. Group - * them by call signature. - */ - if (F_ISSET(conn, WT_CONN_READONLY)) { - wt_session = &session_ret->iface; - wt_session->checkpoint = - (int (*)(WT_SESSION *, const char *))__wt_session_notsup; - wt_session->compact = (int (*)(WT_SESSION *, - const char *, const char *))__wt_session_notsup; - wt_session->create = (int (*)(WT_SESSION *, - const char *, const char *))__wt_session_notsup; - wt_session->drop = (int (*)(WT_SESSION *, - const char *, const char *))__wt_session_notsup; - wt_session->log_flush = - (int (*)(WT_SESSION *, const char *))__wt_session_notsup; - wt_session->log_printf = (int (*)( - WT_SESSION *, const char *, ...))__wt_session_notsup; - wt_session->rebalance = (int (*)(WT_SESSION *, - const char *, const char *))__wt_session_notsup; - wt_session->rename = (int (*)(WT_SESSION *, const char *, - const char *, const char *))__wt_session_notsup; - wt_session->salvage = (int (*)(WT_SESSION *, - const char *, const char *))__wt_session_notsup; - wt_session->transaction_sync = - (int (*)(WT_SESSION *, const char *))__wt_session_notsup; - wt_session->truncate = (int (*)(WT_SESSION *, const char *, - WT_CURSOR *, WT_CURSOR *, const char *))__wt_session_notsup; - wt_session->upgrade = (int (*)(WT_SESSION *, - const char *, const char *))__wt_session_notsup; - } WT_ERR(__wt_cond_alloc(session, "session", false, &session_ret->cond)); diff --git a/src/session/session_compact.c b/src/session/session_compact.c index 5abccbd1366..53faf343f35 100644 --- a/src/session/session_compact.c +++ b/src/session/session_compact.c @@ -193,6 +193,7 @@ err: session->compact_state = WT_COMPACT_NONE; /* * __wt_session_compact -- + * WT_SESSION.compact method. */ int __wt_session_compact( @@ -260,3 +261,17 @@ err: session->compact = NULL; API_END_RET_NOTFOUND_MAP(session, ret); } + +/* + * __wt_session_compact_readonly -- + * WT_SESSION.compact method; readonly version. + */ +int +__wt_session_compact_readonly( + WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_UNUSED(uri); + WT_UNUSED(config); + + return (__wt_session_notsup(wt_session)); +} -- cgit v1.2.1 From 94a269b1b0359b362b506dc6a44693ac3d57e628 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Fri, 19 Feb 2016 09:56:49 -0500 Subject: WT-2349 Fix Jenkins memory failures --- bench/wtperf/config.c | 4 +++- src/conn/conn_api.c | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bench/wtperf/config.c b/bench/wtperf/config.c index 89069a170d8..9be4c2b880e 100644 --- a/bench/wtperf/config.c +++ b/bench/wtperf/config.c @@ -134,9 +134,11 @@ config_free(CONFIG *cfg) } cleanup_truncate_config(cfg); + free(cfg->base_uri); free(cfg->ckptthreads); + free((char *)cfg->partial_config); free(cfg->popthreads); - free(cfg->base_uri); + free((char *)cfg->reopen_config); free(cfg->workers); free(cfg->workload); } diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index e510e147f88..876cefd8496 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -2025,6 +2025,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, */ __conn_config_readonly(cfg); WT_ERR(__wt_config_merge(session, cfg, NULL, &conn->cfg)); + __wt_free(session, merge_cfg); } else conn->cfg = merge_cfg; -- cgit v1.2.1 From 5ff29e127b5281be7a410e23bdf5ee218ef560e7 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 19 Feb 2016 10:13:25 -0500 Subject: WT-2410: Casting function pointers to different types Don't cast function pointers to different types, it's undefined behavior. Fixes for the cursor layer. --- src/cursor/cur_backup.c | 32 ++++++++-------- src/cursor/cur_config.c | 30 +++++++-------- src/cursor/cur_ds.c | 32 ++++++++-------- src/cursor/cur_dump.c | 32 ++++++++-------- src/cursor/cur_file.c | 32 ++++++++-------- src/cursor/cur_index.c | 32 ++++++++-------- src/cursor/cur_join.c | 64 +++++++++++++++---------------- src/cursor/cur_log.c | 32 ++++++++-------- src/cursor/cur_metadata.c | 32 ++++++++-------- src/cursor/cur_stat.c | 32 ++++++++-------- src/cursor/cur_std.c | 95 +++++++++++++++++++++++++++++++++++++++++++---- src/cursor/cur_table.c | 64 +++++++++++++++---------------- src/include/cursor.h | 16 ++++---- src/include/extern.h | 9 ++++- src/lsm/lsm_cursor.c | 32 ++++++++-------- 15 files changed, 326 insertions(+), 240 deletions(-) diff --git a/src/cursor/cur_backup.c b/src/cursor/cur_backup.c index d7d74da48d4..63c4ac6a4ff 100644 --- a/src/cursor/cur_backup.c +++ b/src/cursor/cur_backup.c @@ -103,22 +103,22 @@ __wt_curbackup_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp) { WT_CURSOR_STATIC_INIT(iface, - __wt_cursor_get_key, /* get-key */ - __wt_cursor_notsup, /* get-value */ - __wt_cursor_notsup, /* set-key */ - __wt_cursor_notsup, /* set-value */ - __wt_cursor_notsup, /* compare */ - __wt_cursor_notsup, /* equals */ - __curbackup_next, /* next */ - __wt_cursor_notsup, /* prev */ - __curbackup_reset, /* reset */ - __wt_cursor_notsup, /* search */ - __wt_cursor_notsup, /* search-near */ - __wt_cursor_notsup, /* insert */ - __wt_cursor_notsup, /* update */ - __wt_cursor_notsup, /* remove */ - __wt_cursor_notsup, /* reconfigure */ - __curbackup_close); /* close */ + __wt_cursor_get_key, /* get-key */ + __wt_cursor_get_value_notsup, /* get-value */ + __wt_cursor_set_key_notsup, /* set-key */ + __wt_cursor_set_value_notsup, /* set-value */ + __wt_cursor_compare_notsup, /* compare */ + __wt_cursor_equals_notsup, /* equals */ + __curbackup_next, /* next */ + __wt_cursor_notsup, /* prev */ + __curbackup_reset, /* reset */ + __wt_cursor_notsup, /* search */ + __wt_cursor_search_near_notsup, /* search-near */ + __wt_cursor_notsup, /* insert */ + __wt_cursor_notsup, /* update */ + __wt_cursor_notsup, /* remove */ + __wt_cursor_reconfigure_notsup, /* reconfigure */ + __curbackup_close); /* close */ WT_CURSOR *cursor; WT_CURSOR_BACKUP *cb; WT_DECL_RET; diff --git a/src/cursor/cur_config.c b/src/cursor/cur_config.c index 1b2fec0eb89..e0d270e4245 100644 --- a/src/cursor/cur_config.c +++ b/src/cursor/cur_config.c @@ -27,21 +27,21 @@ __wt_curconfig_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp) { WT_CURSOR_STATIC_INIT(iface, - __wt_cursor_get_key, /* get-key */ - __wt_cursor_get_value, /* get-value */ - __wt_cursor_set_key, /* set-key */ - __wt_cursor_set_value, /* set-value */ - __wt_cursor_notsup, /* compare */ - __wt_cursor_notsup, /* equals */ - __wt_cursor_notsup, /* next */ - __wt_cursor_notsup, /* prev */ - __wt_cursor_noop, /* reset */ - __wt_cursor_notsup, /* search */ - __wt_cursor_notsup, /* search-near */ - __wt_cursor_notsup, /* insert */ - __wt_cursor_notsup, /* update */ - __wt_cursor_notsup, /* remove */ - __wt_cursor_notsup, /* reconfigure */ + __wt_cursor_get_key, /* get-key */ + __wt_cursor_get_value, /* get-value */ + __wt_cursor_set_key, /* set-key */ + __wt_cursor_set_value, /* set-value */ + __wt_cursor_compare_notsup, /* compare */ + __wt_cursor_equals_notsup, /* equals */ + __wt_cursor_notsup, /* next */ + __wt_cursor_notsup, /* prev */ + __wt_cursor_noop, /* reset */ + __wt_cursor_notsup, /* search */ + __wt_cursor_search_near_notsup, /* search-near */ + __wt_cursor_notsup, /* insert */ + __wt_cursor_notsup, /* update */ + __wt_cursor_notsup, /* remove */ + __wt_cursor_reconfigure_notsup, /* reconfigure */ __curconfig_close); WT_CURSOR_CONFIG *cconfig; WT_CURSOR *cursor; diff --git a/src/cursor/cur_ds.c b/src/cursor/cur_ds.c index 2a598c99523..804c24a3d2e 100644 --- a/src/cursor/cur_ds.c +++ b/src/cursor/cur_ds.c @@ -449,22 +449,22 @@ __wt_curds_open( const char *cfg[], WT_DATA_SOURCE *dsrc, WT_CURSOR **cursorp) { WT_CURSOR_STATIC_INIT(iface, - __wt_cursor_get_key, /* get-key */ - __wt_cursor_get_value, /* get-value */ - __wt_cursor_set_key, /* set-key */ - __wt_cursor_set_value, /* set-value */ - __curds_compare, /* compare */ - __wt_cursor_equals, /* equals */ - __curds_next, /* next */ - __curds_prev, /* prev */ - __curds_reset, /* reset */ - __curds_search, /* search */ - __curds_search_near, /* search-near */ - __curds_insert, /* insert */ - __curds_update, /* update */ - __curds_remove, /* remove */ - __wt_cursor_notsup, /* reconfigure */ - __curds_close); /* close */ + __wt_cursor_get_key, /* get-key */ + __wt_cursor_get_value, /* get-value */ + __wt_cursor_set_key, /* set-key */ + __wt_cursor_set_value, /* set-value */ + __curds_compare, /* compare */ + __wt_cursor_equals, /* equals */ + __curds_next, /* next */ + __curds_prev, /* prev */ + __curds_reset, /* reset */ + __curds_search, /* search */ + __curds_search_near, /* search-near */ + __curds_insert, /* insert */ + __curds_update, /* update */ + __curds_remove, /* remove */ + __wt_cursor_reconfigure_notsup, /* reconfigure */ + __curds_close); /* close */ WT_CONFIG_ITEM cval, metadata; WT_CURSOR *cursor, *source; WT_CURSOR_DATA_SOURCE *data_source; diff --git a/src/cursor/cur_dump.c b/src/cursor/cur_dump.c index 3324efd96cc..a7b1c98871a 100644 --- a/src/cursor/cur_dump.c +++ b/src/cursor/cur_dump.c @@ -348,22 +348,22 @@ int __wt_curdump_create(WT_CURSOR *child, WT_CURSOR *owner, WT_CURSOR **cursorp) { WT_CURSOR_STATIC_INIT(iface, - __curdump_get_key, /* get-key */ - __curdump_get_value, /* get-value */ - __curdump_set_key, /* set-key */ - __curdump_set_value, /* set-value */ - __wt_cursor_notsup, /* compare */ - __wt_cursor_notsup, /* equals */ - __curdump_next, /* next */ - __curdump_prev, /* prev */ - __curdump_reset, /* reset */ - __curdump_search, /* search */ - __curdump_search_near, /* search-near */ - __curdump_insert, /* insert */ - __curdump_update, /* update */ - __curdump_remove, /* remove */ - __wt_cursor_notsup, /* reconfigure */ - __curdump_close); /* close */ + __curdump_get_key, /* get-key */ + __curdump_get_value, /* get-value */ + __curdump_set_key, /* set-key */ + __curdump_set_value, /* set-value */ + __wt_cursor_compare_notsup, /* compare */ + __wt_cursor_equals_notsup, /* equals */ + __curdump_next, /* next */ + __curdump_prev, /* prev */ + __curdump_reset, /* reset */ + __curdump_search, /* search */ + __curdump_search_near, /* search-near */ + __curdump_insert, /* insert */ + __curdump_update, /* update */ + __curdump_remove, /* remove */ + __wt_cursor_reconfigure_notsup, /* reconfigure */ + __curdump_close); /* close */ WT_CURSOR *cursor; WT_CURSOR_DUMP *cdump; WT_CURSOR_JSON *json; diff --git a/src/cursor/cur_file.c b/src/cursor/cur_file.c index 8bbe1cc8eda..fac903b4770 100644 --- a/src/cursor/cur_file.c +++ b/src/cursor/cur_file.c @@ -397,22 +397,22 @@ __wt_curfile_create(WT_SESSION_IMPL *session, WT_CURSOR **cursorp) { WT_CURSOR_STATIC_INIT(iface, - __wt_cursor_get_key, /* get-key */ - __wt_cursor_get_value, /* get-value */ - __wt_cursor_set_key, /* set-key */ - __wt_cursor_set_value, /* set-value */ - __curfile_compare, /* compare */ - __curfile_equals, /* equals */ - __curfile_next, /* next */ - __curfile_prev, /* prev */ - __curfile_reset, /* reset */ - __curfile_search, /* search */ - __curfile_search_near, /* search-near */ - __curfile_insert, /* insert */ - __curfile_update, /* update */ - __curfile_remove, /* remove */ - __wt_cursor_reconfigure, /* reconfigure */ - __curfile_close); /* close */ + __wt_cursor_get_key, /* get-key */ + __wt_cursor_get_value, /* get-value */ + __wt_cursor_set_key, /* set-key */ + __wt_cursor_set_value, /* set-value */ + __curfile_compare, /* compare */ + __curfile_equals, /* equals */ + __curfile_next, /* next */ + __curfile_prev, /* prev */ + __curfile_reset, /* reset */ + __curfile_search, /* search */ + __curfile_search_near, /* search-near */ + __curfile_insert, /* insert */ + __curfile_update, /* update */ + __curfile_remove, /* remove */ + __wt_cursor_reconfigure, /* reconfigure */ + __curfile_close); /* close */ WT_BTREE *btree; WT_CONFIG_ITEM cval; WT_CURSOR *cursor; diff --git a/src/cursor/cur_index.c b/src/cursor/cur_index.c index 6822055131a..dbe8046ca21 100644 --- a/src/cursor/cur_index.c +++ b/src/cursor/cur_index.c @@ -386,22 +386,22 @@ __wt_curindex_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) { WT_CURSOR_STATIC_INIT(iface, - __wt_cursor_get_key, /* get-key */ - __curindex_get_value, /* get-value */ - __wt_cursor_set_key, /* set-key */ - __curindex_set_value, /* set-value */ - __curindex_compare, /* compare */ - __wt_cursor_equals, /* equals */ - __curindex_next, /* next */ - __curindex_prev, /* prev */ - __curindex_reset, /* reset */ - __curindex_search, /* search */ - __curindex_search_near, /* search-near */ - __wt_cursor_notsup, /* insert */ - __wt_cursor_notsup, /* update */ - __wt_cursor_notsup, /* remove */ - __wt_cursor_notsup, /* reconfigure */ - __curindex_close); /* close */ + __wt_cursor_get_key, /* get-key */ + __curindex_get_value, /* get-value */ + __wt_cursor_set_key, /* set-key */ + __curindex_set_value, /* set-value */ + __curindex_compare, /* compare */ + __wt_cursor_equals, /* equals */ + __curindex_next, /* next */ + __curindex_prev, /* prev */ + __curindex_reset, /* reset */ + __curindex_search, /* search */ + __curindex_search_near, /* search-near */ + __wt_cursor_notsup, /* insert */ + __wt_cursor_notsup, /* update */ + __wt_cursor_notsup, /* remove */ + __wt_cursor_reconfigure_notsup, /* reconfigure */ + __curindex_close); /* close */ WT_CURSOR_INDEX *cindex; WT_CURSOR *cursor; WT_DECL_ITEM(tmp); diff --git a/src/cursor/cur_join.c b/src/cursor/cur_join.c index 797e6e5879a..fa6dd5c32f7 100644 --- a/src/cursor/cur_join.c +++ b/src/cursor/cur_join.c @@ -583,22 +583,22 @@ __curjoin_entry_member(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, WT_CURJOIN_EXTRACTOR extract_cursor; WT_CURSOR *c; WT_CURSOR_STATIC_INIT(iface, - __wt_cursor_get_key, /* get-key */ - __wt_cursor_get_value, /* get-value */ - __wt_cursor_set_key, /* set-key */ - __wt_cursor_set_value, /* set-value */ - __wt_cursor_notsup, /* compare */ - __wt_cursor_notsup, /* equals */ - __wt_cursor_notsup, /* next */ - __wt_cursor_notsup, /* prev */ - __wt_cursor_notsup, /* reset */ - __wt_cursor_notsup, /* search */ - __wt_cursor_notsup, /* search-near */ - __curjoin_extract_insert, /* insert */ - __wt_cursor_notsup, /* update */ - __wt_cursor_notsup, /* reconfigure */ - __wt_cursor_notsup, /* remove */ - __wt_cursor_notsup); /* close */ + __wt_cursor_get_key, /* get-key */ + __wt_cursor_get_value, /* get-value */ + __wt_cursor_set_key, /* set-key */ + __wt_cursor_set_value, /* set-value */ + __wt_cursor_compare_notsup, /* compare */ + __wt_cursor_equals_notsup, /* equals */ + __wt_cursor_notsup, /* next */ + __wt_cursor_notsup, /* prev */ + __wt_cursor_notsup, /* reset */ + __wt_cursor_notsup, /* search */ + __wt_cursor_search_near_notsup, /* search-near */ + __curjoin_extract_insert, /* insert */ + __wt_cursor_notsup, /* update */ + __wt_cursor_notsup, /* remove */ + __wt_cursor_reconfigure_notsup, /* reconfigure */ + __wt_cursor_notsup); /* close */ WT_DECL_RET; WT_INDEX *idx; WT_ITEM *key, v; @@ -797,22 +797,22 @@ __wt_curjoin_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) { WT_CURSOR_STATIC_INIT(iface, - __curjoin_get_key, /* get-key */ - __curjoin_get_value, /* get-value */ - __wt_cursor_notsup, /* set-key */ - __wt_cursor_notsup, /* set-value */ - __wt_cursor_notsup, /* compare */ - __wt_cursor_notsup, /* equals */ - __curjoin_next, /* next */ - __wt_cursor_notsup, /* prev */ - __curjoin_reset, /* reset */ - __wt_cursor_notsup, /* search */ - __wt_cursor_notsup, /* search-near */ - __wt_cursor_notsup, /* insert */ - __wt_cursor_notsup, /* update */ - __wt_cursor_notsup, /* remove */ - __wt_cursor_notsup, /* reconfigure */ - __curjoin_close); /* close */ + __curjoin_get_key, /* get-key */ + __curjoin_get_value, /* get-value */ + __wt_cursor_set_key_notsup, /* set-key */ + __wt_cursor_set_value_notsup, /* set-value */ + __wt_cursor_compare_notsup, /* compare */ + __wt_cursor_equals_notsup, /* equals */ + __curjoin_next, /* next */ + __wt_cursor_notsup, /* prev */ + __curjoin_reset, /* reset */ + __wt_cursor_notsup, /* search */ + __wt_cursor_search_near_notsup, /* search-near */ + __wt_cursor_notsup, /* insert */ + __wt_cursor_notsup, /* update */ + __wt_cursor_notsup, /* remove */ + __wt_cursor_reconfigure_notsup, /* reconfigure */ + __curjoin_close); /* close */ WT_CURSOR *cursor; WT_CURSOR_JOIN *cjoin; WT_DECL_ITEM(tmp); diff --git a/src/cursor/cur_log.c b/src/cursor/cur_log.c index 3fcd8a86066..47436ac7237 100644 --- a/src/cursor/cur_log.c +++ b/src/cursor/cur_log.c @@ -347,22 +347,22 @@ __wt_curlog_open(WT_SESSION_IMPL *session, { WT_CONNECTION_IMPL *conn; WT_CURSOR_STATIC_INIT(iface, - __wt_cursor_get_key, /* get-key */ - __wt_cursor_get_value, /* get-value */ - __wt_cursor_set_key, /* set-key */ - __wt_cursor_set_value, /* set-value */ - __curlog_compare, /* compare */ - __wt_cursor_equals, /* equals */ - __curlog_next, /* next */ - __wt_cursor_notsup, /* prev */ - __curlog_reset, /* reset */ - __curlog_search, /* search */ - __wt_cursor_notsup, /* search-near */ - __wt_cursor_notsup, /* insert */ - __wt_cursor_notsup, /* update */ - __wt_cursor_notsup, /* remove */ - __wt_cursor_notsup, /* reconfigure */ - __curlog_close); /* close */ + __wt_cursor_get_key, /* get-key */ + __wt_cursor_get_value, /* get-value */ + __wt_cursor_set_key, /* set-key */ + __wt_cursor_set_value, /* set-value */ + __curlog_compare, /* compare */ + __wt_cursor_equals, /* equals */ + __curlog_next, /* next */ + __wt_cursor_notsup, /* prev */ + __curlog_reset, /* reset */ + __curlog_search, /* search */ + __wt_cursor_search_near_notsup, /* search-near */ + __wt_cursor_notsup, /* insert */ + __wt_cursor_notsup, /* update */ + __wt_cursor_notsup, /* remove */ + __wt_cursor_reconfigure_notsup, /* reconfigure */ + __curlog_close); /* close */ WT_CURSOR *cursor; WT_CURSOR_LOG *cl; WT_DECL_RET; diff --git a/src/cursor/cur_metadata.c b/src/cursor/cur_metadata.c index df66ef34ddd..df2cc3f546e 100644 --- a/src/cursor/cur_metadata.c +++ b/src/cursor/cur_metadata.c @@ -448,22 +448,22 @@ __wt_curmetadata_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) { WT_CURSOR_STATIC_INIT(iface, - __wt_cursor_get_key, /* get-key */ - __wt_cursor_get_value, /* get-value */ - __wt_cursor_set_key, /* set-key */ - __wt_cursor_set_value, /* set-value */ - __curmetadata_compare, /* compare */ - __wt_cursor_equals, /* equals */ - __curmetadata_next, /* next */ - __curmetadata_prev, /* prev */ - __curmetadata_reset, /* reset */ - __curmetadata_search, /* search */ - __curmetadata_search_near, /* search-near */ - __curmetadata_insert, /* insert */ - __curmetadata_update, /* update */ - __curmetadata_remove, /* remove */ - __wt_cursor_notsup, /* reconfigure */ - __curmetadata_close); /* close */ + __wt_cursor_get_key, /* get-key */ + __wt_cursor_get_value, /* get-value */ + __wt_cursor_set_key, /* set-key */ + __wt_cursor_set_value, /* set-value */ + __curmetadata_compare, /* compare */ + __wt_cursor_equals, /* equals */ + __curmetadata_next, /* next */ + __curmetadata_prev, /* prev */ + __curmetadata_reset, /* reset */ + __curmetadata_search, /* search */ + __curmetadata_search_near, /* search-near */ + __curmetadata_insert, /* insert */ + __curmetadata_update, /* update */ + __curmetadata_remove, /* remove */ + __wt_cursor_reconfigure_notsup, /* reconfigure */ + __curmetadata_close); /* close */ WT_CURSOR *cursor; WT_CURSOR_METADATA *mdc; WT_DECL_RET; diff --git a/src/cursor/cur_stat.c b/src/cursor/cur_stat.c index bb492c66ace..8528482a009 100644 --- a/src/cursor/cur_stat.c +++ b/src/cursor/cur_stat.c @@ -573,22 +573,22 @@ __wt_curstat_open(WT_SESSION_IMPL *session, { WT_CONNECTION_IMPL *conn; WT_CURSOR_STATIC_INIT(iface, - __curstat_get_key, /* get-key */ - __curstat_get_value, /* get-value */ - __curstat_set_key, /* set-key */ - __curstat_set_value, /* set-value */ - __wt_cursor_notsup, /* compare */ - __wt_cursor_notsup, /* equals */ - __curstat_next, /* next */ - __curstat_prev, /* prev */ - __curstat_reset, /* reset */ - __curstat_search, /* search */ - __wt_cursor_notsup, /* search-near */ - __wt_cursor_notsup, /* insert */ - __wt_cursor_notsup, /* update */ - __wt_cursor_notsup, /* remove */ - __wt_cursor_notsup, /* reconfigure */ - __curstat_close); /* close */ + __curstat_get_key, /* get-key */ + __curstat_get_value, /* get-value */ + __curstat_set_key, /* set-key */ + __curstat_set_value, /* set-value */ + __wt_cursor_compare_notsup, /* compare */ + __wt_cursor_equals_notsup, /* equals */ + __curstat_next, /* next */ + __curstat_prev, /* prev */ + __curstat_reset, /* reset */ + __curstat_search, /* search */ + __wt_cursor_search_near_notsup, /* search-near */ + __wt_cursor_notsup, /* insert */ + __wt_cursor_notsup, /* update */ + __wt_cursor_notsup, /* remove */ + __wt_cursor_reconfigure_notsup, /* reconfigure */ + __curstat_close); /* close */ WT_CONFIG_ITEM cval, sval; WT_CURSOR *cursor; WT_CURSOR_STAT *cst; diff --git a/src/cursor/cur_std.c b/src/cursor/cur_std.c index ec5a2e88331..7839971f975 100644 --- a/src/cursor/cur_std.c +++ b/src/cursor/cur_std.c @@ -8,6 +8,18 @@ #include "wt_internal.h" +/* + * __wt_cursor_noop -- + * Cursor noop. + */ +int +__wt_cursor_noop(WT_CURSOR *cursor) +{ + WT_UNUSED(cursor); + + return (0); +} + /* * __wt_cursor_notsup -- * Unsupported cursor actions. @@ -22,15 +34,83 @@ __wt_cursor_notsup(WT_CURSOR *cursor) } /* - * __wt_cursor_noop -- - * Cursor noop. + * __wt_cursor_get_value_notsup -- + * WT_CURSOR.get_value not-supported. */ int -__wt_cursor_noop(WT_CURSOR *cursor) +__wt_cursor_get_value_notsup(WT_CURSOR *cursor, ...) { - WT_UNUSED(cursor); + return (__wt_cursor_notsup(cursor)); +} - return (0); +/* + * __wt_cursor_set_key_notsup -- + * WT_CURSOR.set_key not-supported. + */ +void +__wt_cursor_set_key_notsup(WT_CURSOR *cursor, ...) +{ + (void)__wt_cursor_notsup(cursor); +} + +/* + * __wt_cursor_set_value_notsup -- + * WT_CURSOR.set_value not-supported. + */ +void +__wt_cursor_set_value_notsup(WT_CURSOR *cursor, ...) +{ + (void)__wt_cursor_notsup(cursor); +} + +/* + * __wt_cursor_compare_notsup -- + * Unsupported cursor comparison. + */ +int +__wt_cursor_compare_notsup(WT_CURSOR *a, WT_CURSOR *b, int *cmpp) +{ + WT_UNUSED(b); + WT_UNUSED(cmpp); + + return (__wt_cursor_notsup(a)); +} + +/* + * __wt_cursor_equals_notsup -- + * Unsupported cursor equality. + */ +int +__wt_cursor_equals_notsup(WT_CURSOR *cursor, WT_CURSOR *other, int *equalp) +{ + WT_UNUSED(other); + WT_UNUSED(equalp); + + return (__wt_cursor_notsup(cursor)); +} + +/* + * __wt_cursor_search_near_notsup -- + * Unsupported cursor search-near. + */ +int +__wt_cursor_search_near_notsup(WT_CURSOR *cursor, int *exact) +{ + WT_UNUSED(exact); + + return (__wt_cursor_notsup(cursor)); +} + +/* + * __wt_cursor_reconfigure_notsup -- + * Unsupported cursor reconfiguration. + */ +int +__wt_cursor_reconfigure_notsup(WT_CURSOR *cursor, const char *config) +{ + WT_UNUSED(config); + + return (__wt_cursor_notsup(cursor)); } /* @@ -47,13 +127,12 @@ __wt_cursor_set_notsup(WT_CURSOR *cursor) * cursors in a session. Reconfigure is left open in case it's possible * in the future to change these configurations. */ - cursor->compare = - (int (*)(WT_CURSOR *, WT_CURSOR *, int *))__wt_cursor_notsup; + cursor->compare = __wt_cursor_compare_notsup; cursor->next = __wt_cursor_notsup; cursor->prev = __wt_cursor_notsup; cursor->reset = __wt_cursor_noop; cursor->search = __wt_cursor_notsup; - cursor->search_near = (int (*)(WT_CURSOR *, int *))__wt_cursor_notsup; + cursor->search_near = __wt_cursor_search_near_notsup; cursor->insert = __wt_cursor_notsup; cursor->update = __wt_cursor_notsup; cursor->remove = __wt_cursor_notsup; diff --git a/src/cursor/cur_table.c b/src/cursor/cur_table.c index d986577f640..9eb88ec6fcd 100644 --- a/src/cursor/cur_table.c +++ b/src/cursor/cur_table.c @@ -79,22 +79,22 @@ __wt_apply_single_idx(WT_SESSION_IMPL *session, WT_INDEX *idx, WT_CURSOR *cur, WT_CURSOR_TABLE *ctable, int (*f)(WT_CURSOR *)) { WT_CURSOR_STATIC_INIT(iface, - __wt_cursor_get_key, /* get-key */ - __wt_cursor_get_value, /* get-value */ - __wt_cursor_set_key, /* set-key */ - __wt_cursor_set_value, /* set-value */ - __wt_cursor_notsup, /* compare */ - __wt_cursor_notsup, /* equals */ - __wt_cursor_notsup, /* next */ - __wt_cursor_notsup, /* prev */ - __wt_cursor_notsup, /* reset */ - __wt_cursor_notsup, /* search */ - __wt_cursor_notsup, /* search-near */ - __curextract_insert, /* insert */ - __wt_cursor_notsup, /* update */ - __wt_cursor_notsup, /* reconfigure */ - __wt_cursor_notsup, /* remove */ - __wt_cursor_notsup); /* close */ + __wt_cursor_get_key, /* get-key */ + __wt_cursor_get_value, /* get-value */ + __wt_cursor_set_key, /* set-key */ + __wt_cursor_set_value, /* set-value */ + __wt_cursor_compare_notsup, /* compare */ + __wt_cursor_equals_notsup, /* equals */ + __wt_cursor_notsup, /* next */ + __wt_cursor_notsup, /* prev */ + __wt_cursor_notsup, /* reset */ + __wt_cursor_notsup, /* search */ + __wt_cursor_search_near_notsup, /* search-near */ + __curextract_insert, /* insert */ + __wt_cursor_notsup, /* update */ + __wt_cursor_notsup, /* remove */ + __wt_cursor_reconfigure_notsup, /* reconfigure */ + __wt_cursor_notsup); /* close */ WT_CURSOR_EXTRACTOR extract_cursor; WT_DECL_RET; WT_ITEM key, value; @@ -842,22 +842,22 @@ __wt_curtable_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) { WT_CURSOR_STATIC_INIT(iface, - __wt_curtable_get_key, /* get-key */ - __wt_curtable_get_value, /* get-value */ - __wt_curtable_set_key, /* set-key */ - __wt_curtable_set_value, /* set-value */ - __curtable_compare, /* compare */ - __wt_cursor_equals, /* equals */ - __curtable_next, /* next */ - __curtable_prev, /* prev */ - __curtable_reset, /* reset */ - __curtable_search, /* search */ - __curtable_search_near, /* search-near */ - __curtable_insert, /* insert */ - __curtable_update, /* update */ - __curtable_remove, /* remove */ - __wt_cursor_reconfigure, /* reconfigure */ - __curtable_close); /* close */ + __wt_curtable_get_key, /* get-key */ + __wt_curtable_get_value, /* get-value */ + __wt_curtable_set_key, /* set-key */ + __wt_curtable_set_value, /* set-value */ + __curtable_compare, /* compare */ + __wt_cursor_equals, /* equals */ + __curtable_next, /* next */ + __curtable_prev, /* prev */ + __curtable_reset, /* reset */ + __curtable_search, /* search */ + __curtable_search_near, /* search-near */ + __curtable_insert, /* insert */ + __curtable_update, /* update */ + __curtable_remove, /* remove */ + __wt_cursor_reconfigure, /* reconfigure */ + __curtable_close); /* close */ WT_CONFIG_ITEM cval; WT_CURSOR *cursor; WT_CURSOR_TABLE *ctable; diff --git a/src/include/cursor.h b/src/include/cursor.h index f9bd20c8ba1..48db8b9ec23 100644 --- a/src/include/cursor.h +++ b/src/include/cursor.h @@ -31,22 +31,22 @@ NULL, /* uri */ \ NULL, /* key_format */ \ NULL, /* value_format */ \ - (int (*)(WT_CURSOR *, ...))(get_key), \ - (int (*)(WT_CURSOR *, ...))(get_value), \ - (void (*)(WT_CURSOR *, ...))(set_key), \ - (void (*)(WT_CURSOR *, ...))(set_value), \ - (int (*)(WT_CURSOR *, WT_CURSOR *, int *))(compare), \ - (int (*)(WT_CURSOR *, WT_CURSOR *, int *))(equals), \ + get_key, \ + get_value, \ + set_key, \ + set_value, \ + compare, \ + equals, \ next, \ prev, \ reset, \ search, \ - (int (*)(WT_CURSOR *, int *))(search_near), \ + search_near, \ insert, \ update, \ remove, \ close, \ - (int (*)(WT_CURSOR *, const char *))(reconfigure), \ + reconfigure, \ { NULL, NULL }, /* TAILQ_ENTRY q */ \ 0, /* recno key */ \ { 0 }, /* recno raw buffer */ \ diff --git a/src/include/extern.h b/src/include/extern.h index de66498692e..0533cb428e0 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -302,8 +302,15 @@ extern int __wt_curmetadata_open(WT_SESSION_IMPL *session, const char *uri, WT_C extern void __wt_curstat_dsrc_final(WT_CURSOR_STAT *cst); extern int __wt_curstat_init(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *curjoin, const char *cfg[], WT_CURSOR_STAT *cst); extern int __wt_curstat_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *other, const char *cfg[], WT_CURSOR **cursorp); -extern int __wt_cursor_notsup(WT_CURSOR *cursor); extern int __wt_cursor_noop(WT_CURSOR *cursor); +extern int __wt_cursor_notsup(WT_CURSOR *cursor); +extern int __wt_cursor_get_value_notsup(WT_CURSOR *cursor, ...); +extern void __wt_cursor_set_key_notsup(WT_CURSOR *cursor, ...); +extern void __wt_cursor_set_value_notsup(WT_CURSOR *cursor, ...); +extern int __wt_cursor_compare_notsup(WT_CURSOR *a, WT_CURSOR *b, int *cmpp); +extern int __wt_cursor_equals_notsup(WT_CURSOR *cursor, WT_CURSOR *other, int *equalp); +extern int __wt_cursor_search_near_notsup(WT_CURSOR *cursor, int *exact); +extern int __wt_cursor_reconfigure_notsup(WT_CURSOR *cursor, const char *config); extern void __wt_cursor_set_notsup(WT_CURSOR *cursor); extern int __wt_cursor_kv_not_set(WT_CURSOR *cursor, bool key); extern int __wt_cursor_get_key(WT_CURSOR *cursor, ...); diff --git a/src/lsm/lsm_cursor.c b/src/lsm/lsm_cursor.c index f76b2bfd9ac..0197b6481f4 100644 --- a/src/lsm/lsm_cursor.c +++ b/src/lsm/lsm_cursor.c @@ -1501,22 +1501,22 @@ __wt_clsm_open(WT_SESSION_IMPL *session, { WT_CONFIG_ITEM cval; WT_CURSOR_STATIC_INIT(iface, - __wt_cursor_get_key, /* get-key */ - __wt_cursor_get_value, /* get-value */ - __wt_cursor_set_key, /* set-key */ - __wt_cursor_set_value, /* set-value */ - __clsm_compare, /* compare */ - __wt_cursor_equals, /* equals */ - __clsm_next, /* next */ - __clsm_prev, /* prev */ - __clsm_reset, /* reset */ - __clsm_search, /* search */ - __clsm_search_near, /* search-near */ - __clsm_insert, /* insert */ - __clsm_update, /* update */ - __clsm_remove, /* remove */ - __wt_cursor_reconfigure, /* reconfigure */ - __wt_clsm_close); /* close */ + __wt_cursor_get_key, /* get-key */ + __wt_cursor_get_value, /* get-value */ + __wt_cursor_set_key, /* set-key */ + __wt_cursor_set_value, /* set-value */ + __clsm_compare, /* compare */ + __wt_cursor_equals, /* equals */ + __clsm_next, /* next */ + __clsm_prev, /* prev */ + __clsm_reset, /* reset */ + __clsm_search, /* search */ + __clsm_search_near, /* search-near */ + __clsm_insert, /* insert */ + __clsm_update, /* update */ + __clsm_remove, /* remove */ + __wt_cursor_reconfigure, /* reconfigure */ + __wt_clsm_close); /* close */ WT_CURSOR *cursor; WT_CURSOR_LSM *clsm; WT_DECL_RET; -- cgit v1.2.1 From 14a3cfe2f55469edf76c833c79a3162dea2a59ac Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 19 Feb 2016 10:16:29 -0500 Subject: WT-2410: Casting function pointers to different types Remove unused variable. --- src/session/session_api.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/session/session_api.c b/src/session/session_api.c index 38e21069009..2414229681b 100644 --- a/src/session/session_api.c +++ b/src/session/session_api.c @@ -1572,7 +1572,6 @@ __open_session(WT_CONNECTION_IMPL *conn, __session_transaction_sync_readonly }; WT_DECL_RET; - WT_SESSION *wt_session; WT_SESSION_IMPL *session, *session_ret; uint32_t i; -- cgit v1.2.1 From 1935326625f661571f322191f83c70d422486c6d Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 19 Feb 2016 11:17:43 -0500 Subject: WT-2405: test utility error handling. Add testutil_checkfmt, a macro that supports additional error output. --- test/format/backup.c | 31 +++++++++++++------------------ test/format/ops.c | 18 +++++++----------- test/format/rebalance.c | 14 +++++--------- test/format/wts.c | 19 ++++++------------- test/utility/test_util.i | 13 ++++++++++++- 5 files changed, 43 insertions(+), 52 deletions(-) diff --git a/test/format/backup.c b/test/format/backup.c index ea6306ccb27..56657940514 100644 --- a/test/format/backup.c +++ b/test/format/backup.c @@ -37,21 +37,18 @@ check_copy(void) { WT_CONNECTION *conn; WT_SESSION *session; - int ret; wts_open(g.home_backup, 0, &conn); - if ((ret = conn->open_session( - conn, NULL, NULL, &session)) != 0) - testutil_die(ret, "connection.open_session: %s", g.home_backup); + testutil_checkfmt( + conn->open_session(conn, NULL, NULL, &session), + "%s", g.home_backup); - ret = session->verify(session, g.uri, NULL); - if (ret != 0) - testutil_die(ret, - "session.verify: %s: %s", g.home_backup, g.uri); + testutil_checkfmt( + session->verify(session, g.uri, NULL), + "%s: %s", g.home_backup, g.uri); - if ((ret = conn->close(conn, NULL)) != 0) - testutil_die(ret, "connection.close: %s", g.home_backup); + testutil_checkfmt(conn->close(conn, NULL), "%s", g.home_backup); } /* @@ -63,14 +60,12 @@ copy_file(const char *name) { size_t len; char *cmd; - int ret; len = strlen(g.home) + strlen(g.home_backup) + strlen(name) * 2 + 20; cmd = dmalloc(len); (void)snprintf(cmd, len, "cp %s/%s %s/%s", g.home, name, g.home_backup, name); - if ((ret = system(cmd)) != 0) - testutil_die(ret, "backup copy: %s", cmd); + testutil_checkfmt(system(cmd), "backup copy: %s", cmd); free(cmd); } @@ -116,8 +111,9 @@ backup(void *arg) testutil_check(pthread_rwlock_wrlock(&g.backup_lock)); /* Re-create the backup directory. */ - if ((ret = system(g.home_backup_init)) != 0) - testutil_die(ret, "backup directory creation failed"); + testutil_checkfmt( + system(g.home_backup_init), + "%s", "backup directory creation failed"); /* * open_cursor can return EBUSY if a metadata operation is @@ -130,9 +126,8 @@ backup(void *arg) testutil_die(ret, "session.open_cursor: backup"); while ((ret = backup_cursor->next(backup_cursor)) == 0) { - if ((ret = - backup_cursor->get_key(backup_cursor, &key)) != 0) - testutil_die(ret, "cursor.get_key"); + testutil_check( + backup_cursor->get_key(backup_cursor, &key)); copy_file(key); } diff --git a/test/format/ops.c b/test/format/ops.c index d248736bb57..5d66f4d5391 100644 --- a/test/format/ops.c +++ b/test/format/ops.c @@ -229,7 +229,7 @@ ops(void *arg) uint32_t op; uint8_t *keybuf, *valbuf; u_int np; - int ckpt_available, dir, insert, intxn, notfound, readonly, ret; + int ckpt_available, dir, insert, intxn, notfound, readonly; char *ckpt_config, ckpt_name[64]; tinfo = arg; @@ -352,11 +352,9 @@ ops(void *arg) testutil_check( pthread_rwlock_wrlock(&g.backup_lock)); - if ((ret = - session->checkpoint(session, ckpt_config)) != 0) - testutil_die(ret, "session.checkpoint%s%s", - ckpt_config == NULL ? "" : ": ", - ckpt_config == NULL ? "" : ckpt_config); + testutil_checkfmt( + session->checkpoint(session, ckpt_config), + "%s", ckpt_config == NULL ? "" : ckpt_config); if (ckpt_config != NULL) testutil_check( @@ -554,7 +552,6 @@ wts_read_scan(void) WT_SESSION *session; uint64_t cnt, last_cnt; uint8_t *keybuf; - int ret; conn = g.wts_conn; @@ -578,8 +575,8 @@ wts_read_scan(void) } key.data = keybuf; - if ((ret = read_row(cursor, &key, cnt, 0)) != 0) - testutil_die(ret, "read_scan"); + testutil_checkfmt( + read_row(cursor, &key, cnt, 0), "%s", "read_scan"); } testutil_check(session->close(session, NULL)); @@ -1074,8 +1071,7 @@ col_insert(TINFO *tinfo, return (WT_ROLLBACK); testutil_die(ret, "cursor.insert"); } - if ((ret = cursor->get_key(cursor, &keyno)) != 0) - testutil_die(ret, "cursor.get_key"); + testutil_check(cursor->get_key(cursor, &keyno)); *keynop = (uint32_t)keyno; table_append(keyno); /* Extend the object. */ diff --git a/test/format/rebalance.c b/test/format/rebalance.c index eea4735a530..d35dcec1d53 100644 --- a/test/format/rebalance.c +++ b/test/format/rebalance.c @@ -33,7 +33,6 @@ wts_rebalance(void) { WT_CONNECTION *conn; WT_SESSION *session; - int ret; char cmd[1024]; if (g.c_rebalance == 0) @@ -45,8 +44,7 @@ wts_rebalance(void) (void)snprintf(cmd, sizeof(cmd), "../../wt -h %s dump -f %s/rebalance.orig %s", g.home, g.home, g.uri); - if ((ret = system(cmd)) != 0) - testutil_die(ret, "command failed: %s", cmd); + testutil_checkfmt(system(cmd), "command failed: %s", cmd); /* Rebalance, then verify the object. */ wts_reopen(); @@ -56,8 +54,8 @@ wts_rebalance(void) (void)g.wt_api->msg_printf(g.wt_api, session, "=============== rebalance start ==============="); - if ((ret = session->rebalance(session, g.uri, NULL)) != 0) - testutil_die(ret, "session.rebalance: %s: %s", g.uri); + testutil_checkfmt( + session->rebalance(session, g.uri, NULL), "%s", g.uri); if (g.logging != 0) (void)g.wt_api->msg_printf(g.wt_api, session, @@ -70,13 +68,11 @@ wts_rebalance(void) (void)snprintf(cmd, sizeof(cmd), "../../wt -h %s dump -f %s/rebalance.new %s", g.home, g.home, g.uri); - if ((ret = system(cmd)) != 0) - testutil_die(ret, "command failed: %s", cmd); + testutil_checkfmt(system(cmd), "command failed: %s", cmd); /* Compare the old/new versions of the object. */ (void)snprintf(cmd, sizeof(cmd), "cmp %s/rebalance.orig %s/rebalance.new > /dev/null", g.home, g.home); - if ((ret = system(cmd)) != 0) - testutil_die(ret, "command failed: %s", cmd); + testutil_checkfmt(system(cmd), "command failed: %s", cmd); } diff --git a/test/format/wts.c b/test/format/wts.c index f1b8eaff3c3..a0e57dc2bee 100644 --- a/test/format/wts.c +++ b/test/format/wts.c @@ -234,8 +234,8 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp) if (strstr(config, "direct_io") != NULL) g.c_backups = 0; - if ((ret = wiredtiger_open(home, &event_handler, config, &conn)) != 0) - testutil_die(ret, "wiredtiger_open: %s", home); + testutil_checkfmt( + wiredtiger_open(home, &event_handler, config, &conn), "%s", home); if (set_api) g.wt_api = conn->get_extension_api(conn); @@ -271,11 +271,8 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp) void wts_reopen(void) { - int ret; - - if ((ret = wiredtiger_open(g.home, - &event_handler, g.wiredtiger_open_config, &g.wts_conn)) != 0) - testutil_die(ret, "wiredtiger_open: %s", g.home); + testutil_checkfmt(wiredtiger_open(g.home, &event_handler, + g.wiredtiger_open_config, &g.wts_conn), "%s", g.home); } /* @@ -288,7 +285,6 @@ wts_create(void) WT_CONNECTION *conn; WT_SESSION *session; uint32_t maxintlpage, maxintlkey, maxleafpage, maxleafkey, maxleafvalue; - int ret; char config[4096], *end, *p; conn = g.wts_conn; @@ -439,8 +435,7 @@ wts_create(void) * Create the underlying store. */ testutil_check(conn->open_session(conn, NULL, NULL, &session)); - if ((ret = session->create(session, g.uri, config)) != 0) - testutil_die(ret, "session.create: %s", g.uri); + testutil_checkfmt(session->create(session, g.uri, config), "%s", g.uri); testutil_check(session->close(session, NULL)); } @@ -464,7 +459,6 @@ wts_dump(const char *tag, int dump_bdb) { #ifdef HAVE_BERKELEY_DB size_t len; - int ret; char *cmd; /* @@ -489,8 +483,7 @@ wts_dump(const char *tag, int dump_bdb) g.uri == NULL ? "" : "-n", g.uri == NULL ? "" : g.uri); - if ((ret = system(cmd)) != 0) - testutil_die(ret, "%s: dump comparison failed", tag); + testutil_checkfmt(system(cmd), "%s: dump comparison failed", tag); free(cmd); #else (void)tag; /* [-Wunused-variable] */ diff --git a/test/utility/test_util.i b/test/utility/test_util.i index f1f0768dfcf..bd7956e460f 100644 --- a/test/utility/test_util.i +++ b/test/utility/test_util.i @@ -76,7 +76,7 @@ testutil_die(int e, const char *fmt, ...) } /* - * check -- + * testutil_check -- * Complain and quit if a function call fails. */ #define testutil_check(call) do { \ @@ -85,6 +85,17 @@ testutil_die(int e, const char *fmt, ...) testutil_die(__r, "%s/%d: %s", __func__, __LINE__, #call);\ } while (0) +/* + * testutil_checkfmt -- + * Complain and quit if a function call fails, with additional arguments. + */ +#define testutil_checkfmt(call, fmt, ...) do { \ + int __r; \ + if ((__r = (call)) != 0) \ + testutil_die(__r, "%s/%d: %s: " fmt, \ + __func__, __LINE__, #call, __VA_ARGS__); \ +} while (0) + /* * testutil_work_dir_from_path -- * Takes a buffer, its size and the intended work directory. -- cgit v1.2.1 From 99add10800fda24b81d0b8143e0e33acef3b641a Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Fri, 19 Feb 2016 11:29:54 -0500 Subject: WT-2349 Free merge_cfg when we're done with it. --- src/conn/conn_api.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 876cefd8496..84f8ad34e4e 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -1887,7 +1887,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, WT_DECL_RET; const WT_NAME_FLAG *ft; WT_SESSION_IMPL *session; - bool config_base_set; + bool config_base_set, free_merge_cfg; const char *enc_cfg[] = { NULL, NULL }, *merge_cfg; char version[64]; @@ -1900,6 +1900,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, conn = NULL; session = NULL; + free_merge_cfg = false; WT_RET(__wt_library_init()); WT_RET(__wt_calloc_one(NULL, &conn)); @@ -2025,7 +2026,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, */ __conn_config_readonly(cfg); WT_ERR(__wt_config_merge(session, cfg, NULL, &conn->cfg)); - __wt_free(session, merge_cfg); + free_merge_cfg = true; } else conn->cfg = merge_cfg; @@ -2200,6 +2201,8 @@ err: /* Discard the scratch buffers. */ __wt_scr_free(session, &i2); __wt_scr_free(session, &i3); + if (free_merge_cfg) + __wt_free(session, merge_cfg); /* * We may have allocated scratch memory when using the dummy session or * the subsequently created real session, and we don't want to tie down -- cgit v1.2.1 From 0649731ce76cf4af4f8be388b2f3f3b002341351 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Fri, 19 Feb 2016 11:44:52 -0500 Subject: WT-2349 Modify use of cfg. --- src/conn/conn_api.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 84f8ad34e4e..1274c203649 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -1887,7 +1887,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, WT_DECL_RET; const WT_NAME_FLAG *ft; WT_SESSION_IMPL *session; - bool config_base_set, free_merge_cfg; + bool config_base_set; const char *enc_cfg[] = { NULL, NULL }, *merge_cfg; char version[64]; @@ -1900,7 +1900,6 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, conn = NULL; session = NULL; - free_merge_cfg = false; WT_RET(__wt_library_init()); WT_RET(__wt_calloc_one(NULL, &conn)); @@ -2025,11 +2024,19 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, * read-only are tested in their individual locations later. */ __conn_config_readonly(cfg); - WT_ERR(__wt_config_merge(session, cfg, NULL, &conn->cfg)); - free_merge_cfg = true; + ret = __wt_config_merge(session, cfg, NULL, &conn->cfg); + __wt_free(session, merge_cfg); + WT_ERR(ret); } else conn->cfg = merge_cfg; + /* + * At this point we've merged the configuration stack and the final + * stack resides in conn->cfg. Set up the cfg array to use that now. + */ + cfg[0] = conn->cfg; + cfg[1] = NULL; + /* * Configuration ... * @@ -2201,8 +2208,6 @@ err: /* Discard the scratch buffers. */ __wt_scr_free(session, &i2); __wt_scr_free(session, &i3); - if (free_merge_cfg) - __wt_free(session, merge_cfg); /* * We may have allocated scratch memory when using the dummy session or * the subsequently created real session, and we don't want to tie down -- cgit v1.2.1 From 2674be5db00c1220554ec1aa8da38a19fc57d373 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Fri, 19 Feb 2016 11:57:39 -0500 Subject: WT-2349 Revert last change. --- src/conn/conn_api.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 1274c203649..25db2379349 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -1887,7 +1887,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, WT_DECL_RET; const WT_NAME_FLAG *ft; WT_SESSION_IMPL *session; - bool config_base_set; + bool config_base_set, free_merge_cfg; const char *enc_cfg[] = { NULL, NULL }, *merge_cfg; char version[64]; @@ -1899,6 +1899,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, conn = NULL; session = NULL; + free_merge_cfg = false; WT_RET(__wt_library_init()); @@ -2024,19 +2025,11 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, * read-only are tested in their individual locations later. */ __conn_config_readonly(cfg); - ret = __wt_config_merge(session, cfg, NULL, &conn->cfg); - __wt_free(session, merge_cfg); - WT_ERR(ret); + free_merge_cfg = true; + WT_ERR(__wt_config_merge(session, cfg, NULL, &conn->cfg)); } else conn->cfg = merge_cfg; - /* - * At this point we've merged the configuration stack and the final - * stack resides in conn->cfg. Set up the cfg array to use that now. - */ - cfg[0] = conn->cfg; - cfg[1] = NULL; - /* * Configuration ... * @@ -2208,6 +2201,8 @@ err: /* Discard the scratch buffers. */ __wt_scr_free(session, &i2); __wt_scr_free(session, &i3); + if (free_merge_cfg) + __wt_free(session, merge_cfg); /* * We may have allocated scratch memory when using the dummy session or * the subsequently created real session, and we don't want to tie down -- cgit v1.2.1 From d1bdd269f6e0bb6c5b798d95bc93008bfe4e9e66 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Fri, 19 Feb 2016 14:20:24 -0500 Subject: WT-2349 Remove boolean. --- src/conn/conn_api.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 25db2379349..bb67185f5c9 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -1887,7 +1887,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, WT_DECL_RET; const WT_NAME_FLAG *ft; WT_SESSION_IMPL *session; - bool config_base_set, free_merge_cfg; + bool config_base_set; const char *enc_cfg[] = { NULL, NULL }, *merge_cfg; char version[64]; @@ -1899,7 +1899,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, conn = NULL; session = NULL; - free_merge_cfg = false; + merge_cfg = NULL; WT_RET(__wt_library_init()); @@ -2025,10 +2025,11 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, * read-only are tested in their individual locations later. */ __conn_config_readonly(cfg); - free_merge_cfg = true; WT_ERR(__wt_config_merge(session, cfg, NULL, &conn->cfg)); - } else + } else { conn->cfg = merge_cfg; + merge_cfg = NULL; + } /* * Configuration ... @@ -2201,8 +2202,7 @@ err: /* Discard the scratch buffers. */ __wt_scr_free(session, &i2); __wt_scr_free(session, &i3); - if (free_merge_cfg) - __wt_free(session, merge_cfg); + __wt_free(session, merge_cfg); /* * We may have allocated scratch memory when using the dummy session or * the subsequently created real session, and we don't want to tie down -- cgit v1.2.1 From 2b6a030eb30fc6744b645d45c8435ddc5ce2cdf1 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Fri, 19 Feb 2016 15:11:08 -0500 Subject: WT-2412 Truncate should not return EBUSY any longer. --- src/include/wiredtiger.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index 91d3ea03617..3fdc966d34b 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -1409,7 +1409,7 @@ struct __wt_session { * if NULL, the truncate continues to the end of the * object * @configempty{WT_SESSION.truncate, see dist/api_data.py} - * @ebusy_errors + * @errors */ int __F(truncate)(WT_SESSION *session, const char *name, -- cgit v1.2.1 From e68f5a2e36cebab58591cb9013224984b3690bdf Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Fri, 19 Feb 2016 15:24:25 -0500 Subject: WT-2349 Clean up casts. --- bench/wtperf/config.c | 4 ++-- bench/wtperf/wtperf.c | 20 ++++++++++---------- bench/wtperf/wtperf.h | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/bench/wtperf/config.c b/bench/wtperf/config.c index 9be4c2b880e..5af2e977a25 100644 --- a/bench/wtperf/config.c +++ b/bench/wtperf/config.c @@ -136,9 +136,9 @@ config_free(CONFIG *cfg) cleanup_truncate_config(cfg); free(cfg->base_uri); free(cfg->ckptthreads); - free((char *)cfg->partial_config); + free(cfg->partial_config); free(cfg->popthreads); - free((char *)cfg->reopen_config); + free(cfg->reopen_config); free(cfg->workers); free(cfg->workload); } diff --git a/bench/wtperf/wtperf.c b/bench/wtperf/wtperf.c index c5c2d294708..38f7688a47a 100644 --- a/bench/wtperf/wtperf.c +++ b/bench/wtperf/wtperf.c @@ -1533,8 +1533,8 @@ close_reopen(CONFIG *cfg) lprintf(cfg, ret, 0, "Closing the connection failed"); return (ret); } - if ((ret = wiredtiger_open( - cfg->home, NULL, cfg->reopen_config, &cfg->conn)) != 0) { + if ((ret = wiredtiger_open(cfg->home, + NULL, (const char *)cfg->reopen_config, &cfg->conn)) != 0) { lprintf(cfg, ret, 0, "Re-opening the connection failed"); return (ret); } @@ -1810,8 +1810,8 @@ create_tables(CONFIG *cfg) for (i = 0; i < cfg->table_count; i++) { if (cfg->log_partial && i > 0) { - if (((ret = session->create(session, - cfg->uris[i], cfg->partial_config)) != 0)) { + if (((ret = session->create(session, cfg->uris[i], + (const char *)cfg->partial_config)) != 0)) { lprintf(cfg, ret, 0, "Error creating table %s", cfg->uris[i]); return (ret); @@ -2298,8 +2298,8 @@ main(int argc, char *argv[]) req_len = strlen(cfg->table_config) + strlen(LOG_PARTIAL_CONFIG) + 1; cfg->partial_config = dcalloc(req_len, 1); - snprintf((char *)cfg->partial_config, req_len, "%s%s", - (char *)cfg->table_config, LOG_PARTIAL_CONFIG); + snprintf(cfg->partial_config, req_len, "%s%s", + cfg->table_config, LOG_PARTIAL_CONFIG); } /* * Set the config for reopen. If readonly add in that string. @@ -2312,11 +2312,11 @@ main(int argc, char *argv[]) req_len = strlen(cfg->conn_config) + 1; cfg->reopen_config = dcalloc(req_len, 1); if (cfg->readonly) - snprintf((char *)cfg->reopen_config, req_len, "%s%s", - (char *)cfg->conn_config, READONLY_CONFIG); + snprintf(cfg->reopen_config, req_len, "%s%s", + cfg->conn_config, READONLY_CONFIG); else - snprintf((char *)cfg->reopen_config, req_len, "%s", - (char *)cfg->conn_config); + snprintf(cfg->reopen_config, req_len, "%s", + cfg->conn_config); /* Sanity-check the configuration. */ if ((ret = config_sanity(cfg)) != 0) diff --git a/bench/wtperf/wtperf.h b/bench/wtperf/wtperf.h index 0b0fb88a99c..65ee4963276 100644 --- a/bench/wtperf/wtperf.h +++ b/bench/wtperf/wtperf.h @@ -146,8 +146,8 @@ typedef struct { struct __config { /* Configuration structure */ const char *home; /* WiredTiger home */ const char *monitor_dir; /* Monitor output dir */ - const char *partial_config; /* Config string for partial logging */ - const char *reopen_config; /* Config string for conn reopen */ + char *partial_config; /* Config string for partial logging */ + char *reopen_config; /* Config string for conn reopen */ char *base_uri; /* Object URI */ char **uris; /* URIs if multiple tables */ const char *helium_mount; /* Optional Helium mount point */ -- cgit v1.2.1 From 4d7ac74204e3ec66bb6eeeb63886c9adf8c61ed7 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sat, 20 Feb 2016 11:15:42 -0500 Subject: WT-2349: Add ability to open databases read-only Remove a couple of "(const char *)" casts I don't think we need. --- bench/wtperf/wtperf.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bench/wtperf/wtperf.c b/bench/wtperf/wtperf.c index 38f7688a47a..b6e10762d8c 100644 --- a/bench/wtperf/wtperf.c +++ b/bench/wtperf/wtperf.c @@ -1533,8 +1533,8 @@ close_reopen(CONFIG *cfg) lprintf(cfg, ret, 0, "Closing the connection failed"); return (ret); } - if ((ret = wiredtiger_open(cfg->home, - NULL, (const char *)cfg->reopen_config, &cfg->conn)) != 0) { + if ((ret = wiredtiger_open( + cfg->home, NULL, cfg->reopen_config, &cfg->conn)) != 0) { lprintf(cfg, ret, 0, "Re-opening the connection failed"); return (ret); } @@ -1810,8 +1810,8 @@ create_tables(CONFIG *cfg) for (i = 0; i < cfg->table_count; i++) { if (cfg->log_partial && i > 0) { - if (((ret = session->create(session, cfg->uris[i], - (const char *)cfg->partial_config)) != 0)) { + if (((ret = session->create(session, + cfg->uris[i], cfg->partial_config)) != 0)) { lprintf(cfg, ret, 0, "Error creating table %s", cfg->uris[i]); return (ret); -- cgit v1.2.1 From b891d6f43c61afeaf8f8f7c847a6e3d6d02c4b11 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sun, 21 Feb 2016 11:37:55 -0500 Subject: WT-2376: Modules should compile without including wt_internal.h header file Exit silently if GNU nm not found, reduce the noise on configuration. --- dist/s_export | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dist/s_export b/dist/s_export index 1212b5b2c1f..429e7667c45 100755 --- a/dist/s_export +++ b/dist/s_export @@ -12,10 +12,7 @@ Darwin) *) # We require GNU nm, which may not be installed. type nm > /dev/null 2>&1 && - (nm --version | grep 'GNU nm') > /dev/null 2>&1 || { - echo 'skipped: GNU nm not found' - exit 0 - } + (nm --version | grep 'GNU nm') > /dev/null 2>&1 || exit 0 NM='nm --extern-only --defined-only --print-file-name $f' ;; esac -- cgit v1.2.1 From 9dcd1205d2b27414fcaba06afba0aee53970e30c Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sun, 21 Feb 2016 11:38:44 -0500 Subject: WT-2376: Modules should compile without including wt_internal.h header file Don't complain about the compression init functions, lz4_extension_init, snappy_extension_init and zlib_extension_init. --- dist/s_export.list | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dist/s_export.list b/dist/s_export.list index c7f088bc2d5..abddd60c462 100644 --- a/dist/s_export.list +++ b/dist/s_export.list @@ -1,4 +1,6 @@ # List of OK external symbols. +lz4_extension_init +snappy_extension_init wiredtiger_config_parser_open wiredtiger_config_validate wiredtiger_open @@ -18,3 +20,4 @@ wiredtiger_unpack_start wiredtiger_unpack_str wiredtiger_unpack_uint wiredtiger_version +zlib_extension_init -- cgit v1.2.1 From 9905036c7033cc5cc0970310a1f82ba2bdbdf24d Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sun, 21 Feb 2016 11:39:28 -0500 Subject: WT-2376: Modules should compile without including wt_internal.h header file Don't include in the compression functions: inline the necessary swap routines, they're pretty simple and there's unlikely to be any performance issue here, we're swapping only a very small amount of information per compression/decompression. --- ext/compressors/lz4/lz4_compress.c | 41 ++++++++++++++++++++++++-------- ext/compressors/snappy/snappy_compress.c | 31 ++++++++++++++++++++---- 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/ext/compressors/lz4/lz4_compress.c b/ext/compressors/lz4/lz4_compress.c index 062307b721a..35159d0fa76 100644 --- a/ext/compressors/lz4/lz4_compress.c +++ b/ext/compressors/lz4/lz4_compress.c @@ -26,13 +26,15 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include - #include #include #include #include +#include +#include +#include + /* Local compressor structure. */ typedef struct { WT_COMPRESSOR compressor; /* Must come first */ @@ -62,6 +64,22 @@ typedef struct { uint32_t unused; /* Guaranteed to be 0 */ } LZ4_PREFIX; +#ifdef WORDS_BIGENDIAN +/* + * lz4_bswap32 -- + * 32-bit unsigned little-endian to/from big-endian value. + */ +static inline uint32_t +lz4_bswap32(uint32_t v) +{ + return ( + ((v << 24) & 0xff000000) | + ((v << 8) & 0x00ff0000) | + ((v >> 8) & 0x0000ff00) | + ((v >> 24) & 0x000000ff) + ); +} + /* * lz4_prefix_swap -- * The additional information is written in little-endian format, handle @@ -70,15 +88,12 @@ typedef struct { static inline void lz4_prefix_swap(LZ4_PREFIX *prefix) { -#ifdef WORDS_BIGENDIAN - prefix->compressed_len = __wt_bswap32(prefix->compressed_len); - prefix->uncompressed_len = __wt_bswap32(prefix->uncompressed_len); - prefix->useful_len = __wt_bswap32(prefix->useful_len); - prefix->unused = __wt_bswap32(prefix->unused); -#else - WT_UNUSED(prefix); -#endif + prefix->compressed_len = lz4_bswap32(prefix->compressed_len); + prefix->uncompressed_len = lz4_bswap32(prefix->uncompressed_len); + prefix->useful_len = lz4_bswap32(prefix->useful_len); + prefix->unused = lz4_bswap32(prefix->unused); } +#endif /* * lz4_error -- @@ -127,7 +142,9 @@ lz4_compress(WT_COMPRESSOR *compressor, WT_SESSION *session, prefix.uncompressed_len = (uint32_t)src_len; prefix.useful_len = (uint32_t)src_len; prefix.unused = 0; +#ifdef WORDS_BIGENDIAN lz4_prefix_swap(&prefix); +#endif memcpy(dst, &prefix, sizeof(LZ4_PREFIX)); *result_lenp = (size_t)lz4_len + sizeof(LZ4_PREFIX); @@ -163,7 +180,9 @@ lz4_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, * decompressed bytes to return from the start of the source buffer. */ memcpy(&prefix, src, sizeof(LZ4_PREFIX)); +#ifdef WORDS_BIGENDIAN lz4_prefix_swap(&prefix); +#endif /* * Decompress, starting after the prefix bytes. Use safe decompression: @@ -278,7 +297,9 @@ lz4_compress_raw(WT_COMPRESSOR *compressor, WT_SESSION *session, prefix.uncompressed_len = (uint32_t)sourceSize; prefix.useful_len = offsets[slot]; prefix.unused = 0; +#ifdef WORDS_BIGENDIAN lz4_prefix_swap(&prefix); +#endif memcpy(dst, &prefix, sizeof(LZ4_PREFIX)); *result_slotsp = slot; diff --git a/ext/compressors/snappy/snappy_compress.c b/ext/compressors/snappy/snappy_compress.c index fcefb8bb575..a95dd665b67 100644 --- a/ext/compressors/snappy/snappy_compress.c +++ b/ext/compressors/snappy/snappy_compress.c @@ -26,13 +26,15 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include - #include #include #include #include +#include +#include +#include + /* Local compressor structure. */ typedef struct { WT_COMPRESSOR compressor; /* Must come first */ @@ -40,6 +42,27 @@ typedef struct { WT_EXTENSION_API *wt_api; /* Extension API */ } SNAPPY_COMPRESSOR; +#ifdef WORDS_BIGENDIAN +/* + * wt_bswap64 -- + * 64-bit unsigned little-endian to/from big-endian value. + */ +static inline uint64_t +wt_bswap64(uint64_t v) +{ + return ( + ((v << 56) & 0xff00000000000000UL) | + ((v << 40) & 0x00ff000000000000UL) | + ((v << 24) & 0x0000ff0000000000UL) | + ((v << 8) & 0x000000ff00000000UL) | + ((v >> 8) & 0x00000000ff000000UL) | + ((v >> 24) & 0x0000000000ff0000UL) | + ((v >> 40) & 0x000000000000ff00UL) | + ((v >> 56) & 0x00000000000000ffUL) + ); +} +#endif + /* * wt_snappy_error -- * Output an error message, and return a standard error code. @@ -109,7 +132,7 @@ wt_snappy_compress(WT_COMPRESSOR *compressor, WT_SESSION *session, * Store the value in little-endian format. */ #ifdef WORDS_BIGENDIAN - snaplen = __wt_bswap64(snaplen); + snaplen = wt_bswap64(snaplen); #endif *(size_t *)dst = snaplen; } else @@ -142,7 +165,7 @@ wt_snappy_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, */ snaplen = *(size_t *)src; #ifdef WORDS_BIGENDIAN - snaplen = __wt_bswap64(snaplen); + snaplen = wt_bswap64(snaplen); #endif if (snaplen + sizeof(size_t) > src_len) { (void)wt_api->err_printf(wt_api, -- cgit v1.2.1 From 9fac72869825dfd4c29241c29759a9e5e017e34e Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sun, 21 Feb 2016 11:52:59 -0500 Subject: WT-2376: WT-2376: Modules should compile without including wt_internal.h header The list of exported symbols in dist/s_export.list is used to build the list of Windows symbols, explicitly discard the compression XXX_init symbols in the s_export script. --- dist/s_export | 4 +++- dist/s_export.list | 3 --- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/dist/s_export b/dist/s_export index 429e7667c45..8a2c701d27f 100755 --- a/dist/s_export +++ b/dist/s_export @@ -25,7 +25,9 @@ check() sed 's/.* //' | egrep -v '^__wt') | sort | - uniq -u > $t + uniq -u | + egrep -v \ + 'zlib_extension_init|lz4_extension_init|snappy_extension_init' > $t test -s $t && { echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" diff --git a/dist/s_export.list b/dist/s_export.list index abddd60c462..c7f088bc2d5 100644 --- a/dist/s_export.list +++ b/dist/s_export.list @@ -1,6 +1,4 @@ # List of OK external symbols. -lz4_extension_init -snappy_extension_init wiredtiger_config_parser_open wiredtiger_config_validate wiredtiger_open @@ -20,4 +18,3 @@ wiredtiger_unpack_start wiredtiger_unpack_str wiredtiger_unpack_uint wiredtiger_version -zlib_extension_init -- cgit v1.2.1 From c4b6096c81a502e5f5d345e3af92817f49afe229 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sun, 21 Feb 2016 12:41:04 -0500 Subject: WT-2107: Add example code including an event handler Add an event handler example to the error handling docs. --- examples/c/ex_all.c | 66 +++++++++++++++++++++++++++++++++++++++++++++ src/docs/error-handling.dox | 11 ++++++++ src/docs/spell.ok | 5 ++-- src/include/wiredtiger.in | 6 ++--- 4 files changed, 83 insertions(+), 5 deletions(-) diff --git a/examples/c/ex_all.c b/examples/c/ex_all.c index 418c99ad6a3..ef77f10f6de 100644 --- a/examples/c/ex_all.c +++ b/examples/c/ex_all.c @@ -36,6 +36,7 @@ #include #include +#include #include #include #include @@ -1044,6 +1045,57 @@ backup(WT_SESSION *session) return (ret); } +#define APPLICATION_ERROR 1 +#define APPLICATION_INFO 2 +static int +application_logging(int which, const char *message) +{ + (void)which; + (void)message; + return (0); +} + +/*! [Function event_handler] */ + +/* + * handle_error -- + * WiredTiger error handler. + */ +int +wiredtiger_handle_error( + WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *error) +{ + int ret; + + (void)(handler); /* Unused variables */ + + /* Timestamp and log the error message. */ + ret = application_logging(APPLICATION_ERROR, error); + + /* Copy and flush the message to stderr. */ + if (fprintf(stderr, "%p: %s\n", session, error) < 0 && ret == 0) + ret = EIO; + if (fflush(stderr) != 0 && ret == 0) + ret = errno; + return (ret); +} + +/* + * handle_message -- + * WiredTiger message handler. + */ +int +wiredtiger_handle_message( + WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *message) +{ + (void)(handler); /* Unused variables */ + (void)(session); /* Unused variables */ + + /* Timestamp and log the informational message. */ + return (application_logging(APPLICATION_INFO, message)); +} +/*! [Function event_handler] */ + int main(void) { @@ -1113,6 +1165,20 @@ main(void) if (ret == 0) (void)conn->close(conn, NULL); #endif + { + /*! [Configure event_handler] */ + WT_EVENT_HANDLER event_handler; + + event_handler.handle_error = wiredtiger_handle_error; + event_handler.handle_message = wiredtiger_handle_message; + event_handler.handle_progress = NULL; + event_handler.handle_close = NULL; + + ret = wiredtiger_open(home, &event_handler, "create", &conn); + /*! [Configure event_handler] */ + if (ret == 0) + (void)conn->close(conn, NULL); + } /*! [Configure file_extend] */ ret = wiredtiger_open( diff --git a/src/docs/error-handling.dox b/src/docs/error-handling.dox index d1291e38ff0..08fecd7f709 100644 --- a/src/docs/error-handling.dox +++ b/src/docs/error-handling.dox @@ -61,8 +61,19 @@ associated with any WiredTiger, ISO C99, or POSIX 1003.1-2001 function: @snippet ex_all.c Display an error @m_if{c} +@section error_handling_event Error handling using the WT_EVENT_HANDLER + More complex error handling can be configured by passing an implementation of WT_EVENT_HANDLER to ::wiredtiger_open or WT_CONNECTION::open_session. + +For example, both informational and error messages might be passed to an +application-specific logging function that added a timestamp and logged +the message to a file, and error messages might additionally be output to +the \c stderr file stream. + +@snippet ex_all.c Function event_handler +@snippet ex_all.c Configure event_handler + @m_endif */ diff --git a/src/docs/spell.ok b/src/docs/spell.ok index 2c4d47f1790..efc306568cd 100644 --- a/src/docs/spell.ok +++ b/src/docs/spell.ok @@ -13,7 +13,6 @@ Christoph Collet's Coverity Coverity's -crc DB's DBTs Datastore @@ -81,7 +80,6 @@ Seward's SiH TXT URIs -vpmsum WiredTiger WiredTiger's WiredTigerCheckpoint @@ -155,6 +153,7 @@ control's copydoc cpp crashless +crc cursortype customerABC cv @@ -424,6 +423,7 @@ src ssd startsync statlog +stderr str strerror strftime @@ -480,6 +480,7 @@ valuefmt vec versa vm +vpmsum warmup whitespace wiredtiger diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index 3fdc966d34b..338b95c8208 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -2477,7 +2477,7 @@ struct __wt_async_callback { struct __wt_event_handler { /*! * Callback to handle error messages; by default, error messages are - * written to the stderr stream. + * written to the stderr stream. See @ref error_handling. * * Errors that require the application to exit and restart will have * their \c error value set to \c WT_PANIC. The application can exit @@ -2500,7 +2500,7 @@ struct __wt_event_handler { /*! * Callback to handle informational messages; by default, informational - * messages are written to the stdout stream. + * messages are written to the stdout stream. See @ref error_handling. * * Message handler returns are not ignored: if the handler returns * non-zero, the error may cause the WiredTiger function posting the @@ -2516,7 +2516,7 @@ struct __wt_event_handler { /*! * Callback to handle progress messages; by default, no progress - * messages are written. + * messages are written. See @ref error_handling. * * Progress handler returns are not ignored: if the handler returns * non-zero, the error may cause the WiredTiger function posting the -- cgit v1.2.1 From 9e2430e56c0b578aa5942ecc31353f351d05864b Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sun, 21 Feb 2016 13:23:51 -0500 Subject: WT-2107: Add example code including an event handler Clean up thread-safe vs. global "display error strings" documentation: the global code snippet was wrong, there wasn't a java thread-safe snippet, add a mention of the thread-safe API to the error-handling page, quit trying to enumerate the underlying strerror(3) standards. --- examples/c/ex_all.c | 6 +++--- examples/java/com/wiredtiger/examples/ex_all.java | 16 ++++++++++++++++ src/docs/error-handling.dox | 11 +++++++++-- src/include/wiredtiger.in | 13 ++++++++----- 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/examples/c/ex_all.c b/examples/c/ex_all.c index ef77f10f6de..3f31ad3e9c5 100644 --- a/examples/c/ex_all.c +++ b/examples/c/ex_all.c @@ -347,8 +347,7 @@ cursor_ops(WT_SESSION *session) cursor->set_key(cursor, key); if ((ret = cursor->remove(cursor)) != 0) { fprintf(stderr, - "cursor.remove: %s\n", - cursor->session->strerror(cursor->session, ret)); + "cursor.remove: %s\n", wiredtiger_strerror(ret)); return (ret); } /*! [Display an error] */ @@ -360,7 +359,8 @@ cursor_ops(WT_SESSION *session) cursor->set_key(cursor, key); if ((ret = cursor->remove(cursor)) != 0) { fprintf(stderr, - "cursor.remove: %s\n", session->strerror(session, ret)); + "cursor.remove: %s\n", + cursor->session->strerror(cursor->session, ret)); return (ret); } /*! [Display an error thread safe] */ diff --git a/examples/java/com/wiredtiger/examples/ex_all.java b/examples/java/com/wiredtiger/examples/ex_all.java index 09db8e0fd56..61092c8be33 100644 --- a/examples/java/com/wiredtiger/examples/ex_all.java +++ b/examples/java/com/wiredtiger/examples/ex_all.java @@ -326,6 +326,22 @@ public static int cursor_ops(Session session) /*! [Display an error] */ } + { + /*! [Display an error thread safe] */ + try { + String key = "non-existent key"; + cursor.putKeyString(key); + if ((ret = cursor.remove()) != 0) { + System.err.println( + "cursor.remove: " + cursor.session.strerror(ret)); + return (ret); + } + } catch (WiredTigerException wte) { /* Catch severe errors. */ + System.err.println("cursor.remove exception: " + wte); + } + /*! [Display an error thread safe] */ + } + /*! [Close the cursor] */ ret = cursor.close(); /*! [Close the cursor] */ diff --git a/src/docs/error-handling.dox b/src/docs/error-handling.dox index 08fecd7f709..8ff3b68d64c 100644 --- a/src/docs/error-handling.dox +++ b/src/docs/error-handling.dox @@ -55,11 +55,18 @@ This error is generated when wiredtiger_open is configured to return an error if @if IGNORE_BUILT_BY_API_ERR_END @endif -The ::wiredtiger_strerror function returns the standard message -associated with any WiredTiger, ISO C99, or POSIX 1003.1-2001 function: +@section error_translation Translating errors + +The WT_SESSION::strerror and ::wiredtiger_strerror functions return the +standard text message associated with any WiredTiger, ISO C, or POSIX +standard API. + +@snippet ex_all.c Display an error thread safe @snippet ex_all.c Display an error +Note that ::wiredtiger_strerror is not thread-safe. + @m_if{c} @section error_handling_event Error handling using the WT_EVENT_HANDLER diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index 338b95c8208..32def98b5e2 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -828,7 +828,8 @@ struct __wt_session { * @snippet ex_all.c Display an error thread safe * * @param session the session handle - * @param error a return value from a WiredTiger function + * @param error a return value from a WiredTiger, ISO C, or POSIX + * standard API * @returns a string representation of the error */ const char *__F(strerror)(WT_SESSION *session, int error); @@ -2434,11 +2435,12 @@ int wiredtiger_open(const char *home, WT_CONNECTION **connectionp); /*! - * Return information about a WiredTiger error as a string, not thread-safe. + * Return information about a WiredTiger error as a string (see + * WT_SESSION::strerror for a thread-safe API). * * @snippet ex_all.c Display an error * - * @param error a return value from a WiredTiger call + * @param error a return value from a WiredTiger, ISO C, or POSIX standard API * @returns a string representation of the error */ const char *wiredtiger_strerror(int error); @@ -2491,8 +2493,9 @@ struct __wt_event_handler { * @param session the WiredTiger session handle in use when the error * was generated. The handle may have been created by the application * or automatically by WiredTiger. - * @param error a WiredTiger, C99 or POSIX error code, which can - * be converted to a string using ::wiredtiger_strerror + * @param error a return value from a WiredTiger, ISO C, or + * POSIX standard API, which can be converted to a string using + * WT_SESSION::strerror * @param message an error string */ int (*handle_error)(WT_EVENT_HANDLER *handler, -- cgit v1.2.1 From 005c17826dbadd4b7fb51149371813768ed2d356 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sun, 21 Feb 2016 13:46:38 -0500 Subject: WT-2107: Add example code including an event handler Fix the error handler function signature. --- examples/c/ex_all.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/c/ex_all.c b/examples/c/ex_all.c index 3f31ad3e9c5..524532d060f 100644 --- a/examples/c/ex_all.c +++ b/examples/c/ex_all.c @@ -1062,18 +1062,19 @@ application_logging(int which, const char *message) * WiredTiger error handler. */ int -wiredtiger_handle_error( - WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *error) +wiredtiger_handle_error(WT_EVENT_HANDLER *handler, + WT_SESSION *session, int error, const char *message) { int ret; (void)(handler); /* Unused variables */ + (void)(error); /* Timestamp and log the error message. */ - ret = application_logging(APPLICATION_ERROR, error); + ret = application_logging(APPLICATION_ERROR, message); /* Copy and flush the message to stderr. */ - if (fprintf(stderr, "%p: %s\n", session, error) < 0 && ret == 0) + if (fprintf(stderr, "%p: %s\n", session, message) < 0 && ret == 0) ret = EIO; if (fflush(stderr) != 0 && ret == 0) ret = errno; -- cgit v1.2.1 From 9263c44575ba95449d1c24d233bf2d01aa512efa Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sun, 21 Feb 2016 14:11:47 -0500 Subject: WT-2107: Add example code including an event handler #2512 Another compiler fix, add prototypes. --- examples/c/ex_all.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/examples/c/ex_all.c b/examples/c/ex_all.c index 524532d060f..ee57d220ea0 100644 --- a/examples/c/ex_all.c +++ b/examples/c/ex_all.c @@ -1047,7 +1047,12 @@ backup(WT_SESSION *session) #define APPLICATION_ERROR 1 #define APPLICATION_INFO 2 -static int +int application_logging(int, const char *); +int wiredtiger_handle_error( + WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); +int wiredtiger_handle_message(WT_EVENT_HANDLER *, WT_SESSION *, const char *); + +int application_logging(int which, const char *message) { (void)which; @@ -1055,6 +1060,7 @@ application_logging(int which, const char *message) return (0); } + /*! [Function event_handler] */ /* -- cgit v1.2.1 From 4c1ae0fe240d9cbc8e88c091d292ce8f2f94c924 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sun, 21 Feb 2016 14:48:25 -0500 Subject: WT-2107: Add example code including an event handler whitespace --- examples/c/ex_all.c | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/c/ex_all.c b/examples/c/ex_all.c index ee57d220ea0..0c33b471843 100644 --- a/examples/c/ex_all.c +++ b/examples/c/ex_all.c @@ -1060,7 +1060,6 @@ application_logging(int which, const char *message) return (0); } - /*! [Function event_handler] */ /* -- cgit v1.2.1 From 6c3446725aa0d3091ab1b79d723264ff28e1ce5d Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Mon, 22 Feb 2016 00:26:49 +0000 Subject: WT-2376 Consistently name snappy byteswap routine. --- ext/compressors/snappy/snappy_compress.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ext/compressors/snappy/snappy_compress.c b/ext/compressors/snappy/snappy_compress.c index a95dd665b67..981e334a2de 100644 --- a/ext/compressors/snappy/snappy_compress.c +++ b/ext/compressors/snappy/snappy_compress.c @@ -44,11 +44,11 @@ typedef struct { #ifdef WORDS_BIGENDIAN /* - * wt_bswap64 -- + * snappy_bswap64 -- * 64-bit unsigned little-endian to/from big-endian value. */ static inline uint64_t -wt_bswap64(uint64_t v) +snappy_bswap64(uint64_t v) { return ( ((v << 56) & 0xff00000000000000UL) | @@ -132,7 +132,7 @@ wt_snappy_compress(WT_COMPRESSOR *compressor, WT_SESSION *session, * Store the value in little-endian format. */ #ifdef WORDS_BIGENDIAN - snaplen = wt_bswap64(snaplen); + snaplen = snappy_bswap64(snaplen); #endif *(size_t *)dst = snaplen; } else @@ -165,7 +165,7 @@ wt_snappy_decompress(WT_COMPRESSOR *compressor, WT_SESSION *session, */ snaplen = *(size_t *)src; #ifdef WORDS_BIGENDIAN - snaplen = wt_bswap64(snaplen); + snaplen = snappy_bswap64(snaplen); #endif if (snaplen + sizeof(size_t) > src_len) { (void)wt_api->err_printf(wt_api, -- cgit v1.2.1 From fc73d64b566c540ea68a156c9e5cdda8b5247b93 Mon Sep 17 00:00:00 2001 From: David Hows Date: Mon, 22 Feb 2016 13:04:26 +1100 Subject: WT-2417 - Fix windows compile error --- src/os_win/os_open.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os_win/os_open.c b/src/os_win/os_open.c index 40f46b001db..f10582c5bd1 100644 --- a/src/os_win/os_open.c +++ b/src/os_win/os_open.c @@ -64,7 +64,7 @@ __wt_open(WT_SESSION_IMPL *session, */ if (F_ISSET(conn, WT_CONN_READONLY) && !WT_STRING_MATCH(name, WT_SINGLETHREAD, - strlen(WT_SINGLETHREAD))); + strlen(WT_SINGLETHREAD))) share_mode = FILE_SHARE_READ; else share_mode = FILE_SHARE_READ | FILE_SHARE_WRITE; -- cgit v1.2.1 From b7cb64083d7192a831f810b60964954608c51045 Mon Sep 17 00:00:00 2001 From: David Hows Date: Mon, 22 Feb 2016 18:48:02 +1100 Subject: WT-2394 - Have pages split as part of compact checkpoints use first-fit --- src/reconcile/rec_write.c | 40 ++++++++++++++++++++++++++++------------ test/suite/test_compact02.py | 2 ++ 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/src/reconcile/rec_write.c b/src/reconcile/rec_write.c index f245ff5d921..d14c29b5348 100644 --- a/src/reconcile/rec_write.c +++ b/src/reconcile/rec_write.c @@ -3112,11 +3112,13 @@ __rec_split_write(WT_SESSION_IMPL *session, uint32_t bnd_slot, i, j; int cmp; uint8_t addr[WT_BTREE_MAX_ADDR_COOKIE]; + bool compact; btree = S2BT(session); dsk = buf->mem; page = r->page; mod = page->modify; + compact = false; WT_RET(__wt_scr_alloc(session, 0, &key)); @@ -3281,19 +3283,22 @@ supd_check_complete: * do this check after calculating the checksums, hopefully the * next write can be skipped. */ - if (session->compact_state == WT_COMPACT_NONE && - mod->rec_result == WT_PM_REC_MULTIBLOCK && - mod->mod_multi_entries > bnd_slot) { - multi = &mod->mod_multi[bnd_slot]; - if (multi->size == bnd->size && - multi->cksum == bnd->cksum) { - multi->addr.reuse = 1; - bnd->addr = multi->addr; - - WT_STAT_FAST_DATA_INCR(session, rec_page_match); - goto done; + if (session->compact_state == WT_COMPACT_NONE) { + if (mod->rec_result == WT_PM_REC_MULTIBLOCK && + mod->mod_multi_entries > bnd_slot) { + multi = &mod->mod_multi[bnd_slot]; + if (multi->size == bnd->size && + multi->cksum == bnd->cksum) { + multi->addr.reuse = 1; + bnd->addr = multi->addr; + + WT_STAT_FAST_DATA_INCR( + session, rec_page_match); + goto done; + } } - } + } else + compact = true; } bnd->entries = r->entries; @@ -3307,8 +3312,19 @@ supd_check_complete: F_ISSET(r, WT_EVICTING) ? "evict" : "checkpoint", r->bnd_state)); + /* + * We should use first-fit if we are doing a compact to avoid writing + * to the end of the file + */ + if (compact) + __wt_block_configure_first_fit(btree->bm->block, true); + WT_ERR(__wt_bt_write(session, buf, addr, &addr_size, false, bnd->already_compressed)); + + if (compact) + __wt_block_configure_first_fit(btree->bm->block, false); + WT_ERR(__wt_strndup(session, addr, addr_size, &bnd->addr.addr)); bnd->addr.size = (uint8_t)addr_size; diff --git a/test/suite/test_compact02.py b/test/suite/test_compact02.py index 14781b0f050..d030b8baa15 100644 --- a/test/suite/test_compact02.py +++ b/test/suite/test_compact02.py @@ -55,6 +55,8 @@ class test_compact02(wttest.WiredTigerTestCase): fileConfig = [ ('default', dict(fileConfig='')), ('8KB', dict(fileConfig='leaf_page_max=8kb')), + ('64KB', dict(fileConfig='leaf_page_max=64KB')), + ('128KB', dict(fileConfig='leaf_page_max=128KB')), ] scenarios = \ number_scenarios(multiply_scenarios('.', types, cacheSize, fileConfig)) -- cgit v1.2.1 From 525b9a6f3058367652d2ef70ae5e514debd85390 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 22 Feb 2016 11:03:47 -0500 Subject: WT-2422: multiple definitions of custom die function Define the custom die function once per program. --- test/bloom/test_bloom.c | 2 ++ test/checkpoint/test_checkpoint.c | 2 ++ test/cursor_order/cursor_order.c | 2 ++ test/fops/t.c | 2 ++ test/format/format.h | 1 - test/format/t.c | 7 ++++--- test/huge/huge.c | 2 ++ test/readonly/readonly.c | 2 ++ test/recovery/random-abort.c | 2 ++ test/recovery/truncated-log.c | 2 ++ test/salvage/salvage.c | 2 ++ test/thread/t.c | 2 ++ test/utility/test_util.i | 5 ++--- 13 files changed, 26 insertions(+), 7 deletions(-) diff --git a/test/bloom/test_bloom.c b/test/bloom/test_bloom.c index 4a3d3372794..ffaddd2173b 100644 --- a/test/bloom/test_bloom.c +++ b/test/bloom/test_bloom.c @@ -55,6 +55,8 @@ void usage(void); extern char *__wt_optarg; extern int __wt_optind; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { diff --git a/test/checkpoint/test_checkpoint.c b/test/checkpoint/test_checkpoint.c index 1914ad0188a..0f28a86b675 100644 --- a/test/checkpoint/test_checkpoint.c +++ b/test/checkpoint/test_checkpoint.c @@ -41,6 +41,8 @@ static int wt_shutdown(void); extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { diff --git a/test/cursor_order/cursor_order.c b/test/cursor_order/cursor_order.c index 14709a2e88e..68d2f092c60 100644 --- a/test/cursor_order/cursor_order.c +++ b/test/cursor_order/cursor_order.c @@ -43,6 +43,8 @@ static void wt_shutdown(SHARED_CONFIG *); extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { diff --git a/test/fops/t.c b/test/fops/t.c index 0881c23d7d4..24994404c7c 100644 --- a/test/fops/t.c +++ b/test/fops/t.c @@ -50,6 +50,8 @@ static void wt_shutdown(void); extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { diff --git a/test/format/format.h b/test/format/format.h index b32877636c8..c54fd061736 100644 --- a/test/format/format.h +++ b/test/format/format.h @@ -310,7 +310,6 @@ void config_single(const char *, int); void *dmalloc(size_t); char *dstrdup(const char *); void fclose_and_clear(FILE **); -void format_die(void); void key_gen(uint8_t *, size_t *, uint64_t); void key_gen_insert(WT_RAND_STATE *, uint8_t *, size_t *, uint64_t); void key_gen_setup(uint8_t **); diff --git a/test/format/t.c b/test/format/t.c index b9dfdaabb33..28c22e23cb8 100644 --- a/test/format/t.c +++ b/test/format/t.c @@ -30,12 +30,15 @@ GLOBAL g; +static void format_die(void); static void startup(void); static void usage(void); extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = format_die; /* Local death handler. */ + int main(int argc, char *argv[]) { @@ -45,8 +48,6 @@ main(int argc, char *argv[]) config = NULL; - /* Register a fatal cleanup handler */ - custom_die = format_die; #ifdef _WIN32 g.progname = "t_format.exe"; #else @@ -310,7 +311,7 @@ startup(void) * die -- * Report an error, dumping the configuration. */ -void +static void format_die(void) { /* diff --git a/test/huge/huge.c b/test/huge/huge.c index d09f6f375fb..ad19035ff99 100644 --- a/test/huge/huge.c +++ b/test/huge/huge.c @@ -167,6 +167,8 @@ run(CONFIG *cp, int bigkey, size_t bytes) extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { diff --git a/test/readonly/readonly.c b/test/readonly/readonly.c index 6f74ee47c5a..100ccbf81b7 100644 --- a/test/readonly/readonly.c +++ b/test/readonly/readonly.c @@ -158,6 +158,8 @@ open_dbs(int op, const char *dir, extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { diff --git a/test/recovery/random-abort.c b/test/recovery/random-abort.c index b356ddc6379..c9cc10d2db3 100644 --- a/test/recovery/random-abort.c +++ b/test/recovery/random-abort.c @@ -136,6 +136,8 @@ fill_db(void) extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { diff --git a/test/recovery/truncated-log.c b/test/recovery/truncated-log.c index d1046064c2f..23269e99d35 100644 --- a/test/recovery/truncated-log.c +++ b/test/recovery/truncated-log.c @@ -169,6 +169,8 @@ fill_db(void) extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { diff --git a/test/salvage/salvage.c b/test/salvage/salvage.c index c2ad6224b11..a1517d70787 100644 --- a/test/salvage/salvage.c +++ b/test/salvage/salvage.c @@ -64,6 +64,8 @@ static int verbose; /* -v flag */ extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { diff --git a/test/thread/t.c b/test/thread/t.c index e72b54bf62a..22334076ee1 100644 --- a/test/thread/t.c +++ b/test/thread/t.c @@ -51,6 +51,8 @@ static void wt_shutdown(void); extern int __wt_optind; extern char *__wt_optarg; +void (*custom_die)(void) = NULL; + int main(int argc, char *argv[]) { diff --git a/test/utility/test_util.i b/test/utility/test_util.i index bd7956e460f..c5cebadcb5c 100644 --- a/test/utility/test_util.i +++ b/test/utility/test_util.i @@ -42,9 +42,8 @@ #define DEFAULT_DIR "WT_TEST" #define MKDIR_COMMAND "mkdir " -/* Setup a function pointer so tests can override the content of die. */ -typedef void (*die_func)(void); -die_func custom_die; +/* Allow tests to add their own death handling. */ +extern void (*custom_die)(void); static void testutil_die(int, const char *, ...) #if defined(__GNUC__) -- cgit v1.2.1 From 029f0ae02346fa7f45bf368e416d757579f0baa5 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 22 Feb 2016 13:04:59 -0500 Subject: WT-2394: Have pages split as part of compact checkpoints use first-fit Move compact start/stop into the session layer so all operations on the file during compaction (including checkpoints) use first-fit allocation. --- src/btree/bt_compact.c | 10 +------- src/reconcile/rec_write.c | 40 ++++++++++--------------------- src/session/session_compact.c | 55 ++++++++++++++++++++++++++++++++++++++++++- test/suite/test_compact02.py | 3 +-- 4 files changed, 68 insertions(+), 40 deletions(-) diff --git a/src/btree/bt_compact.c b/src/btree/bt_compact.c index 12df19a7e04..ac4da42ab08 100644 --- a/src/btree/bt_compact.c +++ b/src/btree/bt_compact.c @@ -96,14 +96,13 @@ __wt_compact(WT_SESSION_IMPL *session, const char *cfg[]) WT_BTREE *btree; WT_DECL_RET; WT_REF *ref; - bool block_manager_begin, skip; + bool skip; WT_UNUSED(cfg); btree = S2BT(session); bm = btree->bm; ref = NULL; - block_manager_begin = false; WT_STAT_FAST_DATA_INCR(session, session_compact); @@ -137,10 +136,6 @@ __wt_compact(WT_SESSION_IMPL *session, const char *cfg[]) */ __wt_spin_lock(session, &btree->flush_lock); - /* Start compaction. */ - WT_ERR(bm->compact_start(bm, session)); - block_manager_begin = true; - /* Walk the tree reviewing pages to see if they should be re-written. */ for (;;) { /* @@ -170,9 +165,6 @@ __wt_compact(WT_SESSION_IMPL *session, const char *cfg[]) err: if (ref != NULL) WT_TRET(__wt_page_release(session, ref, 0)); - if (block_manager_begin) - WT_TRET(bm->compact_end(bm, session)); - /* Unblock threads writing leaf pages. */ __wt_spin_unlock(session, &btree->flush_lock); diff --git a/src/reconcile/rec_write.c b/src/reconcile/rec_write.c index d14c29b5348..f245ff5d921 100644 --- a/src/reconcile/rec_write.c +++ b/src/reconcile/rec_write.c @@ -3112,13 +3112,11 @@ __rec_split_write(WT_SESSION_IMPL *session, uint32_t bnd_slot, i, j; int cmp; uint8_t addr[WT_BTREE_MAX_ADDR_COOKIE]; - bool compact; btree = S2BT(session); dsk = buf->mem; page = r->page; mod = page->modify; - compact = false; WT_RET(__wt_scr_alloc(session, 0, &key)); @@ -3283,22 +3281,19 @@ supd_check_complete: * do this check after calculating the checksums, hopefully the * next write can be skipped. */ - if (session->compact_state == WT_COMPACT_NONE) { - if (mod->rec_result == WT_PM_REC_MULTIBLOCK && - mod->mod_multi_entries > bnd_slot) { - multi = &mod->mod_multi[bnd_slot]; - if (multi->size == bnd->size && - multi->cksum == bnd->cksum) { - multi->addr.reuse = 1; - bnd->addr = multi->addr; - - WT_STAT_FAST_DATA_INCR( - session, rec_page_match); - goto done; - } + if (session->compact_state == WT_COMPACT_NONE && + mod->rec_result == WT_PM_REC_MULTIBLOCK && + mod->mod_multi_entries > bnd_slot) { + multi = &mod->mod_multi[bnd_slot]; + if (multi->size == bnd->size && + multi->cksum == bnd->cksum) { + multi->addr.reuse = 1; + bnd->addr = multi->addr; + + WT_STAT_FAST_DATA_INCR(session, rec_page_match); + goto done; } - } else - compact = true; + } } bnd->entries = r->entries; @@ -3312,19 +3307,8 @@ supd_check_complete: F_ISSET(r, WT_EVICTING) ? "evict" : "checkpoint", r->bnd_state)); - /* - * We should use first-fit if we are doing a compact to avoid writing - * to the end of the file - */ - if (compact) - __wt_block_configure_first_fit(btree->bm->block, true); - WT_ERR(__wt_bt_write(session, buf, addr, &addr_size, false, bnd->already_compressed)); - - if (compact) - __wt_block_configure_first_fit(btree->bm->block, false); - WT_ERR(__wt_strndup(session, addr, addr_size, &bnd->addr.addr)); bnd->addr.size = (uint8_t)addr_size; diff --git a/src/session/session_compact.c b/src/session/session_compact.c index 53faf343f35..564f48094df 100644 --- a/src/session/session_compact.c +++ b/src/session/session_compact.c @@ -138,6 +138,36 @@ __session_compact_check_timeout( return (0); } +/* + * __compact_start -- + * Start objection compaction. + */ +static int +__compact_start(WT_SESSION_IMPL *session, const char *cfg[]) +{ + WT_BM *bm; + + WT_UNUSED(cfg); + + bm = S2BT(session)->bm; + return (bm->compact_start(bm, session)); +} + +/* + * __compact_end -- + * End objection compaction. + */ +static int +__compact_end(WT_SESSION_IMPL *session, const char *cfg[]) +{ + WT_BM *bm; + + WT_UNUSED(cfg); + + bm = S2BT(session)->bm; + return (bm->compact_end(bm, session)); +} + /* * __compact_file -- * Function to alternate between checkpoints and compaction calls. @@ -148,10 +178,28 @@ __compact_file(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) struct timespec start_time; WT_DECL_ITEM(t); WT_DECL_RET; - int i; + int i, tret; const char *checkpoint_cfg[] = { WT_CONFIG_BASE(session, WT_SESSION_checkpoint), NULL, NULL }; + /* + * Start object compaction. + * + * XXX + * There's a bug here. We're configuring compaction on a block manager's + * object, and if we fail in the middle of that process, we would either + * leave a set of objects configured for compaction or try to undo that + * configuration on objects we never configured. There isn't any simple + * solution: we could track objects we've successfully configured for + * later cleanup, but we aren't holding locks to prevent racing with + * another compaction doing the same configuration, or the object being + * dropped in the middle of compaction. + */ + WT_WITH_SCHEMA_LOCK(session, ret, + ret = __wt_schema_worker( + session, uri, __compact_start, NULL, cfg, 0)); + WT_RET(ret); + /* * Force the checkpoint: we don't want to skip it because the work we * need to have done is done in the underlying block manager. @@ -187,6 +235,11 @@ __compact_file(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) err: session->compact_state = WT_COMPACT_NONE; + WT_WITH_SCHEMA_LOCK(session, tret, + tret = __wt_schema_worker( + session, uri, __compact_end, NULL, cfg, 0)); + WT_TRET(tret); + __wt_scr_free(session, &t); return (ret); } diff --git a/test/suite/test_compact02.py b/test/suite/test_compact02.py index d030b8baa15..7ad05cd2536 100644 --- a/test/suite/test_compact02.py +++ b/test/suite/test_compact02.py @@ -50,8 +50,7 @@ class test_compact02(wttest.WiredTigerTestCase): # being stored: compaction doesn't work on tables with many overflow items # because we don't rewrite them. Experimentally, 8KB is as small as the test # can go. Additionally, we can't set the maximum page size too large because - # there won't be enough pages to rewrite. Experimentally, 32KB (the default) - # is as large as the test can go. + # there won't be enough pages to rewrite. Experimentally, 128KB works. fileConfig = [ ('default', dict(fileConfig='')), ('8KB', dict(fileConfig='leaf_page_max=8kb')), -- cgit v1.2.1 From 2f97924854d03be882eb04d85f1374e703794639 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 22 Feb 2016 16:15:42 -0500 Subject: SERVER-22784: Coverity analysis defect 77722: Unused value The variable "couple" is assigned, but we jump to a label where it's assigned again, before it's used. I don't want to change the original assignment, I think it adds clarity to set couple, couple_orig and ref in the initial case. Pull the assignment of couple out of the if/else that holds the jumped-to label, instead. --- src/btree/bt_walk.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/btree/bt_walk.c b/src/btree/bt_walk.c index 55b11d7b2d1..bb8a750d848 100644 --- a/src/btree/bt_walk.c +++ b/src/btree/bt_walk.c @@ -583,14 +583,14 @@ restart: /* break; } WT_ERR(ret); + couple = ref; /* * A new page: configure for traversal of any internal * page's children, else return the leaf page. */ if (WT_PAGE_IS_INTERNAL(ref->page)) { -descend: couple = ref; - empty_internal = true; +descend: empty_internal = true; /* * There's a split race when a cursor is setting @@ -649,7 +649,6 @@ descend: couple = ref; */ if (skipleafcntp != NULL || LF_ISSET(WT_READ_SKIP_LEAF)) { - couple = ref; if (LF_ISSET(WT_READ_SKIP_LEAF)) break; if (*skipleafcntp > 0) { -- cgit v1.2.1 From 166f8f948d3c0fe30597f2ea5e7737c3cba9105b Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 22 Feb 2016 16:19:25 -0500 Subject: SERVER-22784: Coverity analysis defect 77722: Unused value Update spelling list. --- dist/s_string.ok | 1 + 1 file changed, 1 insertion(+) diff --git a/dist/s_string.ok b/dist/s_string.ok index 25e5719130e..ee968b758dc 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -437,6 +437,7 @@ cfkos change's changelog chdir +checkfmt checkpointed checkpointer checkpointing -- cgit v1.2.1 From dd9dbe2f0985a86da7c45f8e8f0d0738d73d0941 Mon Sep 17 00:00:00 2001 From: David Hows Date: Tue, 23 Feb 2016 10:10:34 +1100 Subject: WT-2419 -Fix missing __func__ defintion under MSVC --- test/windows/windows_shim.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/windows/windows_shim.h b/test/windows/windows_shim.h index c35c27cb7b0..dc082f990ce 100644 --- a/test/windows/windows_shim.h +++ b/test/windows/windows_shim.h @@ -44,6 +44,11 @@ typedef int u_int; #define R_OK 04 #define X_OK R_OK +/* MSVC Doesn't provide __func__, it has __FUNCTION__ */ +#ifdef _MSC_VER +#define __func__ __FUNCTION__ +#endif + /* snprintf does not exist on <= VS 2013 */ #if _MSC_VER < 1900 #define snprintf _wt_snprintf -- cgit v1.2.1 From e4e3bbca1669e08dfef753894da5f6e065f895a1 Mon Sep 17 00:00:00 2001 From: David Hows Date: Tue, 23 Feb 2016 10:31:00 +1100 Subject: WT-2419 - Formatting nits --- test/windows/windows_shim.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/windows/windows_shim.h b/test/windows/windows_shim.h index dc082f990ce..f32edce88e7 100644 --- a/test/windows/windows_shim.h +++ b/test/windows/windows_shim.h @@ -46,7 +46,7 @@ typedef int u_int; /* MSVC Doesn't provide __func__, it has __FUNCTION__ */ #ifdef _MSC_VER -#define __func__ __FUNCTION__ +#define __func__ __FUNCTION__ #endif /* snprintf does not exist on <= VS 2013 */ -- cgit v1.2.1 From 75a69bb545f0d2c78ea499edbd19232c0a4cda19 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Tue, 23 Feb 2016 14:05:49 +1100 Subject: WT-2423 Cleanup on error when opening a session handle. It's a bug, but found by inspection. Vanishingly unlikely. --- src/session/session_dhandle.c | 57 ++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/src/session/session_dhandle.c b/src/session/session_dhandle.c index 1ee3342442c..c8346ce2bb2 100644 --- a/src/session/session_dhandle.c +++ b/src/session/session_dhandle.c @@ -10,6 +10,25 @@ static int __session_dhandle_sweep(WT_SESSION_IMPL *); +/* + * __session_discard_dhandle -- + * Remove a data handle from the session cache. + */ +static void +__session_discard_dhandle( + WT_SESSION_IMPL *session, WT_DATA_HANDLE_CACHE *dhandle_cache) +{ + uint64_t bucket; + + bucket = dhandle_cache->dhandle->name_hash % WT_HASH_ARRAY_SIZE; + TAILQ_REMOVE(&session->dhandles, dhandle_cache, q); + TAILQ_REMOVE(&session->dhhash[bucket], dhandle_cache, hashq); + + (void)__wt_atomic_sub32(&dhandle_cache->dhandle->session_ref, 1); + + __wt_overwrite_and_free(session, dhandle_cache); +} + /* * __session_add_dhandle -- * Add a handle to the session's cache. @@ -19,39 +38,33 @@ __session_add_dhandle( WT_SESSION_IMPL *session, WT_DATA_HANDLE_CACHE **dhandle_cachep) { WT_DATA_HANDLE_CACHE *dhandle_cache; + WT_DECL_RET; uint64_t bucket; - WT_RET(__wt_calloc_one(session, &dhandle_cache)); + /* + * Allocate a handle cache entry, fixup the reference count on failure + * since we are part way through adding a handle and have already + * increased the reference count while holding a lock. + */ + if ((ret = __wt_calloc_one(session, &dhandle_cache)) != 0) { + (void)__wt_atomic_sub32(&session->dhandle->session_ref, 1); + return (ret); + } + dhandle_cache->dhandle = session->dhandle; bucket = dhandle_cache->dhandle->name_hash % WT_HASH_ARRAY_SIZE; TAILQ_INSERT_HEAD(&session->dhandles, dhandle_cache, q); TAILQ_INSERT_HEAD(&session->dhhash[bucket], dhandle_cache, hashq); - if (dhandle_cachep != NULL) - *dhandle_cachep = dhandle_cache; - /* Sweep the handle list to remove any dead handles. */ - return (__session_dhandle_sweep(session)); -} - -/* - * __session_discard_dhandle -- - * Remove a data handle from the session cache. - */ -static void -__session_discard_dhandle( - WT_SESSION_IMPL *session, WT_DATA_HANDLE_CACHE *dhandle_cache) -{ - uint64_t bucket; - - bucket = dhandle_cache->dhandle->name_hash % WT_HASH_ARRAY_SIZE; - TAILQ_REMOVE(&session->dhandles, dhandle_cache, q); - TAILQ_REMOVE(&session->dhhash[bucket], dhandle_cache, hashq); + if ((ret = __session_dhandle_sweep(session)) != 0) + __session_discard_dhandle(session, dhandle_cache); - (void)__wt_atomic_sub32(&dhandle_cache->dhandle->session_ref, 1); + if (ret == 0 && dhandle_cachep != NULL) + *dhandle_cachep = dhandle_cache; - __wt_overwrite_and_free(session, dhandle_cache); + return (ret); } /* -- cgit v1.2.1 From 38af01cabfd006894d7e69b991756c06246a708b Mon Sep 17 00:00:00 2001 From: David Hows Date: Tue, 23 Feb 2016 14:27:24 +1100 Subject: WT-2409 - Have LSM checkpoints wait for locks. --- src/lsm/lsm_work_unit.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/lsm/lsm_work_unit.c b/src/lsm/lsm_work_unit.c index 099bde176f7..240152033c5 100644 --- a/src/lsm/lsm_work_unit.c +++ b/src/lsm/lsm_work_unit.c @@ -341,13 +341,11 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, * time, and our checkpoint operation should be very quick. */ WT_ERR(__wt_meta_track_on(session)); - F_SET(session, WT_SESSION_LOCK_NO_WAIT); WT_WITH_CHECKPOINT_LOCK(session, ret, WT_WITH_SCHEMA_LOCK(session, ret, ret = __wt_schema_worker( session, chunk->uri, __wt_checkpoint, NULL, NULL, 0))); WT_TRET(__wt_meta_track_off(session, false, ret != 0)); - F_CLR(session, WT_SESSION_LOCK_NO_WAIT); if (ret != 0) { if (ret == EBUSY) { ret = 0; -- cgit v1.2.1 From f99f833f18d440cedb0a786315dc153b36574b40 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Tue, 23 Feb 2016 14:29:21 +1100 Subject: WT-2421 Fixup error handling in test/bloom. --- test/bloom/test_bloom.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/bloom/test_bloom.c b/test/bloom/test_bloom.c index ffaddd2173b..183dc3d2d42 100644 --- a/test/bloom/test_bloom.c +++ b/test/bloom/test_bloom.c @@ -189,12 +189,16 @@ run(void) */ item.size = g.c_key_max + 10; item.data = calloc(item.size, 1); + if (item.data == NULL) + testutil_die(ENOMEM, "value buffer malloc"); memset((void *)item.data, 'a', item.size); for (i = 0, fp = 0; i < g.c_ops; i++) { ((uint8_t *)item.data)[i % item.size] = 'a' + ((uint8_t)rand() % 26); if ((ret = __wt_bloom_get(bloomp, &item)) == 0) ++fp; + if (ret != 0 && ret != WT_NOTFOUND) + testutil_die(ret, "__wt_bloom_get"); } free((void *)item.data); printf("Out of %d ops, got %d false positives, %.4f%%\n", -- cgit v1.2.1 From 59e0ed7c0a00edcd3449cf7f124b4e08b51fe42d Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Tue, 23 Feb 2016 14:35:42 +1100 Subject: WT-2420 Gather backup metadata in a single pass. Previously, we did one pass to gather entries for the backup file, and a separate pass to gather handles. Since WT-2395, those two passes could be out-of-sync, leading to files missing from a backup. --- src/conn/conn_dhandle.c | 60 ++++++++++++-------- src/conn/conn_stat.c | 2 +- src/cursor/cur_backup.c | 143 ++++++++++++++++-------------------------------- src/include/extern.h | 5 +- src/include/misc.h | 4 -- src/lsm/lsm_tree.c | 3 +- src/meta/meta_apply.c | 34 +++++++----- src/txn/txn_ckpt.c | 4 +- 8 files changed, 111 insertions(+), 144 deletions(-) diff --git a/src/conn/conn_dhandle.c b/src/conn/conn_dhandle.c index 3bea24be508..e3f9871878b 100644 --- a/src/conn/conn_dhandle.c +++ b/src/conn/conn_dhandle.c @@ -355,13 +355,25 @@ err: F_CLR(btree, WT_BTREE_SPECIAL_FLAGS); /* * __conn_btree_apply_internal -- - * Apply a function to the open btree handles. + * Apply a function an open data handle. */ static int __conn_btree_apply_internal(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle, - int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]) + int (*file_func)(WT_SESSION_IMPL *, const char *[]), + int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), + const char *cfg[]) { WT_DECL_RET; + bool skip; + + /* Always apply the name function, if supplied. */ + skip = false; + if (name_func != NULL) + WT_RET(name_func(session, dhandle->name, &skip)); + + /* If there is no file function, don't bother locking the handle */ + if (file_func == NULL || skip) + return (0); /* * We need to pull the handle into the session handle cache and make @@ -372,7 +384,7 @@ __conn_btree_apply_internal(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle, dhandle->name, dhandle->checkpoint, NULL, 0)) != 0) return (ret == EBUSY ? 0 : ret); - WT_SAVE_DHANDLE(session, ret = func(session, cfg)); + WT_SAVE_DHANDLE(session, ret = file_func(session, cfg)); if (WT_META_TRACKING(session)) WT_TRET(__wt_meta_track_handle_lock(session, false)); else @@ -385,9 +397,10 @@ __conn_btree_apply_internal(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle, * Apply a function to all open btree handles with the given URI. */ int -__wt_conn_btree_apply(WT_SESSION_IMPL *session, - bool apply_checkpoints, const char *uri, - int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]) +__wt_conn_btree_apply(WT_SESSION_IMPL *session, const char *uri, + int (*file_func)(WT_SESSION_IMPL *, const char *[]), + int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), + const char *cfg[]) { WT_CONNECTION_IMPL *conn; WT_DATA_HANDLE *dhandle; @@ -404,23 +417,26 @@ __wt_conn_btree_apply(WT_SESSION_IMPL *session, if (uri != NULL) { bucket = __wt_hash_city64(uri, strlen(uri)) % WT_HASH_ARRAY_SIZE; - TAILQ_FOREACH(dhandle, &conn->dhhash[bucket], hashq) - if (F_ISSET(dhandle, WT_DHANDLE_OPEN) && - !F_ISSET(dhandle, WT_DHANDLE_DEAD) && - strcmp(uri, dhandle->name) == 0 && - (apply_checkpoints || dhandle->checkpoint == NULL)) - WT_RET(__conn_btree_apply_internal( - session, dhandle, func, cfg)); + TAILQ_FOREACH(dhandle, &conn->dhhash[bucket], hashq) { + if (!F_ISSET(dhandle, WT_DHANDLE_OPEN) || + F_ISSET(dhandle, WT_DHANDLE_DEAD) || + dhandle->checkpoint != NULL || + strcmp(uri, dhandle->name) != 0) + continue; + WT_RET(__conn_btree_apply_internal( + session, dhandle, file_func, name_func, cfg)); + } } else { - TAILQ_FOREACH(dhandle, &conn->dhqh, q) - if (F_ISSET(dhandle, WT_DHANDLE_OPEN) && - !F_ISSET(dhandle, WT_DHANDLE_DEAD) && - (apply_checkpoints || - dhandle->checkpoint == NULL) && - WT_PREFIX_MATCH(dhandle->name, "file:") && - !WT_IS_METADATA(session, dhandle)) - WT_RET(__conn_btree_apply_internal( - session, dhandle, func, cfg)); + TAILQ_FOREACH(dhandle, &conn->dhqh, q) { + if (!F_ISSET(dhandle, WT_DHANDLE_OPEN) || + F_ISSET(dhandle, WT_DHANDLE_DEAD) || + dhandle->checkpoint != NULL || + !WT_PREFIX_MATCH(dhandle->name, "file:") || + WT_IS_METADATA(session, dhandle)) + continue; + WT_RET(__conn_btree_apply_internal( + session, dhandle, file_func, name_func, cfg)); + } } return (0); diff --git a/src/conn/conn_stat.c b/src/conn/conn_stat.c index e3493ba5eeb..50f8a8b0ca9 100644 --- a/src/conn/conn_stat.c +++ b/src/conn/conn_stat.c @@ -350,7 +350,7 @@ __statlog_log_one(WT_SESSION_IMPL *session, WT_ITEM *path, WT_ITEM *tmp) if (conn->stat_sources != NULL) { WT_WITH_HANDLE_LIST_LOCK(session, ret = __wt_conn_btree_apply( - session, false, NULL, __statlog_apply, NULL)); + session, NULL, __statlog_apply, NULL, NULL)); WT_RET(ret); } diff --git a/src/cursor/cur_backup.c b/src/cursor/cur_backup.c index 63c4ac6a4ff..b097a8c08aa 100644 --- a/src/cursor/cur_backup.c +++ b/src/cursor/cur_backup.c @@ -8,12 +8,12 @@ #include "wt_internal.h" -static int __backup_all(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *); +static int __backup_all(WT_SESSION_IMPL *); static int __backup_cleanup_handles(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *); static int __backup_file_create(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, bool); -static int __backup_list_all_append(WT_SESSION_IMPL *, const char *[]); static int __backup_list_append( WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, const char *); +static int __backup_list_uri_append(WT_SESSION_IMPL *, const char *, bool *); static int __backup_start( WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, const char *[]); static int __backup_stop(WT_SESSION_IMPL *); @@ -241,7 +241,7 @@ __backup_start( if (!target_list) { WT_ERR(__backup_log_append(session, cb, true)); - WT_ERR(__backup_all(session, cb)); + WT_ERR(__backup_all(session)); } /* Add the hot backup and standard WiredTiger files to the list. */ @@ -332,55 +332,14 @@ __backup_stop(WT_SESSION_IMPL *session) * Backup all objects in the database. */ static int -__backup_all(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) +__backup_all(WT_SESSION_IMPL *session) { - WT_CONFIG_ITEM cval; - WT_CURSOR *cursor; WT_DECL_RET; - const char *key, *value; - - cursor = NULL; - - /* Copy all of the metadata entries to the hot backup file. */ - WT_RET(__wt_metadata_cursor(session, &cursor)); - while ((ret = cursor->next(cursor)) == 0) { - WT_ERR(cursor->get_key(cursor, &key)); - WT_ERR(cursor->get_value(cursor, &value)); - WT_ERR(__wt_fprintf(cb->bfp, "%s\n%s\n", key, value)); - - /* - * While reading the metadata file, check there are no "sources" - * or "types" which can't support hot backup. This checks for - * a data source that's non-standard, which can't be backed up, - * but is also sanity checking: if there's an entry backed by - * anything other than a file or lsm entry, we're confused. - */ - if ((ret = __wt_config_getones( - session, value, "type", &cval)) == 0 && - !WT_PREFIX_MATCH_LEN(cval.str, cval.len, "file") && - !WT_PREFIX_MATCH_LEN(cval.str, cval.len, "lsm")) - WT_ERR_MSG(session, ENOTSUP, - "hot backup is not supported for objects of " - "type %.*s", (int)cval.len, cval.str); - WT_ERR_NOTFOUND_OK(ret); - if ((ret =__wt_config_getones( - session, value, "source", &cval)) == 0 && - !WT_PREFIX_MATCH_LEN(cval.str, cval.len, "file:") && - !WT_PREFIX_MATCH_LEN(cval.str, cval.len, "lsm:")) - WT_ERR_MSG(session, ENOTSUP, - "hot backup is not supported for objects of " - "source %.*s", (int)cval.len, cval.str); - WT_ERR_NOTFOUND_OK(ret); - } - WT_ERR_NOTFOUND_OK(ret); - - WT_ERR(__wt_metadata_cursor_release(session, &cursor)); /* Build a list of the file objects that need to be copied. */ WT_WITH_HANDLE_LIST_LOCK(session, ret = - __wt_meta_btree_apply(session, __backup_list_all_append, NULL)); + __wt_meta_apply_all(session, NULL, __backup_list_uri_append, NULL)); -err: WT_TRET(__wt_metadata_cursor_release(session, &cursor)); return (ret); } @@ -430,11 +389,11 @@ __backup_uri(WT_SESSION_IMPL *session, */ if (WT_PREFIX_MATCH(uri, "log:")) { *log_only = !target_list; - WT_ERR(__wt_backup_list_uri_append(session, uri, NULL)); + WT_ERR(__backup_list_uri_append(session, uri, NULL)); } else { *log_only = false; WT_ERR(__wt_schema_worker(session, - uri, NULL, __wt_backup_list_uri_append, cfg, 0)); + uri, NULL, __backup_list_uri_append, cfg, 0)); } } WT_ERR_NOTFOUND_OK(ret); @@ -471,12 +430,12 @@ __wt_backup_file_remove(WT_SESSION_IMPL *session) } /* - * __wt_backup_list_uri_append -- + * __backup_list_uri_append -- * Append a new file name to the list, allocate space as necessary. * Called via the schema_worker function. */ -int -__wt_backup_list_uri_append( +static int +__backup_list_uri_append( WT_SESSION_IMPL *session, const char *name, bool *skip) { WT_CURSOR_BACKUP *cb; @@ -485,11 +444,31 @@ __wt_backup_list_uri_append( cb = session->bkp_cursor; WT_UNUSED(skip); + /* + * While reading the metadata file, check there are no data sources + * that can't support hot backup. This checks for a data source that's + * non-standard, which can't be backed up, but is also sanity checking: + * if there's an entry backed by anything other than a file or lsm + * entry, we're confused. + */ if (WT_PREFIX_MATCH(name, "log:")) { WT_RET(__backup_log_append(session, cb, false)); return (0); } + if (!WT_PREFIX_MATCH(name, "file:") && + !WT_PREFIX_MATCH(name, "colgroup:") && + !WT_PREFIX_MATCH(name, "index:") && + !WT_PREFIX_MATCH(name, "lsm:") && + !WT_PREFIX_MATCH(name, "table:")) + WT_RET_MSG(session, ENOTSUP, + "hot backup is not supported for objects of type %s", + name); + + /* Ignore the lookaside table. */ + if (strcmp(name, WT_LAS_URI) == 0) + return (0); + /* Add the metadata entry to the backup file. */ WT_RET(__wt_metadata_search(session, name, &value)); WT_RET(__wt_fprintf(cb->bfp, "%s\n%s\n", name, value)); @@ -502,34 +481,6 @@ __wt_backup_list_uri_append( return (0); } -/* - * __backup_list_all_append -- - * Append a new file name to the list, allocate space as necessary. - * Called via the __wt_meta_btree_apply function. - */ -static int -__backup_list_all_append(WT_SESSION_IMPL *session, const char *cfg[]) -{ - WT_CURSOR_BACKUP *cb; - const char *name; - - WT_UNUSED(cfg); - - cb = session->bkp_cursor; - name = session->dhandle->name; - - /* Ignore files in the process of being bulk-loaded. */ - if (F_ISSET(S2BT(session), WT_BTREE_BULK)) - return (0); - - /* Ignore the lookaside table. */ - if (strcmp(name, WT_LAS_URI) == 0) - return (0); - - /* Add the file to the list of files to be copied. */ - return (__backup_list_append(session, cb, name)); -} - /* * __backup_list_append -- * Append a new file name to the list, allocate space as necessary. @@ -541,7 +492,6 @@ __backup_list_append( WT_CURSOR_BACKUP_ENTRY *p; WT_DATA_HANDLE *old_dhandle; WT_DECL_RET; - bool need_handle; const char *name; /* Leave a NULL at the end to mark the end of the list. */ @@ -551,11 +501,26 @@ __backup_list_append( p[0].name = p[1].name = NULL; p[0].handle = p[1].handle = NULL; - need_handle = false; name = uri; + + /* + * If it's a file in the database, get a handle for the underlying + * object (this handle blocks schema level operations, for example + * WT_SESSION.drop or an LSM file discard after level merging). + * + * If the handle is busy (e.g., it is being bulk-loaded), silently skip + * it. We have a special fake checkpoint in the metadata, and recovery + * will recreate an empty file. + */ if (WT_PREFIX_MATCH(uri, "file:")) { - need_handle = true; name += strlen("file:"); + + old_dhandle = session->dhandle; + ret = __wt_session_get_btree(session, uri, NULL, NULL, 0); + p->handle = session->dhandle; + session->dhandle = old_dhandle; + if (ret != 0) + return (ret == EBUSY ? 0 : ret); } /* @@ -569,20 +534,6 @@ __backup_list_append( */ WT_RET(__wt_strdup(session, name, &p->name)); - /* - * If it's a file in the database, get a handle for the underlying - * object (this handle blocks schema level operations, for example - * WT_SESSION.drop or an LSM file discard after level merging). - */ - if (need_handle) { - old_dhandle = session->dhandle; - if ((ret = - __wt_session_get_btree(session, uri, NULL, NULL, 0)) == 0) - p->handle = session->dhandle; - session->dhandle = old_dhandle; - WT_RET(ret); - } - ++cb->list_next; return (0); } diff --git a/src/include/extern.h b/src/include/extern.h index b4018eb1191..40b6314aff7 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -252,7 +252,7 @@ extern int __wt_checkpoint_signal(WT_SESSION_IMPL *session, wt_off_t logsize); extern int __wt_conn_dhandle_find( WT_SESSION_IMPL *session, const char *uri, const char *checkpoint); extern int __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force); extern int __wt_conn_btree_open( WT_SESSION_IMPL *session, const char *cfg[], uint32_t flags); -extern int __wt_conn_btree_apply(WT_SESSION_IMPL *session, bool apply_checkpoints, const char *uri, int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]); +extern int __wt_conn_btree_apply(WT_SESSION_IMPL *session, const char *uri, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[]); extern int __wt_conn_dhandle_close_all( WT_SESSION_IMPL *session, const char *uri, bool force); extern int __wt_conn_dhandle_discard_single( WT_SESSION_IMPL *session, bool final, bool force); extern int __wt_conn_dhandle_discard(WT_SESSION_IMPL *session); @@ -276,7 +276,6 @@ extern int __wt_sweep_create(WT_SESSION_IMPL *session); extern int __wt_sweep_destroy(WT_SESSION_IMPL *session); extern int __wt_curbackup_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp); extern int __wt_backup_file_remove(WT_SESSION_IMPL *session); -extern int __wt_backup_list_uri_append( WT_SESSION_IMPL *session, const char *name, bool *skip); extern int __wt_curbulk_init(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool bitmap, bool skip_sort_check); extern int __wt_curconfig_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp); extern int __wt_curds_open( WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_DATA_SOURCE *dsrc, WT_CURSOR **cursorp); @@ -446,7 +445,7 @@ extern int __wt_lsm_work_bloom(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree); extern int __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, WT_LSM_CHUNK *chunk); extern int __wt_lsm_free_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree); extern int __wt_lsm_worker_start(WT_SESSION_IMPL *session, WT_LSM_WORKER_ARGS *args); -extern int __wt_meta_btree_apply(WT_SESSION_IMPL *session, int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]); +extern int __wt_meta_apply_all(WT_SESSION_IMPL *session, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[]); extern int __wt_meta_checkpoint(WT_SESSION_IMPL *session, const char *fname, const char *checkpoint, WT_CKPT *ckpt); extern int __wt_meta_checkpoint_last_name( WT_SESSION_IMPL *session, const char *fname, const char **namep); extern int __wt_meta_checkpoint_clear(WT_SESSION_IMPL *session, const char *fname); diff --git a/src/include/misc.h b/src/include/misc.h index 5dadb1b1484..4d3ca758dc7 100644 --- a/src/include/misc.h +++ b/src/include/misc.h @@ -201,10 +201,6 @@ (((const char *)str)[0] == ((const char *)pfx)[0] && \ strncmp((str), (pfx), strlen(pfx)) == 0) -/* Check if a non-nul-terminated string matches a prefix. */ -#define WT_PREFIX_MATCH_LEN(str, len, pfx) \ - ((len) >= strlen(pfx) && WT_PREFIX_MATCH(str, pfx)) - /* Check if a string matches a prefix, and move past it. */ #define WT_PREFIX_SKIP(str, pfx) \ (WT_PREFIX_MATCH(str, pfx) ? ((str) += strlen(pfx), 1) : 0) diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index 33cfb242a1e..7c188bf3dc7 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -1461,8 +1461,7 @@ __wt_lsm_tree_worker(WT_SESSION_IMPL *session, continue; WT_ERR(__wt_schema_worker(session, chunk->uri, file_func, name_func, cfg, open_flags)); - if (name_func == __wt_backup_list_uri_append && - F_ISSET(chunk, WT_LSM_CHUNK_BLOOM)) + if (F_ISSET(chunk, WT_LSM_CHUNK_BLOOM)) WT_ERR(__wt_schema_worker(session, chunk->bloom_uri, file_func, name_func, cfg, open_flags)); } diff --git a/src/meta/meta_apply.c b/src/meta/meta_apply.c index 7722cd55fbd..fb483c21dd9 100644 --- a/src/meta/meta_apply.c +++ b/src/meta/meta_apply.c @@ -15,22 +15,26 @@ */ static inline int __meta_btree_apply(WT_SESSION_IMPL *session, WT_CURSOR *cursor, - int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]) + int (*file_func)(WT_SESSION_IMPL *, const char *[]), + int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), + const char *cfg[]) { WT_DECL_RET; const char *uri; - int cmp; + bool skip; - cursor->set_key(cursor, "file:"); - if ((ret = cursor->search_near(cursor, &cmp)) == 0 && cmp < 0) - ret = cursor->next(cursor); - for (; ret == 0; ret = cursor->next(cursor)) { + while ((ret = cursor->next(cursor)) == 0) { WT_RET(cursor->get_key(cursor, &uri)); - if (!WT_PREFIX_MATCH(uri, "file:")) - break; if (strcmp(uri, WT_METAFILE_URI) == 0) continue; + skip = false; + if (name_func != NULL) + WT_RET(name_func(session, uri, &skip)); + + if (file_func == NULL || skip || !WT_PREFIX_MATCH(uri, "file:")) + continue; + /* * We need to pull the handle into the session handle cache * and make sure it's referenced to stop other internal code @@ -40,7 +44,7 @@ __meta_btree_apply(WT_SESSION_IMPL *session, WT_CURSOR *cursor, if ((ret = __wt_session_get_btree( session, uri, NULL, NULL, 0)) != 0) return (ret == EBUSY ? 0 : ret); - WT_SAVE_DHANDLE(session, ret = func(session, cfg)); + WT_SAVE_DHANDLE(session, ret = file_func(session, cfg)); if (WT_META_TRACKING(session)) WT_TRET(__wt_meta_track_handle_lock( session, false)); @@ -54,20 +58,22 @@ __meta_btree_apply(WT_SESSION_IMPL *session, WT_CURSOR *cursor, } /* - * __wt_meta_btree_apply -- + * __wt_meta_apply_all -- * Apply a function to all files listed in the metadata, apart from the * metadata file. */ int -__wt_meta_btree_apply(WT_SESSION_IMPL *session, - int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]) +__wt_meta_apply_all(WT_SESSION_IMPL *session, + int (*file_func)(WT_SESSION_IMPL *, const char *[]), + int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), + const char *cfg[]) { WT_CURSOR *cursor; WT_DECL_RET; WT_RET(__wt_metadata_cursor(session, &cursor)); - WT_SAVE_DHANDLE(session, - ret = __meta_btree_apply(session, cursor, func, cfg)); + WT_SAVE_DHANDLE(session, ret = + __meta_btree_apply(session, cursor, file_func, name_func, cfg)); WT_TRET(__wt_metadata_cursor_release(session, &cursor)); return (ret); diff --git a/src/txn/txn_ckpt.c b/src/txn/txn_ckpt.c index 85102ae8cfe..4bb8ccdc6f0 100644 --- a/src/txn/txn_ckpt.c +++ b/src/txn/txn_ckpt.c @@ -155,8 +155,8 @@ __checkpoint_apply_all(WT_SESSION_IMPL *session, const char *cfg[], ckpt_closed = cval.len != 0; } WT_ERR(ckpt_closed ? - __wt_meta_btree_apply(session, op, cfg) : - __wt_conn_btree_apply(session, false, NULL, op, cfg)); + __wt_meta_apply_all(session, op, NULL, cfg) : + __wt_conn_btree_apply(session, NULL, op, NULL, cfg)); } if (fullp != NULL) -- cgit v1.2.1 From 8197cc2c457df8602e576d04ed295d76f6e60918 Mon Sep 17 00:00:00 2001 From: David Hows Date: Tue, 23 Feb 2016 14:41:59 +1100 Subject: WT-2409 - Remove unneeded handling of EBUSY in LSM checkpointing --- src/lsm/lsm_work_unit.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/lsm/lsm_work_unit.c b/src/lsm/lsm_work_unit.c index 240152033c5..7723818f607 100644 --- a/src/lsm/lsm_work_unit.c +++ b/src/lsm/lsm_work_unit.c @@ -346,13 +346,8 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, ret = __wt_schema_worker( session, chunk->uri, __wt_checkpoint, NULL, NULL, 0))); WT_TRET(__wt_meta_track_off(session, false, ret != 0)); - if (ret != 0) { - if (ret == EBUSY) { - ret = 0; - goto err; - } + if (ret != 0) WT_ERR_MSG(session, ret, "LSM checkpoint"); - } /* Now the file is written, get the chunk size. */ WT_ERR(__wt_lsm_tree_set_chunk_size(session, chunk)); -- cgit v1.2.1 From adc1cfbc1815ef8beed04f81c85422e8b15ece49 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Tue, 23 Feb 2016 16:21:19 +1100 Subject: WT-2394 Gather handles for compact at the beginning. This allows us to set the "first fit" flag once and clear it consistently. It also prevents drop or other exclusive operation during compact without reentering the schema lock each time. --- src/btree/bt_compact.c | 14 +---- src/include/extern.h | 1 - src/include/session.h | 15 ++++- src/session/session_compact.c | 136 ++++++++++++++++++++++++------------------ 4 files changed, 95 insertions(+), 71 deletions(-) diff --git a/src/btree/bt_compact.c b/src/btree/bt_compact.c index ac4da42ab08..9cc56c56452 100644 --- a/src/btree/bt_compact.c +++ b/src/btree/bt_compact.c @@ -122,17 +122,9 @@ __wt_compact(WT_SESSION_IMPL *session, const char *cfg[]) * We need to ensure we don't race with page reconciliation as it's * writing the page modify information. * - * There are three ways we call reconciliation: checkpoints, threads - * writing leaf pages (usually in preparation for a checkpoint or if - * closing a file), and eviction. - * - * We're holding the schema lock which serializes with checkpoints. - */ - WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_SCHEMA)); - - /* - * Get the tree handle's flush lock which blocks threads writing leaf - * pages. + * There are two ways we call reconciliation: checkpoints and eviction. + * Get the tree's flush lock which blocks threads writing pages for + * checkpoints. */ __wt_spin_lock(session, &btree->flush_lock); diff --git a/src/include/extern.h b/src/include/extern.h index b4018eb1191..1002c2c0c26 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -627,7 +627,6 @@ extern int __wt_session_drop(WT_SESSION_IMPL *session, const char *uri, const ch extern int __wt_session_range_truncate(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *start, WT_CURSOR *stop); extern int __wt_open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, const char *config, bool open_metadata, WT_SESSION_IMPL **sessionp); extern int __wt_open_internal_session(WT_CONNECTION_IMPL *conn, const char *name, bool open_metadata, uint32_t session_flags, WT_SESSION_IMPL **sessionp); -extern int __wt_compact_uri_analyze(WT_SESSION_IMPL *session, const char *uri, bool *skipp); extern int __wt_session_compact( WT_SESSION *wt_session, const char *uri, const char *config); extern int __wt_session_compact_readonly( WT_SESSION *wt_session, const char *uri, const char *config); extern int __wt_session_lock_dhandle( WT_SESSION_IMPL *session, uint32_t flags, bool *is_deadp); diff --git a/src/include/session.h b/src/include/session.h index b3c475805a4..5c442bf0b81 100644 --- a/src/include/session.h +++ b/src/include/session.h @@ -126,11 +126,24 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { void *block_manager; /* Block-manager support */ int (*block_manager_cleanup)(WT_SESSION_IMPL *); - /* Checkpoint support */ + /* Checkpoint handles */ WT_DATA_HANDLE **ckpt_handle; /* Handle list */ u_int ckpt_handle_next; /* Next empty slot */ size_t ckpt_handle_allocated; /* Bytes allocated */ + /* + * Operations acting on handles. + * + * The preferred pattern is to gather all of the required handles at + * the beginning of an operation, then drop any other locks, perform + * the operation, then release the handles. This cannot be easily + * merged with the list of checkpoint handles because some operation + * (such as compact) do checkpoints internally. + */ + WT_DATA_HANDLE **op_handle; /* Handle list */ + u_int op_handle_next; /* Next empty slot */ + size_t op_handle_allocated; /* Bytes allocated */ + void *reconcile; /* Reconciliation support */ int (*reconcile_cleanup)(WT_SESSION_IMPL *); diff --git a/src/session/session_compact.c b/src/session/session_compact.c index 564f48094df..d2fd995b554 100644 --- a/src/session/session_compact.c +++ b/src/session/session_compact.c @@ -97,13 +97,13 @@ */ /* - * __wt_compact_uri_analyze -- + * __compact_uri_analyze -- * Extract information relevant to deciding what work compact needs to * do from a URI that is part of a table schema. * Called via the schema_worker function. */ -int -__wt_compact_uri_analyze(WT_SESSION_IMPL *session, const char *uri, bool *skipp) +static int +__compact_uri_analyze(WT_SESSION_IMPL *session, const char *uri, bool *skipp) { /* * Add references to schema URI objects to the list of objects to be @@ -120,52 +120,77 @@ __wt_compact_uri_analyze(WT_SESSION_IMPL *session, const char *uri, bool *skipp) } /* - * __session_compact_check_timeout -- - * Check if the timeout has been exceeded. + * __compact_start -- + * Start object compaction. */ static int -__session_compact_check_timeout( - WT_SESSION_IMPL *session, struct timespec begin) +__compact_start(WT_SESSION_IMPL *session) { - struct timespec end; - - if (session->compact->max_time == 0) - return (0); + WT_BM *bm; - WT_RET(__wt_epoch(session, &end)); - if (session->compact->max_time < WT_TIMEDIFF_SEC(end, begin)) - WT_RET(ETIMEDOUT); - return (0); + bm = S2BT(session)->bm; + return (bm->compact_start(bm, session)); } /* - * __compact_start -- - * Start objection compaction. + * __compact_end -- + * End object compaction. */ static int -__compact_start(WT_SESSION_IMPL *session, const char *cfg[]) +__compact_end(WT_SESSION_IMPL *session) { WT_BM *bm; - WT_UNUSED(cfg); - bm = S2BT(session)->bm; - return (bm->compact_start(bm, session)); + return (bm->compact_end(bm, session)); } /* - * __compact_end -- - * End objection compaction. + * __compact_handle_append -- + * Gather a file handle to be compacted. + * Called via the schema_worker function. */ static int -__compact_end(WT_SESSION_IMPL *session, const char *cfg[]) +__compact_handle_append(WT_SESSION_IMPL *session, const char *cfg[]) { - WT_BM *bm; + WT_DECL_RET; WT_UNUSED(cfg); - bm = S2BT(session)->bm; - return (bm->compact_end(bm, session)); + /* Make sure there is space for the next entry. */ + WT_RET(__wt_realloc_def(session, &session->op_handle_allocated, + session->op_handle_next + 1, &session->op_handle)); + + WT_RET(__wt_session_get_btree( + session, session->dhandle->name, NULL, NULL, 0)); + + /* Set compact active on the handle. */ + if ((ret = __compact_start(session)) != 0) { + WT_TRET(__wt_session_release_btree(session)); + return (ret); + } + + session->op_handle[session->op_handle_next++] = session->dhandle; + return (0); +} + +/* + * __session_compact_check_timeout -- + * Check if the timeout has been exceeded. + */ +static int +__session_compact_check_timeout( + WT_SESSION_IMPL *session, struct timespec begin) +{ + struct timespec end; + + if (session->compact->max_time == 0) + return (0); + + WT_RET(__wt_epoch(session, &end)); + if (session->compact->max_time < WT_TIMEDIFF_SEC(end, begin)) + WT_RET(ETIMEDOUT); + return (0); } /* @@ -173,39 +198,25 @@ __compact_end(WT_SESSION_IMPL *session, const char *cfg[]) * Function to alternate between checkpoints and compaction calls. */ static int -__compact_file(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +__compact_file(WT_SESSION_IMPL *session, const char *cfg[]) { struct timespec start_time; + WT_DATA_HANDLE *dhandle; WT_DECL_ITEM(t); WT_DECL_RET; - int i, tret; + int i; const char *checkpoint_cfg[] = { WT_CONFIG_BASE(session, WT_SESSION_checkpoint), NULL, NULL }; - /* - * Start object compaction. - * - * XXX - * There's a bug here. We're configuring compaction on a block manager's - * object, and if we fail in the middle of that process, we would either - * leave a set of objects configured for compaction or try to undo that - * configuration on objects we never configured. There isn't any simple - * solution: we could track objects we've successfully configured for - * later cleanup, but we aren't holding locks to prevent racing with - * another compaction doing the same configuration, or the object being - * dropped in the middle of compaction. - */ - WT_WITH_SCHEMA_LOCK(session, ret, - ret = __wt_schema_worker( - session, uri, __compact_start, NULL, cfg, 0)); - WT_RET(ret); + dhandle = session->dhandle; /* * Force the checkpoint: we don't want to skip it because the work we * need to have done is done in the underlying block manager. */ WT_ERR(__wt_scr_alloc(session, 128, &t)); - WT_ERR(__wt_buf_fmt(session, t, "target=(\"%s\"),force=1", uri)); + WT_ERR(__wt_buf_fmt( + session, t, "target=(\"%s\"),force=1", dhandle->name)); checkpoint_cfg[1] = t->data; WT_ERR(__wt_epoch(session, &start_time)); @@ -221,9 +232,8 @@ __compact_file(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_ERR(__wt_txn_checkpoint(session, checkpoint_cfg)); session->compact_state = WT_COMPACT_RUNNING; - WT_WITH_SCHEMA_LOCK(session, ret, - ret = __wt_schema_worker( - session, uri, __wt_compact, NULL, cfg, 0)); + WT_WITH_DHANDLE(session, dhandle, + ret = __wt_compact(session, cfg)); WT_ERR(ret); if (session->compact_state != WT_COMPACT_SUCCESS) break; @@ -235,11 +245,6 @@ __compact_file(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) err: session->compact_state = WT_COMPACT_NONE; - WT_WITH_SCHEMA_LOCK(session, tret, - tret = __wt_schema_worker( - session, uri, __compact_end, NULL, cfg, 0)); - WT_TRET(tret); - __wt_scr_free(session, &t); return (ret); } @@ -257,6 +262,7 @@ __wt_session_compact( WT_DECL_RET; WT_SESSION_IMPL *session; WT_TXN *txn; + u_int i; session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, compact, config, cfg); @@ -283,8 +289,8 @@ __wt_session_compact( /* Find the types of data sources are being compacted. */ WT_WITH_SCHEMA_LOCK(session, ret, - ret = __wt_schema_worker( - session, uri, NULL, __wt_compact_uri_analyze, cfg, 0)); + ret = __wt_schema_worker(session, uri, + __compact_handle_append, __compact_uri_analyze, cfg, 0)); WT_ERR(ret); if (session->compact->lsm_count != 0) @@ -301,11 +307,25 @@ __wt_session_compact( WT_ERR_MSG(session, EINVAL, " File compaction not permitted in a transaction"); - WT_ERR(__compact_file(session, uri, cfg)); + for (i = 0; i < session->op_handle_next; ++i) { + WT_WITH_DHANDLE(session, session->op_handle[i], + ret = __compact_file(session, cfg)); + WT_ERR(ret); + } } err: session->compact = NULL; + for (i = 0; i < session->op_handle_next; ++i) { + WT_WITH_DHANDLE(session, session->op_handle[i], + WT_TRET(__compact_end(session))); + WT_WITH_DHANDLE(session, session->op_handle[i], + WT_TRET(__wt_session_release_btree(session))); + } + + __wt_free(session, session->op_handle); + session->op_handle_allocated = session->op_handle_next = 0; + /* * Release common session resources (for example, checkpoint may acquire * significant reconciliation structures/memory). -- cgit v1.2.1 From e5a56007153127bc04a680ffff1d6d98e563c6e2 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Tue, 23 Feb 2016 16:25:00 +1100 Subject: WT-2107 Shuffle changes for event handler example. --- examples/c/Makefile.am | 1 + examples/c/ex_all.c | 71 ------------------------- examples/c/ex_event_handler.c | 120 ++++++++++++++++++++++++++++++++++++++++++ src/docs/error-handling.dox | 4 +- src/include/wiredtiger.in | 6 ++- 5 files changed, 127 insertions(+), 75 deletions(-) create mode 100644 examples/c/ex_event_handler.c diff --git a/examples/c/Makefile.am b/examples/c/Makefile.am index 587204efff1..72fd98aff7b 100644 --- a/examples/c/Makefile.am +++ b/examples/c/Makefile.am @@ -12,6 +12,7 @@ noinst_PROGRAMS = \ ex_cursor \ ex_data_source \ ex_encrypt \ + ex_event_handler \ ex_extending \ ex_extractor \ ex_hello \ diff --git a/examples/c/ex_all.c b/examples/c/ex_all.c index 0c33b471843..f9f6292dc86 100644 --- a/examples/c/ex_all.c +++ b/examples/c/ex_all.c @@ -1045,63 +1045,6 @@ backup(WT_SESSION *session) return (ret); } -#define APPLICATION_ERROR 1 -#define APPLICATION_INFO 2 -int application_logging(int, const char *); -int wiredtiger_handle_error( - WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); -int wiredtiger_handle_message(WT_EVENT_HANDLER *, WT_SESSION *, const char *); - -int -application_logging(int which, const char *message) -{ - (void)which; - (void)message; - return (0); -} - -/*! [Function event_handler] */ - -/* - * handle_error -- - * WiredTiger error handler. - */ -int -wiredtiger_handle_error(WT_EVENT_HANDLER *handler, - WT_SESSION *session, int error, const char *message) -{ - int ret; - - (void)(handler); /* Unused variables */ - (void)(error); - - /* Timestamp and log the error message. */ - ret = application_logging(APPLICATION_ERROR, message); - - /* Copy and flush the message to stderr. */ - if (fprintf(stderr, "%p: %s\n", session, message) < 0 && ret == 0) - ret = EIO; - if (fflush(stderr) != 0 && ret == 0) - ret = errno; - return (ret); -} - -/* - * handle_message -- - * WiredTiger message handler. - */ -int -wiredtiger_handle_message( - WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *message) -{ - (void)(handler); /* Unused variables */ - (void)(session); /* Unused variables */ - - /* Timestamp and log the informational message. */ - return (application_logging(APPLICATION_INFO, message)); -} -/*! [Function event_handler] */ - int main(void) { @@ -1171,20 +1114,6 @@ main(void) if (ret == 0) (void)conn->close(conn, NULL); #endif - { - /*! [Configure event_handler] */ - WT_EVENT_HANDLER event_handler; - - event_handler.handle_error = wiredtiger_handle_error; - event_handler.handle_message = wiredtiger_handle_message; - event_handler.handle_progress = NULL; - event_handler.handle_close = NULL; - - ret = wiredtiger_open(home, &event_handler, "create", &conn); - /*! [Configure event_handler] */ - if (ret == 0) - (void)conn->close(conn, NULL); - } /*! [Configure file_extend] */ ret = wiredtiger_open( diff --git a/examples/c/ex_event_handler.c b/examples/c/ex_event_handler.c new file mode 100644 index 00000000000..1e277cab00c --- /dev/null +++ b/examples/c/ex_event_handler.c @@ -0,0 +1,120 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * ex_event_handler.c + * Demonstrate how to use the WiredTiger event handler mechanism. + * + */ +#include +#include + +#include + +static const char * const progname = "ex_event_handler"; +static const char *home; + +int handle_wiredtiger_error( + WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); +int handle_wiredtiger_message(WT_EVENT_HANDLER *, WT_SESSION *, const char *); + +/*! [Function event_handler] */ + +/* + * Setup our own event handler structure to allow us to pass context through + * to event handler callbacks. For this to work the WiredTiger event handler + * must appear first in our custom event handler structure. + */ +typedef struct { + WT_EVENT_HANDLER h; + const char *app_id; +} CUSTOM_EVENT_HANDLER; + +/* + * handle_wiredtiger_error -- + * Function to handle error callbacks from WiredTiger. + */ +int +handle_wiredtiger_error(WT_EVENT_HANDLER *handler, + WT_SESSION *session, int error, const char *message) +{ + CUSTOM_EVENT_HANDLER *custom_handler; + + /* Cast the handler back to our custom handler. */ + custom_handler = (CUSTOM_EVENT_HANDLER *)handler; + + /* Report the error on the console. */ + fprintf(stderr, + "app_id %s, thread context %p, error %d, message %s\n", + custom_handler->app_id, session, error, message); + + return (0); +} + +/* + * handle_wiredtiger_message -- + * Function to handle message callbacks from WiredTiger. + */ +int +handle_wiredtiger_message( + WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *message) +{ + printf("app id %s, thread context %p, message %s\n", + ((CUSTOM_EVENT_HANDLER*)handler)->app_id, session, message); + + return (0); +} +/*! [Function event_handler] */ + +int +main(void) +{ + WT_CONNECTION *conn; + WT_SESSION *session; + int ret; + + /*! [Configure event_handler] */ + CUSTOM_EVENT_HANDLER event_handler; + + event_handler.h.handle_error = handle_wiredtiger_error; + event_handler.h.handle_message = handle_wiredtiger_message; + /* Setup a NULL to use the default handler. */ + event_handler.h.handle_progress = NULL; + event_handler.h.handle_close = NULL; + event_handler.app_id = "example_event_handler"; + + ret = wiredtiger_open(home, + (WT_EVENT_HANDLER*)&event_handler, "create", &conn); + /*! [Configure event_handler] */ + + /* Make an invalid API call, to ensure the event handler works. */ + (void)conn->open_session(conn, NULL, "isolation=invalid", &session); + + if (ret == 0) + (void)conn->close(conn, NULL); + + return (ret); +} diff --git a/src/docs/error-handling.dox b/src/docs/error-handling.dox index 8ff3b68d64c..d91a126ee21 100644 --- a/src/docs/error-handling.dox +++ b/src/docs/error-handling.dox @@ -78,8 +78,8 @@ application-specific logging function that added a timestamp and logged the message to a file, and error messages might additionally be output to the \c stderr file stream. -@snippet ex_all.c Function event_handler -@snippet ex_all.c Configure event_handler +@snippet ex_event_handler.c Function event_handler +@snippet ex_event_handler.c Configure event_handler @m_endif diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index 32def98b5e2..bca9a6a248b 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -1983,7 +1983,8 @@ struct __wt_connection { * * @param connection the connection handle * @param errhandler An error handler. If NULL, the - * connection's error handler is used + * connection's error handler is used. See @ref error_handling_event + * for more information. * @configstart{WT_CONNECTION.open_session, see dist/api_data.py} * @config{isolation, the default isolation level for operations in this * session., a string\, chosen from the following options: \c @@ -2144,7 +2145,8 @@ struct __wt_connection { * @param home The path to the database home directory. See @ref home * for more information. * @param errhandler An error handler. If NULL, a builtin error - * handler is installed that writes error messages to stderr + * handler is installed that writes error messages to stderr. See + * @ref error_handling_event for more information. * @configstart{wiredtiger_open, see dist/api_data.py} * @config{async = (, asynchronous operations configuration options., a set of * related configuration options defined below.} -- cgit v1.2.1 From f90fb74abfd080de37ea92e6091d356d37cf3d25 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 23 Feb 2016 09:37:59 -0500 Subject: WT-2107: Add example code including an event handler Review, minor wording and style cleanup. --- examples/c/ex_event_handler.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/c/ex_event_handler.c b/examples/c/ex_event_handler.c index 1e277cab00c..fae61092687 100644 --- a/examples/c/ex_event_handler.c +++ b/examples/c/ex_event_handler.c @@ -42,9 +42,8 @@ int handle_wiredtiger_error( int handle_wiredtiger_message(WT_EVENT_HANDLER *, WT_SESSION *, const char *); /*! [Function event_handler] */ - /* - * Setup our own event handler structure to allow us to pass context through + * Create our own event handler structure to allow us to pass context through * to event handler callbacks. For this to work the WiredTiger event handler * must appear first in our custom event handler structure. */ @@ -82,8 +81,9 @@ int handle_wiredtiger_message( WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *message) { + /* Cast the handler back to our custom handler. */ printf("app id %s, thread context %p, message %s\n", - ((CUSTOM_EVENT_HANDLER*)handler)->app_id, session, message); + ((CUSTOM_EVENT_HANDLER *)handler)->app_id, session, message); return (0); } @@ -101,13 +101,13 @@ main(void) event_handler.h.handle_error = handle_wiredtiger_error; event_handler.h.handle_message = handle_wiredtiger_message; - /* Setup a NULL to use the default handler. */ + /* Set handlers to NULL to use the default handler. */ event_handler.h.handle_progress = NULL; event_handler.h.handle_close = NULL; event_handler.app_id = "example_event_handler"; ret = wiredtiger_open(home, - (WT_EVENT_HANDLER*)&event_handler, "create", &conn); + (WT_EVENT_HANDLER *)&event_handler, "create", &conn); /*! [Configure event_handler] */ /* Make an invalid API call, to ensure the event handler works. */ -- cgit v1.2.1 From b6a1357f1a3ba992166779ab944aa1774d15243d Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 23 Feb 2016 09:52:18 -0500 Subject: WT-2107: Add example code including an event handler Add WIREDTIGER_HOME handling so "make check" works, remove unused variable. --- examples/c/ex_event_handler.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/examples/c/ex_event_handler.c b/examples/c/ex_event_handler.c index fae61092687..76af3e324f4 100644 --- a/examples/c/ex_event_handler.c +++ b/examples/c/ex_event_handler.c @@ -30,11 +30,11 @@ * */ #include +#include #include #include -static const char * const progname = "ex_event_handler"; static const char *home; int handle_wiredtiger_error( @@ -96,6 +96,17 @@ main(void) WT_SESSION *session; int ret; + /* + * Create a clean test directory for this run of the test program if the + * environment variable isn't already set (as is done by make check). + */ + if (getenv("WIREDTIGER_HOME") == NULL) { + home = "WT_HOME"; + ret = system("rm -rf WT_HOME && mkdir WT_HOME"); + } else + home = NULL; + + { /*! [Configure event_handler] */ CUSTOM_EVENT_HANDLER event_handler; @@ -109,6 +120,7 @@ main(void) ret = wiredtiger_open(home, (WT_EVENT_HANDLER *)&event_handler, "create", &conn); /*! [Configure event_handler] */ + } /* Make an invalid API call, to ensure the event handler works. */ (void)conn->open_session(conn, NULL, "isolation=invalid", &session); -- cgit v1.2.1 From cfcfc53202183cbd314eedd0948baeb1da6734d6 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 23 Feb 2016 10:35:42 -0500 Subject: WT-2107: Add example code including an event handler My last commit introduced a bug, letting the event handler go out of scope before a call that used it. Split out the set of the event handler from the main function, that way we don't have to play games to get the CUSTOM_EVENT_HANDLER declaration next to the code that initializes it. Minor #include cleanup while in the area. --- examples/c/ex_all.c | 1 - examples/c/ex_event_handler.c | 34 +++++++++++++++++++--------------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/examples/c/ex_all.c b/examples/c/ex_all.c index f9f6292dc86..1c036b75461 100644 --- a/examples/c/ex_all.c +++ b/examples/c/ex_all.c @@ -36,7 +36,6 @@ #include #include -#include #include #include #include diff --git a/examples/c/ex_event_handler.c b/examples/c/ex_event_handler.c index 76af3e324f4..ba6807cd56d 100644 --- a/examples/c/ex_event_handler.c +++ b/examples/c/ex_event_handler.c @@ -89,24 +89,13 @@ handle_wiredtiger_message( } /*! [Function event_handler] */ -int -main(void) +static int +config_event_handler() { WT_CONNECTION *conn; WT_SESSION *session; int ret; - /* - * Create a clean test directory for this run of the test program if the - * environment variable isn't already set (as is done by make check). - */ - if (getenv("WIREDTIGER_HOME") == NULL) { - home = "WT_HOME"; - ret = system("rm -rf WT_HOME && mkdir WT_HOME"); - } else - home = NULL; - - { /*! [Configure event_handler] */ CUSTOM_EVENT_HANDLER event_handler; @@ -120,13 +109,28 @@ main(void) ret = wiredtiger_open(home, (WT_EVENT_HANDLER *)&event_handler, "create", &conn); /*! [Configure event_handler] */ - } /* Make an invalid API call, to ensure the event handler works. */ (void)conn->open_session(conn, NULL, "isolation=invalid", &session); if (ret == 0) - (void)conn->close(conn, NULL); + ret = conn->close(conn, NULL); return (ret); } + +int +main(void) +{ + /* + * Create a clean test directory for this run of the test program if the + * environment variable isn't already set (as is done by make check). + */ + if (getenv("WIREDTIGER_HOME") == NULL) { + home = "WT_HOME"; + (void)system("rm -rf WT_HOME && mkdir WT_HOME"); + } else + home = NULL; + + return (config_event_handler()); +} -- cgit v1.2.1 From 23598f1b4a7374cec53c04ccc058292bfab1167f Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 23 Feb 2016 10:53:21 -0500 Subject: WT-2394 Gather handles for compact at the beginning. Fix a typo. --- src/include/session.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/session.h b/src/include/session.h index 5c442bf0b81..7fdb7fc2548 100644 --- a/src/include/session.h +++ b/src/include/session.h @@ -137,7 +137,7 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { * The preferred pattern is to gather all of the required handles at * the beginning of an operation, then drop any other locks, perform * the operation, then release the handles. This cannot be easily - * merged with the list of checkpoint handles because some operation + * merged with the list of checkpoint handles because some operations * (such as compact) do checkpoints internally. */ WT_DATA_HANDLE **op_handle; /* Handle list */ -- cgit v1.2.1 From 65c9ab0105b8198a509cf40281a947e0882ae3f3 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 23 Feb 2016 11:07:01 -0500 Subject: WT-2394 Gather handles for compact at the beginning. Fix a comment typo. --- src/session/session_compact.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/session/session_compact.c b/src/session/session_compact.c index d2fd995b554..2a53ad58f52 100644 --- a/src/session/session_compact.c +++ b/src/session/session_compact.c @@ -287,7 +287,7 @@ __wt_session_compact( WT_ERR(__wt_config_gets(session, cfg, "timeout", &cval)); session->compact->max_time = (uint64_t)cval.val; - /* Find the types of data sources are being compacted. */ + /* Find the types of data sources being compacted. */ WT_WITH_SCHEMA_LOCK(session, ret, ret = __wt_schema_worker(session, uri, __compact_handle_append, __compact_uri_analyze, cfg, 0)); -- cgit v1.2.1 From fb4909a127d046483120743d64d148a4c991fa71 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 23 Feb 2016 11:23:08 -0500 Subject: WT-2420 Typo --- src/conn/conn_dhandle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/conn/conn_dhandle.c b/src/conn/conn_dhandle.c index e3f9871878b..f1b35571533 100644 --- a/src/conn/conn_dhandle.c +++ b/src/conn/conn_dhandle.c @@ -355,7 +355,7 @@ err: F_CLR(btree, WT_BTREE_SPECIAL_FLAGS); /* * __conn_btree_apply_internal -- - * Apply a function an open data handle. + * Apply a function to an open data handle. */ static int __conn_btree_apply_internal(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle, -- cgit v1.2.1 From 628d64f8ef124764615f4f958baf03e450382bc9 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 23 Feb 2016 11:58:39 -0500 Subject: WT-2107: Add example code including an event handler Fix the Java call to strerror. --- examples/java/com/wiredtiger/examples/ex_all.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/java/com/wiredtiger/examples/ex_all.java b/examples/java/com/wiredtiger/examples/ex_all.java index 61092c8be33..5fe767d49bf 100644 --- a/examples/java/com/wiredtiger/examples/ex_all.java +++ b/examples/java/com/wiredtiger/examples/ex_all.java @@ -333,7 +333,7 @@ public static int cursor_ops(Session session) cursor.putKeyString(key); if ((ret = cursor.remove()) != 0) { System.err.println( - "cursor.remove: " + cursor.session.strerror(ret)); + "cursor.remove: " + wiredtiger.wiredtiger_strerror(ret)); return (ret); } } catch (WiredTigerException wte) { /* Catch severe errors. */ -- cgit v1.2.1 From 373c329bb7f1680185f08d98bd9e004cf95678cf Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 23 Feb 2016 13:51:00 -0500 Subject: WT-2423 Cleanup on error when opening a session handle. We're not holding locks when sweeping the session's handle list, move the call of __session_dhandle_sweep() from __session_add_dhandle() to __session_get_dhandle(), simplifying __session_add_dhandle()'s error handling as it no longer calls __sesion_discard_dhandle(). Move the fixup of the handle reference count after __session_add_dhandle() failure to __session_get_dhandle(), that way the increment/decrement of the reference count are relatively close together and logically within a few lines of each other. Remove the final argument of __session_add_dhandle(), it was never used. --- src/session/session_dhandle.c | 71 +++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 39 deletions(-) diff --git a/src/session/session_dhandle.c b/src/session/session_dhandle.c index c8346ce2bb2..242d9ac5cc4 100644 --- a/src/session/session_dhandle.c +++ b/src/session/session_dhandle.c @@ -10,46 +10,18 @@ static int __session_dhandle_sweep(WT_SESSION_IMPL *); -/* - * __session_discard_dhandle -- - * Remove a data handle from the session cache. - */ -static void -__session_discard_dhandle( - WT_SESSION_IMPL *session, WT_DATA_HANDLE_CACHE *dhandle_cache) -{ - uint64_t bucket; - - bucket = dhandle_cache->dhandle->name_hash % WT_HASH_ARRAY_SIZE; - TAILQ_REMOVE(&session->dhandles, dhandle_cache, q); - TAILQ_REMOVE(&session->dhhash[bucket], dhandle_cache, hashq); - - (void)__wt_atomic_sub32(&dhandle_cache->dhandle->session_ref, 1); - - __wt_overwrite_and_free(session, dhandle_cache); -} - /* * __session_add_dhandle -- * Add a handle to the session's cache. */ static int -__session_add_dhandle( - WT_SESSION_IMPL *session, WT_DATA_HANDLE_CACHE **dhandle_cachep) +__session_add_dhandle(WT_SESSION_IMPL *session) { WT_DATA_HANDLE_CACHE *dhandle_cache; - WT_DECL_RET; uint64_t bucket; - /* - * Allocate a handle cache entry, fixup the reference count on failure - * since we are part way through adding a handle and have already - * increased the reference count while holding a lock. - */ - if ((ret = __wt_calloc_one(session, &dhandle_cache)) != 0) { - (void)__wt_atomic_sub32(&session->dhandle->session_ref, 1); - return (ret); - } + /* Allocate a handle cache entry. */ + WT_RET(__wt_calloc_one(session, &dhandle_cache)); dhandle_cache->dhandle = session->dhandle; @@ -57,14 +29,26 @@ __session_add_dhandle( TAILQ_INSERT_HEAD(&session->dhandles, dhandle_cache, q); TAILQ_INSERT_HEAD(&session->dhhash[bucket], dhandle_cache, hashq); - /* Sweep the handle list to remove any dead handles. */ - if ((ret = __session_dhandle_sweep(session)) != 0) - __session_discard_dhandle(session, dhandle_cache); + return (0); +} - if (ret == 0 && dhandle_cachep != NULL) - *dhandle_cachep = dhandle_cache; +/* + * __session_discard_dhandle -- + * Remove a data handle from the session cache. + */ +static void +__session_discard_dhandle( + WT_SESSION_IMPL *session, WT_DATA_HANDLE_CACHE *dhandle_cache) +{ + uint64_t bucket; - return (ret); + bucket = dhandle_cache->dhandle->name_hash % WT_HASH_ARRAY_SIZE; + TAILQ_REMOVE(&session->dhandles, dhandle_cache, q); + TAILQ_REMOVE(&session->dhhash[bucket], dhandle_cache, hashq); + + (void)__wt_atomic_sub32(&dhandle_cache->dhandle->session_ref, 1); + + __wt_overwrite_and_free(session, dhandle_cache); } /* @@ -463,14 +447,23 @@ __session_get_dhandle( return (0); } + /* Sweep the handle list to remove any dead handles. */ + WT_RET(__session_dhandle_sweep(session)); + /* * We didn't find a match in the session cache, search the shared * handle list and cache the handle we find. */ WT_WITH_HANDLE_LIST_LOCK(session, ret = __session_find_shared_dhandle(session, uri, checkpoint)); - if (ret == 0) - ret = __session_add_dhandle(session, NULL); + WT_RET(ret); + + /* + * Fixup the reference count on failure (we incremented the reference + * count while holding the handle-list lock). + */ + if ((ret = __session_add_dhandle(session)) != 0) + (void)__wt_atomic_sub32(&session->dhandle->session_ref, 1); return (ret); } -- cgit v1.2.1 From 585cbf88bec99baa98804c9941c17fc2f0a82051 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 24 Feb 2016 14:16:31 -0500 Subject: WT-2425: Revert "WT-2173: test/format cache stuck full" This reverts commit db58fef15da999a34764b5da4ae6b016f1b8529c. --- src/evict/evict_lru.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index fe503c0777c..8ef7164dbc6 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -1369,12 +1369,10 @@ __evict_get_ref( WT_SESSION_IMPL *session, bool is_server, WT_BTREE **btreep, WT_REF **refp) { WT_CACHE *cache; - WT_CONNECTION_IMPL *conn; WT_EVICT_ENTRY *evict; uint32_t candidates; - conn = S2C(session); - cache = conn->cache; + cache = S2C(session)->cache; *btreep = NULL; *refp = NULL; @@ -1394,14 +1392,11 @@ __evict_get_ref( } /* - * The eviction server normally only tries to evict half of the pages - * before looking for more, unless there isn't anybody else to do the - * work, we're stuck, or there's only a single candidate (that happens - * in very small environments). + * The eviction server only tries to evict half of the pages before + * looking for more. */ candidates = cache->evict_candidates; - if (is_server && conn->evict_workers_max != 0 && - !F_ISSET(cache, WT_CACHE_STUCK) && candidates > 1) + if (is_server && candidates > 1) candidates /= 2; /* Get the next page queued for eviction. */ -- cgit v1.2.1 From 77e71cab199234e102c5288e7d51db89464e9285 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 24 Feb 2016 14:17:08 -0500 Subject: WT-2425: evict-btree read through performance drop Enhance a comment, explain why the eviction server only evicts half of the pages it queues. --- src/evict/evict_lru.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index 8ef7164dbc6..dc1a104a0fc 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -1392,8 +1392,9 @@ __evict_get_ref( } /* - * The eviction server only tries to evict half of the pages before - * looking for more. + * Only evict half of the pages before looking for more. The remainder + * are left to eviction workers (if configured), or application threads + * if necessary. */ candidates = cache->evict_candidates; if (is_server && candidates > 1) -- cgit v1.2.1 From 8d9cab395ddec561646239003b36e348497cc4ca Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 24 Feb 2016 14:18:14 -0500 Subject: WT-2425: evict-btree read through performance drop Update list of strings. --- dist/s_string.ok | 1 + 1 file changed, 1 insertion(+) diff --git a/dist/s_string.ok b/dist/s_string.ok index ee968b758dc..c9c799da5ba 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -113,6 +113,7 @@ FORALL FOREACH FULLFSYNC FindFirstFile +Fixup Fk FlushFileBuffers Fprintf -- cgit v1.2.1 From 58df5c8238f1004eba49241fe9e2ba0b74575808 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Wed, 24 Feb 2016 14:47:19 -0500 Subject: WT-2381 Add new test for table configuration and dump. --- test/suite/test_util13.py | 153 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 test/suite/test_util13.py diff --git a/test/suite/test_util13.py b/test/suite/test_util13.py new file mode 100644 index 00000000000..5a7f217ddfb --- /dev/null +++ b/test/suite/test_util13.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os, re, string +from suite_subprocess import suite_subprocess +import itertools, wiredtiger, wttest + +from helper import complex_populate, complex_populate_lsm, simple_populate +from wtscenario import multiply_scenarios, number_scenarios + +# test_util13.py +# Utilities: wt dump, as well as the dump cursor +class test_util13(wttest.WiredTigerTestCase, suite_subprocess): + """ + Test wt dump. We check for specific output and preservation of + non-default table create parameters. + """ + + pfx = 'test_util13' + nentries = 100 + # + # Select table configuration settings that are not the default. + # + types = [ + ('file-simple', dict(uri='file:' + pfx, pop=simple_populate, + table_config='prefix_compression_min=3', cfg='')), + ('lsm-simple', dict(uri='lsm:' + pfx, pop=simple_populate, + table_config='lsm=(bloom_bit_count=29)', + cfg='bloom_bit_count=29')), + ('table-simple', dict(uri='table:' + pfx, pop=simple_populate, + table_config='split_pct=50', cfg='')), + ('table-complex', + dict(uri='table:' + pfx, pop=complex_populate, + table_config='allocation_size=512B', cfg='')), + ('table-complex-lsm', + dict(uri='table:' + pfx, pop=complex_populate_lsm, + table_config='lsm=(merge_max=5)', + cfg='merge_max=5')), + ] + + scenarios = number_scenarios(multiply_scenarios('.', types)) + + def compare_config(self, expected_cfg, actual_cfg): + # Replace '(' characters so configuration groups don't break parsing. + # If we ever want to look for config groups this will need to change. + #print "compare_config Actual config " + #print actual_cfg + #print "compare_config Expected config " + #print expected_cfg + cfg_orig = actual_cfg + if self.pop != simple_populate: + # + # If we have a complex config, strip out the colgroups and + # columns from the config. Doing so allows us to keep the + # split commands below usable because those two items don't + # have assignments in them. + # + nocolgrp = re.sub("colgroups=\((.+?)\),", '', actual_cfg) + cfg_orig = re.sub("columns=\((.+?)\),", '', nocolgrp) + + #print "Using original config " + #print cfg_orig + da = dict(kv.split('=') for kv in + cfg_orig.strip().replace('(',',').split(',')) + dx = dict(kv.split('=') for kv in + expected_cfg.strip().replace('(',',').split(',')) + + # Check that all items in our expected config subset are in + # the actual configuration and they match. + match = all(item in da.items() for item in dx.items()) + if match == False: + print "MISMATCH:" + print "Original dict: " + print da + print "Expected config: " + print dx + return match + + def compare_files(self, filename1, filename2): + inheader = isconfig = False + for l1, l2 in zip(open(filename1, "rb"), open(filename2, "rb")): + if isconfig: + if not self.compare_config(l1, l2): + return False + elif l1 != l2: + return False + if inheader: + isconfig = not isconfig + if l1.strip() == 'Header': + inheader = True + if l1.strip() == 'Data': + inheader = isconfig = False + return True + + def test_dump_config(self): + # The number of btree_entries reported is influenced by the + # number of column groups and indices. Each insert will have + # a multiplied effect. + self.pop(self, self.uri, + 'key_format=S,value_format=S,' + self.table_config, self.nentries) + + ver = wiredtiger.wiredtiger_version() + verstring = str(ver[1]) + '.' + str(ver[2]) + '.' + str(ver[3]) + with open("expect.out", "w") as expectout: + # Note: this output is sensitive to the precise output format + # generated by wt dump. If this is likely to change, we should + # make this test more accommodating. + expectout.write( + 'WiredTiger Dump (WiredTiger Version ' + verstring + ')\n') + expectout.write('Format=print\n') + expectout.write('Header\n') + expectout.write(self.uri + '\n') + if self.cfg == '': + expectout.write(self.table_config + '\n') + else: + expectout.write(self.cfg + '\n') + expectout.write('Data\n') + + self.pr('calling dump') + with open("dump.out", "w") as dumpout: + dumpargs = ["dump"] + dumpargs.append(self.uri) + self.runWt(dumpargs, outfilename="dump.out") + + self.assertTrue(self.compare_files("expect.out", "dump.out")) + +if __name__ == '__main__': + wttest.run() -- cgit v1.2.1 From a11ce05ac20e1b9b59a69702377e33943f3e820e Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 24 Feb 2016 17:47:21 -0500 Subject: WT-2381: dump utility discards table config whitespace --- test/suite/test_util13.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/suite/test_util13.py b/test/suite/test_util13.py index 5a7f217ddfb..84286dd4dc1 100644 --- a/test/suite/test_util13.py +++ b/test/suite/test_util13.py @@ -95,9 +95,9 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): match = all(item in da.items() for item in dx.items()) if match == False: print "MISMATCH:" - print "Original dict: " + print "Original dict: " print da - print "Expected config: " + print "Expected config: " print dx return match -- cgit v1.2.1 From 5dd31d182c3319b1827f502caa852d39d36d881a Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Thu, 25 Feb 2016 10:13:28 +1100 Subject: WT-2427 Have wtperf use builtin compression when configured. --- bench/wtperf/config.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bench/wtperf/config.c b/bench/wtperf/config.c index 5af2e977a25..2544bbd371c 100644 --- a/bench/wtperf/config.c +++ b/bench/wtperf/config.c @@ -159,13 +159,19 @@ config_compress(CONFIG *cfg) cfg->compress_ext = NULL; cfg->compress_table = NULL; } else if (strcmp(s, "lz4") == 0) { +#ifndef HAVE_BUILTIN_EXTENSION_LZ4 cfg->compress_ext = LZ4_EXT; +#endif cfg->compress_table = LZ4_BLK; } else if (strcmp(s, "snappy") == 0) { +#ifndef HAVE_BUILTIN_EXTENSION_SNAPPY cfg->compress_ext = SNAPPY_EXT; +#endif cfg->compress_table = SNAPPY_BLK; } else if (strcmp(s, "zlib") == 0) { +#ifndef HAVE_BUILTIN_EXTENSION_ZLIB cfg->compress_ext = ZLIB_EXT; +#endif cfg->compress_table = ZLIB_BLK; } else { fprintf(stderr, -- cgit v1.2.1 From 93afecc49cd875b450155a1a03ae5c66e065cade Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Thu, 25 Feb 2016 11:50:55 +1100 Subject: WT-2428 Add a "json" mode to statistics logging. This generates statistics in a format that is compatible with MongoDB so that we can share tools for analysis. --- dist/api_data.py | 6 +- dist/s_string.ok | 2 + dist/stat.py | 12 +- dist/stat_data.py | 6 +- src/config/config_def.c | 21 +- src/conn/conn_stat.c | 99 ++++-- src/include/connection.h | 7 +- src/include/stat.h | 256 +++++++-------- src/include/wiredtiger.in | 806 +++++++++++++++++++++++----------------------- src/support/stat.c | 392 +++++++++++----------- 10 files changed, 833 insertions(+), 774 deletions(-) diff --git a/dist/api_data.py b/dist/api_data.py index 0eee515e851..5575bd9f790 100644 --- a/dist/api_data.py +++ b/dist/api_data.py @@ -522,6 +522,9 @@ connection_runtime_config = [ the statistics log server uses a session from the configured session_max''', type='category', subconfig=[ + Config('json', 'false', r''' + encode statistics in JSON format''', + type='boolean'), Config('on_close', 'false', r'''log statistics on database close''', type='boolean'), Config('path', '"WiredTigerStat.%d.%H"', r''' @@ -538,7 +541,8 @@ connection_runtime_config = [ type='list'), Config('timestamp', '"%b %d %H:%M:%S"', r''' a timestamp prepended to each log record, may contain strftime - conversion specifications'''), + conversion specifications, when \c json is configured, defaults + to \c "%FT%Y.000Z"'''), Config('wait', '0', r''' seconds to wait between each write of the log records; setting this value above 0 configures statistics logging''', diff --git a/dist/s_string.ok b/dist/s_string.ok index c9c799da5ba..c8769abdfbe 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -728,6 +728,7 @@ libwiredtiger llll llu loadtext +localTime localtime logf logmgr @@ -1076,6 +1077,7 @@ waitpid walk's warmup wb +wiredTiger wiredtiger workFactor wrapup diff --git a/dist/stat.py b/dist/stat.py index 6dcfccfeab5..7961bf7053f 100644 --- a/dist/stat.py +++ b/dist/stat.py @@ -98,11 +98,11 @@ for line in open('../src/include/wiredtiger.in', 'r'): f.close() compare_srcfile(tmp_file, '../src/include/wiredtiger.in') -def print_func(name, handle, list): +def print_func(name, handle, statlist): '''Print the structures/functions for the stat.c file.''' f.write('\n') f.write('static const char * const __stats_' + name + '_desc[] = {\n') - for l in list: + for l in statlist: f.write('\t"' + l.desc + '",\n') f.write('};\n') @@ -143,7 +143,7 @@ void __wt_stat_''' + name + '_clear_single(WT_' + name.upper() + '''_STATS *stats) { ''') - for l in sorted(list): + for l in statlist: # no_clear: don't clear the value. if 'no_clear' in l.flags: f.write('\t\t/* not clearing ' + l.name + ' */\n') @@ -170,7 +170,7 @@ __wt_stat_''' + name + '''_aggregate_single( WT_''' + name.upper() + '_STATS *from, WT_' + name.upper() + '''_STATS *to) { ''') - for l in sorted(list): + for l in statlist: if 'max_aggregate' in l.flags: o = '\tif (from->' + l.name + ' > to->' + l.name + ')\n' +\ '\t\tto->' + l.name + ' = from->' + l.name + ';\n' @@ -190,11 +190,11 @@ __wt_stat_''' + name + '''_aggregate( # Connection level aggregation does not currently have any computation # of a maximum value; I'm leaving in support for it, but don't declare # a temporary variable until it's needed. - for l in sorted(list): + for l in statlist: if 'max_aggregate' in l.flags: f.write('\tint64_t v;\n\n') break; - for l in sorted(list): + for l in statlist: if 'max_aggregate' in l.flags: o = '\tif ((v = WT_STAT_READ(from, ' + l.name + ')) > ' +\ 'to->' + l.name + ')\n' diff --git a/dist/stat_data.py b/dist/stat_data.py index 41a93961079..36e9ff4b18c 100644 --- a/dist/stat_data.py +++ b/dist/stat_data.py @@ -315,7 +315,7 @@ connection_stats = [ YieldStat('page_sleep', 'page acquire time sleeping (usecs)'), ] -connection_stats = sorted(connection_stats, key=attrgetter('name')) +connection_stats = sorted(connection_stats, key=attrgetter('desc')) ########################################## # Data source statistics @@ -457,7 +457,7 @@ dsrc_stats = [ TxnStat('txn_update_conflict', 'update conflicts'), ] -dsrc_stats = sorted(dsrc_stats, key=attrgetter('name')) +dsrc_stats = sorted(dsrc_stats, key=attrgetter('desc')) ########################################## # Cursor Join statistics @@ -468,4 +468,4 @@ join_stats = [ JoinStat('bloom_false_positive', 'bloom filter false positives'), ] -join_stats = sorted(join_stats, key=attrgetter('name')) +join_stats = sorted(join_stats, key=attrgetter('desc')) diff --git a/src/config/config_def.c b/src/config/config_def.c index f1e48d85894..6e88f9b4d14 100644 --- a/src/config/config_def.c +++ b/src/config/config_def.c @@ -99,6 +99,7 @@ static const WT_CONFIG_CHECK static const WT_CONFIG_CHECK confchk_wiredtiger_open_statistics_log_subconfigs[] = { + { "json", "boolean", NULL, NULL, NULL, 0 }, { "on_close", "boolean", NULL, NULL, NULL, 0 }, { "path", "string", NULL, NULL, NULL, 0 }, { "sources", "list", NULL, NULL, NULL, 0 }, @@ -146,7 +147,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { NULL, 0 }, { "statistics_log", "category", NULL, NULL, - confchk_wiredtiger_open_statistics_log_subconfigs, 5 }, + confchk_wiredtiger_open_statistics_log_subconfigs, 6 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," "\"evict\",\"evictserver\",\"fileops\",\"log\",\"lsm\"," @@ -555,7 +556,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { NULL, 0 }, { "statistics_log", "category", NULL, NULL, - confchk_wiredtiger_open_statistics_log_subconfigs, 5 }, + confchk_wiredtiger_open_statistics_log_subconfigs, 6 }, { "transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2 }, @@ -636,7 +637,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { NULL, 0 }, { "statistics_log", "category", NULL, NULL, - confchk_wiredtiger_open_statistics_log_subconfigs, 5 }, + confchk_wiredtiger_open_statistics_log_subconfigs, 6 }, { "transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2 }, @@ -714,7 +715,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { NULL, 0 }, { "statistics_log", "category", NULL, NULL, - confchk_wiredtiger_open_statistics_log_subconfigs, 5 }, + confchk_wiredtiger_open_statistics_log_subconfigs, 6 }, { "transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2 }, @@ -790,7 +791,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { NULL, 0 }, { "statistics_log", "category", NULL, NULL, - confchk_wiredtiger_open_statistics_log_subconfigs, 5 }, + confchk_wiredtiger_open_statistics_log_subconfigs, 6 }, { "transaction_sync", "category", NULL, NULL, confchk_wiredtiger_open_transaction_sync_subconfigs, 2 }, @@ -857,7 +858,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { "file_max=100MB,path=,prealloc=,recover=on,zero_fill=0)," "lsm_manager=(merge=,worker_thread_max=4),lsm_merge=," "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," - "statistics=none,statistics_log=(on_close=0," + "statistics=none,statistics_log=(json=0,on_close=0," "path=\"WiredTigerStat.%d.%H\",sources=," "timestamp=\"%b %d %H:%M:%S\",wait=0),verbose=", confchk_WT_CONNECTION_reconfigure, 18 @@ -1024,7 +1025,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,readonly=0," "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" ",name=,quota=0,reserve=0,size=500MB),statistics=none," - "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," + "statistics_log=(json=0,on_close=0,path=\"WiredTigerStat.%d.%H\"," "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "transaction_sync=(enabled=0,method=fsync),use_environment=," "use_environment_priv=0,verbose=,write_through=", @@ -1045,7 +1046,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,readonly=0," "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" ",name=,quota=0,reserve=0,size=500MB),statistics=none," - "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," + "statistics_log=(json=0,on_close=0,path=\"WiredTigerStat.%d.%H\"," "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "transaction_sync=(enabled=0,method=fsync),use_environment=," "use_environment_priv=0,verbose=,version=(major=0,minor=0)," @@ -1066,7 +1067,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,readonly=0," "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" ",name=,quota=0,reserve=0,size=500MB),statistics=none," - "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," + "statistics_log=(json=0,on_close=0,path=\"WiredTigerStat.%d.%H\"," "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "transaction_sync=(enabled=0,method=fsync),verbose=," "version=(major=0,minor=0),write_through=", @@ -1086,7 +1087,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { "worker_thread_max=4),lsm_merge=,mmap=,multiprocess=0,readonly=0," "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" ",name=,quota=0,reserve=0,size=500MB),statistics=none," - "statistics_log=(on_close=0,path=\"WiredTigerStat.%d.%H\"," + "statistics_log=(json=0,on_close=0,path=\"WiredTigerStat.%d.%H\"," "sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "transaction_sync=(enabled=0,method=fsync),verbose=," "write_through=", diff --git a/src/conn/conn_stat.c b/src/conn/conn_stat.c index 50f8a8b0ca9..e89a7e81600 100644 --- a/src/conn/conn_stat.c +++ b/src/conn/conn_stat.c @@ -85,6 +85,11 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp) *runp = cval.val != 0; conn->stat_usecs = (uint64_t)cval.val * WT_MILLION; + WT_RET(__wt_config_gets( + session, cfg, "statistics_log.json", &cval)); + if (cval.val != 0) + FLD_SET(conn->stat_flags, WT_CONN_STAT_JSON); + WT_RET(__wt_config_gets( session, cfg, "statistics_log.on_close", &cval)); if (cval.val != 0) @@ -140,9 +145,23 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp) WT_ERR(__wt_config_gets(session, cfg, "statistics_log.path", &cval)); WT_ERR(__wt_nfilename(session, cval.str, cval.len, &conn->stat_path)); - WT_ERR(__wt_config_gets( - session, cfg, "statistics_log.timestamp", &cval)); - WT_ERR(__wt_strndup(session, cval.str, cval.len, &conn->stat_format)); + /* + * When using JSON format, use the same timestamp format as MongoDB by + * default. + */ + if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_JSON)) { + WT_ERR(__wt_config_gets_def( + session, cfg, "statistics_log.timestamp", 0, &cval)); + if (cval.type == WT_CONFIG_ITEM_NUM) + WT_ERR(__wt_strdup( + session, "%FT%T.000Z", &conn->stat_format)); + } + if (conn->stat_format == NULL) { + WT_ERR(__wt_config_gets( + session, cfg, "statistics_log.timestamp", &cval)); + WT_ERR(__wt_strndup( + session, cval.str, cval.len, &conn->stat_format)); + } err: __stat_sources_free(session, &sources); return (ret); @@ -157,22 +176,25 @@ __statlog_dump(WT_SESSION_IMPL *session, const char *name, bool conn_stats) { WT_CONNECTION_IMPL *conn; WT_CURSOR *cursor; - WT_CURSOR_STAT *cst; WT_DECL_ITEM(tmp); WT_DECL_RET; - int64_t *stats; - int i; - const char *desc, *uri; + int64_t val; + size_t prefixlen; + const char *desc, *endprefix, *valstr, *uri; const char *cfg[] = { WT_CONFIG_BASE(session, WT_SESSION_open_cursor), NULL }; + bool first, groupfirst; conn = S2C(session); + cursor = NULL; + + WT_RET(__wt_scr_alloc(session, 0, &tmp)); + first = groupfirst = true; /* Build URI and configuration string. */ if (conn_stats) uri = "statistics:"; else { - WT_RET(__wt_scr_alloc(session, 0, &tmp)); WT_ERR(__wt_buf_fmt(session, tmp, "statistics:%s", name)); uri = tmp->data; } @@ -183,31 +205,54 @@ __statlog_dump(WT_SESSION_IMPL *session, const char *name, bool conn_stats) * If we don't find an underlying object, silently ignore it, the object * may exist only intermittently. */ - switch (ret = __wt_curstat_open(session, uri, NULL, cfg, &cursor)) { - case 0: - cst = (WT_CURSOR_STAT *)cursor; - for (stats = cst->stats, i = 0; i < cst->stats_count; ++i) { - if (conn_stats) - WT_ERR(__wt_stat_connection_desc(cst, i, - &desc)); - else - WT_ERR(__wt_stat_dsrc_desc(cst, i, &desc)); + if ((ret = __wt_curstat_open(session, uri, NULL, cfg, &cursor)) != 0) { + if (ret == EBUSY || ret == ENOENT || ret == WT_NOTFOUND) + ret = 0; + goto err; + } + + if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_JSON)) { + WT_ERR(__wt_fprintf(conn->stat_fp, + "{\"version\":\"%s\",\"localTime\":\"%s\"", + WIREDTIGER_VERSION_STRING, conn->stat_stamp)); + WT_ERR(__wt_fprintf(conn->stat_fp, ",\"wiredTiger\":{")); + while ((ret = cursor->next(cursor)) == 0) { + WT_ERR(cursor->get_value(cursor, &desc, &valstr, &val)); + /* Check if we are starting a new section. */ + endprefix = strchr(desc, ':'); + prefixlen = WT_PTRDIFF(endprefix, desc); + WT_ASSERT(session, endprefix != NULL); + if (first || + tmp->size != prefixlen || + strncmp(desc, tmp->data, tmp->size) != 0) { + WT_ERR(__wt_buf_set( + session, tmp, desc, prefixlen)); + WT_ERR(__wt_fprintf(conn->stat_fp, + "%s\"%.*s\":{", first ? "" : "},", + (int)prefixlen, desc)); + first = false; + groupfirst = true; + } + WT_ERR(__wt_fprintf(conn->stat_fp, + "%s\"%s\":%" PRId64, + groupfirst ? "" : ",", endprefix + 2, val)); + groupfirst = false; + } + WT_ERR_NOTFOUND_OK(ret); + WT_ERR(__wt_fprintf(conn->stat_fp, "}}}\n")); + } else { + while ((ret = cursor->next(cursor)) == 0) { + WT_ERR(cursor->get_value(cursor, &desc, &valstr, &val)); WT_ERR(__wt_fprintf(conn->stat_fp, "%s %" PRId64 " %s %s\n", - conn->stat_stamp, stats[i], name, desc)); + conn->stat_stamp, val, name, desc)); } - WT_ERR(cursor->close(cursor)); - break; - case EBUSY: - case ENOENT: - case WT_NOTFOUND: - ret = 0; - break; - default: - break; + WT_ERR_NOTFOUND_OK(ret); } err: __wt_scr_free(session, &tmp); + if (cursor != NULL) + WT_TRET(cursor->close(cursor)); return (ret); } diff --git a/src/include/connection.h b/src/include/connection.h index b0edcef718b..2255056fcf6 100644 --- a/src/include/connection.h +++ b/src/include/connection.h @@ -299,9 +299,10 @@ struct __wt_connection_impl { #define WT_CONN_STAT_ALL 0x01 /* "all" statistics configured */ #define WT_CONN_STAT_CLEAR 0x02 /* clear after gathering */ #define WT_CONN_STAT_FAST 0x04 /* "fast" statistics configured */ -#define WT_CONN_STAT_NONE 0x08 /* don't gather statistics */ -#define WT_CONN_STAT_ON_CLOSE 0x10 /* output statistics on close */ -#define WT_CONN_STAT_SIZE 0x20 /* "size" statistics configured */ +#define WT_CONN_STAT_JSON 0x08 /* output JSON format */ +#define WT_CONN_STAT_NONE 0x10 /* don't gather statistics */ +#define WT_CONN_STAT_ON_CLOSE 0x20 /* output statistics on close */ +#define WT_CONN_STAT_SIZE 0x40 /* "size" statistics configured */ uint32_t stat_flags; /* Connection statistics */ diff --git a/src/include/stat.h b/src/include/stat.h index 51d2fa332e7..aa8f8dd4ed5 100644 --- a/src/include/stat.h +++ b/src/include/stat.h @@ -227,12 +227,22 @@ __wt_stats_clear(void *stats_arg, int slot) */ #define WT_CONNECTION_STATS_BASE 1000 struct __wt_connection_stats { - int64_t async_alloc_race; - int64_t async_alloc_view; + int64_t lsm_work_queue_app; + int64_t lsm_work_queue_manager; + int64_t lsm_rows_merged; + int64_t lsm_checkpoint_throttle; + int64_t lsm_merge_throttle; + int64_t lsm_work_queue_switch; + int64_t lsm_work_units_discarded; + int64_t lsm_work_units_done; + int64_t lsm_work_units_created; + int64_t lsm_work_queue_max; int64_t async_cur_queue; + int64_t async_max_queue; + int64_t async_alloc_race; int64_t async_flush; + int64_t async_alloc_view; int64_t async_full; - int64_t async_max_queue; int64_t async_nowork; int64_t async_op_alloc; int64_t async_op_compact; @@ -240,55 +250,63 @@ struct __wt_connection_stats { int64_t async_op_remove; int64_t async_op_search; int64_t async_op_update; - int64_t block_byte_map_read; - int64_t block_byte_read; - int64_t block_byte_write; - int64_t block_map_read; int64_t block_preload; int64_t block_read; int64_t block_write; - int64_t cache_bytes_dirty; - int64_t cache_bytes_internal; + int64_t block_byte_read; + int64_t block_byte_write; + int64_t block_map_read; + int64_t block_byte_map_read; int64_t cache_bytes_inuse; - int64_t cache_bytes_leaf; - int64_t cache_bytes_max; - int64_t cache_bytes_overflow; int64_t cache_bytes_read; int64_t cache_bytes_write; - int64_t cache_eviction_app; int64_t cache_eviction_checkpoint; - int64_t cache_eviction_clean; - int64_t cache_eviction_deepen; - int64_t cache_eviction_dirty; - int64_t cache_eviction_fail; - int64_t cache_eviction_force; - int64_t cache_eviction_force_delete; - int64_t cache_eviction_force_fail; - int64_t cache_eviction_hazard; - int64_t cache_eviction_internal; - int64_t cache_eviction_maximum_page_size; int64_t cache_eviction_queue_empty; int64_t cache_eviction_queue_not_empty; int64_t cache_eviction_server_evicting; int64_t cache_eviction_server_not_evicting; int64_t cache_eviction_slow; - int64_t cache_eviction_split_internal; - int64_t cache_eviction_split_leaf; - int64_t cache_eviction_walk; int64_t cache_eviction_worker_evicting; - int64_t cache_inmem_split; + int64_t cache_eviction_force_fail; + int64_t cache_eviction_hazard; int64_t cache_inmem_splittable; + int64_t cache_inmem_split; + int64_t cache_eviction_internal; + int64_t cache_eviction_split_internal; + int64_t cache_eviction_split_leaf; int64_t cache_lookaside_insert; int64_t cache_lookaside_remove; - int64_t cache_overhead; - int64_t cache_pages_dirty; + int64_t cache_bytes_max; + int64_t cache_eviction_maximum_page_size; + int64_t cache_eviction_dirty; + int64_t cache_eviction_deepen; + int64_t cache_write_lookaside; int64_t cache_pages_inuse; + int64_t cache_eviction_force; + int64_t cache_eviction_force_delete; + int64_t cache_eviction_app; int64_t cache_read; int64_t cache_read_lookaside; + int64_t cache_eviction_fail; + int64_t cache_eviction_walk; int64_t cache_write; - int64_t cache_write_lookaside; int64_t cache_write_restore; + int64_t cache_overhead; + int64_t cache_bytes_internal; + int64_t cache_bytes_leaf; + int64_t cache_bytes_overflow; + int64_t cache_bytes_dirty; + int64_t cache_pages_dirty; + int64_t cache_eviction_clean; + int64_t file_open; + int64_t memory_allocation; + int64_t memory_free; + int64_t memory_grow; int64_t cond_wait; + int64_t rwlock_read; + int64_t rwlock_write; + int64_t read_io; + int64_t write_io; int64_t cursor_create; int64_t cursor_insert; int64_t cursor_next; @@ -298,96 +316,78 @@ struct __wt_connection_stats { int64_t cursor_restart; int64_t cursor_search; int64_t cursor_search_near; - int64_t cursor_truncate; int64_t cursor_update; + int64_t cursor_truncate; int64_t dh_conn_handle_count; - int64_t dh_session_handles; - int64_t dh_session_sweeps; - int64_t dh_sweep_close; int64_t dh_sweep_ref; + int64_t dh_sweep_close; int64_t dh_sweep_remove; int64_t dh_sweep_tod; int64_t dh_sweeps; - int64_t file_open; - int64_t log_buffer_size; + int64_t dh_session_handles; + int64_t dh_session_sweeps; + int64_t log_slot_switch_busy; + int64_t log_slot_closes; + int64_t log_slot_races; + int64_t log_slot_transitions; + int64_t log_slot_joins; + int64_t log_slot_unbuffered; int64_t log_bytes_payload; int64_t log_bytes_written; - int64_t log_close_yields; - int64_t log_compress_len; - int64_t log_compress_mem; - int64_t log_compress_small; - int64_t log_compress_write_fails; - int64_t log_compress_writes; + int64_t log_zero_fills; int64_t log_flush; + int64_t log_compress_writes; + int64_t log_compress_write_fails; + int64_t log_compress_small; + int64_t log_release_write_lsn; + int64_t log_scans; + int64_t log_scan_rereads; + int64_t log_write_lsn; + int64_t log_sync; + int64_t log_sync_dir; + int64_t log_writes; + int64_t log_slot_consolidated; int64_t log_max_filesize; - int64_t log_prealloc_files; int64_t log_prealloc_max; int64_t log_prealloc_missed; + int64_t log_prealloc_files; int64_t log_prealloc_used; - int64_t log_release_write_lsn; int64_t log_scan_records; - int64_t log_scan_rereads; - int64_t log_scans; - int64_t log_slot_closes; + int64_t log_compress_mem; + int64_t log_buffer_size; + int64_t log_compress_len; int64_t log_slot_coalesced; - int64_t log_slot_consolidated; - int64_t log_slot_joins; - int64_t log_slot_races; - int64_t log_slot_switch_busy; - int64_t log_slot_transitions; - int64_t log_slot_unbuffered; - int64_t log_sync; - int64_t log_sync_dir; - int64_t log_write_lsn; - int64_t log_writes; - int64_t log_zero_fills; - int64_t lsm_checkpoint_throttle; - int64_t lsm_merge_throttle; - int64_t lsm_rows_merged; - int64_t lsm_work_queue_app; - int64_t lsm_work_queue_manager; - int64_t lsm_work_queue_max; - int64_t lsm_work_queue_switch; - int64_t lsm_work_units_created; - int64_t lsm_work_units_discarded; - int64_t lsm_work_units_done; - int64_t memory_allocation; - int64_t memory_free; - int64_t memory_grow; - int64_t page_busy_blocked; - int64_t page_forcible_evict_blocked; - int64_t page_locked_blocked; - int64_t page_read_blocked; - int64_t page_sleep; - int64_t read_io; - int64_t rec_page_delete; + int64_t log_close_yields; int64_t rec_page_delete_fast; int64_t rec_pages; int64_t rec_pages_eviction; + int64_t rec_page_delete; int64_t rec_split_stashed_bytes; int64_t rec_split_stashed_objects; - int64_t rwlock_read; - int64_t rwlock_write; int64_t session_cursor_open; int64_t session_open; + int64_t page_busy_blocked; + int64_t page_forcible_evict_blocked; + int64_t page_locked_blocked; + int64_t page_read_blocked; + int64_t page_sleep; + int64_t txn_snapshots_created; + int64_t txn_snapshots_dropped; int64_t txn_begin; - int64_t txn_checkpoint; - int64_t txn_checkpoint_generation; int64_t txn_checkpoint_running; + int64_t txn_checkpoint_generation; int64_t txn_checkpoint_time_max; int64_t txn_checkpoint_time_min; int64_t txn_checkpoint_time_recent; int64_t txn_checkpoint_time_total; - int64_t txn_commit; + int64_t txn_checkpoint; int64_t txn_fail_cache; - int64_t txn_pinned_checkpoint_range; int64_t txn_pinned_range; + int64_t txn_pinned_checkpoint_range; int64_t txn_pinned_snapshot_range; - int64_t txn_rollback; - int64_t txn_snapshots_created; - int64_t txn_snapshots_dropped; int64_t txn_sync; - int64_t write_io; + int64_t txn_commit; + int64_t txn_rollback; }; /* @@ -395,102 +395,102 @@ struct __wt_connection_stats { */ #define WT_DSRC_STATS_BASE 2000 struct __wt_dsrc_stats { - int64_t allocation_size; - int64_t block_alloc; - int64_t block_checkpoint_size; - int64_t block_extension; - int64_t block_free; - int64_t block_magic; - int64_t block_major; - int64_t block_minor; - int64_t block_reuse_bytes; - int64_t block_size; - int64_t bloom_count; int64_t bloom_false_positive; int64_t bloom_hit; int64_t bloom_miss; int64_t bloom_page_evict; int64_t bloom_page_read; + int64_t bloom_count; + int64_t lsm_chunk_count; + int64_t lsm_generation_max; + int64_t lsm_lookup_no_bloom; + int64_t lsm_checkpoint_throttle; + int64_t lsm_merge_throttle; int64_t bloom_size; + int64_t block_extension; + int64_t block_alloc; + int64_t block_free; + int64_t block_checkpoint_size; + int64_t allocation_size; + int64_t block_reuse_bytes; + int64_t block_magic; + int64_t block_major; + int64_t block_size; + int64_t block_minor; int64_t btree_checkpoint_generation; - int64_t btree_column_deleted; int64_t btree_column_fix; int64_t btree_column_internal; int64_t btree_column_rle; + int64_t btree_column_deleted; int64_t btree_column_variable; - int64_t btree_compact_rewrite; - int64_t btree_entries; int64_t btree_fixed_len; - int64_t btree_maximum_depth; int64_t btree_maxintlkey; int64_t btree_maxintlpage; int64_t btree_maxleafkey; int64_t btree_maxleafpage; int64_t btree_maxleafvalue; + int64_t btree_maximum_depth; + int64_t btree_entries; int64_t btree_overflow; + int64_t btree_compact_rewrite; int64_t btree_row_internal; int64_t btree_row_leaf; int64_t cache_bytes_read; int64_t cache_bytes_write; int64_t cache_eviction_checkpoint; - int64_t cache_eviction_clean; - int64_t cache_eviction_deepen; - int64_t cache_eviction_dirty; int64_t cache_eviction_fail; int64_t cache_eviction_hazard; + int64_t cache_inmem_splittable; + int64_t cache_inmem_split; int64_t cache_eviction_internal; int64_t cache_eviction_split_internal; int64_t cache_eviction_split_leaf; - int64_t cache_inmem_split; - int64_t cache_inmem_splittable; + int64_t cache_eviction_dirty; + int64_t cache_read_overflow; int64_t cache_overflow_value; + int64_t cache_eviction_deepen; + int64_t cache_write_lookaside; int64_t cache_read; int64_t cache_read_lookaside; - int64_t cache_read_overflow; int64_t cache_write; - int64_t cache_write_lookaside; int64_t cache_write_restore; - int64_t compress_raw_fail; - int64_t compress_raw_fail_temporary; - int64_t compress_raw_ok; + int64_t cache_eviction_clean; int64_t compress_read; int64_t compress_write; int64_t compress_write_fail; int64_t compress_write_too_small; - int64_t cursor_create; - int64_t cursor_insert; + int64_t compress_raw_fail_temporary; + int64_t compress_raw_fail; + int64_t compress_raw_ok; int64_t cursor_insert_bulk; + int64_t cursor_create; int64_t cursor_insert_bytes; + int64_t cursor_remove_bytes; + int64_t cursor_update_bytes; + int64_t cursor_insert; int64_t cursor_next; int64_t cursor_prev; int64_t cursor_remove; - int64_t cursor_remove_bytes; int64_t cursor_reset; int64_t cursor_restart; int64_t cursor_search; int64_t cursor_search_near; int64_t cursor_truncate; int64_t cursor_update; - int64_t cursor_update_bytes; - int64_t lsm_checkpoint_throttle; - int64_t lsm_chunk_count; - int64_t lsm_generation_max; - int64_t lsm_lookup_no_bloom; - int64_t lsm_merge_throttle; int64_t rec_dictionary; + int64_t rec_page_delete_fast; + int64_t rec_suffix_compression; int64_t rec_multiblock_internal; - int64_t rec_multiblock_leaf; - int64_t rec_multiblock_max; int64_t rec_overflow_key_internal; + int64_t rec_prefix_compression; + int64_t rec_multiblock_leaf; int64_t rec_overflow_key_leaf; + int64_t rec_multiblock_max; int64_t rec_overflow_value; - int64_t rec_page_delete; - int64_t rec_page_delete_fast; int64_t rec_page_match; int64_t rec_pages; int64_t rec_pages_eviction; - int64_t rec_prefix_compression; - int64_t rec_suffix_compression; + int64_t rec_page_delete; int64_t session_compact; int64_t session_cursor_open; int64_t txn_update_conflict; diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index bca9a6a248b..249ecd205ba 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -1894,8 +1894,10 @@ struct __wt_connection { * information. Enabling the statistics log server uses a session from * the configured session_max., a set of related configuration options * defined below.} - * @config{    on_close, log - * statistics on database close., a boolean flag; default \c false.} + * @config{    json, encode + * statistics in JSON format., a boolean flag; default \c false.} + * @config{    on_close, log statistics on database + * close., a boolean flag; default \c false.} * @config{    path, the pathname to a file into * which the log records are written\, may contain ISO C standard * strftime conversion specifications. If the value is not an absolute @@ -1909,7 +1911,8 @@ struct __wt_connection { * empty.} * @config{    timestamp, a timestamp * prepended to each log record\, may contain strftime conversion - * specifications., a string; default \c "%b %d %H:%M:%S".} + * specifications\, when \c json is configured\, defaults to \c + * "%FT%Y.000Z"., a string; default \c "%b %d %H:%M:%S".} * @config{    wait, seconds to wait between each * write of the log records; setting this value above 0 configures * statistics logging., an integer between 0 and 100000; default \c 0.} @@ -2369,23 +2372,26 @@ struct __wt_connection { * maintain\, to a file. See @ref statistics for more information. Enabling * the statistics log server uses a session from the configured session_max., a * set of related configuration options defined below.} - * @config{    on_close, log statistics on database close., - * a boolean flag; default \c false.} - * @config{    path, the - * pathname to a file into which the log records are written\, may contain ISO C - * standard strftime conversion specifications. If the value is not an absolute - * path name\, the file is created relative to the database home., a string; - * default \c "WiredTigerStat.%d.%H".} - * @config{    sources, - * if non-empty\, include statistics for the list of data source URIs\, if they - * are open at the time of the statistics logging. The list may include URIs + * @config{    json, encode statistics in JSON format., a + * boolean flag; default \c false.} + * @config{    on_close, + * log statistics on database close., a boolean flag; default \c false.} + * @config{    path, the pathname to a file into which the + * log records are written\, may contain ISO C standard strftime conversion + * specifications. If the value is not an absolute path name\, the file is + * created relative to the database home., a string; default \c + * "WiredTigerStat.%d.%H".} + * @config{    sources, if + * non-empty\, include statistics for the list of data source URIs\, if they are + * open at the time of the statistics logging. The list may include URIs * matching a single data source ("table:mytable")\, or a URI matching all data * sources of a particular type ("table:")., a list of strings; default empty.} * @config{    timestamp, a timestamp prepended to each log - * record\, may contain strftime conversion specifications., a string; default - * \c "%b %d %H:%M:%S".} - * @config{    wait, seconds to wait - * between each write of the log records; setting this value above 0 configures + * record\, may contain strftime conversion specifications\, when \c json is + * configured\, defaults to \c "%FT%Y.000Z"., a string; default \c "%b %d + * %H:%M:%S".} + * @config{    wait, seconds to wait between + * each write of the log records; setting this value above 0 configures * statistics logging., an integer between 0 and 100000; default \c 0.} * @config{ * ),,} @@ -3700,329 +3706,329 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); * keys. See @ref data_statistics for more information. * @{ */ -/*! async: number of allocation state races */ -#define WT_STAT_CONN_ASYNC_ALLOC_RACE 1000 -/*! async: number of operation slots viewed for allocation */ -#define WT_STAT_CONN_ASYNC_ALLOC_VIEW 1001 +/*! LSM: application work units currently queued */ +#define WT_STAT_CONN_LSM_WORK_QUEUE_APP 1000 +/*! LSM: merge work units currently queued */ +#define WT_STAT_CONN_LSM_WORK_QUEUE_MANAGER 1001 +/*! LSM: rows merged in an LSM tree */ +#define WT_STAT_CONN_LSM_ROWS_MERGED 1002 +/*! LSM: sleep for LSM checkpoint throttle */ +#define WT_STAT_CONN_LSM_CHECKPOINT_THROTTLE 1003 +/*! LSM: sleep for LSM merge throttle */ +#define WT_STAT_CONN_LSM_MERGE_THROTTLE 1004 +/*! LSM: switch work units currently queued */ +#define WT_STAT_CONN_LSM_WORK_QUEUE_SWITCH 1005 +/*! LSM: tree maintenance operations discarded */ +#define WT_STAT_CONN_LSM_WORK_UNITS_DISCARDED 1006 +/*! LSM: tree maintenance operations executed */ +#define WT_STAT_CONN_LSM_WORK_UNITS_DONE 1007 +/*! LSM: tree maintenance operations scheduled */ +#define WT_STAT_CONN_LSM_WORK_UNITS_CREATED 1008 +/*! LSM: tree queue hit maximum */ +#define WT_STAT_CONN_LSM_WORK_QUEUE_MAX 1009 /*! async: current work queue length */ -#define WT_STAT_CONN_ASYNC_CUR_QUEUE 1002 +#define WT_STAT_CONN_ASYNC_CUR_QUEUE 1010 +/*! async: maximum work queue length */ +#define WT_STAT_CONN_ASYNC_MAX_QUEUE 1011 +/*! async: number of allocation state races */ +#define WT_STAT_CONN_ASYNC_ALLOC_RACE 1012 /*! async: number of flush calls */ -#define WT_STAT_CONN_ASYNC_FLUSH 1003 +#define WT_STAT_CONN_ASYNC_FLUSH 1013 +/*! async: number of operation slots viewed for allocation */ +#define WT_STAT_CONN_ASYNC_ALLOC_VIEW 1014 /*! async: number of times operation allocation failed */ -#define WT_STAT_CONN_ASYNC_FULL 1004 -/*! async: maximum work queue length */ -#define WT_STAT_CONN_ASYNC_MAX_QUEUE 1005 +#define WT_STAT_CONN_ASYNC_FULL 1015 /*! async: number of times worker found no work */ -#define WT_STAT_CONN_ASYNC_NOWORK 1006 +#define WT_STAT_CONN_ASYNC_NOWORK 1016 /*! async: total allocations */ -#define WT_STAT_CONN_ASYNC_OP_ALLOC 1007 +#define WT_STAT_CONN_ASYNC_OP_ALLOC 1017 /*! async: total compact calls */ -#define WT_STAT_CONN_ASYNC_OP_COMPACT 1008 +#define WT_STAT_CONN_ASYNC_OP_COMPACT 1018 /*! async: total insert calls */ -#define WT_STAT_CONN_ASYNC_OP_INSERT 1009 +#define WT_STAT_CONN_ASYNC_OP_INSERT 1019 /*! async: total remove calls */ -#define WT_STAT_CONN_ASYNC_OP_REMOVE 1010 +#define WT_STAT_CONN_ASYNC_OP_REMOVE 1020 /*! async: total search calls */ -#define WT_STAT_CONN_ASYNC_OP_SEARCH 1011 +#define WT_STAT_CONN_ASYNC_OP_SEARCH 1021 /*! async: total update calls */ -#define WT_STAT_CONN_ASYNC_OP_UPDATE 1012 -/*! block-manager: mapped bytes read */ -#define WT_STAT_CONN_BLOCK_BYTE_MAP_READ 1013 -/*! block-manager: bytes read */ -#define WT_STAT_CONN_BLOCK_BYTE_READ 1014 -/*! block-manager: bytes written */ -#define WT_STAT_CONN_BLOCK_BYTE_WRITE 1015 -/*! block-manager: mapped blocks read */ -#define WT_STAT_CONN_BLOCK_MAP_READ 1016 +#define WT_STAT_CONN_ASYNC_OP_UPDATE 1022 /*! block-manager: blocks pre-loaded */ -#define WT_STAT_CONN_BLOCK_PRELOAD 1017 +#define WT_STAT_CONN_BLOCK_PRELOAD 1023 /*! block-manager: blocks read */ -#define WT_STAT_CONN_BLOCK_READ 1018 +#define WT_STAT_CONN_BLOCK_READ 1024 /*! block-manager: blocks written */ -#define WT_STAT_CONN_BLOCK_WRITE 1019 -/*! cache: tracked dirty bytes in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1020 -/*! cache: tracked bytes belonging to internal pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1021 +#define WT_STAT_CONN_BLOCK_WRITE 1025 +/*! block-manager: bytes read */ +#define WT_STAT_CONN_BLOCK_BYTE_READ 1026 +/*! block-manager: bytes written */ +#define WT_STAT_CONN_BLOCK_BYTE_WRITE 1027 +/*! block-manager: mapped blocks read */ +#define WT_STAT_CONN_BLOCK_MAP_READ 1028 +/*! block-manager: mapped bytes read */ +#define WT_STAT_CONN_BLOCK_BYTE_MAP_READ 1029 /*! cache: bytes currently in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INUSE 1022 -/*! cache: tracked bytes belonging to leaf pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_LEAF 1023 -/*! cache: maximum bytes configured */ -#define WT_STAT_CONN_CACHE_BYTES_MAX 1024 -/*! cache: tracked bytes belonging to overflow pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_OVERFLOW 1025 +#define WT_STAT_CONN_CACHE_BYTES_INUSE 1030 /*! cache: bytes read into cache */ -#define WT_STAT_CONN_CACHE_BYTES_READ 1026 +#define WT_STAT_CONN_CACHE_BYTES_READ 1031 /*! cache: bytes written from cache */ -#define WT_STAT_CONN_CACHE_BYTES_WRITE 1027 -/*! cache: pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP 1028 +#define WT_STAT_CONN_CACHE_BYTES_WRITE 1032 /*! cache: checkpoint blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_CHECKPOINT 1029 -/*! cache: unmodified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1030 -/*! cache: page split during eviction deepened the tree */ -#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1031 -/*! cache: modified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1032 -/*! cache: pages selected for eviction unable to be evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1033 -/*! cache: pages evicted because they exceeded the in-memory maximum */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1034 -/*! cache: pages evicted because they had chains of deleted items */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1035 -/*! cache: failed eviction of pages that exceeded the in-memory maximum */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1036 -/*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1037 -/*! cache: internal pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1038 -/*! cache: maximum page size at eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1039 +#define WT_STAT_CONN_CACHE_EVICTION_CHECKPOINT 1033 /*! cache: eviction server candidate queue empty when topping up */ -#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_EMPTY 1040 +#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_EMPTY 1034 /*! cache: eviction server candidate queue not empty when topping up */ -#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_NOT_EMPTY 1041 +#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_NOT_EMPTY 1035 /*! cache: eviction server evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_SERVER_EVICTING 1042 +#define WT_STAT_CONN_CACHE_EVICTION_SERVER_EVICTING 1036 /*! cache: eviction server populating queue, but not evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_SERVER_NOT_EVICTING 1043 +#define WT_STAT_CONN_CACHE_EVICTION_SERVER_NOT_EVICTING 1037 /*! cache: eviction server unable to reach eviction goal */ -#define WT_STAT_CONN_CACHE_EVICTION_SLOW 1044 +#define WT_STAT_CONN_CACHE_EVICTION_SLOW 1038 +/*! cache: eviction worker thread evicting pages */ +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1039 +/*! cache: failed eviction of pages that exceeded the in-memory maximum */ +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1040 +/*! cache: hazard pointer blocked page eviction */ +#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1041 +/*! cache: in-memory page passed criteria to be split */ +#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1042 +/*! cache: in-memory page splits */ +#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1043 +/*! cache: internal pages evicted */ +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1044 /*! cache: internal pages split during eviction */ #define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1045 /*! cache: leaf pages split during eviction */ #define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1046 -/*! cache: pages walked for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK 1047 -/*! cache: eviction worker thread evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1048 -/*! cache: in-memory page splits */ -#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1049 -/*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1050 /*! cache: lookaside table insert calls */ -#define WT_STAT_CONN_CACHE_LOOKASIDE_INSERT 1051 +#define WT_STAT_CONN_CACHE_LOOKASIDE_INSERT 1047 /*! cache: lookaside table remove calls */ -#define WT_STAT_CONN_CACHE_LOOKASIDE_REMOVE 1052 -/*! cache: percentage overhead */ -#define WT_STAT_CONN_CACHE_OVERHEAD 1053 -/*! cache: tracked dirty pages in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1054 +#define WT_STAT_CONN_CACHE_LOOKASIDE_REMOVE 1048 +/*! cache: maximum bytes configured */ +#define WT_STAT_CONN_CACHE_BYTES_MAX 1049 +/*! cache: maximum page size at eviction */ +#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1050 +/*! cache: modified pages evicted */ +#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1051 +/*! cache: page split during eviction deepened the tree */ +#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1052 +/*! cache: page written requiring lookaside records */ +#define WT_STAT_CONN_CACHE_WRITE_LOOKASIDE 1053 /*! cache: pages currently held in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_INUSE 1055 +#define WT_STAT_CONN_CACHE_PAGES_INUSE 1054 +/*! cache: pages evicted because they exceeded the in-memory maximum */ +#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1055 +/*! cache: pages evicted because they had chains of deleted items */ +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1056 +/*! cache: pages evicted by application threads */ +#define WT_STAT_CONN_CACHE_EVICTION_APP 1057 /*! cache: pages read into cache */ -#define WT_STAT_CONN_CACHE_READ 1056 +#define WT_STAT_CONN_CACHE_READ 1058 /*! cache: pages read into cache requiring lookaside entries */ -#define WT_STAT_CONN_CACHE_READ_LOOKASIDE 1057 +#define WT_STAT_CONN_CACHE_READ_LOOKASIDE 1059 +/*! cache: pages selected for eviction unable to be evicted */ +#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1060 +/*! cache: pages walked for eviction */ +#define WT_STAT_CONN_CACHE_EVICTION_WALK 1061 /*! cache: pages written from cache */ -#define WT_STAT_CONN_CACHE_WRITE 1058 -/*! cache: page written requiring lookaside records */ -#define WT_STAT_CONN_CACHE_WRITE_LOOKASIDE 1059 +#define WT_STAT_CONN_CACHE_WRITE 1062 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1060 +#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1063 +/*! cache: percentage overhead */ +#define WT_STAT_CONN_CACHE_OVERHEAD 1064 +/*! cache: tracked bytes belonging to internal pages in the cache */ +#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1065 +/*! cache: tracked bytes belonging to leaf pages in the cache */ +#define WT_STAT_CONN_CACHE_BYTES_LEAF 1066 +/*! cache: tracked bytes belonging to overflow pages in the cache */ +#define WT_STAT_CONN_CACHE_BYTES_OVERFLOW 1067 +/*! cache: tracked dirty bytes in the cache */ +#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1068 +/*! cache: tracked dirty pages in the cache */ +#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1069 +/*! cache: unmodified pages evicted */ +#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1070 +/*! connection: files currently open */ +#define WT_STAT_CONN_FILE_OPEN 1071 +/*! connection: memory allocations */ +#define WT_STAT_CONN_MEMORY_ALLOCATION 1072 +/*! connection: memory frees */ +#define WT_STAT_CONN_MEMORY_FREE 1073 +/*! connection: memory re-allocations */ +#define WT_STAT_CONN_MEMORY_GROW 1074 /*! connection: pthread mutex condition wait calls */ -#define WT_STAT_CONN_COND_WAIT 1061 +#define WT_STAT_CONN_COND_WAIT 1075 +/*! connection: pthread mutex shared lock read-lock calls */ +#define WT_STAT_CONN_RWLOCK_READ 1076 +/*! connection: pthread mutex shared lock write-lock calls */ +#define WT_STAT_CONN_RWLOCK_WRITE 1077 +/*! connection: total read I/Os */ +#define WT_STAT_CONN_READ_IO 1078 +/*! connection: total write I/Os */ +#define WT_STAT_CONN_WRITE_IO 1079 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1062 +#define WT_STAT_CONN_CURSOR_CREATE 1080 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1063 +#define WT_STAT_CONN_CURSOR_INSERT 1081 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1064 +#define WT_STAT_CONN_CURSOR_NEXT 1082 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1065 +#define WT_STAT_CONN_CURSOR_PREV 1083 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1066 +#define WT_STAT_CONN_CURSOR_REMOVE 1084 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1067 +#define WT_STAT_CONN_CURSOR_RESET 1085 /*! cursor: cursor restarted searches */ -#define WT_STAT_CONN_CURSOR_RESTART 1068 +#define WT_STAT_CONN_CURSOR_RESTART 1086 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1069 +#define WT_STAT_CONN_CURSOR_SEARCH 1087 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1070 -/*! cursor: truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1071 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1088 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1072 +#define WT_STAT_CONN_CURSOR_UPDATE 1089 +/*! cursor: truncate calls */ +#define WT_STAT_CONN_CURSOR_TRUNCATE 1090 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1073 -/*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1074 -/*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1075 -/*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1076 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1091 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1077 +#define WT_STAT_CONN_DH_SWEEP_REF 1092 +/*! data-handle: connection sweep dhandles closed */ +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1093 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1078 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1094 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1079 +#define WT_STAT_CONN_DH_SWEEP_TOD 1095 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1080 -/*! connection: files currently open */ -#define WT_STAT_CONN_FILE_OPEN 1081 -/*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1082 +#define WT_STAT_CONN_DH_SWEEPS 1096 +/*! data-handle: session dhandles swept */ +#define WT_STAT_CONN_DH_SESSION_HANDLES 1097 +/*! data-handle: session sweep attempts */ +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1098 +/*! log: busy returns attempting to switch slots */ +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1099 +/*! log: consolidated slot closures */ +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1100 +/*! log: consolidated slot join races */ +#define WT_STAT_CONN_LOG_SLOT_RACES 1101 +/*! log: consolidated slot join transitions */ +#define WT_STAT_CONN_LOG_SLOT_TRANSITIONS 1102 +/*! log: consolidated slot joins */ +#define WT_STAT_CONN_LOG_SLOT_JOINS 1103 +/*! log: consolidated slot unbuffered writes */ +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1104 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1083 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1105 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1084 -/*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1085 -/*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1086 -/*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1087 -/*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1088 -/*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1089 -/*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1090 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1106 +/*! log: log files manually zero-filled */ +#define WT_STAT_CONN_LOG_ZERO_FILLS 1107 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1091 +#define WT_STAT_CONN_LOG_FLUSH 1108 +/*! log: log records compressed */ +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1109 +/*! log: log records not compressed */ +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1110 +/*! log: log records too small to compress */ +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1111 +/*! log: log release advances write LSN */ +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1112 +/*! log: log scan operations */ +#define WT_STAT_CONN_LOG_SCANS 1113 +/*! log: log scan records requiring two reads */ +#define WT_STAT_CONN_LOG_SCAN_REREADS 1114 +/*! log: log server thread advances write LSN */ +#define WT_STAT_CONN_LOG_WRITE_LSN 1115 +/*! log: log sync operations */ +#define WT_STAT_CONN_LOG_SYNC 1116 +/*! log: log sync_dir operations */ +#define WT_STAT_CONN_LOG_SYNC_DIR 1117 +/*! log: log write operations */ +#define WT_STAT_CONN_LOG_WRITES 1118 +/*! log: logging bytes consolidated */ +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1119 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1092 -/*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1093 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1120 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1094 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1121 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1095 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1122 +/*! log: pre-allocated log files prepared */ +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1123 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1096 -/*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1097 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1124 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1098 -/*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1099 -/*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1100 -/*! log: consolidated slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1101 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1125 +/*! log: total in-memory size of compressed records */ +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1126 +/*! log: total log buffer size */ +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1127 +/*! log: total size of compressed records */ +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1128 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1102 -/*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1103 -/*! log: consolidated slot joins */ -#define WT_STAT_CONN_LOG_SLOT_JOINS 1104 -/*! log: consolidated slot join races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1105 -/*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1106 -/*! log: consolidated slot join transitions */ -#define WT_STAT_CONN_LOG_SLOT_TRANSITIONS 1107 -/*! log: consolidated slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1108 -/*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1109 -/*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1110 -/*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1111 -/*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1112 -/*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1113 -/*! LSM: sleep for LSM checkpoint throttle */ -#define WT_STAT_CONN_LSM_CHECKPOINT_THROTTLE 1114 -/*! LSM: sleep for LSM merge throttle */ -#define WT_STAT_CONN_LSM_MERGE_THROTTLE 1115 -/*! LSM: rows merged in an LSM tree */ -#define WT_STAT_CONN_LSM_ROWS_MERGED 1116 -/*! LSM: application work units currently queued */ -#define WT_STAT_CONN_LSM_WORK_QUEUE_APP 1117 -/*! LSM: merge work units currently queued */ -#define WT_STAT_CONN_LSM_WORK_QUEUE_MANAGER 1118 -/*! LSM: tree queue hit maximum */ -#define WT_STAT_CONN_LSM_WORK_QUEUE_MAX 1119 -/*! LSM: switch work units currently queued */ -#define WT_STAT_CONN_LSM_WORK_QUEUE_SWITCH 1120 -/*! LSM: tree maintenance operations scheduled */ -#define WT_STAT_CONN_LSM_WORK_UNITS_CREATED 1121 -/*! LSM: tree maintenance operations discarded */ -#define WT_STAT_CONN_LSM_WORK_UNITS_DISCARDED 1122 -/*! LSM: tree maintenance operations executed */ -#define WT_STAT_CONN_LSM_WORK_UNITS_DONE 1123 -/*! connection: memory allocations */ -#define WT_STAT_CONN_MEMORY_ALLOCATION 1124 -/*! connection: memory frees */ -#define WT_STAT_CONN_MEMORY_FREE 1125 -/*! connection: memory re-allocations */ -#define WT_STAT_CONN_MEMORY_GROW 1126 -/*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1127 -/*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1128 -/*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1129 -/*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1130 -/*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1131 -/*! connection: total read I/Os */ -#define WT_STAT_CONN_READ_IO 1132 -/*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1133 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1129 +/*! log: yields waiting for previous log file close */ +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1130 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1134 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1131 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1135 +#define WT_STAT_CONN_REC_PAGES 1132 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1136 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1133 +/*! reconciliation: pages deleted */ +#define WT_STAT_CONN_REC_PAGE_DELETE 1134 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1137 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1135 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1138 -/*! connection: pthread mutex shared lock read-lock calls */ -#define WT_STAT_CONN_RWLOCK_READ 1139 -/*! connection: pthread mutex shared lock write-lock calls */ -#define WT_STAT_CONN_RWLOCK_WRITE 1140 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1136 /*! session: open cursor count */ -#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1141 +#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1137 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1142 +#define WT_STAT_CONN_SESSION_OPEN 1138 +/*! thread-yield: page acquire busy blocked */ +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1139 +/*! thread-yield: page acquire eviction blocked */ +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1140 +/*! thread-yield: page acquire locked blocked */ +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1141 +/*! thread-yield: page acquire read blocked */ +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1142 +/*! thread-yield: page acquire time sleeping (usecs) */ +#define WT_STAT_CONN_PAGE_SLEEP 1143 +/*! transaction: number of named snapshots created */ +#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1144 +/*! transaction: number of named snapshots dropped */ +#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1145 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1143 -/*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1144 -/*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1145 +#define WT_STAT_CONN_TXN_BEGIN 1146 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1146 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1147 +/*! transaction: transaction checkpoint generation */ +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1148 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1147 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1149 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1148 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1150 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1149 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1151 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1150 -/*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1151 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1152 +/*! transaction: transaction checkpoints */ +#define WT_STAT_CONN_TXN_CHECKPOINT 1153 /*! transaction: transaction failures due to cache overflow */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1152 -/*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1153 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1154 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1154 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1155 +/*! transaction: transaction range of IDs currently pinned by a checkpoint */ +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1156 /*! transaction: transaction range of IDs currently pinned by named * snapshots */ -#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1155 -/*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1156 -/*! transaction: number of named snapshots created */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1157 -/*! transaction: number of named snapshots dropped */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1158 +#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1157 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1159 -/*! connection: total write I/Os */ -#define WT_STAT_CONN_WRITE_IO 1160 +#define WT_STAT_CONN_TXN_SYNC 1158 +/*! transaction: transactions committed */ +#define WT_STAT_CONN_TXN_COMMIT 1159 +/*! transaction: transactions rolled back */ +#define WT_STAT_CONN_TXN_ROLLBACK 1160 /*! * @} @@ -4030,200 +4036,200 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); * @anchor statistics_dsrc * @{ */ -/*! block-manager: file allocation unit size */ -#define WT_STAT_DSRC_ALLOCATION_SIZE 2000 -/*! block-manager: blocks allocated */ -#define WT_STAT_DSRC_BLOCK_ALLOC 2001 -/*! block-manager: checkpoint size */ -#define WT_STAT_DSRC_BLOCK_CHECKPOINT_SIZE 2002 -/*! block-manager: allocations requiring file extension */ -#define WT_STAT_DSRC_BLOCK_EXTENSION 2003 -/*! block-manager: blocks freed */ -#define WT_STAT_DSRC_BLOCK_FREE 2004 -/*! block-manager: file magic number */ -#define WT_STAT_DSRC_BLOCK_MAGIC 2005 -/*! block-manager: file major version number */ -#define WT_STAT_DSRC_BLOCK_MAJOR 2006 -/*! block-manager: minor version number */ -#define WT_STAT_DSRC_BLOCK_MINOR 2007 -/*! block-manager: file bytes available for reuse */ -#define WT_STAT_DSRC_BLOCK_REUSE_BYTES 2008 -/*! block-manager: file size in bytes */ -#define WT_STAT_DSRC_BLOCK_SIZE 2009 -/*! LSM: bloom filters in the LSM tree */ -#define WT_STAT_DSRC_BLOOM_COUNT 2010 /*! LSM: bloom filter false positives */ -#define WT_STAT_DSRC_BLOOM_FALSE_POSITIVE 2011 +#define WT_STAT_DSRC_BLOOM_FALSE_POSITIVE 2000 /*! LSM: bloom filter hits */ -#define WT_STAT_DSRC_BLOOM_HIT 2012 +#define WT_STAT_DSRC_BLOOM_HIT 2001 /*! LSM: bloom filter misses */ -#define WT_STAT_DSRC_BLOOM_MISS 2013 +#define WT_STAT_DSRC_BLOOM_MISS 2002 /*! LSM: bloom filter pages evicted from cache */ -#define WT_STAT_DSRC_BLOOM_PAGE_EVICT 2014 +#define WT_STAT_DSRC_BLOOM_PAGE_EVICT 2003 /*! LSM: bloom filter pages read into cache */ -#define WT_STAT_DSRC_BLOOM_PAGE_READ 2015 +#define WT_STAT_DSRC_BLOOM_PAGE_READ 2004 +/*! LSM: bloom filters in the LSM tree */ +#define WT_STAT_DSRC_BLOOM_COUNT 2005 +/*! LSM: chunks in the LSM tree */ +#define WT_STAT_DSRC_LSM_CHUNK_COUNT 2006 +/*! LSM: highest merge generation in the LSM tree */ +#define WT_STAT_DSRC_LSM_GENERATION_MAX 2007 +/*! LSM: queries that could have benefited from a Bloom filter that did + * not exist */ +#define WT_STAT_DSRC_LSM_LOOKUP_NO_BLOOM 2008 +/*! LSM: sleep for LSM checkpoint throttle */ +#define WT_STAT_DSRC_LSM_CHECKPOINT_THROTTLE 2009 +/*! LSM: sleep for LSM merge throttle */ +#define WT_STAT_DSRC_LSM_MERGE_THROTTLE 2010 /*! LSM: total size of bloom filters */ -#define WT_STAT_DSRC_BLOOM_SIZE 2016 +#define WT_STAT_DSRC_BLOOM_SIZE 2011 +/*! block-manager: allocations requiring file extension */ +#define WT_STAT_DSRC_BLOCK_EXTENSION 2012 +/*! block-manager: blocks allocated */ +#define WT_STAT_DSRC_BLOCK_ALLOC 2013 +/*! block-manager: blocks freed */ +#define WT_STAT_DSRC_BLOCK_FREE 2014 +/*! block-manager: checkpoint size */ +#define WT_STAT_DSRC_BLOCK_CHECKPOINT_SIZE 2015 +/*! block-manager: file allocation unit size */ +#define WT_STAT_DSRC_ALLOCATION_SIZE 2016 +/*! block-manager: file bytes available for reuse */ +#define WT_STAT_DSRC_BLOCK_REUSE_BYTES 2017 +/*! block-manager: file magic number */ +#define WT_STAT_DSRC_BLOCK_MAGIC 2018 +/*! block-manager: file major version number */ +#define WT_STAT_DSRC_BLOCK_MAJOR 2019 +/*! block-manager: file size in bytes */ +#define WT_STAT_DSRC_BLOCK_SIZE 2020 +/*! block-manager: minor version number */ +#define WT_STAT_DSRC_BLOCK_MINOR 2021 /*! btree: btree checkpoint generation */ -#define WT_STAT_DSRC_BTREE_CHECKPOINT_GENERATION 2017 -/*! btree: column-store variable-size deleted values */ -#define WT_STAT_DSRC_BTREE_COLUMN_DELETED 2018 +#define WT_STAT_DSRC_BTREE_CHECKPOINT_GENERATION 2022 /*! btree: column-store fixed-size leaf pages */ -#define WT_STAT_DSRC_BTREE_COLUMN_FIX 2019 +#define WT_STAT_DSRC_BTREE_COLUMN_FIX 2023 /*! btree: column-store internal pages */ -#define WT_STAT_DSRC_BTREE_COLUMN_INTERNAL 2020 +#define WT_STAT_DSRC_BTREE_COLUMN_INTERNAL 2024 /*! btree: column-store variable-size RLE encoded values */ -#define WT_STAT_DSRC_BTREE_COLUMN_RLE 2021 +#define WT_STAT_DSRC_BTREE_COLUMN_RLE 2025 +/*! btree: column-store variable-size deleted values */ +#define WT_STAT_DSRC_BTREE_COLUMN_DELETED 2026 /*! btree: column-store variable-size leaf pages */ -#define WT_STAT_DSRC_BTREE_COLUMN_VARIABLE 2022 -/*! btree: pages rewritten by compaction */ -#define WT_STAT_DSRC_BTREE_COMPACT_REWRITE 2023 -/*! btree: number of key/value pairs */ -#define WT_STAT_DSRC_BTREE_ENTRIES 2024 +#define WT_STAT_DSRC_BTREE_COLUMN_VARIABLE 2027 /*! btree: fixed-record size */ -#define WT_STAT_DSRC_BTREE_FIXED_LEN 2025 -/*! btree: maximum tree depth */ -#define WT_STAT_DSRC_BTREE_MAXIMUM_DEPTH 2026 +#define WT_STAT_DSRC_BTREE_FIXED_LEN 2028 /*! btree: maximum internal page key size */ -#define WT_STAT_DSRC_BTREE_MAXINTLKEY 2027 +#define WT_STAT_DSRC_BTREE_MAXINTLKEY 2029 /*! btree: maximum internal page size */ -#define WT_STAT_DSRC_BTREE_MAXINTLPAGE 2028 +#define WT_STAT_DSRC_BTREE_MAXINTLPAGE 2030 /*! btree: maximum leaf page key size */ -#define WT_STAT_DSRC_BTREE_MAXLEAFKEY 2029 +#define WT_STAT_DSRC_BTREE_MAXLEAFKEY 2031 /*! btree: maximum leaf page size */ -#define WT_STAT_DSRC_BTREE_MAXLEAFPAGE 2030 +#define WT_STAT_DSRC_BTREE_MAXLEAFPAGE 2032 /*! btree: maximum leaf page value size */ -#define WT_STAT_DSRC_BTREE_MAXLEAFVALUE 2031 +#define WT_STAT_DSRC_BTREE_MAXLEAFVALUE 2033 +/*! btree: maximum tree depth */ +#define WT_STAT_DSRC_BTREE_MAXIMUM_DEPTH 2034 +/*! btree: number of key/value pairs */ +#define WT_STAT_DSRC_BTREE_ENTRIES 2035 /*! btree: overflow pages */ -#define WT_STAT_DSRC_BTREE_OVERFLOW 2032 +#define WT_STAT_DSRC_BTREE_OVERFLOW 2036 +/*! btree: pages rewritten by compaction */ +#define WT_STAT_DSRC_BTREE_COMPACT_REWRITE 2037 /*! btree: row-store internal pages */ -#define WT_STAT_DSRC_BTREE_ROW_INTERNAL 2033 +#define WT_STAT_DSRC_BTREE_ROW_INTERNAL 2038 /*! btree: row-store leaf pages */ -#define WT_STAT_DSRC_BTREE_ROW_LEAF 2034 +#define WT_STAT_DSRC_BTREE_ROW_LEAF 2039 /*! cache: bytes read into cache */ -#define WT_STAT_DSRC_CACHE_BYTES_READ 2035 +#define WT_STAT_DSRC_CACHE_BYTES_READ 2040 /*! cache: bytes written from cache */ -#define WT_STAT_DSRC_CACHE_BYTES_WRITE 2036 +#define WT_STAT_DSRC_CACHE_BYTES_WRITE 2041 /*! cache: checkpoint blocked page eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_CHECKPOINT 2037 -/*! cache: unmodified pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_CLEAN 2038 -/*! cache: page split during eviction deepened the tree */ -#define WT_STAT_DSRC_CACHE_EVICTION_DEEPEN 2039 -/*! cache: modified pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_DIRTY 2040 +#define WT_STAT_DSRC_CACHE_EVICTION_CHECKPOINT 2042 /*! cache: data source pages selected for eviction unable to be evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_FAIL 2041 +#define WT_STAT_DSRC_CACHE_EVICTION_FAIL 2043 /*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_HAZARD 2042 +#define WT_STAT_DSRC_CACHE_EVICTION_HAZARD 2044 +/*! cache: in-memory page passed criteria to be split */ +#define WT_STAT_DSRC_CACHE_INMEM_SPLITTABLE 2045 +/*! cache: in-memory page splits */ +#define WT_STAT_DSRC_CACHE_INMEM_SPLIT 2046 /*! cache: internal pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_INTERNAL 2043 +#define WT_STAT_DSRC_CACHE_EVICTION_INTERNAL 2047 /*! cache: internal pages split during eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_INTERNAL 2044 +#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_INTERNAL 2048 /*! cache: leaf pages split during eviction */ -#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_LEAF 2045 -/*! cache: in-memory page splits */ -#define WT_STAT_DSRC_CACHE_INMEM_SPLIT 2046 -/*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_DSRC_CACHE_INMEM_SPLITTABLE 2047 +#define WT_STAT_DSRC_CACHE_EVICTION_SPLIT_LEAF 2049 +/*! cache: modified pages evicted */ +#define WT_STAT_DSRC_CACHE_EVICTION_DIRTY 2050 +/*! cache: overflow pages read into cache */ +#define WT_STAT_DSRC_CACHE_READ_OVERFLOW 2051 /*! cache: overflow values cached in memory */ -#define WT_STAT_DSRC_CACHE_OVERFLOW_VALUE 2048 +#define WT_STAT_DSRC_CACHE_OVERFLOW_VALUE 2052 +/*! cache: page split during eviction deepened the tree */ +#define WT_STAT_DSRC_CACHE_EVICTION_DEEPEN 2053 +/*! cache: page written requiring lookaside records */ +#define WT_STAT_DSRC_CACHE_WRITE_LOOKASIDE 2054 /*! cache: pages read into cache */ -#define WT_STAT_DSRC_CACHE_READ 2049 +#define WT_STAT_DSRC_CACHE_READ 2055 /*! cache: pages read into cache requiring lookaside entries */ -#define WT_STAT_DSRC_CACHE_READ_LOOKASIDE 2050 -/*! cache: overflow pages read into cache */ -#define WT_STAT_DSRC_CACHE_READ_OVERFLOW 2051 +#define WT_STAT_DSRC_CACHE_READ_LOOKASIDE 2056 /*! cache: pages written from cache */ -#define WT_STAT_DSRC_CACHE_WRITE 2052 -/*! cache: page written requiring lookaside records */ -#define WT_STAT_DSRC_CACHE_WRITE_LOOKASIDE 2053 +#define WT_STAT_DSRC_CACHE_WRITE 2057 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_DSRC_CACHE_WRITE_RESTORE 2054 -/*! compression: raw compression call failed, no additional data available */ -#define WT_STAT_DSRC_COMPRESS_RAW_FAIL 2055 -/*! compression: raw compression call failed, additional data available */ -#define WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY 2056 -/*! compression: raw compression call succeeded */ -#define WT_STAT_DSRC_COMPRESS_RAW_OK 2057 +#define WT_STAT_DSRC_CACHE_WRITE_RESTORE 2058 +/*! cache: unmodified pages evicted */ +#define WT_STAT_DSRC_CACHE_EVICTION_CLEAN 2059 /*! compression: compressed pages read */ -#define WT_STAT_DSRC_COMPRESS_READ 2058 +#define WT_STAT_DSRC_COMPRESS_READ 2060 /*! compression: compressed pages written */ -#define WT_STAT_DSRC_COMPRESS_WRITE 2059 +#define WT_STAT_DSRC_COMPRESS_WRITE 2061 /*! compression: page written failed to compress */ -#define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2060 +#define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2062 /*! compression: page written was too small to compress */ -#define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2061 -/*! cursor: create calls */ -#define WT_STAT_DSRC_CURSOR_CREATE 2062 -/*! cursor: insert calls */ -#define WT_STAT_DSRC_CURSOR_INSERT 2063 +#define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2063 +/*! compression: raw compression call failed, additional data available */ +#define WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY 2064 +/*! compression: raw compression call failed, no additional data available */ +#define WT_STAT_DSRC_COMPRESS_RAW_FAIL 2065 +/*! compression: raw compression call succeeded */ +#define WT_STAT_DSRC_COMPRESS_RAW_OK 2066 /*! cursor: bulk-loaded cursor-insert calls */ -#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2064 +#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2067 +/*! cursor: create calls */ +#define WT_STAT_DSRC_CURSOR_CREATE 2068 /*! cursor: cursor-insert key and value bytes inserted */ -#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2065 +#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2069 +/*! cursor: cursor-remove key bytes removed */ +#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2070 +/*! cursor: cursor-update value bytes updated */ +#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2071 +/*! cursor: insert calls */ +#define WT_STAT_DSRC_CURSOR_INSERT 2072 /*! cursor: next calls */ -#define WT_STAT_DSRC_CURSOR_NEXT 2066 +#define WT_STAT_DSRC_CURSOR_NEXT 2073 /*! cursor: prev calls */ -#define WT_STAT_DSRC_CURSOR_PREV 2067 +#define WT_STAT_DSRC_CURSOR_PREV 2074 /*! cursor: remove calls */ -#define WT_STAT_DSRC_CURSOR_REMOVE 2068 -/*! cursor: cursor-remove key bytes removed */ -#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2069 +#define WT_STAT_DSRC_CURSOR_REMOVE 2075 /*! cursor: reset calls */ -#define WT_STAT_DSRC_CURSOR_RESET 2070 +#define WT_STAT_DSRC_CURSOR_RESET 2076 /*! cursor: restarted searches */ -#define WT_STAT_DSRC_CURSOR_RESTART 2071 +#define WT_STAT_DSRC_CURSOR_RESTART 2077 /*! cursor: search calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH 2072 +#define WT_STAT_DSRC_CURSOR_SEARCH 2078 /*! cursor: search near calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2073 +#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2079 /*! cursor: truncate calls */ -#define WT_STAT_DSRC_CURSOR_TRUNCATE 2074 +#define WT_STAT_DSRC_CURSOR_TRUNCATE 2080 /*! cursor: update calls */ -#define WT_STAT_DSRC_CURSOR_UPDATE 2075 -/*! cursor: cursor-update value bytes updated */ -#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2076 -/*! LSM: sleep for LSM checkpoint throttle */ -#define WT_STAT_DSRC_LSM_CHECKPOINT_THROTTLE 2077 -/*! LSM: chunks in the LSM tree */ -#define WT_STAT_DSRC_LSM_CHUNK_COUNT 2078 -/*! LSM: highest merge generation in the LSM tree */ -#define WT_STAT_DSRC_LSM_GENERATION_MAX 2079 -/*! LSM: queries that could have benefited from a Bloom filter that did - * not exist */ -#define WT_STAT_DSRC_LSM_LOOKUP_NO_BLOOM 2080 -/*! LSM: sleep for LSM merge throttle */ -#define WT_STAT_DSRC_LSM_MERGE_THROTTLE 2081 +#define WT_STAT_DSRC_CURSOR_UPDATE 2081 /*! reconciliation: dictionary matches */ #define WT_STAT_DSRC_REC_DICTIONARY 2082 +/*! reconciliation: fast-path pages deleted */ +#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2083 +/*! reconciliation: internal page key bytes discarded using suffix + * compression */ +#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2084 /*! reconciliation: internal page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2083 -/*! reconciliation: leaf page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2084 -/*! reconciliation: maximum blocks required for a page */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2085 +#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2085 /*! reconciliation: internal-page overflow keys */ #define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2086 +/*! reconciliation: leaf page key bytes discarded using prefix compression */ +#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2087 +/*! reconciliation: leaf page multi-block writes */ +#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2088 /*! reconciliation: leaf-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2087 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2089 +/*! reconciliation: maximum blocks required for a page */ +#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2090 /*! reconciliation: overflow values written */ -#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2088 -/*! reconciliation: pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE 2089 -/*! reconciliation: fast-path pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2090 +#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2091 /*! reconciliation: page checksum matches */ -#define WT_STAT_DSRC_REC_PAGE_MATCH 2091 +#define WT_STAT_DSRC_REC_PAGE_MATCH 2092 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_DSRC_REC_PAGES 2092 +#define WT_STAT_DSRC_REC_PAGES 2093 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_DSRC_REC_PAGES_EVICTION 2093 -/*! reconciliation: leaf page key bytes discarded using prefix compression */ -#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2094 -/*! reconciliation: internal page key bytes discarded using suffix - * compression */ -#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2095 +#define WT_STAT_DSRC_REC_PAGES_EVICTION 2094 +/*! reconciliation: pages deleted */ +#define WT_STAT_DSRC_REC_PAGE_DELETE 2095 /*! session: object compaction */ #define WT_STAT_DSRC_SESSION_COMPACT 2096 /*! session: open cursor count */ diff --git a/src/support/stat.c b/src/support/stat.c index 7a615131628..e9b89dbbe06 100644 --- a/src/support/stat.c +++ b/src/support/stat.c @@ -3,102 +3,102 @@ #include "wt_internal.h" static const char * const __stats_dsrc_desc[] = { - "block-manager: file allocation unit size", - "block-manager: blocks allocated", - "block-manager: checkpoint size", - "block-manager: allocations requiring file extension", - "block-manager: blocks freed", - "block-manager: file magic number", - "block-manager: file major version number", - "block-manager: minor version number", - "block-manager: file bytes available for reuse", - "block-manager: file size in bytes", - "LSM: bloom filters in the LSM tree", "LSM: bloom filter false positives", "LSM: bloom filter hits", "LSM: bloom filter misses", "LSM: bloom filter pages evicted from cache", "LSM: bloom filter pages read into cache", + "LSM: bloom filters in the LSM tree", + "LSM: chunks in the LSM tree", + "LSM: highest merge generation in the LSM tree", + "LSM: queries that could have benefited from a Bloom filter that did not exist", + "LSM: sleep for LSM checkpoint throttle", + "LSM: sleep for LSM merge throttle", "LSM: total size of bloom filters", + "block-manager: allocations requiring file extension", + "block-manager: blocks allocated", + "block-manager: blocks freed", + "block-manager: checkpoint size", + "block-manager: file allocation unit size", + "block-manager: file bytes available for reuse", + "block-manager: file magic number", + "block-manager: file major version number", + "block-manager: file size in bytes", + "block-manager: minor version number", "btree: btree checkpoint generation", - "btree: column-store variable-size deleted values", "btree: column-store fixed-size leaf pages", "btree: column-store internal pages", "btree: column-store variable-size RLE encoded values", + "btree: column-store variable-size deleted values", "btree: column-store variable-size leaf pages", - "btree: pages rewritten by compaction", - "btree: number of key/value pairs", "btree: fixed-record size", - "btree: maximum tree depth", "btree: maximum internal page key size", "btree: maximum internal page size", "btree: maximum leaf page key size", "btree: maximum leaf page size", "btree: maximum leaf page value size", + "btree: maximum tree depth", + "btree: number of key/value pairs", "btree: overflow pages", + "btree: pages rewritten by compaction", "btree: row-store internal pages", "btree: row-store leaf pages", "cache: bytes read into cache", "cache: bytes written from cache", "cache: checkpoint blocked page eviction", - "cache: unmodified pages evicted", - "cache: page split during eviction deepened the tree", - "cache: modified pages evicted", "cache: data source pages selected for eviction unable to be evicted", "cache: hazard pointer blocked page eviction", + "cache: in-memory page passed criteria to be split", + "cache: in-memory page splits", "cache: internal pages evicted", "cache: internal pages split during eviction", "cache: leaf pages split during eviction", - "cache: in-memory page splits", - "cache: in-memory page passed criteria to be split", + "cache: modified pages evicted", + "cache: overflow pages read into cache", "cache: overflow values cached in memory", + "cache: page split during eviction deepened the tree", + "cache: page written requiring lookaside records", "cache: pages read into cache", "cache: pages read into cache requiring lookaside entries", - "cache: overflow pages read into cache", "cache: pages written from cache", - "cache: page written requiring lookaside records", "cache: pages written requiring in-memory restoration", - "compression: raw compression call failed, no additional data available", - "compression: raw compression call failed, additional data available", - "compression: raw compression call succeeded", + "cache: unmodified pages evicted", "compression: compressed pages read", "compression: compressed pages written", "compression: page written failed to compress", "compression: page written was too small to compress", - "cursor: create calls", - "cursor: insert calls", + "compression: raw compression call failed, additional data available", + "compression: raw compression call failed, no additional data available", + "compression: raw compression call succeeded", "cursor: bulk-loaded cursor-insert calls", + "cursor: create calls", "cursor: cursor-insert key and value bytes inserted", + "cursor: cursor-remove key bytes removed", + "cursor: cursor-update value bytes updated", + "cursor: insert calls", "cursor: next calls", "cursor: prev calls", "cursor: remove calls", - "cursor: cursor-remove key bytes removed", "cursor: reset calls", "cursor: restarted searches", "cursor: search calls", "cursor: search near calls", "cursor: truncate calls", "cursor: update calls", - "cursor: cursor-update value bytes updated", - "LSM: sleep for LSM checkpoint throttle", - "LSM: chunks in the LSM tree", - "LSM: highest merge generation in the LSM tree", - "LSM: queries that could have benefited from a Bloom filter that did not exist", - "LSM: sleep for LSM merge throttle", "reconciliation: dictionary matches", + "reconciliation: fast-path pages deleted", + "reconciliation: internal page key bytes discarded using suffix compression", "reconciliation: internal page multi-block writes", - "reconciliation: leaf page multi-block writes", - "reconciliation: maximum blocks required for a page", "reconciliation: internal-page overflow keys", + "reconciliation: leaf page key bytes discarded using prefix compression", + "reconciliation: leaf page multi-block writes", "reconciliation: leaf-page overflow keys", + "reconciliation: maximum blocks required for a page", "reconciliation: overflow values written", - "reconciliation: pages deleted", - "reconciliation: fast-path pages deleted", "reconciliation: page checksum matches", "reconciliation: page reconciliation calls", "reconciliation: page reconciliation calls for eviction", - "reconciliation: leaf page key bytes discarded using prefix compression", - "reconciliation: internal page key bytes discarded using suffix compression", + "reconciliation: pages deleted", "session: object compaction", "session: open cursor count", "transaction: update conflicts", @@ -132,6 +132,18 @@ __wt_stat_dsrc_init(WT_DATA_HANDLE *handle) void __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) { + stats->bloom_false_positive = 0; + stats->bloom_hit = 0; + stats->bloom_miss = 0; + stats->bloom_page_evict = 0; + stats->bloom_page_read = 0; + stats->bloom_count = 0; + stats->lsm_chunk_count = 0; + stats->lsm_generation_max = 0; + stats->lsm_lookup_no_bloom = 0; + stats->lsm_checkpoint_throttle = 0; + stats->lsm_merge_throttle = 0; + stats->bloom_size = 0; stats->block_extension = 0; stats->block_alloc = 0; stats->block_free = 0; @@ -145,9 +157,9 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) /* not clearing btree_checkpoint_generation */ stats->btree_column_fix = 0; stats->btree_column_internal = 0; + stats->btree_column_rle = 0; stats->btree_column_deleted = 0; stats->btree_column_variable = 0; - stats->btree_column_rle = 0; stats->btree_fixed_len = 0; stats->btree_maxintlkey = 0; stats->btree_maxintlpage = 0; @@ -202,18 +214,6 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->cursor_search_near = 0; stats->cursor_truncate = 0; stats->cursor_update = 0; - stats->bloom_false_positive = 0; - stats->bloom_hit = 0; - stats->bloom_miss = 0; - stats->bloom_page_evict = 0; - stats->bloom_page_read = 0; - stats->bloom_count = 0; - stats->lsm_chunk_count = 0; - stats->lsm_generation_max = 0; - stats->lsm_lookup_no_bloom = 0; - stats->lsm_checkpoint_throttle = 0; - stats->lsm_merge_throttle = 0; - stats->bloom_size = 0; stats->rec_dictionary = 0; stats->rec_page_delete_fast = 0; stats->rec_suffix_compression = 0; @@ -246,6 +246,19 @@ void __wt_stat_dsrc_aggregate_single( WT_DSRC_STATS *from, WT_DSRC_STATS *to) { + to->bloom_false_positive += from->bloom_false_positive; + to->bloom_hit += from->bloom_hit; + to->bloom_miss += from->bloom_miss; + to->bloom_page_evict += from->bloom_page_evict; + to->bloom_page_read += from->bloom_page_read; + to->bloom_count += from->bloom_count; + to->lsm_chunk_count += from->lsm_chunk_count; + if (from->lsm_generation_max > to->lsm_generation_max) + to->lsm_generation_max = from->lsm_generation_max; + to->lsm_lookup_no_bloom += from->lsm_lookup_no_bloom; + to->lsm_checkpoint_throttle += from->lsm_checkpoint_throttle; + to->lsm_merge_throttle += from->lsm_merge_throttle; + to->bloom_size += from->bloom_size; to->block_extension += from->block_extension; to->block_alloc += from->block_alloc; to->block_free += from->block_free; @@ -263,9 +276,9 @@ __wt_stat_dsrc_aggregate_single( to->btree_checkpoint_generation += from->btree_checkpoint_generation; to->btree_column_fix += from->btree_column_fix; to->btree_column_internal += from->btree_column_internal; + to->btree_column_rle += from->btree_column_rle; to->btree_column_deleted += from->btree_column_deleted; to->btree_column_variable += from->btree_column_variable; - to->btree_column_rle += from->btree_column_rle; if (from->btree_fixed_len > to->btree_fixed_len) to->btree_fixed_len = from->btree_fixed_len; if (from->btree_maxintlkey > to->btree_maxintlkey) @@ -328,19 +341,6 @@ __wt_stat_dsrc_aggregate_single( to->cursor_search_near += from->cursor_search_near; to->cursor_truncate += from->cursor_truncate; to->cursor_update += from->cursor_update; - to->bloom_false_positive += from->bloom_false_positive; - to->bloom_hit += from->bloom_hit; - to->bloom_miss += from->bloom_miss; - to->bloom_page_evict += from->bloom_page_evict; - to->bloom_page_read += from->bloom_page_read; - to->bloom_count += from->bloom_count; - to->lsm_chunk_count += from->lsm_chunk_count; - if (from->lsm_generation_max > to->lsm_generation_max) - to->lsm_generation_max = from->lsm_generation_max; - to->lsm_lookup_no_bloom += from->lsm_lookup_no_bloom; - to->lsm_checkpoint_throttle += from->lsm_checkpoint_throttle; - to->lsm_merge_throttle += from->lsm_merge_throttle; - to->bloom_size += from->bloom_size; to->rec_dictionary += from->rec_dictionary; to->rec_page_delete_fast += from->rec_page_delete_fast; to->rec_suffix_compression += from->rec_suffix_compression; @@ -367,6 +367,21 @@ __wt_stat_dsrc_aggregate( { int64_t v; + to->bloom_false_positive += WT_STAT_READ(from, bloom_false_positive); + to->bloom_hit += WT_STAT_READ(from, bloom_hit); + to->bloom_miss += WT_STAT_READ(from, bloom_miss); + to->bloom_page_evict += WT_STAT_READ(from, bloom_page_evict); + to->bloom_page_read += WT_STAT_READ(from, bloom_page_read); + to->bloom_count += WT_STAT_READ(from, bloom_count); + to->lsm_chunk_count += WT_STAT_READ(from, lsm_chunk_count); + if ((v = WT_STAT_READ(from, lsm_generation_max)) > + to->lsm_generation_max) + to->lsm_generation_max = v; + to->lsm_lookup_no_bloom += WT_STAT_READ(from, lsm_lookup_no_bloom); + to->lsm_checkpoint_throttle += + WT_STAT_READ(from, lsm_checkpoint_throttle); + to->lsm_merge_throttle += WT_STAT_READ(from, lsm_merge_throttle); + to->bloom_size += WT_STAT_READ(from, bloom_size); to->block_extension += WT_STAT_READ(from, block_extension); to->block_alloc += WT_STAT_READ(from, block_alloc); to->block_free += WT_STAT_READ(from, block_free); @@ -387,10 +402,10 @@ __wt_stat_dsrc_aggregate( to->btree_column_fix += WT_STAT_READ(from, btree_column_fix); to->btree_column_internal += WT_STAT_READ(from, btree_column_internal); + to->btree_column_rle += WT_STAT_READ(from, btree_column_rle); to->btree_column_deleted += WT_STAT_READ(from, btree_column_deleted); to->btree_column_variable += WT_STAT_READ(from, btree_column_variable); - to->btree_column_rle += WT_STAT_READ(from, btree_column_rle); if ((v = WT_STAT_READ(from, btree_fixed_len)) > to->btree_fixed_len) to->btree_fixed_len = v; if ((v = WT_STAT_READ(from, btree_maxintlkey)) > to->btree_maxintlkey) @@ -467,21 +482,6 @@ __wt_stat_dsrc_aggregate( to->cursor_search_near += WT_STAT_READ(from, cursor_search_near); to->cursor_truncate += WT_STAT_READ(from, cursor_truncate); to->cursor_update += WT_STAT_READ(from, cursor_update); - to->bloom_false_positive += WT_STAT_READ(from, bloom_false_positive); - to->bloom_hit += WT_STAT_READ(from, bloom_hit); - to->bloom_miss += WT_STAT_READ(from, bloom_miss); - to->bloom_page_evict += WT_STAT_READ(from, bloom_page_evict); - to->bloom_page_read += WT_STAT_READ(from, bloom_page_read); - to->bloom_count += WT_STAT_READ(from, bloom_count); - to->lsm_chunk_count += WT_STAT_READ(from, lsm_chunk_count); - if ((v = WT_STAT_READ(from, lsm_generation_max)) > - to->lsm_generation_max) - to->lsm_generation_max = v; - to->lsm_lookup_no_bloom += WT_STAT_READ(from, lsm_lookup_no_bloom); - to->lsm_checkpoint_throttle += - WT_STAT_READ(from, lsm_checkpoint_throttle); - to->lsm_merge_throttle += WT_STAT_READ(from, lsm_merge_throttle); - to->bloom_size += WT_STAT_READ(from, bloom_size); to->rec_dictionary += WT_STAT_READ(from, rec_dictionary); to->rec_page_delete_fast += WT_STAT_READ(from, rec_page_delete_fast); to->rec_suffix_compression += @@ -509,12 +509,22 @@ __wt_stat_dsrc_aggregate( } static const char * const __stats_connection_desc[] = { - "async: number of allocation state races", - "async: number of operation slots viewed for allocation", + "LSM: application work units currently queued", + "LSM: merge work units currently queued", + "LSM: rows merged in an LSM tree", + "LSM: sleep for LSM checkpoint throttle", + "LSM: sleep for LSM merge throttle", + "LSM: switch work units currently queued", + "LSM: tree maintenance operations discarded", + "LSM: tree maintenance operations executed", + "LSM: tree maintenance operations scheduled", + "LSM: tree queue hit maximum", "async: current work queue length", + "async: maximum work queue length", + "async: number of allocation state races", "async: number of flush calls", + "async: number of operation slots viewed for allocation", "async: number of times operation allocation failed", - "async: maximum work queue length", "async: number of times worker found no work", "async: total allocations", "async: total compact calls", @@ -522,55 +532,63 @@ static const char * const __stats_connection_desc[] = { "async: total remove calls", "async: total search calls", "async: total update calls", - "block-manager: mapped bytes read", - "block-manager: bytes read", - "block-manager: bytes written", - "block-manager: mapped blocks read", "block-manager: blocks pre-loaded", "block-manager: blocks read", "block-manager: blocks written", - "cache: tracked dirty bytes in the cache", - "cache: tracked bytes belonging to internal pages in the cache", + "block-manager: bytes read", + "block-manager: bytes written", + "block-manager: mapped blocks read", + "block-manager: mapped bytes read", "cache: bytes currently in the cache", - "cache: tracked bytes belonging to leaf pages in the cache", - "cache: maximum bytes configured", - "cache: tracked bytes belonging to overflow pages in the cache", "cache: bytes read into cache", "cache: bytes written from cache", - "cache: pages evicted by application threads", "cache: checkpoint blocked page eviction", - "cache: unmodified pages evicted", - "cache: page split during eviction deepened the tree", - "cache: modified pages evicted", - "cache: pages selected for eviction unable to be evicted", - "cache: pages evicted because they exceeded the in-memory maximum", - "cache: pages evicted because they had chains of deleted items", - "cache: failed eviction of pages that exceeded the in-memory maximum", - "cache: hazard pointer blocked page eviction", - "cache: internal pages evicted", - "cache: maximum page size at eviction", "cache: eviction server candidate queue empty when topping up", "cache: eviction server candidate queue not empty when topping up", "cache: eviction server evicting pages", "cache: eviction server populating queue, but not evicting pages", "cache: eviction server unable to reach eviction goal", - "cache: internal pages split during eviction", - "cache: leaf pages split during eviction", - "cache: pages walked for eviction", "cache: eviction worker thread evicting pages", - "cache: in-memory page splits", + "cache: failed eviction of pages that exceeded the in-memory maximum", + "cache: hazard pointer blocked page eviction", "cache: in-memory page passed criteria to be split", + "cache: in-memory page splits", + "cache: internal pages evicted", + "cache: internal pages split during eviction", + "cache: leaf pages split during eviction", "cache: lookaside table insert calls", "cache: lookaside table remove calls", - "cache: percentage overhead", - "cache: tracked dirty pages in the cache", + "cache: maximum bytes configured", + "cache: maximum page size at eviction", + "cache: modified pages evicted", + "cache: page split during eviction deepened the tree", + "cache: page written requiring lookaside records", "cache: pages currently held in the cache", + "cache: pages evicted because they exceeded the in-memory maximum", + "cache: pages evicted because they had chains of deleted items", + "cache: pages evicted by application threads", "cache: pages read into cache", "cache: pages read into cache requiring lookaside entries", + "cache: pages selected for eviction unable to be evicted", + "cache: pages walked for eviction", "cache: pages written from cache", - "cache: page written requiring lookaside records", "cache: pages written requiring in-memory restoration", + "cache: percentage overhead", + "cache: tracked bytes belonging to internal pages in the cache", + "cache: tracked bytes belonging to leaf pages in the cache", + "cache: tracked bytes belonging to overflow pages in the cache", + "cache: tracked dirty bytes in the cache", + "cache: tracked dirty pages in the cache", + "cache: unmodified pages evicted", + "connection: files currently open", + "connection: memory allocations", + "connection: memory frees", + "connection: memory re-allocations", "connection: pthread mutex condition wait calls", + "connection: pthread mutex shared lock read-lock calls", + "connection: pthread mutex shared lock write-lock calls", + "connection: total read I/Os", + "connection: total write I/Os", "cursor: cursor create calls", "cursor: cursor insert calls", "cursor: cursor next calls", @@ -580,96 +598,78 @@ static const char * const __stats_connection_desc[] = { "cursor: cursor restarted searches", "cursor: cursor search calls", "cursor: cursor search near calls", - "cursor: truncate calls", "cursor: cursor update calls", + "cursor: truncate calls", "data-handle: connection data handles currently active", - "data-handle: session dhandles swept", - "data-handle: session sweep attempts", - "data-handle: connection sweep dhandles closed", "data-handle: connection sweep candidate became referenced", + "data-handle: connection sweep dhandles closed", "data-handle: connection sweep dhandles removed from hash list", "data-handle: connection sweep time-of-death sets", "data-handle: connection sweeps", - "connection: files currently open", - "log: total log buffer size", + "data-handle: session dhandles swept", + "data-handle: session sweep attempts", + "log: busy returns attempting to switch slots", + "log: consolidated slot closures", + "log: consolidated slot join races", + "log: consolidated slot join transitions", + "log: consolidated slot joins", + "log: consolidated slot unbuffered writes", "log: log bytes of payload data", "log: log bytes written", - "log: yields waiting for previous log file close", - "log: total size of compressed records", - "log: total in-memory size of compressed records", - "log: log records too small to compress", - "log: log records not compressed", - "log: log records compressed", + "log: log files manually zero-filled", "log: log flush operations", + "log: log records compressed", + "log: log records not compressed", + "log: log records too small to compress", + "log: log release advances write LSN", + "log: log scan operations", + "log: log scan records requiring two reads", + "log: log server thread advances write LSN", + "log: log sync operations", + "log: log sync_dir operations", + "log: log write operations", + "log: logging bytes consolidated", "log: maximum log file size", - "log: pre-allocated log files prepared", "log: number of pre-allocated log files to create", "log: pre-allocated log files not ready and missed", + "log: pre-allocated log files prepared", "log: pre-allocated log files used", - "log: log release advances write LSN", "log: records processed by log scan", - "log: log scan records requiring two reads", - "log: log scan operations", - "log: consolidated slot closures", + "log: total in-memory size of compressed records", + "log: total log buffer size", + "log: total size of compressed records", "log: written slots coalesced", - "log: logging bytes consolidated", - "log: consolidated slot joins", - "log: consolidated slot join races", - "log: busy returns attempting to switch slots", - "log: consolidated slot join transitions", - "log: consolidated slot unbuffered writes", - "log: log sync operations", - "log: log sync_dir operations", - "log: log server thread advances write LSN", - "log: log write operations", - "log: log files manually zero-filled", - "LSM: sleep for LSM checkpoint throttle", - "LSM: sleep for LSM merge throttle", - "LSM: rows merged in an LSM tree", - "LSM: application work units currently queued", - "LSM: merge work units currently queued", - "LSM: tree queue hit maximum", - "LSM: switch work units currently queued", - "LSM: tree maintenance operations scheduled", - "LSM: tree maintenance operations discarded", - "LSM: tree maintenance operations executed", - "connection: memory allocations", - "connection: memory frees", - "connection: memory re-allocations", - "thread-yield: page acquire busy blocked", - "thread-yield: page acquire eviction blocked", - "thread-yield: page acquire locked blocked", - "thread-yield: page acquire read blocked", - "thread-yield: page acquire time sleeping (usecs)", - "connection: total read I/Os", - "reconciliation: pages deleted", + "log: yields waiting for previous log file close", "reconciliation: fast-path pages deleted", "reconciliation: page reconciliation calls", "reconciliation: page reconciliation calls for eviction", + "reconciliation: pages deleted", "reconciliation: split bytes currently awaiting free", "reconciliation: split objects currently awaiting free", - "connection: pthread mutex shared lock read-lock calls", - "connection: pthread mutex shared lock write-lock calls", "session: open cursor count", "session: open session count", + "thread-yield: page acquire busy blocked", + "thread-yield: page acquire eviction blocked", + "thread-yield: page acquire locked blocked", + "thread-yield: page acquire read blocked", + "thread-yield: page acquire time sleeping (usecs)", + "transaction: number of named snapshots created", + "transaction: number of named snapshots dropped", "transaction: transaction begins", - "transaction: transaction checkpoints", - "transaction: transaction checkpoint generation", "transaction: transaction checkpoint currently running", + "transaction: transaction checkpoint generation", "transaction: transaction checkpoint max time (msecs)", "transaction: transaction checkpoint min time (msecs)", "transaction: transaction checkpoint most recent time (msecs)", "transaction: transaction checkpoint total time (msecs)", - "transaction: transactions committed", + "transaction: transaction checkpoints", "transaction: transaction failures due to cache overflow", - "transaction: transaction range of IDs currently pinned by a checkpoint", "transaction: transaction range of IDs currently pinned", + "transaction: transaction range of IDs currently pinned by a checkpoint", "transaction: transaction range of IDs currently pinned by named snapshots", - "transaction: transactions rolled back", - "transaction: number of named snapshots created", - "transaction: number of named snapshots dropped", "transaction: transaction sync calls", - "connection: total write I/Os", + "transaction: transactions committed", + "transaction: transactions rolled back", }; int @@ -700,6 +700,16 @@ __wt_stat_connection_init(WT_CONNECTION_IMPL *handle) void __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) { + /* not clearing lsm_work_queue_app */ + /* not clearing lsm_work_queue_manager */ + stats->lsm_rows_merged = 0; + stats->lsm_checkpoint_throttle = 0; + stats->lsm_merge_throttle = 0; + /* not clearing lsm_work_queue_switch */ + stats->lsm_work_units_discarded = 0; + stats->lsm_work_units_done = 0; + stats->lsm_work_units_created = 0; + stats->lsm_work_queue_max = 0; stats->async_cur_queue = 0; /* not clearing async_max_queue */ stats->async_alloc_race = 0; @@ -821,16 +831,6 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->log_compress_len = 0; stats->log_slot_coalesced = 0; stats->log_close_yields = 0; - /* not clearing lsm_work_queue_app */ - /* not clearing lsm_work_queue_manager */ - stats->lsm_rows_merged = 0; - stats->lsm_checkpoint_throttle = 0; - stats->lsm_merge_throttle = 0; - /* not clearing lsm_work_queue_switch */ - stats->lsm_work_units_discarded = 0; - stats->lsm_work_units_done = 0; - stats->lsm_work_units_created = 0; - stats->lsm_work_queue_max = 0; stats->rec_page_delete_fast = 0; stats->rec_pages = 0; stats->rec_pages_eviction = 0; @@ -876,6 +876,21 @@ void __wt_stat_connection_aggregate( WT_CONNECTION_STATS **from, WT_CONNECTION_STATS *to) { + to->lsm_work_queue_app += WT_STAT_READ(from, lsm_work_queue_app); + to->lsm_work_queue_manager += + WT_STAT_READ(from, lsm_work_queue_manager); + to->lsm_rows_merged += WT_STAT_READ(from, lsm_rows_merged); + to->lsm_checkpoint_throttle += + WT_STAT_READ(from, lsm_checkpoint_throttle); + to->lsm_merge_throttle += WT_STAT_READ(from, lsm_merge_throttle); + to->lsm_work_queue_switch += + WT_STAT_READ(from, lsm_work_queue_switch); + to->lsm_work_units_discarded += + WT_STAT_READ(from, lsm_work_units_discarded); + to->lsm_work_units_done += WT_STAT_READ(from, lsm_work_units_done); + to->lsm_work_units_created += + WT_STAT_READ(from, lsm_work_units_created); + to->lsm_work_queue_max += WT_STAT_READ(from, lsm_work_queue_max); to->async_cur_queue += WT_STAT_READ(from, async_cur_queue); to->async_max_queue += WT_STAT_READ(from, async_max_queue); to->async_alloc_race += WT_STAT_READ(from, async_alloc_race); @@ -1018,21 +1033,6 @@ __wt_stat_connection_aggregate( to->log_compress_len += WT_STAT_READ(from, log_compress_len); to->log_slot_coalesced += WT_STAT_READ(from, log_slot_coalesced); to->log_close_yields += WT_STAT_READ(from, log_close_yields); - to->lsm_work_queue_app += WT_STAT_READ(from, lsm_work_queue_app); - to->lsm_work_queue_manager += - WT_STAT_READ(from, lsm_work_queue_manager); - to->lsm_rows_merged += WT_STAT_READ(from, lsm_rows_merged); - to->lsm_checkpoint_throttle += - WT_STAT_READ(from, lsm_checkpoint_throttle); - to->lsm_merge_throttle += WT_STAT_READ(from, lsm_merge_throttle); - to->lsm_work_queue_switch += - WT_STAT_READ(from, lsm_work_queue_switch); - to->lsm_work_units_discarded += - WT_STAT_READ(from, lsm_work_units_discarded); - to->lsm_work_units_done += WT_STAT_READ(from, lsm_work_units_done); - to->lsm_work_units_created += - WT_STAT_READ(from, lsm_work_units_created); - to->lsm_work_queue_max += WT_STAT_READ(from, lsm_work_queue_max); to->rec_page_delete_fast += WT_STAT_READ(from, rec_page_delete_fast); to->rec_pages += WT_STAT_READ(from, rec_pages); to->rec_pages_eviction += WT_STAT_READ(from, rec_pages_eviction); -- cgit v1.2.1 From a54ccf1a03ba1115fa48dd6293c93172305da7f6 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Thu, 25 Feb 2016 12:11:45 +1100 Subject: WT-2428 Fix override of default timestamp for JSON. --- src/conn/conn_stat.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/conn/conn_stat.c b/src/conn/conn_stat.c index e89a7e81600..9ccfa8ad8db 100644 --- a/src/conn/conn_stat.c +++ b/src/conn/conn_stat.c @@ -150,11 +150,12 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp) * default. */ if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_JSON)) { - WT_ERR(__wt_config_gets_def( - session, cfg, "statistics_log.timestamp", 0, &cval)); - if (cval.type == WT_CONFIG_ITEM_NUM) + ret = __wt_config_gets( + session, &cfg[1], "statistics_log.timestamp", &cval); + if (ret == WT_NOTFOUND) WT_ERR(__wt_strdup( session, "%FT%T.000Z", &conn->stat_format)); + WT_ERR_NOTFOUND_OK(ret); } if (conn->stat_format == NULL) { WT_ERR(__wt_config_gets( -- cgit v1.2.1 From 2e708d94104ede9f03412206a7da4ca2d59c087d Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Thu, 25 Feb 2016 03:53:48 +0000 Subject: WT-2429 Add a statistic to track eviction aggressive mode. --- dist/stat_data.py | 1 + src/evict/evict_lru.c | 11 +- src/include/stat.h | 1 + src/include/wiredtiger.in | 268 +++++++++++++++++++++++---------------------- src/support/stat.c | 4 + tools/wtstats/stat_data.py | 2 + 6 files changed, 152 insertions(+), 135 deletions(-) diff --git a/dist/stat_data.py b/dist/stat_data.py index 41a93961079..be33ba4d441 100644 --- a/dist/stat_data.py +++ b/dist/stat_data.py @@ -157,6 +157,7 @@ connection_stats = [ CacheStat('cache_bytes_overflow', 'tracked bytes belonging to overflow pages in the cache', 'no_clear,no_scale'), CacheStat('cache_bytes_read', 'bytes read into cache'), CacheStat('cache_bytes_write', 'bytes written from cache'), + CacheStat('cache_eviction_aggressive_set', 'eviction currently operating in aggressive mode', 'no_clear,no_scale'), CacheStat('cache_eviction_app', 'pages evicted by application threads'), CacheStat('cache_eviction_checkpoint', 'checkpoint blocked page eviction'), CacheStat('cache_eviction_clean', 'unmodified pages evicted'), diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index dc1a104a0fc..c103cab2613 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -477,6 +477,7 @@ __evict_update_work(WT_SESSION_IMPL *session) conn = S2C(session); cache = conn->cache; + WT_STAT_FAST_CONN_SET(session, cache_eviction_aggressive_set, 0); /* Clear previous state. */ cache->state = 0; @@ -534,8 +535,11 @@ __evict_update_work(WT_SESSION_IMPL *session) return (false); -done: if (F_ISSET(cache, WT_CACHE_STUCK)) +done: if (F_ISSET(cache, WT_CACHE_STUCK)) { + WT_STAT_FAST_CONN_SET(session, + cache_eviction_aggressive_set, 1); FLD_SET(cache->state, WT_EVICT_PASS_AGGRESSIVE); + } return (true); } @@ -594,8 +598,11 @@ __evict_pass(WT_SESSION_IMPL *session) if (!__evict_update_work(session)) break; - if (loop > 10) + if (loop > 10) { + WT_STAT_FAST_CONN_SET(session, + cache_eviction_aggressive_set, 1); FLD_SET(cache->state, WT_EVICT_PASS_AGGRESSIVE); + } /* * Start a worker if we have capacity and we haven't reached diff --git a/src/include/stat.h b/src/include/stat.h index 51d2fa332e7..015e930542f 100644 --- a/src/include/stat.h +++ b/src/include/stat.h @@ -255,6 +255,7 @@ struct __wt_connection_stats { int64_t cache_bytes_overflow; int64_t cache_bytes_read; int64_t cache_bytes_write; + int64_t cache_eviction_aggressive_set; int64_t cache_eviction_app; int64_t cache_eviction_checkpoint; int64_t cache_eviction_clean; diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index bca9a6a248b..68799bb26ee 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -3756,273 +3756,275 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_CACHE_BYTES_READ 1026 /*! cache: bytes written from cache */ #define WT_STAT_CONN_CACHE_BYTES_WRITE 1027 +/*! cache: eviction currently operating in aggressive mode */ +#define WT_STAT_CONN_CACHE_EVICTION_AGGRESSIVE_SET 1028 /*! cache: pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP 1028 +#define WT_STAT_CONN_CACHE_EVICTION_APP 1029 /*! cache: checkpoint blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_CHECKPOINT 1029 +#define WT_STAT_CONN_CACHE_EVICTION_CHECKPOINT 1030 /*! cache: unmodified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1030 +#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1031 /*! cache: page split during eviction deepened the tree */ -#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1031 +#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1032 /*! cache: modified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1032 +#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1033 /*! cache: pages selected for eviction unable to be evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1033 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1034 /*! cache: pages evicted because they exceeded the in-memory maximum */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1034 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1035 /*! cache: pages evicted because they had chains of deleted items */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1035 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1036 /*! cache: failed eviction of pages that exceeded the in-memory maximum */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1036 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1037 /*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1037 +#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1038 /*! cache: internal pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1038 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1039 /*! cache: maximum page size at eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1039 +#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1040 /*! cache: eviction server candidate queue empty when topping up */ -#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_EMPTY 1040 +#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_EMPTY 1041 /*! cache: eviction server candidate queue not empty when topping up */ -#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_NOT_EMPTY 1041 +#define WT_STAT_CONN_CACHE_EVICTION_QUEUE_NOT_EMPTY 1042 /*! cache: eviction server evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_SERVER_EVICTING 1042 +#define WT_STAT_CONN_CACHE_EVICTION_SERVER_EVICTING 1043 /*! cache: eviction server populating queue, but not evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_SERVER_NOT_EVICTING 1043 +#define WT_STAT_CONN_CACHE_EVICTION_SERVER_NOT_EVICTING 1044 /*! cache: eviction server unable to reach eviction goal */ -#define WT_STAT_CONN_CACHE_EVICTION_SLOW 1044 +#define WT_STAT_CONN_CACHE_EVICTION_SLOW 1045 /*! cache: internal pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1045 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1046 /*! cache: leaf pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1046 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1047 /*! cache: pages walked for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK 1047 +#define WT_STAT_CONN_CACHE_EVICTION_WALK 1048 /*! cache: eviction worker thread evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1048 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1049 /*! cache: in-memory page splits */ -#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1049 +#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1050 /*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1050 +#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1051 /*! cache: lookaside table insert calls */ -#define WT_STAT_CONN_CACHE_LOOKASIDE_INSERT 1051 +#define WT_STAT_CONN_CACHE_LOOKASIDE_INSERT 1052 /*! cache: lookaside table remove calls */ -#define WT_STAT_CONN_CACHE_LOOKASIDE_REMOVE 1052 +#define WT_STAT_CONN_CACHE_LOOKASIDE_REMOVE 1053 /*! cache: percentage overhead */ -#define WT_STAT_CONN_CACHE_OVERHEAD 1053 +#define WT_STAT_CONN_CACHE_OVERHEAD 1054 /*! cache: tracked dirty pages in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1054 +#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1055 /*! cache: pages currently held in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_INUSE 1055 +#define WT_STAT_CONN_CACHE_PAGES_INUSE 1056 /*! cache: pages read into cache */ -#define WT_STAT_CONN_CACHE_READ 1056 +#define WT_STAT_CONN_CACHE_READ 1057 /*! cache: pages read into cache requiring lookaside entries */ -#define WT_STAT_CONN_CACHE_READ_LOOKASIDE 1057 +#define WT_STAT_CONN_CACHE_READ_LOOKASIDE 1058 /*! cache: pages written from cache */ -#define WT_STAT_CONN_CACHE_WRITE 1058 +#define WT_STAT_CONN_CACHE_WRITE 1059 /*! cache: page written requiring lookaside records */ -#define WT_STAT_CONN_CACHE_WRITE_LOOKASIDE 1059 +#define WT_STAT_CONN_CACHE_WRITE_LOOKASIDE 1060 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1060 +#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1061 /*! connection: pthread mutex condition wait calls */ -#define WT_STAT_CONN_COND_WAIT 1061 +#define WT_STAT_CONN_COND_WAIT 1062 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1062 +#define WT_STAT_CONN_CURSOR_CREATE 1063 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1063 +#define WT_STAT_CONN_CURSOR_INSERT 1064 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1064 +#define WT_STAT_CONN_CURSOR_NEXT 1065 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1065 +#define WT_STAT_CONN_CURSOR_PREV 1066 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1066 +#define WT_STAT_CONN_CURSOR_REMOVE 1067 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1067 +#define WT_STAT_CONN_CURSOR_RESET 1068 /*! cursor: cursor restarted searches */ -#define WT_STAT_CONN_CURSOR_RESTART 1068 +#define WT_STAT_CONN_CURSOR_RESTART 1069 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1069 +#define WT_STAT_CONN_CURSOR_SEARCH 1070 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1070 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1071 /*! cursor: truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1071 +#define WT_STAT_CONN_CURSOR_TRUNCATE 1072 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1072 +#define WT_STAT_CONN_CURSOR_UPDATE 1073 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1073 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1074 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1074 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1075 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1075 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1076 /*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1076 +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1077 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1077 +#define WT_STAT_CONN_DH_SWEEP_REF 1078 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1078 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1079 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1079 +#define WT_STAT_CONN_DH_SWEEP_TOD 1080 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1080 +#define WT_STAT_CONN_DH_SWEEPS 1081 /*! connection: files currently open */ -#define WT_STAT_CONN_FILE_OPEN 1081 +#define WT_STAT_CONN_FILE_OPEN 1082 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1082 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1083 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1083 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1084 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1084 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1085 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1085 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1086 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1086 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1087 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1087 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1088 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1088 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1089 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1089 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1090 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1090 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1091 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1091 +#define WT_STAT_CONN_LOG_FLUSH 1092 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1092 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1093 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1093 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1094 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1094 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1095 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1095 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1096 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1096 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1097 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1097 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1098 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1098 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1099 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1099 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1100 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1100 +#define WT_STAT_CONN_LOG_SCANS 1101 /*! log: consolidated slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1101 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1102 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1102 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1103 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1103 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1104 /*! log: consolidated slot joins */ -#define WT_STAT_CONN_LOG_SLOT_JOINS 1104 +#define WT_STAT_CONN_LOG_SLOT_JOINS 1105 /*! log: consolidated slot join races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1105 +#define WT_STAT_CONN_LOG_SLOT_RACES 1106 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1106 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1107 /*! log: consolidated slot join transitions */ -#define WT_STAT_CONN_LOG_SLOT_TRANSITIONS 1107 +#define WT_STAT_CONN_LOG_SLOT_TRANSITIONS 1108 /*! log: consolidated slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1108 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1109 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1109 +#define WT_STAT_CONN_LOG_SYNC 1110 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1110 +#define WT_STAT_CONN_LOG_SYNC_DIR 1111 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1111 +#define WT_STAT_CONN_LOG_WRITE_LSN 1112 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1112 +#define WT_STAT_CONN_LOG_WRITES 1113 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1113 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1114 /*! LSM: sleep for LSM checkpoint throttle */ -#define WT_STAT_CONN_LSM_CHECKPOINT_THROTTLE 1114 +#define WT_STAT_CONN_LSM_CHECKPOINT_THROTTLE 1115 /*! LSM: sleep for LSM merge throttle */ -#define WT_STAT_CONN_LSM_MERGE_THROTTLE 1115 +#define WT_STAT_CONN_LSM_MERGE_THROTTLE 1116 /*! LSM: rows merged in an LSM tree */ -#define WT_STAT_CONN_LSM_ROWS_MERGED 1116 +#define WT_STAT_CONN_LSM_ROWS_MERGED 1117 /*! LSM: application work units currently queued */ -#define WT_STAT_CONN_LSM_WORK_QUEUE_APP 1117 +#define WT_STAT_CONN_LSM_WORK_QUEUE_APP 1118 /*! LSM: merge work units currently queued */ -#define WT_STAT_CONN_LSM_WORK_QUEUE_MANAGER 1118 +#define WT_STAT_CONN_LSM_WORK_QUEUE_MANAGER 1119 /*! LSM: tree queue hit maximum */ -#define WT_STAT_CONN_LSM_WORK_QUEUE_MAX 1119 +#define WT_STAT_CONN_LSM_WORK_QUEUE_MAX 1120 /*! LSM: switch work units currently queued */ -#define WT_STAT_CONN_LSM_WORK_QUEUE_SWITCH 1120 +#define WT_STAT_CONN_LSM_WORK_QUEUE_SWITCH 1121 /*! LSM: tree maintenance operations scheduled */ -#define WT_STAT_CONN_LSM_WORK_UNITS_CREATED 1121 +#define WT_STAT_CONN_LSM_WORK_UNITS_CREATED 1122 /*! LSM: tree maintenance operations discarded */ -#define WT_STAT_CONN_LSM_WORK_UNITS_DISCARDED 1122 +#define WT_STAT_CONN_LSM_WORK_UNITS_DISCARDED 1123 /*! LSM: tree maintenance operations executed */ -#define WT_STAT_CONN_LSM_WORK_UNITS_DONE 1123 +#define WT_STAT_CONN_LSM_WORK_UNITS_DONE 1124 /*! connection: memory allocations */ -#define WT_STAT_CONN_MEMORY_ALLOCATION 1124 +#define WT_STAT_CONN_MEMORY_ALLOCATION 1125 /*! connection: memory frees */ -#define WT_STAT_CONN_MEMORY_FREE 1125 +#define WT_STAT_CONN_MEMORY_FREE 1126 /*! connection: memory re-allocations */ -#define WT_STAT_CONN_MEMORY_GROW 1126 +#define WT_STAT_CONN_MEMORY_GROW 1127 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1127 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1128 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1128 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1129 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1129 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1130 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1130 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1131 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1131 +#define WT_STAT_CONN_PAGE_SLEEP 1132 /*! connection: total read I/Os */ -#define WT_STAT_CONN_READ_IO 1132 +#define WT_STAT_CONN_READ_IO 1133 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1133 +#define WT_STAT_CONN_REC_PAGE_DELETE 1134 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1134 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1135 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1135 +#define WT_STAT_CONN_REC_PAGES 1136 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1136 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1137 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1137 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1138 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1138 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1139 /*! connection: pthread mutex shared lock read-lock calls */ -#define WT_STAT_CONN_RWLOCK_READ 1139 +#define WT_STAT_CONN_RWLOCK_READ 1140 /*! connection: pthread mutex shared lock write-lock calls */ -#define WT_STAT_CONN_RWLOCK_WRITE 1140 +#define WT_STAT_CONN_RWLOCK_WRITE 1141 /*! session: open cursor count */ -#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1141 +#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1142 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1142 +#define WT_STAT_CONN_SESSION_OPEN 1143 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1143 +#define WT_STAT_CONN_TXN_BEGIN 1144 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1144 +#define WT_STAT_CONN_TXN_CHECKPOINT 1145 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1145 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1146 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1146 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1147 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1147 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1148 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1148 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1149 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1149 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1150 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1150 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1151 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1151 +#define WT_STAT_CONN_TXN_COMMIT 1152 /*! transaction: transaction failures due to cache overflow */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1152 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1153 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1153 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1154 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1154 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1155 /*! transaction: transaction range of IDs currently pinned by named * snapshots */ -#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1155 +#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1156 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1156 +#define WT_STAT_CONN_TXN_ROLLBACK 1157 /*! transaction: number of named snapshots created */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1157 +#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1158 /*! transaction: number of named snapshots dropped */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1158 +#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1159 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1159 +#define WT_STAT_CONN_TXN_SYNC 1160 /*! connection: total write I/Os */ -#define WT_STAT_CONN_WRITE_IO 1160 +#define WT_STAT_CONN_WRITE_IO 1161 /*! * @} diff --git a/src/support/stat.c b/src/support/stat.c index 7a615131628..e355d7dc04c 100644 --- a/src/support/stat.c +++ b/src/support/stat.c @@ -537,6 +537,7 @@ static const char * const __stats_connection_desc[] = { "cache: tracked bytes belonging to overflow pages in the cache", "cache: bytes read into cache", "cache: bytes written from cache", + "cache: eviction currently operating in aggressive mode", "cache: pages evicted by application threads", "cache: checkpoint blocked page eviction", "cache: unmodified pages evicted", @@ -724,6 +725,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_bytes_read = 0; stats->cache_bytes_write = 0; stats->cache_eviction_checkpoint = 0; + /* not clearing cache_eviction_aggressive_set */ stats->cache_eviction_queue_empty = 0; stats->cache_eviction_queue_not_empty = 0; stats->cache_eviction_server_evicting = 0; @@ -901,6 +903,8 @@ __wt_stat_connection_aggregate( to->cache_bytes_write += WT_STAT_READ(from, cache_bytes_write); to->cache_eviction_checkpoint += WT_STAT_READ(from, cache_eviction_checkpoint); + to->cache_eviction_aggressive_set += + WT_STAT_READ(from, cache_eviction_aggressive_set); to->cache_eviction_queue_empty += WT_STAT_READ(from, cache_eviction_queue_empty); to->cache_eviction_queue_not_empty += diff --git a/tools/wtstats/stat_data.py b/tools/wtstats/stat_data.py index 7cee87e49ed..f181aeb09b4 100644 --- a/tools/wtstats/stat_data.py +++ b/tools/wtstats/stat_data.py @@ -3,6 +3,7 @@ no_scale_per_second_list = [ 'async: maximum work queue length', 'cache: bytes currently in the cache', + 'cache: eviction currently operating in aggressive mode', 'cache: maximum bytes configured', 'cache: maximum page size at eviction', 'cache: pages currently held in the cache', @@ -67,6 +68,7 @@ no_scale_per_second_list = [ no_clear_list = [ 'async: maximum work queue length', 'cache: bytes currently in the cache', + 'cache: eviction currently operating in aggressive mode', 'cache: maximum bytes configured', 'cache: maximum page size at eviction', 'cache: pages currently held in the cache', -- cgit v1.2.1 From 319c8f2209eeac78dd2b49765853ddac1b690381 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Thu, 25 Feb 2016 15:08:11 +1100 Subject: SERVER-22831 Queue more leaf pages than internal pages for eviction. Unless we get aggressive, putting internal pages on the LRU queue is counterproductive. That is particularly true for workloads that transition to read-only, where some tables are not being queried. In that case, all of the leaf pages are evicted, and eviction wastes a lot of effort walking and queuing internal pages that are not actually evicted. --- src/evict/evict_lru.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index dc1a104a0fc..02d7b19137f 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -1267,7 +1267,8 @@ __evict_walk_file(WT_SESSION_IMPL *session, u_int *slotp) /* Pages we no longer need (clean or dirty), are found money. */ if (__wt_page_is_empty(page) || - F_ISSET(session->dhandle, WT_DHANDLE_DEAD)) + F_ISSET(session->dhandle, WT_DHANDLE_DEAD) || + page->read_gen == WT_READGEN_OLDEST) goto fast; /* Skip clean pages if appropriate. */ @@ -1280,14 +1281,13 @@ __evict_walk_file(WT_SESSION_IMPL *session, u_int *slotp) * eviction, skip anything that isn't marked. */ if (FLD_ISSET(cache->state, WT_EVICT_PASS_WOULD_BLOCK) && - page->memory_footprint < btree->splitmempage && - page->read_gen != WT_READGEN_OLDEST) + page->memory_footprint < btree->splitmempage) continue; /* Limit internal pages to 50% unless we get aggressive. */ if (WT_PAGE_IS_INTERNAL(page) && - ++internal_pages > WT_EVICT_WALK_PER_FILE / 2 && - !FLD_ISSET(cache->state, WT_EVICT_PASS_AGGRESSIVE)) + !FLD_ISSET(cache->state, WT_EVICT_PASS_AGGRESSIVE) && + internal_pages >= (int)(evict - start) / 2) continue; /* @@ -1332,6 +1332,9 @@ fast: /* If the page can't be evicted, give up. */ __evict_init_candidate(session, evict, ref); ++evict; + if (WT_PAGE_IS_INTERNAL(page)) + ++internal_pages; + WT_RET(__wt_verbose(session, WT_VERB_EVICTSERVER, "select: %p, size %" PRIu64, page, page->memory_footprint)); } -- cgit v1.2.1 From 15bc57796fc6c1cfdbb51523793a063ac9f12396 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 25 Feb 2016 12:47:08 -0500 Subject: WT-2381 Change test to look for config settings in colgroups for complext tables. --- test/suite/helper.py | 33 ++++++++++++++++++--------------- test/suite/test_util13.py | 21 +++++++++++++-------- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/test/suite/helper.py b/test/suite/helper.py index 3c460e23d08..6465283d9ee 100644 --- a/test/suite/helper.py +++ b/test/suite/helper.py @@ -196,31 +196,34 @@ def complex_populate_index_count(): # config: prefix of the session.create configuration string # rows: entries to insert def complex_populate(self, uri, config, rows): - complex_populate_type(self, uri, config, rows, '') + complex_populate_type(self, uri, config, '', rows, '') +def complex_populate_cgconfig(self, uri, config, rows): + complex_populate_type(self, uri, config, config, rows, '') def complex_populate_lsm(self, uri, config, rows): - complex_populate_type(self, uri, config, rows, 'type=lsm') -def complex_populate_type(self, uri, config, rows, type): + complex_populate_type(self, uri, config, '', rows, 'type=lsm') +def complex_populate_type(self, uri, config, cgconfig, rows, type): self.session.create(uri, config + ',value_format=SiSS,' + 'columns=(record,column2,column3,column4,column5),' + 'colgroups=(cgroup1,cgroup2,cgroup3,cgroup4,cgroup5,cgroup6)') cgname = 'colgroup:' + uri.split(":")[1] - self.session.create(cgname + ':cgroup1', 'columns=(column2)' + ',' + type) - self.session.create(cgname + ':cgroup2', 'columns=(column3)' + ',' + type) - self.session.create(cgname + ':cgroup3', 'columns=(column4)' + ',' + type) + cgcfg = ',' + cgconfig + ',' + type + self.session.create(cgname + ':cgroup1', 'columns=(column2)' + ',' + cgcfg) + self.session.create(cgname + ':cgroup2', 'columns=(column3)' + ',' + cgcfg) + self.session.create(cgname + ':cgroup3', 'columns=(column4)' + ',' + cgcfg) self.session.create( - cgname + ':cgroup4', 'columns=(column2,column3)' + ',' + type) + cgname + ':cgroup4', 'columns=(column2,column3)' + ',' + cgcfg) self.session.create( - cgname + ':cgroup5', 'columns=(column3,column4)' + ',' + type) + cgname + ':cgroup5', 'columns=(column3,column4)' + ',' + cgcfg) self.session.create( - cgname + ':cgroup6', 'columns=(column2,column4,column5)' + ',' + type) + cgname + ':cgroup6', 'columns=(column2,column4,column5)' + ',' + cgcfg) indxname = 'index:' + uri.split(":")[1] - self.session.create(indxname + ':indx1', 'columns=(column2)' + ',' + type) - self.session.create(indxname + ':indx2', 'columns=(column3)' + ',' + type) - self.session.create(indxname + ':indx3', 'columns=(column4)' + ',' + type) + self.session.create(indxname + ':indx1', 'columns=(column2)' + ',' + cgcfg) + self.session.create(indxname + ':indx2', 'columns=(column3)' + ',' + cgcfg) + self.session.create(indxname + ':indx3', 'columns=(column4)' + ',' + cgcfg) self.session.create( - indxname + ':indx4', 'columns=(column2,column4)' + ',' + type) + indxname + ':indx4', 'columns=(column2,column4)' + ',' + cgcfg) cursor = self.session.open_cursor(uri, None) for i in range(1, rows + 1): cursor[key_populate(cursor, i)] = \ @@ -228,9 +231,9 @@ def complex_populate_type(self, uri, config, rows, type): cursor.close() # add some indices after populating self.session.create( - indxname + ':indx5', 'columns=(column3,column5)' + ',' + type) + indxname + ':indx5', 'columns=(column3,column5)' + ',' + cgcfg) self.session.create( - indxname + ':indx6', 'columns=(column3,column5,column4)' + ',' + type) + indxname + ':indx6', 'columns=(column3,column5,column4)' + ',' + cgcfg) def complex_populate_colgroup_name(self, uri, i): return 'colgroup:' + uri.split(":")[1] + ':cgroup' + str(i + 1) diff --git a/test/suite/test_util13.py b/test/suite/test_util13.py index 5a7f217ddfb..bb5dfd12800 100644 --- a/test/suite/test_util13.py +++ b/test/suite/test_util13.py @@ -30,7 +30,8 @@ import os, re, string from suite_subprocess import suite_subprocess import itertools, wiredtiger, wttest -from helper import complex_populate, complex_populate_lsm, simple_populate +from helper import complex_populate_cgconfig, complex_populate_lsm +from helper import simple_populate from wtscenario import multiply_scenarios, number_scenarios # test_util13.py @@ -55,7 +56,7 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): ('table-simple', dict(uri='table:' + pfx, pop=simple_populate, table_config='split_pct=50', cfg='')), ('table-complex', - dict(uri='table:' + pfx, pop=complex_populate, + dict(uri='table:' + pfx, pop=complex_populate_cgconfig, table_config='allocation_size=512B', cfg='')), ('table-complex-lsm', dict(uri='table:' + pfx, pop=complex_populate_lsm, @@ -101,20 +102,20 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): print dx return match - def compare_files(self, filename1, filename2): + def compare_files(self, expect_subset, dump_out): inheader = isconfig = False - for l1, l2 in zip(open(filename1, "rb"), open(filename2, "rb")): + for l1, l2 in zip(open(expect_subset, "rb"), open(dump_out, "rb")): if isconfig: if not self.compare_config(l1, l2): return False - elif l1 != l2: - return False if inheader: + # This works because the expected subset has a format + # of URI and config lines alternating. isconfig = not isconfig if l1.strip() == 'Header': inheader = True if l1.strip() == 'Data': - inheader = isconfig = False + break return True def test_dump_config(self): @@ -135,7 +136,11 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): expectout.write('Format=print\n') expectout.write('Header\n') expectout.write(self.uri + '\n') - if self.cfg == '': + # Check the config on the colgroup itself for complex tables. + if self.pop != simple_populate: + expectout.write('key_format=S\n') + expectout.write('colgroup:' + self.pfx + ':cgroup1\n') + if self.cfg == '': expectout.write(self.table_config + '\n') else: expectout.write(self.cfg + '\n') -- cgit v1.2.1 From 9652b0bc96dcf985b15093d87d109667ea6d5c4d Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 25 Feb 2016 12:55:16 -0500 Subject: WT-2381 Whitespace --- test/suite/test_util13.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/suite/test_util13.py b/test/suite/test_util13.py index 5f8cb4e7bc7..7830d0f6a43 100644 --- a/test/suite/test_util13.py +++ b/test/suite/test_util13.py @@ -140,7 +140,7 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): if self.pop != simple_populate: expectout.write('key_format=S\n') expectout.write('colgroup:' + self.pfx + ':cgroup1\n') - if self.cfg == '': + if self.cfg == '': expectout.write(self.table_config + '\n') else: expectout.write(self.cfg + '\n') -- cgit v1.2.1 From 647b6bc1c77199df6defa943fed748d3e6bc526b Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 25 Feb 2016 10:59:15 -0500 Subject: WT-2381: dump utility discards table config indent/whitespace --- test/suite/test_util13.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/suite/test_util13.py b/test/suite/test_util13.py index 7830d0f6a43..5e71bfab10f 100644 --- a/test/suite/test_util13.py +++ b/test/suite/test_util13.py @@ -49,7 +49,7 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): # types = [ ('file-simple', dict(uri='file:' + pfx, pop=simple_populate, - table_config='prefix_compression_min=3', cfg='')), + table_config='prefix_compression_min=3', cfg='')), ('lsm-simple', dict(uri='lsm:' + pfx, pop=simple_populate, table_config='lsm=(bloom_bit_count=29)', cfg='bloom_bit_count=29')), -- cgit v1.2.1 From efc8c1d74d72d020744adb88256b3744070d68a3 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 25 Feb 2016 15:15:22 -0500 Subject: WT-2381 Add complex populate function for colgroup config for LSM. --- test/suite/helper.py | 2 ++ test/suite/test_util13.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/suite/helper.py b/test/suite/helper.py index 6465283d9ee..3895402a0d6 100644 --- a/test/suite/helper.py +++ b/test/suite/helper.py @@ -201,6 +201,8 @@ def complex_populate_cgconfig(self, uri, config, rows): complex_populate_type(self, uri, config, config, rows, '') def complex_populate_lsm(self, uri, config, rows): complex_populate_type(self, uri, config, '', rows, 'type=lsm') +def complex_populate_cgconfig_lsm(self, uri, config, rows): + complex_populate_type(self, uri, config, config, rows, 'type=lsm') def complex_populate_type(self, uri, config, cgconfig, rows, type): self.session.create(uri, config + ',value_format=SiSS,' + diff --git a/test/suite/test_util13.py b/test/suite/test_util13.py index 7830d0f6a43..4f20c647bef 100644 --- a/test/suite/test_util13.py +++ b/test/suite/test_util13.py @@ -30,7 +30,7 @@ import os, re, string from suite_subprocess import suite_subprocess import itertools, wiredtiger, wttest -from helper import complex_populate_cgconfig, complex_populate_lsm +from helper import complex_populate_cgconfig, complex_populate_cgconfig_lsm from helper import simple_populate from wtscenario import multiply_scenarios, number_scenarios @@ -59,7 +59,7 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): dict(uri='table:' + pfx, pop=complex_populate_cgconfig, table_config='allocation_size=512B', cfg='')), ('table-complex-lsm', - dict(uri='table:' + pfx, pop=complex_populate_lsm, + dict(uri='table:' + pfx, pop=complex_populate_cgconfig_lsm, table_config='lsm=(merge_max=5)', cfg='merge_max=5')), ] -- cgit v1.2.1 From dbe722569aef25003d6fac79b02cc58e8900c3d4 Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Fri, 26 Feb 2016 10:35:24 -0500 Subject: WT-2430 Refactor initialization of stats cursor so it is always done completely. --- src/cursor/cur_stat.c | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/cursor/cur_stat.c b/src/cursor/cur_stat.c index 8528482a009..34e64b34ccb 100644 --- a/src/cursor/cur_stat.c +++ b/src/cursor/cur_stat.c @@ -200,8 +200,6 @@ __curstat_next(WT_CURSOR *cursor) if (cst->notinitialized) { WT_ERR(__wt_curstat_init( session, cursor->internal_uri, NULL, cst->cfg, cst)); - if (cst->next_set != NULL) - WT_ERR((*cst->next_set)(session, cst, true, true)); cst->notinitialized = false; } @@ -244,8 +242,6 @@ __curstat_prev(WT_CURSOR *cursor) if (cst->notinitialized) { WT_ERR(__wt_curstat_init( session, cursor->internal_uri, NULL, cst->cfg, cst)); - if (cst->next_set != NULL) - WT_ERR((*cst->next_set)(session, cst, false, true)); cst->notinitialized = false; } @@ -449,7 +445,6 @@ __curstat_join_next_set(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst, WT_JOIN_STATS_GROUP *join_group; ssize_t pos; - WT_ASSERT(session, WT_STREQ(cst->iface.uri, "statistics:join")); join_group = &cst->u.join_stats_group; cjoin = join_group->join_cursor; if (init) @@ -542,25 +537,31 @@ __wt_curstat_init(WT_SESSION_IMPL *session, dsrc_uri = uri + strlen("statistics:"); if (WT_STREQ(dsrc_uri, "join")) - return (__curstat_join_init(session, curjoin, cfg, cst)); + WT_RET(__curstat_join_init(session, curjoin, cfg, cst)); - if (WT_PREFIX_MATCH(dsrc_uri, "colgroup:")) - return ( + else if (WT_PREFIX_MATCH(dsrc_uri, "colgroup:")) + WT_RET( __wt_curstat_colgroup_init(session, dsrc_uri, cfg, cst)); - if (WT_PREFIX_MATCH(dsrc_uri, "file:")) - return (__curstat_file_init(session, dsrc_uri, cfg, cst)); + else if (WT_PREFIX_MATCH(dsrc_uri, "file:")) + WT_RET(__curstat_file_init(session, dsrc_uri, cfg, cst)); - if (WT_PREFIX_MATCH(dsrc_uri, "index:")) - return (__wt_curstat_index_init(session, dsrc_uri, cfg, cst)); + else if (WT_PREFIX_MATCH(dsrc_uri, "index:")) + WT_RET(__wt_curstat_index_init(session, dsrc_uri, cfg, cst)); - if (WT_PREFIX_MATCH(dsrc_uri, "lsm:")) - return (__wt_curstat_lsm_init(session, dsrc_uri, cst)); + else if (WT_PREFIX_MATCH(dsrc_uri, "lsm:")) + WT_RET(__wt_curstat_lsm_init(session, dsrc_uri, cst)); - if (WT_PREFIX_MATCH(dsrc_uri, "table:")) - return (__wt_curstat_table_init(session, dsrc_uri, cfg, cst)); + else if (WT_PREFIX_MATCH(dsrc_uri, "table:")) + WT_RET(__wt_curstat_table_init(session, dsrc_uri, cfg, cst)); - return (__wt_bad_object_type(session, uri)); + else + return (__wt_bad_object_type(session, uri)); + + if (cst->next_set != NULL) + WT_RET((*cst->next_set)(session, cst, false, true)); + + return (0); } /* -- cgit v1.2.1 From 479dc5b4044b3a6cd4bb55235c3dc669e7ba910a Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Fri, 26 Feb 2016 11:37:16 -0500 Subject: WT-2430 Added simple cursor join statistics test to match contributed test case. --- test/suite/test_join01.py | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/test/suite/test_join01.py b/test/suite/test_join01.py index ee4d6e22870..7630706379c 100644 --- a/test/suite/test_join01.py +++ b/test/suite/test_join01.py @@ -33,7 +33,6 @@ from wtscenario import check_scenarios, multiply_scenarios, number_scenarios # Join operations # Basic tests for join class test_join01(wttest.WiredTigerTestCase): - table_name1 = 'test_join01' nentries = 100 scenarios = [ @@ -391,6 +390,7 @@ class test_join01(wttest.WiredTigerTestCase): def test_cursor_close2(self): self.cursor_close_common(False) + # test statistics using the framework set up for this test def test_stats(self): bloomcfg1000 = ',strategy=bloom,count=1000' bloomcfg10 = ',strategy=bloom,count=10' @@ -400,6 +400,40 @@ class test_join01(wttest.WiredTigerTestCase): # statistics should pick up some false positives. self.join_common(bloomcfg10, bloomcfg10, False, True) + # test statistics with a simple one index join cursor + def test_simple_stats(self): + self.session.create("table:join01b", + "key_format=i,value_format=i,columns=(k,v)") + self.session.create("index:join01b:index", "columns=(v)") + + cursor = self.session.open_cursor("table:join01b", None, None) + cursor[1] = 11 + cursor[2] = 12 + cursor[3] = 13 + cursor.close() + + cursor = self.session.open_cursor("index:join01b:index", None, None) + cursor.set_key(11) + cursor.search() + + jcursor = self.session.open_cursor("join:table:join01b", None, None) + self.session.join(jcursor, cursor, "compare=gt") + + while jcursor.next() == 0: + [k] = jcursor.get_keys() + [v] = jcursor.get_values() + + statcur = self.session.open_cursor("statistics:join", jcursor, None) + found = False + while statcur.next() == 0: + [desc, pvalue, value] = statcur.get_values() + #self.tty(str(desc) + "=" + str(pvalue)) + found = True + self.assertEquals(found, True) + + jcursor.close() + cursor.close() + if __name__ == '__main__': wttest.run() -- cgit v1.2.1 From bba6734a180750e156d0b5e11bcac30b36a3ab1d Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Fri, 26 Feb 2016 16:46:33 -0500 Subject: WT-2433 Allow statistics logging in readonly mode. --- src/conn/conn_stat.c | 4 ---- src/docs/readonly.dox | 2 +- test/suite/test_readonly02.py | 4 ---- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/conn/conn_stat.c b/src/conn/conn_stat.c index 9ccfa8ad8db..d6e59a50da5 100644 --- a/src/conn/conn_stat.c +++ b/src/conn/conn_stat.c @@ -106,10 +106,6 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp) * If any statistics logging is done, this must not be a read-only * connection. */ - if (F_ISSET(conn, WT_CONN_READONLY)) - WT_RET_MSG(session, EINVAL, - "Read-only configuration incompatible with statistics " - "logging"); WT_RET(__wt_config_gets(session, cfg, "statistics_log.sources", &cval)); WT_RET(__wt_config_subinit(session, &objectconf, &cval)); for (cnt = 0; (ret = __wt_config_next(&objectconf, &k, &v)) == 0; ++cnt) diff --git a/src/docs/readonly.dox b/src/docs/readonly.dox index 9935f5d1b17..ad4a94a73f1 100644 --- a/src/docs/readonly.dox +++ b/src/docs/readonly.dox @@ -18,7 +18,7 @@ tree merges are turned off when LSM trees are configured, and log file archiving is disabled when logging is configured. Where a user configured setting contradicts read-only operation, WiredTiger -will return an error. For example, statistics logging or zero-filling +will return an error. For example, zero-filling log files is not allowed in read-only mode, and attempting to configure them will return an error. diff --git a/test/suite/test_readonly02.py b/test/suite/test_readonly02.py index e94dd85857d..309c14752ac 100644 --- a/test/suite/test_readonly02.py +++ b/test/suite/test_readonly02.py @@ -54,10 +54,8 @@ class test_readonly02(wttest.WiredTigerTestCase, suite_subprocess): # 1. setting readonly on a new database directory # 2. an unclean shutdown and reopening readonly # 3. logging with zero-fill enabled and readonly - # 4. readonly and statistics logging # badcfg1 = 'log=(enabled,zero_fill=true)' - badcfg2 = 'statistics_log=(wait=3)' def setUpConnectionOpen(self, dir): self.home = dir @@ -111,8 +109,6 @@ class test_readonly02(wttest.WiredTigerTestCase, suite_subprocess): # Close the connection. Reopen readonly with other bad settings. # 3. logging with zero-fill enabled and readonly self.close_checkerror(self.badcfg1) - # 4. readonly and statistics logging - self.close_checkerror(self.badcfg2) if __name__ == '__main__': wttest.run() -- cgit v1.2.1 From 5f131eaeb256d6f132ee25f86ad249d3d81d3826 Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Sun, 28 Feb 2016 17:26:16 -0500 Subject: WT-2431 Add documentation for join cursor statistics. --- src/docs/data-sources.dox | 12 +++++++++--- src/docs/statistics.dox | 9 +++++++++ src/include/wiredtiger.in | 2 +- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/docs/data-sources.dox b/src/docs/data-sources.dox index d09d1cbc1b8..7f1879e0ffe 100644 --- a/src/docs/data-sources.dox +++ b/src/docs/data-sources.dox @@ -38,7 +38,7 @@ cursor types that give access to data managed by WiredTiger: key=string\, value=string\,
see @ref metadata for details} @row{statistics:[\], - database or data source statistics cursor, + database, data source or join statistics cursor, key=int id\,
value=(string description\, string value\, uint64_t value)\,
@@ -106,7 +106,9 @@ WiredTiger database as well as statistics for individual data sources. The statistics are at two levels: per-database and per-individual data source. Database-wide statistics are retrieved with the \c "statistics:" URI; individual data source statistics are available by specifying -\c "statistics:". +\c "statistics:". Additionally, statistics about a +join cursor can be retrieved by specifying \c "statistics:join" and +supplying the join cursor as an argument in the SESSION::open_cursor call. The statistic key is an integer from the list of keys in @ref_single statistics_keys "Statistics Keys". Statistics cursors return @@ -127,7 +129,11 @@ The following is an example of printing statistics about a table: @snippet ex_stat.c statistics table function -Both examples can use a common display routine that iterates through the +The following is an example of printing statistics about a join cursor: + +@snippet ex_stat.c statistics join cursor function + +These three examples can use a common display routine that iterates through the statistics until the cursor returns the end of the list. @snippet ex_stat.c statistics display function diff --git a/src/docs/statistics.dox b/src/docs/statistics.dox index 453da34c51a..0a29e351e4e 100644 --- a/src/docs/statistics.dox +++ b/src/docs/statistics.dox @@ -79,6 +79,15 @@ or logged: @snippet ex_all.c Statistics clear configuration +The following example opens a statistics cursor on an open join cursor: + +@snippet ex_schema.c Statistics cursor join cursor + +The statistics gathered will be organized by reference cursors participating +in the join (see WT_SESSION::join); the uri of each reference cursor appears +as a prefix in the description field returned as a value by the statistics +cursor. + @section statistics_log Statistics logging WiredTiger will optionally log database statistics into a file when the diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index 0c314e0705f..0d328668e10 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -874,7 +874,7 @@ struct __wt_session { * updates). See @ref data_sources for more information. *
* @copydoc doc_cursor_types - * @param to_dup a cursor to duplicate + * @param to_dup a cursor to duplicate or gather statistics on * @configstart{WT_SESSION.open_cursor, see dist/api_data.py} * @config{append, append the value as a new record\, creating a new * record number key; valid only for cursors with record number keys., a -- cgit v1.2.1 From 5083d97a09a9e7dc525db7a4ed322983810fe522 Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Sun, 28 Feb 2016 17:26:53 -0500 Subject: WT-2431 Add examples for join cursor statistics. --- examples/c/ex_schema.c | 10 +++++- examples/c/ex_stat.c | 37 +++++++++++++++++++++- .../java/com/wiredtiger/examples/ex_schema.java | 9 +++++- examples/java/com/wiredtiger/examples/ex_stat.java | 32 ++++++++++++++++++- 4 files changed, 84 insertions(+), 4 deletions(-) diff --git a/examples/c/ex_schema.c b/examples/c/ex_schema.c index fdf02d12302..e7ceb6014d6 100644 --- a/examples/c/ex_schema.c +++ b/examples/c/ex_schema.c @@ -69,7 +69,7 @@ main(void) { POP_RECORD *p; WT_CONNECTION *conn; - WT_CURSOR *cursor, *cursor2, *join_cursor; + WT_CURSOR *cursor, *cursor2, *join_cursor, *stat_cursor; WT_SESSION *session; const char *country; uint64_t recno, population; @@ -354,6 +354,14 @@ main(void) country, year, population); } /*! [Join cursors] */ + + /*! [Statistics cursor join cursor] */ + ret = session->open_cursor(session, + "statistics:join", + join_cursor, NULL, &stat_cursor); + /*! [Statistics cursor join cursor] */ + + ret = stat_cursor->close(stat_cursor); ret = join_cursor->close(join_cursor); ret = cursor2->close(cursor2); ret = cursor->close(cursor); diff --git a/examples/c/ex_stat.c b/examples/c/ex_stat.c index 65402230eb8..6c5c15aacc6 100644 --- a/examples/c/ex_stat.c +++ b/examples/c/ex_stat.c @@ -39,6 +39,7 @@ int print_cursor(WT_CURSOR *); int print_database_stats(WT_SESSION *); int print_file_stats(WT_SESSION *); +int print_join_cursor_stats(WT_SESSION *); int print_overflow_pages(WT_SESSION *); int get_stat(WT_CURSOR *cursor, int stat_field, uint64_t *valuep); int print_derived_stats(WT_SESSION *); @@ -99,6 +100,37 @@ print_file_stats(WT_SESSION *session) } int +print_join_cursor_stats(WT_SESSION *session) +{ + WT_CURSOR *idx_cursor, *join_cursor, *stat_cursor; + int ret; + + ret = session->create( + session, "index:access:idx", "columns=(v)"); + ret = session->open_cursor( + session, "index:access:idx", NULL, NULL, &idx_cursor); + ret = idx_cursor->next(idx_cursor); + ret = session->open_cursor( + session, "join:table:access", NULL, NULL, &join_cursor); + ret = session->join(session, join_cursor, idx_cursor, "compare=gt"); + ret = join_cursor->next(join_cursor); + + /*! [statistics join cursor function] */ + if ((ret = session->open_cursor(session, + "statistics:join", join_cursor, NULL, &stat_cursor)) != 0) + return (ret); + + ret = print_cursor(stat_cursor); + ret = stat_cursor->close(stat_cursor); + /*! [statistics join cursor function] */ + + ret = join_cursor->close(join_cursor); + ret = idx_cursor->close(idx_cursor); + + return (ret); +} + +int print_overflow_pages(WT_SESSION *session) { /*! [statistics retrieve by key] */ @@ -204,7 +236,8 @@ main(void) ret = wiredtiger_open(home, NULL, "create,statistics=(all)", &conn); ret = conn->open_session(conn, NULL, NULL, &session); ret = session->create( - session, "table:access", "key_format=S,value_format=S"); + session, "table:access", + "key_format=S,value_format=S,columns=(k,v)"); ret = session->open_cursor( session, "table:access", NULL, NULL, &cursor); @@ -219,6 +252,8 @@ main(void) ret = print_file_stats(session); + ret = print_join_cursor_stats(session); + ret = print_overflow_pages(session); ret = print_derived_stats(session); diff --git a/examples/java/com/wiredtiger/examples/ex_schema.java b/examples/java/com/wiredtiger/examples/ex_schema.java index be1077ee2df..a4b46d6dfe8 100644 --- a/examples/java/com/wiredtiger/examples/ex_schema.java +++ b/examples/java/com/wiredtiger/examples/ex_schema.java @@ -76,7 +76,7 @@ public class ex_schema { throws WiredTigerException { Connection conn; - Cursor cursor, cursor2, join_cursor; + Cursor cursor, cursor2, join_cursor, stat_cursor; Session session; String country; long recno, population; @@ -368,6 +368,13 @@ public class ex_schema { ", population " + population); } /*! [Join cursors] */ + + /*! [Statistics cursor join cursor] */ + stat_cursor = session.open_cursor( + "statistics:join", join_cursor, null); + /*! [Statistics cursor join cursor] */ + + ret = stat_cursor.close(); ret = join_cursor.close(); ret = cursor2.close(); ret = cursor.close(); diff --git a/examples/java/com/wiredtiger/examples/ex_stat.java b/examples/java/com/wiredtiger/examples/ex_stat.java index b0b83a2d3b2..f8877a4620e 100644 --- a/examples/java/com/wiredtiger/examples/ex_stat.java +++ b/examples/java/com/wiredtiger/examples/ex_stat.java @@ -92,6 +92,33 @@ public class ex_stat { } int + print_join_cursor_stats(Session session) + throws WiredTigerException + { + Cursor idx_cursor, join_cursor, stat_cursor; + int ret; + + ret = session.create("index:access:idx", "columns=(v)"); + idx_cursor = session.open_cursor("index:access:idx", null, null); + ret = idx_cursor.next(); + join_cursor = session.open_cursor("join:table:access", null, null); + ret = session.join(join_cursor, idx_cursor, "compare=gt"); + ret = join_cursor.next(); + + /*! [statistics join cursor function] */ + stat_cursor = session.open_cursor("statistics:join", join_cursor, null); + + ret = print_cursor(stat_cursor); + ret = stat_cursor.close(); + /*! [statistics join cursor function] */ + + ret = join_cursor.close(); + ret = idx_cursor.close(); + + return (ret); + } + + int print_overflow_pages(Session session) throws WiredTigerException { @@ -220,7 +247,8 @@ public class ex_stat { conn = wiredtiger.open(home, "create,statistics=(all)"); session = conn.open_session(null); - ret = session.create("table:access", "key_format=S,value_format=S"); + ret = session.create("table:access", + "key_format=S,value_format=S,columns=(k,v)"); cursor = session.open_cursor("table:access", null, null); cursor.putKeyString("key"); @@ -234,6 +262,8 @@ public class ex_stat { ret = print_file_stats(session); + ret = print_join_cursor_stats(session); + ret = print_overflow_pages(session); ret = print_derived_stats(session); -- cgit v1.2.1 From 57a805d6927022b4dab8e8ec2ae55e3a961e83f2 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Mon, 29 Feb 2016 17:08:42 +1100 Subject: WT-2434 Fix a race between forced drops and sweep. During a forced drop, trees are marked dead but remain in cache temporarily. The sweep thread is aggressive about trying to discard dead trees from cache. If sweep found a tree marked dead before the drop operation had completed, they could race setting / clearing the "no eviction" flag on the tree, allowing eviction to run concurrent with sweep's discard. This could cause a segfault where pages are discarded underneath eviction. The fix is to move setting / clearing the "no eviction" flag inside the spinlock that protects against concurrent close operations. That way, even if sweep does find a tree before drop has finished closing it, sweep will block on the close lock before attempting to discard it. While in the area, tighten up other checks relating to the "no eviction" flag. --- src/conn/conn_dhandle.c | 20 +++++++++----------- src/conn/conn_sweep.c | 8 ++++---- src/evict/evict_lru.c | 22 ++++++++++++++++++---- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src/conn/conn_dhandle.c b/src/conn/conn_dhandle.c index f1b35571533..cd97ee6c2f5 100644 --- a/src/conn/conn_dhandle.c +++ b/src/conn/conn_dhandle.c @@ -134,14 +134,11 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) btree = S2BT(session); bm = btree->bm; dhandle = session->dhandle; - marked_dead = false; + evict_reset = marked_dead = false; if (!F_ISSET(dhandle, WT_DHANDLE_OPEN)) return (0); - /* Ensure that we aren't racing with the eviction server */ - WT_RET(__wt_evict_file_exclusive_on(session, &evict_reset)); - /* * If we don't already have the schema lock, make it an error to try * to acquire it. The problem is that we are holding an exclusive @@ -162,6 +159,9 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) */ __wt_spin_lock(session, &dhandle->close_lock); + /* Ensure that we aren't racing with the eviction server */ + WT_ERR(__wt_evict_file_exclusive_on(session, &evict_reset)); + /* * The close can fail if an update cannot be written, return the EBUSY * error to our caller for eventual retry. @@ -176,23 +176,22 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) WT_BTREE_SALVAGE | WT_BTREE_UPGRADE | WT_BTREE_VERIFY)) { if (force && (bm == NULL || !bm->is_mapped(bm, session))) { F_SET(session->dhandle, WT_DHANDLE_DEAD); + marked_dead = true; /* * Reset the tree's eviction priority, and the tree is * evictable by definition. */ __wt_evict_priority_clear(session); - F_CLR(S2BT(session), WT_BTREE_NO_EVICTION); - - marked_dead = true; } if (!marked_dead || final) WT_ERR(__wt_checkpoint_close(session, final)); } WT_TRET(__wt_btree_close(session)); + /* - * If we marked a handle as dead it will be closed by sweep, via + * If we marked a handle dead it will be closed by sweep, via * another call to sync and close. */ if (!marked_dead) { @@ -204,10 +203,9 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) F_ISSET(dhandle, WT_DHANDLE_DEAD) || !F_ISSET(dhandle, WT_DHANDLE_OPEN)); -err: __wt_spin_unlock(session, &dhandle->close_lock); - - if (evict_reset) +err: if (evict_reset) __wt_evict_file_exclusive_off(session); + __wt_spin_unlock(session, &dhandle->close_lock); if (no_schema_lock) F_CLR(session, WT_SESSION_NO_SCHEMA_LOCK); diff --git a/src/conn/conn_sweep.c b/src/conn/conn_sweep.c index 7628076e605..cc0aa5a1322 100644 --- a/src/conn/conn_sweep.c +++ b/src/conn/conn_sweep.c @@ -91,9 +91,9 @@ __sweep_expire_one(WT_SESSION_IMPL *session) goto err; /* - * Mark the handle as dead and close the underlying file - * handle. Closing the handle decrements the open file count, - * meaning the close loop won't overrun the configured minimum. + * Mark the handle dead and close the underlying file handle. + * Closing the handle decrements the open file count, meaning the close + * loop won't overrun the configured minimum. */ ret = __wt_conn_btree_sync_and_close(session, false, true); @@ -163,7 +163,7 @@ __sweep_discard_trees(WT_SESSION_IMPL *session, u_int *dead_handlesp) !F_ISSET(dhandle, WT_DHANDLE_DEAD)) continue; - /* If the handle is marked "dead", flush it from cache. */ + /* If the handle is marked dead, flush it from cache. */ WT_WITH_DHANDLE(session, dhandle, ret = __wt_conn_btree_sync_and_close(session, false, false)); diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index 35b12e2b685..884c08a02df 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -791,7 +791,7 @@ __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session, bool *evict_resetp) btree = S2BT(session); cache = S2C(session)->cache; - /* If the file wasn't evictable, there's no work to do. */ + /* If the file was never evictable, there's no work to do. */ if (F_ISSET(btree, WT_BTREE_NO_EVICTION)) return (0); @@ -800,9 +800,16 @@ __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session, bool *evict_resetp) * the file will be queued for eviction after this point. */ __wt_spin_lock(session, &cache->evict_walk_lock); - F_SET(btree, WT_BTREE_NO_EVICTION); + if (!F_ISSET(btree, WT_BTREE_NO_EVICTION)) { + F_SET(btree, WT_BTREE_NO_EVICTION); + *evict_resetp = true; + } __wt_spin_unlock(session, &cache->evict_walk_lock); + /* If some other operation has disabled eviction, we're done. */ + if (!*evict_resetp) + return (0); + /* Clear any existing LRU eviction walk for the file. */ WT_ERR(__evict_request_walk_clear(session)); @@ -826,10 +833,10 @@ __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session, bool *evict_resetp) while (btree->evict_busy > 0) __wt_yield(); - *evict_resetp = true; return (0); err: F_CLR(btree, WT_BTREE_NO_EVICTION); + *evict_resetp = false; return (ret); } @@ -844,7 +851,14 @@ __wt_evict_file_exclusive_off(WT_SESSION_IMPL *session) btree = S2BT(session); - WT_ASSERT(session, btree->evict_ref == NULL); + /* + * We have seen subtle bugs with multiple threads racing to turn + * eviction on/off. Make races more likely in diagnostic builds. + */ + WT_DIAGNOSTIC_YIELD; + + WT_ASSERT(session, btree->evict_ref == NULL && + F_ISSET(btree, WT_BTREE_NO_EVICTION)); F_CLR(btree, WT_BTREE_NO_EVICTION); } -- cgit v1.2.1 From 09dcf1586dc1b3bad2bc6926dbe57664a7ca11b5 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 29 Feb 2016 06:49:37 -0500 Subject: WT-2434 Fix a race between forced drops and sweep. Add a comment why we're clearing the tree's eviction flag inside the close lock; fix another commment because code moved. --- src/conn/conn_dhandle.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/conn/conn_dhandle.c b/src/conn/conn_dhandle.c index cd97ee6c2f5..2fab08e3afa 100644 --- a/src/conn/conn_dhandle.c +++ b/src/conn/conn_dhandle.c @@ -159,7 +159,11 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) */ __wt_spin_lock(session, &dhandle->close_lock); - /* Ensure that we aren't racing with the eviction server */ + /* + * Ensure we aren't racing with the eviction server; inside the close + * lock so threads won't race setting/clearing the tree's "no eviction" + * flag. + */ WT_ERR(__wt_evict_file_exclusive_on(session, &evict_reset)); /* @@ -178,10 +182,7 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) F_SET(session->dhandle, WT_DHANDLE_DEAD); marked_dead = true; - /* - * Reset the tree's eviction priority, and the tree is - * evictable by definition. - */ + /* Reset the tree's eviction priority (if any). */ __wt_evict_priority_clear(session); } if (!marked_dead || final) -- cgit v1.2.1 From 8ed209bca54c245412cdcdddfb44120e3389882c Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 29 Feb 2016 07:35:28 -0500 Subject: WT_2435: __wt_evict_file_exclusive_on/off cleanups WT_SESSION.rebalance turns on exclusive access to the file and so should turn it off again. I doubt this is a problem, but calls to on/off should be matched. --- src/btree/bt_rebalance.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/btree/bt_rebalance.c b/src/btree/bt_rebalance.c index 86360e83ddf..e1fb6227779 100644 --- a/src/btree/bt_rebalance.c +++ b/src/btree/bt_rebalance.c @@ -412,6 +412,7 @@ __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) WT_UNUSED(cfg); btree = S2BT(session); + evict_reset = false; /* * If the tree has never been written to disk, we're done, rebalance @@ -470,7 +471,10 @@ __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) btree->root.page = rs->root; rs->root = NULL; -err: /* Discard any leftover root page we created. */ +err: if (evict_reset) + __wt_evict_file_exclusive_off(session); + + /* Discard any leftover root page we created. */ if (rs->root != NULL) { __wt_page_modify_clear(session, rs->root); __wt_page_out(session, &rs->root); -- cgit v1.2.1 From 33991acfa1490ba7ccbe1ab8452858fe849926bd Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Mon, 29 Feb 2016 08:19:40 -0500 Subject: WT-2431 The schema example now needs statistics enabled for the connection. --- examples/c/ex_schema.c | 3 ++- examples/java/com/wiredtiger/examples/ex_schema.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/c/ex_schema.c b/examples/c/ex_schema.c index e7ceb6014d6..f3d3a1b9cb4 100644 --- a/examples/c/ex_schema.c +++ b/examples/c/ex_schema.c @@ -86,7 +86,8 @@ main(void) } else home = NULL; - if ((ret = wiredtiger_open(home, NULL, "create", &conn)) != 0) { + if ((ret = wiredtiger_open( + home, NULL, "create,statistics=(fast)", &conn)) != 0) { fprintf(stderr, "Error connecting to %s: %s\n", home, wiredtiger_strerror(ret)); return (ret); diff --git a/examples/java/com/wiredtiger/examples/ex_schema.java b/examples/java/com/wiredtiger/examples/ex_schema.java index a4b46d6dfe8..7cc26acb479 100644 --- a/examples/java/com/wiredtiger/examples/ex_schema.java +++ b/examples/java/com/wiredtiger/examples/ex_schema.java @@ -106,7 +106,7 @@ public class ex_schema { home = null; try { - conn = wiredtiger.open(home, "create"); + conn = wiredtiger.open(home, "create,statistics=(fast)"); session = conn.open_session(null); } catch (WiredTigerException wte) { System.err.println("WiredTigerException: " + wte); -- cgit v1.2.1 From 0a8aac89fbefb8e376210720d4163f5cde78a134 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 29 Feb 2016 08:55:36 -0500 Subject: WT-2345: __wt_evict_file_exclusive_on/off cleanups Use a per-file spinlock instead of a global cache lock to protect the per-file eviction disabled information, check to see if the lock even needs to be acquired in eviction before doing the work. --- src/btree/bt_handle.c | 4 +++- src/conn/conn_cache.c | 2 -- src/evict/evict_lru.c | 27 +++++++++++++-------------- src/include/btree.h | 9 +++++---- src/include/cache.h | 1 - 5 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/btree/bt_handle.c b/src/btree/bt_handle.c index 52152a2fcac..bf888bfa65f 100644 --- a/src/btree/bt_handle.c +++ b/src/btree/bt_handle.c @@ -164,6 +164,7 @@ __wt_btree_close(WT_SESSION_IMPL *session) /* Destroy locks. */ WT_TRET(__wt_rwlock_destroy(session, &btree->ovfl_lock)); + __wt_spin_destroy(session, &btree->evict_lock); __wt_spin_destroy(session, &btree->flush_lock); /* Free allocated memory. */ @@ -350,7 +351,8 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) /* Initialize locks. */ WT_RET(__wt_rwlock_alloc( session, &btree->ovfl_lock, "btree overflow lock")); - WT_RET(__wt_spin_init(session, &btree->flush_lock, "btree flush lock")); + WT_RET(__wt_spin_init(session, &btree->evict_lock, "btree evict")); + WT_RET(__wt_spin_init(session, &btree->flush_lock, "btree flush")); btree->checkpointing = WT_CKPT_OFF; /* Not checkpointing */ btree->modified = 0; /* Clean */ diff --git a/src/conn/conn_cache.c b/src/conn/conn_cache.c index 1831aad5895..2b2aa9a2e42 100644 --- a/src/conn/conn_cache.c +++ b/src/conn/conn_cache.c @@ -152,7 +152,6 @@ __wt_cache_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR(__wt_cond_alloc(session, "eviction waiters", false, &cache->evict_waiter_cond)); WT_ERR(__wt_spin_init(session, &cache->evict_lock, "cache eviction")); - WT_ERR(__wt_spin_init(session, &cache->evict_walk_lock, "cache walk")); /* Allocate the LRU eviction queue. */ cache->evict_slots = WT_EVICT_WALK_BASE + WT_EVICT_WALK_INCR; @@ -249,7 +248,6 @@ __wt_cache_destroy(WT_SESSION_IMPL *session) WT_TRET(__wt_cond_destroy(session, &cache->evict_cond)); WT_TRET(__wt_cond_destroy(session, &cache->evict_waiter_cond)); __wt_spin_destroy(session, &cache->evict_lock); - __wt_spin_destroy(session, &cache->evict_walk_lock); __wt_free(session, cache->evict_queue); __wt_free(session, conn->cache); diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index 884c08a02df..e65609ab780 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -799,12 +799,12 @@ __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session, bool *evict_resetp) * Hold the walk lock to set the "no eviction" flag: no new pages from * the file will be queued for eviction after this point. */ - __wt_spin_lock(session, &cache->evict_walk_lock); + __wt_spin_lock(session, &btree->evict_lock); if (!F_ISSET(btree, WT_BTREE_NO_EVICTION)) { F_SET(btree, WT_BTREE_NO_EVICTION); *evict_resetp = true; } - __wt_spin_unlock(session, &cache->evict_walk_lock); + __wt_spin_unlock(session, &btree->evict_lock); /* If some other operation has disabled eviction, we're done. */ if (!*evict_resetp) @@ -1127,23 +1127,22 @@ retry: while (slot < max_entries && ret == 0) { __wt_spin_unlock(session, &conn->dhandle_lock); dhandle_locked = false; - __wt_spin_lock(session, &cache->evict_walk_lock); - /* - * Re-check the "no eviction" flag -- it is used to enforce - * exclusive access when a handle is being closed. + * Re-check the "no eviction" flag, used to enforce exclusive + * access when a handle is being closed. If not set, remember + * the file to visit first, next loop. */ if (!F_ISSET(btree, WT_BTREE_NO_EVICTION)) { - /* Remember the file to visit first, next loop. */ - cache->evict_file_next = dhandle; - - WT_WITH_DHANDLE(session, dhandle, - ret = __evict_walk_file(session, &slot)); - WT_ASSERT(session, session->split_gen == 0); + __wt_spin_lock(session, &btree->evict_lock); + if (!F_ISSET(btree, WT_BTREE_NO_EVICTION)) { + cache->evict_file_next = dhandle; + WT_WITH_DHANDLE(session, dhandle, + ret = __evict_walk_file(session, &slot)); + WT_ASSERT(session, session->split_gen == 0); + } + __wt_spin_unlock(session, &btree->evict_lock); } - __wt_spin_unlock(session, &cache->evict_walk_lock); - /* * If we didn't find any candidates in the file, skip it next * time. diff --git a/src/include/btree.h b/src/include/btree.h index 703de0f2fc6..fd2204fd813 100644 --- a/src/include/btree.h +++ b/src/include/btree.h @@ -129,10 +129,11 @@ struct __wt_btree { uint64_t rec_max_txn; /* Maximum txn seen (clean trees) */ uint64_t write_gen; /* Write generation */ - WT_REF *evict_ref; /* Eviction thread's location */ - uint64_t evict_priority; /* Relative priority of cached pages */ - u_int evict_walk_period; /* Skip this many LRU walks */ - u_int evict_walk_skips; /* Number of walks skipped */ + WT_SPINLOCK evict_lock; /* Eviction lock */ + WT_REF *evict_ref; /* Eviction thread's location */ + uint64_t evict_priority; /* Relative priority of cached pages */ + u_int evict_walk_period; /* Skip this many LRU walks */ + u_int evict_walk_skips; /* Number of walks skipped */ volatile uint32_t evict_busy; /* Count of threads in eviction */ enum { diff --git a/src/include/cache.h b/src/include/cache.h index a3961d6043e..0bf98dc3af2 100644 --- a/src/include/cache.h +++ b/src/include/cache.h @@ -84,7 +84,6 @@ struct __wt_cache { */ WT_CONDVAR *evict_cond; /* Eviction server condition */ WT_SPINLOCK evict_lock; /* Eviction LRU queue */ - WT_SPINLOCK evict_walk_lock; /* Eviction walk location */ /* Condition signalled when the eviction server populates the queue */ WT_CONDVAR *evict_waiter_cond; -- cgit v1.2.1 From fcb8e77b0fc1602f90a4d7619461a6d40a88ea4a Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 29 Feb 2016 10:20:33 -0500 Subject: WT-2345: __wt_evict_file_exclusive_on/off cleanups Add a per-file count of the no-eviction requests, only set the flag when the count goes from 0 to 1, and only clear the flag when the count goes from 1 to 0. --- src/btree/bt_rebalance.c | 3 ++- src/btree/bt_sync.c | 6 ++--- src/conn/conn_dhandle.c | 20 ++++++-------- src/evict/evict_file.c | 6 ++--- src/evict/evict_lru.c | 70 ++++++++++++++++++++++++++++-------------------- src/include/btree.h | 1 + src/include/extern.h | 2 +- 7 files changed, 57 insertions(+), 51 deletions(-) diff --git a/src/btree/bt_rebalance.c b/src/btree/bt_rebalance.c index e1fb6227779..d94eb2ddd80 100644 --- a/src/btree/bt_rebalance.c +++ b/src/btree/bt_rebalance.c @@ -439,7 +439,8 @@ __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) * cache is the root page, and that cannot be evicted; however, this way * eviction ignores the tree entirely.) */ - WT_ERR(__wt_evict_file_exclusive_on(session, &evict_reset)); + WT_ERR(__wt_evict_file_exclusive_on(session)); + evict_reset = true; /* Recursively walk the tree. */ switch (rs->type) { diff --git a/src/btree/bt_sync.c b/src/btree/bt_sync.c index 5273f0ee2c3..5d0944bd092 100644 --- a/src/btree/bt_sync.c +++ b/src/btree/bt_sync.c @@ -25,7 +25,6 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) uint64_t internal_bytes, internal_pages, leaf_bytes, leaf_pages; uint64_t oldest_id, saved_snap_min; uint32_t flags; - bool evict_reset; btree = S2BT(session); walk = NULL; @@ -123,9 +122,8 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) */ WT_PUBLISH(btree->checkpointing, WT_CKPT_PREPARE); - WT_ERR(__wt_evict_file_exclusive_on(session, &evict_reset)); - if (evict_reset) - __wt_evict_file_exclusive_off(session); + WT_ERR(__wt_evict_file_exclusive_on(session)); + __wt_evict_file_exclusive_off(session); WT_PUBLISH(btree->checkpointing, WT_CKPT_RUNNING); diff --git a/src/conn/conn_dhandle.c b/src/conn/conn_dhandle.c index 2fab08e3afa..5019ab59fe3 100644 --- a/src/conn/conn_dhandle.c +++ b/src/conn/conn_dhandle.c @@ -129,16 +129,19 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) WT_BTREE *btree; WT_DATA_HANDLE *dhandle; WT_DECL_RET; - bool evict_reset, marked_dead, no_schema_lock; + bool marked_dead, no_schema_lock; btree = S2BT(session); bm = btree->bm; dhandle = session->dhandle; - evict_reset = marked_dead = false; + marked_dead = false; if (!F_ISSET(dhandle, WT_DHANDLE_OPEN)) return (0); + /* Turn off eviction. */ + WT_RET(__wt_evict_file_exclusive_on(session)); + /* * If we don't already have the schema lock, make it an error to try * to acquire it. The problem is that we are holding an exclusive @@ -159,13 +162,6 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) */ __wt_spin_lock(session, &dhandle->close_lock); - /* - * Ensure we aren't racing with the eviction server; inside the close - * lock so threads won't race setting/clearing the tree's "no eviction" - * flag. - */ - WT_ERR(__wt_evict_file_exclusive_on(session, &evict_reset)); - /* * The close can fail if an update cannot be written, return the EBUSY * error to our caller for eventual retry. @@ -204,13 +200,13 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) F_ISSET(dhandle, WT_DHANDLE_DEAD) || !F_ISSET(dhandle, WT_DHANDLE_OPEN)); -err: if (evict_reset) - __wt_evict_file_exclusive_off(session); - __wt_spin_unlock(session, &dhandle->close_lock); +err: __wt_spin_unlock(session, &dhandle->close_lock); if (no_schema_lock) F_CLR(session, WT_SESSION_NO_SCHEMA_LOCK); + __wt_evict_file_exclusive_off(session); + return (ret); } diff --git a/src/evict/evict_file.c b/src/evict/evict_file.c index 641864a8baa..ca98b1bd62a 100644 --- a/src/evict/evict_file.c +++ b/src/evict/evict_file.c @@ -18,13 +18,12 @@ __wt_evict_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) WT_DECL_RET; WT_PAGE *page; WT_REF *next_ref, *ref; - bool evict_reset; /* * We need exclusive access to the file -- disable ordinary eviction * and drain any blocks already queued. */ - WT_RET(__wt_evict_file_exclusive_on(session, &evict_reset)); + WT_RET(__wt_evict_file_exclusive_on(session)); /* Make sure the oldest transaction ID is up-to-date. */ __wt_txn_update_oldest(session, true); @@ -98,8 +97,7 @@ err: /* On error, clear any left-over tree walk. */ session, next_ref, WT_READ_NO_EVICT)); } - if (evict_reset) - __wt_evict_file_exclusive_off(session); + __wt_evict_file_exclusive_off(session); return (ret); } diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index e65609ab780..a061ff352bb 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -778,7 +778,7 @@ __evict_clear_all_walks(WT_SESSION_IMPL *session) * blocks queued for eviction. */ int -__wt_evict_file_exclusive_on(WT_SESSION_IMPL *session, bool *evict_resetp) +__wt_evict_file_exclusive_on(WT_SESSION_IMPL *session) { WT_BTREE *btree; WT_CACHE *cache; @@ -786,40 +786,39 @@ __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session, bool *evict_resetp) WT_EVICT_ENTRY *evict; u_int i, elem; - *evict_resetp = false; - btree = S2BT(session); cache = S2C(session)->cache; - /* If the file was never evictable, there's no work to do. */ - if (F_ISSET(btree, WT_BTREE_NO_EVICTION)) - return (0); - /* - * Hold the walk lock to set the "no eviction" flag: no new pages from - * the file will be queued for eviction after this point. + * Hold the walk lock to set the no-eviction flag. + * + * The no-eviction flag can be set permanently, in which case we never + * increment the no-eviction count. */ __wt_spin_lock(session, &btree->evict_lock); - if (!F_ISSET(btree, WT_BTREE_NO_EVICTION)) { - F_SET(btree, WT_BTREE_NO_EVICTION); - *evict_resetp = true; + if (F_ISSET(btree, WT_BTREE_NO_EVICTION)) { + if (btree->evict_disabled != 0) + ++btree->evict_disabled; + __wt_spin_unlock(session, &btree->evict_lock); + return (0); } - __wt_spin_unlock(session, &btree->evict_lock); + ++btree->evict_disabled; - /* If some other operation has disabled eviction, we're done. */ - if (!*evict_resetp) - return (0); + /* + * Ensure no new pages from the file will be queued for eviction after + * this point. + */ + F_SET(btree, WT_BTREE_NO_EVICTION); + WT_FULL_BARRIER(); /* Clear any existing LRU eviction walk for the file. */ WT_ERR(__evict_request_walk_clear(session)); - /* Hold the evict lock to remove any queued pages from this file. */ - __wt_spin_lock(session, &cache->evict_lock); - /* * The eviction candidate list might reference pages from the file, - * clear it. + * clear it. Hold the evict lock to remove queued pages from a file. */ + __wt_spin_lock(session, &cache->evict_lock); elem = cache->evict_max; for (i = 0, evict = cache->evict_queue; i < elem; i++, evict++) if (evict->btree == btree) @@ -833,10 +832,11 @@ __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session, bool *evict_resetp) while (btree->evict_busy > 0) __wt_yield(); - return (0); - -err: F_CLR(btree, WT_BTREE_NO_EVICTION); - *evict_resetp = false; + if (0) { +err: --btree->evict_disabled; + F_CLR(btree, WT_BTREE_NO_EVICTION); + } + __wt_spin_unlock(session, &btree->evict_lock); return (ret); } @@ -857,10 +857,17 @@ __wt_evict_file_exclusive_off(WT_SESSION_IMPL *session) */ WT_DIAGNOSTIC_YIELD; - WT_ASSERT(session, btree->evict_ref == NULL && - F_ISSET(btree, WT_BTREE_NO_EVICTION)); + WT_ASSERT(session, + btree->evict_ref == NULL && F_ISSET(btree, WT_BTREE_NO_EVICTION)); - F_CLR(btree, WT_BTREE_NO_EVICTION); + /* + * The no-eviction flag can be set permanently, in which case we never + * increment the no-eviction count. + */ + __wt_spin_lock(session, &btree->evict_lock); + if (btree->evict_disabled > 0 && --btree->evict_disabled == 0) + F_CLR(btree, WT_BTREE_NO_EVICTION); + __wt_spin_unlock(session, &btree->evict_lock); } /* @@ -1131,9 +1138,14 @@ retry: while (slot < max_entries && ret == 0) { * Re-check the "no eviction" flag, used to enforce exclusive * access when a handle is being closed. If not set, remember * the file to visit first, next loop. + * + * Only try to acquire the lock and simply continue if we fail; + * the lock is held while the thread turning off eviction clears + * the tree's current eviction point, and part of the process is + * waiting on this thread to acknowledge that action. */ - if (!F_ISSET(btree, WT_BTREE_NO_EVICTION)) { - __wt_spin_lock(session, &btree->evict_lock); + if (!F_ISSET(btree, WT_BTREE_NO_EVICTION) && + !__wt_spin_trylock(session, &btree->evict_lock)) { if (!F_ISSET(btree, WT_BTREE_NO_EVICTION)) { cache->evict_file_next = dhandle; WT_WITH_DHANDLE(session, dhandle, diff --git a/src/include/btree.h b/src/include/btree.h index fd2204fd813..2d8a9ad4607 100644 --- a/src/include/btree.h +++ b/src/include/btree.h @@ -134,6 +134,7 @@ struct __wt_btree { uint64_t evict_priority; /* Relative priority of cached pages */ u_int evict_walk_period; /* Skip this many LRU walks */ u_int evict_walk_skips; /* Number of walks skipped */ + u_int evict_disabled; /* Eviction disabled count */ volatile uint32_t evict_busy; /* Count of threads in eviction */ enum { diff --git a/src/include/extern.h b/src/include/extern.h index 55b0b8cd7ff..e5f3902a268 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -341,7 +341,7 @@ extern void __wt_evict_list_clear_page(WT_SESSION_IMPL *session, WT_REF *ref); extern int __wt_evict_server_wake(WT_SESSION_IMPL *session); extern int __wt_evict_create(WT_SESSION_IMPL *session); extern int __wt_evict_destroy(WT_SESSION_IMPL *session); -extern int __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session, bool *evict_resetp); +extern int __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session); extern void __wt_evict_file_exclusive_off(WT_SESSION_IMPL *session); extern int __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, u_int pct_full); extern void __wt_evict_priority_set(WT_SESSION_IMPL *session, uint64_t v); -- cgit v1.2.1 From 69cfa35548a6b0e971b4fab0704b830a095143f5 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 29 Feb 2016 10:23:13 -0500 Subject: WT-2345: __wt_evict_file_exclusive_on/off cleanups __evict_request_walk_clear is only called from one place, move it next to its caller, and rename it to match other similar functions. --- src/evict/evict_lru.c | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index a061ff352bb..0f31d4d7ae4 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -720,12 +720,32 @@ __evict_clear_walks(WT_SESSION_IMPL *session) } /* - * __evict_request_walk_clear -- + * __evict_clear_all_walks -- + * Clear the eviction walk points for all files a session is waiting on. + */ +static int +__evict_clear_all_walks(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + WT_DATA_HANDLE *dhandle; + WT_DECL_RET; + + conn = S2C(session); + + TAILQ_FOREACH(dhandle, &conn->dhqh, q) + if (WT_PREFIX_MATCH(dhandle->name, "file:")) + WT_WITH_DHANDLE(session, + dhandle, WT_TRET(__evict_clear_walk(session))); + return (ret); +} + +/* + * __evict_request_clear_walk -- * Request that the eviction server clear the tree's current eviction * point. */ static int -__evict_request_walk_clear(WT_SESSION_IMPL *session) +__evict_request_clear_walk(WT_SESSION_IMPL *session) { WT_BTREE *btree; WT_CACHE *cache; @@ -752,26 +772,6 @@ __evict_request_walk_clear(WT_SESSION_IMPL *session) return (ret); } -/* - * __evict_clear_all_walks -- - * Clear the eviction walk points for all files a session is waiting on. - */ -static int -__evict_clear_all_walks(WT_SESSION_IMPL *session) -{ - WT_CONNECTION_IMPL *conn; - WT_DATA_HANDLE *dhandle; - WT_DECL_RET; - - conn = S2C(session); - - TAILQ_FOREACH(dhandle, &conn->dhqh, q) - if (WT_PREFIX_MATCH(dhandle->name, "file:")) - WT_WITH_DHANDLE(session, - dhandle, WT_TRET(__evict_clear_walk(session))); - return (ret); -} - /* * __wt_evict_file_exclusive_on -- * Get exclusive eviction access to a file and discard any of the file's @@ -812,7 +812,7 @@ __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session) WT_FULL_BARRIER(); /* Clear any existing LRU eviction walk for the file. */ - WT_ERR(__evict_request_walk_clear(session)); + WT_ERR(__evict_request_clear_walk(session)); /* * The eviction candidate list might reference pages from the file, -- cgit v1.2.1 From 5a1f23d81421b63e3c56dd66c7389f963bb65d0c Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 29 Feb 2016 10:25:47 -0500 Subject: WT-2345: __wt_evict_file_exclusive_on/off cleanups When checking if application threads should help with eviction, don't ignore threads operating in files that can't be evicted at the moment, only ignore threads that are operating on cache-resident files. (They might be the same thing at the moment, but they're not the same thing in general.) Minor whitespace cleanup. --- src/include/cache.i | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/include/cache.i b/src/include/cache.i index ee13eee84c5..f36ce4ae021 100644 --- a/src/include/cache.i +++ b/src/include/cache.i @@ -119,12 +119,11 @@ __wt_session_can_wait(WT_SESSION_IMPL *session) return (0); /* - * LSM sets the no-eviction flag when holding the LSM tree lock, - * in that case, or when holding the schema lock, we don't want to - * highjack the thread for eviction. + * LSM sets the no-eviction flag when holding the LSM tree lock, in that + * case, or when holding the schema lock, we don't want to highjack the + * thread for eviction. */ - if (F_ISSET(session, - WT_SESSION_NO_EVICTION | WT_SESSION_LOCKED_SCHEMA)) + if (F_ISSET(session, WT_SESSION_NO_EVICTION | WT_SESSION_LOCKED_SCHEMA)) return (0); return (1); @@ -224,11 +223,11 @@ __wt_cache_eviction_check(WT_SESSION_IMPL *session, bool busy, bool *didworkp) return (0); /* - * Threads operating on trees that cannot be evicted are ignored, - * mostly because they're not contributing to the problem. + * Threads operating on cache-resident trees are ignored because they're + * not contributing to the problem. */ btree = S2BT_SAFE(session); - if (btree != NULL && F_ISSET(btree, WT_BTREE_NO_EVICTION)) + if (btree != NULL && F_ISSET(btree, WT_BTREE_IN_MEMORY)) return (0); /* Check if eviction is needed. */ -- cgit v1.2.1 From 1257c17932350321a798bf79d4efb864e5dad70d Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 29 Feb 2016 10:32:33 -0500 Subject: WT-2345: __wt_evict_file_exclusive_on/off cleanups Once the underlying btree handle is closed, don't turn eviction back on, that function operates on the underlying btree handle. --- src/conn/conn_dhandle.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/conn/conn_dhandle.c b/src/conn/conn_dhandle.c index 5019ab59fe3..07be66ecdb0 100644 --- a/src/conn/conn_dhandle.c +++ b/src/conn/conn_dhandle.c @@ -129,18 +129,19 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) WT_BTREE *btree; WT_DATA_HANDLE *dhandle; WT_DECL_RET; - bool marked_dead, no_schema_lock; + bool evict_reset, marked_dead, no_schema_lock; btree = S2BT(session); bm = btree->bm; dhandle = session->dhandle; - marked_dead = false; + evict_reset = marked_dead = false; if (!F_ISSET(dhandle, WT_DHANDLE_OPEN)) return (0); /* Turn off eviction. */ WT_RET(__wt_evict_file_exclusive_on(session)); + evict_reset = true; /* * If we don't already have the schema lock, make it an error to try @@ -185,11 +186,16 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) WT_ERR(__wt_checkpoint_close(session, final)); } + /* + * Once the underlying btree handle is closed, don't turn eviction back + * on, that function operates on the underlying btree handle. + */ WT_TRET(__wt_btree_close(session)); + evict_reset = false; /* - * If we marked a handle dead it will be closed by sweep, via - * another call to sync and close. + * If we marked a handle dead it will be closed by sweep, via another + * call to sync and close. */ if (!marked_dead) { F_CLR(dhandle, WT_DHANDLE_OPEN); @@ -205,7 +211,8 @@ err: __wt_spin_unlock(session, &dhandle->close_lock); if (no_schema_lock) F_CLR(session, WT_SESSION_NO_SCHEMA_LOCK); - __wt_evict_file_exclusive_off(session); + if (evict_reset) + __wt_evict_file_exclusive_off(session); return (ret); } -- cgit v1.2.1 From b4f20f40a25bcc8e51c12923b9660d44236d1eb6 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 29 Feb 2016 13:43:49 -0500 Subject: WT-2345: __wt_evict_file_exclusive_on/off cleanups When closing a btree handle, clear the eviction information. --- src/btree/bt_handle.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/btree/bt_handle.c b/src/btree/bt_handle.c index bf888bfa65f..04f55f61e47 100644 --- a/src/btree/bt_handle.c +++ b/src/btree/bt_handle.c @@ -147,6 +147,7 @@ __wt_btree_close(WT_SESSION_IMPL *session) btree = S2BT(session); + /* Close the block manager. */ if ((bm = btree->bm) != NULL) { /* Unload the checkpoint, unless it's a special command. */ if (!F_ISSET(btree, @@ -162,10 +163,7 @@ __wt_btree_close(WT_SESSION_IMPL *session) /* Close the Huffman tree. */ __wt_btree_huffman_close(session); - /* Destroy locks. */ WT_TRET(__wt_rwlock_destroy(session, &btree->ovfl_lock)); - __wt_spin_destroy(session, &btree->evict_lock); - __wt_spin_destroy(session, &btree->flush_lock); /* Free allocated memory. */ __wt_free(session, btree->key_format); @@ -182,6 +180,17 @@ __wt_btree_close(WT_SESSION_IMPL *session) btree->bulk_load_ok = false; + /* Reset eviction information. */ + __wt_spin_destroy(session, &btree->evict_lock); + btree->evict_ref = NULL; + btree->evict_walk_period = 0; + btree->evict_walk_skips = 0; + btree->evict_disabled = 0; + btree->evict_busy = 0; + F_CLR(btree, WT_BTREE_NO_EVICTION); + + __wt_spin_destroy(session, &btree->flush_lock); + F_CLR(btree, WT_BTREE_SPECIAL_FLAGS); return (ret); -- cgit v1.2.1 From 8a0ffcb3e4e59ee3822df6e559f24ea719e3bca1 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Tue, 1 Mar 2016 05:46:47 +0000 Subject: WT-2366 Extend wtperf to support changing record sizes in update ops Via a new update_delta setting in the thread group which takes either a signed number of "rand" as an argument. Defaults to 0 (disabled). There is also a new value_sz_max configuration option to limit how large values can grow. --- bench/wtperf/config.c | 43 +++++++++--- bench/wtperf/runners/update-grow-stress.wtperf | 15 ++++ bench/wtperf/wtperf.c | 95 ++++++++++++++++++++++---- bench/wtperf/wtperf.h | 1 + bench/wtperf/wtperf_opt.i | 6 +- src/docs/wtperf.dox | 9 ++- 6 files changed, 142 insertions(+), 27 deletions(-) create mode 100644 bench/wtperf/runners/update-grow-stress.wtperf diff --git a/bench/wtperf/config.c b/bench/wtperf/config.c index 2544bbd371c..fc122eaf987 100644 --- a/bench/wtperf/config.c +++ b/bench/wtperf/config.c @@ -241,10 +241,6 @@ config_threads(CONFIG *cfg, const char *config, size_t len) goto err; continue; } - if (STRING_MATCH("throttle", k.str, k.len)) { - workp->throttle = (uint64_t)v.val; - continue; - } if (STRING_MATCH("insert", k.str, k.len) || STRING_MATCH("inserts", k.str, k.len)) { if ((workp->insert = v.val) < 0) @@ -262,10 +258,8 @@ config_threads(CONFIG *cfg, const char *config, size_t len) goto err; continue; } - if (STRING_MATCH("update", k.str, k.len) || - STRING_MATCH("updates", k.str, k.len)) { - if ((workp->update = v.val) < 0) - goto err; + if (STRING_MATCH("throttle", k.str, k.len)) { + workp->throttle = (uint64_t)v.val; continue; } if (STRING_MATCH("truncate", k.str, k.len)) { @@ -290,6 +284,23 @@ config_threads(CONFIG *cfg, const char *config, size_t len) workp->truncate_count = (uint64_t)v.val; continue; } + if (STRING_MATCH("update", k.str, k.len) || + STRING_MATCH("updates", k.str, k.len)) { + if ((workp->update = v.val) < 0) + goto err; + continue; + } + if (STRING_MATCH("update_delta", k.str, k.len)) { + if (v.type == WT_CONFIG_ITEM_STRING || + v.type == WT_CONFIG_ITEM_ID) { + if (strncmp(v.str, "rand", 4) != 0) + goto err; + /* Special random value */ + workp->update_delta = INT64_MAX; + } else + workp->update_delta = v.val; + continue; + } goto err; } if (ret == WT_NOTFOUND) @@ -409,7 +420,12 @@ config_opt(CONFIG *cfg, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v) *(uint32_t *)valueloc = (uint32_t)v->val; break; case CONFIG_STRING_TYPE: - if (v->type != WT_CONFIG_ITEM_STRING) { + /* + * Configuration parsing uses string/ID to distinguish + * between quoted and unquoted values. + */ + if (v->type != WT_CONFIG_ITEM_STRING && + v->type != WT_CONFIG_ITEM_ID) { fprintf(stderr, "wtperf: Error: " "bad string value for \'%.*s=%.*s\'\n", (int)k->len, k->str, (int)v->len, v->str); @@ -438,7 +454,8 @@ config_opt(CONFIG *cfg, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v) STRING_MATCH("threads", k->str, k->len)) return (config_threads(cfg, v->str, v->len)); - if (v->type != WT_CONFIG_ITEM_STRING) { + if (v->type != WT_CONFIG_ITEM_STRING && + v->type != WT_CONFIG_ITEM_ID) { fprintf(stderr, "wtperf: Error: " "bad string value for \'%.*s=%.*s\'\n", (int)k->len, k->str, (int)v->len, v->str); @@ -672,6 +689,12 @@ config_sanity(CONFIG *cfg) return (EINVAL); } + if (cfg->value_sz_max < cfg->value_sz) { + fprintf(stderr, + "value_sz_max must greater than or equal to value_sz\n"); + return (EINVAL); + } + if (cfg->readonly && cfg->workload != NULL) for (i = 0, workp = cfg->workload; i < cfg->workload_cnt; ++i, ++workp) diff --git a/bench/wtperf/runners/update-grow-stress.wtperf b/bench/wtperf/runners/update-grow-stress.wtperf new file mode 100644 index 00000000000..4ec039af4b4 --- /dev/null +++ b/bench/wtperf/runners/update-grow-stress.wtperf @@ -0,0 +1,15 @@ +# wtperf options file: Grow the size of documents while there is cache +# pressure and appends are happening as well. +conn_config="cache_size=2GB,checkpoint=(wait=60)" +table_config="type=file,leaf_page_max=32k,leaf_value_max=128k,split_pct=90" +# The values are starting small, insert a lot so our database grows larger than +# cache quickly. +icount=20000000 +report_interval=5 +run_time=240 +populate_threads=1 +# Continue inserting new records. +threads=((count=1,inserts=1,throttle=1000),(count=4,update=1,update_delta=100)) +# Start with small values and let them grow slowly to large values. +value_sz=20 +value_sz_max=65536 diff --git a/bench/wtperf/wtperf.c b/bench/wtperf/wtperf.c index b6e10762d8c..3d6f662c762 100644 --- a/bench/wtperf/wtperf.c +++ b/bench/wtperf/wtperf.c @@ -87,6 +87,7 @@ static int start_threads(CONFIG *, WORKLOAD *, CONFIG_THREAD *, u_int, void *(*)(void *)); static int stop_threads(CONFIG *, u_int, CONFIG_THREAD *); static void *thread_run_wtperf(void *); +static void update_value_delta(CONFIG_THREAD *); static void *worker(void *); static uint64_t wtperf_rand(CONFIG_THREAD *); @@ -105,24 +106,91 @@ get_next_incr(CONFIG *cfg) return (__wt_atomic_add64(&cfg->insert_key, 1)); } +/* + * Each time this function is called we will overwrite the first and one + * other element in the value buffer. + */ static void randomize_value(CONFIG_THREAD *thread, char *value_buf) { uint8_t *vb; - uint32_t i; + uint32_t i, max_range, rand_val; + + /* + * Limit how much of the buffer we validate for length, this means + * that only threads that do growing updates will ever make changes to + * values outside of the initial value size, but that's a fair trade + * off for avoiding figuring out how long the value is more accurately + * in this performance sensitive function. + */ + if (thread->workload->update_delta == 0) + max_range = thread->cfg->value_sz; + else + max_range = thread->cfg->value_sz_max; + + /* + * Generate a single random value and re-use it. We generally only + * have small ranges in this function, so avoiding a bunch of calls + * is worthwhile. + */ + rand_val = __wt_random(&thread->rnd); + i = rand_val % (max_range - 1); /* - * Each time we're called overwrite value_buf[0] and one other - * randomly chosen byte (other than the trailing NUL). - * Make sure we don't write a NUL: keep the value the same length. + * Ensure we don't write past the end of a value when configured for + * randomly sized values. */ - i = __wt_random(&thread->rnd) % (thread->cfg->value_sz - 1); while (value_buf[i] == '\0' && i > 0) --i; - if (i > 0) { - vb = (uint8_t *)value_buf; - vb[0] = (__wt_random(&thread->rnd) % 255) + 1; - vb[i] = (__wt_random(&thread->rnd) % 255) + 1; + + vb = (uint8_t *)value_buf; + vb[0] = ((rand_val >> 8) % 255) + 1; + /* + * If i happened to be 0, we'll be re-writing the same value + * twice, but that doesn't matter. + */ + vb[i] = ((rand_val >> 16) % 255) + 1; +} + +/* + * Figure out and extend the size of the value string, used for growing + * updates. We know that the value to be updated is in the threads value + * scratch buffer. + */ +static inline void +update_value_delta(CONFIG_THREAD *thread) +{ + CONFIG *cfg; + char * value; + int64_t delta, len, new_len; + + cfg = thread->cfg; + value = thread->value_buf; + delta = thread->workload->update_delta; + len = (int64_t)strlen(value); + + if (delta == INT64_MAX) + delta = __wt_random(&thread->rnd) % + (cfg->value_sz_max - cfg->value_sz); + + /* Ensure we aren't changing across boundaries */ + if (delta > 0 && len + delta > cfg->value_sz_max) + delta = cfg->value_sz_max - len; + else if (delta < 0 && -delta > len) + delta = -(len - 1); + + /* Bail if there isn't anything to do */ + if (delta == 0) + return; + + if (delta < 0) { + value[len + delta] = '\0'; + } else { + /* Extend the value by the configured amount. */ + for (new_len = len; + new_len < cfg->value_sz_max && new_len - len < delta; + new_len++) + value[new_len] = 'a'; } } @@ -624,8 +692,10 @@ worker(void *arg) * Copy as much of the previous value as is * safe, and be sure to NUL-terminate. */ - strncpy(value_buf, value, cfg->value_sz); - value_buf[cfg->value_sz - 1] = '\0'; + strncpy(value_buf, + value, cfg->value_sz_max - 1); + if (thread->workload->update_delta != 0) + update_value_delta(thread); if (value_buf[0] == 'a') value_buf[0] = 'b'; else @@ -2374,7 +2444,8 @@ start_threads(CONFIG *cfg, * strings: trailing NUL is included in the size. */ thread->key_buf = dcalloc(cfg->key_sz, 1); - thread->value_buf = dcalloc(cfg->value_sz, 1); + thread->value_buf = dcalloc(cfg->value_sz_max, 1); + /* * Initialize and then toss in a bit of random values if needed. */ diff --git a/bench/wtperf/wtperf.h b/bench/wtperf/wtperf.h index 65ee4963276..6107eaf721d 100644 --- a/bench/wtperf/wtperf.h +++ b/bench/wtperf/wtperf.h @@ -94,6 +94,7 @@ typedef struct { int64_t truncate; /* Truncate ratio */ uint64_t truncate_pct; /* Truncate Percent */ uint64_t truncate_count; /* Truncate Count */ + int64_t update_delta; /* Value size change on update */ #define WORKER_INSERT 1 /* Insert */ #define WORKER_INSERT_RMW 2 /* Insert with read-modify-write */ diff --git a/bench/wtperf/wtperf_opt.i b/bench/wtperf/wtperf_opt.i index ecc1f216299..4fd1936b8b7 100644 --- a/bench/wtperf/wtperf_opt.i +++ b/bench/wtperf/wtperf_opt.i @@ -184,13 +184,15 @@ DEF_OPT_AS_STRING(threads, "", "workload configuration: each 'count' " "'threads=((count=2,reads=1)(count=8,reads=1,inserts=2,updates=1))' " "which would create 2 threads doing nothing but reads and 8 threads " "each doing 50% inserts and 25% reads and updates. Allowed configuration " - "values are 'count', 'throttle', 'reads', 'inserts', 'updates', 'truncate'," - " 'truncate_pct' and 'truncate_count'. There are " + "values are 'count', 'throttle', 'update_delta', 'reads', 'inserts', " + "'updates', 'truncate', 'truncate_pct' and 'truncate_count'. There are " "also behavior modifiers, supported modifiers are 'ops_per_txn'") DEF_OPT_AS_CONFIG_STRING(transaction_config, "", "transaction configuration string, relevant when populate_opts_per_txn " "is nonzero") DEF_OPT_AS_STRING(table_name, "test", "table name") +DEF_OPT_AS_UINT32(value_sz_max, 1000, + "maximum value size when delta updates are present. Default disabled") DEF_OPT_AS_UINT32(value_sz, 100, "value size") DEF_OPT_AS_UINT32(verbose, 1, "verbosity") DEF_OPT_AS_UINT32(warmup, 0, diff --git a/src/docs/wtperf.dox b/src/docs/wtperf.dox index 6f3d2f87ee0..59176f7909a 100644 --- a/src/docs/wtperf.dox +++ b/src/docs/wtperf.dox @@ -251,14 +251,17 @@ threads configuration might be 'threads=((count=2,reads=1)(count=8,reads=1,inserts=2,updates=1))' which would create 2 threads doing nothing but reads and 8 threads each doing 50% inserts and 25% reads and updates. Allowed -configuration values are 'count', 'throttle', 'reads', 'inserts', -'updates', 'truncate', 'truncate_pct' and 'truncate_count'. There are -also behavior modifiers, supported modifiers are 'ops_per_txn' +configuration values are 'count', 'throttle', 'update_delta', 'reads', +'inserts', 'updates', 'truncate', 'truncate_pct' and 'truncate_count'. +There are also behavior modifiers, supported modifiers are +'ops_per_txn' @par transaction_config (string, default=) transaction configuration string, relevant when populate_opts_per_txn is nonzero @par table_name (string, default=test) table name +@par value_sz_max (unsigned int, default=1000) +maximum value size when delta updates are present. Default disabled @par value_sz (unsigned int, default=100) value size @par verbose (unsigned int, default=1) -- cgit v1.2.1 From e750ee98f2851846e19f9e754b1fff3a1a8e2407 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Tue, 1 Mar 2016 06:34:53 +0000 Subject: WT-2438 Add size identifier to statistics based on timeseries classification. --- dist/stat_data.py | 61 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/dist/stat_data.py b/dist/stat_data.py index be4bacaece7..37238b8525d 100644 --- a/dist/stat_data.py +++ b/dist/stat_data.py @@ -12,6 +12,7 @@ # max_aggregate Take the maximum value when aggregating statistics # no_clear Value not cleared when statistics cleared # no_scale Don't scale value per second in the logging tool script +# size Used by timeseries tool, indicates value is a byte count # # The no_clear and no_scale flags are normally always set together (values that # are maintained over time are normally not scaled per second). @@ -138,9 +139,9 @@ connection_stats = [ ########################################## # Block manager statistics ########################################## - BlockStat('block_byte_map_read', 'mapped bytes read'), - BlockStat('block_byte_read', 'bytes read'), - BlockStat('block_byte_write', 'bytes written'), + BlockStat('block_byte_map_read', 'mapped bytes read', 'size'), + BlockStat('block_byte_read', 'bytes read', 'size'), + BlockStat('block_byte_write', 'bytes written', 'size'), BlockStat('block_map_read', 'mapped blocks read'), BlockStat('block_preload', 'blocks pre-loaded'), BlockStat('block_read', 'blocks read'), @@ -149,14 +150,14 @@ connection_stats = [ ########################################## # Cache and eviction statistics ########################################## - CacheStat('cache_bytes_dirty', 'tracked dirty bytes in the cache', 'no_clear,no_scale'), - CacheStat('cache_bytes_internal', 'tracked bytes belonging to internal pages in the cache', 'no_clear,no_scale'), - CacheStat('cache_bytes_inuse', 'bytes currently in the cache', 'no_clear,no_scale'), - CacheStat('cache_bytes_leaf', 'tracked bytes belonging to leaf pages in the cache', 'no_clear,no_scale'), - CacheStat('cache_bytes_max', 'maximum bytes configured', 'no_clear,no_scale'), - CacheStat('cache_bytes_overflow', 'tracked bytes belonging to overflow pages in the cache', 'no_clear,no_scale'), - CacheStat('cache_bytes_read', 'bytes read into cache'), - CacheStat('cache_bytes_write', 'bytes written from cache'), + CacheStat('cache_bytes_dirty', 'tracked dirty bytes in the cache', 'no_clear,no_scale,size'), + CacheStat('cache_bytes_internal', 'tracked bytes belonging to internal pages in the cache', 'no_clear,no_scale,size'), + CacheStat('cache_bytes_inuse', 'bytes currently in the cache', 'no_clear,no_scale,size'), + CacheStat('cache_bytes_leaf', 'tracked bytes belonging to leaf pages in the cache', 'no_clear,no_scale,size'), + CacheStat('cache_bytes_max', 'maximum bytes configured', 'no_clear,no_scale,size'), + CacheStat('cache_bytes_overflow', 'tracked bytes belonging to overflow pages in the cache', 'no_clear,no_scale,size'), + CacheStat('cache_bytes_read', 'bytes read into cache', 'size'), + CacheStat('cache_bytes_write', 'bytes written from cache', 'size'), CacheStat('cache_eviction_aggressive_set', 'eviction currently operating in aggressive mode', 'no_clear,no_scale'), CacheStat('cache_eviction_app', 'pages evicted by application threads'), CacheStat('cache_eviction_checkpoint', 'checkpoint blocked page eviction'), @@ -169,7 +170,7 @@ connection_stats = [ CacheStat('cache_eviction_force_fail', 'failed eviction of pages that exceeded the in-memory maximum'), CacheStat('cache_eviction_hazard', 'hazard pointer blocked page eviction'), CacheStat('cache_eviction_internal', 'internal pages evicted'), - CacheStat('cache_eviction_maximum_page_size', 'maximum page size at eviction', 'no_clear,no_scale'), + CacheStat('cache_eviction_maximum_page_size', 'maximum page size at eviction', 'no_clear,no_scale,size'), CacheStat('cache_eviction_queue_empty', 'eviction server candidate queue empty when topping up'), CacheStat('cache_eviction_queue_not_empty', 'eviction server candidate queue not empty when topping up'), CacheStat('cache_eviction_server_evicting', 'eviction server evicting pages'), @@ -207,17 +208,17 @@ connection_stats = [ ########################################## # Logging statistics ########################################## - LogStat('log_buffer_size', 'total log buffer size', 'no_clear,no_scale'), - LogStat('log_bytes_payload', 'log bytes of payload data'), - LogStat('log_bytes_written', 'log bytes written'), + LogStat('log_buffer_size', 'total log buffer size', 'no_clear,no_scale,size'), + LogStat('log_bytes_payload', 'log bytes of payload data', 'size'), + LogStat('log_bytes_written', 'log bytes written', 'size'), LogStat('log_close_yields', 'yields waiting for previous log file close'), - LogStat('log_compress_len', 'total size of compressed records'), - LogStat('log_compress_mem', 'total in-memory size of compressed records'), + LogStat('log_compress_len', 'total size of compressed records', 'size'), + LogStat('log_compress_mem', 'total in-memory size of compressed records', 'size'), LogStat('log_compress_small', 'log records too small to compress'), LogStat('log_compress_write_fails', 'log records not compressed'), LogStat('log_compress_writes', 'log records compressed'), LogStat('log_flush', 'log flush operations'), - LogStat('log_max_filesize', 'maximum log file size', 'no_clear,no_scale'), + LogStat('log_max_filesize', 'maximum log file size', 'no_clear,no_scale,size'), LogStat('log_prealloc_files', 'pre-allocated log files prepared'), LogStat('log_prealloc_max', 'number of pre-allocated log files to create', 'no_clear,no_scale'), LogStat('log_prealloc_missed', 'pre-allocated log files not ready and missed'), @@ -228,7 +229,7 @@ connection_stats = [ LogStat('log_scans', 'log scan operations'), LogStat('log_slot_closes', 'consolidated slot closures'), LogStat('log_slot_coalesced', 'written slots coalesced'), - LogStat('log_slot_consolidated', 'logging bytes consolidated'), + LogStat('log_slot_consolidated', 'logging bytes consolidated', 'size'), LogStat('log_slot_joins', 'consolidated slot joins'), LogStat('log_slot_races', 'consolidated slot join races'), LogStat('log_slot_switch_busy', 'busy returns attempting to switch slots'), @@ -247,7 +248,7 @@ connection_stats = [ RecStat('rec_page_delete_fast', 'fast-path pages deleted'), RecStat('rec_pages', 'page reconciliation calls'), RecStat('rec_pages_eviction', 'page reconciliation calls for eviction'), - RecStat('rec_split_stashed_bytes', 'split bytes currently awaiting free', 'no_clear,no_scale'), + RecStat('rec_split_stashed_bytes', 'split bytes currently awaiting free', 'no_clear,no_scale,size'), RecStat('rec_split_stashed_objects', 'split objects currently awaiting free', 'no_clear,no_scale'), ########################################## @@ -334,18 +335,18 @@ dsrc_stats = [ CursorStat('cursor_create', 'create calls'), CursorStat('cursor_insert', 'insert calls'), CursorStat('cursor_insert_bulk', 'bulk-loaded cursor-insert calls'), - CursorStat('cursor_insert_bytes', 'cursor-insert key and value bytes inserted'), + CursorStat('cursor_insert_bytes', 'cursor-insert key and value bytes inserted', 'size'), CursorStat('cursor_next', 'next calls'), CursorStat('cursor_prev', 'prev calls'), CursorStat('cursor_remove', 'remove calls'), - CursorStat('cursor_remove_bytes', 'cursor-remove key bytes removed'), + CursorStat('cursor_remove_bytes', 'cursor-remove key bytes removed', 'size'), CursorStat('cursor_reset', 'reset calls'), CursorStat('cursor_restart', 'restarted searches'), CursorStat('cursor_search', 'search calls'), CursorStat('cursor_search_near', 'search near calls'), CursorStat('cursor_truncate', 'truncate calls'), CursorStat('cursor_update', 'update calls'), - CursorStat('cursor_update_bytes', 'cursor-update value bytes updated'), + CursorStat('cursor_update_bytes', 'cursor-update value bytes updated', 'size'), ########################################## # Btree statistics @@ -390,20 +391,20 @@ dsrc_stats = [ ########################################## BlockStat('allocation_size', 'file allocation unit size', 'max_aggregate,no_scale'), BlockStat('block_alloc', 'blocks allocated'), - BlockStat('block_checkpoint_size', 'checkpoint size', 'no_scale'), + BlockStat('block_checkpoint_size', 'checkpoint size', 'no_scale,size'), BlockStat('block_extension', 'allocations requiring file extension'), BlockStat('block_free', 'blocks freed'), BlockStat('block_magic', 'file magic number', 'max_aggregate,no_scale'), BlockStat('block_major', 'file major version number', 'max_aggregate,no_scale'), BlockStat('block_minor', 'minor version number', 'max_aggregate,no_scale'), - BlockStat('block_reuse_bytes', 'file bytes available for reuse'), - BlockStat('block_size', 'file size in bytes', 'no_scale'), + BlockStat('block_reuse_bytes', 'file bytes available for reuse', 'size'), + BlockStat('block_size', 'file size in bytes', 'no_scale,size'), ########################################## # Cache and eviction statistics ########################################## - CacheStat('cache_bytes_read', 'bytes read into cache'), - CacheStat('cache_bytes_write', 'bytes written from cache'), + CacheStat('cache_bytes_read', 'bytes read into cache', 'size'), + CacheStat('cache_bytes_write', 'bytes written from cache', 'size'), CacheStat('cache_eviction_checkpoint', 'checkpoint blocked page eviction'), CacheStat('cache_eviction_clean', 'unmodified pages evicted'), CacheStat('cache_eviction_deepen', 'page split during eviction deepened the tree'), @@ -449,8 +450,8 @@ dsrc_stats = [ RecStat('rec_page_match', 'page checksum matches'), RecStat('rec_pages', 'page reconciliation calls'), RecStat('rec_pages_eviction', 'page reconciliation calls for eviction'), - RecStat('rec_prefix_compression', 'leaf page key bytes discarded using prefix compression'), - RecStat('rec_suffix_compression', 'internal page key bytes discarded using suffix compression'), + RecStat('rec_prefix_compression', 'leaf page key bytes discarded using prefix compression', 'size'), + RecStat('rec_suffix_compression', 'internal page key bytes discarded using suffix compression', 'size'), ########################################## # Transaction statistics -- cgit v1.2.1 From 89b39adc1e63a5ee6a1aeaadeea5fa5c16356245 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Tue, 1 Mar 2016 06:38:02 +0000 Subject: WT-2438 Add size identifier to other statistics as required. --- dist/stat_data.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dist/stat_data.py b/dist/stat_data.py index 37238b8525d..6d91a87a13e 100644 --- a/dist/stat_data.py +++ b/dist/stat_data.py @@ -359,13 +359,13 @@ dsrc_stats = [ BtreeStat('btree_column_variable', 'column-store variable-size leaf pages', 'no_scale'), BtreeStat('btree_compact_rewrite', 'pages rewritten by compaction'), BtreeStat('btree_entries', 'number of key/value pairs', 'no_scale'), - BtreeStat('btree_fixed_len', 'fixed-record size', 'max_aggregate,no_scale'), + BtreeStat('btree_fixed_len', 'fixed-record size', 'max_aggregate,no_scale,size'), BtreeStat('btree_maximum_depth', 'maximum tree depth', 'max_aggregate,no_scale'), - BtreeStat('btree_maxintlkey', 'maximum internal page key size', 'max_aggregate,no_scale'), - BtreeStat('btree_maxintlpage', 'maximum internal page size', 'max_aggregate,no_scale'), - BtreeStat('btree_maxleafkey', 'maximum leaf page key size', 'max_aggregate,no_scale'), - BtreeStat('btree_maxleafpage', 'maximum leaf page size', 'max_aggregate,no_scale'), - BtreeStat('btree_maxleafvalue', 'maximum leaf page value size', 'max_aggregate,no_scale'), + BtreeStat('btree_maxintlkey', 'maximum internal page key size', 'max_aggregate,no_scale,size'), + BtreeStat('btree_maxintlpage', 'maximum internal page size', 'max_aggregate,no_scale,size'), + BtreeStat('btree_maxleafkey', 'maximum leaf page key size', 'max_aggregate,no_scale,size'), + BtreeStat('btree_maxleafpage', 'maximum leaf page size', 'max_aggregate,no_scale,size'), + BtreeStat('btree_maxleafvalue', 'maximum leaf page value size', 'max_aggregate,no_scale,size'), BtreeStat('btree_overflow', 'overflow pages', 'no_scale'), BtreeStat('btree_row_internal', 'row-store internal pages', 'no_scale'), BtreeStat('btree_row_leaf', 'row-store leaf pages', 'no_scale'), @@ -379,7 +379,7 @@ dsrc_stats = [ LSMStat('bloom_miss', 'bloom filter misses'), LSMStat('bloom_page_evict', 'bloom filter pages evicted from cache'), LSMStat('bloom_page_read', 'bloom filter pages read into cache'), - LSMStat('bloom_size', 'total size of bloom filters', 'no_scale'), + LSMStat('bloom_size', 'total size of bloom filters', 'no_scale,size'), LSMStat('lsm_checkpoint_throttle', 'sleep for LSM checkpoint throttle'), LSMStat('lsm_chunk_count', 'chunks in the LSM tree', 'no_scale'), LSMStat('lsm_generation_max', 'highest merge generation in the LSM tree', 'max_aggregate,no_scale'), @@ -389,7 +389,7 @@ dsrc_stats = [ ########################################## # Block manager statistics ########################################## - BlockStat('allocation_size', 'file allocation unit size', 'max_aggregate,no_scale'), + BlockStat('allocation_size', 'file allocation unit size', 'max_aggregate,no_scale,size'), BlockStat('block_alloc', 'blocks allocated'), BlockStat('block_checkpoint_size', 'checkpoint size', 'no_scale,size'), BlockStat('block_extension', 'allocations requiring file extension'), -- cgit v1.2.1 From 07209b0e327445d022d2df674c31ec75ec3d9d12 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 1 Mar 2016 10:04:18 -0500 Subject: Revert "WT-2345: __wt_evict_file_exclusive_on/off cleanups" This reverts commit 1257c17932350321a798bf79d4efb864e5dad70d. --- src/conn/conn_dhandle.c | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/conn/conn_dhandle.c b/src/conn/conn_dhandle.c index 07be66ecdb0..5019ab59fe3 100644 --- a/src/conn/conn_dhandle.c +++ b/src/conn/conn_dhandle.c @@ -129,19 +129,18 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) WT_BTREE *btree; WT_DATA_HANDLE *dhandle; WT_DECL_RET; - bool evict_reset, marked_dead, no_schema_lock; + bool marked_dead, no_schema_lock; btree = S2BT(session); bm = btree->bm; dhandle = session->dhandle; - evict_reset = marked_dead = false; + marked_dead = false; if (!F_ISSET(dhandle, WT_DHANDLE_OPEN)) return (0); /* Turn off eviction. */ WT_RET(__wt_evict_file_exclusive_on(session)); - evict_reset = true; /* * If we don't already have the schema lock, make it an error to try @@ -186,16 +185,11 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) WT_ERR(__wt_checkpoint_close(session, final)); } - /* - * Once the underlying btree handle is closed, don't turn eviction back - * on, that function operates on the underlying btree handle. - */ WT_TRET(__wt_btree_close(session)); - evict_reset = false; /* - * If we marked a handle dead it will be closed by sweep, via another - * call to sync and close. + * If we marked a handle dead it will be closed by sweep, via + * another call to sync and close. */ if (!marked_dead) { F_CLR(dhandle, WT_DHANDLE_OPEN); @@ -211,8 +205,7 @@ err: __wt_spin_unlock(session, &dhandle->close_lock); if (no_schema_lock) F_CLR(session, WT_SESSION_NO_SCHEMA_LOCK); - if (evict_reset) - __wt_evict_file_exclusive_off(session); + __wt_evict_file_exclusive_off(session); return (ret); } -- cgit v1.2.1 From b96befb2529e870b41d8e2e2edda7788acab463b Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 1 Mar 2016 10:04:28 -0500 Subject: Revert "WT-2345: __wt_evict_file_exclusive_on/off cleanups" This reverts commit b4f20f40a25bcc8e51c12923b9660d44236d1eb6. --- src/btree/bt_handle.c | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/btree/bt_handle.c b/src/btree/bt_handle.c index 04f55f61e47..bf888bfa65f 100644 --- a/src/btree/bt_handle.c +++ b/src/btree/bt_handle.c @@ -147,7 +147,6 @@ __wt_btree_close(WT_SESSION_IMPL *session) btree = S2BT(session); - /* Close the block manager. */ if ((bm = btree->bm) != NULL) { /* Unload the checkpoint, unless it's a special command. */ if (!F_ISSET(btree, @@ -163,7 +162,10 @@ __wt_btree_close(WT_SESSION_IMPL *session) /* Close the Huffman tree. */ __wt_btree_huffman_close(session); + /* Destroy locks. */ WT_TRET(__wt_rwlock_destroy(session, &btree->ovfl_lock)); + __wt_spin_destroy(session, &btree->evict_lock); + __wt_spin_destroy(session, &btree->flush_lock); /* Free allocated memory. */ __wt_free(session, btree->key_format); @@ -180,17 +182,6 @@ __wt_btree_close(WT_SESSION_IMPL *session) btree->bulk_load_ok = false; - /* Reset eviction information. */ - __wt_spin_destroy(session, &btree->evict_lock); - btree->evict_ref = NULL; - btree->evict_walk_period = 0; - btree->evict_walk_skips = 0; - btree->evict_disabled = 0; - btree->evict_busy = 0; - F_CLR(btree, WT_BTREE_NO_EVICTION); - - __wt_spin_destroy(session, &btree->flush_lock); - F_CLR(btree, WT_BTREE_SPECIAL_FLAGS); return (ret); -- cgit v1.2.1 From 5229ef8d6b2e42b7a19e701b58e549f1b8b9bafd Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 1 Mar 2016 12:50:15 -0500 Subject: WT-2437 Adjust expected error messages to support Windows. Look for WT-specific portion of the message or detect Posix/Windows. --- test/suite/test_backup05.py | 8 -------- test/suite/test_checkpoint01.py | 2 +- test/suite/test_cursor06.py | 2 +- test/suite/test_cursor_random.py | 2 +- test/suite/test_join01.py | 2 +- test/suite/test_readonly02.py | 4 +++- test/suite/test_readonly03.py | 2 +- 7 files changed, 8 insertions(+), 14 deletions(-) diff --git a/test/suite/test_backup05.py b/test/suite/test_backup05.py index 8ffeb6752df..991a9f71b19 100644 --- a/test/suite/test_backup05.py +++ b/test/suite/test_backup05.py @@ -44,14 +44,6 @@ class test_backup05(wttest.WiredTigerTestCase, suite_subprocess): create_params = 'key_format=i,value_format=i' freq = 5 - def copy_windows(self, olddir, newdir): - os.mkdir(newdir) - for fname in os.listdir(olddir): - fullname = os.path.join(olddir, fname) - # Skip lock file on Windows since it is locked - if os.path.isfile(fullname) and "WiredTiger.lock" not in fullname: - shutil.copy(fullname, newdir) - def check_manual_backup(self, i, olddir, newdir): ''' Simulate a manual backup from olddir and restart in newdir. ''' self.session.checkpoint() diff --git a/test/suite/test_checkpoint01.py b/test/suite/test_checkpoint01.py index 36f1ef733a4..9955944f73d 100644 --- a/test/suite/test_checkpoint01.py +++ b/test/suite/test_checkpoint01.py @@ -265,7 +265,7 @@ class test_checkpoint_cursor_update(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt") cursor.set_key(key_populate(cursor, 10)) cursor.set_value("XXX") - msg = "/not supported/" + msg = "/Unsupported cursor/" self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cursor.insert(), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, diff --git a/test/suite/test_cursor06.py b/test/suite/test_cursor06.py index d702f97c5dd..5545c862dd7 100644 --- a/test/suite/test_cursor06.py +++ b/test/suite/test_cursor06.py @@ -89,7 +89,7 @@ class test_cursor06(wttest.WiredTigerTestCase): self.session.drop(uri, "force") self.populate(uri) cursor = self.session.open_cursor(uri, None, open_config) - msg = '/not supported/' + msg = '/Unsupported cursor/' if open_config == "readonly=1": self.set_kv(cursor) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, diff --git a/test/suite/test_cursor_random.py b/test/suite/test_cursor_random.py index cd91a925b0c..16ce5cae685 100644 --- a/test/suite/test_cursor_random.py +++ b/test/suite/test_cursor_random.py @@ -51,7 +51,7 @@ class test_cursor_random(wttest.WiredTigerTestCase): uri = self.type self.session.create(uri, 'key_format=S,value_format=S') cursor = self.session.open_cursor(uri, None, self.config) - msg = "/not supported/" + msg = "/Unsupported cursor/" self.assertRaisesWithMessage( wiredtiger.WiredTigerError, lambda: cursor.compare(cursor), msg) self.assertRaisesWithMessage( diff --git a/test/suite/test_join01.py b/test/suite/test_join01.py index 7630706379c..539a3a3ae57 100644 --- a/test/suite/test_join01.py +++ b/test/suite/test_join01.py @@ -341,7 +341,7 @@ class test_join01(wttest.WiredTigerTestCase): '/index cursor is being used in a join/') # Only a small number of operations allowed on a join cursor - msg = "/not supported/" + msg = "/Unsupported cursor/" self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: jc.search(), msg) diff --git a/test/suite/test_readonly02.py b/test/suite/test_readonly02.py index 309c14752ac..0df5465642d 100644 --- a/test/suite/test_readonly02.py +++ b/test/suite/test_readonly02.py @@ -66,8 +66,10 @@ class test_readonly02(wttest.WiredTigerTestCase, suite_subprocess): if self.create: # 1. setting readonly on a new database directory # Setting readonly prevents creation so we should see an - # ENOENT error because the lock file does not exist. + # error because the lock file does not exist. msg = '/No such file/' + if os.name != 'posix': + msg = '/cannot find the file/' os.mkdir(rdonlydir) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.wiredtiger_open( diff --git a/test/suite/test_readonly03.py b/test/suite/test_readonly03.py index 981a21d51ac..d9930e8f553 100644 --- a/test/suite/test_readonly03.py +++ b/test/suite/test_readonly03.py @@ -68,7 +68,7 @@ class test_readonly03(wttest.WiredTigerTestCase, suite_subprocess): # Now close and reopen. Note that the connection function # above will reopen it readonly. self.reopen_conn() - msg = '/not supported/' + msg = '/Unsupported/' c = self.session.open_cursor(self.uri, None, None) for op in self.cursor_ops: c.set_key(1) -- cgit v1.2.1 From c8f122e9e7010d06c0e3b84927bb3750ea7783c8 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 1 Mar 2016 14:02:10 -0500 Subject: WT-2437 Skip the chmod'ed portion of the test on Windows. --- test/suite/test_readonly01.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/suite/test_readonly01.py b/test/suite/test_readonly01.py index 86604eb6bfb..0a4bc219700 100644 --- a/test/suite/test_readonly01.py +++ b/test/suite/test_readonly01.py @@ -96,7 +96,10 @@ class test_readonly01(wttest.WiredTigerTestCase, suite_subprocess): # connection with readonly. # self.close_conn() - if self.dirchmod: + # + # The chmod command is not fully portable to windows. + # + if self.dirchmod and os.name == 'posix': for f in os.listdir(self.home): if os.path.isfile(f): os.chmod(f, 0444) -- cgit v1.2.1 From 745fd23ce3e0a95a380f97387320aa4bf9fa8867 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 1 Mar 2016 14:25:01 -0500 Subject: WT-2432 Add eviction worker threads to eviction configs. Turn off mmap for readonly. --- bench/wtperf/runners/evict-btree-readonly.wtperf | 2 +- bench/wtperf/runners/evict-btree.wtperf | 2 +- bench/wtperf/runners/evict-lsm-readonly.wtperf | 2 +- bench/wtperf/runners/evict-lsm.wtperf | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bench/wtperf/runners/evict-btree-readonly.wtperf b/bench/wtperf/runners/evict-btree-readonly.wtperf index d79af2b762b..25599fadd8d 100644 --- a/bench/wtperf/runners/evict-btree-readonly.wtperf +++ b/bench/wtperf/runners/evict-btree-readonly.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict btree configuration -conn_config="cache_size=50M" +conn_config="cache_size=50M,eviction=(threads_max=4),mmap=false" table_config="type=file" icount=10000000 report_interval=5 diff --git a/bench/wtperf/runners/evict-btree.wtperf b/bench/wtperf/runners/evict-btree.wtperf index 24da4dd7902..e7d967e5c63 100644 --- a/bench/wtperf/runners/evict-btree.wtperf +++ b/bench/wtperf/runners/evict-btree.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict btree configuration -conn_config="cache_size=50M" +conn_config="cache_size=50M,eviction=(threads_max=4)" table_config="type=file" icount=10000000 report_interval=5 diff --git a/bench/wtperf/runners/evict-lsm-readonly.wtperf b/bench/wtperf/runners/evict-lsm-readonly.wtperf index fe45c0e93b6..661b8e21924 100644 --- a/bench/wtperf/runners/evict-lsm-readonly.wtperf +++ b/bench/wtperf/runners/evict-lsm-readonly.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict lsm configuration -conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6)" +conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6),eviction=(threads_max=4)" table_config="type=lsm,lsm=(chunk_size=2M),os_cache_dirty_max=16MB" compact=true icount=10000000 diff --git a/bench/wtperf/runners/evict-lsm.wtperf b/bench/wtperf/runners/evict-lsm.wtperf index ad885d98eb7..b872d429046 100644 --- a/bench/wtperf/runners/evict-lsm.wtperf +++ b/bench/wtperf/runners/evict-lsm.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict lsm configuration -conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6)" +conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6),eviction=(threads_max=4)" table_config="type=lsm,lsm=(chunk_size=2M),os_cache_dirty_max=16MB" compact=true icount=10000000 -- cgit v1.2.1 From a833cddcb3f50221a86881c3bb4661b0b99a531c Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Tue, 1 Mar 2016 14:46:30 -0500 Subject: WT-2384 For cursor joins, specifying the first index with only compare=le requires another cursor internally to get all results in order. --- src/cursor/cur_join.c | 76 +++++++++++++++++++++++++++++++++++++++++---------- src/include/cursor.h | 1 + 2 files changed, 62 insertions(+), 15 deletions(-) diff --git a/src/cursor/cur_join.c b/src/cursor/cur_join.c index fa6dd5c32f7..b37dcc95726 100644 --- a/src/cursor/cur_join.c +++ b/src/cursor/cur_join.c @@ -8,6 +8,10 @@ #include "wt_internal.h" +static int +__curjoin_insert_endpoint(WT_SESSION_IMPL *, WT_CURSOR_JOIN_ENTRY *, u_int, + WT_CURSOR_JOIN_ENDPOINT **); + /* * __curjoin_entry_iter_init -- * Initialize an iteration for the index managed by a join entry. @@ -401,8 +405,13 @@ __curjoin_init_iter(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin) { WT_BLOOM *bloom; WT_DECL_RET; + WT_CURSOR *origcur; WT_CURSOR_JOIN_ENTRY *je, *jeend, *je2; WT_CURSOR_JOIN_ENDPOINT *end; + const char *def_cfg[] = { WT_CONFIG_BASE( + session, WT_SESSION_open_cursor), NULL }; + const char *raw_cfg[] = { WT_CONFIG_BASE( + session, WT_SESSION_open_cursor), "raw", NULL }; uint32_t f, k; if (cjoin->entries_next == 0) @@ -411,9 +420,27 @@ __curjoin_init_iter(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin) "cursors"); je = &cjoin->entries[0]; + jeend = &cjoin->entries[cjoin->entries_next]; + + /* + * For a single compare=le endpoint in the first iterated entry, + * construct a companion compare=ge endpoint that will actually + * be iterated. + */ + if (((je = cjoin->entries) != jeend) && + je->ends_next == 1 && F_ISSET(&je->ends[0], WT_CURJOIN_END_LT)) { + origcur = je->ends[0].cursor; + WT_RET(__curjoin_insert_endpoint(session, je, 0, &end)); + WT_RET(__wt_open_cursor(session, origcur->uri, + (WT_CURSOR *)cjoin, + F_ISSET(origcur, WT_CURSTD_RAW) ? raw_cfg : def_cfg, + &end->cursor)); + WT_RET(end->cursor->next(end->cursor)); + end->flags = WT_CURJOIN_END_GT | WT_CURJOIN_END_EQ | + WT_CURJOIN_END_OWN_CURSOR; + } WT_RET(__curjoin_entry_iter_init(session, cjoin, je, &cjoin->iter)); - jeend = &cjoin->entries[cjoin->entries_next]; for (je = cjoin->entries; je < jeend; je++) { __wt_stat_join_init_single(&je->stats); for (end = &je->ends[0]; end < &je->ends[je->ends_next]; @@ -709,7 +736,8 @@ nextkey: skip_left = false; WT_ERR(ret); } - } + } else if (ret != WT_NOTFOUND) + WT_ERR(ret); if (0) { err: F_SET(cjoin, WT_CURJOIN_ERROR); @@ -772,8 +800,11 @@ __curjoin_close(WT_CURSOR *cursor) if (F_ISSET(entry, WT_CURJOIN_ENTRY_OWN_BLOOM)) WT_TRET(__wt_bloom_close(entry->bloom)); for (end = &entry->ends[0]; - end < &entry->ends[entry->ends_next]; end++) + end < &entry->ends[entry->ends_next]; end++) { F_CLR(end->cursor, WT_CURSTD_JOINED); + if (F_ISSET(end, WT_CURJOIN_END_OWN_CURSOR)) + WT_TRET(end->cursor->close(end->cursor)); + } __wt_free(session, entry->ends); __wt_free(session, entry->repack_format); } @@ -879,7 +910,7 @@ __wt_curjoin_join(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, uint64_t count, uint32_t bloom_bit_count, uint32_t bloom_hash_count) { WT_CURSOR_INDEX *cindex; - WT_CURSOR_JOIN_ENDPOINT *end, *newend; + WT_CURSOR_JOIN_ENDPOINT *end; WT_CURSOR_JOIN_ENTRY *entry; WT_DECL_RET; bool hasins, needbloom, range_eq; @@ -1000,17 +1031,10 @@ __wt_curjoin_join(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, entry->bloom_hash_count = WT_MAX(entry->bloom_hash_count, bloom_hash_count); } - WT_ERR(__wt_realloc_def(session, &entry->ends_allocated, - entry->ends_next + 1, &entry->ends)); - if (!hasins) - ins = entry->ends_next; - newend = &entry->ends[ins]; - memmove(newend + 1, newend, - (entry->ends_next - ins) * sizeof(WT_CURSOR_JOIN_ENDPOINT)); - memset(newend, 0, sizeof(WT_CURSOR_JOIN_ENDPOINT)); - entry->ends_next++; - newend->cursor = ref_cursor; - F_SET(newend, range); + WT_ERR(__curjoin_insert_endpoint(session, entry, + hasins ? ins : entry->ends_next, &end)); + end->cursor = ref_cursor; + F_SET(end, range); /* Open the main file with a projection of the indexed columns. */ if (entry->main == NULL && idx != NULL) { @@ -1053,3 +1077,25 @@ err: if (main_uri != NULL) __wt_free(session, main_uri); return (ret); } + +/* + * __curjoin_insert_endpoint -- + * Insert a new entry into the endpoint array for the join entry. + */ +static int +__curjoin_insert_endpoint(WT_SESSION_IMPL *session, WT_CURSOR_JOIN_ENTRY *entry, + u_int pos, WT_CURSOR_JOIN_ENDPOINT **newendp) +{ + WT_CURSOR_JOIN_ENDPOINT *newend; + + WT_RET(__wt_realloc_def(session, &entry->ends_allocated, + entry->ends_next + 1, &entry->ends)); + newend = &entry->ends[pos]; + memmove(newend + 1, newend, + (entry->ends_next - pos) * sizeof(WT_CURSOR_JOIN_ENDPOINT)); + memset(newend, 0, sizeof(WT_CURSOR_JOIN_ENDPOINT)); + entry->ends_next++; + *newendp = newend; + + return (0); +} diff --git a/src/include/cursor.h b/src/include/cursor.h index 48db8b9ec23..2665ff83df3 100644 --- a/src/include/cursor.h +++ b/src/include/cursor.h @@ -303,6 +303,7 @@ struct __wt_cursor_join_endpoint { #define WT_CURJOIN_END_GT 0x04 /* include values > cursor */ #define WT_CURJOIN_END_GE (WT_CURJOIN_END_GT | WT_CURJOIN_END_EQ) #define WT_CURJOIN_END_LE (WT_CURJOIN_END_LT | WT_CURJOIN_END_EQ) +#define WT_CURJOIN_END_OWN_CURSOR 0x08 /* must close cursor */ uint8_t flags; /* range for this endpoint */ }; #define WT_CURJOIN_END_RANGE(endp) \ -- cgit v1.2.1 From 5ca497fafbda79167e26e8560dcf86c9cb39de53 Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Tue, 1 Mar 2016 15:41:34 -0500 Subject: WT-2384 Added test corresponding to submitted test case. --- test/suite/test_join05.py | 66 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 test/suite/test_join05.py diff --git a/test/suite/test_join05.py b/test/suite/test_join05.py new file mode 100644 index 00000000000..ef2be4c6460 --- /dev/null +++ b/test/suite/test_join05.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import check_scenarios, multiply_scenarios, number_scenarios + +# test_join05.py +# Tests based on JIRA reports +class test_join05(wttest.WiredTigerTestCase): + + # test join having the first index just be lt/le + def test_wt_2384(self): + self.session.create("table:test_2384", + "key_format=i,value_format=i,columns=(k,v)") + self.session.create("index:test_2384:index", "columns=(v)") + cursor = self.session.open_cursor("table:test_2384", None, None) + cursor[1] = 11 + cursor[2] = 12 + cursor[3] = 13 + cursor.close() + + cursor = self.session.open_cursor("index:test_2384:index", None, None) + cursor.set_key(13) + self.assertEquals(cursor.search(), 0) + + jcursor = self.session.open_cursor("join:table:test_2384", None, None) + self.session.join(jcursor, cursor, "compare=lt") + + nr_found = 0 + while jcursor.next() == 0: + [k] = jcursor.get_keys() + [v] = jcursor.get_values() + #self.tty("jcursor: k=" + str(k) + ", v=" + str(v)) + nr_found += 1 + + self.assertEquals(nr_found, 2) + jcursor.close() + cursor.close() + +if __name__ == '__main__': + wttest.run() -- cgit v1.2.1 From 5a47fd78102f26ff85ba353154617772acd8036b Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 1 Mar 2016 16:54:31 -0500 Subject: WT-2366 Add shrink changes. --- bench/wtperf/config.c | 31 ++++++++++++++++++------ bench/wtperf/runners/update-shrink-stress.wtperf | 14 +++++++++++ bench/wtperf/wtperf.c | 20 ++++++++------- bench/wtperf/wtperf.h | 5 +++- bench/wtperf/wtperf_opt.i | 2 ++ 5 files changed, 55 insertions(+), 17 deletions(-) create mode 100644 bench/wtperf/runners/update-shrink-stress.wtperf diff --git a/bench/wtperf/config.c b/bench/wtperf/config.c index fc122eaf987..8a514bd794b 100644 --- a/bench/wtperf/config.c +++ b/bench/wtperf/config.c @@ -266,10 +266,9 @@ config_threads(CONFIG *cfg, const char *config, size_t len) if ((workp->truncate = v.val) != 1) goto err; /* There can only be one Truncate thread. */ - if (cfg->has_truncate != 0) { + if (F_ISSET(cfg, CFG_TRUNCATE)) goto err; - } - cfg->has_truncate = 1; + F_SET(cfg, CFG_TRUNCATE); continue; } if (STRING_MATCH("truncate_pct", k.str, k.len)) { @@ -297,8 +296,13 @@ config_threads(CONFIG *cfg, const char *config, size_t len) goto err; /* Special random value */ workp->update_delta = INT64_MAX; - } else + } else { workp->update_delta = v.val; + if (v.val > 0) + F_SET(cfg, CFG_GROW); + if (v.val < 0) + F_SET(cfg, CFG_SHRINK); + } continue; } goto err; @@ -690,9 +694,22 @@ config_sanity(CONFIG *cfg) } if (cfg->value_sz_max < cfg->value_sz) { - fprintf(stderr, - "value_sz_max must greater than or equal to value_sz\n"); - return (EINVAL); + if (F_ISSET(cfg, CFG_GROW)) { + fprintf(stderr, "value_sz_max %" PRIu32 + " must be greater than or equal to value_sz %" + PRIu32 "\n", cfg->value_sz_max, cfg->value_sz); + return (EINVAL); + } else + cfg->value_sz_max = cfg->value_sz; + } + if (cfg->value_sz_min > cfg->value_sz) { + if (F_ISSET(cfg, CFG_SHRINK)) { + fprintf(stderr, "value_sz_min %" PRIu32 + " must be less than or equal to value_sz %" + PRIu32 "\n", cfg->value_sz_min, cfg->value_sz); + return (EINVAL); + } else + cfg->value_sz_min = cfg->value_sz; } if (cfg->readonly && cfg->workload != NULL) diff --git a/bench/wtperf/runners/update-shrink-stress.wtperf b/bench/wtperf/runners/update-shrink-stress.wtperf new file mode 100644 index 00000000000..0bba3d67b70 --- /dev/null +++ b/bench/wtperf/runners/update-shrink-stress.wtperf @@ -0,0 +1,14 @@ +# wtperf options file: Grow the size of documents while there is cache +# pressure and appends are happening as well. +conn_config="cache_size=2GB,checkpoint=(wait=60)" +table_config="type=file,leaf_page_max=32k,leaf_value_max=128k,split_pct=90" +# Since we're continually inserting, start with a small number initially. +icount=200000 +report_interval=5 +run_time=240 +populate_threads=1 +# Continue inserting new records. +threads=((count=1,inserts=1,throttle=1000),(count=4,update=1,update_delta=-100)) +# Start with moderate values and let them shrink slowly. +value_sz_min=1000 +value_sz=20000 diff --git a/bench/wtperf/wtperf.c b/bench/wtperf/wtperf.c index 3d6f662c762..3e43467a3e2 100644 --- a/bench/wtperf/wtperf.c +++ b/bench/wtperf/wtperf.c @@ -60,7 +60,7 @@ static const CONFIG default_cfg = { 0, /* in warmup phase */ false, /* Signal for idle cycle thread */ 0, /* total seconds running */ - 0, /* has truncate */ + 0, /* flags */ {NULL, NULL}, /* the truncate queue */ {NULL, NULL}, /* the config queue */ @@ -125,8 +125,10 @@ randomize_value(CONFIG_THREAD *thread, char *value_buf) */ if (thread->workload->update_delta == 0) max_range = thread->cfg->value_sz; - else + else if (thread->workload->update_delta > 0) max_range = thread->cfg->value_sz_max; + else + max_range = thread->cfg->value_sz_min; /* * Generate a single random value and re-use it. We generally only @@ -176,16 +178,16 @@ update_value_delta(CONFIG_THREAD *thread) /* Ensure we aren't changing across boundaries */ if (delta > 0 && len + delta > cfg->value_sz_max) delta = cfg->value_sz_max - len; - else if (delta < 0 && -delta > len) - delta = -(len - 1); + else if (delta < 0 && len + delta < cfg->value_sz_min) + delta = cfg->value_sz_min - len; /* Bail if there isn't anything to do */ if (delta == 0) return; - if (delta < 0) { + if (delta < 0) value[len + delta] = '\0'; - } else { + else { /* Extend the value by the configured amount. */ for (new_len = len; new_len < cfg->value_sz_max && new_len - len < delta; @@ -2265,7 +2267,7 @@ main(int argc, char *argv[]) * the compact operation, but not for the workloads. */ if (cfg->async_threads > 0) { - if (cfg->has_truncate > 0) { + if (F_ISSET(cfg, CFG_TRUNCATE) > 0) { lprintf(cfg, 1, 0, "Cannot run truncate and async\n"); goto err; } @@ -2290,13 +2292,13 @@ main(int argc, char *argv[]) goto err; /* You can't have truncate on a random collection. */ - if (cfg->has_truncate && cfg->random_range) { + if (F_ISSET(cfg, CFG_TRUNCATE) && cfg->random_range) { lprintf(cfg, 1, 0, "Cannot run truncate and random_range\n"); goto err; } /* We can't run truncate with more than one table. */ - if (cfg->has_truncate && cfg->table_count > 1) { + if (F_ISSET(cfg, CFG_TRUNCATE) && cfg->table_count > 1) { lprintf(cfg, 1, 0, "Cannot truncate more than 1 table\n"); goto err; } diff --git a/bench/wtperf/wtperf.h b/bench/wtperf/wtperf.h index 6107eaf721d..c591499b907 100644 --- a/bench/wtperf/wtperf.h +++ b/bench/wtperf/wtperf.h @@ -191,7 +191,10 @@ struct __config { /* Configuration structure */ volatile uint32_t totalsec; /* total seconds running */ - u_int has_truncate; /* if there is a truncate workload */ +#define CFG_GROW 0x0001 /* There is a grow workload */ +#define CFG_SHRINK 0x0002 /* There is a shrink workload */ +#define CFG_TRUNCATE 0x0004 /* There is a truncate workload */ + uint32_t flags; /* flags */ /* Queue head for use with the Truncate Logic */ TAILQ_HEAD(__truncate_qh, __truncate_queue_entry) stone_head; diff --git a/bench/wtperf/wtperf_opt.i b/bench/wtperf/wtperf_opt.i index 4fd1936b8b7..b5e274a17c2 100644 --- a/bench/wtperf/wtperf_opt.i +++ b/bench/wtperf/wtperf_opt.i @@ -193,6 +193,8 @@ DEF_OPT_AS_CONFIG_STRING(transaction_config, "", DEF_OPT_AS_STRING(table_name, "test", "table name") DEF_OPT_AS_UINT32(value_sz_max, 1000, "maximum value size when delta updates are present. Default disabled") +DEF_OPT_AS_UINT32(value_sz_min, 1, + "minimum value size when delta updates are present. Default disabled") DEF_OPT_AS_UINT32(value_sz, 100, "value size") DEF_OPT_AS_UINT32(verbose, 1, "verbosity") DEF_OPT_AS_UINT32(warmup, 0, -- cgit v1.2.1 From 784e7bd75ddf29be1eef1b5289c0673942c84256 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 1 Mar 2016 16:57:47 -0500 Subject: WT-2366 Run s_all --- dist/s_string.ok | 1 + src/docs/wtperf.dox | 2 ++ 2 files changed, 3 insertions(+) diff --git a/dist/s_string.ok b/dist/s_string.ok index c8769abdfbe..2caaddcc15a 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -983,6 +983,7 @@ superset sw sy sys +sz t's tV tablename diff --git a/src/docs/wtperf.dox b/src/docs/wtperf.dox index 59176f7909a..6d8dcab8f65 100644 --- a/src/docs/wtperf.dox +++ b/src/docs/wtperf.dox @@ -262,6 +262,8 @@ is nonzero table name @par value_sz_max (unsigned int, default=1000) maximum value size when delta updates are present. Default disabled +@par value_sz_min (unsigned int, default=1) +minimum value size when delta updates are present. Default disabled @par value_sz (unsigned int, default=100) value size @par verbose (unsigned int, default=1) -- cgit v1.2.1 From dd41bbeea51a4633e0411ab6cf2f614d1670f833 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 1 Mar 2016 19:32:55 -0500 Subject: WT-2437 Missed a place to check for posix and dirchmod. --- test/suite/test_readonly01.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/suite/test_readonly01.py b/test/suite/test_readonly01.py index 0a4bc219700..59e9743ab7e 100644 --- a/test/suite/test_readonly01.py +++ b/test/suite/test_readonly01.py @@ -136,7 +136,7 @@ class test_readonly01(wttest.WiredTigerTestCase, suite_subprocess): self.create = True def test_readonly(self): - if self.dirchmod: + if self.dirchmod and os.name == 'posix': with self.expectedStderrPattern('Permission'): self.readonly() else: -- cgit v1.2.1 From 7e18c2380eb25119252cba5044df9e7568ccee14 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Wed, 2 Mar 2016 04:04:43 +0000 Subject: whitespace --- dist/stat_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/stat_data.py b/dist/stat_data.py index 6d91a87a13e..09e5643a5d6 100644 --- a/dist/stat_data.py +++ b/dist/stat_data.py @@ -12,7 +12,7 @@ # max_aggregate Take the maximum value when aggregating statistics # no_clear Value not cleared when statistics cleared # no_scale Don't scale value per second in the logging tool script -# size Used by timeseries tool, indicates value is a byte count +# size Used by timeseries tool, indicates value is a byte count # # The no_clear and no_scale flags are normally always set together (values that # are maintained over time are normally not scaled per second). -- cgit v1.2.1 From 4e1a8bdc2c6083956fdf9b6c971e472c65d38f5d Mon Sep 17 00:00:00 2001 From: Mark Benvenuto Date: Wed, 2 Mar 2016 14:55:30 +1100 Subject: WT-2440 Per the PPC64 ABI, v20-v31 are non-volatile registers See http://refspecs.linuxfoundation.org/ELF/ppc64/PPC-elf64abi.html#REG for calling convention. In experiments under GDB, we see that after crc32_vpmsum is called that registers v20-v31 are modified. We discovered this issue after integrating the crc32_vpmsum code into our C++ code base which targeted POWER8 for code generation, see https://jira.mongodb.org/browse/SERVER-22773 for repro. --- src/support/power8/crc32.S | 50 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/support/power8/crc32.S b/src/support/power8/crc32.S index 4bc1fad416d..3ef2928aaa1 100644 --- a/src/support/power8/crc32.S +++ b/src/support/power8/crc32.S @@ -90,6 +90,31 @@ FUNC_START(__crc32_vpmsum) std r26,-48(r1) std r25,-56(r1) + li r31, -256 + stvx v20, r31, r1 + li r31, -240 + stvx v21, r31, r1 + li r31, -224 + stvx v22, r31, r1 + li r31, -208 + stvx v23, r31, r1 + li r31, -192 + stvx v24, r31, r1 + li r31, -176 + stvx v25, r31, r1 + li r31, -160 + stvx v26, r31, r1 + li r31, -144 + stvx v27, r31, r1 + li r31, -128 + stvx v28, r31, r1 + li r31, -112 + stvx v29, r31, r1 + li r31, -96 + stvx v30, r31, r1 + li r31, -80 + stvx v31, r31, r1 + li off16,16 li off32,32 li off48,48 @@ -571,6 +596,31 @@ FUNC_START(__crc32_vpmsum) /* Get it into r3 */ MFVRD(r3, v0) + li r31, -256 + lvx v20, r31, r1 + li r31, -240 + lvx v21, r31, r1 + li r31, -224 + lvx v22, r31, r1 + li r31, -208 + lvx v23, r31, r1 + li r31, -192 + lvx v24, r31, r1 + li r31, -176 + lvx v25, r31, r1 + li r31, -160 + lvx v26, r31, r1 + li r31, -144 + lvx v27, r31, r1 + li r31, -128 + lvx v28, r31, r1 + li r31, -112 + lvx v29, r31, r1 + li r31, -96 + lvx v30, r31, r1 + li r31, -80 + lvx v31, r31, r1 + ld r31,-8(r1) ld r30,-16(r1) ld r29,-24(r1) -- cgit v1.2.1 From a57d033bf9f84164a846364f127377141d063c7e Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Wed, 2 Mar 2016 09:57:46 -0500 Subject: WT-2436 For cursor joins with single compare='lt/le', fix how Bloom filters are populated. --- src/cursor/cur_join.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/cursor/cur_join.c b/src/cursor/cur_join.c index fa6dd5c32f7..7646ddfc059 100644 --- a/src/cursor/cur_join.c +++ b/src/cursor/cur_join.c @@ -295,11 +295,17 @@ __curjoin_init_bloom(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, /* Initially position the cursor if necessary. */ endmax = &entry->ends[entry->ends_next]; - if ((end = &entry->ends[0]) < endmax && - F_ISSET(end, WT_CURJOIN_END_GE)) { - WT_ERR(__wt_cursor_dup_position(end->cursor, c)); - if (WT_CURJOIN_END_RANGE(end) == WT_CURJOIN_END_GE) - skip = 1; + if ((end = &entry->ends[0]) < endmax) { + if (F_ISSET(end, WT_CURJOIN_END_GT) || + WT_CURJOIN_END_RANGE(end) == WT_CURJOIN_END_EQ) { + WT_ERR(__wt_cursor_dup_position(end->cursor, c)); + if (WT_CURJOIN_END_RANGE(end) == WT_CURJOIN_END_GE) + skip = 1; + } else if (F_ISSET(end, WT_CURJOIN_END_LT)) { + if ((ret = c->next(c)) == WT_NOTFOUND) + goto done; + WT_ERR(ret); + } } collator = (entry->index == NULL) ? NULL : entry->index->collator; while (ret == 0) { -- cgit v1.2.1 From f0bc281964df3633d271546ddade4e339b1c241c Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Wed, 2 Mar 2016 10:53:17 -0500 Subject: WT-2366 Minor changes to grow/shrink workloads --- bench/wtperf/runners/update-grow-stress.wtperf | 2 +- bench/wtperf/runners/update-shrink-stress.wtperf | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/bench/wtperf/runners/update-grow-stress.wtperf b/bench/wtperf/runners/update-grow-stress.wtperf index 4ec039af4b4..5d2ab1d3f7e 100644 --- a/bench/wtperf/runners/update-grow-stress.wtperf +++ b/bench/wtperf/runners/update-grow-stress.wtperf @@ -1,6 +1,6 @@ # wtperf options file: Grow the size of documents while there is cache # pressure and appends are happening as well. -conn_config="cache_size=2GB,checkpoint=(wait=60)" +conn_config="cache_size=2GB,checkpoint=(wait=30)" table_config="type=file,leaf_page_max=32k,leaf_value_max=128k,split_pct=90" # The values are starting small, insert a lot so our database grows larger than # cache quickly. diff --git a/bench/wtperf/runners/update-shrink-stress.wtperf b/bench/wtperf/runners/update-shrink-stress.wtperf index 0bba3d67b70..2b9887449f4 100644 --- a/bench/wtperf/runners/update-shrink-stress.wtperf +++ b/bench/wtperf/runners/update-shrink-stress.wtperf @@ -1,8 +1,9 @@ -# wtperf options file: Grow the size of documents while there is cache -# pressure and appends are happening as well. -conn_config="cache_size=2GB,checkpoint=(wait=60)" +# wtperf options file: Shrink the size of values. Checkpoint frequently +# and insert new records too. +# +conn_config="cache_size=2GB,checkpoint=(wait=30)" table_config="type=file,leaf_page_max=32k,leaf_value_max=128k,split_pct=90" -# Since we're continually inserting, start with a small number initially. +# Since we're continually inserting, start with a smaller number initially. icount=200000 report_interval=5 run_time=240 -- cgit v1.2.1 From 027fc3282621e5007a0e4c30a17f68be7a283385 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Wed, 2 Mar 2016 10:53:51 -0500 Subject: WT-2366 Add new wtperf workloads mixing grow and shrink. --- bench/wtperf/runners/update-delta-mix1.wtperf | 18 ++++++++++++++++++ bench/wtperf/runners/update-delta-mix2.wtperf | 18 ++++++++++++++++++ bench/wtperf/runners/update-delta-mix3.wtperf | 18 ++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 bench/wtperf/runners/update-delta-mix1.wtperf create mode 100644 bench/wtperf/runners/update-delta-mix2.wtperf create mode 100644 bench/wtperf/runners/update-delta-mix3.wtperf diff --git a/bench/wtperf/runners/update-delta-mix1.wtperf b/bench/wtperf/runners/update-delta-mix1.wtperf new file mode 100644 index 00000000000..0f5e75f5347 --- /dev/null +++ b/bench/wtperf/runners/update-delta-mix1.wtperf @@ -0,0 +1,18 @@ +# wtperf options file: Mixed workload where we grow some values and shrink +# others. Mixed load leaning toward growing the dataset. +# +conn_config="cache_size=2GB,checkpoint=(wait=30)" +table_config="type=file,leaf_page_max=32k,leaf_value_max=128k,split_pct=90" +# The values are starting small, insert a lot so our database grows larger than +# cache quickly. +icount=200000 +report_interval=5 +run_time=300 +populate_threads=1 +# +# Run more grow workload threads than shrink threads. +# +threads=((count=4,update=1,update_delta=100),(count=2,update=1,update_delta=-150)) +value_sz=20000 +value_sz_min=1000 +value_sz_max=65536 diff --git a/bench/wtperf/runners/update-delta-mix2.wtperf b/bench/wtperf/runners/update-delta-mix2.wtperf new file mode 100644 index 00000000000..f3ce2a455cc --- /dev/null +++ b/bench/wtperf/runners/update-delta-mix2.wtperf @@ -0,0 +1,18 @@ +# wtperf options file: Mixed workload where we grow some values and shrink +# others. Mixed load leaning toward shrinking the dataset. +# +conn_config="cache_size=2GB,checkpoint=(wait=30)" +table_config="type=file,leaf_page_max=32k,leaf_value_max=128k,split_pct=90" +# The values are starting small, insert a lot so our database grows larger than +# cache quickly. +icount=200000 +report_interval=5 +run_time=300 +populate_threads=1 +# +# Run more shrink workload threads than grow threads. +# +threads=((count=2,update=1,update_delta=150),(count=4,update=1,update_delta=-100)) +value_sz=20000 +value_sz_min=1000 +value_sz_max=65536 diff --git a/bench/wtperf/runners/update-delta-mix3.wtperf b/bench/wtperf/runners/update-delta-mix3.wtperf new file mode 100644 index 00000000000..606eb727eef --- /dev/null +++ b/bench/wtperf/runners/update-delta-mix3.wtperf @@ -0,0 +1,18 @@ +# wtperf options file: Mixed workload where we grow some values and shrink +# others. Mixed load leaning toward mostly a balance. +# +conn_config="cache_size=2GB,checkpoint=(wait=30)" +table_config="type=file,leaf_page_max=32k,leaf_value_max=128k,split_pct=90" +# The values are starting small, insert a lot so our database grows larger than +# cache quickly. +icount=200000 +report_interval=5 +run_time=300 +populate_threads=1 +# +# Run a balance of threads. +# +threads=((count=3,update=1,update_delta=100),(count=3,update=1,update_delta=-100)) +value_sz=20000 +value_sz_min=1000 +value_sz_max=65536 -- cgit v1.2.1 From 8619b643dd1c50e5a3dc57973f06a9875bc4776f Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Wed, 2 Mar 2016 11:06:55 -0500 Subject: WT-2366 Set CFG_GROW flag on random delta path. --- bench/wtperf/config.c | 1 + 1 file changed, 1 insertion(+) diff --git a/bench/wtperf/config.c b/bench/wtperf/config.c index 8a514bd794b..e83d6fcceed 100644 --- a/bench/wtperf/config.c +++ b/bench/wtperf/config.c @@ -296,6 +296,7 @@ config_threads(CONFIG *cfg, const char *config, size_t len) goto err; /* Special random value */ workp->update_delta = INT64_MAX; + F_SET(cfg, CFG_GROW); } else { workp->update_delta = v.val; if (v.val > 0) -- cgit v1.2.1 From 7c0e7943f681daf7c27038d33661db1cb8737208 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Wed, 2 Mar 2016 13:16:44 -0500 Subject: WT-2366 Adjust grow/shrink workloads to be comparable to each other. --- bench/wtperf/runners/update-grow-stress.wtperf | 4 ++-- bench/wtperf/runners/update-shrink-stress.wtperf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bench/wtperf/runners/update-grow-stress.wtperf b/bench/wtperf/runners/update-grow-stress.wtperf index 5d2ab1d3f7e..f7403e1578d 100644 --- a/bench/wtperf/runners/update-grow-stress.wtperf +++ b/bench/wtperf/runners/update-grow-stress.wtperf @@ -4,12 +4,12 @@ conn_config="cache_size=2GB,checkpoint=(wait=30)" table_config="type=file,leaf_page_max=32k,leaf_value_max=128k,split_pct=90" # The values are starting small, insert a lot so our database grows larger than # cache quickly. -icount=20000000 +icount=200000 report_interval=5 run_time=240 populate_threads=1 # Continue inserting new records. threads=((count=1,inserts=1,throttle=1000),(count=4,update=1,update_delta=100)) # Start with small values and let them grow slowly to large values. -value_sz=20 +value_sz=10000 value_sz_max=65536 diff --git a/bench/wtperf/runners/update-shrink-stress.wtperf b/bench/wtperf/runners/update-shrink-stress.wtperf index 2b9887449f4..bbdd9593b59 100644 --- a/bench/wtperf/runners/update-shrink-stress.wtperf +++ b/bench/wtperf/runners/update-shrink-stress.wtperf @@ -12,4 +12,4 @@ populate_threads=1 threads=((count=1,inserts=1,throttle=1000),(count=4,update=1,update_delta=-100)) # Start with moderate values and let them shrink slowly. value_sz_min=1000 -value_sz=20000 +value_sz=10000 -- cgit v1.2.1 From 97391150d51cde7a74ce0f0ec18091bc0268c516 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Wed, 2 Mar 2016 13:43:01 -0500 Subject: WT-2366 Check for NULL workload pointer. --- bench/wtperf/wtperf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/wtperf/wtperf.c b/bench/wtperf/wtperf.c index 3e43467a3e2..5755e22dd2f 100644 --- a/bench/wtperf/wtperf.c +++ b/bench/wtperf/wtperf.c @@ -123,7 +123,7 @@ randomize_value(CONFIG_THREAD *thread, char *value_buf) * off for avoiding figuring out how long the value is more accurately * in this performance sensitive function. */ - if (thread->workload->update_delta == 0) + if (thread->workload == NULL || thread->workload->update_delta == 0) max_range = thread->cfg->value_sz; else if (thread->workload->update_delta > 0) max_range = thread->cfg->value_sz_max; -- cgit v1.2.1 From 77d492d721495a26b459f5c4cd1cbbd620c10bc1 Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Wed, 2 Mar 2016 15:16:14 -0500 Subject: WT-2322 Do not permit read-uncommitted with bloom filters, it would be prone to errors. --- src/cursor/cur_join.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/cursor/cur_join.c b/src/cursor/cur_join.c index fa6dd5c32f7..3a030c558dd 100644 --- a/src/cursor/cur_join.c +++ b/src/cursor/cur_join.c @@ -431,6 +431,11 @@ __curjoin_init_iter(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin) F_SET(cjoin, WT_CURJOIN_SKIP_FIRST_LEFT); if (F_ISSET(je, WT_CURJOIN_ENTRY_BLOOM)) { + if (session->txn.isolation == WT_ISO_READ_UNCOMMITTED) + WT_RET_MSG(session, EINVAL, + "join cursors with Bloom filters cannot be " + "used with WT_ISO_READ_UNCOMMITTED " + "isolation"); if (je->bloom == NULL) { /* * Look for compatible filters to be shared, -- cgit v1.2.1 From 1083bd43803734d5e69d8b284c6f5deee8212315 Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Wed, 2 Mar 2016 15:17:37 -0500 Subject: WT-2322 On transaction rollback, do not try to reset joined cursors, as it will fail. The parent 'join' cursor will be reset, which is enough. --- src/session/session_api.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/session/session_api.c b/src/session/session_api.c index 2414229681b..3153ca458eb 100644 --- a/src/session/session_api.c +++ b/src/session/session_api.c @@ -40,7 +40,8 @@ __wt_session_reset_cursors(WT_SESSION_IMPL *session, bool free_buffers) /* Stop when there are no positioned cursors. */ if (session->ncursors == 0) break; - WT_TRET(cursor->reset(cursor)); + if (!F_ISSET(cursor, WT_CURSTD_JOINED)) + WT_TRET(cursor->reset(cursor)); /* Optionally, free the cursor buffers */ if (free_buffers) { __wt_buf_free(session, &cursor->key); -- cgit v1.2.1 From b9ac15cbcf352cfd78a133cf72e58dbc8c6d42cd Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Wed, 2 Mar 2016 15:19:24 -0500 Subject: WT-2322 Add test for uncommitted reads with join cursors. --- test/suite/test_join06.py | 158 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 test/suite/test_join06.py diff --git a/test/suite/test_join06.py b/test/suite/test_join06.py new file mode 100644 index 00000000000..4a0af809087 --- /dev/null +++ b/test/suite/test_join06.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os +import wiredtiger, wttest, run +from wtscenario import check_scenarios, multiply_scenarios, number_scenarios + +# test_join06.py +# Join operations +# Joins with a read-uncommitted +class test_join06(wttest.WiredTigerTestCase): + nentries = 1000 + + isoscen = [ + ('isolation_read_uncommitted', dict(uncommitted=True)), + ('isolation_default', dict(uncommitted=False)) + ] + + bloomscen = [ + ('bloom', dict(bloom=True)), + ('nobloom', dict(bloom=False)) + ] + + scenarios = number_scenarios(multiply_scenarios('.', isoscen, bloomscen)) + + def gen_values(self, i): + s = str(i) # 345 => "345" + f = s[0:1] + s[0:1] + s[0:1] # 345 => "333" + return [s, f] + + def gen_values2(self, i): + s = str(i) # 345 => "345" + l = s[-1:] + s[-1:] + s[-1:] # 345 => "555" + return [s, l] + + def populate(self, s, gen_values): + c = s.open_cursor('table:join06', None, None) + for i in range(0, self.nentries): + c.set_key(i) + c.set_value(*gen_values(i)) + c.insert() + c.close() + + # Common function for testing the most basic functionality + # of joins + def test_join(self): + self.session.create('table:join06', + 'columns=(k,v0,v1),key_format=i,value_format=SS') + self.session.create('index:join06:index0','columns=(v0)') + self.session.create('index:join06:index1','columns=(v1)') + + self.populate(self.session, self.gen_values) + + # TODO: needed? + #self.reopen_conn() + + if self.uncommitted: + self.session.begin_transaction('isolation=read-uncommitted') + + jc = self.session.open_cursor('join:table:join06', None, None) + c0 = self.session.open_cursor('index:join06:index0', None, None) + c0.set_key('520') + self.assertEquals(0, c0.search()) + self.session.join(jc, c0, 'compare=ge') + + joinconfig = 'compare=eq' + if self.bloom: + joinconfig += ',strategy=bloom,count=1000' + c1 = self.session.open_cursor('index:join06:index1', None, None) + c1.set_key('555') + self.assertEquals(0, c1.search()) + self.session.join(jc, c1, joinconfig) + + if self.uncommitted and self.bloom: + # Make sure that read-uncommitted with Bloom is not allowed. + # This is detected on the first next() operation. + msg = '/cannot be used with WT_ISO_READ_UNCOMMITTED/' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: jc.next(), msg) + return + + # Changes made in another session may or may not be visible to us, + # depending on the isolation level. + if self.uncommitted: + # isolation level is read-uncommitted, so we will see + # additions deletions made in our other session. + mbr = set(range(525,1000,10)) | set(range(55,100,10)) | set([520]) + else: + # default isolation level, so we should see a consistent + # set at the time we begin iteration. + mbr = set(range(520,600)) | set(range(53,60)) + + altered = False + + while jc.next() == 0: + [k] = jc.get_keys() + [v0,v1] = jc.get_values() + #self.tty('GOT: ' + str(k) + ': ' + str(jc.get_values())) + if altered and self.uncommitted: + self.assertEquals(self.gen_values2(k), [v0, v1]) + else: + self.assertEquals(self.gen_values(k), [v0, v1]) + if not k in mbr: + self.tty('**** ERROR: result ' + str(k) + ' is not in: ' + + str(mbr)) + self.assertTrue(k in mbr) + mbr.remove(k) + + # In another session, we remove entries for keys ending in 6, + # and add entries for keys ending in 5. Depending on the + # isolation level for the transaction, these changes may or + # may not be visible for the original session. + if not altered: + s = self.conn.open_session(None) + s.begin_transaction(None) + self.populate(s, self.gen_values2) + s.commit_transaction() + s.close() + altered = True + + if len(mbr) != 0: + self.tty('**** ERROR: did not see these: ' + str(mbr)) + self.assertEquals(0, len(mbr)) + + jc.close() + c1.close() + c0.close() + if self.uncommitted: + self.session.commit_transaction() + self.session.drop('table:join06') + +if __name__ == '__main__': + wttest.run() -- cgit v1.2.1 From 8801bc6c622636fa88200595f765fb2d143b3fea Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Wed, 2 Mar 2016 15:20:10 -0500 Subject: WT-2322 Remove spurious period from error message. --- src/cursor/cur_join.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cursor/cur_join.c b/src/cursor/cur_join.c index 3a030c558dd..6c1ad622c33 100644 --- a/src/cursor/cur_join.c +++ b/src/cursor/cur_join.c @@ -638,7 +638,7 @@ __curjoin_entry_member(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, ret = c->get_value(c, &v); else if (ret == WT_NOTFOUND) WT_ERR_MSG(session, WT_ERROR, - "main table for join is missing entry."); + "main table for join is missing entry"); WT_TRET(c->reset(c)); WT_ERR(ret); } else -- cgit v1.2.1 From 880203a063177e4733739d1082ea6901306b2b9b Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Wed, 2 Mar 2016 16:36:53 -0500 Subject: WT-2322 white space --- src/cursor/cur_join.c | 4 ++-- test/suite/test_join06.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/cursor/cur_join.c b/src/cursor/cur_join.c index 6c1ad622c33..57497b5ce2a 100644 --- a/src/cursor/cur_join.c +++ b/src/cursor/cur_join.c @@ -431,8 +431,8 @@ __curjoin_init_iter(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin) F_SET(cjoin, WT_CURJOIN_SKIP_FIRST_LEFT); if (F_ISSET(je, WT_CURJOIN_ENTRY_BLOOM)) { - if (session->txn.isolation == WT_ISO_READ_UNCOMMITTED) - WT_RET_MSG(session, EINVAL, + if (session->txn.isolation == WT_ISO_READ_UNCOMMITTED) + WT_RET_MSG(session, EINVAL, "join cursors with Bloom filters cannot be " "used with WT_ISO_READ_UNCOMMITTED " "isolation"); diff --git a/test/suite/test_join06.py b/test/suite/test_join06.py index 4a0af809087..1910b98d9bf 100644 --- a/test/suite/test_join06.py +++ b/test/suite/test_join06.py @@ -81,7 +81,7 @@ class test_join06(wttest.WiredTigerTestCase): if self.uncommitted: self.session.begin_transaction('isolation=read-uncommitted') - + jc = self.session.open_cursor('join:table:join06', None, None) c0 = self.session.open_cursor('index:join06:index0', None, None) c0.set_key('520') @@ -130,7 +130,7 @@ class test_join06(wttest.WiredTigerTestCase): str(mbr)) self.assertTrue(k in mbr) mbr.remove(k) - + # In another session, we remove entries for keys ending in 6, # and add entries for keys ending in 5. Depending on the # isolation level for the transaction, these changes may or @@ -142,7 +142,7 @@ class test_join06(wttest.WiredTigerTestCase): s.commit_transaction() s.close() altered = True - + if len(mbr) != 0: self.tty('**** ERROR: did not see these: ' + str(mbr)) self.assertEquals(0, len(mbr)) -- cgit v1.2.1 From 6990bcc1917ce5f241a2553ae9a362dc7e14ba2e Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Thu, 3 Mar 2016 12:17:40 +1100 Subject: Update error message per review feedback --- src/cursor/cur_join.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/cursor/cur_join.c b/src/cursor/cur_join.c index 57497b5ce2a..1dee7f56bd5 100644 --- a/src/cursor/cur_join.c +++ b/src/cursor/cur_join.c @@ -434,8 +434,7 @@ __curjoin_init_iter(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin) if (session->txn.isolation == WT_ISO_READ_UNCOMMITTED) WT_RET_MSG(session, EINVAL, "join cursors with Bloom filters cannot be " - "used with WT_ISO_READ_UNCOMMITTED " - "isolation"); + "used with read-uncommitted isolation"); if (je->bloom == NULL) { /* * Look for compatible filters to be shared, -- cgit v1.2.1 From bbb1352e1667176348b3c5c820e63611f0c4c3f6 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Thu, 3 Mar 2016 15:45:55 +1100 Subject: Fixup test case error message to match new error message. --- test/suite/test_join06.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/suite/test_join06.py b/test/suite/test_join06.py index 1910b98d9bf..9af6f93792f 100644 --- a/test/suite/test_join06.py +++ b/test/suite/test_join06.py @@ -99,7 +99,7 @@ class test_join06(wttest.WiredTigerTestCase): if self.uncommitted and self.bloom: # Make sure that read-uncommitted with Bloom is not allowed. # This is detected on the first next() operation. - msg = '/cannot be used with WT_ISO_READ_UNCOMMITTED/' + msg = '/cannot be used with read-uncommitted/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: jc.next(), msg) return -- cgit v1.2.1 From 9a19889494447752612aaae6d463cc2cd96338c8 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Thu, 3 Mar 2016 16:04:17 +1100 Subject: WT-2381 Rewrite LSM metadata to fix dump / load. Include all of the WT_SESSION::create config in the ordinary LSM metadata so it is merged correctly into the dump header. Provide an upgrade path for LSM metadata in the old format. ** Backwards bracking change for LSM: ** once metadata is upgraded to the new format, LSM trees cannot be opened with older versions of WiredTiger. --- dist/api_data.py | 23 +++- src/config/config_def.c | 147 +++++++++++++++++++++ src/include/config.h | 16 ++- src/lsm/lsm_meta.c | 332 +++++++++++++++++++++++++++++++++++++++++++----- src/lsm/lsm_tree.c | 143 ++------------------- 5 files changed, 484 insertions(+), 177 deletions(-) diff --git a/dist/api_data.py b/dist/api_data.py index 0eee515e851..e57146fc4a5 100644 --- a/dist/api_data.py +++ b/dist/api_data.py @@ -76,12 +76,12 @@ lsm_config = [ Config('bloom', 'true', r''' create bloom filters on LSM tree chunks as they are merged''', type='boolean'), - Config('bloom_config', '', r''' - config string used when creating Bloom filter files, passed - to WT_SESSION::create'''), Config('bloom_bit_count', '16', r''' the number of bits used per item for LSM bloom filters''', min='2', max='1000'), + Config('bloom_config', '', r''' + config string used when creating Bloom filter files, passed + to WT_SESSION::create'''), Config('bloom_hash_count', '8', r''' the number of hash values per item used for LSM bloom filters''', @@ -299,6 +299,15 @@ file_meta = file_config + [ the file version'''), ] +lsm_meta = file_config + lsm_config + [ + Config('last', '', r''' + the last allocated chunk ID'''), + Config('chunks', '', r''' + active chunks in the LSM tree'''), + Config('old_chunks', '', r''' + obsolete chunks in the LSM tree'''), +] + table_only_config = [ Config('colgroups', '', r''' comma-separated list of names of column groups. Each column @@ -737,12 +746,16 @@ cursor_runtime_config = [ ] methods = { -'file.meta' : Method(file_meta), - 'colgroup.meta' : Method(colgroup_meta), +'file.config' : Method(file_config), + +'file.meta' : Method(file_meta), + 'index.meta' : Method(index_meta), +'lsm.meta' : Method(lsm_meta), + 'table.meta' : Method(table_meta), 'WT_CURSOR.close' : Method([]), diff --git a/src/config/config_def.c b/src/config/config_def.c index f1e48d85894..a8a91c29e9e 100644 --- a/src/config/config_def.c +++ b/src/config/config_def.c @@ -390,6 +390,61 @@ static const WT_CONFIG_CHECK confchk_colgroup_meta[] = { { NULL, NULL, NULL, NULL, NULL, 0 } }; +static const WT_CONFIG_CHECK confchk_file_config[] = { + { "allocation_size", "int", + NULL, "min=512B,max=128MB", + NULL, 0 }, + { "app_metadata", "string", NULL, NULL, NULL, 0 }, + { "block_allocation", "string", + NULL, "choices=[\"first\",\"best\"]", + NULL, 0 }, + { "block_compressor", "string", NULL, NULL, NULL, 0 }, + { "cache_resident", "boolean", NULL, NULL, NULL, 0 }, + { "checksum", "string", + NULL, "choices=[\"on\",\"off\",\"uncompressed\"]", + NULL, 0 }, + { "collator", "string", NULL, NULL, NULL, 0 }, + { "columns", "list", NULL, NULL, NULL, 0 }, + { "dictionary", "int", NULL, "min=0", NULL, 0 }, + { "encryption", "category", + NULL, NULL, + confchk_WT_SESSION_create_encryption_subconfigs, 2 }, + { "format", "string", NULL, "choices=[\"btree\"]", NULL, 0 }, + { "huffman_key", "string", NULL, NULL, NULL, 0 }, + { "huffman_value", "string", NULL, NULL, NULL, 0 }, + { "internal_item_max", "int", NULL, "min=0", NULL, 0 }, + { "internal_key_max", "int", NULL, "min=0", NULL, 0 }, + { "internal_key_truncate", "boolean", NULL, NULL, NULL, 0 }, + { "internal_page_max", "int", + NULL, "min=512B,max=512MB", + NULL, 0 }, + { "key_format", "format", __wt_struct_confchk, NULL, NULL, 0 }, + { "key_gap", "int", NULL, "min=0", NULL, 0 }, + { "leaf_item_max", "int", NULL, "min=0", NULL, 0 }, + { "leaf_key_max", "int", NULL, "min=0", NULL, 0 }, + { "leaf_page_max", "int", + NULL, "min=512B,max=512MB", + NULL, 0 }, + { "leaf_value_max", "int", NULL, "min=0", NULL, 0 }, + { "log", "category", + NULL, NULL, + confchk_WT_SESSION_create_log_subconfigs, 1 }, + { "memory_page_max", "int", + NULL, "min=512B,max=10TB", + NULL, 0 }, + { "os_cache_dirty_max", "int", NULL, "min=0", NULL, 0 }, + { "os_cache_max", "int", NULL, "min=0", NULL, 0 }, + { "prefix_compression", "boolean", NULL, NULL, NULL, 0 }, + { "prefix_compression_min", "int", NULL, "min=0", NULL, 0 }, + { "split_deepen_min_child", "int", NULL, NULL, NULL, 0 }, + { "split_deepen_per_child", "int", NULL, NULL, NULL, 0 }, + { "split_pct", "int", NULL, "min=25,max=100", NULL, 0 }, + { "value_format", "format", + __wt_struct_confchk, NULL, + NULL, 0 }, + { NULL, NULL, NULL, NULL, NULL, 0 } +}; + static const WT_CONFIG_CHECK confchk_file_meta[] = { { "allocation_size", "int", NULL, "min=512B,max=128MB", @@ -465,6 +520,67 @@ static const WT_CONFIG_CHECK confchk_index_meta[] = { { NULL, NULL, NULL, NULL, NULL, 0 } }; +static const WT_CONFIG_CHECK confchk_lsm_meta[] = { + { "allocation_size", "int", + NULL, "min=512B,max=128MB", + NULL, 0 }, + { "app_metadata", "string", NULL, NULL, NULL, 0 }, + { "block_allocation", "string", + NULL, "choices=[\"first\",\"best\"]", + NULL, 0 }, + { "block_compressor", "string", NULL, NULL, NULL, 0 }, + { "cache_resident", "boolean", NULL, NULL, NULL, 0 }, + { "checksum", "string", + NULL, "choices=[\"on\",\"off\",\"uncompressed\"]", + NULL, 0 }, + { "chunks", "string", NULL, NULL, NULL, 0 }, + { "collator", "string", NULL, NULL, NULL, 0 }, + { "columns", "list", NULL, NULL, NULL, 0 }, + { "dictionary", "int", NULL, "min=0", NULL, 0 }, + { "encryption", "category", + NULL, NULL, + confchk_WT_SESSION_create_encryption_subconfigs, 2 }, + { "format", "string", NULL, "choices=[\"btree\"]", NULL, 0 }, + { "huffman_key", "string", NULL, NULL, NULL, 0 }, + { "huffman_value", "string", NULL, NULL, NULL, 0 }, + { "internal_item_max", "int", NULL, "min=0", NULL, 0 }, + { "internal_key_max", "int", NULL, "min=0", NULL, 0 }, + { "internal_key_truncate", "boolean", NULL, NULL, NULL, 0 }, + { "internal_page_max", "int", + NULL, "min=512B,max=512MB", + NULL, 0 }, + { "key_format", "format", __wt_struct_confchk, NULL, NULL, 0 }, + { "key_gap", "int", NULL, "min=0", NULL, 0 }, + { "last", "string", NULL, NULL, NULL, 0 }, + { "leaf_item_max", "int", NULL, "min=0", NULL, 0 }, + { "leaf_key_max", "int", NULL, "min=0", NULL, 0 }, + { "leaf_page_max", "int", + NULL, "min=512B,max=512MB", + NULL, 0 }, + { "leaf_value_max", "int", NULL, "min=0", NULL, 0 }, + { "log", "category", + NULL, NULL, + confchk_WT_SESSION_create_log_subconfigs, 1 }, + { "lsm", "category", + NULL, NULL, + confchk_WT_SESSION_create_lsm_subconfigs, 11 }, + { "memory_page_max", "int", + NULL, "min=512B,max=10TB", + NULL, 0 }, + { "old_chunks", "string", NULL, NULL, NULL, 0 }, + { "os_cache_dirty_max", "int", NULL, "min=0", NULL, 0 }, + { "os_cache_max", "int", NULL, "min=0", NULL, 0 }, + { "prefix_compression", "boolean", NULL, NULL, NULL, 0 }, + { "prefix_compression_min", "int", NULL, "min=0", NULL, 0 }, + { "split_deepen_min_child", "int", NULL, NULL, NULL, 0 }, + { "split_deepen_per_child", "int", NULL, NULL, NULL, 0 }, + { "split_pct", "int", NULL, "min=25,max=100", NULL, 0 }, + { "value_format", "format", + __wt_struct_confchk, NULL, + NULL, 0 }, + { NULL, NULL, NULL, NULL, NULL, 0 } +}; + static const WT_CONFIG_CHECK confchk_table_meta[] = { { "app_metadata", "string", NULL, NULL, NULL, 0 }, { "colgroups", "list", NULL, NULL, NULL, 0 }, @@ -984,6 +1100,20 @@ static const WT_CONFIG_ENTRY config_entries[] = { "app_metadata=,collator=,columns=,source=,type=file", confchk_colgroup_meta, 5 }, + { "file.config", + "allocation_size=4KB,app_metadata=,block_allocation=best," + "block_compressor=,cache_resident=0,checksum=uncompressed," + "collator=,columns=,dictionary=0,encryption=(keyid=,name=)," + "format=btree,huffman_key=,huffman_value=,internal_item_max=0," + "internal_key_max=0,internal_key_truncate=,internal_page_max=4KB," + "key_format=u,key_gap=10,leaf_item_max=0,leaf_key_max=0," + "leaf_page_max=32KB,leaf_value_max=0,log=(enabled=)," + "memory_page_max=5MB,os_cache_dirty_max=0,os_cache_max=0," + "prefix_compression=0,prefix_compression_min=4," + "split_deepen_min_child=0,split_deepen_per_child=0,split_pct=75," + "value_format=u", + confchk_file_config, 33 + }, { "file.meta", "allocation_size=4KB,app_metadata=,block_allocation=best," "block_compressor=,cache_resident=0,checkpoint=,checkpoint_lsn=," @@ -1004,6 +1134,23 @@ static const WT_CONFIG_ENTRY config_entries[] = { "index_key_columns=,key_format=u,source=,type=file,value_format=u", confchk_index_meta, 10 }, + { "lsm.meta", + "allocation_size=4KB,app_metadata=,block_allocation=best," + "block_compressor=,cache_resident=0,checksum=uncompressed,chunks=" + ",collator=,columns=,dictionary=0,encryption=(keyid=,name=)," + "format=btree,huffman_key=,huffman_value=,internal_item_max=0," + "internal_key_max=0,internal_key_truncate=,internal_page_max=4KB," + "key_format=u,key_gap=10,last=,leaf_item_max=0,leaf_key_max=0," + "leaf_page_max=32KB,leaf_value_max=0,log=(enabled=)," + "lsm=(auto_throttle=,bloom=,bloom_bit_count=16,bloom_config=," + "bloom_hash_count=8,bloom_oldest=0,chunk_count_limit=0," + "chunk_max=5GB,chunk_size=10MB,merge_max=15,merge_min=0)," + "memory_page_max=5MB,old_chunks=,os_cache_dirty_max=0," + "os_cache_max=0,prefix_compression=0,prefix_compression_min=4," + "split_deepen_min_child=0,split_deepen_per_child=0,split_pct=75," + "value_format=u", + confchk_lsm_meta, 37 + }, { "table.meta", "app_metadata=,colgroups=,collator=,columns=,key_format=u," "value_format=u", diff --git a/src/include/config.h b/src/include/config.h index e63db0e76cf..48a255134af 100644 --- a/src/include/config.h +++ b/src/include/config.h @@ -85,13 +85,15 @@ struct __wt_config_parser_impl { #define WT_CONFIG_ENTRY_WT_SESSION_upgrade 33 #define WT_CONFIG_ENTRY_WT_SESSION_verify 34 #define WT_CONFIG_ENTRY_colgroup_meta 35 -#define WT_CONFIG_ENTRY_file_meta 36 -#define WT_CONFIG_ENTRY_index_meta 37 -#define WT_CONFIG_ENTRY_table_meta 38 -#define WT_CONFIG_ENTRY_wiredtiger_open 39 -#define WT_CONFIG_ENTRY_wiredtiger_open_all 40 -#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 41 -#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 42 +#define WT_CONFIG_ENTRY_file_config 36 +#define WT_CONFIG_ENTRY_file_meta 37 +#define WT_CONFIG_ENTRY_index_meta 38 +#define WT_CONFIG_ENTRY_lsm_meta 39 +#define WT_CONFIG_ENTRY_table_meta 40 +#define WT_CONFIG_ENTRY_wiredtiger_open 41 +#define WT_CONFIG_ENTRY_wiredtiger_open_all 42 +#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 43 +#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 44 /* * configuration section: END * DO NOT EDIT: automatically built by dist/flags.py. diff --git a/src/lsm/lsm_meta.c b/src/lsm/lsm_meta.c index d76b2a48aa7..3c7de0e6a60 100644 --- a/src/lsm/lsm_meta.c +++ b/src/lsm/lsm_meta.c @@ -9,17 +9,17 @@ #include "wt_internal.h" /* - * __wt_lsm_meta_read -- - * Read the metadata for an LSM tree. + * __lsm_meta_read_v0 -- + * Read v0 of LSM metadata. */ -int -__wt_lsm_meta_read(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) +static int +__lsm_meta_read_v0( + WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, const char *lsmconf) { WT_CONFIG cparser, lparser; WT_CONFIG_ITEM ck, cv, fileconf, lk, lv, metadata; WT_DECL_RET; WT_LSM_CHUNK *chunk; - char *lsmconfig; u_int nchunks; chunk = NULL; /* -Wconditional-uninitialized */ @@ -28,8 +28,7 @@ __wt_lsm_meta_read(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) if (F_ISSET(S2C(session), WT_CONN_LSM_MERGE)) F_SET(lsm_tree, WT_LSM_TREE_MERGES); - WT_RET(__wt_metadata_search(session, lsm_tree->name, &lsmconfig)); - WT_ERR(__wt_config_init(session, &cparser, lsmconfig)); + WT_ERR(__wt_config_init(session, &cparser, lsmconf)); while ((ret = __wt_config_next(&cparser, &ck, &cv)) == 0) { if (WT_STRING_MATCH("key_format", ck.str, ck.len)) { __wt_free(session, lsm_tree->key_format); @@ -48,7 +47,7 @@ __wt_lsm_meta_read(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) * from the file configuration. */ WT_ERR(__wt_config_getones( - session, lsmconfig, "file_config", &fileconf)); + session, lsmconf, "file_config", &fileconf)); WT_CLEAR(metadata); WT_ERR_NOTFOUND_OK(__wt_config_subgets( session, &fileconf, "app_metadata", &metadata)); @@ -169,7 +168,287 @@ __wt_lsm_meta_read(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) if (lsm_tree->merge_min < 2) lsm_tree->merge_min = WT_MAX(2, lsm_tree->merge_max / 2); -err: __wt_free(session, lsmconfig); +err: return (ret); +} + +/* + * __lsm_meta_read_v1 -- + * Read v1 of LSM metadata. + */ +static int +__lsm_meta_read_v1( + WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, const char *lsmconf) +{ + WT_CONFIG lparser; + WT_CONFIG_ITEM cv, lk, lv, metadata; + WT_DECL_ITEM(buf); + WT_DECL_RET; + WT_LSM_CHUNK *chunk; + const char *file_cfg[] = { + WT_CONFIG_BASE(session, file_config), NULL, NULL, NULL }; + char *fileconf; + u_int nchunks; + + chunk = NULL; /* -Wconditional-uninitialized */ + + /* + * Set up the config for each chunk. + * + * Make the memory_page_max double the chunk size, so application + * threads don't immediately try to force evict the chunk when the + * worker thread clears the NO_EVICTION flag. + */ + file_cfg[1] = lsmconf; + WT_ERR(__wt_scr_alloc(session, 0, &buf)); + WT_ERR(__wt_buf_fmt(session, buf, + "key_format=u,value_format=u,memory_page_max=%" PRIu64, + 2 * lsm_tree->chunk_max)); + file_cfg[2] = buf->data; + WT_ERR(__wt_config_collapse(session, file_cfg, &fileconf)); + lsm_tree->file_config = fileconf; + + WT_ERR(__wt_config_getones(session, lsmconf, "key_format", &cv)); + WT_ERR(__wt_strndup(session, cv.str, cv.len, &lsm_tree->key_format)); + WT_ERR(__wt_config_getones(session, lsmconf, "value_format", &cv)); + WT_ERR(__wt_strndup(session, cv.str, cv.len, &lsm_tree->value_format)); + + WT_ERR(__wt_config_getones(session, lsmconf, "collator", &cv)); + if (cv.len != 0 && !WT_STRING_MATCH("none", cv.str, cv.len)) { + /* Extract the application-supplied metadata (if any). */ + WT_CLEAR(metadata); + WT_ERR_NOTFOUND_OK(__wt_config_getones( + session, lsmconf, "app_metadata", &metadata)); + WT_ERR(__wt_collator_config(session, lsm_tree->name, + &cv, &metadata, + &lsm_tree->collator, &lsm_tree->collator_owned)); + WT_ERR(__wt_strndup(session, + cv.str, cv.len, &lsm_tree->collator_name)); + } + + WT_ERR(__wt_config_getones(session, lsmconf, "lsm.auto_throttle", &cv)); + if (cv.val) + F_SET(lsm_tree, WT_LSM_TREE_THROTTLE); + else + F_CLR(lsm_tree, WT_LSM_TREE_THROTTLE); + + WT_ERR(__wt_config_getones(session, lsmconf, "lsm.bloom", &cv)); + FLD_SET(lsm_tree->bloom, + (cv.val == 0 ? WT_LSM_BLOOM_OFF : WT_LSM_BLOOM_MERGED)); + WT_ERR(__wt_config_getones(session, lsmconf, "lsm.bloom_oldest", &cv)); + if (cv.val != 0) + FLD_SET(lsm_tree->bloom, WT_LSM_BLOOM_OLDEST); + + if (FLD_ISSET(lsm_tree->bloom, WT_LSM_BLOOM_OFF) && + FLD_ISSET(lsm_tree->bloom, WT_LSM_BLOOM_OLDEST)) + WT_ERR_MSG(session, EINVAL, + "Bloom filters can only be created on newest and oldest " + "chunks if bloom filters are enabled"); + + WT_ERR(__wt_config_getones( + session, lsmconf, "lsm.bloom_bit_count", &cv)); + lsm_tree->bloom_bit_count = (uint32_t)cv.val; + WT_ERR(__wt_config_getones(session, lsmconf, "lsm.bloom_config", &cv)); + /* Don't include the brackets. */ + if (cv.type == WT_CONFIG_ITEM_STRUCT) { + cv.str++; + cv.len -= 2; + } + WT_ERR(__wt_config_check(session, + WT_CONFIG_REF(session, WT_SESSION_create), cv.str, cv.len)); + WT_ERR(__wt_strndup( + session, cv.str, cv.len, &lsm_tree->bloom_config)); + WT_ERR(__wt_config_getones( + session, lsmconf, "lsm.bloom_hash_count", &cv)); + lsm_tree->bloom_hash_count = (uint32_t)cv.val; + + WT_ERR(__wt_config_getones( + session, lsmconf, "lsm.chunk_count_limit", &cv)); + lsm_tree->chunk_count_limit = (uint32_t)cv.val; + if (cv.val == 0) + F_SET(lsm_tree, WT_LSM_TREE_MERGES); + else + F_CLR(lsm_tree, WT_LSM_TREE_MERGES); + WT_ERR(__wt_config_getones(session, lsmconf, "lsm.chunk_max", &cv)); + lsm_tree->chunk_max = (uint64_t)cv.val; + WT_ERR(__wt_config_getones(session, lsmconf, "lsm.chunk_size", &cv)); + lsm_tree->chunk_size = (uint64_t)cv.val; + + if (lsm_tree->chunk_size > lsm_tree->chunk_max) + WT_ERR_MSG(session, EINVAL, + "Chunk size (chunk_size) must be smaller than or equal to " + "the maximum chunk size (chunk_max)"); + + WT_ERR(__wt_config_getones(session, lsmconf, "lsm.merge_max", &cv)); + lsm_tree->merge_max = (uint32_t)cv.val; + WT_ERR(__wt_config_getones(session, lsmconf, "lsm.merge_min", &cv)); + lsm_tree->merge_min = (uint32_t)cv.val; + + if (lsm_tree->merge_min > lsm_tree->merge_max) + WT_ERR_MSG(session, EINVAL, + "LSM merge_min must be less than or equal to merge_max"); + + WT_ERR(__wt_config_getones(session, lsmconf, "last", &cv)); + lsm_tree->last = (u_int)cv.val; + WT_ERR(__wt_config_getones(session, lsmconf, "chunks", &cv)); + WT_ERR(__wt_config_subinit(session, &lparser, &cv)); + for (nchunks = 0; (ret = + __wt_config_next(&lparser, &lk, &lv)) == 0; ) { + if (WT_STRING_MATCH("id", lk.str, lk.len)) { + WT_ERR(__wt_realloc_def(session, + &lsm_tree->chunk_alloc, + nchunks + 1, &lsm_tree->chunk)); + WT_ERR( + __wt_calloc_one(session, &chunk)); + lsm_tree->chunk[nchunks++] = chunk; + chunk->id = (uint32_t)lv.val; + WT_ERR(__wt_lsm_tree_chunk_name(session, + lsm_tree, chunk->id, &chunk->uri)); + F_SET(chunk, + WT_LSM_CHUNK_ONDISK | + WT_LSM_CHUNK_STABLE); + } else if (WT_STRING_MATCH( + "bloom", lk.str, lk.len)) { + WT_ERR(__wt_lsm_tree_bloom_name( + session, lsm_tree, + chunk->id, &chunk->bloom_uri)); + F_SET(chunk, WT_LSM_CHUNK_BLOOM); + continue; + } else if (WT_STRING_MATCH( + "chunk_size", lk.str, lk.len)) { + chunk->size = (uint64_t)lv.val; + continue; + } else if (WT_STRING_MATCH( + "count", lk.str, lk.len)) { + chunk->count = (uint64_t)lv.val; + continue; + } else if (WT_STRING_MATCH( + "generation", lk.str, lk.len)) { + chunk->generation = (uint32_t)lv.val; + continue; + } + } + WT_ERR_NOTFOUND_OK(ret); + lsm_tree->nchunks = nchunks; + + WT_ERR(__wt_config_getones(session, lsmconf, "old_chunks", &cv)); + WT_ERR(__wt_config_subinit(session, &lparser, &cv)); + for (nchunks = 0; (ret = + __wt_config_next(&lparser, &lk, &lv)) == 0; ) { + if (WT_STRING_MATCH("bloom", lk.str, lk.len)) { + WT_ERR(__wt_strndup(session, + lv.str, lv.len, &chunk->bloom_uri)); + F_SET(chunk, WT_LSM_CHUNK_BLOOM); + continue; + } + WT_ERR(__wt_realloc_def(session, + &lsm_tree->old_alloc, nchunks + 1, + &lsm_tree->old_chunks)); + WT_ERR(__wt_calloc_one(session, &chunk)); + lsm_tree->old_chunks[nchunks++] = chunk; + WT_ERR(__wt_strndup(session, + lk.str, lk.len, &chunk->uri)); + F_SET(chunk, WT_LSM_CHUNK_ONDISK); + } + WT_ERR_NOTFOUND_OK(ret); + lsm_tree->nold_chunks = nchunks; + + /* + * Ignore any other values: the metadata entry might have been + * created by a future release, with unknown options. + */ + +err: __wt_scr_free(session, &buf); + return (ret); +} + +/* + * __lsm_meta_upgrade_v1 -- + * Upgrade to v1 of LSM metadata. + */ +static int +__lsm_meta_upgrade_v1(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) +{ + WT_DECL_ITEM(buf); + WT_DECL_RET; + const char *new_cfg[] = { + WT_CONFIG_BASE(session, lsm_meta), NULL, NULL, NULL }; + + /* Include the custom config that used to be embedded in file_config. */ + new_cfg[1] = lsm_tree->file_config; + + WT_ERR(__wt_scr_alloc(session, 0, &buf)); + WT_ERR(__wt_buf_fmt(session, buf, + "key_format=%s,value_format=%s", + lsm_tree->key_format, lsm_tree->value_format)); + + WT_ERR(__wt_buf_catfmt(session, buf, ",collator=%s", + lsm_tree->collator_name != NULL ? lsm_tree->collator_name : "")); + + WT_ERR(__wt_buf_catfmt(session, buf, ",lsm=(")); + + WT_ERR(__wt_buf_catfmt(session, buf, "auto_throttle=%d", + F_ISSET(lsm_tree, WT_LSM_TREE_THROTTLE))); + + WT_ERR(__wt_buf_catfmt(session, buf, ",bloom=%d", + FLD_ISSET(lsm_tree->bloom, WT_LSM_BLOOM_MERGED))); + WT_ERR(__wt_buf_catfmt(session, buf, ",bloom_oldest=%d", + FLD_ISSET(lsm_tree->bloom, WT_LSM_BLOOM_OLDEST))); + WT_ERR(__wt_buf_catfmt(session, buf, ",bloom_bit_count=%" PRIu32, + lsm_tree->bloom_bit_count)); + if (lsm_tree->bloom_config != NULL && + strlen(lsm_tree->bloom_config) > 0) + WT_ERR(__wt_buf_catfmt(session, buf, ",bloom_config=(%s)", + lsm_tree->bloom_config)); + else + WT_ERR(__wt_buf_catfmt(session, buf, ",bloom_config=")); + WT_ERR(__wt_buf_catfmt(session, buf, ",bloom_hash_count=%" PRIu32, + lsm_tree->bloom_hash_count)); + + WT_ERR(__wt_buf_catfmt(session, buf, ",chunk_count_limit=%" PRIu32, + lsm_tree->chunk_count_limit)); + WT_ERR(__wt_buf_catfmt(session, buf, ",chunk_max=%" PRIu64, + lsm_tree->chunk_max)); + WT_ERR(__wt_buf_catfmt(session, buf, ",merge_max=%" PRIu32, + lsm_tree->merge_max)); + WT_ERR(__wt_buf_catfmt(session, buf, ",merge_min=%" PRIu32, + lsm_tree->merge_min)); + + WT_ERR(__wt_buf_catfmt(session, buf, ")")); + + new_cfg[2] = buf->data; + WT_ERR(__wt_config_merge(session, new_cfg, NULL, &lsm_tree->config)); + +err: __wt_scr_free(session, &buf); + return (ret); +} +/* + * __wt_lsm_meta_read -- + * Read the metadata for an LSM tree. + */ +int +__wt_lsm_meta_read(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) +{ + WT_CONFIG_ITEM cval; + WT_DECL_RET; + char *lsmconf; + + /* LSM trees inherit the merge setting from the connection. */ + if (F_ISSET(S2C(session), WT_CONN_LSM_MERGE)) + F_SET(lsm_tree, WT_LSM_TREE_MERGES); + + WT_RET(__wt_metadata_search(session, lsm_tree->name, &lsmconf)); + + ret = __wt_config_getones(session, lsmconf, "file_config", &cval); + if (ret == 0) { + ret = __lsm_meta_read_v0(session, lsm_tree, lsmconf); + __wt_free(session, lsmconf); + WT_RET(ret); + return (__lsm_meta_upgrade_v1(session, lsm_tree)); + } else if (ret == WT_NOTFOUND) { + lsm_tree->config = lsmconf; + return (__lsm_meta_read_v1(session, lsm_tree, lsmconf)); + } + return (ret); } @@ -183,33 +462,16 @@ __wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_DECL_ITEM(buf); WT_DECL_RET; WT_LSM_CHUNK *chunk; + const char *new_cfg[] = { NULL, NULL, NULL }; + char *new_metadata; u_int i; bool first; + new_metadata = NULL; + WT_RET(__wt_scr_alloc(session, 0, &buf)); - WT_ERR(__wt_buf_fmt(session, buf, - "key_format=%s,value_format=%s,bloom_config=(%s),file_config=(%s)", - lsm_tree->key_format, lsm_tree->value_format, - lsm_tree->bloom_config, lsm_tree->file_config)); - if (lsm_tree->collator_name != NULL) - WT_ERR(__wt_buf_catfmt( - session, buf, ",collator=%s", lsm_tree->collator_name)); WT_ERR(__wt_buf_catfmt(session, buf, - ",last=%" PRIu32 - ",chunk_count_limit=%" PRIu32 - ",chunk_max=%" PRIu64 - ",chunk_size=%" PRIu64 - ",auto_throttle=%" PRIu32 - ",merge_max=%" PRIu32 - ",merge_min=%" PRIu32 - ",bloom=%" PRIu32 - ",bloom_bit_count=%" PRIu32 - ",bloom_hash_count=%" PRIu32, - lsm_tree->last, lsm_tree->chunk_count_limit, - lsm_tree->chunk_max, lsm_tree->chunk_size, - F_ISSET(lsm_tree, WT_LSM_TREE_THROTTLE) ? 1 : 0, - lsm_tree->merge_max, lsm_tree->merge_min, lsm_tree->bloom, - lsm_tree->bloom_bit_count, lsm_tree->bloom_hash_count)); + ",last=%" PRIu32, lsm_tree->last)); WT_ERR(__wt_buf_catfmt(session, buf, ",chunks=[")); for (i = 0; i < lsm_tree->nchunks; i++) { chunk = lsm_tree->chunk[i]; @@ -243,9 +505,15 @@ __wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) session, buf, ",bloom=\"%s\"", chunk->bloom_uri)); } WT_ERR(__wt_buf_catfmt(session, buf, "]")); - ret = __wt_metadata_update(session, lsm_tree->name, buf->data); + + /* Update the existing configuration with the new values. */ + new_cfg[0] = lsm_tree->config; + new_cfg[1] = buf->data; + WT_ERR(__wt_config_collapse(session, new_cfg, &new_metadata)); + ret = __wt_metadata_update(session, lsm_tree->name, new_metadata); WT_ERR(ret); err: __wt_scr_free(session, &buf); + __wt_free(session, new_metadata); return (ret); } diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index 7c188bf3dc7..f34f737baae 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -306,15 +306,15 @@ int __wt_lsm_tree_create(WT_SESSION_IMPL *session, const char *uri, bool exclusive, const char *config) { - WT_CONFIG_ITEM cval; - WT_DECL_ITEM(buf); WT_DECL_RET; WT_LSM_TREE *lsm_tree; const char *cfg[] = - { WT_CONFIG_BASE(session, WT_SESSION_create), config, NULL }; - char *tmpconfig; + { WT_CONFIG_BASE(session, lsm_meta), config, NULL }; + const char *metadata; + + metadata = NULL; - /* If the tree is open, it already exists. */ + /* If the tree can be opened, it already exists. */ WT_WITH_HANDLE_LIST_LOCK(session, ret = __wt_lsm_tree_get(session, uri, false, &lsm_tree)); if (ret == 0) { @@ -323,128 +323,9 @@ __wt_lsm_tree_create(WT_SESSION_IMPL *session, } WT_RET_NOTFOUND_OK(ret); - /* - * If the tree has metadata, it already exists. - * - * !!! - * Use a local variable: we don't care what the existing configuration - * is, but we don't want to overwrite the real config. - */ - if (__wt_metadata_search(session, uri, &tmpconfig) == 0) { - __wt_free(session, tmpconfig); - return (exclusive ? EEXIST : 0); - } - WT_RET_NOTFOUND_OK(ret); - - /* In-memory configurations don't make sense for LSM. */ - if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) - WT_RET_MSG(session, EINVAL, - "LSM trees not supported by in-memory configurations"); - - WT_RET(__wt_config_gets(session, cfg, "key_format", &cval)); - if (WT_STRING_MATCH("r", cval.str, cval.len)) - WT_RET_MSG(session, EINVAL, - "LSM trees cannot be configured as column stores"); - - WT_RET(__wt_calloc_one(session, &lsm_tree)); - - WT_ERR(__lsm_tree_set_name(session, lsm_tree, uri)); - - WT_ERR(__wt_config_gets(session, cfg, "key_format", &cval)); - WT_ERR(__wt_strndup( - session, cval.str, cval.len, &lsm_tree->key_format)); - WT_ERR(__wt_config_gets(session, cfg, "value_format", &cval)); - WT_ERR(__wt_strndup( - session, cval.str, cval.len, &lsm_tree->value_format)); - - WT_ERR(__wt_config_gets_none(session, cfg, "collator", &cval)); - WT_ERR(__wt_strndup( - session, cval.str, cval.len, &lsm_tree->collator_name)); - - WT_ERR(__wt_config_gets(session, cfg, "cache_resident", &cval)); - if (cval.val != 0) - WT_ERR_MSG(session, EINVAL, - "The cache_resident flag is not compatible with LSM"); - - WT_ERR(__wt_config_gets(session, cfg, "lsm.auto_throttle", &cval)); - if (cval.val) - F_SET(lsm_tree, WT_LSM_TREE_THROTTLE); - else - F_CLR(lsm_tree, WT_LSM_TREE_THROTTLE); - WT_ERR(__wt_config_gets(session, cfg, "lsm.bloom", &cval)); - FLD_SET(lsm_tree->bloom, - (cval.val == 0 ? WT_LSM_BLOOM_OFF : WT_LSM_BLOOM_MERGED)); - WT_ERR(__wt_config_gets(session, cfg, "lsm.bloom_oldest", &cval)); - if (cval.val != 0) - FLD_SET(lsm_tree->bloom, WT_LSM_BLOOM_OLDEST); - - if (FLD_ISSET(lsm_tree->bloom, WT_LSM_BLOOM_OFF) && - FLD_ISSET(lsm_tree->bloom, WT_LSM_BLOOM_OLDEST)) - WT_ERR_MSG(session, EINVAL, - "Bloom filters can only be created on newest and oldest " - "chunks if bloom filters are enabled"); - - WT_ERR(__wt_config_gets(session, cfg, "lsm.bloom_config", &cval)); - if (cval.type == WT_CONFIG_ITEM_STRUCT) { - cval.str++; - cval.len -= 2; - } - WT_ERR(__wt_config_check(session, - WT_CONFIG_REF(session, WT_SESSION_create), cval.str, cval.len)); - WT_ERR(__wt_strndup( - session, cval.str, cval.len, &lsm_tree->bloom_config)); - - WT_ERR(__wt_config_gets(session, cfg, "lsm.bloom_bit_count", &cval)); - lsm_tree->bloom_bit_count = (uint32_t)cval.val; - WT_ERR(__wt_config_gets(session, cfg, "lsm.bloom_hash_count", &cval)); - lsm_tree->bloom_hash_count = (uint32_t)cval.val; - WT_ERR(__wt_config_gets(session, cfg, "lsm.chunk_count_limit", &cval)); - lsm_tree->chunk_count_limit = (uint32_t)cval.val; - if (cval.val == 0) - F_SET(lsm_tree, WT_LSM_TREE_MERGES); - else - F_CLR(lsm_tree, WT_LSM_TREE_MERGES); - WT_ERR(__wt_config_gets(session, cfg, "lsm.chunk_max", &cval)); - lsm_tree->chunk_max = (uint64_t)cval.val; - WT_ERR(__wt_config_gets(session, cfg, "lsm.chunk_size", &cval)); - lsm_tree->chunk_size = (uint64_t)cval.val; - if (lsm_tree->chunk_size > lsm_tree->chunk_max) - WT_ERR_MSG(session, EINVAL, - "Chunk size (chunk_size) must be smaller than or equal to " - "the maximum chunk size (chunk_max)"); - WT_ERR(__wt_config_gets(session, cfg, "lsm.merge_max", &cval)); - lsm_tree->merge_max = (uint32_t)cval.val; - WT_ERR(__wt_config_gets(session, cfg, "lsm.merge_min", &cval)); - lsm_tree->merge_min = (uint32_t)cval.val; - if (lsm_tree->merge_min > lsm_tree->merge_max) - WT_ERR_MSG(session, EINVAL, - "LSM merge_min must be less than or equal to merge_max"); - if (!F_ISSET(S2C(session), WT_CONN_READONLY)) { - /* - * Set up the config for each chunk. - * - * Make the memory_page_max double the chunk size, so - * application threads don't immediately try to force evict - * the chunk when the worker thread clears the NO_EVICTION flag. - */ - WT_ERR(__wt_scr_alloc(session, 0, &buf)); - WT_ERR(__wt_buf_fmt(session, buf, - "%s,key_format=u,value_format=u,memory_page_max=%" PRIu64, - config, 2 * lsm_tree->chunk_max)); - WT_ERR(__wt_strndup( - session, buf->data, buf->size, &lsm_tree->file_config)); - - /* Create the first chunk and flush the metadata. */ - WT_ERR(__wt_lsm_meta_write(session, lsm_tree)); - - /* Discard our partially populated handle. */ - ret = __lsm_tree_discard(session, lsm_tree, false); - lsm_tree = NULL; - } else { - F_CLR(lsm_tree, WT_LSM_TREE_MERGES); - FLD_SET(lsm_tree->bloom, WT_LSM_BLOOM_OFF); - FLD_CLR(lsm_tree->bloom, WT_LSM_BLOOM_OLDEST); + WT_ERR(__wt_config_merge(session, cfg, NULL, &metadata)); + WT_ERR(__wt_metadata_insert(session, uri, metadata)); } /* @@ -452,16 +333,12 @@ __wt_lsm_tree_create(WT_SESSION_IMPL *session, * error: the returned handle is NULL on error, and the metadata * tracking macros handle cleaning up on failure. */ - if (ret == 0) - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __lsm_tree_open(session, uri, true, &lsm_tree)); + WT_WITH_HANDLE_LIST_LOCK(session, + ret = __lsm_tree_open(session, uri, true, &lsm_tree)); if (ret == 0) __wt_lsm_tree_release(session, lsm_tree); - if (0) { -err: WT_TRET(__lsm_tree_discard(session, lsm_tree, false)); - } - __wt_scr_free(session, &buf); +err: __wt_free(session, metadata); return (ret); } -- cgit v1.2.1 From b3b6c5383f78b6bff81062f5b40739a2f5ce4ab8 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Thu, 3 Mar 2016 16:16:29 +1100 Subject: WT-2381 Add upgrading notes. --- src/docs/upgrading.dox | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/docs/upgrading.dox b/src/docs/upgrading.dox index df0a22ba0fe..8b3d61e4c19 100644 --- a/src/docs/upgrading.dox +++ b/src/docs/upgrading.dox @@ -2,6 +2,14 @@ @section version_271 Upgrading to Version 2.7.1
+
LSM metadata
+
+There is a change to the format of LSM metadata in this release to fix bugs +in dump / load of tables of type LSM. Tables created with the old LSM metadata +format will be upgraded automatically, but once updated to the new version +are no longer compatible with older releases of WiredTiger. +
+
Column-store bulk-load cursors
Historically, bulk-load of a column-store object ignored any key set in the -- cgit v1.2.1 From 841de414da06be0293d38586595df93885c16b42 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 3 Mar 2016 08:43:17 -0500 Subject: WT-2444: broken flag test in wtperf, whitespace --- bench/wtperf/wtperf.c | 2 +- src/cursor/cur_join.c | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/bench/wtperf/wtperf.c b/bench/wtperf/wtperf.c index 5755e22dd2f..b04c6ba662d 100644 --- a/bench/wtperf/wtperf.c +++ b/bench/wtperf/wtperf.c @@ -2267,7 +2267,7 @@ main(int argc, char *argv[]) * the compact operation, but not for the workloads. */ if (cfg->async_threads > 0) { - if (F_ISSET(cfg, CFG_TRUNCATE) > 0) { + if (F_ISSET(cfg, CFG_TRUNCATE)) { lprintf(cfg, 1, 0, "Cannot run truncate and async\n"); goto err; } diff --git a/src/cursor/cur_join.c b/src/cursor/cur_join.c index 68d06b58013..2ef65b5bddf 100644 --- a/src/cursor/cur_join.c +++ b/src/cursor/cur_join.c @@ -8,9 +8,8 @@ #include "wt_internal.h" -static int -__curjoin_insert_endpoint(WT_SESSION_IMPL *, WT_CURSOR_JOIN_ENTRY *, u_int, - WT_CURSOR_JOIN_ENDPOINT **); +static int __curjoin_insert_endpoint(WT_SESSION_IMPL *, + WT_CURSOR_JOIN_ENTRY *, u_int, WT_CURSOR_JOIN_ENDPOINT **); /* * __curjoin_entry_iter_init -- @@ -464,7 +463,7 @@ __curjoin_init_iter(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin) F_SET(cjoin, WT_CURJOIN_SKIP_FIRST_LEFT); if (F_ISSET(je, WT_CURJOIN_ENTRY_BLOOM)) { - if (session->txn.isolation == WT_ISO_READ_UNCOMMITTED) + if (session->txn.isolation == WT_ISO_READ_UNCOMMITTED) WT_RET_MSG(session, EINVAL, "join cursors with Bloom filters cannot be " "used with read-uncommitted isolation"); -- cgit v1.2.1 From c24e88f35a7e50177b322a26ca64e3aec453c38d Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Thu, 3 Mar 2016 11:25:07 -0500 Subject: WT-2443 Fix how a statistics 'set' is initialized. --- src/cursor/cur_stat.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/cursor/cur_stat.c b/src/cursor/cur_stat.c index 34e64b34ccb..f7a8f5fc866 100644 --- a/src/cursor/cur_stat.c +++ b/src/cursor/cur_stat.c @@ -207,6 +207,8 @@ __curstat_next(WT_CURSOR *cursor) if (cst->notpositioned) { cst->notpositioned = false; cst->key = WT_STAT_KEY_MIN(cst); + if (cst->next_set != NULL) + WT_ERR((*cst->next_set)(session, cst, true, true)); } else if (cst->key < WT_STAT_KEY_MAX(cst)) ++cst->key; else if (cst->next_set != NULL) @@ -249,6 +251,8 @@ __curstat_prev(WT_CURSOR *cursor) if (cst->notpositioned) { cst->notpositioned = false; cst->key = WT_STAT_KEY_MAX(cst); + if (cst->next_set != NULL) + WT_ERR((*cst->next_set)(session, cst, false, true)); } else if (cst->key > WT_STAT_KEY_MIN(cst)) --cst->key; else if (cst->next_set != NULL) @@ -558,9 +562,6 @@ __wt_curstat_init(WT_SESSION_IMPL *session, else return (__wt_bad_object_type(session, uri)); - if (cst->next_set != NULL) - WT_RET((*cst->next_set)(session, cst, false, true)); - return (0); } -- cgit v1.2.1 From 879d36efed8a0acc9a190dc65dbad6864b565abd Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Thu, 3 Mar 2016 11:26:20 -0500 Subject: WT-2443 Added more thorough tests for statistics for join cursors. --- test/suite/test_join01.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/test/suite/test_join01.py b/test/suite/test_join01.py index 539a3a3ae57..27d35808b52 100644 --- a/test/suite/test_join01.py +++ b/test/suite/test_join01.py @@ -74,8 +74,18 @@ class test_join01(wttest.WiredTigerTestCase): # the join cursor and iterating again. def stats(self, jc, which): statcur = self.session.open_cursor('statistics:join', jc, None) - self.check_stats(statcur, 0, 'join: index:join01:index1: ' + - 'bloom filter false positives') + # pick a stat we always expect to see + statdesc = 'bloom filter false positives' + expectstats = [ + 'join: index:join01:index1: ' + statdesc, + 'join: index:join01:index2: ' + statdesc ] + if self.ref == 'index': + expectstats.append('join: index:join01:index0: ' + statdesc) + else: + expectstats.append('join: table:join01: ' + statdesc) + self.check_stats(statcur, expectstats) + statcur.reset() + self.check_stats(statcur, expectstats) statcur.close() def statstr_to_int(self, str): @@ -86,16 +96,14 @@ class test_join01(wttest.WiredTigerTestCase): parts = str.rpartition('(') return int(parts[2].rstrip(')')) - # string should appear with a minimum value of least "min". - def check_stats(self, statcursor, min, lookfor): + # All of the expect strings should appear + def check_stats(self, statcursor, expectstats): stringclass = ''.__class__ intclass = (0).__class__ # Reset the cursor, we're called multiple times. statcursor.reset() - found = False - foundval = 0 self.printVerbose(3, 'statistics:') for id, desc, valstr, val in statcursor: self.assertEqual(type(desc), stringclass) @@ -104,12 +112,11 @@ class test_join01(wttest.WiredTigerTestCase): self.assertEqual(val, self.statstr_to_int(valstr)) self.printVerbose(3, ' stat: \'' + desc + '\', \'' + valstr + '\', ' + str(val)) - if desc == lookfor: - found = True - foundval = val + if desc in expectstats: + expectstats.remove(desc) - self.assertTrue(found, 'in stats, did not see: ' + lookfor) - self.assertTrue(foundval >= min) + self.assertTrue(len(expectstats) == 0, + 'missing expected values in stats: ' + str(expectstats)) # Common function for testing the most basic functionality # of joins -- cgit v1.2.1 From 935f61f9245a6e1f81327a7e9b20c322d06ce1b5 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 3 Mar 2016 13:38:54 -0500 Subject: WT-2381 Add LSM to dump test. --- test/suite/test_dump.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/suite/test_dump.py b/test/suite/test_dump.py index c850d1b5d3f..752e4d6a36a 100644 --- a/test/suite/test_dump.py +++ b/test/suite/test_dump.py @@ -57,6 +57,9 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): ('file', dict(type='file:', populate=simple_populate, populate_check=simple_populate_check_cursor)), + ('lsm', dict(type='lsm:', + populate=simple_populate, + populate_check=simple_populate_check_cursor)), ('table-simple', dict(type='table:', populate=simple_populate, populate_check=simple_populate_check_cursor)), -- cgit v1.2.1 From 71f0c4dcfdfb53cb364cc8ab779be735d5c3bfa0 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 3 Mar 2016 15:58:28 -0500 Subject: WT-2381: dump utility discards table config Allow any URI through with an underlying type of file, that allows the creation of top-level lsm:XXX objects, that is, LSM objects that aren't underneath tables. --- src/session/session_api.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/session/session_api.c b/src/session/session_api.c index 3153ca458eb..b3478554450 100644 --- a/src/session/session_api.c +++ b/src/session/session_api.c @@ -493,10 +493,13 @@ __session_create(WT_SESSION *wt_session, const char *uri, const char *config) /* * We can't disallow type entirely, a configuration string might * innocently include it, for example, a dump/load pair. If the - * URI type prefix and the type are the same, let it go. + * underlying type is "file", it's OK ("file" is the underlying + * type for every type); if the URI type prefix and the type are + * the same, let it go. */ if ((ret = __wt_config_getones(session, config, "type", &cval)) == 0 && + !WT_STRING_MATCH("file", cval.str, cval.len) && (strncmp(uri, cval.str, cval.len) != 0 || uri[cval.len] != ':')) WT_ERR_MSG(session, EINVAL, -- cgit v1.2.1 From 73d925b40cbd2143d9a705de81a6cc4a262aae22 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Fri, 4 Mar 2016 15:43:07 +1100 Subject: WT-2318 Add implementation for auto adjusting condition variables. Where the amount of time spent waiting varies depending on how whether work is being done between timeouts. --- dist/filelist | 1 + src/conn/conn_cache.c | 6 +-- src/evict/evict_lru.c | 9 ++-- src/include/extern.h | 4 ++ src/include/mutex.h | 7 +++ src/support/cond_auto.c | 115 ++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 135 insertions(+), 7 deletions(-) create mode 100644 src/support/cond_auto.c diff --git a/dist/filelist b/dist/filelist index 4ed7d7e3beb..350e0c50087 100644 --- a/dist/filelist +++ b/dist/filelist @@ -153,6 +153,7 @@ src/session/session_compact.c src/session/session_dhandle.c src/session/session_salvage.c src/support/cksum.c +src/support/cond_auto.c src/support/crypto.c src/support/err.c src/support/filename.c diff --git a/src/conn/conn_cache.c b/src/conn/conn_cache.c index 1831aad5895..0f370856881 100644 --- a/src/conn/conn_cache.c +++ b/src/conn/conn_cache.c @@ -147,8 +147,8 @@ __wt_cache_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR_MSG(session, EINVAL, "eviction target must be lower than the eviction trigger"); - WT_ERR(__wt_cond_alloc(session, - "cache eviction server", false, &cache->evict_cond)); + WT_ERR(__wt_cond_auto_alloc(session, "cache eviction server", + false, 10000, WT_MILLION, &cache->evict_cond)); WT_ERR(__wt_cond_alloc(session, "eviction waiters", false, &cache->evict_waiter_cond)); WT_ERR(__wt_spin_init(session, &cache->evict_lock, "cache eviction")); @@ -246,7 +246,7 @@ __wt_cache_destroy(WT_SESSION_IMPL *session) " bytes dirty and %" PRIu64 " pages dirty", cache->bytes_dirty, cache->pages_dirty); - WT_TRET(__wt_cond_destroy(session, &cache->evict_cond)); + WT_TRET(__wt_cond_auto_destroy(session, &cache->evict_cond)); WT_TRET(__wt_cond_destroy(session, &cache->evict_waiter_cond)); __wt_spin_destroy(session, &cache->evict_lock); __wt_spin_destroy(session, &cache->evict_walk_lock); diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index 884c08a02df..46a40dc124f 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -175,8 +175,8 @@ __evict_server(void *arg) WT_SESSION_IMPL *session; #ifdef HAVE_DIAGNOSTIC struct timespec now, stuck_ts; - uint64_t pages_evicted = 0; #endif + uint64_t pages_evicted = 0; u_int spins; session = arg; @@ -219,11 +219,11 @@ __evict_server(void *arg) /* Next time we wake up, reverse the sweep direction. */ cache->flags ^= WT_CACHE_WALK_REVERSE; -#ifdef HAVE_DIAGNOSTIC pages_evicted = 0; } else if (pages_evicted != cache->pages_evict) { - WT_ERR(__wt_epoch(session, &stuck_ts)); pages_evicted = cache->pages_evict; +#ifdef HAVE_DIAGNOSTIC + WT_ERR(__wt_epoch(session, &stuck_ts)); } else { /* After being stuck for 5 minutes, give up. */ WT_ERR(__wt_epoch(session, &now)); @@ -238,7 +238,8 @@ __evict_server(void *arg) WT_ERR(__wt_verbose(session, WT_VERB_EVICTSERVER, "sleeping")); /* Don't rely on signals: check periodically. */ - WT_ERR(__wt_cond_wait(session, cache->evict_cond, 100000)); + WT_ERR(__wt_cond_auto_wait( + session, cache->evict_cond, pages_evicted != 0)); WT_ERR(__wt_verbose(session, WT_VERB_EVICTSERVER, "waking")); } diff --git a/src/include/extern.h b/src/include/extern.h index 55b0b8cd7ff..e52cf51779b 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -637,6 +637,10 @@ extern int __wt_session_lock_checkpoint(WT_SESSION_IMPL *session, const char *ch extern int __wt_salvage(WT_SESSION_IMPL *session, const char *cfg[]); extern uint32_t __wt_cksum(const void *chunk, size_t len); extern void __wt_cksum_init(void); +extern int __wt_cond_auto_alloc( WT_SESSION_IMPL *session, const char *name, bool is_signalled, uint64_t min, uint64_t max, WT_CONDVAR **condp); +extern int __wt_cond_auto_wait_signal( WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress, bool *signalled); +extern int __wt_cond_auto_wait( WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress); +extern int __wt_cond_auto_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp); extern int __wt_decrypt(WT_SESSION_IMPL *session, WT_ENCRYPTOR *encryptor, size_t skip, WT_ITEM *in, WT_ITEM *out); extern int __wt_encrypt(WT_SESSION_IMPL *session, WT_KEYED_ENCRYPTOR *kencryptor, size_t skip, WT_ITEM *in, WT_ITEM *out); extern void __wt_encrypt_size(WT_SESSION_IMPL *session, WT_KEYED_ENCRYPTOR *kencryptor, size_t incoming_size, size_t *sizep); diff --git a/src/include/mutex.h b/src/include/mutex.h index f798bfb3ece..04679884930 100644 --- a/src/include/mutex.h +++ b/src/include/mutex.h @@ -20,6 +20,13 @@ struct __wt_condvar { int waiters; /* Numbers of waiters, or -1 if signalled with no waiters. */ + /* + * The following fields are only used for automatically adjusting + * condition variables. They could be in a separate structure. + */ + uint64_t min_wait; /* Minimum wait duration */ + uint64_t max_wait; /* Maximum wait duration */ + uint64_t prev_wait; /* Wait duration used last time */ }; /* diff --git a/src/support/cond_auto.c b/src/support/cond_auto.c new file mode 100644 index 00000000000..0c255bd1c98 --- /dev/null +++ b/src/support/cond_auto.c @@ -0,0 +1,115 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "wt_internal.h" + +/* + * This is an implementation of condition variables that automatically adjust + * the wait time depending on whether the wake is resulting in useful work. + */ + +/* + * __wt_cond_auto_alloc -- + * Allocate and initialize an automatically adjusting condition variable. + */ +int +__wt_cond_auto_alloc( + WT_SESSION_IMPL *session, const char *name, + bool is_signalled, uint64_t min, uint64_t max, WT_CONDVAR **condp) +{ + WT_CONDVAR *cond; + + WT_RET(__wt_cond_alloc(session, name, is_signalled, condp)); + cond = *condp; + + cond->min_wait = min; + cond->max_wait = max; + cond->prev_wait = min; + + return (0); +} + +/* + * __wt_cond_auto_wait_signal -- + * Wait on a mutex, optionally timing out. If we get it before the time + * out period expires, let the caller know. + * TODO: Can this version of the API be removed, now that we have the + * auto adjusting condition variables? + */ +int +__wt_cond_auto_wait_signal( + WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress, bool *signalled) +{ + uint64_t delta; + + if (progress) + cond->prev_wait = cond->min_wait; + else { + delta = WT_MAX(1, (cond->max_wait - cond->min_wait) / 10); + cond->prev_wait = WT_MIN( + cond->max_wait, cond->prev_wait + delta); + } + + WT_RET(__wt_cond_wait_signal( + session, cond, cond->prev_wait, signalled)); + + if (*signalled) + cond->prev_wait = cond->min_wait; + + return (0); +} + +/* + * __wt_cond_auto_wait -- + * Wait on a mutex, optionally timing out. If we get it before the time + * out period expires, let the caller know. + */ +int +__wt_cond_auto_wait( + WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress) +{ + bool signalled; + + /* + * Call the signal version so the wait period is reset if the + * condition is woken explicitly. + */ + WT_RET(__wt_cond_auto_wait_signal(session, cond, progress, &signalled)); + + return (0); +} + +/* + * __wt_cond_auto_destroy -- + * Destroy a condition variable. + */ +int +__wt_cond_auto_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp) +{ + return (__wt_cond_destroy(session, condp)); +} -- cgit v1.2.1 From 73733f9f70b145765fe48ba2cf916ab43dd158fa Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Fri, 4 Mar 2016 15:49:38 +1100 Subject: Fix Windows build. --- build_win/filelist.win | 1 + 1 file changed, 1 insertion(+) diff --git a/build_win/filelist.win b/build_win/filelist.win index 0a313026793..b6a9caf4a74 100644 --- a/build_win/filelist.win +++ b/build_win/filelist.win @@ -155,6 +155,7 @@ src/session/session_compact.c src/session/session_dhandle.c src/session/session_salvage.c src/support/cksum.c +src/support/cond_auto.c src/support/crypto.c src/support/err.c src/support/filename.c -- cgit v1.2.1 From 99012f223e4fc081937704ca202b9cf8874aeed1 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Fri, 4 Mar 2016 15:49:48 +1100 Subject: WT-2318 Add a safety check to auto condition variables. --- src/support/cond_auto.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/support/cond_auto.c b/src/support/cond_auto.c index 0c255bd1c98..934752ffe00 100644 --- a/src/support/cond_auto.c +++ b/src/support/cond_auto.c @@ -67,6 +67,12 @@ __wt_cond_auto_wait_signal( { uint64_t delta; + /* + * Catch cases where this function is called with a condition variable + * that was initialized non-auto. + */ + WT_ASSERT(session, cond->min_wait != 0); + if (progress) cond->prev_wait = cond->min_wait; else { -- cgit v1.2.1 From 1e656a4f69fa9dc1d7229240cc20ae137944eecd Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Fri, 4 Mar 2016 17:48:26 +1100 Subject: WT-2404 Add streaming pack and unpack APIs to the extension API. Will allow us to deprecate the incomplete pack/unpack there now. --- src/conn/conn_api.c | 13 +++++ src/include/extern.h | 11 +++++ src/include/wiredtiger_ext.h | 113 +++++++++++++++++++++++++++++++++++++++++++ src/packing/pack_stream.c | 112 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 249 insertions(+) diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index bb67185f5c9..9cd7dd2f44e 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -772,6 +772,19 @@ __conn_get_extension_api(WT_CONNECTION *wt_conn) conn->extension_api.transaction_visible = __wt_ext_transaction_visible; conn->extension_api.version = wiredtiger_version; + /* Streaming pack/unpack API */ + conn->extension_api.pack_start = __wt_ext_pack_start; + conn->extension_api.unpack_start = __wt_ext_unpack_start; + conn->extension_api.pack_close = __wt_ext_pack_close; + conn->extension_api.pack_item = __wt_ext_pack_item; + conn->extension_api.pack_int = __wt_ext_pack_int; + conn->extension_api.pack_str = __wt_ext_pack_str; + conn->extension_api.pack_uint = __wt_ext_pack_uint; + conn->extension_api.unpack_item = __wt_ext_unpack_item; + conn->extension_api.unpack_int = __wt_ext_unpack_int; + conn->extension_api.unpack_str = __wt_ext_unpack_str; + conn->extension_api.unpack_uint = __wt_ext_unpack_uint; + return (&conn->extension_api); } diff --git a/src/include/extern.h b/src/include/extern.h index 55b0b8cd7ff..0774a14f382 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -558,6 +558,17 @@ extern int __wt_struct_size(WT_SESSION_IMPL *session, size_t *sizep, const char extern int __wt_struct_pack(WT_SESSION_IMPL *session, void *buffer, size_t size, const char *fmt, ...); extern int __wt_struct_unpack(WT_SESSION_IMPL *session, const void *buffer, size_t size, const char *fmt, ...); extern int __wt_struct_repack(WT_SESSION_IMPL *session, const char *infmt, const char *outfmt, const WT_ITEM *inbuf, WT_ITEM *outbuf); +extern int __wt_ext_pack_start(WT_SESSION *session, const char *format, void *buffer, size_t size, WT_PACK_STREAM **psp); +extern int __wt_ext_unpack_start(WT_SESSION *session, const char *format, const void *buffer, size_t size, WT_PACK_STREAM **psp); +extern int __wt_ext_pack_close(WT_PACK_STREAM *ps, size_t *usedp); +extern int __wt_ext_pack_item(WT_PACK_STREAM *ps, WT_ITEM *item); +extern int __wt_ext_pack_int(WT_PACK_STREAM *ps, int64_t i); +extern int __wt_ext_pack_str(WT_PACK_STREAM *ps, const char *s); +extern int __wt_ext_pack_uint(WT_PACK_STREAM *ps, uint64_t u); +extern int __wt_ext_unpack_item(WT_PACK_STREAM *ps, WT_ITEM *item); +extern int __wt_ext_unpack_int(WT_PACK_STREAM *ps, int64_t *ip); +extern int __wt_ext_unpack_str(WT_PACK_STREAM *ps, const char **sp); +extern int __wt_ext_unpack_uint(WT_PACK_STREAM *ps, uint64_t *up); extern int __wt_ovfl_discard_add(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL *cell); extern void __wt_ovfl_discard_free(WT_SESSION_IMPL *session, WT_PAGE *page); extern int __wt_ovfl_reuse_search(WT_SESSION_IMPL *session, WT_PAGE *page, uint8_t **addrp, size_t *addr_sizep, const void *value, size_t value_size); diff --git a/src/include/wiredtiger_ext.h b/src/include/wiredtiger_ext.h index 0db876b56f3..7271bf6ba9d 100644 --- a/src/include/wiredtiger_ext.h +++ b/src/include/wiredtiger_ext.h @@ -309,6 +309,119 @@ struct __wt_extension_api { int (*struct_unpack)(WT_EXTENSION_API *wt_api, WT_SESSION *session, const void *buffer, size_t size, const char *format, ...); + /* + * Streaming pack/unpack API. + */ + /*! + * Start a packing operation into a buffer. + * See ::wiredtiger_pack_start for details. + * + * @param session the session handle + * @param format the data format, see @ref packing + * @param buffer a pointer to memory to hold the packed data + * @param size the size of the buffer + * @param[out] psp the new packing stream handle + * @errors + */ + int (*pack_start)(WT_SESSION *session, const char *format, + void *buffer, size_t size, WT_PACK_STREAM **psp); + + /*! + * Start an unpacking operation from a buffer. + * See ::wiredtiger_unpack_start for details. + * + * @param session the session handle + * @param format the data format, see @ref packing + * @param buffer a pointer to memory holding the packed data + * @param size the size of the buffer + * @param[out] psp the new packing stream handle + * @errors + */ + int (*unpack_start)(WT_SESSION *session, const char *format, + const void *buffer, size_t size, WT_PACK_STREAM **psp); + + /*! + * Close a packing stream. + * + * @param ps the packing stream handle + * @param[out] usedp the number of bytes in the buffer used by the + * stream + * @errors + */ + int (*pack_close)(WT_PACK_STREAM *ps, size_t *usedp); + + /*! + * Pack an item into a packing stream. + * + * @param ps the packing stream handle + * @param item an item to pack + * @errors + */ + int (*pack_item)(WT_PACK_STREAM *ps, WT_ITEM *item); + + /*! + * Pack a signed integer into a packing stream. + * + * @param ps the packing stream handle + * @param i a signed integer to pack + * @errors + */ + int (*pack_int)(WT_PACK_STREAM *ps, int64_t i); + + /*! + * Pack a string into a packing stream. + * + * @param ps the packing stream handle + * @param s a string to pack + * @errors + */ + int (*pack_str)(WT_PACK_STREAM *ps, const char *s); + + /*! + * Pack an unsigned integer into a packing stream. + * + * @param ps the packing stream handle + * @param u an unsigned integer to pack + * @errors + */ + int (*pack_uint)(WT_PACK_STREAM *ps, uint64_t u); + + /*! + * Unpack an item from a packing stream. + * + * @param ps the packing stream handle + * @param item an item to unpack + * @errors + */ + int (*unpack_item)(WT_PACK_STREAM *ps, WT_ITEM *item); + + /*! + * Unpack a signed integer from a packing stream. + * + * @param ps the packing stream handle + * @param[out] ip the unpacked signed integer + * @errors + */ + int (*unpack_int)(WT_PACK_STREAM *ps, int64_t *ip); + + /*! + * Unpack a string from a packing stream. + * + * @param ps the packing stream handle + * @param[out] sp the unpacked string + * @errors + */ + int (*unpack_str)(WT_PACK_STREAM *ps, const char **sp); + + /*! + * Unpack an unsigned integer from a packing stream. + * + * @param ps the packing stream handle + * @param[out] up the unpacked unsigned integer + * @errors + */ + int (*unpack_uint)(WT_PACK_STREAM *ps, uint64_t *up); + /*! * Return the current transaction ID. * diff --git a/src/packing/pack_stream.c b/src/packing/pack_stream.c index 98da5b405c3..6ab44c123c8 100644 --- a/src/packing/pack_stream.c +++ b/src/packing/pack_stream.c @@ -327,3 +327,115 @@ wiredtiger_unpack_uint(WT_PACK_STREAM *ps, uint64_t *up) } return (0); } + +/* + * __wt_ext_pack_start -- + * WT_EXTENSION.pack_start method. + */ +int +__wt_ext_pack_start(WT_SESSION *session, const char *format, + void *buffer, size_t size, WT_PACK_STREAM **psp) +{ + return (wiredtiger_pack_start(session, format, buffer, size, psp)); +} + +/* + * __wt_ext_unpack_start -- + * WT_EXTENSION.unpack_start + */ +int +__wt_ext_unpack_start(WT_SESSION *session, const char *format, + const void *buffer, size_t size, WT_PACK_STREAM **psp) +{ + return (wiredtiger_unpack_start(session, format, buffer, size, psp)); +} + +/* + * __wt_ext_pack_close -- + * WT_EXTENSION.pack_close + */ +int +__wt_ext_pack_close(WT_PACK_STREAM *ps, size_t *usedp) +{ + return (wiredtiger_pack_close(ps, usedp)); +} + +/* + * __wt_ext_pack_item -- + * WT_EXTENSION.pack_item + */ +int +__wt_ext_pack_item(WT_PACK_STREAM *ps, WT_ITEM *item) +{ + return (wiredtiger_pack_item(ps, item)); +} + +/* + * __wt_ext_pack_int -- + * WT_EXTENSION.pack_int + */ +int +__wt_ext_pack_int(WT_PACK_STREAM *ps, int64_t i) +{ + return (wiredtiger_pack_int(ps, i)); +} + +/* + * __wt_ext_pack_str -- + * WT_EXTENSION.pack_str + */ +int +__wt_ext_pack_str(WT_PACK_STREAM *ps, const char *s) +{ + return (wiredtiger_pack_str(ps, s)); +} + +/* + * __wt_ext_pack_uint -- + * WT_EXTENSION.pack_uint + */ +int +__wt_ext_pack_uint(WT_PACK_STREAM *ps, uint64_t u) +{ + return (wiredtiger_pack_uint(ps, u)); +} + +/* + * __wt_ext_unpack_item -- + * WT_EXTENSION.unpack_item + */ +int +__wt_ext_unpack_item(WT_PACK_STREAM *ps, WT_ITEM *item) +{ + return (wiredtiger_unpack_item(ps, item)); +} + +/* + * __wt_ext_unpack_int -- + * WT_EXTENSION.unpack_int + */ +int +__wt_ext_unpack_int(WT_PACK_STREAM *ps, int64_t *ip) +{ + return (wiredtiger_unpack_int(ps, ip)); +} + +/* + * __wt_ext_unpack_str -- + * WT_EXTENSION.unpack_str + */ +int +__wt_ext_unpack_str(WT_PACK_STREAM *ps, const char **sp) +{ + return (wiredtiger_unpack_str(ps, sp)); +} + +/* + * __wt_ext_unpack_uint -- + * WT_EXTENSION.unpack_uint + */ +int +__wt_ext_unpack_uint(WT_PACK_STREAM *ps, uint64_t *up) +{ + return (wiredtiger_unpack_uint(ps, up)); +} -- cgit v1.2.1 From a43114647a18e8b3bf9df7b823acca197c1829ad Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 4 Mar 2016 07:49:23 -0500 Subject: WT-2381: dump utility discards table config LSM doesn't support column-store keys, don't try to test that combination. --- test/suite/test_dump.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/suite/test_dump.py b/test/suite/test_dump.py index 752e4d6a36a..5ecc670bea7 100644 --- a/test/suite/test_dump.py +++ b/test/suite/test_dump.py @@ -97,6 +97,10 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): # Dump, re-load and do a content comparison. def test_dump(self): + # LSM and column-store isn't a valid combination. + if self.type == 'lsm:' and self.keyfmt == 'r': + return + # Create the object. uri = self.type + self.name self.populate(self, uri, 'key_format=' + self.keyfmt, self.nentries) -- cgit v1.2.1 From c279bb2d2c70249b076190974c68722d09f32215 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 4 Mar 2016 07:54:44 -0500 Subject: WT-2381: dump utility discards table config whitespace --- test/suite/test_dump.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/suite/test_dump.py b/test/suite/test_dump.py index 5ecc670bea7..a9795dde903 100644 --- a/test/suite/test_dump.py +++ b/test/suite/test_dump.py @@ -97,9 +97,9 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): # Dump, re-load and do a content comparison. def test_dump(self): - # LSM and column-store isn't a valid combination. - if self.type == 'lsm:' and self.keyfmt == 'r': - return + # LSM and column-store isn't a valid combination. + if self.type == 'lsm:' and self.keyfmt == 'r': + return # Create the object. uri = self.type + self.name -- cgit v1.2.1 From 91055b7d5e093ef1e83187b61cb6de9654ce8b22 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Fri, 4 Mar 2016 17:13:31 -0500 Subject: WT-2318 Add use of auto adjusting condition for some log threads. --- bench/wtperf/runners/log.wtperf | 2 +- dist/stat_data.py | 5 ++ src/conn/conn_log.c | 55 ++++++++---- src/cursor/cur_log.c | 2 +- src/include/extern.h | 2 +- src/include/stat.h | 5 ++ src/include/wiredtiger.in | 190 +++++++++++++++++++++------------------- src/log/log.c | 19 +++- src/support/cond_auto.c | 3 + src/support/stat.c | 15 ++++ 10 files changed, 186 insertions(+), 112 deletions(-) diff --git a/bench/wtperf/runners/log.wtperf b/bench/wtperf/runners/log.wtperf index 32a9cc3b0a6..1fc195eebf4 100644 --- a/bench/wtperf/runners/log.wtperf +++ b/bench/wtperf/runners/log.wtperf @@ -1,6 +1,6 @@ # wtperf options file: Test performance with a log file enabled. # Set the log file small to catch log-swtich bottlenecks. -conn_config="cache_size=1G,log=(enabled=true,file_max=200K),checkpoint=(log_size=500MB)" +conn_config="cache_size=1G,log=(enabled=true,file_max=200K),checkpoint=(log_size=500MB),statistics=(fast),statistics_log=(wait=10)" table_config="type=file" icount=50000 report_interval=5 diff --git a/dist/stat_data.py b/dist/stat_data.py index 09e5643a5d6..636f3398742 100644 --- a/dist/stat_data.py +++ b/dist/stat_data.py @@ -109,6 +109,8 @@ connection_stats = [ ########################################## # System statistics ########################################## + ConnStat('cond_auto_wait', 'auto adjusting condition wait calls'), + ConnStat('cond_auto_wait_reset', 'auto adjusting condition resets'), ConnStat('cond_wait', 'pthread mutex condition wait calls'), ConnStat('file_open', 'files currently open', 'no_clear,no_scale'), ConnStat('memory_allocation', 'memory allocations'), @@ -218,6 +220,8 @@ connection_stats = [ LogStat('log_compress_write_fails', 'log records not compressed'), LogStat('log_compress_writes', 'log records compressed'), LogStat('log_flush', 'log flush operations'), + LogStat('log_force_write', 'log force write operations'), + LogStat('log_force_write_skip', 'log force write operations skipped'), LogStat('log_max_filesize', 'maximum log file size', 'no_clear,no_scale,size'), LogStat('log_prealloc_files', 'pre-allocated log files prepared'), LogStat('log_prealloc_max', 'number of pre-allocated log files to create', 'no_clear,no_scale'), @@ -238,6 +242,7 @@ connection_stats = [ LogStat('log_sync', 'log sync operations'), LogStat('log_sync_dir', 'log sync_dir operations'), LogStat('log_write_lsn', 'log server thread advances write LSN'), + LogStat('log_write_lsn_skip', 'log server thread write LSN walk skipped'), LogStat('log_writes', 'log write operations'), LogStat('log_zero_fills', 'log files manually zero-filled'), diff --git a/src/conn/conn_log.c b/src/conn/conn_log.c index 5999cf20b3b..662ef1a7f09 100644 --- a/src/conn/conn_log.c +++ b/src/conn/conn_log.c @@ -667,31 +667,54 @@ __log_wrlsn_server(void *arg) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; + WT_LOG *log; + WT_LSN prev; WT_SESSION_IMPL *session; int yield; + bool did_work; session = arg; conn = S2C(session); + log = conn->log; yield = 0; + WT_INIT_LSN(&prev); + did_work = false; while (F_ISSET(conn, WT_CONN_LOG_SERVER_RUN)) { /* - * Write out any log record buffers. + * Write out any log record buffers if anything was done + * since last time. Only call the function to walk the + * slots if the system is not idle. On an idle system + * the alloc_lsn will not advance and the written lsn will + * match the alloc_lsn. */ - WT_ERR(__wt_log_wrlsn(session, &yield)); + if (__wt_log_cmp(&prev, &log->alloc_lsn) != 0 || + __wt_log_cmp(&log->write_lsn, &log->alloc_lsn) != 0) + WT_ERR(__wt_log_wrlsn(session, &yield)); + else + WT_STAT_FAST_CONN_INCR(session, log_write_lsn_skip); + prev = log->alloc_lsn; + if (yield == 0) + did_work = true; + else + did_work = false; /* * If __wt_log_wrlsn did work we want to yield instead of sleep. */ if (yield++ < WT_THOUSAND) __wt_yield(); else - WT_ERR(__wt_cond_wait( - session, conn->log_wrlsn_cond, 10000)); + /* + * Send in false because if we did any work we would + * not be on this path. + */ + WT_ERR(__wt_cond_auto_wait( + session, conn->log_wrlsn_cond, did_work)); } /* * On close we need to do this one more time because there could * be straggling log writes that need to be written. */ - WT_ERR(__wt_log_force_write(session, 1)); + WT_ERR(__wt_log_force_write(session, 1, NULL)); WT_ERR(__wt_log_wrlsn(session, NULL)); if (0) { err: __wt_err(session, ret, "log wrlsn server error"); @@ -711,7 +734,7 @@ __log_server(void *arg) WT_LOG *log; WT_SESSION_IMPL *session; int freq_per_sec; - bool locked, signalled; + bool did_work, locked, signalled; session = arg; conn = S2C(session); @@ -736,6 +759,7 @@ __log_server(void *arg) * don't want log records sitting in the buffer over the time it * takes to sync out an earlier file. */ + did_work = true; while (F_ISSET(conn, WT_CONN_LOG_SERVER_RUN)) { /* * Slots depend on future activity. Force out buffered @@ -744,7 +768,7 @@ __log_server(void *arg) * and a buffer may need to wait for the write_lsn to advance * in the case of a synchronous buffer. We end up with a hang. */ - WT_ERR_BUSY_OK(__wt_log_force_write(session, 0)); + WT_ERR_BUSY_OK(__wt_log_force_write(session, 0, &did_work)); /* * We don't want to archive or pre-allocate files as often as @@ -793,8 +817,8 @@ __log_server(void *arg) } /* Wait until the next event. */ - WT_ERR(__wt_cond_wait_signal(session, conn->log_cond, - WT_MILLION / WT_FORCE_PER_SECOND, &signalled)); + WT_ERR(__wt_cond_auto_wait_signal(session, conn->log_cond, + did_work || signalled, &signalled)); } if (0) { @@ -906,8 +930,9 @@ __wt_logmgr_open(WT_SESSION_IMPL *session) */ WT_RET(__wt_open_internal_session(conn, "log-wrlsn-server", false, session_flags, &conn->log_wrlsn_session)); - WT_RET(__wt_cond_alloc(conn->log_wrlsn_session, - "log write lsn server", false, &conn->log_wrlsn_cond)); + WT_RET(__wt_cond_auto_alloc(conn->log_wrlsn_session, + "log write lsn server", false, 10000, WT_MILLION, + &conn->log_wrlsn_cond)); WT_RET(__wt_thread_create(conn->log_wrlsn_session, &conn->log_wrlsn_tid, __log_wrlsn_server, conn->log_wrlsn_session)); conn->log_wrlsn_tid_set = true; @@ -926,8 +951,8 @@ __wt_logmgr_open(WT_SESSION_IMPL *session) /* The log server gets its own session. */ WT_RET(__wt_open_internal_session(conn, "log-server", false, session_flags, &conn->log_session)); - WT_RET(__wt_cond_alloc(conn->log_session, - "log server", false, &conn->log_cond)); + WT_RET(__wt_cond_auto_alloc(conn->log_session, + "log server", false, 50000, WT_MILLION, &conn->log_cond)); /* * Start the thread. @@ -999,9 +1024,9 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session) } /* Destroy the condition variables now that all threads are stopped */ - WT_TRET(__wt_cond_destroy(session, &conn->log_cond)); + WT_TRET(__wt_cond_auto_destroy(session, &conn->log_cond)); WT_TRET(__wt_cond_destroy(session, &conn->log_file_cond)); - WT_TRET(__wt_cond_destroy(session, &conn->log_wrlsn_cond)); + WT_TRET(__wt_cond_auto_destroy(session, &conn->log_wrlsn_cond)); WT_TRET(__wt_cond_destroy(session, &conn->log->log_sync_cond)); WT_TRET(__wt_cond_destroy(session, &conn->log->log_write_cond)); diff --git a/src/cursor/cur_log.c b/src/cursor/cur_log.c index 47436ac7237..0a13803da5d 100644 --- a/src/cursor/cur_log.c +++ b/src/cursor/cur_log.c @@ -397,7 +397,7 @@ __wt_curlog_open(WT_SESSION_IMPL *session, * The user may be trying to read a log record they just wrote. * Log records may be buffered, so force out any now. */ - WT_ERR(__wt_log_force_write(session, 1)); + WT_ERR(__wt_log_force_write(session, 1, NULL)); /* Log cursors block archiving. */ WT_ERR(__wt_readlock(session, log->log_archive_lock)); diff --git a/src/include/extern.h b/src/include/extern.h index e52cf51779b..83fb132774e 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -364,7 +364,7 @@ extern int __wt_log_open(WT_SESSION_IMPL *session); extern int __wt_log_close(WT_SESSION_IMPL *session); extern int __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep); extern int __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, int (*func)(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, WT_LSN *next_lsnp, void *cookie, int firstrecord), void *cookie); -extern int __wt_log_force_write(WT_SESSION_IMPL *session, bool retry); +extern int __wt_log_force_write(WT_SESSION_IMPL *session, bool retry, bool *did_work); extern int __wt_log_write(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, uint32_t flags); extern int __wt_log_vprintf(WT_SESSION_IMPL *session, const char *fmt, va_list ap); extern int __wt_log_flush(WT_SESSION_IMPL *session, uint32_t flags); diff --git a/src/include/stat.h b/src/include/stat.h index 8bc6c37b53e..f9170dc1a79 100644 --- a/src/include/stat.h +++ b/src/include/stat.h @@ -299,6 +299,8 @@ struct __wt_connection_stats { int64_t cache_bytes_dirty; int64_t cache_pages_dirty; int64_t cache_eviction_clean; + int64_t cond_auto_wait_reset; + int64_t cond_auto_wait; int64_t file_open; int64_t memory_allocation; int64_t memory_free; @@ -337,6 +339,8 @@ struct __wt_connection_stats { int64_t log_bytes_written; int64_t log_zero_fills; int64_t log_flush; + int64_t log_force_write; + int64_t log_force_write_skip; int64_t log_compress_writes; int64_t log_compress_write_fails; int64_t log_compress_small; @@ -344,6 +348,7 @@ struct __wt_connection_stats { int64_t log_scans; int64_t log_scan_rereads; int64_t log_write_lsn; + int64_t log_write_lsn_skip; int64_t log_sync; int64_t log_sync_dir; int64_t log_writes; diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index 0d328668e10..1e263f22880 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -3850,187 +3850,197 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_CACHE_PAGES_DIRTY 1070 /*! cache: unmodified pages evicted */ #define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1071 +/*! connection: auto adjusting condition resets */ +#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1072 +/*! connection: auto adjusting condition wait calls */ +#define WT_STAT_CONN_COND_AUTO_WAIT 1073 /*! connection: files currently open */ -#define WT_STAT_CONN_FILE_OPEN 1072 +#define WT_STAT_CONN_FILE_OPEN 1074 /*! connection: memory allocations */ -#define WT_STAT_CONN_MEMORY_ALLOCATION 1073 +#define WT_STAT_CONN_MEMORY_ALLOCATION 1075 /*! connection: memory frees */ -#define WT_STAT_CONN_MEMORY_FREE 1074 +#define WT_STAT_CONN_MEMORY_FREE 1076 /*! connection: memory re-allocations */ -#define WT_STAT_CONN_MEMORY_GROW 1075 +#define WT_STAT_CONN_MEMORY_GROW 1077 /*! connection: pthread mutex condition wait calls */ -#define WT_STAT_CONN_COND_WAIT 1076 +#define WT_STAT_CONN_COND_WAIT 1078 /*! connection: pthread mutex shared lock read-lock calls */ -#define WT_STAT_CONN_RWLOCK_READ 1077 +#define WT_STAT_CONN_RWLOCK_READ 1079 /*! connection: pthread mutex shared lock write-lock calls */ -#define WT_STAT_CONN_RWLOCK_WRITE 1078 +#define WT_STAT_CONN_RWLOCK_WRITE 1080 /*! connection: total read I/Os */ -#define WT_STAT_CONN_READ_IO 1079 +#define WT_STAT_CONN_READ_IO 1081 /*! connection: total write I/Os */ -#define WT_STAT_CONN_WRITE_IO 1080 +#define WT_STAT_CONN_WRITE_IO 1082 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1081 +#define WT_STAT_CONN_CURSOR_CREATE 1083 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1082 +#define WT_STAT_CONN_CURSOR_INSERT 1084 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1083 +#define WT_STAT_CONN_CURSOR_NEXT 1085 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1084 +#define WT_STAT_CONN_CURSOR_PREV 1086 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1085 +#define WT_STAT_CONN_CURSOR_REMOVE 1087 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1086 +#define WT_STAT_CONN_CURSOR_RESET 1088 /*! cursor: cursor restarted searches */ -#define WT_STAT_CONN_CURSOR_RESTART 1087 +#define WT_STAT_CONN_CURSOR_RESTART 1089 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1088 +#define WT_STAT_CONN_CURSOR_SEARCH 1090 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1089 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1091 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1090 +#define WT_STAT_CONN_CURSOR_UPDATE 1092 /*! cursor: truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1091 +#define WT_STAT_CONN_CURSOR_TRUNCATE 1093 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1092 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1094 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1093 +#define WT_STAT_CONN_DH_SWEEP_REF 1095 /*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1094 +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1096 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1095 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1097 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1096 +#define WT_STAT_CONN_DH_SWEEP_TOD 1098 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1097 +#define WT_STAT_CONN_DH_SWEEPS 1099 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1098 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1100 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1099 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1101 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1100 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1102 /*! log: consolidated slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1101 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1103 /*! log: consolidated slot join races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1102 +#define WT_STAT_CONN_LOG_SLOT_RACES 1104 /*! log: consolidated slot join transitions */ -#define WT_STAT_CONN_LOG_SLOT_TRANSITIONS 1103 +#define WT_STAT_CONN_LOG_SLOT_TRANSITIONS 1105 /*! log: consolidated slot joins */ -#define WT_STAT_CONN_LOG_SLOT_JOINS 1104 +#define WT_STAT_CONN_LOG_SLOT_JOINS 1106 /*! log: consolidated slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1105 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1107 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1106 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1108 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1107 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1109 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1108 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1110 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1109 +#define WT_STAT_CONN_LOG_FLUSH 1111 +/*! log: log force write operations */ +#define WT_STAT_CONN_LOG_FORCE_WRITE 1112 +/*! log: log force write operations skipped */ +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1113 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1110 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1114 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1111 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1115 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1112 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1116 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1113 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1117 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1114 +#define WT_STAT_CONN_LOG_SCANS 1118 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1115 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1119 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1116 +#define WT_STAT_CONN_LOG_WRITE_LSN 1120 +/*! log: log server thread write LSN walk skipped */ +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1121 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1117 +#define WT_STAT_CONN_LOG_SYNC 1122 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1118 +#define WT_STAT_CONN_LOG_SYNC_DIR 1123 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1119 +#define WT_STAT_CONN_LOG_WRITES 1124 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1120 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1125 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1121 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1126 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1122 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1127 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1123 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1128 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1124 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1129 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1125 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1130 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1126 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1131 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1127 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1132 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1128 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1133 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1129 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1134 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1130 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1135 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1131 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1136 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1132 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1137 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1133 +#define WT_STAT_CONN_REC_PAGES 1138 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1134 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1139 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1135 +#define WT_STAT_CONN_REC_PAGE_DELETE 1140 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1136 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1141 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1137 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1142 /*! session: open cursor count */ -#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1138 +#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1143 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1139 +#define WT_STAT_CONN_SESSION_OPEN 1144 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1140 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1145 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1141 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1146 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1142 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1147 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1143 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1148 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1144 +#define WT_STAT_CONN_PAGE_SLEEP 1149 /*! transaction: number of named snapshots created */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1145 +#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1150 /*! transaction: number of named snapshots dropped */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1146 +#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1151 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1147 +#define WT_STAT_CONN_TXN_BEGIN 1152 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1148 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1153 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1149 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1154 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1150 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1155 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1151 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1156 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1152 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1157 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1153 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1158 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1154 +#define WT_STAT_CONN_TXN_CHECKPOINT 1159 /*! transaction: transaction failures due to cache overflow */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1155 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1160 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1156 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1161 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1157 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1162 /*! transaction: transaction range of IDs currently pinned by named * snapshots */ -#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1158 +#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1163 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1159 +#define WT_STAT_CONN_TXN_SYNC 1164 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1160 +#define WT_STAT_CONN_TXN_COMMIT 1165 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1161 +#define WT_STAT_CONN_TXN_ROLLBACK 1166 /*! * @} diff --git a/src/log/log.c b/src/log/log.c index 03145d8408c..1a7612ccd8a 100644 --- a/src/log/log.c +++ b/src/log/log.c @@ -46,7 +46,7 @@ __wt_log_flush_lsn(WT_SESSION_IMPL *session, WT_LSN *lsn, bool start) conn = S2C(session); log = conn->log; - WT_RET(__wt_log_force_write(session, 1)); + WT_RET(__wt_log_force_write(session, 1, NULL)); WT_RET(__wt_log_wrlsn(session, NULL)); if (start) *lsn = log->write_start_lsn; @@ -273,7 +273,7 @@ __wt_log_get_all_files(WT_SESSION_IMPL *session, * These may be files needed by backup. Force the current slot * to get written to the file. */ - WT_RET(__wt_log_force_write(session, 1)); + WT_RET(__wt_log_force_write(session, 1, NULL)); WT_RET(__log_get_files(session, WT_LOG_FILENAME, &files, &count)); /* Filter out any files that are below the checkpoint LSN. */ @@ -1758,14 +1758,25 @@ err: WT_STAT_FAST_CONN_INCR(session, log_scans); * Wrapper function that takes the lock. */ int -__wt_log_force_write(WT_SESSION_IMPL *session, bool retry) +__wt_log_force_write(WT_SESSION_IMPL *session, bool retry, bool *did_work) { WT_LOG *log; WT_MYSLOT myslot; + uint32_t joined; log = S2C(session)->log; memset(&myslot, 0, sizeof(myslot)); + WT_STAT_FAST_CONN_INCR(session, log_force_write); + if (did_work != NULL) + *did_work = true; myslot.slot = log->active_slot; + joined = WT_LOG_SLOT_JOINED(log->active_slot->slot_state); + if (joined == 0) { + WT_STAT_FAST_CONN_INCR(session, log_force_write_skip); + if (did_work != NULL) + *did_work = false; + return (0); + } return (__wt_log_slot_switch(session, &myslot, retry, true)); } @@ -2001,7 +2012,7 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, WT_ERR(__wt_cond_signal(session, conn->log_cond)); __wt_yield(); } else - WT_ERR(__wt_log_force_write(session, 1)); + WT_ERR(__wt_log_force_write(session, 1, NULL)); } if (LF_ISSET(WT_LOG_FLUSH)) { /* Wait for our writes to reach the OS */ diff --git a/src/support/cond_auto.c b/src/support/cond_auto.c index 934752ffe00..56f844e5240 100644 --- a/src/support/cond_auto.c +++ b/src/support/cond_auto.c @@ -73,6 +73,7 @@ __wt_cond_auto_wait_signal( */ WT_ASSERT(session, cond->min_wait != 0); + WT_STAT_FAST_CONN_INCR(session, cond_auto_wait); if (progress) cond->prev_wait = cond->min_wait; else { @@ -84,6 +85,8 @@ __wt_cond_auto_wait_signal( WT_RET(__wt_cond_wait_signal( session, cond, cond->prev_wait, signalled)); + if (progress || *signalled) + WT_STAT_FAST_CONN_INCR(session, cond_auto_wait_reset); if (*signalled) cond->prev_wait = cond->min_wait; diff --git a/src/support/stat.c b/src/support/stat.c index 0df38bfe6b0..2a826eda962 100644 --- a/src/support/stat.c +++ b/src/support/stat.c @@ -581,6 +581,8 @@ static const char * const __stats_connection_desc[] = { "cache: tracked dirty bytes in the cache", "cache: tracked dirty pages in the cache", "cache: unmodified pages evicted", + "connection: auto adjusting condition resets", + "connection: auto adjusting condition wait calls", "connection: files currently open", "connection: memory allocations", "connection: memory frees", @@ -619,6 +621,8 @@ static const char * const __stats_connection_desc[] = { "log: log bytes written", "log: log files manually zero-filled", "log: log flush operations", + "log: log force write operations", + "log: log force write operations skipped", "log: log records compressed", "log: log records not compressed", "log: log records too small to compress", @@ -626,6 +630,7 @@ static const char * const __stats_connection_desc[] = { "log: log scan operations", "log: log scan records requiring two reads", "log: log server thread advances write LSN", + "log: log server thread write LSN walk skipped", "log: log sync operations", "log: log sync_dir operations", "log: log write operations", @@ -773,6 +778,8 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) /* not clearing cache_bytes_dirty */ /* not clearing cache_pages_dirty */ stats->cache_eviction_clean = 0; + stats->cond_auto_wait_reset = 0; + stats->cond_auto_wait = 0; /* not clearing file_open */ stats->memory_allocation = 0; stats->memory_free = 0; @@ -811,6 +818,8 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->log_bytes_written = 0; stats->log_zero_fills = 0; stats->log_flush = 0; + stats->log_force_write = 0; + stats->log_force_write_skip = 0; stats->log_compress_writes = 0; stats->log_compress_write_fails = 0; stats->log_compress_small = 0; @@ -818,6 +827,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->log_scans = 0; stats->log_scan_rereads = 0; stats->log_write_lsn = 0; + stats->log_write_lsn_skip = 0; stats->log_sync = 0; stats->log_sync_dir = 0; stats->log_writes = 0; @@ -974,6 +984,8 @@ __wt_stat_connection_aggregate( to->cache_bytes_dirty += WT_STAT_READ(from, cache_bytes_dirty); to->cache_pages_dirty += WT_STAT_READ(from, cache_pages_dirty); to->cache_eviction_clean += WT_STAT_READ(from, cache_eviction_clean); + to->cond_auto_wait_reset += WT_STAT_READ(from, cond_auto_wait_reset); + to->cond_auto_wait += WT_STAT_READ(from, cond_auto_wait); to->file_open += WT_STAT_READ(from, file_open); to->memory_allocation += WT_STAT_READ(from, memory_allocation); to->memory_free += WT_STAT_READ(from, memory_free); @@ -1012,6 +1024,8 @@ __wt_stat_connection_aggregate( to->log_bytes_written += WT_STAT_READ(from, log_bytes_written); to->log_zero_fills += WT_STAT_READ(from, log_zero_fills); to->log_flush += WT_STAT_READ(from, log_flush); + to->log_force_write += WT_STAT_READ(from, log_force_write); + to->log_force_write_skip += WT_STAT_READ(from, log_force_write_skip); to->log_compress_writes += WT_STAT_READ(from, log_compress_writes); to->log_compress_write_fails += WT_STAT_READ(from, log_compress_write_fails); @@ -1021,6 +1035,7 @@ __wt_stat_connection_aggregate( to->log_scans += WT_STAT_READ(from, log_scans); to->log_scan_rereads += WT_STAT_READ(from, log_scan_rereads); to->log_write_lsn += WT_STAT_READ(from, log_write_lsn); + to->log_write_lsn_skip += WT_STAT_READ(from, log_write_lsn_skip); to->log_sync += WT_STAT_READ(from, log_sync); to->log_sync_dir += WT_STAT_READ(from, log_sync_dir); to->log_writes += WT_STAT_READ(from, log_writes); -- cgit v1.2.1 From 3f8bab585b09eb60c41953aec482f6c3de88160e Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Fri, 4 Mar 2016 17:14:15 -0500 Subject: WT-2318 Revert wtperf config change. --- bench/wtperf/runners/log.wtperf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/wtperf/runners/log.wtperf b/bench/wtperf/runners/log.wtperf index 1fc195eebf4..32a9cc3b0a6 100644 --- a/bench/wtperf/runners/log.wtperf +++ b/bench/wtperf/runners/log.wtperf @@ -1,6 +1,6 @@ # wtperf options file: Test performance with a log file enabled. # Set the log file small to catch log-swtich bottlenecks. -conn_config="cache_size=1G,log=(enabled=true,file_max=200K),checkpoint=(log_size=500MB),statistics=(fast),statistics_log=(wait=10)" +conn_config="cache_size=1G,log=(enabled=true,file_max=200K),checkpoint=(log_size=500MB)" table_config="type=file" icount=50000 report_interval=5 -- cgit v1.2.1 From 45ffd7785f27d2cefc429f8edbeb1d0db1901796 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sun, 6 Mar 2016 13:52:23 -0500 Subject: WT-2391: De-prioritize eviction from indexes With commit 7f84e3b it's unlikely for a page to NOT have a read generation, it only happens if a reader configured with WT_READ_NO_GEN set were to read a page into cache, and that won't happen because those readers set either WT_READ_CACHE or WT_READ_WONT_NEED. The eviction code still has special code to handle pages without a read generation set, and it maintains a "oldest page generation" value to support that. However, the oldest page generation value isn't really right -- it's simply that page generation value from the first slot in the queue, which can be special (for example, if the page generation was set to WT_READGEN_OLDEST because the page was empty). Add a test in the page-read code to set the page generation if it's never been set, regardless of WT_READ_NO_GEN. Remove the eviction code to maintain an "oldest page generation". Leave the eviction code to handle WT_READGEN_NOTSET (it can still see that value if there's a race), but change it to set the page read generation to the "standard" value instead of trying anything fancy. That ensure that if there's a bug (we read the page but for some reason never set the page generation), the page won't stay in that state forever. --- src/btree/bt_read.c | 17 +++++++++++------ src/evict/evict_lru.c | 11 +++++------ src/include/cache.h | 2 -- src/include/cache.i | 13 ------------- 4 files changed, 16 insertions(+), 27 deletions(-) diff --git a/src/btree/bt_read.c b/src/btree/bt_read.c index ac9faef4ff2..fd84ad8c7c8 100644 --- a/src/btree/bt_read.c +++ b/src/btree/bt_read.c @@ -575,18 +575,23 @@ __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags } /* - * If we read the page and we are configured to not - * trash the cache, set the oldest read generation so - * the page is forcibly evicted as soon as possible. + * If we read the page and are configured to not trash + * the cache, and no other thread has already used the + * page, set the oldest read generation so the page is + * forcibly evicted as soon as possible. * - * Otherwise, update the page's read generation. + * Otherwise, if we read the page, or, if configured to + * update the page's read generation and the page isn't + * already flagged for forced eviction, update the page + * read generation. */ page = ref->page; if (oldgen && page->read_gen == WT_READGEN_NOTSET) __wt_page_evict_soon(page); - else if (!LF_ISSET(WT_READ_NO_GEN) && + else if (page->read_gen == WT_READGEN_NOTSET || + (!LF_ISSET(WT_READ_NO_GEN) && page->read_gen != WT_READGEN_OLDEST && - page->read_gen < __wt_cache_read_gen(session)) + page->read_gen < __wt_cache_read_gen(session))) page->read_gen = __wt_cache_read_gen_bump(session); skip_evict: diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index 884c08a02df..581637f1396 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -933,9 +933,6 @@ __evict_lru_walk(WT_SESSION_IMPL *session) WT_ASSERT(session, cache->evict_queue[0].ref != NULL); - /* Track the oldest read generation we have in the queue. */ - cache->read_gen_oldest = cache->evict_queue[0].ref->page->read_gen; - if (FLD_ISSET(cache->state, WT_EVICT_PASS_AGGRESSIVE | WT_EVICT_PASS_WOULD_BLOCK)) /* @@ -1312,11 +1309,13 @@ __evict_walk_file(WT_SESSION_IMPL *session, u_int *slotp) continue; /* - * If this page has never been considered for eviction, set its - * read generation to somewhere in the middle of the LRU list. + * It's possible (but unlikely) to visit a page without a read + * generation, if we race with the read instantiating the page. + * Be safe and set the page's read generation here to ensure a + * bug doesn't somehow leave a page without a read generation. */ if (page->read_gen == WT_READGEN_NOTSET) - page->read_gen = __wt_cache_read_gen_new(session); + page->read_gen = __wt_cache_read_gen_bump(session); fast: /* If the page can't be evicted, give up. */ if (!__wt_page_can_evict(session, ref, NULL)) diff --git a/src/include/cache.h b/src/include/cache.h index a3961d6043e..fb221fd9af5 100644 --- a/src/include/cache.h +++ b/src/include/cache.h @@ -76,8 +76,6 @@ struct __wt_cache { * Read information. */ uint64_t read_gen; /* Page read generation (LRU) */ - uint64_t read_gen_oldest; /* The oldest read generation that - eviction knows about */ /* * Eviction thread information. diff --git a/src/include/cache.i b/src/include/cache.i index ee13eee84c5..46809598d5c 100644 --- a/src/include/cache.i +++ b/src/include/cache.i @@ -45,19 +45,6 @@ __wt_cache_read_gen_bump(WT_SESSION_IMPL *session) return (__wt_cache_read_gen(session) + WT_READGEN_STEP); } -/* - * __wt_cache_read_gen_new -- - * Get the read generation for a new page in memory. - */ -static inline uint64_t -__wt_cache_read_gen_new(WT_SESSION_IMPL *session) -{ - WT_CACHE *cache; - - cache = S2C(session)->cache; - return (__wt_cache_read_gen(session) + cache->read_gen_oldest) / 2; -} - /* * __wt_cache_pages_inuse -- * Return the number of pages in use. -- cgit v1.2.1 From 2147fac6813949cf33b7e430db0c213736ee8d8a Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sun, 6 Mar 2016 13:55:24 -0500 Subject: WT-2391: De-prioritize eviction from indexes Rename "oldgen" to "evict_soon" for clarity. --- src/btree/bt_read.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/btree/bt_read.c b/src/btree/bt_read.c index fd84ad8c7c8..7cd2d9458df 100644 --- a/src/btree/bt_read.c +++ b/src/btree/bt_read.c @@ -460,12 +460,12 @@ __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags WT_DECL_RET; WT_PAGE *page; u_int sleep_cnt, wait_cnt; - bool busy, cache_work, oldgen, stalled; + bool busy, cache_work, evict_soon, stalled; int force_attempts; btree = S2BT(session); - for (oldgen = stalled = false, + for (evict_soon = stalled = false, force_attempts = 0, sleep_cnt = wait_cnt = 0;;) { switch (ref->state) { case WT_REF_DELETED: @@ -486,7 +486,7 @@ __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags WT_RET(__wt_cache_eviction_check( session, 1, NULL)); WT_RET(__page_read(session, ref)); - oldgen = LF_ISSET(WT_READ_WONT_NEED) || + evict_soon = LF_ISSET(WT_READ_WONT_NEED) || F_ISSET(session, WT_SESSION_NO_CACHE); continue; case WT_REF_READING: @@ -586,7 +586,7 @@ __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags * read generation. */ page = ref->page; - if (oldgen && page->read_gen == WT_READGEN_NOTSET) + if (evict_soon && page->read_gen == WT_READGEN_NOTSET) __wt_page_evict_soon(page); else if (page->read_gen == WT_READGEN_NOTSET || (!LF_ISSET(WT_READ_NO_GEN) && -- cgit v1.2.1 From 0d847085452e889a58b2a0c72cd821754406a101 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sun, 6 Mar 2016 13:59:17 -0500 Subject: WT-2391: De-prioritize eviction from indexes Add a comment. --- src/btree/bt_read.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/btree/bt_read.c b/src/btree/bt_read.c index 7cd2d9458df..3070716631f 100644 --- a/src/btree/bt_read.c +++ b/src/btree/bt_read.c @@ -486,6 +486,15 @@ __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags WT_RET(__wt_cache_eviction_check( session, 1, NULL)); WT_RET(__page_read(session, ref)); + + /* + * If configured to not trash the cache, leave the page + * generation unset, we'll set it before returning to + * the oldest read generation, so the page is forcibly + * evicted as soon as possible. We don't do that set + * here because we don't want to evict the page before + * we "acquire" it. + */ evict_soon = LF_ISSET(WT_READ_WONT_NEED) || F_ISSET(session, WT_SESSION_NO_CACHE); continue; -- cgit v1.2.1 From 4afb380882b2534f4ac6272a8f724ccb4a97dd23 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sun, 6 Mar 2016 14:06:47 -0500 Subject: WT-2391: De-prioritize eviction from indexes Move the tests for whether or not a page generation number is updated into __wt_cache_read_gen_bump, it gives us consistent behavior in all callers of not updating any page generation that's set for forcible eviction (all callers already checked that), and not updating any page generation that doesn't need to be updated (eviction didn't check that). Make the code a bit simpler, change __wt_cache_read_gen_bump to take a page reference instead of returning a generation number requiring an assignment. --- src/btree/bt_read.c | 9 +++------ src/evict/evict_lru.c | 7 ++----- src/include/cache.i | 30 +++++++++++++++++++----------- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/btree/bt_read.c b/src/btree/bt_read.c index 3070716631f..738726efd1a 100644 --- a/src/btree/bt_read.c +++ b/src/btree/bt_read.c @@ -597,12 +597,9 @@ __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags page = ref->page; if (evict_soon && page->read_gen == WT_READGEN_NOTSET) __wt_page_evict_soon(page); - else if (page->read_gen == WT_READGEN_NOTSET || - (!LF_ISSET(WT_READ_NO_GEN) && - page->read_gen != WT_READGEN_OLDEST && - page->read_gen < __wt_cache_read_gen(session))) - page->read_gen = - __wt_cache_read_gen_bump(session); + else if (!LF_ISSET(WT_READ_NO_GEN) || + page->read_gen == WT_READGEN_NOTSET) + __wt_cache_read_gen_bump(session, page); skip_evict: /* * Check if we need an autocommit transaction. diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index 581637f1396..be699a737e6 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -1315,7 +1315,7 @@ __evict_walk_file(WT_SESSION_IMPL *session, u_int *slotp) * bug doesn't somehow leave a page without a read generation. */ if (page->read_gen == WT_READGEN_NOTSET) - page->read_gen = __wt_cache_read_gen_bump(session); + __wt_cache_read_gen_bump(session, page); fast: /* If the page can't be evicted, give up. */ if (!__wt_page_can_evict(session, ref, NULL)) @@ -1476,7 +1476,6 @@ __evict_page(WT_SESSION_IMPL *session, bool is_server) { WT_BTREE *btree; WT_DECL_RET; - WT_PAGE *page; WT_REF *ref; WT_RET(__evict_get_ref(session, is_server, &btree, &ref)); @@ -1505,9 +1504,7 @@ __evict_page(WT_SESSION_IMPL *session, bool is_server) * the page and some other thread may have evicted it by the time we * look at it. */ - page = ref->page; - if (page->read_gen != WT_READGEN_OLDEST) - page->read_gen = __wt_cache_read_gen_bump(session); + __wt_cache_read_gen_bump(session, ref->page); WT_WITH_BTREE(session, btree, ret = __wt_evict(session, ref, false)); diff --git a/src/include/cache.i b/src/include/cache.i index 46809598d5c..1f0af8ab16f 100644 --- a/src/include/cache.i +++ b/src/include/cache.i @@ -28,21 +28,29 @@ __wt_cache_read_gen_incr(WT_SESSION_IMPL *session) /* * __wt_cache_read_gen_bump -- - * Get the read generation to keep a page in memory. + * Update the page's read generation. */ -static inline uint64_t -__wt_cache_read_gen_bump(WT_SESSION_IMPL *session) +static inline void +__wt_cache_read_gen_bump(WT_SESSION_IMPL *session, WT_PAGE *page) { + /* Ignore pages set for forcible eviction. */ + if (page->read_gen == WT_READGEN_OLDEST) + return; + + /* Ignore pages already in the future. */ + if (page->read_gen > __wt_cache_read_gen(session)) + return; + /* - * We return read-generations from the future (where "the future" is - * measured by increments of the global read generation). The reason - * is because when acquiring a new hazard pointer for a page, we can - * check its read generation, and if the read generation isn't less - * than the current global generation, we don't bother updating the - * page. In other words, the goal is to avoid some number of updates - * immediately after each update we have to make. + * We set read-generations in the future (where "the future" is measured + * by increments of the global read generation). The reason is because + * when acquiring a new hazard pointer for a page, we can check its read + * generation, and if the read generation isn't less than the current + * global generation, we don't bother updating the page. In other + * words, the goal is to avoid some number of updates immediately after + * each update we have to make. */ - return (__wt_cache_read_gen(session) + WT_READGEN_STEP); + page->read_gen = __wt_cache_read_gen(session) + WT_READGEN_STEP; } /* -- cgit v1.2.1 From 334b3a1cdef7e8b6a0bf27a81694611f085642dd Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sun, 6 Mar 2016 14:16:41 -0500 Subject: WT-2391: De-prioritize eviction from indexes Eviction should ignore pages without a page read generation, if it happens across one. Move the check for a page without a page read generation to before other uses of the page's read generation. (Not a bug fix, the test wasn't wrong, but it seems slightly cleaner.) --- src/evict/evict_lru.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index be699a737e6..01e490fb907 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -1283,6 +1283,18 @@ __evict_walk_file(WT_SESSION_IMPL *session, u_int *slotp) if (F_ISSET_ATOMIC(page, WT_PAGE_EVICT_LRU)) continue; + /* + * It's possible (but unlikely) to visit a page without a read + * generation, if we race with the read instantiating the page. + * Ignore those pages, but set the page's read generation here + * to ensure a bug doesn't somehow leave a page without a read + * generation. + */ + if (page->read_gen == WT_READGEN_NOTSET) { + __wt_cache_read_gen_bump(session, page); + continue; + } + /* Pages we no longer need (clean or dirty), are found money. */ if (__wt_page_is_empty(page) || F_ISSET(session->dhandle, WT_DHANDLE_DEAD) || @@ -1308,15 +1320,6 @@ __evict_walk_file(WT_SESSION_IMPL *session, u_int *slotp) internal_pages >= (int)(evict - start) / 2) continue; - /* - * It's possible (but unlikely) to visit a page without a read - * generation, if we race with the read instantiating the page. - * Be safe and set the page's read generation here to ensure a - * bug doesn't somehow leave a page without a read generation. - */ - if (page->read_gen == WT_READGEN_NOTSET) - __wt_cache_read_gen_bump(session, page); - fast: /* If the page can't be evicted, give up. */ if (!__wt_page_can_evict(session, ref, NULL)) continue; -- cgit v1.2.1 From a012fca0b0c924d4c2136a804a587284acb5c297 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Mon, 7 Mar 2016 10:04:12 +1100 Subject: WT-2448 Add no_scale flag into a couple of statistics. --- dist/stat_data.py | 4 ++-- tools/wtstats/stat_data.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dist/stat_data.py b/dist/stat_data.py index 09e5643a5d6..43d4474ac61 100644 --- a/dist/stat_data.py +++ b/dist/stat_data.py @@ -124,7 +124,7 @@ connection_stats = [ ########################################## AsyncStat('async_alloc_race', 'number of allocation state races'), AsyncStat('async_alloc_view', 'number of operation slots viewed for allocation'), - AsyncStat('async_cur_queue', 'current work queue length'), + AsyncStat('async_cur_queue', 'current work queue length', 'no_scale'), AsyncStat('async_flush', 'number of flush calls'), AsyncStat('async_full', 'number of times operation allocation failed'), AsyncStat('async_max_queue', 'maximum work queue length', 'no_clear,no_scale'), @@ -397,7 +397,7 @@ dsrc_stats = [ BlockStat('block_magic', 'file magic number', 'max_aggregate,no_scale'), BlockStat('block_major', 'file major version number', 'max_aggregate,no_scale'), BlockStat('block_minor', 'minor version number', 'max_aggregate,no_scale'), - BlockStat('block_reuse_bytes', 'file bytes available for reuse', 'size'), + BlockStat('block_reuse_bytes', 'file bytes available for reuse', 'no_scale,size'), BlockStat('block_size', 'file size in bytes', 'no_scale,size'), ########################################## diff --git a/tools/wtstats/stat_data.py b/tools/wtstats/stat_data.py index f181aeb09b4..c75e4f194dd 100644 --- a/tools/wtstats/stat_data.py +++ b/tools/wtstats/stat_data.py @@ -1,6 +1,7 @@ # DO NOT EDIT: automatically built by dist/stat.py. */ no_scale_per_second_list = [ + 'async: current work queue length', 'async: maximum work queue length', 'cache: bytes currently in the cache', 'cache: eviction currently operating in aggressive mode', @@ -36,6 +37,7 @@ no_scale_per_second_list = [ 'transaction: transaction range of IDs currently pinned by named snapshots', 'block-manager: checkpoint size', 'block-manager: file allocation unit size', + 'block-manager: file bytes available for reuse', 'block-manager: file magic number', 'block-manager: file major version number', 'block-manager: file size in bytes', -- cgit v1.2.1 From 799ed27d8fdc6b802eebf7720d3f24626cd1bafb Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sun, 6 Mar 2016 18:31:18 -0500 Subject: WT-2391: De-prioritize eviction from indexes Add the cache's notion of the oldest read generation back in, but instead of always taking the first slot of the queue, walk the queue until we find a read generation that's not WT_READGEN_OLDEST. Rework the 50% cutoff code: First, if at least 50% of the whole queue is WT_READGEN_OLDEST entries, take all of the WT_READGEN_OLDEST entries, second, use the calculated oldest generation instead of the first slot in the queue. When we've just read a page and are setting the read generation for the first time, try and put it in the middle of the cache. When eviction hits a page without a read generation number, restore the previous behavior of putting it in the middle of the cache. --- src/btree/bt_read.c | 10 +++++--- src/evict/evict_lru.c | 67 +++++++++++++++++++++++++++++++++++++-------------- src/include/cache.h | 4 ++- src/include/cache.i | 14 +++++++++++ 4 files changed, 72 insertions(+), 23 deletions(-) diff --git a/src/btree/bt_read.c b/src/btree/bt_read.c index 738726efd1a..f80eb09153c 100644 --- a/src/btree/bt_read.c +++ b/src/btree/bt_read.c @@ -595,10 +595,12 @@ __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags * read generation. */ page = ref->page; - if (evict_soon && page->read_gen == WT_READGEN_NOTSET) - __wt_page_evict_soon(page); - else if (!LF_ISSET(WT_READ_NO_GEN) || - page->read_gen == WT_READGEN_NOTSET) + if (page->read_gen == WT_READGEN_NOTSET) { + if (evict_soon) + __wt_page_evict_soon(page); + else + __wt_cache_read_gen_new(session, page); + } else if (!LF_ISSET(WT_READ_NO_GEN)) __wt_cache_read_gen_bump(session, page); skip_evict: /* diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index 01e490fb907..3b9a7c9db62 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -890,7 +890,7 @@ __evict_lru_walk(WT_SESSION_IMPL *session) { WT_CACHE *cache; WT_DECL_RET; - uint64_t cutoff; + uint64_t cutoff, read_gen_oldest; uint32_t candidates, entries; cache = S2C(session)->cache; @@ -931,31 +931,62 @@ __evict_lru_walk(WT_SESSION_IMPL *session) return (0); } - WT_ASSERT(session, cache->evict_queue[0].ref != NULL); - + /* Decide how many of the candidates we're going to try and evict. */ if (FLD_ISSET(cache->state, - WT_EVICT_PASS_AGGRESSIVE | WT_EVICT_PASS_WOULD_BLOCK)) + WT_EVICT_PASS_AGGRESSIVE | WT_EVICT_PASS_WOULD_BLOCK)) { /* * Take all candidates if we only gathered pages with an oldest * read generation set. */ cache->evict_candidates = entries; - else { - /* Find the bottom 25% of read generations. */ - cutoff = (3 * __evict_read_gen(&cache->evict_queue[0]) + - __evict_read_gen(&cache->evict_queue[entries - 1])) / 4; + } else { /* - * Don't take less than 10% or more than 50% of entries, - * regardless. That said, if there is only one entry, which is - * normal when populating an empty file, don't exclude it. + * Find the oldest read generation we have in the queue, used + * to set the initial value for pages read into the system. + * The queue is sorted, find the first "normal" generation. */ - for (candidates = 1 + entries / 10; - candidates < entries / 2; - candidates++) - if (__evict_read_gen( - &cache->evict_queue[candidates]) > cutoff) + read_gen_oldest = WT_READGEN_OLDEST; + for (candidates = 0; candidates < entries; ++candidates) { + read_gen_oldest = + __evict_read_gen(&cache->evict_queue[candidates]); + if (read_gen_oldest != WT_READGEN_OLDEST) break; - cache->evict_candidates = candidates; + } + + /* + * Take all candidates if we only gathered pages with an oldest + * read generation set. + * + * We normally never take more than 50% of the entries; if 50% + * of the entries were at the oldest read generation, take them. + */ + if (read_gen_oldest == WT_READGEN_OLDEST) + cache->evict_candidates = entries; + else if (candidates >= entries / 2) + cache->evict_candidates = candidates; + else { + /* Save the calculated oldest generation. */ + cache->read_gen_oldest = read_gen_oldest; + + /* Find the bottom 25% of read generations. */ + cutoff = + (3 * read_gen_oldest + __evict_read_gen( + &cache->evict_queue[entries - 1])) / 4; + + /* + * Don't take less than 10% or more than 50% of entries, + * regardless. That said, if there is only one entry, + * which is normal when populating an empty file, don't + * exclude it. + */ + for (candidates = 1 + entries / 10; + candidates < entries / 2; + candidates++) + if (__evict_read_gen( + &cache->evict_queue[candidates]) > cutoff) + break; + cache->evict_candidates = candidates; + } } cache->evict_current = cache->evict_queue; @@ -1291,7 +1322,7 @@ __evict_walk_file(WT_SESSION_IMPL *session, u_int *slotp) * generation. */ if (page->read_gen == WT_READGEN_NOTSET) { - __wt_cache_read_gen_bump(session, page); + __wt_cache_read_gen_new(session, page); continue; } diff --git a/src/include/cache.h b/src/include/cache.h index fb221fd9af5..9184a2fe6ed 100644 --- a/src/include/cache.h +++ b/src/include/cache.h @@ -75,7 +75,9 @@ struct __wt_cache { /* * Read information. */ - uint64_t read_gen; /* Page read generation (LRU) */ + uint64_t read_gen; /* Current page read generation */ + uint64_t read_gen_oldest; /* Oldest read generation the eviction + * server saw in its last queue load */ /* * Eviction thread information. diff --git a/src/include/cache.i b/src/include/cache.i index 1f0af8ab16f..76729685eae 100644 --- a/src/include/cache.i +++ b/src/include/cache.i @@ -53,6 +53,20 @@ __wt_cache_read_gen_bump(WT_SESSION_IMPL *session, WT_PAGE *page) page->read_gen = __wt_cache_read_gen(session) + WT_READGEN_STEP; } +/* + * __wt_cache_read_gen_new -- + * Get the read generation for a new page in memory. + */ +static inline void +__wt_cache_read_gen_new(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_CACHE *cache; + + cache = S2C(session)->cache; + page->read_gen = + (__wt_cache_read_gen(session) + cache->read_gen_oldest) / 2; +} + /* * __wt_cache_pages_inuse -- * Return the number of pages in use. -- cgit v1.2.1 From a5c7704620ac227c2447bb95e7a4467491884a18 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Mon, 7 Mar 2016 10:58:11 +1100 Subject: WT-2449 Check for a 64-bit build during configure. --- build_posix/configure.ac.in | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build_posix/configure.ac.in b/build_posix/configure.ac.in index 06d73e2fe12..fc13ab2b183 100644 --- a/build_posix/configure.ac.in +++ b/build_posix/configure.ac.in @@ -97,6 +97,11 @@ AC_SYS_LARGEFILE AC_C_BIGENDIAN +AC_CHECK_SIZEOF(void *) +if test "$ac_cv_sizeof_void_p" != "8" ; then + AC_MSG_ERROR([WiredTiger requires a 64-bit build.]) +fi + # Linux requires _GNU_SOURCE to be defined case "$host_os" in linux*) AM_CFLAGS="$AM_CFLAGS -D_GNU_SOURCE" ;; -- cgit v1.2.1 From 34c29af1c992a435a382b973787a5760f4d95c71 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sun, 6 Mar 2016 19:05:33 -0500 Subject: WT-2391: De-prioritize eviction from indexes The lowest possible page read-generation has a special meaning, it marks a page for forcible eviction. If we read in page before doing anything else, we can end up with a page that looks like it needs forcible eviction, and salvage isn't prepared to deal with that. It's probably a bug in salvage that it doesn't handle the page being "released" and then disappearing, but this gets us most of the way there. --- src/conn/conn_cache.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/conn/conn_cache.c b/src/conn/conn_cache.c index 1831aad5895..9e8daaf9f7e 100644 --- a/src/conn/conn_cache.c +++ b/src/conn/conn_cache.c @@ -139,6 +139,12 @@ __wt_cache_create(WT_SESSION_IMPL *session, const char *cfg[]) /* Use a common routine for run-time configuration options. */ WT_RET(__wt_cache_config(session, false, cfg)); + /* + * The lowest possible page read-generation has a special meaning, it + * marks a page for forcible eviction; don't let it happen by accident. + */ + cache->read_gen = 10; + /* * The target size must be lower than the trigger size or we will never * get any work done. -- cgit v1.2.1 From 66d3dc3ca9ef18c37264291dc1ebca1468751abf Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sun, 6 Mar 2016 19:18:56 -0500 Subject: WT-2391: De-prioritize eviction from indexes Create a new #define for the starting page read generation value, instead of a magic constant in the code. --- src/conn/conn_cache.c | 2 +- src/include/btmem.h | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/conn/conn_cache.c b/src/conn/conn_cache.c index 9e8daaf9f7e..8a7064cbb35 100644 --- a/src/conn/conn_cache.c +++ b/src/conn/conn_cache.c @@ -143,7 +143,7 @@ __wt_cache_create(WT_SESSION_IMPL *session, const char *cfg[]) * The lowest possible page read-generation has a special meaning, it * marks a page for forcible eviction; don't let it happen by accident. */ - cache->read_gen = 10; + cache->read_gen = WT_READGEN_START_VALUE; /* * The target size must be lower than the trigger size or we will never diff --git a/src/include/btmem.h b/src/include/btmem.h index ee495c52fc8..7cdf2bef43a 100644 --- a/src/include/btmem.h +++ b/src/include/btmem.h @@ -598,9 +598,14 @@ struct __wt_page { * read generation is incremented by the eviction server each time it * becomes active. To avoid incrementing a page's read generation too * frequently, it is set to a future point. + * + * Because low read generation values have special meaning, and there + * are places where we manipulate the value, use an initial value well + * outside of the special range. */ #define WT_READGEN_NOTSET 0 #define WT_READGEN_OLDEST 1 +#define WT_READGEN_START_VALUE 100 #define WT_READGEN_STEP 100 uint64_t read_gen; -- cgit v1.2.1 From 79c6bba9e6e9dfb75615293126048dabdd326838 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Mon, 7 Mar 2016 14:56:42 +1100 Subject: WT-2318 Add __wt_cond_auto_signal for consistency. --- src/conn/conn_log.c | 8 ++++---- src/evict/evict_lru.c | 2 +- src/include/extern.h | 1 + src/log/log.c | 8 ++++---- src/log/log_slot.c | 2 +- src/support/cond_auto.c | 12 ++++++++++++ 6 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/conn/conn_log.c b/src/conn/conn_log.c index 662ef1a7f09..f78b52f2d27 100644 --- a/src/conn/conn_log.c +++ b/src/conn/conn_log.c @@ -468,7 +468,7 @@ __log_file_server(void *arg) locked = false; __wt_spin_unlock(session, &log->log_sync_lock); } else { - WT_ERR(__wt_cond_signal( + WT_ERR(__wt_cond_auto_signal( session, conn->log_wrlsn_cond)); /* * We do not want to wait potentially a second @@ -946,7 +946,7 @@ __wt_logmgr_open(WT_SESSION_IMPL *session) if (conn->log_session != NULL) { WT_ASSERT(session, conn->log_cond != NULL); WT_ASSERT(session, conn->log_tid_set == true); - WT_RET(__wt_cond_signal(session, conn->log_cond)); + WT_RET(__wt_cond_auto_signal(session, conn->log_cond)); } else { /* The log server gets its own session. */ WT_RET(__wt_open_internal_session(conn, @@ -988,7 +988,7 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session) return (0); } if (conn->log_tid_set) { - WT_TRET(__wt_cond_signal(session, conn->log_cond)); + WT_TRET(__wt_cond_auto_signal(session, conn->log_cond)); WT_TRET(__wt_thread_join(session, conn->log_tid)); conn->log_tid_set = false; } @@ -1003,7 +1003,7 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session) conn->log_file_session = NULL; } if (conn->log_wrlsn_tid_set) { - WT_TRET(__wt_cond_signal(session, conn->log_wrlsn_cond)); + WT_TRET(__wt_cond_auto_signal(session, conn->log_wrlsn_cond)); WT_TRET(__wt_thread_join(session, conn->log_wrlsn_tid)); conn->log_wrlsn_tid_set = false; } diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index 46a40dc124f..59db872ad9d 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -159,7 +159,7 @@ __wt_evict_server_wake(WT_SESSION_IMPL *session) bytes_max / WT_MEGABYTE)); } - return (__wt_cond_signal(session, cache->evict_cond)); + return (__wt_cond_auto_signal(session, cache->evict_cond)); } /* diff --git a/src/include/extern.h b/src/include/extern.h index 83fb132774e..a59aa67f944 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -638,6 +638,7 @@ extern int __wt_salvage(WT_SESSION_IMPL *session, const char *cfg[]); extern uint32_t __wt_cksum(const void *chunk, size_t len); extern void __wt_cksum_init(void); extern int __wt_cond_auto_alloc( WT_SESSION_IMPL *session, const char *name, bool is_signalled, uint64_t min, uint64_t max, WT_CONDVAR **condp); +extern int __wt_cond_auto_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond); extern int __wt_cond_auto_wait_signal( WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress, bool *signalled); extern int __wt_cond_auto_wait( WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress); extern int __wt_cond_auto_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp); diff --git a/src/log/log.c b/src/log/log.c index 1a7612ccd8a..0b84b5b2b19 100644 --- a/src/log/log.c +++ b/src/log/log.c @@ -29,7 +29,7 @@ __wt_log_ckpt(WT_SESSION_IMPL *session, WT_LSN *ckp_lsn) log = conn->log; log->ckpt_lsn = *ckp_lsn; if (conn->log_cond != NULL) - WT_RET(__wt_cond_signal(session, conn->log_cond)); + WT_RET(__wt_cond_auto_signal(session, conn->log_cond)); return (0); } @@ -824,7 +824,7 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) if (create_log) { WT_STAT_FAST_CONN_INCR(session, log_prealloc_missed); if (conn->log_cond != NULL) - WT_RET(__wt_cond_signal( + WT_RET(__wt_cond_auto_signal( session, conn->log_cond)); } } @@ -1338,7 +1338,7 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) */ if (F_ISSET(session, WT_SESSION_LOCKED_SLOT)) __wt_spin_unlock(session, &log->log_slot_lock); - WT_ERR(__wt_cond_signal(session, conn->log_wrlsn_cond)); + WT_ERR(__wt_cond_auto_signal(session, conn->log_wrlsn_cond)); if (++yield_count < WT_THOUSAND) __wt_yield(); else @@ -2009,7 +2009,7 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, * XXX I've seen times when conditions are NULL. */ if (conn->log_cond != NULL) { - WT_ERR(__wt_cond_signal(session, conn->log_cond)); + WT_ERR(__wt_cond_auto_signal(session, conn->log_cond)); __wt_yield(); } else WT_ERR(__wt_log_force_write(session, 1, NULL)); diff --git a/src/log/log_slot.c b/src/log/log_slot.c index 2844516e78f..570d1c9ce48 100644 --- a/src/log/log_slot.c +++ b/src/log/log_slot.c @@ -253,7 +253,7 @@ __wt_log_slot_new(WT_SESSION_IMPL *session) /* * If we didn't find any free slots signal the worker thread. */ - (void)__wt_cond_signal(session, conn->log_wrlsn_cond); + (void)__wt_cond_auto_signal(session, conn->log_wrlsn_cond); __wt_yield(); } /* NOTREACHED */ diff --git a/src/support/cond_auto.c b/src/support/cond_auto.c index 56f844e5240..ec95622f333 100644 --- a/src/support/cond_auto.c +++ b/src/support/cond_auto.c @@ -54,6 +54,18 @@ __wt_cond_auto_alloc( return (0); } +/* + * __wt_cond_auto_signal -- + * Signal a condition variable. + */ +int +__wt_cond_auto_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond) +{ + + WT_ASSERT(session, cond->min_wait != 0); + return (__wt_cond_signal(session, cond)); +} + /* * __wt_cond_auto_wait_signal -- * Wait on a mutex, optionally timing out. If we get it before the time -- cgit v1.2.1 From ea81053275589fcf58050b642409c667d2b4dd46 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Mon, 7 Mar 2016 15:13:37 +1100 Subject: WT-2451 Allow eviction of WiredTiger metadata. Switch from making eviction impossible to merely unlikely. This allows metadata pages to split and for eviction to discard deleted data. Without this change, the trivial test added in test_schema07.py hangs because the metadata fills the cache. --- src/meta/meta_table.c | 16 ++++++-------- test/suite/test_schema07.py | 54 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 9 deletions(-) create mode 100644 test/suite/test_schema07.py diff --git a/src/meta/meta_table.c b/src/meta/meta_table.c index 61cc009c983..e5f2727b5b6 100644 --- a/src/meta/meta_table.c +++ b/src/meta/meta_table.c @@ -67,18 +67,16 @@ __wt_metadata_cursor_open( btree = ((WT_CURSOR_BTREE *)(*cursorp))->btree; /* - * Set special flags for the metadata file: eviction (the metadata file - * is in-memory and never evicted), logging (the metadata file is always - * logged if possible). + * Special settings for metadata: skew eviction so metadata almost + * always stays in cache and make sure metadata is logged if possible. * - * Test flags before setting them so updates can't race in subsequent - * opens (the first update is safe because it's single-threaded from + * Test before setting so updates can't race in subsequent opens (the + * first update is safe because it's single-threaded from * wiredtiger_open). */ - if (!F_ISSET(btree, WT_BTREE_IN_MEMORY)) - F_SET(btree, WT_BTREE_IN_MEMORY); - if (!F_ISSET(btree, WT_BTREE_NO_EVICTION)) - F_SET(btree, WT_BTREE_NO_EVICTION); + if (btree->evict_priority == 0) + WT_WITH_BTREE(session, btree, + __wt_evict_priority_set(session, WT_EVICT_INT_SKEW)); if (F_ISSET(btree, WT_BTREE_NO_LOGGING)) F_CLR(btree, WT_BTREE_NO_LOGGING); diff --git a/test/suite/test_schema07.py b/test/suite/test_schema07.py new file mode 100644 index 00000000000..ac397c6e1a1 --- /dev/null +++ b/test/suite/test_schema07.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest + +# test_schema07.py +# Test that long-running tests don't fill the cache with metadata +class test_schema07(wttest.WiredTigerTestCase): + tablename = 'table:test_schema07' + + def conn_config(self, dir): + return 'cache_size=10MB' + + @wttest.longtest("Creating many tables shouldn't fill the cache") + def test_many_tables(self): + s = self.session + # We have a 10MB cache, metadata is (well) over 512B per table, + # if we can create 20K tables, something must be cleaning up. + for i in xrange(20000): + uri = '%s-%06d' % (self.tablename, i) + s.create(uri) + c = s.open_cursor(uri) + # This will block if the metadata fills the cache + c["key"] = "value" + c.close() + self.session.drop(uri) + +if __name__ == '__main__': + wttest.run() -- cgit v1.2.1 From cc751a7610d5e0ed400d0edfad0a00a61a538bac Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Mon, 7 Mar 2016 11:21:35 -0500 Subject: WT-2318 Fix test suite failures. Signal on reconfigure. Log_server needs to check elapsed time instead of frequency since the thread auto adjusts now. --- src/conn/conn_log.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/conn/conn_log.c b/src/conn/conn_log.c index f78b52f2d27..7a88fc124d6 100644 --- a/src/conn/conn_log.c +++ b/src/conn/conn_log.c @@ -142,6 +142,8 @@ __logmgr_config( } WT_RET(__logmgr_sync_cfg(session, cfg)); + if (conn->log_cond != NULL) + WT_RET(__wt_cond_auto_signal(session, conn->log_cond)); return (0); } @@ -729,11 +731,12 @@ err: __wt_err(session, ret, "log wrlsn server error"); static WT_THREAD_RET __log_server(void *arg) { + struct timespec start, now; WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_LOG *log; WT_SESSION_IMPL *session; - int freq_per_sec; + uint64_t timediff; bool did_work, locked, signalled; session = arg; @@ -742,11 +745,10 @@ __log_server(void *arg) locked = signalled = false; /* - * Set this to the number of times per second we want to force out the - * log slot buffer. + * Set this to the number of milliseconds we want to run archive and + * pre-allocation. Start it so that we run on the first time through. */ -#define WT_FORCE_PER_SECOND 20 - freq_per_sec = WT_FORCE_PER_SECOND; + timediff = WT_THOUSAND; /* * The log server thread does a variety of work. It forces out any @@ -775,8 +777,7 @@ __log_server(void *arg) * we want to force out log buffers. Only do it once per second * or if the condition was signalled. */ - if (--freq_per_sec <= 0 || signalled) { - freq_per_sec = WT_FORCE_PER_SECOND; + if (timediff >= WT_THOUSAND || signalled) { /* * Perform log pre-allocation. @@ -817,8 +818,12 @@ __log_server(void *arg) } /* Wait until the next event. */ + + WT_ERR(__wt_epoch(session, &start)); WT_ERR(__wt_cond_auto_wait_signal(session, conn->log_cond, did_work || signalled, &signalled)); + WT_ERR(__wt_epoch(session, &now)); + timediff = WT_TIMEDIFF_MS(now, start); } if (0) { -- cgit v1.2.1 From 63f0404d38be4783a6ee57e643815a491a4df55d Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Mon, 7 Mar 2016 11:41:23 -0500 Subject: WT-2318 Remove unnecessary boolean usage. --- src/conn/conn_log.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/conn/conn_log.c b/src/conn/conn_log.c index 7a88fc124d6..757d69bf240 100644 --- a/src/conn/conn_log.c +++ b/src/conn/conn_log.c @@ -821,7 +821,7 @@ __log_server(void *arg) WT_ERR(__wt_epoch(session, &start)); WT_ERR(__wt_cond_auto_wait_signal(session, conn->log_cond, - did_work || signalled, &signalled)); + did_work, &signalled)); WT_ERR(__wt_epoch(session, &now)); timediff = WT_TIMEDIFF_MS(now, start); } -- cgit v1.2.1 From 77eefaf730675e35e3da99aaccd69939b40a66bb Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 7 Mar 2016 12:09:27 -0500 Subject: WT-2418: test_rebalance failing with EBUSY Fix a test cut-and-paste error, we're testing WT_SESSION.rebalance here, not WT_SESSION.drop. --- test/suite/test_rebalance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/suite/test_rebalance.py b/test/suite/test_rebalance.py index 80cce6ed514..f2167e864c9 100644 --- a/test/suite/test_rebalance.py +++ b/test/suite/test_rebalance.py @@ -59,7 +59,7 @@ class test_rebalance(wttest.WiredTigerTestCase): if with_cursor: cursor = self.session.open_cursor(uri, None, None) self.assertRaises(wiredtiger.WiredTigerError, - lambda: self.session.drop(uri, None)) + lambda: self.session.rebalance(uri, None)) cursor.close() self.session.rebalance(uri, None) -- cgit v1.2.1 From 2f3259aba177c72a47d8e6ed3577ee16e4ad41bb Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 7 Mar 2016 15:36:39 -0500 Subject: WT-2123: Don't clear allocated memory if not required Switch to __wt_realloc (allocate and clear), __wt_realloc_aligned allocate aligned if possible, don't clear), __wt_realloc_noclear (allocate and don't clear). --- src/include/extern.h | 3 ++- src/os_posix/os_alloc.c | 34 ++++++++++++++++++++-------------- src/support/scratch.c | 10 ++++++++-- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/src/include/extern.h b/src/include/extern.h index 998936e4b72..9ebfc399f59 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -487,7 +487,8 @@ extern void __wt_abort(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((nor extern int __wt_calloc(WT_SESSION_IMPL *session, size_t number, size_t size, void *retp); extern int __wt_malloc(WT_SESSION_IMPL *session, size_t bytes_to_allocate, void *retp); extern int __wt_realloc(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp); -extern int __wt_realloc_item( WT_SESSION_IMPL *session, WT_ITEM *buf, size_t bytes_to_allocate); +extern int __wt_realloc_noclear(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp); +extern int __wt_realloc_aligned(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp); extern int __wt_strndup(WT_SESSION_IMPL *session, const void *str, size_t len, void *retp); extern void __wt_free_int(WT_SESSION_IMPL *session, const void *p_arg); extern int __wt_dirlist(WT_SESSION_IMPL *session, const char *dir, const char *prefix, uint32_t flags, char ***dirlist, u_int *countp); diff --git a/src/os_posix/os_alloc.c b/src/os_posix/os_alloc.c index dcc16fde666..7eef9838107 100644 --- a/src/os_posix/os_alloc.c +++ b/src/os_posix/os_alloc.c @@ -159,30 +159,36 @@ __wt_realloc(WT_SESSION_IMPL *session, } /* - * __wt_realloc_item -- - * Allocate or reallocate memory for a WT_ITEM, optionally aligning to - * buffer boundaries (configured with "buffer_alignment" to wiredtiger_open). + * __wt_realloc_noclear -- + * WiredTiger's realloc API, not clearing allocated memory. */ int -__wt_realloc_item( - WT_SESSION_IMPL *session, WT_ITEM *buf, size_t bytes_to_allocate) +__wt_realloc_noclear(WT_SESSION_IMPL *session, + size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp) { - size_t *bytes_allocated_ret; - void *retp; - - bytes_allocated_ret = &buf->memsize; - retp = &buf->mem; + return (__realloc_func( + session, bytes_allocated_ret, bytes_to_allocate, false, retp)); +} +/* + * __wt_realloc_aligned -- + * ANSI realloc function that aligns to buffer boundaries, configured with + * the "buffer_alignment" key to wiredtiger_open. + */ +int +__wt_realloc_aligned(WT_SESSION_IMPL *session, + size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp) +{ #if defined(HAVE_POSIX_MEMALIGN) + WT_DECL_RET; + /* * !!! * This function MUST handle a NULL WT_SESSION_IMPL handle. */ - if (F_ISSET(buf, WT_ITEM_ALIGNED) && - session != NULL && S2C(session)->buffer_alignment > 0) { - WT_DECL_RET; - size_t bytes_allocated; + if (session != NULL && S2C(session)->buffer_alignment > 0) { void *p, *newp; + size_t bytes_allocated; /* * Sometimes we're allocating memory and we don't care about the diff --git a/src/support/scratch.c b/src/support/scratch.c index 0e23d0dd695..94020ba2621 100644 --- a/src/support/scratch.c +++ b/src/support/scratch.c @@ -40,8 +40,14 @@ __wt_buf_grow_worker(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size) * This function is also used to ensure data is local to the buffer, * check to see if we actually need to grow anything. */ - if (size > buf->memsize) - WT_RET(__wt_realloc_item(session, buf, size)); + if (size > buf->memsize) { + if (F_ISSET(buf, WT_ITEM_ALIGNED)) + WT_RET(__wt_realloc_aligned( + session, &buf->memsize, size, &buf->mem)); + else + WT_RET(__wt_realloc( + session, &buf->memsize, size, &buf->mem)); + } if (buf->data == NULL) { buf->data = buf->mem; -- cgit v1.2.1 From 6056ef6871b39c27c353bcbdfecc03a4ae4e0dc3 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 7 Mar 2016 15:53:43 -0500 Subject: WT-2123: Don't clear allocated memory if not required Scratch buffers don't need to clear memory. --- dist/s_string.ok | 1 + src/support/scratch.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dist/s_string.ok b/dist/s_string.ok index 2caaddcc15a..9e091d89c4d 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -806,6 +806,7 @@ nfilename nhex nlpo nocase +noclear nocrypto nolock nonliteral diff --git a/src/support/scratch.c b/src/support/scratch.c index 94020ba2621..aea98dc49ef 100644 --- a/src/support/scratch.c +++ b/src/support/scratch.c @@ -45,7 +45,7 @@ __wt_buf_grow_worker(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size) WT_RET(__wt_realloc_aligned( session, &buf->memsize, size, &buf->mem)); else - WT_RET(__wt_realloc( + WT_RET(__wt_realloc_noclear( session, &buf->memsize, size, &buf->mem)); } -- cgit v1.2.1 From 70deef0535cc9031ba8f9782de0bb3a36d4cc0de Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Mon, 7 Mar 2016 15:56:33 -0500 Subject: WT-2318 Add test. --- build_posix/Make.subdirs | 1 + dist/s_string.ok | 3 ++ test/manydbs/Makefile.am | 13 +++++ test/manydbs/manydbs.c | 132 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 149 insertions(+) create mode 100644 test/manydbs/Makefile.am create mode 100644 test/manydbs/manydbs.c diff --git a/build_posix/Make.subdirs b/build_posix/Make.subdirs index 14258666d84..775df0c203a 100644 --- a/build_posix/Make.subdirs +++ b/build_posix/Make.subdirs @@ -30,6 +30,7 @@ test/cursor_order test/fops test/format test/huge +test/manydbs test/packing test/readonly test/recovery diff --git a/dist/s_string.ok b/dist/s_string.ok index 2caaddcc15a..ab9aefccf06 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -844,8 +844,11 @@ parserp patchp pathname pathnames +pclose +pcpu perf pfx +popen poptable popthreads portably diff --git a/test/manydbs/Makefile.am b/test/manydbs/Makefile.am new file mode 100644 index 00000000000..a04e6a7250b --- /dev/null +++ b/test/manydbs/Makefile.am @@ -0,0 +1,13 @@ +AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \ + -I$(top_srcdir)/test/utility + +noinst_PROGRAMS = t +t_SOURCES = manydbs.c +t_LDADD = $(top_builddir)/libwiredtiger.la +t_LDFLAGS = -static + +# Run this during a "make check" smoke test. +TESTS = $(noinst_PROGRAMS) + +clean-local: + rm -rf WiredTiger* *.core __* diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c new file mode 100644 index 00000000000..9ed31ca3edb --- /dev/null +++ b/test/manydbs/manydbs.c @@ -0,0 +1,132 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#ifndef _WIN32 +#include +#endif + +#include + +#include "test_util.i" + +#define HOME_SIZE 512 +#define HOME_BASE "WT_HOME" +static char home[HOME_SIZE]; /* Base home directory */ +static char hometmp[HOME_SIZE]; /* Each conn home directory */ +static const char *progname; /* Program name */ + +#define ENV_CONFIG \ + "create,log=(file_max=10M,archive=false,enabled)," + +#define MAX_CPU 1.0 +#define MAX_DBS 10 + +static void +usage(void) +{ + fprintf(stderr, "usage: %s [-h dir]\n", progname); + exit(EXIT_FAILURE); +} + +extern int __wt_optind; +extern char *__wt_optarg; + +void (*custom_die)(void) = NULL; + +int +main(int argc, char *argv[]) +{ + FILE *fp; + WT_CONNECTION **conn; + float cpu; + int ch; + u_int dbs, i; + const char *working_dir; + char cmd[128]; + + if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) + progname = argv[0]; + else + ++progname; + dbs = MAX_DBS; + working_dir = HOME_BASE; + while ((ch = __wt_getopt(progname, argc, argv, "D:h:")) != EOF) + switch (ch) { + case 'D': + dbs = (u_int)atoi(__wt_optarg); + break; + case 'h': + working_dir = __wt_optarg; + break; + default: + usage(); + } + argc -= __wt_optind; + argv += __wt_optind; + if (argc != 0) + usage(); + + if ((conn = calloc(sizeof(WT_CONNECTION *), dbs)) == NULL) + testutil_die(ENOMEM, "connection array malloc"); + memset(cmd, 0, sizeof(cmd)); + /* + * Set up all the directory names. + */ + testutil_work_dir_from_path(home, HOME_SIZE, working_dir); + testutil_make_work_dir(home); + for (i = 0; i < dbs; ++i) { + snprintf(hometmp, HOME_SIZE, "%s/%s.%d", home, HOME_BASE, i); + testutil_make_work_dir(hometmp); + testutil_check(wiredtiger_open( + hometmp, NULL, ENV_CONFIG, &conn[i])); + } + + sleep(10); + (void)snprintf(cmd, sizeof(cmd), + "ps -p %lu -o pcpu=", (unsigned long)getpid()); + if ((fp = popen(cmd, "r")) == NULL) + testutil_die(errno, "popen"); + fscanf(fp, "%f", &cpu); + if (cpu > MAX_CPU) { + fprintf(stderr, "CPU usage: %f, max %f\n", cpu, MAX_CPU); + testutil_die(ERANGE, "CPU Usage"); + } + if (pclose(fp) != 0) + testutil_die(errno, "pclose"); + + for (i = 0; i < dbs; ++i) + testutil_check(conn[i]->close(conn[i], NULL)); + + return (EXIT_SUCCESS); +} -- cgit v1.2.1 From d5fbb1d0bc052b106612c54e37e31c171a23d79b Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Mon, 7 Mar 2016 16:55:07 -0500 Subject: WT-2318 Increase max CPU default and add arg to pass it in. --- test/manydbs/manydbs.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index 9ed31ca3edb..16dcec5e078 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -49,7 +49,7 @@ static const char *progname; /* Program name */ #define ENV_CONFIG \ "create,log=(file_max=10M,archive=false,enabled)," -#define MAX_CPU 1.0 +#define MAX_CPU 10.0 #define MAX_DBS 10 static void @@ -69,7 +69,7 @@ main(int argc, char *argv[]) { FILE *fp; WT_CONNECTION **conn; - float cpu; + float cpu, max; int ch; u_int dbs, i; const char *working_dir; @@ -81,8 +81,12 @@ main(int argc, char *argv[]) ++progname; dbs = MAX_DBS; working_dir = HOME_BASE; - while ((ch = __wt_getopt(progname, argc, argv, "D:h:")) != EOF) + max = MAX_CPU; + while ((ch = __wt_getopt(progname, argc, argv, "C:D:h:")) != EOF) switch (ch) { + case 'C': + max = (float)atof(__wt_optarg); + break; case 'D': dbs = (u_int)atoi(__wt_optarg); break; @@ -118,8 +122,8 @@ main(int argc, char *argv[]) if ((fp = popen(cmd, "r")) == NULL) testutil_die(errno, "popen"); fscanf(fp, "%f", &cpu); - if (cpu > MAX_CPU) { - fprintf(stderr, "CPU usage: %f, max %f\n", cpu, MAX_CPU); + if (cpu > max) { + fprintf(stderr, "CPU usage: %f, max %f\n", cpu, max); testutil_die(ERANGE, "CPU Usage"); } if (pclose(fp) != 0) -- cgit v1.2.1 From 83b7117b10cd290d5271b873d3fb2612934f7805 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Mon, 7 Mar 2016 17:15:35 -0500 Subject: WT-2318 Add loop to check CPU for 30 seconds. --- test/manydbs/manydbs.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index 16dcec5e078..2d6baa89a86 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -119,15 +119,17 @@ main(int argc, char *argv[]) sleep(10); (void)snprintf(cmd, sizeof(cmd), "ps -p %lu -o pcpu=", (unsigned long)getpid()); - if ((fp = popen(cmd, "r")) == NULL) - testutil_die(errno, "popen"); - fscanf(fp, "%f", &cpu); - if (cpu > max) { - fprintf(stderr, "CPU usage: %f, max %f\n", cpu, max); - testutil_die(ERANGE, "CPU Usage"); + for (i = 0; i < 30; i += 5) { + if ((fp = popen(cmd, "r")) == NULL) + testutil_die(errno, "popen"); + fscanf(fp, "%f", &cpu); + if (cpu > max) { + fprintf(stderr, "CPU usage: %f, max %f\n", cpu, max); + testutil_die(ERANGE, "CPU Usage"); + } + if (pclose(fp) != 0) + testutil_die(errno, "pclose"); } - if (pclose(fp) != 0) - testutil_die(errno, "pclose"); for (i = 0; i < dbs; ++i) testutil_check(conn[i]->close(conn[i], NULL)); -- cgit v1.2.1 From 634c9dcd9649afcc5c1db4eca08347f9e554152d Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 8 Mar 2016 12:14:51 -0500 Subject: WT-2318 Enhance test. Add different configs and some operations. --- test/manydbs/Makefile.am | 2 +- test/manydbs/manydbs.c | 121 +++++++++++++++++++++++++++++++++++++++++++---- test/manydbs/smoke.sh | 20 ++++++++ 3 files changed, 132 insertions(+), 11 deletions(-) create mode 100755 test/manydbs/smoke.sh diff --git a/test/manydbs/Makefile.am b/test/manydbs/Makefile.am index a04e6a7250b..53559b25243 100644 --- a/test/manydbs/Makefile.am +++ b/test/manydbs/Makefile.am @@ -7,7 +7,7 @@ t_LDADD = $(top_builddir)/libwiredtiger.la t_LDFLAGS = -static # Run this during a "make check" smoke test. -TESTS = $(noinst_PROGRAMS) +TESTS = smoke.sh clean-local: rm -rf WiredTiger* *.core __* diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index 2d6baa89a86..28a3dbd5ac9 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -45,17 +45,34 @@ static char home[HOME_SIZE]; /* Base home directory */ static char hometmp[HOME_SIZE]; /* Each conn home directory */ static const char *progname; /* Program name */ +static const char * const uri = "table:main"; -#define ENV_CONFIG \ - "create,log=(file_max=10M,archive=false,enabled)," +#define WTOPEN_CFG_COMMON \ + "create,log=(file_max=10M,archive=false,enabled)," \ + "statistics=(fast),statistics_log=(wait=5)," +#define WT_CONFIG0 \ + WTOPEN_CFG_COMMON \ + "transaction_sync=(enabled=false)" +#define WT_CONFIG1 \ + WTOPEN_CFG_COMMON \ + "transaction_sync=(enabled,method=none)" +#define WT_CONFIG2 \ + WTOPEN_CFG_COMMON \ + "transaction_sync=(enabled,method=fsync)" #define MAX_CPU 10.0 #define MAX_DBS 10 +#define MAX_IDLE_TIME 30 +#define IDLE_INCR 5 + +#define MAX_KV 100 +#define MAX_VAL 128 static void usage(void) { - fprintf(stderr, "usage: %s [-h dir]\n", progname); + fprintf(stderr, + "usage: %s [-I] [-C maxcpu%%] [-D maxdbs] [-h dir]\n", progname); exit(EXIT_FAILURE); } @@ -64,15 +81,67 @@ extern char *__wt_optarg; void (*custom_die)(void) = NULL; +WT_CONNECTION **conn = NULL; +WT_CURSOR **cursor = NULL; +WT_RAND_STATE rnd; +WT_SESSION **session = NULL; + +static int +run_ops(int dbs) +{ + WT_ITEM data; + int db, db_set, i, key, ret; + uint8_t buf[MAX_VAL]; + + memset(buf, 0, sizeof(buf)); + /* + * First time through, set up sessions, create the tables and + * open cursors. + */ + if (session == NULL) { + __wt_random_init(&rnd); + if ((session = calloc(sizeof(WT_SESSION *), dbs)) == NULL) + testutil_die(ENOMEM, "session array malloc"); + if ((cursor = calloc(sizeof(WT_CURSOR *), dbs)) == NULL) + testutil_die(ENOMEM, "cursor array malloc"); + for (i = 0; i < dbs; ++i) { + testutil_check(conn[i]->open_session(conn[i], + NULL, NULL, &session[i])); + testutil_check(session[i]->create(session[i], + uri, "key_format=Q,value_format=u")); + testutil_check(session[i]->open_cursor(session[i], + uri, NULL, NULL, &cursor[i])); + } + } + for (i = 0; i < MAX_VAL; ++i) + buf[i] = (uint8_t)__wt_random(&rnd); + data.data = buf; + /* + * Write a small amount of data into a random subset of the databases. + */ + db_set = dbs / 4; + for (i = 0; i < db_set; ++i) { + db = __wt_random(&rnd) % dbs; + printf("Write to database %d\n", db); + for (key = 0; key < MAX_KV; ++key) { + data.size = __wt_random(&rnd) % MAX_VAL; + cursor[db]->set_key(cursor[db], key); + cursor[db]->set_value(cursor[db], &data); + testutil_check(cursor[db]->insert(cursor[db])); + } + } + return (0); +} + int main(int argc, char *argv[]) { FILE *fp; - WT_CONNECTION **conn; float cpu, max; - int ch; - u_int dbs, i; + int cfg, ch, dbs, i; const char *working_dir; + bool idle; + const char *wt_cfg; char cmd[128]; if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) @@ -82,17 +151,21 @@ main(int argc, char *argv[]) dbs = MAX_DBS; working_dir = HOME_BASE; max = MAX_CPU; - while ((ch = __wt_getopt(progname, argc, argv, "C:D:h:")) != EOF) + idle = false; + while ((ch = __wt_getopt(progname, argc, argv, "C:D:h:I")) != EOF) switch (ch) { case 'C': max = (float)atof(__wt_optarg); break; case 'D': - dbs = (u_int)atoi(__wt_optarg); + dbs = atoi(__wt_optarg); break; case 'h': working_dir = __wt_optarg; break; + case 'I': + idle = true; + break; default: usage(); } @@ -112,14 +185,25 @@ main(int argc, char *argv[]) for (i = 0; i < dbs; ++i) { snprintf(hometmp, HOME_SIZE, "%s/%s.%d", home, HOME_BASE, i); testutil_make_work_dir(hometmp); + /* + * Open each database. Rotate different configurations + * among them. + */ + cfg = i % 3; + if (cfg == 0) + wt_cfg = WT_CONFIG0; + else if (cfg == 1) + wt_cfg = WT_CONFIG1; + else + wt_cfg = WT_CONFIG2; testutil_check(wiredtiger_open( - hometmp, NULL, ENV_CONFIG, &conn[i])); + hometmp, NULL, wt_cfg, &conn[i])); } sleep(10); (void)snprintf(cmd, sizeof(cmd), "ps -p %lu -o pcpu=", (unsigned long)getpid()); - for (i = 0; i < 30; i += 5) { + for (i = 0; i < MAX_IDLE_TIME; i += IDLE_INCR) { if ((fp = popen(cmd, "r")) == NULL) testutil_die(errno, "popen"); fscanf(fp, "%f", &cpu); @@ -129,8 +213,25 @@ main(int argc, char *argv[]) } if (pclose(fp) != 0) testutil_die(errno, "pclose"); + if (!idle) + testutil_check(run_ops(dbs)); + printf("Sleep %d (%d of %d)\n", IDLE_INCR, i, MAX_IDLE_TIME); + sleep(IDLE_INCR); } + /* + * Check CPU usage one last time. + */ + if ((fp = popen(cmd, "r")) == NULL) + testutil_die(errno, "popen"); + fscanf(fp, "%f", &cpu); + printf("Final CPU %f, max %f\n", cpu, max); + if (cpu > max) { + fprintf(stderr, "CPU usage: %f, max %f\n", cpu, max); + testutil_die(ERANGE, "CPU Usage"); + } + if (pclose(fp) != 0) + testutil_die(errno, "pclose"); for (i = 0; i < dbs; ++i) testutil_check(conn[i]->close(conn[i], NULL)); diff --git a/test/manydbs/smoke.sh b/test/manydbs/smoke.sh new file mode 100755 index 00000000000..41d9dd778a7 --- /dev/null +++ b/test/manydbs/smoke.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +trap 'chmod -R u+w WT_*; exit 0' 0 1 2 3 13 15 + +set -e + +# Smoke-test format as part of running "make check". +# Run with: +# 1. The defaults +# 2. Set idle flag to turn off operations. +# 3. More dbs. +# +echo "manydbs: default with operations turned on" +$TEST_WRAPPER ./t +echo "manydbs: totally idle databases" +$TEST_WRAPPER ./t -I +echo "manydbs: 40 databases with operations" +$TEST_WRAPPER ./t -D 40 +echo "manydbs: 40 idle databases" +$TEST_WRAPPER ./t -I -D 40 -- cgit v1.2.1 From 681f607ab11ba44dc55153736d642edf6fbe99a2 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 8 Mar 2016 12:17:46 -0500 Subject: WT-2318 Strings and whitespace. --- dist/s_string.ok | 2 ++ test/manydbs/manydbs.c | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dist/s_string.ok b/dist/s_string.ok index ab9aefccf06..db7f0032934 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -758,6 +758,8 @@ majorp malloc marshall marshalled +maxcpu +maxdbs mbll mbss mem diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index 28a3dbd5ac9..eeb8a93d07c 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -52,18 +52,18 @@ static const char * const uri = "table:main"; "statistics=(fast),statistics_log=(wait=5)," #define WT_CONFIG0 \ WTOPEN_CFG_COMMON \ - "transaction_sync=(enabled=false)" + "transaction_sync=(enabled=false)" #define WT_CONFIG1 \ WTOPEN_CFG_COMMON \ - "transaction_sync=(enabled,method=none)" + "transaction_sync=(enabled,method=none)" #define WT_CONFIG2 \ WTOPEN_CFG_COMMON \ - "transaction_sync=(enabled,method=fsync)" + "transaction_sync=(enabled,method=fsync)" #define MAX_CPU 10.0 #define MAX_DBS 10 #define MAX_IDLE_TIME 30 -#define IDLE_INCR 5 +#define IDLE_INCR 5 #define MAX_KV 100 #define MAX_VAL 128 -- cgit v1.2.1 From 87a33a9854cf70f113836b664055f062dbd31956 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 8 Mar 2016 12:22:54 -0500 Subject: WT-2318 Unused variable. --- test/manydbs/manydbs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index eeb8a93d07c..59bd5903b80 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -90,7 +90,7 @@ static int run_ops(int dbs) { WT_ITEM data; - int db, db_set, i, key, ret; + int db, db_set, i, key; uint8_t buf[MAX_VAL]; memset(buf, 0, sizeof(buf)); -- cgit v1.2.1 From ff0442671375025b1728d60a9a12dfe5c6d6862a Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 8 Mar 2016 13:06:19 -0500 Subject: WT-2318 Adjust CPU based on how many dbs are configured. --- test/manydbs/manydbs.c | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index 59bd5903b80..9b80771cbfe 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -140,7 +140,7 @@ main(int argc, char *argv[]) float cpu, max; int cfg, ch, dbs, i; const char *working_dir; - bool idle; + bool idle, setmax; const char *wt_cfg; char cmd[128]; @@ -150,12 +150,13 @@ main(int argc, char *argv[]) ++progname; dbs = MAX_DBS; working_dir = HOME_BASE; - max = MAX_CPU; - idle = false; + max = (float)dbs; + idle = setmax = false; while ((ch = __wt_getopt(progname, argc, argv, "C:D:h:I")) != EOF) switch (ch) { case 'C': max = (float)atof(__wt_optarg); + setmax = true; break; case 'D': dbs = atoi(__wt_optarg); @@ -171,6 +172,13 @@ main(int argc, char *argv[]) } argc -= __wt_optind; argv += __wt_optind; + /* + * Adjust the max cpu in relation to the number of databases, unless + * the user set it explicitly. If we're doing operations add in + * another 10%. + */ + if (!setmax) + max = (float)dbs + (idle ? 0.0 : (float)dbs * 0.10); if (argc != 0) usage(); @@ -208,8 +216,9 @@ main(int argc, char *argv[]) testutil_die(errno, "popen"); fscanf(fp, "%f", &cpu); if (cpu > max) { - fprintf(stderr, "CPU usage: %f, max %f\n", cpu, max); - testutil_die(ERANGE, "CPU Usage"); + fprintf(stderr, + "ERROR: CPU usage: %f, max %f\n", cpu, max); + testutil_die(ERANGE, "CPU"); } if (pclose(fp) != 0) testutil_die(errno, "pclose"); @@ -227,8 +236,8 @@ main(int argc, char *argv[]) fscanf(fp, "%f", &cpu); printf("Final CPU %f, max %f\n", cpu, max); if (cpu > max) { - fprintf(stderr, "CPU usage: %f, max %f\n", cpu, max); - testutil_die(ERANGE, "CPU Usage"); + fprintf(stderr, "ERROR: CPU usage: %f, max %f\n", cpu, max); + testutil_die(ERANGE, "CPU"); } if (pclose(fp) != 0) testutil_die(errno, "pclose"); -- cgit v1.2.1 From 0cdf0f718a8ef63cfc534e62b2faedbf4e01c24a Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 8 Mar 2016 13:20:40 -0500 Subject: WT-2318 Fix smoke.sh script so it correctly fails. --- test/manydbs/manydbs.c | 2 +- test/manydbs/smoke.sh | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index 9b80771cbfe..7d66b17c8bd 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -173,7 +173,7 @@ main(int argc, char *argv[]) argc -= __wt_optind; argv += __wt_optind; /* - * Adjust the max cpu in relation to the number of databases, unless + * Adjust the maxcpu in relation to the number of databases, unless * the user set it explicitly. If we're doing operations add in * another 10%. */ diff --git a/test/manydbs/smoke.sh b/test/manydbs/smoke.sh index 41d9dd778a7..c0e2976f154 100755 --- a/test/manydbs/smoke.sh +++ b/test/manydbs/smoke.sh @@ -1,7 +1,5 @@ #!/bin/sh -trap 'chmod -R u+w WT_*; exit 0' 0 1 2 3 13 15 - set -e # Smoke-test format as part of running "make check". -- cgit v1.2.1 From bc5a8ac83b2e1331bc039cc7981751aad09f60e6 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 8 Mar 2016 13:56:52 -0500 Subject: WT-2381: dump utility discards table config Parenthesize a macro argument. --- src/include/misc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/misc.h b/src/include/misc.h index 4d3ca758dc7..07d52c61eac 100644 --- a/src/include/misc.h +++ b/src/include/misc.h @@ -198,7 +198,7 @@ /* Check if a string matches a prefix. */ #define WT_PREFIX_MATCH(str, pfx) \ - (((const char *)str)[0] == ((const char *)pfx)[0] && \ + (((const char *)(str))[0] == ((const char *)pfx)[0] && \ strncmp((str), (pfx), strlen(pfx)) == 0) /* Check if a string matches a prefix, and move past it. */ -- cgit v1.2.1 From b3a40e79a018e0b7a5330a4e0db575c5f7c9b5f1 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 8 Mar 2016 13:59:34 -0500 Subject: WT-2381: dump utility discards table config Add LSM tables to the dump test. After dumping/re-loading the database, confirm that the contents of the database are the same by comparing the objects returned by wt list. Replace simple_populate_check_cursor/complex_populate_check_cursor with simple_populate_check/complex_populate_check, then we don't have to open a cursor. --- test/suite/test_dump.py | 50 ++++++++++++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/test/suite/test_dump.py b/test/suite/test_dump.py index a9795dde903..fc1422155e2 100644 --- a/test/suite/test_dump.py +++ b/test/suite/test_dump.py @@ -29,8 +29,8 @@ import os import wiredtiger, wttest from helper import \ - complex_populate, complex_populate_check_cursor,\ - simple_populate, simple_populate_check_cursor + complex_populate, complex_populate_check, \ + simple_populate, simple_populate_check from suite_subprocess import suite_subprocess from wtscenario import multiply_scenarios, number_scenarios @@ -54,18 +54,24 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): ('string', dict(keyfmt='S')) ] types = [ - ('file', dict(type='file:', + ('file', dict(uri='file:', config='', lsm=False, populate=simple_populate, - populate_check=simple_populate_check_cursor)), - ('lsm', dict(type='lsm:', + populate_check=simple_populate_check)), + ('lsm', dict(uri='lsm:', config='', lsm=True, populate=simple_populate, - populate_check=simple_populate_check_cursor)), - ('table-simple', dict(type='table:', + populate_check=simple_populate_check)), + ('table-simple', dict(uri='table:', config='', lsm=False, populate=simple_populate, - populate_check=simple_populate_check_cursor)), - ('table-complex', dict(type='table:', + populate_check=simple_populate_check)), + ('table-simple-lsm', dict(uri='table:', config='type=lsm', lsm=True, + populate=simple_populate, + populate_check=simple_populate_check)), + ('table-complex', dict(uri='table:', config='', lsm=False, + populate=complex_populate, + populate_check=complex_populate_check)), + ('table-complex-lsm', dict(uri='table:', config='type=lsm', lsm=True, populate=complex_populate, - populate_check=complex_populate_check_cursor)) + populate_check=complex_populate_check)) ] scenarios = number_scenarios( multiply_scenarios('.', types, keyfmt, dumpfmt)) @@ -98,12 +104,13 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): # Dump, re-load and do a content comparison. def test_dump(self): # LSM and column-store isn't a valid combination. - if self.type == 'lsm:' and self.keyfmt == 'r': + if self.lsm and self.keyfmt == 'r': return # Create the object. - uri = self.type + self.name - self.populate(self, uri, 'key_format=' + self.keyfmt, self.nentries) + uri = self.uri + self.name + self.populate(self, uri, + self.config + ',key_format=' + self.keyfmt, self.nentries) # Dump the object. os.mkdir(self.dir) @@ -115,11 +122,17 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): # Re-load the object. self.runWt(['-h', self.dir, 'load', '-f', 'dump.out']) - # Check the contents + # Check the database contents + self.runWt(['list'], outfilename='list.out') + self.runWt(['-h', self.dir, 'list'], outfilename='list.out.new') + s1 = set(open('list.out').read().split()) + s2 = set(open('list.out.new').read().split()) + self.assertEqual(not s1.symmetric_difference(s2), True) + + # Check the object's contents conn = self.wiredtiger_open(self.dir) session = conn.open_session() - cursor = session.open_cursor(uri, None, None) - self.populate_check(self, cursor, self.nentries) + self.populate_check(self, uri, self.nentries) conn.close() # Re-load the object again. @@ -128,8 +141,7 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): # Check the contents, they shouldn't have changed. conn = self.wiredtiger_open(self.dir) session = conn.open_session() - cursor = session.open_cursor(uri, None, None) - self.populate_check(self, cursor, self.nentries) + self.populate_check(self, uri, self.nentries) conn.close() # Re-load the object again, but confirm -n (no overwrite) fails. @@ -137,7 +149,7 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): 'load', '-n', '-f', 'dump.out'], errfilename='errfile.out') self.check_non_empty_file('errfile.out') - # If there is are indices, dump one of them and check the output. + # If there are indices, dump one of them and check the output. if self.populate == complex_populate: indexuri = 'index:' + self.name + ':indx1' hexopt = ['-x'] if self.hex == 1 else [] -- cgit v1.2.1 From 5a2c04f7a5c4e99f7f45d32f0218634f6bed984c Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 8 Mar 2016 14:46:20 -0500 Subject: WT-2318 Only check CPU usage at end of test. It can spike and report higher at the start. --- test/manydbs/manydbs.c | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index 7d66b17c8bd..07fcb06adff 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -174,11 +174,10 @@ main(int argc, char *argv[]) argv += __wt_optind; /* * Adjust the maxcpu in relation to the number of databases, unless - * the user set it explicitly. If we're doing operations add in - * another 10%. + * the user set it explicitly. */ if (!setmax) - max = (float)dbs + (idle ? 0.0 : (float)dbs * 0.10); + max = (float)dbs; if (argc != 0) usage(); @@ -209,19 +208,7 @@ main(int argc, char *argv[]) } sleep(10); - (void)snprintf(cmd, sizeof(cmd), - "ps -p %lu -o pcpu=", (unsigned long)getpid()); for (i = 0; i < MAX_IDLE_TIME; i += IDLE_INCR) { - if ((fp = popen(cmd, "r")) == NULL) - testutil_die(errno, "popen"); - fscanf(fp, "%f", &cpu); - if (cpu > max) { - fprintf(stderr, - "ERROR: CPU usage: %f, max %f\n", cpu, max); - testutil_die(ERANGE, "CPU"); - } - if (pclose(fp) != 0) - testutil_die(errno, "pclose"); if (!idle) testutil_check(run_ops(dbs)); printf("Sleep %d (%d of %d)\n", IDLE_INCR, i, MAX_IDLE_TIME); @@ -229,8 +216,10 @@ main(int argc, char *argv[]) } /* - * Check CPU usage one last time. + * Check CPU after all idling or work is done. */ + (void)snprintf(cmd, sizeof(cmd), + "ps -p %lu -o pcpu=", (unsigned long)getpid()); if ((fp = popen(cmd, "r")) == NULL) testutil_die(errno, "popen"); fscanf(fp, "%f", &cpu); -- cgit v1.2.1 From 77ac1479ad83a79c6ecb1fa049e78b5bf8eccf2d Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 8 Mar 2016 16:27:24 -0500 Subject: WT-2381: dump utility discards table config Rework the dump utility and the dump library support. Previously, the metadata cursor returned a full set of WT_SESSION.create configuration values, basically, the requested configuration values plus the default values, where the requested configuration values overrode the default configuration values. This doesn't work because dump takes a few configuration strings and collapses them into a single string, and the default configuration values start overriding real configurtion values. Change the metadata cursor to return only the requested configuration values instead, and change dump to add in the default values when it's collapsing the strings into a single string. Add a function __wt_schema_create_final; it takes a set of configuration strings, adds in the default WT_SESSION.create configuration values, and collapses them into a single string. Add a function __wt_config_strip_and_collapse; it behaves similarly to __wt_config_collapse, except it doesn't add in the default strings. Fix bugs where we weren't copying returned metadata strings into local memory. Fix bugs where we weren't correctly parsing the URIs in the metadata file. --- src/config/config_collapse.c | 48 +++++++++++++++++--- src/cursor/cur_metadata.c | 56 +++++++++++++++++++++-- src/include/extern.h | 3 +- src/schema/schema_create.c | 16 ------- src/utilities/util_dump.c | 105 +++++++++++++++++++++++++------------------ 5 files changed, 159 insertions(+), 69 deletions(-) diff --git a/src/config/config_collapse.c b/src/config/config_collapse.c index 27bd6255a0a..890f1f16ed5 100644 --- a/src/config/config_collapse.c +++ b/src/config/config_collapse.c @@ -9,7 +9,7 @@ #include "wt_internal.h" /* - * __wt_config_collapse -- + * __config_collapse -- * Collapse a set of configuration strings into newly allocated memory. * * This function takes a NULL-terminated list of configuration strings (where @@ -28,10 +28,14 @@ * "key=(k4=v4)", the result will be "key=(k4=v4)", as we search for and * use the final value of "key", regardless of field overlap or missing * fields in the nested value. + * + * Optionally, values found only in the first string aren't included in the + * final result, that is, the first string is only used as a list of valid + * configuration strings. */ -int -__wt_config_collapse( - WT_SESSION_IMPL *session, const char **cfg, char **config_ret) +static int +__config_collapse(WT_SESSION_IMPL *session, + const char **cfg, bool include_defaults, char **config_ret) { WT_CONFIG cparser; WT_CONFIG_ITEM k, v; @@ -41,12 +45,22 @@ __wt_config_collapse( WT_RET(__wt_scr_alloc(session, 0, &tmp)); WT_ERR(__wt_config_init(session, &cparser, cfg[0])); + if (!include_defaults) + ++cfg; while ((ret = __wt_config_next(&cparser, &k, &v)) == 0) { if (k.type != WT_CONFIG_ITEM_STRING && k.type != WT_CONFIG_ITEM_ID) WT_ERR_MSG(session, EINVAL, "Invalid configuration key found: '%s'\n", k.str); - WT_ERR(__wt_config_get(session, cfg, &k, &v)); + /* + * Not-found is an expected error when not including entries + * from the first configuration string. + */ + if ((ret = __wt_config_get(session, cfg, &k, &v)) != 0) { + WT_ERR_NOTFOUND_OK(ret); + continue; + } + /* Include the quotes around string keys/values. */ if (k.type == WT_CONFIG_ITEM_STRING) { --k.str; @@ -77,6 +91,30 @@ err: __wt_scr_free(session, &tmp); return (ret); } +/* + * __wt_config_collapse -- + * Collapse a set of configuration strings into newly allocated memory, + * including the defaults from the first configuration string. + */ +int +__wt_config_collapse( + WT_SESSION_IMPL *session, const char **cfg, char **config_ret) +{ + return (__config_collapse(session, cfg, true, config_ret)); +} + +/* + * __wt_config_strip_and_collapse -- + * Collapse a set of configuration strings into newly allocated memory, + * NOT including the defaults from the first configuration string. + */ +int +__wt_config_strip_and_collapse( + WT_SESSION_IMPL *session, const char **cfg, char **config_ret) +{ + return (__config_collapse(session, cfg, false, config_ret)); +} + /* * We need a character that can't appear in a key as a separator. */ diff --git a/src/cursor/cur_metadata.c b/src/cursor/cur_metadata.c index df2cc3f546e..c25c7d1bdee 100644 --- a/src/cursor/cur_metadata.c +++ b/src/cursor/cur_metadata.c @@ -30,6 +30,56 @@ WT_CURSTD_VALUE_EXT); \ } while (0) +/* + * __wt_schema_create_final -- + * Create a single configuration line from a set of configuration strings, + * including all of the defaults declared for a session.create. Here for the wt + * dump command utility, which reads a set of configuration strings and needs to + * add in the defaults and then collapse them into single string for load. + */ +int +__wt_schema_create_final( + WT_SESSION_IMPL *session, char *cfg_arg[], char **value_ret) +{ + WT_DECL_RET; + u_int i; + const char **cfg; + + /* + * Count the entries in the original, + * Allocate a copy with the defaults as the first entry, + * Collapse the whole thing into a single configuration string. + */ + for (i = 0; cfg_arg[i] != NULL; ++i) + ; + WT_RET(__wt_calloc_def(session, i + 2, &cfg)); + cfg[0] = WT_CONFIG_BASE(session, WT_SESSION_create); + for (i = 0; cfg_arg[i] != NULL; ++i) + cfg[i + 1] = cfg_arg[i]; + cfg[i + 1] = NULL; + + ret = __wt_config_collapse(session, cfg, value_ret); + + __wt_free(session, cfg); + return (ret); +} + +/* + * __schema_create_strip -- + * Discard any configuration information from a schema entry that is not + * applicable to an session.create call. Here for the wt dump command utility, + * which only wants to dump the schema information needed for load. + */ +static int +__schema_create_strip( + WT_SESSION_IMPL *session, const char *value, char **value_ret) +{ + const char *cfg[] = + { WT_CONFIG_BASE(session, WT_SESSION_create), value, NULL }; + + return (__wt_config_strip_and_collapse(session, cfg, value_ret)); +} + /* * __curmetadata_setkv -- * Copy key/value into the public cursor, stripping internal metadata for @@ -49,8 +99,7 @@ __curmetadata_setkv(WT_CURSOR_METADATA *mdc, WT_CURSOR *fc) c->key.data = fc->key.data; c->key.size = fc->key.size; if (F_ISSET(mdc, WT_MDC_CREATEONLY)) { - WT_RET(__wt_schema_create_strip( - session, fc->value.data, NULL, &value)); + WT_RET(__schema_create_strip(session, fc->value.data, &value)); ret = __wt_buf_set( session, &c->value, value, strlen(value) + 1); __wt_free(session, value); @@ -92,8 +141,7 @@ __curmetadata_metadata_search(WT_SESSION_IMPL *session, WT_CURSOR *cursor) WT_RET(__wt_metadata_search(session, WT_METAFILE_URI, &value)); if (F_ISSET(mdc, WT_MDC_CREATEONLY)) { - ret = __wt_schema_create_strip( - session, value, NULL, &stripped); + ret = __schema_create_strip(session, value, &stripped); __wt_free(session, value); WT_RET(ret); value = stripped; diff --git a/src/include/extern.h b/src/include/extern.h index 55b0b8cd7ff..8a5e6630df9 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -220,6 +220,7 @@ extern void __wt_conn_foc_discard(WT_SESSION_IMPL *session); extern int __wt_configure_method(WT_SESSION_IMPL *session, const char *method, const char *uri, const char *config, const char *type, const char *check); extern int __wt_config_check(WT_SESSION_IMPL *session, const WT_CONFIG_ENTRY *entry, const char *config, size_t config_len); extern int __wt_config_collapse( WT_SESSION_IMPL *session, const char **cfg, char **config_ret); +extern int __wt_config_strip_and_collapse( WT_SESSION_IMPL *session, const char **cfg, char **config_ret); extern int __wt_config_merge(WT_SESSION_IMPL *session, const char **cfg, const char *cfg_strip, const char **config_ret); extern int __wt_conn_config_init(WT_SESSION_IMPL *session); extern void __wt_conn_config_discard(WT_SESSION_IMPL *session); @@ -297,6 +298,7 @@ extern int __wt_json_to_item(WT_SESSION_IMPL *session, const char *jstr, const c extern ssize_t __wt_json_strlen(const char *src, size_t srclen); extern int __wt_json_strncpy(char **pdst, size_t dstlen, const char *src, size_t srclen); extern int __wt_curlog_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp); +extern int __wt_schema_create_final( WT_SESSION_IMPL *session, char *cfg_arg[], char **value_ret); extern int __wt_curmetadata_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp); extern void __wt_curstat_dsrc_final(WT_CURSOR_STAT *cst); extern int __wt_curstat_init(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *curjoin, const char *cfg[], WT_CURSOR_STAT *cst); @@ -576,7 +578,6 @@ extern int __wt_bulk_insert_row(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) extern int __wt_bulk_insert_fix( WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool deleted); extern int __wt_bulk_insert_fix_bitmap(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk); extern int __wt_bulk_insert_var( WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool deleted); -extern int __wt_schema_create_strip(WT_SESSION_IMPL *session, const char *v1, const char *v2, char **value_ret); extern int __wt_direct_io_size_check(WT_SESSION_IMPL *session, const char **cfg, const char *config_name, uint32_t *allocsizep); extern int __wt_schema_colgroup_source(WT_SESSION_IMPL *session, WT_TABLE *table, const char *cgname, const char *config, WT_ITEM *buf); extern int __wt_schema_index_source(WT_SESSION_IMPL *session, WT_TABLE *table, const char *idxname, const char *config, WT_ITEM *buf); diff --git a/src/schema/schema_create.c b/src/schema/schema_create.c index 9b3b76b62de..756f1fdcc6c 100644 --- a/src/schema/schema_create.c +++ b/src/schema/schema_create.c @@ -8,22 +8,6 @@ #include "wt_internal.h" -/* - * __wt_schema_create_strip -- - * Discard any configuration information from a schema entry that is not - * applicable to an session.create call, here for the wt dump command utility, - * which only wants to dump the schema information needed for load. - */ -int -__wt_schema_create_strip(WT_SESSION_IMPL *session, - const char *v1, const char *v2, char **value_ret) -{ - const char *cfg[] = - { WT_CONFIG_BASE(session, WT_SESSION_create), v1, v2, NULL }; - - return (__wt_config_collapse(session, cfg, value_ret)); -} - /* * __wt_direct_io_size_check -- * Return a size from the configuration, complaining if it's insufficient diff --git a/src/utilities/util_dump.c b/src/utilities/util_dump.c index 287f784770c..95515931a65 100644 --- a/src/utilities/util_dump.c +++ b/src/utilities/util_dump.c @@ -25,7 +25,7 @@ static int dump_table_config(WT_SESSION *, WT_CURSOR *, const char *); static int dump_table_config_type( WT_SESSION *, WT_CURSOR *, WT_CURSOR *, const char *, const char *); static int dup_json_string(const char *, char **); -static int print_config(WT_SESSION *, const char *, const char *, const char *); +static int print_config(WT_SESSION *, const char *, char *[]); static int usage(void); int @@ -459,10 +459,11 @@ dump_table_config(WT_SESSION *session, WT_CURSOR *cursor, const char *uri) WT_DECL_RET; size_t len; int tret; - const char *name, *value; - char *p; + const char *name, *v; + char *p, **cfg, *_cfg[4] = {NULL, NULL, NULL, NULL}; p = NULL; + cfg = &_cfg[3]; /* Get the table name. */ if ((name = strchr(uri, ':')) == NULL) { @@ -472,13 +473,24 @@ dump_table_config(WT_SESSION *session, WT_CURSOR *cursor, const char *uri) ++name; /* - * Dump out the config information: first, dump the uri entry itself. - * + * Dump out the config information: first, dump the uri entry itself, + * it overrides all subsequent configurations. + */ + cursor->set_key(cursor, uri); + if ((ret = cursor->search(cursor)) != 0) + return (util_cerr(cursor, "search", ret)); + if ((ret = cursor->get_value(cursor, &v)) != 0) + return (util_cerr(cursor, "get_value", ret)); + if ((*--cfg = strdup(v)) == NULL) + return (util_err(session, errno, NULL)); + + /* * Workaround for WiredTiger "simple" table handling. Simple tables * have column-group entries, but they aren't listed in the metadata's * table entry. Figure out if it's a simple table and in that case, - * retrieve the column-group entry and use the value from its "source" - * file. + * retrieve the column-group entry and then retrieve the value from + * its "source" file, where the column-group entry configuration + * overrides the source value. */ if (WT_PREFIX_MATCH(uri, "table:")) { len = strlen(uri) + strlen("colgroup:"); @@ -487,11 +499,13 @@ dump_table_config(WT_SESSION *session, WT_CURSOR *cursor, const char *uri) (void)snprintf(p, len, "colgroup:%s", name); cursor->set_key(cursor, p); if ((ret = cursor->search(cursor)) == 0) { - if ((ret = cursor->get_value(cursor, &value)) != 0) + if ((ret = cursor->get_value(cursor, &v)) != 0) return (util_cerr(cursor, "get_value", ret)); + if ((*--cfg = strdup(v)) == NULL) + return (util_err(session, errno, NULL)); if ((ret =__wt_config_getones( (WT_SESSION_IMPL *)session, - value, "source", &cval)) != 0) + *cfg, "source", &cval)) != 0) return (util_err( session, ret, "%s: source entry", p)); free(p); @@ -500,24 +514,24 @@ dump_table_config(WT_SESSION *session, WT_CURSOR *cursor, const char *uri) return (util_err(session, errno, NULL)); (void)snprintf(p, len, "%.*s", (int)cval.len, cval.str); cursor->set_key(cursor, p); - } else - cursor->set_key(cursor, uri); - } else - cursor->set_key(cursor, uri); + if ((ret = cursor->search(cursor)) != 0) + return (util_cerr(cursor, "search", ret)); + if ((ret = cursor->get_value(cursor, &v)) != 0) + return (util_cerr(cursor, "get_value", ret)); + if ((*--cfg = strdup(v)) == NULL) + return (util_err(session, errno, NULL)); + } + } - if ((ret = cursor->search(cursor)) != 0) - return (util_cerr(cursor, "search", ret)); - if ((ret = cursor->get_value(cursor, &value)) != 0) - return (util_cerr(cursor, "get_value", ret)); - if (print_config(session, uri, value, NULL) != 0) + if (print_config(session, uri, cfg) != 0) return (1); /* * The underlying table configuration function needs a second cursor: * open one before calling it, it makes error handling hugely simpler. */ - if ((ret = - session->open_cursor(session, NULL, cursor, NULL, &srch)) != 0) + if ((ret = session->open_cursor( + session, "metadata:create", NULL, NULL, &srch)) != 0) return (util_cerr(cursor, "open_cursor", ret)); if ((ret = dump_table_config_type( @@ -532,6 +546,9 @@ dump_table_config(WT_SESSION *session, WT_CURSOR *cursor, const char *uri) } free(p); + free(_cfg[0]); + free(_cfg[1]); + free(_cfg[2]); return (ret); } @@ -545,15 +562,17 @@ dump_table_config_type(WT_SESSION *session, { WT_CONFIG_ITEM cval; WT_DECL_RET; - const char *key, *skip, *value, *value_source; + const char *key; size_t len; int exact; - char *p; + const char *v; + char *p, *cfg[3] = {NULL, NULL, NULL}; /* * Search the file looking for column group and index key/value pairs: * for each one, look up the related source information and append it - * to the base record. + * to the base record, where the column group and index configuration + * overrides the source configuration. */ cursor->set_key(cursor, entry); if ((ret = cursor->search_near(cursor, &exact)) != 0) { @@ -572,18 +591,18 @@ match: if ((ret = cursor->get_key(cursor, &key)) != 0) return (0); /* Check for a table name match. */ - skip = key + strlen(entry); - if (strncmp( - skip, name, strlen(name)) != 0 || skip[strlen(name)] != ':') + if (!WT_PREFIX_MATCH(key + strlen(entry), name)) continue; /* Get the value. */ - if ((ret = cursor->get_value(cursor, &value)) != 0) + if ((ret = cursor->get_value(cursor, &v)) != 0) return (util_cerr(cursor, "get_value", ret)); + if ((cfg[1] = strdup(v)) == NULL) + return (util_err(session, errno, NULL)); /* Crack it and get the underlying source. */ if ((ret = __wt_config_getones( - (WT_SESSION_IMPL *)session, value, "source", &cval)) != 0) + (WT_SESSION_IMPL *)session, cfg[1], "source", &cval)) != 0) return ( util_err(session, ret, "%s: source entry", key)); @@ -600,16 +619,22 @@ match: if ((ret = cursor->get_key(cursor, &key)) != 0) return (1); /* Get the source's value. */ - if ((ret = srch->get_value(srch, &value_source)) != 0) + if ((ret = srch->get_value(srch, &v)) != 0) return (util_cerr(cursor, "get_value", ret)); + if ((cfg[0] = strdup(v)) == NULL) + return (util_err(session, errno, NULL)); /* * The dumped configuration string is the original key plus the - * source's configuration. + * source's configuration, where the values of the original key + * override any source configurations of the same name. */ - if (print_config(session, key, value, value_source) != 0) + if (print_config(session, key, cfg) != 0) return (util_err(session, EIO, NULL)); } + free(cfg[0]); + free(cfg[1]); + if (ret == 0 || ret == WT_NOTFOUND) return (0); return (util_cerr(cursor, "next", ret)); @@ -723,27 +748,21 @@ dup_json_string(const char *str, char **result) * Output a key/value URI pair by combining v1 and v2. */ static int -print_config(WT_SESSION *session, - const char *key, const char *v1, const char *v2) +print_config(WT_SESSION *session, const char *key, char *cfg[]) { WT_DECL_RET; char *value_ret; - const char *cfg[] = { v1, v2, NULL }; /* - * The underlying call will stop if the first string is NULL -- check - * here and swap in that case. + * We have all of the object configuration, but don't have the default + * session.create configuration. Have the underlying library add in the + * defaults and collapse it all into one load configuration string. */ - if (cfg[0] == NULL) { - cfg[0] = cfg[1]; - cfg[1] = NULL; - } - - if ((ret = __wt_config_collapse( + if ((ret = __wt_schema_create_final( (WT_SESSION_IMPL *)session, cfg, &value_ret)) != 0) return (util_err(session, ret, NULL)); ret = printf("%s\n%s\n", key, value_ret); - free((char *)value_ret); + free(value_ret); if (ret < 0) return (util_err(session, EIO, NULL)); return (0); -- cgit v1.2.1 From 840c45242a4c5870c0b3618d20c0b610bc801cde Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 8 Mar 2016 17:12:16 -0500 Subject: WT-2381: dump utility discards table config One of the changes in 77ac147 changed the test for column-group and index names, and the changed version matches both simple and complex entries. Leave the changed test alone, instead don't look for separate column-group and index entries in the case of a simple table. --- src/utilities/util_dump.c | 63 ++++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/src/utilities/util_dump.c b/src/utilities/util_dump.c index 95515931a65..ca9b43c1647 100644 --- a/src/utilities/util_dump.c +++ b/src/utilities/util_dump.c @@ -22,7 +22,7 @@ static int dump_prefix(WT_SESSION *, bool); static int dump_record(WT_CURSOR *, bool, bool); static int dump_suffix(WT_SESSION *); static int dump_table_config(WT_SESSION *, WT_CURSOR *, const char *); -static int dump_table_config_type( +static int dump_table_config_complex( WT_SESSION *, WT_CURSOR *, WT_CURSOR *, const char *, const char *); static int dup_json_string(const char *, char **); static int print_config(WT_SESSION *, const char *, char *[]); @@ -459,6 +459,7 @@ dump_table_config(WT_SESSION *session, WT_CURSOR *cursor, const char *uri) WT_DECL_RET; size_t len; int tret; + bool complex_table; const char *name, *v; char *p, **cfg, *_cfg[4] = {NULL, NULL, NULL, NULL}; @@ -487,11 +488,12 @@ dump_table_config(WT_SESSION *session, WT_CURSOR *cursor, const char *uri) /* * Workaround for WiredTiger "simple" table handling. Simple tables * have column-group entries, but they aren't listed in the metadata's - * table entry. Figure out if it's a simple table and in that case, - * retrieve the column-group entry and then retrieve the value from - * its "source" file, where the column-group entry configuration - * overrides the source value. + * table entry, and the name is different from other column-groups. + * Figure out if it's a simple table and in that case, retrieve the + * column-group's configuration value and the column-group's "source" + * entry, where the column-group entry overrides the source's. */ + complex_table = false; if (WT_PREFIX_MATCH(uri, "table:")) { len = strlen(uri) + strlen("colgroup:"); if ((p = malloc(len)) == NULL) @@ -520,29 +522,33 @@ dump_table_config(WT_SESSION *session, WT_CURSOR *cursor, const char *uri) return (util_cerr(cursor, "get_value", ret)); if ((*--cfg = strdup(v)) == NULL) return (util_err(session, errno, NULL)); - } + } else + complex_table = true; } if (print_config(session, uri, cfg) != 0) return (1); - /* - * The underlying table configuration function needs a second cursor: - * open one before calling it, it makes error handling hugely simpler. - */ - if ((ret = session->open_cursor( - session, "metadata:create", NULL, NULL, &srch)) != 0) - return (util_cerr(cursor, "open_cursor", ret)); - - if ((ret = dump_table_config_type( - session, cursor, srch, name, "colgroup:")) == 0) - ret = dump_table_config_type( - session, cursor, srch, name, "index:"); - - if ((tret = srch->close(srch)) != 0) { - tret = util_cerr(cursor, "close", tret); - if (ret == 0) - ret = tret; + if (complex_table) { + /* + * The underlying table configuration function needs a second + * cursor: open one before calling it, it makes error handling + * hugely simpler. + */ + if ((ret = session->open_cursor( + session, "metadata:create", NULL, NULL, &srch)) != 0) + return (util_cerr(cursor, "open_cursor", ret)); + + if ((ret = dump_table_config_complex( + session, cursor, srch, name, "colgroup:")) == 0) + ret = dump_table_config_complex( + session, cursor, srch, name, "index:"); + + if ((tret = srch->close(srch)) != 0) { + tret = util_cerr(cursor, "close", tret); + if (ret == 0) + ret = tret; + } } free(p); @@ -553,11 +559,11 @@ dump_table_config(WT_SESSION *session, WT_CURSOR *cursor, const char *uri) } /* - * dump_table_config_type -- + * dump_table_config_complex -- * Dump the column groups or indices for a table. */ static int -dump_table_config_type(WT_SESSION *session, +dump_table_config_complex(WT_SESSION *session, WT_CURSOR *cursor, WT_CURSOR *srch, const char *name, const char *entry) { WT_CONFIG_ITEM cval; @@ -590,7 +596,12 @@ match: if ((ret = cursor->get_key(cursor, &key)) != 0) if (!WT_PREFIX_MATCH(key, entry)) return (0); - /* Check for a table name match. */ + /* + * Check for a table name match. This test will match "simple" + * table column-groups as well as the more complex ones, but + * the previous version of the test was wrong and we're only + * in this function in the case of complex tables. + */ if (!WT_PREFIX_MATCH(key + strlen(entry), name)) continue; -- cgit v1.2.1 From 3cdcf361b6c60356610279bb30b8d59722ebc7d8 Mon Sep 17 00:00:00 2001 From: Mark Benvenuto Date: Wed, 9 Mar 2016 23:11:23 +1100 Subject: WT-2456 Update CRC32 Power8 Code --- src/support/power8/crc32.S | 100 ++++++++++++++++++--------------------------- 1 file changed, 40 insertions(+), 60 deletions(-) diff --git a/src/support/power8/crc32.S b/src/support/power8/crc32.S index 3ef2928aaa1..c0b81143f07 100644 --- a/src/support/power8/crc32.S +++ b/src/support/power8/crc32.S @@ -65,14 +65,13 @@ #define off96 r30 #define off112 r31 -#define const1 v25 -#define const2 v26 +#define const1 v24 +#define const2 v25 -#define byteswap v27 -#define mask_32bit v28 -#define mask_64bit v29 -#define zeroes v30 -#define ones v31 +#define byteswap v26 +#define mask_32bit v27 +#define mask_64bit v28 +#define zeroes v29 #ifdef BYTESWAP_DATA #define VPERM(A, B, C, D) vperm A, B, C, D @@ -90,31 +89,6 @@ FUNC_START(__crc32_vpmsum) std r26,-48(r1) std r25,-56(r1) - li r31, -256 - stvx v20, r31, r1 - li r31, -240 - stvx v21, r31, r1 - li r31, -224 - stvx v22, r31, r1 - li r31, -208 - stvx v23, r31, r1 - li r31, -192 - stvx v24, r31, r1 - li r31, -176 - stvx v25, r31, r1 - li r31, -160 - stvx v26, r31, r1 - li r31, -144 - stvx v27, r31, r1 - li r31, -128 - stvx v28, r31, r1 - li r31, -112 - stvx v29, r31, r1 - li r31, -96 - stvx v30, r31, r1 - li r31, -80 - stvx v31, r31, r1 - li off16,16 li off32,32 li off48,48 @@ -124,13 +98,28 @@ FUNC_START(__crc32_vpmsum) li off112,112 li r0,0 + /* Enough room for saving 10 non volatile VMX registers */ + subi r6,r1,56+10*16 + subi r7,r1,56+2*16 + + stvx v20,0,r6 + stvx v21,off16,r6 + stvx v22,off32,r6 + stvx v23,off48,r6 + stvx v24,off64,r6 + stvx v25,off80,r6 + stvx v26,off96,r6 + stvx v27,off112,r6 + stvx v28,0,r7 + stvx v29,off16,r7 + mr r10,r3 vxor zeroes,zeroes,zeroes - vspltisw ones,-1 + vspltisw v0,-1 - vsldoi mask_32bit,zeroes,ones,4 - vsldoi mask_64bit,zeroes,ones,8 + vsldoi mask_32bit,zeroes,v0,4 + vsldoi mask_64bit,zeroes,v0,8 /* Get the initial value into v8 */ vxor v8,v8,v8 @@ -596,30 +585,20 @@ FUNC_START(__crc32_vpmsum) /* Get it into r3 */ MFVRD(r3, v0) - li r31, -256 - lvx v20, r31, r1 - li r31, -240 - lvx v21, r31, r1 - li r31, -224 - lvx v22, r31, r1 - li r31, -208 - lvx v23, r31, r1 - li r31, -192 - lvx v24, r31, r1 - li r31, -176 - lvx v25, r31, r1 - li r31, -160 - lvx v26, r31, r1 - li r31, -144 - lvx v27, r31, r1 - li r31, -128 - lvx v28, r31, r1 - li r31, -112 - lvx v29, r31, r1 - li r31, -96 - lvx v30, r31, r1 - li r31, -80 - lvx v31, r31, r1 +.Lout: + subi r6,r1,56+10*16 + subi r7,r1,56+2*16 + + lvx v20,0,r6 + lvx v21,off16,r6 + lvx v22,off32,r6 + lvx v23,off48,r6 + lvx v24,off64,r6 + lvx v25,off80,r6 + lvx v26,off96,r6 + lvx v27,off112,r6 + lvx v28,0,r7 + lvx v29,off16,r7 ld r31,-8(r1) ld r30,-16(r1) @@ -786,6 +765,7 @@ FUNC_START(__crc32_vpmsum) .Lzero: mr r3,r10 - blr + b .Lout + FUNC_END(__crc32_vpmsum) #endif -- cgit v1.2.1 From 7cd409dff1cc2cfa9cc6e3e18269ec7e24328012 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 9 Mar 2016 08:28:22 -0500 Subject: WT-2454: checkpoint_sync=false does not prevent flushes/sync to disk WiredTiger was doing leaf page flushes even when checkpoint_sync=false, as well as one other less-interesting checkpoint flush (the checkpoint of the metadata). Don't flush leaf pages if checkpoint_sync=false, and check for checkpoint_sync=false in the underlying flush routine instead of all of the callers -- this isn't worth optimizing out a function call and it makes things simpler. --- src/btree/bt_sync.c | 7 +++++-- src/txn/txn_ckpt.c | 16 +++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/btree/bt_sync.c b/src/btree/bt_sync.c index 5273f0ee2c3..c0647467323 100644 --- a/src/btree/bt_sync.c +++ b/src/btree/bt_sync.c @@ -17,6 +17,7 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) { struct timespec end, start; WT_BTREE *btree; + WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_PAGE *page; WT_PAGE_MODIFY *mod; @@ -27,6 +28,7 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) uint32_t flags; bool evict_reset; + conn = S2C(session); btree = S2BT(session); walk = NULL; txn = &session->txn; @@ -223,7 +225,7 @@ err: /* On error, clear any left-over tree walk. */ * so that eviction knows that the checkpoint has completed. */ WT_PUBLISH(btree->checkpoint_gen, - S2C(session)->txn_global.checkpoint_gen); + conn->txn_global.checkpoint_gen); WT_STAT_FAST_DATA_SET(session, btree_checkpoint_generation, btree->checkpoint_gen); @@ -257,7 +259,8 @@ err: /* On error, clear any left-over tree walk. */ * before checkpointing the file). Start a flush to stable storage, * but don't wait for it. */ - if (ret == 0 && syncop == WT_SYNC_WRITE_LEAVES) + if (ret == 0 && + syncop == WT_SYNC_WRITE_LEAVES && F_ISSET(conn, WT_CONN_CKPT_SYNC)) WT_RET(btree->bm->sync(btree->bm, session, true)); return (ret); diff --git a/src/txn/txn_ckpt.c b/src/txn/txn_ckpt.c index 4bb8ccdc6f0..de8f0d84951 100644 --- a/src/txn/txn_ckpt.c +++ b/src/txn/txn_ckpt.c @@ -408,8 +408,7 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) * completion. Do it after flushing the pages to give the * asynchronous flush as much time as possible before we wait. */ - if (F_ISSET(conn, WT_CONN_CKPT_SYNC)) - WT_ERR(__checkpoint_apply(session, cfg, __wt_checkpoint_sync)); + WT_ERR(__checkpoint_apply(session, cfg, __wt_checkpoint_sync)); /* Start the checkpoint for real. */ WT_ERR(__wt_meta_track_on(session)); @@ -509,8 +508,7 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) * Checkpoints have to hit disk (it would be reasonable to configure for * lazy checkpoints, but we don't support them yet). */ - if (F_ISSET(conn, WT_CONN_CKPT_SYNC)) - WT_ERR(__checkpoint_apply(session, cfg, __wt_checkpoint_sync)); + WT_ERR(__checkpoint_apply(session, cfg, __wt_checkpoint_sync)); WT_ERR(__checkpoint_verbose_track(session, "sync completed", &verb_timer)); @@ -1120,9 +1118,8 @@ fake: /* * sync the file here or we could roll forward the metadata in * recovery and open a checkpoint that isn't yet durable. */ - if (F_ISSET(conn, WT_CONN_CKPT_SYNC) && - (WT_IS_METADATA(session, dhandle) || - !F_ISSET(&session->txn, WT_TXN_RUNNING))) + if (WT_IS_METADATA(session, dhandle) || + !F_ISSET(&session->txn, WT_TXN_RUNNING)) WT_ERR(__wt_checkpoint_sync(session, NULL)); WT_ERR(__wt_meta_ckptlist_set( @@ -1197,8 +1194,9 @@ __wt_checkpoint_sync(WT_SESSION_IMPL *session, const char *cfg[]) /* Should not be called with a checkpoint handle. */ WT_ASSERT(session, session->dhandle->checkpoint == NULL); - /* Should have an underlying block manager reference. */ - WT_ASSERT(session, bm != NULL); + /* Unnecessary if checkpoint_sync has been configured "off". */ + if (!F_ISSET(S2C(session), WT_CONN_CKPT_SYNC)) + return (0); return (bm->sync(bm, session, false)); } -- cgit v1.2.1 From 34b9ee3248fb36acf4ab7a384d05f115a1e59122 Mon Sep 17 00:00:00 2001 From: David Hows Date: Thu, 10 Mar 2016 18:18:42 +1100 Subject: WT-2459 - Force the CC tag under powerPC --- build_posix/configure.ac.in | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/build_posix/configure.ac.in b/build_posix/configure.ac.in index fc13ab2b183..f64f68e4491 100644 --- a/build_posix/configure.ac.in +++ b/build_posix/configure.ac.in @@ -34,6 +34,15 @@ AC_PROG_CC(cc gcc) AC_PROG_CXX(c++ g++) AM_PROG_AS(as gas) +# Set the libtool tag flag on PowerPC +AS_CASE([$host_cpu], + [ppc64*], [AM_LIBTOOLFLAGS="--tag=cc"], + [elf64lppc], [AM_LIBTOOLFLAGS="--tag=cc"], + [powerpc*], [AM_LIBTOOLFLAGS="--tag=cc"], + []) + +AC_SUBST(AM_LIBTOOLFLAGS) + if test "$GCC" = "yes"; then # The Solaris gcc compiler gets the additional -pthreads flag. if test "`uname -s`" = "SunOS"; then -- cgit v1.2.1 From 76c7399dd2e802cd2efacb164498564f2177473e Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 10 Mar 2016 08:56:57 -0500 Subject: WT-2381: dump utility discards table config Whitespace. --- src/lsm/lsm_meta.c | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/src/lsm/lsm_meta.c b/src/lsm/lsm_meta.c index 3c7de0e6a60..e57be0c6459 100644 --- a/src/lsm/lsm_meta.c +++ b/src/lsm/lsm_meta.c @@ -185,7 +185,7 @@ __lsm_meta_read_v1( WT_DECL_RET; WT_LSM_CHUNK *chunk; const char *file_cfg[] = { - WT_CONFIG_BASE(session, file_config), NULL, NULL, NULL }; + WT_CONFIG_BASE(session, file_config), NULL, NULL, NULL }; char *fileconf; u_int nchunks; @@ -255,8 +255,7 @@ __lsm_meta_read_v1( } WT_ERR(__wt_config_check(session, WT_CONFIG_REF(session, WT_SESSION_create), cv.str, cv.len)); - WT_ERR(__wt_strndup( - session, cv.str, cv.len, &lsm_tree->bloom_config)); + WT_ERR(__wt_strndup(session, cv.str, cv.len, &lsm_tree->bloom_config)); WT_ERR(__wt_config_getones( session, lsmconf, "lsm.bloom_hash_count", &cv)); lsm_tree->bloom_hash_count = (uint32_t)cv.val; @@ -297,8 +296,7 @@ __lsm_meta_read_v1( WT_ERR(__wt_realloc_def(session, &lsm_tree->chunk_alloc, nchunks + 1, &lsm_tree->chunk)); - WT_ERR( - __wt_calloc_one(session, &chunk)); + WT_ERR(__wt_calloc_one(session, &chunk)); lsm_tree->chunk[nchunks++] = chunk; chunk->id = (uint32_t)lv.val; WT_ERR(__wt_lsm_tree_chunk_name(session, @@ -306,23 +304,18 @@ __lsm_meta_read_v1( F_SET(chunk, WT_LSM_CHUNK_ONDISK | WT_LSM_CHUNK_STABLE); - } else if (WT_STRING_MATCH( - "bloom", lk.str, lk.len)) { + } else if (WT_STRING_MATCH("bloom", lk.str, lk.len)) { WT_ERR(__wt_lsm_tree_bloom_name( - session, lsm_tree, - chunk->id, &chunk->bloom_uri)); + session, lsm_tree, chunk->id, &chunk->bloom_uri)); F_SET(chunk, WT_LSM_CHUNK_BLOOM); continue; - } else if (WT_STRING_MATCH( - "chunk_size", lk.str, lk.len)) { + } else if (WT_STRING_MATCH("chunk_size", lk.str, lk.len)) { chunk->size = (uint64_t)lv.val; continue; - } else if (WT_STRING_MATCH( - "count", lk.str, lk.len)) { + } else if (WT_STRING_MATCH("count", lk.str, lk.len)) { chunk->count = (uint64_t)lv.val; continue; - } else if (WT_STRING_MATCH( - "generation", lk.str, lk.len)) { + } else if (WT_STRING_MATCH("generation", lk.str, lk.len)) { chunk->generation = (uint32_t)lv.val; continue; } @@ -371,7 +364,7 @@ __lsm_meta_upgrade_v1(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_DECL_ITEM(buf); WT_DECL_RET; const char *new_cfg[] = { - WT_CONFIG_BASE(session, lsm_meta), NULL, NULL, NULL }; + WT_CONFIG_BASE(session, lsm_meta), NULL, NULL, NULL }; /* Include the custom config that used to be embedded in file_config. */ new_cfg[1] = lsm_tree->file_config; @@ -462,9 +455,9 @@ __wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_DECL_ITEM(buf); WT_DECL_RET; WT_LSM_CHUNK *chunk; + u_int i; const char *new_cfg[] = { NULL, NULL, NULL }; char *new_metadata; - u_int i; bool first; new_metadata = NULL; -- cgit v1.2.1 From 09d17833f87fc89b55481fc57c700a503163bff7 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 10 Mar 2016 08:58:27 -0500 Subject: WT-2381: dump utility discards table config Michael notes dump no longer needs to use the metadata:create URI, that simplifies the change, most importantly, we no longer need two versions of config_collapse. --- src/config/config_collapse.c | 48 +++++--------------------------------------- src/cursor/cur_metadata.c | 16 ++++++++------- src/include/extern.h | 1 - src/utilities/util_dump.c | 6 +++--- 4 files changed, 17 insertions(+), 54 deletions(-) diff --git a/src/config/config_collapse.c b/src/config/config_collapse.c index 890f1f16ed5..27bd6255a0a 100644 --- a/src/config/config_collapse.c +++ b/src/config/config_collapse.c @@ -9,7 +9,7 @@ #include "wt_internal.h" /* - * __config_collapse -- + * __wt_config_collapse -- * Collapse a set of configuration strings into newly allocated memory. * * This function takes a NULL-terminated list of configuration strings (where @@ -28,14 +28,10 @@ * "key=(k4=v4)", the result will be "key=(k4=v4)", as we search for and * use the final value of "key", regardless of field overlap or missing * fields in the nested value. - * - * Optionally, values found only in the first string aren't included in the - * final result, that is, the first string is only used as a list of valid - * configuration strings. */ -static int -__config_collapse(WT_SESSION_IMPL *session, - const char **cfg, bool include_defaults, char **config_ret) +int +__wt_config_collapse( + WT_SESSION_IMPL *session, const char **cfg, char **config_ret) { WT_CONFIG cparser; WT_CONFIG_ITEM k, v; @@ -45,22 +41,12 @@ __config_collapse(WT_SESSION_IMPL *session, WT_RET(__wt_scr_alloc(session, 0, &tmp)); WT_ERR(__wt_config_init(session, &cparser, cfg[0])); - if (!include_defaults) - ++cfg; while ((ret = __wt_config_next(&cparser, &k, &v)) == 0) { if (k.type != WT_CONFIG_ITEM_STRING && k.type != WT_CONFIG_ITEM_ID) WT_ERR_MSG(session, EINVAL, "Invalid configuration key found: '%s'\n", k.str); - /* - * Not-found is an expected error when not including entries - * from the first configuration string. - */ - if ((ret = __wt_config_get(session, cfg, &k, &v)) != 0) { - WT_ERR_NOTFOUND_OK(ret); - continue; - } - + WT_ERR(__wt_config_get(session, cfg, &k, &v)); /* Include the quotes around string keys/values. */ if (k.type == WT_CONFIG_ITEM_STRING) { --k.str; @@ -91,30 +77,6 @@ err: __wt_scr_free(session, &tmp); return (ret); } -/* - * __wt_config_collapse -- - * Collapse a set of configuration strings into newly allocated memory, - * including the defaults from the first configuration string. - */ -int -__wt_config_collapse( - WT_SESSION_IMPL *session, const char **cfg, char **config_ret) -{ - return (__config_collapse(session, cfg, true, config_ret)); -} - -/* - * __wt_config_strip_and_collapse -- - * Collapse a set of configuration strings into newly allocated memory, - * NOT including the defaults from the first configuration string. - */ -int -__wt_config_strip_and_collapse( - WT_SESSION_IMPL *session, const char **cfg, char **config_ret) -{ - return (__config_collapse(session, cfg, false, config_ret)); -} - /* * We need a character that can't appear in a key as a separator. */ diff --git a/src/cursor/cur_metadata.c b/src/cursor/cur_metadata.c index c25c7d1bdee..3d702e2ea8c 100644 --- a/src/cursor/cur_metadata.c +++ b/src/cursor/cur_metadata.c @@ -33,9 +33,11 @@ /* * __wt_schema_create_final -- * Create a single configuration line from a set of configuration strings, - * including all of the defaults declared for a session.create. Here for the wt - * dump command utility, which reads a set of configuration strings and needs to - * add in the defaults and then collapse them into single string for load. + * including all of the defaults declared for a session.create, and stripping + * any configuration strings that don't belong in a session.create. Here for + * the wt dump command utility, which reads a set of configuration strings and + * needs to add in the defaults and then collapse them into single string for + * a subsequent load. */ int __wt_schema_create_final( @@ -48,7 +50,8 @@ __wt_schema_create_final( /* * Count the entries in the original, * Allocate a copy with the defaults as the first entry, - * Collapse the whole thing into a single configuration string. + * Collapse the whole thing into a single configuration string (which + * also strips any entries that don't appear in the first entry). */ for (i = 0; cfg_arg[i] != NULL; ++i) ; @@ -67,8 +70,7 @@ __wt_schema_create_final( /* * __schema_create_strip -- * Discard any configuration information from a schema entry that is not - * applicable to an session.create call. Here for the wt dump command utility, - * which only wants to dump the schema information needed for load. + * applicable to an session.create call. Here for the metadata:create URI. */ static int __schema_create_strip( @@ -77,7 +79,7 @@ __schema_create_strip( const char *cfg[] = { WT_CONFIG_BASE(session, WT_SESSION_create), value, NULL }; - return (__wt_config_strip_and_collapse(session, cfg, value_ret)); + return (__wt_config_collapse(session, cfg, value_ret)); } /* diff --git a/src/include/extern.h b/src/include/extern.h index 6fc68310070..71cc106c00a 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -220,7 +220,6 @@ extern void __wt_conn_foc_discard(WT_SESSION_IMPL *session); extern int __wt_configure_method(WT_SESSION_IMPL *session, const char *method, const char *uri, const char *config, const char *type, const char *check); extern int __wt_config_check(WT_SESSION_IMPL *session, const WT_CONFIG_ENTRY *entry, const char *config, size_t config_len); extern int __wt_config_collapse( WT_SESSION_IMPL *session, const char **cfg, char **config_ret); -extern int __wt_config_strip_and_collapse( WT_SESSION_IMPL *session, const char **cfg, char **config_ret); extern int __wt_config_merge(WT_SESSION_IMPL *session, const char **cfg, const char *cfg_strip, const char **config_ret); extern int __wt_conn_config_init(WT_SESSION_IMPL *session); extern void __wt_conn_config_discard(WT_SESSION_IMPL *session); diff --git a/src/utilities/util_dump.c b/src/utilities/util_dump.c index ca9b43c1647..015fd9830a0 100644 --- a/src/utilities/util_dump.c +++ b/src/utilities/util_dump.c @@ -150,9 +150,9 @@ dump_config(WT_SESSION *session, const char *uri, bool hex) /* Open a metadata cursor. */ if ((ret = session->open_cursor( - session, "metadata:create", NULL, NULL, &cursor)) != 0) { + session, "metadata:", NULL, NULL, &cursor)) != 0) { fprintf(stderr, "%s: %s: session.open_cursor: %s\n", progname, - "metadata:create", session->strerror(session, ret)); + "metadata:", session->strerror(session, ret)); return (1); } /* @@ -536,7 +536,7 @@ dump_table_config(WT_SESSION *session, WT_CURSOR *cursor, const char *uri) * hugely simpler. */ if ((ret = session->open_cursor( - session, "metadata:create", NULL, NULL, &srch)) != 0) + session, "metadata:", NULL, NULL, &srch)) != 0) return (util_cerr(cursor, "open_cursor", ret)); if ((ret = dump_table_config_complex( -- cgit v1.2.1 From 7750e39b2c656fd7016ec29263ac89302be25721 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 10 Mar 2016 09:24:56 -0500 Subject: WT-2318 Fix warnings. --- test/manydbs/manydbs.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index 07fcb06adff..4627dc703fe 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -90,7 +90,8 @@ static int run_ops(int dbs) { WT_ITEM data; - int db, db_set, i, key; + int db_set, i, key; + uint32_t db; uint8_t buf[MAX_VAL]; memset(buf, 0, sizeof(buf)); @@ -100,9 +101,9 @@ run_ops(int dbs) */ if (session == NULL) { __wt_random_init(&rnd); - if ((session = calloc(sizeof(WT_SESSION *), dbs)) == NULL) + if ((session = calloc(dbs, sizeof(WT_SESSION *))) == NULL) testutil_die(ENOMEM, "session array malloc"); - if ((cursor = calloc(sizeof(WT_CURSOR *), dbs)) == NULL) + if ((cursor = calloc(dbs, sizeof(WT_CURSOR *))) == NULL) testutil_die(ENOMEM, "cursor array malloc"); for (i = 0; i < dbs; ++i) { testutil_check(conn[i]->open_session(conn[i], @@ -122,7 +123,7 @@ run_ops(int dbs) db_set = dbs / 4; for (i = 0; i < db_set; ++i) { db = __wt_random(&rnd) % dbs; - printf("Write to database %d\n", db); + printf("Write to database %" PRIu32 "\n", db); for (key = 0; key < MAX_KV; ++key) { data.size = __wt_random(&rnd) % MAX_VAL; cursor[db]->set_key(cursor[db], key); @@ -181,7 +182,7 @@ main(int argc, char *argv[]) if (argc != 0) usage(); - if ((conn = calloc(sizeof(WT_CONNECTION *), dbs)) == NULL) + if ((conn = calloc(dbs, sizeof(WT_CONNECTION *))) == NULL) testutil_die(ENOMEM, "connection array malloc"); memset(cmd, 0, sizeof(cmd)); /* -- cgit v1.2.1 From 10ee3a4a1910745bc09fcc6d057343c345672b7d Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 10 Mar 2016 09:51:00 -0500 Subject: WT-2318 Fix warnings. --- test/manydbs/manydbs.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index 4627dc703fe..5ca0bcbbb5f 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -101,9 +101,9 @@ run_ops(int dbs) */ if (session == NULL) { __wt_random_init(&rnd); - if ((session = calloc(dbs, sizeof(WT_SESSION *))) == NULL) + if ((session = calloc((size_t)dbs, sizeof(WT_SESSION *))) == NULL) testutil_die(ENOMEM, "session array malloc"); - if ((cursor = calloc(dbs, sizeof(WT_CURSOR *))) == NULL) + if ((cursor = calloc((size_t)dbs, sizeof(WT_CURSOR *))) == NULL) testutil_die(ENOMEM, "cursor array malloc"); for (i = 0; i < dbs; ++i) { testutil_check(conn[i]->open_session(conn[i], @@ -122,7 +122,7 @@ run_ops(int dbs) */ db_set = dbs / 4; for (i = 0; i < db_set; ++i) { - db = __wt_random(&rnd) % dbs; + db = __wt_random(&rnd) % (uint32_t)dbs; printf("Write to database %" PRIu32 "\n", db); for (key = 0; key < MAX_KV; ++key) { data.size = __wt_random(&rnd) % MAX_VAL; @@ -182,7 +182,7 @@ main(int argc, char *argv[]) if (argc != 0) usage(); - if ((conn = calloc(dbs, sizeof(WT_CONNECTION *))) == NULL) + if ((conn = calloc((size_t)dbs, sizeof(WT_CONNECTION *))) == NULL) testutil_die(ENOMEM, "connection array malloc"); memset(cmd, 0, sizeof(cmd)); /* -- cgit v1.2.1 From 01ad929001cd7516f0740e4f184966e8321fdfc0 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 10 Mar 2016 10:40:26 -0500 Subject: WT-2381 Add load phase and dump newly loaded data to check for table config retention. --- test/suite/test_util13.py | 42 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/test/suite/test_util13.py b/test/suite/test_util13.py index 0e4a42243f7..222f42cd7f1 100644 --- a/test/suite/test_util13.py +++ b/test/suite/test_util13.py @@ -32,10 +32,13 @@ import itertools, wiredtiger, wttest from helper import complex_populate_cgconfig, complex_populate_cgconfig_lsm from helper import simple_populate +from helper import complex_populate_check, simple_populate_check from wtscenario import multiply_scenarios, number_scenarios # test_util13.py # Utilities: wt dump, as well as the dump cursor +# Test that dump and load retain table configuration information. +# class test_util13(wttest.WiredTigerTestCase, suite_subprocess): """ Test wt dump. We check for specific output and preservation of @@ -44,22 +47,28 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): pfx = 'test_util13' nentries = 100 + dir = "dump_dir" # # Select table configuration settings that are not the default. # types = [ ('file-simple', dict(uri='file:' + pfx, pop=simple_populate, + populate_check=simple_populate_check, table_config='prefix_compression_min=3', cfg='')), ('lsm-simple', dict(uri='lsm:' + pfx, pop=simple_populate, + populate_check=simple_populate_check, table_config='lsm=(bloom_bit_count=29)', cfg='bloom_bit_count=29')), ('table-simple', dict(uri='table:' + pfx, pop=simple_populate, + populate_check=simple_populate_check, table_config='split_pct=50', cfg='')), ('table-complex', dict(uri='table:' + pfx, pop=complex_populate_cgconfig, + populate_check=complex_populate_check, table_config='allocation_size=512B', cfg='')), ('table-complex-lsm', dict(uri='table:' + pfx, pop=complex_populate_cgconfig_lsm, + populate_check=complex_populate_check, table_config='lsm=(merge_max=5)', cfg='merge_max=5')), ] @@ -118,6 +127,25 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): break return True + def load_recheck(self, expect_subset, dump_out): + newdump = "newdump.out" + os.mkdir(self.dir) + self.runWt(['-h', self.dir, 'load', '-f', dump_out]) + # Check the contents + conn = self.wiredtiger_open(self.dir) + session = conn.open_session() + cursor = session.open_cursor(self.uri, None, None) + self.populate_check + conn.close() + dumpargs = ["-h"] + dumpargs.append(self.dir) + dumpargs.append("dump") + dumpargs.append(self.uri) + self.runWt(dumpargs, outfilename=newdump) + + self.assertTrue(self.compare_files(expect_subset, newdump)) + return True + def test_dump_config(self): # The number of btree_entries reported is influenced by the # number of column groups and indices. Each insert will have @@ -127,7 +155,8 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): ver = wiredtiger.wiredtiger_version() verstring = str(ver[1]) + '.' + str(ver[2]) + '.' + str(ver[3]) - with open("expect.out", "w") as expectout: + expectfile="expect.out" + with open(expectfile, "w") as expectout: # Note: this output is sensitive to the precise output format # generated by wt dump. If this is likely to change, we should # make this test more accommodating. @@ -147,12 +176,13 @@ class test_util13(wttest.WiredTigerTestCase, suite_subprocess): expectout.write('Data\n') self.pr('calling dump') - with open("dump.out", "w") as dumpout: - dumpargs = ["dump"] - dumpargs.append(self.uri) - self.runWt(dumpargs, outfilename="dump.out") + outfile="dump.out" + dumpargs = ["dump"] + dumpargs.append(self.uri) + self.runWt(dumpargs, outfilename=outfile) - self.assertTrue(self.compare_files("expect.out", "dump.out")) + self.assertTrue(self.compare_files(expectfile, outfile)) + self.assertTrue(self.load_recheck(expectfile, outfile)) if __name__ == '__main__': wttest.run() -- cgit v1.2.1 From f318cc1e949be274745a226852cdd3df66a27b44 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 10 Mar 2016 13:35:23 -0500 Subject: WT-2461 Adjust sweep timeout values. --- test/suite/test_sweep01.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/suite/test_sweep01.py b/test/suite/test_sweep01.py index f996dbfa06d..5c279c8f074 100644 --- a/test/suite/test_sweep01.py +++ b/test/suite/test_sweep01.py @@ -87,7 +87,7 @@ class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): # # We've configured checkpoints to run every 5 seconds, sweep server to # run every 2 seconds and idle time to be 6 seconds. It should take - # about 8 seconds for a handle to be closed. Sleep for 12 seconds to be + # about 8 seconds for a handle to be closed. Sleep for double to be # safe. # uri = '%s.test' % self.uri @@ -105,7 +105,7 @@ class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri, None) k = 0 sleep = 0 - while sleep < 12: + while sleep < 16: self.session.checkpoint() k = k+1 c[k] = 1 -- cgit v1.2.1 From bcd26f82e60e493b8a3e847feea93072e4d851a1 Mon Sep 17 00:00:00 2001 From: David Hows Date: Fri, 11 Mar 2016 11:15:29 +1100 Subject: WT-2459 - Make the PPC tag override only occur when we are using different C and ASM compilers --- build_posix/configure.ac.in | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/build_posix/configure.ac.in b/build_posix/configure.ac.in index f64f68e4491..3db445fa562 100644 --- a/build_posix/configure.ac.in +++ b/build_posix/configure.ac.in @@ -34,12 +34,17 @@ AC_PROG_CC(cc gcc) AC_PROG_CXX(c++ g++) AM_PROG_AS(as gas) -# Set the libtool tag flag on PowerPC -AS_CASE([$host_cpu], - [ppc64*], [AM_LIBTOOLFLAGS="--tag=cc"], - [elf64lppc], [AM_LIBTOOLFLAGS="--tag=cc"], - [powerpc*], [AM_LIBTOOLFLAGS="--tag=cc"], - []) +# When we are compiling with overridden C and assembly compilers on PPC +# then we can run into an issue where libtool doesn't see the needed tag +# for compiling and will error out. We set a bad tag here, which still +# lets the compile pass without overriding anything. +if test "$CC" != "$CCAS"; then + AS_CASE([$host_cpu], + [ppc64*], [AM_LIBTOOLFLAGS="--tag=cc"], + [elf64lppc], [AM_LIBTOOLFLAGS="--tag=cc"], + [powerpc*], [AM_LIBTOOLFLAGS="--tag=cc"], + []) +fi AC_SUBST(AM_LIBTOOLFLAGS) -- cgit v1.2.1 From 1f9ccd49f1040161aaa94acda4ba7e7359d5978a Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Fri, 11 Mar 2016 12:19:52 +1100 Subject: WT-2404 Fix a bug in error handling through row search routine. If a collator returns an error the page could be released twice. Restructure and simplify the page release logic in row search. --- src/btree/row_srch.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/btree/row_srch.c b/src/btree/row_srch.c index 9d68c8e0ce7..6169a0a810a 100644 --- a/src/btree/row_srch.c +++ b/src/btree/row_srch.c @@ -460,6 +460,12 @@ leaf_only: page = current->page; cbt->ref = current; + /* + * Clear current now that we have moved the reference into the btree + * cursor, so that cleanup never releases twice. + */ + current = NULL; + /* * In the case of a right-side tree descent during an insert, do a fast * check for an append to the page, try to catch cursors appending data @@ -614,14 +620,7 @@ leaf_match: cbt->compare = 0; return (0); -err: /* - * Release the current page if the search started at the root. If the - * search didn't start at the root we should never have gone looking - * beyond the start page. - */ - WT_ASSERT(session, leaf == NULL || leaf == current); - if (leaf == NULL) - WT_TRET(__wt_page_release(session, current, 0)); +err: WT_TRET(__wt_page_release(session, current, 0)); return (ret); } -- cgit v1.2.1 From ccb81f359eb56649978621c6f1ddcfe7e68e1f48 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Fri, 11 Mar 2016 12:21:51 +1100 Subject: WT-2404 Make pack/unpack extension API consistent. --- src/include/extern.h | 22 +++++++++---------- src/include/wiredtiger_ext.h | 33 +++++++++++++++++++---------- src/packing/pack_stream.c | 50 ++++++++++++++++++++++++++++++++------------ 3 files changed, 70 insertions(+), 35 deletions(-) diff --git a/src/include/extern.h b/src/include/extern.h index 0774a14f382..e6b00189de1 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -558,17 +558,17 @@ extern int __wt_struct_size(WT_SESSION_IMPL *session, size_t *sizep, const char extern int __wt_struct_pack(WT_SESSION_IMPL *session, void *buffer, size_t size, const char *fmt, ...); extern int __wt_struct_unpack(WT_SESSION_IMPL *session, const void *buffer, size_t size, const char *fmt, ...); extern int __wt_struct_repack(WT_SESSION_IMPL *session, const char *infmt, const char *outfmt, const WT_ITEM *inbuf, WT_ITEM *outbuf); -extern int __wt_ext_pack_start(WT_SESSION *session, const char *format, void *buffer, size_t size, WT_PACK_STREAM **psp); -extern int __wt_ext_unpack_start(WT_SESSION *session, const char *format, const void *buffer, size_t size, WT_PACK_STREAM **psp); -extern int __wt_ext_pack_close(WT_PACK_STREAM *ps, size_t *usedp); -extern int __wt_ext_pack_item(WT_PACK_STREAM *ps, WT_ITEM *item); -extern int __wt_ext_pack_int(WT_PACK_STREAM *ps, int64_t i); -extern int __wt_ext_pack_str(WT_PACK_STREAM *ps, const char *s); -extern int __wt_ext_pack_uint(WT_PACK_STREAM *ps, uint64_t u); -extern int __wt_ext_unpack_item(WT_PACK_STREAM *ps, WT_ITEM *item); -extern int __wt_ext_unpack_int(WT_PACK_STREAM *ps, int64_t *ip); -extern int __wt_ext_unpack_str(WT_PACK_STREAM *ps, const char **sp); -extern int __wt_ext_unpack_uint(WT_PACK_STREAM *ps, uint64_t *up); +extern int __wt_ext_pack_start(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *format, void *buffer, size_t size, WT_PACK_STREAM **psp); +extern int __wt_ext_unpack_start(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *format, const void *buffer, size_t size, WT_PACK_STREAM **psp); +extern int __wt_ext_pack_close(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, size_t *usedp); +extern int __wt_ext_pack_item(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, WT_ITEM *item); +extern int __wt_ext_pack_int(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, int64_t i); +extern int __wt_ext_pack_str(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, const char *s); +extern int __wt_ext_pack_uint(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, uint64_t u); +extern int __wt_ext_unpack_item(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, WT_ITEM *item); +extern int __wt_ext_unpack_int(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, int64_t *ip); +extern int __wt_ext_unpack_str(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, const char **sp); +extern int __wt_ext_unpack_uint(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, uint64_t *up); extern int __wt_ovfl_discard_add(WT_SESSION_IMPL *session, WT_PAGE *page, WT_CELL *cell); extern void __wt_ovfl_discard_free(WT_SESSION_IMPL *session, WT_PAGE *page); extern int __wt_ovfl_reuse_search(WT_SESSION_IMPL *session, WT_PAGE *page, uint8_t **addrp, size_t *addr_sizep, const void *value, size_t value_size); diff --git a/src/include/wiredtiger_ext.h b/src/include/wiredtiger_ext.h index 7271bf6ba9d..011d85fda02 100644 --- a/src/include/wiredtiger_ext.h +++ b/src/include/wiredtiger_ext.h @@ -323,7 +323,8 @@ struct __wt_extension_api { * @param[out] psp the new packing stream handle * @errors */ - int (*pack_start)(WT_SESSION *session, const char *format, + int (*pack_start)(WT_EXTENSION_API *wt_api, + WT_SESSION *session, const char *format, void *buffer, size_t size, WT_PACK_STREAM **psp); /*! @@ -337,7 +338,8 @@ struct __wt_extension_api { * @param[out] psp the new packing stream handle * @errors */ - int (*unpack_start)(WT_SESSION *session, const char *format, + int (*unpack_start)(WT_EXTENSION_API *wt_api, + WT_SESSION *session, const char *format, const void *buffer, size_t size, WT_PACK_STREAM **psp); /*! @@ -348,7 +350,8 @@ struct __wt_extension_api { * stream * @errors */ - int (*pack_close)(WT_PACK_STREAM *ps, size_t *usedp); + int (*pack_close)(WT_EXTENSION_API *wt_api, + WT_PACK_STREAM *ps, size_t *usedp); /*! * Pack an item into a packing stream. @@ -357,7 +360,8 @@ struct __wt_extension_api { * @param item an item to pack * @errors */ - int (*pack_item)(WT_PACK_STREAM *ps, WT_ITEM *item); + int (*pack_item)(WT_EXTENSION_API *wt_api, + WT_PACK_STREAM *ps, WT_ITEM *item); /*! * Pack a signed integer into a packing stream. @@ -366,7 +370,8 @@ struct __wt_extension_api { * @param i a signed integer to pack * @errors */ - int (*pack_int)(WT_PACK_STREAM *ps, int64_t i); + int (*pack_int)(WT_EXTENSION_API *wt_api, + WT_PACK_STREAM *ps, int64_t i); /*! * Pack a string into a packing stream. @@ -375,7 +380,8 @@ struct __wt_extension_api { * @param s a string to pack * @errors */ - int (*pack_str)(WT_PACK_STREAM *ps, const char *s); + int (*pack_str)(WT_EXTENSION_API *wt_api, + WT_PACK_STREAM *ps, const char *s); /*! * Pack an unsigned integer into a packing stream. @@ -384,7 +390,8 @@ struct __wt_extension_api { * @param u an unsigned integer to pack * @errors */ - int (*pack_uint)(WT_PACK_STREAM *ps, uint64_t u); + int (*pack_uint)(WT_EXTENSION_API *wt_api, + WT_PACK_STREAM *ps, uint64_t u); /*! * Unpack an item from a packing stream. @@ -393,7 +400,8 @@ struct __wt_extension_api { * @param item an item to unpack * @errors */ - int (*unpack_item)(WT_PACK_STREAM *ps, WT_ITEM *item); + int (*unpack_item)(WT_EXTENSION_API *wt_api, + WT_PACK_STREAM *ps, WT_ITEM *item); /*! * Unpack a signed integer from a packing stream. @@ -402,7 +410,8 @@ struct __wt_extension_api { * @param[out] ip the unpacked signed integer * @errors */ - int (*unpack_int)(WT_PACK_STREAM *ps, int64_t *ip); + int (*unpack_int)(WT_EXTENSION_API *wt_api, + WT_PACK_STREAM *ps, int64_t *ip); /*! * Unpack a string from a packing stream. @@ -411,7 +420,8 @@ struct __wt_extension_api { * @param[out] sp the unpacked string * @errors */ - int (*unpack_str)(WT_PACK_STREAM *ps, const char **sp); + int (*unpack_str)(WT_EXTENSION_API *wt_api, + WT_PACK_STREAM *ps, const char **sp); /*! * Unpack an unsigned integer from a packing stream. @@ -420,7 +430,8 @@ struct __wt_extension_api { * @param[out] up the unpacked unsigned integer * @errors */ - int (*unpack_uint)(WT_PACK_STREAM *ps, uint64_t *up); + int (*unpack_uint)(WT_EXTENSION_API *wt_api, + WT_PACK_STREAM *ps, uint64_t *up); /*! * Return the current transaction ID. diff --git a/src/packing/pack_stream.c b/src/packing/pack_stream.c index 6ab44c123c8..5716c31a9c2 100644 --- a/src/packing/pack_stream.c +++ b/src/packing/pack_stream.c @@ -333,10 +333,16 @@ wiredtiger_unpack_uint(WT_PACK_STREAM *ps, uint64_t *up) * WT_EXTENSION.pack_start method. */ int -__wt_ext_pack_start(WT_SESSION *session, const char *format, +__wt_ext_pack_start(WT_EXTENSION_API *wt_api, + WT_SESSION *wt_session, const char *format, void *buffer, size_t size, WT_PACK_STREAM **psp) { - return (wiredtiger_pack_start(session, format, buffer, size, psp)); + WT_CONNECTION_IMPL *conn; + + conn = (WT_CONNECTION_IMPL *)wt_api->conn; + if (wt_session == NULL) + wt_session = (WT_SESSION *)conn->default_session; + return (wiredtiger_pack_start(wt_session, format, buffer, size, psp)); } /* @@ -344,10 +350,16 @@ __wt_ext_pack_start(WT_SESSION *session, const char *format, * WT_EXTENSION.unpack_start */ int -__wt_ext_unpack_start(WT_SESSION *session, const char *format, +__wt_ext_unpack_start(WT_EXTENSION_API *wt_api, + WT_SESSION *wt_session, const char *format, const void *buffer, size_t size, WT_PACK_STREAM **psp) { - return (wiredtiger_unpack_start(session, format, buffer, size, psp)); + WT_CONNECTION_IMPL *conn; + + conn = (WT_CONNECTION_IMPL *)wt_api->conn; + if (wt_session == NULL) + wt_session = (WT_SESSION *)conn->default_session; + return (wiredtiger_unpack_start(wt_session, format, buffer, size, psp)); } /* @@ -355,8 +367,9 @@ __wt_ext_unpack_start(WT_SESSION *session, const char *format, * WT_EXTENSION.pack_close */ int -__wt_ext_pack_close(WT_PACK_STREAM *ps, size_t *usedp) +__wt_ext_pack_close(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, size_t *usedp) { + WT_UNUSED(wt_api); return (wiredtiger_pack_close(ps, usedp)); } @@ -365,8 +378,9 @@ __wt_ext_pack_close(WT_PACK_STREAM *ps, size_t *usedp) * WT_EXTENSION.pack_item */ int -__wt_ext_pack_item(WT_PACK_STREAM *ps, WT_ITEM *item) +__wt_ext_pack_item(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, WT_ITEM *item) { + WT_UNUSED(wt_api); return (wiredtiger_pack_item(ps, item)); } @@ -375,8 +389,9 @@ __wt_ext_pack_item(WT_PACK_STREAM *ps, WT_ITEM *item) * WT_EXTENSION.pack_int */ int -__wt_ext_pack_int(WT_PACK_STREAM *ps, int64_t i) +__wt_ext_pack_int(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, int64_t i) { + WT_UNUSED(wt_api); return (wiredtiger_pack_int(ps, i)); } @@ -385,8 +400,9 @@ __wt_ext_pack_int(WT_PACK_STREAM *ps, int64_t i) * WT_EXTENSION.pack_str */ int -__wt_ext_pack_str(WT_PACK_STREAM *ps, const char *s) +__wt_ext_pack_str(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, const char *s) { + WT_UNUSED(wt_api); return (wiredtiger_pack_str(ps, s)); } @@ -395,8 +411,9 @@ __wt_ext_pack_str(WT_PACK_STREAM *ps, const char *s) * WT_EXTENSION.pack_uint */ int -__wt_ext_pack_uint(WT_PACK_STREAM *ps, uint64_t u) +__wt_ext_pack_uint(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, uint64_t u) { + WT_UNUSED(wt_api); return (wiredtiger_pack_uint(ps, u)); } @@ -405,8 +422,10 @@ __wt_ext_pack_uint(WT_PACK_STREAM *ps, uint64_t u) * WT_EXTENSION.unpack_item */ int -__wt_ext_unpack_item(WT_PACK_STREAM *ps, WT_ITEM *item) +__wt_ext_unpack_item(WT_EXTENSION_API *wt_api, + WT_PACK_STREAM *ps, WT_ITEM *item) { + WT_UNUSED(wt_api); return (wiredtiger_unpack_item(ps, item)); } @@ -415,8 +434,10 @@ __wt_ext_unpack_item(WT_PACK_STREAM *ps, WT_ITEM *item) * WT_EXTENSION.unpack_int */ int -__wt_ext_unpack_int(WT_PACK_STREAM *ps, int64_t *ip) +__wt_ext_unpack_int(WT_EXTENSION_API *wt_api, + WT_PACK_STREAM *ps, int64_t *ip) { + WT_UNUSED(wt_api); return (wiredtiger_unpack_int(ps, ip)); } @@ -425,8 +446,10 @@ __wt_ext_unpack_int(WT_PACK_STREAM *ps, int64_t *ip) * WT_EXTENSION.unpack_str */ int -__wt_ext_unpack_str(WT_PACK_STREAM *ps, const char **sp) +__wt_ext_unpack_str(WT_EXTENSION_API *wt_api, + WT_PACK_STREAM *ps, const char **sp) { + WT_UNUSED(wt_api); return (wiredtiger_unpack_str(ps, sp)); } @@ -435,7 +458,8 @@ __wt_ext_unpack_str(WT_PACK_STREAM *ps, const char **sp) * WT_EXTENSION.unpack_uint */ int -__wt_ext_unpack_uint(WT_PACK_STREAM *ps, uint64_t *up) +__wt_ext_unpack_uint(WT_EXTENSION_API *wt_api, WT_PACK_STREAM *ps, uint64_t *up) { + WT_UNUSED(wt_api); return (wiredtiger_unpack_uint(ps, up)); } -- cgit v1.2.1 From 8279190c7fb1a6b02aadc32e7f6c48e516aaec91 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Fri, 11 Mar 2016 12:30:17 +1100 Subject: WT-2404 Document old WT_EXTENSION_API pack/unpack API as deprecated. --- src/include/wiredtiger_ext.h | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/include/wiredtiger_ext.h b/src/include/wiredtiger_ext.h index 011d85fda02..7d97d97dcf5 100644 --- a/src/include/wiredtiger_ext.h +++ b/src/include/wiredtiger_ext.h @@ -268,8 +268,9 @@ struct __wt_extension_api { WT_SESSION *session, const char *key, const char *value); /*! - * Pack a structure into a buffer. - * See ::wiredtiger_struct_pack for details. + * Pack a structure into a buffer. Deprecated in favor of stream + * based pack and unpack API. See WT_EXTENSION_API::pack_start for + * details. * * @param wt_api the extension handle * @param session the session handle @@ -282,8 +283,8 @@ struct __wt_extension_api { void *buffer, size_t size, const char *format, ...); /*! - * Calculate the size required to pack a structure. - * See ::wiredtiger_struct_size for details. + * Calculate the size required to pack a structure. Deprecated in + * favor of stream based pack and unpack API. * * @param wt_api the extension handle * @param session the session handle @@ -296,8 +297,9 @@ struct __wt_extension_api { size_t *sizep, const char *format, ...); /*! - * Unpack a structure from a buffer. - * See ::wiredtiger_struct_unpack for details. + * Unpack a structure from a buffer. Deprecated in favor of stream + * based pack and unpack API. See WT_EXTENSION_API::unpack_start for + * details. * * @param wt_api the extension handle * @param session the session handle -- cgit v1.2.1 From 49d78ac41a1cdfffafcb4d5eb8315ca3d6225235 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Fri, 11 Mar 2016 14:29:07 +1100 Subject: WT-2463 Disable manydbs failure mode for now. It's causing problems with valgrind testing. --- test/manydbs/smoke.sh | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/manydbs/smoke.sh b/test/manydbs/smoke.sh index c0e2976f154..291c02bd1e6 100755 --- a/test/manydbs/smoke.sh +++ b/test/manydbs/smoke.sh @@ -2,6 +2,10 @@ set -e +# XXX Disable more complex tests, and bump the CPU usage check up to 100 for +# now, since running under valgrind consumes a lot more CPU. A different +# measuring mechanism would be ideal. + # Smoke-test format as part of running "make check". # Run with: # 1. The defaults @@ -9,10 +13,10 @@ set -e # 3. More dbs. # echo "manydbs: default with operations turned on" -$TEST_WRAPPER ./t -echo "manydbs: totally idle databases" -$TEST_WRAPPER ./t -I -echo "manydbs: 40 databases with operations" -$TEST_WRAPPER ./t -D 40 -echo "manydbs: 40 idle databases" -$TEST_WRAPPER ./t -I -D 40 +$TEST_WRAPPER ./t -C 100 +#echo "manydbs: totally idle databases" +#$TEST_WRAPPER ./t -I +#echo "manydbs: 40 databases with operations" +#$TEST_WRAPPER ./t -D 40 +#echo "manydbs: 40 idle databases" +#$TEST_WRAPPER ./t -I -D 40 -- cgit v1.2.1 From d480863cbb55910b77322bca2a07c5a1cc5ceada Mon Sep 17 00:00:00 2001 From: David Hows Date: Fri, 11 Mar 2016 14:31:08 +1100 Subject: WT-2459 - Fix comments, change casing --- build_posix/configure.ac.in | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/build_posix/configure.ac.in b/build_posix/configure.ac.in index 3db445fa562..3d0ba479cbd 100644 --- a/build_posix/configure.ac.in +++ b/build_posix/configure.ac.in @@ -36,13 +36,12 @@ AM_PROG_AS(as gas) # When we are compiling with overridden C and assembly compilers on PPC # then we can run into an issue where libtool doesn't see the needed tag -# for compiling and will error out. We set a bad tag here, which still -# lets the compile pass without overriding anything. +# for compiling and will error out. if test "$CC" != "$CCAS"; then AS_CASE([$host_cpu], - [ppc64*], [AM_LIBTOOLFLAGS="--tag=cc"], - [elf64lppc], [AM_LIBTOOLFLAGS="--tag=cc"], - [powerpc*], [AM_LIBTOOLFLAGS="--tag=cc"], + [ppc64*], [AM_LIBTOOLFLAGS+="--tag=CC"], + [elf64lppc], [AM_LIBTOOLFLAGS+="--tag=CC"], + [powerpc*], [AM_LIBTOOLFLAGS+="--tag=CC"], []) fi -- cgit v1.2.1 From fcdc497ff3b96bcae41656ea42867571c193e418 Mon Sep 17 00:00:00 2001 From: David Hows Date: Fri, 11 Mar 2016 16:09:36 +1100 Subject: WT-2459 - Update comment --- build_posix/configure.ac.in | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/build_posix/configure.ac.in b/build_posix/configure.ac.in index 3d0ba479cbd..3280a4e9af3 100644 --- a/build_posix/configure.ac.in +++ b/build_posix/configure.ac.in @@ -34,9 +34,13 @@ AC_PROG_CC(cc gcc) AC_PROG_CXX(c++ g++) AM_PROG_AS(as gas) -# When we are compiling with overridden C and assembly compilers on PPC -# then we can run into an issue where libtool doesn't see the needed tag -# for compiling and will error out. +# This is a workaround as part of WT-2459. Currently, clang (v3.7) does not +# support compiling the ASM code we have to perform the CRC checks on PowerPC. +# To compile with clang we need to override the ASM compiler with CCAS to use +# gcc. Unfortunately, doing the compilation in this manner means libtool can't +# determine what tag to use for that one .S file. If we catch that we are using +# two different compilers for CC and CCAS and we are on a PowerPC system we +# overload the libtool flags to provide CC by default. if test "$CC" != "$CCAS"; then AS_CASE([$host_cpu], [ppc64*], [AM_LIBTOOLFLAGS+="--tag=CC"], @@ -44,7 +48,6 @@ if test "$CC" != "$CCAS"; then [powerpc*], [AM_LIBTOOLFLAGS+="--tag=CC"], []) fi - AC_SUBST(AM_LIBTOOLFLAGS) if test "$GCC" = "yes"; then -- cgit v1.2.1 From 5e4766ff71886b0a00d4c56315dd55759cdf3811 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 11 Mar 2016 08:52:22 -0500 Subject: SERVER-23040: Coverity analysis defect 98151: Dereference after null check We always truncate in cursor-forward direction, there's never a case where we don't have a "start" cursor. Simplify some code and stop checking if the start cursor is set or not. --- src/btree/bt_cursor.c | 12 ++---------- src/schema/schema_truncate.c | 11 ++++------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/src/btree/bt_cursor.c b/src/btree/bt_cursor.c index c11b7d35de6..d64b6064268 100644 --- a/src/btree/bt_cursor.c +++ b/src/btree/bt_cursor.c @@ -1162,21 +1162,13 @@ int __wt_btcur_range_truncate(WT_CURSOR_BTREE *start, WT_CURSOR_BTREE *stop) { WT_BTREE *btree; - WT_CURSOR_BTREE *cbt; WT_DECL_RET; WT_SESSION_IMPL *session; - cbt = (start != NULL) ? start : stop; - session = (WT_SESSION_IMPL *)cbt->iface.session; - btree = cbt->btree; + session = (WT_SESSION_IMPL *)start->iface.session; + btree = start->btree; WT_STAT_FAST_DATA_INCR(session, cursor_truncate); - /* - * We always delete in a forward direction because it's faster, assert - * our caller provided us with a start cursor. - */ - WT_ASSERT(session, start != NULL); - /* * For recovery, log the start and stop keys for a truncate operation, * not the individual records removed. On the other hand, for rollback diff --git a/src/schema/schema_truncate.c b/src/schema/schema_truncate.c index e7752b60ca4..d9a798b6ed8 100644 --- a/src/schema/schema_truncate.c +++ b/src/schema/schema_truncate.c @@ -131,22 +131,19 @@ int __wt_schema_range_truncate( WT_SESSION_IMPL *session, WT_CURSOR *start, WT_CURSOR *stop) { - WT_CURSOR *cursor; WT_DATA_SOURCE *dsrc; WT_DECL_RET; const char *uri; - cursor = (start != NULL) ? start : stop; - uri = cursor->internal_uri; + uri = start->internal_uri; if (WT_PREFIX_MATCH(uri, "file:")) { - if (start != NULL) - WT_CURSOR_NEEDKEY(start); + WT_CURSOR_NEEDKEY(start); if (stop != NULL) WT_CURSOR_NEEDKEY(stop); - WT_WITH_BTREE(session, ((WT_CURSOR_BTREE *)cursor)->btree, + WT_WITH_BTREE(session, ((WT_CURSOR_BTREE *)start)->btree, ret = __wt_btcur_range_truncate( - (WT_CURSOR_BTREE *)start, (WT_CURSOR_BTREE *)stop)); + (WT_CURSOR_BTREE *)start, (WT_CURSOR_BTREE *)stop)); } else if (WT_PREFIX_MATCH(uri, "table:")) ret = __wt_table_range_truncate( (WT_CURSOR_TABLE *)start, (WT_CURSOR_TABLE *)stop); -- cgit v1.2.1 From 5e659cd82c272cdf80df89e10d0e3977075883f9 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 11 Mar 2016 09:45:23 -0500 Subject: WT-2465: Coverity 1352899: Dereference before null check Don't check for NULL after a dereference, don't bother checking for NULL, dcalloc doesn't return NULL. Change dcalloc arguments to match C library calloc names and usage. --- bench/wtperf/wtperf.h | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/bench/wtperf/wtperf.h b/bench/wtperf/wtperf.h index c591499b907..60efce650aa 100644 --- a/bench/wtperf/wtperf.h +++ b/bench/wtperf/wtperf.h @@ -370,11 +370,11 @@ dmalloc(size_t len) * Call calloc, dying on failure. */ static inline void * -dcalloc(size_t num, size_t len) +dcalloc(size_t num, size_t size) { void *p; - if ((p = calloc(len, num)) == NULL) + if ((p = calloc(num, size)) == NULL) die(errno, "calloc"); return (p); } @@ -416,11 +416,9 @@ static inline char * dstrndup(const char *str, const size_t len) { char *p; - p = dcalloc(len + 1, 1); - strncpy(p, str, len); - if (p == NULL) - die(errno, "dstrndup"); + p = dcalloc(len + 1, sizeof(char)); + memcpy(p, str, len); return (p); } #endif -- cgit v1.2.1 From 2807e4509e1ef4fe06af29dce36437a8bdc1f9ef Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 11 Mar 2016 10:09:32 -0500 Subject: WT-2466: Coverity 1352893 Buffer not null terminated Don't overflow directory buffers, allocate space for directory name suffixes. --- test/readonly/readonly.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/test/readonly/readonly.c b/test/readonly/readonly.c index 100ccbf81b7..11da5ea5fcd 100644 --- a/test/readonly/readonly.c +++ b/test/readonly/readonly.c @@ -42,9 +42,13 @@ #define HOME_SIZE 512 static char home[HOME_SIZE]; /* Program working dir lock file */ -static char home_wr[HOME_SIZE]; /* Writable dir copy no lock file */ -static char home_rd[HOME_SIZE]; /* Read-only dir */ -static char home_rd2[HOME_SIZE]; /* Read-only dir no lock file */ +#define HOME_WR_SUFFIX ".WRNOLOCK" /* Writable dir copy no lock file */ +static char home_wr[HOME_SIZE + sizeof(HOME_WR_SUFFIX)]; +#define HOME_RD_SUFFIX ".RD" /* Read-only dir */ +static char home_rd[HOME_SIZE + sizeof(HOME_RD_SUFFIX)]; +#define HOME_RD2_SUFFIX ".RDNOLOCK" /* Read-only dir no lock file */ +static char home_rd2[HOME_SIZE + sizeof(HOME_RD2_SUFFIX)]; + static const char *progname; /* Program name */ static const char *saved_argv0; /* Program command */ static const char * const uri = "table:main"; @@ -207,17 +211,14 @@ main(int argc, char *argv[]) if (argc != 0) usage(); - memset(buf, 0, sizeof(buf)); /* * Set up all the directory names. */ - testutil_work_dir_from_path(home, 512, working_dir); - strncpy(home_wr, home, HOME_SIZE); - strcat(home_wr, ".WRNOLOCK"); - strncpy(home_rd, home, HOME_SIZE); - strcat(home_rd, ".RD"); - strncpy(home_rd2, home, HOME_SIZE); - strcat(home_rd2, ".RDNOLOCK"); + testutil_work_dir_from_path(home, sizeof(home), working_dir); + (void)snprintf(home_wr, sizeof(home_wr), "%s%s", home, HOME_WR_SUFFIX); + (void)snprintf(home_rd, sizeof(home_rd), "%s%s", home, HOME_RD_SUFFIX); + (void)snprintf( + home_rd2, sizeof(home_rd2), "%s%s", home, HOME_RD2_SUFFIX); if (!child) { testutil_make_work_dir(home); testutil_make_work_dir(home_wr); @@ -260,6 +261,7 @@ main(int argc, char *argv[]) /* * Write data into the table and then cleanly shut down connection. */ + memset(buf, 0, sizeof(buf)); data.data = buf; data.size = MAX_VAL; for (i = 0; i < MAX_KV; ++i) { -- cgit v1.2.1 From 96d266a085c48e283eafc953a245c99c20ca30a4 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 11 Mar 2016 10:26:36 -0500 Subject: WT-2466: Coverity 1352893 Buffer not null terminated #2566 Add the test directory to the local-clean target. --- test/readonly/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/readonly/Makefile.am b/test/readonly/Makefile.am index 384e197a1f8..3abcd2386a1 100644 --- a/test/readonly/Makefile.am +++ b/test/readonly/Makefile.am @@ -10,4 +10,4 @@ t_LDFLAGS = -static TESTS = smoke.sh clean-local: - rm -rf WiredTiger* *.core __* + rm -rf WT_RD* WiredTiger* *.core __* -- cgit v1.2.1 From 4faf2bede2cbc454c173d2b63de3276c5c2020f2 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 11 Mar 2016 10:27:11 -0500 Subject: WT-2466: Coverity 1352893 Buffer not null terminated Make the -h argument work, it has to be passed to the child process. --- test/readonly/readonly.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test/readonly/readonly.c b/test/readonly/readonly.c index 11da5ea5fcd..eac657d51c1 100644 --- a/test/readonly/readonly.c +++ b/test/readonly/readonly.c @@ -331,7 +331,8 @@ main(int argc, char *argv[]) * the child even though it should not be. So use 'system' to spawn * an entirely new process. */ - (void)snprintf(cmd, sizeof(cmd), "%s -R", saved_argv0); + (void)snprintf( + cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir); if ((status = system(cmd)) < 0) testutil_die(status, "system"); /* @@ -343,7 +344,8 @@ main(int argc, char *argv[]) /* * Scenario 2. Run child with writable config. */ - (void)snprintf(cmd, sizeof(cmd), "%s -W", saved_argv0); + (void)snprintf( + cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir); if ((status = system(cmd)) < 0) testutil_die(status, "system"); @@ -364,7 +366,8 @@ main(int argc, char *argv[]) /* * Scenario 3. Child read-only. */ - (void)snprintf(cmd, sizeof(cmd), "%s -R", saved_argv0); + (void)snprintf( + cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir); if ((status = system(cmd)) < 0) testutil_die(status, "system"); if (WEXITSTATUS(status) != 0) @@ -373,7 +376,8 @@ main(int argc, char *argv[]) /* * Scenario 4. Run child with writable config. */ - (void)snprintf(cmd, sizeof(cmd), "%s -W", saved_argv0); + (void)snprintf( + cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir); if ((status = system(cmd)) < 0) testutil_die(status, "system"); if (WEXITSTATUS(status) != 0) -- cgit v1.2.1 From 8269d006d98506e80eb36f364ec8837ab6c52fb0 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 11 Mar 2016 10:34:55 -0500 Subject: WT-2467: Coverity 1352894: Logically dead code Update the mod value to reflect the addition of a new operation. --- test/fops/fops.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fops/fops.c b/test/fops/fops.c index fbc9d9c6048..3333ff16858 100644 --- a/test/fops/fops.c +++ b/test/fops/fops.c @@ -109,7 +109,7 @@ fop(void *arg) __wt_random_init(&rnd); for (i = 0; i < nops; ++i, __wt_yield()) - switch (__wt_random(&rnd) % 9) { + switch (__wt_random(&rnd) % 10) { case 0: ++s->bulk; obj_bulk(); -- cgit v1.2.1 From f02cca788bfd19c41a398e5587f19ed79036b192 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 11 Mar 2016 10:51:07 -0500 Subject: WT-2468: Coverity 1352896: Explicit null dereferenced This isn't really a bug, but Coverity can't figure that out -- restructure slightly so it's clear we only use the returned WT_CONNECTION if wiredtiger_open returned 0. --- test/readonly/readonly.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/readonly/readonly.c b/test/readonly/readonly.c index 100ccbf81b7..ea005bb3b7f 100644 --- a/test/readonly/readonly.c +++ b/test/readonly/readonly.c @@ -87,13 +87,14 @@ run_child(const char *homedir, int op, int expect) cfg = ENV_CONFIG_RD; else cfg = ENV_CONFIG_WR; - ret = wiredtiger_open(homedir, NULL, cfg, &conn); - if (expect == EXPECT_SUCCESS && ret != 0) - testutil_die(ret, "wiredtiger_open success err"); - if (expect == EXPECT_ERR) { - if (ret == 0) + if ((ret = wiredtiger_open(homedir, NULL, cfg, &conn)) == 0) { + if (expect == EXPECT_ERR) testutil_die( - ret, "wiredtiger_open expected err succeeded"); + ret, "wiredtiger_open expected error, succeeded"); + } else { + if (expect == EXPECT_SUCCESS) + testutil_die( + ret, "wiredtiger_open expected success, error"); /* * If we expect an error and got one, we're done. */ -- cgit v1.2.1 From c0dbe962eb38220c50a0b2ea290e4c69a66b6cee Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 11 Mar 2016 11:04:45 -0500 Subject: WT-2469: Coverity 1352897: Integer overflowed argument Don't overflow the file offset. Don't use %u to print PRIu64 and PRIu32 types. --- test/recovery/truncated-log.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/recovery/truncated-log.c b/test/recovery/truncated-log.c index 23269e99d35..4dd7db2b984 100644 --- a/test/recovery/truncated-log.c +++ b/test/recovery/truncated-log.c @@ -243,8 +243,10 @@ main(int argc, char *argv[]) * The offset is the beginning of the last record. Truncate to * the middle of that last record (i.e. ahead of that offset). */ + if (offset > UINT64_MAX - V_SIZE) + testutil_die(ERANGE, "offset"); new_offset = offset + V_SIZE; - printf("Parent: Truncate to %u\n", (uint32_t)new_offset); + printf("Parent: Truncate to %" PRIu64 "\n", new_offset); if ((ret = truncate(LOG_FILE_1, (wt_off_t)new_offset)) != 0) testutil_die(errno, "truncate"); @@ -267,9 +269,10 @@ main(int argc, char *argv[]) if ((ret = conn->close(conn, NULL)) != 0) testutil_die(ret, "WT_CONNECTION:close"); if (count > max_key) { - printf("expected %u records found %u\n", max_key, count); + printf("expected %" PRIu32 " records found %" PRIu32 "\n", + max_key, count); return (EXIT_FAILURE); } - printf("%u records verified\n", count); + printf("%" PRIu32 " records verified\n", count); return (EXIT_SUCCESS); } -- cgit v1.2.1 From 62d8c52f06a12a371a0926529c9bc50c51542446 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 11 Mar 2016 12:17:34 -0500 Subject: WT-2470: Coverity 1352898: Resource leak Coverity is seeing a resource leak; minor restructing of the code to clarify that can't happen. --- test/recovery/truncated-log.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/recovery/truncated-log.c b/test/recovery/truncated-log.c index 23269e99d35..1227b21b0ec 100644 --- a/test/recovery/truncated-log.c +++ b/test/recovery/truncated-log.c @@ -156,14 +156,16 @@ fill_db(void) "%" PRIu32 " %" PRIu32 "\n", save_lsn.l.offset, i - 1) == -1) testutil_die(errno, "fprintf"); - if (fclose(fp) != 0) - testutil_die(errno, "fclose"); - abort(); + break; } } first = false; } } + if (fclose(fp) != 0) + testutil_die(errno, "fclose"); + abort(); + /* NOTREACHED */ } extern int __wt_optind; -- cgit v1.2.1 From cc0d34b09350d73702eb6d76560dff0b9448db77 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 11 Mar 2016 16:02:41 -0500 Subject: WT-2471: review WiredTiger "int" printf formats --- bench/wtperf/wtperf.c | 4 ++-- dist/s_string.ok | 1 + examples/c/ex_async.c | 2 +- examples/c/ex_config_parse.c | 4 ++-- examples/c/ex_extractor.c | 13 ++++++++----- examples/c/ex_schema.c | 32 +++++++++++++++++++------------- src/block/block_ckpt.c | 3 +-- src/block/block_open.c | 2 +- src/config/config.c | 9 ++++++--- src/log/log.c | 23 +++++++++++++---------- src/lsm/lsm_manager.c | 6 ++++-- src/lsm/lsm_merge.c | 10 ++++++---- src/lsm/lsm_tree.c | 8 ++++---- src/lsm/lsm_work_unit.c | 4 ++-- src/lsm/lsm_worker.c | 9 ++++----- src/schema/schema_plan.c | 12 ++++++------ src/session/session_api.c | 2 +- src/support/huffman.c | 9 +++++---- src/txn/txn.c | 2 +- src/txn/txn_recover.c | 14 +++++++------- test/bloom/test_bloom.c | 8 +++++--- test/checkpoint/test_checkpoint.c | 2 +- test/cursor_order/cursor_order.c | 6 ++++-- test/fops/file.c | 2 +- test/format/wts.c | 16 +++++++++------- test/manydbs/manydbs.c | 3 ++- test/recovery/random-abort.c | 5 +++-- test/recovery/truncated-log.c | 7 ++++--- 28 files changed, 123 insertions(+), 95 deletions(-) diff --git a/bench/wtperf/wtperf.c b/bench/wtperf/wtperf.c index b04c6ba662d..340c400ba7e 100644 --- a/bench/wtperf/wtperf.c +++ b/bench/wtperf/wtperf.c @@ -1668,7 +1668,7 @@ execute_workload(CONFIG *cfg) for (threads = cfg->workers, i = 0, workp = cfg->workload; i < cfg->workload_cnt; ++i, ++workp) { lprintf(cfg, 0, 1, - "Starting workload #%d: %" PRId64 " threads, inserts=%" + "Starting workload #%u: %" PRId64 " threads, inserts=%" PRId64 ", reads=%" PRId64 ", updates=%" PRId64 ", truncate=%" PRId64 ", throttle=%" PRId64, i + 1, workp->threads, workp->insert, @@ -2285,7 +2285,7 @@ main(int argc, char *argv[]) req_len = strlen(",async=(enabled=true,threads=)") + 4; cfg->async_config = dcalloc(req_len, 1); snprintf(cfg->async_config, req_len, - ",async=(enabled=true,threads=%d)", + ",async=(enabled=true,threads=%" PRIu32 ")", cfg->async_threads); } if ((ret = config_compress(cfg)) != 0) diff --git a/dist/s_string.ok b/dist/s_string.ok index c582f3380bd..44e2442a348 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -994,6 +994,7 @@ t's tV tablename tcbench +td testutil th tid diff --git a/examples/c/ex_async.c b/examples/c/ex_async.c index 584c3e54b87..ecdbd2f4fea 100644 --- a/examples/c/ex_async.c +++ b/examples/c/ex_async.c @@ -218,7 +218,7 @@ main(void) */ ret = conn->close(conn, NULL); - printf("Searched for %d keys\n", ex_asynckeys.num_keys); + printf("Searched for %" PRIu32 " keys\n", ex_asynckeys.num_keys); return (ret); } diff --git a/examples/c/ex_config_parse.c b/examples/c/ex_config_parse.c index 124eff21130..caa7ffa8fe0 100644 --- a/examples/c/ex_config_parse.c +++ b/examples/c/ex_config_parse.c @@ -99,7 +99,7 @@ main(void) while ((ret = parser->next(parser, &k, &v)) == 0) { printf("%.*s:", (int)k.len, k.str); if (v.type == WT_CONFIG_ITEM_NUM) - printf("%d\n", (int)v.val); + printf("%" PRId64 "\n", v.val); else printf("%.*s\n", (int)v.len, v.str); } @@ -126,7 +126,7 @@ main(void) "log.file_max configuration: %s", wiredtiger_strerror(ret)); return (ret); } - printf("log file max: %d\n", (int)v.val); + printf("log file max: %" PRId64 "\n", v.val); /*! [nested get] */ ret = parser->close(parser); diff --git a/examples/c/ex_extractor.c b/examples/c/ex_extractor.c index fff9c79f8e0..8623f4759fc 100644 --- a/examples/c/ex_extractor.c +++ b/examples/c/ex_extractor.c @@ -99,11 +99,13 @@ my_extract(WT_EXTRACTOR *extractor, WT_SESSION *session, * key(s). WiredTiger will perform the required operation * (such as a remove()). */ - fprintf(stderr, "EXTRACTOR: index op for year %d: %s %s\n", + fprintf(stderr, + "EXTRACTOR: index op for year %" PRIu16 ": %s %s\n", year, first_name, last_name); result_cursor->set_key(result_cursor, year); if ((ret = result_cursor->insert(result_cursor)) != 0) { - fprintf(stderr, "EXTRACTOR: op year %d: error %d\n", + fprintf(stderr, + "EXTRACTOR: op year %" PRIu16 ": error %d\n", year, ret); return (ret); } @@ -157,7 +159,7 @@ read_index(WT_SESSION *session) */ for (i = 0; i < 10 && RET_OK(ret); i++) { year = (uint16_t)((rand() % YEAR_SPAN) + YEAR_BASE); - printf("Year %d:\n", year); + printf("Year %" PRIu16 ":\n", year); cursor->set_key(cursor, year); if ((ret = cursor->search(cursor)) != 0) break; @@ -181,7 +183,7 @@ read_index(WT_SESSION *session) } } if (!RET_OK(ret)) - fprintf(stderr, "Error %d for year %d\n", ret, year); + fprintf(stderr, "Error %d for year %" PRIu16 "\n", ret, year); ret = cursor->close(cursor); return (ret); @@ -245,7 +247,8 @@ setup_table(WT_SESSION *session) cursor->set_key(cursor, p.id); cursor->set_value(cursor, p.last_name, p.first_name, p.term_start, p.term_end); - fprintf(stderr, "SETUP: table insert %d-%d: %s %s\n", + fprintf(stderr, + "SETUP: table insert %" PRIu16 "-%" PRIu16 ": %s %s\n", p.term_start, p.term_end, p.first_name, p.last_name); ret = cursor->insert(cursor); diff --git a/examples/c/ex_schema.c b/examples/c/ex_schema.c index f3d3a1b9cb4..70fc7eb2e62 100644 --- a/examples/c/ex_schema.c +++ b/examples/c/ex_schema.c @@ -165,7 +165,8 @@ main(void) ret = cursor->get_key(cursor, &recno); ret = cursor->get_value(cursor, &country, &year, &population); printf("ID %" PRIu64, recno); - printf(": country %s, year %u, population %" PRIu64 "\n", + printf( + ": country %s, year %" PRIu16 ", population %" PRIu64 "\n", country, year, population); } ret = cursor->close(cursor); @@ -186,7 +187,8 @@ main(void) ret = wiredtiger_struct_unpack(session, value.data, value.size, "5sHQ", &country, &year, &population); - printf(": country %s, year %u, population %" PRIu64 "\n", + printf( + ": country %s, year %" PRIu16 ", population %" PRIu64 "\n", country, year, population); } /*! [List the records in the table using raw mode.] */ @@ -202,7 +204,9 @@ main(void) cursor->set_key(cursor, 2); if ((ret = cursor->search(cursor)) == 0) { ret = cursor->get_value(cursor, &country, &year, &population); - printf("ID 2: country %s, year %u, population %" PRIu64 "\n", + printf( + "ID 2: " + "country %s, year %" PRIu16 ", population %" PRIu64 "\n", country, year, population); } /*! [Read population from the primary column group] */ @@ -230,8 +234,8 @@ main(void) cursor->set_key(cursor, "AU\0\0\0"); ret = cursor->search(cursor); ret = cursor->get_value(cursor, &country, &year, &population); - printf("AU: country %s, year %u, population %" PRIu64 "\n", - country, (unsigned int)year, population); + printf("AU: country %s, year %" PRIu16 ", population %" PRIu64 "\n", + country, year, population); /*! [Search in a simple index] */ ret = cursor->close(cursor); @@ -242,8 +246,9 @@ main(void) cursor->set_key(cursor, "USA\0\0", (uint16_t)1900); ret = cursor->search(cursor); ret = cursor->get_value(cursor, &country, &year, &population); - printf("US 1900: country %s, year %u, population %" PRIu64 "\n", - country, (unsigned int)year, population); + printf( + "US 1900: country %s, year %" PRIu16 ", population %" PRIu64 "\n", + country, year, population); /*! [Search in a composite index] */ ret = cursor->close(cursor); @@ -256,7 +261,7 @@ main(void) "table:poptable(country,year)", NULL, NULL, &cursor); while ((ret = cursor->next(cursor)) == 0) { ret = cursor->get_value(cursor, &country, &year); - printf("country %s, year %u\n", country, year); + printf("country %s, year %" PRIu16 "\n", country, year); } /*! [Return a subset of values from the table] */ ret = cursor->close(cursor); @@ -274,7 +279,7 @@ main(void) ret = cursor->get_value(cursor, &value); ret = wiredtiger_struct_unpack( session, value.data, value.size, "5sH", &country, &year); - printf("country %s, year %u\n", country, year); + printf("country %s, year %" PRIu16 "\n", country, year); } /*! [Return a subset of values from the table using raw mode] */ ret = cursor->close(cursor); @@ -289,7 +294,7 @@ main(void) while ((ret = cursor->next(cursor)) == 0) { ret = cursor->get_key(cursor, &country, &year); ret = cursor->get_value(cursor, &recno); - printf("row ID %" PRIu64 ": country %s, year %u\n", + printf("row ID %" PRIu64 ": country %s, year %" PRIu16 "\n", recno, country, year); } /*! [Return the table's record number key using an index] */ @@ -306,7 +311,7 @@ main(void) while ((ret = cursor->next(cursor)) == 0) { ret = cursor->get_key(cursor, &country, &year); ret = cursor->get_value(cursor, &population); - printf("population %" PRIu64 ": country %s, year %u\n", + printf("population %" PRIu64 ": country %s, year %" PRIu16 "\n", population, country, year); } /*! [Return a subset of the value columns from an index] */ @@ -321,7 +326,7 @@ main(void) "index:poptable:country_plus_year()", NULL, NULL, &cursor); while ((ret = cursor->next(cursor)) == 0) { ret = cursor->get_key(cursor, &country, &year); - printf("country %s, year %u\n", country, year); + printf("country %s, year %" PRIu16 "\n", country, year); } /*! [Access only the index] */ ret = cursor->close(cursor); @@ -351,7 +356,8 @@ main(void) ret = join_cursor->get_value(join_cursor, &country, &year, &population); printf("ID %" PRIu64, recno); - printf(": country %s, year %u, population %" PRIu64 "\n", + printf( + ": country %s, year %" PRIu16 ", population %" PRIu64 "\n", country, year, population); } /*! [Join cursors] */ diff --git a/src/block/block_ckpt.c b/src/block/block_ckpt.c index 03059c8f23a..812bf99acfb 100644 --- a/src/block/block_ckpt.c +++ b/src/block/block_ckpt.c @@ -812,8 +812,7 @@ __ckpt_string(WT_SESSION_IMPL *session, WT_RET(__wt_block_buffer_to_ckpt(session, block, addr, ci)); WT_RET(__wt_buf_fmt(session, buf, - "version=%d", - ci->version)); + "version=%" PRIu8, ci->version)); if (ci->root_offset == WT_BLOCK_INVALID_OFFSET) WT_RET(__wt_buf_catfmt(session, buf, ", root=[Empty]")); else diff --git a/src/block/block_open.c b/src/block/block_open.c index d9b2f908737..adb745c99e7 100644 --- a/src/block/block_open.c +++ b/src/block/block_open.c @@ -369,7 +369,7 @@ __desc_read(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_ERR_MSG(session, WT_ERROR, "unsupported WiredTiger file version: this build only " "supports major/minor versions up to %d/%d, and the file " - "is version %d/%d", + "is version %" PRIu16 "/%" PRIu16, WT_BLOCK_MAJOR_VERSION, WT_BLOCK_MINOR_VERSION, desc->majorv, desc->minorv); diff --git a/src/config/config.c b/src/config/config.c index f480ab83dbd..9d14353f730 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -15,10 +15,13 @@ static int __config_err(WT_CONFIG *conf, const char *msg, int err) { + ptrdiff_t d; + + d = conf->cur - conf->orig; + WT_RET_MSG(conf->session, err, - "Error parsing '%.*s' at byte %u: %s", - (int)(conf->end - conf->orig), conf->orig, - (u_int)(conf->cur - conf->orig), msg); + "Error parsing '%.*s' at offset %td: %s", + (int)(conf->end - conf->orig), conf->orig, d, msg); } /* diff --git a/src/log/log.c b/src/log/log.c index 0b84b5b2b19..0b47308cfd8 100644 --- a/src/log/log.c +++ b/src/log/log.c @@ -118,9 +118,9 @@ __wt_log_force_sync(WT_SESSION_IMPL *session, WT_LSN *min_lsn) */ if (log->sync_dir_lsn.l.file < min_lsn->l.file) { WT_ERR(__wt_verbose(session, WT_VERB_LOG, - "log_force_sync: sync directory %s to LSN %d/%lu", - log->log_dir_fh->name, - min_lsn->l.file, min_lsn->l.offset)); + "log_force_sync: sync directory %s to LSN %" PRIu32 + "/%" PRIu32, + log->log_dir_fh->name, min_lsn->l.file, min_lsn->l.offset)); WT_ERR(__wt_directory_sync_fh(session, log->log_dir_fh)); log->sync_dir_lsn = *min_lsn; WT_STAT_FAST_CONN_INCR(session, log_sync_dir); @@ -130,7 +130,7 @@ __wt_log_force_sync(WT_SESSION_IMPL *session, WT_LSN *min_lsn) */ if (__wt_log_cmp(&log->sync_lsn, min_lsn) < 0) { WT_ERR(__wt_verbose(session, WT_VERB_LOG, - "log_force_sync: sync %s to LSN %d/%lu", + "log_force_sync: sync %s to LSN %" PRIu32 "/%" PRIu32, log->log_fh->name, min_lsn->l.file, min_lsn->l.offset)); WT_ERR(__wt_fsync(session, log->log_fh)); log->sync_lsn = *min_lsn; @@ -697,7 +697,7 @@ __log_openfile(WT_SESSION_IMPL *session, WT_ERR_MSG(session, WT_ERROR, "unsupported WiredTiger file version: this build " " only supports major/minor versions up to %d/%d, " - " and the file is version %d/%d", + " and the file is version %" PRIu16 "/%" PRIu16, WT_LOG_MAJOR_VERSION, WT_LOG_MINOR_VERSION, desc->majorv, desc->minorv); } @@ -1129,7 +1129,8 @@ __wt_log_open(WT_SESSION_IMPL *session) } log->fileid = lastlog; WT_ERR(__wt_verbose(session, WT_VERB_LOG, - "log_open: first log %d last log %d", firstlog, lastlog)); + "log_open: first log %" PRIu32 " last log %" PRIu32, + firstlog, lastlog)); if (firstlog == UINT32_MAX) { WT_ASSERT(session, logcount == 0); WT_INIT_LSN(&log->first_lsn); @@ -1395,7 +1396,8 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) (log->sync_dir_lsn.l.file < sync_lsn.l.file)) { WT_ASSERT(session, log->log_dir_fh != NULL); WT_ERR(__wt_verbose(session, WT_VERB_LOG, - "log_release: sync directory %s to LSN %u/%lu", + "log_release: sync directory %s to LSN %" PRIu32 + "/%" PRIu32, log->log_dir_fh->name, sync_lsn.l.file, sync_lsn.l.offset)); WT_ERR(__wt_directory_sync_fh( @@ -1410,7 +1412,8 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) if (F_ISSET(slot, WT_SLOT_SYNC) && __wt_log_cmp(&log->sync_lsn, &slot->slot_end_lsn) < 0) { WT_ERR(__wt_verbose(session, WT_VERB_LOG, - "log_release: sync log %s to LSN %u/%lu", + "log_release: sync log %s to LSN %" PRIu32 + "/%" PRIu32, log->log_fh->name, sync_lsn.l.file, sync_lsn.l.offset)); WT_STAT_FAST_CONN_INCR(session, log_sync); @@ -1477,7 +1480,7 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, if (LF_ISSET(WT_LOGSCAN_RECOVER)) WT_RET(__wt_verbose(session, WT_VERB_LOG, - "__wt_log_scan truncating to %u/%u", + "__wt_log_scan truncating to %" PRIu32 "/%" PRIu32, log->trunc_lsn.l.file, log->trunc_lsn.l.offset)); if (log != NULL) { @@ -2139,7 +2142,7 @@ __wt_log_flush(WT_SESSION_IMPL *session, uint32_t flags) WT_RET(__wt_log_flush_lsn(session, &lsn, false)); WT_RET(__wt_verbose(session, WT_VERB_LOG, - "log_flush: flags %d LSN %u/%lu", + "log_flush: flags %#" PRIx32 " LSN %" PRIu32 "/%" PRIu32, flags, lsn.l.file, lsn.l.offset)); /* * If the user wants write-no-sync, there is nothing more to do. diff --git a/src/lsm/lsm_manager.c b/src/lsm/lsm_manager.c index cf581475d2c..24707abdb5a 100644 --- a/src/lsm/lsm_manager.c +++ b/src/lsm/lsm_manager.c @@ -433,8 +433,10 @@ __lsm_manager_run_server(WT_SESSION_IMPL *session) session, WT_LSM_WORK_BLOOM, 0, lsm_tree)); WT_ERR(__wt_verbose(session, WT_VERB_LSM_MANAGER, - "MGR %s: queue %d mod %d nchunks %d" - " flags 0x%x aggressive %d pushms %" PRIu64 + "MGR %s: queue %" PRIu32 " mod %d " + "nchunks %" PRIu32 + " flags %#" PRIx32 " aggressive %" PRIu32 + " pushms %" PRIu64 " fillms %" PRIu64, lsm_tree->name, lsm_tree->queue_ref, lsm_tree->modified, lsm_tree->nchunks, diff --git a/src/lsm/lsm_merge.c b/src/lsm/lsm_merge.c index 29325066da7..f7176067f19 100644 --- a/src/lsm/lsm_merge.c +++ b/src/lsm/lsm_merge.c @@ -60,7 +60,7 @@ __lsm_merge_aggressive_update(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) { struct timespec now; uint64_t msec_since_last_merge, msec_to_create_merge; - u_int new_aggressive; + uint32_t new_aggressive; new_aggressive = 0; @@ -124,8 +124,9 @@ __lsm_merge_aggressive_update(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) if (new_aggressive > lsm_tree->merge_aggressiveness) { WT_RET(__wt_verbose(session, WT_VERB_LSM, - "LSM merge %s got aggressive (old %u new %u), " - "merge_min %d, %u / %" PRIu64, + "LSM merge %s got aggressive " + "(old %" PRIu32 " new %" PRIu32 "), " + "merge_min %u, %" PRIu64 " / %" PRIu64, lsm_tree->name, lsm_tree->merge_aggressiveness, new_aggressive, lsm_tree->merge_min, msec_since_last_merge, lsm_tree->chunk_fill_ms)); @@ -410,7 +411,8 @@ __wt_lsm_merge(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int id) start_chunk, end_chunk, dest_id, record_count, generation)); for (verb = start_chunk; verb <= end_chunk; verb++) WT_ERR(__wt_verbose(session, WT_VERB_LSM, - "Merging %s: Chunk[%u] id %u, gen: %" PRIu32 + "Merging %s: Chunk[%u] id %" PRIu32 + ", gen: %" PRIu32 ", size: %" PRIu64 ", records: %" PRIu64, lsm_tree->name, verb, lsm_tree->chunk[verb]->id, lsm_tree->chunk[verb]->generation, diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index 7c188bf3dc7..91339501663 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -1314,8 +1314,8 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, bool *skipp) if (chunk != NULL) { WT_ERR(__wt_verbose(session, WT_VERB_LSM, "Compact force flush %s flags 0x%" PRIx32 - " chunk %u flags 0x%" - PRIx32, name, lsm_tree->flags, chunk->id, chunk->flags)); + " chunk %" PRIu32 " flags 0x%" PRIx32, + name, lsm_tree->flags, chunk->id, chunk->flags)); flushing = true; /* * Make sure the in-memory chunk gets flushed do not push a @@ -1348,7 +1348,7 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, bool *skipp) if (F_ISSET(chunk, WT_LSM_CHUNK_ONDISK)) { WT_ERR(__wt_verbose(session, WT_VERB_LSM, - "Compact flush done %s chunk %u. " + "Compact flush done %s chunk %" PRIu32 ". " "Start compacting progress %" PRIu64, name, chunk->id, lsm_tree->merge_progressing)); @@ -1359,7 +1359,7 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, bool *skipp) progress = lsm_tree->merge_progressing; } else { WT_ERR(__wt_verbose(session, WT_VERB_LSM, - "Compact flush retry %s chunk %u", + "Compact flush retry %s chunk %" PRIu32, name, chunk->id)); WT_ERR(__wt_lsm_manager_push_entry(session, WT_LSM_WORK_FLUSH, WT_LSM_WORK_FORCE, diff --git a/src/lsm/lsm_work_unit.c b/src/lsm/lsm_work_unit.c index 7723818f607..88054f86d65 100644 --- a/src/lsm/lsm_work_unit.c +++ b/src/lsm/lsm_work_unit.c @@ -72,7 +72,7 @@ __wt_lsm_get_chunk_to_flush(WT_SESSION_IMPL *session, { WT_DECL_RET; WT_LSM_CHUNK *chunk, *evict_chunk, *flush_chunk; - u_int i; + uint32_t i; *chunkp = NULL; chunk = evict_chunk = flush_chunk = NULL; @@ -118,7 +118,7 @@ __wt_lsm_get_chunk_to_flush(WT_SESSION_IMPL *session, if (chunk != NULL) { WT_ERR(__wt_verbose(session, WT_VERB_LSM, - "Flush%s: return chunk %u of %u: %s", + "Flush%s: return chunk %" PRIu32 " of %" PRIu32 ": %s", force ? " w/ force" : "", i, lsm_tree->nchunks, chunk->uri)); diff --git a/src/lsm/lsm_worker.c b/src/lsm/lsm_worker.c index 7562cb1cae3..b8a85e4a5e5 100644 --- a/src/lsm/lsm_worker.c +++ b/src/lsm/lsm_worker.c @@ -20,7 +20,7 @@ int __wt_lsm_worker_start(WT_SESSION_IMPL *session, WT_LSM_WORKER_ARGS *args) { WT_RET(__wt_verbose(session, WT_VERB_LSM_MANAGER, - "Start LSM worker %d type 0x%x", args->id, args->type)); + "Start LSM worker %u type %#" PRIx32, args->id, args->type)); return (__wt_thread_create(session, &args->tid, __lsm_worker, args)); } @@ -59,9 +59,8 @@ __lsm_worker_general_op( */ if (chunk != NULL) { WT_ERR(__wt_verbose(session, WT_VERB_LSM, - "Flush%s chunk %d %s", - force ? " w/ force" : "", - chunk->id, chunk->uri)); + "Flush%s chunk %" PRIu32 " %s", + force ? " w/ force" : "", chunk->id, chunk->uri)); ret = __wt_lsm_checkpoint_chunk( session, entry->lsm_tree, chunk); WT_ASSERT(session, chunk->refcnt > 0); @@ -164,7 +163,7 @@ __lsm_worker(void *arg) if (ret != 0) { err: __wt_lsm_manager_free_work_unit(session, entry); WT_PANIC_MSG(session, ret, - "Error in LSM worker thread %d", cookie->id); + "Error in LSM worker thread %u", cookie->id); } return (WT_THREAD_RET_VALUE); } diff --git a/src/schema/schema_plan.c b/src/schema/schema_plan.c index 612a2d2d192..12a1aa9c22f 100644 --- a/src/schema/schema_plan.c +++ b/src/schema/schema_plan.c @@ -212,7 +212,7 @@ __wt_struct_plan(WT_SESSION_IMPL *session, WT_TABLE *table, WT_ASSERT(session, !value_only || coltype == WT_PROJ_VALUE); WT_RET(__wt_buf_catfmt( - session, plan, "%d%c", cg, coltype)); + session, plan, "%u%c", cg, coltype)); /* * Set the current column group and column @@ -226,7 +226,7 @@ __wt_struct_plan(WT_SESSION_IMPL *session, WT_TABLE *table, if (current_col < col) { if (col - current_col > 1) WT_RET(__wt_buf_catfmt(session, - plan, "%d", col - current_col)); + plan, "%u", col - current_col)); WT_RET(__wt_buf_catfmt(session, plan, "%c", WT_PROJ_SKIP)); } @@ -375,8 +375,8 @@ __wt_struct_reformat(WT_SESSION_IMPL *session, WT_TABLE *table, pv.type = 'u'; if (pv.havesize) - WT_RET(__wt_buf_catfmt( - session, format, "%d%c", (int)pv.size, pv.type)); + WT_RET(__wt_buf_catfmt(session, + format, "%" PRIu32 "%c", pv.size, pv.type)); else WT_RET(__wt_buf_catfmt(session, format, "%c", pv.type)); } while (have_next); @@ -399,8 +399,8 @@ __wt_struct_truncate(WT_SESSION_IMPL *session, while (ncols-- > 0) { WT_RET(__pack_next(&pack, &pv)); if (pv.havesize) - WT_RET(__wt_buf_catfmt( - session, format, "%d%c", (int)pv.size, pv.type)); + WT_RET(__wt_buf_catfmt(session, + format, "%" PRIu32 "%c", pv.size, pv.type)); else WT_RET(__wt_buf_catfmt(session, format, "%c", pv.type)); } diff --git a/src/session/session_api.c b/src/session/session_api.c index 3153ca458eb..a03fa586a9a 100644 --- a/src/session/session_api.c +++ b/src/session/session_api.c @@ -1598,7 +1598,7 @@ __open_session(WT_CONNECTION_IMPL *conn, if (i == conn->session_size) WT_ERR_MSG(session, ENOMEM, "only configured to support %" PRIu32 " sessions" - " (including %d additional internal sessions)", + " (including %" PRIu32 " additional internal sessions)", conn->session_size, WT_EXTRA_INTERNAL_SESSIONS); /* diff --git a/src/support/huffman.c b/src/support/huffman.c index edd0bc9f648..1e1aaeab5b5 100644 --- a/src/support/huffman.c +++ b/src/support/huffman.c @@ -492,11 +492,12 @@ __wt_huffman_open(WT_SESSION_IMPL *session, uint8_t symbol; uint32_t weighted_length; - printf("leaf depth %" PRIu16 "..%" PRIu16 ", memory use: " - "codes %u# * %uB + code2symbol %u# * %uB\n", + printf("leaf depth %" PRIu16 "..%" PRIu16 + ", memory use: codes %u# * %" WT_SIZET_FMT + "B + code2symbol %u# * %" WT_SIZET_FMT "B\n", huffman->min_depth, huffman->max_depth, - huffman->numSymbols, (u_int)sizeof(WT_HUFFMAN_CODE), - 1U << huffman->max_depth, (u_int)sizeof(uint16_t)); + huffman->numSymbols, sizeof(WT_HUFFMAN_CODE), + 1U << huffman->max_depth, sizeof(uint16_t)); /* * measure quality of computed Huffman codes, for different max bit diff --git a/src/txn/txn.c b/src/txn/txn.c index e8fd8c0c119..7a768a8fe20 100644 --- a/src/txn/txn.c +++ b/src/txn/txn.c @@ -344,7 +344,7 @@ retry: current_id - oldest_id > 10000 && oldest_session != NULL) { (void)__wt_verbose(session, WT_VERB_TRANSACTION, "old snapshot %" PRIu64 - " pinned in session %d [%s]" + " pinned in session %" PRIu32 " [%s]" " with snap_min %" PRIu64 "\n", oldest_id, oldest_session->id, oldest_session->lastop, diff --git a/src/txn/txn_recover.c b/src/txn/txn_recover.c index f41691bbc3b..1ea4dba1152 100644 --- a/src/txn/txn_recover.c +++ b/src/txn/txn_recover.c @@ -88,11 +88,11 @@ __recovery_cursor(WT_SESSION_IMPL *session, WT_RECOVERY *r, * Helper to a cursor if this operation is to be applied during recovery. */ #define GET_RECOVERY_CURSOR(session, r, lsnp, fileid, cp) \ - WT_ERR(__recovery_cursor( \ - (session), (r), (lsnp), (fileid), false, (cp))); \ - WT_ERR(__wt_verbose((session), WT_VERB_RECOVERY, \ - "%s op %d to file %d at LSN %u/%u", \ - (cursor == NULL) ? "Skipping" : "Applying", \ + WT_ERR(__recovery_cursor(session, r, lsnp, fileid, false, cp)); \ + WT_ERR(__wt_verbose(session, WT_VERB_RECOVERY, \ + "%s op %" PRIu32 " to file %" PRIu32 " at LSN %" PRIu32 \ + "/%" PRIu32, \ + cursor == NULL ? "Skipping" : "Applying", \ optype, fileid, lsnp->l.file, lsnp->l.offset)); \ if (cursor == NULL) \ break @@ -334,7 +334,7 @@ __recovery_setup_file(WT_RECOVERY *r, const char *uri, const char *config) r->files[fileid].ckpt_lsn = lsn; WT_RET(__wt_verbose(r->session, WT_VERB_RECOVERY, - "Recovering %s with id %u @ (%" PRIu32 ", %" PRIu32 ")", + "Recovering %s with id %" PRIu32 " @ (%" PRIu32 ", %" PRIu32 ")", uri, fileid, lsn.l.file, lsn.l.offset)); return (0); @@ -496,7 +496,7 @@ __wt_txn_recover(WT_SESSION_IMPL *session) */ r.metadata_only = false; WT_ERR(__wt_verbose(session, WT_VERB_RECOVERY, - "Main recovery loop: starting at %u/%u", + "Main recovery loop: starting at %" PRIu32 "/%" PRIu32, r.ckpt_lsn.l.file, r.ckpt_lsn.l.offset)); WT_ERR(__wt_log_needs_recovery(session, &r.ckpt_lsn, &needs_rec)); /* diff --git a/test/bloom/test_bloom.c b/test/bloom/test_bloom.c index 183dc3d2d42..f95bc7faaf9 100644 --- a/test/bloom/test_bloom.c +++ b/test/bloom/test_bloom.c @@ -160,7 +160,7 @@ run(void) for (i = 0; i < g.c_ops; i++) { item.data = g.entries[i]; if ((ret = __wt_bloom_insert(bloomp, &item)) != 0) - testutil_die(ret, "__wt_bloom_insert: %d", i); + testutil_die(ret, "__wt_bloom_insert: %" PRIu32, i); } testutil_check(__wt_bloom_finalize(bloomp)); @@ -168,7 +168,8 @@ run(void) for (i = 0; i < g.c_ops; i++) { item.data = g.entries[i]; if ((ret = __wt_bloom_get(bloomp, &item)) != 0) { - fprintf(stderr, "get failed at record: %d\n", i); + fprintf(stderr, + "get failed at record: %" PRIu32 "\n", i); testutil_die(ret, "__wt_bloom_get"); } } @@ -201,7 +202,8 @@ run(void) testutil_die(ret, "__wt_bloom_get"); } free((void *)item.data); - printf("Out of %d ops, got %d false positives, %.4f%%\n", + printf( + "Out of %" PRIu32 " ops, got %" PRIu32 " false positives, %.4f%%\n", g.c_ops, fp, 100.0 * fp/g.c_ops); testutil_check(__wt_bloom_drop(bloomp, NULL)); } diff --git a/test/checkpoint/test_checkpoint.c b/test/checkpoint/test_checkpoint.c index 0f28a86b675..c5524b3c63e 100644 --- a/test/checkpoint/test_checkpoint.c +++ b/test/checkpoint/test_checkpoint.c @@ -136,7 +136,7 @@ main(int argc, char *argv[]) printf("%s: process %" PRIu64 "\n", g.progname, (uint64_t)getpid()); for (cnt = 1; (runs == 0 || cnt <= runs) && g.status == 0; ++cnt) { - printf(" %d: %u workers, %u tables\n", + printf(" %d: %d workers, %d tables\n", cnt, g.nworkers, g.ntables); (void)cleanup(); /* Clean up previous runs */ diff --git a/test/cursor_order/cursor_order.c b/test/cursor_order/cursor_order.c index 68d2f092c60..d8cfc0c1421 100644 --- a/test/cursor_order/cursor_order.c +++ b/test/cursor_order/cursor_order.c @@ -154,8 +154,10 @@ main(int argc, char *argv[]) printf("%s: process %" PRIu64 "\n", progname, (uint64_t)getpid()); for (cnt = 1; runs == 0 || cnt <= runs; ++cnt) { - printf(" %d: %u reverse scanners, %u writers\n", cnt, - (int)cfg->reverse_scanners, (int)cfg->append_inserters); + printf( + " %d: %" PRIu64 + " reverse scanners, %" PRIu64 " writers\n", + cnt, cfg->reverse_scanners, cfg->append_inserters); shutdown(); /* Clean up previous runs */ diff --git a/test/fops/file.c b/test/fops/file.c index 4cd92e7b590..ea15f1ee80d 100644 --- a/test/fops/file.c +++ b/test/fops/file.c @@ -147,7 +147,7 @@ obj_create_unique(int force) /* Generate a unique object name. */ if ((ret = pthread_rwlock_wrlock(&single)) != 0) testutil_die(ret, "pthread_rwlock_wrlock single"); - (void)snprintf(new_uri, sizeof(new_uri), "%s.%d", uri, ++uid); + (void)snprintf(new_uri, sizeof(new_uri), "%s.%u", uri, ++uid); if ((ret = pthread_rwlock_unlock(&single)) != 0) testutil_die(ret, "pthread_rwlock_unlock single"); diff --git a/test/format/wts.c b/test/format/wts.c index a0e57dc2bee..81e484296e2 100644 --- a/test/format/wts.c +++ b/test/format/wts.c @@ -53,7 +53,8 @@ compressor(uint32_t compress_flag) default: break; } - testutil_die(EINVAL, "illegal compression flag: 0x%x", compress_flag); + testutil_die(EINVAL, + "illegal compression flag: %#" PRIx32, compress_flag); } /* @@ -71,7 +72,8 @@ encryptor(uint32_t encrypt_flag) default: break; } - testutil_die(EINVAL, "illegal encryption flag: 0x%x", encrypt_flag); + testutil_die(EINVAL, + "illegal encryption flag: %#" PRIx32, encrypt_flag); } static int @@ -313,7 +315,7 @@ wts_create(void) p += snprintf(p, REMAIN(p, end), "key_format=%s," "allocation_size=512,%s" - "internal_page_max=%d,leaf_page_max=%d", + "internal_page_max=%" PRIu32 ",leaf_page_max=%" PRIu32, (g.type == ROW) ? "u" : "r", g.c_firstfit ? "block_allocation=first," : "", maxintlpage, maxleafpage); @@ -325,15 +327,15 @@ wts_create(void) maxintlkey = mmrand(NULL, maxintlpage / 50, maxintlpage / 40); if (maxintlkey > 20) p += snprintf(p, REMAIN(p, end), - ",internal_key_max=%d", maxintlkey); + ",internal_key_max=%" PRIu32, maxintlkey); maxleafkey = mmrand(NULL, maxleafpage / 50, maxleafpage / 40); if (maxleafkey > 20) p += snprintf(p, REMAIN(p, end), - ",leaf_key_max=%d", maxleafkey); + ",leaf_key_max=%" PRIu32, maxleafkey); maxleafvalue = mmrand(NULL, maxleafpage * 10, maxleafpage / 40); if (maxleafvalue > 40 && maxleafvalue < 100 * 1024) p += snprintf(p, REMAIN(p, end), - ",leaf_value_max=%d", maxleafvalue); + ",leaf_value_max=%" PRIu32, maxleafvalue); switch (g.type) { case FIX: @@ -361,7 +363,7 @@ wts_create(void) ",huffman_value=english"); if (g.c_dictionary) p += snprintf(p, REMAIN(p, end), - ",dictionary=%d", mmrand(NULL, 123, 517)); + ",dictionary=%" PRIu32, mmrand(NULL, 123, 517)); break; } diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index 5ca0bcbbb5f..80d4be2ca3d 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -101,7 +101,8 @@ run_ops(int dbs) */ if (session == NULL) { __wt_random_init(&rnd); - if ((session = calloc((size_t)dbs, sizeof(WT_SESSION *))) == NULL) + if ((session = + calloc((size_t)dbs, sizeof(WT_SESSION *))) == NULL) testutil_die(ENOMEM, "session array malloc"); if ((cursor = calloc((size_t)dbs, sizeof(WT_CURSOR *))) == NULL) testutil_die(ENOMEM, "cursor array malloc"); diff --git a/test/recovery/random-abort.c b/test/recovery/random-abort.c index c9cc10d2db3..f9c3ed28814 100644 --- a/test/recovery/random-abort.c +++ b/test/recovery/random-abort.c @@ -249,9 +249,10 @@ main(int argc, char *argv[]) if ((ret = conn->close(conn, NULL)) != 0) testutil_die(ret, "WT_CONNECTION:close"); if (absent) { - printf("%u record(s) absent from %u\n", absent, count); + printf("%" PRIu32 " record(s) absent from %" PRIu32 "\n", + absent, count); return (EXIT_FAILURE); } - printf("%u records verified\n", count); + printf("%" PRIu32 " records verified\n", count); return (EXIT_SUCCESS); } diff --git a/test/recovery/truncated-log.c b/test/recovery/truncated-log.c index 23269e99d35..24e05b5482b 100644 --- a/test/recovery/truncated-log.c +++ b/test/recovery/truncated-log.c @@ -244,7 +244,7 @@ main(int argc, char *argv[]) * the middle of that last record (i.e. ahead of that offset). */ new_offset = offset + V_SIZE; - printf("Parent: Truncate to %u\n", (uint32_t)new_offset); + printf("Parent: Truncate to %" PRIu64 "\n", new_offset); if ((ret = truncate(LOG_FILE_1, (wt_off_t)new_offset)) != 0) testutil_die(errno, "truncate"); @@ -267,9 +267,10 @@ main(int argc, char *argv[]) if ((ret = conn->close(conn, NULL)) != 0) testutil_die(ret, "WT_CONNECTION:close"); if (count > max_key) { - printf("expected %u records found %u\n", max_key, count); + printf("expected %" PRIu32 " records found %" PRIu32 "\n", + max_key, count); return (EXIT_FAILURE); } - printf("%u records verified\n", count); + printf("%" PRIu32 " records verified\n", count); return (EXIT_SUCCESS); } -- cgit v1.2.1 From b56ef1594e879263de256a49eb96377027dc0b25 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 11 Mar 2016 16:13:03 -0500 Subject: WT-2471: review WiredTiger "int" printf formats MAX_CPU not used. --- test/manydbs/manydbs.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index 80d4be2ca3d..897f57d4b11 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -60,13 +60,12 @@ static const char * const uri = "table:main"; WTOPEN_CFG_COMMON \ "transaction_sync=(enabled,method=fsync)" -#define MAX_CPU 10.0 -#define MAX_DBS 10 +#define MAX_DBS 10 #define MAX_IDLE_TIME 30 #define IDLE_INCR 5 -#define MAX_KV 100 -#define MAX_VAL 128 +#define MAX_KV 100 +#define MAX_VAL 128 static void usage(void) -- cgit v1.2.1 From 6bc033048f0369e63bf50253f3e2c755dbc2b0db Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 11 Mar 2016 16:48:50 -0500 Subject: WT-2471: review WiredTiger "int" printf formats Sue notes WT_LSM_TREE.modified should be a boolean, not an "int". --- src/include/lsm.h | 2 +- src/lsm/lsm_tree.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/include/lsm.h b/src/include/lsm.h index 7cb3ccc895d..5a3a63a03eb 100644 --- a/src/include/lsm.h +++ b/src/include/lsm.h @@ -215,7 +215,7 @@ struct __wt_lsm_tree { size_t chunk_alloc; /* Space allocated for chunks */ uint32_t nchunks; /* Number of active chunks */ uint32_t last; /* Last allocated ID */ - int modified; /* Have there been updates? */ + bool modified; /* Have there been updates? */ WT_LSM_CHUNK **old_chunks; /* Array of old LSM chunks */ size_t old_alloc; /* Space allocated for old chunks */ diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index 91339501663..b82eb6c0f5c 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -874,7 +874,7 @@ __wt_lsm_tree_switch(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) F_CLR(lsm_tree, WT_LSM_TREE_NEED_SWITCH); ++lsm_tree->dsk_gen; - lsm_tree->modified = 1; + lsm_tree->modified = true; /* * Set the switch transaction in the previous chunk unless this is -- cgit v1.2.1 From 0b2766c1db03e96d5c7bfbb1eae66a9df790e301 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 11 Mar 2016 18:10:23 -0500 Subject: WT-2473: MSVC doesn't support PRId64 --- examples/c/ex_config_parse.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/c/ex_config_parse.c b/examples/c/ex_config_parse.c index caa7ffa8fe0..7e9bc6104e8 100644 --- a/examples/c/ex_config_parse.c +++ b/examples/c/ex_config_parse.c @@ -99,7 +99,7 @@ main(void) while ((ret = parser->next(parser, &k, &v)) == 0) { printf("%.*s:", (int)k.len, k.str); if (v.type == WT_CONFIG_ITEM_NUM) - printf("%" PRId64 "\n", v.val); + printf("%lld\n", (long long)v.val); else printf("%.*s\n", (int)v.len, v.str); } -- cgit v1.2.1 From c2e2203c8b289f0d4f2ec5708a0305e98ec42c62 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 11 Mar 2016 19:04:42 -0500 Subject: Revert "WT-2473: MSVC doesn't support PRId64" This reverts commit 0b2766c1db03e96d5c7bfbb1eae66a9df790e301. --- examples/c/ex_config_parse.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/c/ex_config_parse.c b/examples/c/ex_config_parse.c index 7e9bc6104e8..caa7ffa8fe0 100644 --- a/examples/c/ex_config_parse.c +++ b/examples/c/ex_config_parse.c @@ -99,7 +99,7 @@ main(void) while ((ret = parser->next(parser, &k, &v)) == 0) { printf("%.*s:", (int)k.len, k.str); if (v.type == WT_CONFIG_ITEM_NUM) - printf("%lld\n", (long long)v.val); + printf("%" PRId64 "\n", v.val); else printf("%.*s\n", (int)v.len, v.str); } -- cgit v1.2.1 From 4ca8e2430b7a4798a4a570f82b17b2b96a0dbbec Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sat, 12 Mar 2016 16:58:28 -0500 Subject: WT-2464: Valgrind errors. Clear at clear the entire disk-image buffer in the case of fixed-length column-store, fixed-length column-store sets bits in bytes, and the bytes are assumed to be initially 0. --- src/reconcile/rec_write.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/reconcile/rec_write.c b/src/reconcile/rec_write.c index f245ff5d921..6047020ff26 100644 --- a/src/reconcile/rec_write.c +++ b/src/reconcile/rec_write.c @@ -1960,12 +1960,19 @@ __rec_split_init(WT_SESSION_IMPL *session, WT_RET(__wt_buf_init(session, &r->disk_image, corrected_page_size)); /* - * Clear the disk page's header and block-manager space, set the page - * type (the type doesn't change, and setting it later would require - * additional code in a few different places). + * Clear at least the disk page's header and block-manager space, but + * in the case of fixed-length column-store, clear the entire buffer. + * (Fixed-length column-store sets bits in bytes, and the bytes are + * assumed to be initially 0.) + */ + memset(r->disk_image.mem, 0, page->type == WT_PAGE_COL_FIX ? + corrected_page_size : WT_PAGE_HEADER_BYTE_SIZE(btree)); + + /* + * Set the page type (the type doesn't change, and setting it later + * would require additional code in a few different places). */ dsk = r->disk_image.mem; - memset(dsk, 0, WT_PAGE_HEADER_BYTE_SIZE(btree)); dsk->type = page->type; /* -- cgit v1.2.1 From b2ee661a91b6a00b64a46454f63a0e128e9f7813 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sun, 13 Mar 2016 13:00:02 +0000 Subject: WT-2464: Valgrind errors. When creating the scratch buffer copy of the original disk image, copy the block-manager space into the scratch buffer (not just the disk page's header), otherwise it's uninitialized memory. --- src/reconcile/rec_write.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/reconcile/rec_write.c b/src/reconcile/rec_write.c index 6047020ff26..9b1ff9ede74 100644 --- a/src/reconcile/rec_write.c +++ b/src/reconcile/rec_write.c @@ -3033,13 +3033,13 @@ __rec_split_fixup(WT_SESSION_IMPL *session, WT_RECONCILE *r) * The data isn't laid out on a page boundary or nul padded; copy it to * a clean, aligned, padded buffer before writing it. * - * Allocate a scratch buffer to hold the new disk image. Copy the - * WT_PAGE_HEADER header onto the scratch buffer, most of the header - * information remains unchanged between the pages. + * Allocate a scratch buffer to hold the new disk image. Copy the disk + * page's header and block-manager space into the scratch buffer, most + * of the header information remains unchanged between the pages. */ WT_RET(__wt_scr_alloc(session, r->disk_image.memsize, &tmp)); dsk = tmp->mem; - memcpy(dsk, r->disk_image.mem, WT_PAGE_HEADER_SIZE); + memcpy(dsk, r->disk_image.mem, WT_PAGE_HEADER_BYTE_SIZE(btree)); /* * For each split chunk we've created, update the disk image and copy -- cgit v1.2.1 From 7116ddeb62ba3984d18814143b4df750eba9bbed Mon Sep 17 00:00:00 2001 From: David Hows Date: Mon, 14 Mar 2016 10:49:43 +1100 Subject: WT-2473 - Add inttypes.h to ex_config_parse --- examples/c/ex_config_parse.c | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/c/ex_config_parse.c b/examples/c/ex_config_parse.c index caa7ffa8fe0..be3c78bedd4 100644 --- a/examples/c/ex_config_parse.c +++ b/examples/c/ex_config_parse.c @@ -30,6 +30,7 @@ * configuration strings. */ +#include #include #include -- cgit v1.2.1 From bf32b9f6ddb2941afd9504cbffb9ee9933c747e4 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Mon, 14 Mar 2016 03:29:25 +0000 Subject: WT-2375 Update reverse integer collator to use streaming pack API. --- ext/collators/revint/revint_collator.c | 39 +++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/ext/collators/revint/revint_collator.c b/ext/collators/revint/revint_collator.c index 1e432842caf..30b5dc67556 100644 --- a/ext/collators/revint/revint_collator.c +++ b/ext/collators/revint/revint_collator.c @@ -53,6 +53,7 @@ revint_compare(WT_COLLATOR *collator, { const REVINT_COLLATOR *revint_collator; WT_EXTENSION_API *wtapi; + WT_PACK_STREAM *pstream; int ret; int64_t i1, i2, p1, p2; @@ -78,22 +79,26 @@ revint_compare(WT_COLLATOR *collator, * To keep this code simple, we do not reverse the ordering * when comparing primary keys. */ - if ((ret = wtapi->struct_unpack(wtapi, session, - k1->data, k1->size, "ii", &i1, &p1)) != 0) { - /* could be missing primary key, try again without */ - if ((ret = wtapi->struct_unpack(wtapi, session, - k1->data, k1->size, "i", &i1)) != 0) - return (ret); - p1 = INT64_MIN; /* sort this first */ - } - if ((ret = wtapi->struct_unpack(wtapi, session, - k2->data, k2->size, "ii", &i2, &p2)) != 0) { - /* could be missing primary key, try again without */ - if ((ret = wtapi->struct_unpack(wtapi, session, - k2->data, k2->size, "i", &i2)) != 0) - return (ret); - p2 = INT64_MIN; /* sort this first */ - } + if ((ret = wtapi->unpack_start( + wtapi, session, "ii", k1->data, k1->size, &pstream)) != 0 || + (ret = wtapi->unpack_int(wtapi, pstream, &i1)) != 0) + goto err; + if ((ret = wtapi->unpack_int(wtapi, pstream, &p1)) != 0) + /* A missing primary key is OK and sorts first. */ + p1 = INT64_MIN; + if ((ret = wtapi->pack_close(wtapi, pstream, NULL)) != 0) + goto err; + + /* Unpack the second pair of numbers. */ + if ((ret = wtapi->unpack_start( + wtapi, session, "ii", k2->data, k2->size, &pstream)) != 0 || + (ret = wtapi->unpack_int(wtapi, pstream, &i2)) != 0) + goto err; + if ((ret = wtapi->unpack_int(wtapi, pstream, &p2)) != 0) + /* A missing primary key is OK and sorts first. */ + p2 = INT64_MIN; + if ((ret = wtapi->pack_close(wtapi, pstream, NULL)) != 0) + goto err; /* sorting is reversed */ if (i1 < i2) @@ -108,7 +113,7 @@ revint_compare(WT_COLLATOR *collator, else *cmp = 0; /* index key and primary key are same */ - return (0); +err: return (ret); } /* -- cgit v1.2.1 From cc484cebade699bb40c5d3b600dcbe11633d166a Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Mon, 14 Mar 2016 03:33:01 +0000 Subject: WT-2475 Have reconf script remove config.cache. --- build_posix/reconf | 1 + 1 file changed, 1 insertion(+) diff --git a/build_posix/reconf b/build_posix/reconf index 8700c5da43d..16d4002d9b9 100755 --- a/build_posix/reconf +++ b/build_posix/reconf @@ -24,6 +24,7 @@ clean() aclocal.m4 \ auto-includes.chk \ autom4te.cache \ + config.cache \ config.hin \ config.hin~ \ config.log \ -- cgit v1.2.1 From 827e107f21697be612c2b93a2fe401808cfc4e8d Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Mon, 14 Mar 2016 14:35:36 +1100 Subject: WT-2460 Fix a flags test in LAS code to use mask not is-set. It was attempting to save which flags were set by checking F_ISSET (which returns a boolean). Should have been using WT_MASK. This was leading to the WT_SESSION_NO_EVICTION flag being cleared unexpectedly and WT_SESSION_CAN_WAIT being set as well. --- src/cache/cache_las.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cache/cache_las.c b/src/cache/cache_las.c index 3549e41e80d..8796ec6b2fc 100644 --- a/src/cache/cache_las.c +++ b/src/cache/cache_las.c @@ -205,7 +205,7 @@ __wt_las_cursor( * useful more than once. */ *session_flags = - F_ISSET(session, WT_SESSION_NO_CACHE | WT_SESSION_NO_EVICTION); + F_MASK(session, WT_SESSION_NO_CACHE | WT_SESSION_NO_EVICTION); conn = S2C(session); -- cgit v1.2.1 From cd34e23f1d59a665a65b7b3e4c718bdccf9aa72a Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Mon, 14 Mar 2016 15:18:23 +1100 Subject: WT-2375 Revert some error handling changes to packing.i The semantic is slightly odd, but relied on internally. --- src/include/packing.i | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/include/packing.i b/src/include/packing.i index 7f277596345..203ea8f2026 100644 --- a/src/include/packing.i +++ b/src/include/packing.i @@ -677,17 +677,13 @@ __wt_struct_unpackv(WT_SESSION_IMPL *session, if (fmt[0] != '\0' && fmt[1] == '\0') { pv.type = fmt[0]; - if (p >= end && pv.type != 'u') - return (ENOMEM); - if ((ret = __unpack_read(session, &pv, &p, size)) == 0) - WT_UNPACK_PUT(session, pv, ap); + WT_RET(__unpack_read(session, &pv, &p, size)); + WT_UNPACK_PUT(session, pv, ap); return (0); } WT_RET(__pack_init(session, &pack, fmt)); while ((ret = __pack_next(&pack, &pv)) == 0) { - if (p >= end && pv.type != 'u') - return (ENOMEM); WT_RET(__unpack_read(session, &pv, &p, (size_t)(end - p))); WT_UNPACK_PUT(session, pv, ap); } -- cgit v1.2.1 From ab5138cc655b19c502ffae9bc5fceb319f1dcb96 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Mon, 14 Mar 2016 15:22:20 +1100 Subject: whitespace --- src/include/packing.i | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/packing.i b/src/include/packing.i index 203ea8f2026..35b2ddc43db 100644 --- a/src/include/packing.i +++ b/src/include/packing.i @@ -678,7 +678,7 @@ __wt_struct_unpackv(WT_SESSION_IMPL *session, if (fmt[0] != '\0' && fmt[1] == '\0') { pv.type = fmt[0]; WT_RET(__unpack_read(session, &pv, &p, size)); - WT_UNPACK_PUT(session, pv, ap); + WT_UNPACK_PUT(session, pv, ap); return (0); } -- cgit v1.2.1 From 880021aea3dfae1a398bcec4922afe0c5755e14d Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Mon, 14 Mar 2016 16:45:49 +1100 Subject: WT-2477 Checks for a 64-bit build without polluting config.h files. --- build_posix/configure.ac.in | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build_posix/configure.ac.in b/build_posix/configure.ac.in index 3280a4e9af3..9251873be73 100644 --- a/build_posix/configure.ac.in +++ b/build_posix/configure.ac.in @@ -113,10 +113,12 @@ AC_SYS_LARGEFILE AC_C_BIGENDIAN -AC_CHECK_SIZEOF(void *) +AC_MSG_CHECKING([for a 64-bit build]) +AC_COMPUTE_INT(ac_cv_sizeof_void_p, [sizeof(void *)]) if test "$ac_cv_sizeof_void_p" != "8" ; then AC_MSG_ERROR([WiredTiger requires a 64-bit build.]) fi +AC_MSG_RESULT(yes) # Linux requires _GNU_SOURCE to be defined case "$host_os" in -- cgit v1.2.1 From 94fd9955def2dc1a243b3504acf7f0c65580de1e Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 14 Mar 2016 12:42:15 +0000 Subject: WT-2478: Valgrind test failures The bit-set macros assume the underlying initial byte values are 0, clear the allocated temporary buffer. --- src/btree/row_key.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/btree/row_key.c b/src/btree/row_key.c index 8b9e858ec18..1ffde52809e 100644 --- a/src/btree/row_key.c +++ b/src/btree/row_key.c @@ -52,7 +52,8 @@ __wt_row_leaf_keys(WT_SESSION_IMPL *session, WT_PAGE *page) WT_RET(__wt_scr_alloc(session, 0, &key)); WT_RET(__wt_scr_alloc(session, (uint32_t)__bitstr_size(page->pg_row_entries), &tmp)); - + memset(tmp->mem, 0, tmp->memsize); + if ((gap = btree->key_gap) == 0) gap = 1; __inmem_row_leaf_slots(tmp->mem, 0, page->pg_row_entries, gap); -- cgit v1.2.1 From dee838189e67a4a68313450c98042ec17321285f Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 14 Mar 2016 12:55:54 +0000 Subject: WT-2478: Valgrind test failures Whitespace --- src/btree/row_key.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/btree/row_key.c b/src/btree/row_key.c index 1ffde52809e..9fff092d079 100644 --- a/src/btree/row_key.c +++ b/src/btree/row_key.c @@ -53,7 +53,7 @@ __wt_row_leaf_keys(WT_SESSION_IMPL *session, WT_PAGE *page) WT_RET(__wt_scr_alloc(session, (uint32_t)__bitstr_size(page->pg_row_entries), &tmp)); memset(tmp->mem, 0, tmp->memsize); - + if ((gap = btree->key_gap) == 0) gap = 1; __inmem_row_leaf_slots(tmp->mem, 0, page->pg_row_entries, gap); -- cgit v1.2.1 From 1fd5955e494b17617211778acb1a17d59f412b3c Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 14 Mar 2016 13:23:03 +0000 Subject: WT-2478: Valgrind test failures Too many places create buffers we write (for example, encryption). Quit trying to initialize the block header information in any place other than the block header code. --- src/block/block_write.c | 8 +++++++- src/reconcile/rec_write.c | 12 +++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/block/block_write.c b/src/block/block_write.c index 4c6ac198fe4..e05a430832e 100644 --- a/src/block/block_write.c +++ b/src/block/block_write.c @@ -206,9 +206,15 @@ __wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t cksum; bool local_locked; - blk = WT_BLOCK_HEADER_REF(buf->mem); fh = block->fh; + /* + * Clear the block header to ensure all of it is initialized, even the + * unused fields. + */ + blk = WT_BLOCK_HEADER_REF(buf->mem); + memset(blk, 0, sizeof(*blk)); + /* * Swap the page-header as needed; this doesn't belong here, but it's * the best place to catch all callers. diff --git a/src/reconcile/rec_write.c b/src/reconcile/rec_write.c index 9b1ff9ede74..a69f335c9b3 100644 --- a/src/reconcile/rec_write.c +++ b/src/reconcile/rec_write.c @@ -1960,13 +1960,15 @@ __rec_split_init(WT_SESSION_IMPL *session, WT_RET(__wt_buf_init(session, &r->disk_image, corrected_page_size)); /* - * Clear at least the disk page's header and block-manager space, but - * in the case of fixed-length column-store, clear the entire buffer. - * (Fixed-length column-store sets bits in bytes, and the bytes are - * assumed to be initially 0.) + * Clear the disk page header to ensure all of it is initialized, even + * the unused fields. + * + * In the case of fixed-length column-store, clear the entire buffer: + * fixed-length column-store sets bits in bytes, where the bytes are + * assumed to initially be 0. */ memset(r->disk_image.mem, 0, page->type == WT_PAGE_COL_FIX ? - corrected_page_size : WT_PAGE_HEADER_BYTE_SIZE(btree)); + corrected_page_size : WT_PAGE_HEADER_SIZE); /* * Set the page type (the type doesn't change, and setting it later -- cgit v1.2.1 From 181e25dfa3968e48e63430f2f9ace996bc75553c Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Mon, 14 Mar 2016 10:16:32 -0400 Subject: WT-2447 Avoid main table reads. During iteration, walk the first joined index entries using a cursor with a null projection, when we find an entry, set the position on a full projection. For bloom filters on indices, use the raw index. --- src/cursor/cur_join.c | 186 ++++++++++++++++++++++++++++++-------------------- src/include/cursor.h | 3 +- 2 files changed, 115 insertions(+), 74 deletions(-) diff --git a/src/cursor/cur_join.c b/src/cursor/cur_join.c index 2ef65b5bddf..62aef27e3d1 100644 --- a/src/cursor/cur_join.c +++ b/src/cursor/cur_join.c @@ -20,42 +20,46 @@ static int __curjoin_entry_iter_init(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, WT_CURSOR_JOIN_ENTRY *entry, WT_CURSOR_JOIN_ITER **iterp) { - WT_CURSOR *newcur; WT_CURSOR *to_dup; WT_DECL_RET; const char *raw_cfg[] = { WT_CONFIG_BASE( session, WT_SESSION_open_cursor), "raw", NULL }; const char *def_cfg[] = { WT_CONFIG_BASE( session, WT_SESSION_open_cursor), NULL }; - const char *uri, **config; - char *uribuf; + const char *urifull, **config; + char *fullbuf, *urimin; WT_CURSOR_JOIN_ITER *iter; size_t size; iter = NULL; - uribuf = NULL; + fullbuf = urimin = NULL; to_dup = entry->ends[0].cursor; - uri = to_dup->uri; if (F_ISSET((WT_CURSOR *)cjoin, WT_CURSTD_RAW)) config = &raw_cfg[0]; else config = &def_cfg[0]; + urifull = to_dup->uri; + size = strlen(urifull) + 3; + WT_ERR(__wt_calloc(session, size, 1, &urimin)); + snprintf(urimin, size, "%s()", urifull); if (cjoin->projection != NULL) { - size = strlen(uri) + strlen(cjoin->projection) + 1; - WT_ERR(__wt_calloc(session, size, 1, &uribuf)); - snprintf(uribuf, size, "%s%s", uri, cjoin->projection); - uri = uribuf; + size = strlen(urifull) + strlen(cjoin->projection) + 1; + WT_ERR(__wt_calloc(session, size, 1, &fullbuf)); + snprintf(fullbuf, size, "%s%s", urifull, cjoin->projection); + urifull = fullbuf; } - WT_ERR(__wt_open_cursor(session, uri, (WT_CURSOR *)cjoin, config, - &newcur)); - WT_ERR(__wt_cursor_dup_position(to_dup, newcur)); + WT_ERR(__wt_calloc_one(session, &iter)); + WT_ERR(__wt_open_cursor(session, urimin, (WT_CURSOR *)cjoin, config, + &iter->cursor)); + WT_ERR(__wt_cursor_dup_position(to_dup, iter->cursor)); + WT_ERR(__wt_open_cursor(session, urifull, (WT_CURSOR *)cjoin, config, + &iter->full)); iter->cjoin = cjoin; iter->session = session; iter->entry = entry; - iter->cursor = newcur; iter->positioned = false; iter->isequal = (entry->ends_next == 1 && WT_CURJOIN_END_RANGE(&entry->ends[0]) == WT_CURJOIN_END_EQ); @@ -64,7 +68,8 @@ __curjoin_entry_iter_init(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, if (0) { err: __wt_free(session, iter); } - __wt_free(session, uribuf); + __wt_free(session, fullbuf); + __wt_free(session, urimin); return (ret); } @@ -90,45 +95,77 @@ __curjoin_pack_recno(WT_SESSION_IMPL *session, uint64_t r, uint8_t *buf, } /* - * __curjoin_entry_iter_next -- - * Get the next item in an iteration. + * __curjoin_copy_primary_key -- + * Copy the primary key from a cursor (either main table or index) + * to another cursory. * */ static int -__curjoin_entry_iter_next(WT_CURSOR_JOIN_ITER *iter, WT_ITEM *primkey, - uint64_t *rp) +__curjoin_copy_primary_key(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, + WT_CURSOR *tocur, WT_CURSOR *fromcur, const char *repack_fmt, bool isindex) { WT_CURSOR *firstcg_cur; - WT_CURSOR_JOIN *cjoin; - WT_SESSION_IMPL *session; - uint64_t r; + WT_CURSOR_INDEX *cindex; + WT_ITEM key, *keyp; + const uint8_t *p; + + if (isindex) { + cindex = ((WT_CURSOR_INDEX *)fromcur); + firstcg_cur = cindex->cg_cursors[0]; + /* + * Repack tells us where the index key ends; advance past + * that to get where the raw primary key starts. + */ + WT_RET(__wt_struct_repack(session, cindex->child->key_format, + repack_fmt != NULL ? repack_fmt : cindex->iface.key_format, + &cindex->child->key, &key)); + WT_ASSERT(session, cindex->child->key.size > key.size); + key.data = (uint8_t *)key.data + key.size; + key.size = cindex->child->key.size - key.size; + WT_ITEM_SET(tocur->key, key); + if (WT_CURSOR_RECNO(tocur)) { + p = (const uint8_t *)key.data; + WT_RET(__wt_vunpack_uint(&p, key.size, &tocur->recno)); + } else + tocur->recno = 0; + } else { + firstcg_cur = ((WT_CURSOR_TABLE *)fromcur)->cg_cursors[0]; + keyp = &firstcg_cur->key; + if (WT_CURSOR_RECNO(tocur)) { + WT_ASSERT(session, keyp->size == sizeof(uint64_t)); + tocur->recno = *(uint64_t *)keyp->data; + WT_RET(__curjoin_pack_recno(session, tocur->recno, + cjoin->recno_buf, sizeof(cjoin->recno_buf), + &tocur->key)); + } else { + WT_ITEM_SET(tocur->key, *keyp); + tocur->recno = 0; + } + } + return (0); +} +/* + * __curjoin_entry_iter_next -- + * Get the next item in an iteration. + * + */ +static int +__curjoin_entry_iter_next(WT_CURSOR_JOIN_ITER *iter, WT_CURSOR *cursor) +{ if (iter->positioned) WT_RET(iter->cursor->next(iter->cursor)); else iter->positioned = true; - session = iter->session; - cjoin = iter->cjoin; - /* * Set our key to the primary key, we'll also need this * to check membership. */ - if (iter->entry->index != NULL) - firstcg_cur = ((WT_CURSOR_INDEX *)iter->cursor)->cg_cursors[0]; - else - firstcg_cur = ((WT_CURSOR_TABLE *)iter->cursor)->cg_cursors[0]; - if (WT_CURSOR_RECNO(&cjoin->iface)) { - r = *(uint64_t *)firstcg_cur->key.data; - WT_RET(__curjoin_pack_recno(session, r, cjoin->recno_buf, - sizeof(cjoin->recno_buf), primkey)); - *rp = r; - } else { - WT_ITEM_SET(*primkey, firstcg_cur->key); - *rp = 0; - } - iter->curkey = primkey; + WT_RET(__curjoin_copy_primary_key(iter->session, iter->cjoin, cursor, + iter->cursor, iter->entry->repack_format, + iter->entry->index != NULL)); + iter->curkey = &cursor->key; iter->entry->stats.actual_count++; iter->entry->stats.accesses++; return (0); @@ -144,6 +181,7 @@ __curjoin_entry_iter_reset(WT_CURSOR_JOIN_ITER *iter) { if (iter->positioned) { WT_RET(iter->cursor->reset(iter->cursor)); + WT_RET(iter->full->reset(iter->full)); WT_RET(__wt_cursor_dup_position( iter->cjoin->entries[0].ends[0].cursor, iter->cursor)); iter->positioned = false; @@ -175,6 +213,8 @@ __curjoin_entry_iter_close(WT_CURSOR_JOIN_ITER *iter) if (iter->cursor != NULL) WT_TRET(iter->cursor->close(iter->cursor)); + if (iter->full != NULL) + WT_TRET(iter->full->close(iter->full)); __wt_free(iter->session, iter); return (ret); @@ -230,10 +270,11 @@ __curjoin_get_value(WT_CURSOR *cursor, ...) !__curjoin_entry_iter_ready(iter)) WT_ERR_MSG(session, EINVAL, "join cursor must be advanced with next()"); + if (iter->entry->index != NULL) - WT_ERR(__wt_curindex_get_valuev(iter->cursor, ap)); + WT_ERR(__wt_curindex_get_valuev(iter->full, ap)); else - WT_ERR(__wt_curtable_get_valuev(iter->cursor, ap)); + WT_ERR(__wt_curtable_get_valuev(iter->full, ap)); err: va_end(ap); API_END_RET(session, ret); @@ -249,41 +290,26 @@ __curjoin_init_bloom(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, { WT_COLLATOR *collator; WT_CURSOR *c; - WT_CURSOR_INDEX *cindex; WT_CURSOR_JOIN_ENDPOINT *end, *endmax; WT_DECL_RET; WT_DECL_ITEM(uribuf); WT_ITEM curkey, curvalue; - WT_TABLE *maintable; const char *raw_cfg[] = { WT_CONFIG_BASE( session, WT_SESSION_open_cursor), "raw", NULL }; - const char *mainkey_str, *p; - size_t mainkey_len, size; - u_int i; + const char *uri; + size_t size; int cmp, skip; c = NULL; skip = 0; - if (entry->index != NULL) { + if (entry->index != NULL) /* - * Open a cursor having a projection of the keys of the - * index we're comparing against. Open it raw, we're - * going to compare it to the raw keys of the - * reference cursors. + * Open the raw index. We're avoiding any references + * to the main table, they may be expensive. */ - maintable = ((WT_CURSOR_TABLE *)entry->main)->table; - mainkey_str = maintable->colconf.str + 1; - for (p = mainkey_str, i = 0; - p != NULL && i < maintable->nkey_columns; i++) - p = strchr(p + 1, ','); - WT_ASSERT(session, p != 0); - mainkey_len = WT_PTRDIFF(p, mainkey_str); - size = strlen(entry->index->name) + mainkey_len + 3; - WT_ERR(__wt_scr_alloc(session, size, &uribuf)); - WT_ERR(__wt_buf_fmt(session, uribuf, "%s(%.*s)", - entry->index->name, (int)mainkey_len, mainkey_str)); - } else { + uri = entry->index->source; + else { /* * For joins on the main table, we just need the primary * key for comparison, we don't need any values. @@ -292,9 +318,9 @@ __curjoin_init_bloom(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, WT_ERR(__wt_scr_alloc(session, size, &uribuf)); WT_ERR(__wt_buf_fmt(session, uribuf, "%s()", cjoin->table->name)); + uri = uribuf->data; } - WT_ERR(__wt_open_cursor( - session, uribuf->data, &cjoin->iface, raw_cfg, &c)); + WT_ERR(__wt_open_cursor(session, uri, &cjoin->iface, raw_cfg, &c)); /* Initially position the cursor if necessary. */ endmax = &entry->ends[entry->ends_next]; @@ -308,22 +334,23 @@ __curjoin_init_bloom(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, if ((ret = c->next(c)) == WT_NOTFOUND) goto done; WT_ERR(ret); - } + } else + /* There should not be any other cases */ + WT_ASSERT(session, false); } collator = (entry->index == NULL) ? NULL : entry->index->collator; while (ret == 0) { WT_ERR(c->get_key(c, &curkey)); if (entry->index != NULL) { - cindex = (WT_CURSOR_INDEX *)c; /* * Repack so it's comparable to the * reference endpoints. */ WT_ERR(__wt_struct_repack(session, - cindex->child->key_format, + c->key_format, (entry->repack_format != NULL ? - entry->repack_format : cindex->iface.key_format), - &cindex->child->key, &curkey)); + entry->repack_format : entry->index->idxkey_format), + &c->key, &curkey)); } for (end = &entry->ends[skip]; end < endmax; end++) { WT_ERR(__wt_compare(session, collator, &curkey, @@ -344,8 +371,12 @@ __curjoin_init_bloom(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, goto done; } } - if (entry->index != NULL) - WT_ERR(c->get_value(c, &curvalue)); + if (entry->index != NULL) { + curvalue.data = + (unsigned char *)curkey.data + curkey.size; + WT_ASSERT(session, c->key.size > curkey.size); + curvalue.size = c->key.size - curkey.size; + } else WT_ERR(c->get_key(c, &curvalue)); WT_ERR(__wt_bloom_insert(bloom, &curvalue)); @@ -640,6 +671,8 @@ __curjoin_entry_member(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, WT_ITEM *key, v; bool bloom_found; + if (skip_left && entry->ends_next == 1) + return (0); /* no checks to make */ key = cjoin->iter->curkey; entry->stats.accesses++; bloom_found = false; @@ -719,9 +752,10 @@ __curjoin_next(WT_CURSOR *cursor) if (!F_ISSET(cjoin, WT_CURJOIN_INITIALIZED)) WT_ERR(__curjoin_init_iter(session, cjoin)); + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + nextkey: - if ((ret = __curjoin_entry_iter_next(cjoin->iter, &cursor->key, - &cursor->recno)) == 0) { + if ((ret = __curjoin_entry_iter_next(cjoin->iter, cursor)) == 0) { F_SET(cursor, WT_CURSTD_KEY_EXT); /* @@ -748,6 +782,12 @@ nextkey: } else if (ret != WT_NOTFOUND) WT_ERR(ret); + if (ret == 0) { + WT_ERR(__wt_cursor_dup_position(cjoin->iter->cursor, + cjoin->iter->full)); + F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); + } + if (0) { err: F_SET(cjoin, WT_CURJOIN_ERROR); } diff --git a/src/include/cursor.h b/src/include/cursor.h index 2665ff83df3..aba9d2bb1a0 100644 --- a/src/include/cursor.h +++ b/src/include/cursor.h @@ -287,7 +287,8 @@ struct __wt_cursor_join_iter { WT_SESSION_IMPL *session; WT_CURSOR_JOIN *cjoin; WT_CURSOR_JOIN_ENTRY *entry; - WT_CURSOR *cursor; + WT_CURSOR *cursor; /* has null projection */ + WT_CURSOR *full; /* has requested projection */ WT_ITEM *curkey; bool positioned; bool isequal; /* advancing means we're done */ -- cgit v1.2.1 From 97e86ed17a93f3c2a0a43007b4e7127b1040bada Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Mon, 14 Mar 2016 11:07:52 -0400 Subject: WT-2463 Test validity via WT stats instead of system CPU usage. --- test/manydbs/manydbs.c | 125 +++++++++++++++++++++++++------------------------ test/manydbs/smoke.sh | 2 +- 2 files changed, 66 insertions(+), 61 deletions(-) diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index 897f57d4b11..e35ac72d231 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -60,18 +60,18 @@ static const char * const uri = "table:main"; WTOPEN_CFG_COMMON \ "transaction_sync=(enabled,method=fsync)" -#define MAX_DBS 10 +#define MAX_DBS 10 #define MAX_IDLE_TIME 30 #define IDLE_INCR 5 -#define MAX_KV 100 -#define MAX_VAL 128 +#define MAX_KV 100 +#define MAX_VAL 128 static void usage(void) { fprintf(stderr, - "usage: %s [-I] [-C maxcpu%%] [-D maxdbs] [-h dir]\n", progname); + "usage: %s [-I] [-D maxdbs] [-h dir]\n", progname); exit(EXIT_FAILURE); } @@ -82,9 +82,23 @@ void (*custom_die)(void) = NULL; WT_CONNECTION **conn = NULL; WT_CURSOR **cursor = NULL; +WT_CURSOR **statc = NULL; WT_RAND_STATE rnd; WT_SESSION **session = NULL; +static int +get_stat(WT_CURSOR *cursor, int stat_field, uint64_t *valuep) +{ + const char *desc, *pvalue; + int ret; + + cursor->set_key(cursor, stat_field); + if ((ret = cursor->search(cursor)) != 0) + return (ret); + + return (cursor->get_value(cursor, &desc, &pvalue, valuep)); +} + static int run_ops(int dbs) { @@ -94,26 +108,6 @@ run_ops(int dbs) uint8_t buf[MAX_VAL]; memset(buf, 0, sizeof(buf)); - /* - * First time through, set up sessions, create the tables and - * open cursors. - */ - if (session == NULL) { - __wt_random_init(&rnd); - if ((session = - calloc((size_t)dbs, sizeof(WT_SESSION *))) == NULL) - testutil_die(ENOMEM, "session array malloc"); - if ((cursor = calloc((size_t)dbs, sizeof(WT_CURSOR *))) == NULL) - testutil_die(ENOMEM, "cursor array malloc"); - for (i = 0; i < dbs; ++i) { - testutil_check(conn[i]->open_session(conn[i], - NULL, NULL, &session[i])); - testutil_check(session[i]->create(session[i], - uri, "key_format=Q,value_format=u")); - testutil_check(session[i]->open_cursor(session[i], - uri, NULL, NULL, &cursor[i])); - } - } for (i = 0; i < MAX_VAL; ++i) buf[i] = (uint8_t)__wt_random(&rnd); data.data = buf; @@ -137,12 +131,10 @@ run_ops(int dbs) int main(int argc, char *argv[]) { - FILE *fp; - float cpu, max; + uint64_t cond_reset, cond_wait; int cfg, ch, dbs, i; - const char *working_dir; - bool idle, setmax; - const char *wt_cfg; + bool idle; + const char *working_dir, *wt_cfg; char cmd[128]; if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) @@ -151,14 +143,9 @@ main(int argc, char *argv[]) ++progname; dbs = MAX_DBS; working_dir = HOME_BASE; - max = (float)dbs; - idle = setmax = false; - while ((ch = __wt_getopt(progname, argc, argv, "C:D:h:I")) != EOF) + idle = false; + while ((ch = __wt_getopt(progname, argc, argv, "D:h:I")) != EOF) switch (ch) { - case 'C': - max = (float)atof(__wt_optarg); - setmax = true; - break; case 'D': dbs = atoi(__wt_optarg); break; @@ -173,29 +160,37 @@ main(int argc, char *argv[]) } argc -= __wt_optind; argv += __wt_optind; - /* - * Adjust the maxcpu in relation to the number of databases, unless - * the user set it explicitly. - */ - if (!setmax) - max = (float)dbs; if (argc != 0) usage(); + /* + * Allocate arrays for connection handles, sessions, statistics + * cursors and, if needed, data cursors. + */ if ((conn = calloc((size_t)dbs, sizeof(WT_CONNECTION *))) == NULL) testutil_die(ENOMEM, "connection array malloc"); + if ((session = calloc( + (size_t)dbs, sizeof(WT_SESSION *))) == NULL) + testutil_die(ENOMEM, "session array malloc"); + if ((statc = calloc((size_t)dbs, sizeof(WT_CURSOR *))) == NULL) + testutil_die(ENOMEM, "cursor array malloc"); + if (!idle && ((cursor = calloc( + (size_t)dbs, sizeof(WT_CURSOR *))) == NULL)) + testutil_die(ENOMEM, "cursor array malloc"); memset(cmd, 0, sizeof(cmd)); /* * Set up all the directory names. */ testutil_work_dir_from_path(home, HOME_SIZE, working_dir); testutil_make_work_dir(home); + __wt_random_init(&rnd); for (i = 0; i < dbs; ++i) { snprintf(hometmp, HOME_SIZE, "%s/%s.%d", home, HOME_BASE, i); testutil_make_work_dir(hometmp); /* * Open each database. Rotate different configurations - * among them. + * among them. Open a session and statistics cursor. + * If writing data, create the table and open a data cursor. */ cfg = i % 3; if (cfg == 0) @@ -206,33 +201,43 @@ main(int argc, char *argv[]) wt_cfg = WT_CONFIG2; testutil_check(wiredtiger_open( hometmp, NULL, wt_cfg, &conn[i])); + testutil_check(conn[i]->open_session(conn[i], + NULL, NULL, &session[i])); + testutil_check(session[i]->open_cursor(session[i], + "statistics:", NULL, NULL, &statc[i])); + if (!idle) { + testutil_check(session[i]->create(session[i], + uri, "key_format=Q,value_format=u")); + testutil_check(session[i]->open_cursor(session[i], + uri, NULL, NULL, &cursor[i])); + } } sleep(10); + for (i = 0; i < MAX_IDLE_TIME; i += IDLE_INCR) { if (!idle) testutil_check(run_ops(dbs)); printf("Sleep %d (%d of %d)\n", IDLE_INCR, i, MAX_IDLE_TIME); sleep(IDLE_INCR); } - - /* - * Check CPU after all idling or work is done. - */ - (void)snprintf(cmd, sizeof(cmd), - "ps -p %lu -o pcpu=", (unsigned long)getpid()); - if ((fp = popen(cmd, "r")) == NULL) - testutil_die(errno, "popen"); - fscanf(fp, "%f", &cpu); - printf("Final CPU %f, max %f\n", cpu, max); - if (cpu > max) { - fprintf(stderr, "ERROR: CPU usage: %f, max %f\n", cpu, max); - testutil_die(ERANGE, "CPU"); - } - if (pclose(fp) != 0) - testutil_die(errno, "pclose"); - for (i = 0; i < dbs; ++i) + for (i = 0; i < dbs; ++i) { + testutil_check(get_stat(statc[i], + WT_STAT_CONN_COND_AUTO_WAIT_RESET, &cond_reset)); + testutil_check(get_stat(statc[i], + WT_STAT_CONN_COND_AUTO_WAIT, &cond_wait)); + /* + * On an idle workload there should be no resets of condition + * variables. Even with a light workload, resets should not be + * very common. We look for 5%. + */ + if (idle && cond_reset != 0) + testutil_die(ERANGE, + "condvar reset on idle connection"); + if (!idle && cond_reset > cond_wait / 20) + testutil_die(ERANGE, "condvar reset exceeds 5%"); testutil_check(conn[i]->close(conn[i], NULL)); + } return (EXIT_SUCCESS); } diff --git a/test/manydbs/smoke.sh b/test/manydbs/smoke.sh index 291c02bd1e6..5d8d1473a3a 100755 --- a/test/manydbs/smoke.sh +++ b/test/manydbs/smoke.sh @@ -13,7 +13,7 @@ set -e # 3. More dbs. # echo "manydbs: default with operations turned on" -$TEST_WRAPPER ./t -C 100 +$TEST_WRAPPER ./t #echo "manydbs: totally idle databases" #$TEST_WRAPPER ./t -I #echo "manydbs: 40 databases with operations" -- cgit v1.2.1 From 541480893952e25030820d518a4c9be921cb5cd3 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Mon, 14 Mar 2016 11:11:14 -0400 Subject: WT-2463 Reinstate smoke.sh tests. --- test/manydbs/smoke.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/manydbs/smoke.sh b/test/manydbs/smoke.sh index 5d8d1473a3a..b829a8b6994 100755 --- a/test/manydbs/smoke.sh +++ b/test/manydbs/smoke.sh @@ -14,9 +14,9 @@ set -e # echo "manydbs: default with operations turned on" $TEST_WRAPPER ./t -#echo "manydbs: totally idle databases" -#$TEST_WRAPPER ./t -I -#echo "manydbs: 40 databases with operations" -#$TEST_WRAPPER ./t -D 40 -#echo "manydbs: 40 idle databases" -#$TEST_WRAPPER ./t -I -D 40 +echo "manydbs: totally idle databases" +$TEST_WRAPPER ./t -I +echo "manydbs: 40 databases with operations" +$TEST_WRAPPER ./t -D 40 +echo "manydbs: 40 idle databases" +$TEST_WRAPPER ./t -I -D 40 -- cgit v1.2.1 From c238cb5fccec4a29d7759d4145fc90ab4eba7566 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Mon, 14 Mar 2016 11:29:52 -0400 Subject: WT-2463 Fix warning. --- test/manydbs/manydbs.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index e35ac72d231..60a04340736 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -87,16 +87,16 @@ WT_RAND_STATE rnd; WT_SESSION **session = NULL; static int -get_stat(WT_CURSOR *cursor, int stat_field, uint64_t *valuep) +get_stat(WT_CURSOR *statcurs, int stat_field, uint64_t *valuep) { const char *desc, *pvalue; int ret; - cursor->set_key(cursor, stat_field); - if ((ret = cursor->search(cursor)) != 0) + statcurs->set_key(statcurs, stat_field); + if ((ret = statcurs->search(statcurs)) != 0) return (ret); - return (cursor->get_value(cursor, &desc, &pvalue, valuep)); + return (statcurs->get_value(statcurs, &desc, &pvalue, valuep)); } static int -- cgit v1.2.1 From be8c7dcb39c70b3b5afe5010a2a5afad017c81e0 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Mon, 14 Mar 2016 11:37:13 -0400 Subject: WT-2463 Remove outdated comment in script. --- test/manydbs/smoke.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/manydbs/smoke.sh b/test/manydbs/smoke.sh index b829a8b6994..c0e2976f154 100755 --- a/test/manydbs/smoke.sh +++ b/test/manydbs/smoke.sh @@ -2,10 +2,6 @@ set -e -# XXX Disable more complex tests, and bump the CPU usage check up to 100 for -# now, since running under valgrind consumes a lot more CPU. A different -# measuring mechanism would be ideal. - # Smoke-test format as part of running "make check". # Run with: # 1. The defaults -- cgit v1.2.1 From 5c93381d6065054642daef03b1e0a86eeb5573fd Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Mon, 14 Mar 2016 12:10:20 -0400 Subject: WT-2463 Record original number of resets after initial period. --- test/manydbs/manydbs.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index 60a04340736..60881694d6a 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -132,6 +132,7 @@ int main(int argc, char *argv[]) { uint64_t cond_reset, cond_wait; + uint64_t *cond_reset_orig; int cfg, ch, dbs, i; bool idle; const char *working_dir, *wt_cfg; @@ -174,6 +175,8 @@ main(int argc, char *argv[]) testutil_die(ENOMEM, "session array malloc"); if ((statc = calloc((size_t)dbs, sizeof(WT_CURSOR *))) == NULL) testutil_die(ENOMEM, "cursor array malloc"); + if ((cond_reset_orig = calloc((size_t)dbs, sizeof(uint64_t))) == NULL) + testutil_die(ENOMEM, "orig stat malloc"); if (!idle && ((cursor = calloc( (size_t)dbs, sizeof(WT_CURSOR *))) == NULL)) testutil_die(ENOMEM, "cursor array malloc"); @@ -215,6 +218,13 @@ main(int argc, char *argv[]) sleep(10); + /* + * Record original reset setting. There could have been some + * activity during the creation period. + */ + for (i = 0; i < dbs; ++i) + testutil_check(get_stat(statc[i], + WT_STAT_CONN_COND_AUTO_WAIT_RESET, &cond_reset_orig[i])); for (i = 0; i < MAX_IDLE_TIME; i += IDLE_INCR) { if (!idle) testutil_check(run_ops(dbs)); @@ -228,10 +238,10 @@ main(int argc, char *argv[]) WT_STAT_CONN_COND_AUTO_WAIT, &cond_wait)); /* * On an idle workload there should be no resets of condition - * variables. Even with a light workload, resets should not be - * very common. We look for 5%. + * variables during the idle period. Even with a light + * workload, resets should not be very common. We look for 5%. */ - if (idle && cond_reset != 0) + if (idle && cond_reset != cond_reset_orig[i]) testutil_die(ERANGE, "condvar reset on idle connection"); if (!idle && cond_reset > cond_wait / 20) -- cgit v1.2.1 From f705399db7cd35413fe3940b3e21d6e34bdaffb3 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 14 Mar 2016 12:47:58 -0400 Subject: WT-2482: Coverity 1353015, 1353016, out-of-bounds access Coverity thinks we can overflow a buffer -- we can't, but clarify the code to avoid the false positive. --- src/utilities/util_dump.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utilities/util_dump.c b/src/utilities/util_dump.c index 015fd9830a0..aedd9168fbd 100644 --- a/src/utilities/util_dump.c +++ b/src/utilities/util_dump.c @@ -390,7 +390,7 @@ dump_json_table_config(WT_SESSION *session, const char *uri) * file. */ if (WT_PREFIX_MATCH(uri, "table:")) { - len = strlen(uri) + strlen("colgroup:"); + len = strlen("colgroup:") + strlen(name) + 1; if ((p = malloc(len)) == NULL) return (util_err(session, errno, NULL)); (void)snprintf(p, len, "colgroup:%s", name); @@ -495,7 +495,7 @@ dump_table_config(WT_SESSION *session, WT_CURSOR *cursor, const char *uri) */ complex_table = false; if (WT_PREFIX_MATCH(uri, "table:")) { - len = strlen(uri) + strlen("colgroup:"); + len = strlen("colgroup:") + strlen(name) + 1; if ((p = malloc(len)) == NULL) return (util_err(session, errno, NULL)); (void)snprintf(p, len, "colgroup:%s", name); -- cgit v1.2.1 From 99e2a663aa0c2d4fc7cd1055cde140ce0a06169c Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 14 Mar 2016 12:54:21 -0400 Subject: WT-2484: Coverity 1345809: unchecked return value Explicitly discard the sscanf return value, hopefully that will quiet Coverity. --- bench/wtperf/wtperf.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/wtperf/wtperf.h b/bench/wtperf/wtperf.h index 60efce650aa..a2b497b3142 100644 --- a/bench/wtperf/wtperf.h +++ b/bench/wtperf/wtperf.h @@ -337,7 +337,7 @@ generate_key(CONFIG *cfg, char *key_buf, uint64_t keyno) static inline void extract_key(char *key_buf, uint64_t *keynop) { - sscanf(key_buf, "%" SCNu64, keynop); + (void)sscanf(key_buf, "%" SCNu64, keynop); } /* -- cgit v1.2.1 From 2ca6076281100a779234f9a6e03fda721795d812 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Mon, 14 Mar 2016 12:54:24 -0400 Subject: WT-2483 Skip temp log files when copying WT directory. --- test/suite/helper.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/suite/helper.py b/test/suite/helper.py index 3895402a0d6..f85d708880f 100644 --- a/test/suite/helper.py +++ b/test/suite/helper.py @@ -107,7 +107,10 @@ def copy_wiredtiger_home(olddir, newdir, aligned=True): for fname in os.listdir(olddir): fullname = os.path.join(olddir, fname) # Skip lock file, on Windows it is locked. - if os.path.isfile(fullname) and "WiredTiger.lock" not in fullname: + # Skip temporary log files. + if os.path.isfile(fullname) and "WiredTiger.lock" not in fullname and \ + "WiredTigerTmplog" not in fullname and \ + "WiredTigerPreplog" not in fullname: # Use a dd command that does not align on a block boundary. if aligned: shutil.copy(fullname, newdir) -- cgit v1.2.1 From 02e26ac5fea1f8ca7c3f3942f8fc77008f8f0dec Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Mon, 14 Mar 2016 14:13:49 -0400 Subject: WT-2447. A performance gain and simplification: use a cursor based on the main table instead of the index table to retrieve the value needed for the join cursor, since the iteration already has the primary key. --- src/cursor/cur_join.c | 54 ++++++++++++++++++++++++++++----------------------- src/include/cursor.h | 2 +- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/src/cursor/cur_join.c b/src/cursor/cur_join.c index 62aef27e3d1..a72f08fd9d3 100644 --- a/src/cursor/cur_join.c +++ b/src/cursor/cur_join.c @@ -26,13 +26,13 @@ __curjoin_entry_iter_init(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, session, WT_SESSION_open_cursor), "raw", NULL }; const char *def_cfg[] = { WT_CONFIG_BASE( session, WT_SESSION_open_cursor), NULL }; - const char *urifull, **config; - char *fullbuf, *urimin; + const char *urimain, **config; + char *mainbuf, *urimin; WT_CURSOR_JOIN_ITER *iter; size_t size; iter = NULL; - fullbuf = urimin = NULL; + mainbuf = urimin = NULL; to_dup = entry->ends[0].cursor; if (F_ISSET((WT_CURSOR *)cjoin, WT_CURSTD_RAW)) @@ -40,23 +40,23 @@ __curjoin_entry_iter_init(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, else config = &def_cfg[0]; - urifull = to_dup->uri; - size = strlen(urifull) + 3; + urimain = cjoin->table->name; + size = strlen(to_dup->uri) + 3; WT_ERR(__wt_calloc(session, size, 1, &urimin)); - snprintf(urimin, size, "%s()", urifull); + snprintf(urimin, size, "%s()", to_dup->uri); if (cjoin->projection != NULL) { - size = strlen(urifull) + strlen(cjoin->projection) + 1; - WT_ERR(__wt_calloc(session, size, 1, &fullbuf)); - snprintf(fullbuf, size, "%s%s", urifull, cjoin->projection); - urifull = fullbuf; + size = strlen(urimain) + strlen(cjoin->projection) + 1; + WT_ERR(__wt_calloc(session, size, 1, &mainbuf)); + snprintf(mainbuf, size, "%s%s", urimain, cjoin->projection); + urimain = mainbuf; } WT_ERR(__wt_calloc_one(session, &iter)); WT_ERR(__wt_open_cursor(session, urimin, (WT_CURSOR *)cjoin, config, &iter->cursor)); WT_ERR(__wt_cursor_dup_position(to_dup, iter->cursor)); - WT_ERR(__wt_open_cursor(session, urifull, (WT_CURSOR *)cjoin, config, - &iter->full)); + WT_ERR(__wt_open_cursor(session, urimain, (WT_CURSOR *)cjoin, config, + &iter->main)); iter->cjoin = cjoin; iter->session = session; iter->entry = entry; @@ -68,7 +68,7 @@ __curjoin_entry_iter_init(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, if (0) { err: __wt_free(session, iter); } - __wt_free(session, fullbuf); + __wt_free(session, mainbuf); __wt_free(session, urimin); return (ret); } @@ -181,7 +181,7 @@ __curjoin_entry_iter_reset(WT_CURSOR_JOIN_ITER *iter) { if (iter->positioned) { WT_RET(iter->cursor->reset(iter->cursor)); - WT_RET(iter->full->reset(iter->full)); + WT_RET(iter->main->reset(iter->main)); WT_RET(__wt_cursor_dup_position( iter->cjoin->entries[0].ends[0].cursor, iter->cursor)); iter->positioned = false; @@ -213,8 +213,8 @@ __curjoin_entry_iter_close(WT_CURSOR_JOIN_ITER *iter) if (iter->cursor != NULL) WT_TRET(iter->cursor->close(iter->cursor)); - if (iter->full != NULL) - WT_TRET(iter->full->close(iter->full)); + if (iter->main != NULL) + WT_TRET(iter->main->close(iter->main)); __wt_free(iter->session, iter); return (ret); @@ -271,10 +271,7 @@ __curjoin_get_value(WT_CURSOR *cursor, ...) WT_ERR_MSG(session, EINVAL, "join cursor must be advanced with next()"); - if (iter->entry->index != NULL) - WT_ERR(__wt_curindex_get_valuev(iter->full, ap)); - else - WT_ERR(__wt_curtable_get_valuev(iter->full, ap)); + WT_ERR(__wt_curtable_get_valuev(iter->main, ap)); err: va_end(ap); API_END_RET(session, ret); @@ -736,7 +733,9 @@ err: if (ret == WT_NOTFOUND && bloom_found) static int __curjoin_next(WT_CURSOR *cursor) { + WT_CURSOR *c; WT_CURSOR_JOIN *cjoin; + WT_CURSOR_JOIN_ITER *iter; WT_DECL_RET; WT_SESSION_IMPL *session; bool skip_left; @@ -753,9 +752,10 @@ __curjoin_next(WT_CURSOR *cursor) WT_ERR(__curjoin_init_iter(session, cjoin)); F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + iter = cjoin->iter; nextkey: - if ((ret = __curjoin_entry_iter_next(cjoin->iter, cursor)) == 0) { + if ((ret = __curjoin_entry_iter_next(iter, cursor)) == 0) { F_SET(cursor, WT_CURSTD_KEY_EXT); /* @@ -772,7 +772,7 @@ nextkey: * If this is compare=eq on our outer iterator, * and we've moved past it, we're done. */ - if (cjoin->iter->isequal && i == 0) + if (iter->isequal && i == 0) break; goto nextkey; } @@ -783,8 +783,14 @@ nextkey: WT_ERR(ret); if (ret == 0) { - WT_ERR(__wt_cursor_dup_position(cjoin->iter->cursor, - cjoin->iter->full)); + /* + * Position the 'main' cursor, this will be used to + * retrieve values from the cursor join. + */ + c = iter->main; + c->set_key(c, iter->curkey); + if ((ret = c->search(c)) != 0) + WT_ERR(c->search(c)); F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); } diff --git a/src/include/cursor.h b/src/include/cursor.h index aba9d2bb1a0..a2110465355 100644 --- a/src/include/cursor.h +++ b/src/include/cursor.h @@ -288,7 +288,7 @@ struct __wt_cursor_join_iter { WT_CURSOR_JOIN *cjoin; WT_CURSOR_JOIN_ENTRY *entry; WT_CURSOR *cursor; /* has null projection */ - WT_CURSOR *full; /* has requested projection */ + WT_CURSOR *main; /* main table with projection */ WT_ITEM *curkey; bool positioned; bool isequal; /* advancing means we're done */ -- cgit v1.2.1 From c7deca02e486d7c2ac3d8044b038997f20595d69 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Mon, 14 Mar 2016 14:21:30 -0400 Subject: WT-2463 Add more information to error message. --- test/manydbs/manydbs.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index 60881694d6a..1d7ee798f2f 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -243,9 +243,12 @@ main(int argc, char *argv[]) */ if (idle && cond_reset != cond_reset_orig[i]) testutil_die(ERANGE, - "condvar reset on idle connection"); + "condvar reset on idle connection %d of %" PRIu64, + i, cond_reset); if (!idle && cond_reset > cond_wait / 20) - testutil_die(ERANGE, "condvar reset exceeds 5%"); + testutil_die(ERANGE, "connection %d condvar reset %" + PRIu64 " exceeds 5%% of %" PRIu64, + i, cond_reset, cond_wait); testutil_check(conn[i]->close(conn[i], NULL)); } -- cgit v1.2.1 From e52c6e6ad35338fc55a78ebfbe142c9568565960 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Mon, 14 Mar 2016 15:32:39 -0400 Subject: WT-2463 Open/close stat cursor every time we want stats. --- test/manydbs/manydbs.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index 1d7ee798f2f..e71ebd936f0 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -82,21 +82,25 @@ void (*custom_die)(void) = NULL; WT_CONNECTION **conn = NULL; WT_CURSOR **cursor = NULL; -WT_CURSOR **statc = NULL; WT_RAND_STATE rnd; WT_SESSION **session = NULL; static int -get_stat(WT_CURSOR *statcurs, int stat_field, uint64_t *valuep) +get_stat(WT_SESSION *session, int stat_field, uint64_t *valuep) { + WT_CURSOR *statc; const char *desc, *pvalue; int ret; - statcurs->set_key(statcurs, stat_field); - if ((ret = statcurs->search(statcurs)) != 0) + testutil_check(session->open_cursor(session, + "statistics:", NULL, NULL, &statc)); + statc->set_key(statc, stat_field); + if ((ret = statc->search(statc)) != 0) return (ret); - return (statcurs->get_value(statcurs, &desc, &pvalue, valuep)); + ret = statc->get_value(statc, &desc, &pvalue, valuep); + statc->close(statc); + return (ret); } static int @@ -173,8 +177,6 @@ main(int argc, char *argv[]) if ((session = calloc( (size_t)dbs, sizeof(WT_SESSION *))) == NULL) testutil_die(ENOMEM, "session array malloc"); - if ((statc = calloc((size_t)dbs, sizeof(WT_CURSOR *))) == NULL) - testutil_die(ENOMEM, "cursor array malloc"); if ((cond_reset_orig = calloc((size_t)dbs, sizeof(uint64_t))) == NULL) testutil_die(ENOMEM, "orig stat malloc"); if (!idle && ((cursor = calloc( @@ -206,8 +208,6 @@ main(int argc, char *argv[]) hometmp, NULL, wt_cfg, &conn[i])); testutil_check(conn[i]->open_session(conn[i], NULL, NULL, &session[i])); - testutil_check(session[i]->open_cursor(session[i], - "statistics:", NULL, NULL, &statc[i])); if (!idle) { testutil_check(session[i]->create(session[i], uri, "key_format=Q,value_format=u")); @@ -223,7 +223,7 @@ main(int argc, char *argv[]) * activity during the creation period. */ for (i = 0; i < dbs; ++i) - testutil_check(get_stat(statc[i], + testutil_check(get_stat(session[i], WT_STAT_CONN_COND_AUTO_WAIT_RESET, &cond_reset_orig[i])); for (i = 0; i < MAX_IDLE_TIME; i += IDLE_INCR) { if (!idle) @@ -232,9 +232,9 @@ main(int argc, char *argv[]) sleep(IDLE_INCR); } for (i = 0; i < dbs; ++i) { - testutil_check(get_stat(statc[i], + testutil_check(get_stat(session[i], WT_STAT_CONN_COND_AUTO_WAIT_RESET, &cond_reset)); - testutil_check(get_stat(statc[i], + testutil_check(get_stat(session[i], WT_STAT_CONN_COND_AUTO_WAIT, &cond_wait)); /* * On an idle workload there should be no resets of condition -- cgit v1.2.1 From 4e9c4ccfd5280e0b7c0e1a31aa4dd84b5b8d41c2 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Mon, 14 Mar 2016 15:37:06 -0400 Subject: WT-2463 Fix warning. --- test/manydbs/manydbs.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index e71ebd936f0..1472fd9de17 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -86,13 +86,13 @@ WT_RAND_STATE rnd; WT_SESSION **session = NULL; static int -get_stat(WT_SESSION *session, int stat_field, uint64_t *valuep) +get_stat(WT_SESSION *stat_session, int stat_field, uint64_t *valuep) { WT_CURSOR *statc; const char *desc, *pvalue; int ret; - testutil_check(session->open_cursor(session, + testutil_check(stat_session->open_cursor(stat_session, "statistics:", NULL, NULL, &statc)); statc->set_key(statc, stat_field); if ((ret = statc->search(statc)) != 0) -- cgit v1.2.1 From 71ddb40def71f5aa9134a9a74e476220441113ca Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Mon, 14 Mar 2016 15:54:18 -0400 Subject: WT-2463 KNF --- test/manydbs/manydbs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index 1472fd9de17..cfb4b7f5081 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -60,12 +60,12 @@ static const char * const uri = "table:main"; WTOPEN_CFG_COMMON \ "transaction_sync=(enabled,method=fsync)" -#define MAX_DBS 10 +#define MAX_DBS 10 #define MAX_IDLE_TIME 30 #define IDLE_INCR 5 -#define MAX_KV 100 -#define MAX_VAL 128 +#define MAX_KV 100 +#define MAX_VAL 128 static void usage(void) -- cgit v1.2.1 From f4f14ff8bbef932167890b600d8b07cf055cec82 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Tue, 15 Mar 2016 15:58:38 +1100 Subject: WT-2487 Release memory at the end of the test to make leak checkers happy. --- test/manydbs/manydbs.c | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index cfb4b7f5081..9191ec728bd 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -80,10 +80,10 @@ extern char *__wt_optarg; void (*custom_die)(void) = NULL; -WT_CONNECTION **conn = NULL; -WT_CURSOR **cursor = NULL; +WT_CONNECTION **connections = NULL; +WT_CURSOR **cursors = NULL; WT_RAND_STATE rnd; -WT_SESSION **session = NULL; +WT_SESSION **sessions = NULL; static int get_stat(WT_SESSION *stat_session, int stat_field, uint64_t *valuep) @@ -124,9 +124,9 @@ run_ops(int dbs) printf("Write to database %" PRIu32 "\n", db); for (key = 0; key < MAX_KV; ++key) { data.size = __wt_random(&rnd) % MAX_VAL; - cursor[db]->set_key(cursor[db], key); - cursor[db]->set_value(cursor[db], &data); - testutil_check(cursor[db]->insert(cursor[db])); + cursors[db]->set_key(cursors[db], key); + cursors[db]->set_value(cursors[db], &data); + testutil_check(cursors[db]->insert(cursors[db])); } } return (0); @@ -172,14 +172,14 @@ main(int argc, char *argv[]) * Allocate arrays for connection handles, sessions, statistics * cursors and, if needed, data cursors. */ - if ((conn = calloc((size_t)dbs, sizeof(WT_CONNECTION *))) == NULL) + if ((connections = calloc((size_t)dbs, sizeof(WT_CONNECTION *))) == NULL) testutil_die(ENOMEM, "connection array malloc"); - if ((session = calloc( + if ((sessions = calloc( (size_t)dbs, sizeof(WT_SESSION *))) == NULL) testutil_die(ENOMEM, "session array malloc"); if ((cond_reset_orig = calloc((size_t)dbs, sizeof(uint64_t))) == NULL) testutil_die(ENOMEM, "orig stat malloc"); - if (!idle && ((cursor = calloc( + if (!idle && ((cursors = calloc( (size_t)dbs, sizeof(WT_CURSOR *))) == NULL)) testutil_die(ENOMEM, "cursor array malloc"); memset(cmd, 0, sizeof(cmd)); @@ -205,14 +205,14 @@ main(int argc, char *argv[]) else wt_cfg = WT_CONFIG2; testutil_check(wiredtiger_open( - hometmp, NULL, wt_cfg, &conn[i])); - testutil_check(conn[i]->open_session(conn[i], - NULL, NULL, &session[i])); + hometmp, NULL, wt_cfg, &connections[i])); + testutil_check(connections[i]->open_session(connections[i], + NULL, NULL, &sessions[i])); if (!idle) { - testutil_check(session[i]->create(session[i], + testutil_check(sessions[i]->create(sessions[i], uri, "key_format=Q,value_format=u")); - testutil_check(session[i]->open_cursor(session[i], - uri, NULL, NULL, &cursor[i])); + testutil_check(sessions[i]->open_cursor(sessions[i], + uri, NULL, NULL, &cursors[i])); } } @@ -223,7 +223,7 @@ main(int argc, char *argv[]) * activity during the creation period. */ for (i = 0; i < dbs; ++i) - testutil_check(get_stat(session[i], + testutil_check(get_stat(sessions[i], WT_STAT_CONN_COND_AUTO_WAIT_RESET, &cond_reset_orig[i])); for (i = 0; i < MAX_IDLE_TIME; i += IDLE_INCR) { if (!idle) @@ -232,9 +232,9 @@ main(int argc, char *argv[]) sleep(IDLE_INCR); } for (i = 0; i < dbs; ++i) { - testutil_check(get_stat(session[i], + testutil_check(get_stat(sessions[i], WT_STAT_CONN_COND_AUTO_WAIT_RESET, &cond_reset)); - testutil_check(get_stat(session[i], + testutil_check(get_stat(sessions[i], WT_STAT_CONN_COND_AUTO_WAIT, &cond_wait)); /* * On an idle workload there should be no resets of condition @@ -249,8 +249,15 @@ main(int argc, char *argv[]) testutil_die(ERANGE, "connection %d condvar reset %" PRIu64 " exceeds 5%% of %" PRIu64, i, cond_reset, cond_wait); - testutil_check(conn[i]->close(conn[i], NULL)); + testutil_check(connections[i]->close(connections[i], NULL)); } + /* Cleanup allocated memory. */ + free(connections); + free(sessions); + free(cond_reset_orig); + if (!idle) + free(cursors); + return (EXIT_SUCCESS); } -- cgit v1.2.1 From 0675443be5d8a4a2b2c90f9f017a0e046cc0c80b Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Tue, 15 Mar 2016 16:13:05 +1100 Subject: Whitespace. --- test/manydbs/manydbs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index 9191ec728bd..7cadfb0a11b 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -172,7 +172,8 @@ main(int argc, char *argv[]) * Allocate arrays for connection handles, sessions, statistics * cursors and, if needed, data cursors. */ - if ((connections = calloc((size_t)dbs, sizeof(WT_CONNECTION *))) == NULL) + if ((connections = calloc( + (size_t)dbs, sizeof(WT_CONNECTION *))) == NULL) testutil_die(ENOMEM, "connection array malloc"); if ((sessions = calloc( (size_t)dbs, sizeof(WT_SESSION *))) == NULL) -- cgit v1.2.1 From 0d2fad0dcfaa6ac07b91ebc7bb414ba9a720d702 Mon Sep 17 00:00:00 2001 From: David Hows Date: Tue, 15 Mar 2016 16:35:48 +1100 Subject: WT-2476 - Revert changing the cache evict_walk_lock to the btree evict_lock --- src/btree/bt_handle.c | 2 -- src/conn/conn_cache.c | 2 ++ src/evict/evict_lru.c | 16 +++++++++------- src/include/btree.h | 1 - src/include/cache.h | 1 + 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/btree/bt_handle.c b/src/btree/bt_handle.c index bf888bfa65f..1d33a7e7c9a 100644 --- a/src/btree/bt_handle.c +++ b/src/btree/bt_handle.c @@ -164,7 +164,6 @@ __wt_btree_close(WT_SESSION_IMPL *session) /* Destroy locks. */ WT_TRET(__wt_rwlock_destroy(session, &btree->ovfl_lock)); - __wt_spin_destroy(session, &btree->evict_lock); __wt_spin_destroy(session, &btree->flush_lock); /* Free allocated memory. */ @@ -351,7 +350,6 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) /* Initialize locks. */ WT_RET(__wt_rwlock_alloc( session, &btree->ovfl_lock, "btree overflow lock")); - WT_RET(__wt_spin_init(session, &btree->evict_lock, "btree evict")); WT_RET(__wt_spin_init(session, &btree->flush_lock, "btree flush")); btree->checkpointing = WT_CKPT_OFF; /* Not checkpointing */ diff --git a/src/conn/conn_cache.c b/src/conn/conn_cache.c index 9850854a1de..9a2c394e9a6 100644 --- a/src/conn/conn_cache.c +++ b/src/conn/conn_cache.c @@ -158,6 +158,7 @@ __wt_cache_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR(__wt_cond_alloc(session, "eviction waiters", false, &cache->evict_waiter_cond)); WT_ERR(__wt_spin_init(session, &cache->evict_lock, "cache eviction")); + WT_ERR(__wt_spin_init(session, &cache->evict_walk_lock, "cache walk")); /* Allocate the LRU eviction queue. */ cache->evict_slots = WT_EVICT_WALK_BASE + WT_EVICT_WALK_INCR; @@ -254,6 +255,7 @@ __wt_cache_destroy(WT_SESSION_IMPL *session) WT_TRET(__wt_cond_auto_destroy(session, &cache->evict_cond)); WT_TRET(__wt_cond_destroy(session, &cache->evict_waiter_cond)); __wt_spin_destroy(session, &cache->evict_lock); + __wt_spin_destroy(session, &cache->evict_walk_lock); __wt_free(session, cache->evict_queue); __wt_free(session, conn->cache); diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index 855824930f8..50a00787f35 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -796,11 +796,11 @@ __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session) * The no-eviction flag can be set permanently, in which case we never * increment the no-eviction count. */ - __wt_spin_lock(session, &btree->evict_lock); + __wt_spin_lock(session, &cache->evict_walk_lock); if (F_ISSET(btree, WT_BTREE_NO_EVICTION)) { if (btree->evict_disabled != 0) ++btree->evict_disabled; - __wt_spin_unlock(session, &btree->evict_lock); + __wt_spin_unlock(session, &cache->evict_walk_lock); return (0); } ++btree->evict_disabled; @@ -837,7 +837,7 @@ __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session) err: --btree->evict_disabled; F_CLR(btree, WT_BTREE_NO_EVICTION); } - __wt_spin_unlock(session, &btree->evict_lock); + __wt_spin_unlock(session, &cache->evict_walk_lock); return (ret); } @@ -849,8 +849,10 @@ void __wt_evict_file_exclusive_off(WT_SESSION_IMPL *session) { WT_BTREE *btree; + WT_CACHE *cache; btree = S2BT(session); + cache = S2C(session)->cache; /* * We have seen subtle bugs with multiple threads racing to turn @@ -865,10 +867,10 @@ __wt_evict_file_exclusive_off(WT_SESSION_IMPL *session) * The no-eviction flag can be set permanently, in which case we never * increment the no-eviction count. */ - __wt_spin_lock(session, &btree->evict_lock); + __wt_spin_lock(session, &cache->evict_walk_lock); if (btree->evict_disabled > 0 && --btree->evict_disabled == 0) F_CLR(btree, WT_BTREE_NO_EVICTION); - __wt_spin_unlock(session, &btree->evict_lock); + __wt_spin_unlock(session, &cache->evict_walk_lock); } /* @@ -1174,14 +1176,14 @@ retry: while (slot < max_entries && ret == 0) { * waiting on this thread to acknowledge that action. */ if (!F_ISSET(btree, WT_BTREE_NO_EVICTION) && - !__wt_spin_trylock(session, &btree->evict_lock)) { + !__wt_spin_trylock(session, &cache->evict_walk_lock)) { if (!F_ISSET(btree, WT_BTREE_NO_EVICTION)) { cache->evict_file_next = dhandle; WT_WITH_DHANDLE(session, dhandle, ret = __evict_walk_file(session, &slot)); WT_ASSERT(session, session->split_gen == 0); } - __wt_spin_unlock(session, &btree->evict_lock); + __wt_spin_unlock(session, &cache->evict_walk_lock); } /* diff --git a/src/include/btree.h b/src/include/btree.h index 2d8a9ad4607..fd921677751 100644 --- a/src/include/btree.h +++ b/src/include/btree.h @@ -129,7 +129,6 @@ struct __wt_btree { uint64_t rec_max_txn; /* Maximum txn seen (clean trees) */ uint64_t write_gen; /* Write generation */ - WT_SPINLOCK evict_lock; /* Eviction lock */ WT_REF *evict_ref; /* Eviction thread's location */ uint64_t evict_priority; /* Relative priority of cached pages */ u_int evict_walk_period; /* Skip this many LRU walks */ diff --git a/src/include/cache.h b/src/include/cache.h index af819cf250a..9184a2fe6ed 100644 --- a/src/include/cache.h +++ b/src/include/cache.h @@ -84,6 +84,7 @@ struct __wt_cache { */ WT_CONDVAR *evict_cond; /* Eviction server condition */ WT_SPINLOCK evict_lock; /* Eviction LRU queue */ + WT_SPINLOCK evict_walk_lock; /* Eviction walk location */ /* Condition signalled when the eviction server populates the queue */ WT_CONDVAR *evict_waiter_cond; -- cgit v1.2.1 From ba37e960fd628f4313bdd30733a16453c3ae5c96 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Tue, 15 Mar 2016 18:02:46 +1100 Subject: WT-2457 Wait for LSM work units to drain when getting exclusive access. Otherwise we can return EBUSY unexpectedly. Fixed several outstanding bugs along the way. --- src/include/lsm.h | 2 +- src/lsm/lsm_tree.c | 53 ++++++++++++++++++++++++---------------------------- src/lsm/lsm_worker.c | 2 +- 3 files changed, 26 insertions(+), 31 deletions(-) diff --git a/src/include/lsm.h b/src/include/lsm.h index 5a3a63a03eb..7251b43d2bd 100644 --- a/src/include/lsm.h +++ b/src/include/lsm.h @@ -179,7 +179,7 @@ struct __wt_lsm_tree { int collator_owned; uint32_t refcnt; /* Number of users of the tree */ - uint8_t exclusive; /* Tree is locked exclusively */ + WT_SESSION_IMPL *exclusive; /* Tree is locked exclusively */ #define LSM_TREE_MAX_QUEUE 100 uint32_t queue_ref; diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index 45994bae42b..48648b99d2f 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -114,11 +114,14 @@ __lsm_tree_close(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) if (i % WT_THOUSAND == 0) { WT_WITHOUT_LOCKS(session, ret = __wt_lsm_manager_clear_tree(session, lsm_tree)); - WT_RET(ret); + WT_ERR(ret); } __wt_yield(); } return (0); + +err: F_SET(lsm_tree, WT_LSM_TREE_ACTIVE); + return (ret); } /* @@ -360,27 +363,25 @@ __lsm_tree_find(WT_SESSION_IMPL *session, /* See if the tree is already open. */ TAILQ_FOREACH(lsm_tree, &S2C(session)->lsmqh, q) if (strcmp(uri, lsm_tree->name) == 0) { - /* - * Short circuit if the handle is already held - * exclusively or exclusive access is requested and - * there are references held. - */ - if ((exclusive && lsm_tree->refcnt > 0) || - lsm_tree->exclusive) - return (EBUSY); - if (exclusive) { /* * Make sure we win the race to switch on the * exclusive flag. */ - if (!__wt_atomic_cas8( - &lsm_tree->exclusive, 0, 1)) + if (!__wt_atomic_cas_ptr( + &lsm_tree->exclusive, NULL, session)) return (EBUSY); - /* Make sure there are no readers */ - if (!__wt_atomic_cas32( + + /* + * Drain the work queue before checking for + * open cursors - otherwise we can generate + * spurious busy returns. + */ + if (__lsm_tree_close(session, lsm_tree) != 0 || + !__wt_atomic_cas32( &lsm_tree->refcnt, 0, 1)) { - lsm_tree->exclusive = 0; + F_SET(lsm_tree, WT_LSM_TREE_ACTIVE); + lsm_tree->exclusive = NULL; return (EBUSY); } } else { @@ -390,7 +391,7 @@ __lsm_tree_find(WT_SESSION_IMPL *session, * We got a reference, check if an exclusive * lock beat us to it. */ - if (lsm_tree->exclusive) { + if (lsm_tree->exclusive != NULL) { WT_ASSERT(session, lsm_tree->refcnt > 0); (void)__wt_atomic_sub32( @@ -486,7 +487,7 @@ __lsm_tree_open(WT_SESSION_IMPL *session, * with getting handles exclusive. */ lsm_tree->refcnt = 1; - lsm_tree->exclusive = exclusive ? 1 : 0; + lsm_tree->exclusive = exclusive ? session : NULL; lsm_tree->queue_ref = 0; /* Set a flush timestamp as a baseline. */ @@ -521,7 +522,7 @@ __wt_lsm_tree_get(WT_SESSION_IMPL *session, ret = __lsm_tree_open(session, uri, exclusive, treep); WT_ASSERT(session, ret != 0 || - (exclusive ? 1 : 0) == (*treep)->exclusive); + (exclusive ? session : NULL) == (*treep)->exclusive); return (ret); } @@ -533,8 +534,11 @@ void __wt_lsm_tree_release(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) { WT_ASSERT(session, lsm_tree->refcnt > 0); - if (lsm_tree->exclusive) - lsm_tree->exclusive = 0; + if (lsm_tree->exclusive == session) { + /* We cleared the active flag when getting exclusive access. */ + F_SET(lsm_tree, WT_LSM_TREE_ACTIVE); + lsm_tree->exclusive = NULL; + } (void)__wt_atomic_sub32(&lsm_tree->refcnt, 1); } @@ -848,9 +852,6 @@ __wt_lsm_tree_drop( ret = __wt_lsm_tree_get(session, name, true, &lsm_tree)); WT_RET(ret); - /* Shut down the LSM worker. */ - WT_ERR(__lsm_tree_close(session, lsm_tree)); - /* Prevent any new opens. */ WT_ERR(__wt_lsm_tree_writelock(session, lsm_tree)); locked = true; @@ -910,9 +911,6 @@ __wt_lsm_tree_rename(WT_SESSION_IMPL *session, ret = __wt_lsm_tree_get(session, olduri, true, &lsm_tree)); WT_RET(ret); - /* Shut down the LSM worker. */ - WT_ERR(__lsm_tree_close(session, lsm_tree)); - /* Prevent any new opens. */ WT_ERR(__wt_lsm_tree_writelock(session, lsm_tree)); locked = true; @@ -985,9 +983,6 @@ __wt_lsm_tree_truncate( ret = __wt_lsm_tree_get(session, name, true, &lsm_tree)); WT_RET(ret); - /* Shut down the LSM worker. */ - WT_ERR(__lsm_tree_close(session, lsm_tree)); - /* Prevent any new opens. */ WT_ERR(__wt_lsm_tree_writelock(session, lsm_tree)); locked = true; diff --git a/src/lsm/lsm_worker.c b/src/lsm/lsm_worker.c index b8a85e4a5e5..0874da8db13 100644 --- a/src/lsm/lsm_worker.c +++ b/src/lsm/lsm_worker.c @@ -139,7 +139,7 @@ __lsm_worker(void *arg) if (ret == WT_NOTFOUND) { F_CLR(entry->lsm_tree, WT_LSM_TREE_COMPACTING); ret = 0; - } else if (ret == EBUSY) + } else if (ret == EBUSY || ret == EINTR) ret = 0; /* Paranoia: clear session state. */ -- cgit v1.2.1 From da220f8d099733a984f314d4da56781549a3dc73 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 15 Mar 2016 09:01:41 -0400 Subject: WT-2489 Minor cleanup and string fixes. --- test/manydbs/manydbs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c index 7cadfb0a11b..1d3412a7b06 100644 --- a/test/manydbs/manydbs.c +++ b/test/manydbs/manydbs.c @@ -99,7 +99,7 @@ get_stat(WT_SESSION *stat_session, int stat_field, uint64_t *valuep) return (ret); ret = statc->get_value(statc, &desc, &pvalue, valuep); - statc->close(statc); + testutil_check(statc->close(statc)); return (ret); } @@ -244,10 +244,10 @@ main(int argc, char *argv[]) */ if (idle && cond_reset != cond_reset_orig[i]) testutil_die(ERANGE, - "condvar reset on idle connection %d of %" PRIu64, + "condition reset on idle connection %d of %" PRIu64, i, cond_reset); if (!idle && cond_reset > cond_wait / 20) - testutil_die(ERANGE, "connection %d condvar reset %" + testutil_die(ERANGE, "connection %d condition reset %" PRIu64 " exceeds 5%% of %" PRIu64, i, cond_reset, cond_wait); testutil_check(connections[i]->close(connections[i], NULL)); -- cgit v1.2.1 From d6247e8bc54c8639881e2006cfe957ca24b454a8 Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Tue, 15 Mar 2016 14:06:52 -0400 Subject: WT-2447 Modified some join tests to use projections on reference cursors. --- test/suite/test_join01.py | 7 ++++--- test/suite/test_join02.py | 11 ++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/test/suite/test_join01.py b/test/suite/test_join01.py index 27d35808b52..4aa2bc6e269 100644 --- a/test/suite/test_join01.py +++ b/test/suite/test_join01.py @@ -148,7 +148,8 @@ class test_join01(wttest.WiredTigerTestCase): # and examine primary keys 2,5,8,...,95,98,1,4,7,...,94,97. jc = self.session.open_cursor('join:table:join01' + proj_suffix, None, None) - c2 = self.session.open_cursor('index:join01:index2', None, None) + # Adding a projection to a reference cursor should be allowed. + c2 = self.session.open_cursor('index:join01:index2(v1)', None, None) c2.set_key(99) # skips all entries w/ primary key divisible by three self.assertEquals(0, c2.search()) self.session.join(jc, c2, 'compare=gt') @@ -166,12 +167,12 @@ class test_join01(wttest.WiredTigerTestCase): # Then select all numbers whose reverse string representation # is in '20' < x < '40'. - c1a = self.session.open_cursor('index:join01:index1', None, None) + c1a = self.session.open_cursor('index:join01:index1(v1)', None, None) c1a.set_key('21') self.assertEquals(0, c1a.search()) self.session.join(jc, c1a, 'compare=gt' + joincfg1) - c1b = self.session.open_cursor('index:join01:index1', None, None) + c1b = self.session.open_cursor('index:join01:index1(v1)', None, None) c1b.set_key('41') self.assertEquals(0, c1b.search()) self.session.join(jc, c1b, 'compare=lt' + joincfg1) diff --git a/test/suite/test_join02.py b/test/suite/test_join02.py index d122de8a0eb..a691c499cf6 100644 --- a/test/suite/test_join02.py +++ b/test/suite/test_join02.py @@ -179,15 +179,16 @@ class test_join02(wttest.WiredTigerTestCase): c.close() # Use the primary table in one of the joins. + # Use various projections, which should not matter for ref cursors c0a = self.session.open_cursor('table:join02', None, None) - c0b = self.session.open_cursor('table:join02', None, None) - c1a = self.session.open_cursor('index:join02:index1', None, None) + c0b = self.session.open_cursor('table:join02(v4)', None, None) + c1a = self.session.open_cursor('index:join02:index1(v0)', None, None) c1b = self.session.open_cursor('index:join02:index1', None, None) c2a = self.session.open_cursor('index:join02:index2', None, None) c2b = self.session.open_cursor('index:join02:index2', None, None) - c3a = self.session.open_cursor('index:join02:index3', None, None) - c3b = self.session.open_cursor('index:join02:index3', None, None) - c4a = self.session.open_cursor('index:join02:index4', None, None) + c3a = self.session.open_cursor('index:join02:index3(v4)', None, None) + c3b = self.session.open_cursor('index:join02:index3(v0)', None, None) + c4a = self.session.open_cursor('index:join02:index4(v1)', None, None) # Attach extra properties to each cursor. For cursors that # may appear on the 'left' side of a range CA < x < CB, -- cgit v1.2.1 From 8935c408ac9decf8d8f7312abd6eb032bfc617c9 Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Tue, 15 Mar 2016 14:08:22 -0400 Subject: WT-2447 In cursor joins, when opening an iterator cursor, always force to a null projection, even when the reference cursor has a projection. Also a small naming change, and give a better message for an illegal case. --- src/cursor/cur_join.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/cursor/cur_join.c b/src/cursor/cur_join.c index a72f08fd9d3..8f69f748f9d 100644 --- a/src/cursor/cur_join.c +++ b/src/cursor/cur_join.c @@ -27,12 +27,12 @@ __curjoin_entry_iter_init(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, const char *def_cfg[] = { WT_CONFIG_BASE( session, WT_SESSION_open_cursor), NULL }; const char *urimain, **config; - char *mainbuf, *urimin; + char *mainbuf, *uri; WT_CURSOR_JOIN_ITER *iter; size_t size; iter = NULL; - mainbuf = urimin = NULL; + mainbuf = uri = NULL; to_dup = entry->ends[0].cursor; if (F_ISSET((WT_CURSOR *)cjoin, WT_CURSTD_RAW)) @@ -40,10 +40,10 @@ __curjoin_entry_iter_init(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, else config = &def_cfg[0]; + size = strlen(to_dup->internal_uri) + 3; + WT_ERR(__wt_calloc(session, size, 1, &uri)); + snprintf(uri, size, "%s()", to_dup->internal_uri); urimain = cjoin->table->name; - size = strlen(to_dup->uri) + 3; - WT_ERR(__wt_calloc(session, size, 1, &urimin)); - snprintf(urimin, size, "%s()", to_dup->uri); if (cjoin->projection != NULL) { size = strlen(urimain) + strlen(cjoin->projection) + 1; WT_ERR(__wt_calloc(session, size, 1, &mainbuf)); @@ -52,7 +52,7 @@ __curjoin_entry_iter_init(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, } WT_ERR(__wt_calloc_one(session, &iter)); - WT_ERR(__wt_open_cursor(session, urimin, (WT_CURSOR *)cjoin, config, + WT_ERR(__wt_open_cursor(session, uri, (WT_CURSOR *)cjoin, config, &iter->cursor)); WT_ERR(__wt_cursor_dup_position(to_dup, iter->cursor)); WT_ERR(__wt_open_cursor(session, urimain, (WT_CURSOR *)cjoin, config, @@ -69,7 +69,7 @@ __curjoin_entry_iter_init(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, err: __wt_free(session, iter); } __wt_free(session, mainbuf); - __wt_free(session, urimin); + __wt_free(session, uri); return (ret); } @@ -332,8 +332,7 @@ __curjoin_init_bloom(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, goto done; WT_ERR(ret); } else - /* There should not be any other cases */ - WT_ASSERT(session, false); + WT_ERR(__wt_illegal_value(session, NULL)); } collator = (entry->index == NULL) ? NULL : entry->index->collator; while (ret == 0) { -- cgit v1.2.1 From 4077c50345f35ab2b921a1ad528926412e698409 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 15 Mar 2016 15:17:13 -0400 Subject: WT-2461 Fix sweep loop to look for progress. Reduce number of tables. --- test/suite/test_sweep01.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/test/suite/test_sweep01.py b/test/suite/test_sweep01.py index 5c279c8f074..33d4f2398db 100644 --- a/test/suite/test_sweep01.py +++ b/test/suite/test_sweep01.py @@ -40,7 +40,7 @@ import wttest class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): tablebase = 'test_sweep01' uri = 'table:' + tablebase - numfiles = 50 + numfiles = 30 numkv = 1000 conn_config = 'file_manager=(close_handle_minimum=0,' + \ 'close_idle_time=6,close_scan_interval=2),' + \ @@ -105,13 +105,27 @@ class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): c = self.session.open_cursor(uri, None) k = 0 sleep = 0 - while sleep < 16: + max = 60 + last_nfile = nfile1 + while sleep < max: self.session.checkpoint() k = k+1 c[k] = 1 sleep += 2 time.sleep(2) + # Give slow machines time to process files. As long as progress + # is made, keep going. + stat_cursor = self.session.open_cursor('statistics:', None, None) + this_nfile = stat_cursor[stat.conn.file_open][2] + stat_cursor.close() + self.pr("==== loop " + str(sleep)) + self.pr("last_nfile " + str(last_nfile)) + self.pr("this_nfile " + str(this_nfile)) + if this_nfile == last_nfile and this_nfile < nfile1: + break + last_nfile = this_nfile c.close() + self.pr("Sweep loop took " + str(sleep)) stat_cursor = self.session.open_cursor('statistics:', None, None) close2 = stat_cursor[stat.conn.dh_sweep_close][2] -- cgit v1.2.1 From 6f1c39bfd77fc7905144e7a39899ed37c0c98e51 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 15 Mar 2016 17:09:57 -0400 Subject: WT-2485 Adjust lsm_tree->merge_min when we read the metadata. --- src/lsm/lsm_merge.c | 1 + src/lsm/lsm_meta.c | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/src/lsm/lsm_merge.c b/src/lsm/lsm_merge.c index f7176067f19..973043f334f 100644 --- a/src/lsm/lsm_merge.c +++ b/src/lsm/lsm_merge.c @@ -64,6 +64,7 @@ __lsm_merge_aggressive_update(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) new_aggressive = 0; + WT_ASSERT(session, lsm_tree->merge_min != 0); /* * If the tree is open read-only or we are compacting, be very * aggressive. Otherwise, we can spend a long time waiting for merges diff --git a/src/lsm/lsm_meta.c b/src/lsm/lsm_meta.c index e57be0c6459..0ef78204a2b 100644 --- a/src/lsm/lsm_meta.c +++ b/src/lsm/lsm_meta.c @@ -349,6 +349,13 @@ __lsm_meta_read_v1( * Ignore any other values: the metadata entry might have been * created by a future release, with unknown options. */ + /* + * If the default merge_min was not overridden, calculate it now. We + * do this here so that trees created before merge_min was added get a + * sane value. + */ + if (lsm_tree->merge_min < 2) + lsm_tree->merge_min = WT_MAX(2, lsm_tree->merge_max / 2); err: __wt_scr_free(session, &buf); return (ret); -- cgit v1.2.1 From 01f26d900ae55247495eb40b9a1fbab6ccfb1462 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Wed, 16 Mar 2016 14:04:32 +1100 Subject: WT-2457 Add a test case for LSM not returning EBUSY surprisingly. --- test/suite/test_lsm03.py | 60 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 test/suite/test_lsm03.py diff --git a/test/suite/test_lsm03.py b/test/suite/test_lsm03.py new file mode 100644 index 00000000000..37d1f761744 --- /dev/null +++ b/test/suite/test_lsm03.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wtscenario, wttest +from helper import simple_populate + +# test_lsm03.py +# Check to make sure that LSM schema operations don't get EBUSY when +# there are no user operations active. +class test_lsm03(wttest.WiredTigerTestCase): + name = 'test_rebalance' + + # Use small pages so we generate some internal layout + # Setup LSM so multiple chunks are present + config = 'key_format=S,allocation_size=512,internal_page_max=512' + \ + ',leaf_page_max=1k,lsm=(chunk_size=512k,merge_min=10)' + + # Populate an object, then rebalance it. + def test_lsm_drop_active(self): + uri = 'lsm:' + self.name + simple_populate(self, uri, self.config, 10000) + + # Force to disk + self.reopen_conn() + + # An open cursors should cause failure. + cursor = self.session.open_cursor(uri, None, None) + self.assertRaises(wiredtiger.WiredTigerError, + lambda: self.session.drop(uri, None)) + cursor.close() + + # Add enough records that a merge should be running + simple_populate(self, uri, self.config, 50000) + # The drop should succeed even when LSM work units are active + self.session.drop(uri) -- cgit v1.2.1 From 1a698c5f5495555d51cc7bf5d3209433e3b9a083 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Wed, 16 Mar 2016 18:24:36 +1100 Subject: WT-2492 Change printf format specifier to be Windows compatible. Windows doesn't like "td" for ptrdiff_t --- src/config/config.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index 9d14353f730..4eadaf2f008 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -15,13 +15,14 @@ static int __config_err(WT_CONFIG *conf, const char *msg, int err) { - ptrdiff_t d; + uint64_t offset; - d = conf->cur - conf->orig; + /* Cast because printing a pointer diff isn't platform portable */ + offset = (uint64_t)(conf->cur - conf->orig); WT_RET_MSG(conf->session, err, - "Error parsing '%.*s' at offset %td: %s", - (int)(conf->end - conf->orig), conf->orig, d, msg); + "Error parsing '%.*s' at offset %" PRIu64 ": %s", + (int)(conf->end - conf->orig), conf->orig, offset, msg); } /* -- cgit v1.2.1 From 6bf871c9cc0379a15bf1aa550df6a1ea2dfd4521 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 16 Mar 2016 09:54:34 -0400 Subject: WT-2490: search_near() returns wrong key for column-store Update string list. --- dist/s_string.ok | 1 + 1 file changed, 1 insertion(+) diff --git a/dist/s_string.ok b/dist/s_string.ok index 43eb7861b23..9a02537d055 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -467,6 +467,7 @@ comparep compat concat cond +condvar conf confchk config -- cgit v1.2.1 From bb7bd4c719f7fae1cdece402251653d8d8576aab Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 16 Mar 2016 10:07:04 -0400 Subject: WT-2490: search_near() returns wrong key for column-store Lint: Last value assigned to variable 'firstcg_cur' not used in one code path. --- src/cursor/cur_join.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cursor/cur_join.c b/src/cursor/cur_join.c index 8f69f748f9d..dfdbe1ae2ff 100644 --- a/src/cursor/cur_join.c +++ b/src/cursor/cur_join.c @@ -111,7 +111,6 @@ __curjoin_copy_primary_key(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, if (isindex) { cindex = ((WT_CURSOR_INDEX *)fromcur); - firstcg_cur = cindex->cg_cursors[0]; /* * Repack tells us where the index key ends; advance past * that to get where the raw primary key starts. -- cgit v1.2.1 From 78042b7435619490d96ce1ff6120675f962ed903 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 16 Mar 2016 10:12:40 -0400 Subject: WT-2490: search_near() returns wrong key for column-store Enhance the variable-length column-store search/search-near tests to catch WT-2490. --- test/suite/test_bug008.py | 237 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 189 insertions(+), 48 deletions(-) diff --git a/test/suite/test_bug008.py b/test/suite/test_bug008.py index 8f0526d9cef..0243887e258 100644 --- a/test/suite/test_bug008.py +++ b/test/suite/test_bug008.py @@ -33,65 +33,208 @@ import wiredtiger, wttest from helper import simple_populate, key_populate, value_populate from wtscenario import check_scenarios -# Tests for invisible updates. +# Test search/search-near operations, including invisible values and keys +# past the end of the table. class test_bug008(wttest.WiredTigerTestCase): + uri = 'file:test_bug008' # This is a btree layer test. scenarios = check_scenarios([ - ('fix', dict(fmt='key_format=r,value_format=8t', empty=1)), - ('row', dict(fmt='key_format=S', empty=0)), - ('var', dict(fmt='key_format=r', empty=0)) + ('fix', dict(fmt='key_format=r,value_format=8t', empty=1, colvar=0)), + ('row', dict(fmt='key_format=S', empty=0, colvar=0)), + ('var', dict(fmt='key_format=r', empty=0, colvar=1)) ]) + # Verify cursor search and search-near operations in an empty table. + def test_search_empty(self): + # Create the object and open a cursor. + self.session.create(self.uri, self.fmt) + cursor = self.session.open_cursor(self.uri, None) + + # Search for a record past the end of the table, which should fail. + cursor.set_key(key_populate(cursor, 100)) + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + + # Search-near for a record past the end of the table, which should fail. + cursor.set_key(key_populate(cursor, 100)) + self.assertEqual(cursor.search_near(), wiredtiger.WT_NOTFOUND) + + # Verify cursor search and search-near operations at and past the end of + # a file, with a set of on-page visible records. + def test_search_eot(self): + # Populate the tree and reopen the connection, forcing it to disk + # and moving the records to an on-page format. + simple_populate(self, self.uri, self.fmt, 100) + self.reopen_conn() + + # Open a cursor. + cursor = self.session.open_cursor(self.uri, None) + + # Search for a record at the end of the table, which should succeed. + cursor.set_key(key_populate(cursor, 100)) + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) + self.assertEqual(cursor.get_value(), value_populate(cursor, 100)) + + # Search-near for a record at the end of the table, which should + # succeed, returning the last record. + cursor.set_key(key_populate(cursor, 100)) + self.assertEqual(cursor.search_near(), 0) + self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) + self.assertEqual(cursor.get_value(), value_populate(cursor, 100)) + + # Search for a record past the end of the table, which should fail. + cursor.set_key(key_populate(cursor, 200)) + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + + # Search-near for a record past the end of the table, which should + # succeed, returning the last record. + cursor.set_key(key_populate(cursor, 200)) + self.assertEqual(cursor.search_near(), -1) + self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) + self.assertEqual(cursor.get_value(), value_populate(cursor, 100)) + + # Verify cursor search-near operations before and after a set of + # column-store duplicates. + def test_search_duplicate(self): + if self.colvar == 0: + return + + # Populate the tree. + simple_populate(self, self.uri, self.fmt, 105) + + # Set up deleted records before and after a set of duplicate records, + # and make sure search/search-near returns the correct record. + cursor = self.session.open_cursor(self.uri, None) + for i in range(20, 100): + cursor[key_populate(cursor, i)] = '=== IDENTICAL VALUE ===' + for i in range(15, 25): + cursor.set_key(key_populate(cursor, i)) + self.assertEqual(cursor.remove(), 0) + for i in range(95, 106): + cursor.set_key(key_populate(cursor, i)) + self.assertEqual(cursor.remove(), 0) + cursor.close() + + # Reopen the connection, forcing it to disk and moving the records to + # an on-page format. + self.reopen_conn() + + # Open a cursor. + cursor = self.session.open_cursor(self.uri, None) + + # Search-near for a record in the deleted set before the duplicate set, + # which should succeed, returning the first record in the duplicate set. + cursor.set_key(key_populate(cursor, 18)) + self.assertEqual(cursor.search_near(), 1) + self.assertEqual(cursor.get_key(), key_populate(cursor, 25)) + + # Search-near for a record in the deleted set after the duplicate set, + # which should succeed, returning the last record in the duplicate set. + cursor.set_key(key_populate(cursor, 98)) + self.assertEqual(cursor.search_near(), -1) + self.assertEqual(cursor.get_key(), key_populate(cursor, 94)) + # Verify cursor search and search-near operations on a file with a set of # on-page visible records, and a set of insert-list invisible records. def test_search_invisible_one(self): - uri = 'file:test_bug008' # This is a btree layer test. + # Populate the tree. + simple_populate(self, self.uri, self.fmt, 100) - # Populate the tree and reopen the connection, forcing it to disk - # and moving the records to an on-page format. - simple_populate(self, uri, self.fmt, 100) + # Delete a range of records. + for i in range(5, 10): + cursor = self.session.open_cursor(self.uri, None) + cursor.set_key(key_populate(cursor, i)) + self.assertEqual(cursor.remove(), 0) + + # Reopen the connection, forcing it to disk and moving the records to + # an on-page format. self.reopen_conn() - # Begin a transaction, and add some additional records. + # Add updates to the existing records (in both the deleted an undeleted + # range), as well as some new records after the end. Put the updates in + # a separate transaction so they're invisible to another cursor. self.session.begin_transaction() - cursor = self.session.open_cursor(uri, None) + cursor = self.session.open_cursor(self.uri, None) + for i in range(5, 10): + cursor[key_populate(cursor, i)] = value_populate(cursor, i + 1000) + for i in range(30, 40): + cursor[key_populate(cursor, i)] = value_populate(cursor, i + 1000) for i in range(100, 140): - cursor[key_populate(cursor, i)] = value_populate(cursor, i) + cursor[key_populate(cursor, i)] = value_populate(cursor, i + 1000) # Open a separate session and cursor. s = self.conn.open_session() - cursor = s.open_cursor(uri, None) + cursor = s.open_cursor(self.uri, None) - # Search for an invisible record. - cursor.set_key(key_populate(cursor, 130)) - if self.empty: - # Invisible updates to fixed-length column-store objects are - # invisible to the reader, but the fact that they exist past - # the end of the initial records causes the instantiation of - # empty records: confirm successful return of an empty row. - cursor.search() - self.assertEqual(cursor.get_key(), 130) - self.assertEqual(cursor.get_value(), 0) - else: - # Otherwise, we should not find any matching records. - self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + # Search for an existing record in the deleted range, should not find + # it. + for i in range(5, 10): + cursor.set_key(key_populate(cursor, i)) + if self.empty: + # Fixed-length column-store rows always exist. + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_key(), i) + self.assertEqual(cursor.get_value(), 0) + else: + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) - # Search-near for an invisible record, which should succeed, returning - # the last visible record. - cursor.set_key(key_populate(cursor, 130)) - cursor.search_near() - if self.empty: - # Invisible updates to fixed-length column-store objects are - # invisible to the reader, but the fact that they exist past - # the end of the initial records causes the instantiation of - # empty records: confirm successful return of an empty row. - cursor.search() - self.assertEqual(cursor.get_key(), 130) - self.assertEqual(cursor.get_value(), 0) - else: - # Otherwise, we should find the closest record for which we can see - # the value. - self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) - self.assertEqual(cursor.get_value(), value_populate(cursor, 100)) + # Search for an existing record in the updated range, should see the + # original value. + for i in range(30, 40): + cursor.set_key(key_populate(cursor, i)) + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_key(), key_populate(cursor, i)) + + # Search for a added record, should not find it. + for i in range(120, 130): + cursor.set_key(key_populate(cursor, i)) + if self.empty: + # Invisible updates to fixed-length column-store objects are + # invisible to the reader, but the fact that they exist past + # the end of the initial records causes the instantiation of + # empty records: confirm successful return of an empty row. + self.assertEqual(cursor.search(), 0) + self.assertEqual(cursor.get_key(), i) + self.assertEqual(cursor.get_value(), 0) + else: + # Otherwise, we should not find any matching records. + self.assertEqual(cursor.search(), wiredtiger.WT_NOTFOUND) + + # Search-near for an existing record in the deleted range, should find + # the next largest record. (This depends on the implementation behavior + # which currently includes a bias to prefix search.) + for i in range(5, 10): + cursor.set_key(key_populate(cursor, i)) + if self.empty: + # Fixed-length column-store rows always exist. + self.assertEqual(cursor.search_near(), 0) + self.assertEqual(cursor.get_key(), i) + self.assertEqual(cursor.get_value(), 0) + else: + self.assertEqual(cursor.search_near(), 1) + self.assertEqual(cursor.get_key(), key_populate(cursor, 10)) + + # Search-near for an existing record in the updated range, should see + # the original value. + for i in range(30, 40): + cursor.set_key(key_populate(cursor, i)) + self.assertEqual(cursor.search_near(), 0) + self.assertEqual(cursor.get_key(), key_populate(cursor, i)) + + # Search-near for an added record, should find the previous largest + # record. + for i in range(120, 130): + cursor.set_key(key_populate(cursor, i)) + if self.empty: + # Invisible updates to fixed-length column-store objects are + # invisible to the reader, but the fact that they exist past + # the end of the initial records causes the instantiation of + # empty records: confirm successful return of an empty row. + self.assertEqual(cursor.search_near(), 0) + self.assertEqual(cursor.get_key(), i) + self.assertEqual(cursor.get_value(), 0) + else: + self.assertEqual(cursor.search_near(), -1) + self.assertEqual(cursor.get_key(), key_populate(cursor, 100)) # Verify cursor search and search-near operations on a file with a set of # on-page visible records, a set of insert-list visible records, and a set @@ -101,28 +244,26 @@ class test_bug008(wttest.WiredTigerTestCase): # fallback happens, whether the correct position is in the page slots or # the insert list.) def test_search_invisible_two(self): - uri = 'file:test_bug008' # This is a btree layer test. - # Populate the tree and reopen the connection, forcing it to disk # and moving the records to an on-page format. - simple_populate(self, uri, self.fmt, 100) + simple_populate(self, self.uri, self.fmt, 100) self.reopen_conn() # Add some additional visible records. - cursor = self.session.open_cursor(uri, None) + cursor = self.session.open_cursor(self.uri, None) for i in range(100, 120): cursor[key_populate(cursor, i)] = value_populate(cursor, i) cursor.close() # Begin a transaction, and add some additional records. self.session.begin_transaction() - cursor = self.session.open_cursor(uri, None) + cursor = self.session.open_cursor(self.uri, None) for i in range(120, 140): cursor[key_populate(cursor, i)] = value_populate(cursor, i) # Open a separate session and cursor. s = self.conn.open_session() - cursor = s.open_cursor(uri, None) + cursor = s.open_cursor(self.uri, None) # Search for an invisible record. cursor.set_key(key_populate(cursor, 130)) -- cgit v1.2.1 From 56eeee379caf134a2a21e2f4679563147849f71c Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 16 Mar 2016 10:26:55 -0400 Subject: WT-2490: search_near() returns wrong key for column-store In variable-length column-store, updates/deletes are stored in the insert list. (In short, there needs to be a place to put references for individual records because we might update/delete a single entry in a large duplicate set, and the insert list has to exist anyway, so we used it.) However, if we find a match in the insert list, but it's not visible to the current transaction for some reason, we need to know if there's an on-page record with the same record number with a previous version of the data we can use instead. We were using an illegal cursor slot value for this purpose, but that messes up search-near, which needs a valid slot should the entry on the insert list not be visible. Add a new flag to the btree cursor, WT_CBT_VAR_ONPAGE_MATCH, that's set if we find a matching on-page variable length column-store record. Clarify the column-search routines, specifically set the cursor's slot and record number for each code path. --- src/btree/bt_cursor.c | 29 +++++++++++++++-------------- src/btree/col_srch.c | 22 ++++++++++++---------- src/include/cursor.h | 3 ++- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/btree/bt_cursor.c b/src/btree/bt_cursor.c index d64b6064268..1f3ac443495 100644 --- a/src/btree/bt_cursor.c +++ b/src/btree/bt_cursor.c @@ -173,13 +173,18 @@ __cursor_valid(WT_CURSOR_BTREE *cbt, WT_UPDATE **updp) */ break; case BTREE_COL_VAR: + /* The search function doesn't check for empty pages. */ + if (page->pg_var_entries == 0) + return (false); + WT_ASSERT(session, cbt->slot < page->pg_var_entries); + /* - * If search returned an insert object, there may or may not be - * a matching on-page object, we have to check. Variable-length - * column-store pages don't map one-to-one to keys, but have - * "slots", check if search returned a valid slot. + * Column-store updates aren't stored on the page, instead they + * are stored as "insert" objects. If search returned an insert + * object we can't return, the returned on-page object must be + * checked for a match. */ - if (cbt->slot >= page->pg_var_entries) + if (cbt->ins != NULL && !F_ISSET(cbt, WT_CBT_VAR_ONPAGE_MATCH)) return (false); /* @@ -194,6 +199,11 @@ __cursor_valid(WT_CURSOR_BTREE *cbt, WT_UPDATE **updp) return (false); break; case BTREE_ROW: + /* The search function doesn't check for empty pages. */ + if (page->pg_row_entries == 0) + return (false); + WT_ASSERT(session, cbt->slot < page->pg_row_entries); + /* * See above: for row-store, no insert object can have the same * key as an on-page object, we're done. @@ -201,15 +211,6 @@ __cursor_valid(WT_CURSOR_BTREE *cbt, WT_UPDATE **updp) if (cbt->ins != NULL) return (false); - /* - * Check if searched returned a valid slot (the failure mode is - * an empty page, the search function doesn't check, and so the - * more exact test is "page->pg_row_entries == 0", but this test - * mirrors the column-store test). - */ - if (cbt->slot >= page->pg_row_entries) - return (false); - /* Updates are stored on the page, check for a delete. */ if (page->pg_row_upd != NULL && (upd = __wt_txn_read( session, page->pg_row_upd[cbt->slot])) != NULL) { diff --git a/src/btree/col_srch.c b/src/btree/col_srch.c index 23eae75ec2b..4730267a545 100644 --- a/src/btree/col_srch.c +++ b/src/btree/col_srch.c @@ -211,7 +211,6 @@ descend: /* leaf_only: page = current->page; cbt->ref = current; - cbt->recno = recno; /* * Don't bother searching if the caller is appending a new record where @@ -224,13 +223,6 @@ leaf_only: return (0); } - /* - * Set the on-page slot to an impossible value larger than any possible - * slot (it's used to interpret the search function's return after the - * search returns an insert list for a page that has no entries). - */ - cbt->slot = UINT32_MAX; - /* * Search the leaf page. * @@ -244,28 +236,38 @@ leaf_only: * that's impossibly large for the page. We do have additional setup to * do in that case, the record may be appended to the page. */ - cbt->compare = 0; if (page->type == WT_PAGE_COL_FIX) { if (recno < page->pg_fix_recno) { + cbt->recno = page->pg_fix_recno; cbt->compare = 1; return (0); } if (recno >= page->pg_fix_recno + page->pg_fix_entries) { cbt->recno = page->pg_fix_recno + page->pg_fix_entries; goto past_end; - } else + } else { + cbt->recno = recno; + cbt->compare = 0; ins_head = WT_COL_UPDATE_SINGLE(page); + } } else { if (recno < page->pg_var_recno) { + cbt->recno = page->pg_var_recno; + cbt->slot = 0; cbt->compare = 1; return (0); } if ((cip = __col_var_search(page, recno, NULL)) == NULL) { cbt->recno = __col_var_last_recno(page); + cbt->slot = page->pg_var_entries == 0 ? + 0 : page->pg_var_entries - 1; goto past_end; } else { + cbt->recno = recno; cbt->slot = WT_COL_SLOT(page, cip); + cbt->compare = 0; ins_head = WT_COL_UPDATE_SLOT(page, cbt->slot); + F_SET(cbt, WT_CBT_VAR_ONPAGE_MATCH); } } diff --git a/src/include/cursor.h b/src/include/cursor.h index a2110465355..b72d8de9e11 100644 --- a/src/include/cursor.h +++ b/src/include/cursor.h @@ -213,10 +213,11 @@ struct __wt_cursor_btree { #define WT_CBT_NO_TXN 0x10 /* Non-transactional cursor (e.g. on a checkpoint) */ #define WT_CBT_SEARCH_SMALLEST 0x20 /* Row-store: small-key insert list */ +#define WT_CBT_VAR_ONPAGE_MATCH 0x40 /* Var-store: on-page recno match */ #define WT_CBT_POSITION_MASK /* Flags associated with position */ \ (WT_CBT_ITERATE_APPEND | WT_CBT_ITERATE_NEXT | WT_CBT_ITERATE_PREV | \ - WT_CBT_SEARCH_SMALLEST) + WT_CBT_SEARCH_SMALLEST | WT_CBT_VAR_ONPAGE_MATCH) uint8_t flags; }; -- cgit v1.2.1 From e782e1ce92c9786f3ed41277f386192cb683abba Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 16 Mar 2016 11:34:27 -0400 Subject: WT-2492 Change printf format specifier to be Windows compatible. Windows doesn't like %td, use the maximum unsigned integer format instead, PRIuMAX. Get rid of the temporary variable I introduced in cc0d34b, there's no need for one. --- dist/s_string.ok | 1 + src/config/config.c | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/dist/s_string.ok b/dist/s_string.ok index 43eb7861b23..6762521ca76 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -878,6 +878,7 @@ ps psp pthread ptr +ptrdiff pushms putK putV diff --git a/src/config/config.c b/src/config/config.c index 4eadaf2f008..a50c4d29f65 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -15,14 +15,14 @@ static int __config_err(WT_CONFIG *conf, const char *msg, int err) { - uint64_t offset; - - /* Cast because printing a pointer diff isn't platform portable */ - offset = (uint64_t)(conf->cur - conf->orig); - + /* + * Cast the string offset to uintmax_t because the %td format to print + * a type ptrdiff_t isn't supported under MSVC. + */ WT_RET_MSG(conf->session, err, - "Error parsing '%.*s' at offset %" PRIu64 ": %s", - (int)(conf->end - conf->orig), conf->orig, offset, msg); + "Error parsing '%.*s' at offset %" PRIuMAX ": %s", + (int)(conf->end - conf->orig), conf->orig, + (uintmax_t)(conf->cur - conf->orig), msg); } /* -- cgit v1.2.1 From 4cfc6158c29e1e614a28212058d994077cb95fd9 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Wed, 16 Mar 2016 11:58:25 -0400 Subject: WT-2493 Add lsm_manager to verbose settings. --- src/conn/conn_api.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 9cd7dd2f44e..6d115c8fdcd 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -1694,6 +1694,7 @@ __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) { "fileops", WT_VERB_FILEOPS }, { "log", WT_VERB_LOG }, { "lsm", WT_VERB_LSM }, + { "lsm_manager", WT_VERB_LSM_MANAGER }, { "metadata", WT_VERB_METADATA }, { "mutex", WT_VERB_MUTEX }, { "overflow", WT_VERB_OVERFLOW }, -- cgit v1.2.1 From c82bed6dc12e6819e73203c034191bf8a1335204 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 16 Mar 2016 13:07:59 -0400 Subject: WT-2494: review calls to __wt_free, plus minor bug in an error path. Remove checks for "pointer != NULL" before calling __wt_free(), they're not needed. In __wt_async_op_init(), don't bother clearing the pointer values after calling __wt_free(), they get cleared by __wt_free(). In __lsm_tree_set_name(), don't leave the tree without a name, by freeing the name and then failing to allocate room for the new one. In __ckpt_last_name(), don't free the wrong pointer in the error path. --- src/async/async_op.c | 12 +++--------- src/btree/bt_discard.c | 3 +-- src/btree/bt_read.c | 6 ++---- src/btree/bt_slvg.c | 6 ++---- src/btree/bt_split.c | 3 +-- src/conn/conn_open.c | 6 ++---- src/cursor/cur_join.c | 3 +-- src/log/log.c | 6 ++---- src/lsm/lsm_tree.c | 13 ++++++++----- src/meta/meta_ckpt.c | 5 ++--- src/os_posix/os_stdio.c | 3 +-- src/packing/pack_stream.c | 3 +-- src/schema/schema_open.c | 3 +-- 13 files changed, 27 insertions(+), 45 deletions(-) diff --git a/src/async/async_op.c b/src/async/async_op.c index 130c704757b..970c33c3360 100644 --- a/src/async/async_op.c +++ b/src/async/async_op.c @@ -349,14 +349,8 @@ __wt_async_op_init(WT_SESSION_IMPL *session) WT_ERR(__async_op_init(conn, op, i)); } return (0); -err: - if (async->async_ops != NULL) { - __wt_free(session, async->async_ops); - async->async_ops = NULL; - } - if (async->async_queue != NULL) { - __wt_free(session, async->async_queue); - async->async_queue = NULL; - } + +err: __wt_free(session, async->async_ops); + __wt_free(session, async->async_queue); return (ret); } diff --git a/src/btree/bt_discard.c b/src/btree/bt_discard.c index 795111d53f9..1f739c9572e 100644 --- a/src/btree/bt_discard.c +++ b/src/btree/bt_discard.c @@ -337,8 +337,7 @@ __free_page_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page) copy = WT_ROW_KEY_COPY(rip); (void)__wt_row_leaf_key_info( page, copy, &ikey, NULL, NULL, NULL); - if (ikey != NULL) - __wt_free(session, ikey); + __wt_free(session, ikey); } /* diff --git a/src/btree/bt_read.c b/src/btree/bt_read.c index f80eb09153c..5cf6a9bf2bc 100644 --- a/src/btree/bt_read.c +++ b/src/btree/bt_read.c @@ -281,10 +281,8 @@ err: WT_TRET(__wt_las_cursor_close(session, &cursor, session_flags)); * On error, upd points to a single unlinked WT_UPDATE structure, * first_upd points to a list. */ - if (upd != NULL) - __wt_free(session, upd); - if (first_upd != NULL) - __wt_free_update_list(session, first_upd); + __wt_free(session, upd); + __wt_free_update_list(session, first_upd); __wt_scr_free(session, ¤t_key); __wt_scr_free(session, &las_addr); diff --git a/src/btree/bt_slvg.c b/src/btree/bt_slvg.c index 8d78bda79fb..0e064d306b6 100644 --- a/src/btree/bt_slvg.c +++ b/src/btree/bt_slvg.c @@ -1206,8 +1206,7 @@ __slvg_col_build_internal( __wt_root_ref_init(&ss->root_ref, page, true); if (0) { -err: if (addr != NULL) - __wt_free(session, addr); +err: __wt_free(session, addr); __wt_page_out(session, &page); } return (ret); @@ -1868,8 +1867,7 @@ __slvg_row_build_internal( __wt_root_ref_init(&ss->root_ref, page, false); if (0) { -err: if (addr != NULL) - __wt_free(session, addr); +err: __wt_free(session, addr); __wt_page_out(session, &page); } return (ret); diff --git a/src/btree/bt_split.c b/src/btree/bt_split.c index 3dea03316ce..4f16a290958 100644 --- a/src/btree/bt_split.c +++ b/src/btree/bt_split.c @@ -151,8 +151,7 @@ __wt_split_stash_discard_all( for (i = 0, stash = session->split_stash; i < session->split_stash_cnt; ++i, ++stash) - if (stash->p != NULL) - __wt_free(session_safe, stash->p); + __wt_free(session_safe, stash->p); __wt_free(session_safe, session->split_stash); session->split_stash_cnt = session->split_stash_alloc = 0; diff --git a/src/conn/conn_open.c b/src/conn/conn_open.c index 58577b4587d..aff422654d7 100644 --- a/src/conn/conn_open.c +++ b/src/conn/conn_open.c @@ -210,10 +210,8 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) /* * If hash arrays were allocated, free them now. */ - if (s->dhhash != NULL) - __wt_free(session, s->dhhash); - if (s->tablehash != NULL) - __wt_free(session, s->tablehash); + __wt_free(session, s->dhhash); + __wt_free(session, s->tablehash); __wt_free(session, s->hazard); } diff --git a/src/cursor/cur_join.c b/src/cursor/cur_join.c index 8f69f748f9d..33e4f2580b6 100644 --- a/src/cursor/cur_join.c +++ b/src/cursor/cur_join.c @@ -1127,8 +1127,7 @@ __wt_curjoin_join(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, cindex->iface.key_format); } -err: if (main_uri != NULL) - __wt_free(session, main_uri); +err: __wt_free(session, main_uri); return (ret); } diff --git a/src/log/log.c b/src/log/log.c index 0b47308cfd8..e41073299a8 100644 --- a/src/log/log.c +++ b/src/log/log.c @@ -1252,10 +1252,8 @@ __log_has_hole(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t offset, bool *hole) } } -err: if (buf != NULL) - __wt_free(session, buf); - if (zerobuf != NULL) - __wt_free(session, zerobuf); +err: __wt_free(session, buf); + __wt_free(session, zerobuf); return (ret); } diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index 45994bae42b..70ac912f333 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -157,9 +157,12 @@ static int __lsm_tree_set_name(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, const char *uri) { - if (lsm_tree->name != NULL) - __wt_free(session, lsm_tree->name); - WT_RET(__wt_strdup(session, uri, &lsm_tree->name)); + void *p; + + WT_RET(__wt_strdup(session, uri, &p)); + + __wt_free(session, lsm_tree->name); + lsm_tree->name = p; lsm_tree->filename = lsm_tree->name + strlen("lsm:"); return (0); } @@ -950,8 +953,8 @@ __wt_lsm_tree_rename(WT_SESSION_IMPL *session, err: if (locked) WT_TRET(__wt_lsm_tree_writeunlock(session, lsm_tree)); - if (old != NULL) - __wt_free(session, old); + __wt_free(session, old); + /* * Discard this LSM tree structure. The first operation on the renamed * tree will create a new one. diff --git a/src/meta/meta_ckpt.c b/src/meta/meta_ckpt.c index df4cd2cb4d6..0a864432daf 100644 --- a/src/meta/meta_ckpt.c +++ b/src/meta/meta_ckpt.c @@ -212,8 +212,7 @@ __ckpt_last_name( if (found && a.val < found) continue; - if (*namep != NULL) - __wt_free(session, *namep); + __wt_free(session, *namep); WT_ERR(__wt_strndup(session, k.str, k.len, namep)); found = a.val; } @@ -221,7 +220,7 @@ __ckpt_last_name( ret = WT_NOTFOUND; if (0) { -err: __wt_free(session, namep); +err: __wt_free(session, *namep); } return (ret); } diff --git a/src/os_posix/os_stdio.c b/src/os_posix/os_stdio.c index 7ab107eda1e..65a0f40a659 100644 --- a/src/os_posix/os_stdio.c +++ b/src/os_posix/os_stdio.c @@ -46,8 +46,7 @@ __wt_fopen(WT_SESSION_IMPL *session, if (*fpp == NULL) ret = __wt_errno(); - if (pathbuf != NULL) - __wt_free(session, pathbuf); + __wt_free(session, pathbuf); if (ret == 0) return (0); diff --git a/src/packing/pack_stream.c b/src/packing/pack_stream.c index 5716c31a9c2..1393eb9a9c1 100644 --- a/src/packing/pack_stream.c +++ b/src/packing/pack_stream.c @@ -65,8 +65,7 @@ wiredtiger_pack_close(WT_PACK_STREAM *ps, size_t *usedp) if (usedp != NULL) *usedp = WT_PTRDIFF(ps->p, ps->start); - if (ps != NULL) - __wt_free(ps->pack.session, ps); + __wt_free(ps->pack.session, ps); return (0); } diff --git a/src/schema/schema_open.c b/src/schema/schema_open.c index 49318f80959..e7ce4e42498 100644 --- a/src/schema/schema_open.c +++ b/src/schema/schema_open.c @@ -109,8 +109,7 @@ __wt_schema_open_colgroups(WT_SESSION_IMPL *session, WT_TABLE *table) err: __wt_scr_free(session, &buf); __wt_schema_destroy_colgroup(session, &colgroup); - if (cgconfig != NULL) - __wt_free(session, cgconfig); + __wt_free(session, cgconfig); return (ret); } -- cgit v1.2.1 From 72beb86483fc6e637952034bb12c8c672b47c8bc Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Wed, 16 Mar 2016 13:55:18 -0400 Subject: WT-2481 Set memory_page_max after we read chunk_max. --- src/lsm/lsm_meta.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/lsm/lsm_meta.c b/src/lsm/lsm_meta.c index e57be0c6459..52a174f644c 100644 --- a/src/lsm/lsm_meta.c +++ b/src/lsm/lsm_meta.c @@ -191,22 +191,6 @@ __lsm_meta_read_v1( chunk = NULL; /* -Wconditional-uninitialized */ - /* - * Set up the config for each chunk. - * - * Make the memory_page_max double the chunk size, so application - * threads don't immediately try to force evict the chunk when the - * worker thread clears the NO_EVICTION flag. - */ - file_cfg[1] = lsmconf; - WT_ERR(__wt_scr_alloc(session, 0, &buf)); - WT_ERR(__wt_buf_fmt(session, buf, - "key_format=u,value_format=u,memory_page_max=%" PRIu64, - 2 * lsm_tree->chunk_max)); - file_cfg[2] = buf->data; - WT_ERR(__wt_config_collapse(session, file_cfg, &fileconf)); - lsm_tree->file_config = fileconf; - WT_ERR(__wt_config_getones(session, lsmconf, "key_format", &cv)); WT_ERR(__wt_strndup(session, cv.str, cv.len, &lsm_tree->key_format)); WT_ERR(__wt_config_getones(session, lsmconf, "value_format", &cv)); @@ -345,6 +329,22 @@ __lsm_meta_read_v1( WT_ERR_NOTFOUND_OK(ret); lsm_tree->nold_chunks = nchunks; + /* + * Set up the config for each chunk. + * + * Make the memory_page_max double the chunk size, so application + * threads don't immediately try to force evict the chunk when the + * worker thread clears the NO_EVICTION flag. + */ + file_cfg[1] = lsmconf; + WT_ERR(__wt_scr_alloc(session, 0, &buf)); + WT_ERR(__wt_buf_fmt(session, buf, + "key_format=u,value_format=u,memory_page_max=%" PRIu64, + 2 * lsm_tree->chunk_max)); + file_cfg[2] = buf->data; + WT_ERR(__wt_config_collapse(session, file_cfg, &fileconf)); + lsm_tree->file_config = fileconf; + /* * Ignore any other values: the metadata entry might have been * created by a future release, with unknown options. -- cgit v1.2.1 From a9b9e2ec66bdf199e73215c7ef404fa1b8ca319f Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Wed, 16 Mar 2016 14:21:26 -0400 Subject: WT-2485 Refactor and consolidate to fix merge_min. --- src/lsm/lsm_meta.c | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/src/lsm/lsm_meta.c b/src/lsm/lsm_meta.c index 0ef78204a2b..66e3226b26e 100644 --- a/src/lsm/lsm_meta.c +++ b/src/lsm/lsm_meta.c @@ -159,15 +159,6 @@ __lsm_meta_read_v0( */ } WT_ERR_NOTFOUND_OK(ret); - - /* - * If the default merge_min was not overridden, calculate it now. We - * do this here so that trees created before merge_min was added get a - * sane value. - */ - if (lsm_tree->merge_min < 2) - lsm_tree->merge_min = WT_MAX(2, lsm_tree->merge_max / 2); - err: return (ret); } @@ -349,14 +340,6 @@ __lsm_meta_read_v1( * Ignore any other values: the metadata entry might have been * created by a future release, with unknown options. */ - /* - * If the default merge_min was not overridden, calculate it now. We - * do this here so that trees created before merge_min was added get a - * sane value. - */ - if (lsm_tree->merge_min < 2) - lsm_tree->merge_min = WT_MAX(2, lsm_tree->merge_max / 2); - err: __wt_scr_free(session, &buf); return (ret); } @@ -431,6 +414,7 @@ __wt_lsm_meta_read(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_CONFIG_ITEM cval; WT_DECL_RET; char *lsmconf; + bool upgrade; /* LSM trees inherit the merge setting from the connection. */ if (F_ISSET(S2C(session), WT_CONN_LSM_MERGE)) @@ -438,17 +422,29 @@ __wt_lsm_meta_read(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_RET(__wt_metadata_search(session, lsm_tree->name, &lsmconf)); + upgrade = false; ret = __wt_config_getones(session, lsmconf, "file_config", &cval); if (ret == 0) { ret = __lsm_meta_read_v0(session, lsm_tree, lsmconf); __wt_free(session, lsmconf); WT_RET(ret); - return (__lsm_meta_upgrade_v1(session, lsm_tree)); + upgrade = true; } else if (ret == WT_NOTFOUND) { lsm_tree->config = lsmconf; - return (__lsm_meta_read_v1(session, lsm_tree, lsmconf)); + ret = 0; + WT_RET(__lsm_meta_read_v1(session, lsm_tree, lsmconf)); } - + /* + * If the default merge_min was not overridden, calculate it now. + */ + if (lsm_tree->merge_min < 2) + lsm_tree->merge_min = WT_MAX(2, lsm_tree->merge_max / 2); + /* + * If needed, upgrade the configuration. We need to do this after + * we have fixed the merge_min value. + */ + if (upgrade) + WT_RET(__lsm_meta_upgrade_v1(session, lsm_tree)); return (ret); } -- cgit v1.2.1 From 689f3f95534c9f1f67478f8c702303274d988594 Mon Sep 17 00:00:00 2001 From: Don Anderson Date: Wed, 16 Mar 2016 15:08:00 -0400 Subject: WT-2414 When iterating cursor joins, take advantage of the fact that we have the index key for the first index available. We don't need to extract it again. --- src/cursor/cur_join.c | 66 +++++++++++++++++++++++++++++++-------------------- src/include/cursor.h | 3 ++- 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/src/cursor/cur_join.c b/src/cursor/cur_join.c index 8f69f748f9d..0ea42b14048 100644 --- a/src/cursor/cur_join.c +++ b/src/cursor/cur_join.c @@ -95,18 +95,20 @@ __curjoin_pack_recno(WT_SESSION_IMPL *session, uint64_t r, uint8_t *buf, } /* - * __curjoin_copy_primary_key -- + * __curjoin_split_key -- * Copy the primary key from a cursor (either main table or index) - * to another cursory. + * to another cursor. When copying from an index file, the index + * key is also returned. * */ static int -__curjoin_copy_primary_key(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, - WT_CURSOR *tocur, WT_CURSOR *fromcur, const char *repack_fmt, bool isindex) +__curjoin_split_key(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, + WT_ITEM *idxkey, WT_CURSOR *tocur, WT_CURSOR *fromcur, + const char *repack_fmt, bool isindex) { WT_CURSOR *firstcg_cur; WT_CURSOR_INDEX *cindex; - WT_ITEM key, *keyp; + WT_ITEM *keyp; const uint8_t *p; if (isindex) { @@ -118,14 +120,14 @@ __curjoin_copy_primary_key(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, */ WT_RET(__wt_struct_repack(session, cindex->child->key_format, repack_fmt != NULL ? repack_fmt : cindex->iface.key_format, - &cindex->child->key, &key)); - WT_ASSERT(session, cindex->child->key.size > key.size); - key.data = (uint8_t *)key.data + key.size; - key.size = cindex->child->key.size - key.size; - WT_ITEM_SET(tocur->key, key); + &cindex->child->key, idxkey)); + WT_ASSERT(session, cindex->child->key.size > idxkey->size); + tocur->key.data = (uint8_t *)idxkey->data + idxkey->size; + tocur->key.size = cindex->child->key.size - idxkey->size; if (WT_CURSOR_RECNO(tocur)) { - p = (const uint8_t *)key.data; - WT_RET(__wt_vunpack_uint(&p, key.size, &tocur->recno)); + p = (const uint8_t *)tocur->key.data; + WT_RET(__wt_vunpack_uint(&p, tocur->key.size, + &tocur->recno)); } else tocur->recno = 0; } else { @@ -141,6 +143,8 @@ __curjoin_copy_primary_key(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, WT_ITEM_SET(tocur->key, *keyp); tocur->recno = 0; } + idxkey->data = NULL; + idxkey->size = 0; } return (0); } @@ -162,8 +166,8 @@ __curjoin_entry_iter_next(WT_CURSOR_JOIN_ITER *iter, WT_CURSOR *cursor) * Set our key to the primary key, we'll also need this * to check membership. */ - WT_RET(__curjoin_copy_primary_key(iter->session, iter->cjoin, cursor, - iter->cursor, iter->entry->repack_format, + WT_RET(__curjoin_split_key(iter->session, iter->cjoin, &iter->idxkey, + cursor, iter->cursor, iter->entry->repack_format, iter->entry->index != NULL)); iter->curkey = &cursor->key; iter->entry->stats.actual_count++; @@ -691,20 +695,30 @@ __curjoin_entry_member(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, bloom_found = true; } if (entry->index != NULL) { - memset(&v, 0, sizeof(v)); /* Keep lint quiet. */ - c = entry->main; - c->set_key(c, key); - if ((ret = c->search(c)) == 0) - ret = c->get_value(c, &v); - else if (ret == WT_NOTFOUND) - WT_ERR_MSG(session, WT_ERROR, - "main table for join is missing entry"); - WT_TRET(c->reset(c)); - WT_ERR(ret); + /* + * If this entry is used by the iterator, then we already + * have the index key, and we won't have to do any extraction + * either. + */ + if (entry == cjoin->iter->entry) + WT_ITEM_SET(v, cjoin->iter->idxkey); + else { + memset(&v, 0, sizeof(v)); /* Keep lint quiet. */ + c = entry->main; + c->set_key(c, key); + if ((ret = c->search(c)) == 0) + ret = c->get_value(c, &v); + else if (ret == WT_NOTFOUND) + WT_ERR_MSG(session, WT_ERROR, + "main table for join is missing entry"); + WT_TRET(c->reset(c)); + WT_ERR(ret); + } } else - v = *key; + WT_ITEM_SET(v, *key); - if ((idx = entry->index) != NULL && idx->extractor != NULL) { + if ((idx = entry->index) != NULL && idx->extractor != NULL && + entry != cjoin->iter->entry) { WT_CLEAR(extract_cursor); extract_cursor.iface = iface; extract_cursor.iface.session = &session->iface; diff --git a/src/include/cursor.h b/src/include/cursor.h index a2110465355..7ce0159c608 100644 --- a/src/include/cursor.h +++ b/src/include/cursor.h @@ -289,7 +289,8 @@ struct __wt_cursor_join_iter { WT_CURSOR_JOIN_ENTRY *entry; WT_CURSOR *cursor; /* has null projection */ WT_CURSOR *main; /* main table with projection */ - WT_ITEM *curkey; + WT_ITEM *curkey; /* primary key */ + WT_ITEM idxkey; bool positioned; bool isequal; /* advancing means we're done */ }; -- cgit v1.2.1 From 2feb9792dca6815dcc2f4317755216994b76b936 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Wed, 16 Mar 2016 15:43:44 -0400 Subject: WT-2461 Look explicitly for the final number of expected files. --- test/suite/test_sweep01.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/test/suite/test_sweep01.py b/test/suite/test_sweep01.py index 33d4f2398db..bccd2bce012 100644 --- a/test/suite/test_sweep01.py +++ b/test/suite/test_sweep01.py @@ -106,24 +106,21 @@ class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): k = 0 sleep = 0 max = 60 - last_nfile = nfile1 + final_nfile = 4 while sleep < max: self.session.checkpoint() k = k+1 c[k] = 1 sleep += 2 time.sleep(2) - # Give slow machines time to process files. As long as progress - # is made, keep going. + # Give slow machines time to process files. stat_cursor = self.session.open_cursor('statistics:', None, None) this_nfile = stat_cursor[stat.conn.file_open][2] stat_cursor.close() self.pr("==== loop " + str(sleep)) - self.pr("last_nfile " + str(last_nfile)) self.pr("this_nfile " + str(this_nfile)) - if this_nfile == last_nfile and this_nfile < nfile1: + if this_nfile == final_nfile: break - last_nfile = this_nfile c.close() self.pr("Sweep loop took " + str(sleep)) @@ -191,7 +188,7 @@ class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): self.assertEqual(nfile2 < nfile1, True) # The only files that should be left are the metadata, the lookaside # file, the lock file, and the active file. - if (nfile2 != 4): + if (nfile2 != final_nfile): print "close1: " + str(close1) + " close2: " + str(close2) print "remove1: " + str(remove1) + " remove2: " + str(remove2) print "sweep1: " + str(sweep1) + " sweep2: " + str(sweep2) @@ -200,7 +197,7 @@ class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): print "tod1: " + str(tod1) + " tod2: " + str(tod2) print "ref1: " + str(ref1) + " ref2: " + str(ref2) print "XX2: nfile1: " + str(nfile1) + " nfile2: " + str(nfile2) - self.assertEqual(nfile2 == 4, True) + self.assertEqual(nfile2 == final_nfile, True) if __name__ == '__main__': wttest.run() -- cgit v1.2.1 From d870b48d42135f0a6ce06a8a174126bb2f55077d Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 16 Mar 2016 15:43:46 -0400 Subject: WT-2490: search_near() returns wrong key for column-store This reverts commit 6bf871c9cc0379a15bf1aa550df6a1ea2dfd4521. --- dist/s_string.ok | 1 - 1 file changed, 1 deletion(-) diff --git a/dist/s_string.ok b/dist/s_string.ok index 9a02537d055..43eb7861b23 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -467,7 +467,6 @@ comparep compat concat cond -condvar conf confchk config -- cgit v1.2.1 From be18e7cf192c5ef7a6b782370a3765a0883a5186 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Thu, 17 Mar 2016 10:42:15 +1100 Subject: WT-2492 Add WT_PTRDIFFT_FMT for print statements. Since there isn't a specifier that works on all platforms. --- src/config/config.c | 8 ++------ src/include/gcc.h | 1 + src/include/lint.h | 1 + src/include/msvc.h | 1 + 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/config/config.c b/src/config/config.c index a50c4d29f65..96ef7a4e62a 100644 --- a/src/config/config.c +++ b/src/config/config.c @@ -15,14 +15,10 @@ static int __config_err(WT_CONFIG *conf, const char *msg, int err) { - /* - * Cast the string offset to uintmax_t because the %td format to print - * a type ptrdiff_t isn't supported under MSVC. - */ WT_RET_MSG(conf->session, err, - "Error parsing '%.*s' at offset %" PRIuMAX ": %s", + "Error parsing '%.*s' at offset %" WT_PTRDIFFT_FMT ": %s", (int)(conf->end - conf->orig), conf->orig, - (uintmax_t)(conf->cur - conf->orig), msg); + conf->cur - conf->orig, msg); } /* diff --git a/src/include/gcc.h b/src/include/gcc.h index 6ccc0de3c03..ce6afdd6e9c 100644 --- a/src/include/gcc.h +++ b/src/include/gcc.h @@ -6,6 +6,7 @@ * See the file LICENSE for redistribution information. */ +#define WT_PTRDIFFT_FMT "td" /* ptrdiff_t format string */ #define WT_SIZET_FMT "zu" /* size_t format string */ /* Add GCC-specific attributes to types and function declarations. */ diff --git a/src/include/lint.h b/src/include/lint.h index f8b17022968..1b64186cbab 100644 --- a/src/include/lint.h +++ b/src/include/lint.h @@ -6,6 +6,7 @@ * See the file LICENSE for redistribution information. */ +#define WT_PTRDIFFT_FMT "td" /* ptrdiff_t format string */ #define WT_SIZET_FMT "zu" /* size_t format string */ #define WT_COMPILER_TYPE_ALIGN(x) diff --git a/src/include/msvc.h b/src/include/msvc.h index 99260a44875..d5be5bd8c60 100644 --- a/src/include/msvc.h +++ b/src/include/msvc.h @@ -13,6 +13,7 @@ #define inline __inline +#define WT_PTRDIFFT_FMT "Id" /* ptrdiff_t format string */ #define WT_SIZET_FMT "Iu" /* size_t format string */ /* -- cgit v1.2.1 From 0b8745ea5e5fedd4b173914add7746b99f66aec2 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Thu, 17 Mar 2016 10:56:30 +1100 Subject: WT-2426 Lock all handles for checkpoints in one pass. In particular, gather all locks (including exclusive locks on old checkpoints) under the schema / handle list lock so we never reenter those locks once we have a handle exclusive. This extends the duration that we hold exclusive locks on checkpoints to include the write leaves phase. If that turns out to be a problem, we could further split out write leaves into a separate pass (that releases all handles at the end). --- src/btree/bt_sync.c | 37 ++--- src/btree/bt_vrfy.c | 2 +- src/include/extern.h | 4 +- src/lsm/lsm_work_unit.c | 2 +- src/schema/schema_worker.c | 2 +- src/session/session_dhandle.c | 2 +- src/txn/txn_ckpt.c | 311 +++++++++++++++++++++++----------------- test/suite/test_checkpoint01.py | 2 +- 8 files changed, 195 insertions(+), 167 deletions(-) diff --git a/src/btree/bt_sync.c b/src/btree/bt_sync.c index 8b54087794f..57056eb5c99 100644 --- a/src/btree/bt_sync.c +++ b/src/btree/bt_sync.c @@ -269,24 +269,18 @@ err: /* On error, clear any left-over tree walk. */ * Cache operations. */ int -__wt_cache_op(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, WT_CACHE_OP op) +__wt_cache_op(WT_SESSION_IMPL *session, WT_CACHE_OP op) { - WT_DECL_RET; - WT_BTREE *btree; - - btree = S2BT(session); - switch (op) { case WT_SYNC_CHECKPOINT: case WT_SYNC_CLOSE: /* - * Set the checkpoint reference for reconciliation; it's ugly, - * but drilling a function parameter path from our callers to - * the reconciliation of the tree's root page is going to be - * worse. + * Make sure the checkpoint reference is set for + * reconciliation; it's ugly, but drilling a function parameter + * path from our callers to the reconciliation of the tree's + * root page is going to be worse. */ - WT_ASSERT(session, btree->ckpt == NULL); - btree->ckpt = ckptbase; + WT_ASSERT(session, S2BT(session)->ckpt != NULL); break; case WT_SYNC_DISCARD: case WT_SYNC_WRITE_LEAVES: @@ -296,23 +290,10 @@ __wt_cache_op(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, WT_CACHE_OP op) switch (op) { case WT_SYNC_CHECKPOINT: case WT_SYNC_WRITE_LEAVES: - WT_ERR(__sync_file(session, op)); - break; + return (__sync_file(session, op)); case WT_SYNC_CLOSE: case WT_SYNC_DISCARD: - WT_ERR(__wt_evict_file(session, op)); - break; + return (__wt_evict_file(session, op)); + WT_ILLEGAL_VALUE(session); } - -err: switch (op) { - case WT_SYNC_CHECKPOINT: - case WT_SYNC_CLOSE: - btree->ckpt = NULL; - break; - case WT_SYNC_DISCARD: - case WT_SYNC_WRITE_LEAVES: - break; - } - - return (ret); } diff --git a/src/btree/bt_vrfy.c b/src/btree/bt_vrfy.c index ae2c20be1b6..952298f2456 100644 --- a/src/btree/bt_vrfy.c +++ b/src/btree/bt_vrfy.c @@ -226,7 +226,7 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) WT_WITH_PAGE_INDEX(session, ret = __verify_tree(session, &btree->root, vs)); - WT_TRET(__wt_cache_op(session, NULL, WT_SYNC_DISCARD)); + WT_TRET(__wt_cache_op(session, WT_SYNC_DISCARD)); } /* Unload the checkpoint. */ diff --git a/src/include/extern.h b/src/include/extern.h index 6f5d3ff66d5..48c52d4a109 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -168,7 +168,7 @@ extern int __wt_split_multi(WT_SESSION_IMPL *session, WT_REF *ref, int closing); extern int __wt_split_reverse(WT_SESSION_IMPL *session, WT_REF *ref); extern int __wt_split_rewrite(WT_SESSION_IMPL *session, WT_REF *ref); extern int __wt_btree_stat_init(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst); -extern int __wt_cache_op(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, WT_CACHE_OP op); +extern int __wt_cache_op(WT_SESSION_IMPL *session, WT_CACHE_OP op); extern int __wt_upgrade(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_verify_dsk_image(WT_SESSION_IMPL *session, const char *tag, const WT_PAGE_HEADER *dsk, size_t size, bool empty_page_ok); @@ -754,7 +754,7 @@ extern void __wt_txn_destroy(WT_SESSION_IMPL *session); extern int __wt_txn_global_init(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_txn_global_destroy(WT_SESSION_IMPL *session); extern int __wt_checkpoint_name_ok(WT_SESSION_IMPL *session, const char *name, size_t len); -extern int __wt_checkpoint_list(WT_SESSION_IMPL *session, const char *cfg[]); +extern int __wt_checkpoint_get_handles(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_checkpoint_sync(WT_SESSION_IMPL *session, const char *cfg[]); diff --git a/src/lsm/lsm_work_unit.c b/src/lsm/lsm_work_unit.c index 88054f86d65..545e711f4ad 100644 --- a/src/lsm/lsm_work_unit.c +++ b/src/lsm/lsm_work_unit.c @@ -322,7 +322,7 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, */ saved_isolation = session->txn.isolation; session->txn.isolation = WT_ISO_READ_UNCOMMITTED; - ret = __wt_cache_op(session, NULL, WT_SYNC_WRITE_LEAVES); + ret = __wt_cache_op(session, WT_SYNC_WRITE_LEAVES); session->txn.isolation = saved_isolation; WT_TRET(__wt_session_release_btree(session)); } diff --git a/src/schema/schema_worker.c b/src/schema/schema_worker.c index e60a7107786..52be76bb7a5 100644 --- a/src/schema/schema_worker.c +++ b/src/schema/schema_worker.c @@ -126,7 +126,7 @@ __wt_schema_worker(WT_SESSION_IMPL *session, dsrc, wt_session, uri, (WT_CONFIG_ARG *)cfg)); else if (file_func == __wt_checkpoint) ; - else if (file_func == __wt_checkpoint_list) + else if (file_func == __wt_checkpoint_get_handles) ; else if (file_func == __wt_checkpoint_sync) ; diff --git a/src/session/session_dhandle.c b/src/session/session_dhandle.c index 242d9ac5cc4..ddf4d3dfa33 100644 --- a/src/session/session_dhandle.c +++ b/src/session/session_dhandle.c @@ -577,7 +577,7 @@ __wt_session_lock_checkpoint(WT_SESSION_IMPL *session, const char *checkpoint) * files, since changes to the underlying file are visible to the in * memory pages. */ - WT_ERR(__wt_cache_op(session, NULL, WT_SYNC_DISCARD)); + WT_ERR(__wt_cache_op(session, WT_SYNC_DISCARD)); /* * We lock checkpoint handles that we are overwriting, so the handle diff --git a/src/txn/txn_ckpt.c b/src/txn/txn_ckpt.c index de8f0d84951..09d5f54e801 100644 --- a/src/txn/txn_ckpt.c +++ b/src/txn/txn_ckpt.c @@ -8,6 +8,10 @@ #include "wt_internal.h" +static int __checkpoint_lock_tree( + WT_SESSION_IMPL *, bool, bool, const char *[]); +static int __checkpoint_tree_helper(WT_SESSION_IMPL *, const char *[]); + /* * __wt_checkpoint_name_ok -- * Complain if the checkpoint name isn't acceptable. @@ -224,11 +228,11 @@ __checkpoint_data_source(WT_SESSION_IMPL *session, const char *cfg[]) } /* - * __wt_checkpoint_list -- + * __wt_checkpoint_get_handles -- * Get a list of handles to flush. */ int -__wt_checkpoint_list(WT_SESSION_IMPL *session, const char *cfg[]) +__wt_checkpoint_get_handles(WT_SESSION_IMPL *session, const char *cfg[]) { WT_DECL_RET; const char *name; @@ -254,6 +258,13 @@ __wt_checkpoint_list(WT_SESSION_IMPL *session, const char *cfg[]) if ((ret = __wt_session_get_btree(session, name, NULL, NULL, 0)) != 0) return (ret == EBUSY ? 0 : ret); + WT_SAVE_DHANDLE(session, + ret = __checkpoint_lock_tree(session, true, true, cfg)); + if (ret != 0) { + WT_TRET(__wt_session_release_btree(session)); + return (ret); + } + session->ckpt_handle[session->ckpt_handle_next++] = session->dhandle; return (0); } @@ -267,7 +278,7 @@ __checkpoint_write_leaves(WT_SESSION_IMPL *session, const char *cfg[]) { WT_UNUSED(cfg); - return (__wt_cache_op(session, NULL, WT_SYNC_WRITE_LEAVES)); + return (__wt_cache_op(session, WT_SYNC_WRITE_LEAVES)); } /* @@ -371,15 +382,20 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) /* Configure logging only if doing a full checkpoint. */ logging = FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED); + /* Keep track of handles acquired for locking. */ + WT_ERR(__wt_meta_track_on(session)); + tracking = true; + /* * Get a list of handles we want to flush; this may pull closed objects * into the session cache, but we're going to do that eventually anyway. */ + WT_ASSERT(session, session->ckpt_handle_next == 0); WT_WITH_SCHEMA_LOCK(session, ret, WT_WITH_TABLE_LOCK(session, ret, WT_WITH_HANDLE_LIST_LOCK(session, ret = __checkpoint_apply_all( - session, cfg, __wt_checkpoint_list, NULL)))); + session, cfg, __wt_checkpoint_get_handles, NULL)))); WT_ERR(ret); /* @@ -410,10 +426,6 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) */ WT_ERR(__checkpoint_apply(session, cfg, __wt_checkpoint_sync)); - /* Start the checkpoint for real. */ - WT_ERR(__wt_meta_track_on(session)); - tracking = true; - /* Tell logging that we are about to start a database checkpoint. */ if (full && logging) WT_ERR(__wt_txn_checkpoint_log( @@ -426,6 +438,8 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR(__wt_epoch(session, &start)); /* + * Start the checkpoint for real. + * * Bump the global checkpoint generation, used to figure out whether * checkpoint has visited a tree. There is no need for this to be * atomic: it is only written while holding the checkpoint lock. @@ -489,7 +503,7 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR(__wt_txn_checkpoint_log( session, full, WT_TXN_LOG_CKPT_START, NULL)); - WT_ERR(__checkpoint_apply(session, cfg, __wt_checkpoint)); + WT_ERR(__checkpoint_apply(session, cfg, __checkpoint_tree_helper)); /* * Clear the dhandle so the visibility check doesn't get confused about @@ -752,14 +766,13 @@ __drop_to(WT_CKPT *ckptbase, const char *name, size_t len) } /* - * __checkpoint_worker -- - * Checkpoint a tree. + * __checkpoint_lock_tree -- + * Acquire the locks required to checkpoint a tree. */ static int -__checkpoint_worker(WT_SESSION_IMPL *session, - const char *cfg[], bool is_checkpoint, bool need_tracking) +__checkpoint_lock_tree(WT_SESSION_IMPL *session, + bool is_checkpoint, bool need_tracking, const char *cfg[]) { - WT_BM *bm; WT_BTREE *btree; WT_CKPT *ckpt, *ckptbase; WT_CONFIG dropconf; @@ -767,19 +780,15 @@ __checkpoint_worker(WT_SESSION_IMPL *session, WT_CONNECTION_IMPL *conn; WT_DATA_HANDLE *dhandle; WT_DECL_RET; - WT_LSN ckptlsn; - int deleted, was_modified; - bool fake_ckpt, force, hot_backup_locked; - const char *name; char *name_alloc; + const char *name; + bool hot_backup_locked; btree = S2BT(session); - bm = btree->bm; conn = S2C(session); ckpt = ckptbase = NULL; dhandle = session->dhandle; - was_modified = btree->modified; - fake_ckpt = hot_backup_locked = false; + hot_backup_locked = false; name_alloc = NULL; /* @@ -798,15 +807,6 @@ __checkpoint_worker(WT_SESSION_IMPL *session, WT_ASSERT(session, !need_tracking || WT_IS_METADATA(session, dhandle) || WT_META_TRACKING(session)); - /* - * Set the checkpoint LSN to the maximum LSN so that if logging is - * disabled, recovery will never roll old changes forward over the - * non-logged changes in this checkpoint. If logging is enabled, a - * real checkpoint LSN will be assigned later for this checkpoint and - * overwrite this. - */ - WT_MAX_LSN(&ckptlsn); - /* Get the list of checkpoints for this file. */ WT_RET(__wt_meta_ckptlist_get(session, dhandle->name, &ckptbase)); @@ -857,70 +857,6 @@ __checkpoint_worker(WT_SESSION_IMPL *session, /* Drop checkpoints with the same name as the one we're taking. */ __drop(ckptbase, name, strlen(name)); - /* - * Check for clean objects not requiring a checkpoint. - * - * If we're closing a handle, and the object is clean, we can skip the - * checkpoint, whatever checkpoints we have are sufficient. (We might - * not have any checkpoints if the object was never modified, and that's - * OK: the object creation code doesn't mark the tree modified so we can - * skip newly created trees here.) - * - * If the application repeatedly checkpoints an object (imagine hourly - * checkpoints using the same explicit or internal name), there's no - * reason to repeat the checkpoint for clean objects. The test is if - * the only checkpoint we're deleting is the last one in the list and - * it has the same name as the checkpoint we're about to take, skip the - * work. (We can't skip checkpoints that delete more than the last - * checkpoint because deleting those checkpoints might free up space in - * the file.) This means an application toggling between two (or more) - * checkpoint names will repeatedly take empty checkpoints, but that's - * not likely enough to make detection worthwhile. - * - * Checkpoint read-only objects otherwise: the application must be able - * to open the checkpoint in a cursor after taking any checkpoint, which - * means it must exist. - */ - force = false; - F_CLR(btree, WT_BTREE_SKIP_CKPT); - if (!btree->modified && cfg != NULL) { - ret = __wt_config_gets(session, cfg, "force", &cval); - if (ret != 0 && ret != WT_NOTFOUND) - WT_ERR(ret); - if (ret == 0 && cval.val != 0) - force = true; - } - if (!btree->modified && !force) { - if (!is_checkpoint) - goto nockpt; - - deleted = 0; - WT_CKPT_FOREACH(ckptbase, ckpt) - if (F_ISSET(ckpt, WT_CKPT_DELETE)) - ++deleted; - /* - * Complicated test: if the last checkpoint in the object has - * the same name as the checkpoint we're taking (correcting for - * internal checkpoint names with their generational suffix - * numbers), we can skip the checkpoint, there's nothing to do. - * The exception is if we're deleting two or more checkpoints: - * then we may save space. - */ - if (ckpt > ckptbase && - (strcmp(name, (ckpt - 1)->name) == 0 || - (WT_PREFIX_MATCH(name, WT_CHECKPOINT) && - WT_PREFIX_MATCH((ckpt - 1)->name, WT_CHECKPOINT))) && - deleted < 2) { -nockpt: F_SET(btree, WT_BTREE_SKIP_CKPT); - WT_PUBLISH(btree->checkpoint_gen, - S2C(session)->txn_global.checkpoint_gen); - WT_STAT_FAST_DATA_SET(session, - btree_checkpoint_generation, - btree->checkpoint_gen); - goto done; - } - } - /* Add a new checkpoint entry at the end of the list. */ WT_CKPT_FOREACH(ckptbase, ckpt) ; @@ -1005,32 +941,119 @@ nockpt: F_SET(btree, WT_BTREE_SKIP_CKPT); * copy instead of forcing checkpoints on clean objects to associate * names with checkpoints. */ - if (is_checkpoint) - switch (F_MASK(btree, WT_BTREE_SPECIAL_FLAGS)) { - case 0: - break; - case WT_BTREE_BULK: - /* - * The only checkpoints a bulk-loaded file should have - * are fake ones we created without the underlying block - * manager. I'm leaving this code here because it's a - * cheap test and a nasty race. - */ - WT_CKPT_FOREACH(ckptbase, ckpt) - if (!F_ISSET(ckpt, WT_CKPT_ADD | WT_CKPT_FAKE)) - WT_ERR_MSG(session, ret, - "block-manager checkpoint found " - "for a bulk-loaded file"); - fake_ckpt = true; - goto fake; - case WT_BTREE_REBALANCE: - case WT_BTREE_SALVAGE: - case WT_BTREE_UPGRADE: - case WT_BTREE_VERIFY: - WT_ERR_MSG(session, EINVAL, - "checkpoints are blocked during rebalance, " - "salvage, upgrade or verify operations"); + WT_ASSERT(session, + !is_checkpoint || !F_ISSET(btree, WT_BTREE_SPECIAL_FLAGS)); + + hot_backup_locked = false; + WT_ERR(__wt_readunlock(session, conn->hot_backup_lock)); + + WT_ASSERT(session, btree->ckpt == NULL); + btree->ckpt = ckptbase; + + return (0); + +err: if (hot_backup_locked) + WT_TRET(__wt_readunlock(session, conn->hot_backup_lock)); + + __wt_meta_ckptlist_free(session, ckptbase); + __wt_free(session, name_alloc); + + return (ret); +} + +/* + * __checkpoint_tree -- + * Checkpoint a single tree. + * Assumes all necessary locks have been acquired by the caller. + */ +static int +__checkpoint_tree( + WT_SESSION_IMPL *session, bool is_checkpoint, const char *cfg[]) +{ + WT_BM *bm; + WT_BTREE *btree; + WT_CKPT *ckpt, *ckptbase; + WT_CONFIG_ITEM cval; + WT_CONNECTION_IMPL *conn; + WT_DATA_HANDLE *dhandle; + WT_DECL_RET; + WT_LSN ckptlsn; + const char *name; + int deleted, was_modified; + bool fake_ckpt, force; + + btree = S2BT(session); + bm = btree->bm; + ckptbase = btree->ckpt; + conn = S2C(session); + dhandle = session->dhandle; + fake_ckpt = false; + was_modified = btree->modified; + + /* + * Check for clean objects not requiring a checkpoint. + * + * If we're closing a handle, and the object is clean, we can skip the + * checkpoint, whatever checkpoints we have are sufficient. (We might + * not have any checkpoints if the object was never modified, and that's + * OK: the object creation code doesn't mark the tree modified so we can + * skip newly created trees here.) + * + * If the application repeatedly checkpoints an object (imagine hourly + * checkpoints using the same explicit or internal name), there's no + * reason to repeat the checkpoint for clean objects. The test is if + * the only checkpoint we're deleting is the last one in the list and + * it has the same name as the checkpoint we're about to take, skip the + * work. (We can't skip checkpoints that delete more than the last + * checkpoint because deleting those checkpoints might free up space in + * the file.) This means an application toggling between two (or more) + * checkpoint names will repeatedly take empty checkpoints, but that's + * not likely enough to make detection worthwhile. + * + * Checkpoint read-only objects otherwise: the application must be able + * to open the checkpoint in a cursor after taking any checkpoint, which + * means it must exist. + */ + force = false; + F_CLR(btree, WT_BTREE_SKIP_CKPT); + if (!btree->modified && cfg != NULL) { + ret = __wt_config_gets(session, cfg, "force", &cval); + if (ret != 0 && ret != WT_NOTFOUND) + WT_ERR(ret); + if (ret == 0 && cval.val != 0) + force = true; + } + if (!btree->modified && !force) { + if (!is_checkpoint) + goto nockpt; + + deleted = 0; + WT_CKPT_FOREACH(ckptbase, ckpt) + if (F_ISSET(ckpt, WT_CKPT_DELETE)) + ++deleted; + /* + * Complicated test: if the tree is clean and last two + * checkpoints have the same name (correcting for internal + * checkpoint names with their generational suffix numbers), we + * can skip the checkpoint, there's nothing to do. The + * exception is if we're deleting two or more checkpoints: then + * we may save space. + */ + name = (ckpt - 1)->name; + if (ckpt > ckptbase + 1 && deleted < 2 && + (strcmp(name, (ckpt - 2)->name) == 0 || + (WT_PREFIX_MATCH(name, WT_CHECKPOINT) && + WT_PREFIX_MATCH((ckpt - 2)->name, WT_CHECKPOINT)))) { +nockpt: F_SET(btree, WT_BTREE_SKIP_CKPT); + WT_PUBLISH(btree->checkpoint_gen, + S2C(session)->txn_global.checkpoint_gen); + WT_STAT_FAST_DATA_SET(session, + btree_checkpoint_generation, + btree->checkpoint_gen); + ret = 0; + goto err; } + } /* * If an object has never been used (in other words, if it could become @@ -1077,6 +1100,15 @@ nockpt: F_SET(btree, WT_BTREE_SKIP_CKPT); btree->modified = 0; WT_FULL_BARRIER(); + /* + * Set the checkpoint LSN to the maximum LSN so that if logging is + * disabled, recovery will never roll old changes forward over the + * non-logged changes in this checkpoint. If logging is enabled, a + * real checkpoint LSN will be assigned for this checkpoint and + * overwrite this. + */ + WT_MAX_LSN(&ckptlsn); + /* Tell logging that a file checkpoint is starting. */ if (FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) WT_ERR(__wt_txn_checkpoint_log( @@ -1084,9 +1116,9 @@ nockpt: F_SET(btree, WT_BTREE_SKIP_CKPT); /* Flush the file from the cache, creating the checkpoint. */ if (is_checkpoint) - WT_ERR(__wt_cache_op(session, ckptbase, WT_SYNC_CHECKPOINT)); + WT_ERR(__wt_cache_op(session, WT_SYNC_CHECKPOINT)); else - WT_ERR(__wt_cache_op(session, ckptbase, WT_SYNC_CLOSE)); + WT_ERR(__wt_cache_op(session, WT_SYNC_CLOSE)); /* * All blocks being written have been written; set the object's write @@ -1144,7 +1176,6 @@ fake: /* WT_ERR(__wt_txn_checkpoint_log( session, false, WT_TXN_LOG_CKPT_STOP, NULL)); -done: err: /* * If the checkpoint didn't complete successfully, make sure the * tree is marked dirty. @@ -1152,15 +1183,22 @@ err: /* if (ret != 0 && !btree->modified && was_modified) btree->modified = 1; - if (hot_backup_locked) - WT_TRET(__wt_readunlock(session, conn->hot_backup_lock)); - __wt_meta_ckptlist_free(session, ckptbase); - __wt_free(session, name_alloc); + btree->ckpt = NULL; return (ret); } +/* + * __checkpoint_tree_helper -- + * Checkpoint a tree (suitable for use in *_apply functions). + */ +static int +__checkpoint_tree_helper(WT_SESSION_IMPL *session, const char *cfg[]) +{ + return (__checkpoint_tree(session, true, cfg)); +} + /* * __wt_checkpoint -- * Checkpoint a file. @@ -1168,14 +1206,19 @@ err: /* int __wt_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) { + WT_DECL_RET; + /* Should not be called with a checkpoint handle. */ WT_ASSERT(session, session->dhandle->checkpoint == NULL); - /* Should be holding the schema lock. */ + /* We must hold the metadata lock if checkpointing the metadata. */ WT_ASSERT(session, !WT_IS_METADATA(session, session->dhandle) || F_ISSET(session, WT_SESSION_LOCKED_METADATA)); - return (__checkpoint_worker(session, cfg, true, true)); + WT_SAVE_DHANDLE(session, + ret = __checkpoint_lock_tree(session, true, true, cfg)); + WT_RET(ret); + return (__checkpoint_tree(session, true, cfg)); } /* @@ -1225,7 +1268,7 @@ __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) if (F_ISSET(btree, WT_BTREE_NO_CHECKPOINT)) F_SET(session->dhandle, WT_DHANDLE_DEAD); if (F_ISSET(session->dhandle, WT_DHANDLE_DEAD)) - return (__wt_cache_op(session, NULL, WT_SYNC_DISCARD)); + return (__wt_cache_op(session, WT_SYNC_DISCARD)); /* * If closing an unmodified file, check that no update is required @@ -1234,7 +1277,7 @@ __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) if (!btree->modified && !bulk) { __wt_txn_update_oldest(session, true); return (__wt_txn_visible_all(session, btree->rec_max_txn) ? - __wt_cache_op(session, NULL, WT_SYNC_DISCARD) : EBUSY); + __wt_cache_op(session, WT_SYNC_DISCARD) : EBUSY); } /* @@ -1248,10 +1291,14 @@ __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) if (need_tracking) WT_RET(__wt_meta_track_on(session)); - WT_TRET(__checkpoint_worker(session, NULL, false, need_tracking)); + WT_SAVE_DHANDLE(session, + ret = __checkpoint_lock_tree(session, false, need_tracking, NULL)); + WT_ASSERT(session, ret == 0); + if (ret == 0) + ret = __checkpoint_tree(session, false, NULL); if (need_tracking) - WT_RET(__wt_meta_track_off(session, true, ret != 0)); + WT_TRET(__wt_meta_track_off(session, true, ret != 0)); return (ret); } diff --git a/test/suite/test_checkpoint01.py b/test/suite/test_checkpoint01.py index 9955944f73d..6e1ad7814ed 100644 --- a/test/suite/test_checkpoint01.py +++ b/test/suite/test_checkpoint01.py @@ -185,7 +185,7 @@ class test_checkpoint_cursor(wttest.WiredTigerTestCase): # Check dropping all checkpoints fails. msg = '/checkpoints cannot be dropped/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.session.checkpoint("name=checkpoint-2"), msg) + lambda: self.session.checkpoint("force,name=checkpoint-2"), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.checkpoint("drop=(checkpoint-2)"), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, -- cgit v1.2.1 From 18aac3b0f142dc0d5bc644343f0e8cb2a63503e8 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Thu, 17 Mar 2016 15:44:07 +1100 Subject: WT-2495 Make malloc calls use tcmalloc when configured. Otherwise we can free memoy with tcmalloc that was allocated with the system allocator. --- src/os_posix/os_alloc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/os_posix/os_alloc.c b/src/os_posix/os_alloc.c index 7eef9838107..cfc7b80450e 100644 --- a/src/os_posix/os_alloc.c +++ b/src/os_posix/os_alloc.c @@ -18,6 +18,7 @@ #include #define calloc tc_calloc +#define malloc tc_malloc #define realloc tc_realloc #define posix_memalign tc_posix_memalign #define free tc_free -- cgit v1.2.1 From b15c7d5d682305e3b4088e19b4171c45556598d6 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Thu, 17 Mar 2016 16:13:00 +1100 Subject: WT-2457 Review changes. Rename LSM field "excl_session" to match WT_DATA_HANDLE, make new test URI match test name. --- src/include/lsm.h | 2 +- src/lsm/lsm_cursor.c | 2 +- src/lsm/lsm_tree.c | 14 +++++++------- test/suite/test_lsm03.py | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/include/lsm.h b/src/include/lsm.h index 7251b43d2bd..eacb4d52d3e 100644 --- a/src/include/lsm.h +++ b/src/include/lsm.h @@ -179,7 +179,7 @@ struct __wt_lsm_tree { int collator_owned; uint32_t refcnt; /* Number of users of the tree */ - WT_SESSION_IMPL *exclusive; /* Tree is locked exclusively */ + WT_SESSION_IMPL *excl_session; /* Session has exclusive lock */ #define LSM_TREE_MAX_QUEUE 100 uint32_t queue_ref; diff --git a/src/lsm/lsm_cursor.c b/src/lsm/lsm_cursor.c index 0197b6481f4..e023b2b407e 100644 --- a/src/lsm/lsm_cursor.c +++ b/src/lsm/lsm_cursor.c @@ -1556,7 +1556,7 @@ __wt_clsm_open(WT_SESSION_IMPL *session, WT_ERR(ret); /* Make sure we have exclusive access if and only if we want it */ - WT_ASSERT(session, !bulk || lsm_tree->exclusive); + WT_ASSERT(session, !bulk || lsm_tree->excl_session != NULL); WT_ERR(__wt_calloc_one(session, &clsm)); diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index 48648b99d2f..f5c51a3fbda 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -369,7 +369,7 @@ __lsm_tree_find(WT_SESSION_IMPL *session, * exclusive flag. */ if (!__wt_atomic_cas_ptr( - &lsm_tree->exclusive, NULL, session)) + &lsm_tree->excl_session, NULL, session)) return (EBUSY); /* @@ -381,7 +381,7 @@ __lsm_tree_find(WT_SESSION_IMPL *session, !__wt_atomic_cas32( &lsm_tree->refcnt, 0, 1)) { F_SET(lsm_tree, WT_LSM_TREE_ACTIVE); - lsm_tree->exclusive = NULL; + lsm_tree->excl_session = NULL; return (EBUSY); } } else { @@ -391,7 +391,7 @@ __lsm_tree_find(WT_SESSION_IMPL *session, * We got a reference, check if an exclusive * lock beat us to it. */ - if (lsm_tree->exclusive != NULL) { + if (lsm_tree->excl_session != NULL) { WT_ASSERT(session, lsm_tree->refcnt > 0); (void)__wt_atomic_sub32( @@ -487,7 +487,7 @@ __lsm_tree_open(WT_SESSION_IMPL *session, * with getting handles exclusive. */ lsm_tree->refcnt = 1; - lsm_tree->exclusive = exclusive ? session : NULL; + lsm_tree->excl_session = exclusive ? session : NULL; lsm_tree->queue_ref = 0; /* Set a flush timestamp as a baseline. */ @@ -522,7 +522,7 @@ __wt_lsm_tree_get(WT_SESSION_IMPL *session, ret = __lsm_tree_open(session, uri, exclusive, treep); WT_ASSERT(session, ret != 0 || - (exclusive ? session : NULL) == (*treep)->exclusive); + (*treep)->excl_session == (exclusive ? session : NULL)); return (ret); } @@ -534,10 +534,10 @@ void __wt_lsm_tree_release(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) { WT_ASSERT(session, lsm_tree->refcnt > 0); - if (lsm_tree->exclusive == session) { + if (lsm_tree->excl_session == session) { /* We cleared the active flag when getting exclusive access. */ F_SET(lsm_tree, WT_LSM_TREE_ACTIVE); - lsm_tree->exclusive = NULL; + lsm_tree->excl_session = NULL; } (void)__wt_atomic_sub32(&lsm_tree->refcnt, 1); } diff --git a/test/suite/test_lsm03.py b/test/suite/test_lsm03.py index 37d1f761744..448d864c646 100644 --- a/test/suite/test_lsm03.py +++ b/test/suite/test_lsm03.py @@ -33,14 +33,14 @@ from helper import simple_populate # Check to make sure that LSM schema operations don't get EBUSY when # there are no user operations active. class test_lsm03(wttest.WiredTigerTestCase): - name = 'test_rebalance' + name = 'test_lsm03' # Use small pages so we generate some internal layout # Setup LSM so multiple chunks are present config = 'key_format=S,allocation_size=512,internal_page_max=512' + \ ',leaf_page_max=1k,lsm=(chunk_size=512k,merge_min=10)' - # Populate an object, then rebalance it. + # Populate an object then drop it. def test_lsm_drop_active(self): uri = 'lsm:' + self.name simple_populate(self, uri, self.config, 10000) -- cgit v1.2.1 From 20b1a6ed80bd342a957c91acd090760de6ba3d24 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 17 Mar 2016 10:39:32 -0400 Subject: WT-2497 Create second backup copy. --- test/format/backup.c | 7 +++++++ test/format/format.h | 1 + test/format/util.c | 14 ++++++++++---- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/test/format/backup.c b/test/format/backup.c index 56657940514..2b1463bd0e3 100644 --- a/test/format/backup.c +++ b/test/format/backup.c @@ -67,6 +67,13 @@ copy_file(const char *name) "cp %s/%s %s/%s", g.home, name, g.home_backup, name); testutil_checkfmt(system(cmd), "backup copy: %s", cmd); free(cmd); + + len = strlen(g.home) + strlen(g.home_backup2) + strlen(name) * 2 + 20; + cmd = dmalloc(len); + (void)snprintf(cmd, len, + "cp %s/%s %s/%s", g.home, name, g.home_backup2, name); + testutil_checkfmt(system(cmd), "backup copy: %s", cmd); + free(cmd); } /* diff --git a/test/format/format.h b/test/format/format.h index c54fd061736..a129c5395fd 100644 --- a/test/format/format.h +++ b/test/format/format.h @@ -109,6 +109,7 @@ typedef struct { char *home; /* Home directory */ char *home_backup; /* Hot-backup directory */ + char *home_backup2; /* Saved Hot-backup directory */ char *home_backup_init; /* Initialize backup command */ char *home_bdb; /* BDB directory */ char *home_config; /* Run CONFIG file path */ diff --git a/test/format/util.c b/test/format/util.c index 347b2ea1db3..2e4c869366c 100644 --- a/test/format/util.c +++ b/test/format/util.c @@ -310,6 +310,10 @@ path_setup(const char *home) g.home_backup = dmalloc(len); snprintf(g.home_backup, len, "%s/%s", g.home, "BACKUP"); + len = strlen(g.home) + strlen("BACKUP2") + 2; + g.home_backup2 = dmalloc(len); + snprintf(g.home_backup2, len, "%s/%s", g.home, "BACKUP2"); + /* BDB directory. */ len = strlen(g.home) + strlen("bdb") + 2; g.home_bdb = dmalloc(len); @@ -340,13 +344,15 @@ path_setup(const char *home) /* Backup directory initialize command, remove and re-create it. */ #undef CMD #ifdef _WIN32 -#define CMD "del /s /q >:nul && mkdir %s" +#define CMD "del /s /q >:nul && mkdir %s %s" #else -#define CMD "rm -rf %s && mkdir %s" +#define CMD "rm -rf %s %s && mkdir %s %s" #endif - len = strlen(g.home_backup) * 2 + strlen(CMD) + 1; + len = strlen(g.home_backup) * 2 + + strlen(g.home_backup2) * 2 + strlen(CMD) + 1; g.home_backup_init = dmalloc(len); - snprintf(g.home_backup_init, len, CMD, g.home_backup, g.home_backup); + snprintf(g.home_backup_init, len, CMD, g.home_backup, g.home_backup2, + g.home_backup, g.home_backup2); /* * Salvage command, save the interesting files so we can replay the -- cgit v1.2.1 From 5b4e57c4b5d3651a6d8a7d5fbc63f32a345b6e17 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Fri, 18 Mar 2016 14:18:38 +1100 Subject: WT-2498 Fix a bug in LSM where table drop could hang. If there are open cursors. --- src/lsm/lsm_tree.c | 22 ++++++++++++++-------- test/suite/test_drop.py | 15 +++++++-------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index e79d52c18e9..bd9d0671bde 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -85,7 +85,7 @@ __lsm_tree_discard(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool final) * Close an LSM tree structure. */ static int -__lsm_tree_close(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) +__lsm_tree_close(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool wait) { WT_DECL_RET; int i; @@ -94,10 +94,13 @@ __lsm_tree_close(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) F_CLR(lsm_tree, WT_LSM_TREE_ACTIVE); /* - * Wait for all LSM operations and work units that were in flight to - * finish. + * Wait for all LSM operations to drain. If WiredTiger is shutting + * down also wait for the tree reference count to go to zero, otherwise + * we know a user is holding a reference to the tree, so exclusive + * access is not available. */ - for (i = 0; lsm_tree->refcnt > 1 || lsm_tree->queue_ref > 0; ++i) { + for (i = 0; + lsm_tree->refcnt > 1 && (wait || lsm_tree->queue_ref > 0); ++i) { /* * Remove any work units from the manager queues. Do this step * repeatedly in case a work unit was in the process of being @@ -145,7 +148,7 @@ __wt_lsm_tree_close_all(WT_SESSION_IMPL *session) * is unconditional. */ (void)__wt_atomic_add32(&lsm_tree->refcnt, 1); - WT_TRET(__lsm_tree_close(session, lsm_tree)); + WT_TRET(__lsm_tree_close(session, lsm_tree, true)); WT_TRET(__lsm_tree_discard(session, lsm_tree, true)); } @@ -380,9 +383,12 @@ __lsm_tree_find(WT_SESSION_IMPL *session, * open cursors - otherwise we can generate * spurious busy returns. */ - if (__lsm_tree_close(session, lsm_tree) != 0 || - !__wt_atomic_cas32( - &lsm_tree->refcnt, 0, 1)) { + (void)__wt_atomic_add32(&lsm_tree->refcnt, 1); + if (__lsm_tree_close( + session, lsm_tree, false) != 0 || + lsm_tree->refcnt != 1) { + (void)__wt_atomic_sub32( + &lsm_tree->refcnt, 1); F_SET(lsm_tree, WT_LSM_TREE_ACTIVE); lsm_tree->excl_session = NULL; return (EBUSY); diff --git a/test/suite/test_drop.py b/test/suite/test_drop.py index 5663b85d661..52ea7251ab5 100644 --- a/test/suite/test_drop.py +++ b/test/suite/test_drop.py @@ -41,12 +41,11 @@ class test_drop(wttest.WiredTigerTestCase): scenarios = check_scenarios([ ('file', dict(uri='file:')), ('table', dict(uri='table:')), - #Not yet: drop failing with an open cursor needs handle locking - #('table-lsm', dict(uri='table:', extra_config=',type=lsm')), + ('table-lsm', dict(uri='table:', extra_config=',type=lsm')), ]) # Populate an object, remove it and confirm it no longer exists. - def drop(self, populate, with_cursor, close_session, drop_index): + def drop(self, populate, with_cursor, reopen, drop_index): uri = self.uri + self.name populate(self, uri, 'key_format=S' + self.extra_config, 10) @@ -57,7 +56,7 @@ class test_drop(wttest.WiredTigerTestCase): lambda: self.session.drop(uri, None)) cursor.close() - if close_session: + if reopen: self.reopen_conn() if drop_index: @@ -73,17 +72,17 @@ class test_drop(wttest.WiredTigerTestCase): # Try all combinations except dropping the index, the simple # case has no indices. for with_cursor in [False, True]: - for close_session in [False, True]: - self.drop(simple_populate, with_cursor, close_session, False) + for reopen in [False, True]: + self.drop(simple_populate, with_cursor, reopen, False) # A complex, multi-file table object. # Try all test combinations. if self.uri == "table:": for with_cursor in [False, True]: - for close_session in [False, True]: + for reopen in [False, True]: for drop_index in [False, True]: self.drop(complex_populate, with_cursor, - close_session, drop_index) + reopen, drop_index) # Test drop of a non-existent object: force succeeds, without force fails. def test_drop_dne(self): -- cgit v1.2.1 From 558672a31ded61187599dbf545a4e9f42ba0dd37 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Fri, 18 Mar 2016 14:28:45 +1100 Subject: WT-2496 Fix a race between checkpoint and starting a hot backup. In particular, a checkpoint could mark and lock an old checkpoint for deletion before a hotbackup starts. The hotbackup could then write metadata referencing that old backup which the checkpoint goes on to delete. The fix is crude: don't allow a hotbackup to start while a checkpoint is in progress. --- src/cursor/cur_backup.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/cursor/cur_backup.c b/src/cursor/cur_backup.c index b097a8c08aa..2fb0c464a76 100644 --- a/src/cursor/cur_backup.c +++ b/src/cursor/cur_backup.c @@ -140,8 +140,9 @@ __wt_curbackup_open(WT_SESSION_IMPL *session, * Start the backup and fill in the cursor's list. Acquire the schema * lock, we need a consistent view when creating a copy. */ - WT_WITH_SCHEMA_LOCK(session, ret, - ret = __backup_start(session, cb, cfg)); + WT_WITH_CHECKPOINT_LOCK(session, ret, + WT_WITH_SCHEMA_LOCK(session, ret, + ret = __backup_start(session, cb, cfg))); WT_ERR(ret); /* __wt_cursor_init is last so we don't have to clean up on error. */ -- cgit v1.2.1 From 0eeaedd5c2d1158bdd734fad9456cb52181c5e2b Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Fri, 18 Mar 2016 15:08:25 +1100 Subject: WT-2498 Fix LSM close logic. Always wait for internal operations, optionally wait for application operations. --- src/lsm/lsm_tree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index bd9d0671bde..b64b72256cc 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -100,7 +100,7 @@ __lsm_tree_close(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool wait) * access is not available. */ for (i = 0; - lsm_tree->refcnt > 1 && (wait || lsm_tree->queue_ref > 0); ++i) { + lsm_tree->queue_ref > 0 || (wait && lsm_tree->refcnt > 1); ++i) { /* * Remove any work units from the manager queues. Do this step * repeatedly in case a work unit was in the process of being -- cgit v1.2.1 From b5a7e1742035dceeb842cd465fa09338e5a72cfa Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Mon, 21 Mar 2016 12:16:18 +1100 Subject: WT-2499 Fix a race in LSM between setting and checking WT_LSM_TREE_ACTIVE. Tree close relies on any state change being visible immediately. Found by inspection, but could cause the symptom seen in this ticket. --- src/include/lsm.h | 19 ++++++++++++------- src/lsm/lsm_manager.c | 4 ++-- src/lsm/lsm_merge.c | 2 +- src/lsm/lsm_tree.c | 20 +++++++++++++------- src/lsm/lsm_work_unit.c | 4 ++-- 5 files changed, 30 insertions(+), 19 deletions(-) diff --git a/src/include/lsm.h b/src/include/lsm.h index eacb4d52d3e..444073087df 100644 --- a/src/include/lsm.h +++ b/src/include/lsm.h @@ -242,13 +242,18 @@ struct __wt_lsm_tree { int64_t lsm_lookup_no_bloom; int64_t lsm_merge_throttle; -#define WT_LSM_TREE_ACTIVE 0x01 /* Workers are active */ -#define WT_LSM_TREE_AGGRESSIVE_TIMER 0x02 /* Timer for merge aggression */ -#define WT_LSM_TREE_COMPACTING 0x04 /* Tree being compacted */ -#define WT_LSM_TREE_MERGES 0x08 /* Tree should run merges */ -#define WT_LSM_TREE_NEED_SWITCH 0x10 /* New chunk needs creating */ -#define WT_LSM_TREE_OPEN 0x20 /* The tree is open */ -#define WT_LSM_TREE_THROTTLE 0x40 /* Throttle updates */ + /* + * The tree is open for business. This used to be a flag, but it is + * susceptible to races. + */ + bool active; + +#define WT_LSM_TREE_AGGRESSIVE_TIMER 0x01 /* Timer for merge aggression */ +#define WT_LSM_TREE_COMPACTING 0x02 /* Tree being compacted */ +#define WT_LSM_TREE_MERGES 0x04 /* Tree should run merges */ +#define WT_LSM_TREE_NEED_SWITCH 0x08 /* New chunk needs creating */ +#define WT_LSM_TREE_OPEN 0x10 /* The tree is open */ +#define WT_LSM_TREE_THROTTLE 0x20 /* Throttle updates */ uint32_t flags; }; diff --git a/src/lsm/lsm_manager.c b/src/lsm/lsm_manager.c index 24707abdb5a..943a5894ab3 100644 --- a/src/lsm/lsm_manager.c +++ b/src/lsm/lsm_manager.c @@ -390,7 +390,7 @@ __lsm_manager_run_server(WT_SESSION_IMPL *session) F_SET(session, WT_SESSION_LOCKED_HANDLE_LIST); dhandle_locked = true; TAILQ_FOREACH(lsm_tree, &S2C(session)->lsmqh, q) { - if (!F_ISSET(lsm_tree, WT_LSM_TREE_ACTIVE)) + if (!lsm_tree->active) continue; WT_ERR(__wt_epoch(session, &now)); pushms = lsm_tree->work_push_ts.tv_sec == 0 ? 0 : @@ -650,7 +650,7 @@ __wt_lsm_manager_push_entry(WT_SESSION_IMPL *session, * is checked. */ (void)__wt_atomic_add32(&lsm_tree->queue_ref, 1); - if (!F_ISSET(lsm_tree, WT_LSM_TREE_ACTIVE)) { + if (!lsm_tree->active) { (void)__wt_atomic_sub32(&lsm_tree->queue_ref, 1); return (0); } diff --git a/src/lsm/lsm_merge.c b/src/lsm/lsm_merge.c index 973043f334f..6d907284546 100644 --- a/src/lsm/lsm_merge.c +++ b/src/lsm/lsm_merge.c @@ -463,7 +463,7 @@ __wt_lsm_merge(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int id) #define LSM_MERGE_CHECK_INTERVAL WT_THOUSAND for (insert_count = 0; (ret = src->next(src)) == 0; insert_count++) { if (insert_count % LSM_MERGE_CHECK_INTERVAL == 0) { - if (!F_ISSET(lsm_tree, WT_LSM_TREE_ACTIVE)) + if (!lsm_tree->active) WT_ERR(EINTR); WT_STAT_FAST_CONN_INCRV(session, diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index b64b72256cc..73466a53215 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -90,8 +90,13 @@ __lsm_tree_close(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool wait) WT_DECL_RET; int i; - /* Stop any active merges. */ - F_CLR(lsm_tree, WT_LSM_TREE_ACTIVE); + /* + * Stop any new work units being added. The barrier is necessary + * because we rely on the state change being visible before checking + * the tree queue state. + */ + lsm_tree->active = false; + WT_READ_BARRIER(); /* * Wait for all LSM operations to drain. If WiredTiger is shutting @@ -123,7 +128,7 @@ __lsm_tree_close(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool wait) } return (0); -err: F_SET(lsm_tree, WT_LSM_TREE_ACTIVE); +err: lsm_tree->active = true; return (ret); } @@ -389,7 +394,7 @@ __lsm_tree_find(WT_SESSION_IMPL *session, lsm_tree->refcnt != 1) { (void)__wt_atomic_sub32( &lsm_tree->refcnt, 1); - F_SET(lsm_tree, WT_LSM_TREE_ACTIVE); + lsm_tree->active = true; lsm_tree->excl_session = NULL; return (EBUSY); } @@ -504,7 +509,8 @@ __lsm_tree_open(WT_SESSION_IMPL *session, /* Now the tree is setup, make it visible to others. */ TAILQ_INSERT_HEAD(&S2C(session)->lsmqh, lsm_tree, q); - F_SET(lsm_tree, WT_LSM_TREE_ACTIVE | WT_LSM_TREE_OPEN); + lsm_tree->active = true; + F_SET(lsm_tree, WT_LSM_TREE_OPEN); *treep = lsm_tree; @@ -545,7 +551,7 @@ __wt_lsm_tree_release(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_ASSERT(session, lsm_tree->refcnt > 0); if (lsm_tree->excl_session == session) { /* We cleared the active flag when getting exclusive access. */ - F_SET(lsm_tree, WT_LSM_TREE_ACTIVE); + lsm_tree->active = true; lsm_tree->excl_session = NULL; } (void)__wt_atomic_sub32(&lsm_tree->refcnt, 1); @@ -1218,7 +1224,7 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, bool *skipp) } /* Wait for the work unit queues to drain. */ - while (F_ISSET(lsm_tree, WT_LSM_TREE_ACTIVE)) { + while (lsm_tree->active) { /* * The flush flag is cleared when the chunk has been flushed. * Continue to push forced flushes until the chunk is on disk. diff --git a/src/lsm/lsm_work_unit.c b/src/lsm/lsm_work_unit.c index 88054f86d65..2af0c11e37e 100644 --- a/src/lsm/lsm_work_unit.c +++ b/src/lsm/lsm_work_unit.c @@ -29,7 +29,7 @@ __lsm_copy_chunks(WT_SESSION_IMPL *session, cookie->nchunks = 0; WT_RET(__wt_lsm_tree_readlock(session, lsm_tree)); - if (!F_ISSET(lsm_tree, WT_LSM_TREE_ACTIVE)) + if (!lsm_tree->active) return (__wt_lsm_tree_readunlock(session, lsm_tree)); /* Take a copy of the current state of the LSM tree. */ @@ -79,7 +79,7 @@ __wt_lsm_get_chunk_to_flush(WT_SESSION_IMPL *session, WT_ASSERT(session, lsm_tree->queue_ref > 0); WT_RET(__wt_lsm_tree_readlock(session, lsm_tree)); - if (!F_ISSET(lsm_tree, WT_LSM_TREE_ACTIVE) || lsm_tree->nchunks == 0) + if (!lsm_tree->active || lsm_tree->nchunks == 0) return (__wt_lsm_tree_readunlock(session, lsm_tree)); /* Search for a chunk to evict and/or a chunk to flush. */ -- cgit v1.2.1 From bad1f3a25baf4fe80358ce2304388fba402fc479 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Mon, 21 Mar 2016 17:23:21 +1100 Subject: WT-2501 LSM drop can violate access assumptions. Add a test case demonstrating the problem. --- src/lsm/lsm_tree.c | 3 +++ test/suite/test_drop02.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 test/suite/test_drop02.py diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index b64b72256cc..9d311a6edf3 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -27,6 +27,7 @@ __lsm_tree_discard(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool final) WT_UNUSED(final); /* Only used in diagnostic builds */ + WT_ASSERT(session, !F_ISSET(lsm_tree, WT_LSM_TREE_ACTIVE)); /* * The work unit queue should be empty, but it's worth checking * since work units use a different locking scheme to regular tree @@ -860,6 +861,7 @@ __wt_lsm_tree_drop( WT_WITH_HANDLE_LIST_LOCK(session, ret = __wt_lsm_tree_get(session, name, true, &lsm_tree)); WT_RET(ret); + WT_ASSERT(session, !F_ISSET(lsm_tree, WT_LSM_TREE_ACTIVE)); /* Prevent any new opens. */ WT_ERR(__wt_lsm_tree_writelock(session, lsm_tree)); @@ -888,6 +890,7 @@ __wt_lsm_tree_drop( WT_ERR(__wt_lsm_tree_writeunlock(session, lsm_tree)); ret = __wt_metadata_remove(session, name); + WT_ASSERT(session, !F_ISSET(lsm_tree, WT_LSM_TREE_ACTIVE)); err: if (locked) WT_TRET(__wt_lsm_tree_writeunlock(session, lsm_tree)); WT_WITH_HANDLE_LIST_LOCK(session, diff --git a/test/suite/test_drop02.py b/test/suite/test_drop02.py new file mode 100644 index 00000000000..677ba3866b2 --- /dev/null +++ b/test/suite/test_drop02.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from helper import simple_populate + +# test_drop02.py +# Test dropping an LSM tree on first open. There was a bug where this +# would cause an assertion failure: WT-2501 +class test_drop02(wttest.WiredTigerTestCase): + name = 'test_drop02' + + # Populate an object, remove it and confirm it no longer exists. + def test_drop(self): + uri = 'lsm:' + self.name + simple_populate(self, uri, 'key_format=S', 100000) + self.reopen_conn() + + self.session.drop(uri, None) + +if __name__ == '__main__': + wttest.run() -- cgit v1.2.1 From 9121562bebd77b82dd3fe6e467e735864232efe1 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Mon, 21 Mar 2016 17:36:49 +1100 Subject: WT-2499 Fix LSM get with exclusive access set. We should only set active when not opening an exclusive handle. --- src/lsm/lsm_tree.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index 73466a53215..fc3126a3823 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -509,7 +509,8 @@ __lsm_tree_open(WT_SESSION_IMPL *session, /* Now the tree is setup, make it visible to others. */ TAILQ_INSERT_HEAD(&S2C(session)->lsmqh, lsm_tree, q); - lsm_tree->active = true; + if (!exclusive) + lsm_tree->active = true; F_SET(lsm_tree, WT_LSM_TREE_OPEN); *treep = lsm_tree; -- cgit v1.2.1 From 3a1990ac352b38d5b8355108f9af93a58826e94f Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Mon, 21 Mar 2016 15:42:00 -0400 Subject: WT-2502 Free local allocation. --- src/txn/txn_ckpt.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/txn/txn_ckpt.c b/src/txn/txn_ckpt.c index 09d5f54e801..2dab7eef183 100644 --- a/src/txn/txn_ckpt.c +++ b/src/txn/txn_ckpt.c @@ -861,6 +861,12 @@ __checkpoint_lock_tree(WT_SESSION_IMPL *session, WT_CKPT_FOREACH(ckptbase, ckpt) ; WT_ERR(__wt_strdup(session, name, &ckpt->name)); + /* + * We are now done with the local use of the name. Free the local + * allocation, if needed. + */ + __wt_free(session, name_alloc); + name_alloc = NULL; F_SET(ckpt, WT_CKPT_ADD); /* -- cgit v1.2.1 From 4e62bc3b92ba439d0b817ba87a67ab7fb84e5bc1 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Tue, 22 Mar 2016 09:08:47 +1100 Subject: WT-2502 __wt_free clears the pointer; don't do it again. --- src/txn/txn_ckpt.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/txn/txn_ckpt.c b/src/txn/txn_ckpt.c index 2dab7eef183..a4a24b41a89 100644 --- a/src/txn/txn_ckpt.c +++ b/src/txn/txn_ckpt.c @@ -866,7 +866,6 @@ __checkpoint_lock_tree(WT_SESSION_IMPL *session, * allocation, if needed. */ __wt_free(session, name_alloc); - name_alloc = NULL; F_SET(ckpt, WT_CKPT_ADD); /* -- cgit v1.2.1 From 300d2882c1f3334b75bfec82e08f5a5274668eb3 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Tue, 22 Mar 2016 11:36:58 +1100 Subject: WT-2499 Cleanup LSM tree release --- src/lsm/lsm_tree.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index fc3126a3823..2b7e68b3f4e 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -392,10 +392,8 @@ __lsm_tree_find(WT_SESSION_IMPL *session, if (__lsm_tree_close( session, lsm_tree, false) != 0 || lsm_tree->refcnt != 1) { - (void)__wt_atomic_sub32( - &lsm_tree->refcnt, 1); - lsm_tree->active = true; - lsm_tree->excl_session = NULL; + __wt_lsm_tree_release( + session, lsm_tree); return (EBUSY); } } else { @@ -408,8 +406,8 @@ __lsm_tree_find(WT_SESSION_IMPL *session, if (lsm_tree->excl_session != NULL) { WT_ASSERT(session, lsm_tree->refcnt > 0); - (void)__wt_atomic_sub32( - &lsm_tree->refcnt, 1); + __wt_lsm_tree_release( + session, lsm_tree); return (EBUSY); } } @@ -1307,7 +1305,6 @@ err: __wt_lsm_tree_release(session, lsm_tree); return (ret); - } /* -- cgit v1.2.1 From 80341eae7116416f3da689fc28d0ce6a357d59c3 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Tue, 22 Mar 2016 13:16:47 +1100 Subject: WT-2501 Fix assertions to use new active field. --- src/lsm/lsm_tree.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index 7fb3cfc0e95..5ede47c0c43 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -27,7 +27,7 @@ __lsm_tree_discard(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool final) WT_UNUSED(final); /* Only used in diagnostic builds */ - WT_ASSERT(session, !F_ISSET(lsm_tree, WT_LSM_TREE_ACTIVE)); + WT_ASSERT(session, !lsm_tree->active); /* * The work unit queue should be empty, but it's worth checking * since work units use a different locking scheme to regular tree @@ -866,7 +866,7 @@ __wt_lsm_tree_drop( WT_WITH_HANDLE_LIST_LOCK(session, ret = __wt_lsm_tree_get(session, name, true, &lsm_tree)); WT_RET(ret); - WT_ASSERT(session, !F_ISSET(lsm_tree, WT_LSM_TREE_ACTIVE)); + WT_ASSERT(session, !lsm_tree->active); /* Prevent any new opens. */ WT_ERR(__wt_lsm_tree_writelock(session, lsm_tree)); @@ -895,7 +895,7 @@ __wt_lsm_tree_drop( WT_ERR(__wt_lsm_tree_writeunlock(session, lsm_tree)); ret = __wt_metadata_remove(session, name); - WT_ASSERT(session, !F_ISSET(lsm_tree, WT_LSM_TREE_ACTIVE)); + WT_ASSERT(session, !lsm_tree->active); err: if (locked) WT_TRET(__wt_lsm_tree_writeunlock(session, lsm_tree)); WT_WITH_HANDLE_LIST_LOCK(session, -- cgit v1.2.1 From 1f21bcdca279c22c688cdd810c9b9ad9f2f012b5 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Tue, 22 Mar 2016 13:48:14 +1100 Subject: WT-2506 Reorder to fix uninitialized variable warning. --- src/txn/txn_ckpt.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/txn/txn_ckpt.c b/src/txn/txn_ckpt.c index a4a24b41a89..1eebc9e9d04 100644 --- a/src/txn/txn_ckpt.c +++ b/src/txn/txn_ckpt.c @@ -995,6 +995,15 @@ __checkpoint_tree( fake_ckpt = false; was_modified = btree->modified; + /* + * Set the checkpoint LSN to the maximum LSN so that if logging is + * disabled, recovery will never roll old changes forward over the + * non-logged changes in this checkpoint. If logging is enabled, a + * real checkpoint LSN will be assigned for this checkpoint and + * overwrite this. + */ + WT_MAX_LSN(&ckptlsn); + /* * Check for clean objects not requiring a checkpoint. * @@ -1105,15 +1114,6 @@ nockpt: F_SET(btree, WT_BTREE_SKIP_CKPT); btree->modified = 0; WT_FULL_BARRIER(); - /* - * Set the checkpoint LSN to the maximum LSN so that if logging is - * disabled, recovery will never roll old changes forward over the - * non-logged changes in this checkpoint. If logging is enabled, a - * real checkpoint LSN will be assigned for this checkpoint and - * overwrite this. - */ - WT_MAX_LSN(&ckptlsn); - /* Tell logging that a file checkpoint is starting. */ if (FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) WT_ERR(__wt_txn_checkpoint_log( -- cgit v1.2.1 From e3a043eed98859f1585b0854c179c81f7486f1e7 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Tue, 22 Mar 2016 13:56:31 +1100 Subject: WT-2503 Rename "wait" to avoid shadowing system call. --- src/lsm/lsm_tree.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index 2b7e68b3f4e..c69f5416621 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -85,7 +85,7 @@ __lsm_tree_discard(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool final) * Close an LSM tree structure. */ static int -__lsm_tree_close(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool wait) +__lsm_tree_close(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool final) { WT_DECL_RET; int i; @@ -105,7 +105,7 @@ __lsm_tree_close(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool wait) * access is not available. */ for (i = 0; - lsm_tree->queue_ref > 0 || (wait && lsm_tree->refcnt > 1); ++i) { + lsm_tree->queue_ref > 0 || (final && lsm_tree->refcnt > 1); ++i) { /* * Remove any work units from the manager queues. Do this step * repeatedly in case a work unit was in the process of being -- cgit v1.2.1