diff options
author | Susan LoVerso <sue@mongodb.com> | 2016-03-02 09:48:47 -0500 |
---|---|---|
committer | Susan LoVerso <sue@mongodb.com> | 2016-03-02 09:48:47 -0500 |
commit | a26aeb83e92be584040f1f09c5ab02f04d39c7c5 (patch) | |
tree | 91e9ee6f7a08f78afc32203558b0fd11a196a9d3 | |
parent | 784e7bd75ddf29be1eef1b5289c0673942c84256 (diff) | |
parent | c2342bd27c07b5059e5ab7c34523685ed0985174 (diff) | |
download | mongo-a26aeb83e92be584040f1f09c5ab02f04d39c7c5.tar.gz |
Merge branch 'develop' into wt-2366-simpler
-rw-r--r-- | bench/wtperf/runners/evict-btree-readonly.wtperf | 2 | ||||
-rw-r--r-- | bench/wtperf/runners/evict-btree.wtperf | 2 | ||||
-rw-r--r-- | bench/wtperf/runners/evict-lsm-readonly.wtperf | 2 | ||||
-rw-r--r-- | bench/wtperf/runners/evict-lsm.wtperf | 2 | ||||
-rw-r--r-- | dist/stat_data.py | 77 | ||||
-rw-r--r-- | src/conn/conn_dhandle.c | 29 | ||||
-rw-r--r-- | src/conn/conn_stat.c | 4 | ||||
-rw-r--r-- | src/conn/conn_sweep.c | 8 | ||||
-rw-r--r-- | src/cursor/cur_stat.c | 35 | ||||
-rw-r--r-- | src/docs/readonly.dox | 2 | ||||
-rw-r--r-- | src/evict/evict_lru.c | 22 | ||||
-rw-r--r-- | src/support/power8/crc32.S | 50 | ||||
-rw-r--r-- | test/suite/test_backup05.py | 8 | ||||
-rw-r--r-- | test/suite/test_checkpoint01.py | 2 | ||||
-rw-r--r-- | test/suite/test_cursor06.py | 2 | ||||
-rw-r--r-- | test/suite/test_cursor_random.py | 2 | ||||
-rw-r--r-- | test/suite/test_join01.py | 38 | ||||
-rw-r--r-- | test/suite/test_readonly01.py | 7 | ||||
-rw-r--r-- | test/suite/test_readonly02.py | 8 | ||||
-rw-r--r-- | test/suite/test_readonly03.py | 2 |
20 files changed, 196 insertions, 108 deletions
diff --git a/bench/wtperf/runners/evict-btree-readonly.wtperf b/bench/wtperf/runners/evict-btree-readonly.wtperf index d79af2b762b..25599fadd8d 100644 --- a/bench/wtperf/runners/evict-btree-readonly.wtperf +++ b/bench/wtperf/runners/evict-btree-readonly.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict btree configuration -conn_config="cache_size=50M" +conn_config="cache_size=50M,eviction=(threads_max=4),mmap=false" table_config="type=file" icount=10000000 report_interval=5 diff --git a/bench/wtperf/runners/evict-btree.wtperf b/bench/wtperf/runners/evict-btree.wtperf index 24da4dd7902..e7d967e5c63 100644 --- a/bench/wtperf/runners/evict-btree.wtperf +++ b/bench/wtperf/runners/evict-btree.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict btree configuration -conn_config="cache_size=50M" +conn_config="cache_size=50M,eviction=(threads_max=4)" table_config="type=file" icount=10000000 report_interval=5 diff --git a/bench/wtperf/runners/evict-lsm-readonly.wtperf b/bench/wtperf/runners/evict-lsm-readonly.wtperf index fe45c0e93b6..661b8e21924 100644 --- a/bench/wtperf/runners/evict-lsm-readonly.wtperf +++ b/bench/wtperf/runners/evict-lsm-readonly.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict lsm configuration -conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6)" +conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6),eviction=(threads_max=4)" table_config="type=lsm,lsm=(chunk_size=2M),os_cache_dirty_max=16MB" compact=true icount=10000000 diff --git a/bench/wtperf/runners/evict-lsm.wtperf b/bench/wtperf/runners/evict-lsm.wtperf index ad885d98eb7..b872d429046 100644 --- a/bench/wtperf/runners/evict-lsm.wtperf +++ b/bench/wtperf/runners/evict-lsm.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict lsm configuration -conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6)" +conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6),eviction=(threads_max=4)" table_config="type=lsm,lsm=(chunk_size=2M),os_cache_dirty_max=16MB" compact=true icount=10000000 diff --git a/dist/stat_data.py b/dist/stat_data.py index be4bacaece7..09e5643a5d6 100644 --- a/dist/stat_data.py +++ b/dist/stat_data.py @@ -12,6 +12,7 @@ # max_aggregate Take the maximum value when aggregating statistics # no_clear Value not cleared when statistics cleared # no_scale Don't scale value per second in the logging tool script +# size Used by timeseries tool, indicates value is a byte count # # The no_clear and no_scale flags are normally always set together (values that # are maintained over time are normally not scaled per second). @@ -138,9 +139,9 @@ connection_stats = [ ########################################## # Block manager statistics ########################################## - BlockStat('block_byte_map_read', 'mapped bytes read'), - BlockStat('block_byte_read', 'bytes read'), - BlockStat('block_byte_write', 'bytes written'), + BlockStat('block_byte_map_read', 'mapped bytes read', 'size'), + BlockStat('block_byte_read', 'bytes read', 'size'), + BlockStat('block_byte_write', 'bytes written', 'size'), BlockStat('block_map_read', 'mapped blocks read'), BlockStat('block_preload', 'blocks pre-loaded'), BlockStat('block_read', 'blocks read'), @@ -149,14 +150,14 @@ connection_stats = [ ########################################## # Cache and eviction statistics ########################################## - CacheStat('cache_bytes_dirty', 'tracked dirty bytes in the cache', 'no_clear,no_scale'), - CacheStat('cache_bytes_internal', 'tracked bytes belonging to internal pages in the cache', 'no_clear,no_scale'), - CacheStat('cache_bytes_inuse', 'bytes currently in the cache', 'no_clear,no_scale'), - CacheStat('cache_bytes_leaf', 'tracked bytes belonging to leaf pages in the cache', 'no_clear,no_scale'), - CacheStat('cache_bytes_max', 'maximum bytes configured', 'no_clear,no_scale'), - CacheStat('cache_bytes_overflow', 'tracked bytes belonging to overflow pages in the cache', 'no_clear,no_scale'), - CacheStat('cache_bytes_read', 'bytes read into cache'), - CacheStat('cache_bytes_write', 'bytes written from cache'), + CacheStat('cache_bytes_dirty', 'tracked dirty bytes in the cache', 'no_clear,no_scale,size'), + CacheStat('cache_bytes_internal', 'tracked bytes belonging to internal pages in the cache', 'no_clear,no_scale,size'), + CacheStat('cache_bytes_inuse', 'bytes currently in the cache', 'no_clear,no_scale,size'), + CacheStat('cache_bytes_leaf', 'tracked bytes belonging to leaf pages in the cache', 'no_clear,no_scale,size'), + CacheStat('cache_bytes_max', 'maximum bytes configured', 'no_clear,no_scale,size'), + CacheStat('cache_bytes_overflow', 'tracked bytes belonging to overflow pages in the cache', 'no_clear,no_scale,size'), + CacheStat('cache_bytes_read', 'bytes read into cache', 'size'), + CacheStat('cache_bytes_write', 'bytes written from cache', 'size'), CacheStat('cache_eviction_aggressive_set', 'eviction currently operating in aggressive mode', 'no_clear,no_scale'), CacheStat('cache_eviction_app', 'pages evicted by application threads'), CacheStat('cache_eviction_checkpoint', 'checkpoint blocked page eviction'), @@ -169,7 +170,7 @@ connection_stats = [ CacheStat('cache_eviction_force_fail', 'failed eviction of pages that exceeded the in-memory maximum'), CacheStat('cache_eviction_hazard', 'hazard pointer blocked page eviction'), CacheStat('cache_eviction_internal', 'internal pages evicted'), - CacheStat('cache_eviction_maximum_page_size', 'maximum page size at eviction', 'no_clear,no_scale'), + CacheStat('cache_eviction_maximum_page_size', 'maximum page size at eviction', 'no_clear,no_scale,size'), CacheStat('cache_eviction_queue_empty', 'eviction server candidate queue empty when topping up'), CacheStat('cache_eviction_queue_not_empty', 'eviction server candidate queue not empty when topping up'), CacheStat('cache_eviction_server_evicting', 'eviction server evicting pages'), @@ -207,17 +208,17 @@ connection_stats = [ ########################################## # Logging statistics ########################################## - LogStat('log_buffer_size', 'total log buffer size', 'no_clear,no_scale'), - LogStat('log_bytes_payload', 'log bytes of payload data'), - LogStat('log_bytes_written', 'log bytes written'), + LogStat('log_buffer_size', 'total log buffer size', 'no_clear,no_scale,size'), + LogStat('log_bytes_payload', 'log bytes of payload data', 'size'), + LogStat('log_bytes_written', 'log bytes written', 'size'), LogStat('log_close_yields', 'yields waiting for previous log file close'), - LogStat('log_compress_len', 'total size of compressed records'), - LogStat('log_compress_mem', 'total in-memory size of compressed records'), + LogStat('log_compress_len', 'total size of compressed records', 'size'), + LogStat('log_compress_mem', 'total in-memory size of compressed records', 'size'), LogStat('log_compress_small', 'log records too small to compress'), LogStat('log_compress_write_fails', 'log records not compressed'), LogStat('log_compress_writes', 'log records compressed'), LogStat('log_flush', 'log flush operations'), - LogStat('log_max_filesize', 'maximum log file size', 'no_clear,no_scale'), + LogStat('log_max_filesize', 'maximum log file size', 'no_clear,no_scale,size'), LogStat('log_prealloc_files', 'pre-allocated log files prepared'), LogStat('log_prealloc_max', 'number of pre-allocated log files to create', 'no_clear,no_scale'), LogStat('log_prealloc_missed', 'pre-allocated log files not ready and missed'), @@ -228,7 +229,7 @@ connection_stats = [ LogStat('log_scans', 'log scan operations'), LogStat('log_slot_closes', 'consolidated slot closures'), LogStat('log_slot_coalesced', 'written slots coalesced'), - LogStat('log_slot_consolidated', 'logging bytes consolidated'), + LogStat('log_slot_consolidated', 'logging bytes consolidated', 'size'), LogStat('log_slot_joins', 'consolidated slot joins'), LogStat('log_slot_races', 'consolidated slot join races'), LogStat('log_slot_switch_busy', 'busy returns attempting to switch slots'), @@ -247,7 +248,7 @@ connection_stats = [ RecStat('rec_page_delete_fast', 'fast-path pages deleted'), RecStat('rec_pages', 'page reconciliation calls'), RecStat('rec_pages_eviction', 'page reconciliation calls for eviction'), - RecStat('rec_split_stashed_bytes', 'split bytes currently awaiting free', 'no_clear,no_scale'), + RecStat('rec_split_stashed_bytes', 'split bytes currently awaiting free', 'no_clear,no_scale,size'), RecStat('rec_split_stashed_objects', 'split objects currently awaiting free', 'no_clear,no_scale'), ########################################## @@ -334,18 +335,18 @@ dsrc_stats = [ CursorStat('cursor_create', 'create calls'), CursorStat('cursor_insert', 'insert calls'), CursorStat('cursor_insert_bulk', 'bulk-loaded cursor-insert calls'), - CursorStat('cursor_insert_bytes', 'cursor-insert key and value bytes inserted'), + CursorStat('cursor_insert_bytes', 'cursor-insert key and value bytes inserted', 'size'), CursorStat('cursor_next', 'next calls'), CursorStat('cursor_prev', 'prev calls'), CursorStat('cursor_remove', 'remove calls'), - CursorStat('cursor_remove_bytes', 'cursor-remove key bytes removed'), + CursorStat('cursor_remove_bytes', 'cursor-remove key bytes removed', 'size'), CursorStat('cursor_reset', 'reset calls'), CursorStat('cursor_restart', 'restarted searches'), CursorStat('cursor_search', 'search calls'), CursorStat('cursor_search_near', 'search near calls'), CursorStat('cursor_truncate', 'truncate calls'), CursorStat('cursor_update', 'update calls'), - CursorStat('cursor_update_bytes', 'cursor-update value bytes updated'), + CursorStat('cursor_update_bytes', 'cursor-update value bytes updated', 'size'), ########################################## # Btree statistics @@ -358,13 +359,13 @@ dsrc_stats = [ BtreeStat('btree_column_variable', 'column-store variable-size leaf pages', 'no_scale'), BtreeStat('btree_compact_rewrite', 'pages rewritten by compaction'), BtreeStat('btree_entries', 'number of key/value pairs', 'no_scale'), - BtreeStat('btree_fixed_len', 'fixed-record size', 'max_aggregate,no_scale'), + BtreeStat('btree_fixed_len', 'fixed-record size', 'max_aggregate,no_scale,size'), BtreeStat('btree_maximum_depth', 'maximum tree depth', 'max_aggregate,no_scale'), - BtreeStat('btree_maxintlkey', 'maximum internal page key size', 'max_aggregate,no_scale'), - BtreeStat('btree_maxintlpage', 'maximum internal page size', 'max_aggregate,no_scale'), - BtreeStat('btree_maxleafkey', 'maximum leaf page key size', 'max_aggregate,no_scale'), - BtreeStat('btree_maxleafpage', 'maximum leaf page size', 'max_aggregate,no_scale'), - BtreeStat('btree_maxleafvalue', 'maximum leaf page value size', 'max_aggregate,no_scale'), + BtreeStat('btree_maxintlkey', 'maximum internal page key size', 'max_aggregate,no_scale,size'), + BtreeStat('btree_maxintlpage', 'maximum internal page size', 'max_aggregate,no_scale,size'), + BtreeStat('btree_maxleafkey', 'maximum leaf page key size', 'max_aggregate,no_scale,size'), + BtreeStat('btree_maxleafpage', 'maximum leaf page size', 'max_aggregate,no_scale,size'), + BtreeStat('btree_maxleafvalue', 'maximum leaf page value size', 'max_aggregate,no_scale,size'), BtreeStat('btree_overflow', 'overflow pages', 'no_scale'), BtreeStat('btree_row_internal', 'row-store internal pages', 'no_scale'), BtreeStat('btree_row_leaf', 'row-store leaf pages', 'no_scale'), @@ -378,7 +379,7 @@ dsrc_stats = [ LSMStat('bloom_miss', 'bloom filter misses'), LSMStat('bloom_page_evict', 'bloom filter pages evicted from cache'), LSMStat('bloom_page_read', 'bloom filter pages read into cache'), - LSMStat('bloom_size', 'total size of bloom filters', 'no_scale'), + LSMStat('bloom_size', 'total size of bloom filters', 'no_scale,size'), LSMStat('lsm_checkpoint_throttle', 'sleep for LSM checkpoint throttle'), LSMStat('lsm_chunk_count', 'chunks in the LSM tree', 'no_scale'), LSMStat('lsm_generation_max', 'highest merge generation in the LSM tree', 'max_aggregate,no_scale'), @@ -388,22 +389,22 @@ dsrc_stats = [ ########################################## # Block manager statistics ########################################## - BlockStat('allocation_size', 'file allocation unit size', 'max_aggregate,no_scale'), + BlockStat('allocation_size', 'file allocation unit size', 'max_aggregate,no_scale,size'), BlockStat('block_alloc', 'blocks allocated'), - BlockStat('block_checkpoint_size', 'checkpoint size', 'no_scale'), + BlockStat('block_checkpoint_size', 'checkpoint size', 'no_scale,size'), BlockStat('block_extension', 'allocations requiring file extension'), BlockStat('block_free', 'blocks freed'), BlockStat('block_magic', 'file magic number', 'max_aggregate,no_scale'), BlockStat('block_major', 'file major version number', 'max_aggregate,no_scale'), BlockStat('block_minor', 'minor version number', 'max_aggregate,no_scale'), - BlockStat('block_reuse_bytes', 'file bytes available for reuse'), - BlockStat('block_size', 'file size in bytes', 'no_scale'), + BlockStat('block_reuse_bytes', 'file bytes available for reuse', 'size'), + BlockStat('block_size', 'file size in bytes', 'no_scale,size'), ########################################## # Cache and eviction statistics ########################################## - CacheStat('cache_bytes_read', 'bytes read into cache'), - CacheStat('cache_bytes_write', 'bytes written from cache'), + CacheStat('cache_bytes_read', 'bytes read into cache', 'size'), + CacheStat('cache_bytes_write', 'bytes written from cache', 'size'), CacheStat('cache_eviction_checkpoint', 'checkpoint blocked page eviction'), CacheStat('cache_eviction_clean', 'unmodified pages evicted'), CacheStat('cache_eviction_deepen', 'page split during eviction deepened the tree'), @@ -449,8 +450,8 @@ dsrc_stats = [ RecStat('rec_page_match', 'page checksum matches'), RecStat('rec_pages', 'page reconciliation calls'), RecStat('rec_pages_eviction', 'page reconciliation calls for eviction'), - RecStat('rec_prefix_compression', 'leaf page key bytes discarded using prefix compression'), - RecStat('rec_suffix_compression', 'internal page key bytes discarded using suffix compression'), + RecStat('rec_prefix_compression', 'leaf page key bytes discarded using prefix compression', 'size'), + RecStat('rec_suffix_compression', 'internal page key bytes discarded using suffix compression', 'size'), ########################################## # Transaction statistics diff --git a/src/conn/conn_dhandle.c b/src/conn/conn_dhandle.c index f1b35571533..2fab08e3afa 100644 --- a/src/conn/conn_dhandle.c +++ b/src/conn/conn_dhandle.c @@ -134,14 +134,11 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) btree = S2BT(session); bm = btree->bm; dhandle = session->dhandle; - marked_dead = false; + evict_reset = marked_dead = false; if (!F_ISSET(dhandle, WT_DHANDLE_OPEN)) return (0); - /* Ensure that we aren't racing with the eviction server */ - WT_RET(__wt_evict_file_exclusive_on(session, &evict_reset)); - /* * If we don't already have the schema lock, make it an error to try * to acquire it. The problem is that we are holding an exclusive @@ -163,6 +160,13 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) __wt_spin_lock(session, &dhandle->close_lock); /* + * Ensure we aren't racing with the eviction server; inside the close + * lock so threads won't race setting/clearing the tree's "no eviction" + * flag. + */ + WT_ERR(__wt_evict_file_exclusive_on(session, &evict_reset)); + + /* * The close can fail if an update cannot be written, return the EBUSY * error to our caller for eventual retry. * @@ -176,23 +180,19 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) WT_BTREE_SALVAGE | WT_BTREE_UPGRADE | WT_BTREE_VERIFY)) { if (force && (bm == NULL || !bm->is_mapped(bm, session))) { F_SET(session->dhandle, WT_DHANDLE_DEAD); + marked_dead = true; - /* - * Reset the tree's eviction priority, and the tree is - * evictable by definition. - */ + /* Reset the tree's eviction priority (if any). */ __wt_evict_priority_clear(session); - F_CLR(S2BT(session), WT_BTREE_NO_EVICTION); - - marked_dead = true; } if (!marked_dead || final) WT_ERR(__wt_checkpoint_close(session, final)); } WT_TRET(__wt_btree_close(session)); + /* - * If we marked a handle as dead it will be closed by sweep, via + * If we marked a handle dead it will be closed by sweep, via * another call to sync and close. */ if (!marked_dead) { @@ -204,10 +204,9 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) F_ISSET(dhandle, WT_DHANDLE_DEAD) || !F_ISSET(dhandle, WT_DHANDLE_OPEN)); -err: __wt_spin_unlock(session, &dhandle->close_lock); - - if (evict_reset) +err: if (evict_reset) __wt_evict_file_exclusive_off(session); + __wt_spin_unlock(session, &dhandle->close_lock); if (no_schema_lock) F_CLR(session, WT_SESSION_NO_SCHEMA_LOCK); diff --git a/src/conn/conn_stat.c b/src/conn/conn_stat.c index 9ccfa8ad8db..d6e59a50da5 100644 --- a/src/conn/conn_stat.c +++ b/src/conn/conn_stat.c @@ -106,10 +106,6 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp) * If any statistics logging is done, this must not be a read-only * connection. */ - if (F_ISSET(conn, WT_CONN_READONLY)) - WT_RET_MSG(session, EINVAL, - "Read-only configuration incompatible with statistics " - "logging"); WT_RET(__wt_config_gets(session, cfg, "statistics_log.sources", &cval)); WT_RET(__wt_config_subinit(session, &objectconf, &cval)); for (cnt = 0; (ret = __wt_config_next(&objectconf, &k, &v)) == 0; ++cnt) diff --git a/src/conn/conn_sweep.c b/src/conn/conn_sweep.c index 7628076e605..cc0aa5a1322 100644 --- a/src/conn/conn_sweep.c +++ b/src/conn/conn_sweep.c @@ -91,9 +91,9 @@ __sweep_expire_one(WT_SESSION_IMPL *session) goto err; /* - * Mark the handle as dead and close the underlying file - * handle. Closing the handle decrements the open file count, - * meaning the close loop won't overrun the configured minimum. + * Mark the handle dead and close the underlying file handle. + * Closing the handle decrements the open file count, meaning the close + * loop won't overrun the configured minimum. */ ret = __wt_conn_btree_sync_and_close(session, false, true); @@ -163,7 +163,7 @@ __sweep_discard_trees(WT_SESSION_IMPL *session, u_int *dead_handlesp) !F_ISSET(dhandle, WT_DHANDLE_DEAD)) continue; - /* If the handle is marked "dead", flush it from cache. */ + /* If the handle is marked dead, flush it from cache. */ WT_WITH_DHANDLE(session, dhandle, ret = __wt_conn_btree_sync_and_close(session, false, false)); diff --git a/src/cursor/cur_stat.c b/src/cursor/cur_stat.c index 8528482a009..34e64b34ccb 100644 --- a/src/cursor/cur_stat.c +++ b/src/cursor/cur_stat.c @@ -200,8 +200,6 @@ __curstat_next(WT_CURSOR *cursor) if (cst->notinitialized) { WT_ERR(__wt_curstat_init( session, cursor->internal_uri, NULL, cst->cfg, cst)); - if (cst->next_set != NULL) - WT_ERR((*cst->next_set)(session, cst, true, true)); cst->notinitialized = false; } @@ -244,8 +242,6 @@ __curstat_prev(WT_CURSOR *cursor) if (cst->notinitialized) { WT_ERR(__wt_curstat_init( session, cursor->internal_uri, NULL, cst->cfg, cst)); - if (cst->next_set != NULL) - WT_ERR((*cst->next_set)(session, cst, false, true)); cst->notinitialized = false; } @@ -449,7 +445,6 @@ __curstat_join_next_set(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst, WT_JOIN_STATS_GROUP *join_group; ssize_t pos; - WT_ASSERT(session, WT_STREQ(cst->iface.uri, "statistics:join")); join_group = &cst->u.join_stats_group; cjoin = join_group->join_cursor; if (init) @@ -542,25 +537,31 @@ __wt_curstat_init(WT_SESSION_IMPL *session, dsrc_uri = uri + strlen("statistics:"); if (WT_STREQ(dsrc_uri, "join")) - return (__curstat_join_init(session, curjoin, cfg, cst)); + WT_RET(__curstat_join_init(session, curjoin, cfg, cst)); - if (WT_PREFIX_MATCH(dsrc_uri, "colgroup:")) - return ( + else if (WT_PREFIX_MATCH(dsrc_uri, "colgroup:")) + WT_RET( __wt_curstat_colgroup_init(session, dsrc_uri, cfg, cst)); - if (WT_PREFIX_MATCH(dsrc_uri, "file:")) - return (__curstat_file_init(session, dsrc_uri, cfg, cst)); + else if (WT_PREFIX_MATCH(dsrc_uri, "file:")) + WT_RET(__curstat_file_init(session, dsrc_uri, cfg, cst)); - if (WT_PREFIX_MATCH(dsrc_uri, "index:")) - return (__wt_curstat_index_init(session, dsrc_uri, cfg, cst)); + else if (WT_PREFIX_MATCH(dsrc_uri, "index:")) + WT_RET(__wt_curstat_index_init(session, dsrc_uri, cfg, cst)); - if (WT_PREFIX_MATCH(dsrc_uri, "lsm:")) - return (__wt_curstat_lsm_init(session, dsrc_uri, cst)); + else if (WT_PREFIX_MATCH(dsrc_uri, "lsm:")) + WT_RET(__wt_curstat_lsm_init(session, dsrc_uri, cst)); - if (WT_PREFIX_MATCH(dsrc_uri, "table:")) - return (__wt_curstat_table_init(session, dsrc_uri, cfg, cst)); + else if (WT_PREFIX_MATCH(dsrc_uri, "table:")) + WT_RET(__wt_curstat_table_init(session, dsrc_uri, cfg, cst)); - return (__wt_bad_object_type(session, uri)); + else + return (__wt_bad_object_type(session, uri)); + + if (cst->next_set != NULL) + WT_RET((*cst->next_set)(session, cst, false, true)); + + return (0); } /* diff --git a/src/docs/readonly.dox b/src/docs/readonly.dox index 9935f5d1b17..ad4a94a73f1 100644 --- a/src/docs/readonly.dox +++ b/src/docs/readonly.dox @@ -18,7 +18,7 @@ tree merges are turned off when LSM trees are configured, and log file archiving is disabled when logging is configured. Where a user configured setting contradicts read-only operation, WiredTiger -will return an error. For example, statistics logging or zero-filling +will return an error. For example, zero-filling log files is not allowed in read-only mode, and attempting to configure them will return an error. diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index 35b12e2b685..884c08a02df 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -791,7 +791,7 @@ __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session, bool *evict_resetp) btree = S2BT(session); cache = S2C(session)->cache; - /* If the file wasn't evictable, there's no work to do. */ + /* If the file was never evictable, there's no work to do. */ if (F_ISSET(btree, WT_BTREE_NO_EVICTION)) return (0); @@ -800,9 +800,16 @@ __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session, bool *evict_resetp) * the file will be queued for eviction after this point. */ __wt_spin_lock(session, &cache->evict_walk_lock); - F_SET(btree, WT_BTREE_NO_EVICTION); + if (!F_ISSET(btree, WT_BTREE_NO_EVICTION)) { + F_SET(btree, WT_BTREE_NO_EVICTION); + *evict_resetp = true; + } __wt_spin_unlock(session, &cache->evict_walk_lock); + /* If some other operation has disabled eviction, we're done. */ + if (!*evict_resetp) + return (0); + /* Clear any existing LRU eviction walk for the file. */ WT_ERR(__evict_request_walk_clear(session)); @@ -826,10 +833,10 @@ __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session, bool *evict_resetp) while (btree->evict_busy > 0) __wt_yield(); - *evict_resetp = true; return (0); err: F_CLR(btree, WT_BTREE_NO_EVICTION); + *evict_resetp = false; return (ret); } @@ -844,7 +851,14 @@ __wt_evict_file_exclusive_off(WT_SESSION_IMPL *session) btree = S2BT(session); - WT_ASSERT(session, btree->evict_ref == NULL); + /* + * We have seen subtle bugs with multiple threads racing to turn + * eviction on/off. Make races more likely in diagnostic builds. + */ + WT_DIAGNOSTIC_YIELD; + + WT_ASSERT(session, btree->evict_ref == NULL && + F_ISSET(btree, WT_BTREE_NO_EVICTION)); F_CLR(btree, WT_BTREE_NO_EVICTION); } diff --git a/src/support/power8/crc32.S b/src/support/power8/crc32.S index 4bc1fad416d..3ef2928aaa1 100644 --- a/src/support/power8/crc32.S +++ b/src/support/power8/crc32.S @@ -90,6 +90,31 @@ FUNC_START(__crc32_vpmsum) std r26,-48(r1) std r25,-56(r1) + li r31, -256 + stvx v20, r31, r1 + li r31, -240 + stvx v21, r31, r1 + li r31, -224 + stvx v22, r31, r1 + li r31, -208 + stvx v23, r31, r1 + li r31, -192 + stvx v24, r31, r1 + li r31, -176 + stvx v25, r31, r1 + li r31, -160 + stvx v26, r31, r1 + li r31, -144 + stvx v27, r31, r1 + li r31, -128 + stvx v28, r31, r1 + li r31, -112 + stvx v29, r31, r1 + li r31, -96 + stvx v30, r31, r1 + li r31, -80 + stvx v31, r31, r1 + li off16,16 li off32,32 li off48,48 @@ -571,6 +596,31 @@ FUNC_START(__crc32_vpmsum) /* Get it into r3 */ MFVRD(r3, v0) + li r31, -256 + lvx v20, r31, r1 + li r31, -240 + lvx v21, r31, r1 + li r31, -224 + lvx v22, r31, r1 + li r31, -208 + lvx v23, r31, r1 + li r31, -192 + lvx v24, r31, r1 + li r31, -176 + lvx v25, r31, r1 + li r31, -160 + lvx v26, r31, r1 + li r31, -144 + lvx v27, r31, r1 + li r31, -128 + lvx v28, r31, r1 + li r31, -112 + lvx v29, r31, r1 + li r31, -96 + lvx v30, r31, r1 + li r31, -80 + lvx v31, r31, r1 + ld r31,-8(r1) ld r30,-16(r1) ld r29,-24(r1) diff --git a/test/suite/test_backup05.py b/test/suite/test_backup05.py index 8ffeb6752df..991a9f71b19 100644 --- a/test/suite/test_backup05.py +++ b/test/suite/test_backup05.py @@ -44,14 +44,6 @@ class test_backup05(wttest.WiredTigerTestCase, suite_subprocess): create_params = 'key_format=i,value_format=i' freq = 5 - def copy_windows(self, olddir, newdir): - os.mkdir(newdir) - for fname in os.listdir(olddir): - fullname = os.path.join(olddir, fname) - # Skip lock file on Windows since it is locked - if os.path.isfile(fullname) and "WiredTiger.lock" not in fullname: - shutil.copy(fullname, newdir) - def check_manual_backup(self, i, olddir, newdir): ''' Simulate a manual backup from olddir and restart in newdir. ''' self.session.checkpoint() diff --git a/test/suite/test_checkpoint01.py b/test/suite/test_checkpoint01.py index 36f1ef733a4..9955944f73d 100644 --- a/test/suite/test_checkpoint01.py +++ b/test/suite/test_checkpoint01.py @@ -265,7 +265,7 @@ class test_checkpoint_cursor_update(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(self.uri, None, "checkpoint=ckpt") cursor.set_key(key_populate(cursor, 10)) cursor.set_value("XXX") - msg = "/not supported/" + msg = "/Unsupported cursor/" self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cursor.insert(), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, diff --git a/test/suite/test_cursor06.py b/test/suite/test_cursor06.py index d702f97c5dd..5545c862dd7 100644 --- a/test/suite/test_cursor06.py +++ b/test/suite/test_cursor06.py @@ -89,7 +89,7 @@ class test_cursor06(wttest.WiredTigerTestCase): self.session.drop(uri, "force") self.populate(uri) cursor = self.session.open_cursor(uri, None, open_config) - msg = '/not supported/' + msg = '/Unsupported cursor/' if open_config == "readonly=1": self.set_kv(cursor) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, diff --git a/test/suite/test_cursor_random.py b/test/suite/test_cursor_random.py index cd91a925b0c..16ce5cae685 100644 --- a/test/suite/test_cursor_random.py +++ b/test/suite/test_cursor_random.py @@ -51,7 +51,7 @@ class test_cursor_random(wttest.WiredTigerTestCase): uri = self.type self.session.create(uri, 'key_format=S,value_format=S') cursor = self.session.open_cursor(uri, None, self.config) - msg = "/not supported/" + msg = "/Unsupported cursor/" self.assertRaisesWithMessage( wiredtiger.WiredTigerError, lambda: cursor.compare(cursor), msg) self.assertRaisesWithMessage( diff --git a/test/suite/test_join01.py b/test/suite/test_join01.py index ee4d6e22870..539a3a3ae57 100644 --- a/test/suite/test_join01.py +++ b/test/suite/test_join01.py @@ -33,7 +33,6 @@ from wtscenario import check_scenarios, multiply_scenarios, number_scenarios # Join operations # Basic tests for join class test_join01(wttest.WiredTigerTestCase): - table_name1 = 'test_join01' nentries = 100 scenarios = [ @@ -342,7 +341,7 @@ class test_join01(wttest.WiredTigerTestCase): '/index cursor is being used in a join/') # Only a small number of operations allowed on a join cursor - msg = "/not supported/" + msg = "/Unsupported cursor/" self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: jc.search(), msg) @@ -391,6 +390,7 @@ class test_join01(wttest.WiredTigerTestCase): def test_cursor_close2(self): self.cursor_close_common(False) + # test statistics using the framework set up for this test def test_stats(self): bloomcfg1000 = ',strategy=bloom,count=1000' bloomcfg10 = ',strategy=bloom,count=10' @@ -400,6 +400,40 @@ class test_join01(wttest.WiredTigerTestCase): # statistics should pick up some false positives. self.join_common(bloomcfg10, bloomcfg10, False, True) + # test statistics with a simple one index join cursor + def test_simple_stats(self): + self.session.create("table:join01b", + "key_format=i,value_format=i,columns=(k,v)") + self.session.create("index:join01b:index", "columns=(v)") + + cursor = self.session.open_cursor("table:join01b", None, None) + cursor[1] = 11 + cursor[2] = 12 + cursor[3] = 13 + cursor.close() + + cursor = self.session.open_cursor("index:join01b:index", None, None) + cursor.set_key(11) + cursor.search() + + jcursor = self.session.open_cursor("join:table:join01b", None, None) + self.session.join(jcursor, cursor, "compare=gt") + + while jcursor.next() == 0: + [k] = jcursor.get_keys() + [v] = jcursor.get_values() + + statcur = self.session.open_cursor("statistics:join", jcursor, None) + found = False + while statcur.next() == 0: + [desc, pvalue, value] = statcur.get_values() + #self.tty(str(desc) + "=" + str(pvalue)) + found = True + self.assertEquals(found, True) + + jcursor.close() + cursor.close() + if __name__ == '__main__': wttest.run() diff --git a/test/suite/test_readonly01.py b/test/suite/test_readonly01.py index 86604eb6bfb..59e9743ab7e 100644 --- a/test/suite/test_readonly01.py +++ b/test/suite/test_readonly01.py @@ -96,7 +96,10 @@ class test_readonly01(wttest.WiredTigerTestCase, suite_subprocess): # connection with readonly. # self.close_conn() - if self.dirchmod: + # + # The chmod command is not fully portable to windows. + # + if self.dirchmod and os.name == 'posix': for f in os.listdir(self.home): if os.path.isfile(f): os.chmod(f, 0444) @@ -133,7 +136,7 @@ class test_readonly01(wttest.WiredTigerTestCase, suite_subprocess): self.create = True def test_readonly(self): - if self.dirchmod: + if self.dirchmod and os.name == 'posix': with self.expectedStderrPattern('Permission'): self.readonly() else: diff --git a/test/suite/test_readonly02.py b/test/suite/test_readonly02.py index e94dd85857d..0df5465642d 100644 --- a/test/suite/test_readonly02.py +++ b/test/suite/test_readonly02.py @@ -54,10 +54,8 @@ class test_readonly02(wttest.WiredTigerTestCase, suite_subprocess): # 1. setting readonly on a new database directory # 2. an unclean shutdown and reopening readonly # 3. logging with zero-fill enabled and readonly - # 4. readonly and statistics logging # badcfg1 = 'log=(enabled,zero_fill=true)' - badcfg2 = 'statistics_log=(wait=3)' def setUpConnectionOpen(self, dir): self.home = dir @@ -68,8 +66,10 @@ class test_readonly02(wttest.WiredTigerTestCase, suite_subprocess): if self.create: # 1. setting readonly on a new database directory # Setting readonly prevents creation so we should see an - # ENOENT error because the lock file does not exist. + # error because the lock file does not exist. msg = '/No such file/' + if os.name != 'posix': + msg = '/cannot find the file/' os.mkdir(rdonlydir) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.wiredtiger_open( @@ -111,8 +111,6 @@ class test_readonly02(wttest.WiredTigerTestCase, suite_subprocess): # Close the connection. Reopen readonly with other bad settings. # 3. logging with zero-fill enabled and readonly self.close_checkerror(self.badcfg1) - # 4. readonly and statistics logging - self.close_checkerror(self.badcfg2) if __name__ == '__main__': wttest.run() diff --git a/test/suite/test_readonly03.py b/test/suite/test_readonly03.py index 981a21d51ac..d9930e8f553 100644 --- a/test/suite/test_readonly03.py +++ b/test/suite/test_readonly03.py @@ -68,7 +68,7 @@ class test_readonly03(wttest.WiredTigerTestCase, suite_subprocess): # Now close and reopen. Note that the connection function # above will reopen it readonly. self.reopen_conn() - msg = '/not supported/' + msg = '/Unsupported/' c = self.session.open_cursor(self.uri, None, None) for op in self.cursor_ops: c.set_key(1) |