From e099f346b351dcc94ff4e068a0519d88b5803663 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Thu, 4 Sep 2014 12:14:16 +1000 Subject: Move LSM manager initialization into main thread. --- src/conn/conn_handle.c | 10 +++++ src/include/lsm.h | 2 - src/lsm/lsm_manager.c | 102 +++++++++++++++++++++---------------------------- 3 files changed, 54 insertions(+), 60 deletions(-) diff --git a/src/conn/conn_handle.c b/src/conn/conn_handle.c index 36c53133325..af1513b3014 100644 --- a/src/conn/conn_handle.c +++ b/src/conn/conn_handle.c @@ -51,6 +51,16 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn) WT_RET( __wt_spin_init(session, &conn->page_lock[i], "btree page")); + /* Setup the spin locks for the LSM manager queues. */ + WT_RET(__wt_spin_init( + session, &conn->lsm_manager.app_lock, "LSM application queue lock")); + WT_RET(__wt_spin_init( + session, &conn->lsm_manager.manager_lock, "LSM manager queue lock")); + WT_RET(__wt_spin_init( + session, &conn->lsm_manager.switch_lock, "LSM switch queue lock")); + WT_RET(__wt_cond_alloc( + session, "LSM worker cond", 0, &conn->lsm_manager.work_cond)); + /* * Generation numbers. * diff --git a/src/include/lsm.h b/src/include/lsm.h index fceed7987a7..e0197e01af6 100644 --- a/src/include/lsm.h +++ b/src/include/lsm.h @@ -125,8 +125,6 @@ struct __wt_lsm_manager { uint32_t lsm_workers; /* Current number of LSM workers */ uint32_t lsm_workers_max; WT_LSM_WORKER_ARGS *lsm_worker_cookies; -#define WT_LSM_MANAGER_RUNNING 0x01 - uint32_t flags; }; /* diff --git a/src/lsm/lsm_manager.c b/src/lsm/lsm_manager.c index 7052d16c451..1c9240d6af6 100644 --- a/src/lsm/lsm_manager.c +++ b/src/lsm/lsm_manager.c @@ -15,7 +15,8 @@ static void * __lsm_worker_manager(void *); /* * __wt_lsm_manager_start -- - * Start the LSM management infrastructure. + * Start the LSM management infrastructure. Our queues and locks were + * initialized when the connection was intialized. */ int __wt_lsm_manager_start(WT_SESSION_IMPL *session) @@ -54,8 +55,6 @@ __wt_lsm_manager_start(WT_SESSION_IMPL *session) WT_ERR(__wt_thread_create( session, &cookies[0].tid, __lsm_worker_manager, &cookies[0])); - while (!F_ISSET(manager, WT_LSM_MANAGER_RUNNING)) - __wt_yield(); F_SET(S2C(session), WT_CONN_SERVER_LSM); if (0) { @@ -104,56 +103,59 @@ __wt_lsm_manager_destroy(WT_CONNECTION_IMPL *conn) manager = &conn->lsm_manager; removed = 0; - if (manager->lsm_worker_cookies == NULL) - return (0); + if (manager->lsm_worker_cookies != NULL) { + /* Wait for the server to notice and wrap up. */ + while (F_ISSET(conn, WT_CONN_SERVER_LSM)) + __wt_yield(); - /* Wait for the server to notice and wrap up. */ - while (F_ISSET(conn, WT_CONN_SERVER_LSM)) - __wt_yield(); + /* Clean up open LSM handles. */ + ret = __wt_lsm_tree_close_all(conn->default_session); - /* Clean up open LSM handles. */ - ret = __wt_lsm_tree_close_all(conn->default_session); - - WT_TRET(__wt_thread_join(session, manager->lsm_worker_cookies[0].tid)); - manager->lsm_worker_cookies[0].tid = 0; + WT_TRET(__wt_thread_join( + session, manager->lsm_worker_cookies[0].tid)); + manager->lsm_worker_cookies[0].tid = 0; + + /* Release memory from any operations left on the queue. */ + for (current = TAILQ_FIRST(&manager->switchqh); + current != NULL; current = next) { + next = TAILQ_NEXT(current, q); + TAILQ_REMOVE(&manager->switchqh, current, q); + ++removed; + __wt_lsm_manager_free_work_unit(session, current); + } + for (current = TAILQ_FIRST(&manager->appqh); + current != NULL; current = next) { + next = TAILQ_NEXT(current, q); + TAILQ_REMOVE(&manager->appqh, current, q); + ++removed; + __wt_lsm_manager_free_work_unit(session, current); + } + for (current = TAILQ_FIRST(&manager->managerqh); + current != NULL; current = next) { + next = TAILQ_NEXT(current, q); + TAILQ_REMOVE(&manager->managerqh, current, q); + ++removed; + __wt_lsm_manager_free_work_unit(session, current); + } - /* Release memory from any operations left on the queue. */ - for (current = TAILQ_FIRST(&manager->switchqh); - current != NULL; current = next) { - next = TAILQ_NEXT(current, q); - TAILQ_REMOVE(&manager->switchqh, current, q); - ++removed; - __wt_lsm_manager_free_work_unit(session, current); - } - for (current = TAILQ_FIRST(&manager->appqh); - current != NULL; current = next) { - next = TAILQ_NEXT(current, q); - TAILQ_REMOVE(&manager->appqh, current, q); - ++removed; - __wt_lsm_manager_free_work_unit(session, current); - } - for (current = TAILQ_FIRST(&manager->managerqh); - current != NULL; current = next) { - next = TAILQ_NEXT(current, q); - TAILQ_REMOVE(&manager->managerqh, current, q); - ++removed; - __wt_lsm_manager_free_work_unit(session, current); - } + /* Close all LSM worker sessions. */ + for (i = 0; i < manager->lsm_workers_max; i++) { + wt_session = + &manager->lsm_worker_cookies[i].session->iface; + WT_TRET(wt_session->close(wt_session, NULL)); + } - /* Close all LSM worker sessions. */ - for (i = 0; i < manager->lsm_workers_max; i++) { - wt_session = &manager->lsm_worker_cookies[i].session->iface; - WT_TRET(wt_session->close(wt_session, NULL)); + WT_STAT_FAST_CONN_INCRV(session, + lsm_work_units_discarded, removed); + __wt_free(session, manager->lsm_worker_cookies); } - WT_STAT_FAST_CONN_INCRV(session, lsm_work_units_discarded, removed); - + /* Free resources that are allocated in connection initialize */ __wt_spin_destroy(session, &manager->switch_lock); __wt_spin_destroy(session, &manager->app_lock); __wt_spin_destroy(session, &manager->manager_lock); WT_TRET(__wt_cond_destroy(session, &manager->work_cond)); - __wt_free(session, manager->lsm_worker_cookies); return (ret); } @@ -208,16 +210,6 @@ __lsm_manager_worker_setup(WT_SESSION_IMPL *session) WT_ASSERT(session, manager->lsm_workers == 1); - /* Setup the spin locks for the queues. */ - WT_RET(__wt_spin_init( - session, &manager->app_lock, "LSM application queue lock")); - WT_RET(__wt_spin_init( - session, &manager->manager_lock, "LSM manager queue lock")); - WT_RET(__wt_spin_init( - session, &manager->switch_lock, "LSM switch queue lock")); - WT_RET(__wt_cond_alloc( - session, "LSM worker cond", 0, &manager->work_cond)); - worker_args = &manager->lsm_worker_cookies[1]; worker_args->work_cond = manager->work_cond; worker_args->id = manager->lsm_workers++; @@ -252,12 +244,6 @@ __lsm_manager_worker_setup(WT_SESSION_IMPL *session) F_SET(worker_args, WT_LSM_WORK_MERGE); WT_RET(__wt_lsm_worker_start(session, worker_args)); } - /* - * Yield to give new threads a chance to get started. Indicate that - * we have allocated resources and are running now. - */ - __wt_yield(); - F_SET(manager, WT_LSM_MANAGER_RUNNING); return (0); } -- cgit v1.2.1 From d3eecc260fb9bf7ea35b28c997100c8265c3a650 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 4 Sep 2014 07:17:58 -0400 Subject: long lines, whitespace --- src/conn/conn_handle.c | 8 ++++---- src/lsm/lsm_manager.c | 3 +-- src/lsm/lsm_worker.c | 3 ++- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/conn/conn_handle.c b/src/conn/conn_handle.c index af1513b3014..66517f9c2de 100644 --- a/src/conn/conn_handle.c +++ b/src/conn/conn_handle.c @@ -52,10 +52,10 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn) __wt_spin_init(session, &conn->page_lock[i], "btree page")); /* Setup the spin locks for the LSM manager queues. */ - WT_RET(__wt_spin_init( - session, &conn->lsm_manager.app_lock, "LSM application queue lock")); - WT_RET(__wt_spin_init( - session, &conn->lsm_manager.manager_lock, "LSM manager queue lock")); + WT_RET(__wt_spin_init(session, + &conn->lsm_manager.app_lock, "LSM application queue lock")); + WT_RET(__wt_spin_init(session, + &conn->lsm_manager.manager_lock, "LSM manager queue lock")); WT_RET(__wt_spin_init( session, &conn->lsm_manager.switch_lock, "LSM switch queue lock")); WT_RET(__wt_cond_alloc( diff --git a/src/lsm/lsm_manager.c b/src/lsm/lsm_manager.c index 1c9240d6af6..34cd13554d1 100644 --- a/src/lsm/lsm_manager.c +++ b/src/lsm/lsm_manager.c @@ -16,7 +16,7 @@ static void * __lsm_worker_manager(void *); /* * __wt_lsm_manager_start -- * Start the LSM management infrastructure. Our queues and locks were - * initialized when the connection was intialized. + * initialized when the connection was initialized. */ int __wt_lsm_manager_start(WT_SESSION_IMPL *session) @@ -156,7 +156,6 @@ __wt_lsm_manager_destroy(WT_CONNECTION_IMPL *conn) __wt_spin_destroy(session, &manager->manager_lock); WT_TRET(__wt_cond_destroy(session, &manager->work_cond)); - return (ret); } diff --git a/src/lsm/lsm_worker.c b/src/lsm/lsm_worker.c index 0bb6cfb9c08..257fda03f3b 100644 --- a/src/lsm/lsm_worker.c +++ b/src/lsm/lsm_worker.c @@ -93,7 +93,8 @@ __lsm_worker(void *arg) (ret = __wt_lsm_manager_pop_entry( session, WT_LSM_WORK_SWITCH, &entry)) == 0 && entry != NULL) - WT_ERR(__wt_lsm_work_switch(session, &entry, &progress)); + WT_ERR( + __wt_lsm_work_switch(session, &entry, &progress)); /* Flag an error if the pop failed. */ WT_ERR(ret); -- cgit v1.2.1 From b2542b7800942e1c6ed3d8eb3a4b976a2310fee1 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 4 Sep 2014 07:33:39 -0400 Subject: Don't remove the wiredtiger directory, it has source code, reference #1188. --- lang/python/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/python/Makefile.am b/lang/python/Makefile.am index 0ac56138e29..ae4e6cc6441 100644 --- a/lang/python/Makefile.am +++ b/lang/python/Makefile.am @@ -19,6 +19,6 @@ install-exec-local: clean-local: $(PYTHON) $(PYSRC)/setup.py clean - rm -rf _wiredtiger.so WT_TEST build wiredtiger + rm -rf _wiredtiger.so WT_TEST build TESTS = run-ex_access -- cgit v1.2.1 From ed5c5d33892f1a8d044b4b009ca7c00eed3ce21e Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 4 Sep 2014 07:41:32 -0400 Subject: Add wiredtiger_wrap.o to the clean-local label. --- lang/python/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/python/Makefile.am b/lang/python/Makefile.am index ae4e6cc6441..223d9695f76 100644 --- a/lang/python/Makefile.am +++ b/lang/python/Makefile.am @@ -19,6 +19,6 @@ install-exec-local: clean-local: $(PYTHON) $(PYSRC)/setup.py clean - rm -rf _wiredtiger.so WT_TEST build + rm -rf _wiredtiger.so wiredtiger_wrap.o WT_TEST build TESTS = run-ex_access -- cgit v1.2.1 From eece553c0be260e8e51012b29fa302807aa85e25 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 4 Sep 2014 07:58:20 -0400 Subject: As far as I can tell, there's no "build" directory, so quit trying to remove it, reference #1188. --- lang/python/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/python/Makefile.am b/lang/python/Makefile.am index 223d9695f76..a6b44932d60 100644 --- a/lang/python/Makefile.am +++ b/lang/python/Makefile.am @@ -19,6 +19,6 @@ install-exec-local: clean-local: $(PYTHON) $(PYSRC)/setup.py clean - rm -rf _wiredtiger.so wiredtiger_wrap.o WT_TEST build + rm -rf _wiredtiger.so wiredtiger_wrap.o WT_TEST TESTS = run-ex_access -- cgit v1.2.1 From a5a290e95ab0209cef905637cbb563017745f200 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 14 Aug 2014 11:04:56 -0400 Subject: It's "PREDEFINED", not "PREDEFINE" --- dist/s_docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/s_docs b/dist/s_docs index 815d27d7b11..c0c8885e1b5 100755 --- a/dist/s_docs +++ b/dist/s_docs @@ -113,7 +113,7 @@ valid_build() } classf=`ls ../docs/struct___* 2>/dev/null` for c in $classf; do - echo "$c: Need to add class to PREDEFINE in src/docs/Doxyfile" + echo "$c: Need to add class to PREDEFINED in src/docs/Doxyfile" done } -- cgit v1.2.1 From d4b42f0563b46c51bee85d60035724ef41412387 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 15 Aug 2014 12:37:09 -0400 Subject: Minor cleanups: "onpage_ovfl" is never interesting if the overflow key has already been removed, factor that into the original test and set of the variable. --- src/btree/rec_write.c | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/src/btree/rec_write.c b/src/btree/rec_write.c index 4687505df11..bae29929aa5 100644 --- a/src/btree/rec_write.c +++ b/src/btree/rec_write.c @@ -3859,7 +3859,7 @@ __rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) WT_REF *ref; size_t size; u_int vtype; - int hazard, onpage_ovfl, ovfl_key, state; + int hazard, key_onpage_ovfl, ovfl_key, state; const void *p; btree = S2BT(session); @@ -3907,11 +3907,12 @@ __rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) ikey = __wt_ref_key_instantiated(ref); if (ikey == NULL || ikey->cell_offset == 0) { cell = NULL; - onpage_ovfl = 0; + key_onpage_ovfl = 0; } else { cell = WT_PAGE_REF_OFFSET(page, ikey->cell_offset); __wt_cell_unpack(cell, kpack); - onpage_ovfl = kpack->ovfl == 1 ? 1 : 0; + key_onpage_ovfl = + kpack->ovfl && kpack->raw != WT_CELL_KEY_OVFL_RM; } WT_ERR(__rec_child_modify(session, r, ref, &hazard, &state)); @@ -3928,7 +3929,7 @@ __rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) * always instantiated. Don't worry about reuse, * reusing this key in this reconciliation is unlikely. */ - if (onpage_ovfl && kpack->raw != WT_CELL_KEY_OVFL_RM) + if (key_onpage_ovfl) WT_ERR(__wt_ovfl_discard_add( session, page, kpack->cell)); CHILD_RELEASE_ERR(session, hazard, ref); @@ -3954,8 +3955,7 @@ __rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) * worry about reuse, reusing this key in this * reconciliation is unlikely. */ - if (onpage_ovfl && - kpack->raw != WT_CELL_KEY_OVFL_RM) + if (key_onpage_ovfl) WT_ERR(__wt_ovfl_discard_add( session, page, kpack->cell)); CHILD_RELEASE_ERR(session, hazard, ref); @@ -3970,8 +3970,7 @@ __rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) * worry about reuse, reusing this key in this * reconciliation is unlikely. */ - if (onpage_ovfl && - kpack->raw != WT_CELL_KEY_OVFL_RM) + if (key_onpage_ovfl) WT_ERR(__wt_ovfl_discard_add( session, page, kpack->cell)); @@ -4010,19 +4009,11 @@ __rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) __rec_cell_build_addr(r, p, size, vtype, 0); CHILD_RELEASE_ERR(session, hazard, ref); - /* - * If the key is an overflow key, check to see if the backing - * blocks have been freed; in that case, we have to build a new - * key. - */ - if (onpage_ovfl && kpack->raw == WT_CELL_KEY_OVFL_RM) - onpage_ovfl = 0; - /* * Build key cell. * Truncate any 0th key, internal pages don't need 0th keys. */ - if (onpage_ovfl) { + if (key_onpage_ovfl) { key->buf.data = cell; key->buf.size = __wt_cell_total_len(kpack); key->cell_len = 0; @@ -4048,10 +4039,10 @@ __rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) * case, we have to build the actual key now because we * are about to promote it. */ - if (onpage_ovfl) { + if (key_onpage_ovfl) { WT_ERR(__wt_buf_set(session, r->cur, WT_IKEY_DATA(ikey), ikey->size)); - onpage_ovfl = 0; + key_onpage_ovfl = 0; } WT_ERR(__rec_split(session, r)); } -- cgit v1.2.1 From a012e4829e15461f172f78a1d1bfbb3fe9992b0f Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 4 Sep 2014 11:40:50 -0400 Subject: Send t_ret error value to lprintf. --- bench/wtperf/wtperf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/wtperf/wtperf.c b/bench/wtperf/wtperf.c index ad645e19596..38352fd3ef6 100644 --- a/bench/wtperf/wtperf.c +++ b/bench/wtperf/wtperf.c @@ -1871,7 +1871,7 @@ err: if (ret == 0) if (cfg->conn != NULL && (t_ret = cfg->conn->close(cfg->conn, NULL)) != 0) { - lprintf(cfg, ret, 0, + lprintf(cfg, t_ret, 0, "Error closing connection to %s", cfg->home); if (ret == 0) ret = t_ret; -- cgit v1.2.1 From 5604615b1053ec20dfa37dfac66642423cd8cb18 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Fri, 5 Sep 2014 11:03:05 +1000 Subject: Document the autogen.sh script in the top level of the tree: that is common practise on GitHub. refs #1198 --- src/docs/install.dox | 7 +++---- src/docs/spell.ok | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/docs/install.dox b/src/docs/install.dox index f3f5094eb5d..eae566f4291 100644 --- a/src/docs/install.dox +++ b/src/docs/install.dox @@ -21,15 +21,14 @@ First, clone the repository: git clone git://github.com/wiredtiger/wiredtiger.git @endcode -Second, run the \c build_posix/reconf script: +Second, run \c autogen.sh to create the \c configure script: @code cd wiredtiger -sh build_posix/reconf +sh autogen.sh @endcode -This creates the \c configure script, and you can now proceed with @ref -building. +Now proceed with @ref building. @section building Building WiredTiger diff --git a/src/docs/spell.ok b/src/docs/spell.ok index 95bdf58fc06..857f89cef05 100644 --- a/src/docs/spell.ok +++ b/src/docs/spell.ok @@ -88,6 +88,7 @@ ar archiver arg async +autogen atomicity autoconf automake @@ -339,7 +340,6 @@ realloc'd recno recnoN recnum -reconf recoverability recs rectype -- cgit v1.2.1 From fedef3e83f881596e83a1df196fda233b251b021 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Fri, 5 Sep 2014 12:21:59 +1000 Subject: Rearrange the Python build so we generate object files under the build directory. Clean up manually: don't rely on setup.py. refs #1188 --- lang/python/Makefile.am | 18 +++++++++++------- lang/python/setup.py | 10 +++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lang/python/Makefile.am b/lang/python/Makefile.am index a6b44932d60..03c65a57028 100644 --- a/lang/python/Makefile.am +++ b/lang/python/Makefile.am @@ -1,5 +1,5 @@ PYSRC = $(top_srcdir)/lang/python -PY_INCLUDE_DIRS = $(top_srcdir) +PYDIRS = -t $(abs_builddir) -I $(abs_top_srcdir):$(abs_top_builddir) -L $(abs_top_builddir)/.libs all-local: _wiredtiger.so # We keep generated Python sources under lang/python: that's where they live @@ -10,15 +10,19 @@ $(PYSRC)/wiredtiger_wrap.c: $(top_srcdir)/src/include/wiredtiger.in $(PYSRC)/wir mv wiredtiger.py wiredtiger/__init__.py) _wiredtiger.so: $(top_builddir)/libwiredtiger.la $(PYSRC)/wiredtiger_wrap.c - $(PYTHON) $(PYSRC)/setup.py build_ext -b . -t . -f -I $(PY_INCLUDE_DIRS) + (cd $(PYSRC) && \ + $(PYTHON) setup.py build_ext -f -b $(abs_builddir) $(PYDIRS)) install-exec-local: - $(PYTHON) $(PYSRC)/setup.py build_py -d build - $(PYTHON) $(PYSRC)/setup.py build_ext -b build -t . -f -I $(PY_INCLUDE_DIRS) - $(PYTHON) $(PYSRC)/setup.py install_lib -b build --skip-build $(PYTHON_INSTALL_ARG) + (cd $(PYSRC) && \ + $(PYTHON) setup.py build_py -d $(abs_builddir)/build && \ + $(PYTHON) setup.py build_ext -f -b $(abs_builddir)/build $(PYDIRS) && \ + $(PYTHON) setup.py install_lib -b $(abs_builddir)/build --skip-build $(PYTHON_INSTALL_ARG)) +# We build in different places for an install vs running from the tree: +# clean up both. Don't rely on "setup.py clean" -- everything that should +# be removed is created under the build directory. clean-local: - $(PYTHON) $(PYSRC)/setup.py clean - rm -rf _wiredtiger.so wiredtiger_wrap.o WT_TEST + rm -rf build _wiredtiger.so wiredtiger_wrap.o WT_TEST TESTS = run-ex_access diff --git a/lang/python/setup.py b/lang/python/setup.py index 1c6ebc71387..1057006ce50 100644 --- a/lang/python/setup.py +++ b/lang/python/setup.py @@ -35,9 +35,7 @@ if not 'ARCHFLAGS' in os.environ: os.environ['ARCHFLAGS'] = '' # Suppress warnings building SWIG generated code -extra_cflags = [ - '-w', -] +extra_cflags = [ '-w' ] dir = os.path.dirname(__file__) @@ -50,12 +48,10 @@ wt_ver = '%d.%d' % (WIREDTIGER_VERSION_MAJOR, WIREDTIGER_VERSION_MINOR) setup(name='wiredtiger', version=wt_ver, ext_modules=[Extension('_wiredtiger', - [os.path.join(dir, 'wiredtiger_wrap.c')], - include_dirs=['../..'], - library_dirs=['../../.libs'], + [os.path.join(dir, 'wiredtiger_wrap.c')], libraries=['wiredtiger'], extra_compile_args=extra_cflags, )], - package_dir={'' : dir}, + package_dir={'' : dir}, packages=['wiredtiger'], ) -- cgit v1.2.1 From 7bbaf0f70683dd7cf91ad3bd728885d263f973d0 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 5 Sep 2014 07:38:27 -0400 Subject: whitespace --- src/include/dhandle.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/dhandle.h b/src/include/dhandle.h index 9a05620c74c..5556627c74d 100644 --- a/src/include/dhandle.h +++ b/src/include/dhandle.h @@ -18,7 +18,7 @@ #define WT_SET_BTREE_IN_SESSION(s, b) ((s)->dhandle = b->dhandle) #define WT_CLEAR_BTREE_IN_SESSION(s) ((s)->dhandle = NULL) -#define WT_WITH_DHANDLE(s, d, e) do { \ +#define WT_WITH_DHANDLE(s, d, e) do { \ WT_DATA_HANDLE *__saved_dhandle = (s)->dhandle; \ (s)->dhandle = (d); \ e; \ -- cgit v1.2.1 From 7a009dc0aad7d82f666d5ceadfac479bc65773d7 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 5 Sep 2014 07:39:02 -0400 Subject: WT_WITH_SCHEMA_LOCK, WT_SESSION_SCHEMA_LOCKED don't need to spin, #1202. --- src/include/schema.h | 47 +++++++++++++++++++++-------------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/src/include/schema.h b/src/include/schema.h index 038404f5ea5..c8c80dcf9c1 100644 --- a/src/include/schema.h +++ b/src/include/schema.h @@ -65,43 +65,38 @@ struct __wt_table { */ #define WT_COLGROUPS(t) WT_MAX((t)->ncolgroups, 1) +/* + * WT_WITH_SCHEMA_LOCK -- + * Acquire the schema lock, perform an operation, drop the lock. + */ #define WT_WITH_SCHEMA_LOCK(session, op) do { \ - int __schema_locked = 0; \ - WT_DECL_SPINLOCK_ID(__id); /* Must appear last */ \ WT_ASSERT(session, \ F_ISSET(session, WT_SESSION_SCHEMA_LOCKED) || \ !F_ISSET(session, WT_SESSION_NO_SCHEMA_LOCK)); \ - while (!F_ISSET(session, WT_SESSION_SCHEMA_LOCKED)) \ - if (session->skip_schema_lock || __wt_spin_trylock( \ - session, &S2C(session)->schema_lock, &__id) == 0) { \ - F_SET(session, WT_SESSION_SCHEMA_LOCKED); \ - __schema_locked = 1; \ - } else \ - __wt_yield(); \ - (op); \ - if (__schema_locked) { \ + if (F_ISSET(session, WT_SESSION_SCHEMA_LOCKED) || \ + session->skip_schema_lock) { \ + (op); \ + } else { \ + __wt_spin_lock(session, &S2C(session)->schema_lock); \ + F_SET(session, WT_SESSION_SCHEMA_LOCKED); \ + (op); \ + __wt_spin_unlock(session, &S2C(session)->schema_lock); \ F_CLR(session, WT_SESSION_SCHEMA_LOCKED); \ - if (!session->skip_schema_lock) \ - __wt_spin_unlock( \ - session, &S2C(session)->schema_lock); \ } \ } while (0) -/* Drop the schema lock, and re-acquire after operation. */ +/* + * WT_WITHOUT_SCHEMA_LOCK -- + * Drop the schema lock, perform an operation, re-acquire the lock. + */ #define WT_WITHOUT_SCHEMA_LOCK(session, op) do { \ - WT_DECL_SPINLOCK_ID(__id); /* Must appear last */ \ - if (!F_ISSET(session, WT_SESSION_SCHEMA_LOCKED)) \ - (op); \ - else { \ + if (F_ISSET(session, WT_SESSION_SCHEMA_LOCKED)) { \ __wt_spin_unlock(session, &S2C(session)->schema_lock); \ F_CLR(session, WT_SESSION_SCHEMA_LOCKED); \ (op); \ - while (!F_ISSET(session, WT_SESSION_SCHEMA_LOCKED)) { \ - if (__wt_spin_trylock(session, \ - &S2C(session)->schema_lock, &__id) == 0) \ - F_SET(session, WT_SESSION_SCHEMA_LOCKED);\ - else \ - __wt_yield(); \ - } \ + __wt_spin_lock(session, &S2C(session)->schema_lock); \ + F_SET(session, WT_SESSION_SCHEMA_LOCKED); \ + } else { \ + (op); \ } \ } while (0) -- cgit v1.2.1 From 99ce38eef9b3b6828e5769d56cd0d53b272358e8 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sat, 6 Sep 2014 15:59:51 -0400 Subject: Write a new version of config-collapse that merges nested structures so reconfiguration doesn't lose previous configuration information. Reference #1172. --- src/config/config_collapse.c | 282 +++++++++++++++++++++++++++++++++++++++---- src/conn/conn_api.c | 57 +++++---- src/conn/conn_handle.c | 1 + src/include/connection.h | 2 + src/include/extern.h | 5 +- src/meta/meta_ckpt.c | 2 +- src/meta/meta_turtle.c | 2 +- src/schema/schema_create.c | 8 +- src/session/session_api.c | 2 +- 9 files changed, 304 insertions(+), 57 deletions(-) diff --git a/src/config/config_collapse.c b/src/config/config_collapse.c index 8d245680f69..8b56e57bc74 100644 --- a/src/config/config_collapse.c +++ b/src/config/config_collapse.c @@ -8,29 +8,64 @@ #include "wt_internal.h" /* - * __wt_config_collapse -- - * Given a NULL-terminated list of configuration strings, where the first - * one contains all the defaults, collapse them into newly allocated - * memory. + * We need a character that can't appear in a key as a separator. + * + * XXX + * I'm not using '.' although that seems like the natural one to use because + * default checkpoints are named "WiredTiger.#" where dot is part of the key. + * I think it's wrong, we should not have used a dot in that name, but that's + * a format change. */ -int -__wt_config_collapse( - WT_SESSION_IMPL *session, const char **cfg, const char **config_ret) +#undef SEP /* separator key, character */ +#define SEP "," +#undef SEPC +#define SEPC ',' + +/* + * Individual configuration entries, including a generation number used to make + * the qsort stable. + */ +typedef struct { + char *k, *v; /* key, value */ + size_t gen; /* generation */ +} WT_COLLAPSE_ENTRY; + +/* + * The array of configuration entries. + */ +typedef struct { + size_t entries_allocated; /* allocated */ + size_t entries_next; /* next slot */ + + int nested_replace; /* replace nested values */ + + WT_COLLAPSE_ENTRY *entries; /* array of entries */ +} WT_COLLAPSE; + +/* + * __collapse_scan -- + * Walk a configuration string, inserting entries into the collapse array. + */ +static int +__collapse_scan(WT_SESSION_IMPL *session, + const char *key, const char *value, WT_COLLAPSE *cp) { WT_CONFIG cparser; WT_CONFIG_ITEM k, v; - WT_DECL_ITEM(tmp); + WT_DECL_ITEM(kb); + WT_DECL_ITEM(vb); WT_DECL_RET; - WT_RET(__wt_scr_alloc(session, 0, &tmp)); + WT_ERR(__wt_scr_alloc(session, 0, &kb)); + WT_ERR(__wt_scr_alloc(session, 0, &vb)); - WT_ERR(__wt_config_init(session, &cparser, cfg[0])); + WT_ERR(__wt_config_init(session, &cparser, value)); while ((ret = __wt_config_next(&cparser, &k, &v)) == 0) { if (k.type != WT_CONFIG_ITEM_STRING && k.type != WT_CONFIG_ITEM_ID) WT_ERR_MSG(session, EINVAL, "Invalid configuration key found: '%s'\n", k.str); - WT_ERR(__wt_config_get(session, cfg, &k, &v)); + /* Include the quotes around string keys/values. */ if (k.type == WT_CONFIG_ITEM_STRING) { --k.str; @@ -40,23 +75,222 @@ __wt_config_collapse( --v.str; v.len += 2; } - WT_ERR(__wt_buf_catfmt(session, tmp, "%.*s=%.*s,", - (int)k.len, k.str, (int)v.len, v.str)); + + /* Build the key/value strings. */ + WT_ERR(__wt_buf_fmt(session, + kb, "%s%s%.*s", + key == NULL ? "" : key, + key == NULL ? "" : SEP, + (int)k.len, k.str)); + WT_ERR(__wt_buf_fmt(session, + vb, "%.*s", (int)v.len, v.str)); + + /* + * If the value is a structure, recursively parse it. + * + * XXX + * Problem #1: we store "checkpoint_lsn=(1,0)" in the metadata + * file, where the key is type WT_CONFIG_ITEM_ID, the value is + * type WT_CONFIG_ITEM_STRUCT. Other nested structures have + * field names, should this have been "(file=1,offset=0)"? + * + * Problem #2: the configuration collapse functions are used by + * checkpoint to replace the previous entry in its entirety, + * that is, the work we're doing to integrate nested changes + * into previous values breaks it. + * + * We're currently turning off merging nested structures in most + * places (including the checkpoint code). + */ + if (!cp->nested_replace && + v.type == WT_CONFIG_ITEM_STRUCT && + strchr(vb->data, '=') != NULL) { + WT_ERR( + __collapse_scan(session, kb->data, vb->data, cp)); + continue; + } + + /* Insert the value into the array. */ + WT_ERR(__wt_realloc_def(session, + &cp->entries_allocated, + cp->entries_next + 1, &cp->entries)); + WT_ERR(__wt_strndup(session, + kb->data, kb->size, &cp->entries[cp->entries_next].k)); + WT_ERR(__wt_strndup(session, + vb->data, vb->size, &cp->entries[cp->entries_next].v)); + cp->entries[cp->entries_next].gen = cp->entries_next; + ++cp->entries_next; + } + +err: __wt_scr_free(&kb); + __wt_scr_free(&vb); + return (0); +} + +/* + * __strip_comma -- + * Strip a trailing comma. + */ +static inline void +__strip_comma(WT_ITEM *buf) +{ + if (buf->size != 0 && ((char *)buf->data)[buf->size - 1] == ',') + --buf->size; +} + +/* + * __collapse_format_next -- + * Walk the array, building entries. + */ +static int +__collapse_format_next(WT_SESSION_IMPL *session, const char *prefix, + size_t plen, size_t *enp, WT_COLLAPSE *cp, WT_ITEM *build) +{ + WT_COLLAPSE_ENTRY *ep; + size_t len1, len2, next; + char *p; + + for (; *enp < cp->entries_next; ++*enp) { + ep = &cp->entries[*enp]; + + /* + * The entries are in sorted order, take the last entry for any + * key. + */ + if (*enp < (cp->entries_next - 1)) { + len1 = strlen(ep->k); + len2 = strlen((ep + 1)->k); + + /* Choose the last of identical keys. */ + if (len1 == len2 && + memcmp(ep->k, (ep + 1)->k, len1) == 0) + continue; + + /* + * The test is complicated by matching empty entries + * "foo=" against nested structures "foo,bar=", where + * the latter is a replacement for the former. + */ + if (len2 > len1 && + (ep + 1)->k[len1] == SEPC && + memcmp(ep->k, (ep + 1)->k, len1) == 0) + continue; + } + + /* + * If we're skipping a prefix and this entry doesn't match it, + * back off one entry and pop up a level. + */ + if (plen != 0 && memcmp(ep->k, prefix, plen) != 0) { + --*enp; + break; + } + + /* + * If the entry introduces a new level, recurse through that + * new level. + */ + if ((p = strchr(ep->k + plen, SEPC)) != NULL) { + next = WT_PTRDIFF(p, ep->k); + WT_RET(__wt_buf_catfmt(session, + build, "%.*s=(", (int)(next - plen), ep->k + plen)); + WT_RET(__collapse_format_next( + session, ep->k, next + 1, enp, cp, build)); + __strip_comma(build); + WT_RET(__wt_buf_catfmt(session, build, "),")); + continue; + } + + /* Append the entry to the buffer. */ + WT_RET(__wt_buf_catfmt( + session, build, "%s=%s,", ep->k + plen, ep->v)); } - if (ret != WT_NOTFOUND) - goto err; + + return (0); +} + +/* + * __collapse_format -- + * Take the sorted array of entries, and format them into allocated memory. + */ +static int +__collapse_format( + WT_SESSION_IMPL *session, WT_COLLAPSE *cp, const char **config_ret) +{ + WT_DECL_ITEM(build); + WT_DECL_RET; + size_t entries; + + WT_RET(__wt_scr_alloc(session, 4 * 1024, &build)); + + entries = 0; + WT_ERR(__collapse_format_next(session, "", 0, &entries, cp, build)); + + __strip_comma(build); + + ret = __wt_strndup(session, build->data, build->size, config_ret); + +err: __wt_scr_free(&build); + return (ret); +} + +/* + * __collapse_cmp -- + * Qsort function: sort the collapse array. + */ +static int +__collapse_cmp(const void *a, const void *b) +{ + WT_COLLAPSE_ENTRY *ae, *be; + int cmp; + + ae = (WT_COLLAPSE_ENTRY *)a; + be = (WT_COLLAPSE_ENTRY *)b; + + if ((cmp = strcmp(ae->k, be->k)) != 0) + return (cmp); + return (ae->gen > be->gen ? 1 : -1); +} + +/* + * __wt_config_collapse -- + * Given a NULL-terminated list of configuration strings, in reverse order + * of preference (the first set of strings are the least preferred), collapse + * them into allocated memory. + */ +int +__wt_config_collapse(WT_SESSION_IMPL *session, + const char **cfg, const char **config_ret, int nested_replace) +{ + WT_COLLAPSE collapse; + WT_DECL_RET; + size_t i; + + /* Start out with a reasonable number of entries. */ + WT_CLEAR(collapse); + collapse.nested_replace = nested_replace; + + WT_RET(__wt_realloc_def( + session, &collapse.entries_allocated, 100, &collapse.entries)); + + /* Scan the configuration strings, entering them into the array. */ + for (; *cfg != NULL; ++cfg) + WT_ERR(__collapse_scan(session, NULL, *cfg, &collapse)); /* - * If the caller passes us no valid configuration strings, we get here - * with no bytes to copy -- that's OK, the underlying string copy can - * handle empty strings. - * - * Strip any trailing comma. + * Sort the array by key and, in the case of identical keys, by + * generation. */ - if (tmp->size != 0) - --tmp->size; - ret = __wt_strndup(session, tmp->data, tmp->size, config_ret); + qsort(collapse.entries, + collapse.entries_next, sizeof(WT_COLLAPSE_ENTRY), __collapse_cmp); -err: __wt_scr_free(&tmp); + /* Convert the array of entries into a string. */ + ret = __collapse_format(session, &collapse, config_ret); + +err: for (i = 0; i < collapse.entries_next; ++i) { + __wt_free(session, collapse.entries[i].k); + __wt_free(session, collapse.entries[i].v); + } + __wt_free(session, collapse.entries); return (ret); } diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index e7826e9fd56..41bf351f2a5 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -630,30 +630,36 @@ __conn_reconfigure(WT_CONNECTION *wt_conn, const char *config) WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_SESSION_IMPL *session; - - /* - * Special version of cfg that doesn't include the default config: used - * to limit changes to values that the application sets explicitly. - * Note that any function using this value has to be prepared to handle - * not-found as a valid option return. - */ - const char *raw_cfg[] = { config, NULL }; + const char *p, *config_cfg[] = { NULL, NULL, NULL }; conn = (WT_CONNECTION_IMPL *)wt_conn; CONNECTION_API_CALL(conn, session, reconfigure, config, cfg); + WT_UNUSED(cfg); - WT_ERR(__wt_conn_cache_pool_config(session, cfg)); - WT_ERR(__wt_cache_config(conn, raw_cfg)); + /* + * The configuration argument has been checked for validity, replace the + * previous connection configuration. + * + * DO NOT collapse the configuration before the reconfigure calls. Some + * of the underlying reconfiguration functions do explicit checks for + * the second element of the configuration array, knowing the defaults + * are in slot #1 and the application's modifications are in slot #2. + */ + config_cfg[0] = conn->cfg; + config_cfg[1] = config; - WT_ERR(__wt_async_reconfig(conn, raw_cfg)); - WT_ERR(__conn_statistics_config(session, raw_cfg)); - WT_ERR(__wt_conn_verbose_config(session, raw_cfg)); - WT_ERR(__wt_checkpoint_server_create(conn, cfg)); - WT_ERR(__wt_statlog_create(conn, cfg)); + WT_ERR(__wt_conn_cache_pool_config(session, config_cfg)); + WT_ERR(__wt_cache_config(conn, config_cfg)); + + WT_ERR(__wt_async_reconfig(conn, config_cfg)); + WT_ERR(__conn_statistics_config(session, config_cfg)); + WT_ERR(__wt_conn_verbose_config(session, config_cfg)); + WT_ERR(__wt_checkpoint_server_create(conn, config_cfg)); + WT_ERR(__wt_statlog_create(conn, config_cfg)); WT_ERR(__wt_config_gets( - session, cfg, "lsm_manager.worker_thread_max", &cval)); + session, config_cfg, "lsm_manager.worker_thread_max", &cval)); if (cval.val) conn->lsm_manager.lsm_workers_max = (uint32_t)cval.val; @@ -662,6 +668,10 @@ __conn_reconfigure(WT_CONNECTION *wt_conn, const char *config) WT_ERR(__wt_cond_signal( session, __wt_process.cache_pool->cache_pool_cond)); + WT_ERR(__wt_config_collapse(session, config_cfg, &p, 0)); + __wt_free(session, conn->cfg); + conn->cfg = p; + err: API_END_RET(session, ret); } @@ -1399,18 +1409,17 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, */ WT_ERR(__wt_connection_workers(session, cfg)); + /* Take a copy of the final configuration for later reconfiguration. */ + WT_ERR(__wt_config_collapse(session, cfg, &conn->cfg, 0)); + STATIC_ASSERT(offsetof(WT_CONNECTION_IMPL, iface) == 0); *wt_connp = &conn->iface; - /* - * Destroying the connection on error will destroy our session handle, - * cleanup using the session handle first, then discard the connection. - */ -err: __wt_buf_free(session, &cbbuf); - __wt_buf_free(session, &cubuf); - - if (ret != 0 && conn != NULL) +err: if (ret != 0 && conn != NULL) WT_TRET(__wt_connection_close(conn)); + __wt_buf_free(session, &cbbuf); + __wt_buf_free(session, &cubuf); + return (ret); } diff --git a/src/conn/conn_handle.c b/src/conn/conn_handle.c index 66517f9c2de..03c22b04e30 100644 --- a/src/conn/conn_handle.c +++ b/src/conn/conn_handle.c @@ -130,6 +130,7 @@ __wt_connection_destroy(WT_CONNECTION_IMPL *conn) __wt_free(session, conn->page_lock); /* Free allocated memory. */ + __wt_free(session, conn->cfg); __wt_free(session, conn->home); __wt_free(session, conn->error_prefix); __wt_free(session, conn->sessions); diff --git a/src/include/connection.h b/src/include/connection.h index 03feef68e56..1f1f8be88ea 100644 --- a/src/include/connection.h +++ b/src/include/connection.h @@ -70,6 +70,8 @@ struct __wt_connection_impl { WT_SESSION_IMPL *default_session; WT_SESSION_IMPL dummy_session; + const char *cfg; /* Connection configuration */ + WT_SPINLOCK api_lock; /* Connection API spinlock */ WT_SPINLOCK checkpoint_lock; /* Checkpoint spinlock */ WT_SPINLOCK fh_lock; /* File handle queue spinlock */ diff --git a/src/include/extern.h b/src/include/extern.h index a213a0e4bfa..a462815a93d 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -572,9 +572,10 @@ extern int __wt_config_check(WT_SESSION_IMPL *session, const WT_CONFIG_ENTRY *entry, const char *config, size_t config_len); -extern int __wt_config_collapse( WT_SESSION_IMPL *session, +extern int __wt_config_collapse(WT_SESSION_IMPL *session, const char **cfg, - const char **config_ret); + const char **config_ret, + int nested_replace); extern int __wt_config_concat( WT_SESSION_IMPL *session, const char **cfg, const char **config_ret); diff --git a/src/meta/meta_ckpt.c b/src/meta/meta_ckpt.c index 998ae7e0d02..ec42a8032ae 100644 --- a/src/meta/meta_ckpt.c +++ b/src/meta/meta_ckpt.c @@ -117,7 +117,7 @@ __ckpt_set(WT_SESSION_IMPL *session, const char *fname, const char *v) cfg[0] = config; cfg[1] = v == NULL ? "checkpoint=()" : v; cfg[2] = NULL; - WT_ERR(__wt_config_collapse(session, cfg, &newcfg)); + WT_ERR(__wt_config_collapse(session, cfg, &newcfg, 1)); WT_ERR(__wt_metadata_update(session, fname, newcfg)); err: __wt_free(session, config); diff --git a/src/meta/meta_turtle.c b/src/meta/meta_turtle.c index 2e3eca10833..b0914165837 100644 --- a/src/meta/meta_turtle.c +++ b/src/meta/meta_turtle.c @@ -29,7 +29,7 @@ __metadata_config(WT_SESSION_IMPL *session, const char **metaconfp) "key_format=S,value_format=S,id=0,version=(major=%d,minor=%d)", WT_BTREE_MAJOR_VERSION_MAX, WT_BTREE_MINOR_VERSION_MAX)); cfg[1] = buf->data; - WT_ERR(__wt_config_collapse(session, cfg, &metaconf)); + WT_ERR(__wt_config_collapse(session, cfg, &metaconf, 1)); *metaconfp = metaconf; diff --git a/src/schema/schema_create.c b/src/schema/schema_create.c index 398fea4476f..50005753559 100644 --- a/src/schema/schema_create.c +++ b/src/schema/schema_create.c @@ -102,7 +102,7 @@ __create_file(WT_SESSION_IMPL *session, for (p = filecfg; *p != NULL; ++p) ; *p = val->data; - WT_ERR(__wt_config_collapse(session, filecfg, &fileconf)); + WT_ERR(__wt_config_collapse(session, filecfg, &fileconf, 1)); WT_ERR(__wt_metadata_insert(session, uri, fileconf)); } @@ -248,7 +248,7 @@ __create_colgroup(WT_SESSION_IMPL *session, WT_ERR(__wt_schema_create(session, source, sourceconf)); - WT_ERR(__wt_config_collapse(session, cfg, &cgconf)); + WT_ERR(__wt_config_collapse(session, cfg, &cgconf, 1)); if ((ret = __wt_metadata_insert(session, name, cgconf)) != 0) { /* * If the entry already exists in the metadata, we're done. @@ -416,7 +416,7 @@ __create_index(WT_SESSION_IMPL *session, cfg[1] = sourceconf; cfg[2] = confbuf.data; - WT_ERR(__wt_config_collapse(session, cfg, &idxconf)); + WT_ERR(__wt_config_collapse(session, cfg, &idxconf, 1)); if ((ret = __wt_metadata_insert(session, name, idxconf)) != 0) { /* * If the entry already exists in the metadata, we're done. @@ -480,7 +480,7 @@ __create_table(WT_SESSION_IMPL *session, ; WT_RET_NOTFOUND_OK(ret); - WT_RET(__wt_config_collapse(session, cfg, &tableconf)); + WT_RET(__wt_config_collapse(session, cfg, &tableconf, 1)); if ((ret = __wt_metadata_insert(session, name, tableconf)) != 0) { /* * If the entry already exists in the metadata, we're done. diff --git a/src/session/session_api.c b/src/session/session_api.c index e63e2c0284a..36f7afc075e 100644 --- a/src/session/session_api.c +++ b/src/session/session_api.c @@ -300,7 +300,7 @@ __wt_session_create_strip(WT_SESSION *wt_session, const char *cfg[] = { WT_CONFIG_BASE(session, session_create), v1, v2, NULL }; - return (__wt_config_collapse(session, cfg, value_ret)); + return (__wt_config_collapse(session, cfg, value_ret, 1)); } /* -- cgit v1.2.1 From 021f259d74db789d8271d8f6fdcf9bac5b166663 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Sun, 7 Sep 2014 20:02:55 -0400 Subject: Get rid of WT_SESSION.skip_schema_lock, it's no longer used anywhere. --- src/include/schema.h | 3 +-- src/include/session.h | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/include/schema.h b/src/include/schema.h index c8c80dcf9c1..e24a19b03ca 100644 --- a/src/include/schema.h +++ b/src/include/schema.h @@ -73,8 +73,7 @@ struct __wt_table { WT_ASSERT(session, \ F_ISSET(session, WT_SESSION_SCHEMA_LOCKED) || \ !F_ISSET(session, WT_SESSION_NO_SCHEMA_LOCK)); \ - if (F_ISSET(session, WT_SESSION_SCHEMA_LOCKED) || \ - session->skip_schema_lock) { \ + if (F_ISSET(session, WT_SESSION_SCHEMA_LOCKED)) { \ (op); \ } else { \ __wt_spin_lock(session, &S2C(session)->schema_lock); \ diff --git a/src/include/session.h b/src/include/session.h index 5d566f8b62d..31d58ff61e5 100644 --- a/src/include/session.h +++ b/src/include/session.h @@ -113,8 +113,6 @@ struct __wt_session_impl { int (*reconcile_cleanup)(WT_SESSION_IMPL *); int compaction; /* Compaction did some work */ - int skip_schema_lock; /* Another thread holds the schema lock - * on our behalf */ /* * The split stash memory and hazard information persist past session -- cgit v1.2.1 From c7ddaaeef8cf7483111c7266c99bf490b4e88016 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Mon, 8 Sep 2014 22:20:49 +1000 Subject: Fixes for the LevelDB JNI build. Make the fields of Slice public, rename the config file and only include the necessary defines. --- api/leveldb/Makefile.am | 2 +- api/leveldb/basho/perf_count.h | 2 +- api/leveldb/config.hin | 22 ++++++++++++++++++++++ api/leveldb/hyperleveldb/replay_iterator.h | 2 +- api/leveldb/leveldb/include/leveldb/cache.h | 2 +- api/leveldb/leveldb/include/leveldb/comparator.h | 2 +- api/leveldb/leveldb/include/leveldb/db.h | 8 +++++++- api/leveldb/leveldb/include/leveldb/env.h | 2 +- .../leveldb/include/leveldb/filter_policy.h | 2 +- api/leveldb/leveldb/include/leveldb/iterator.h | 2 +- api/leveldb/leveldb/include/leveldb/options.h | 2 +- api/leveldb/leveldb/include/leveldb/slice.h | 5 +++-- api/leveldb/leveldb/include/leveldb/status.h | 2 +- api/leveldb/leveldb/include/leveldb/write_batch.h | 2 +- api/leveldb/leveldb_wt.cc | 14 ++------------ api/leveldb/leveldb_wt.h | 3 ++- build_posix/configure.ac.in | 2 +- 17 files changed, 48 insertions(+), 28 deletions(-) create mode 100644 api/leveldb/config.hin diff --git a/api/leveldb/Makefile.am b/api/leveldb/Makefile.am index 44aa69bbd48..2cfd9d945a5 100644 --- a/api/leveldb/Makefile.am +++ b/api/leveldb/Makefile.am @@ -16,7 +16,7 @@ leveldbincludedir = $(includedir)/wiredtiger/leveldb endif endif leveldbinclude_HEADERS = \ - wiredtiger_config.h \ + leveldb_wt_config.h \ leveldb/include/leveldb/cache.h \ leveldb/include/leveldb/comparator.h\ leveldb/include/leveldb/db.h \ diff --git a/api/leveldb/basho/perf_count.h b/api/leveldb/basho/perf_count.h index 0edf1b96549..b0f4abf9b66 100644 --- a/api/leveldb/basho/perf_count.h +++ b/api/leveldb/basho/perf_count.h @@ -23,7 +23,7 @@ #ifndef STORAGE_LEVELDB_INCLUDE_PERF_COUNT_H_ #define STORAGE_LEVELDB_INCLUDE_PERF_COUNT_H_ -#include "wiredtiger_config.h" +#include "leveldb_wt_config.h" #include #include diff --git a/api/leveldb/config.hin b/api/leveldb/config.hin new file mode 100644 index 00000000000..131b68969d3 --- /dev/null +++ b/api/leveldb/config.hin @@ -0,0 +1,22 @@ +/* api/leveldb/config.hin. Generated by autoheader, then hand-edited. */ + +/* Build the LevelDB API with Basho LevelDB support. */ +#undef HAVE_BASHOLEVELDB + +/* Snappy support automatically loaded. */ +#undef HAVE_BUILTIN_EXTENSION_SNAPPY + +/* Zlib support automatically loaded. */ +#undef HAVE_BUILTIN_EXTENSION_ZLIB + +/* Define to 1 for diagnostic tests. */ +#undef HAVE_DIAGNOSTIC + +/* Build the LevelDB API with HyperLevelDB support. */ +#undef HAVE_HYPERLEVELDB + +/* Define to 1 if you have the `snappy' library (-lsnappy). */ +#undef HAVE_LIBSNAPPY + +/* Build the LevelDB API with RocksDB support. */ +#undef HAVE_ROCKSDB diff --git a/api/leveldb/hyperleveldb/replay_iterator.h b/api/leveldb/hyperleveldb/replay_iterator.h index 6e2f562c6c4..397acdfd889 100644 --- a/api/leveldb/hyperleveldb/replay_iterator.h +++ b/api/leveldb/hyperleveldb/replay_iterator.h @@ -5,7 +5,7 @@ #ifndef STORAGE_LEVELDB_INCLUDE_REPLAY_ITERATOR_H_ #define STORAGE_LEVELDB_INCLUDE_REPLAY_ITERATOR_H_ -#include "wiredtiger_config.h" +#include "leveldb_wt_config.h" #include "slice.h" #include "status.h" diff --git a/api/leveldb/leveldb/include/leveldb/cache.h b/api/leveldb/leveldb/include/leveldb/cache.h index 6ae25122133..94be8e919a8 100644 --- a/api/leveldb/leveldb/include/leveldb/cache.h +++ b/api/leveldb/leveldb/include/leveldb/cache.h @@ -18,7 +18,7 @@ #ifndef STORAGE_LEVELDB_INCLUDE_CACHE_H_ #define STORAGE_LEVELDB_INCLUDE_CACHE_H_ -#include "wiredtiger_config.h" +#include "leveldb_wt_config.h" #if defined(HAVE_ROCKSDB) && !defined(leveldb) #define leveldb rocksdb #endif diff --git a/api/leveldb/leveldb/include/leveldb/comparator.h b/api/leveldb/leveldb/include/leveldb/comparator.h index 23e0ba84559..78d83a4d08e 100644 --- a/api/leveldb/leveldb/include/leveldb/comparator.h +++ b/api/leveldb/leveldb/include/leveldb/comparator.h @@ -5,7 +5,7 @@ #ifndef STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_ #define STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_ -#include "wiredtiger_config.h" +#include "leveldb_wt_config.h" #if defined(HAVE_ROCKSDB) && !defined(leveldb) #define leveldb rocksdb #endif diff --git a/api/leveldb/leveldb/include/leveldb/db.h b/api/leveldb/leveldb/include/leveldb/db.h index c1818d28a7a..df8fcbbe9f8 100644 --- a/api/leveldb/leveldb/include/leveldb/db.h +++ b/api/leveldb/leveldb/include/leveldb/db.h @@ -5,7 +5,7 @@ #ifndef STORAGE_LEVELDB_INCLUDE_DB_H_ #define STORAGE_LEVELDB_INCLUDE_DB_H_ -#include "wiredtiger_config.h" +#include "leveldb_wt_config.h" #if defined(HAVE_ROCKSDB) && !defined(leveldb) #define leveldb rocksdb #endif @@ -292,6 +292,12 @@ class DB { // db->CompactRange(NULL, NULL); virtual void CompactRange(const Slice* begin, const Slice* end) = 0; + // Suspends the background compaction thread. This methods + // returns once suspended. + virtual void SuspendCompactions() = 0; + // Resumes a suspended background compation thread. + virtual void ResumeCompactions() = 0; + #ifdef HAVE_HYPERLEVELDB // Create a live backup of a live LevelDB instance. // The backup is stored in a directory named "backup-" under the top diff --git a/api/leveldb/leveldb/include/leveldb/env.h b/api/leveldb/leveldb/include/leveldb/env.h index 0d043307736..4ad67d36fea 100644 --- a/api/leveldb/leveldb/include/leveldb/env.h +++ b/api/leveldb/leveldb/include/leveldb/env.h @@ -13,7 +13,7 @@ #ifndef STORAGE_LEVELDB_INCLUDE_ENV_H_ #define STORAGE_LEVELDB_INCLUDE_ENV_H_ -#include "wiredtiger_config.h" +#include "leveldb_wt_config.h" #if defined(HAVE_ROCKSDB) && !defined(leveldb) #define leveldb rocksdb #endif diff --git a/api/leveldb/leveldb/include/leveldb/filter_policy.h b/api/leveldb/leveldb/include/leveldb/filter_policy.h index 2d970e709d6..e434ef4b241 100644 --- a/api/leveldb/leveldb/include/leveldb/filter_policy.h +++ b/api/leveldb/leveldb/include/leveldb/filter_policy.h @@ -16,7 +16,7 @@ #ifndef STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_ #define STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_ -#include "wiredtiger_config.h" +#include "leveldb_wt_config.h" #if defined(HAVE_ROCKSDB) && !defined(leveldb) #define leveldb rocksdb #endif diff --git a/api/leveldb/leveldb/include/leveldb/iterator.h b/api/leveldb/leveldb/include/leveldb/iterator.h index 3845d553a4e..2d97d180b17 100644 --- a/api/leveldb/leveldb/include/leveldb/iterator.h +++ b/api/leveldb/leveldb/include/leveldb/iterator.h @@ -15,7 +15,7 @@ #ifndef STORAGE_LEVELDB_INCLUDE_ITERATOR_H_ #define STORAGE_LEVELDB_INCLUDE_ITERATOR_H_ -#include "wiredtiger_config.h" +#include "leveldb_wt_config.h" #if defined(HAVE_ROCKSDB) && !defined(leveldb) #define leveldb rocksdb #endif diff --git a/api/leveldb/leveldb/include/leveldb/options.h b/api/leveldb/leveldb/include/leveldb/options.h index a14503fe086..9dcf73fc2a0 100644 --- a/api/leveldb/leveldb/include/leveldb/options.h +++ b/api/leveldb/leveldb/include/leveldb/options.h @@ -5,7 +5,7 @@ #ifndef STORAGE_LEVELDB_INCLUDE_OPTIONS_H_ #define STORAGE_LEVELDB_INCLUDE_OPTIONS_H_ -#include "wiredtiger_config.h" +#include "leveldb_wt_config.h" #if defined(HAVE_ROCKSDB) && !defined(leveldb) #define leveldb rocksdb #endif diff --git a/api/leveldb/leveldb/include/leveldb/slice.h b/api/leveldb/leveldb/include/leveldb/slice.h index d7c20cfcaac..1eb66dd825f 100644 --- a/api/leveldb/leveldb/include/leveldb/slice.h +++ b/api/leveldb/leveldb/include/leveldb/slice.h @@ -15,7 +15,7 @@ #ifndef STORAGE_LEVELDB_INCLUDE_SLICE_H_ #define STORAGE_LEVELDB_INCLUDE_SLICE_H_ -#include "wiredtiger_config.h" +#include "leveldb_wt_config.h" #if defined(HAVE_ROCKSDB) && !defined(leveldb) #define leveldb rocksdb #endif @@ -82,7 +82,8 @@ class Slice { (memcmp(data_, x.data_, x.size_) == 0)); } - private: +// The LevelDB JNI layer peeks in here +// private: const char* data_; size_t size_; diff --git a/api/leveldb/leveldb/include/leveldb/status.h b/api/leveldb/leveldb/include/leveldb/status.h index 8b2cbb9b422..3c21f64462b 100644 --- a/api/leveldb/leveldb/include/leveldb/status.h +++ b/api/leveldb/leveldb/include/leveldb/status.h @@ -13,7 +13,7 @@ #ifndef STORAGE_LEVELDB_INCLUDE_STATUS_H_ #define STORAGE_LEVELDB_INCLUDE_STATUS_H_ -#include "wiredtiger_config.h" +#include "leveldb_wt_config.h" #if defined(HAVE_ROCKSDB) && !defined(leveldb) #define leveldb rocksdb #endif diff --git a/api/leveldb/leveldb/include/leveldb/write_batch.h b/api/leveldb/leveldb/include/leveldb/write_batch.h index 9184d42c24c..293b41ad818 100644 --- a/api/leveldb/leveldb/include/leveldb/write_batch.h +++ b/api/leveldb/leveldb/include/leveldb/write_batch.h @@ -21,7 +21,7 @@ #ifndef STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_ #define STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_ -#include "wiredtiger_config.h" +#include "leveldb_wt_config.h" #if defined(HAVE_ROCKSDB) && !defined(leveldb) #define leveldb rocksdb #endif diff --git a/api/leveldb/leveldb_wt.cc b/api/leveldb/leveldb_wt.cc index cfeb0549db4..6425a5a8dfd 100644 --- a/api/leveldb/leveldb_wt.cc +++ b/api/leveldb/leveldb_wt.cc @@ -755,13 +755,8 @@ IteratorImpl::Next() int ret; WT_ITEM item; - if (!Status().ok()) - return; - - if (!valid_) { - SetError(EINVAL); + if (!Status().ok() || !valid_) return; - } ret = cursor_->next(cursor_); if (ret != 0) { @@ -791,13 +786,8 @@ IteratorImpl::Prev() { WT_ITEM item; - if (!Status().ok()) - return; - - if (!valid_) { - SetError(EINVAL); + if (!Status().ok() || !valid_) return; - } int ret = cursor_->prev(cursor_); if (ret != 0) { diff --git a/api/leveldb/leveldb_wt.h b/api/leveldb/leveldb_wt.h index 301fa250e85..683482ad23c 100644 --- a/api/leveldb/leveldb_wt.h +++ b/api/leveldb/leveldb_wt.h @@ -27,7 +27,7 @@ #ifndef _INCLUDE_LEVELDB_WT_H #define _INCLUDE_LEVELDB_WT_H 1 -#include "wiredtiger_config.h" +#include "leveldb_wt_config.h" #include "leveldb/cache.h" #include "leveldb/comparator.h" @@ -171,6 +171,7 @@ private: class CacheImpl : public Cache { public: CacheImpl(size_t capacity) : Cache(), capacity_(capacity) {} + virtual ~CacheImpl() {} virtual Handle* Insert(const Slice&, void*, size_t, void (*)(const Slice&, void*)) { return 0; } diff --git a/build_posix/configure.ac.in b/build_posix/configure.ac.in index dd5bce738ea..268ed586232 100644 --- a/build_posix/configure.ac.in +++ b/build_posix/configure.ac.in @@ -165,7 +165,7 @@ AC_CONFIG_HEADERS([wiredtiger_config.h:build_posix/config.hin]) # The LevelDB API needs some configuration knowledge AM_COND_IF([LEVELDB], - AC_CONFIG_HEADERS([api/leveldb/wiredtiger_config.h:build_posix/config.hin])) + AC_CONFIG_HEADERS([api/leveldb/leveldb_wt_config.h:api/leveldb/config.hin])) # BEGIN check existence -- maintained by reconf and Make.subdirs # END check existence -- cgit v1.2.1 From be4dce95a559e66578795b41e914fd6660a3a72f Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 8 Sep 2014 11:12:22 -0400 Subject: Create __wt_config_merge for the merge functionality, called only from reconfigure at the moment. Clean up some comments, explain how config-collapse and config-merge differ. --- src/config/config_collapse.c | 216 +++++++++++++++++++++++++++++-------------- src/conn/conn_api.c | 4 +- src/include/extern.h | 8 +- src/meta/meta_ckpt.c | 2 +- src/meta/meta_turtle.c | 2 +- src/schema/schema_create.c | 8 +- src/session/session_api.c | 2 +- 7 files changed, 163 insertions(+), 79 deletions(-) diff --git a/src/config/config_collapse.c b/src/config/config_collapse.c index 8b56e57bc74..2d708521d20 100644 --- a/src/config/config_collapse.c +++ b/src/config/config_collapse.c @@ -8,18 +8,81 @@ #include "wt_internal.h" /* - * We need a character that can't appear in a key as a separator. + * __wt_config_collapse -- + * Collapse a set of configuration strings into newly allocated memory. + * + * This function takes a NULL-terminated list of configuration strings (where + * the first one contains all the defaults and the values are in order from + * least to most preferred, that is, the default values are least preferred), + * and collapses them into newly allocated memory. The algorithm is to walk + * the first of the configuration strings, and for each entry, search all of + * the configuration strings for a final value, keeping the last value found. * - * XXX - * I'm not using '.' although that seems like the natural one to use because - * default checkpoints are named "WiredTiger.#" where dot is part of the key. - * I think it's wrong, we should not have used a dot in that name, but that's - * a format change. + * Notes: + * Any key not appearing in the first configuration string is discarded + * from the final result, because we'll never search for it. + * + * Nested structures aren't parsed. For example, imagine a configuration + * string contains "key=(k2=v2,k3=v3)", and a subsequent string has + * "key=(k4=v4)", the result will be "key=(k4=v4)", as we search for and + * use the final value of "key", regardless of field overlap or missing + * fields in the nested value. + */ +int +__wt_config_collapse( + WT_SESSION_IMPL *session, const char **cfg, const char **config_ret) +{ + WT_CONFIG cparser; + WT_CONFIG_ITEM k, v; + WT_DECL_ITEM(tmp); + WT_DECL_RET; + + WT_RET(__wt_scr_alloc(session, 0, &tmp)); + + WT_ERR(__wt_config_init(session, &cparser, cfg[0])); + while ((ret = __wt_config_next(&cparser, &k, &v)) == 0) { + if (k.type != WT_CONFIG_ITEM_STRING && + k.type != WT_CONFIG_ITEM_ID) + WT_ERR_MSG(session, EINVAL, + "Invalid configuration key found: '%s'\n", k.str); + WT_ERR(__wt_config_get(session, cfg, &k, &v)); + /* Include the quotes around string keys/values. */ + if (k.type == WT_CONFIG_ITEM_STRING) { + --k.str; + k.len += 2; + } + if (v.type == WT_CONFIG_ITEM_STRING) { + --v.str; + v.len += 2; + } + WT_ERR(__wt_buf_catfmt(session, tmp, "%.*s=%.*s,", + (int)k.len, k.str, (int)v.len, v.str)); + } + if (ret != WT_NOTFOUND) + goto err; + + /* + * If the caller passes us no valid configuration strings, we get here + * with no bytes to copy -- that's OK, the underlying string copy can + * handle empty strings. + * + * Strip any trailing comma. + */ + if (tmp->size != 0) + --tmp->size; + ret = __wt_strndup(session, tmp->data, tmp->size, config_ret); + +err: __wt_scr_free(&tmp); + return (ret); +} + +/* + * We need a character that can't appear in a key as a separator. */ #undef SEP /* separator key, character */ -#define SEP "," +#define SEP "." #undef SEPC -#define SEPC ',' +#define SEPC '.' /* * Individual configuration entries, including a generation number used to make @@ -28,7 +91,7 @@ typedef struct { char *k, *v; /* key, value */ size_t gen; /* generation */ -} WT_COLLAPSE_ENTRY; +} WT_CONFIG_MERGE_ENTRY; /* * The array of configuration entries. @@ -37,18 +100,16 @@ typedef struct { size_t entries_allocated; /* allocated */ size_t entries_next; /* next slot */ - int nested_replace; /* replace nested values */ - - WT_COLLAPSE_ENTRY *entries; /* array of entries */ -} WT_COLLAPSE; + WT_CONFIG_MERGE_ENTRY *entries; /* array of entries */ +} WT_CONFIG_MERGE; /* - * __collapse_scan -- - * Walk a configuration string, inserting entries into the collapse array. + * __config_merge_scan -- + * Walk a configuration string, inserting entries into the merged array. */ static int -__collapse_scan(WT_SESSION_IMPL *session, - const char *key, const char *value, WT_COLLAPSE *cp) +__config_merge_scan(WT_SESSION_IMPL *session, + const char *key, const char *value, WT_CONFIG_MERGE *cp) { WT_CONFIG cparser; WT_CONFIG_ITEM k, v; @@ -85,28 +146,40 @@ __collapse_scan(WT_SESSION_IMPL *session, WT_ERR(__wt_buf_fmt(session, vb, "%.*s", (int)v.len, v.str)); + /* + * !!! + * WiredTiger names its internal checkpoints with a trailing + * dot and a number, for example, "WiredTigerCheckpoint.37". + * We're using dot to separate names in nested structures, + * and there's an obvious conflict. This works for now because + * that's the only case of a dot in a key name, and we never + * merge configuration strings that contain checkpoint names, + * for historic reasons. For now, return an error if there's + * ever a problem. (Note, it's probably safe if the dot is in + * a quoted key, that is, a key of type WT_CONFIG_ITEM_STRING, + * but since this isn't ever supposed to happen, I'm leaving + * the test simple.) + */ + if (strchr(kb->data, SEPC) != NULL) + WT_RET_MSG(session, EINVAL, + "key %s contains a separator character (%s)", + kb->data, SEP); + /* * If the value is a structure, recursively parse it. * - * XXX - * Problem #1: we store "checkpoint_lsn=(1,0)" in the metadata - * file, where the key is type WT_CONFIG_ITEM_ID, the value is - * type WT_CONFIG_ITEM_STRUCT. Other nested structures have - * field names, should this have been "(file=1,offset=0)"? - * - * Problem #2: the configuration collapse functions are used by - * checkpoint to replace the previous entry in its entirety, - * that is, the work we're doing to integrate nested changes - * into previous values breaks it. - * - * We're currently turning off merging nested structures in most - * places (including the checkpoint code). + * !!! + * Don't merge unless the structure has field names. WiredTiger + * stores checkpoint LSNs in the metadata file using nested + * structures without field names: "checkpoint_lsn=(1,0)", not + * "checkpoint_lsn=(file=1,offset=0)". The value type is still + * WT_CONFIG_ITEM_STRUCT, so we check for a field name in the + * value. */ - if (!cp->nested_replace && - v.type == WT_CONFIG_ITEM_STRUCT && + if (v.type == WT_CONFIG_ITEM_STRUCT && strchr(vb->data, '=') != NULL) { - WT_ERR( - __collapse_scan(session, kb->data, vb->data, cp)); + WT_ERR(__config_merge_scan( + session, kb->data, vb->data, cp)); continue; } @@ -139,14 +212,14 @@ __strip_comma(WT_ITEM *buf) } /* - * __collapse_format_next -- + * __config_merge_format_next -- * Walk the array, building entries. */ static int -__collapse_format_next(WT_SESSION_IMPL *session, const char *prefix, - size_t plen, size_t *enp, WT_COLLAPSE *cp, WT_ITEM *build) +__config_merge_format_next(WT_SESSION_IMPL *session, const char *prefix, + size_t plen, size_t *enp, WT_CONFIG_MERGE *cp, WT_ITEM *build) { - WT_COLLAPSE_ENTRY *ep; + WT_CONFIG_MERGE_ENTRY *ep; size_t len1, len2, next; char *p; @@ -194,7 +267,7 @@ __collapse_format_next(WT_SESSION_IMPL *session, const char *prefix, next = WT_PTRDIFF(p, ep->k); WT_RET(__wt_buf_catfmt(session, build, "%.*s=(", (int)(next - plen), ep->k + plen)); - WT_RET(__collapse_format_next( + WT_RET(__config_merge_format_next( session, ep->k, next + 1, enp, cp, build)); __strip_comma(build); WT_RET(__wt_buf_catfmt(session, build, "),")); @@ -210,12 +283,12 @@ __collapse_format_next(WT_SESSION_IMPL *session, const char *prefix, } /* - * __collapse_format -- + * __config_merge_format -- * Take the sorted array of entries, and format them into allocated memory. */ static int -__collapse_format( - WT_SESSION_IMPL *session, WT_COLLAPSE *cp, const char **config_ret) +__config_merge_format( + WT_SESSION_IMPL *session, WT_CONFIG_MERGE *cp, const char **config_ret) { WT_DECL_ITEM(build); WT_DECL_RET; @@ -224,7 +297,7 @@ __collapse_format( WT_RET(__wt_scr_alloc(session, 4 * 1024, &build)); entries = 0; - WT_ERR(__collapse_format_next(session, "", 0, &entries, cp, build)); + WT_ERR(__config_merge_format_next(session, "", 0, &entries, cp, build)); __strip_comma(build); @@ -235,17 +308,17 @@ err: __wt_scr_free(&build); } /* - * __collapse_cmp -- - * Qsort function: sort the collapse array. + * __config_merge_cmp -- + * Qsort function: sort the config merge array. */ static int -__collapse_cmp(const void *a, const void *b) +__config_merge_cmp(const void *a, const void *b) { - WT_COLLAPSE_ENTRY *ae, *be; + WT_CONFIG_MERGE_ENTRY *ae, *be; int cmp; - ae = (WT_COLLAPSE_ENTRY *)a; - be = (WT_COLLAPSE_ENTRY *)b; + ae = (WT_CONFIG_MERGE_ENTRY *)a; + be = (WT_CONFIG_MERGE_ENTRY *)b; if ((cmp = strcmp(ae->k, be->k)) != 0) return (cmp); @@ -253,44 +326,53 @@ __collapse_cmp(const void *a, const void *b) } /* - * __wt_config_collapse -- - * Given a NULL-terminated list of configuration strings, in reverse order - * of preference (the first set of strings are the least preferred), collapse - * them into allocated memory. + * __wt_config_merge -- + * Merge a set of configuration strings into newly allocated memory. + * + * This function takes a NULL-terminated list of configuration strings (where + * the values are in order from least to most preferred), and merges them into + * newly allocated memory. The algorithm is to walk the configuration strings + * and build a table of each key/value pair. The pairs are sorted based on the + * name and the configuration string in which they were found, and a final + * configuration string is built from the result. + * + * Note: + * Nested structures are parsed and merge. For example, if configuration + * strings "key=(k1=v1,k2=v2)" and "key=(k1=v2)" appear, the result will + * be "key=(k1=v2,k2=v2)" because the nested values are merged. */ int -__wt_config_collapse(WT_SESSION_IMPL *session, - const char **cfg, const char **config_ret, int nested_replace) +__wt_config_merge( + WT_SESSION_IMPL *session, const char **cfg, const char **config_ret) { - WT_COLLAPSE collapse; + WT_CONFIG_MERGE merge; WT_DECL_RET; size_t i; /* Start out with a reasonable number of entries. */ - WT_CLEAR(collapse); - collapse.nested_replace = nested_replace; + WT_CLEAR(merge); WT_RET(__wt_realloc_def( - session, &collapse.entries_allocated, 100, &collapse.entries)); + session, &merge.entries_allocated, 100, &merge.entries)); /* Scan the configuration strings, entering them into the array. */ for (; *cfg != NULL; ++cfg) - WT_ERR(__collapse_scan(session, NULL, *cfg, &collapse)); + WT_ERR(__config_merge_scan(session, NULL, *cfg, &merge)); /* * Sort the array by key and, in the case of identical keys, by * generation. */ - qsort(collapse.entries, - collapse.entries_next, sizeof(WT_COLLAPSE_ENTRY), __collapse_cmp); + qsort(merge.entries, merge.entries_next, + sizeof(WT_CONFIG_MERGE_ENTRY), __config_merge_cmp); /* Convert the array of entries into a string. */ - ret = __collapse_format(session, &collapse, config_ret); + ret = __config_merge_format(session, &merge, config_ret); -err: for (i = 0; i < collapse.entries_next; ++i) { - __wt_free(session, collapse.entries[i].k); - __wt_free(session, collapse.entries[i].v); +err: for (i = 0; i < merge.entries_next; ++i) { + __wt_free(session, merge.entries[i].k); + __wt_free(session, merge.entries[i].v); } - __wt_free(session, collapse.entries); + __wt_free(session, merge.entries); return (ret); } diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 41bf351f2a5..955933f81cc 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -668,7 +668,7 @@ __conn_reconfigure(WT_CONNECTION *wt_conn, const char *config) WT_ERR(__wt_cond_signal( session, __wt_process.cache_pool->cache_pool_cond)); - WT_ERR(__wt_config_collapse(session, config_cfg, &p, 0)); + WT_ERR(__wt_config_merge(session, config_cfg, &p)); __wt_free(session, conn->cfg); conn->cfg = p; @@ -1410,7 +1410,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, WT_ERR(__wt_connection_workers(session, cfg)); /* Take a copy of the final configuration for later reconfiguration. */ - WT_ERR(__wt_config_collapse(session, cfg, &conn->cfg, 0)); + WT_ERR(__wt_config_collapse(session, cfg, &conn->cfg)); STATIC_ASSERT(offsetof(WT_CONNECTION_IMPL, iface) == 0); *wt_connp = &conn->iface; diff --git a/src/include/extern.h b/src/include/extern.h index a462815a93d..43821086bf5 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -572,10 +572,12 @@ extern int __wt_config_check(WT_SESSION_IMPL *session, const WT_CONFIG_ENTRY *entry, const char *config, size_t config_len); -extern int __wt_config_collapse(WT_SESSION_IMPL *session, +extern int __wt_config_collapse( WT_SESSION_IMPL *session, const char **cfg, - const char **config_ret, - int nested_replace); + const char **config_ret); +extern int __wt_config_merge( WT_SESSION_IMPL *session, + const char **cfg, + const char **config_ret); extern int __wt_config_concat( WT_SESSION_IMPL *session, const char **cfg, const char **config_ret); diff --git a/src/meta/meta_ckpt.c b/src/meta/meta_ckpt.c index ec42a8032ae..998ae7e0d02 100644 --- a/src/meta/meta_ckpt.c +++ b/src/meta/meta_ckpt.c @@ -117,7 +117,7 @@ __ckpt_set(WT_SESSION_IMPL *session, const char *fname, const char *v) cfg[0] = config; cfg[1] = v == NULL ? "checkpoint=()" : v; cfg[2] = NULL; - WT_ERR(__wt_config_collapse(session, cfg, &newcfg, 1)); + WT_ERR(__wt_config_collapse(session, cfg, &newcfg)); WT_ERR(__wt_metadata_update(session, fname, newcfg)); err: __wt_free(session, config); diff --git a/src/meta/meta_turtle.c b/src/meta/meta_turtle.c index b0914165837..2e3eca10833 100644 --- a/src/meta/meta_turtle.c +++ b/src/meta/meta_turtle.c @@ -29,7 +29,7 @@ __metadata_config(WT_SESSION_IMPL *session, const char **metaconfp) "key_format=S,value_format=S,id=0,version=(major=%d,minor=%d)", WT_BTREE_MAJOR_VERSION_MAX, WT_BTREE_MINOR_VERSION_MAX)); cfg[1] = buf->data; - WT_ERR(__wt_config_collapse(session, cfg, &metaconf, 1)); + WT_ERR(__wt_config_collapse(session, cfg, &metaconf)); *metaconfp = metaconf; diff --git a/src/schema/schema_create.c b/src/schema/schema_create.c index 50005753559..398fea4476f 100644 --- a/src/schema/schema_create.c +++ b/src/schema/schema_create.c @@ -102,7 +102,7 @@ __create_file(WT_SESSION_IMPL *session, for (p = filecfg; *p != NULL; ++p) ; *p = val->data; - WT_ERR(__wt_config_collapse(session, filecfg, &fileconf, 1)); + WT_ERR(__wt_config_collapse(session, filecfg, &fileconf)); WT_ERR(__wt_metadata_insert(session, uri, fileconf)); } @@ -248,7 +248,7 @@ __create_colgroup(WT_SESSION_IMPL *session, WT_ERR(__wt_schema_create(session, source, sourceconf)); - WT_ERR(__wt_config_collapse(session, cfg, &cgconf, 1)); + WT_ERR(__wt_config_collapse(session, cfg, &cgconf)); if ((ret = __wt_metadata_insert(session, name, cgconf)) != 0) { /* * If the entry already exists in the metadata, we're done. @@ -416,7 +416,7 @@ __create_index(WT_SESSION_IMPL *session, cfg[1] = sourceconf; cfg[2] = confbuf.data; - WT_ERR(__wt_config_collapse(session, cfg, &idxconf, 1)); + WT_ERR(__wt_config_collapse(session, cfg, &idxconf)); if ((ret = __wt_metadata_insert(session, name, idxconf)) != 0) { /* * If the entry already exists in the metadata, we're done. @@ -480,7 +480,7 @@ __create_table(WT_SESSION_IMPL *session, ; WT_RET_NOTFOUND_OK(ret); - WT_RET(__wt_config_collapse(session, cfg, &tableconf, 1)); + WT_RET(__wt_config_collapse(session, cfg, &tableconf)); if ((ret = __wt_metadata_insert(session, name, tableconf)) != 0) { /* * If the entry already exists in the metadata, we're done. diff --git a/src/session/session_api.c b/src/session/session_api.c index 36f7afc075e..e63e2c0284a 100644 --- a/src/session/session_api.c +++ b/src/session/session_api.c @@ -300,7 +300,7 @@ __wt_session_create_strip(WT_SESSION *wt_session, const char *cfg[] = { WT_CONFIG_BASE(session, session_create), v1, v2, NULL }; - return (__wt_config_collapse(session, cfg, value_ret, 1)); + return (__wt_config_collapse(session, cfg, value_ret)); } /* -- cgit v1.2.1 From 77769ec0bb3e5d3323ffa1d6e0ad77963eaa731c Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 8 Sep 2014 15:33:34 -0400 Subject: Messed up the test for a separator character (dot) in the key, the check has to appear before we build the key we're using for the merge. --- src/config/config_collapse.c | 29 ++++++++++++++++------------- src/conn/conn_api.c | 4 ++-- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/config/config_collapse.c b/src/config/config_collapse.c index 2d708521d20..e56c6a62800 100644 --- a/src/config/config_collapse.c +++ b/src/config/config_collapse.c @@ -116,6 +116,8 @@ __config_merge_scan(WT_SESSION_IMPL *session, WT_DECL_ITEM(kb); WT_DECL_ITEM(vb); WT_DECL_RET; + size_t len; + char *str; WT_ERR(__wt_scr_alloc(session, 0, &kb)); WT_ERR(__wt_scr_alloc(session, 0, &vb)); @@ -137,15 +139,6 @@ __config_merge_scan(WT_SESSION_IMPL *session, v.len += 2; } - /* Build the key/value strings. */ - WT_ERR(__wt_buf_fmt(session, - kb, "%s%s%.*s", - key == NULL ? "" : key, - key == NULL ? "" : SEP, - (int)k.len, k.str)); - WT_ERR(__wt_buf_fmt(session, - vb, "%.*s", (int)v.len, v.str)); - /* * !!! * WiredTiger names its internal checkpoints with a trailing @@ -160,10 +153,20 @@ __config_merge_scan(WT_SESSION_IMPL *session, * but since this isn't ever supposed to happen, I'm leaving * the test simple.) */ - if (strchr(kb->data, SEPC) != NULL) - WT_RET_MSG(session, EINVAL, - "key %s contains a separator character (%s)", - kb->data, SEP); + for (str = k.str, len = k.len; len > 0; ++str, --len) + if (*str == SEPC) + WT_RET_MSG(session, EINVAL, + "key %s contains a separator character " + "(%s)", kb->data, SEP); + + /* Build the key/value strings. */ + WT_ERR(__wt_buf_fmt(session, + kb, "%s%s%.*s", + key == NULL ? "" : key, + key == NULL ? "" : SEP, + (int)k.len, k.str)); + WT_ERR(__wt_buf_fmt(session, + vb, "%.*s", (int)v.len, v.str)); /* * If the value is a structure, recursively parse it. diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 955933f81cc..6ec87d2293a 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -641,8 +641,8 @@ __conn_reconfigure(WT_CONNECTION *wt_conn, const char *config) * The configuration argument has been checked for validity, replace the * previous connection configuration. * - * DO NOT collapse the configuration before the reconfigure calls. Some - * of the underlying reconfiguration functions do explicit checks for + * DO NOT merge the configuration before the reconfigure calls. Some + * of the underlying reconfiguration functions do explicit checks with * the second element of the configuration array, knowing the defaults * are in slot #1 and the application's modifications are in slot #2. */ -- cgit v1.2.1 From 929b579f4267c54587955457770b6b584c78a140 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 8 Sep 2014 15:45:41 -0400 Subject: Statistics logging requires a wait time, reference #1172. --- examples/c/ex_all.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/c/ex_all.c b/examples/c/ex_all.c index 8f9fba093de..8928736570e 100644 --- a/examples/c/ex_all.c +++ b/examples/c/ex_all.c @@ -1055,7 +1055,7 @@ main(void) /*! [Statistics logging with a table] */ ret = wiredtiger_open(home, NULL, - "create," + "create,wait=5" "statistics_log=(sources=(\"table:table1\",\"table:table2\"))", &conn); /*! [Statistics logging with a table] */ @@ -1064,7 +1064,7 @@ main(void) /*! [Statistics logging with all tables] */ ret = wiredtiger_open(home, NULL, - "create,statistics_log=(sources=(\"table:\"))", + "create,wait=5,statistics_log=(sources=(\"table:\"))", &conn); /*! [Statistics logging with all tables] */ if (ret == 0) -- cgit v1.2.1 From d08de00193efb8ba4a150a4c90bd91d3f61a43f4 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Mon, 8 Sep 2014 16:24:27 -0400 Subject: Add force flag to checkpoint last chunk. #1200 --- src/include/extern.h | 3 ++- src/include/lsm.h | 2 ++ src/lsm/lsm_manager.c | 2 +- src/lsm/lsm_tree.c | 2 +- src/lsm/lsm_work_unit.c | 9 +++++---- src/lsm/lsm_worker.c | 7 +++++-- 6 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/include/extern.h b/src/include/extern.h index a213a0e4bfa..76b9f350566 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -1010,8 +1010,9 @@ extern int __wt_lsm_tree_worker(WT_SESSION_IMPL *session, int *), const char *cfg[], uint32_t open_flags); -extern int __wt_lsm_get_chunk_to_flush( WT_SESSION_IMPL *session, +extern int __wt_lsm_get_chunk_to_flush(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, + int force, WT_LSM_CHUNK **chunkp); extern int __wt_lsm_work_switch( WT_SESSION_IMPL *session, WT_LSM_WORK_UNIT **entryp, diff --git a/src/include/lsm.h b/src/include/lsm.h index e0197e01af6..b4581b2b094 100644 --- a/src/include/lsm.h +++ b/src/include/lsm.h @@ -88,6 +88,8 @@ struct __wt_lsm_chunk { #define WT_LSM_WORK_FLUSH 0x04 /* Flush a chunk to disk */ #define WT_LSM_WORK_MERGE 0x08 /* Look for a tree merge */ #define WT_LSM_WORK_SWITCH 0x10 /* Switch to a new in memory chunk */ +#define WT_LSM_WORK_FORCE 0x10000 /* Force last chunk flush */ +#define WT_LSM_WORK_MASK 0xffff /* Mask for work types */ /* * WT_LSM_WORK_UNIT -- diff --git a/src/lsm/lsm_manager.c b/src/lsm/lsm_manager.c index 34cd13554d1..6f532111905 100644 --- a/src/lsm/lsm_manager.c +++ b/src/lsm/lsm_manager.c @@ -523,7 +523,7 @@ __wt_lsm_manager_push_entry( (void)WT_ATOMIC_ADD(lsm_tree->queue_ref, 1); WT_STAT_FAST_CONN_INCR(session, lsm_work_units_created); - switch (type) { + switch (type & WT_LSM_WORK_MASK) { case WT_LSM_WORK_SWITCH: __wt_spin_lock(session, &manager->switch_lock); TAILQ_INSERT_TAIL(&manager->switchqh, entry, q); diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index 30f97821a8c..f8a7083efac 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -1058,7 +1058,7 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) /* Make sure the in-memory chunk gets flushed but not switched. */ WT_ERR(__wt_lsm_manager_push_entry( - session, WT_LSM_WORK_FLUSH, lsm_tree)); + session, WT_LSM_WORK_FLUSH | WT_LSM_WORK_FORCE, lsm_tree)); /* Wait for the work unit queues to drain. */ while (F_ISSET(lsm_tree, WT_LSM_TREE_ACTIVE)) { diff --git a/src/lsm/lsm_work_unit.c b/src/lsm/lsm_work_unit.c index 5c96c82f84c..e0b4a6a808b 100644 --- a/src/lsm/lsm_work_unit.c +++ b/src/lsm/lsm_work_unit.c @@ -66,10 +66,10 @@ err: WT_TRET(__wt_lsm_tree_unlock(session, lsm_tree)); * Find and pin a chunk in the LSM tree that is likely to need flushing. */ int -__wt_lsm_get_chunk_to_flush( - WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, WT_LSM_CHUNK **chunkp) +__wt_lsm_get_chunk_to_flush(WT_SESSION_IMPL *session, + WT_LSM_TREE *lsm_tree, int force, WT_LSM_CHUNK **chunkp) { - u_int i; + u_int i, end; *chunkp = NULL; @@ -78,7 +78,8 @@ __wt_lsm_get_chunk_to_flush( if (!F_ISSET(lsm_tree, WT_LSM_TREE_ACTIVE)) return (__wt_lsm_tree_unlock(session, lsm_tree)); - for (i = 0; i < lsm_tree->nchunks - 1; i++) { + end = force ? lsm_tree->nchunks : lsm_tree->nchunks - 1; + for (i = 0; i < end; i++) { if (!F_ISSET(lsm_tree->chunk[i], WT_LSM_CHUNK_ONDISK)) { (void)WT_ATOMIC_ADD(lsm_tree->chunk[i]->refcnt, 1); *chunkp = lsm_tree->chunk[i]; diff --git a/src/lsm/lsm_worker.c b/src/lsm/lsm_worker.c index 257fda03f3b..4aab508896c 100644 --- a/src/lsm/lsm_worker.c +++ b/src/lsm/lsm_worker.c @@ -32,6 +32,7 @@ __lsm_worker_general_op( WT_DECL_RET; WT_LSM_CHUNK *chunk; WT_LSM_WORK_UNIT *entry; + int force; *completed = 0; if (!F_ISSET(cookie, WT_LSM_WORK_FLUSH) && @@ -43,9 +44,11 @@ __lsm_worker_general_op( cookie->flags, &entry)) != 0 || entry == NULL) return (ret); - if (entry->flags == WT_LSM_WORK_FLUSH) { + if ((entry->flags & WT_LSM_WORK_MASK) == WT_LSM_WORK_FLUSH) { + force = F_ISSET(entry, WT_LSM_WORK_FORCE); + F_CLR(entry, WT_LSM_WORK_FORCE); WT_ERR(__wt_lsm_get_chunk_to_flush( - session, entry->lsm_tree, &chunk)); + session, entry->lsm_tree, force, &chunk)); if (chunk != NULL) { ret = __wt_lsm_checkpoint_chunk( session, entry->lsm_tree, chunk); -- cgit v1.2.1 From bcd1fe29870f3d9200e9426320eb47d5b03bc682 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 8 Sep 2014 17:43:31 -0400 Subject: config_collapse.c:156:12: error: assignment discards 'const' qualifier from pointer target type [-Werror] config_collapse.c:158:5: error: format '%s' expects argument of type 'char *', but argument 4 has type 'const void *' [-Werror=format=] --- src/config/config_collapse.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config/config_collapse.c b/src/config/config_collapse.c index e56c6a62800..6773b06dd09 100644 --- a/src/config/config_collapse.c +++ b/src/config/config_collapse.c @@ -117,7 +117,7 @@ __config_merge_scan(WT_SESSION_IMPL *session, WT_DECL_ITEM(vb); WT_DECL_RET; size_t len; - char *str; + const char *str; WT_ERR(__wt_scr_alloc(session, 0, &kb)); WT_ERR(__wt_scr_alloc(session, 0, &vb)); @@ -157,7 +157,7 @@ __config_merge_scan(WT_SESSION_IMPL *session, if (*str == SEPC) WT_RET_MSG(session, EINVAL, "key %s contains a separator character " - "(%s)", kb->data, SEP); + "(%s)", (char *)kb->data, SEP); /* Build the key/value strings. */ WT_ERR(__wt_buf_fmt(session, -- cgit v1.2.1 From 04204363208e7ff13d0483e44d212990ca26148d Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 8 Sep 2014 18:27:04 -0400 Subject: typo --- examples/c/ex_all.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/c/ex_all.c b/examples/c/ex_all.c index 8928736570e..6ddd76bca46 100644 --- a/examples/c/ex_all.c +++ b/examples/c/ex_all.c @@ -1055,7 +1055,7 @@ main(void) /*! [Statistics logging with a table] */ ret = wiredtiger_open(home, NULL, - "create,wait=5" + "create,wait=5," "statistics_log=(sources=(\"table:table1\",\"table:table2\"))", &conn); /*! [Statistics logging with a table] */ -- cgit v1.2.1 From 19a4da2b049010deff2d6423f31809c2787a5bd8 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 8 Sep 2014 18:33:37 -0400 Subject: Statistics logging of sources only currently supports lsm: and file: URIs. --- examples/c/ex_all.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/c/ex_all.c b/examples/c/ex_all.c index 6ddd76bca46..ea5d26ce133 100644 --- a/examples/c/ex_all.c +++ b/examples/c/ex_all.c @@ -1055,8 +1055,8 @@ main(void) /*! [Statistics logging with a table] */ ret = wiredtiger_open(home, NULL, - "create,wait=5," - "statistics_log=(sources=(\"table:table1\",\"table:table2\"))", + "create, statistics_log=(" + "sources=(\"lsm:table1\",\"lsm:table2\"), wait=5)", &conn); /*! [Statistics logging with a table] */ if (ret == 0) @@ -1064,7 +1064,7 @@ main(void) /*! [Statistics logging with all tables] */ ret = wiredtiger_open(home, NULL, - "create,wait=5,statistics_log=(sources=(\"table:\"))", + "create, statistics_log=(sources=(\"lsm:\"), wait=5)", &conn); /*! [Statistics logging with all tables] */ if (ret == 0) -- cgit v1.2.1 From 852ed7b98520cf06bf4866863da4257124ed8046 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Mon, 8 Sep 2014 18:35:58 -0400 Subject: The wiredtiger_open final config array contains multiple entries (the defaults, the config file, the environment variable and the application-supplied config), merge them instead of collapse so we don't lose fields in nested structures. --- src/conn/conn_api.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 6ec87d2293a..0b210627df5 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -1409,8 +1409,8 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, */ WT_ERR(__wt_connection_workers(session, cfg)); - /* Take a copy of the final configuration for later reconfiguration. */ - WT_ERR(__wt_config_collapse(session, cfg, &conn->cfg)); + /* Merge the final configuration for later reconfiguration. */ + WT_ERR(__wt_config_merge(session, cfg, &conn->cfg)); STATIC_ASSERT(offsetof(WT_CONNECTION_IMPL, iface) == 0); *wt_connp = &conn->iface; -- cgit v1.2.1 From 9f396e9572ce0356505eee6755c11b62f8006a13 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Tue, 9 Sep 2014 17:24:20 +1000 Subject: Fix a bug in the shared cache implementation. We could decrement an unsigned number into negative. Also add a test case and some better documentation to the source code. --- src/conn/conn_cache_pool.c | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/src/conn/conn_cache_pool.c b/src/conn/conn_cache_pool.c index ab7e2cc48ee..9b540c147d4 100644 --- a/src/conn/conn_cache_pool.c +++ b/src/conn/conn_cache_pool.c @@ -488,26 +488,46 @@ __cache_pool_adjust(WT_SESSION_IMPL *session, if (cache->cp_skip_count > 0 && --cache->cp_skip_count > 0) continue; /* - * TODO: Use __wt_cache_bytes_inuse instead of eviction_target - * which doesn't do the right thing at the moment. + * If the entry is currently allocated less than the reserved + * size, increase it's allocation. This should only happen if: + * - It's the first time we've seen this member + * - The reserved size has been adjusted */ if (entry->cache_size < reserved) { grew = 1; adjusted = reserved - entry->cache_size; + /* + * Conditions for reducing the amount of resources for an + * entry: + * - If we are forcing and this entry has more than the + * minimum amount of space in use. + * - If the read pressure in this entry is below the + * threshold, other entries need more cache, the entry has + * more than the minimum space and there is no available + * space in the pool. + */ } else if ((force && entry->cache_size > reserved) || (read_pressure < WT_CACHE_POOL_REDUCE_THRESHOLD && highest > 1 && entry->cache_size > reserved && cp->currently_used >= cp->size)) { + grew = 0; /* - * If a connection isn't actively using it's assigned - * cache and is assigned a reasonable amount - reduce - * it. + * Shrink by a chunk size if that doesn't drop us + * below the reserved size. */ - grew = 0; - if (entry->cache_size - cp->chunk > reserved) + if (entry->cache_size > cp->chunk + reserved) adjusted = cp->chunk; else adjusted = entry->cache_size - reserved; + /* + * Conditions for increasing the amount of resources for an + * entry: + * - There was some activity across the pool + * - This entry is using less than the entire cache pool + * - The connection is using enough cache to require eviction + * - There is space available in the pool + * - Additional cache would benefit the connection + */ } else if (highest > 1 && entry->cache_size < cp->size && cache->bytes_inmem >= @@ -527,6 +547,9 @@ __cache_pool_adjust(WT_SESSION_IMPL *session, } else { cache->cp_skip_count = WT_CACHE_POOL_REDUCE_SKIPS; + WT_ASSERT(session, + entry->cache_size >= adjusted && + cp->currently_used >= adjusted); entry->cache_size -= adjusted; cp->currently_used -= adjusted; } -- cgit v1.2.1 From 127d0f0bb229516648b6ef60795765d2723e80d8 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Tue, 9 Sep 2014 17:49:21 +1000 Subject: The WT_NOTFOUND return should work the same as other non-zero returns and leave cursors pointing to the original key/value pair. refs #1209 --- src/cursor/cur_file.c | 21 +++++++-------------- src/include/misc.h | 6 ++++++ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/cursor/cur_file.c b/src/cursor/cur_file.c index e5a1d8a68b6..08129e668f5 100644 --- a/src/cursor/cur_file.c +++ b/src/cursor/cur_file.c @@ -10,29 +10,22 @@ /* * WT_BTREE_CURSOR_SAVE_AND_RESTORE * Save the cursor's key/value data/size fields, call an underlying btree - * function, and then consistently handle failure and success. + * function, and then consistently handle failure and success. */ #define WT_BTREE_CURSOR_SAVE_AND_RESTORE(cursor, f, ret) do { \ - const void *__key_data = (cursor)->key.data; \ - const void *__value_data = (cursor)->value.data; \ + WT_ITEM __key_copy = (cursor)->key; \ uint64_t __recno = (cursor)->recno; \ - size_t __key_size = (cursor)->key.size; \ - size_t __value_size = (cursor)->value.size; \ + WT_ITEM __value_copy = (cursor)->value; \ if (((ret) = (f)) == 0) { \ F_CLR(cursor, WT_CURSTD_KEY_EXT | WT_CURSTD_VALUE_EXT); \ F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); \ - } else if ((ret) == WT_NOTFOUND) \ - F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); \ - else { \ + } else { \ if (F_ISSET(cursor, WT_CURSTD_KEY_EXT)) { \ (cursor)->recno = __recno; \ - (cursor)->key.data = __key_data; \ - (cursor)->key.size = __key_size; \ - } \ - if (F_ISSET(cursor, WT_CURSTD_VALUE_EXT)) { \ - (cursor)->value.data = __value_data; \ - (cursor)->value.size = __value_size; \ + WT_ITEM_SET((cursor)->key, __key_copy); \ } \ + if (F_ISSET(cursor, WT_CURSTD_VALUE_EXT)) \ + WT_ITEM_SET((cursor)->value, __value_copy); \ F_CLR(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); \ } \ } while (0) diff --git a/src/include/misc.h b/src/include/misc.h index e50038b2c66..d28de81a6aa 100644 --- a/src/include/misc.h +++ b/src/include/misc.h @@ -186,6 +186,12 @@ ((i)->mem != NULL && (i)->data >= (i)->mem && \ WT_PTRDIFF((i)->data, (i)->mem) < (i)->memsize) +/* Copy the data and size fields of an item. */ +#define WT_ITEM_SET(dst, src) do { \ + (dst).data = (src).data; \ + (dst).size = (src).size; \ +} while (0) + /* * In diagnostic mode we track the locations from which hazard pointers and * scratch buffers were acquired. -- cgit v1.2.1 From c3d9cd2a77b886aeaad8bdaeb60599784cba6423 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Tue, 9 Sep 2014 17:49:21 +1000 Subject: api/leveldb/leveldb_wt_config.in: copyright information is incorrect --- dist/s_copyright.list | 1 + 1 file changed, 1 insertion(+) diff --git a/dist/s_copyright.list b/dist/s_copyright.list index ca2ba425ad5..d66be5a1ba7 100644 --- a/dist/s_copyright.list +++ b/dist/s_copyright.list @@ -1,3 +1,4 @@ +skip api/leveldb/leveldb_wt_config.in skip dist/api_config.py skip dist/api_data.py skip dist/api_err.py -- cgit v1.2.1 From 0a3061750aee368a5aa57e4bd6a367536c466122 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Tue, 9 Sep 2014 17:57:13 +1000 Subject: Have test/format always reset cursors at the end of each operation. --- test/format/ops.c | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/test/format/ops.c b/test/format/ops.c index 8c5a75e57a3..3ceaaf5e33f 100644 --- a/test/format/ops.c +++ b/test/format/ops.c @@ -220,21 +220,13 @@ ops(void *arg) /* * We can't checkpoint or swap sessions/cursors while in a * transaction, resolve any running transaction. - * - * Reset the cursor regardless: we may block waiting for a lock - * and there is no reason to keep pages pinned. */ - if (cnt == ckpt_op || cnt == session_op) { - if (intxn) { - if ((ret = session->commit_transaction( - session, NULL)) != 0) - die(ret, "session.commit_transaction"); - ++tinfo->commit; - intxn = 0; - } - if (cursor != NULL && - (ret = cursor->reset(cursor)) != 0) - die(ret, "cursor.reset"); + if (intxn && (cnt == ckpt_op || cnt == session_op)) { + if ((ret = session->commit_transaction( + session, NULL)) != 0) + die(ret, "session.commit_transaction"); + ++tinfo->commit; + intxn = 0; } /* Open up a new session and cursors. */ @@ -372,13 +364,6 @@ ops(void *arg) if (g.append_cnt >= g.append_max) goto skip_insert; - /* - * Reset the standard cursor so it doesn't keep - * pages pinned. - */ - if ((ret = cursor->reset(cursor)) != 0) - die(ret, "cursor.reset"); - /* Insert, then reset the insert cursor. */ if (col_insert( cursor_insert, &key, &value, &keyno)) @@ -430,6 +415,10 @@ skip_insert: if (col_update(cursor, &key, &value, keyno)) ++tinfo->search; if (read_row(cursor, &key, keyno)) goto deadlock; + + /* Reset the cursor: there is no reason to keep pages pinned. */ + if (cursor != NULL && (ret = cursor->reset(cursor)) != 0) + die(ret, "cursor.reset"); /* * If we're in the transaction, commit 40% of the time and -- cgit v1.2.1 From 5cd6a2685fcdce853ed29f8e7b75e2c9e83bbbbe Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Tue, 9 Sep 2014 18:09:34 +1000 Subject: It turns out that LSM depended on a `WT_NOTFOUND` return clearing `WT_CURSTD_KEY_SET`. Change LSM for now to keep test/format running, but we should review the way that LSM tracks whether chunk cursors are positioned. --- src/lsm/lsm_cursor.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lsm/lsm_cursor.c b/src/lsm/lsm_cursor.c index bbc5de7f13f..df2f7bba271 100644 --- a/src/lsm/lsm_cursor.c +++ b/src/lsm/lsm_cursor.c @@ -700,7 +700,7 @@ __clsm_get_current( multiple = 0; WT_FORALL_CURSORS(clsm, c, i) { - if (!F_ISSET(c, WT_CURSTD_KEY_SET)) + if (!F_ISSET(c, WT_CURSTD_KEY_INT)) continue; if (current == NULL) { current = c; @@ -823,7 +823,7 @@ retry: /* if (F_ISSET(clsm, WT_CLSM_MULTIPLE)) { check = 0; WT_FORALL_CURSORS(clsm, c, i) { - if (!F_ISSET(c, WT_CURSTD_KEY_SET)) + if (!F_ISSET(c, WT_CURSTD_KEY_INT)) continue; if (check) { WT_ERR(WT_LSM_CURCMP(session, @@ -906,7 +906,7 @@ retry: /* if (F_ISSET(clsm, WT_CLSM_MULTIPLE)) { check = 0; WT_FORALL_CURSORS(clsm, c, i) { - if (!F_ISSET(c, WT_CURSTD_KEY_SET)) + if (!F_ISSET(c, WT_CURSTD_KEY_INT)) continue; if (check) { WT_ERR(WT_LSM_CURCMP(session, @@ -958,7 +958,7 @@ __clsm_reset_cursors(WT_CURSOR_LSM *clsm, WT_CURSOR *skip) WT_FORALL_CURSORS(clsm, c, i) { if (c == skip) continue; - if (F_ISSET(c, WT_CURSTD_KEY_SET)) + if (F_ISSET(c, WT_CURSTD_KEY_INT)) WT_TRET(c->reset(c)); } -- cgit v1.2.1 From 10a92203f9708f35ebcfbed67ecfc34544c6c75b Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Tue, 9 Sep 2014 18:14:06 +1000 Subject: whitespace --- test/format/ops.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/format/ops.c b/test/format/ops.c index 3ceaaf5e33f..e38b75f0deb 100644 --- a/test/format/ops.c +++ b/test/format/ops.c @@ -415,7 +415,7 @@ skip_insert: if (col_update(cursor, &key, &value, keyno)) ++tinfo->search; if (read_row(cursor, &key, keyno)) goto deadlock; - + /* Reset the cursor: there is no reason to keep pages pinned. */ if (cursor != NULL && (ret = cursor->reset(cursor)) != 0) die(ret, "cursor.reset"); -- cgit v1.2.1 From 08c75ef4d13f3a6f5c6d3f3c4dd345d301af4f34 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 9 Sep 2014 08:44:10 -0400 Subject: Fix return after branch to error. --- src/config/config_collapse.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/config_collapse.c b/src/config/config_collapse.c index 6773b06dd09..98b32a109bc 100644 --- a/src/config/config_collapse.c +++ b/src/config/config_collapse.c @@ -155,7 +155,7 @@ __config_merge_scan(WT_SESSION_IMPL *session, */ for (str = k.str, len = k.len; len > 0; ++str, --len) if (*str == SEPC) - WT_RET_MSG(session, EINVAL, + WT_ERR_MSG(session, EINVAL, "key %s contains a separator character " "(%s)", (char *)kb->data, SEP); -- cgit v1.2.1 From 7ef2e3adfb6da3c7da6c51e36b4775fad72443d5 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 9 Sep 2014 09:06:39 -0400 Subject: Disallow JSON quoting characters in checkpoint names, it's not a good idea. --- src/include/extern.h | 5 +++- src/schema/schema_util.c | 26 +++++++++++++++--- src/session/session_api.c | 12 ++++----- src/txn/txn_ckpt.c | 59 ++++++++++++++++++++++------------------- test/suite/test_checkpoint01.py | 10 +++++-- 5 files changed, 73 insertions(+), 39 deletions(-) diff --git a/src/include/extern.h b/src/include/extern.h index 3528ef38283..6be8d20bf2b 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -1379,7 +1379,10 @@ extern int __wt_schema_range_truncate( WT_SESSION_IMPL *session, WT_CURSOR *stop); extern WT_DATA_SOURCE *__wt_schema_get_source(WT_SESSION_IMPL *session, const char *name); -extern int __wt_schema_name_check(WT_SESSION_IMPL *session, const char *uri); +extern int __wt_str_name_check(WT_SESSION_IMPL *session, const char *str); +extern int __wt_name_check(WT_SESSION_IMPL *session, + const char *str, + size_t len); extern int __wt_schema_worker(WT_SESSION_IMPL *session, const char *uri, int (*file_func)(WT_SESSION_IMPL *, diff --git a/src/schema/schema_util.c b/src/schema/schema_util.c index 90e5fb42dc1..263f56f1c41 100644 --- a/src/schema/schema_util.c +++ b/src/schema/schema_util.c @@ -23,11 +23,11 @@ __wt_schema_get_source(WT_SESSION_IMPL *session, const char *name) } /* - * __wt_schema_name_check -- + * __wt_str_name_check -- * Disallow any use of the WiredTiger name space. */ int -__wt_schema_name_check(WT_SESSION_IMPL *session, const char *uri) +__wt_str_name_check(WT_SESSION_IMPL *session, const char *str) { const char *name, *sep; int skipped; @@ -37,7 +37,7 @@ __wt_schema_name_check(WT_SESSION_IMPL *session, const char *uri) * "bad" if the application truncated the metadata file. Skip any * leading URI prefix, check and then skip over a table name. */ - name = uri; + name = str; for (skipped = 0; skipped < 2; skipped++) { if ((sep = strchr(name, ':')) == NULL) break; @@ -62,3 +62,23 @@ __wt_schema_name_check(WT_SESSION_IMPL *session, const char *uri) return (0); } + +/* + * __wt_name_check -- + * Disallow any use of the WiredTiger name space. + */ +int +__wt_name_check(WT_SESSION_IMPL *session, const char *str, size_t len) +{ + WT_DECL_RET; + WT_DECL_ITEM(tmp); + + WT_RET(__wt_scr_alloc(session, len, &tmp)); + + WT_ERR(__wt_buf_fmt(session, tmp, "%.*s", (int)len, str)); + + ret = __wt_str_name_check(session, tmp->data); + +err: __wt_scr_free(&tmp); + return (ret); +} diff --git a/src/session/session_api.c b/src/session/session_api.c index e63e2c0284a..922aa3e9c25 100644 --- a/src/session/session_api.c +++ b/src/session/session_api.c @@ -319,7 +319,7 @@ __session_create(WT_SESSION *wt_session, const char *uri, const char *config) WT_UNUSED(cfg); /* Disallow objects in the WiredTiger name space. */ - WT_ERR(__wt_schema_name_check(session, uri)); + WT_ERR(__wt_str_name_check(session, uri)); /* * Type configuration only applies to tables, column groups and indexes. @@ -387,8 +387,8 @@ __session_rename(WT_SESSION *wt_session, SESSION_API_CALL(session, rename, config, cfg); /* Disallow objects in the WiredTiger name space. */ - WT_ERR(__wt_schema_name_check(session, uri)); - WT_ERR(__wt_schema_name_check(session, newuri)); + WT_ERR(__wt_str_name_check(session, uri)); + WT_ERR(__wt_str_name_check(session, newuri)); WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_rename(session, uri, newuri, cfg)); @@ -408,7 +408,7 @@ __session_compact(WT_SESSION *wt_session, const char *uri, const char *config) session = (WT_SESSION_IMPL *)wt_session; /* Disallow objects in the WiredTiger name space. */ - WT_RET(__wt_schema_name_check(session, uri)); + WT_RET(__wt_str_name_check(session, uri)); if (!WT_PREFIX_MATCH(uri, "colgroup:") && !WT_PREFIX_MATCH(uri, "file:") && @@ -434,7 +434,7 @@ __session_drop(WT_SESSION *wt_session, const char *uri, const char *config) SESSION_API_CALL(session, drop, config, cfg); /* Disallow objects in the WiredTiger name space. */ - WT_ERR(__wt_schema_name_check(session, uri)); + WT_ERR(__wt_str_name_check(session, uri)); WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_drop(session, uri, cfg)); @@ -496,7 +496,7 @@ __session_truncate(WT_SESSION *wt_session, if (uri != NULL) { /* Disallow objects in the WiredTiger name space. */ - WT_ERR(__wt_schema_name_check(session, uri)); + WT_ERR(__wt_str_name_check(session, uri)); WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_truncate(session, uri, cfg)); diff --git a/src/txn/txn_ckpt.c b/src/txn/txn_ckpt.c index 71f1c8bb2ae..c7ddbcfd604 100644 --- a/src/txn/txn_ckpt.c +++ b/src/txn/txn_ckpt.c @@ -7,6 +7,30 @@ #include "wt_internal.h" +/* + * __checkpoint_name_ok -- + * Complain if the checkpoint name isn't acceptable. + */ +static int +__checkpoint_name_ok(WT_SESSION_IMPL *session, const char *name, size_t len) +{ + /* Check for characters we don't want to see in a metadata file. */ + WT_RET(__wt_name_check(session, name, len)); + + /* + * The internal checkpoint name is special, applications aren't allowed + * to use it. Be aggressive and disallow any matching prefix, it makes + * things easier when checking in other places. + */ + if (len < strlen(WT_CHECKPOINT)) + return (0); + if (!WT_PREFIX_MATCH(name, WT_CHECKPOINT)) + return (0); + + WT_RET_MSG(session, EINVAL, + "the checkpoint name \"%s\" is reserved", WT_CHECKPOINT); +} + /* * __checkpoint_name_check -- * Check for an attempt to name a checkpoint that includes anything @@ -75,9 +99,11 @@ __checkpoint_apply(WT_SESSION_IMPL *session, const char *cfg[], target_list = 0; - /* Flag if this is a named checkpoint. */ - WT_ERR(__wt_config_gets(session, cfg, "name", &cval)); + /* Flag if this is a named checkpoint, and check if the name is OK. */ + WT_RET(__wt_config_gets(session, cfg, "name", &cval)); named = cval.len != 0; + if (named) + WT_RET(__checkpoint_name_ok(session, cval.str, cval.len)); /* Step through the targets and optionally operate on each one. */ WT_ERR(__wt_config_gets(session, cfg, "target", &cval)); @@ -412,27 +438,6 @@ err: /* return (ret); } -/* - * __ckpt_name_ok -- - * Complain if our reserved checkpoint name is used. - */ -static int -__ckpt_name_ok(WT_SESSION_IMPL *session, const char *name, size_t len) -{ - /* - * The internal checkpoint name is special, applications aren't allowed - * to use it. Be aggressive and disallow any matching prefix, it makes - * things easier when checking in other places. - */ - if (len < strlen(WT_CHECKPOINT)) - return (0); - if (!WT_PREFIX_MATCH(name, WT_CHECKPOINT)) - return (0); - - WT_RET_MSG(session, EINVAL, - "the checkpoint name \"%s\" is reserved", WT_CHECKPOINT); -} - /* * __drop -- * Drop all checkpoints with a specific name. @@ -575,7 +580,7 @@ __checkpoint_worker( if (cval.len == 0) name = WT_CHECKPOINT; else { - WT_ERR(__ckpt_name_ok(session, cval.str, cval.len)); + WT_ERR(__checkpoint_name_ok(session, cval.str, cval.len)); WT_ERR(__wt_strndup(session, cval.str, cval.len, &name_alloc)); name = name_alloc; } @@ -588,12 +593,12 @@ __checkpoint_worker( WT_ERR(__wt_config_subinit(session, &dropconf, &cval)); while ((ret = __wt_config_next(&dropconf, &k, &v)) == 0) { - /* Disallow the reserved checkpoint name. */ + /* Disallow unsafe checkpoint names. */ if (v.len == 0) - WT_ERR(__ckpt_name_ok( + WT_ERR(__checkpoint_name_ok( session, k.str, k.len)); else - WT_ERR(__ckpt_name_ok( + WT_ERR(__checkpoint_name_ok( session, v.str, v.len)); if (v.len == 0) diff --git a/test/suite/test_checkpoint01.py b/test/suite/test_checkpoint01.py index 153ea015cf5..ab4dbe18bd6 100644 --- a/test/suite/test_checkpoint01.py +++ b/test/suite/test_checkpoint01.py @@ -308,8 +308,8 @@ class test_checkpoint_last(wttest.WiredTigerTestCase): # Check we can't use the reserved name as an application checkpoint name. -class test_checkpoint_last_name(wttest.WiredTigerTestCase): - def test_checkpoint_last_name(self): +class test_checkpoint_illegal_name(wttest.WiredTigerTestCase): + def test_checkpoint_illegal_name(self): simple_populate(self, "file:checkpoint", 'key_format=S', 100) msg = '/the checkpoint name.*is reserved/' for conf in ( @@ -324,6 +324,12 @@ class test_checkpoint_last_name(wttest.WiredTigerTestCase): 'drop=(to=WiredTigerCheckpointX)'): self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.checkpoint(conf), msg) + msg = '/WiredTiger objects should not include grouping/' + for conf in ( + 'name=check{point', + 'name=check\\point'): + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.checkpoint(conf), msg) # Check we can't name checkpoints that include LSM tables. -- cgit v1.2.1 From 95e84d49bbf4a571e2790836b656ba2a14e995cf Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 9 Sep 2014 09:33:35 -0400 Subject: Don't check the key for a prefix match if the prefix is longer than the key. --- src/config/config_collapse.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/config/config_collapse.c b/src/config/config_collapse.c index 98b32a109bc..425ab963e5b 100644 --- a/src/config/config_collapse.c +++ b/src/config/config_collapse.c @@ -228,13 +228,13 @@ __config_merge_format_next(WT_SESSION_IMPL *session, const char *prefix, for (; *enp < cp->entries_next; ++*enp) { ep = &cp->entries[*enp]; + len1 = strlen(ep->k); /* * The entries are in sorted order, take the last entry for any * key. */ if (*enp < (cp->entries_next - 1)) { - len1 = strlen(ep->k); len2 = strlen((ep + 1)->k); /* Choose the last of identical keys. */ @@ -257,7 +257,8 @@ __config_merge_format_next(WT_SESSION_IMPL *session, const char *prefix, * If we're skipping a prefix and this entry doesn't match it, * back off one entry and pop up a level. */ - if (plen != 0 && memcmp(ep->k, prefix, plen) != 0) { + if (plen != 0 && + (plen < len1 || memcmp(ep->k, prefix, plen) != 0)) { --*enp; break; } -- cgit v1.2.1 From bdebc7c4c054b7aef2e09a883b14bcfd3b773066 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 9 Sep 2014 09:50:40 -0400 Subject: I flipped the comparison, fix it. --- src/config/config_collapse.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/config_collapse.c b/src/config/config_collapse.c index 425ab963e5b..5621ad492f6 100644 --- a/src/config/config_collapse.c +++ b/src/config/config_collapse.c @@ -258,7 +258,7 @@ __config_merge_format_next(WT_SESSION_IMPL *session, const char *prefix, * back off one entry and pop up a level. */ if (plen != 0 && - (plen < len1 || memcmp(ep->k, prefix, plen) != 0)) { + (plen > len1 || memcmp(ep->k, prefix, plen) != 0)) { --*enp; break; } -- cgit v1.2.1 From 0841c3b7cebc35de0977a87969ebacc742bbfc77 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Tue, 9 Sep 2014 12:26:26 -0400 Subject: Don't check for a checkpoint name match if none was specified (yeah, it works, but it's not a reasonable check). Don't let the application specify an illegal checkpoint name, it's going to fail eventually. Don't roll our own __wt_buf_fmt() call, it's just wasted effort. Don't leak memory if the checkpoint server's name is reconfigured. --- src/conn/conn_ckpt.c | 24 ++++++++++++++++-------- src/include/extern.h | 3 +++ src/txn/txn_ckpt.c | 14 +++++++------- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/src/conn/conn_ckpt.c b/src/conn/conn_ckpt.c index d1ee647d08a..f948cd64d2e 100644 --- a/src/conn/conn_ckpt.c +++ b/src/conn/conn_ckpt.c @@ -20,6 +20,7 @@ __ckpt_server_config(WT_SESSION_IMPL *session, const char **cfg, int *startp) WT_CONNECTION_IMPL *conn; WT_DECL_ITEM(tmp); WT_DECL_RET; + char *p; conn = S2C(session); @@ -40,19 +41,26 @@ __ckpt_server_config(WT_SESSION_IMPL *session, const char **cfg, int *startp) } *startp = 1; + /* + * The application can specify a checkpoint name, which we ignore if + * it's our default. + */ WT_RET(__wt_config_gets(session, cfg, "checkpoint.name", &cval)); + if (cval.len != 0 && + !WT_STRING_MATCH(WT_CHECKPOINT, cval.str, cval.len)) { + WT_RET(__wt_checkpoint_name_ok(session, cval.str, cval.len)); - if (!WT_STRING_MATCH(WT_CHECKPOINT, cval.str, cval.len)) { WT_RET(__wt_scr_alloc(session, cval.len + 20, &tmp)); - strcpy((char *)tmp->data, "name="); - strncat((char *)tmp->data, cval.str, cval.len); - ret = __wt_strndup(session, - tmp->data, strlen("name=") + cval.len, &conn->ckpt_config); - __wt_scr_free(&tmp); - WT_RET(ret); + WT_ERR(__wt_buf_fmt( + session, tmp, "name=%.*s", (int)cval.len, cval.str)); + WT_ERR(__wt_strdup(session, tmp->data, &p)); + + __wt_free(session, conn->ckpt_config); + conn->ckpt_config = p; } - return (0); +err: __wt_scr_free(&tmp); + return (ret); } /* diff --git a/src/include/extern.h b/src/include/extern.h index 6be8d20bf2b..ed285db6395 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -1611,6 +1611,9 @@ extern int __wt_txn_init(WT_SESSION_IMPL *session); extern void __wt_txn_destroy(WT_SESSION_IMPL *session); extern int __wt_txn_global_init(WT_CONNECTION_IMPL *conn, const char *cfg[]); extern void __wt_txn_global_destroy(WT_CONNECTION_IMPL *conn); +extern int __wt_checkpoint_name_ok(WT_SESSION_IMPL *session, + const char *name, + size_t len); extern int __wt_checkpoint_list(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]); diff --git a/src/txn/txn_ckpt.c b/src/txn/txn_ckpt.c index c7ddbcfd604..0bebce927fe 100644 --- a/src/txn/txn_ckpt.c +++ b/src/txn/txn_ckpt.c @@ -8,11 +8,11 @@ #include "wt_internal.h" /* - * __checkpoint_name_ok -- + * __wt_checkpoint_name_ok -- * Complain if the checkpoint name isn't acceptable. */ -static int -__checkpoint_name_ok(WT_SESSION_IMPL *session, const char *name, size_t len) +int +__wt_checkpoint_name_ok(WT_SESSION_IMPL *session, const char *name, size_t len) { /* Check for characters we don't want to see in a metadata file. */ WT_RET(__wt_name_check(session, name, len)); @@ -103,7 +103,7 @@ __checkpoint_apply(WT_SESSION_IMPL *session, const char *cfg[], WT_RET(__wt_config_gets(session, cfg, "name", &cval)); named = cval.len != 0; if (named) - WT_RET(__checkpoint_name_ok(session, cval.str, cval.len)); + WT_RET(__wt_checkpoint_name_ok(session, cval.str, cval.len)); /* Step through the targets and optionally operate on each one. */ WT_ERR(__wt_config_gets(session, cfg, "target", &cval)); @@ -580,7 +580,7 @@ __checkpoint_worker( if (cval.len == 0) name = WT_CHECKPOINT; else { - WT_ERR(__checkpoint_name_ok(session, cval.str, cval.len)); + WT_ERR(__wt_checkpoint_name_ok(session, cval.str, cval.len)); WT_ERR(__wt_strndup(session, cval.str, cval.len, &name_alloc)); name = name_alloc; } @@ -595,10 +595,10 @@ __checkpoint_worker( __wt_config_next(&dropconf, &k, &v)) == 0) { /* Disallow unsafe checkpoint names. */ if (v.len == 0) - WT_ERR(__checkpoint_name_ok( + WT_ERR(__wt_checkpoint_name_ok( session, k.str, k.len)); else - WT_ERR(__checkpoint_name_ok( + WT_ERR(__wt_checkpoint_name_ok( session, v.str, v.len)); if (v.len == 0) -- cgit v1.2.1 From 4d361acba912a98ea7dcfbad4ef1ea5778f3e0c7 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 9 Sep 2014 13:28:00 -0400 Subject: Add and enhance LSM verbose messages. #1200 --- src/lsm/lsm_merge.c | 6 ++++++ src/lsm/lsm_tree.c | 1 + src/lsm/lsm_work_unit.c | 23 +++++++++++++++++++---- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/lsm/lsm_merge.c b/src/lsm/lsm_merge.c index bf758abd6b1..6b8c4b65dea 100644 --- a/src/lsm/lsm_merge.c +++ b/src/lsm/lsm_merge.c @@ -61,6 +61,7 @@ __wt_lsm_merge( uint32_t aggressive, generation, max_gap, max_gen, max_level, start_id; uint64_t insert_count, record_count, chunk_size; u_int dest_id, end_chunk, i, merge_max, merge_min, nchunks, start_chunk; + u_int verb; int create_bloom, locked, tret; const char *cfg[3]; const char *drop_cfg[] = @@ -253,6 +254,11 @@ __wt_lsm_merge( "Merging chunks %u-%u into %u (%" PRIu64 " records)" ", generation %" PRIu32, start_chunk, end_chunk, dest_id, record_count, generation)); + if (WT_VERBOSE_ISSET(session, WT_VERB_LSM)) + for (verb = start_chunk; verb <= end_chunk; verb++) + WT_RET(__wt_verbose(session, WT_VERB_LSM, + "Chunk[%u] id %u", + verb, lsm_tree->chunk[verb]->id)); WT_RET(__wt_calloc_def(session, 1, &chunk)); chunk->id = dest_id; diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index f8a7083efac..51e357fd33d 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -1057,6 +1057,7 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) WT_ERR(__wt_lsm_tree_unlock(session, lsm_tree)); /* Make sure the in-memory chunk gets flushed but not switched. */ + WT_ERR(__wt_verbose(session, WT_VERB_LSM, "Compact force flush")); WT_ERR(__wt_lsm_manager_push_entry( session, WT_LSM_WORK_FLUSH | WT_LSM_WORK_FORCE, lsm_tree)); diff --git a/src/lsm/lsm_work_unit.c b/src/lsm/lsm_work_unit.c index e0b4a6a808b..ac18e133846 100644 --- a/src/lsm/lsm_work_unit.c +++ b/src/lsm/lsm_work_unit.c @@ -78,10 +78,22 @@ __wt_lsm_get_chunk_to_flush(WT_SESSION_IMPL *session, if (!F_ISSET(lsm_tree, WT_LSM_TREE_ACTIVE)) return (__wt_lsm_tree_unlock(session, lsm_tree)); - end = force ? lsm_tree->nchunks : lsm_tree->nchunks - 1; + /* + * Normally we don't want to force out the last chunk. But if we're + * doing a forced flush, likely from a compact call, then we do want + * to include the final chunk. + */ + if (force) { + end = lsm_tree->nchunks; + } else + end = lsm_tree->nchunks - 1; for (i = 0; i < end; i++) { if (!F_ISSET(lsm_tree->chunk[i], WT_LSM_CHUNK_ONDISK)) { (void)WT_ATOMIC_ADD(lsm_tree->chunk[i]->refcnt, 1); + WT_RET(__wt_verbose(session, WT_VERB_LSM, + "Flush%s: return chunk %u of %u: %s", + force ? " w/ force" : "", i, end - 1, + lsm_tree->chunk[i]->uri)); *chunkp = lsm_tree->chunk[i]; break; } @@ -226,7 +238,8 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, !__wt_txn_visible_all(session, chunk->switch_txn)) return (0); - WT_RET(__wt_verbose(session, WT_VERB_LSM, "LSM worker flushing")); + WT_RET(__wt_verbose(session, WT_VERB_LSM, "LSM worker flushing %s", + chunk->uri)); /* * Flush the file before checkpointing: this is the expensive part in @@ -249,7 +262,8 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, } WT_RET(ret); - WT_RET(__wt_verbose(session, WT_VERB_LSM, "LSM worker checkpointing")); + WT_RET(__wt_verbose(session, WT_VERB_LSM, "LSM worker checkpointing %s", + chunk->uri)); WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, chunk->uri, @@ -290,7 +304,8 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, /* Make sure we aren't pinning a transaction ID. */ __wt_txn_release_snapshot(session); - WT_RET(__wt_verbose(session, WT_VERB_LSM, "LSM worker checkpointed")); + WT_RET(__wt_verbose(session, WT_VERB_LSM, "LSM worker checkpointed %s", + chunk->uri)); /* * Schedule a bloom filter create for our newly flushed chunk */ if (!FLD_ISSET(lsm_tree->bloom, WT_LSM_BLOOM_OFF)) -- cgit v1.2.1 From e9df44477ce16f80a19a96b30b6ba34bafdbc798 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 9 Sep 2014 13:51:41 -0400 Subject: If forcing a flush, look for all chunks to process. #1200 --- src/lsm/lsm_worker.c | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/lsm/lsm_worker.c b/src/lsm/lsm_worker.c index 4aab508896c..70ec016db01 100644 --- a/src/lsm/lsm_worker.c +++ b/src/lsm/lsm_worker.c @@ -32,7 +32,7 @@ __lsm_worker_general_op( WT_DECL_RET; WT_LSM_CHUNK *chunk; WT_LSM_WORK_UNIT *entry; - int force; + int force, count; *completed = 0; if (!F_ISSET(cookie, WT_LSM_WORK_FLUSH) && @@ -47,15 +47,26 @@ __lsm_worker_general_op( if ((entry->flags & WT_LSM_WORK_MASK) == WT_LSM_WORK_FLUSH) { force = F_ISSET(entry, WT_LSM_WORK_FORCE); F_CLR(entry, WT_LSM_WORK_FORCE); - WT_ERR(__wt_lsm_get_chunk_to_flush( - session, entry->lsm_tree, force, &chunk)); - if (chunk != NULL) { - ret = __wt_lsm_checkpoint_chunk( - session, entry->lsm_tree, chunk); - WT_ASSERT(session, chunk->refcnt > 0); - (void)WT_ATOMIC_SUB(chunk->refcnt, 1); - WT_ERR(ret); - } + /* + * If this is a force flush, we want to force out all + * possible chunks, not just the first one we find. + */ + count = 0; + do { + WT_ERR(__wt_lsm_get_chunk_to_flush( + session, entry->lsm_tree, force, &chunk)); + if (chunk != NULL) { + count++; + __wt_errx(session, "Got chunk %d", chunk->id); + ret = __wt_lsm_checkpoint_chunk( + session, entry->lsm_tree, chunk); + WT_ASSERT(session, chunk->refcnt > 0); + (void)WT_ATOMIC_SUB(chunk->refcnt, 1); + WT_ERR(ret); + } + while (force &&chunk != NULL); + if (count > 1) + __wt_errx(session, "Processed %d chunks", count); } else if (entry->flags == WT_LSM_WORK_DROP) WT_ERR(__wt_lsm_free_chunks(session, entry->lsm_tree)); else if (entry->flags == WT_LSM_WORK_BLOOM) { -- cgit v1.2.1 From 8c7ed677e61e18572440aaebe9b4151f8e4d738d Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 9 Sep 2014 13:54:09 -0400 Subject: Fix typo --- src/lsm/lsm_worker.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lsm/lsm_worker.c b/src/lsm/lsm_worker.c index 70ec016db01..3b13cb639e8 100644 --- a/src/lsm/lsm_worker.c +++ b/src/lsm/lsm_worker.c @@ -64,7 +64,7 @@ __lsm_worker_general_op( (void)WT_ATOMIC_SUB(chunk->refcnt, 1); WT_ERR(ret); } - while (force &&chunk != NULL); + } while (force &&chunk != NULL); if (count > 1) __wt_errx(session, "Processed %d chunks", count); } else if (entry->flags == WT_LSM_WORK_DROP) -- cgit v1.2.1 From c4fb36e6262f818b714865508c1050c69de6c342 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 9 Sep 2014 16:27:46 -0400 Subject: Add table name to many LSM verbose statements. Add arg to indicate if this is the last chunk. #1200 --- src/include/extern.h | 1 + src/lsm/lsm_manager.c | 8 ++++---- src/lsm/lsm_merge.c | 5 +++-- src/lsm/lsm_tree.c | 7 ++++--- src/lsm/lsm_work_unit.c | 5 ++++- src/lsm/lsm_worker.c | 19 ++++++++++--------- 6 files changed, 26 insertions(+), 19 deletions(-) diff --git a/src/include/extern.h b/src/include/extern.h index ed285db6395..a0cb087e5e2 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -1016,6 +1016,7 @@ extern int __wt_lsm_tree_worker(WT_SESSION_IMPL *session, extern int __wt_lsm_get_chunk_to_flush(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, int force, + int *last, WT_LSM_CHUNK **chunkp); extern int __wt_lsm_work_switch( WT_SESSION_IMPL *session, WT_LSM_WORK_UNIT **entryp, diff --git a/src/lsm/lsm_manager.c b/src/lsm/lsm_manager.c index 6f532111905..10e61176c82 100644 --- a/src/lsm/lsm_manager.c +++ b/src/lsm/lsm_manager.c @@ -185,10 +185,10 @@ __lsm_manager_aggressive_update(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) if (lsm_tree->merge_aggressiveness > old_aggressive) WT_RET(__wt_verbose(session, WT_VERB_LSM, - "LSM merge got aggressive (%u), " - "%u / %" PRIu64, - lsm_tree->merge_aggressiveness, stallms, - lsm_tree->chunk_fill_ms)); + "LSM merge %s got aggressive (%u), " + "%u / %" PRIu64, + lsm_tree->name, lsm_tree->merge_aggressiveness, stallms, + lsm_tree->chunk_fill_ms)); return (0); } diff --git a/src/lsm/lsm_merge.c b/src/lsm/lsm_merge.c index 6b8c4b65dea..2da69b3601c 100644 --- a/src/lsm/lsm_merge.c +++ b/src/lsm/lsm_merge.c @@ -73,7 +73,6 @@ __wt_lsm_merge( dest = src = NULL; locked = 0; start_id = 0; - aggressive = lsm_tree->merge_aggressiveness; /* * If the tree is open read-only be very aggressive. Otherwise, we can @@ -83,6 +82,7 @@ __wt_lsm_merge( if (!lsm_tree->modified) lsm_tree->merge_aggressiveness = 10; + aggressive = lsm_tree->merge_aggressiveness; merge_max = (aggressive > 5) ? 100 : lsm_tree->merge_min; merge_min = (aggressive > 5) ? 2 : lsm_tree->merge_min; max_gap = (aggressive + 4) / 5; @@ -251,8 +251,9 @@ __wt_lsm_merge( dest_id = WT_ATOMIC_ADD(lsm_tree->last, 1); WT_RET(__wt_verbose(session, WT_VERB_LSM, - "Merging chunks %u-%u into %u (%" PRIu64 " records)" + "Merging %s chunks %u-%u into %u (%" PRIu64 " records)" ", generation %" PRIu32, + lsm_tree->name, start_chunk, end_chunk, dest_id, record_count, generation)); if (WT_VERBOSE_ISSET(session, WT_VERB_LSM)) for (verb = start_chunk; verb <= end_chunk; verb++) diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index 51e357fd33d..ef9f859b59a 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -744,8 +744,8 @@ __wt_lsm_tree_switch(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) nchunks + 1, &lsm_tree->chunk)); WT_ERR(__wt_verbose(session, WT_VERB_LSM, - "Tree switch to: %" PRIu32 ", checkpoint throttle %ld, " - "merge throttle %ld", + "Tree %s switch to: %" PRIu32 ", checkpoint throttle %ld, " + "merge throttle %ld", lsm_tree->name, new_id, lsm_tree->ckpt_throttle, lsm_tree->merge_throttle)); WT_ERR(__wt_calloc_def(session, 1, &chunk)); @@ -1057,7 +1057,8 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) WT_ERR(__wt_lsm_tree_unlock(session, lsm_tree)); /* Make sure the in-memory chunk gets flushed but not switched. */ - WT_ERR(__wt_verbose(session, WT_VERB_LSM, "Compact force flush")); + WT_ERR(__wt_verbose(session, WT_VERB_LSM, "Compact %s force flush", + name)); WT_ERR(__wt_lsm_manager_push_entry( session, WT_LSM_WORK_FLUSH | WT_LSM_WORK_FORCE, lsm_tree)); diff --git a/src/lsm/lsm_work_unit.c b/src/lsm/lsm_work_unit.c index ac18e133846..c7c78a70f44 100644 --- a/src/lsm/lsm_work_unit.c +++ b/src/lsm/lsm_work_unit.c @@ -67,11 +67,12 @@ err: WT_TRET(__wt_lsm_tree_unlock(session, lsm_tree)); */ int __wt_lsm_get_chunk_to_flush(WT_SESSION_IMPL *session, - WT_LSM_TREE *lsm_tree, int force, WT_LSM_CHUNK **chunkp) + WT_LSM_TREE *lsm_tree, int force, int *last, WT_LSM_CHUNK **chunkp) { u_int i, end; *chunkp = NULL; + *last = 0; WT_ASSERT(session, lsm_tree->queue_ref > 0); WT_RET(__wt_lsm_tree_lock(session, lsm_tree, 0)); @@ -95,6 +96,8 @@ __wt_lsm_get_chunk_to_flush(WT_SESSION_IMPL *session, force ? " w/ force" : "", i, end - 1, lsm_tree->chunk[i]->uri)); *chunkp = lsm_tree->chunk[i]; + if (i == end - 1) + *last = 1; break; } } diff --git a/src/lsm/lsm_worker.c b/src/lsm/lsm_worker.c index 3b13cb639e8..d1eb9ca9ee6 100644 --- a/src/lsm/lsm_worker.c +++ b/src/lsm/lsm_worker.c @@ -32,7 +32,7 @@ __lsm_worker_general_op( WT_DECL_RET; WT_LSM_CHUNK *chunk; WT_LSM_WORK_UNIT *entry; - int force, count; + int force, last; *completed = 0; if (!F_ISSET(cookie, WT_LSM_WORK_FLUSH) && @@ -51,22 +51,23 @@ __lsm_worker_general_op( * If this is a force flush, we want to force out all * possible chunks, not just the first one we find. */ - count = 0; + last = 0; do { - WT_ERR(__wt_lsm_get_chunk_to_flush( - session, entry->lsm_tree, force, &chunk)); + WT_ERR(__wt_lsm_get_chunk_to_flush(session, + entry->lsm_tree, force, &last, &chunk)); if (chunk != NULL) { - count++; - __wt_errx(session, "Got chunk %d", chunk->id); + WT_ERR(__wt_verbose(session, WT_VERB_LSM, + "Flush%s%s chunk %d %s", + force ? " w/ force" : "", + last ? " last" : "", + chunk->id, chunk->uri)); ret = __wt_lsm_checkpoint_chunk( session, entry->lsm_tree, chunk); WT_ASSERT(session, chunk->refcnt > 0); (void)WT_ATOMIC_SUB(chunk->refcnt, 1); WT_ERR(ret); } - } while (force &&chunk != NULL); - if (count > 1) - __wt_errx(session, "Processed %d chunks", count); + } while (force && chunk != NULL && !last); } else if (entry->flags == WT_LSM_WORK_DROP) WT_ERR(__wt_lsm_free_chunks(session, entry->lsm_tree)); else if (entry->flags == WT_LSM_WORK_BLOOM) { -- cgit v1.2.1 From c4893022e076d47db4d552b3960adf85a5292ed5 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Tue, 9 Sep 2014 16:39:38 -0400 Subject: Fix conditional after removing verbose code. --- src/lsm/lsm_work_unit.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/lsm/lsm_work_unit.c b/src/lsm/lsm_work_unit.c index c7c78a70f44..525fb57b5b3 100644 --- a/src/lsm/lsm_work_unit.c +++ b/src/lsm/lsm_work_unit.c @@ -84,10 +84,7 @@ __wt_lsm_get_chunk_to_flush(WT_SESSION_IMPL *session, * doing a forced flush, likely from a compact call, then we do want * to include the final chunk. */ - if (force) { - end = lsm_tree->nchunks; - } else - end = lsm_tree->nchunks - 1; + end = force ? lsm_tree->nchunks : lsm_tree->nchunks - 1; for (i = 0; i < end; i++) { if (!F_ISSET(lsm_tree->chunk[i], WT_LSM_CHUNK_ONDISK)) { (void)WT_ATOMIC_ADD(lsm_tree->chunk[i]->refcnt, 1); -- cgit v1.2.1 From 92a06bb0c94321e1ddab940cb77e3c0a3d5f5704 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Wed, 10 Sep 2014 09:08:25 +1000 Subject: Add wtperf configuration for shared cache testing. --- bench/wtperf/runners/shared-cache-stress.wtperf | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 bench/wtperf/runners/shared-cache-stress.wtperf diff --git a/bench/wtperf/runners/shared-cache-stress.wtperf b/bench/wtperf/runners/shared-cache-stress.wtperf new file mode 100644 index 00000000000..eee73b5d8e3 --- /dev/null +++ b/bench/wtperf/runners/shared-cache-stress.wtperf @@ -0,0 +1,11 @@ +# Stress out the shared cache. +conn_config="statistics=(none),shared_cache=(name=wt-cache,size=536870912,reserve=10MB,chunk=20MB,)" +table_config="allocation_size=4KB,key_gap=10,split_pct=75,internal_page_max=4KB,internal_key_truncate=false,prefix_compression=false,leaf_item_max=1433,type=file,internal_item_max=1433,exclusive=true,leaf_page_max=4KB,block_compressor=," +checkpoint_interval=100 +checkpoint_threads=1 +icount=50000 +report_interval=5 +run_time=600 +populate_threads=1 +threads=((count=1,inserts=1),(count=1,reads=1)) +database_count=25 -- cgit v1.2.1 From 0177e63ff2b68caceddefe237daa2e55d7be2454 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Wed, 10 Sep 2014 13:55:44 +1000 Subject: Fix for doxygen-1.8.8: it doesn't know how to treat "wiredtiger.in" without help. --- src/docs/Doxyfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/docs/Doxyfile b/src/docs/Doxyfile index 792b255f4ab..5492905f7e9 100644 --- a/src/docs/Doxyfile +++ b/src/docs/Doxyfile @@ -268,7 +268,7 @@ OPTIMIZE_OUTPUT_VHDL = NO # that for custom extensions you also need to set FILE_PATTERNS otherwise the # files are not read by doxygen. -EXTENSION_MAPPING = +EXTENSION_MAPPING = in=C # If MARKDOWN_SUPPORT is enabled (the default) then doxygen pre-processes all # comments according to the Markdown format, which allows for more readable -- cgit v1.2.1 From 03495cf56795eef5011c2440678025912c42d119 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Wed, 10 Sep 2014 13:55:44 +1000 Subject: Document that bulk load cursors are non-transactional. --- src/docs/tune-bulk-load.dox | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/docs/tune-bulk-load.dox b/src/docs/tune-bulk-load.dox index 9e89fb7ceea..8ee1061c76c 100644 --- a/src/docs/tune-bulk-load.dox +++ b/src/docs/tune-bulk-load.dox @@ -11,7 +11,9 @@ be used on newly created objects, and an object being bulk-loaded is not accessible from other cursors. Cursors configured for bulk-load only support the WT_CURSOR::insert and -WT_CURSOR::close methods. +WT_CURSOR::close methods. Bulk load inserts are non-transactional: they +cannot be rolled back and ignore the transactional state of the WT_SESSION +in which they are opened. When bulk-loading row-store objects, keys must be loaded in sorted order. -- cgit v1.2.1 From 72d4e708e6a1d4e84db117ae3f098ad970c098ba Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Wed, 10 Sep 2014 14:30:02 +1000 Subject: Make "-O3 -g" the default build flags (again), including for C++ (api/leveldb). --- build_posix/configure.ac.in | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/build_posix/configure.ac.in b/build_posix/configure.ac.in index 268ed586232..6352fa6d0df 100644 --- a/build_posix/configure.ac.in +++ b/build_posix/configure.ac.in @@ -9,6 +9,10 @@ AC_CONFIG_AUX_DIR([build_posix/gnu-support]) AC_CONFIG_MACRO_DIR([build_posix/aclocal]) AC_CONFIG_SRCDIR([RELEASE]) +# If CFLAGS/CXXFLAGS were not set on entry, default to "-O3 -g" +: ${CFLAGS=-O3 -g} +: ${CXXFLAGS=-O3 -g} + AM_INIT_AUTOMAKE([1.11 foreign parallel-tests subdir-objects]) m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([no])]) @@ -24,9 +28,6 @@ LT_PREREQ(2.2.6) LT_INIT([pic-only]) AC_SUBST([LIBTOOL_DEPS]) -# If CFLAGS was not set on entry, default to "-O3 -g" -: ${CFLAGS="-O3 -g"} - AC_PROG_CC(cc gcc) # AC_PROG_CXX(c++ g++) -- cgit v1.2.1 From c07c5cb34aae56fb20e30fa6586fa660326a6d5d Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Wed, 10 Sep 2014 14:30:24 +1000 Subject: Limit the maximum compression ratio our raw zlib implementation will allow. Once we have taken 20x the maximum page size, stop. This prevents pathological behavior in sythetic workloads where a page is forced out of cache, then compresses into a single page on disk, and we repeat for every update. --- ext/compressors/zlib/zlib_compress.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ext/compressors/zlib/zlib_compress.c b/ext/compressors/zlib/zlib_compress.c index 33bb9bf8810..3532ecf16cd 100644 --- a/ext/compressors/zlib/zlib_compress.c +++ b/ext/compressors/zlib/zlib_compress.c @@ -225,8 +225,15 @@ zlib_compress_raw(WT_COMPRESSOR *compressor, WT_SESSION *session, * Strategy: take the available output size and compress that much * input. Continue until there is no input small enough or the * compression fails to fit. + * + * Don't let the compression ratio become insanely good (which can + * happen with synthetic workloads). Once we hit a limit, stop so that + * the in-memory size of pages isn't totally different to the on-disk + * size. Otherwise we can get into trouble where every update to a + * page results in forced eviction based on in-memory size, even though + * the data fits into a single on-disk block. */ - while (zs.avail_out > 0) { + while (zs.avail_out > 0 && zs.total_in <= zs.total_out * 20) { /* Find the slot we will try to compress up to. */ if ((curr_slot = zlib_find_slot( zs.total_in + zs.avail_out, offsets, slots)) <= last_slot) -- cgit v1.2.1 From 8599b1a45ae20ba3cd586442d9faf7de0b5d56ae Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Wed, 10 Sep 2014 14:50:04 +1000 Subject: Limit range in shared cache wtperf configuration. Avoids using too much disk space. --- bench/wtperf/runners/shared-cache-stress.wtperf | 1 + 1 file changed, 1 insertion(+) diff --git a/bench/wtperf/runners/shared-cache-stress.wtperf b/bench/wtperf/runners/shared-cache-stress.wtperf index eee73b5d8e3..87d14f4f5c1 100644 --- a/bench/wtperf/runners/shared-cache-stress.wtperf +++ b/bench/wtperf/runners/shared-cache-stress.wtperf @@ -4,6 +4,7 @@ table_config="allocation_size=4KB,key_gap=10,split_pct=75,internal_page_max=4KB, checkpoint_interval=100 checkpoint_threads=1 icount=50000 +random_range=500000 report_interval=5 run_time=600 populate_threads=1 -- cgit v1.2.1 From 4e7ec366cd20667ba873a9f1dd3edf9144031fbf Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Wed, 10 Sep 2014 13:26:16 -0400 Subject: Add flushing stage to compact. Restore COMPACTING check during merges to force aggressiveness. #1200 --- src/include/lsm.h | 11 ++++++----- src/lsm/lsm_merge.c | 9 +++++---- src/lsm/lsm_tree.c | 54 ++++++++++++++++++++++++++++++++++------------------ src/lsm/lsm_worker.c | 37 +++++++++++++++++++---------------- 4 files changed, 68 insertions(+), 43 deletions(-) diff --git a/src/include/lsm.h b/src/include/lsm.h index b4581b2b094..48984399acd 100644 --- a/src/include/lsm.h +++ b/src/include/lsm.h @@ -192,11 +192,12 @@ struct __wt_lsm_tree { int freeing_old_chunks; /* Whether chunks are being freed */ uint32_t merge_aggressiveness; /* Increase amount of work per merge */ -#define WT_LSM_TREE_ACTIVE 0x01 /* Workers are active */ -#define WT_LSM_TREE_COMPACTING 0x02 /* Tree is being compacted */ -#define WT_LSM_TREE_NEED_SWITCH 0x04 /* A new chunk should be created */ -#define WT_LSM_TREE_OPEN 0x08 /* The tree is open */ -#define WT_LSM_TREE_THROTTLE 0x10 /* Throttle updates */ +#define WT_LSM_TREE_ACTIVE 0x01 /* Workers are active */ +#define WT_LSM_TREE_COMPACT_FLUSH 0x02 /* Flushed for compact */ +#define WT_LSM_TREE_COMPACTING 0x04 /* Tree being compacted */ +#define WT_LSM_TREE_NEED_SWITCH 0x08 /* New chunk needs creating */ +#define WT_LSM_TREE_OPEN 0x10 /* The tree is open */ +#define WT_LSM_TREE_THROTTLE 0x20 /* Throttle updates */ uint32_t flags; #define WT_LSM_TREE_EXCLUSIVE 0x01 /* Tree is opened exclusively */ diff --git a/src/lsm/lsm_merge.c b/src/lsm/lsm_merge.c index 2da69b3601c..8de7d1350df 100644 --- a/src/lsm/lsm_merge.c +++ b/src/lsm/lsm_merge.c @@ -75,11 +75,12 @@ __wt_lsm_merge( start_id = 0; /* - * If the tree is open read-only be very aggressive. Otherwise, we can - * spend a long time waiting for merges to start in read-only - * applications. + * If the tree is open read-only or we are compacting, be very + * aggressive. Otherwise, we can spend a long time waiting for merges + * to start in read-only applications. */ - if (!lsm_tree->modified) + if (!lsm_tree->modified || + F_ISSET(lsm_tree, WT_LSM_TREE_COMPACTING)) lsm_tree->merge_aggressiveness = 10; aggressive = lsm_tree->merge_aggressiveness; diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index ef9f859b59a..8a5a8ca1f3d 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -1007,7 +1007,7 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) WT_LSM_CHUNK *chunk; WT_LSM_TREE *lsm_tree; time_t begin, end; - int i, compacting, locked; + int i, compacting, flushing, locked, ref; compacting = locked = 0; /* @@ -1040,30 +1040,42 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) if (F_ISSET(lsm_tree, WT_LSM_TREE_COMPACTING)) goto err; - compacting = 1; - F_SET(lsm_tree, WT_LSM_TREE_COMPACTING); + compacting = flushing = 1; + F_SET(lsm_tree, WT_LSM_TREE_COMPACT_FLUSH | WT_LSM_TREE_COMPACTING); /* * Set the switch transaction on the current chunk, if it * hasn't been set before. This prevents further writes, so it * can be flushed by the checkpoint worker. */ + ref = 0; if (lsm_tree->nchunks > 0 && - (chunk = lsm_tree->chunk[lsm_tree->nchunks - 1]) != NULL && - chunk->switch_txn == WT_TXN_NONE) - chunk->switch_txn = __wt_txn_current_id(session); + (chunk = lsm_tree->chunk[lsm_tree->nchunks - 1]) != NULL) { + if (chunk->switch_txn == WT_TXN_NONE) + chunk->switch_txn = __wt_txn_current_id(session); + (void)WT_ATOMIC_ADD(chunk->refcnt, 1); + ref = 1; + } locked = 0; WT_ERR(__wt_lsm_tree_unlock(session, lsm_tree)); - /* Make sure the in-memory chunk gets flushed but not switched. */ - WT_ERR(__wt_verbose(session, WT_VERB_LSM, "Compact %s force flush", - name)); - WT_ERR(__wt_lsm_manager_push_entry( - session, WT_LSM_WORK_FLUSH | WT_LSM_WORK_FORCE, lsm_tree)); + WT_ERR(__wt_verbose(session, WT_VERB_LSM, + "Compact force flush %s flags 0x%" PRIx32 " chunk %u flags 0x%" + PRIx32, name, lsm_tree->flags, chunk->id, chunk->flags)); + /* Make sure the in-memory chunk gets flushed but not switched. */ + WT_ERR(__wt_lsm_manager_push_entry(session, + WT_LSM_WORK_FLUSH | WT_LSM_WORK_FORCE, lsm_tree)); /* Wait for the work unit queues to drain. */ while (F_ISSET(lsm_tree, WT_LSM_TREE_ACTIVE)) { + if (flushing && !F_ISSET(lsm_tree, WT_LSM_TREE_COMPACT_FLUSH)) { + WT_ERR(__wt_verbose(session, WT_VERB_LSM, + "Compact flush complete %s chunk %u", + name, chunk->id)); + flushing = ref = 0; + (void)WT_ATOMIC_SUB(chunk->refcnt, 1); + } /* * The compacting flag is cleared when no merges can be done. * Ensure that we push through some aggressive merges before @@ -1074,7 +1086,7 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) if (lsm_tree->merge_aggressiveness < 10) { F_SET(lsm_tree, WT_LSM_TREE_COMPACTING); lsm_tree->merge_aggressiveness = 10; - } else + } else if (!flushing) break; } __wt_sleep(1, 0); @@ -1088,21 +1100,27 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) * done. If we are pushing merges, make sure they are * aggressive, to avoid duplicating effort. */ + if (!flushing) #define COMPACT_PARALLEL_MERGES 5 - for (i = lsm_tree->queue_ref; - i < COMPACT_PARALLEL_MERGES; i++) { - lsm_tree->merge_aggressiveness = 10; - WT_ERR(__wt_lsm_manager_push_entry( - session, WT_LSM_WORK_MERGE, lsm_tree)); - } + for (i = lsm_tree->queue_ref; + i < COMPACT_PARALLEL_MERGES; i++) { + lsm_tree->merge_aggressiveness = 10; + WT_ERR(__wt_lsm_manager_push_entry( + session, WT_LSM_WORK_MERGE, lsm_tree)); + } } err: if (locked) WT_ERR(__wt_lsm_tree_unlock(session, lsm_tree)); /* Ensure the compacting flag is cleared if we set it. */ + if (flushing) + F_CLR(lsm_tree, WT_LSM_TREE_COMPACT_FLUSH); + if (ref) + (void)WT_ATOMIC_SUB(chunk->refcnt, 1); if (compacting) { F_CLR(lsm_tree, WT_LSM_TREE_COMPACTING); lsm_tree->merge_aggressiveness = 0; } + WT_ERR(__wt_verbose(session, WT_VERB_LSM, "Compact complete %s", name)); __wt_lsm_tree_release(session, lsm_tree); return (ret); diff --git a/src/lsm/lsm_worker.c b/src/lsm/lsm_worker.c index d1eb9ca9ee6..68ebe34fe91 100644 --- a/src/lsm/lsm_worker.c +++ b/src/lsm/lsm_worker.c @@ -52,22 +52,27 @@ __lsm_worker_general_op( * possible chunks, not just the first one we find. */ last = 0; - do { - WT_ERR(__wt_lsm_get_chunk_to_flush(session, - entry->lsm_tree, force, &last, &chunk)); - if (chunk != NULL) { - WT_ERR(__wt_verbose(session, WT_VERB_LSM, - "Flush%s%s chunk %d %s", - force ? " w/ force" : "", - last ? " last" : "", - chunk->id, chunk->uri)); - ret = __wt_lsm_checkpoint_chunk( - session, entry->lsm_tree, chunk); - WT_ASSERT(session, chunk->refcnt > 0); - (void)WT_ATOMIC_SUB(chunk->refcnt, 1); - WT_ERR(ret); - } - } while (force && chunk != NULL && !last); + WT_ERR(__wt_lsm_get_chunk_to_flush(session, + entry->lsm_tree, force, &last, &chunk)); + if (chunk != NULL) { + WT_ERR(__wt_verbose(session, WT_VERB_LSM, + "Flush%s%s chunk %d %s", + force ? " w/ force" : "", + last ? " last" : "", + chunk->id, chunk->uri)); + ret = __wt_lsm_checkpoint_chunk( + session, entry->lsm_tree, chunk); + WT_ASSERT(session, chunk->refcnt > 0); + (void)WT_ATOMIC_SUB(chunk->refcnt, 1); + WT_ERR(ret); + } + /* + * If we flushed the last chunk for a compact, clear the + * flag so compact knows that is complete. + */ + if (last && force && + F_ISSET(entry->lsm_tree, WT_LSM_TREE_COMPACT_FLUSH)) + F_CLR(entry->lsm_tree, WT_LSM_TREE_COMPACT_FLUSH); } else if (entry->flags == WT_LSM_WORK_DROP) WT_ERR(__wt_lsm_free_chunks(session, entry->lsm_tree)); else if (entry->flags == WT_LSM_WORK_BLOOM) { -- cgit v1.2.1 From 44581e8f5c7cf73af1a2e30de55395d5eb00c4df Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Wed, 10 Sep 2014 13:55:57 -0400 Subject: Fix warnings. --- src/lsm/lsm_tree.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index 8a5a8ca1f3d..681b194ed9d 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -1009,7 +1009,7 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) time_t begin, end; int i, compacting, flushing, locked, ref; - compacting = locked = 0; + compacting = flushing = locked = ref = 0; /* * This function is applied to all matching sources: ignore anything * that is not an LSM tree. @@ -1048,7 +1048,7 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) * hasn't been set before. This prevents further writes, so it * can be flushed by the checkpoint worker. */ - ref = 0; + chunk = NULL; if (lsm_tree->nchunks > 0 && (chunk = lsm_tree->chunk[lsm_tree->nchunks - 1]) != NULL) { if (chunk->switch_txn == WT_TXN_NONE) @@ -1069,12 +1069,15 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) /* Wait for the work unit queues to drain. */ while (F_ISSET(lsm_tree, WT_LSM_TREE_ACTIVE)) { - if (flushing && !F_ISSET(lsm_tree, WT_LSM_TREE_COMPACT_FLUSH)) { - WT_ERR(__wt_verbose(session, WT_VERB_LSM, - "Compact flush complete %s chunk %u", - name, chunk->id)); + if (flushing && ref && + !F_ISSET(lsm_tree, WT_LSM_TREE_COMPACT_FLUSH)) { flushing = ref = 0; - (void)WT_ATOMIC_SUB(chunk->refcnt, 1); + if (chunk != NULL) { + WT_ERR(__wt_verbose(session, WT_VERB_LSM, + "Compact flush complete %s chunk %u", + name, chunk->id)); + (void)WT_ATOMIC_SUB(chunk->refcnt, 1); + } } /* * The compacting flag is cleared when no merges can be done. -- cgit v1.2.1 From ccb500510aa03e6679286c278625156ec3a82c9d Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 10 Sep 2014 16:26:29 -0400 Subject: Add support to run zlib-noraw compression. --- test/format/config.c | 21 +++++++++++++++------ test/format/config.h | 2 +- test/format/format.h | 1 + test/format/wts.c | 4 ++++ 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/test/format/config.c b/test/format/config.c index 509dc7684ec..519c5db7890 100644 --- a/test/format/config.c +++ b/test/format/config.c @@ -243,26 +243,32 @@ config_compression(void) cp = config_find("compression", strlen("compression")); if (!(cp->flags & C_PERM)) { cstr = "compression=none"; - switch (MMRAND(1, 10)) { - case 1: case 2: case 3: /* 30% */ + switch (MMRAND(1, 20)) { + case 1: case 2: case 3: /* 30% no compression */ + case 4: case 5: case 6: break; - case 4: case 5: /* 20% */ + case 7: case 8: case 9: case 10: /* 20% bzip */ if (access(BZIP_PATH, R_OK) == 0) cstr = "compression=bzip"; break; - case 6: /* 10% */ + case 11: /* 5% bzip-raw */ if (access(BZIP_PATH, R_OK) == 0) cstr = "compression=bzip-raw"; break; - case 7: case 8: /* 20% */ + case 12: case 13: case 14: case 15: /* 20% snappy */ if (access(SNAPPY_PATH, R_OK) == 0) cstr = "compression=snappy"; break; - case 9: case 10: /* 20% */ + case 16: case 17: case 18: case 19: /* 20% zlib */ if (access(ZLIB_PATH, R_OK) == 0) cstr = "compression=zlib"; break; + case 20: /* 5% zlib-no-raw */ + if (access(ZLIB_PATH, R_OK) == 0) + cstr = "compression=zlib-noraw"; + break; } + config_single(cstr, 0); } @@ -281,6 +287,7 @@ config_compression(void) die(0, "snappy library not found or not readable"); break; case COMPRESS_ZLIB: + case COMPRESS_ZLIB_NO_RAW: if (access(ZLIB_PATH, R_OK) != 0) die(0, "zlib library not found or not readable"); break; @@ -549,6 +556,8 @@ config_map_compression(const char *s, u_int *vp) *vp = COMPRESS_SNAPPY; else if (strcmp(s, "zlib") == 0) *vp = COMPRESS_ZLIB; + else if (strcmp(s, "zlib-noraw") == 0) + *vp = COMPRESS_ZLIB_NO_RAW; else die(EINVAL, "illegal compression configuration: %s", s); } diff --git a/test/format/config.h b/test/format/config.h index a32df3de95c..4bc1493a681 100644 --- a/test/format/config.h +++ b/test/format/config.h @@ -115,7 +115,7 @@ static CONFIG c[] = { C_BOOL, 10, 0, 0, &g.c_compact, NULL }, { "compression", - "type of compression (none | bzip | bzip-raw | lzo | snappy | zlib)", + "type of compression (none | bzip | bzip-raw | lzo | snappy | zlib | zlib-noraw)", C_IGNORE|C_STRING, 1, 5, 5, NULL, &g.c_compression }, { "data_extend", diff --git a/test/format/format.h b/test/format/format.h index 0e45a28b3ef..1f2b363e9a4 100644 --- a/test/format/format.h +++ b/test/format/format.h @@ -205,6 +205,7 @@ typedef struct { #define COMPRESS_LZO 4 #define COMPRESS_SNAPPY 5 #define COMPRESS_ZLIB 6 +#define COMPRESS_ZLIB_NO_RAW 7 u_int c_compression_flag; /* Compression flag value */ #define ISOLATION_RANDOM 1 diff --git a/test/format/wts.c b/test/format/wts.c index 1a83fa92894..e495956fd2e 100644 --- a/test/format/wts.c +++ b/test/format/wts.c @@ -272,6 +272,10 @@ wts_create(void) p += snprintf(p, (size_t)(end - p), ",block_compressor=\"zlib\""); break; + case COMPRESS_ZLIB_NO_RAW: + p += snprintf(p, (size_t)(end - p), + ",block_compressor=\"zlib-noraw\""); + break; } /* Configure Btree internal key truncation. */ -- cgit v1.2.1 From 2ff6bbfc499db45087a85494d5d390634d2be2ab Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 10 Sep 2014 17:25:44 -0400 Subject: Rename __wt_conn_cache_pool_config to be __wt_cache_pool_config, the other cache_XXX API doesn't have _conn_ in its name. --- src/conn/conn_api.c | 3 +-- src/conn/conn_cache_pool.c | 4 ++-- src/conn/conn_open.c | 2 +- src/include/extern.h | 3 +-- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 0b210627df5..d9e83db4e37 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -649,9 +649,8 @@ __conn_reconfigure(WT_CONNECTION *wt_conn, const char *config) config_cfg[0] = conn->cfg; config_cfg[1] = config; - WT_ERR(__wt_conn_cache_pool_config(session, config_cfg)); + WT_ERR(__wt_cache_pool_config(session, config_cfg)); WT_ERR(__wt_cache_config(conn, config_cfg)); - WT_ERR(__wt_async_reconfig(conn, config_cfg)); WT_ERR(__conn_statistics_config(session, config_cfg)); WT_ERR(__wt_conn_verbose_config(session, config_cfg)); diff --git a/src/conn/conn_cache_pool.c b/src/conn/conn_cache_pool.c index 9b540c147d4..57d343d0d46 100644 --- a/src/conn/conn_cache_pool.c +++ b/src/conn/conn_cache_pool.c @@ -24,11 +24,11 @@ static int __cache_pool_assess(WT_SESSION_IMPL *, uint64_t *); static int __cache_pool_balance(WT_SESSION_IMPL *); /* - * __wt_conn_cache_pool_config -- + * __wt_cache_pool_config -- * Parse and setup the cache pool options. */ int -__wt_conn_cache_pool_config(WT_SESSION_IMPL *session, const char **cfg) +__wt_cache_pool_config(WT_SESSION_IMPL *session, const char **cfg) { WT_CACHE_POOL *cp; WT_CONFIG_ITEM cval; diff --git a/src/conn/conn_open.c b/src/conn/conn_open.c index 3a2f1cb51a4..e20931e7028 100644 --- a/src/conn/conn_open.c +++ b/src/conn/conn_open.c @@ -55,7 +55,7 @@ __wt_connection_open(WT_CONNECTION_IMPL *conn, const char *cfg[]) WT_WRITE_BARRIER(); /* Connect to a cache pool. */ - WT_RET(__wt_conn_cache_pool_config(session, cfg)); + WT_RET(__wt_cache_pool_config(session, cfg)); /* Create the cache. */ WT_RET(__wt_cache_create(conn, cfg)); diff --git a/src/include/extern.h b/src/include/extern.h index ed285db6395..67decaee28b 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -609,8 +609,7 @@ extern int __wt_cache_config(WT_CONNECTION_IMPL *conn, const char *cfg[]); extern int __wt_cache_create(WT_CONNECTION_IMPL *conn, const char *cfg[]); extern void __wt_cache_stats_update(WT_SESSION_IMPL *session); extern int __wt_cache_destroy(WT_CONNECTION_IMPL *conn); -extern int __wt_conn_cache_pool_config(WT_SESSION_IMPL *session, - const char **cfg); +extern int __wt_cache_pool_config(WT_SESSION_IMPL *session, const char **cfg); extern int __wt_conn_cache_pool_open(WT_SESSION_IMPL *session); extern int __wt_conn_cache_pool_destroy(WT_CONNECTION_IMPL *conn); extern void *__wt_cache_pool_server(void *arg); -- cgit v1.2.1 From 65e598ddc1031b4803281644a0ac8023d944e5b2 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 10 Sep 2014 17:27:17 -0400 Subject: Rename __wt_conn_verbose_config to be __wt_verbose_config, the _conn_ isn't adding anything useful. --- src/btree/bt_debug.c | 2 +- src/conn/conn_api.c | 14 +++++++------- src/include/extern.h | 3 +-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/btree/bt_debug.c b/src/btree/bt_debug.c index 8a069cc4bdf..84c4565eafe 100644 --- a/src/btree/bt_debug.c +++ b/src/btree/bt_debug.c @@ -62,7 +62,7 @@ __wt_debug_set_verbose(WT_SESSION_IMPL *session, const char *v) snprintf(buf, sizeof(buf), "verbose=[%s]", v); cfg[0] = buf; - return (__wt_conn_verbose_config(session, cfg)); + return (__wt_verbose_config(session, cfg)); } /* diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index d9e83db4e37..b05a2f87f60 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -649,12 +649,12 @@ __conn_reconfigure(WT_CONNECTION *wt_conn, const char *config) config_cfg[0] = conn->cfg; config_cfg[1] = config; - WT_ERR(__wt_cache_pool_config(session, config_cfg)); - WT_ERR(__wt_cache_config(conn, config_cfg)); - WT_ERR(__wt_async_reconfig(conn, config_cfg)); WT_ERR(__conn_statistics_config(session, config_cfg)); - WT_ERR(__wt_conn_verbose_config(session, config_cfg)); + WT_ERR(__wt_async_reconfig(conn, config_cfg)); + WT_ERR(__wt_cache_config(conn, config_cfg)); + WT_ERR(__wt_cache_pool_config(session, config_cfg)); WT_ERR(__wt_checkpoint_server_create(conn, config_cfg)); + WT_ERR(__wt_verbose_config(session, config_cfg)); WT_ERR(__wt_statlog_create(conn, config_cfg)); WT_ERR(__wt_config_gets( @@ -1062,11 +1062,11 @@ __conn_statistics_config(WT_SESSION_IMPL *session, const char *cfg[]) } /* - * __wt_conn_verbose_config -- + * __wt_verbose_config -- * Set verbose configuration. */ int -__wt_conn_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) +__wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) { WT_CONFIG_ITEM cval, sval; WT_CONNECTION_IMPL *conn; @@ -1328,7 +1328,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, if (cval.val) F_SET(conn, WT_CONN_CKPT_SYNC); - WT_ERR(__wt_conn_verbose_config(session, cfg)); + WT_ERR(__wt_verbose_config(session, cfg)); WT_ERR(__wt_config_gets(session, cfg, "buffer_alignment", &cval)); if (cval.val == -1) diff --git a/src/include/extern.h b/src/include/extern.h index 67decaee28b..b7479f586b9 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -603,8 +603,7 @@ extern int __wt_conn_remove_compressor( WT_CONNECTION_IMPL *conn, WT_NAMED_COMPRESSOR *ncomp); extern int __wt_conn_remove_data_source( WT_CONNECTION_IMPL *conn, WT_NAMED_DATA_SOURCE *ndsrc); -extern int __wt_conn_verbose_config(WT_SESSION_IMPL *session, - const char *cfg[]); +extern int __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_cache_config(WT_CONNECTION_IMPL *conn, const char *cfg[]); extern int __wt_cache_create(WT_CONNECTION_IMPL *conn, const char *cfg[]); extern void __wt_cache_stats_update(WT_SESSION_IMPL *session); -- cgit v1.2.1 From 91d63c3b70a91933547f8d7ce510994afd2e8b50 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 10 Sep 2014 17:40:37 -0400 Subject: Fix some places we were passing the WT_CONNECTION_IMPL to underlying routines, that means they have to directly grab WT_CONNECTION_IMPL.default_sesion which isn't good. Pass the WT_SESSION_IMPL instead, then functions do S2C() like the rest of the code. Not "right", but "better". --- src/async/async_api.c | 15 +++++++-------- src/conn/conn_api.c | 8 ++++---- src/conn/conn_cache.c | 14 +++++++------- src/conn/conn_ckpt.c | 6 ++++-- src/conn/conn_log.c | 6 +++--- src/conn/conn_open.c | 10 +++++----- src/conn/conn_stat.c | 6 +++--- src/include/extern.h | 14 +++++++------- 8 files changed, 40 insertions(+), 39 deletions(-) diff --git a/src/async/async_api.c b/src/async/async_api.c index ae567466e8e..294662defab 100644 --- a/src/async/async_api.c +++ b/src/async/async_api.c @@ -228,14 +228,14 @@ __wt_async_stats_update(WT_SESSION_IMPL *session) * Start the async subsystem and worker threads. */ int -__wt_async_create(WT_CONNECTION_IMPL *conn, const char *cfg[]) +__wt_async_create(WT_SESSION_IMPL *session, const char *cfg[]) { WT_ASYNC *async; - WT_SESSION_IMPL *session; + WT_CONNECTION_IMPL *conn; int run; uint32_t i; - session = conn->default_session; + conn = S2C(session); /* Handle configuration. */ run = 0; @@ -288,17 +288,16 @@ __wt_async_create(WT_CONNECTION_IMPL *conn, const char *cfg[]) * Start the async subsystem and worker threads. */ int -__wt_async_reconfig(WT_CONNECTION_IMPL *conn, const char *cfg[]) +__wt_async_reconfig(WT_SESSION_IMPL *session, const char *cfg[]) { WT_ASYNC *async; - WT_CONNECTION_IMPL tmp_conn; + WT_CONNECTION_IMPL *conn, tmp_conn; WT_DECL_RET; WT_SESSION *wt_session; - WT_SESSION_IMPL *session; int run; uint32_t i; - session = conn->default_session; + conn = S2C(session); async = conn->async; memset(&tmp_conn, 0, sizeof(tmp_conn)); tmp_conn.async_cfg = conn->async_cfg; @@ -338,7 +337,7 @@ __wt_async_reconfig(WT_CONNECTION_IMPL *conn, const char *cfg[]) return (ret); } else if (conn->async_cfg == 0 && run) /* Case 2 */ - return (__wt_async_create(conn, cfg)); + return (__wt_async_create(session, cfg)); else if (conn->async_cfg == 0) /* Case 3 */ return (0); diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index b05a2f87f60..22d3b9d4411 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -650,12 +650,12 @@ __conn_reconfigure(WT_CONNECTION *wt_conn, const char *config) config_cfg[1] = config; WT_ERR(__conn_statistics_config(session, config_cfg)); - WT_ERR(__wt_async_reconfig(conn, config_cfg)); - WT_ERR(__wt_cache_config(conn, config_cfg)); + WT_ERR(__wt_async_reconfig(session, config_cfg)); + WT_ERR(__wt_cache_config(session, config_cfg)); WT_ERR(__wt_cache_pool_config(session, config_cfg)); - WT_ERR(__wt_checkpoint_server_create(conn, config_cfg)); + WT_ERR(__wt_checkpoint_server_create(session, config_cfg)); WT_ERR(__wt_verbose_config(session, config_cfg)); - WT_ERR(__wt_statlog_create(conn, config_cfg)); + WT_ERR(__wt_statlog_create(session, config_cfg)); WT_ERR(__wt_config_gets( session, config_cfg, "lsm_manager.worker_thread_max", &cval)); diff --git a/src/conn/conn_cache.c b/src/conn/conn_cache.c index 42e45a9c58b..b5f94f137ab 100644 --- a/src/conn/conn_cache.c +++ b/src/conn/conn_cache.c @@ -12,14 +12,14 @@ * Configure the underlying cache. */ int -__wt_cache_config(WT_CONNECTION_IMPL *conn, const char *cfg[]) +__wt_cache_config(WT_SESSION_IMPL *session, const char *cfg[]) { WT_CACHE *cache; WT_CONFIG_ITEM cval; + WT_CONNECTION_IMPL *conn; WT_DECL_RET; - WT_SESSION_IMPL *session; - session = conn->default_session; + conn = S2C(session); cache = conn->cache; /* @@ -85,13 +85,13 @@ __wt_cache_config(WT_CONNECTION_IMPL *conn, const char *cfg[]) * Create the underlying cache. */ int -__wt_cache_create(WT_CONNECTION_IMPL *conn, const char *cfg[]) +__wt_cache_create(WT_SESSION_IMPL *session, const char *cfg[]) { WT_CACHE *cache; + WT_CONNECTION_IMPL *conn; WT_DECL_RET; - WT_SESSION_IMPL *session; - session = conn->default_session; + conn = S2C(session); WT_ASSERT(session, conn->cache == NULL || (F_ISSET(conn, WT_CONN_CACHE_POOL) && conn->cache != NULL)); @@ -101,7 +101,7 @@ __wt_cache_create(WT_CONNECTION_IMPL *conn, const char *cfg[]) cache = conn->cache; /* Use a common routine for run-time configuration options. */ - WT_RET(__wt_cache_config(conn, cfg)); + WT_RET(__wt_cache_config(session, cfg)); /* Add the configured cache to the cache pool. */ if (F_ISSET(conn, WT_CONN_CACHE_POOL)) diff --git a/src/conn/conn_ckpt.c b/src/conn/conn_ckpt.c index f948cd64d2e..101877a3ddb 100644 --- a/src/conn/conn_ckpt.c +++ b/src/conn/conn_ckpt.c @@ -147,17 +147,19 @@ __ckpt_server_start(WT_CONNECTION_IMPL *conn) * Configure and start the checkpoint server. */ int -__wt_checkpoint_server_create(WT_CONNECTION_IMPL *conn, const char *cfg[]) +__wt_checkpoint_server_create(WT_SESSION_IMPL *session, const char *cfg[]) { + WT_CONNECTION_IMPL *conn; int start; + conn = S2C(session); start = 0; /* If there is already a server running, shut it down. */ if (conn->ckpt_session != NULL) WT_RET(__wt_checkpoint_server_destroy(conn)); - WT_RET(__ckpt_server_config(conn->default_session, cfg, &start)); + WT_RET(__ckpt_server_config(session, cfg, &start)); if (start) WT_RET(__ckpt_server_start(conn)); diff --git a/src/conn/conn_log.c b/src/conn/conn_log.c index 0ecf48c6628..114e44ea193 100644 --- a/src/conn/conn_log.c +++ b/src/conn/conn_log.c @@ -166,13 +166,13 @@ err: __wt_err(session, ret, "log archive server error"); * Start the log subsystem and archive server thread. */ int -__wt_logmgr_create(WT_CONNECTION_IMPL *conn, const char *cfg[]) +__wt_logmgr_create(WT_SESSION_IMPL *session, const char *cfg[]) { - WT_SESSION_IMPL *session; + WT_CONNECTION_IMPL *conn; WT_LOG *log; int run; - session = conn->default_session; + conn = S2C(session); /* Handle configuration. */ WT_RET(__logmgr_config(session, cfg, &run)); diff --git a/src/conn/conn_open.c b/src/conn/conn_open.c index e20931e7028..78351ce4bd7 100644 --- a/src/conn/conn_open.c +++ b/src/conn/conn_open.c @@ -58,7 +58,7 @@ __wt_connection_open(WT_CONNECTION_IMPL *conn, const char *cfg[]) WT_RET(__wt_cache_pool_config(session, cfg)); /* Create the cache. */ - WT_RET(__wt_cache_create(conn, cfg)); + WT_RET(__wt_cache_create(session, cfg)); /* Initialize transaction support. */ WT_RET(__wt_txn_global_init(conn, cfg)); @@ -225,20 +225,20 @@ __wt_connection_workers(WT_SESSION_IMPL *session, const char *cfg[]) * Start the optional statistics thread. Start statistics first so that * other optional threads can know if statistics are enabled or not. */ - WT_RET(__wt_statlog_create(conn, cfg)); + WT_RET(__wt_statlog_create(session, cfg)); /* Start the optional async threads. */ - WT_RET(__wt_async_create(conn, cfg)); + WT_RET(__wt_async_create(session, cfg)); /* * Start the optional logging/archive thread. * NOTE: The log manager must be started before checkpoints so that the * checkpoint server knows if logging is enabled. */ - WT_RET(__wt_logmgr_create(conn, cfg)); + WT_RET(__wt_logmgr_create(session, cfg)); /* Start the optional checkpoint thread. */ - WT_RET(__wt_checkpoint_server_create(conn, cfg)); + WT_RET(__wt_checkpoint_server_create(session, cfg)); return (0); } diff --git a/src/conn/conn_stat.c b/src/conn/conn_stat.c index eaee410ad0a..12b9567b6e8 100644 --- a/src/conn/conn_stat.c +++ b/src/conn/conn_stat.c @@ -441,12 +441,12 @@ __statlog_start(WT_CONNECTION_IMPL *conn) * Start the statistics server thread. */ int -__wt_statlog_create(WT_CONNECTION_IMPL *conn, const char *cfg[]) +__wt_statlog_create(WT_SESSION_IMPL *session, const char *cfg[]) { - WT_SESSION_IMPL *session; + WT_CONNECTION_IMPL *conn; int start; - session = conn->default_session; + conn = S2C(session); start = 0; /* diff --git a/src/include/extern.h b/src/include/extern.h index b7479f586b9..8208c021d47 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -1,8 +1,8 @@ /* DO NOT EDIT: automatically built by dist/s_prototypes. */ extern void __wt_async_stats_update(WT_SESSION_IMPL *session); -extern int __wt_async_create(WT_CONNECTION_IMPL *conn, const char *cfg[]); -extern int __wt_async_reconfig(WT_CONNECTION_IMPL *conn, const char *cfg[]); +extern int __wt_async_create(WT_SESSION_IMPL *session, const char *cfg[]); +extern int __wt_async_reconfig(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_async_destroy(WT_CONNECTION_IMPL *conn); extern int __wt_async_flush(WT_CONNECTION_IMPL *conn); extern int __wt_async_new_op(WT_CONNECTION_IMPL *conn, @@ -604,15 +604,15 @@ extern int __wt_conn_remove_compressor( WT_CONNECTION_IMPL *conn, extern int __wt_conn_remove_data_source( WT_CONNECTION_IMPL *conn, WT_NAMED_DATA_SOURCE *ndsrc); extern int __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]); -extern int __wt_cache_config(WT_CONNECTION_IMPL *conn, const char *cfg[]); -extern int __wt_cache_create(WT_CONNECTION_IMPL *conn, const char *cfg[]); +extern int __wt_cache_config(WT_SESSION_IMPL *session, const char *cfg[]); +extern int __wt_cache_create(WT_SESSION_IMPL *session, const char *cfg[]); extern void __wt_cache_stats_update(WT_SESSION_IMPL *session); extern int __wt_cache_destroy(WT_CONNECTION_IMPL *conn); extern int __wt_cache_pool_config(WT_SESSION_IMPL *session, const char **cfg); extern int __wt_conn_cache_pool_open(WT_SESSION_IMPL *session); extern int __wt_conn_cache_pool_destroy(WT_CONNECTION_IMPL *conn); extern void *__wt_cache_pool_server(void *arg); -extern int __wt_checkpoint_server_create(WT_CONNECTION_IMPL *conn, +extern int __wt_checkpoint_server_create(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_checkpoint_server_destroy(WT_CONNECTION_IMPL *conn); extern int __wt_checkpoint_signal(WT_SESSION_IMPL *session, off_t logsize); @@ -642,14 +642,14 @@ extern int __wt_conn_dhandle_discard_single( WT_SESSION_IMPL *session, extern int __wt_conn_dhandle_discard(WT_CONNECTION_IMPL *conn); extern int __wt_connection_init(WT_CONNECTION_IMPL *conn); extern int __wt_connection_destroy(WT_CONNECTION_IMPL *conn); -extern int __wt_logmgr_create(WT_CONNECTION_IMPL *conn, const char *cfg[]); +extern int __wt_logmgr_create(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_logmgr_destroy(WT_CONNECTION_IMPL *conn); extern int __wt_connection_open(WT_CONNECTION_IMPL *conn, const char *cfg[]); extern int __wt_connection_close(WT_CONNECTION_IMPL *conn); extern int __wt_connection_workers(WT_SESSION_IMPL *session, const char *cfg[]); extern void __wt_conn_stat_init(WT_SESSION_IMPL *session); extern int __wt_statlog_log_one(WT_SESSION_IMPL *session); -extern int __wt_statlog_create(WT_CONNECTION_IMPL *conn, const char *cfg[]); +extern int __wt_statlog_create(WT_SESSION_IMPL *session, const char *cfg[]); extern int __wt_statlog_destroy(WT_CONNECTION_IMPL *conn, int is_close); extern int __wt_sweep_create(WT_CONNECTION_IMPL *conn); extern int __wt_sweep_destroy(WT_CONNECTION_IMPL *conn); -- cgit v1.2.1 From 90d6d4173aa1957e3b4d633c5691f7853c2701a8 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 10 Sep 2014 17:43:30 -0400 Subject: Merging the wiredtiger_open configuration means we should never see not-found returned when retrieving the "statistics" configuration. Don't reset the connection's statistics flags until the entire config function is successful, it would be strange to mix-and-match old and new flags. --- src/conn/conn_api.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 22d3b9d4411..3793d30b335 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -1018,46 +1018,48 @@ __conn_statistics_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_CONFIG_ITEM cval, sval; WT_CONNECTION_IMPL *conn; WT_DECL_RET; + uint32_t flags; int set; conn = S2C(session); - if ((ret = __wt_config_gets(session, cfg, "statistics", &cval)) != 0) - return (ret == WT_NOTFOUND ? 0 : ret); - - /* Configuring statistics clears any existing values. */ - conn->stat_flags = 0; + WT_RET(__wt_config_gets(session, cfg, "statistics", &cval)); + flags = 0; set = 0; if ((ret = __wt_config_subgets( session, &cval, "none", &sval)) == 0 && sval.val != 0) { - FLD_SET(conn->stat_flags, WT_CONN_STAT_NONE); + LF_SET(WT_CONN_STAT_NONE); ++set; } WT_RET_NOTFOUND_OK(ret); if ((ret = __wt_config_subgets( session, &cval, "fast", &sval)) == 0 && sval.val != 0) { - FLD_SET(conn->stat_flags, WT_CONN_STAT_FAST); + LF_SET(WT_CONN_STAT_FAST); ++set; } WT_RET_NOTFOUND_OK(ret); if ((ret = __wt_config_subgets( session, &cval, "all", &sval)) == 0 && sval.val != 0) { - FLD_SET(conn->stat_flags, WT_CONN_STAT_ALL | WT_CONN_STAT_FAST); + LF_SET(WT_CONN_STAT_ALL | WT_CONN_STAT_FAST); ++set; } WT_RET_NOTFOUND_OK(ret); if ((ret = __wt_config_subgets( session, &cval, "clear", &sval)) == 0 && sval.val != 0) - FLD_SET(conn->stat_flags, WT_CONN_STAT_CLEAR); + LF_SET(WT_CONN_STAT_CLEAR); WT_RET_NOTFOUND_OK(ret); if (set > 1) WT_RET_MSG(session, EINVAL, "only one statistics configuration value may be specified"); + + /* Configuring statistics clears any existing values. */ + conn->stat_flags = flags; + return (0); } -- cgit v1.2.1 From 8582cd5db8ae729deef7aa8e67cdfdb29f08c848 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 10 Sep 2014 17:46:19 -0400 Subject: Merging the wiredtiger_open configuration means we should never see not-found returned when retrieving the async.enabled, async.ops_max and async.threads configurations. --- src/async/async_api.c | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/src/async/async_api.c b/src/async/async_api.c index 294662defab..e7c2c101bb1 100644 --- a/src/async/async_api.c +++ b/src/async/async_api.c @@ -172,32 +172,24 @@ __async_config(WT_SESSION_IMPL *session, WT_CONNECTION_IMPL *conn, const char **cfg, int *runp) { WT_CONFIG_ITEM cval; - WT_DECL_RET; /* * The async configuration is off by default. */ - if ((ret = __wt_config_gets( - session, cfg, "async.enabled", &cval)) == 0) - *runp = cval.val != 0; - WT_RET_NOTFOUND_OK(ret); + WT_RET(__wt_config_gets(session, cfg, "async.enabled", &cval)); + *runp = cval.val != 0; /* * Even if async is turned off, we want to parse and store the * default values so that reconfigure can just enable them. */ - if ((ret = __wt_config_gets( - session, cfg, "async.ops_max", &cval)) == 0) - conn->async_size = (uint32_t)cval.val; - WT_RET_NOTFOUND_OK(ret); - - if ((ret = __wt_config_gets( - session, cfg, "async.threads", &cval)) == 0) { - conn->async_workers = (uint32_t)cval.val; - /* Sanity check that api_data.py is in sync with async.h */ - WT_ASSERT(session, conn->async_workers <= WT_ASYNC_MAX_WORKERS); - } - WT_RET_NOTFOUND_OK(ret); + WT_RET(__wt_config_gets(session, cfg, "async.ops_max", &cval)); + conn->async_size = (uint32_t)cval.val; + + WT_RET(__wt_config_gets(session, cfg, "async.threads", &cval)); + conn->async_workers = (uint32_t)cval.val; + /* Sanity check that api_data.py is in sync with async.h */ + WT_ASSERT(session, conn->async_workers <= WT_ASYNC_MAX_WORKERS); return (0); } -- cgit v1.2.1 From 45746528302a833ff16435477f789411fd4ac907 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 10 Sep 2014 17:49:09 -0400 Subject: Merging the wiredtiger_open configuration means we should never see not-found returned when retrieving the eviction_target, eviction_trigger, eviction_dirty_target, eviction.threads_max and eviction.threads_min configuration strings. --- src/conn/conn_cache.c | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/src/conn/conn_cache.c b/src/conn/conn_cache.c index b5f94f137ab..fabff374319 100644 --- a/src/conn/conn_cache.c +++ b/src/conn/conn_cache.c @@ -39,38 +39,27 @@ __wt_cache_config(WT_SESSION_IMPL *session, const char *cfg[]) cache->cp_reserved = (uint64_t)cval.val; WT_RET_NOTFOUND_OK(ret); - if ((ret = - __wt_config_gets(session, cfg, "eviction_target", &cval)) == 0) - cache->eviction_target = (u_int)cval.val; - WT_RET_NOTFOUND_OK(ret); + WT_RET(__wt_config_gets(session, cfg, "eviction_target", &cval)); + cache->eviction_target = (u_int)cval.val; - if ((ret = - __wt_config_gets(session, cfg, "eviction_trigger", &cval)) == 0) - cache->eviction_trigger = (u_int)cval.val; - WT_RET_NOTFOUND_OK(ret); + WT_RET(__wt_config_gets(session, cfg, "eviction_trigger", &cval)); + cache->eviction_trigger = (u_int)cval.val; - if ((ret = __wt_config_gets( - session, cfg, "eviction_dirty_target", &cval)) == 0) - cache->eviction_dirty_target = (u_int)cval.val; - WT_RET_NOTFOUND_OK(ret); + WT_RET(__wt_config_gets(session, cfg, "eviction_dirty_target", &cval)); + cache->eviction_dirty_target = (u_int)cval.val; /* * The eviction thread configuration options include the main eviction * thread and workers. Our implementation splits them out. Adjust for * the difference when parsing the configuration. */ - if ((ret = __wt_config_gets( - session, cfg, "eviction.threads_max", &cval)) == 0) { - WT_ASSERT(session, cval.val > 0); - conn->evict_workers_max = (u_int)cval.val - 1; - } - WT_RET_NOTFOUND_OK(ret); - if ((ret = __wt_config_gets( - session, cfg, "eviction.threads_min", &cval)) == 0) { - WT_ASSERT(session, cval.val > 0); - conn->evict_workers_min = (u_int)cval.val - 1; - } - WT_RET_NOTFOUND_OK(ret); + WT_RET(__wt_config_gets(session, cfg, "eviction.threads_max", &cval)); + WT_ASSERT(session, cval.val > 0); + conn->evict_workers_max = (u_int)cval.val - 1; + + WT_RET(__wt_config_gets(session, cfg, "eviction.threads_min", &cval)); + WT_ASSERT(session, cval.val > 0); + conn->evict_workers_min = (u_int)cval.val - 1; if (conn->evict_workers_min > conn->evict_workers_max) WT_RET_MSG(session, EINVAL, -- cgit v1.2.1 From 9b35bee31185f06a4ce883648fd478bd46893e67 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 10 Sep 2014 17:52:47 -0400 Subject: Merging the wiredtiger_open configuration means we should never see not-found returned when retrieving the verbose configuation. Don't reset the connection's verbose flags until the entire config function is successful. It's not a bug yet, but makes sure an error case introduced later doesn't also introduce the possibility of mixing-and-matching old/new flags. --- src/conn/conn_api.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 3793d30b335..b406a69b9c6 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -1073,6 +1073,7 @@ __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_CONFIG_ITEM cval, sval; WT_CONNECTION_IMPL *conn; WT_DECL_RET; + uint32_t flags; static const struct { const char *name; uint32_t flag; @@ -1103,14 +1104,14 @@ __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) conn = S2C(session); - if ((ret = __wt_config_gets(session, cfg, "verbose", &cval)) != 0) - return (ret == WT_NOTFOUND ? 0 : ret); + WT_RET(__wt_config_gets(session, cfg, "verbose", &cval)); + flags = 0; for (ft = verbtypes; ft->name != NULL; ft++) { if ((ret = __wt_config_subgets( session, &cval, ft->name, &sval)) == 0 && sval.val != 0) { #ifdef HAVE_VERBOSE - FLD_SET(conn->verbose, ft->flag); + LF_SET(ft->flag); #else WT_RET_MSG(session, EINVAL, "Verbose option specified when WiredTiger built " @@ -1118,11 +1119,11 @@ __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) "configure command and rebuild to include support " "for verbose messages"); #endif - } else - FLD_CLR(conn->verbose, ft->flag); - + } WT_RET_NOTFOUND_OK(ret); } + + conn->verbose = flags; return (0); } -- cgit v1.2.1 From 93de0739cf20574897f94e1e198e68efcf9fd69c Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 10 Sep 2014 18:02:20 -0400 Subject: Merging the wiredtiger_open configuration means we should never see not-found returned when retrieving the statistics-logs configuration. Don't leak memory if we're unable to build the list of sources for the statistics-logging. Cleanup a couple more places where we were passing a connection handle instead of a session handle. --- src/conn/conn_open.c | 2 +- src/conn/conn_stat.c | 59 +++++++++++++++++++++++++++++++++------------------- src/include/extern.h | 2 +- 3 files changed, 40 insertions(+), 23 deletions(-) diff --git a/src/conn/conn_open.c b/src/conn/conn_open.c index 78351ce4bd7..956b944ec19 100644 --- a/src/conn/conn_open.c +++ b/src/conn/conn_open.c @@ -102,7 +102,7 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) WT_TRET(__wt_async_destroy(conn)); WT_TRET(__wt_lsm_manager_destroy(conn)); WT_TRET(__wt_checkpoint_server_destroy(conn)); - WT_TRET(__wt_statlog_destroy(conn, 1)); + WT_TRET(__wt_statlog_destroy(session, 1)); WT_TRET(__wt_sweep_destroy(conn)); /* Close open data handles. */ diff --git a/src/conn/conn_stat.c b/src/conn/conn_stat.c index 12b9567b6e8..fbd9b3835b4 100644 --- a/src/conn/conn_stat.c +++ b/src/conn/conn_stat.c @@ -18,6 +18,22 @@ #endif #endif +/* + * __stat_sources_free -- + * Free the array of statistics sources. + */ +static void +__stat_sources_free(WT_SESSION_IMPL *session, char ***sources) +{ + char **p; + + if ((p = (*sources)) != NULL) { + for (; *p != NULL; ++p) + __wt_free(session, *p); + __wt_free(session, *sources); + } +} + /* * __wt_conn_stat_init -- * Initialize the per-connection statistics. @@ -41,8 +57,10 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, int *runp) WT_CONNECTION_IMPL *conn; WT_DECL_RET; int cnt; + char **sources; conn = S2C(session); + sources = NULL; WT_RET(__wt_config_gets(session, cfg, "statistics_log.wait", &cval)); /* Only start the server if wait time is non-zero */ @@ -67,7 +85,7 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, int *runp) ; WT_RET_NOTFOUND_OK(ret); if (cnt != 0) { - WT_RET(__wt_calloc_def(session, cnt + 1, &conn->stat_sources)); + WT_RET(__wt_calloc_def(session, cnt + 1, &sources)); WT_RET(__wt_config_subinit(session, &objectconf, &cval)); for (cnt = 0; (ret = __wt_config_next(&objectconf, &k, &v)) == 0; ++cnt) { @@ -80,24 +98,28 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, int *runp) */ if (!WT_PREFIX_MATCH(k.str, "file:") && !WT_PREFIX_MATCH(k.str, "lsm:")) - WT_RET_MSG(session, EINVAL, + WT_ERR_MSG(session, EINVAL, "statistics_log sources configuration only " "supports objects of type \"file\" or " "\"lsm\""); - WT_RET(__wt_strndup(session, - k.str, k.len, &conn->stat_sources[cnt])); + WT_ERR( + __wt_strndup(session, k.str, k.len, &sources[cnt])); } - WT_RET_NOTFOUND_OK(ret); + WT_ERR_NOTFOUND_OK(ret); + + conn->stat_sources = sources; + sources = NULL; } - WT_RET(__wt_config_gets(session, cfg, "statistics_log.path", &cval)); - WT_RET(__wt_nfilename(session, cval.str, cval.len, &conn->stat_path)); + WT_ERR(__wt_config_gets(session, cfg, "statistics_log.path", &cval)); + WT_ERR(__wt_nfilename(session, cval.str, cval.len, &conn->stat_path)); - WT_RET(__wt_config_gets( + WT_ERR(__wt_config_gets( session, cfg, "statistics_log.timestamp", &cval)); - WT_RET(__wt_strndup(session, cval.str, cval.len, &conn->stat_format)); + WT_ERR(__wt_strndup(session, cval.str, cval.len, &conn->stat_format)); - return (0); +err: __stat_sources_free(session, &sources); + return (ret); } /* @@ -455,9 +477,9 @@ __wt_statlog_create(WT_SESSION_IMPL *session, const char *cfg[]) * configuration changes - but that makes our lives easier. */ if (conn->stat_session != NULL) - WT_RET(__wt_statlog_destroy(conn, 0)); + WT_RET(__wt_statlog_destroy(session, 0)); - WT_RET_NOTFOUND_OK(__statlog_config(session, cfg, &start)); + WT_RET(__statlog_config(session, cfg, &start)); if (start) WT_RET(__statlog_start(conn)); @@ -469,14 +491,13 @@ __wt_statlog_create(WT_SESSION_IMPL *session, const char *cfg[]) * Destroy the statistics server thread. */ int -__wt_statlog_destroy(WT_CONNECTION_IMPL *conn, int is_close) +__wt_statlog_destroy(WT_SESSION_IMPL *session, int is_close) { + WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_SESSION *wt_session; - WT_SESSION_IMPL *session; - char **p; - session = conn->default_session; + conn = S2C(session); F_CLR(conn, WT_CONN_SERVER_STATISTICS); if (conn->stat_tid_set) { @@ -491,11 +512,7 @@ __wt_statlog_destroy(WT_CONNECTION_IMPL *conn, int is_close) WT_TRET(__wt_cond_destroy(session, &conn->stat_cond)); - if ((p = conn->stat_sources) != NULL) { - for (; *p != NULL; ++p) - __wt_free(session, *p); - __wt_free(session, conn->stat_sources); - } + __stat_sources_free(session, &conn->stat_sources); __wt_free(session, conn->stat_path); __wt_free(session, conn->stat_format); diff --git a/src/include/extern.h b/src/include/extern.h index 8208c021d47..1d52262446b 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -650,7 +650,7 @@ extern int __wt_connection_workers(WT_SESSION_IMPL *session, const char *cfg[]); extern void __wt_conn_stat_init(WT_SESSION_IMPL *session); extern int __wt_statlog_log_one(WT_SESSION_IMPL *session); extern int __wt_statlog_create(WT_SESSION_IMPL *session, const char *cfg[]); -extern int __wt_statlog_destroy(WT_CONNECTION_IMPL *conn, int is_close); +extern int __wt_statlog_destroy(WT_SESSION_IMPL *session, int is_close); extern int __wt_sweep_create(WT_CONNECTION_IMPL *conn); extern int __wt_sweep_destroy(WT_CONNECTION_IMPL *conn); extern int __wt_curbackup_open(WT_SESSION_IMPL *session, -- cgit v1.2.1 From 1613c2661598f7bd72f1a5e8e8925339ec312b9a Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 10 Sep 2014 18:07:10 -0400 Subject: Move the LSM manager re-configuration code into the LSM manager file. --- src/conn/conn_api.c | 9 ++------- src/include/extern.h | 1 + src/lsm/lsm_manager.c | 19 +++++++++++++++++++ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index b406a69b9c6..b681876a4b2 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -626,7 +626,6 @@ err: /* static int __conn_reconfigure(WT_CONNECTION *wt_conn, const char *config) { - WT_CONFIG_ITEM cval; WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_SESSION_IMPL *session; @@ -654,13 +653,9 @@ __conn_reconfigure(WT_CONNECTION *wt_conn, const char *config) WT_ERR(__wt_cache_config(session, config_cfg)); WT_ERR(__wt_cache_pool_config(session, config_cfg)); WT_ERR(__wt_checkpoint_server_create(session, config_cfg)); - WT_ERR(__wt_verbose_config(session, config_cfg)); + WT_ERR(__wt_lsm_manager_config(session, config_cfg)); WT_ERR(__wt_statlog_create(session, config_cfg)); - - WT_ERR(__wt_config_gets( - session, config_cfg, "lsm_manager.worker_thread_max", &cval)); - if (cval.val) - conn->lsm_manager.lsm_workers_max = (uint32_t)cval.val; + WT_ERR(__wt_verbose_config(session, config_cfg)); /* Wake up the cache pool server so any changes are noticed. */ if (F_ISSET(conn, WT_CONN_CACHE_POOL)) diff --git a/src/include/extern.h b/src/include/extern.h index 1d52262446b..c5e6a49fdf8 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -930,6 +930,7 @@ extern int __wt_clsm_open(WT_SESSION_IMPL *session, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp); +extern int __wt_lsm_manager_config(WT_SESSION_IMPL *session, const char **cfg); extern int __wt_lsm_manager_start(WT_SESSION_IMPL *session); extern void __wt_lsm_manager_free_work_unit( WT_SESSION_IMPL *session, WT_LSM_WORK_UNIT *entry); diff --git a/src/lsm/lsm_manager.c b/src/lsm/lsm_manager.c index 6f532111905..2ac20b9b92d 100644 --- a/src/lsm/lsm_manager.c +++ b/src/lsm/lsm_manager.c @@ -13,6 +13,25 @@ static int __lsm_manager_worker_setup(WT_SESSION_IMPL *); static void * __lsm_worker_manager(void *); +/* + * __wt_lsm_manager_config -- + * Re-configure the LSM manager. + */ +int +__wt_lsm_manager_config(WT_SESSION_IMPL *session, const char **cfg) +{ + WT_CONNECTION_IMPL *conn; + WT_CONFIG_ITEM cval; + + conn = S2C(session); + + WT_RET(__wt_config_gets( + session, cfg, "lsm_manager.worker_thread_max", &cval)); + if (cval.val) + conn->lsm_manager.lsm_workers_max = (uint32_t)cval.val; + return (0); +} + /* * __wt_lsm_manager_start -- * Start the LSM management infrastructure. Our queues and locks were -- cgit v1.2.1 From 87a90096360c83584e2a3cb8785dc6dbed031b46 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 10 Sep 2014 18:16:51 -0400 Subject: KNF --- src/async/async_api.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/async/async_api.c b/src/async/async_api.c index e7c2c101bb1..2e788b772fd 100644 --- a/src/async/async_api.c +++ b/src/async/async_api.c @@ -201,8 +201,8 @@ __async_config(WT_SESSION_IMPL *session, void __wt_async_stats_update(WT_SESSION_IMPL *session) { - WT_CONNECTION_IMPL *conn; WT_ASYNC *async; + WT_CONNECTION_IMPL *conn; WT_CONNECTION_STATS *stats; conn = S2C(session); -- cgit v1.2.1 From 59f5c4e4376e5b6450e9fb87b11fd270f18569fc Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 10 Sep 2014 18:29:21 -0400 Subject: long line complaint --- test/format/config.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/format/config.h b/test/format/config.h index 4bc1493a681..9852fafabf7 100644 --- a/test/format/config.h +++ b/test/format/config.h @@ -115,7 +115,8 @@ static CONFIG c[] = { C_BOOL, 10, 0, 0, &g.c_compact, NULL }, { "compression", - "type of compression (none | bzip | bzip-raw | lzo | snappy | zlib | zlib-noraw)", + "type of compression " + "(none | bzip | bzip-raw | lzo | snappy | zlib | zlib-noraw)", C_IGNORE|C_STRING, 1, 5, 5, NULL, &g.c_compression }, { "data_extend", -- cgit v1.2.1 From cfab614cbb4d2c9f3443a0289eb47476344fb5a3 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Wed, 10 Sep 2014 19:40:44 -0400 Subject: Wait for ondisk flag to get set on compact. #1200 --- src/lsm/lsm_tree.c | 19 +++++++++++++++---- src/lsm/lsm_work_unit.c | 11 ++++++++++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index 681b194ed9d..5db017514ae 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -1071,12 +1071,23 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) while (F_ISSET(lsm_tree, WT_LSM_TREE_ACTIVE)) { if (flushing && ref && !F_ISSET(lsm_tree, WT_LSM_TREE_COMPACT_FLUSH)) { - flushing = ref = 0; - if (chunk != NULL) { + if (chunk != NULL && + !F_ISSET(chunk, WT_LSM_CHUNK_ONDISK)) { WT_ERR(__wt_verbose(session, WT_VERB_LSM, - "Compact flush complete %s chunk %u", + "Compact flush retry %s chunk %u", name, chunk->id)); - (void)WT_ATOMIC_SUB(chunk->refcnt, 1); + WT_ERR(__wt_lsm_manager_push_entry(session, + WT_LSM_WORK_FLUSH | WT_LSM_WORK_FORCE, + lsm_tree)); + } else { + flushing = ref = 0; + if (chunk != NULL) { + WT_ERR(__wt_verbose(session, + WT_VERB_LSM, + "Compact flush done %s chunk %u", + name, chunk->id)); + (void)WT_ATOMIC_SUB(chunk->refcnt, 1); + } } } /* diff --git a/src/lsm/lsm_work_unit.c b/src/lsm/lsm_work_unit.c index 525fb57b5b3..f3ae9f04219 100644 --- a/src/lsm/lsm_work_unit.c +++ b/src/lsm/lsm_work_unit.c @@ -229,11 +229,20 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, else WT_RET_MSG(session, ret, "discard handle"); } - if (F_ISSET(chunk, WT_LSM_CHUNK_ONDISK)) + if (F_ISSET(chunk, WT_LSM_CHUNK_ONDISK)) { + WT_RET(__wt_verbose(session, WT_VERB_LSM, + "LSM worker %s already on disk", + chunk->uri)); return (0); + } /* Stop if a running transaction needs the chunk. */ __wt_txn_update_oldest(session); + if (chunk->switch_txn == WT_TXN_NONE) + WT_RET(__wt_verbose(session, WT_VERB_LSM, + "LSM ckp txn needs chunk %s: switch %" PRIu64 + " oldest %" PRIu64, chunk->uri, + chunk->switch_txn, S2C(session)->txn_global.oldest_id)); if (chunk->switch_txn == WT_TXN_NONE || !__wt_txn_visible_all(session, chunk->switch_txn)) return (0); -- cgit v1.2.1 From e535a5202a60050f84cc2a0f8f5cc8c7392b9b20 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Thu, 11 Sep 2014 10:30:52 +1000 Subject: Change transaction ID allocation so that if transactions stop, the last ID becomes globally visible. We want post-increment semantics but our atomic primitive is pre-increment. refs #1200 --- src/btree/rec_track.c | 2 +- src/include/txn.i | 21 +++++++++------------ src/lsm/lsm_tree.c | 4 ++-- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/btree/rec_track.c b/src/btree/rec_track.c index a4ef0aaa100..165df9d61e5 100644 --- a/src/btree/rec_track.c +++ b/src/btree/rec_track.c @@ -807,7 +807,7 @@ __wt_ovfl_txnc_add(WT_SESSION_IMPL *session, WT_PAGE *page, txnc->value_offset = WT_PTRDIFF32(p, txnc); txnc->value_size = WT_STORE_SIZE(value_size); memcpy(p, value, value_size); - txnc->current = __wt_txn_current_id(session); + txnc->current = __wt_txn_new_id(session); __wt_cache_page_inmem_incr(session, page, WT_OVFL_SIZE(WT_OVFL_TXNC) + addr_size + value_size); diff --git a/src/include/txn.i b/src/include/txn.i index 3854429f8e4..81559bfe490 100644 --- a/src/include/txn.i +++ b/src/include/txn.i @@ -179,7 +179,7 @@ __wt_txn_read(WT_SESSION_IMPL *session, WT_UPDATE *upd) /* * __wt_txn_autocommit_check -- - * If an auto-commit transaction is required, start one. + * If an auto-commit transaction is required, start one. */ static inline int __wt_txn_autocommit_check(WT_SESSION_IMPL *session) @@ -194,16 +194,6 @@ __wt_txn_autocommit_check(WT_SESSION_IMPL *session) return (0); } -/* - * __wt_txn_current_id -- - * Get the current transaction ID. - */ -static inline uint64_t -__wt_txn_current_id(WT_SESSION_IMPL *session) -{ - return (S2C(session)->txn_global.current); -} - /* * __wt_txn_new_id -- * Allocate a new transaction ID. @@ -211,7 +201,14 @@ __wt_txn_current_id(WT_SESSION_IMPL *session) static inline uint64_t __wt_txn_new_id(WT_SESSION_IMPL *session) { - return WT_ATOMIC_ADD(S2C(session)->txn_global.current, 1); + /* + * We want the global value to lead the allocated values, so that any + * allocated transaction ID eventually becomes globally visible. When + * there are no transactions running, the oldest_id will reach the + * global current ID, so we want post-increment semantics. Our atomic + * add primitive does pre-increment, so adjust the result here. + */ + return WT_ATOMIC_ADD(S2C(session)->txn_global.current, 1) - 1; } /* diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index 5db017514ae..c3082ca4a92 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -733,7 +733,7 @@ __wt_lsm_tree_switch(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) /* Set the switch transaction in the previous chunk, if necessary. */ if (chunk != NULL && chunk->switch_txn == WT_TXN_NONE) - chunk->switch_txn = __wt_txn_current_id(session); + chunk->switch_txn = __wt_txn_new_id(session); /* Update the throttle time. */ __wt_lsm_tree_throttle(session, lsm_tree, 0); @@ -1052,7 +1052,7 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) if (lsm_tree->nchunks > 0 && (chunk = lsm_tree->chunk[lsm_tree->nchunks - 1]) != NULL) { if (chunk->switch_txn == WT_TXN_NONE) - chunk->switch_txn = __wt_txn_current_id(session); + chunk->switch_txn = __wt_txn_new_id(session); (void)WT_ATOMIC_ADD(chunk->refcnt, 1); ref = 1; } -- cgit v1.2.1 From 4a901b42ab03a708de27c31a0bc6c9ad582191aa Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Thu, 11 Sep 2014 10:31:22 +1000 Subject: On error in wiredtiger_open, don't try to free buffers after closing the connection handle. --- src/conn/conn_api.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 0b210627df5..1865fb5c047 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -1415,11 +1415,11 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, STATIC_ASSERT(offsetof(WT_CONNECTION_IMPL, iface) == 0); *wt_connp = &conn->iface; -err: if (ret != 0 && conn != NULL) - WT_TRET(__wt_connection_close(conn)); - - __wt_buf_free(session, &cbbuf); +err: __wt_buf_free(session, &cbbuf); __wt_buf_free(session, &cubuf); + if (ret != 0 && conn != NULL) + WT_TRET(__wt_connection_close(conn)); + return (ret); } -- cgit v1.2.1 From c63a2ebf84982758b9183e50b6f778ac0cff70d0 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 10 Sep 2014 20:38:44 -0400 Subject: Clean up the first test in __wt_cache_config (not-found is no longer an option). --- src/conn/conn_cache.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/conn/conn_cache.c b/src/conn/conn_cache.c index fabff374319..2c21e5f1fbc 100644 --- a/src/conn/conn_cache.c +++ b/src/conn/conn_cache.c @@ -17,7 +17,6 @@ __wt_cache_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_CACHE *cache; WT_CONFIG_ITEM cval; WT_CONNECTION_IMPL *conn; - WT_DECL_RET; conn = S2C(session); cache = conn->cache; @@ -26,18 +25,17 @@ __wt_cache_config(WT_SESSION_IMPL *session, const char *cfg[]) * If not using a shared cache configure the cache size, otherwise * check for a reserved size. */ - if (!F_ISSET(conn, WT_CONN_CACHE_POOL) && - (ret = __wt_config_gets(session, cfg, "cache_size", &cval)) == 0) + if (!F_ISSET(conn, WT_CONN_CACHE_POOL)) { + WT_RET(__wt_config_gets(session, cfg, "cache_size", &cval)); conn->cache_size = (uint64_t)cval.val; - - if (F_ISSET(conn, WT_CONN_CACHE_POOL) && - (ret = __wt_config_gets(session, cfg, - "shared_cache.reserve", &cval)) == 0 && cval.val != 0) - cache->cp_reserved = (uint64_t)cval.val; - else if ((ret = __wt_config_gets(session, cfg, - "shared_cache.chunk", &cval)) == 0) + } else { + WT_RET(__wt_config_gets( + session, cfg, "shared_cache.reserve", &cval)); + if (cval.val == 0) + WT_RET(__wt_config_gets( + session, cfg, "shared_cache.chunk", &cval)); cache->cp_reserved = (uint64_t)cval.val; - WT_RET_NOTFOUND_OK(ret); + } WT_RET(__wt_config_gets(session, cfg, "eviction_target", &cval)); cache->eviction_target = (u_int)cval.val; -- cgit v1.2.1 From cf8454b55bfdb177a53660297ca2be511a29d7c1 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Wed, 10 Sep 2014 20:51:32 -0400 Subject: Serialize re-configuration, everything is going to go pear-shaped if we reconfigure in multiple threads at the same time. --- src/conn/conn_api.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index b681876a4b2..28509a71be6 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -636,6 +636,9 @@ __conn_reconfigure(WT_CONNECTION *wt_conn, const char *config) CONNECTION_API_CALL(conn, session, reconfigure, config, cfg); WT_UNUSED(cfg); + /* Serialize reconfiguration. */ + __wt_spin_lock(session, &conn->api_lock); + /* * The configuration argument has been checked for validity, replace the * previous connection configuration. @@ -666,7 +669,9 @@ __conn_reconfigure(WT_CONNECTION *wt_conn, const char *config) __wt_free(session, conn->cfg); conn->cfg = p; -err: API_END_RET(session, ret); +err: __wt_spin_unlock(session, &conn->api_lock); + + API_END_RET(session, ret); } /* -- cgit v1.2.1 From ba56917bcedaba6fdc3d697ce740fc5654480f26 Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Thu, 11 Sep 2014 11:47:20 +1000 Subject: Add a new lock to serialize reconfigure. Don't overload the API lock, since reconfigure can acquire the api lock while doing a reconfigure. --- src/conn/conn_api.c | 4 ++-- src/conn/conn_handle.c | 1 + src/include/connection.h | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 28509a71be6..019463f44b7 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -637,7 +637,7 @@ __conn_reconfigure(WT_CONNECTION *wt_conn, const char *config) WT_UNUSED(cfg); /* Serialize reconfiguration. */ - __wt_spin_lock(session, &conn->api_lock); + __wt_spin_lock(session, &conn->reconfig_lock); /* * The configuration argument has been checked for validity, replace the @@ -669,7 +669,7 @@ __conn_reconfigure(WT_CONNECTION *wt_conn, const char *config) __wt_free(session, conn->cfg); conn->cfg = p; -err: __wt_spin_unlock(session, &conn->api_lock); +err: __wt_spin_unlock(session, &conn->reconfig_lock); API_END_RET(session, ret); } diff --git a/src/conn/conn_handle.c b/src/conn/conn_handle.c index 03c22b04e30..8deae14ce0c 100644 --- a/src/conn/conn_handle.c +++ b/src/conn/conn_handle.c @@ -44,6 +44,7 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn) WT_RET(__wt_spin_init(session, &conn->checkpoint_lock, "checkpoint")); WT_RET(__wt_spin_init(session, &conn->dhandle_lock, "data handle")); WT_RET(__wt_spin_init(session, &conn->fh_lock, "file list")); + WT_RET(__wt_spin_init(session, &conn->reconfig_lock, "reconfigure")); WT_RET(__wt_spin_init(session, &conn->hot_backup_lock, "hot backup")); WT_RET(__wt_spin_init(session, &conn->schema_lock, "schema")); WT_RET(__wt_calloc_def(session, WT_PAGE_LOCKS(conn), &conn->page_lock)); diff --git a/src/include/connection.h b/src/include/connection.h index 1f1f8be88ea..97314759342 100644 --- a/src/include/connection.h +++ b/src/include/connection.h @@ -76,6 +76,7 @@ struct __wt_connection_impl { WT_SPINLOCK checkpoint_lock; /* Checkpoint spinlock */ WT_SPINLOCK fh_lock; /* File handle queue spinlock */ WT_SPINLOCK schema_lock; /* Schema operation spinlock */ + WT_SPINLOCK reconfig_lock; /* Single thread reconfigure */ /* * We distribute the btree page locks across a set of spin locks; it -- cgit v1.2.1 From 06d9d5ca346f9cb075773224dec93fe296ccdcae Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Thu, 11 Sep 2014 12:40:50 +1000 Subject: Fixup cache pool reconfigure now that options are being folded together. Try to make the code more obvious as well. --- src/conn/conn_api.c | 5 ---- src/conn/conn_cache_pool.c | 65 ++++++++++++++++++++++++++++++---------------- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 019463f44b7..81d06881eb5 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -660,11 +660,6 @@ __conn_reconfigure(WT_CONNECTION *wt_conn, const char *config) WT_ERR(__wt_statlog_create(session, config_cfg)); WT_ERR(__wt_verbose_config(session, config_cfg)); - /* Wake up the cache pool server so any changes are noticed. */ - if (F_ISSET(conn, WT_CONN_CACHE_POOL)) - WT_ERR(__wt_cond_signal( - session, __wt_process.cache_pool->cache_pool_cond)); - WT_ERR(__wt_config_merge(session, config_cfg, &p)); __wt_free(session, conn->cfg); conn->cfg = p; diff --git a/src/conn/conn_cache_pool.c b/src/conn/conn_cache_pool.c index 57d343d0d46..5148229db51 100644 --- a/src/conn/conn_cache_pool.c +++ b/src/conn/conn_cache_pool.c @@ -102,6 +102,7 @@ __wt_cache_pool_config(WT_SESSION_IMPL *session, const char **cfg) pool_name); cp = __wt_process.cache_pool; + /* * The cache pool requires a reference count to avoid a race between * configuration/open and destroy. @@ -110,39 +111,54 @@ __wt_cache_pool_config(WT_SESSION_IMPL *session, const char **cfg) ++cp->refs; /* - * Retrieve the pool configuration options. The values are optional if - * we are re-configuring. + * Cache pool configurations are optional when not creating. If + * values aren't being changed, retrieve the current value so that + * validation of settings works. */ - ret = __wt_config_gets(session, cfg, "shared_cache.size", &cval); - if (reconfiguring && ret == WT_NOTFOUND) - /* Not being changed; use the old value. */ - size = cp->size; - else { - WT_ERR(ret); + if (!created) { + if (__wt_config_gets(session, &cfg[1], + "shared_cache.size", &cval) == 0 && cval.val != 0) + size = (uint64_t)cval.val; + else + size = cp->size; + if (__wt_config_gets(session, &cfg[1], + "shared_cache.chunk", &cval) == 0 && cval.val != 0) + chunk = (uint64_t)cval.val; + else + chunk = cp->chunk; + } else { + /* + * The only time shared cache configuration uses default + * values is when we are creating the pool. + */ + WT_ERR(__wt_config_gets( + session, cfg, "shared_cache.size", &cval)); + WT_ASSERT(session, cval.val != 0); size = (uint64_t)cval.val; - } - ret = __wt_config_gets(session, cfg, "shared_cache.chunk", &cval); - if (reconfiguring && ret == WT_NOTFOUND) - /* Not being changed; use the old value. */ - chunk = cp->chunk; - else { - WT_ERR(ret); + WT_ERR(__wt_config_gets( + session, cfg, "shared_cache.chunk", &cval)); + WT_ASSERT(session, cval.val != 0); chunk = (uint64_t)cval.val; } + /* * Retrieve the reserve size here for validation of configuration. * Don't save it yet since the connections cache is not created if * we are opening. Cache configuration is responsible for saving the * setting. + * The different conditions when reserved size are set are: + * - It's part of the users configuration - use that value. + * - We are reconfiguring - keep the previous value. + * - We are joining a cache pool for the first time (including + * creating the pool) - use the chunk size; that's the default. */ - ret = __wt_config_gets(session, cfg, "shared_cache.reserve", &cval); - if (reconfiguring && ret == WT_NOTFOUND) - /* It is safe to access the cache during reconfigure. */ - reserve = conn->cache->cp_reserved; - else { - WT_ERR(ret); + if (__wt_config_gets(session, &cfg[1], + "shared_cache.reserve", &cval) == 0 && cval.val != 0) reserve = (uint64_t)cval.val; - } + else if (reconfiguring) + reserve = conn->cache->cp_reserved; + else + reserve = chunk; /* * Validate that size and reserve values don't cause the cache @@ -163,6 +179,11 @@ __wt_cache_pool_config(WT_SESSION_IMPL *session, const char **cfg) cp->size = size; cp->chunk = chunk; + /* Wake up the cache pool server so any changes are noticed. */ + if (reconfiguring) + WT_ERR(__wt_cond_signal( + session, __wt_process.cache_pool->cache_pool_cond)); + WT_ERR(__wt_verbose(session, WT_VERB_SHARED_CACHE, "Configured cache pool %s. Size: %" PRIu64 ", chunk size: %" PRIu64, cp->name, cp->size, cp->chunk)); -- cgit v1.2.1 From c97794bc9e0a9a42f2b25b402424ce9a6a63feea Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 11 Sep 2014 09:41:13 -0400 Subject: Add a spin_destroy call for the new reconfig_lock, sort the lists of locks. --- src/conn/conn_handle.c | 3 ++- src/include/connection.h | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/conn/conn_handle.c b/src/conn/conn_handle.c index 8deae14ce0c..e4f0a6ddd73 100644 --- a/src/conn/conn_handle.c +++ b/src/conn/conn_handle.c @@ -44,8 +44,8 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn) WT_RET(__wt_spin_init(session, &conn->checkpoint_lock, "checkpoint")); WT_RET(__wt_spin_init(session, &conn->dhandle_lock, "data handle")); WT_RET(__wt_spin_init(session, &conn->fh_lock, "file list")); - WT_RET(__wt_spin_init(session, &conn->reconfig_lock, "reconfigure")); WT_RET(__wt_spin_init(session, &conn->hot_backup_lock, "hot backup")); + WT_RET(__wt_spin_init(session, &conn->reconfig_lock, "reconfigure")); WT_RET(__wt_spin_init(session, &conn->schema_lock, "schema")); WT_RET(__wt_calloc_def(session, WT_PAGE_LOCKS(conn), &conn->page_lock)); for (i = 0; i < WT_PAGE_LOCKS(conn); ++i) @@ -125,6 +125,7 @@ __wt_connection_destroy(WT_CONNECTION_IMPL *conn) __wt_spin_destroy(session, &conn->dhandle_lock); __wt_spin_destroy(session, &conn->fh_lock); __wt_spin_destroy(session, &conn->hot_backup_lock); + __wt_spin_destroy(session, &conn->reconfig_lock); __wt_spin_destroy(session, &conn->schema_lock); for (i = 0; i < WT_PAGE_LOCKS(conn); ++i) __wt_spin_destroy(session, &conn->page_lock[i]); diff --git a/src/include/connection.h b/src/include/connection.h index 97314759342..9af23f95cbf 100644 --- a/src/include/connection.h +++ b/src/include/connection.h @@ -75,8 +75,8 @@ struct __wt_connection_impl { WT_SPINLOCK api_lock; /* Connection API spinlock */ WT_SPINLOCK checkpoint_lock; /* Checkpoint spinlock */ WT_SPINLOCK fh_lock; /* File handle queue spinlock */ - WT_SPINLOCK schema_lock; /* Schema operation spinlock */ WT_SPINLOCK reconfig_lock; /* Single thread reconfigure */ + WT_SPINLOCK schema_lock; /* Schema operation spinlock */ /* * We distribute the btree page locks across a set of spin locks; it -- cgit v1.2.1 From 60751aaba624cf364acc1a45f4007f2b031135d3 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 11 Sep 2014 10:05:45 -0400 Subject: LSM tombstones are no longer empty values, remove that caveat from the LSM page. --- src/docs/lsm.dox | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/docs/lsm.dox b/src/docs/lsm.dox index 0313862afdf..d5439d6209e 100644 --- a/src/docs/lsm.dox +++ b/src/docs/lsm.dox @@ -107,12 +107,6 @@ there are chunks in the tree for each cursor that is open on the LSM tree. The number of hazard pointers is configured with the \c "hazard_max" configuration key to ::wiredtiger_open. -@subsection lsm_tombstones Empty values - -Internally, WiredTiger's LSM trees use an empty value to represent a -record that has been removed (also known as a "tombstone"). For this -reason, applications cannot store records in LSM trees with empty values. - @subsection lsm_checkpoints Named checkpoints Named checkpoints are not supported on LSM trees, and cursors cannot be opened -- cgit v1.2.1 From 539f01b310867a5dfb7d4056a305fe11ad57f423 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 11 Sep 2014 10:08:43 -0400 Subject: I don't expect to fix named checkpoints in LSM, tombstones were the caveat we planned to fix. --- src/docs/lsm.dox | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/docs/lsm.dox b/src/docs/lsm.dox index d5439d6209e..b71fccd7151 100644 --- a/src/docs/lsm.dox +++ b/src/docs/lsm.dox @@ -109,9 +109,8 @@ configuration key to ::wiredtiger_open. @subsection lsm_checkpoints Named checkpoints -Named checkpoints are not supported on LSM trees, and cursors cannot be opened -with a non-empty \c "checkpoint" configuration. - -We intend to address these limitations in future releases. +Named checkpoints are not supported on LSM trees, and cursors cannot be +opened with a non-empty \c "checkpoint" configuration (that is, only the +most recent standard checkpoint can be read). */ -- cgit v1.2.1 From 0fa396cb4ad1d2d8fdf8f4d2fd9eb9907a1c3a46 Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Thu, 11 Sep 2014 10:40:10 -0400 Subject: Add a section on choosing a storage option. --- src/docs/file-formats.dox | 31 ++++++++++++++++++++++++++++--- src/docs/programming.dox | 2 +- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/docs/file-formats.dox b/src/docs/file-formats.dox index 46865da4811..bc747433172 100644 --- a/src/docs/file-formats.dox +++ b/src/docs/file-formats.dox @@ -3,7 +3,8 @@ @section file_formats_formats File formats WiredTiger supports two underlying file formats: row-store and -column-store, both are key/value stores. +column-store, where both are B+tree implementations of key/value stores. +WiredTiger also supports @ref lsm, implemented as a tree of B+trees. In a row-store, both keys and data are variable-length byte strings. In a column-store, keys are 64-bit record numbers (key_format type 'r'), @@ -28,14 +29,38 @@ deleting a value is the same as storing a value of 0. For the same reason, storing a value of 0 will cause cursor scans to skip the record. WiredTiger does not support duplicate data items: there can be only a -single value for any given key, and applications are responsible for -creating unique key/value pairs. +single value associated with any given key, and applications are +responsible for creating unique key/value pairs. WiredTiger allocates space from the underlying files in block units. The minimum file allocation unit WiredTiger supports is 512B and the maximum file allocation unit is 512MB. File block offsets are 64-bit (meaning the maximum file size is very, very large). +@section file_formats_choice Choosing a file format + +The row-store format is the default choice for most applications. When +the primary key is a record number, there are advantages to storing +columns in separate files, or the underlying data is a set of bits, +column-store format may be a better choice. + +Both row- and column-store formats can maintain high volumes of writes, +but for data sets requiring sustained, extreme write throughput, @ref +lsm are usually a better choice. For applications that do not require +extreme write throughput, row- or column-store is likely to be a better +choice because the read throughput is better than with LSM trees (an +effect that becomes more pronounced as additional read threads are added). + +Applications with complex schemas may also benefit from using multiple +storage formats, that is, using a combination of different formats in +the database, and even in individual tables (for example, a sparse, wide +table configured with a column-store primary, where indexes are stored +in an LSM tree). + +Finally, as WiredTiger makes it easy to switch back-and-forth between +storage configurations, it's usually worthwhile benchmarking possible +configurations when there is any question. + @section file_formats_compression File formats and compression Row-stores support four types of compression: key prefix compression, diff --git a/src/docs/programming.dox b/src/docs/programming.dox index 59eeab7705d..4add19c833b 100644 --- a/src/docs/programming.dox +++ b/src/docs/programming.dox @@ -18,8 +18,8 @@ each of which is ordered by one or more columns.

Storage options

- @subpage schema -- @subpage lsm - @subpage file_formats +- @subpage lsm - @subpage compression

Programming notes

-- cgit v1.2.1 From d3984707e53bf1037b455d9243d2177645e3ba76 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 11 Sep 2014 11:08:13 -0400 Subject: Refactor to only parse async config once. #1172 --- src/async/async_api.c | 43 ++++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/src/async/async_api.c b/src/async/async_api.c index 2e788b772fd..23a176653c7 100644 --- a/src/async/async_api.c +++ b/src/async/async_api.c @@ -216,27 +216,18 @@ __wt_async_stats_update(WT_SESSION_IMPL *session) } /* - * __wt_async_create -- - * Start the async subsystem and worker threads. + * __async_start -- + * Start the async subsystem. All configuration processing has + * already been done by the caller. */ -int -__wt_async_create(WT_SESSION_IMPL *session, const char *cfg[]) +static int +__async_start(WT_SESSION_IMPL *session) { WT_ASYNC *async; WT_CONNECTION_IMPL *conn; - int run; uint32_t i; conn = S2C(session); - - /* Handle configuration. */ - run = 0; - WT_RET(__async_config(session, conn, cfg, &run)); - - /* If async is not configured, we're done. */ - if (!run) - return (0); - conn->async_cfg = 1; /* * Async is on, allocate the WT_ASYNC structure and initialize the ops. @@ -275,6 +266,28 @@ __wt_async_create(WT_SESSION_IMPL *session, const char *cfg[]) return (0); } +/* + * __wt_async_create -- + * Start the async subsystem and worker threads. + */ +int +__wt_async_create(WT_SESSION_IMPL *session, const char *cfg[]) +{ + WT_CONNECTION_IMPL *conn; + int run; + + conn = S2C(session); + + /* Handle configuration. */ + run = 0; + WT_RET(__async_config(session, conn, cfg, &run)); + + /* If async is not configured, we're done. */ + if (!run) + return (0); + return (__async_start(session)); +} + /* * __wt_async_reconfig -- * Start the async subsystem and worker threads. @@ -329,7 +342,7 @@ __wt_async_reconfig(WT_SESSION_IMPL *session, const char *cfg[]) return (ret); } else if (conn->async_cfg == 0 && run) /* Case 2 */ - return (__wt_async_create(session, cfg)); + return (__async_start(session)); else if (conn->async_cfg == 0) /* Case 3 */ return (0); -- cgit v1.2.1 From 13549c7ab002533a3c79ea546cc1f3571e2f037f Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 11 Sep 2014 11:46:44 -0400 Subject: Split flushing versus compacting phases of compact. Cleanup. #1200 --- src/lsm/lsm_merge.c | 22 ++++++++++++++-------- src/lsm/lsm_tree.c | 15 +++++++++------ src/lsm/lsm_work_unit.c | 11 +++++------ 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/src/lsm/lsm_merge.c b/src/lsm/lsm_merge.c index 8de7d1350df..04e0bf19254 100644 --- a/src/lsm/lsm_merge.c +++ b/src/lsm/lsm_merge.c @@ -251,16 +251,22 @@ __wt_lsm_merge( /* Allocate an ID for the merge. */ dest_id = WT_ATOMIC_ADD(lsm_tree->last, 1); - WT_RET(__wt_verbose(session, WT_VERB_LSM, - "Merging %s chunks %u-%u into %u (%" PRIu64 " records)" - ", generation %" PRIu32, - lsm_tree->name, - start_chunk, end_chunk, dest_id, record_count, generation)); - if (WT_VERBOSE_ISSET(session, WT_VERB_LSM)) + /* + * We only want to do the chunk loop if we're running + * with verbose, we wrap these statements in the conditional. + * Avoid that in the normal path. + */ + if (WT_VERBOSE_ISSET(session, WT_VERB_LSM)) { + WT_RET(__wt_verbose(session, WT_VERB_LSM, + "Merging %s chunks %u-%u into %u (%" PRIu64 " records)" + ", generation %" PRIu32, + lsm_tree->name, + start_chunk, end_chunk, dest_id, record_count, generation)); for (verb = start_chunk; verb <= end_chunk; verb++) WT_RET(__wt_verbose(session, WT_VERB_LSM, - "Chunk[%u] id %u", - verb, lsm_tree->chunk[verb]->id)); + "%s: Chunk[%u] id %u", + lsm_tree->name, verb, lsm_tree->chunk[verb]->id)); + } WT_RET(__wt_calloc_def(session, 1, &chunk)); chunk->id = dest_id; diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index c3082ca4a92..a54c4d227a7 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -1040,8 +1040,8 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) if (F_ISSET(lsm_tree, WT_LSM_TREE_COMPACTING)) goto err; - compacting = flushing = 1; - F_SET(lsm_tree, WT_LSM_TREE_COMPACT_FLUSH | WT_LSM_TREE_COMPACTING); + flushing = 1; + F_SET(lsm_tree, WT_LSM_TREE_COMPACT_FLUSH); /* * Set the switch transaction on the current chunk, if it @@ -1069,25 +1069,28 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) /* Wait for the work unit queues to drain. */ while (F_ISSET(lsm_tree, WT_LSM_TREE_ACTIVE)) { - if (flushing && ref && - !F_ISSET(lsm_tree, WT_LSM_TREE_COMPACT_FLUSH)) { + if (flushing && !F_ISSET(lsm_tree, WT_LSM_TREE_COMPACT_FLUSH)) { if (chunk != NULL && !F_ISSET(chunk, WT_LSM_CHUNK_ONDISK)) { WT_ERR(__wt_verbose(session, WT_VERB_LSM, "Compact flush retry %s chunk %u", name, chunk->id)); + F_SET(lsm_tree, WT_LSM_TREE_COMPACT_FLUSH); WT_ERR(__wt_lsm_manager_push_entry(session, WT_LSM_WORK_FLUSH | WT_LSM_WORK_FORCE, lsm_tree)); } else { - flushing = ref = 0; - if (chunk != NULL) { + if (ref) { + WT_ASSERT(session, chunk != NULL); WT_ERR(__wt_verbose(session, WT_VERB_LSM, "Compact flush done %s chunk %u", name, chunk->id)); (void)WT_ATOMIC_SUB(chunk->refcnt, 1); } + flushing = ref = 0; + compacting = 1; + F_SET(lsm_tree, WT_LSM_TREE_COMPACTING); } } /* diff --git a/src/lsm/lsm_work_unit.c b/src/lsm/lsm_work_unit.c index f3ae9f04219..a34bf41969c 100644 --- a/src/lsm/lsm_work_unit.c +++ b/src/lsm/lsm_work_unit.c @@ -238,14 +238,13 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, /* Stop if a running transaction needs the chunk. */ __wt_txn_update_oldest(session); - if (chunk->switch_txn == WT_TXN_NONE) - WT_RET(__wt_verbose(session, WT_VERB_LSM, - "LSM ckp txn needs chunk %s: switch %" PRIu64 - " oldest %" PRIu64, chunk->uri, - chunk->switch_txn, S2C(session)->txn_global.oldest_id)); if (chunk->switch_txn == WT_TXN_NONE || - !__wt_txn_visible_all(session, chunk->switch_txn)) + !__wt_txn_visible_all(session, chunk->switch_txn)) { + WT_RET(__wt_verbose(session, WT_VERB_LSM, + "LSM worker %s: running transaction, return", + chunk->uri)); return (0); + } WT_RET(__wt_verbose(session, WT_VERB_LSM, "LSM worker flushing %s", chunk->uri)); -- cgit v1.2.1 From 7179a227e579b49d91608e53c2df4a0d2a0a3030 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 11 Sep 2014 12:35:29 -0400 Subject: Comments and cleanup. #1200 --- src/lsm/lsm_merge.c | 6 +++--- src/lsm/lsm_tree.c | 52 +++++++++++++++++++++++++++++++++++++------------ src/lsm/lsm_work_unit.c | 6 +++++- src/lsm/lsm_worker.c | 7 +++---- 4 files changed, 51 insertions(+), 20 deletions(-) diff --git a/src/lsm/lsm_merge.c b/src/lsm/lsm_merge.c index 04e0bf19254..363fe77b93e 100644 --- a/src/lsm/lsm_merge.c +++ b/src/lsm/lsm_merge.c @@ -252,9 +252,9 @@ __wt_lsm_merge( dest_id = WT_ATOMIC_ADD(lsm_tree->last, 1); /* - * We only want to do the chunk loop if we're running - * with verbose, we wrap these statements in the conditional. - * Avoid that in the normal path. + * We only want to do the chunk loop if we're running with verbose, + * so we wrap these statements in the conditional. Avoid the loop + * in the normal path. */ if (WT_VERBOSE_ISSET(session, WT_VERB_LSM)) { WT_RET(__wt_verbose(session, WT_VERB_LSM, diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index a54c4d227a7..4eec6a9b559 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -1028,6 +1028,19 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) WT_ERR(__wt_seconds(session, &begin)); + /* + * Compacting has two distinct phases. + * 1. All in-memory chunks up to and including the current + * current chunk must be flushed. Normally, the flush code + * does not flush the last, in-use chunk, so we set a force + * flag to include that last chunk. We monitor the state of the + * last chunk and periodically push another forced flush work + * unit until it is complete. + * 2. After all flushing is done, we move onto the merging + * phase for compaction. Again, we monitor the state and + * continue to push merge work units until all merging is done. + */ + /* Lock the tree: single-thread compaction. */ WT_ERR(__wt_lsm_tree_lock(session, lsm_tree, 1)); locked = 1; @@ -1036,8 +1049,9 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) lsm_tree->merge_throttle = 0; lsm_tree->merge_aggressiveness = 0; - /* If another thread started compacting this tree, we're done. */ - if (F_ISSET(lsm_tree, WT_LSM_TREE_COMPACTING)) + /* If another thread started a compact on this tree, we're done. */ + if (F_ISSET(lsm_tree, + WT_LSM_TREE_COMPACT_FLUSH | WT_LSM_TREE_COMPACTING)) goto err; flushing = 1; @@ -1053,6 +1067,10 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) (chunk = lsm_tree->chunk[lsm_tree->nchunks - 1]) != NULL) { if (chunk->switch_txn == WT_TXN_NONE) chunk->switch_txn = __wt_txn_new_id(session); + /* + * If we have a chunk, we want to look for it to be on-disk. + * So we need to add a reference to keep it available. + */ (void)WT_ATOMIC_ADD(chunk->refcnt, 1); ref = 1; } @@ -1060,15 +1078,23 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) locked = 0; WT_ERR(__wt_lsm_tree_unlock(session, lsm_tree)); - WT_ERR(__wt_verbose(session, WT_VERB_LSM, - "Compact force flush %s flags 0x%" PRIx32 " chunk %u flags 0x%" - PRIx32, name, lsm_tree->flags, chunk->id, chunk->flags)); + if (chunk != NULL) + WT_ERR(__wt_verbose(session, WT_VERB_LSM, + "Compact force flush %s flags 0x%" PRIx32 + " chunk %u flags 0x%" + PRIx32, name, lsm_tree->flags, chunk->id, chunk->flags)); /* Make sure the in-memory chunk gets flushed but not switched. */ WT_ERR(__wt_lsm_manager_push_entry(session, WT_LSM_WORK_FLUSH | WT_LSM_WORK_FORCE, lsm_tree)); /* Wait for the work unit queues to drain. */ while (F_ISSET(lsm_tree, WT_LSM_TREE_ACTIVE)) { + /* + * The flush flag is cleared when the the chunk has been + * flushed. Continue to push forced flushes until the + * chunk is on disk. Once it is on disk move to the compacting + * phase. + */ if (flushing && !F_ISSET(lsm_tree, WT_LSM_TREE_COMPACT_FLUSH)) { if (chunk != NULL && !F_ISSET(chunk, WT_LSM_CHUNK_ONDISK)) { @@ -1099,11 +1125,11 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) * stopping otherwise we might not do merges that would * span chunks with different generations. */ - if (!F_ISSET(lsm_tree, WT_LSM_TREE_COMPACTING)) { + if (compacting && !F_ISSET(lsm_tree, WT_LSM_TREE_COMPACTING)) { if (lsm_tree->merge_aggressiveness < 10) { F_SET(lsm_tree, WT_LSM_TREE_COMPACTING); lsm_tree->merge_aggressiveness = 10; - } else if (!flushing) + } else break; } __wt_sleep(1, 0); @@ -1117,7 +1143,7 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) * done. If we are pushing merges, make sure they are * aggressive, to avoid duplicating effort. */ - if (!flushing) + if (compacting) #define COMPACT_PARALLEL_MERGES 5 for (i = lsm_tree->queue_ref; i < COMPACT_PARALLEL_MERGES; i++) { @@ -1126,9 +1152,8 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) session, WT_LSM_WORK_MERGE, lsm_tree)); } } -err: if (locked) - WT_ERR(__wt_lsm_tree_unlock(session, lsm_tree)); - /* Ensure the compacting flag is cleared if we set it. */ +err: + /* Ensure anything we set is cleared. */ if (flushing) F_CLR(lsm_tree, WT_LSM_TREE_COMPACT_FLUSH); if (ref) @@ -1136,8 +1161,11 @@ err: if (locked) if (compacting) { F_CLR(lsm_tree, WT_LSM_TREE_COMPACTING); lsm_tree->merge_aggressiveness = 0; + if (locked) + WT_ERR(__wt_lsm_tree_unlock(session, lsm_tree)); } - WT_ERR(__wt_verbose(session, WT_VERB_LSM, "Compact complete %s", name)); + WT_ERR(__wt_verbose(session, WT_VERB_LSM, + "Compact %s complete, return %d", name, ret)); __wt_lsm_tree_release(session, lsm_tree); return (ret); diff --git a/src/lsm/lsm_work_unit.c b/src/lsm/lsm_work_unit.c index a34bf41969c..eb791f98f5f 100644 --- a/src/lsm/lsm_work_unit.c +++ b/src/lsm/lsm_work_unit.c @@ -81,7 +81,7 @@ __wt_lsm_get_chunk_to_flush(WT_SESSION_IMPL *session, /* * Normally we don't want to force out the last chunk. But if we're - * doing a forced flush, likely from a compact call, then we do want + * doing a forced flush, likely from a compact call, then we want * to include the final chunk. */ end = force ? lsm_tree->nchunks : lsm_tree->nchunks - 1; @@ -93,6 +93,10 @@ __wt_lsm_get_chunk_to_flush(WT_SESSION_IMPL *session, force ? " w/ force" : "", i, end - 1, lsm_tree->chunk[i]->uri)); *chunkp = lsm_tree->chunk[i]; + /* + * Let the caller know if this is the last chunk we + * could have selected or an earlier one. + */ if (i == end - 1) *last = 1; break; diff --git a/src/lsm/lsm_worker.c b/src/lsm/lsm_worker.c index 68ebe34fe91..1f2b76ba720 100644 --- a/src/lsm/lsm_worker.c +++ b/src/lsm/lsm_worker.c @@ -47,13 +47,12 @@ __lsm_worker_general_op( if ((entry->flags & WT_LSM_WORK_MASK) == WT_LSM_WORK_FLUSH) { force = F_ISSET(entry, WT_LSM_WORK_FORCE); F_CLR(entry, WT_LSM_WORK_FORCE); - /* - * If this is a force flush, we want to force out all - * possible chunks, not just the first one we find. - */ last = 0; WT_ERR(__wt_lsm_get_chunk_to_flush(session, entry->lsm_tree, force, &last, &chunk)); + /* + * If we got a chunk to flush, checkpoint it. + */ if (chunk != NULL) { WT_ERR(__wt_verbose(session, WT_VERB_LSM, "Flush%s%s chunk %d %s", -- cgit v1.2.1 From 5e2285ec4ad0ab2f2e462263c1cd14322891ca19 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 11 Sep 2014 15:41:39 -0400 Subject: Restore load time to floating point for better granularity. Restore runner script to expect floating point for load time. --- bench/wtperf/runners/wtperf_run.sh | 26 ++++++++++++++++++++------ bench/wtperf/wtperf.c | 15 +++++++-------- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/bench/wtperf/runners/wtperf_run.sh b/bench/wtperf/runners/wtperf_run.sh index 16470f35dfa..3296a4072b5 100755 --- a/bench/wtperf/runners/wtperf_run.sh +++ b/bench/wtperf/runners/wtperf_run.sh @@ -79,7 +79,7 @@ while test "$run" -le "$runmax"; do if test "$?" -ne "0"; then exit 1 fi - # Load uses different text. Handle separately. + # Load is always using floating point, so handle separately l=`grep "^Load time:" ./WT_TEST/test.stat` if test "$?" -eq "0"; then load=`echo $l | cut -d ' ' -f 3` @@ -87,7 +87,7 @@ while test "$run" -le "$runmax"; do load=0 fi cur[$loadindex]=$load - sum[$loadindex]=`expr $load + ${sum[$loadindex]}` + sum[$loadindex]=`echo "${sum[$loadindex]} + $load" | bc` echo "cur ${cur[$loadindex]} sum ${sum[$loadindex]}" >> $outfile for i in ${!ops[*]}; do l=`grep "Executed.*${ops[$i]} operations" ./WT_TEST/test.stat` @@ -109,8 +109,17 @@ while test "$run" -le "$runmax"; do done else for i in ${!cur[*]}; do - min[$i]=$(getval $getmin ${cur[$i]} ${min[$i]}) - max[$i]=$(getval $getmax ${cur[$i]} ${max[$i]}) + if test "$i" -eq "$loadindex"; then + if (($(bc <<< "${cur[$i]} < ${min[$i]}") )); then + min[$i]=${cur[$i]} + fi + if (($(bc <<< "${cur[$i]} > ${max[$i]}") )); then + max[$i]=${cur[$i]} + fi + else + min[$i]=$(getval $getmin ${cur[$i]} ${min[$i]}) + max[$i]=$(getval $getmax ${cur[$i]} ${max[$i]}) + fi done fi # @@ -145,8 +154,13 @@ fi # Average the remaining and write it out to the file. # for i in ${!min[*]}; do - s=`expr ${sum[$i]} - ${min[$i]} - ${max[$i]}` - avg[$i]=`expr $s / $numruns` + if test "$i" -eq "$loadindex"; then + s=`echo "scale=3; ${sum[$i]} - ${min[$i]} - ${max[$i]}" | bc` + avg[$i]=`echo "scale=3; $s / $numruns" | bc` + else + s=`expr ${sum[$i]} - ${min[$i]} - ${max[$i]}` + avg[$i]=`expr $s / $numruns` + fi done for i in ${!outp[*]}; do echo "${outp[$i]} ${avg[$i]}" >> $outfile diff --git a/bench/wtperf/wtperf.c b/bench/wtperf/wtperf.c index 38352fd3ef6..3469f0f1599 100644 --- a/bench/wtperf/wtperf.c +++ b/bench/wtperf/wtperf.c @@ -1200,7 +1200,7 @@ execute_populate(CONFIG *cfg) CONFIG_THREAD *popth; WT_ASYNC_OP *asyncop; size_t i; - uint64_t last_ops, secs; + uint64_t last_ops, msecs; uint32_t interval, tables; int elapsed, ret; void *(*pfunc)(void *); @@ -1278,12 +1278,11 @@ execute_populate(CONFIG *cfg) } lprintf(cfg, 0, 1, "Finished load of %" PRIu32 " items", cfg->icount); - secs = WT_TIMEDIFF(stop, start) / BILLION; - if (secs == 0) - ++secs; + msecs = ns_to_ms(WT_TIMEDIFF(stop, start)); lprintf(cfg, 0, 1, - "Load time: %" PRIu64 "\n" "load ops/sec: %" PRIu64, - secs, cfg->icount / secs); + "Load time: %.2f\n" "load ops/sec: %" PRIu64, + (double)msecs / (double)THOUSAND, + (cfg->icount / msecs) / THOUSAND); /* * If configured, compact to allow LSM merging to complete. We @@ -1323,9 +1322,9 @@ execute_populate(CONFIG *cfg) lprintf(cfg, ret, 0, "Get time failed in populate."); return (ret); } - secs = WT_TIMEDIFF(stop, start) / BILLION; lprintf(cfg, 0, 1, - "Compact completed in %" PRIu64 " seconds", secs); + "Compact completed in %" PRIu64 " seconds", + ns_to_sec(WT_TIMEDIFF(stop, start))); assert(tables == 0); } return (0); -- cgit v1.2.1 From 34222b005ee0f5eab62e4e05dc108d4351e0dc78 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Thu, 11 Sep 2014 15:50:39 -0400 Subject: Fix warning. --- bench/wtperf/wtperf.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bench/wtperf/wtperf.c b/bench/wtperf/wtperf.c index 3469f0f1599..b6d65761b5a 100644 --- a/bench/wtperf/wtperf.c +++ b/bench/wtperf/wtperf.c @@ -1282,7 +1282,7 @@ execute_populate(CONFIG *cfg) lprintf(cfg, 0, 1, "Load time: %.2f\n" "load ops/sec: %" PRIu64, (double)msecs / (double)THOUSAND, - (cfg->icount / msecs) / THOUSAND); + (uint64_t)((cfg->icount / msecs) / THOUSAND)); /* * If configured, compact to allow LSM merging to complete. We @@ -1324,7 +1324,7 @@ execute_populate(CONFIG *cfg) } lprintf(cfg, 0, 1, "Compact completed in %" PRIu64 " seconds", - ns_to_sec(WT_TIMEDIFF(stop, start))); + (uint64_t)(ns_to_sec(WT_TIMEDIFF(stop, start)))); assert(tables == 0); } return (0); -- cgit v1.2.1 From 2a594958c1f65d32f85e3b7d19c26d2a4919deff Mon Sep 17 00:00:00 2001 From: Alex Gorrod Date: Fri, 12 Sep 2014 09:58:24 +1000 Subject: Fix build warning: ../src/lsm/lsm_tree.c:1007: warning: 'chunk' may be used uninitialized in this function --- src/lsm/lsm_tree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index 4eec6a9b559..59baf038fd1 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -1010,6 +1010,7 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) int i, compacting, flushing, locked, ref; compacting = flushing = locked = ref = 0; + chunk = NULL; /* * This function is applied to all matching sources: ignore anything * that is not an LSM tree. @@ -1062,7 +1063,6 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) * hasn't been set before. This prevents further writes, so it * can be flushed by the checkpoint worker. */ - chunk = NULL; if (lsm_tree->nchunks > 0 && (chunk = lsm_tree->chunk[lsm_tree->nchunks - 1]) != NULL) { if (chunk->switch_txn == WT_TXN_NONE) -- cgit v1.2.1 From b82fa31a5c3f27e7d975bc1eb16a76e3181e85cb Mon Sep 17 00:00:00 2001 From: Keith Bostic Date: Fri, 12 Sep 2014 07:32:32 -0400 Subject: typo. --- src/lsm/lsm_tree.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index 59baf038fd1..fac47ff0465 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -1090,10 +1090,9 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, int *skip) /* Wait for the work unit queues to drain. */ while (F_ISSET(lsm_tree, WT_LSM_TREE_ACTIVE)) { /* - * The flush flag is cleared when the the chunk has been - * flushed. Continue to push forced flushes until the - * chunk is on disk. Once it is on disk move to the compacting - * phase. + * The flush flag is cleared when the chunk has been flushed. + * Continue to push forced flushes until the chunk is on disk. + * Once it is on disk move to the compacting phase. */ if (flushing && !F_ISSET(lsm_tree, WT_LSM_TREE_COMPACT_FLUSH)) { if (chunk != NULL && -- cgit v1.2.1