diff options
author | Etienne Petrel <etienne.petrel@mongodb.com> | 2022-07-26 00:07:58 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-07-26 00:39:38 +0000 |
commit | 776258cfc9026a3d5dec8c07db74fdd579957132 (patch) | |
tree | d22abbcd9bac0029aa06fca132d234f43c16f3fc | |
parent | 1351f3febd548f2ee721dfb2a393988296544d58 (diff) | |
download | mongo-776258cfc9026a3d5dec8c07db74fdd579957132.tar.gz |
Import wiredtiger: 783aa1f7df44e1d8e5ff4a2b15475bf57946b3ec from branch mongodb-master
ref: 3ae4a6657c..783aa1f7df
for: 6.1.0-rc0
WT-9474 Implement cursor bound checking for cursor next / prev for column store. (#8080)
15 files changed, 401 insertions, 96 deletions
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index bd1a360b72a..d09c16fa7e3 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "3ae4a6657c32da5ec485857e3c8a9025f3677834" + "commit": "783aa1f7df44e1d8e5ff4a2b15475bf57946b3ec" } diff --git a/src/third_party/wiredtiger/src/btree/bt_curnext.c b/src/third_party/wiredtiger/src/btree/bt_curnext.c index 71fb544b8b6..2b99c6fbae3 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curnext.c +++ b/src/third_party/wiredtiger/src/btree/bt_curnext.c @@ -149,8 +149,10 @@ restart_read: * Return the next variable-length entry on the append list. */ static inline int -__cursor_var_append_next(WT_CURSOR_BTREE *cbt, bool newpage, bool restart, size_t *skippedp) +__cursor_var_append_next( + WT_CURSOR_BTREE *cbt, bool newpage, bool restart, size_t *skippedp, bool *key_out_of_boundsp) { + WT_DECL_RET; WT_SESSION_IMPL *session; session = CUR2S(cbt); @@ -170,9 +172,18 @@ __cursor_var_append_next(WT_CURSOR_BTREE *cbt, bool newpage, bool restart, size_ new_page: if (cbt->ins == NULL) return (WT_NOTFOUND); - __cursor_set_recno(cbt, WT_INSERT_RECNO(cbt->ins)); + restart_read: + /* + * If an upper bound has been set ensure that the key is within the range, otherwise early + * exit. + */ + if ((ret = __wt_btcur_bounds_early_exit(session, cbt, true, key_out_of_boundsp)) == + WT_NOTFOUND) + WT_STAT_CONN_DATA_INCR(session, cursor_bounds_next_early_exit); + WT_RET(ret); + WT_RET(__wt_txn_read_upd_list(session, cbt, cbt->ins->upd)); if (cbt->upd_value->type == WT_UPDATE_INVALID) { @@ -197,11 +208,13 @@ restart_read: * Move to the next, variable-length column-store item. */ static inline int -__cursor_var_next(WT_CURSOR_BTREE *cbt, bool newpage, bool restart, size_t *skippedp) +__cursor_var_next( + WT_CURSOR_BTREE *cbt, bool newpage, bool restart, size_t *skippedp, bool *key_out_of_boundsp) { WT_CELL *cell; WT_CELL_UNPACK_KV unpack; WT_COL *cip; + WT_DECL_RET; WT_INSERT *ins; WT_PAGE *page; WT_SESSION_IMPL *session; @@ -240,6 +253,19 @@ __cursor_var_next(WT_CURSOR_BTREE *cbt, bool newpage, bool restart, size_t *skip new_page: restart_read: + /* + * If an upper bound has been set ensure that the key is within the range, otherwise early + * exit. In the case where there is a large set of RLE deleted records it is possible that + * calculated recno will be off the end of the page. We don't need to add an additional + * check for this case as the next iteration, either on a page or append list will check the + * recno and early exit. It does present a potential optimization but to keep the bounded + * cursor logic simple we will forego it for now. + */ + if ((ret = __wt_btcur_bounds_early_exit(session, cbt, true, key_out_of_boundsp)) == + WT_NOTFOUND) + WT_STAT_CONN_DATA_INCR(session, cursor_bounds_next_early_exit); + WT_RET(ret); + /* Find the matching WT_COL slot. */ if ((cip = __col_var_search(cbt->ref, cbt->recno, &rle_start)) == NULL) return (WT_NOTFOUND); @@ -780,7 +806,7 @@ __wt_btcur_next_prefix(WT_CURSOR_BTREE *cbt, WT_ITEM *prefix, bool truncating) * bounds, continue the next traversal logic. */ if (F_ISSET(cursor, WT_CURSTD_BOUND_LOWER) && !WT_CURSOR_IS_POSITIONED(cbt)) { - WT_ERR(__wt_btcur_bounds_row_position(session, cbt, true, &need_walk)); + WT_ERR(__wt_btcur_bounds_position(session, cbt, true, &need_walk)); if (!need_walk) { __wt_value_return(cbt, cbt->upd_value); goto done; @@ -810,7 +836,7 @@ __wt_btcur_next_prefix(WT_CURSOR_BTREE *cbt, WT_ITEM *prefix, bool truncating) ret = __cursor_fix_append_next(cbt, newpage, restart); break; case WT_PAGE_COL_VAR: - ret = __cursor_var_append_next(cbt, newpage, restart, &skipped); + ret = __cursor_var_append_next(cbt, newpage, restart, &skipped, &key_out_of_bounds); total_skipped += skipped; break; default: @@ -821,13 +847,22 @@ __wt_btcur_next_prefix(WT_CURSOR_BTREE *cbt, WT_ITEM *prefix, bool truncating) F_CLR(cbt, WT_CBT_ITERATE_APPEND); if (ret != WT_NOTFOUND) break; + + /* + * If we are doing an operation when the cursor has bounds set, we need to check if we + * have exited the next function due to the key being out of bounds. If so, we break + * instead of walking onto the next page. We're not directly returning here to allow the + * cursor to be reset first before we return WT_NOTFOUND. + */ + if (key_out_of_bounds) + break; } else if (page != NULL) { switch (page->type) { case WT_PAGE_COL_FIX: ret = __cursor_fix_next(cbt, newpage, restart); break; case WT_PAGE_COL_VAR: - ret = __cursor_var_next(cbt, newpage, restart, &skipped); + ret = __cursor_var_next(cbt, newpage, restart, &skipped, &key_out_of_bounds); total_skipped += skipped; break; case WT_PAGE_ROW_LEAF: diff --git a/src/third_party/wiredtiger/src/btree/bt_curprev.c b/src/third_party/wiredtiger/src/btree/bt_curprev.c index d4c32386bc2..20a0172ba7e 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curprev.c +++ b/src/third_party/wiredtiger/src/btree/bt_curprev.c @@ -296,8 +296,10 @@ restart_read: * Return the previous variable-length entry on the append list. */ static inline int -__cursor_var_append_prev(WT_CURSOR_BTREE *cbt, bool newpage, bool restart, size_t *skippedp) +__cursor_var_append_prev( + WT_CURSOR_BTREE *cbt, bool newpage, bool restart, size_t *skippedp, bool *key_out_of_boundsp) { + WT_DECL_RET; WT_SESSION_IMPL *session; session = CUR2S(cbt); @@ -324,6 +326,15 @@ new_page: return (0); restart_read: + /* + * If a lower bound has been set ensure that the key is within the range, otherwise early + * exit. + */ + if ((ret = __wt_btcur_bounds_early_exit(session, cbt, false, key_out_of_boundsp)) == + WT_NOTFOUND) + WT_STAT_CONN_DATA_INCR(session, cursor_bounds_prev_early_exit); + WT_RET(ret); + WT_RET(__wt_txn_read_upd_list(session, cbt, cbt->ins->upd)); if (cbt->upd_value->type == WT_UPDATE_INVALID) { ++*skippedp; @@ -347,11 +358,13 @@ restart_read: * Move to the previous, variable-length column-store item. */ static inline int -__cursor_var_prev(WT_CURSOR_BTREE *cbt, bool newpage, bool restart, size_t *skippedp) +__cursor_var_prev( + WT_CURSOR_BTREE *cbt, bool newpage, bool restart, size_t *skippedp, bool *key_out_of_boundsp) { WT_CELL *cell; WT_CELL_UNPACK_KV unpack; WT_COL *cip; + WT_DECL_RET; WT_INSERT *ins; WT_PAGE *page; WT_SESSION_IMPL *session; @@ -391,6 +404,19 @@ new_page: return (WT_NOTFOUND); restart_read: + /* + * If an lower bound has been set ensure that the key is within the range, otherwise early + * exit. In the case where there is a large set of RLE deleted records it is possible that + * calculated recno will be off the end of the page. We don't need to add an additional + * check for this case as the prev iteration, either on a page or append list will check the + * recno and early exit. It does present a potential optimization but to keep the bounded + * cursor logic simple we will forego it for now. + */ + if ((ret = __wt_btcur_bounds_early_exit(session, cbt, false, key_out_of_boundsp)) == + WT_NOTFOUND) + WT_STAT_CONN_DATA_INCR(session, cursor_bounds_prev_early_exit); + WT_RET(ret); + /* Find the matching WT_COL slot. */ if ((cip = __col_var_search(cbt->ref, cbt->recno, &rle_start)) == NULL) return (WT_NOTFOUND); @@ -718,7 +744,7 @@ __wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) * bounds, continue the prev traversal logic. */ if (F_ISSET(cursor, WT_CURSTD_BOUND_UPPER) && !WT_CURSOR_IS_POSITIONED(cbt)) { - WT_ERR(__wt_btcur_bounds_row_position(session, cbt, false, &need_walk)); + WT_ERR(__wt_btcur_bounds_position(session, cbt, false, &need_walk)); if (!need_walk) { __wt_value_return(cbt, cbt->upd_value); goto done; @@ -758,7 +784,7 @@ __wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) ret = __cursor_fix_append_prev(cbt, newpage, restart); break; case WT_PAGE_COL_VAR: - ret = __cursor_var_append_prev(cbt, newpage, restart, &skipped); + ret = __cursor_var_append_prev(cbt, newpage, restart, &skipped, &key_out_of_bounds); total_skipped += skipped; break; default: @@ -777,7 +803,7 @@ __wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) ret = __cursor_fix_prev(cbt, newpage, restart); break; case WT_PAGE_COL_VAR: - ret = __cursor_var_prev(cbt, newpage, restart, &skipped); + ret = __cursor_var_prev(cbt, newpage, restart, &skipped, &key_out_of_bounds); total_skipped += skipped; break; case WT_PAGE_ROW_LEAF: @@ -789,16 +815,17 @@ __wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) } if (ret != WT_NOTFOUND) break; - /* - * If we are doing a search near with prefix key configured or the cursor has bounds - * set, we need to check if we have exited the prev function due to a prefix key - * mismatch or the key is out of bounds. If so, we break instead of walking onto the - * next page. We're not directly returning here to allow the cursor to be reset first - * before we return WT_NOTFOUND. - */ - if (key_out_of_bounds) - break; } + + /* + * If we are doing an operation with bounds set, we need to check if we have exited the prev + * function due to the key being out of bounds. If so, we break instead of walking onto the + * prev page. We're not directly returning here to allow the cursor to be reset first before + * we return WT_NOTFOUND. + */ + if (key_out_of_bounds) + break; + /* * If we saw a lot of deleted records on this page, or we went all the way through a page * and only saw deleted records, try to evict the page when we release it. Otherwise diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index ff26d98c0a3..2f6111a9ff8 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -28,22 +28,29 @@ static int __btcur_bounds_contains_key( WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_ITEM *key, bool *key_out_of_boundsp, bool *upperp) { + WT_CURSOR_BTREE *cbt; + + cbt = (WT_CURSOR_BTREE *)cursor; *key_out_of_boundsp = false; +#ifndef HAVE_DIAGNOSTIC + WT_UNUSED(cbt); +#endif + if (upperp != NULL) *upperp = false; WT_ASSERT(session, WT_CURSOR_BOUNDS_SET(cursor)); - WT_ASSERT(session, key != NULL); - - if (CUR2BT(cursor)->type != BTREE_ROW) - WT_RET(ENOTSUP); + if (CUR2BT(cursor)->type == BTREE_ROW) + WT_ASSERT(session, key != NULL); + else + WT_ASSERT(session, cbt->recno != 0); if (F_ISSET(cursor, WT_CURSTD_BOUND_LOWER)) - WT_RET(__wt_row_compare_bounds(session, cursor, key, false, key_out_of_boundsp)); + WT_RET(__wt_compare_bounds(session, cursor, key, false, key_out_of_boundsp)); if (!(*key_out_of_boundsp) && F_ISSET(cursor, WT_CURSTD_BOUND_UPPER)) { - WT_RET(__wt_row_compare_bounds(session, cursor, key, true, key_out_of_boundsp)); + WT_RET(__wt_compare_bounds(session, cursor, key, true, key_out_of_boundsp)); if (*key_out_of_boundsp && upperp != NULL) *upperp = true; } @@ -65,8 +72,6 @@ __btcur_bounds_search_near_reposition(WT_SESSION_IMPL *session, WT_CURSOR_BTREE key_out_of_bounds = upper = false; WT_ASSERT(session, WT_CURSOR_BOUNDS_SET(cursor)); - if (CUR2BT(cbt)->type != BTREE_ROW) - WT_RET(ENOTSUP); /* * Suppose a caller calls with the search key set to the lower bound but also specifies that the @@ -301,9 +306,9 @@ __wt_cursor_valid(WT_CURSOR_BTREE *cbt, WT_ITEM *key, uint64_t recno, bool *vali * update that's been deleted is not a valid key/value pair). */ if (cbt->ins != NULL) { - if (WT_CURSOR_BOUNDS_SET(&cbt->iface) && btree->type == BTREE_ROW) { + if (WT_CURSOR_BOUNDS_SET(&cbt->iface)) { /* Get the insert list key. */ - if (key == NULL) { + if (key == NULL && btree->type == BTREE_ROW) { tmp_key.data = WT_INSERT_KEY(cbt->ins); tmp_key.size = WT_INSERT_KEY_SIZE(cbt->ins); } @@ -374,6 +379,14 @@ __wt_cursor_valid(WT_CURSOR_BTREE *cbt, WT_ITEM *key, uint64_t recno, bool *vali if (__wt_cell_type(cell) == WT_CELL_DEL) return (0); + if (WT_CURSOR_BOUNDS_SET(&cbt->iface)) { + WT_RET( + __btcur_bounds_contains_key(session, &cbt->iface, key, &key_out_of_bounds, NULL)); + /* The key value pair we were trying to return weren't within the given bounds. */ + if (key_out_of_bounds) + return (0); + } + /* * Check for an update. For column store, modifications are handled with insert lists, so an * insert can have the same key as an on-page or history store object. Setting update here, @@ -2223,13 +2236,13 @@ __wt_btcur_close(WT_CURSOR_BTREE *cbt, bool lowlevel) } /* - * __wt_btcur_bounds_row_position -- - * A unpositioned bounded cursor need to start its cursor next and prev walk from the lower or + * __wt_btcur_bounds_position -- + * An unpositioned bounded cursor need to start its cursor next and prev walk from the lower or * upper bound depending on which direction it is going. This function calls cursor row search - * to position the cursor appropriately. + * or cursor col search to position the cursor appropriately. */ int -__wt_btcur_bounds_row_position( +__wt_btcur_bounds_position( WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, bool next, bool *need_walkp) { @@ -2248,12 +2261,18 @@ __wt_btcur_bounds_row_position( WT_ASSERT(session, WT_DATA_IN_ITEM(bound)); __wt_cursor_set_raw_key(cursor, bound); - WT_RET(__cursor_row_search(cbt, false, NULL, NULL)); - /* - * If there's an exact match, the row-store search function built the key in the cursor's - * temporary buffer. - */ - WT_RET(__wt_cursor_valid(cbt, (cbt->compare == 0 ? cbt->tmp : NULL), WT_RECNO_OOB, &valid)); + + if (S2BT(session)->type == BTREE_ROW) { + WT_RET(__cursor_row_search(cbt, false, NULL, NULL)); + /* + * If there's an exact match, the row-store search function built the key in the cursor's + * temporary buffer. + */ + WT_RET(__wt_cursor_valid(cbt, (cbt->compare == 0 ? cbt->tmp : NULL), WT_RECNO_OOB, &valid)); + } else { + WT_RET(__cursor_col_search(cbt, NULL, NULL)); + WT_RET(__wt_cursor_valid(cbt, NULL, cbt->recno, &valid)); + } /* * Clear the cursor key set flag, as we don't want to return the internal set key to the user. diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c index 5c7d5a1b7eb..ff8131d5167 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_std.c +++ b/src/third_party/wiredtiger/src/cursor/cur_std.c @@ -1181,8 +1181,8 @@ __wt_cursor_bound(WT_CURSOR *cursor, const char *config) CURSOR_API_CALL_CONF(cursor, session, bound, config, cfg, NULL); - if (WT_STREQ(cursor->key_format, "r")) - WT_ERR_MSG(session, EINVAL, "setting bounds is not compatible with column store yet."); + if (CUR2BT(cursor)->type == BTREE_COL_FIX) + WT_ERR_MSG(session, EINVAL, "setting bounds is not compatible with fixed column store."); if (F_ISSET(cursor, WT_CURSTD_PREFIX_SEARCH)) WT_ERR_MSG(session, EINVAL, "setting bounds is not compatible with prefix search."); diff --git a/src/third_party/wiredtiger/src/include/btree_cmp_inline.h b/src/third_party/wiredtiger/src/include/btree_cmp_inline.h index d4f90915b8a..3767bc1873e 100644 --- a/src/third_party/wiredtiger/src/include/btree_cmp_inline.h +++ b/src/third_party/wiredtiger/src/include/btree_cmp_inline.h @@ -138,31 +138,55 @@ __wt_prefix_match(const WT_ITEM *prefix, const WT_ITEM *tree_item) } /* - * __wt_row_compare_bounds -- - * Return if the cursor key is within the bounded range. If next is True, this indicates a next - * call and the key is checked against the upper bound. If next is False, this indicates a prev + * __wt_compare_bounds -- + * Return if the cursor key is within the bounded range. If upper is True, this indicates a next + * call and the key is checked against the upper bound. If upper is False, this indicates a prev * call and the key is then checked against the lower bound. */ static inline int -__wt_row_compare_bounds( - WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_ITEM *key, bool upper, bool *key_out_of_boundsp) +__wt_compare_bounds( + WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_ITEM *key, bool upper, bool *key_out_of_bounds) { + WT_CURSOR_BTREE *cbt; + uint64_t recno_bound; int cmpp; + cbt = (WT_CURSOR_BTREE *)cursor; + cmpp = 0; + recno_bound = 0; + if (upper) { WT_ASSERT(session, WT_DATA_IN_ITEM(&cursor->upper_bound)); - WT_RET(__wt_compare(session, S2BT(session)->collator, key, &cursor->upper_bound, &cmpp)); + if (S2BT(session)->type == BTREE_ROW) + WT_RET( + __wt_compare(session, S2BT(session)->collator, key, &cursor->upper_bound, &cmpp)); + else + /* Unpack the raw recno buffer into integer variable. */ + WT_RET(__wt_struct_unpack( + session, cursor->upper_bound.data, cursor->upper_bound.size, "q", &recno_bound)); + if (F_ISSET(cursor, WT_CURSTD_BOUND_UPPER_INCLUSIVE)) - *key_out_of_boundsp = (cmpp > 0); + *key_out_of_bounds = + S2BT(session)->type == BTREE_ROW ? (cmpp > 0) : (cbt->recno > recno_bound); else - *key_out_of_boundsp = (cmpp >= 0); + *key_out_of_bounds = + S2BT(session)->type == BTREE_ROW ? (cmpp >= 0) : (cbt->recno >= recno_bound); } else { WT_ASSERT(session, WT_DATA_IN_ITEM(&cursor->lower_bound)); - WT_RET(__wt_compare(session, S2BT(session)->collator, key, &cursor->lower_bound, &cmpp)); + if (S2BT(session)->type == BTREE_ROW) + WT_RET( + __wt_compare(session, S2BT(session)->collator, key, &cursor->lower_bound, &cmpp)); + else + /* Unpack the raw recno buffer into integer variable. */ + WT_RET(__wt_struct_unpack( + session, cursor->lower_bound.data, cursor->lower_bound.size, "q", &recno_bound)); + if (F_ISSET(cursor, WT_CURSTD_BOUND_LOWER_INCLUSIVE)) - *key_out_of_boundsp = (cmpp < 0); + *key_out_of_bounds = + S2BT(session)->type == BTREE_ROW ? (cmpp < 0) : (cbt->recno < recno_bound); else - *key_out_of_boundsp = (cmpp <= 0); + *key_out_of_bounds = + S2BT(session)->type == BTREE_ROW ? (cmpp <= 0) : (cbt->recno <= recno_bound); } return (0); } diff --git a/src/third_party/wiredtiger/src/include/btree_inline.h b/src/third_party/wiredtiger/src/include/btree_inline.h index 4195db24427..e0b3f9fe535 100644 --- a/src/third_party/wiredtiger/src/include/btree_inline.h +++ b/src/third_party/wiredtiger/src/include/btree_inline.h @@ -2136,8 +2136,7 @@ __wt_btcur_bounds_early_exit( if (!F_ISSET((&cbt->iface), bound_flag)) return (0); - WT_RET( - __wt_row_compare_bounds(session, &cbt->iface, &cbt->iface.key, next, key_out_of_boundsp)); + WT_RET(__wt_compare_bounds(session, &cbt->iface, &cbt->iface.key, next, key_out_of_boundsp)); if (*key_out_of_boundsp) return (WT_NOTFOUND); diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 73f3d2632ed..c5d45a6249e 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -258,7 +258,7 @@ extern int __wt_bm_corrupt(WT_BM *bm, WT_SESSION_IMPL *session, const uint8_t *a size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bm_read(WT_BM *bm, WT_SESSION_IMPL *session, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_btcur_bounds_row_position(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, bool next, +extern int __wt_btcur_bounds_position(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, bool next, bool *need_walkp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btcur_close(WT_CURSOR_BTREE *cbt, bool lowlevel) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -2071,6 +2071,8 @@ static inline int __wt_col_append_serial(WT_SESSION_IMPL *session, WT_PAGE *page static inline int __wt_compare(WT_SESSION_IMPL *session, WT_COLLATOR *collator, const WT_ITEM *user_item, const WT_ITEM *tree_item, int *cmpp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +static inline int __wt_compare_bounds(WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_ITEM *key, + bool upper, bool *key_out_of_bounds) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_compare_skip(WT_SESSION_IMPL *session, WT_COLLATOR *collator, const WT_ITEM *user_item, const WT_ITEM *tree_item, int *cmpp, size_t *matchp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -2163,8 +2165,6 @@ static inline int __wt_rec_dict_replace( WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_ref_block_free(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -static inline int __wt_row_compare_bounds(WT_SESSION_IMPL *session, WT_CURSOR *cursor, WT_ITEM *key, - bool upper, bool *key_out_of_boundsp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_row_leaf_key(WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW *rip, WT_ITEM *key, bool instantiate) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); static inline int __wt_row_leaf_key_instantiate(WT_SESSION_IMPL *session, WT_PAGE *page) diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_bound01.py b/src/third_party/wiredtiger/test/suite/test_cursor_bound01.py index c0baf30ee22..d45ebe186f1 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor_bound01.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_bound01.py @@ -28,10 +28,11 @@ import wiredtiger, wttest from wtscenario import make_scenarios +from wtbound import bound_base # test_cursor_bound01.py # Basic cursor bound API validation. -class test_cursor_bound01(wttest.WiredTigerTestCase): +class test_cursor_bound01(bound_base): file_name = 'test_cursor_bound01' types = [ @@ -42,11 +43,22 @@ class test_cursor_bound01(wttest.WiredTigerTestCase): #FIXME: Turn on once index cursor bound implementation is done. #('index', dict(uri='table:', use_index = True)), ] - scenarios = make_scenarios(types) + + format_values = [ + ('string', dict(key_format='S',value_format='S')), + ('var', dict(key_format='r',value_format='S')), + ('fix', dict(key_format='r',value_format='8t')) + ] + + scenarios = make_scenarios(types,format_values) def test_bound_api(self): + # LSM doesn't support column store type, therefore we can just return early here. + if (self.key_format == 'r' and self.uri == 'lsm:'): + return + uri = self.uri + self.file_name - create_params = 'value_format=S,key_format=i' + create_params = 'value_format={},key_format={}'.format(self.value_format, self.key_format) if self.use_index or self.use_colgroup: create_params += ",columns=(k,v)" if self.use_colgroup: @@ -68,9 +80,13 @@ class test_cursor_bound01(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri) # LSM format is not supported with range cursors. - if self.uri == 'lsm:': + if self.uri == 'lsm:': + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cursor.bound("bound=lower"), + '/Operation not supported/') + return + if self.value_format == '8t': self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: cursor.bound("bound=lower"), - '/Unsupported cursor operation/') + '/Invalid argument/') return # Cursor bound API should return EINVAL if no configurations are passed in. @@ -78,9 +94,9 @@ class test_cursor_bound01(wttest.WiredTigerTestCase): '/Invalid argument/') # Check that bound configuration works properly. - cursor.set_key(0) + cursor.set_key(self.gen_key(1)) cursor.bound("bound=lower") - cursor.set_key(10) + cursor.set_key(self.gen_key(10)) cursor.bound("bound=upper") # Clear and inclusive configuration are not compatible with each other. diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_bound02.py b/src/third_party/wiredtiger/test/suite/test_cursor_bound02.py index 1e2af6c69a2..5d32cea503d 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor_bound02.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_bound02.py @@ -45,8 +45,7 @@ class test_cursor_bound02(bound_base): key_format_values = [ ('string', dict(key_format='S')), - # FIXME-WT-9474: Uncomment once column store is implemented. - #('var', dict(key_format='r')), + ('var', dict(key_format='r')), ('int', dict(key_format='i')), ('bytes', dict(key_format='u')), ('composite_string', dict(key_format='SSS')), diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_bound03.py b/src/third_party/wiredtiger/test/suite/test_cursor_bound03.py index 91945724348..2ae55ec3576 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor_bound03.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_bound03.py @@ -44,8 +44,7 @@ class test_cursor_bound03(bound_base): key_format_values = [ ('string', dict(key_format='S')), - # FIXME-WT-9474: Uncomment once column store is implemented. - # ('var', dict(key_format='r')), + ('var', dict(key_format='r')), ('int', dict(key_format='i')), ('bytes', dict(key_format='u')), ('composite_string', dict(key_format='SSS')), diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_bound04.py b/src/third_party/wiredtiger/test/suite/test_cursor_bound04.py index c36207c6429..f8c4c5cc900 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor_bound04.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_bound04.py @@ -46,8 +46,7 @@ class test_cursor_bound04(bound_base): key_format_values = [ ('string', dict(key_format='S')), - # FIXME-WT-9474: Uncomment once column store is implemented. - #('var', dict(key_format='r')), + ('var', dict(key_format='r')), ('int', dict(key_format='i')), ('bytes', dict(key_format='u')), ('composite_string', dict(key_format='SSS')), diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_bound07.py b/src/third_party/wiredtiger/test/suite/test_cursor_bound07.py new file mode 100644 index 00000000000..d87a97e2552 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_cursor_bound07.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python +# +# Public Domain 2014-present MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios +from wtbound import bound_base + +# test_cursor_bound07.py +# Test column store related scenarios with the bounds API. +class test_cursor_bound07(bound_base): + file_name = 'test_cursor_bound07' + start_key = 10 + end_key = 100 + key_format = 'r' + lower_inclusive = True + upper_inclusive = True + + types = [ + ('file', dict(uri='file:')), + ('table', dict(uri='table:')), + ] + + evict = [ + ('evict', dict(evict=True)), + ('no-evict', dict(evict=True)), + ] + + record_values = [ + ('all-records-rle', dict(records_rle=True,deleted_rle=True)), + ('only-deleted-records-rle', dict(records_rle=False,deleted_rle=True)), + ('only-normal-records-rle', dict(records_rle=True,deleted_rle=False)), + ('no-records-rle', dict(records_rle=False,deleted_rle=False)), + ] + + direction = [ + ('prev', dict(next=False)), + ('next', dict(next=True)), + ] + scenarios = make_scenarios(types, direction, evict, record_values) + + def create_session_and_cursor(self): + uri = self.uri + self.file_name + create_params = 'value_format=S,key_format={}'.format(self.key_format) + self.session.create(uri, create_params) + + cursor = self.session.open_cursor(uri) + self.session.begin_transaction() + for i in range(10, 31): + value = "value" + str(i) if not self.records_rle else "value" + cursor[self.gen_key(i)] = value + self.session.commit_transaction() + + self.session.begin_transaction() + for i in range(71, 101): + value = "value" + str(i) if not self.records_rle else "value" + cursor[self.gen_key(i)] = value + self.session.commit_transaction() + + self.session.begin_transaction() + for i in range(31, 71): + value = "value" + str(i) if not self.deleted_rle else "value" + cursor[self.gen_key(i)] = value + cursor.set_key(self.gen_key(i)) + self.assertEqual(cursor.remove(), 0) + self.session.commit_transaction() + + if (self.evict): + evict_cursor = self.session.open_cursor(uri, None, "debug=(release_evict)") + for i in range(self.start_key, self.end_key): + evict_cursor.set_key(self.gen_key(i)) + evict_cursor.search() + evict_cursor.reset() + evict_cursor.close() + return cursor + + def test_bound_next_scenario(self): + cursor = self.create_session_and_cursor() + + # Test bound api: Test early exit works with upper bound. + self.set_bounds(cursor, 15, "upper", self.upper_inclusive) + self.cursor_traversal_bound(cursor, None, 15, self.next) + self.assertEqual(cursor.bound("action=clear"), 0) + + # Test bound api: Test traversal works with lower bound. + self.set_bounds(cursor, 90, "lower", self.lower_inclusive) + self.cursor_traversal_bound(cursor, 90, None, self.next) + self.assertEqual(cursor.bound("action=clear"), 0) + + # Test bound api: Test traversal with both bounds. + self.set_bounds(cursor, 80, "lower", self.lower_inclusive) + self.set_bounds(cursor, 90, "upper", self.upper_inclusive) + self.cursor_traversal_bound(cursor, 80, 90, self.next) + self.assertEqual(cursor.bound("action=clear"), 0) + + # Test bound api: Test traversal with lower over deleted records. + self.set_bounds(cursor, 50, "lower", self.lower_inclusive) + self.cursor_traversal_bound(cursor, 50, None, self.next, 29) + self.assertEqual(cursor.bound("action=clear"), 0) + + self.set_bounds(cursor, 50, "upper", self.upper_inclusive) + self.cursor_traversal_bound(cursor, None, 50, self.next, 20) + self.assertEqual(cursor.bound("action=clear"), 0) + + # Test bound api: Test column store insert list. + self.session.begin_transaction() + for i in range(51, 61): + cursor[self.gen_key(i)] = "value" + str(i) + self.session.commit_transaction() + + self.set_bounds(cursor, 55, "upper", self.upper_inclusive) + self.cursor_traversal_bound(cursor, None, 55, self.next, 25) + self.assertEqual(cursor.bound("action=clear"), 0) + + self.set_bounds(cursor, 55, "lower", self.lower_inclusive) + self.cursor_traversal_bound(cursor, 55, None, self.next, 35) + self.assertEqual(cursor.bound("action=clear"), 0) + + # Test bound api: Test inclusive option between a RLE record and a normal record. + self.session.begin_transaction() + cursor[101] = "value_normal" + str(i) + self.session.commit_transaction() + + # FIX-ME-WT-9475: cursor_traversal_bound has a bug, therefore e-enable this check when the function is fixed. + # self.set_bounds(cursor, 100, "lower", False) + # self.cursor_traversal_bound(cursor, 100, None, self.next, 1) + # self.assertEqual(cursor.bound("action=clear"), 0) + + self.set_bounds(cursor, 101, "upper", False) + self.cursor_traversal_bound(cursor, None, 101, self.next, 60) + self.assertEqual(cursor.bound("action=clear"), 0) + + self.session.begin_transaction() + cursor[102] = "value_normal" + str(i) + self.session.commit_transaction() + + # FIX-ME-WT-9475: cursor_traversal_bound has a bug, therefore e-enable this check when the function is fixed. + # self.set_bounds(cursor, 100, "lower", False) + # self.cursor_traversal_bound(cursor, 100, None, self.next, 2) + # self.assertEqual(cursor.bound("action=clear"), 0) + + self.set_bounds(cursor, 101, "upper", False) + self.cursor_traversal_bound(cursor, None, 101, self.next, 60) + self.assertEqual(cursor.bound("action=clear"), 0) + + cursor.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_bound_fuzz.py b/src/third_party/wiredtiger/test/suite/test_cursor_bound_fuzz.py index 20846d0e003..e6e57679a56 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor_bound_fuzz.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_bound_fuzz.py @@ -91,18 +91,21 @@ class test_cursor_bound_fuzz(wttest.WiredTigerTestCase): types = [ ('file', dict(uri='file:')), - ('table', dict(uri='table:')) + #('table', dict(uri='table:')) ] data_format = [ ('row', dict(key_format='i')), - # FIXME Enable once column store is completed. - #('column', dict(key_format='r')) + ('column', dict(key_format='r')) ] scenarios = make_scenarios(types, data_format) + def key_range_iter(self): + for i in range(self.min_key, self.max_key): + yield i + def dump_key_range(self): - for i in range(0, self.key_count): + for i in self.key_range_iter(): self.pr(self.key_range[i].to_string()) def generate_value(self): @@ -127,7 +130,7 @@ class test_cursor_bound_fuzz(wttest.WiredTigerTestCase): def apply_ops(self, cursor): op_count = self.key_count - for i in range(0, op_count): + for i in self.key_range_iter(): op = random.choice(list(operations)) if (op is operations.UPSERT): self.apply_update(cursor, i) @@ -141,7 +144,7 @@ class test_cursor_bound_fuzz(wttest.WiredTigerTestCase): checked_keys = [] if (next): self.verbose(2, "Running scenario: NEXT") - key_range_it = -1 + key_range_it = self.min_key - 1 while (cursor.next() != wiredtiger.WT_NOTFOUND): current_key = cursor.get_key() current_value = cursor.get_value() @@ -154,18 +157,20 @@ class test_cursor_bound_fuzz(wttest.WiredTigerTestCase): # Check that the key range between key_range_it and current_key isn't visible if (current_key != key_range_it + 1): for i in range(key_range_it + 1, current_key): + self.verbose(3, "Checking key is deleted or oob: " + str(i)) checked_keys.append(i) self.assertTrue(self.key_range[i].is_deleted_or_oob(bound_set)) key_range_it = current_key # If key_range_it is < key_count then the rest of the range was deleted # Remember to increment it by one to get it to the first not in bounds key. key_range_it = key_range_it + 1 - for i in range(key_range_it, self.key_count): + for i in range(key_range_it, self.max_key): checked_keys.append(i) + self.verbose(3, "Checking key is deleted or oob: " + str(i)) self.assertTrue(self.key_range[i].is_deleted_or_oob(bound_set)) else: self.verbose(2, "Running scenario: PREV") - key_range_it = self.key_count + key_range_it = self.max_key while (cursor.prev() != wiredtiger.WT_NOTFOUND): current_key = cursor.get_key() current_value = cursor.get_value() @@ -176,12 +181,14 @@ class test_cursor_bound_fuzz(wttest.WiredTigerTestCase): if (current_key != key_range_it - 1): # Check that the key range between key_range_it and current_key isn't visible for i in range(current_key + 1, key_range_it): + self.verbose(3, "Checking key is deleted or oob: " + str(i)) checked_keys.append(i) self.assertTrue(self.key_range[i].is_deleted_or_oob(bound_set)) key_range_it = current_key # If key_range_it is > key_count then the rest of the range was deleted - for i in range(0, key_range_it): + for i in range(self.min_key, key_range_it): checked_keys.append(i) + self.verbose(3, "Checking key is deleted or oob: " + str(i)) self.assertTrue(self.key_range[i].is_deleted_or_oob(bound_set)) self.assertTrue(len(checked_keys) == self.key_count) @@ -212,7 +219,7 @@ class test_cursor_bound_fuzz(wttest.WiredTigerTestCase): def run_search_near(self, bound_set, cursor): # Choose N random keys and perform a search near. for i in range(0, self.search_count): - search_key = random.randrange(self.key_count) + search_key = random.randrange(self.min_key, self.max_key) cursor.set_key(search_key) self.verbose(2, "Searching for key: " + str(search_key)) ret = cursor.search_near() @@ -280,7 +287,12 @@ class test_cursor_bound_fuzz(wttest.WiredTigerTestCase): raise Exception('Illegal state found in search_near') def run_bound_scenarios(self, bound_set, cursor): - scenario = random.choice(list(bound_scenarios)) + if (self.data_format == 'column'): + scenario = random.choice([bound_scenarios.NEXT, bound_scenarios.PREV]) + else: + scenario = random.choice(list(bound_scenarios)) + + scenario = bound_scenarios.PREV if (scenario is bound_scenarios.NEXT): self.run_next_prev(bound_set, True, cursor) elif (scenario is bound_scenarios.PREV): @@ -294,8 +306,8 @@ class test_cursor_bound_fuzz(wttest.WiredTigerTestCase): def apply_bounds(self, cursor): cursor.reset() - lower = bound(random.randrange(0, self.key_count), bool(random.getrandbits(1)), bool(random.getrandbits(1))) - upper = bound(random.randrange(lower.key, self.key_count), bool(random.getrandbits(1)), bool(random.getrandbits(1))) + lower = bound(random.randrange(self.min_key, self.max_key), bool(random.getrandbits(1)), bool(random.getrandbits(1))) + upper = bound(random.randrange(lower.key, self.max_key), bool(random.getrandbits(1)), bool(random.getrandbits(1))) # Prevent invalid bounds being generated. if (lower.key == upper.key and lower.enabled and upper.enabled): lower.inclusive = upper.inclusive = True @@ -312,11 +324,11 @@ class test_cursor_bound_fuzz(wttest.WiredTigerTestCase): uri = self.uri + self.file_name create_params = 'value_format=S,key_format={}'.format(self.key_format) # Reset the key range for every scenario. - self.key_range = [] + self.key_range = {} # Setup a reproducible random seed. # If this test fails inspect the file WT_TEST/results.txt and replace the time.time() # with a given seed. e.g.: - #seed = 1657150934.9222412 + #seed = 1657676799.777366 # Additionally this test is configured for verbose logging which can make debugging a bit # easier. seed = time.time() @@ -327,16 +339,18 @@ class test_cursor_bound_fuzz(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri) # Initialize the value array. + self.verbose(2, "Generating value array") for i in range(0, self.value_array_size): self.value_array.append(self.generate_value()) # Initialize the key range. - for i in range(0, self.key_count): + for i in self.key_range_iter(): key_value = self.get_value() - self.key_range.append(key(i, key_value, key_states.UPSERTED)) + self.key_range[i] = key(i, key_value, key_states.UPSERTED) cursor[i] = key_value - self.session.checkpoint() + #self.dump_key_range() + self.session.checkpoint() # Begin main loop for i in range(0, self.iteration_count): self.verbose(2, "Iteration: " + str(i)) @@ -353,14 +367,16 @@ class test_cursor_bound_fuzz(wttest.WiredTigerTestCase): # A lot of time was spent generating values, to achieve some amount of randomness we pre # generate N values and keep them in memory. value_array = [] - iteration_count = 1000 if wttest.islongtest() else 50 - value_size = 10000 + iteration_count = 200 if wttest.islongtest() else 50 + value_size = 1000000 if wttest.islongtest() else 100 value_array_size = 20 - key_count = 1000 + key_count = 10000 if wttest.islongtest() else 1000 + min_key = 1 + max_key = min_key + key_count # For each iteration we do search_count searches that way we test more cases without having to # generate as many key ranges. search_count = 20 - key_range = [] + key_range = {} if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/wtbound.py b/src/third_party/wiredtiger/test/suite/wtbound.py index 9ec34d21b3f..3496b627de7 100644 --- a/src/third_party/wiredtiger/test/suite/wtbound.py +++ b/src/third_party/wiredtiger/test/suite/wtbound.py @@ -173,7 +173,6 @@ class bound_base(wttest.WiredTigerTestCase): count = ret = 0 while True: - self.session.breakpoint() if (next): ret = cursor.next() else: @@ -183,7 +182,7 @@ class bound_base(wttest.WiredTigerTestCase): break count += 1 key = cursor.get_key() - + if (self.lower_inclusive and lower_key): self.assertTrue(self.check_key(lower_key) <= key) elif (lower_key): |