diff options
author | Luke Chen <luke.chen@mongodb.com> | 2021-05-21 16:40:54 +1000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-05-21 07:15:11 +0000 |
commit | 10e91f7dc954023e5ff73683e655c7d90e0052bf (patch) | |
tree | edfc327716ea265e572832000c387c3bddd2f048 /src | |
parent | c29ac114551d425a34409ffb2bee052bfcc36f91 (diff) | |
download | mongo-10e91f7dc954023e5ff73683e655c7d90e0052bf.tar.gz |
Import wiredtiger: cb8077e2b6d4c1d60ef71305e36f28c4866693b6 from branch mongodb-5.0
ref: 02da3990fb..cb8077e2b6
for: 5.0.0
WT-7549 clean up block manager identifiers to use object id naming
Diffstat (limited to 'src')
21 files changed, 174 insertions, 149 deletions
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 6d65d9d217a..e1c95a1c8bd 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -279,9 +279,8 @@ file_config = format_meta + file_runtime_config + tiered_config + [ Config('block_allocation', 'best', r''' configure block allocation. Permitted values are \c "best" or \c "first"; the \c "best" configuration uses a best-fit algorithm, - the \c "first" configuration uses a first-available algorithm during block allocation, - the \c "log-structure" configuration allocates a new file for each checkpoint''', - choices=['best', 'first', 'log-structured',]), + the \c "first" configuration uses a first-available algorithm during block allocation''', + choices=['best', 'first',]), Config('allocation_size', '4KB', r''' the file unit allocation size, in bytes, must a power-of-two; smaller values decrease the file space required by overflow diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 59e4dafe850..e948c0db9ed 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-5.0", - "commit": "02da3990fb1d9da925ff5ba5e0ba7042b48f5c46" + "commit": "cb8077e2b6d4c1d60ef71305e36f28c4866693b6" } diff --git a/src/third_party/wiredtiger/src/block/block_addr.c b/src/third_party/wiredtiger/src/block/block_addr.c index 15295bc02d3..f594d30e83e 100644 --- a/src/third_party/wiredtiger/src/block/block_addr.c +++ b/src/third_party/wiredtiger/src/block/block_addr.c @@ -14,12 +14,12 @@ * reference so it can be called repeatedly to load a buffer. */ static int -__block_buffer_to_addr(WT_BLOCK *block, const uint8_t **pp, uint32_t *logidp, wt_off_t *offsetp, +__block_buffer_to_addr(WT_BLOCK *block, const uint8_t **pp, uint32_t *objectidp, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump) { uint64_t l, o, s, c; - if (block->log_structured) + if (block->has_objects) WT_RET(__wt_vunpack_uint(pp, 0, &l)); else l = 0; @@ -38,9 +38,9 @@ __block_buffer_to_addr(WT_BLOCK *block, const uint8_t **pp, uint32_t *logidp, wt */ if (s == 0) { *offsetp = 0; - *logidp = *sizep = *checksump = 0; + *objectidp = *sizep = *checksump = 0; } else { - *logidp = (uint32_t)l; + *objectidp = (uint32_t)l; *offsetp = (wt_off_t)(o + 1) * block->allocsize; *sizep = (uint32_t)s * block->allocsize; *checksump = (uint32_t)c; @@ -53,8 +53,8 @@ __block_buffer_to_addr(WT_BLOCK *block, const uint8_t **pp, uint32_t *logidp, wt * Convert the filesystem components into its address cookie. */ int -__wt_block_addr_to_buffer( - WT_BLOCK *block, uint8_t **pp, uint32_t logid, wt_off_t offset, uint32_t size, uint32_t checksum) +__wt_block_addr_to_buffer(WT_BLOCK *block, uint8_t **pp, uint32_t objectid, wt_off_t offset, + uint32_t size, uint32_t checksum) { uint64_t l, o, s, c; @@ -63,12 +63,12 @@ __wt_block_addr_to_buffer( o = WT_BLOCK_INVALID_OFFSET; l = s = c = 0; } else { - l = logid; + l = objectid; o = (uint64_t)offset / block->allocsize - 1; s = size / block->allocsize; c = checksum; } - if (block->log_structured) + if (block->has_objects) WT_RET(__wt_vpack_uint(pp, 0, l)); WT_RET(__wt_vpack_uint(pp, 0, o)); WT_RET(__wt_vpack_uint(pp, 0, s)); @@ -82,10 +82,10 @@ __wt_block_addr_to_buffer( * reference. */ int -__wt_block_buffer_to_addr(WT_BLOCK *block, const uint8_t *p, uint32_t *logidp, wt_off_t *offsetp, +__wt_block_buffer_to_addr(WT_BLOCK *block, const uint8_t *p, uint32_t *objectidp, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump) { - return (__block_buffer_to_addr(block, &p, logidp, offsetp, sizep, checksump)); + return (__block_buffer_to_addr(block, &p, objectidp, offsetp, sizep, checksump)); } /* @@ -97,14 +97,14 @@ __wt_block_addr_invalid( WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, bool live) { wt_off_t offset; - uint32_t checksum, logid, size; + uint32_t checksum, objectid, size; WT_UNUSED(session); WT_UNUSED(addr_size); WT_UNUSED(live); /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); #ifdef HAVE_DIAGNOSTIC /* @@ -116,7 +116,7 @@ __wt_block_addr_invalid( #endif /* Check if the address is past the end of the file. */ - return (logid == block->logid && offset + size > block->size ? EINVAL : 0); + return (objectid == block->objectid && offset + size > block->size ? EINVAL : 0); } /* @@ -128,16 +128,16 @@ __wt_block_addr_string( WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) { wt_off_t offset; - uint32_t checksum, logid, size; + uint32_t checksum, objectid, size; WT_UNUSED(addr_size); /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); /* Printable representation. */ WT_RET(__wt_buf_fmt(session, buf, - "[%" PRIu32 ": %" PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]", logid, + "[%" PRIu32 ": %" PRIuMAX "-%" PRIuMAX ", %" PRIu32 ", %" PRIu32 "]", objectid, (uintmax_t)offset, (uintmax_t)offset + size, size, checksum)); return (0); @@ -160,12 +160,12 @@ __block_buffer_to_ckpt( pp = &p; WT_RET(__block_buffer_to_addr( - block, pp, &ci->root_logid, &ci->root_offset, &ci->root_size, &ci->root_checksum)); + block, pp, &ci->root_objectid, &ci->root_offset, &ci->root_size, &ci->root_checksum)); WT_RET(__block_buffer_to_addr( - block, pp, &ci->alloc.logid, &ci->alloc.offset, &ci->alloc.size, &ci->alloc.checksum)); + block, pp, &ci->alloc.objectid, &ci->alloc.offset, &ci->alloc.size, &ci->alloc.checksum)); WT_RET(__block_buffer_to_addr( - block, pp, &ci->avail.logid, &ci->avail.offset, &ci->avail.size, &ci->avail.checksum)); - WT_RET(__block_buffer_to_addr(block, pp, &ci->discard.logid, &ci->discard.offset, + block, pp, &ci->avail.objectid, &ci->avail.offset, &ci->avail.size, &ci->avail.checksum)); + WT_RET(__block_buffer_to_addr(block, pp, &ci->discard.objectid, &ci->discard.offset, &ci->discard.size, &ci->discard.checksum)); WT_RET(__wt_vunpack_uint(pp, 0, &a)); ci->file_size = (wt_off_t)a; @@ -209,9 +209,9 @@ __wt_block_ckpt_to_buffer( WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t **pp, WT_BLOCK_CKPT *ci, bool skip_avail) { uint64_t a; - uint32_t logid; + uint32_t objectid; - logid = block->logid; + objectid = block->objectid; if (ci->version != WT_BM_CHECKPOINT_VERSION) WT_RET_MSG(session, WT_ERROR, "unsupported checkpoint version"); @@ -220,16 +220,16 @@ __wt_block_ckpt_to_buffer( (*pp)++; WT_RET(__wt_block_addr_to_buffer( - block, pp, logid, ci->root_offset, ci->root_size, ci->root_checksum)); + block, pp, objectid, ci->root_offset, ci->root_size, ci->root_checksum)); WT_RET(__wt_block_addr_to_buffer( - block, pp, logid, ci->alloc.offset, ci->alloc.size, ci->alloc.checksum)); + block, pp, objectid, ci->alloc.offset, ci->alloc.size, ci->alloc.checksum)); if (skip_avail) WT_RET(__wt_block_addr_to_buffer(block, pp, 0, 0, 0, 0)); else WT_RET(__wt_block_addr_to_buffer( - block, pp, logid, ci->avail.offset, ci->avail.size, ci->avail.checksum)); + block, pp, objectid, ci->avail.offset, ci->avail.size, ci->avail.checksum)); WT_RET(__wt_block_addr_to_buffer( - block, pp, logid, ci->discard.offset, ci->discard.size, ci->discard.checksum)); + block, pp, objectid, ci->discard.offset, ci->discard.size, ci->discard.checksum)); a = (uint64_t)ci->file_size; WT_RET(__wt_vpack_uint(pp, 0, a)); a = ci->ckpt_size; diff --git a/src/third_party/wiredtiger/src/block/block_ckpt.c b/src/third_party/wiredtiger/src/block/block_ckpt.c index 0f4af019b52..8194b8212de 100644 --- a/src/third_party/wiredtiger/src/block/block_ckpt.c +++ b/src/third_party/wiredtiger/src/block/block_ckpt.c @@ -95,7 +95,7 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint if (ci->root_offset != WT_BLOCK_INVALID_OFFSET) { endp = root_addr; WT_ERR(__wt_block_addr_to_buffer( - block, &endp, ci->root_logid, ci->root_offset, ci->root_size, ci->root_checksum)); + block, &endp, ci->root_objectid, ci->root_offset, ci->root_size, ci->root_checksum)); *root_addr_sizep = WT_PTRDIFF(endp, root_addr); WT_ERR(__wt_block_tiered_load(session, block, ci)); @@ -115,7 +115,7 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint * the end of the file, that was done when the checkpoint was first written (re-writing the * checkpoint might possibly make it relevant here, but it's unlikely enough I don't bother). */ - if (!checkpoint && !block->log_structured) + if (!checkpoint && !block->has_objects) WT_ERR(__wt_block_truncate(session, block, ci->file_size)); if (0) { @@ -239,9 +239,9 @@ __wt_block_checkpoint( */ if (buf == NULL) { ci->root_offset = WT_BLOCK_INVALID_OFFSET; - ci->root_logid = ci->root_size = ci->root_checksum = 0; + ci->root_objectid = ci->root_size = ci->root_checksum = 0; } else - WT_ERR(__wt_block_write_off(session, block, buf, &ci->root_logid, &ci->root_offset, + WT_ERR(__wt_block_write_off(session, block, buf, &ci->root_objectid, &ci->root_offset, &ci->root_size, &ci->root_checksum, data_checksum, true, false)); /* @@ -612,7 +612,7 @@ __ckpt_process(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase) * lists, and the freed blocks will then be included when writing the live extent lists. */ WT_CKPT_FOREACH (ckptbase, ckpt) { - if (F_ISSET(ckpt, WT_CKPT_FAKE) || !F_ISSET(ckpt, WT_CKPT_DELETE) || block->log_structured) + if (F_ISSET(ckpt, WT_CKPT_FAKE) || !F_ISSET(ckpt, WT_CKPT_DELETE) || block->has_objects) continue; if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) @@ -750,7 +750,7 @@ live_update: * TODO: tiered: for now we are switching files on a checkpoint, we'll want to do it only on * flush_tier. */ - if (block->log_structured) + if (block->has_objects) WT_ERR(__wt_block_tiered_newfile(session, block)); #ifdef HAVE_DIAGNOSTIC diff --git a/src/third_party/wiredtiger/src/block/block_ckpt_scan.c b/src/third_party/wiredtiger/src/block/block_ckpt_scan.c index 0b2629cce83..8b87fb5ab62 100644 --- a/src/third_party/wiredtiger/src/block/block_ckpt_scan.c +++ b/src/third_party/wiredtiger/src/block/block_ckpt_scan.c @@ -220,15 +220,15 @@ __wt_block_checkpoint_last(WT_SESSION_IMPL *session, WT_BLOCK *block, char **met const WT_PAGE_HEADER *dsk; wt_off_t ext_off, ext_size, offset; uint64_t len, nblocks, write_gen; - uint32_t checksum, logid, size; + uint32_t checksum, objectid, size; const uint8_t *p, *t; bool found; *metadatap = *checkpoint_listp = NULL; WT_RET(__wt_buf_init(session, checkpoint, WT_BLOCK_CHECKPOINT_BUFFER)); - /* TODO: scan all log IDs. */ - logid = 0; + /* TODO: tiered: scan all object IDs. */ + objectid = 0; /* * Initialize a pair of structures that track the best and current checkpoints found so far. @@ -282,7 +282,7 @@ __wt_block_checkpoint_last(WT_SESSION_IMPL *session, WT_BLOCK *block, char **met * block isn't valid, skip to the next possible block. */ if (__wt_block_offset_invalid(block, offset, size) || - __wt_block_read_off(session, block, tmp, logid, offset, size, checksum) != 0) { + __wt_block_read_off(session, block, tmp, objectid, offset, size, checksum) != 0) { size = WT_BTREE_MIN_ALLOC_SIZE; continue; } diff --git a/src/third_party/wiredtiger/src/block/block_compact.c b/src/third_party/wiredtiger/src/block/block_compact.c index 740abcf6141..735b4d5c4f4 100644 --- a/src/third_party/wiredtiger/src/block/block_compact.c +++ b/src/third_party/wiredtiger/src/block/block_compact.c @@ -136,13 +136,13 @@ __wt_block_compact_page_skip( WT_EXT *ext; WT_EXTLIST *el; wt_off_t limit, offset; - uint32_t checksum, logid, size; + uint32_t checksum, objectid, size; WT_UNUSED(addr_size); *skipp = true; /* Return a default skip. */ /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); /* * If this block is in the chosen percentage of the file and there's a block on the available diff --git a/src/third_party/wiredtiger/src/block/block_ext.c b/src/third_party/wiredtiger/src/block/block_ext.c index 310330da831..2d95ed973cd 100644 --- a/src/third_party/wiredtiger/src/block/block_ext.c +++ b/src/third_party/wiredtiger/src/block/block_ext.c @@ -563,24 +563,24 @@ __wt_block_free(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, { WT_DECL_RET; wt_off_t offset; - uint32_t checksum, logid, size; + uint32_t checksum, objectid, size; WT_UNUSED(addr_size); WT_STAT_DATA_INCR(session, block_free); /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); - __wt_verbose(session, WT_VERB_BLOCK, "free %" PRIu32 ": %" PRIdMAX "/%" PRIdMAX, logid, + __wt_verbose(session, WT_VERB_BLOCK, "free %" PRIu32 ": %" PRIdMAX "/%" PRIdMAX, objectid, (intmax_t)offset, (intmax_t)size); #ifdef HAVE_DIAGNOSTIC WT_RET(__wt_block_misplaced(session, block, "free", offset, size, true, __func__, __LINE__)); #endif - if (logid == block->logid) { + if (objectid == block->objectid) { WT_RET(__wt_block_ext_prealloc(session, 5)); __wt_spin_lock(session, &block->live_lock); - ret = __wt_block_off_free(session, block, logid, offset, (wt_off_t)size); + ret = __wt_block_off_free(session, block, objectid, offset, (wt_off_t)size); __wt_spin_unlock(session, &block->live_lock); } else { /* TODO: update stats about older files to drive garbage collection. */ @@ -595,7 +595,7 @@ __wt_block_free(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, */ int __wt_block_off_free( - WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t logid, wt_off_t offset, wt_off_t size) + WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t objectid, wt_off_t offset, wt_off_t size) { WT_DECL_RET; @@ -603,7 +603,7 @@ __wt_block_off_free( WT_ASSERT(session, WT_SESSION_BTREE_SYNC_SAFE(session, S2BT(session))); /* TODO: track stats for old files to drive garbage collection. */ - if (logid != block->logid) + if (objectid != block->objectid) return (0); /* @@ -1103,7 +1103,8 @@ __wt_block_extlist_read( return (0); WT_RET(__wt_scr_alloc(session, el->size, &tmp)); - WT_ERR(__wt_block_read_off(session, block, tmp, el->logid, el->offset, el->size, el->checksum)); + WT_ERR( + __wt_block_read_off(session, block, tmp, el->objectid, el->offset, el->size, el->checksum)); p = WT_BLOCK_HEADER_BYTE(tmp->mem); WT_ERR(__wt_extlist_read_pair(&p, &off, &size)); @@ -1163,7 +1164,7 @@ __wt_block_extlist_write( WT_EXT *ext; WT_PAGE_HEADER *dsk; size_t size; - uint32_t logid, entries; + uint32_t objectid, entries; uint8_t *p; WT_RET(__block_extlist_dump(session, block, el, "write")); @@ -1221,8 +1222,8 @@ __wt_block_extlist_write( /* Write the extent list to disk. */ WT_ERR(__wt_block_write_off( - session, block, tmp, &logid, &el->offset, &el->size, &el->checksum, true, true, true)); - WT_UNUSED(logid); /* TODO check */ + session, block, tmp, &objectid, &el->offset, &el->size, &el->checksum, true, true, true)); + WT_UNUSED(objectid); /* TODO: tiered: check */ /* * Remove the allocated blocks from the system's allocation list, extent blocks never appear on diff --git a/src/third_party/wiredtiger/src/block/block_open.c b/src/third_party/wiredtiger/src/block/block_open.c index 169c1cd5d87..c41de4aaaaa 100644 --- a/src/third_party/wiredtiger/src/block/block_open.c +++ b/src/third_party/wiredtiger/src/block/block_open.c @@ -99,10 +99,10 @@ __block_destroy(WT_SESSION_IMPL *session, WT_BLOCK *block) __wt_free(session, block->name); - if (block->log_structured && block->lfh != NULL) { - for (i = 0; i < block->max_logid; i++) - WT_TRET(__wt_close(session, &block->lfh[i])); - __wt_free(session, block->lfh); + if (block->has_objects && block->ofh != NULL) { + for (i = 0; i < block->max_objectid; i++) + WT_TRET(__wt_close(session, &block->ofh[i])); + __wt_free(session, block->ofh); } if (block->fh != NULL) @@ -182,7 +182,6 @@ __wt_block_open(WT_SESSION_IMPL *session, const char *filename, const char *cfg[ WT_ERR(__wt_config_gets(session, cfg, "block_allocation", &cval)); block->allocfirst = WT_STRING_MATCH("first", cval.str, cval.len); - block->log_structured = WT_STRING_MATCH("log-structured", cval.str, cval.len); /* Configuration: optional OS buffer cache maximum size. */ WT_ERR(__wt_config_gets(session, cfg, "os_cache_max", &cval)); diff --git a/src/third_party/wiredtiger/src/block/block_read.c b/src/third_party/wiredtiger/src/block/block_read.c index 80dbb3aac21..08c1ce8067f 100644 --- a/src/third_party/wiredtiger/src/block/block_read.c +++ b/src/third_party/wiredtiger/src/block/block_read.c @@ -21,7 +21,7 @@ __wt_bm_preload(WT_BM *bm, WT_SESSION_IMPL *session, const uint8_t *addr, size_t WT_FH *fh; WT_FILE_HANDLE *handle; wt_off_t offset; - uint32_t checksum, logid, size; + uint32_t checksum, objectid, size; bool mapped; block = bm->block; @@ -29,9 +29,9 @@ __wt_bm_preload(WT_BM *bm, WT_SESSION_IMPL *session, const uint8_t *addr, size_t WT_STAT_CONN_INCR(session, block_preload); /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); - WT_RET(__wt_block_fh(session, block, logid, &fh)); + WT_RET(__wt_block_fh(session, block, objectid, &fh)); handle = fh->handle; mapped = bm->map != NULL && offset + size <= (wt_off_t)bm->maplen; if (mapped && handle->fh_map_preload != NULL) @@ -64,19 +64,19 @@ __wt_bm_read( WT_FH *fh; WT_FILE_HANDLE *handle; wt_off_t offset; - uint32_t checksum, logid, size; + uint32_t checksum, objectid, size; bool mapped; WT_UNUSED(addr_size); block = bm->block; /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); /* * Map the block if it's possible. */ - WT_RET(__wt_block_fh(session, block, logid, &fh)); + WT_RET(__wt_block_fh(session, block, objectid, &fh)); handle = fh->handle; mapped = bm->map != NULL && offset + size <= (wt_off_t)bm->maplen; if (mapped && handle->fh_map_preload != NULL) { @@ -100,7 +100,7 @@ __wt_bm_read( #endif /* Read the block. */ __wt_capacity_throttle(session, size, WT_THROTTLE_READ); - WT_RET(__wt_block_read_off(session, block, buf, logid, offset, size, checksum)); + WT_RET(__wt_block_read_off(session, block, buf, objectid, offset, size, checksum)); /* Optionally discard blocks from the system's buffer cache. */ WT_RET(__wt_block_discard(session, block, (size_t)size)); @@ -113,7 +113,7 @@ __wt_bm_read( * Dump a block into the log in 1KB chunks. */ static int -__wt_bm_corrupt_dump(WT_SESSION_IMPL *session, WT_ITEM *buf, uint32_t logid, wt_off_t offset, +__wt_bm_corrupt_dump(WT_SESSION_IMPL *session, WT_ITEM *buf, uint32_t objectid, wt_off_t offset, uint32_t size, uint32_t checksum) WT_GCC_FUNC_ATTRIBUTE((cold)) { WT_DECL_ITEM(tmp); @@ -122,7 +122,7 @@ __wt_bm_corrupt_dump(WT_SESSION_IMPL *session, WT_ITEM *buf, uint32_t logid, wt_ #define WT_CORRUPT_FMT "{%" PRIu32 ": %" PRIuMAX ", %" PRIu32 ", %#" PRIx32 "}" if (buf->size == 0) { - __wt_errx(session, WT_CORRUPT_FMT ": empty buffer, no dump available", logid, + __wt_errx(session, WT_CORRUPT_FMT ": empty buffer, no dump available", objectid, (uintmax_t)offset, size, checksum); return (0); } @@ -134,7 +134,7 @@ __wt_bm_corrupt_dump(WT_SESSION_IMPL *session, WT_ITEM *buf, uint32_t logid, wt_ WT_ERR(__wt_buf_catfmt(session, tmp, "%02x ", ((uint8_t *)buf->data)[i])); if (++i == buf->size || i % 1024 == 0) { __wt_errx(session, - WT_CORRUPT_FMT ": (chunk %" WT_SIZET_FMT " of %" WT_SIZET_FMT "): %.*s", logid, + WT_CORRUPT_FMT ": (chunk %" WT_SIZET_FMT " of %" WT_SIZET_FMT "): %.*s", objectid, (uintmax_t)offset, size, checksum, ++chunk, nchunks, (int)tmp->size, (char *)tmp->data); if (i == buf->size) @@ -158,15 +158,15 @@ __wt_bm_corrupt(WT_BM *bm, WT_SESSION_IMPL *session, const uint8_t *addr, size_t WT_DECL_ITEM(tmp); WT_DECL_RET; wt_off_t offset; - uint32_t checksum, logid, size; + uint32_t checksum, objectid, size; /* Read the block. */ WT_RET(__wt_scr_alloc(session, 0, &tmp)); WT_ERR(__wt_bm_read(bm, session, tmp, addr, addr_size)); /* Crack the cookie, dump the block. */ - WT_ERR(__wt_block_buffer_to_addr(bm->block, addr, &logid, &offset, &size, &checksum)); - WT_ERR(__wt_bm_corrupt_dump(session, tmp, logid, offset, size, checksum)); + WT_ERR(__wt_block_buffer_to_addr(bm->block, addr, &objectid, &offset, &size, &checksum)); + WT_ERR(__wt_bm_corrupt_dump(session, tmp, objectid, offset, size, checksum)); err: __wt_scr_free(session, &tmp); @@ -211,39 +211,40 @@ err: * Get a block file handle. */ int -__wt_block_fh(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t logid, WT_FH **fhp) +__wt_block_fh(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t objectid, WT_FH **fhp) { WT_DECL_ITEM(tmp); WT_DECL_RET; const char *filename; - if (!block->log_structured || logid == block->logid) { + if (!block->has_objects || objectid == block->objectid) { *fhp = block->fh; return (0); } /* TODO: tiered: fh readlock; we may want a reference count on each file handle given out. */ - if (logid * sizeof(WT_FILE_HANDLE *) < block->lfh_alloc && (*fhp = block->lfh[logid]) != NULL) + if (objectid * sizeof(WT_FILE_HANDLE *) < block->ofh_alloc && + (*fhp = block->ofh[objectid]) != NULL) return (0); /* TODO: tiered: fh writelock */ /* Ensure the array goes far enough. */ - WT_RET(__wt_realloc_def(session, &block->lfh_alloc, logid + 1, &block->lfh)); - if (logid >= block->max_logid) - block->max_logid = logid + 1; - if ((*fhp = block->lfh[logid]) != NULL) + WT_RET(__wt_realloc_def(session, &block->ofh_alloc, objectid + 1, &block->ofh)); + if (objectid >= block->max_objectid) + block->max_objectid = objectid + 1; + if ((*fhp = block->ofh[objectid]) != NULL) return (0); WT_RET(__wt_scr_alloc(session, 0, &tmp)); - if (logid == 0) + if (objectid == 0) filename = block->name; else { - WT_ERR(__wt_buf_fmt(session, tmp, "%s.%08" PRIu32, block->name, logid)); + WT_ERR(__wt_buf_fmt(session, tmp, "%s.%08" PRIu32, block->name, objectid)); filename = tmp->data; } WT_ERR(__wt_open(session, filename, WT_FS_OPEN_FILE_TYPE_DATA, - WT_FS_OPEN_READONLY | block->file_flags, &block->lfh[logid])); - *fhp = block->lfh[logid]; + WT_FS_OPEN_READONLY | block->file_flags, &block->ofh[objectid])); + *fhp = block->ofh[objectid]; WT_ASSERT(session, *fhp != NULL); err: @@ -256,7 +257,7 @@ err: * Read an addr/size pair referenced block into a buffer. */ int -__wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint32_t logid, +__wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint32_t objectid, wt_off_t offset, uint32_t size, uint32_t checksum) { WT_BLOCK_HEADER *blk, swap; @@ -293,7 +294,7 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uin block->name, size, block->allocsize); WT_RET(__wt_buf_init(session, buf, bufsize)); - WT_RET(__wt_block_fh(session, block, logid, &fh)); + WT_RET(__wt_block_fh(session, block, objectid, &fh)); WT_RET(__wt_read(session, fh, offset, size, buf->mem)); buf->size = size; @@ -327,7 +328,7 @@ __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uin block->name, size, (uintmax_t)offset, swap.checksum, checksum); if (!F_ISSET(session, WT_SESSION_QUIET_CORRUPT_FILE)) - WT_IGNORE_RET(__wt_bm_corrupt_dump(session, buf, logid, offset, size, checksum)); + WT_IGNORE_RET(__wt_bm_corrupt_dump(session, buf, objectid, offset, size, checksum)); /* Panic if a checksum fails during an ordinary read. */ F_SET(S2C(session), WT_CONN_DATA_CORRUPTION); diff --git a/src/third_party/wiredtiger/src/block/block_slvg.c b/src/third_party/wiredtiger/src/block/block_slvg.c index 7e09d8ab147..280d520d1dd 100644 --- a/src/third_party/wiredtiger/src/block/block_slvg.c +++ b/src/third_party/wiredtiger/src/block/block_slvg.c @@ -104,13 +104,13 @@ __wt_block_salvage_next( WT_DECL_RET; WT_FH *fh; wt_off_t max, offset; - uint32_t allocsize, checksum, logid, size; + uint32_t allocsize, checksum, objectid, size; uint8_t *endp; *eofp = 0; - /* FIXME: salvage across all chunks in a log-structured tree. */ - logid = 0; + /* TODO: tiered: salvage across all objects in a tiered tree. */ + objectid = 0; fh = block->fh; allocsize = block->allocsize; @@ -140,19 +140,19 @@ __wt_block_salvage_next( * otherwise, move past it. */ if (!__wt_block_offset_invalid(block, offset, size) && - __wt_block_read_off(session, block, tmp, logid, offset, size, checksum) == 0) + __wt_block_read_off(session, block, tmp, objectid, offset, size, checksum) == 0) break; /* Free the allocation-size block. */ __wt_verbose(session, WT_VERB_SALVAGE, "skipping %" PRIu32 "B at file offset %" PRIuMAX, allocsize, (uintmax_t)offset); - WT_ERR(__wt_block_off_free(session, block, logid, offset, (wt_off_t)allocsize)); + WT_ERR(__wt_block_off_free(session, block, objectid, offset, (wt_off_t)allocsize)); block->slvg_off += allocsize; } /* Re-create the address cookie that should reference this block. */ endp = addr; - WT_ERR(__wt_block_addr_to_buffer(block, &endp, logid, offset, size, checksum)); + WT_ERR(__wt_block_addr_to_buffer(block, &endp, objectid, offset, size, checksum)); *addr_sizep = WT_PTRDIFF(endp, addr); done: @@ -170,7 +170,7 @@ __wt_block_salvage_valid( WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t *addr, size_t addr_size, bool valid) { wt_off_t offset; - uint32_t size, logid, checksum; + uint32_t size, objectid, checksum; WT_UNUSED(addr_size); @@ -178,11 +178,11 @@ __wt_block_salvage_valid( * Crack the cookie. If the upper layer took the block, move past it; if the upper layer * rejected the block, move past an allocation size chunk and free it. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); if (valid) block->slvg_off = offset + size; else { - WT_RET(__wt_block_off_free(session, block, logid, offset, (wt_off_t)block->allocsize)); + WT_RET(__wt_block_off_free(session, block, objectid, offset, (wt_off_t)block->allocsize)); block->slvg_off = offset + block->allocsize; } diff --git a/src/third_party/wiredtiger/src/block/block_tiered.c b/src/third_party/wiredtiger/src/block/block_tiered.c index d922a663e03..b275ccd95a7 100644 --- a/src/third_party/wiredtiger/src/block/block_tiered.c +++ b/src/third_party/wiredtiger/src/block/block_tiered.c @@ -34,8 +34,8 @@ __wt_block_tiered_load(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_BLOCK_CKPT * TODO: tiered: this call currently advances the object id, that's probably not appropriate for * readonly opens. Perhaps it's also not appropriate for opening at an older checkpoint? */ - if (block->log_structured) { - block->logid = ci->root_logid; + if (block->has_objects) { + block->objectid = ci->root_objectid; /* Advance to the next file for future changes. */ WT_RET(__wt_block_tiered_newfile(session, block)); @@ -64,8 +64,8 @@ __wt_block_tiered_newfile(WT_SESSION_IMPL *session, WT_BLOCK *block) WT_ERR(__wt_close(session, &block->fh)); /* Bump to a new file ID. */ - ++block->logid; - WT_ERR(__wt_buf_fmt(session, tmp, "%s.%08" PRIu32, block->name, block->logid)); + ++block->objectid; + WT_ERR(__wt_buf_fmt(session, tmp, "%s.%08" PRIu32, block->name, block->objectid)); filename = tmp->data; WT_WITH_BUCKET_STORAGE(session->bucket_storage, session, { diff --git a/src/third_party/wiredtiger/src/block/block_vrfy.c b/src/third_party/wiredtiger/src/block/block_vrfy.c index 9785be9d12d..28c95415254 100644 --- a/src/third_party/wiredtiger/src/block/block_vrfy.c +++ b/src/third_party/wiredtiger/src/block/block_vrfy.c @@ -316,12 +316,12 @@ __wt_block_verify_addr( WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size) { wt_off_t offset; - uint32_t checksum, logid, size; + uint32_t checksum, objectid, size; WT_UNUSED(addr_size); /* Crack the cookie. */ - WT_RET(__wt_block_buffer_to_addr(block, addr, &logid, &offset, &size, &checksum)); + WT_RET(__wt_block_buffer_to_addr(block, addr, &objectid, &offset, &size, &checksum)); /* Add to the per-file list. */ WT_RET(__verify_filefrag_add(session, block, NULL, offset, size, false)); diff --git a/src/third_party/wiredtiger/src/block/block_write.c b/src/third_party/wiredtiger/src/block/block_write.c index 283b2f2a12d..ecbc3d02553 100644 --- a/src/third_party/wiredtiger/src/block/block_write.c +++ b/src/third_party/wiredtiger/src/block/block_write.c @@ -189,14 +189,14 @@ __wt_block_write(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint8_ size_t *addr_sizep, bool data_checksum, bool checkpoint_io) { wt_off_t offset; - uint32_t checksum, logid, size; + uint32_t checksum, objectid, size; uint8_t *endp; - WT_RET(__wt_block_write_off( - session, block, buf, &logid, &offset, &size, &checksum, data_checksum, checkpoint_io, false)); + WT_RET(__wt_block_write_off(session, block, buf, &objectid, &offset, &size, &checksum, + data_checksum, checkpoint_io, false)); endp = addr; - WT_RET(__wt_block_addr_to_buffer(block, &endp, logid, offset, size, checksum)); + WT_RET(__wt_block_addr_to_buffer(block, &endp, objectid, offset, size, checksum)); *addr_sizep = WT_PTRDIFF(endp, addr); return (0); @@ -207,7 +207,7 @@ __wt_block_write(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint8_ * Write a buffer into a block, returning the block's offset, size and checksum. */ static int -__block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint32_t *logidp, +__block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint32_t *objectidp, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump, bool data_checksum, bool checkpoint_io, bool caller_locked) { @@ -216,7 +216,7 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint3 WT_FH *fh; wt_off_t offset; size_t align_size; - uint32_t checksum, logid; + uint32_t checksum, objectid; uint8_t *file_sizep; bool local_locked; @@ -225,7 +225,7 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint3 *checksump = 0; /* -Werror=maybe-uninitialized */ fh = block->fh; - logid = block->logid; + objectid = block->objectid; /* Buffers should be aligned for writing. */ if (!F_ISSET(buf, WT_ITEM_ALIGNED)) { @@ -327,7 +327,7 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint3 if ((ret = __wt_write(session, fh, offset, align_size, buf->mem)) != 0) { if (!caller_locked) __wt_spin_lock(session, &block->live_lock); - WT_TRET(__wt_block_off_free(session, block, logid, offset, (wt_off_t)align_size)); + WT_TRET(__wt_block_off_free(session, block, objectid, offset, (wt_off_t)align_size)); if (!caller_locked) __wt_spin_unlock(session, &block->live_lock); WT_RET(ret); @@ -361,7 +361,7 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint3 __wt_verbose(session, WT_VERB_WRITE, "off %" PRIuMAX ", size %" PRIuMAX ", checksum %#" PRIx32, (uintmax_t)offset, (uintmax_t)align_size, checksum); - *logidp = logid; + *objectidp = objectid; *offsetp = offset; *sizep = WT_STORE_SIZE(align_size); *checksump = checksum; @@ -374,7 +374,7 @@ __block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint3 * Write a buffer into a block, returning the block's offset, size and checksum. */ int -__wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint32_t *logidp, +__wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint32_t *objectidp, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump, bool data_checksum, bool checkpoint_io, bool caller_locked) { @@ -386,8 +386,8 @@ __wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, ui * never see anything other than their original content. */ __wt_page_header_byteswap(buf->mem); - ret = __block_write_off(session, block, buf, logidp, offsetp, sizep, checksump, data_checksum, - checkpoint_io, caller_locked); + ret = __block_write_off(session, block, buf, objectidp, offsetp, sizep, checksump, + data_checksum, checkpoint_io, caller_locked); __wt_page_header_byteswap(buf->mem); return (ret); } diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index c68855988a2..1093ed26b57 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -390,7 +390,7 @@ __wt_debug_offset( */ block = S2BT(session)->bm->block; endp = addr; - WT_RET(__wt_block_addr_to_buffer(block, &endp, block->logid, offset, size, checksum)); + WT_RET(__wt_block_addr_to_buffer(block, &endp, block->objectid, offset, size, checksum)); /* * Read the address through the btree I/O functions (so the block is decompressed as necessary). diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index 654abaf40d5..ca7b6bca448 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -270,7 +270,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create[] = { {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 4}, - {"block_allocation", "string", NULL, "choices=[\"best\",\"first\",\"log-structured\"]", NULL, 0}, + {"block_allocation", "string", NULL, "choices=[\"best\",\"first\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"checksum", "string", NULL, "choices=[\"on\",\"off\",\"uncompressed\"]", NULL, 0}, @@ -424,7 +424,7 @@ static const WT_CONFIG_CHECK confchk_file_config[] = { {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 4}, - {"block_allocation", "string", NULL, "choices=[\"best\",\"first\",\"log-structured\"]", NULL, 0}, + {"block_allocation", "string", NULL, "choices=[\"best\",\"first\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"checksum", "string", NULL, "choices=[\"on\",\"off\",\"uncompressed\"]", NULL, 0}, @@ -469,7 +469,7 @@ static const WT_CONFIG_CHECK confchk_file_meta[] = { {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 4}, - {"block_allocation", "string", NULL, "choices=[\"best\",\"first\",\"log-structured\"]", NULL, 0}, + {"block_allocation", "string", NULL, "choices=[\"best\",\"first\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"checkpoint", "string", NULL, NULL, NULL, 0}, {"checkpoint_backup_info", "string", NULL, NULL, NULL, 0}, @@ -534,7 +534,7 @@ static const WT_CONFIG_CHECK confchk_lsm_meta[] = { {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 4}, - {"block_allocation", "string", NULL, "choices=[\"best\",\"first\",\"log-structured\"]", NULL, 0}, + {"block_allocation", "string", NULL, "choices=[\"best\",\"first\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"checksum", "string", NULL, "choices=[\"on\",\"off\",\"uncompressed\"]", NULL, 0}, @@ -581,7 +581,7 @@ static const WT_CONFIG_CHECK confchk_object_meta[] = { {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 4}, - {"block_allocation", "string", NULL, "choices=[\"best\",\"first\",\"log-structured\"]", NULL, 0}, + {"block_allocation", "string", NULL, "choices=[\"best\",\"first\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"checkpoint", "string", NULL, NULL, NULL, 0}, {"checkpoint_backup_info", "string", NULL, NULL, NULL, 0}, @@ -645,7 +645,7 @@ static const WT_CONFIG_CHECK confchk_tier_meta[] = { {"allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0}, {"app_metadata", "string", NULL, NULL, NULL, 0}, {"assert", "category", NULL, NULL, confchk_assert_subconfigs, 4}, - {"block_allocation", "string", NULL, "choices=[\"best\",\"first\",\"log-structured\"]", NULL, 0}, + {"block_allocation", "string", NULL, "choices=[\"best\",\"first\"]", NULL, 0}, {"block_compressor", "string", NULL, NULL, NULL, 0}, {"bucket", "string", NULL, NULL, NULL, 0}, {"bucket_prefix", "string", NULL, NULL, NULL, 0}, {"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"checkpoint", "string", NULL, NULL, NULL, 0}, diff --git a/src/third_party/wiredtiger/src/include/block.h b/src/third_party/wiredtiger/src/include/block.h index b8a982e1713..2006be2f9f4 100644 --- a/src/third_party/wiredtiger/src/include/block.h +++ b/src/third_party/wiredtiger/src/include/block.h @@ -51,7 +51,7 @@ struct __wt_extlist { uint64_t bytes; /* Byte count */ uint32_t entries; /* Entry count */ - uint32_t logid; /* Written log ID */ + uint32_t objectid; /* Written object ID */ wt_off_t offset; /* Written extent offset */ uint32_t checksum; /* Written extent checksum */ uint32_t size; /* Written extent size */ @@ -141,7 +141,7 @@ struct __wt_size { struct __wt_block_ckpt { uint8_t version; /* Version */ - uint32_t root_logid; + uint32_t root_objectid; wt_off_t root_offset; /* The root */ uint32_t root_checksum, root_size; @@ -239,17 +239,17 @@ struct __wt_block { /* Configuration information, set when the file is opened. */ uint32_t allocfirst; /* Allocation is first-fit */ uint32_t allocsize; /* Allocation size */ - bool log_structured; /* Write checkpoint as separate files */ + bool has_objects; /* Address cookies contain object id */ size_t os_cache; /* System buffer cache flush max */ size_t os_cache_max; size_t os_cache_dirty_max; u_int block_header; /* Header length */ - /* Log-structured tracking. */ - uint32_t file_flags, logid, max_logid; - WT_FH **lfh; - size_t lfh_alloc; + /* Object file tracking. */ + uint32_t file_flags, objectid, max_objectid; + WT_FH **ofh; + size_t ofh_alloc; /* * There is only a single checkpoint in a file that can be written. The information could diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 1389361119c..3d0eb0554fd 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -87,11 +87,12 @@ extern int __wt_block_addr_invalid(WT_SESSION_IMPL *session, WT_BLOCK *block, co size_t addr_size, bool live) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_addr_string(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_addr_to_buffer(WT_BLOCK *block, uint8_t **pp, uint32_t logid, wt_off_t offset, - uint32_t size, uint32_t checksum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_addr_to_buffer(WT_BLOCK *block, uint8_t **pp, uint32_t objectid, + wt_off_t offset, uint32_t size, uint32_t checksum) + WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_alloc(WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t *offp, wt_off_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_buffer_to_addr(WT_BLOCK *block, const uint8_t *p, uint32_t *logidp, +extern int __wt_block_buffer_to_addr(WT_BLOCK *block, const uint8_t *p, uint32_t *objectidp, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_buffer_to_ckpt(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *p, @@ -153,7 +154,7 @@ extern int __wt_block_extlist_truncate(WT_SESSION_IMPL *session, WT_BLOCK *block WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_extlist_write(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *el, WT_EXTLIST *additional) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_fh(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t logid, WT_FH **fhp) +extern int __wt_block_fh(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t objectid, WT_FH **fhp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_free(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -175,7 +176,7 @@ extern int __wt_block_map(WT_SESSION_IMPL *session, WT_BLOCK *block, void *mappe extern int __wt_block_misplaced(WT_SESSION_IMPL *session, WT_BLOCK *block, const char *list, wt_off_t offset, uint32_t size, bool live, const char *func, int line) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_off_free(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t logid, +extern int __wt_block_off_free(WT_SESSION_IMPL *session, WT_BLOCK *block, uint32_t objectid, wt_off_t offset, wt_off_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_off_remove_overlap(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_EXTLIST *el, wt_off_t off, wt_off_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -183,7 +184,7 @@ extern int __wt_block_open(WT_SESSION_IMPL *session, const char *filename, const bool forced_salvage, bool readonly, uint32_t allocsize, WT_BLOCK **blockp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_read_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, - uint32_t logid, wt_off_t offset, uint32_t size, uint32_t checksum) + uint32_t objectid, wt_off_t offset, uint32_t size, uint32_t checksum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_read_off_blind(WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t offset, uint32_t *sizep, uint32_t *checksump) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -217,7 +218,7 @@ extern int __wt_block_write(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM * size_t *addr_sizep, bool data_checksum, bool checkpoint_io) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, - uint32_t *logidp, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump, bool data_checksum, + uint32_t *objectidp, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump, bool data_checksum, bool checkpoint_io, bool caller_locked) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_write_size(WT_SESSION_IMPL *session, WT_BLOCK *block, size_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index c21c3dac748..14ee53f536a 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -1068,10 +1068,8 @@ struct __wt_session { * empty.} * @config{block_allocation, configure block allocation. Permitted values are \c "best" or * \c "first"; the \c "best" configuration uses a best-fit algorithm\, the \c "first" - * configuration uses a first-available algorithm during block allocation\, the \c - * "log-structure" configuration allocates a new file for each checkpoint., a string\, - * chosen from the following options: \c "best"\, \c "first"\, \c "log-structured"; default - * \c best.} + * configuration uses a first-available algorithm during block allocation., a string\, + * chosen from the following options: \c "best"\, \c "first"; default \c best.} * @config{block_compressor, configure a compressor for file blocks. Permitted values are * \c "none" or custom compression engine name created with WT_CONNECTION::add_compressor. * If WiredTiger has builtin support for \c "lz4"\, \c "snappy"\, \c "zlib" or \c "zstd" diff --git a/src/third_party/wiredtiger/src/utilities/util_list.c b/src/third_party/wiredtiger/src/utilities/util_list.c index eec4fe99cc2..e0953c0abfe 100644 --- a/src/third_party/wiredtiger/src/utilities/util_list.c +++ b/src/third_party/wiredtiger/src/utilities/util_list.c @@ -101,8 +101,8 @@ list_init_block(WT_SESSION *session, const char *key, WT_BLOCK *block) else if (ret != WT_NOTFOUND) WT_ERR(util_err(session, ret, "WT_CONFIG_PARSER.get")); - if ((ret = parser->get(parser, "block_allocation", &cval)) == 0) - block->log_structured = WT_STRING_MATCH("log_structured", cval.str, cval.len); + if (WT_PREFIX_MATCH(key, "tiered:")) + block->has_objects = true; err: if (parser != NULL && (tret = parser->close(parser)) != 0) { diff --git a/src/third_party/wiredtiger/test/suite/test_tiered02.py b/src/third_party/wiredtiger/test/suite/test_tiered02.py index 4b638a4015f..515e84388e5 100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered02.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered02.py @@ -30,23 +30,29 @@ import os, wiredtiger, wtscenario, wttest from wtdataset import SimpleDataSet # test_tiered02.py -# Test block-log-structured tree configuration options. +# Test tiered tree class test_tiered02(wttest.WiredTigerTestCase): K = 1024 M = 1024 * K G = 1024 * M + # TODO: tiered: change this to a table: URI, otherwise we are + # not using tiered files. uri = "file:test_tiered02" auth_token = "test_token" bucket = "mybucket" bucket_prefix = "pfx_" extension_name = "local_store" + prefix = "pfx-" def conn_config(self): os.makedirs(self.bucket, exist_ok=True) return \ - 'tiered_storage=(auth_token={},bucket={},bucket_prefix={},name={})'.format( \ - self.auth_token, self.bucket, self.bucket_prefix, self.extension_name) + 'statistics=(all),' + \ + 'tiered_storage=(auth_token=%s,' % self.auth_token + \ + 'bucket=%s,' % self.bucket + \ + 'bucket_prefix=%s,' % self.prefix + \ + 'name=%s)' % self.extension_name # Load the local store extension, but skip the test if it is missing. def conn_extensions(self, extlist): @@ -72,8 +78,7 @@ class test_tiered02(wttest.WiredTigerTestCase): # replaced with flush_tier. def test_tiered(self): self.flushed_objects = 0 - args = 'key_format=S,block_allocation=log-structured' - self.verbose(3, 'Test log-structured allocation with config: ' + args) + args = 'key_format=S' ds = SimpleDataSet(self, self.uri, 10, config=args) ds.populate() diff --git a/src/third_party/wiredtiger/test/suite/test_tiered03.py b/src/third_party/wiredtiger/test/suite/test_tiered03.py index 670b2e4202c..4c870e366c2 100644..100755 --- a/src/third_party/wiredtiger/test/suite/test_tiered03.py +++ b/src/third_party/wiredtiger/test/suite/test_tiered03.py @@ -36,6 +36,9 @@ class test_tiered03(wttest.WiredTigerTestCase): K = 1024 M = 1024 * K G = 1024 * M + # TODO: tiered: change this to a table: URI, otherwise we are + # not using tiered files. The use of a second directory for + # sharing would probably need to be reworked. uri = 'file:test_tiered03' # Occasionally add a lot of records, so that merges (and bloom) happen. @@ -44,12 +47,30 @@ class test_tiered03(wttest.WiredTigerTestCase): scenarios = wtscenario.make_scenarios(record_count_scenarios, prune=100, prunelong=500) + auth_token = "test_token" + bucket = "mybucket" + bucket_prefix = "pfx_" + extension_name = "local_store" + prefix = "pfx-" + + def conn_config(self): + if not os.path.exists(self.bucket): + os.mkdir(self.bucket) + return \ + 'statistics=(all),' + \ + 'tiered_storage=(auth_token=%s,' % self.auth_token + \ + 'bucket=%s,' % self.bucket + \ + 'bucket_prefix=%s,' % self.prefix + \ + 'name=%s)' % self.extension_name + + # Load the local store extension, but skip the test if it is missing. + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('storage_sources', self.extension_name) + # Test sharing data between a primary and a secondary def test_sharing(self): - args = 'block_allocation=log-structured' - self.verbose(3, - 'Test log-structured allocation with config: ' + args + ' count: ' + str(self.nrecs)) - ds = SimpleDataSet(self, self.uri, 10, config=args) + ds = SimpleDataSet(self, self.uri, 10) ds.populate() ds.check() self.session.checkpoint() @@ -71,7 +92,7 @@ class test_tiered03(wttest.WiredTigerTestCase): ds.check_cursor(cursor2) cursor2.close() - newds = SimpleDataSet(self, self.uri, 10000, config=args) + newds = SimpleDataSet(self, self.uri, 10000) newds.populate() newds.check() self.session.checkpoint() |