summaryrefslogtreecommitdiff
path: root/src/third_party/wiredtiger
diff options
context:
space:
mode:
authorLuke Chen <luke.chen@mongodb.com>2020-02-07 02:24:42 +0000
committerevergreen <evergreen@mongodb.com>2020-02-07 02:24:42 +0000
commitb720ff3c5a6e14eac458f539bb6b83ef46e4435a (patch)
treedf90e17fec0ff96acddf1ef46296e7e0241cdc84 /src/third_party/wiredtiger
parentf60e5094bb0e582b1eef502b00580d3072ef73d4 (diff)
downloadmongo-b720ff3c5a6e14eac458f539bb6b83ef46e4435a.tar.gz
Import wiredtiger: 94c6c16014292f335a5094669e850125d7432b14 from branch mongodb-4.4
ref: 168c7f1b38..94c6c16014 for: 4.3.4 WT-4999 Migrate Jenkins “wiredtiger-test-format-stress-zseries” job to Evergreen WT-5206 Return the correct checkpoint-modified list of blocks WT-5489 page-read can race with threads locking in-memory page structures WT-5534 Incremental backup needs to accept older metadata WT-5537 Use correct WT_ITEM fields per memory sanitizer
Diffstat (limited to 'src/third_party/wiredtiger')
-rw-r--r--src/third_party/wiredtiger/dist/api_data.py2
-rw-r--r--src/third_party/wiredtiger/dist/s_string.ok1
-rw-r--r--src/third_party/wiredtiger/examples/c/ex_backup_block.c127
-rw-r--r--src/third_party/wiredtiger/import.data2
-rw-r--r--src/third_party/wiredtiger/src/block/block_ckpt.c83
-rw-r--r--src/third_party/wiredtiger/src/btree/bt_import.c6
-rw-r--r--src/third_party/wiredtiger/src/config/config_def.c11
-rw-r--r--src/third_party/wiredtiger/src/conn/conn_api.c6
-rw-r--r--src/third_party/wiredtiger/src/conn/conn_dhandle.c32
-rw-r--r--src/third_party/wiredtiger/src/conn/conn_handle.c8
-rw-r--r--src/third_party/wiredtiger/src/conn/conn_open.c2
-rw-r--r--src/third_party/wiredtiger/src/cursor/cur_backup.c220
-rw-r--r--src/third_party/wiredtiger/src/cursor/cur_backup_incr.c260
-rw-r--r--src/third_party/wiredtiger/src/include/btree.i2
-rw-r--r--src/third_party/wiredtiger/src/include/connection.h2
-rw-r--r--src/third_party/wiredtiger/src/include/cursor.h51
-rw-r--r--src/third_party/wiredtiger/src/include/extern.h5
-rw-r--r--src/third_party/wiredtiger/src/include/meta.h50
-rw-r--r--src/third_party/wiredtiger/src/include/wt_internal.h4
-rw-r--r--src/third_party/wiredtiger/src/meta/meta_ckpt.c208
-rw-r--r--src/third_party/wiredtiger/src/meta/meta_turtle.c13
-rw-r--r--src/third_party/wiredtiger/src/txn/txn_ckpt.c1
-rw-r--r--src/third_party/wiredtiger/src/utilities/util_load.c2
-rwxr-xr-xsrc/third_party/wiredtiger/test/evergreen.yml20
-rw-r--r--src/third_party/wiredtiger/test/suite/test_backup12.py8
25 files changed, 739 insertions, 387 deletions
diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py
index cf942be2f84..45cca0ef829 100644
--- a/src/third_party/wiredtiger/dist/api_data.py
+++ b/src/third_party/wiredtiger/dist/api_data.py
@@ -377,6 +377,8 @@ file_config = format_meta + file_runtime_config + [
file_meta = file_config + [
Config('checkpoint', '', r'''
the file checkpoint entries'''),
+ Config('checkpoint_backup_info', '', r'''
+ the incremental backup durable information'''),
Config('checkpoint_lsn', '', r'''
LSN of the last checkpoint'''),
Config('id', '', r'''
diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok
index 6cda20d9b3d..185ed5aac8d 100644
--- a/src/third_party/wiredtiger/dist/s_string.ok
+++ b/src/third_party/wiredtiger/dist/s_string.ok
@@ -510,6 +510,7 @@ bitpos
bitstring
bitwise
blk
+blkmod
bm
bnd
bool
diff --git a/src/third_party/wiredtiger/examples/c/ex_backup_block.c b/src/third_party/wiredtiger/examples/c/ex_backup_block.c
index fbae6e6da5d..422bf728540 100644
--- a/src/third_party/wiredtiger/examples/c/ex_backup_block.c
+++ b/src/third_party/wiredtiger/examples/c/ex_backup_block.c
@@ -136,11 +136,11 @@ setup_directories(void)
}
static void
-add_work(WT_SESSION *session, int iter)
+add_work(WT_SESSION *session, int iter, int iterj)
{
WT_CURSOR *cursor, *cursor2;
int i;
- char k[32], v[32];
+ char k[64], v[64];
error_check(session->open_cursor(session, uri, NULL, NULL, &cursor));
/*
@@ -154,8 +154,8 @@ add_work(WT_SESSION *session, int iter)
* Perform some operations with individual auto-commit transactions.
*/
for (i = 0; i < MAX_KEYS; i++) {
- (void)snprintf(k, sizeof(k), "key.%d.%d", iter, i);
- (void)snprintf(v, sizeof(v), "value.%d.%d", iter, i);
+ (void)snprintf(k, sizeof(k), "key.%d.%d.%d", iter, iterj, i);
+ (void)snprintf(v, sizeof(v), "value.%d.%d.%d", iter, iterj, i);
cursor->set_key(cursor, k);
cursor->set_value(cursor, v);
error_check(cursor->insert(cursor));
@@ -260,7 +260,8 @@ take_full_backup(WT_SESSION *session, int i)
} else
hdir = home_incr;
if (i == 0) {
- (void)snprintf(buf, sizeof(buf), "incremental=(enabled=true,this_id=ID%d)", i);
+ (void)snprintf(
+ buf, sizeof(buf), "incremental=(granularity=1M,enabled=true,this_id=ID%d)", i);
error_check(session->open_cursor(session, "backup:", NULL, buf, &cursor));
} else
error_check(session->open_cursor(session, "backup:", NULL, NULL, &cursor));
@@ -279,13 +280,17 @@ take_full_backup(WT_SESSION *session, int i)
for (j = 0; j < MAX_ITERATIONS; j++) {
(void)snprintf(h, sizeof(h), "%s.%d", home_incr, j);
(void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename);
+#if 0
printf("FULL: Copy: %s\n", buf);
+#endif
error_check(system(buf));
}
else {
(void)snprintf(h, sizeof(h), "%s.%d", home_full, i);
(void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, hdir, filename);
+#if 0
printf("FULL %d: Copy: %s\n", i, buf);
+#endif
error_check(system(buf));
}
}
@@ -300,13 +305,16 @@ take_incr_backup(WT_SESSION *session, int i)
FILELIST *flist;
WT_CURSOR *backup_cur, *incr_cur;
uint64_t offset, size, type;
- size_t alloc, count;
+ size_t alloc, count, rdsize, tmp_sz;
int j, ret, rfd, wfd;
- char buf[1024], h[256];
+ char buf[1024], h[256], *tmp;
const char *filename;
+ bool first;
/*! [incremental backup using block transfer]*/
+ tmp = NULL;
+ tmp_sz = 0;
/* Open the backup data source for incremental backup. */
(void)snprintf(buf, sizeof(buf), "incremental=(src_id=ID%d,this_id=ID%d)", i - 1, i);
error_check(session->open_cursor(session, "backup:", NULL, buf, &backup_cur));
@@ -321,51 +329,61 @@ take_incr_backup(WT_SESSION *session, int i)
error_check(process_file(&flist, &count, &alloc, filename));
(void)snprintf(h, sizeof(h), "%s.0", home_incr);
(void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename);
- printf("Copying backup: %s\n", buf);
- error_check(system(buf));
#if 0
- (void)snprintf(buf, sizeof(buf), "%s/%s", home, filename);
- printf("Open source %s for reading\n", buf);
- error_check(rfd = open(buf, O_RDONLY, 0));
- (void)snprintf(h, sizeof(h), "%s.%d", home_incr, i);
- (void)snprintf(buf, sizeof(buf), "%s/%s", h, filename);
- printf("Open dest %s for writing\n", buf);
- error_check(wfd = open(buf, O_WRONLY, 0));
+ printf("Copying backup: %s\n", buf);
#endif
+ error_check(system(buf));
+ first = true;
(void)snprintf(buf, sizeof(buf), "incremental=(file=%s)", filename);
error_check(session->open_cursor(session, NULL, backup_cur, buf, &incr_cur));
+#if 0
printf("Taking incremental %d: File %s\n", i, filename);
+#endif
while ((ret = incr_cur->next(incr_cur)) == 0) {
error_check(incr_cur->get_key(incr_cur, &offset, &size, &type));
- printf("Incremental %s: KEY: Off %" PRIu64 " Size: %" PRIu64 " Type: %" PRIu64 "\n",
- filename, offset, size, type);
scan_end_check(type == WT_BACKUP_FILE || type == WT_BACKUP_RANGE);
+#if 0
+ printf("Incremental %s: KEY: Off %" PRIu64 " Size: %" PRIu64 " %s\n", filename, offset,
+ size, type == WT_BACKUP_FILE ? "WT_BACKUP_FILE" : "WT_BACKUP_RANGE");
+#endif
if (type == WT_BACKUP_RANGE) {
/*
* We should never get a range key after a whole file so the read file descriptor
- * should be valid. If the read descriptor is valid, so it the write one.
+ * should be valid. If the read descriptor is valid, so is the write one.
*/
- scan_end_check(rfd != -1);
- printf("Incremental %s: Range Offset: %" PRIu64 " Size: %" PRIu64 "\n", filename,
- offset, size);
+ if (tmp_sz < size) {
+ tmp = realloc(tmp, size);
+ testutil_assert(tmp != NULL);
+ tmp_sz = size;
+ }
+ if (first) {
+ (void)snprintf(buf, sizeof(buf), "%s/%s", home, filename);
+ error_sys_check(rfd = open(buf, O_RDONLY, 0));
+ (void)snprintf(h, sizeof(h), "%s.%d", home_incr, i);
+ (void)snprintf(buf, sizeof(buf), "%s/%s", h, filename);
+ error_sys_check(wfd = open(buf, O_WRONLY, 0));
+ first = false;
+ }
+
error_sys_check(lseek(rfd, (wt_off_t)offset, SEEK_SET));
- error_sys_check(read(rfd, buf, (size_t)size));
+ error_sys_check(rdsize = (size_t)read(rfd, tmp, (size_t)size));
error_sys_check(lseek(wfd, (wt_off_t)offset, SEEK_SET));
- error_sys_check(write(wfd, buf, (size_t)size));
+ /* Use the read size since we may have read less than the granularity. */
+ error_sys_check(write(wfd, tmp, rdsize));
} else {
-/* Whole file, so close both files and just copy the whole thing. */
-#if 0
- error_check(close(rfd));
- error_check(close(wfd));
-#endif
+ /* Whole file, so close both files and just copy the whole thing. */
+ testutil_assert(first == true);
rfd = wfd = -1;
(void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename);
+#if 0
printf("Incremental: Whole file copy: %s\n", buf);
+#endif
error_check(system(buf));
}
}
scan_end_check(ret == WT_NOTFOUND);
+ /* Done processing this file. Close incremental cursor. */
error_check(incr_cur->close(incr_cur));
/* Close file descriptors if they're open. */
@@ -386,18 +404,21 @@ take_incr_backup(WT_SESSION *session, int i)
}
scan_end_check(ret == WT_NOTFOUND);
+ /* Done processing all files. Close backup cursor. */
error_check(backup_cur->close(backup_cur));
error_check(finalize_files(flist, count));
+ free(tmp);
/*! [incremental backup using block transfer]*/
}
int
main(int argc, char *argv[])
{
+ struct stat sb;
WT_CONNECTION *wt_conn;
WT_CURSOR *backup_cur;
WT_SESSION *session;
- int i;
+ int i, j, ret;
char cmd_buf[256];
(void)argc; /* Unused variable */
@@ -412,7 +433,7 @@ main(int argc, char *argv[])
error_check(session->create(session, uri, "key_format=S,value_format=S"));
error_check(session->create(session, uri2, "key_format=S,value_format=S"));
printf("Adding initial data\n");
- add_work(session, 0);
+ add_work(session, 0, 0);
printf("Taking initial backup\n");
take_full_backup(session, 0);
@@ -421,8 +442,12 @@ main(int argc, char *argv[])
for (i = 1; i < MAX_ITERATIONS; i++) {
printf("Iteration %d: adding data\n", i);
- add_work(session, i);
- error_check(session->checkpoint(session, NULL));
+ /* For each iteration we may add work and checkpoint multiple times. */
+ for (j = 0; j < i; j++) {
+ add_work(session, i, j);
+ error_check(session->checkpoint(session, NULL));
+ }
+
/*
* The full backup here is only needed for testing and comparison purposes. A normal
* incremental backup procedure would not include this.
@@ -440,6 +465,20 @@ main(int argc, char *argv[])
error_check(compare_backups(i));
}
+ printf("Close and reopen the connection\n");
+ /*
+ * Close and reopen the connection to illustrate the durability of id information.
+ */
+ error_check(wt_conn->close(wt_conn, NULL));
+ error_check(wiredtiger_open(home, NULL, CONN_CONFIG, &wt_conn));
+ error_check(wt_conn->open_session(wt_conn, NULL, NULL, &session));
+ /*
+ * We should have an entry for i-1 and i-2. Use the older one.
+ */
+ (void)snprintf(cmd_buf, sizeof(cmd_buf), "incremental=(src_id=ID%d,this_id=ID%d)", i - 2, i);
+ error_check(session->open_cursor(session, "backup:", NULL, cmd_buf, &backup_cur));
+ error_check(backup_cur->close(backup_cur));
+
/*
* After we're done, release resources. Test the force stop setting.
*/
@@ -455,6 +494,28 @@ main(int argc, char *argv[])
printf("Final comparison: dumping and comparing data\n");
error_check(compare_backups(0));
+ for (i = 0; i < (int)filelist_count; ++i) {
+ if (last_flist[i].name == NULL)
+ break;
+ free((void *)last_flist[i].name);
+ }
+ free(last_flist);
+
+ /*
+ * Reopen the connection to verify that the forced stop should remove incremental information.
+ */
+ error_check(wiredtiger_open(home, NULL, CONN_CONFIG, &wt_conn));
+ error_check(wt_conn->open_session(wt_conn, NULL, NULL, &session));
+ /*
+ * We should not have any information.
+ */
+ (void)snprintf(cmd_buf, sizeof(cmd_buf), "incremental=(src_id=ID%d,this_id=ID%d)", i - 2, i);
+ testutil_assert(session->open_cursor(session, "backup:", NULL, cmd_buf, &backup_cur) == ENOENT);
+ error_check(wt_conn->close(wt_conn, NULL));
+
+ (void)snprintf(cmd_buf, sizeof(cmd_buf), "%s/WiredTiger.backup.block", home);
+ ret = stat(cmd_buf, &sb);
+ testutil_assert(ret == -1 && errno == ENOENT);
return (EXIT_SUCCESS);
}
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 6f3e41094c5..195d9c6f867 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
"vendor": "wiredtiger",
"github": "wiredtiger/wiredtiger.git",
"branch": "mongodb-4.4",
- "commit": "168c7f1b3859516115721868f312544abb0caf5b"
+ "commit": "94c6c16014292f335a5094669e850125d7432b14"
}
diff --git a/src/third_party/wiredtiger/src/block/block_ckpt.c b/src/third_party/wiredtiger/src/block/block_ckpt.c
index 158fc919820..e1cf8982daf 100644
--- a/src/third_party/wiredtiger/src/block/block_ckpt.c
+++ b/src/third_party/wiredtiger/src/block/block_ckpt.c
@@ -9,8 +9,7 @@
#include "wt_internal.h"
static int __ckpt_process(WT_SESSION_IMPL *, WT_BLOCK *, WT_CKPT *);
-static int __ckpt_update(
- WT_SESSION_IMPL *, WT_BLOCK *, WT_CKPT *, WT_CKPT *, WT_BLOCK_CKPT *, bool);
+static int __ckpt_update(WT_SESSION_IMPL *, WT_BLOCK *, WT_CKPT *, WT_CKPT *, WT_BLOCK_CKPT *);
/*
* __wt_block_ckpt_init --
@@ -331,6 +330,7 @@ __ckpt_verify(WT_SESSION_IMPL *session, WT_CKPT *ckptbase)
case WT_CKPT_DELETE | WT_CKPT_FAKE:
case WT_CKPT_FAKE:
break;
+ case WT_CKPT_ADD | WT_CKPT_BLOCK_MODS:
case WT_CKPT_ADD:
if (ckpt[1].name == NULL)
break;
@@ -570,7 +570,7 @@ __ckpt_process(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase)
/* Update checkpoints marked for update. */
WT_CKPT_FOREACH (ckptbase, ckpt)
if (F_ISSET(ckpt, WT_CKPT_UPDATE))
- WT_ERR(__ckpt_update(session, block, ckptbase, ckpt, ckpt->bpriv, false));
+ WT_ERR(__ckpt_update(session, block, ckptbase, ckpt, ckpt->bpriv));
live_update:
/* Truncate the file if that's possible. */
@@ -607,7 +607,7 @@ live_update:
*/
ci->ckpt_size = WT_MIN(ckpt_size, (uint64_t)block->size);
- WT_ERR(__ckpt_update(session, block, ckptbase, ckpt, ci, true));
+ WT_ERR(__ckpt_update(session, block, ckptbase, ckpt, ci));
}
/*
@@ -654,16 +654,80 @@ err:
}
/*
+ * __ckpt_add_blkmod_entry --
+ * Add an offset/length entry to the bitstring based on granularity.
+ */
+static int
+__ckpt_add_blkmod_entry(
+ WT_SESSION_IMPL *session, WT_BLOCK_MODS *blk_mod, wt_off_t offset, wt_off_t len)
+{
+ uint64_t end, start;
+ uint32_t end_rdup;
+
+ WT_ASSERT(session, blk_mod->granularity != 0);
+ start = (uint64_t)offset / blk_mod->granularity;
+ end = (uint64_t)(offset + len) / blk_mod->granularity;
+ WT_ASSERT(session, end < UINT32_MAX);
+ end_rdup = WT_MAX(__wt_rduppo2((uint32_t)end, 8), WT_BLOCK_MODS_LIST_MIN);
+ if ((end_rdup << 3) > blk_mod->nbits) {
+ /* If we don't have enough, extend the buffer. */
+ if (blk_mod->nbits == 0) {
+ WT_RET(__wt_buf_initsize(session, &blk_mod->bitstring, end_rdup));
+ memset(blk_mod->bitstring.mem, 0, end_rdup);
+ } else
+ WT_RET(__wt_buf_extend(session, &blk_mod->bitstring, end_rdup));
+ blk_mod->nbits = end_rdup << 3;
+ }
+
+ /* Set all the bits needed to record this offset/length pair. */
+ __bit_nset(blk_mod->bitstring.mem, start, end);
+ return (0);
+}
+
+/*
+ * __ckpt_add_blk_mods --
+ * Add the blocks to all valid incremental backup source identifiers.
+ */
+static int
+__ckpt_add_blk_mods(WT_SESSION_IMPL *session, WT_CKPT *ckpt, WT_BLOCK_CKPT *ci)
+{
+ WT_BLOCK_MODS *blk_mod;
+ WT_EXT *ext;
+ u_int i;
+
+ for (i = 0; i < WT_BLKINCR_MAX; ++i) {
+ blk_mod = &ckpt->backup_blocks[i];
+ /* If there is no information at this entry, we're done. */
+ if (!F_ISSET(blk_mod, WT_BLOCK_MODS_VALID))
+ continue;
+
+ WT_EXT_FOREACH (ext, ci->alloc.off)
+ WT_RET(__ckpt_add_blkmod_entry(session, blk_mod, ext->off, ext->size));
+
+ if (ci->alloc.offset != WT_BLOCK_INVALID_OFFSET)
+ WT_RET(__ckpt_add_blkmod_entry(session, blk_mod, ci->alloc.offset, ci->alloc.size));
+ if (ci->discard.offset != WT_BLOCK_INVALID_OFFSET)
+ WT_RET(__ckpt_add_blkmod_entry(session, blk_mod, ci->discard.offset, ci->discard.size));
+ if (ci->avail.offset != WT_BLOCK_INVALID_OFFSET)
+ WT_RET(__ckpt_add_blkmod_entry(session, blk_mod, ci->avail.offset, ci->avail.size));
+ }
+ return (0);
+}
+
+/*
* __ckpt_update --
* Update a checkpoint.
*/
static int
-__ckpt_update(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase, WT_CKPT *ckpt,
- WT_BLOCK_CKPT *ci, bool is_live)
+__ckpt_update(
+ WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase, WT_CKPT *ckpt, WT_BLOCK_CKPT *ci)
{
WT_DECL_ITEM(a);
WT_DECL_RET;
uint8_t *endp;
+ bool is_live;
+
+ is_live = F_ISSET(ckpt, WT_CKPT_ADD);
#ifdef HAVE_DIAGNOSTIC
/* Check the extent list combinations for overlaps. */
@@ -723,6 +787,13 @@ __ckpt_update(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_CKPT *ckptbase, WT_C
}
/*
+ * If this is the live system, we need to record the list of blocks written for this checkpoint
+ * (including the blocks we allocated to write the extent lists).
+ */
+ if (F_ISSET(ckpt, WT_CKPT_BLOCK_MODS))
+ WT_RET(__ckpt_add_blk_mods(session, ckpt, ci));
+
+ /*
* Set the file size for the live system.
*
* !!!
diff --git a/src/third_party/wiredtiger/src/btree/bt_import.c b/src/third_party/wiredtiger/src/btree/bt_import.c
index 02f023567f5..9d63a3cc959 100644
--- a/src/third_party/wiredtiger/src/btree/bt_import.c
+++ b/src/third_party/wiredtiger/src/btree/bt_import.c
@@ -91,12 +91,14 @@ __wt_import(WT_SESSION_IMPL *session, const char *uri)
* Build and flatten the metadata and the checkpoint list, then insert it into the metadata for
* this file.
*
- * Strip out the checkpoint-LSN, an imported file isn't associated with any log files. Assign a
+ * Strip out any incremental backup information, an imported file has not been part of a backup.
+ * Strip out the checkpoint LSN, an imported file isn't associated with any log files. Assign a
* unique file ID.
*/
filecfg[1] = a->data;
filecfg[2] = checkpoint_list;
- filecfg[3] = "checkpoint_lsn=";
+ filecfg[3] = "checkpoint_backup_info=";
+ filecfg[4] = "checkpoint_lsn=";
WT_WITH_SCHEMA_LOCK(session,
ret = __wt_snprintf(fileid, sizeof(fileid), "id=%" PRIu32, ++S2C(session)->next_file_id));
WT_ERR(ret);
diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c
index 629c5316e53..bba4a9b914b 100644
--- a/src/third_party/wiredtiger/src/config/config_def.c
+++ b/src/third_party/wiredtiger/src/config/config_def.c
@@ -415,6 +415,7 @@ static const WT_CONFIG_CHECK confchk_file_meta[] = {
{"block_allocation", "string", NULL, "choices=[\"first\",\"best\"]", NULL, 0},
{"block_compressor", "string", NULL, NULL, NULL, 0},
{"cache_resident", "boolean", NULL, NULL, NULL, 0}, {"checkpoint", "string", NULL, NULL, NULL, 0},
+ {"checkpoint_backup_info", "string", NULL, NULL, NULL, 0},
{"checkpoint_lsn", "string", NULL, NULL, NULL, 0},
{"checksum", "string", NULL, "choices=[\"on\",\"off\",\"uncompressed\"]", NULL, 0},
{"collator", "string", NULL, NULL, NULL, 0}, {"columns", "list", NULL, NULL, NULL, 0},
@@ -941,10 +942,10 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator",
"access_pattern_hint=none,allocation_size=4KB,app_metadata=,"
"assert=(commit_timestamp=none,durable_timestamp=none,"
"read_timestamp=none),block_allocation=best,block_compressor=,"
- "cache_resident=false,checkpoint=,checkpoint_lsn=,"
- "checksum=uncompressed,collator=,columns=,dictionary=0,"
- "encryption=(keyid=,name=),format=btree,huffman_key=,"
- "huffman_value=,id=,ignore_in_memory_cache_size=false,"
+ "cache_resident=false,checkpoint=,checkpoint_backup_info=,"
+ "checkpoint_lsn=,checksum=uncompressed,collator=,columns=,"
+ "dictionary=0,encryption=(keyid=,name=),format=btree,huffman_key="
+ ",huffman_value=,id=,ignore_in_memory_cache_size=false,"
"internal_item_max=0,internal_key_max=0,"
"internal_key_truncate=true,internal_page_max=4KB,key_format=u,"
"key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB,"
@@ -953,7 +954,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {{"WT_CONNECTION.add_collator",
"prefix_compression=false,prefix_compression_min=4,"
"split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90,"
"value_format=u,version=(major=0,minor=0)",
- confchk_file_meta, 41},
+ confchk_file_meta, 42},
{"index.meta",
"app_metadata=,collator=,columns=,extractor=,immutable=false,"
"index_key_columns=,key_format=u,source=,type=file,value_format=u",
diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c
index ace1505e6dd..fcf46b48f95 100644
--- a/src/third_party/wiredtiger/src/conn/conn_api.c
+++ b/src/third_party/wiredtiger/src/conn/conn_api.c
@@ -2671,6 +2671,12 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, const char *c
WT_ERR(__wt_metadata_cursor(session, NULL));
+ /*
+ * Load any incremental backup information. This reads the metadata so must be done after the
+ * turtle file is initialized.
+ */
+ WT_ERR(__wt_backup_open(session));
+
/* Start the worker threads and run recovery. */
WT_ERR(__wt_connection_workers(session, cfg));
diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c
index c947954e75c..1d79064186c 100644
--- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c
+++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c
@@ -38,7 +38,7 @@ __conn_dhandle_config_set(WT_SESSION_IMPL *session)
WT_DATA_HANDLE *dhandle;
WT_DECL_RET;
char *metaconf, *tmp;
- const char *base, *cfg[3];
+ const char *base, *cfg[4];
dhandle = session->dhandle;
base = NULL;
@@ -68,30 +68,32 @@ __conn_dhandle_config_set(WT_SESSION_IMPL *session)
switch (dhandle->type) {
case WT_DHANDLE_TYPE_BTREE:
/*
- * We are stripping out the checkpoint and checkpoint_lsn information from the config
- * string. We save the rest of the metadata string, that is essentially static and
- * unchanging and then concatenate the new checkpoint and LSN information on each
- * checkpoint. The reason is performance and avoiding a lot of calls to the config parsing
- * functions during a checkpoint for information that changes in a very well known way.
+ * We are stripping out all checkpoint related information from the config string. We save
+ * the rest of the metadata string, that is essentially static and unchanging and then
+ * concatenate the new checkpoint related information on each checkpoint. The reason is
+ * performance and avoiding a lot of calls to the config parsing functions during a
+ * checkpoint for information that changes in a very well known way.
+ *
+ * First collapse and overwrite checkpoint information because we do not know the name of or
+ * how many checkpoints may be in this metadata. Similarly, for backup information, we want
+ * an empty category to strip out since we don't know any backup ids. Set them empty and
+ * call collapse to overwrite anything existing.
*/
cfg[0] = metaconf;
cfg[1] = "checkpoint=()";
- cfg[2] = NULL;
+ cfg[2] = "checkpoint_backup_info=()";
+ cfg[3] = NULL;
WT_ERR(__wt_strdup(session, WT_CONFIG_BASE(session, file_meta), &dhandle->cfg[0]));
WT_ASSERT(session, dhandle->meta_base == NULL);
- /*
- * First collapse and overwrite any checkpoint information because we do not know the name
- * or how many checkpoints may be in this metadata. So first we have to set the string to
- * the empty checkpoint string and call collapse to overwrite anything existing.
- */
WT_ERR(__wt_config_collapse(session, cfg, &tmp));
/*
- * Now strip out the checkpoint and checkpoint LSN items from the configuration string and
- * that is now our base metadata string.
+ * Now strip out the checkpoint related items from the configuration string and that is now
+ * our base metadata string.
*/
cfg[0] = tmp;
cfg[1] = NULL;
- WT_ERR(__wt_config_merge(session, cfg, "checkpoint=,checkpoint_lsn=", &base));
+ WT_ERR(__wt_config_merge(
+ session, cfg, "checkpoint=,checkpoint_backup_info=,checkpoint_lsn=", &base));
__wt_free(session, tmp);
break;
case WT_DHANDLE_TYPE_TABLE:
diff --git a/src/third_party/wiredtiger/src/conn/conn_handle.c b/src/third_party/wiredtiger/src/conn/conn_handle.c
index 37136926060..e5c82d49a48 100644
--- a/src/third_party/wiredtiger/src/conn/conn_handle.c
+++ b/src/third_party/wiredtiger/src/conn/conn_handle.c
@@ -94,7 +94,6 @@ void
__wt_connection_destroy(WT_CONNECTION_IMPL *conn)
{
WT_SESSION_IMPL *session;
- u_int i;
/* Check there's something to destroy. */
if (conn == NULL)
@@ -132,13 +131,6 @@ __wt_connection_destroy(WT_CONNECTION_IMPL *conn)
__wt_cond_destroy(session, &conn->lsm_manager.work_cond);
/* Free allocated memory. */
- /*
- * XXX we need to persist this information when we are working on making incremental backups
- * persistent across restarts.
- */
- for (i = 0; i < WT_BLKINCR_MAX; ++i)
- __wt_free(session, conn->incr_backups[i].id_str);
-
__wt_free(session, conn->cfg);
__wt_free(session, conn->debug_ckpt);
__wt_free(session, conn->error_prefix);
diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c
index f7e338ac9bb..df6e6b79300 100644
--- a/src/third_party/wiredtiger/src/conn/conn_open.c
+++ b/src/third_party/wiredtiger/src/conn/conn_open.c
@@ -141,6 +141,8 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn)
/* Close operation tracking */
WT_TRET(__wt_conn_optrack_teardown(session, false));
+ __wt_backup_destroy(session);
+
/* Close any file handles left open. */
WT_TRET(__wt_close_connection_close(session));
diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c
index be781118895..3fdbafce445 100644
--- a/src/third_party/wiredtiger/src/cursor/cur_backup.c
+++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c
@@ -19,6 +19,101 @@ static int __backup_stop(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *);
WT_ERR(F_ISSET(((WT_CURSOR_BACKUP *)(cursor)), WT_CURBACKUP_FORCE_STOP) ? EINVAL : 0);
/*
+ * __wt_backup_destroy --
+ * Destroy any backup information.
+ */
+void
+__wt_backup_destroy(WT_SESSION_IMPL *session)
+{
+ WT_BLKINCR *blkincr;
+ WT_CONNECTION_IMPL *conn;
+ uint64_t i;
+
+ conn = S2C(session);
+ /* Free any incremental backup information. */
+ for (i = 0; i < WT_BLKINCR_MAX; ++i) {
+ blkincr = &conn->incr_backups[i];
+ __wt_free(session, blkincr->id_str);
+ F_CLR(blkincr, WT_BLKINCR_VALID);
+ }
+ conn->incr_granularity = 0;
+ F_CLR(conn, WT_CONN_INCR_BACKUP);
+}
+
+/*
+ * __wt_backup_open --
+ * Restore any incremental backup information. We use the metadata's block information as the
+ * authority on whether incremental backup was in use on last shutdown.
+ */
+int
+__wt_backup_open(WT_SESSION_IMPL *session)
+{
+ WT_BLKINCR *blkincr;
+ WT_CONFIG blkconf;
+ WT_CONFIG_ITEM b, k, v;
+ WT_CONNECTION_IMPL *conn;
+ WT_DECL_RET;
+ uint64_t i;
+ char *config;
+
+ conn = S2C(session);
+ config = NULL;
+
+ WT_RET(__wt_metadata_search(session, WT_METAFILE_URI, &config));
+ WT_ERR(__wt_config_getones(session, config, "checkpoint_backup_info", &v));
+ __wt_config_subinit(session, &blkconf, &v);
+ /*
+ * Walk each item in the metadata and set up our last known global incremental information.
+ */
+ F_CLR(conn, WT_CONN_INCR_BACKUP);
+ i = 0;
+ while (__wt_config_next(&blkconf, &k, &v) == 0) {
+ WT_ASSERT(session, i < WT_BLKINCR_MAX);
+ /*
+ * If we get here, we have at least one valid incremental backup. We want to set up its
+ * general configuration in the global table.
+ */
+ blkincr = &conn->incr_backups[i++];
+ F_SET(conn, WT_CONN_INCR_BACKUP);
+ WT_ERR(__wt_strndup(session, k.str, k.len, &blkincr->id_str));
+ WT_ERR(__wt_config_subgets(session, &v, "granularity", &b));
+ /*
+ * NOTE: For now the granularity is in the connection because it cannot change. We may be
+ * able to relax that.
+ */
+ conn->incr_granularity = blkincr->granularity = (uint64_t)b.val;
+ F_SET(blkincr, WT_BLKINCR_VALID);
+ }
+
+err:
+ if (ret != 0 && ret != WT_NOTFOUND)
+ __wt_backup_destroy(session);
+ __wt_free(session, config);
+ return (ret == WT_NOTFOUND ? 0 : ret);
+}
+
+/*
+ * __wt_backup_file_remove --
+ * Remove the incremental and meta-data backup files.
+ */
+int
+__wt_backup_file_remove(WT_SESSION_IMPL *session)
+{
+ WT_DECL_RET;
+
+ /*
+ * Note that order matters for removing the incremental files. We must remove the backup file
+ * before removing the source file so that we always know we were a source directory while
+ * there's any chance of an incremental backup file existing.
+ */
+ WT_TRET(__wt_remove_if_exists(session, WT_BACKUP_TMP, true));
+ WT_TRET(__wt_remove_if_exists(session, WT_LOGINCR_BACKUP, true));
+ WT_TRET(__wt_remove_if_exists(session, WT_LOGINCR_SRC, true));
+ WT_TRET(__wt_remove_if_exists(session, WT_METADATA_BACKUP, true));
+ return (ret);
+}
+
+/*
* __curbackup_next --
* WT_CURSOR->next method for the backup cursor type.
*/
@@ -71,35 +166,6 @@ err:
}
/*
- * __backup_incr_release --
- * Free all resources relating to incremental backup.
- */
-static int
-__backup_incr_release(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, bool force)
-{
- WT_BLKINCR *blk;
- WT_CONNECTION_IMPL *conn;
- u_int i;
-
- WT_UNUSED(cb);
- WT_UNUSED(force);
- conn = S2C(session);
- /*
- * Clear flags. Remove file. Release any memory information.
- */
- F_CLR(conn, WT_CONN_INCR_BACKUP);
- for (i = 0; i < WT_BLKINCR_MAX; ++i) {
- blk = &conn->incr_backups[i];
- F_CLR(blk, WT_BLKINCR_VALID);
- }
- /* __wt_block_backup_remove... */
- conn->ckpt_incr_granularity = 0;
- WT_RET(__wt_remove_if_exists(session, WT_BLKINCR_BACKUP, true));
-
- return (0);
-}
-
-/*
* __backup_free --
* Free list resources for a backup cursor.
*/
@@ -115,10 +181,6 @@ __backup_free(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb)
}
if (cb->incr_file != NULL)
__wt_free(session, cb->incr_file);
- if (cb->incr_src != NULL)
- __wt_free(session, cb->incr_src);
- if (cb->incr_this != NULL)
- __wt_free(session, cb->incr_this);
__wt_curbackup_free_incr(session, cb);
}
@@ -140,7 +202,7 @@ err:
if (F_ISSET(cb, WT_CURBACKUP_FORCE_STOP)) {
__wt_verbose(
session, WT_VERB_BACKUP, "%s", "Releasing resources from forced stop incremental");
- __backup_incr_release(session, cb, true);
+ __wt_backup_destroy(session);
}
/*
@@ -232,26 +294,6 @@ err:
}
/*
- * __backup_get_ckpt --
- * Get the most recent checkpoint information and store it in the structure.
- *
- * XXX - Currently set return to static void for the compiler, when this function has real content
- * it should be static int.
- */
-static void
-__backup_get_ckpt(WT_SESSION_IMPL *session, WT_BLKINCR *incr)
-{
- WT_UNUSED(session);
- WT_UNUSED(incr);
- /*
- * Look up the most recent checkpoint and store information about it in incr.
- *
- * XXX When this function has content, return a real value. return (0);
- */
- return;
-}
-
-/*
* __backup_add_id --
* Add the identifier for block based incremental backup.
*/
@@ -262,11 +304,13 @@ __backup_add_id(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cval)
WT_CONNECTION_IMPL *conn;
WT_DECL_RET;
u_int i;
+ const char *ckpt;
conn = S2C(session);
blk = NULL;
for (i = 0; i < WT_BLKINCR_MAX; ++i) {
blk = &conn->incr_backups[i];
+ __wt_verbose(session, WT_VERB_BACKUP, "blk[%u] flags 0x%" PRIx64, i, blk->flags);
/* If it isn't use, we can use it. */
if (!F_ISSET(blk, WT_BLKINCR_INUSE))
break;
@@ -281,16 +325,28 @@ __backup_add_id(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cval)
if (blk->id_str != NULL)
__wt_verbose(
session, WT_VERB_BACKUP, "Freeing and reusing backup slot with old id %s", blk->id_str);
- /* Free any string that was there. */
+ /* Free anything that was there. */
__wt_free(session, blk->id_str);
WT_ERR(__wt_strndup(session, cval->str, cval->len, &blk->id_str));
- __wt_verbose(session, WT_VERB_BACKUP, "Using backup slot %u for id %s", i, blk->id_str);
/*
- * XXX This function can error in the future.
- *
- * WT_ERR(__backup_get_ckpt(session, blk));
+ * Get the most recent checkpoint name. For now just use the one that is part of the metadata.
+ * We only care whether or not a checkpoint exists, so immediately free it.
*/
- __backup_get_ckpt(session, blk);
+ ret = __wt_meta_checkpoint_last_name(session, WT_METAFILE_URI, &ckpt);
+ __wt_free(session, ckpt);
+ if (ret != 0 && ret != WT_NOTFOUND)
+ WT_ERR(ret);
+ if (ret == WT_NOTFOUND) {
+ /*
+ * If we don't find any checkpoint, backup files need to be full copy.
+ */
+ __wt_verbose(session, WT_VERB_BACKUP, "ID %s: Did not find any metadata checkpoint for %s.",
+ blk->id_str, WT_METAFILE_URI);
+ F_SET(blk, WT_BLKINCR_FULL);
+ } else {
+ __wt_verbose(session, WT_VERB_BACKUP, "Using backup slot %u for id %s", i, blk->id_str);
+ F_CLR(blk, WT_BLKINCR_FULL);
+ }
F_SET(blk, WT_BLKINCR_VALID);
return (0);
@@ -400,12 +456,11 @@ __backup_config(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, const char *cfg[
if (cval.val) {
if (!F_ISSET(conn, WT_CONN_INCR_BACKUP)) {
WT_RET(__wt_config_gets(session, cfg, "incremental.granularity", &cval));
- /* XXX may not need cb->incr_granularity */
- if (conn->ckpt_incr_granularity != 0)
+ if (conn->incr_granularity != 0)
WT_RET_MSG(session, EINVAL, "Cannot change the incremental backup granularity");
- conn->ckpt_incr_granularity = cb->incr_granularity = (uint64_t)cval.val;
+ conn->incr_granularity = (uint64_t)cval.val;
}
- /* XXX Granularity can only be set once at the beginning */
+ /* Granularity can only be set once at the beginning */
F_SET(conn, WT_CONN_INCR_BACKUP);
incremental_config = true;
}
@@ -432,10 +487,8 @@ __backup_config(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, const char *cfg[
if (is_dup)
WT_RET_MSG(session, EINVAL,
"Incremental source identifier can only be specified on a primary backup cursor");
- WT_RET(__backup_find_id(session, &cval, &cb->incr));
- /* XXX might not need this incr_src field */
- WT_RET(__wt_strndup(session, cval.str, cval.len, &cb->incr_src));
- F_SET(cb->incr, WT_BLKINCR_INUSE);
+ WT_RET(__backup_find_id(session, &cval, &cb->incr_src));
+ F_SET(cb->incr_src, WT_BLKINCR_INUSE);
incremental_config = true;
}
/*
@@ -455,8 +508,6 @@ __backup_config(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, const char *cfg[
WT_ERR_MSG(session, EINVAL, "Incremental identifier already exists");
WT_ERR(__backup_add_id(session, &cval));
- /* XXX might not need this incr_this field */
- WT_ERR(__wt_strndup(session, cval.str, cval.len, &cb->incr_this));
incremental_config = true;
}
@@ -530,8 +581,8 @@ __backup_config(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, const char *cfg[
F_SET(cb, WT_CURBACKUP_INCR);
}
err:
- if (ret != 0 && cb->incr != NULL)
- F_CLR(cb->incr, WT_BLKINCR_INUSE);
+ if (ret != 0 && cb->incr_src != NULL)
+ F_CLR(cb->incr_src, WT_BLKINCR_INUSE);
__wt_scr_free(session, &tmp);
return (ret);
}
@@ -716,8 +767,8 @@ __backup_stop(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb)
/* If it's not a dup backup cursor, make sure one isn't open. */
WT_ASSERT(session, !F_ISSET(session, WT_SESSION_BACKUP_DUP));
WT_WITH_HOTBACKUP_WRITE_LOCK(session, conn->hot_backup_list = NULL);
- if (cb->incr != NULL)
- F_CLR(cb->incr, WT_BLKINCR_INUSE);
+ if (cb->incr_src != NULL)
+ F_CLR(cb->incr_src, WT_BLKINCR_INUSE);
__backup_free(session, cb);
/* Remove any backup specific file. */
@@ -742,27 +793,6 @@ __backup_all(WT_SESSION_IMPL *session)
}
/*
- * __wt_backup_file_remove --
- * Remove the incremental and meta-data backup files.
- */
-int
-__wt_backup_file_remove(WT_SESSION_IMPL *session)
-{
- WT_DECL_RET;
-
- /*
- * Note that order matters for removing the incremental files. We must remove the backup file
- * before removing the source file so that we always know we were a source directory while
- * there's any chance of an incremental backup file existing.
- */
- WT_TRET(__wt_remove_if_exists(session, WT_BACKUP_TMP, true));
- WT_TRET(__wt_remove_if_exists(session, WT_LOGINCR_BACKUP, true));
- WT_TRET(__wt_remove_if_exists(session, WT_LOGINCR_SRC, true));
- WT_TRET(__wt_remove_if_exists(session, WT_METADATA_BACKUP, true));
- return (ret);
-}
-
-/*
* __backup_list_uri_append --
* Append a new file name to the list, allocate space as necessary. Called via the schema_worker
* function.
diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c b/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c
index c8c59d3f9db..5403e2308ff 100644
--- a/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c
+++ b/src/third_party/wiredtiger/src/cursor/cur_backup_incr.c
@@ -9,53 +9,74 @@
#include "wt_internal.h"
/*
- * __alloc_merge --
- * Merge two allocation lists.
+ * __wt_backup_load_incr --
+ * Load the incremental.
*/
-static void
-__alloc_merge(
- uint64_t *a, uint64_t a_cnt, uint64_t *b, uint64_t b_cnt, uint64_t *res, uint64_t *res_cnt)
+int
+__wt_backup_load_incr(
+ WT_SESSION_IMPL *session, WT_CONFIG_ITEM *blkcfg, WT_ITEM *bitstring, uint64_t nbits)
{
- uint64_t total;
-
- for (total = 0; a_cnt > 0 || b_cnt > 0; ++total, res += 2) {
- if (a_cnt > 0 && b_cnt > 0) {
- if (a[0] <= b[0]) {
- res[0] = a[0];
- if (a[0] + a[1] < b[0])
- res[1] = a[1];
- else {
- res[1] = (b[0] + b[1]) - a[0];
- b += 2;
- --b_cnt;
- }
- a += 2;
- --a_cnt;
- } else if (b[0] <= a[0]) {
- res[0] = b[0];
- if (b[0] + b[1] < a[0])
- res[1] = b[1];
- else {
- res[1] = (a[0] + a[1]) - b[0];
- a += 2;
- --a_cnt;
- }
- b += 2;
- --b_cnt;
- }
- } else if (a_cnt > 0) {
- res[0] = a[0];
- res[1] = a[1];
- a += 2;
- --a_cnt;
- } else if (b_cnt > 0) {
- res[0] = b[0];
- res[1] = b[1];
- b += 2;
- --b_cnt;
+ if (blkcfg->len != 0)
+ WT_RET(__wt_nhex_to_raw(session, blkcfg->str, blkcfg->len, bitstring));
+ if (bitstring->size != (nbits >> 3))
+ WT_RET_MSG(session, WT_ERROR, "corrupted modified block list");
+ return (0);
+}
+
+/*
+ * __curbackup_incr_blkmod --
+ * Get the block modifications for a tree from its metadata and fill in the backup cursor's
+ * information with it.
+ */
+static int
+__curbackup_incr_blkmod(WT_SESSION_IMPL *session, WT_BTREE *btree, WT_CURSOR_BACKUP *cb)
+{
+ WT_CONFIG blkconf;
+ WT_CONFIG_ITEM b, k, v;
+ WT_DECL_RET;
+ char *config;
+
+ WT_ASSERT(session, btree != NULL);
+ WT_ASSERT(session, btree->dhandle != NULL);
+ WT_ASSERT(session, cb->incr_src != NULL);
+
+ WT_RET(__wt_metadata_search(session, btree->dhandle->name, &config));
+ WT_ERR(__wt_config_getones(session, config, "checkpoint_backup_info", &v));
+ __wt_config_subinit(session, &blkconf, &v);
+ while ((ret = __wt_config_next(&blkconf, &k, &v)) == 0) {
+ /*
+ * First see if we have information for this source identifier.
+ */
+ if (WT_STRING_MATCH(cb->incr_src->id_str, k.str, k.len) == 0)
+ continue;
+
+ /*
+ * We found a match. If we have a name, then there should be granularity and nbits. The
+ * granularity should be set to something. But nbits may be 0 if there are no blocks
+ * currently modified.
+ */
+ WT_ERR(__wt_config_subgets(session, &v, "granularity", &b));
+ cb->granularity = (uint64_t)b.val;
+ WT_ERR(__wt_config_subgets(session, &v, "nbits", &b));
+ cb->nbits = (uint64_t)b.val;
+ WT_ERR(__wt_config_subgets(session, &v, "offset", &b));
+ cb->offset = (uint64_t)b.val;
+
+ /*
+ * We found a match. Load the block information into the cursor.
+ */
+ ret = __wt_config_subgets(session, &v, "blocks", &b);
+ if (ret != WT_NOTFOUND) {
+ WT_ERR(__wt_backup_load_incr(session, &b, &cb->bitstring, cb->nbits));
+ cb->bit_offset = 0;
+ cb->incr_init = true;
}
}
- *res_cnt = total;
+ WT_ERR_NOTFOUND_OK(ret);
+
+err:
+ __wt_free(session, config);
+ return (ret == WT_NOTFOUND ? 0 : ret);
}
/*
@@ -66,18 +87,11 @@ static int
__curbackup_incr_next(WT_CURSOR *cursor)
{
WT_BTREE *btree;
- WT_CKPT *ckpt, *ckptbase;
WT_CURSOR_BACKUP *cb;
WT_DECL_RET;
WT_SESSION_IMPL *session;
wt_off_t size;
- uint64_t *a, *b, *current, *next;
- uint64_t entries, total;
uint32_t raw;
- bool start, stop;
-
- ckptbase = NULL;
- a = b = NULL;
cb = (WT_CURSOR_BACKUP *)cursor;
btree = cb->incr_cursor == NULL ? NULL : ((WT_CURSOR_BTREE *)cb->incr_cursor)->btree;
@@ -86,104 +100,46 @@ __curbackup_incr_next(WT_CURSOR *cursor)
F_CLR(cursor, WT_CURSTD_RAW);
if (cb->incr_init) {
- /* We have this object's incremental information, Check if we're done. */
- if (cb->incr_list_offset >= cb->incr_list_count - WT_BACKUP_INCR_COMPONENTS)
- return (WT_NOTFOUND);
+ /* Look for the next chunk that had modifications. */
+ while (cb->bit_offset < cb->nbits)
+ if (__bit_test(cb->bitstring.mem, cb->bit_offset))
+ break;
+ else
+ ++cb->bit_offset;
- /*
- * If we returned all of the data, step to the next block, otherwise return the next chunk
- * of the current block.
- */
- if (cb->incr_list[cb->incr_list_offset + 1] <= cb->incr_granularity)
- cb->incr_list_offset += WT_BACKUP_INCR_COMPONENTS;
- else {
- cb->incr_list[cb->incr_list_offset] += cb->incr_granularity;
- cb->incr_list[cb->incr_list_offset + 1] -= cb->incr_granularity;
- cb->incr_list[cb->incr_list_offset + 2] = WT_BACKUP_RANGE;
- }
- } else if (btree == NULL) {
+ /* We either have this object's incremental information or we're done. */
+ if (cb->bit_offset >= cb->nbits)
+ WT_ERR(WT_NOTFOUND);
+ __wt_cursor_set_key(cursor, cb->offset + cb->granularity * cb->bit_offset++,
+ cb->granularity, WT_BACKUP_RANGE);
+ } else if (btree == NULL || F_ISSET(cb, WT_CURBACKUP_FORCE_FULL)) {
/* We don't have this object's incremental information, and it's a full file copy. */
WT_ERR(__wt_fs_size(session, cb->incr_file, &size));
- cb->incr_list_count = WT_BACKUP_INCR_COMPONENTS;
+ cb->nbits = 0;
+ cb->offset = 0;
+ cb->bit_offset = 0;
cb->incr_init = true;
- cb->incr_list_offset = 0;
__wt_cursor_set_key(cursor, 0, size, WT_BACKUP_FILE);
} else {
/*
* We don't have this object's incremental information, and it's not a full file copy. Get a
- * list of the checkpoints available for the file and flag the starting/stopping ones. It
- * shouldn't be possible to specify checkpoints that no longer exist, but check anyway.
+ * list of the block modifications for the file. The block modifications are from the
+ * incremental identifier starting point. Walk the list looking for one with a source of our
+ * id.
*/
- ret = __wt_meta_ckptlist_get(session, cb->incr_file, false, &ckptbase);
- WT_ERR(ret == WT_NOTFOUND ? ENOENT : ret);
-
+ WT_ERR(__curbackup_incr_blkmod(session, btree, cb));
/*
- * Count up the maximum number of block entries we might have to merge, and allocate a pair
- * of temporary arrays in which to do the merge.
+ * If there is no block modification information for this file, there is no information to
+ * return to the user.
*/
- entries = 0;
- WT_CKPT_FOREACH (ckptbase, ckpt)
- entries += ckpt->alloc_list_entries;
- WT_ERR(__wt_calloc_def(session, entries * WT_BACKUP_INCR_COMPONENTS, &a));
- WT_ERR(__wt_calloc_def(session, entries * WT_BACKUP_INCR_COMPONENTS, &b));
-
- /* Merge the block lists into a final list of blocks to copy. */
- start = stop = false;
- total = 0;
- current = NULL;
- next = a;
- WT_CKPT_FOREACH (ckptbase, ckpt) {
- if (strcmp(ckpt->name, cb->incr_checkpoint_start) == 0) {
- start = true;
- WT_ERR_ASSERT(session, ckpt->alloc_list_entries == 0, __wt_panic(session),
- "incremental backup start checkpoint has allocation list blocks");
- continue;
- }
- if (start == true) {
- if (strcmp(ckpt->name, cb->incr_checkpoint_stop) == 0)
- stop = true;
-
- __alloc_merge(
- current, total, ckpt->alloc_list, ckpt->alloc_list_entries, next, &total);
- current = next;
- next = next == a ? b : a;
- }
-
- if (stop == true)
- break;
- }
-
- if (!start)
- WT_ERR_MSG(session, ENOENT, "incremental backup start checkpoint %s not found",
- cb->incr_checkpoint_start);
- if (!stop)
- WT_ERR_MSG(session, ENOENT, "incremental backup stop checkpoint %s not found",
- cb->incr_checkpoint_stop);
-
- /* There may be nothing that needs copying. */
- if (total == 0)
+ if (cb->bitstring.mem == NULL)
WT_ERR(WT_NOTFOUND);
-
- if (next == a) {
- cb->incr_list = b;
- b = NULL;
- } else {
- cb->incr_list = a;
- a = NULL;
- }
- cb->incr_list_count = total;
- cb->incr_list_offset = 0;
- WT_ERR(__wt_scr_alloc(session, 0, &cb->incr_block));
- cb->incr_init = true;
-
- F_SET(cursor, WT_CURSTD_KEY_EXT | WT_CURSTD_VALUE_EXT);
+ __wt_cursor_set_key(cursor, cb->offset + cb->granularity * cb->bit_offset++,
+ cb->granularity, WT_BACKUP_RANGE);
}
err:
- __wt_free(session, a);
- __wt_free(session, b);
- __wt_meta_ckptlist_free(session, &ckptbase);
F_SET(cursor, raw);
API_END_RET(session, ret);
}
@@ -198,10 +154,7 @@ __wt_curbackup_free_incr(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb)
__wt_free(session, cb->incr_file);
if (cb->incr_cursor != NULL)
__wt_cursor_close(cb->incr_cursor);
- __wt_free(session, cb->incr_checkpoint_start);
- __wt_free(session, cb->incr_checkpoint_stop);
- __wt_free(session, cb->incr_list);
- __wt_scr_free(session, &cb->incr_block);
+ __wt_buf_free(session, &cb->bitstring);
}
/*
@@ -213,21 +166,48 @@ __wt_curbackup_open_incr(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *o
WT_CURSOR *cursor, const char *cfg[], WT_CURSOR **cursorp)
{
WT_CURSOR_BACKUP *cb, *other_cb;
+ WT_DECL_ITEM(open_uri);
+ WT_DECL_RET;
cb = (WT_CURSOR_BACKUP *)cursor;
other_cb = (WT_CURSOR_BACKUP *)other;
- WT_UNUSED(session);
cursor->key_format = WT_UNCHECKED_STRING(qqq);
cursor->value_format = "";
+ WT_ASSERT(session, other_cb->incr_src != NULL);
+
/*
* Inherit from the backup cursor but reset specific functions for incremental.
*/
cursor->next = __curbackup_incr_next;
cursor->get_key = __wt_cursor_get_key;
cursor->get_value = __wt_cursor_get_value_notsup;
- cb->incr_granularity = other_cb->incr_granularity;
+ cb->incr_src = other_cb->incr_src;
+
+ /* All WiredTiger owned files are full file copies. */
+ if (F_ISSET(other_cb->incr_src, WT_BLKINCR_FULL) ||
+ WT_PREFIX_MATCH(cb->incr_file, "WiredTiger")) {
+ __wt_verbose(session, WT_VERB_BACKUP, "Forcing full file copies for id %s",
+ other_cb->incr_src->id_str);
+ F_SET(cb, WT_CURBACKUP_FORCE_FULL);
+ }
+ /*
+ * Set up the incremental backup information, if we are not forcing a full file copy. We need an
+ * open cursor on the file. Open the backup checkpoint, confirming it exists.
+ */
+ if (!F_ISSET(cb, WT_CURBACKUP_FORCE_FULL)) {
+ WT_ERR(__wt_scr_alloc(session, 0, &open_uri));
+ WT_ERR(__wt_buf_fmt(session, open_uri, "file:%s", cb->incr_file));
+ __wt_free(session, cb->incr_file);
+ WT_ERR(__wt_strdup(session, open_uri->data, &cb->incr_file));
+
+ WT_ERR(__wt_curfile_open(session, cb->incr_file, NULL, cfg, &cb->incr_cursor));
+ WT_ERR(__wt_cursor_init(cursor, uri, NULL, cfg, cursorp));
+ WT_ERR(__wt_strdup(session, cb->incr_cursor->internal_uri, &cb->incr_cursor->internal_uri));
+ } else
+ WT_ERR(__wt_cursor_init(cursor, uri, NULL, cfg, cursorp));
- /* XXX Return full file info for all files for now. */
- return (__wt_cursor_init(cursor, uri, NULL, cfg, cursorp));
+err:
+ __wt_scr_free(session, &open_uri);
+ return (ret);
}
diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i
index dd1dc11bb6e..50f6fcf9759 100644
--- a/src/third_party/wiredtiger/src/include/btree.i
+++ b/src/third_party/wiredtiger/src/include/btree.i
@@ -1111,7 +1111,7 @@ __wt_ref_info_lock(
*/
for (;; __wt_yield()) {
previous_state = ref->state;
- if (previous_state != WT_REF_LOCKED &&
+ if (previous_state != WT_REF_LOCKED && previous_state != WT_REF_READING &&
WT_REF_CAS_STATE(session, ref, previous_state, WT_REF_LOCKED))
break;
}
diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h
index 9498eb5d6c6..2a275449284 100644
--- a/src/third_party/wiredtiger/src/include/connection.h
+++ b/src/third_party/wiredtiger/src/include/connection.h
@@ -299,7 +299,7 @@ struct __wt_connection_impl {
uint64_t ckpt_write_pages;
/* Checkpoint and incremental backup data */
- uint64_t ckpt_incr_granularity;
+ uint64_t incr_granularity;
WT_BLKINCR incr_backups[WT_BLKINCR_MAX];
/* Connection's maximum and base write generations. */
diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h
index 45a4ac01a9f..3ea011b8fc9 100644
--- a/src/third_party/wiredtiger/src/include/cursor.h
+++ b/src/third_party/wiredtiger/src/include/cursor.h
@@ -32,21 +32,6 @@
0 /* uint32_t flags */ \
}
-/*
- * Block based incremental backup structure. These live in the connection.
- */
-#define WT_BLKINCR_MAX 2
-struct __wt_blkincr {
- const char *id_str; /* User's name for this backup. */
- const char *ckpt_name; /* Requires WT-5115. All checkpoints must be this name */
- void *data;
-/* AUTOMATIC FLAG VALUE GENERATION START */
-#define WT_BLKINCR_INUSE 0x1u /* This entry is active */
-#define WT_BLKINCR_VALID 0x2u /* This entry is valid */
- /* AUTOMATIC FLAG VALUE GENERATION STOP */
- uint64_t flags;
-};
-
struct __wt_cursor_backup {
WT_CURSOR iface;
@@ -61,31 +46,25 @@ struct __wt_cursor_backup {
size_t list_next;
/* File offset-based incremental backup. */
- WT_BLKINCR *incr; /* Incremental backup in use */
- char *incr_file; /* File name */
- char *incr_src; /* Source identifier */
- char *incr_this; /* New base identifier */
- uint64_t incr_granularity; /* Maximum transfer size */
+ WT_BLKINCR *incr_src; /* Incremental backup source */
+ char *incr_file; /* File name */
WT_CURSOR *incr_cursor; /* File cursor */
- /* Start/stop checkpoints */
- char *incr_checkpoint_start;
- char *incr_checkpoint_stop;
-
-#define WT_BACKUP_INCR_COMPONENTS 3
- bool incr_init; /* Cursor traversal initialized */
- uint64_t *incr_list; /* List of file offset/size/type triples */
- uint64_t incr_list_count; /* Count of file offset/size/type triples */
- uint64_t incr_list_offset; /* Current offset */
- uint64_t incr_size; /* Maximum transfer size */
- WT_ITEM *incr_block; /* Current block of data */
+
+ bool incr_init; /* Cursor traversal initialized */
+ WT_ITEM bitstring; /* List of modified blocks */
+ uint64_t nbits; /* Number of bits in bitstring */
+ uint64_t offset; /* Zero bit offset in bitstring */
+ uint64_t bit_offset; /* Current offset */
+ uint64_t granularity; /* Length, transfer size */
/* AUTOMATIC FLAG VALUE GENERATION START */
-#define WT_CURBACKUP_DUP 0x1u /* Duplicated backup cursor */
-#define WT_CURBACKUP_FORCE_STOP 0x2u /* Force stop incremental backup */
-#define WT_CURBACKUP_INCR 0x4u /* Incremental backup cursor */
-#define WT_CURBACKUP_LOCKER 0x8u /* Hot-backup started */
- /* AUTOMATIC FLAG VALUE GENERATION STOP */
+#define WT_CURBACKUP_DUP 0x01u /* Duplicated backup cursor */
+#define WT_CURBACKUP_FORCE_FULL 0x02u /* Force full file copy for this cursor */
+#define WT_CURBACKUP_FORCE_STOP 0x04u /* Force stop incremental backup */
+#define WT_CURBACKUP_INCR 0x08u /* Incremental backup cursor */
+#define WT_CURBACKUP_LOCKER 0x10u /* Hot-backup started */
+ /* AUTOMATIC FLAG VALUE GENERATION STOP */
uint8_t flags;
};
diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h
index c206781116c..2cf1408525e 100644
--- a/src/third_party/wiredtiger/src/include/extern.h
+++ b/src/third_party/wiredtiger/src/include/extern.h
@@ -84,6 +84,10 @@ extern int __wt_async_reconfig(WT_SESSION_IMPL *session, const char *cfg[])
WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_backup_file_remove(WT_SESSION_IMPL *session)
WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+extern int __wt_backup_load_incr(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *blkcfg,
+ WT_ITEM *bitstring, uint64_t nbits) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+extern int __wt_backup_open(WT_SESSION_IMPL *session)
+ WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_bad_object_type(WT_SESSION_IMPL *session, const char *uri)
WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_block_addr_invalid(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr,
@@ -1564,6 +1568,7 @@ extern void *__wt_ext_scr_alloc(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session
extern void __wt_abort(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn))
WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")));
extern void __wt_async_stats_update(WT_SESSION_IMPL *session);
+extern void __wt_backup_destroy(WT_SESSION_IMPL *session);
extern void __wt_block_ckpt_destroy(WT_SESSION_IMPL *session, WT_BLOCK_CKPT *ci);
extern void __wt_block_configure_first_fit(WT_BLOCK *block, bool on);
extern void __wt_block_ext_free(WT_SESSION_IMPL *session, WT_EXT *ext);
diff --git a/src/third_party/wiredtiger/src/include/meta.h b/src/third_party/wiredtiger/src/include/meta.h
index 9845dbd7f7d..b29d9665069 100644
--- a/src/third_party/wiredtiger/src/include/meta.h
+++ b/src/third_party/wiredtiger/src/include/meta.h
@@ -17,11 +17,10 @@
/*
* Backup related WiredTiger files.
*/
-#define WT_BACKUP_TMP "WiredTiger.backup.tmp" /* Backup tmp file */
-#define WT_BLKINCR_BACKUP "WiredTiger.backup.block" /* Block incremental durable file */
-#define WT_METADATA_BACKUP "WiredTiger.backup" /* Hot backup file */
-#define WT_LOGINCR_BACKUP "WiredTiger.ibackup" /* Log incremental backup */
-#define WT_LOGINCR_SRC "WiredTiger.isrc" /* Log incremental source */
+#define WT_BACKUP_TMP "WiredTiger.backup.tmp" /* Backup tmp file */
+#define WT_METADATA_BACKUP "WiredTiger.backup" /* Hot backup file */
+#define WT_LOGINCR_BACKUP "WiredTiger.ibackup" /* Log incremental backup */
+#define WT_LOGINCR_SRC "WiredTiger.isrc" /* Log incremental source */
#define WT_METADATA_TURTLE "WiredTiger.turtle" /* Metadata metadata */
#define WT_METADATA_TURTLE_SET "WiredTiger.turtle.set" /* Turtle temp file */
@@ -58,6 +57,42 @@
} while (0)
/*
+ * Block based incremental backup structure. These live in the connection.
+ */
+#define WT_BLKINCR_MAX 2
+struct __wt_blkincr {
+ const char *id_str; /* User's name for this backup. */
+ uint64_t granularity; /* Granularity of this backup. */
+/* AUTOMATIC FLAG VALUE GENERATION START */
+#define WT_BLKINCR_FULL 0x1u /* There is no checkpoint, always do full file */
+#define WT_BLKINCR_INUSE 0x2u /* This entry is active */
+#define WT_BLKINCR_VALID 0x4u /* This entry is valid */
+ /* AUTOMATIC FLAG VALUE GENERATION STOP */
+ uint64_t flags;
+};
+
+/*
+ * Block modifications from an incremental identifier going forward.
+ */
+/*
+ * At the default granularity, this is enough for blocks in a 2G file.
+ */
+#define WT_BLOCK_MODS_LIST_MIN 16 /* Initial bytes for bitmap. */
+struct __wt_block_mods {
+ const char *id_str;
+
+ WT_ITEM bitstring;
+ uint64_t nbits; /* Number of bits in bitstring */
+
+ uint64_t offset; /* Zero bit offset for bitstring */
+ uint64_t granularity;
+/* AUTOMATIC FLAG VALUE GENERATION START */
+#define WT_BLOCK_MODS_VALID 0x1u /* Entry is valid */
+ /* AUTOMATIC FLAG VALUE GENERATION STOP */
+ uint32_t flags;
+};
+
+/*
* WT_CKPT --
* Encapsulation of checkpoint information, shared by the metadata, the
* btree engine, and the block manager.
@@ -88,6 +123,8 @@ struct __wt_ckpt {
char *block_metadata; /* Block-stored metadata */
char *block_checkpoint; /* Block-stored checkpoint */
+ WT_BLOCK_MODS backup_blocks[WT_BLKINCR_MAX];
+
/* Validity window */
wt_timestamp_t newest_durable_ts;
wt_timestamp_t oldest_start_ts;
@@ -95,9 +132,6 @@ struct __wt_ckpt {
wt_timestamp_t newest_stop_ts;
uint64_t newest_stop_txn;
- uint64_t *alloc_list; /* Checkpoint allocation list */
- uint64_t alloc_list_entries;
-
WT_ITEM addr; /* Checkpoint cookie string */
WT_ITEM raw; /* Checkpoint cookie raw */
diff --git a/src/third_party/wiredtiger/src/include/wt_internal.h b/src/third_party/wiredtiger/src/include/wt_internal.h
index 1cd8753a0ac..7ae570d3e59 100644
--- a/src/third_party/wiredtiger/src/include/wt_internal.h
+++ b/src/third_party/wiredtiger/src/include/wt_internal.h
@@ -87,6 +87,8 @@ struct __wt_block_desc;
typedef struct __wt_block_desc WT_BLOCK_DESC;
struct __wt_block_header;
typedef struct __wt_block_header WT_BLOCK_HEADER;
+struct __wt_block_mods;
+typedef struct __wt_block_mods WT_BLOCK_MODS;
struct __wt_bloom;
typedef struct __wt_bloom WT_BLOOM;
struct __wt_bloom_hash;
@@ -390,7 +392,7 @@ typedef uint64_t wt_timestamp_t;
#include "error.h"
#include "log.h"
#include "lsm.h"
-#include "meta.h"
+#include "meta.h" /* required by block.h */
#include "optrack.h"
#include "os.h"
#include "reconcile.h"
diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c
index 93a01fa6abb..34c8b643c08 100644
--- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c
+++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c
@@ -11,11 +11,79 @@
static int __ckpt_last(WT_SESSION_IMPL *, const char *, WT_CKPT *);
static int __ckpt_last_name(WT_SESSION_IMPL *, const char *, const char **);
static int __ckpt_load(WT_SESSION_IMPL *, WT_CONFIG_ITEM *, WT_CONFIG_ITEM *, WT_CKPT *);
+static int __ckpt_load_blk_mods(WT_SESSION_IMPL *, const char *, WT_CKPT *);
static int __ckpt_named(WT_SESSION_IMPL *, const char *, const char *, WT_CKPT *);
static int __ckpt_set(WT_SESSION_IMPL *, const char *, const char *, bool);
static int __ckpt_version_chk(WT_SESSION_IMPL *, const char *, const char *);
/*
+ * __ckpt_load_blk_mods --
+ * Load the block information from the config string.
+ */
+static int
+__ckpt_load_blk_mods(WT_SESSION_IMPL *session, const char *config, WT_CKPT *ckpt)
+{
+ WT_BLKINCR *blkincr;
+ WT_BLOCK_MODS *blk_mod;
+ WT_CONFIG blkconf;
+ WT_CONFIG_ITEM b, k, v;
+ WT_CONNECTION_IMPL *conn;
+ WT_DECL_RET;
+ uint64_t i;
+
+ conn = S2C(session);
+ if (config == NULL)
+ return (0);
+ /*
+ * We could be reading in a configuration from an earlier release. If the string doesn't exist
+ * then we're done.
+ */
+ if ((ret = __wt_config_getones(session, config, "checkpoint_backup_info", &v)) != 0)
+ return (ret == WT_NOTFOUND ? 0 : ret);
+ __wt_config_subinit(session, &blkconf, &v);
+ /*
+ * Load block lists. Ignore any that have an id string that is not known.
+ *
+ * Remove those not known (TODO).
+ */
+ blkincr = NULL;
+ while ((ret = __wt_config_next(&blkconf, &k, &v)) == 0) {
+ /*
+ * See if this is a valid backup string.
+ */
+ for (i = 0; i < WT_BLKINCR_MAX; ++i) {
+ blkincr = &conn->incr_backups[i];
+ if (blkincr->id_str != NULL && WT_STRING_MATCH(blkincr->id_str, k.str, k.len))
+ break;
+ }
+ if (i == WT_BLKINCR_MAX)
+ /*
+ * This is the place to note that we want to remove an unknown id.
+ */
+ continue;
+
+ /*
+ * We have a valid entry. Load the block information.
+ */
+ blk_mod = &ckpt->backup_blocks[i];
+ WT_RET(__wt_strdup(session, blkincr->id_str, &blk_mod->id_str));
+ WT_RET(__wt_config_subgets(session, &v, "granularity", &b));
+ blk_mod->granularity = (uint64_t)b.val;
+ WT_RET(__wt_config_subgets(session, &v, "nbits", &b));
+ blk_mod->nbits = (uint64_t)b.val;
+ WT_RET(__wt_config_subgets(session, &v, "offset", &b));
+ blk_mod->offset = (uint64_t)b.val;
+ ret = __wt_config_subgets(session, &v, "blocks", &b);
+ WT_RET_NOTFOUND_OK(ret);
+ if (ret != WT_NOTFOUND) {
+ WT_RET(__wt_backup_load_incr(session, &b, &blk_mod->bitstring, blk_mod->nbits));
+ F_SET(blk_mod, WT_BLOCK_MODS_VALID);
+ }
+ }
+ return (ret == WT_NOTFOUND ? 0 : ret);
+}
+
+/*
* __wt_meta_checkpoint --
* Return a file's checkpoint information.
*/
@@ -118,7 +186,7 @@ __ckpt_set(WT_SESSION_IMPL *session, const char *fname, const char *v, bool use_
* use the slower path through configuration parsing functions.
*/
config = newcfg = NULL;
- str = v == NULL ? "checkpoint=(),checkpoint_lsn=" : v;
+ str = v == NULL ? "checkpoint=(),checkpoint_backup_info=(),checkpoint_lsn=" : v;
if (use_base && session->dhandle != NULL) {
WT_ERR(__wt_scr_alloc(session, 0, &tmp));
WT_ASSERT(session, strcmp(session->dhandle->name, fname) == 0);
@@ -315,6 +383,75 @@ __ckpt_compare_order(const void *a, const void *b)
}
/*
+ * __ckpt_valid_blk_mods --
+ * Make sure that this set of block mods reflects the current valid backup identifiers. If so,
+ * there is nothing to do. If not, free up old information and set it up for the current
+ * information.
+ */
+static int
+__ckpt_valid_blk_mods(WT_SESSION_IMPL *session, WT_CKPT *ckpt)
+{
+ WT_BLKINCR *blk;
+ WT_BLOCK_MODS *blk_mod;
+ uint64_t i;
+ bool free, setup;
+
+ WT_ASSERT(session, F_ISSET(ckpt, WT_CKPT_ADD));
+ for (i = 0; i < WT_BLKINCR_MAX; ++i) {
+ blk = &S2C(session)->incr_backups[i];
+ blk_mod = &ckpt->backup_blocks[i];
+
+ /*
+ * Check the state of our block list array compared to the global one. There are
+ * several possibilities:
+ * - There is no global information for this index, nothing to do but free our resources.
+ * - We don't have any backup information locally. Set up our entry.
+ * - Our entry's id string matches the current global information. We just want to add our
+ * information to the existing list.
+ * - Our entry's id string does not match the current one. It is outdated. Free old
+ * resources
+ * and then set up our entry.
+ */
+
+ /* Check if the global entry is valid at our index. */
+ if (!F_ISSET(blk, WT_BLKINCR_VALID)) {
+ free = true;
+ setup = false;
+ } else if (F_ISSET(blk_mod, WT_BLOCK_MODS_VALID) &&
+ WT_STRING_MATCH(blk_mod->id_str, blk->id_str, strlen(blk->id_str))) {
+ /* We match, keep our entry and don't set up. */
+ setup = false;
+ free = false;
+ } else {
+ /* We don't match, free any old information. */
+ free = true;
+ setup = true;
+ }
+
+ /* Free any old information if we need to do so. */
+ if (free && F_ISSET(blk_mod, WT_BLOCK_MODS_VALID)) {
+ __wt_free(session, blk_mod->id_str);
+ __wt_buf_free(session, &blk_mod->bitstring);
+ blk_mod->nbits = 0;
+ blk_mod->granularity = 0;
+ blk_mod->offset = 0;
+ F_CLR(blk_mod, WT_BLOCK_MODS_VALID);
+ }
+
+ /* Set up the block list to point to the current information. */
+ if (setup) {
+ WT_RET(__wt_strdup(session, blk->id_str, &blk_mod->id_str));
+ WT_CLEAR(blk_mod->bitstring);
+ blk_mod->granularity = S2C(session)->incr_granularity;
+ blk_mod->nbits = 0;
+ blk_mod->offset = 0;
+ F_SET(blk_mod, WT_BLOCK_MODS_VALID);
+ }
+ }
+ return (0);
+}
+
+/*
* __wt_meta_ckptlist_get --
* Load all available checkpoint information for a file.
*/
@@ -378,10 +515,22 @@ __wt_meta_ckptlist_get(
maxorder = ckpt->order;
ckpt->order = maxorder + 1;
__wt_seconds(session, &ckpt->sec);
+ /*
+ * Load most recent checkpoint backup blocks to this checkpoint.
+ */
+ WT_ERR(__ckpt_load_blk_mods(session, config, ckpt));
WT_ERR(__wt_meta_block_metadata(session, config, ckpt));
+ /*
+ * Set the add-a-checkpoint flag, and if we're doing incremental backups, request a list of
+ * the checkpoint's modified blocks from the block manager.
+ */
F_SET(ckpt, WT_CKPT_ADD);
+ if (F_ISSET(S2C(session), WT_CONN_INCR_BACKUP)) {
+ F_SET(ckpt, WT_CKPT_BLOCK_MODS);
+ WT_ERR(__ckpt_valid_blk_mods(session, ckpt));
+ }
}
/* Return the array to our caller. */
@@ -578,6 +727,50 @@ __wt_meta_ckptlist_to_meta(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, WT_ITEM
}
/*
+ * __ckpt_blkmod_to_meta --
+ * Add in any modification block string needed, including an empty one.
+ */
+static int
+__ckpt_blkmod_to_meta(WT_SESSION_IMPL *session, WT_ITEM *buf, WT_CKPT *ckpt)
+{
+ WT_BLOCK_MODS *blk;
+ WT_ITEM bitstring;
+ u_int i;
+ bool valid;
+
+ WT_CLEAR(bitstring);
+ valid = false;
+ for (i = 0, blk = &ckpt->backup_blocks[0]; i < WT_BLKINCR_MAX; ++i, ++blk)
+ if (F_ISSET(blk, WT_BLOCK_MODS_VALID))
+ valid = true;
+
+ /*
+ * If the existing block modifications are not valid, there is nothing to do.
+ */
+ if (!valid) {
+ WT_RET(__wt_buf_catfmt(session, buf, ",checkpoint_backup_info="));
+ return (0);
+ }
+
+ /*
+ * We have at least one valid modified block list.
+ */
+ WT_RET(__wt_buf_catfmt(session, buf, ",checkpoint_backup_info=("));
+ for (i = 0, blk = &ckpt->backup_blocks[0]; i < WT_BLKINCR_MAX; ++i, ++blk) {
+ if (!F_ISSET(blk, WT_BLOCK_MODS_VALID))
+ continue;
+ WT_RET(__wt_raw_to_hex(session, blk->bitstring.data, blk->bitstring.size, &bitstring));
+ WT_RET(__wt_buf_catfmt(session, buf, "%s%s=(id=%" PRIu32 ",granularity=%" PRIu64
+ ",nbits=%" PRIu64 ",offset=%" PRIu64 ",blocks=%.*s)",
+ i == 0 ? "" : ",", blk->id_str, i, blk->granularity, blk->nbits, blk->offset,
+ (int)bitstring.size, (char *)bitstring.data));
+ __wt_buf_free(session, &bitstring);
+ }
+ WT_RET(__wt_buf_catfmt(session, buf, ")"));
+ return (0);
+}
+
+/*
* __wt_meta_ckptlist_set --
* Set a file's checkpoint value from the WT_CKPT list.
*/
@@ -593,6 +786,10 @@ __wt_meta_ckptlist_set(
WT_RET(__wt_scr_alloc(session, 1024, &buf));
WT_ERR(__wt_meta_ckptlist_to_meta(session, ckptbase, buf));
+ /* Add backup block modifications for any added checkpoint. */
+ WT_CKPT_FOREACH (ckptbase, ckpt)
+ if (F_ISSET(ckpt, WT_CKPT_ADD))
+ WT_ERR(__ckpt_blkmod_to_meta(session, buf, ckpt));
has_lsn = ckptlsn != NULL;
if (ckptlsn != NULL)
@@ -634,6 +831,9 @@ __wt_meta_ckptlist_free(WT_SESSION_IMPL *session, WT_CKPT **ckptbasep)
void
__wt_meta_checkpoint_free(WT_SESSION_IMPL *session, WT_CKPT *ckpt)
{
+ WT_BLOCK_MODS *blk_mod;
+ uint64_t i;
+
if (ckpt == NULL)
return;
@@ -643,6 +843,12 @@ __wt_meta_checkpoint_free(WT_SESSION_IMPL *session, WT_CKPT *ckpt)
__wt_buf_free(session, &ckpt->addr);
__wt_buf_free(session, &ckpt->raw);
__wt_free(session, ckpt->bpriv);
+ for (i = 0; i < WT_BLKINCR_MAX; ++i) {
+ blk_mod = &ckpt->backup_blocks[i];
+ __wt_buf_free(session, &blk_mod->bitstring);
+ __wt_free(session, blk_mod->id_str);
+ F_CLR(blk_mod, WT_BLOCK_MODS_VALID);
+ }
WT_CLEAR(*ckpt); /* Clear to prepare for re-use. */
}
diff --git a/src/third_party/wiredtiger/src/meta/meta_turtle.c b/src/third_party/wiredtiger/src/meta/meta_turtle.c
index 044094133ce..a7b2e740caf 100644
--- a/src/third_party/wiredtiger/src/meta/meta_turtle.c
+++ b/src/third_party/wiredtiger/src/meta/meta_turtle.c
@@ -185,7 +185,7 @@ __wt_turtle_init(WT_SESSION_IMPL *session)
{
WT_DECL_RET;
char *metaconf, *unused_value;
- bool exist_backup, exist_bincr, exist_incr, exist_isrc, exist_turtle;
+ bool exist_backup, exist_incr, exist_isrc, exist_turtle;
bool load, loadTurtle;
load = loadTurtle = false;
@@ -212,17 +212,6 @@ __wt_turtle_init(WT_SESSION_IMPL *session)
WT_RET(__wt_fs_exist(session, WT_LOGINCR_SRC, &exist_isrc));
WT_RET(__wt_fs_exist(session, WT_METADATA_BACKUP, &exist_backup));
WT_RET(__wt_fs_exist(session, WT_METADATA_TURTLE, &exist_turtle));
- /*
- * Block incremental is different. If it exists, then we have block incremental information we
- * need to keep. Mark the connection as having block-based incremental backup turned on. XXX -
- * Need to call something to read it in and set this up. Maybe here, maybe not.
- */
- WT_RET(__wt_fs_exist(session, WT_BLKINCR_BACKUP, &exist_bincr));
- if (exist_bincr) {
- F_SET(S2C(session), WT_CONN_INCR_BACKUP);
- /* Load content into some structure. Not sure this is the right place. It may be too early.
- */
- }
if (exist_turtle) {
/*
diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c
index e960ec03d48..200f84cc8c1 100644
--- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c
+++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c
@@ -1226,6 +1226,7 @@ __checkpoint_lock_dirty_tree_int(WT_SESSION_IMPL *session, bool is_checkpoint, b
"cannot be deleted during a hot backup",
ckpt->name);
}
+
/*
* Mark old checkpoints that are being deleted and figure out which trees we can skip in this
* checkpoint.
diff --git a/src/third_party/wiredtiger/src/utilities/util_load.c b/src/third_party/wiredtiger/src/utilities/util_load.c
index 4f1d1bcb1f1..44672a66854 100644
--- a/src/third_party/wiredtiger/src/utilities/util_load.c
+++ b/src/third_party/wiredtiger/src/utilities/util_load.c
@@ -449,7 +449,7 @@ config_update(WT_SESSION *session, char **list)
if ((ret = __wt_config_merge((WT_SESSION_IMPL *)session, cfg,
"filename=,id=,"
- "checkpoint=,checkpoint_lsn=,version=,source=,",
+ "checkpoint=,checkpoint_backup_info=,checkpoint_lsn=,version=,source=,",
&p)) != 0)
break;
diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml
index cdc98884023..e7eaaa15de5 100755
--- a/src/third_party/wiredtiger/test/evergreen.yml
+++ b/src/third_party/wiredtiger/test/evergreen.yml
@@ -1996,22 +1996,6 @@ tasks:
# At the time of writing this script, one call to underlying scripts takes about ~15 mins to finish in worst case.
# We are giving an extra ~20% room for vairance in execution time.
times: 80
-
- # This is special task until lz4 issues are resolved for zSeries distros
- - name: recovery-stress-test-without-lz4
- #set a 25 hours timeout
- exec_timeout_secs: 90000
- commands:
- - func: "get project"
- - func: "compile wiredtiger"
- vars:
- posix_configure_flags: --enable-strict --enable-diagnostic --with-builtins=snappy,zlib
- - func: "recovery stress test script"
- vars:
- # Repeat this script enough times to make this task run for 24 hours
- # At the time of writing this script, one call to underlying scripts takes about 8 mins to finish in worst case.
- # We are giving an extra ~20% room for vairance in execution time.
- times: 120
- name: split-stress-test
commands:
@@ -2360,5 +2344,7 @@ buildvariants:
tasks:
- name: compile
- name: unit-test
- - name: recovery-stress-test-without-lz4
+ - name: recovery-stress-test
- name: split-stress-test
+ - name: format-stress-ppc-zseries-test
+ - name: format-stress-smoke-test
diff --git a/src/third_party/wiredtiger/test/suite/test_backup12.py b/src/third_party/wiredtiger/test/suite/test_backup12.py
index 6726164d038..1148ed84a45 100644
--- a/src/third_party/wiredtiger/test/suite/test_backup12.py
+++ b/src/third_party/wiredtiger/test/suite/test_backup12.py
@@ -83,6 +83,9 @@ class test_backup12(wttest.WiredTigerTestCase, suite_subprocess):
# That log file is not part of the list returned. This is a full backup
# primary cursor with incremental configured.
os.mkdir(self.dir)
+ #
+ # Note, this first backup is actually done before a checkpoint is taken.
+ #
config = 'incremental=(enabled,this_id="ID1")'
bkup_c = self.session.open_cursor('backup:', None, config)
@@ -168,12 +171,9 @@ class test_backup12(wttest.WiredTigerTestCase, suite_subprocess):
offset = incrlist[0]
size = incrlist[1]
curtype = incrlist[2]
- self.assertEqual(offset, 0)
- # For now assert WT_BACKUP_FILE (which is 1).
- self.assertEqual(curtype, 1)
+ self.assertTrue(curtype == 1 or curtype == 2)
dup_cnt += 1
dupc.close()
- self.assertEqual(dup_cnt, 1)
self.pr('Copy from: ' + newfile + ' (' + str(sz) + ') to ' + self.dir)
shutil.copy(newfile, self.dir)
self.assertEqual(ret, wiredtiger.WT_NOTFOUND)