summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLuke Chen <luke.chen@mongodb.com>2022-11-14 12:27:22 +1100
committerEvergreen Agent <no-reply@evergreen.mongodb.com>2022-11-14 02:04:31 +0000
commit4bf7c5d7cb66be76956685783fc9014b05b2baf4 (patch)
treeea4c28ce0c21de3f3df385c14d70881bf613e411
parent553067485afc74087de94ef11298a02298524fae (diff)
downloadmongo-4bf7c5d7cb66be76956685783fc9014b05b2baf4.tar.gz
Import wiredtiger: f33fe0274f33f378b77f015c976889b0eed05197 from branch mongodb-6.1
ref: 005ca70527..f33fe0274f for: 6.1.1 WT-10030 Mark internal page with deleted pages as dirty when reading it into memory (#8414) (#8481)
-rw-r--r--src/third_party/wiredtiger/import.data2
-rw-r--r--src/third_party/wiredtiger/src/btree/bt_page.c27
-rw-r--r--src/third_party/wiredtiger/test/suite/test_truncate19.py119
3 files changed, 147 insertions, 1 deletions
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index c42c9995517..1955b4480fe 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
"vendor": "wiredtiger",
"github": "wiredtiger/wiredtiger.git",
"branch": "mongodb-6.1",
- "commit": "005ca70527d68c4ad868ea50091413f6b99f272b"
+ "commit": "f33fe0274f33f378b77f015c976889b0eed05197"
}
diff --git a/src/third_party/wiredtiger/src/btree/bt_page.c b/src/third_party/wiredtiger/src/btree/bt_page.c
index 87abfebf041..f5baea54d18 100644
--- a/src/third_party/wiredtiger/src/btree/bt_page.c
+++ b/src/third_party/wiredtiger/src/btree/bt_page.c
@@ -634,6 +634,10 @@ static int
__inmem_col_int_init_ref(WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE *home, uint32_t hint,
void *addr, uint64_t recno, bool internal, bool deleted, WT_PAGE_DELETED *page_del)
{
+ WT_BTREE *btree;
+
+ btree = S2BT(session);
+
ref->home = home;
ref->pindex_hint = hint;
ref->addr = addr;
@@ -653,6 +657,16 @@ __inmem_col_int_init_ref(WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE *home, u
*ref->page_del = *page_del;
}
WT_REF_SET_STATE(ref, WT_REF_DELETED);
+
+ /*
+ * If the tree is already dirty and so will be written, mark the page dirty. (We want to
+ * free the deleted pages, but if the handle is read-only or if the application never
+ * modifies the tree, we're not able to do so.)
+ */
+ if (btree->modified) {
+ WT_RET(__wt_page_modify_init(session, home));
+ __wt_page_only_modify_set(session, home);
+ }
}
return (0);
@@ -807,6 +821,7 @@ __inmem_col_var(
static int
__inmem_row_int(WT_SESSION_IMPL *session, WT_PAGE *page, size_t *sizep)
{
+ WT_BTREE *btree;
WT_CELL_UNPACK_ADDR unpack;
WT_DECL_ITEM(current);
WT_DECL_RET;
@@ -815,6 +830,8 @@ __inmem_row_int(WT_SESSION_IMPL *session, WT_PAGE *page, size_t *sizep)
uint32_t hint;
bool overflow_keys;
+ btree = S2BT(session);
+
WT_RET(__wt_scr_alloc(session, 0, &current));
/*
@@ -876,6 +893,16 @@ __inmem_row_int(WT_SESSION_IMPL *session, WT_PAGE *page, size_t *sizep)
}
WT_REF_SET_STATE(ref, WT_REF_DELETED);
+ /*
+ * If the tree is already dirty and so will be written, mark the page dirty. (We want to
+ * free the deleted pages, but if the handle is read-only or if the application never
+ * modifies the tree, we're not able to do so.)
+ */
+ if (btree->modified) {
+ WT_ERR(__wt_page_modify_init(session, page));
+ __wt_page_only_modify_set(session, page);
+ }
+
ref->addr = unpack.cell;
++refp;
break;
diff --git a/src/third_party/wiredtiger/test/suite/test_truncate19.py b/src/third_party/wiredtiger/test/suite/test_truncate19.py
new file mode 100644
index 00000000000..1afda92e738
--- /dev/null
+++ b/src/third_party/wiredtiger/test/suite/test_truncate19.py
@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-present MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+import os, wttest, suite_random
+from wiredtiger import stat
+from wtdataset import SimpleDataSet
+from wtscenario import make_scenarios
+
+# test_truncate19.py
+#
+# Test to mimic oplog workload in MongoDB. Ensure the deleted pages are
+# cleaned up on disk and we are not using excessive disk space.
+class test_truncate19(wttest.WiredTigerTestCase):
+ conn_config = 'statistics=(all)'
+
+ format_values = [
+ ('string_row', dict(key_format='S', value_format='S')),
+ ]
+ scenarios = make_scenarios(format_values)
+
+ def append_rows(self, uri, ds, start_row, nrows, value):
+ cursor = self.session.open_cursor(uri)
+ self.session.begin_transaction()
+ for i in range(start_row, start_row + nrows + 1):
+ cursor[ds.key(i)] = value
+ if i % 2 == 0:
+ self.session.commit_transaction()
+ self.session.begin_transaction()
+ self.session.commit_transaction()
+ cursor.close()
+
+ def do_truncate(self, ds, start_row, nrows):
+ self.session.begin_transaction()
+ hicursor = self.session.open_cursor(ds.uri)
+ hicursor.set_key(ds.key(start_row + nrows))
+ self.session.truncate(None, None, hicursor, None)
+ self.session.commit_transaction()
+
+ def test_truncate19(self):
+ uri = 'table:oplog'
+ nrows = 1000000
+
+ # Create a table.
+ ds = SimpleDataSet(self, uri, 0, key_format=self.key_format, value_format=self.value_format)
+ ds.populate()
+ ds_dummy = SimpleDataSet(self, 'table:dummy', 0, key_format=self.key_format, value_format=self.value_format)
+ ds_dummy.populate()
+
+ value_a = "aaaaa" * 100
+
+ # Write some data
+ self.append_rows(uri, ds, 1, nrows, value_a)
+ self.session.checkpoint()
+
+ # Reopen the database.
+ self.reopen_conn()
+
+ # Session for checkpoint
+ session2 = self.conn.open_session()
+ # Session for long running transaction, to make truncate not globally visible
+ session3 = self.conn.open_session()
+
+ trunc_rows = 0
+ start_num = 1
+ end_num = nrows
+ for i in range(1, 50):
+ # Start a long running transaction
+ session3.begin_transaction()
+ trunc_rows = 10000
+
+ self.do_truncate(ds, start_num, trunc_rows)
+
+ # Check stats to make sure we fast-deleted at least one page.
+ stat_cursor = self.session.open_cursor('statistics:', None, None)
+ fastdelete_pages = stat_cursor[stat.conn.rec_page_delete_fast][2]
+
+ self.assertGreater(fastdelete_pages, 0)
+
+ # Take a checkpoint.
+ session2.checkpoint()
+ # Ensure the datasize is smaller than 600M
+ self.assertGreater(600000000, os.path.getsize("oplog.wt"))
+ session3.rollback_transaction()
+
+ self.append_rows(uri, ds, end_num, trunc_rows, value_a)
+
+ end_num = end_num + trunc_rows
+ start_num = start_num + trunc_rows
+
+ session2.checkpoint()
+ # Ensure the datasize is smaller than 600M
+ self.assertGreater(600000000, os.path.getsize("oplog.wt"))
+
+if __name__ == '__main__':
+ wttest.run()