diff options
author | Etienne Petrel <etienne.petrel@mongodb.com> | 2021-10-19 06:19:34 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-10-19 07:35:46 +0000 |
commit | 69fbb9eccbd73b300dfab89ab0282d2eefbebf03 (patch) | |
tree | c3585863fbf204feed656f37ea919cd2952e9e61 | |
parent | 3529fd75f5bdf8092ccc6fb7ff33609470c2d19b (diff) | |
download | mongo-69fbb9eccbd73b300dfab89ab0282d2eefbebf03.tar.gz |
Import wiredtiger: 4692a642021eff5b40438a8b4ee0bf85be9d9dc8 from branch mongodb-master
ref: 0d11c31127..4692a64202
for: 5.2.0
WT-8189 Add useful information for compact analysis to WT verbose messaging
-rw-r--r-- | src/third_party/wiredtiger/import.data | 2 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/block/block_compact.c | 84 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/session/session_compact.c | 18 |
3 files changed, 80 insertions, 24 deletions
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index d99f4370a11..cbadb94e170 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "0d11c31127fdeefad6a85b9567923e484c28dd0e" + "commit": "4692a642021eff5b40438a8b4ee0bf85be9d9dc8" } diff --git a/src/third_party/wiredtiger/src/block/block_compact.c b/src/third_party/wiredtiger/src/block/block_compact.c index 51614c4aae7..f6be40cfa78 100644 --- a/src/third_party/wiredtiger/src/block/block_compact.c +++ b/src/third_party/wiredtiger/src/block/block_compact.c @@ -8,7 +8,7 @@ #include "wt_internal.h" -static void __block_dump_avail(WT_SESSION_IMPL *, WT_BLOCK *, bool); +static void __block_dump_file_stat(WT_SESSION_IMPL *, WT_BLOCK *, bool); /* * __wt_block_compact_start -- @@ -44,7 +44,7 @@ __wt_block_compact_end(WT_SESSION_IMPL *session, WT_BLOCK *block) /* Dump the results of the compaction pass. */ if (WT_VERBOSE_ISSET(session, WT_VERB_COMPACT)) { __wt_spin_lock(session, &block->live_lock); - __block_dump_avail(session, block, false); + __block_dump_file_stat(session, block, false); __wt_spin_unlock(session, &block->live_lock); } return (0); @@ -68,14 +68,19 @@ __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, bool *skipp) * we need some metrics to decide if it's worth doing. Ignore small files, and files where we * are unlikely to recover 10% of the file. */ - if (block->size <= WT_MEGABYTE) + if (block->size <= WT_MEGABYTE) { + __wt_verbose(session, WT_VERB_COMPACT, + "%s: skipping because the file size must be greater than 1MB: %" PRIuMAX "B.", + block->name, (uintmax_t)block->size); + return (0); + } __wt_spin_lock(session, &block->live_lock); /* Dump the current state of the file. */ if (WT_VERBOSE_ISSET(session, WT_VERB_COMPACT)) - __block_dump_avail(session, block, true); + __block_dump_file_stat(session, block, true); /* Sum the available bytes in the initial 80% and 90% of the file. */ avail_eighty = avail_ninety = 0; @@ -109,6 +114,11 @@ __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, bool *skipp) } __wt_verbose(session, WT_VERB_COMPACT, + "%s: total reviewed %" PRIu64 " pages, total skipped %" PRIu64 " pages, total wrote %" PRIu64 + " pages", + block->name, block->compact_pages_reviewed, block->compact_pages_skipped, + block->compact_pages_written); + __wt_verbose(session, WT_VERB_COMPACT, "%s: %" PRIuMAX "MB (%" PRIuMAX ") available space in the first 80%% of the file", block->name, (uintmax_t)avail_eighty / WT_MEGABYTE, (uintmax_t)avail_eighty); __wt_verbose(session, WT_VERB_COMPACT, @@ -175,15 +185,45 @@ __wt_block_compact_page_skip( } /* - * __block_dump_avail -- - * Dump out the avail list so we can see what compaction will look like. + * __block_dump_bucket_stat -- + * Dump out the information about available and used blocks in the given bucket (part of the + * file). */ static void -__block_dump_avail(WT_SESSION_IMPL *session, WT_BLOCK *block, bool start) +__block_dump_bucket_stat(WT_SESSION_IMPL *session, uintmax_t file_size, uintmax_t file_free, + uintmax_t bucket_size, uintmax_t bucket_free, u_int bucket_pct) +{ + uintmax_t bucket_used, free_pct, used_pct; + + free_pct = used_pct = 0; + + /* Handle rounding error in which case bucket used size can be negative. */ + bucket_used = (bucket_size > bucket_free) ? (bucket_size - bucket_free) : 0; + + if (file_free != 0) + free_pct = (bucket_free * 100) / file_free; + + if (file_size > file_free) + used_pct = (bucket_used * 100) / (file_size - file_free); + + __wt_verbose(session, WT_VERB_COMPACT, + "%2u%%: %12" PRIuMAX "MB, (free: %" PRIuMAX "B, %" PRIuMAX "%%), (used: %" PRIuMAX + "MB, %" PRIuMAX "B, %" PRIuMAX "%%)", + bucket_pct, bucket_free / WT_MEGABYTE, bucket_free, free_pct, bucket_used / WT_MEGABYTE, + bucket_used, used_pct); +} + +/* + * __block_dump_file_stat -- + * Dump out the avail/used list so we can see what compaction will look like. + */ +static void +__block_dump_file_stat(WT_SESSION_IMPL *session, WT_BLOCK *block, bool start) { WT_EXT *ext; WT_EXTLIST *el; - wt_off_t decile[10], percentile[100], size, v; + wt_off_t decile[10], percentile[100], size; + uintmax_t bucket_size; u_int i; el = &block->live.avail; @@ -226,19 +266,21 @@ __block_dump_avail(WT_SESSION_IMPL *session, WT_BLOCK *block, bool start) #ifdef __VERBOSE_OUTPUT_PERCENTILE /* * The verbose output always displays 10% buckets, running this code as well also displays 1% - * buckets. + * buckets. There will be rounding error in the `used` stats because of the bucket size + * calculation. Adding 50 to minimize the rounding error. */ - for (i = 0; i < WT_ELEMENTS(percentile); ++i) { - v = percentile[i] * 512; - __wt_verbose(session, WT_VERB_COMPACT, - "%2u%%: %12" PRIuMAX "MB, (%" PRIuMAX "B, %" PRIuMAX "%%)", i, (uintmax_t)v / WT_MEGABYTE, - (uintmax_t)v, (uintmax_t)((v * 100) / (wt_off_t)el->bytes)); - } + bucket_size = (uintmax_t)((size + 50) / 100); + for (i = 0; i < WT_ELEMENTS(percentile); ++i) + __block_dump_bucket_stat(session, (uintmax_t)size, (uintmax_t)el->bytes, bucket_size, + (uintmax_t)percentile[i] * 512, i); #endif - for (i = 0; i < WT_ELEMENTS(decile); ++i) { - v = decile[i] * 512; - __wt_verbose(session, WT_VERB_COMPACT, - "%2u%%: %12" PRIuMAX "MB, (%" PRIuMAX "B, %" PRIuMAX "%%)", i * 10, - (uintmax_t)v / WT_MEGABYTE, (uintmax_t)v, (uintmax_t)((v * 100) / (wt_off_t)el->bytes)); - } + + /* + * There will be rounding error in the `used` stats because of the bucket size calculation. + * Adding 5 to minimize the rounding error. + */ + bucket_size = (uintmax_t)((size + 5) / 10); + for (i = 0; i < WT_ELEMENTS(decile); ++i) + __block_dump_bucket_stat(session, (uintmax_t)size, (uintmax_t)el->bytes, bucket_size, + (uintmax_t)decile[i] * 512, i * 10); } diff --git a/src/third_party/wiredtiger/src/session/session_compact.c b/src/third_party/wiredtiger/src/session/session_compact.c index b57a734b29b..285bacf49f9 100644 --- a/src/third_party/wiredtiger/src/session/session_compact.c +++ b/src/third_party/wiredtiger/src/session/session_compact.c @@ -189,10 +189,17 @@ __wt_session_compact_check_timeout(WT_SESSION_IMPL *session) return (0); __wt_epoch(session, &end); + ret = session->compact->max_time > WT_TIMEDIFF_SEC(end, session->compact->begin) ? 0 : ETIMEDOUT; - if (ret != 0) + if (ret != 0) { WT_STAT_CONN_INCR(session, session_table_compact_timeout); + + __wt_verbose(session, WT_VERB_COMPACT, + "Compact has timed out! The operation has been running for %" PRIu64 + " second(s). Configured timeout is %" PRIu64 " second(s).", + WT_TIMEDIFF_SEC(end, session->compact->begin), session->compact->max_time); + } return (ret); } @@ -279,6 +286,10 @@ __compact_worker(WT_SESSION_IMPL *session) } ret = 0; another_pass = true; + + __wt_verbose(session, WT_VERB_COMPACT, "%s", + "Data handle compaction failed with EBUSY but the cache is not stuck. " + "Will give it another go."); } } if (!another_pass) @@ -329,8 +340,11 @@ __wt_session_compact(WT_SESSION *wt_session, const char *uri, const char *config } /* In-memory ignores compaction operations. */ - if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) + if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) { + __wt_verbose( + session, WT_VERB_COMPACT, "%s", "Compact does not work for in-memory databases."); goto err; + } /* * Non-LSM object compaction requires checkpoints, which are impossible in transactional |