diff options
-rwxr-xr-x | src/third_party/wiredtiger/dist/s_all | 1 | ||||
-rw-r--r-- | src/third_party/wiredtiger/dist/s_define.list | 3 | ||||
-rw-r--r-- | src/third_party/wiredtiger/dist/s_string.ok | 4 | ||||
-rwxr-xr-x | src/third_party/wiredtiger/dist/verbose.py | 88 | ||||
-rw-r--r-- | src/third_party/wiredtiger/import.data | 2 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/include/verbose.h | 121 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/include/wiredtiger.in | 97 | ||||
-rw-r--r-- | src/third_party/wiredtiger/src/support/err.c | 7 | ||||
-rwxr-xr-x | src/third_party/wiredtiger/test/suite/test_verbose01.py | 14 | ||||
-rwxr-xr-x | src/third_party/wiredtiger/test/suite/test_verbose03.py | 16 |
10 files changed, 270 insertions, 83 deletions
diff --git a/src/third_party/wiredtiger/dist/s_all b/src/third_party/wiredtiger/dist/s_all index b5f25c05519..916115ca3ee 100755 --- a/src/third_party/wiredtiger/dist/s_all +++ b/src/third_party/wiredtiger/dist/s_all @@ -86,6 +86,7 @@ run "python api_err.py" run "python flags.py" run "python log.py" run "python stat.py" +run "python verbose.py" run "sh ./s_copyright" run "sh ./s_style" run "./s_clang-format" diff --git a/src/third_party/wiredtiger/dist/s_define.list b/src/third_party/wiredtiger/dist/s_define.list index 14cd37f2e58..96b4fa27f54 100644 --- a/src/third_party/wiredtiger/dist/s_define.list +++ b/src/third_party/wiredtiger/dist/s_define.list @@ -36,9 +36,6 @@ WT_ERR_ASSERT WT_ERR_ERROR_OK WT_EXT_FOREACH_OFF WT_FILE_HANDLE_WILLNEED -WT_GEN_VERBOSE_CATEGORIES -WT_GEN_VERBOSE_ENUM -WT_GEN_VERBOSE_ENUM_STR WT_HANDLE_CLOSED WT_HANDLE_NULLABLE WT_HS_COMPRESSOR diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 692d5c40592..8a171ba9090 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -130,11 +130,13 @@ ENCRYPTOR ENOENT ENOMEM ENOTSUP +ENUM EOF ERET ESET ETIME ETIMEDOUT +EVICTSERVER EXTLIST EXTSUBPATH Encryptor @@ -144,6 +146,7 @@ Eron FALLOC FALLTHROUGH FH +FILEOPS FIXME FLCS FLD @@ -184,6 +187,7 @@ GetModuleHandleExW GetProcAddress Getters Google +HANDLEOPS HFS HHHH HHHHLL diff --git a/src/third_party/wiredtiger/dist/verbose.py b/src/third_party/wiredtiger/dist/verbose.py new file mode 100755 index 00000000000..07f166f1bf7 --- /dev/null +++ b/src/third_party/wiredtiger/dist/verbose.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python + +import re +from dist import compare_srcfile + +verbose_categories = [] +filename = '../src/include/wiredtiger.in' +start_tag = 'VERBOSE ENUM START' +stop_tag = 'VERBOSE ENUM STOP' + +# Retrieve all verbose categories. +with open(filename, 'r') as f: + in_section = False + pattern = re.compile("^WT_VERB_[A-Z_]+$") + for line in f: + if line.find(start_tag) != -1: + in_section = True + continue + if line.find(stop_tag) != -1: + break + if in_section: + # Remove any leading and trailing whitespaces. + line = line.strip() + content = line.split(',') + + # The length will be <= 1 if no comma is present. + assert len(content) > 1, content + verbose_category = content[0] + # Check the value follows the expected format. + assert pattern.match(verbose_category), \ + 'The category ' + verbose_category + ' does not follow the expected syntax.' + + # Save the category. + verbose_categories.append(content[0]) + + assert len(verbose_categories) > 0, 'No verbose categories have been found in ' + filename +f.close() + +filename = '../src/include/verbose.h' +start_tag = 'AUTOMATIC VERBOSE ENUM STRING GENERATION START' +stop_tag = 'AUTOMATIC VERBOSE ENUM STRING GENERATION STOP' + +# Generate all verbose categories as strings +tmp_file = '__tmp_verbose' +with open(filename, 'r') as f: + + tfile = open(tmp_file, 'w') + end_found = False + in_section = False + start_found = False + + for line in f: + + line_tmp = line + + if line.find(stop_tag) != -1: + assert start_found, 'Missing start tag: ' + start_tag + assert not end_found, 'Found end tag again: ' + stop_tag + + end_found = True + in_section = False + + elif line.find(start_tag) != -1: + assert not start_found and not end_found, 'Duplicate tag' + + start_found = True + in_section = True + + escape_char = '' + if line.strip().endswith('\\'): + escape_char = '\\' + + indentation = len(line) - len(line.lstrip()) + + for category in verbose_categories: + line_tmp += indentation * ' ' + "\"" + category + "\", %s\n" % escape_char + + elif in_section: + continue + + tfile.write(line_tmp) + + assert not in_section and start_found and end_found, 'File syntax incorrect' + + tfile.close() + compare_srcfile(tmp_file, filename) + +f.close() diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 4b1c0fffcaa..e2ed874b823 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "7d2c1389a470ccc0db0f61a703191a6f7fab3f97" + "commit": "bd9148d45bc5f7b8e2a9b59f96d4785d0db30f13" } diff --git a/src/third_party/wiredtiger/src/include/verbose.h b/src/third_party/wiredtiger/src/include/verbose.h index 96f2995ca23..1102e47db4f 100644 --- a/src/third_party/wiredtiger/src/include/verbose.h +++ b/src/third_party/wiredtiger/src/include/verbose.h @@ -6,82 +6,57 @@ * See the file LICENSE for redistribution information. */ -#define WT_GEN_VERBOSE_ENUM(value) value, -#define WT_GEN_VERBOSE_ENUM_STR(value_str) #value_str, - -/* Disable clang format for the following macro to keep the categories formatted on a new line. */ /* clang-format off */ -#define WT_GEN_VERBOSE_CATEGORIES(gen) \ - gen(WT_VERB_API) \ - gen(WT_VERB_BACKUP) \ - gen(WT_VERB_BLKCACHE) \ - gen(WT_VERB_BLOCK) \ - gen(WT_VERB_CHECKPOINT) \ - gen(WT_VERB_CHECKPOINT_CLEANUP) \ - gen(WT_VERB_CHECKPOINT_PROGRESS) \ - gen(WT_VERB_COMPACT) \ - gen(WT_VERB_COMPACT_PROGRESS) \ - gen(WT_VERB_DEFAULT) \ - gen(WT_VERB_ERROR_RETURNS) \ - gen(WT_VERB_EVICT) \ - gen(WT_VERB_EVICTSERVER) \ - gen(WT_VERB_EVICT_STUCK) \ - gen(WT_VERB_EXTENSION) \ - gen(WT_VERB_FILEOPS) \ - gen(WT_VERB_GENERATION) \ - gen(WT_VERB_HANDLEOPS) \ - gen(WT_VERB_HS) \ - gen(WT_VERB_HS_ACTIVITY) \ - gen(WT_VERB_LOG) \ - gen(WT_VERB_LSM) \ - gen(WT_VERB_LSM_MANAGER) \ - gen(WT_VERB_METADATA) \ - gen(WT_VERB_MUTEX) \ - gen(WT_VERB_OUT_OF_ORDER) \ - gen(WT_VERB_OVERFLOW) \ - gen(WT_VERB_READ) \ - gen(WT_VERB_RECONCILE) \ - gen(WT_VERB_RECOVERY) \ - gen(WT_VERB_RECOVERY_PROGRESS) \ - gen(WT_VERB_RTS) \ - gen(WT_VERB_SALVAGE) \ - gen(WT_VERB_SHARED_CACHE) \ - gen(WT_VERB_SPLIT) \ - gen(WT_VERB_TEMPORARY) \ - gen(WT_VERB_THREAD_GROUP) \ - gen(WT_VERB_TIERED) \ - gen(WT_VERB_TIMESTAMP) \ - gen(WT_VERB_TRANSACTION) \ - gen(WT_VERB_VERIFY) \ - gen(WT_VERB_VERSION) \ - gen(WT_VERB_WRITE) +#define WT_VERBOSE_CATEGORY_STR_INIT \ + { \ + /* AUTOMATIC VERBOSE ENUM STRING GENERATION START */ \ + "WT_VERB_API", \ + "WT_VERB_BACKUP", \ + "WT_VERB_BLKCACHE", \ + "WT_VERB_BLOCK", \ + "WT_VERB_CHECKPOINT", \ + "WT_VERB_CHECKPOINT_CLEANUP", \ + "WT_VERB_CHECKPOINT_PROGRESS", \ + "WT_VERB_COMPACT", \ + "WT_VERB_COMPACT_PROGRESS", \ + "WT_VERB_DEFAULT", \ + "WT_VERB_ERROR_RETURNS", \ + "WT_VERB_EVICT", \ + "WT_VERB_EVICTSERVER", \ + "WT_VERB_EVICT_STUCK", \ + "WT_VERB_EXTENSION", \ + "WT_VERB_FILEOPS", \ + "WT_VERB_GENERATION", \ + "WT_VERB_HANDLEOPS", \ + "WT_VERB_HS", \ + "WT_VERB_HS_ACTIVITY", \ + "WT_VERB_LOG", \ + "WT_VERB_LSM", \ + "WT_VERB_LSM_MANAGER", \ + "WT_VERB_MUTEX", \ + "WT_VERB_METADATA", \ + "WT_VERB_OUT_OF_ORDER", \ + "WT_VERB_OVERFLOW", \ + "WT_VERB_READ", \ + "WT_VERB_RECONCILE", \ + "WT_VERB_RECOVERY", \ + "WT_VERB_RECOVERY_PROGRESS", \ + "WT_VERB_RTS", \ + "WT_VERB_SALVAGE", \ + "WT_VERB_SHARED_CACHE", \ + "WT_VERB_SPLIT", \ + "WT_VERB_TEMPORARY", \ + "WT_VERB_THREAD_GROUP", \ + "WT_VERB_TIERED", \ + "WT_VERB_TIMESTAMP", \ + "WT_VERB_TRANSACTION", \ + "WT_VERB_VERIFY", \ + "WT_VERB_VERSION", \ + "WT_VERB_WRITE", \ + /* AUTOMATIC VERBOSE ENUM STRING GENERATION STOP */ \ + } /* clang-format on */ -/* Permitted verbose event categories that can be used when defining a verbose message. */ -typedef enum { - WT_GEN_VERBOSE_CATEGORIES(WT_GEN_VERBOSE_ENUM) - /* This entry needs to be the last in order to track the number of category items. */ - WT_VERB_NUM_CATEGORIES, -} WT_VERBOSE_CATEGORY; - -/* Convert a category to its string representation. */ -#define WT_VERBOSE_CATEGORY_STR(category) \ - (const char *[]){WT_GEN_VERBOSE_CATEGORIES(WT_GEN_VERBOSE_ENUM_STR)}[category] - -/* - * Permitted verbosity levels; to be used when defining verbose messages. The levels define a range - * of severity categories, with WT_VERBOSE_ERROR being the lowest, most critical level (used by - * messages on critical error paths) and WT_VERBOSE_DEBUG being the highest verbosity/informational - * level (mostly adopted for debugging). - */ -typedef enum { - WT_VERBOSE_ERROR = -3, - WT_VERBOSE_WARNING, - WT_VERBOSE_NOTICE, - WT_VERBOSE_INFO, - WT_VERBOSE_DEBUG -} WT_VERBOSE_LEVEL; - /* Convert a verbose level to its string representation. */ #define WT_VERBOSE_LEVEL_STR(level, level_str) \ do { \ diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index aa33185db64..ac031ba0dc9 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -7090,6 +7090,103 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); */ /*! @} */ +/******************************************* + * Verbose categories + *******************************************/ +/*! + * @addtogroup wt + * @{ + */ +/*! + * @name Verbose categories + * @anchor verbose_categories + * @{ + */ +/*! + * WiredTiger verbose event categories. + * Note that the verbose categories cover a wide set of sub-systems and operations + * within WiredTiger. As such, the categories are subject to change and evolve + * between different WiredTiger releases. + */ +typedef enum { +/* VERBOSE ENUM START */ + WT_VERB_API, /*!< API messages. */ + WT_VERB_BACKUP, /*!< Backup messages. */ + WT_VERB_BLKCACHE, + WT_VERB_BLOCK, /*!< Block manager messages. */ + WT_VERB_CHECKPOINT, /*!< Checkpoint messages. */ + WT_VERB_CHECKPOINT_CLEANUP, + WT_VERB_CHECKPOINT_PROGRESS, /*!< Checkpoint progress messages. */ + WT_VERB_COMPACT, /*!< Compact messages. */ + WT_VERB_COMPACT_PROGRESS, /*!< Compact progress messages. */ + WT_VERB_DEFAULT, + WT_VERB_ERROR_RETURNS, + WT_VERB_EVICT, /*!< Eviction messages. */ + WT_VERB_EVICTSERVER, /*!< Eviction server messages. */ + WT_VERB_EVICT_STUCK, + WT_VERB_EXTENSION, /*!< Extension messages. */ + WT_VERB_FILEOPS, + WT_VERB_GENERATION, + WT_VERB_HANDLEOPS, + WT_VERB_HS, /*!< History store messages. */ + WT_VERB_HS_ACTIVITY, /*!< History store activity messages. */ + WT_VERB_LOG, /*!< Log messages. */ + WT_VERB_LSM, /*!< LSM messages. */ + WT_VERB_LSM_MANAGER, + WT_VERB_MUTEX, + WT_VERB_METADATA, /*!< Metadata messages. */ + WT_VERB_OUT_OF_ORDER, + WT_VERB_OVERFLOW, + WT_VERB_READ, + WT_VERB_RECONCILE, /*!< Reconcile messages. */ + WT_VERB_RECOVERY, /*!< Recovery messages. */ + WT_VERB_RECOVERY_PROGRESS, /*!< Recovery progress messages. */ + WT_VERB_RTS, /*!< RTS messages. */ + WT_VERB_SALVAGE, /*!< Salvage messages. */ + WT_VERB_SHARED_CACHE, + WT_VERB_SPLIT, + WT_VERB_TEMPORARY, + WT_VERB_THREAD_GROUP, + WT_VERB_TIERED, /*!< Tiered messages. */ + WT_VERB_TIMESTAMP, /*!< Timestamp messages. */ + WT_VERB_TRANSACTION, /*!< Transaction messages. */ + WT_VERB_VERIFY, /*!< Verify messages. */ + WT_VERB_VERSION, /*!< Version messages. */ + WT_VERB_WRITE, +/* VERBOSE ENUM STOP */ + WT_VERB_NUM_CATEGORIES +} WT_VERBOSE_CATEGORY; +/*! @} */ + +/******************************************* + * Verbose levels + *******************************************/ +/*! + * @name Verbose levels + * @anchor verbose_levels + * @{ + */ +/*! + * WiredTiger verbosity levels. The levels define a range of severity categories, with + * \c WT_VERBOSE_ERROR being the lowest, most critical level (used by messages on critical error + * paths) and \c WT_VERBOSE_DEBUG being the highest verbosity/informational level (mostly adopted + * for debugging). + */ +typedef enum { + WT_VERBOSE_ERROR = -3, /*!< Error conditions triggered in WiredTiger. */ + WT_VERBOSE_WARNING, /*!< Warning conditions potentially signaling non-imminent errors and + behaviors. */ + WT_VERBOSE_NOTICE, /*!< Messages for significant events in WiredTiger, usually worth + noting. */ + WT_VERBOSE_INFO, /*!< Informational style messages. */ + WT_VERBOSE_DEBUG /*!< Low severity messages, useful for debugging purposes. */ +} WT_VERBOSE_LEVEL; +/*! @} */ +/* + * Verbose section: END + */ +/*! @} */ + #undef __F #if defined(__cplusplus) diff --git a/src/third_party/wiredtiger/src/support/err.c b/src/third_party/wiredtiger/src/support/err.c index a03bb4f30c1..869652ea75c 100644 --- a/src/third_party/wiredtiger/src/support/err.c +++ b/src/third_party/wiredtiger/src/support/err.c @@ -8,6 +8,9 @@ #include "wt_internal.h" +/* Define the string representation of each verbose category. */ +static const char *verbose_category_strings[] = WT_VERBOSE_CATEGORY_STR_INIT; + /* * __handle_error_default -- * Default WT_EVENT_HANDLER->handle_error implementation: send to stderr. @@ -295,7 +298,7 @@ __eventv(WT_SESSION_IMPL *session, bool is_json, int error, const char *func, in err = error == 0 ? NULL : __wt_strerror(session, error, NULL, 0); if (is_json) { /* Category and verbosity level. */ - WT_ERROR_APPEND(p, remain, "\"category\":\"%s\",", WT_VERBOSE_CATEGORY_STR(category)); + WT_ERROR_APPEND(p, remain, "\"category\":\"%s\",", verbose_category_strings[category]); WT_ERROR_APPEND(p, remain, "\"category_id\":%" PRIu32 ",", category); WT_ERROR_APPEND(p, remain, "\"verbose_level\":\"%s\",", verbosity_level_tag); WT_ERROR_APPEND(p, remain, "\"verbose_level_id\":%d,", level); @@ -361,7 +364,7 @@ __eventv(WT_SESSION_IMPL *session, bool is_json, int error, const char *func, in } else { /* Category and verbosity level. */ WT_ERROR_APPEND( - p, remain, ": [%s][%s]", WT_VERBOSE_CATEGORY_STR(category), verbosity_level_tag); + p, remain, ": [%s][%s]", verbose_category_strings[category], verbosity_level_tag); if (func != NULL) WT_ERROR_APPEND(p, remain, ": %s, %d", func, line); diff --git a/src/third_party/wiredtiger/test/suite/test_verbose01.py b/src/third_party/wiredtiger/test/suite/test_verbose01.py index 9a96f8dabe7..31d9e85957f 100755 --- a/src/third_party/wiredtiger/test/suite/test_verbose01.py +++ b/src/third_party/wiredtiger/test/suite/test_verbose01.py @@ -59,20 +59,34 @@ class test_verbose_base(wttest.WiredTigerTestCase, suite_subprocess): # Validates the JSON schema of a given event handler message, ensuring the schema is consistent and expected. def validate_json_schema(self, json_msg): expected_schema = dict(self.expected_json_schema) + for field in json_msg: # Assert the JSON field is valid and expected. self.assertTrue(field in expected_schema, 'Unexpected field "%s" in JSON message: %s' % (field, str(json_msg))) + # Assert the type of the JSON field is expected. self.assertEqual(type(json_msg[field]), expected_schema[field]['type'], 'Unexpected type of field "%s" in JSON message, expected "%s" but got "%s": %s' % (field, str(expected_schema[field]['type']), str(type(json_msg[field])), str(json_msg))) + expected_schema.pop(field, None) + # Go through the remaining fields in the schema and ensure we've seen all the fields that are always expected be present # in the JSON message for field in expected_schema: self.assertFalse(expected_schema[field]['always_expected'], 'Expected field "%s" in JSON message, but not found: %s' % (field, str(json_msg))) + # Validates the verbose category (and ID) in a JSON message is expected. + def validate_json_category(self, json_msg, expected_categories): + # Assert the category field is in the JSON message. + self.assertTrue('category' in json_msg, 'JSON message missing "category" field') + self.assertTrue('category_id' in json_msg, 'JSON message missing "category_id" field') + # Assert the category field values in the JSON message are expected. + self.assertTrue(json_msg['category'] in expected_categories, 'Unexpected verbose category "%s"' % json_msg['category']) + self.assertTrue(json_msg['category_id'] == expected_categories[json_msg['category']], + 'The category ID received in the message "%d" does not match its expected definition "%d"' % (json_msg['category_id'], expected_categories[json_msg['category']])) + def create_verbose_configuration(self, categories): if len(categories) == 0: return '' diff --git a/src/third_party/wiredtiger/test/suite/test_verbose03.py b/src/third_party/wiredtiger/test/suite/test_verbose03.py index b6bdff9bc59..eaff25d8460 100755 --- a/src/third_party/wiredtiger/test/suite/test_verbose03.py +++ b/src/third_party/wiredtiger/test/suite/test_verbose03.py @@ -42,7 +42,7 @@ class test_verbose03(test_verbose_base): nlines = 50000 @contextmanager - def expect_event_handler_json(self, config, stdErr=False): + def expect_event_handler_json(self, config, expected_categories, stdErr=False): # Clean the stdout/stderr resource before yielding the context to the execution block. We only want to # capture the verbose output of the using context (ignoring any previous output up to this point). if stdErr: @@ -80,6 +80,7 @@ class test_verbose03(test_verbose_base): self.pr('Unable to parse JSON message format: %s' % line) raise e self.validate_json_schema(msg) + self.validate_json_category(msg, expected_categories) # Close the connection resource and clean up the contents of the stdout/stderr file, flushing out the # verbose output that occurred during the execution of this context. @@ -95,8 +96,12 @@ class test_verbose03(test_verbose_base): # this test. self.close_conn() + expected_verbose_categories = { + 'WT_VERB_API': wiredtiger.WT_VERB_API, + 'WT_VERB_VERSION': wiredtiger.WT_VERB_VERSION, + } # Test passing a single verbose category, 'api'. - with self.expect_event_handler_json(self.create_verbose_configuration(['api'])) as conn: + with self.expect_event_handler_json(self.create_verbose_configuration(['api']), expected_verbose_categories) as conn: # Perform a set of simple API operations (table creations and cursor operations) to generate verbose API # messages. uri = 'table:test_verbose03_api' @@ -108,7 +113,7 @@ class test_verbose03(test_verbose_base): session.close() # Test passing multiple verbose categories, being 'api' & 'version'. - with self.expect_event_handler_json(self.create_verbose_configuration(['api','version'])) as conn: + with self.expect_event_handler_json(self.create_verbose_configuration(['api','version']), expected_verbose_categories) as conn: # Perform a set of simple API operations (table creations and cursor operations) to generate verbose API # messages. Beyond opening the connection resource, we shouldn't need to do anything special for the version # category. @@ -125,8 +130,11 @@ class test_verbose03(test_verbose_base): # this test. self.close_conn() + expected_verbose_categories = { + 'WT_VERB_DEFAULT': wiredtiger.WT_VERB_DEFAULT, + } # Test generating an error message, ensuring the JSON output is valid. - with self.expect_event_handler_json('', stdErr=True) as conn: + with self.expect_event_handler_json('', expected_verbose_categories, stdErr=True) as conn: # Attempt to begin a read transaction with an invalid timestamp, inorder to produce an error message. uri = 'table:test_verbose03_error' session = conn.open_session() |