diff options
Diffstat (limited to 'src/third_party/wiredtiger')
25 files changed, 356 insertions, 174 deletions
diff --git a/src/third_party/wiredtiger/bench/workgen/workgen.cxx b/src/third_party/wiredtiger/bench/workgen/workgen.cxx index a319e335255..ea2d0529e9f 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen.cxx +++ b/src/third_party/wiredtiger/bench/workgen/workgen.cxx @@ -111,10 +111,11 @@ struct WorkloadRunnerConnection { WT_CONNECTION *connection; }; -// The number of contexts. Normally there is one context created, but it will -// be possible to use several eventually. More than one is not yet -// implemented, but we must at least guard against the caller creating more -// than one. +/* + * The number of contexts. Normally there is one context created, but it will be possible to use + * several eventually. More than one is not yet implemented, but we must at least guard against the + * caller creating more than one. + */ static uint32_t context_count = 0; static void * @@ -288,9 +289,10 @@ monitor_main(void *arg) return (nullptr); } -// Exponentiate (like the pow function), except that it returns an exact -// integral 64 bit value, and if it overflows, returns the maximum possible -// value for the return type. +/* + * Exponentiate (like the pow function), except that it returns an exact integral 64 bit value, and + * if it overflows, returns the maximum possible value for the return type. + */ static uint64_t power64(int base, int exp) { @@ -887,10 +889,11 @@ ThreadRunner::op_create_all(Operation *op, size_t &keysize, size_t &valuesize) #define PARETO_SHAPE 1.5 -// Return a value within the interval [ 0, recno_max ) -// that is weighted toward lower numbers with pareto_param at 0 (the minimum), -// and more evenly distributed with pareto_param at 100 (the maximum). -// +/* + * Return a value within the interval [ 0, recno_max ) that is weighted toward lower numbers with + * pareto_param at 0 (the minimum), and more evenly distributed with pareto_param at 100 (the + * maximum). + */ static uint64_t pareto_calculation(uint32_t randint, uint64_t recno_max, ParetoOptions &pareto) { @@ -907,17 +910,16 @@ pareto_calculation(uint32_t randint, uint64_t recno_max, ParetoOptions &pareto) double U = 1 - r / static_cast<double>(UINT32_MAX); // interval [0, 1) uint32_t result = (uint64_t)((pow(U, S1) - 1) * S2); - // This Pareto calculation chooses out of range values less than 20% - // of the time, depending on pareto_param. For param of 0, it is - // never out of range, for param of 100, 19.2%. For the default - // pareto_param of 20, it will be out of range 2.7% of the time. - // Out of range values are channeled into the first key, - // making it "hot". Unfortunately, that means that using a higher - // param can get a lot lumped into the first bucket. - // - // XXX This matches the behavior of wtperf, we may consider instead - // retrying (modifying the random number) until we get a good value. - // + /* + * This Pareto calculation chooses out of range values less than 20% of the time, depending on + * pareto_param. For param of 0, it is never out of range, for param of 100, 19.2%. For the + * default pareto_param of 20, it will be out of range 2.7% of the time. Out of range values are + * channeled into the first key, making it "hot". Unfortunately, that means that using a higher + * param can get a lot lumped into the first bucket. + * + * XXX This matches the behavior of wtperf, we may consider instead retrying (modifying the + * random number) until we get a good value. + */ if (result > recno_max) result = 0; return (result); @@ -972,10 +974,11 @@ ThreadRunner::op_run(Operation *op) range = op->_table.options.range; if (_throttle != nullptr) { while (_throttle_ops >= _throttle_limit && !_in_transaction && !_stop) { - // Calling throttle causes a sleep until the next time division, - // and we are given a new batch of operations to do before calling - // throttle again. If the number of operations in the batch is - // zero, we'll need to go around and throttle again. + /* + * Calling throttle causes a sleep until the next time division, and we are given a new + * batch of operations to do before calling throttle again. If the number of operations + * in the batch is zero, we'll need to go around and throttle again. + */ WT_ERR(_throttle->throttle(_throttle_ops, &_throttle_limit)); _throttle_ops = 0; if (_throttle_limit != 0) @@ -1014,14 +1017,14 @@ ThreadRunner::op_run(Operation *op) table_uri = op->_table._uri; } - // A potential race: thread1 is inserting, and increments - // Context->_recno[] for fileX.wt. thread2 is doing one of - // remove/search/update and grabs the new value of Context->_recno[] - // for fileX.wt. thread2 randomly chooses the highest recno (which - // has not yet been inserted by thread1), and when it accesses - // the record will get WT_NOTFOUND. It should be somewhat rare - // (and most likely when the threads are first beginning). Any - // WT_NOTFOUND returns are allowed and get their own statistic bumped. + /* + * A potential race: thread1 is inserting, and increments Context->_recno[] for fileX.wt. + * thread2 is doing one of remove/search/update and grabs the new value of Context->_recno[] for + * fileX.wt. thread2 randomly chooses the highest recno (which has not yet been inserted by + * thread1), and when it accesses the record will get WT_NOTFOUND. It should be somewhat rare + * (and most likely when the threads are first beginning). Any WT_NOTFOUND returns are allowed + * and get their own statistic bumped. + */ switch (op->_optype) { case Operation::OP_CHECKPOINT: recno = 0; @@ -1271,32 +1274,33 @@ Throttle::Throttle(ThreadRunner &runner, double throttle, double throttle_burst) _started(false) { - // Our throttling is done by dividing each second into THROTTLE_PER_SEC - // parts (we call the parts divisions). In each division, we perform - // a certain number of operations. This number is approximately - // throttle/THROTTLE_PER_SEC, except that throttle is not necessarily - // a multiple of THROTTLE_PER_SEC, nor is it even necessarily an integer. - // (That way we can have 1000 threads each inserting 0.5 a second). + /* + * Our throttling is done by dividing each second into THROTTLE_PER_SEC parts (we call the parts + * divisions). In each division, we perform a certain number of operations. This number is + * approximately throttle/THROTTLE_PER_SEC, except that throttle is not necessarily a multiple + * of THROTTLE_PER_SEC, nor is it even necessarily an integer. (That way we can have 1000 + * threads each inserting 0.5 a second). + */ ts_clear(_next_div); ASSERT(1000 % THROTTLE_PER_SEC == 0); // must evenly divide _ms_per_div = 1000 / THROTTLE_PER_SEC; _ops_per_div = (uint64_t)ceill(_throttle / THROTTLE_PER_SEC); } -// Each time throttle is called, we sleep and return a number of operations to -// perform next. To implement this we keep a time calculation in _next_div set -// initially to the current time + 1/THROTTLE_PER_SEC. Each call to throttle -// advances _next_div by 1/THROTTLE_PER_SEC, and if _next_div is in the future, -// we sleep for the difference between the _next_div and the current_time. We -// we return (Thread.options.throttle / THROTTLE_PER_SEC) as the number of -// operations, if it does not divide evenly, we'll make sure to not exceed -// the number of operations requested per second. -// -// The only variation is that the amount of individual sleeps is modified by a -// random amount (which varies more widely as Thread.options.throttle_burst is -// greater). This has the effect of randomizing how much clumping happens, and -// ensures that multiple threads aren't executing in lock step. -// +/* + * Each time throttle is called, we sleep and return a number of operations to perform next. To + * implement this we keep a time calculation in _next_div set initially to the current time + + * 1/THROTTLE_PER_SEC. Each call to throttle advances _next_div by 1/THROTTLE_PER_SEC, and if + * _next_div is in the future, we sleep for the difference between the _next_div and the + * current_time. We we return (Thread.options.throttle / THROTTLE_PER_SEC) as the number of + * operations, if it does not divide evenly, we'll make sure to not exceed the number of operations + * requested per second. + * + * The only variation is that the amount of individual sleeps is modified by a random amount (which + * varies more widely as Thread.options.throttle_burst is greater). This has the effect of + * randomizing how much clumping happens, and ensures that multiple threads aren't executing in lock + * step. + */ int Throttle::throttle(uint64_t op_count, uint64_t *op_limit) { @@ -1988,16 +1992,18 @@ Track::complete_with_latency(uint64_t usecs) sec[LATENCY_SEC_BUCKETS - 1]++; } -// Return the latency for which the given percent is lower than it. -// E.g. for percent == 95, returns the latency for which 95% of latencies -// are faster (lower), and 5% are slower (higher). +/* + * Return the latency for which the given percent is lower than it. E.g. for percent == 95, returns + * the latency for which 95% of latencies are faster (lower), and 5% are slower (higher). + */ uint64_t Track::percentile_latency(int percent) const { - // Get the total number of operations in the latency buckets. - // We can't reliably use latency_ops, because this struct was - // added up from Track structures that were being copied while - // being updated. + /* + * Get the total number of operations in the latency buckets. We can't reliably use latency_ops, + * because this struct was added up from Track structures that were being copied while being + * updated. + */ uint64_t total = 0; for (int i = 0; i < LATENCY_SEC_BUCKETS; i++) total += sec[i]; diff --git a/src/third_party/wiredtiger/bench/workgen/workgen.h b/src/third_party/wiredtiger/bench/workgen/workgen.h index 773d2d87693..cb7dcb30f67 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen.h +++ b/src/third_party/wiredtiger/bench/workgen/workgen.h @@ -65,30 +65,31 @@ private: }; #endif -// These classes are all exposed to Python via SWIG. While they may contain -// data that is private to C++, such data must not prevent the objects from -// being shared. Tables, Keys, Values, Operations and Threads can be shared: a -// single Key object might appear in many operations; Operations may appear -// multiple times in a Thread or in different Threads; the same Thread may -// appear multiple times in a Workload list, etc. -// -// Certain kinds of state are allowed: A Table contains a unique pointer that -// is used within the internal part of the Context. Stats contain lots -// of state, but is made available after a Workload.run(). -// -// Python controls the lifetime of (nearly) all objects of these classes. -// The exception is Stat/Track objects, which are also created/used -// internally to calculate and show statistics during a run. -// +/* + * These classes are all exposed to Python via SWIG. While they may contain + * data that is private to C++, such data must not prevent the objects from + * being shared. Tables, Keys, Values, Operations and Threads can be shared: a + * single Key object might appear in many operations; Operations may appear + * multiple times in a Thread or in different Threads; the same Thread may + * appear multiple times in a Workload list, etc. + * + * Certain kinds of state are allowed: A Table contains a unique pointer that + * is used within the internal part of the Context. Stats contain lots + * of state, but is made available after a Workload.run(). + * + * Python controls the lifetime of (nearly) all objects of these classes. + * The exception is Stat/Track objects, which are also created/used + * internally to calculate and show statistics during a run. + */ struct Track { // Threads maintain the total thread operation and total latency they've // experienced. - uint64_t ops_in_progress; // Total operations not completed */ - uint64_t ops; // Total operations completed */ - uint64_t rollbacks; // Total operations rolled back */ + uint64_t ops_in_progress; // Total operations not completed + uint64_t ops; // Total operations completed + uint64_t rollbacks; // Total operations rolled back uint64_t latency_ops; // Total ops sampled for latency - uint64_t latency; // Total latency */ + uint64_t latency; // Total latency uint64_t bucket_ops; // Computed for percentile_latency // Minimum/maximum latency, shared with the monitor thread, that is, the @@ -157,7 +158,6 @@ private: // A Context tracks the current record number for each uri, used // for key generation. -// struct Context { bool _verbose; ContextInternal *_internal; @@ -175,7 +175,6 @@ struct Context { // To prevent silent errors, this class is set up in Python so that new // properties are prevented, only existing properties can be set. -// struct TableOptions { uint_t key_size; uint_t value_size; @@ -336,7 +335,6 @@ struct Operation { // To prevent silent errors, this class is set up in Python so that new // properties are prevented, only existing properties can be set. -// struct ThreadOptions { std::string name; std::string session_config; @@ -365,12 +363,14 @@ private: OptionsList _options; }; -// This is a list of threads, which may be used in the Workload constructor. -// It participates with ThreadList defined on the SWIG/Python side and -// some Python operators added to Thread to allow Threads to be easily -// composed using '+' and multiplied (by integer counts) using '*'. -// Users of the workgen API in Python don't ever need to use -// ThreadListWrapper or ThreadList. +/* + * This is a list of threads, which may be used in the Workload constructor. + * It participates with ThreadList defined on the SWIG/Python side and + * some Python operators added to Thread to allow Threads to be easily + * composed using '+' and multiplied (by integer counts) using '*'. + * Users of the workgen API in Python don't ever need to use + * ThreadListWrapper or ThreadList. + */ struct ThreadListWrapper { std::vector<Thread> _threads; @@ -432,7 +432,6 @@ struct Transaction { // To prevent silent errors, this class is set up in Python so that new // properties are prevented, only existing properties can be set. -// struct WorkloadOptions { int max_latency; std::string report_file; diff --git a/src/third_party/wiredtiger/bench/workgen/workgen_int.h b/src/third_party/wiredtiger/bench/workgen/workgen_int.h index 3f48f8fea19..7df97af587d 100644 --- a/src/third_party/wiredtiger/bench/workgen/workgen_int.h +++ b/src/third_party/wiredtiger/bench/workgen/workgen_int.h @@ -41,9 +41,11 @@ extern "C" { namespace workgen { -// A 'tint' or ('table integer') is a unique small value integer -// assigned to each table URI in use. Currently, we assign it once, -// and its value persists through the lifetime of the Context. +/* + * A 'tint' or ('table integer') is a unique small value integer + * assigned to each table URI in use. Currently, we assign it once, + * and its value persists through the lifetime of the Context. + */ typedef uint32_t tint_t; struct ThreadRunner; @@ -70,9 +72,11 @@ struct WorkgenTimeStamp { } }; -// A exception generated by the workgen classes. Methods generally return an -// int errno, so this is useful primarily for notifying the caller about -// failures in constructors. +/* + * A exception generated by the workgen classes. Methods generally return an + * int errno, so this is useful primarily for notifying the caller about + * failures in constructors. + */ struct WorkgenException { WorkgenException() = default; WorkgenException(int err, const std::string& msg) { @@ -104,9 +108,11 @@ struct Throttle { Throttle(ThreadRunner &runner, double throttle, double burst); ~Throttle() = default; - // Called with the number of operations since the last throttle. - // Sleeps for any needed amount and returns the number operations the - // caller should perform before the next call to throttle. + /* + * Called with the number of operations since the last throttle. + * Sleeps for any needed amount and returns the number operations the + * caller should perform before the next call to throttle. + */ int throttle(uint64_t op_count, uint64_t *op_limit); }; diff --git a/src/third_party/wiredtiger/dist/comment_style.py b/src/third_party/wiredtiger/dist/comment_style.py new file mode 100755 index 00000000000..9d44294a2f3 --- /dev/null +++ b/src/third_party/wiredtiger/dist/comment_style.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 + +import argparse, os, re, subprocess, sys + +def report_illegal_comment(file_name, line_num, line, multiline): + print("Illegal " + multiline + "comment in " + file_name + ":" + str(line_num - 1) + " " + + line.strip('\n')) + +def check_c_comments(file_name): + count = 0 + with open(file_name) as f: + line_num = 0 + for line in f: + stripped = line.strip() + if has_single_line_comment(stripped): + report_illegal_comment(file_name, line_num, line, "") + count +=1 + line_num += 1 + return count + +def has_single_line_comment(stripped_line): + # We don't worry about whitespace in matches as we strip the line of whitespace. + # URLs are a false positive that we want to ignore. + url = re.compile(r'https?:\/\/') + + # // Style comments. The gist of it is: match anything except certain + # combinations of characters before a //. The trick is in the "certain combinations + # of characters" part, which consists of an odd number of un-escaped string + # markers, while ignoring everything else. As an aside, the "?:" notation indicates + # a non-capturing group, and speeds up the script considerably since we're saving + # any captures. + single_line_comment = re.compile(r'^(?:[^"\\]|"(?:[^"\\]|\\.)*"|\\.)*//(?:.*)$') + + return single_line_comment.match(stripped_line) and not url.search(stripped_line) + +def check_cpp_comments(file_name): + # Text before comments + text_check = re.compile(r'^[^\/\n]+') + + with open(file_name) as f: + count = 0 + line_num = 0 + length = 0 + for line in f: + stripped = line.strip() + if has_single_line_comment(stripped): + # Detecting multi-line comments of // is not easy as technically they can occur + # on contiguous lines without being multi-line. + # E.g: + # int height; // the height of our object + # int width; // the width of our object + # So we can't just count the number of lines we need to check that there is no text + # preceding the comment in subsequent lines. + if length != 0 and not text_check.match(line.strip()): + # If the comment is length 2 and we found another matching line then we have + # found an illegal comment. + if length == 2: + report_illegal_comment(file_name, line_num, line, "multiline ") + count += 1 + length += 1 + # Try and print only one error per comment, just keep incrementing the count and + # the above if will only run once. + else: + length += 1 + else: + length = 0 + line_num += 1 + return count + +def file_is_cpp(name): + if re.search('(.cpp|.cxx|.hpp)$', name) is not None: + return True + if re.search('(.c|.i|.in)$', name) is not None: + return False + + # House style is that C++ header files use ".h", which unfortunately makes + # this sort of code rather difficult. Luckily, libmagic can identify C/C++ + # based on content. Don't import it because Python packaging is a disaster + # and this script needs to run reliably. + result = subprocess.run("file {}".format(name), + shell=True, + capture_output=True, + text=True).stdout.strip('\n') + return "C++" in result + +# A collection of cases we handle. +tests = [ + (r'// hello world', True), + (r' // hello world', True), + (r'hello world', False), + (r'printf("Hello, World!\n"); // prints hello world', True), + (r'String url = "http://www.example.com"', False), + (r'// hello world', True), + (r'//\\', True), + (r'// "some comment"', True), + (r'new URI("http://www.google.com")', False), + (r'printf("Escaped quote\""); // Comment', True), + (r' * http://www.google.com', False) +] + +def validate(line, expect_match): + if (expect_match and not has_single_line_comment(line.strip())) \ + or (not expect_match and has_single_line_comment(line.strip())): + print("Test failed:" + line) + else: + print("Test success") + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-t', '--test', default=False, + help='Run self tests', action='store_true') + parser.add_argument('-f', '--fast', default=False, + help='Use fast mode by only checking files changed in git', + action='store_true') + args = parser.parse_args() + + if args.test: + for line, expected in tests: + validate(line, expected) + sys.exit(0) + + # Move up to root dir. + os.chdir("..") + + # Some files aren't expected to comply with WiredTiger style. Ignore them. + ignore_files = [ + 'src/support/mtx_rw.c', + ] + + command = "find bench examples ext src test -name \"*.[ch]\" -o -name \"*.in\" -o -name \ + \"*.cxx\" -o -name \"*.cpp\" -o -name \"*.i\" " + if args.fast: + command = "git diff --name-only $(git merge-base --fork-point develop) bench \ + examples ext src test | grep -E '(.c|.h|.cpp|.in|.cxx|.i)$'" + + result = subprocess.run(command, shell=True, capture_output=True, text=True).stdout.strip('\n') + count = 0 + for file_name in result.split('\n'): + if file_name in ignore_files: + continue + + if file_is_cpp(file_name): + count += check_cpp_comments(file_name) + else: + count += check_c_comments(file_name) + + if (count != 0): + print('Detected ' + str(count) +' comment format issues!') + diff --git a/src/third_party/wiredtiger/dist/s_all b/src/third_party/wiredtiger/dist/s_all index 7b6e291f62c..b259683f663 100755 --- a/src/third_party/wiredtiger/dist/s_all +++ b/src/third_party/wiredtiger/dist/s_all @@ -110,8 +110,9 @@ COMMANDS=" 2>&1 ./s_typedef -c > ${t_pfx}s_typedef_c 2>&1 ./s_void > ${t_pfx}s_void 2>&1 ./s_whitespace > ${t_pfx}s_whitespace -2>&1 python function.py > ${t_pfx}py_function -2>&1 python style.py > ${t_pfx}py_style" +2>&1 python3 function.py > ${t_pfx}py_function +2>&1 python3 style.py > ${t_pfx}py_style +2>&1 python3 comment_style.py ${fast} > ${t_pfx}py_comment_style" # Parallelize if possible. xp="" diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 7e7d807657b..4cbe71ac288 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -1072,6 +1072,7 @@ iteratively iters jjj jprx +js json kB kb diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_aws_manager.h b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_aws_manager.h index ed0c2cc7fc9..141be5ff381 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_aws_manager.h +++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_aws_manager.h @@ -31,12 +31,14 @@ #include <mutex> #include <aws/core/Aws.h> -// The AWS SDK must only be initialized once and must call initialization and shutdown in the -// correct order. The AwsManager handles multiple calls from the S3 extension initialization and -// uses a reference counter to check how many instances of the S3 extension are using the SDK. The -// first call to the extension will initiate the SDK while subsequent calls will increment the -// reference counter. Then each call to terminate will decrement the reference counter until it -// reaches 0 at which point SDK shutdown will be called. +/* + * The AWS SDK must only be initialized once and must call initialization and shutdown in the + * correct order. The AwsManager handles multiple calls from the S3 extension initialization and + * uses a reference counter to check how many instances of the S3 extension are using the SDK. The + * first call to the extension will initiate the SDK while subsequent calls will increment the + * reference counter. Then each call to terminate will decrement the reference counter until it + * reaches 0 at which point SDK shutdown will be called. + */ class AwsManager { public: static AwsManager & diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.cpp b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.cpp index 13674d73fb2..21dd3333c6c 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.cpp +++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.cpp @@ -66,10 +66,11 @@ S3Connection::S3Connection(const Aws::S3Crt::ClientConfiguration &config, throw std::invalid_argument(_bucketName + " : Unable to access bucket."); } -// Builds a list of object names, with prefix matching, from an S3 bucket into a vector. The -// batchSize parameter specifies the maximum number of objects returned in each AWS response, up -// to 1000. Return an errno value given an HTTP response code if the aws request does not -// succeed. +/* + * Builds a list of object names, with prefix matching, from an S3 bucket into a vector. The + * batchSize parameter specifies the maximum number of objects returned in each AWS response, up to + * 1000. Return an errno value given an HTTP response code if the aws request does not succeed. + */ int S3Connection::ListObjects(const std::string &prefix, std::vector<std::string> &objects, uint32_t batchSize, bool listSingle) const @@ -202,9 +203,11 @@ S3Connection::ObjectExists(const std::string &objectKey, bool &exists, size_t &o request.SetKey(_objectPrefix + objectKey); Aws::S3Crt::Model::HeadObjectOutcome outcome = _s3CrtClient.HeadObject(request); - // If an object with the given key does not exist the HEAD request will return a 404. - // Do not fail in this case as it is an expected response. Otherwise return an errno value - // for any other HTTP response code. + /* + * If an object with the given key does not exist the HEAD request will return a 404. Do not + * fail in this case as it is an expected response. Otherwise return an errno value for any + * other HTTP response code. + */ if (outcome.IsSuccess()) { exists = true; objectSize = outcome.GetResult().GetContentLength(); @@ -230,9 +233,11 @@ S3Connection::BucketExists(bool &exists) const request.WithBucket(_bucketName); Aws::S3Crt::Model::HeadBucketOutcome outcome = _s3CrtClient.HeadBucket(request); - // If an object with the given key does not exist the HEAD request will return a 404. - // Do not fail in this case as it is an expected response. Otherwise return an errno value - // for any other HTTP response code. + /* + * If an object with the given key does not exist the HEAD request will return a 404. Do not + * fail in this case as it is an expected response. Otherwise return an errno value for any + * other HTTP response code. + */ if (outcome.IsSuccess()) { exists = true; return (0); diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.h b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.h index 6da8c50da6e..4323a7dae9c 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.h +++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.h @@ -45,18 +45,21 @@ static const std::map<Aws::Http::HttpResponseCode, int32_t> toErrno = { {Aws::Http::HttpResponseCode::BAD_REQUEST, EINVAL}, {Aws::Http::HttpResponseCode::INTERNAL_SERVER_ERROR, EAGAIN}}; -// This class represents an active connection to the AWS S3 endpoint and allows for interaction with -// S3-Crt client. The S3Connection exposes an API to list the bucket contents filtered by a -// directory and a prefix, check for an object's existence in the bucket, put an object to the -// cloud, and get the object from the cloud. Though not required for the file system's -// implementation, the class also provides the means to delete the objects to clean up artifacts -// from the internal unit testing. Note we are using S3-Crt client in this class, which differs to -// the S3 client. +/* + * This class represents an active connection to the AWS S3 endpoint and allows for interaction with + * S3-Crt client. The S3Connection exposes an API to list the bucket contents filtered by a + * directory and a prefix, check for an object's existence in the bucket, put an object to the + * cloud, and get the object from the cloud. Though not required for the file system's + * implementation, the class also provides the means to delete the objects to clean up artifacts + * from the internal unit testing. Note we are using S3-Crt client in this class, which differs to + * the S3 client. + */ class S3Connection { public: - // We have two constructors for the two different ways to start a S3 connection. - // First constructor uses provided credentials, the following uses credentials stored in a local - // file. + /* + * We have two constructors for the two different ways to start a S3 connection. First + * constructor uses provided credentials, the following uses credentials stored in a local file. + */ S3Connection(const Aws::Auth::AWSCredentials &credentials, const Aws::S3Crt::ClientConfiguration &config, const std::string &bucketName, const std::string &objPrefix = ""); diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.h b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.h index 3e6b2c42e21..44ddc3e1d89 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.h +++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.h @@ -49,10 +49,12 @@ static const std::map<int32_t, Aws::Utils::Logging::LogLevel> verbosityMapping = {WT_VERBOSE_DEBUG_4, Aws::Utils::Logging::LogLevel::Debug}, {WT_VERBOSE_DEBUG_5, Aws::Utils::Logging::LogLevel::Trace}}; -// Provides the S3 Store with a logger implementation that redirects the generated logs to -// WiredTiger's logging streams. This class implements AWS's LogSystemInterface class, an interface -// for logging implementations. Functions are derived from the interface to incorporate the -// logging with WiredTiger's logging system. +/* + * Provides the S3 Store with a logger implementation that redirects the generated logs to + * WiredTiger's logging streams. This class implements AWS's LogSystemInterface class, an interface + * for logging implementations. Functions are derived from the interface to incorporate the logging + * with WiredTiger's logging system. + */ class S3LogSystem : public Aws::Utils::Logging::LogSystemInterface { public: S3LogSystem(WT_EXTENSION_API *wtApi, uint32_t wtVerbosityLevel); diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp index 4010c5bcf99..14a09f6b63f 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp +++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp @@ -82,9 +82,11 @@ struct S3FileSystem { // Must come first - this is the interface for the file system we are implementing. WT_FILE_SYSTEM fileSystem; S3Storage *storage; - // The S3_FILE_SYSTEM is built on top of the WT_FILE_SYSTEM. We require an instance of the - // WT_FILE_SYSTEM in order to access the native WiredTiger filesystem functionality, such as the - // native WT file handle open. + /* + * The S3_FILE_SYSTEM is built on top of the WT_FILE_SYSTEM. We require an instance of the + * WT_FILE_SYSTEM in order to access the native WiredTiger filesystem functionality, such as the + * native WT file handle open. + */ WT_FILE_SYSTEM *wtFileSystem; S3Connection *connection; std::string cacheDir; // Directory for cached objects @@ -95,9 +97,11 @@ struct S3FileHandle { WT_FILE_HANDLE iface; // Must come first S3Storage *storage; // Enclosing storage source - // Similarly, The S3FileHandle is built on top of the WT_FILE_HANDLE. We require an instance of - // the WT_FILE_HANDLE in order to access the native WiredTiger filehandle functionality, such as - // the native WT file handle read and close. + /* + * Similarly, The S3FileHandle is built on top of the WT_FILE_HANDLE. We require an instance of + * the WT_FILE_HANDLE in order to access the native WiredTiger filehandle functionality, such as + * the native WT file handle read and close. + */ WT_FILE_HANDLE *wtFileHandle; }; @@ -527,9 +531,11 @@ S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, con awsConfig.region = region; awsConfig.throughputTargetGbps = throughputTargetGbps; - // Get the directory to setup the cache, or use the default one. The default cache directory is - // named "cache-<name>", where name is the last component of the bucket name's path. We'll - // create it if it doesn't exist. + /* + * Get the directory to setup the cache, or use the default one. The default cache directory is + * named "cache-<name>", where name is the last component of the bucket name's path. We'll + * create it if it doesn't exist. + */ WT_CONFIG_ITEM cacheDirConf; std::string cacheDir; std::string cacheStr; @@ -736,17 +742,21 @@ S3Terminate(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session) if (--s3->referenceCount != 0) return (0); - // It is currently unclear at the moment what the multi-threading will look like in the - // extension. The current implementation is NOT thread-safe, and needs to be addressed in the - // future, as multiple threads could call terminate leading to a race condition. + /* + * It is currently unclear at the moment what the multi-threading will look like in the + * extension. The current implementation is NOT thread-safe, and needs to be addressed in the + * future, as multiple threads could call terminate leading to a race condition. + */ while (!s3->fhList.empty()) { S3FileHandle *fs = s3->fhList.front(); S3FileClose((WT_FILE_HANDLE *)fs, session); } - // Terminate any active filesystems. There are no references to the storage source, so it is - // safe to walk the active filesystem list without a lock. The removal from the list happens - // under a lock. Also, removal happens from the front and addition at the end, so we are safe. + /* + * Terminate any active filesystems. There are no references to the storage source, so it is + * safe to walk the active filesystem list without a lock. The removal from the list happens + * under a lock. Also, removal happens from the front and addition at the end, so we are safe. + */ while (!s3->fsList.empty()) { S3FileSystem *fs = s3->fsList.front(); S3FileSystemTerminate(&fs->fileSystem, session); diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/test/test_s3_connection.cpp b/src/third_party/wiredtiger/ext/storage_sources/s3_store/test/test_s3_connection.cpp index af9a815e3b7..423a56d4f1d 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/s3_store/test/test_s3_connection.cpp +++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/test/test_s3_connection.cpp @@ -41,9 +41,11 @@ const double throughputTargetGbps = 5; const uint64_t partSize = 8 * 1024 * 1024; // 8 MB. static std::string bucketName("s3testext"); // Can be overridden with environment variables. -// Objects with the prefex pattern "s3test/*" are deleted after a certain period of time according -// to the lifecycle rule on the S3 bucket. Should you wish to make any changes to the prefix pattern -// or lifecycle of the object, please speak to the release manager. +/* + * Objects with the prefex pattern "s3test/*" are deleted after a certain period of time according + * to the lifecycle rule on the S3 bucket. Should you wish to make any changes to the prefix pattern + * or lifecycle of the object, please speak to the release manager. + */ static std::string objPrefix("s3test/unit/"); // To be concatenated with a random string. } // namespace TestDefaults diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index b2a38aef17d..5d18f8728a1 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "8f0a6e229b438ba4fe660beaddb9f6a52725855b" + "commit": "f71af27c5a39b1490a5f78029d205280cfd03174" } diff --git a/src/third_party/wiredtiger/src/config/config.c b/src/third_party/wiredtiger/src/config/config.c index cd7c60cb356..1a5f48a32b8 100644 --- a/src/third_party/wiredtiger/src/config/config.c +++ b/src/third_party/wiredtiger/src/config/config.c @@ -99,6 +99,9 @@ typedef enum { } CONFIG_ACTION; /* + * Some inspiration taken from https://github.com/quartzjer/js0n (public domain). These + * comments map from constructs in js0n to the equivalent in WiredTiger. + * * static void *gostruct[] = { * [0 ... 255] = &&l_bad, * ['\t'] = &&l_loop, [' '] = &&l_loop, @@ -106,12 +109,9 @@ typedef enum { * ['"'] = &&l_qup, * [':'] = &&l_value, ['='] = &&l_value, * [','] = &&l_next, - * // tracking [] and {} individually would allow fuller - * // validation but is really messy * ['('] = &&l_up, [')'] = &&l_down, * ['['] = &&l_up, [']'] = &&l_down, * ['{'] = &&l_up, ['}'] = &&l_down, - * // bare identifiers * ['-'] = &&l_numbare, * ['0' ... '9'] = &&l_numbare, * ['_'] = &&l_bare, @@ -144,7 +144,6 @@ static const int8_t gostruct[256] = {A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A_BAD, A * static void *gobare[] = * { * [0 ... 31] = &&l_bad, - * // could be more pedantic/validation-checking * [32 ... 126] = &&l_loop, * ['\t'] = &&l_unbare, [' '] = &&l_unbare, * ['\r'] = &&l_unbare, ['\n'] = &&l_unbare, diff --git a/src/third_party/wiredtiger/src/include/log.h b/src/third_party/wiredtiger/src/include/log.h index 5a4ee2599e1..9fcccd2e89c 100644 --- a/src/third_party/wiredtiger/src/include/log.h +++ b/src/third_party/wiredtiger/src/include/log.h @@ -366,9 +366,10 @@ struct __wt_log_desc { /* * WiredTiger release version where log format version changed. + * + * FIXME WT-8681 - According to WT_MIN_STARTUP_VERSION any WT version less then 3.2.0 will not + * start. Can we drop V2, V3 here? */ -// FIXME WT-8681 - According to WT_MIN_STARTUP_VERSION any WT version less then 3.2.0 will not -// start. Can we drop V2, V3 here? #define WT_LOG_V2_VERSION ((WT_VERSION){3, 0, 0}) #define WT_LOG_V3_VERSION ((WT_VERSION){3, 1, 0}) #define WT_LOG_V4_VERSION ((WT_VERSION){3, 3, 0}) diff --git a/src/third_party/wiredtiger/src/os_win/os_dlopen.c b/src/third_party/wiredtiger/src/os_win/os_dlopen.c index 73ac06b56f4..ebed3ed1ab7 100644 --- a/src/third_party/wiredtiger/src/os_win/os_dlopen.c +++ b/src/third_party/wiredtiger/src/os_win/os_dlopen.c @@ -33,7 +33,7 @@ __wt_dlopen(WT_SESSION_IMPL *session, const char *path, WT_DLH **dlhp) WT_ERR(ret); } } else { - // TODO: load dll here + /* TODO: load dll here */ DebugBreak(); } diff --git a/src/third_party/wiredtiger/src/reconcile/rec_child.c b/src/third_party/wiredtiger/src/reconcile/rec_child.c index 87fcd4ff1ea..5214ac9a848 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_child.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_child.c @@ -198,7 +198,6 @@ __wt_rec_child_modify( switch (r->tested_ref_state = ref->state) { case WT_REF_DISK: /* On disk, not modified by definition. */ - // 9417 IGNORE WT_ASSERT(session, ref->addr != NULL); /* DISK pages do not have fast-truncate info. */ WT_ASSERT(session, ref->page_del == NULL); diff --git a/src/third_party/wiredtiger/src/reconcile/rec_col.c b/src/third_party/wiredtiger/src/reconcile/rec_col.c index 9d6fe320ed8..07ed2fb22c4 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_col.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_col.c @@ -8,8 +8,6 @@ #include "wt_internal.h" -// 9417 IGNORE asserts in this file. Column store is not related to BF-25011 - /* * __rec_col_fix_bulk_insert_split_check -- * Check if a bulk-loaded fixed-length column store page needs to split. diff --git a/src/third_party/wiredtiger/src/support/hash_city.c b/src/third_party/wiredtiger/src/support/hash_city.c index 01462cececa..3cf09ca51a7 100644 --- a/src/third_party/wiredtiger/src/support/hash_city.c +++ b/src/third_party/wiredtiger/src/support/hash_city.c @@ -94,7 +94,7 @@ static uint32_t UNALIGNED_LOAD32(const char *p) { #elif defined(__APPLE__) -// Mac OS X / Darwin features +/* Mac OS X / Darwin features */ #include <libkern/OSByteOrder.h> #define bswap_32(x) OSSwapInt32(x) #define bswap_64(x) OSSwapInt64(x) diff --git a/src/third_party/wiredtiger/test/csuite/rwlock/main.c b/src/third_party/wiredtiger/test/csuite/rwlock/main.c index 43e4b308ccc..7dabeff7ddc 100644 --- a/src/third_party/wiredtiger/test/csuite/rwlock/main.c +++ b/src/third_party/wiredtiger/test/csuite/rwlock/main.c @@ -32,11 +32,8 @@ */ #define MAX_THREADS 1000 #define READS_PER_WRITE 10000 -//#define READS_PER_WRITE 1000000 -//#define READS_PER_WRITE 100 #define CHECK_CORRECTNESS 1 -//#define USE_POSIX 1 static WT_RWLOCK rwlock; static pthread_rwlock_t p_rwlock; diff --git a/src/third_party/wiredtiger/test/csuite/wt2999_join_extractor/main.c b/src/third_party/wiredtiger/test/csuite/wt2999_join_extractor/main.c index ab10d11111f..0de3dda9fbc 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2999_join_extractor/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2999_join_extractor/main.c @@ -156,7 +156,7 @@ main(int argc, char *argv[]) testutil_check(session->join(session, jcursor, cursor1, "compare=gt")); testutil_check(session->join(session, jcursor, cursor2, "compare=gt")); - while ((ret = jcursor->next(jcursor)) == 0) // leak + while ((ret = jcursor->next(jcursor)) == 0) /* leak */ ; testutil_assert(ret == WT_NOTFOUND); diff --git a/src/third_party/wiredtiger/test/syscall/wt2336_base/main.c b/src/third_party/wiredtiger/test/syscall/wt2336_base/main.c index 56d75dc835e..31fa857eb62 100644 --- a/src/third_party/wiredtiger/test/syscall/wt2336_base/main.c +++ b/src/third_party/wiredtiger/test/syscall/wt2336_base/main.c @@ -28,8 +28,8 @@ #include <stdlib.h> -#include <unistd.h> // TODO -#include <fcntl.h> // TODO +#include <unistd.h> +#include <fcntl.h> #include <wt_internal.h> static void fail(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); diff --git a/src/third_party/wiredtiger/test/unittest/tests/block/test_block_ckpt.cpp b/src/third_party/wiredtiger/test/unittest/tests/block/test_block_ckpt.cpp index 81b6260cb1d..8057629b032 100644 --- a/src/third_party/wiredtiger/test/unittest/tests/block/test_block_ckpt.cpp +++ b/src/third_party/wiredtiger/test/unittest/tests/block/test_block_ckpt.cpp @@ -64,8 +64,10 @@ TEST_CASE("Block manager: __ckpt_add_blkmod_entry", "[block]") // Edge case, this should just fit in 256 bits test_ckpt_add_blkmod_entry(255, 1, 256); - // This case relies on the "+ 1" introduced in WT-6366 in __ckpt_add_blkmod_entry. - // Without it, this test would fail as it would only allocate 256 bits. - // We expect an extra 8 bytes (64 bits to be added), 256 + 64 = 320; + /* + * This case relies on the "+ 1" introduced in WT-6366 in __ckpt_add_blkmod_entry. Without it, + * this test would fail as it would only allocate 256 bits. We expect an extra 8 bytes (64 bits + * to be added), 256 + 64 = 320; + */ test_ckpt_add_blkmod_entry(256, 1, 320); } diff --git a/src/third_party/wiredtiger/test/unittest/tests/test_extent_list.cpp b/src/third_party/wiredtiger/test/unittest/tests/test_extent_list.cpp index bddd8b3edd5..934396ae85c 100644 --- a/src/third_party/wiredtiger/test/unittest/tests/test_extent_list.cpp +++ b/src/third_party/wiredtiger/test/unittest/tests/test_extent_list.cpp @@ -312,9 +312,10 @@ TEST_CASE("Extent Lists: block_off_srch", "[extent_list]") __ut_block_off_srch(&head[0], 2, &stack[0], true); - // For each level of the extent list, if the searched-for element was - // visible, we should point to it. otherwise, we should point to the - // next-largest item. + /* + * For each level of the extent list, if the searched-for element was visible, we should + * point to it. otherwise, we should point to the next-largest item. + */ REQUIRE((*stack[0])->off == 2); REQUIRE((*stack[1])->off == 2); REQUIRE((*stack[2])->off == 3); diff --git a/src/third_party/wiredtiger/test/utility/misc.c b/src/third_party/wiredtiger/test/utility/misc.c index c43da94e7f4..98b788834cc 100644 --- a/src/third_party/wiredtiger/test/utility/misc.c +++ b/src/third_party/wiredtiger/test/utility/misc.c @@ -415,7 +415,6 @@ testutil_wiredtiger_open(TEST_OPTS *opts, const char *home, const char *config, testutil_check(__wt_snprintf(buf, sizeof(buf), "%s%s%s%s", config, (rerun ? TESTUTIL_ENV_CONFIG_REC : ""), (opts->compat ? TESTUTIL_ENV_CONFIG_COMPAT : ""), (opts->tiered_storage ? tiered_ext_cfg : ""))); - // printf("wiredtiger_open configuration: %s\n", buf); testutil_check(wiredtiger_open(home, event_handler, buf, connectionp)); } |