diff options
| author | Chenhao Qu <chenhao.qu@mongodb.com> | 2022-04-13 01:46:12 +0000 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-04-13 02:12:41 +0000 |
| commit | b3cded481b3b70c766338a729014e64bf71a97e6 (patch) | |
| tree | 5bcd0f836bfc241d3061ad26e1c46707b47d1c62 /src | |
| parent | 583df21242096af2467a4b94ff9cb2a86d55bbaf (diff) | |
| download | mongo-b3cded481b3b70c766338a729014e64bf71a97e6.tar.gz | |
Import wiredtiger: 7a7d6bf9ab40cd5635ee960fac1b21edf118a007 from branch mongodb-master
ref: 07cee370d8..7a7d6bf9ab
for: 6.0.0-rc0
WT-8998 S3 extension - Uniform coding standards, error handling and logging
Diffstat (limited to 'src')
8 files changed, 595 insertions, 552 deletions
diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 8e94b159ae8..da396a97162 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -17,6 +17,7 @@ ASAN ASM AUS AWS +AWS's Addr Ailamaki Alakuijala @@ -99,6 +100,7 @@ Coverity CreateFileMapping CreateFileMappingW CreateFileW +Crt Crummey CustomersPhone DAX @@ -268,6 +270,7 @@ LibFuzzer LmRrSVv LoadLoad LockFile +LogSystemInterface Lookaside Lookup MADV @@ -392,6 +395,7 @@ RocksDB Rogerio Runtime SDK +SDK's SDKOptions SIMD SLIST diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.cpp b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.cpp index a9983ea5f94..1ee103a0690 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.cpp +++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.cpp @@ -1,4 +1,32 @@ -#include <aws/core/Aws.h> +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "s3_connection.h" + #include <aws/s3-crt/model/DeleteObjectRequest.h> #include <aws/s3-crt/model/ListObjectsV2Request.h> #include <aws/s3-crt/model/PutObjectRequest.h> @@ -6,52 +34,41 @@ #include <aws/s3-crt/model/HeadObjectRequest.h> #include <aws/s3-crt/model/HeadBucketRequest.h> -#include "s3_connection.h" - #include <fstream> #include <iostream> -#include <string> -#include <vector> -#define S3_ALLOCATION_TAG "" -/* - * S3Connection -- - * Constructor for AWS S3 bucket connection with provided credentials. - */ +// Constructor for AWS S3 bucket connection with provided credentials. S3Connection::S3Connection(const Aws::Auth::AWSCredentials &credentials, const Aws::S3Crt::ClientConfiguration &config, const std::string &bucketName, const std::string &objPrefix) : _s3CrtClient(credentials, config), _bucketName(bucketName), _objectPrefix(objPrefix) { - /* Confirm that we can access the bucket, else fail. */ + // Confirm that we can access the bucket, else fail. bool exists; int ret = BucketExists(exists); - if (ret != 0 || !exists) + if (!exists) throw std::invalid_argument(_bucketName + " : No such bucket."); + if (ret != 0) + throw std::invalid_argument(_bucketName + " :Unable to access bucket."); } -/* - * S3Connection -- - * Constructor for AWS S3 bucket connection with credentials in local file. - * https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html - */ +// Constructor for AWS S3 bucket connection with credentials in local file. S3Connection::S3Connection(const Aws::S3Crt::ClientConfiguration &config, const std::string &bucketName, const std::string &objPrefix) : _s3CrtClient(config), _bucketName(bucketName), _objectPrefix(objPrefix) { - /* Confirm that we can access the bucket, else fail. */ + // Confirm that we can access the bucket, else fail. bool exists; int ret = BucketExists(exists); - if (ret != 0 || !exists) + if (!exists) throw std::invalid_argument(_bucketName + " : No such bucket."); + if (ret != 0) + throw std::invalid_argument(_bucketName + " : Unable to access bucket."); } -/* - * ListObjects -- - * Builds a list of object names, with prefix matching, from an S3 bucket into a vector. The - * batchSize parameter specifies the maximum number of objects returned in each AWS response, up - * to 1000. Returns 0 if success, otherwise 1. - */ +// Builds a list of object names, with prefix matching, from an S3 bucket into a vector. The +// batchSize parameter specifies the maximum number of objects returned in each AWS response, up +// to 1000. Returns 0 if success, otherwise 1. int S3Connection::ListObjects(const std::string &prefix, std::vector<std::string> &objects, uint32_t batchSize, bool listSingle) const @@ -66,14 +83,14 @@ S3Connection::ListObjects(const std::string &prefix, std::vector<std::string> &o return (1); auto result = outcomes.GetResult(); - /* Returning the object name with the prefix stripped. */ + // Returning the object name with the prefix stripped. for (const auto &object : result.GetContents()) objects.push_back(object.GetKey().substr(_objectPrefix.length())); if (listSingle) return (0); - /* Continuation token will be an empty string if we have returned all possible objects. */ + // Continuation token will be an empty string if we have returned all possible objects. std::string continuationToken = result.GetNextContinuationToken(); while (continuationToken != "") { request.SetContinuationToken(continuationToken); @@ -88,15 +105,12 @@ S3Connection::ListObjects(const std::string &prefix, std::vector<std::string> &o return (0); } -/* - * PutObject -- - * Puts an object into an S3 bucket. Returns 0 if success, otherwise 1. - */ +// Puts an object into an S3 bucket. Returns 0 if success, otherwise 1. int S3Connection::PutObject(const std::string &objectKey, const std::string &fileName) const { std::shared_ptr<Aws::IOStream> inputData = Aws::MakeShared<Aws::FStream>( - "s3-source", fileName.c_str(), std::ios_base::in | std::ios_base::binary); + s3AllocationTag, fileName.c_str(), std::ios_base::in | std::ios_base::binary); Aws::S3Crt::Model::PutObjectRequest request; request.SetBucket(_bucketName); @@ -111,10 +125,7 @@ S3Connection::PutObject(const std::string &objectKey, const std::string &fileNam return (1); } -/* - * DeleteObject -- - * Deletes an object from S3 bucket. Returns 0 if success, otherwise 1. - */ +// Deletes an object from S3 bucket. Returns 0 if success, otherwise 1. int S3Connection::DeleteObject(const std::string &objectKey) const { @@ -129,24 +140,19 @@ S3Connection::DeleteObject(const std::string &objectKey) const return (1); } -/* - * GetObject -- - * Retrieves an object from S3. The object is downloaded to disk at the specified location. - */ +// Retrieves an object from S3. The object is downloaded to disk at the specified location. int S3Connection::GetObject(const std::string &objectKey, const std::string &path) const { Aws::S3Crt::Model::GetObjectRequest request; request.SetBucket(_bucketName); request.SetKey(_objectPrefix + objectKey); - /* - * The S3 Object should be downloaded to disk rather than into an in-memory buffer. Use a custom - * response stream factory to specify how the response should be downloaded. - * https://sdk.amazonaws.com/cpp/api/0.14.3/class_aws_1_1_utils_1_1_stream_1_1_response_stream.html - */ + + // The S3 Object should be downloaded to disk rather than into an in-memory buffer. Use a custom + // response stream factory to specify how the response should be downloaded. request.SetResponseStreamFactory([=]() { return (Aws::New<Aws::FStream>( - S3_ALLOCATION_TAG, path, std::ios_base::out | std::ios_base::binary)); + s3AllocationTag, path, std::ios_base::out | std::ios_base::binary)); }); if (!_s3CrtClient.GetObject(request).IsSuccess()) @@ -155,11 +161,8 @@ S3Connection::GetObject(const std::string &objectKey, const std::string &path) c return (0); } -/* - * ObjectExists -- - * Checks whether an object with the given key exists in the S3 bucket and also retrieves - * size of the object. - */ +// Checks whether an object with the given key exists in the S3 bucket and also retrieves +// size of the object. int S3Connection::ObjectExists(const std::string &objectKey, bool &exists, size_t &objectSize) const { @@ -171,10 +174,8 @@ S3Connection::ObjectExists(const std::string &objectKey, bool &exists, size_t &o request.SetKey(_objectPrefix + objectKey); Aws::S3Crt::Model::HeadObjectOutcome outcome = _s3CrtClient.HeadObject(request); - /* - * If an object with the given key does not exist the HEAD request will return a 404. - * https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html Do not fail in this case. - */ + // If an object with the given key does not exist the HEAD request will return a 404. + // Do not fail in this case. if (outcome.IsSuccess()) { exists = true; objectSize = outcome.GetResult().GetContentLength(); @@ -182,17 +183,12 @@ S3Connection::ObjectExists(const std::string &objectKey, bool &exists, size_t &o } else if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) return (0); - /* - * Fix later, return a proper error code. Not sure if we always have - * outcome.GetError().GetResponseCode() - */ + // Fix later, return a proper error code. Not sure if we always have + // outcome.GetError().GetResponseCode() return (1); } -/* - * BucketExists -- - * Checks whether the bucket configured for the class is accessible to us or not. - */ +// Checks whether the bucket configured for the class is accessible to us or not. int S3Connection::BucketExists(bool &exists) const { @@ -202,19 +198,15 @@ S3Connection::BucketExists(bool &exists) const request.WithBucket(_bucketName); Aws::S3Crt::Model::HeadBucketOutcome outcome = _s3CrtClient.HeadBucket(request); - /* - * If an object with the given key does not exist the HEAD request will return a 404. - * https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadBucket.html Do not fail in this case. - */ + // If an object with the given key does not exist the HEAD request will return a 404. + // Do not fail in this case. if (outcome.IsSuccess()) { exists = true; return (0); } else if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) return (0); - /* - * Fix later, return a proper error code. Not sure if we always have - * outcome.GetError().GetResponseCode() - */ + // Fix later, return a proper error code. Not sure if we always have + // outcome.GetError().GetResponseCode() return (1); } diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.h b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.h index 51085988225..314173f9928 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.h +++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.h @@ -1,4 +1,31 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ #ifndef S3CONNECTION #define S3CONNECTION @@ -10,12 +37,18 @@ #include <string> #include <vector> -/* - * Class to represent an active connection to the AWS S3 endpoint. Allows for interaction with S3 - * client. - */ +// This class represents an active connection to the AWS S3 endpoint and allows for interaction with +// S3-Crt client. The S3Connection exposes an API to list the bucket contents filtered by a +// directory and a prefix, check for an object's existence in the bucket, put an object to the +// cloud, and get the object from the cloud. Though not required for the file system's +// implementation, the class also provides the means to delete the objects to clean up artifacts +// from the internal unit testing. Note we are using S3-Crt client in this class, which differs to +// the S3 client. class S3Connection { public: + // We have two constructors for the two different ways to start a S3 connection. + // First constructor uses provided credentials, the following uses credentials stored in a local + // file. S3Connection(const Aws::Auth::AWSCredentials &credentials, const Aws::S3Crt::ClientConfiguration &config, const std::string &bucketName, const std::string &objPrefix = ""); @@ -35,6 +68,10 @@ class S3Connection { const std::string _bucketName; const std::string _objectPrefix; + // Tag that can be set and used when uploading or retrieving objects from the S3. + // Tagging in S3 allows for categorization of objects, as well as other benefits. + static inline const char *const s3AllocationTag = "s3-source"; + int BucketExists(bool &exists) const; }; #endif diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.cpp b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.cpp index 2dceece391b..0445b8f3880 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.cpp +++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.cpp @@ -1,12 +1,42 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ #include <aws/core/Aws.h> #include "s3_log_system.h" #include <cstdarg> +// Constructor for S3LogSystem that calls to set the WiredTiger verbosity level. S3LogSystem::S3LogSystem(WT_EXTENSION_API *wtApi, uint32_t wtVerbosityLevel) : _wtApi(wtApi) { SetWtVerbosityLevel(wtVerbosityLevel); } +// Overrides the interface's Log method to write the AWS SDK log stream to WiredTiger's log stream +// using variadic style through a helper function. Inherited from AWS's LogSystemInterface. void S3LogSystem::Log(Aws::Utils::Logging::LogLevel logLevel, const char *tag, const char *format, ...) { @@ -33,6 +63,8 @@ S3LogSystem::Log(Aws::Utils::Logging::LogLevel logLevel, const char *tag, const va_end(args); } +// Overrides the interface's LogStream method to write the AWS SDK log stream to WiredTiger's log +// stream through a helper function. Inherited from AWS's LogSystemInterface. void S3LogSystem::LogStream( Aws::Utils::Logging::LogLevel logLevel, const char *tag, const Aws::OStringStream &messageStream) @@ -40,50 +72,35 @@ S3LogSystem::LogStream( LogAwsMessage(tag, messageStream.rdbuf()->str().c_str()); } +// Directs the message to WiredTiger's log streams. void S3LogSystem::LogAwsMessage(const char *tag, const std::string &message) const { _wtApi->err_printf(_wtApi, NULL, "%s : %s", tag, message.c_str()); } +// Directs the message to WiredTiger's log streams matched at WiredTiger's log stream levels. void S3LogSystem::LogVerboseMessage(int32_t verbosityLevel, const std::string &message) const { if (verbosityLevel <= _wtVerbosityLevel) { - /* Use err_printf for error and warning messages and use msg_printf for notice, info and - * debug messages. */ - if (verbosityLevel < -1) + // Use err_printf for error and warning messages and use msg_printf for notice, info and + // debug messages. + if (verbosityLevel < WT_VERBOSE_NOTICE) _wtApi->err_printf(_wtApi, NULL, "%s", message.c_str()); else _wtApi->msg_printf(_wtApi, NULL, "%s", message.c_str()); } } -void -S3LogSystem::LogErrorMessage(const std::string &message) const -{ - LogVerboseMessage(WT_VERBOSE_ERROR, message); -} - -void -S3LogSystem::LogDebugMessage(const std::string &message) const -{ - LogVerboseMessage(WT_VERBOSE_DEBUG, message); -} - +// Sets the WiredTiger verbosity level by mapping the AWS SDK log level. void S3LogSystem::SetWtVerbosityLevel(int32_t wtVerbosityLevel) { _wtVerbosityLevel = wtVerbosityLevel; - /* If the verbosity level is out of range it will default to AWS SDK Error level. */ + // If the verbosity level is out of range it will default to AWS SDK Error level. if (verbosityMapping.find(_wtVerbosityLevel) != verbosityMapping.end()) _awsLogLevel = verbosityMapping.at(_wtVerbosityLevel); else _awsLogLevel = Aws::Utils::Logging::LogLevel::Error; } - -void -S3LogSystem::Flush() -{ - return; -} diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.h b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.h index 06202dbe84f..e6e8ddf2d5b 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.h +++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.h @@ -1,3 +1,33 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef S3LOGSYSTEM +#define S3LOGSYSTEM + #include <wiredtiger.h> #include <wiredtiger_ext.h> @@ -13,8 +43,11 @@ static const std::map<int32_t, Aws::Utils::Logging::LogLevel> verbosityMapping = {-1, Aws::Utils::Logging::LogLevel::Info}, {0, Aws::Utils::Logging::LogLevel::Info}, {1, Aws::Utils::Logging::LogLevel::Debug}}; +// Provides the S3 Store with a logger implementation that redirects the generated logs to +// WiredTiger's logging streams. This class implements AWS's LogSystemInterface class, an interface +// for logging implementations. Functions are derived from the interface to incorporate the +// logging with WiredTiger's logging system. class S3LogSystem : public Aws::Utils::Logging::LogSystemInterface { - public: S3LogSystem(WT_EXTENSION_API *wtApi, uint32_t wtVerbosityLevel); Aws::Utils::Logging::LogLevel @@ -26,10 +59,30 @@ class S3LogSystem : public Aws::Utils::Logging::LogSystemInterface { Aws::Utils::Logging::LogLevel logLevel, const char *tag, const char *format, ...) override; void LogStream(Aws::Utils::Logging::LogLevel logLevel, const char *tag, const Aws::OStringStream &messageStream) override; - void LogErrorMessage(const std::string &message) const; - void LogDebugMessage(const std::string &message) const; + + // Sends error messages to WiredTiger's error level log stream. + void + LogErrorMessage(const std::string &message) const + { + LogVerboseMessage(WT_VERBOSE_ERROR, message); + } + + // Sends error messages to WiredTiger's debug level log stream. + void + LogDebugMessage(const std::string &message) const + { + LogVerboseMessage(WT_VERBOSE_DEBUG, message); + } + + // Sets the WiredTiger Extension's verbosity level and matches the AWS log levels + // to this. void SetWtVerbosityLevel(int32_t wtVerbosityLevel); - void Flush() override; + + // Inherited from AWS LogSystemInterface and is not used. + void + Flush() + { + } private: void LogAwsMessage(const char *tag, const std::string &message) const; @@ -38,3 +91,4 @@ class S3LogSystem : public Aws::Utils::Logging::LogSystemInterface { WT_EXTENSION_API *_wtApi; int32_t _wtVerbosityLevel; }; +#endif diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp index 0ad2517ac62..d2547110fc4 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp +++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp @@ -25,104 +25,100 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ + #include <wiredtiger.h> #include <wiredtiger_ext.h> -#include <sys/stat.h> #include <fstream> #include <list> #include <errno.h> -#include <unistd.h> +#include <filesystem> #include "s3_connection.h" #include "s3_log_system.h" #include <aws/auth/credentials.h> #include <aws/core/Aws.h> -#include <aws/core/auth/AWSCredentialsProvider.h> -#include <aws/core/utils/memory/stl/AWSString.h> #include <aws/core/utils/logging/AWSLogging.h> #define UNUSED(x) (void)(x) -#define FS2S3(fs) (((S3_FILE_SYSTEM *)(fs))->storage) - -struct S3_FILE_HANDLE; -struct S3_FILE_SYSTEM; - -/* Statistics to be collected for the S3 storage. */ -struct S3_STATISTICS { - /* Operations using AWS SDK. */ - uint64_t listObjectsCount; /* Number of S3 list objects requests */ - uint64_t putObjectCount; /* Number of S3 put object requests */ - uint64_t getObjectCount; /* Number of S3 get object requests */ - uint64_t objectExistsCount; /* Number of S3 object exists requests */ - - /* Operations using WiredTiger's native file handle operations. */ - uint64_t fhOps; /* Number of non read/write file handle operations */ - uint64_t fhReadOps; /* Number of file handle read operations */ +#define FS2S3(fs) (((S3FileSystem *)(fs))->storage) + +struct S3FileHandle; +struct S3FileSystem; + +// Statistics to be collected for the S3 storage. +struct S3Statistics { + // Operations using AWS SDK. + uint64_t listObjectsCount; // Number of S3 list objects requests + uint64_t putObjectCount; // Number of S3 put object requests + uint64_t getObjectCount; // Number of S3 get object requests + uint64_t objectExistsCount; // Number of S3 object exists requests + + // Operations using WiredTiger's native file handle operations. + uint64_t fhOps; // Number of non read/write file handle operations + uint64_t fhReadOps; // Number of file handle read operations }; -/* S3 storage source structure. */ -struct S3_STORAGE { - WT_STORAGE_SOURCE storageSource; /* Must come first */ - WT_EXTENSION_API *wtApi; /* Extension API */ +// S3 storage source structure. +struct S3Storage { + WT_STORAGE_SOURCE storageSource; // Must come first + WT_EXTENSION_API *wtApi; // Extension API std::shared_ptr<S3LogSystem> log; - std::mutex fsListMutex; /* Protect the file system list */ - std::list<S3_FILE_SYSTEM *> fsList; /* List of initiated file systems */ - std::mutex fhMutex; /* Protect the file handle list*/ - std::list<S3_FILE_HANDLE *> fhList; /* List of open file handles */ + std::mutex fsListMutex; // Protect the file system list + std::list<S3FileSystem *> fsList; // List of initiated file systems + std::mutex fhMutex; // Protect the file handle list + std::list<S3FileHandle *> fhList; // List of open file handles - uint32_t referenceCount; /* Number of references to this storage source */ + uint32_t referenceCount; // Number of references to this storage source int32_t verbose; - S3_STATISTICS statistics; + S3Statistics statistics; }; -struct S3_FILE_SYSTEM { - /* Must come first - this is the interface for the file system we are implementing. */ +struct S3FileSystem { + // Must come first - this is the interface for the file system we are implementing. WT_FILE_SYSTEM fileSystem; - S3_STORAGE *storage; - /* - * The S3_FILE_SYSTEM is built on top of the WT_FILE_SYSTEM. We require an instance of the - * WT_FILE_SYSTEM in order to access the native WiredTiger filesystem functionality, such as the - * native WT file handle open. - */ + S3Storage *storage; + // The S3_FILE_SYSTEM is built on top of the WT_FILE_SYSTEM. We require an instance of the + // WT_FILE_SYSTEM in order to access the native WiredTiger filesystem functionality, such as the + // native WT file handle open. WT_FILE_SYSTEM *wtFileSystem; S3Connection *connection; - std::string cacheDir; /* Directory for cached objects */ - std::string homeDir; /* Owned by the connection */ + std::string cacheDir; // Directory for cached objects + std::string homeDir; // Owned by the connection }; -struct S3_FILE_HANDLE { - WT_FILE_HANDLE iface; /* Must come first */ - S3_STORAGE *storage; /* Enclosing storage source */ - /* - * Similarly, The S3_FILE_HANDLE is built on top of the WT_FILE_HANDLE. We require an instance - * of the WT_FILE_HANDLE in order to access the native WiredTiger filehandle functionality, such - * as the native WT file handle read and close. - */ +struct S3FileHandle { + WT_FILE_HANDLE iface; // Must come first + S3Storage *storage; // Enclosing storage source + + // Similarly, The S3FileHandle is built on top of the WT_FILE_HANDLE. We require an instance of + // the WT_FILE_HANDLE in order to access the native WiredTiger filehandle functionality, such as + // the native WT file handle read and close. + WT_FILE_HANDLE *wtFileHandle; }; -/* Configuration variables for connecting to S3CrtClient. */ +// Configuration variables for connecting to S3CrtClient. const double throughputTargetGbps = 5; -const uint64_t partSize = 8 * 1024 * 1024; /* 8 MB. */ +const uint64_t partSize = 8 * 1024 * 1024; // 8 MB. -/* Setting SDK options. */ +// Setting SDK options. Aws::SDKOptions options; static int S3GetDirectory( - const S3_STORAGE &, const std::string &, const std::string &, bool, std::string &); + const S3Storage &, const std::string &, const std::string &, bool, std::string &); static bool S3CacheExists(WT_FILE_SYSTEM *, const std::string &); static std::string S3Path(const std::string &, const std::string &); static std::string S3HomePath(WT_FILE_SYSTEM *, const char *); static std::string S3CachePath(WT_FILE_SYSTEM *, const char *); -static int S3Exist(WT_FILE_SYSTEM *, WT_SESSION *, const char *, bool *); +static int S3FileExists(WT_FILE_SYSTEM *, WT_SESSION *, const char *, bool *); static int S3CustomizeFileSystem( WT_STORAGE_SOURCE *, WT_SESSION *, const char *, const char *, const char *, WT_FILE_SYSTEM **); static int S3AddReference(WT_STORAGE_SOURCE *); static int S3FileSystemTerminate(WT_FILE_SYSTEM *, WT_SESSION *); -static int S3Open( +static int S3FileOpen( WT_FILE_SYSTEM *, WT_SESSION *, const char *, WT_FS_OPEN_FILE_TYPE, uint32_t, WT_FILE_HANDLE **); static int S3Remove(WT_FILE_SYSTEM *, WT_SESSION *, const char *, uint32_t); static int S3Rename(WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, uint32_t); @@ -131,25 +127,22 @@ static int S3FileRead(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t, size_t, void *); static int S3ObjectList( WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, char ***, uint32_t *); static int S3ObjectListAdd( - const S3_STORAGE &, char ***, const std::vector<std::string> &, const uint32_t); + const S3Storage &, char ***, const std::vector<std::string> &, const uint32_t); static int S3ObjectListSingle( WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, char ***, uint32_t *); static int S3ObjectListFree(WT_FILE_SYSTEM *, WT_SESSION *, char **, uint32_t); -static void S3ShowStatistics(const S3_STORAGE &); +static void S3LogStatistics(const S3Storage &); static int S3FileClose(WT_FILE_HANDLE *, WT_SESSION *); static int S3FileSize(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t *); static int S3FileLock(WT_FILE_HANDLE *, WT_SESSION *, bool); -static int S3Size(WT_FILE_SYSTEM *, WT_SESSION *, const char *, wt_off_t *); +static int S3ObjectSize(WT_FILE_SYSTEM *, WT_SESSION *, const char *, wt_off_t *); -/* - * S3Path -- - * Construct a pathname from the directory and the object name. - */ +// Construct a pathname from the directory and the object name. static std::string S3Path(const std::string &dir, const std::string &name) { - /* Skip over "./" and variations (".//", ".///./././//") at the beginning of the name. */ + // Skip over "./" and variations (".//", ".///./././//") at the beginning of the name. int i = 0; while (name[i] == '.') { if (name[1] != '/') @@ -162,46 +155,41 @@ S3Path(const std::string &dir, const std::string &name) return (dir + "/" + strippedName); } -/* - * S3Exist-- - * Return if the file exists. First checks the cache, and then the S3 Bucket. - */ +// Return if the file exists. First checks the cache, and then the S3 Bucket. static int -S3Exist(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *name, bool *exist) +S3FileExists(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *name, bool *fileExists) { - size_t objectSize; - S3_FILE_SYSTEM *fs = (S3_FILE_SYSTEM *)fileSystem; - S3_STORAGE *s3 = FS2S3(fileSystem); int ret = 0; + S3FileSystem *fs = (S3FileSystem *)fileSystem; + S3Storage *s3 = FS2S3(fileSystem); - /* Check if file exists in the cache. */ - *exist = S3CacheExists(fileSystem, name); - if (*exist) + // Check if file exists in the cache. + *fileExists = S3CacheExists(fileSystem, name); + if (*fileExists) { + s3->log->LogDebugMessage("S3FileExists: Found file in cache."); return (ret); + } - /* It's not in the cache, try the S3 bucket. */ + // It's not in the cache, try the S3 bucket. + size_t objectSize; s3->statistics.objectExistsCount++; - if ((ret = fs->connection->ObjectExists(name, *exist, objectSize)) != 0) - s3->log->LogErrorMessage("S3Exist: ObjectExists request to S3 failed."); + if ((ret = fs->connection->ObjectExists(name, *fileExists, objectSize)) != 0) + s3->log->LogErrorMessage("S3FileExists: ObjectExists request to S3 failed."); + else + s3->log->LogDebugMessage("S3FileExists: Found file in S3."); return (ret); } -/* - * S3CacheExists -- - * Checks whether the given file exists in the cache. - */ +// Checks whether the given file exists in the cache. static bool S3CacheExists(WT_FILE_SYSTEM *fileSystem, const std::string &name) { - const std::string path = S3Path(((S3_FILE_SYSTEM *)fileSystem)->cacheDir, name); + const std::string path = S3Path(((S3FileSystem *)fileSystem)->cacheDir, name); return (LocalFileExists(path)); } -/* - * LocalFileExists -- - * Checks whether a file corresponding to the provided path exists locally. - */ +// Checks whether a file corresponding to the provided path exists locally. static bool LocalFileExists(const std::string &path) { @@ -209,35 +197,44 @@ LocalFileExists(const std::string &path) return (f.good()); } -/* - * S3GetDirectory -- - * Return a copy of a directory name after verifying that it is a directory. - */ +// Return a copy of a directory name after verifying that it is a directory. static int -S3GetDirectory(const S3_STORAGE &s3, const std::string &home, const std::string &name, bool create, +S3GetDirectory(const S3Storage &s3, const std::string &home, const std::string &name, bool create, std::string ©) { + // copy must be initialised before the function returns. copy = ""; - struct stat sb; int ret; std::string dirName; - /* For relative pathnames, the path is considered to be relative to the home directory. */ + // For relative pathnames, the path is considered to be relative to the home directory. if (name[0] == '/') dirName = name; else dirName = home + "/" + name; - ret = stat(dirName.c_str(), &sb); - if (ret != 0 && errno == ENOENT && create) { - mkdir(dirName.c_str(), 0777); - ret = stat(dirName.c_str(), &sb); + // Use filesystem status to find if directory exists. + std::error_code ec; + std::filesystem::file_status status = std::filesystem::status(dirName.c_str(), ec); + + if (!std::filesystem::exists(status) && create) { + try { + std::filesystem::create_directory(dirName.c_str()); + std::filesystem::permissions(dirName.c_str(), std::filesystem::perms::all); + } catch (std::filesystem::filesystem_error const &e) { + s3.log->LogErrorMessage(std::string("S3GetDirectory: ") + e.what()); + } + + s3.log->LogDebugMessage("S3GetDirectory: Successfully created directory."); } + + status = std::filesystem::status(dirName.c_str(), ec); + ret = ec.value(); + if (ret != 0) { - ret = errno; s3.log->LogErrorMessage("S3GetDirectory: No such file or directory"); - } else if ((sb.st_mode & S_IFMT) != S_IFDIR) { + } else if (!std::filesystem::is_directory(status)) { s3.log->LogErrorMessage("S3GetDirectory: invalid directory name"); ret = EINVAL; } @@ -246,29 +243,27 @@ S3GetDirectory(const S3_STORAGE &s3, const std::string &home, const std::string return (ret); } -/* - * S3FileClose -- - * File handle close. - */ +// File handle close. static int S3FileClose(WT_FILE_HANDLE *fileHandle, WT_SESSION *session) { int ret = 0; - S3_FILE_HANDLE *s3FileHandle = (S3_FILE_HANDLE *)fileHandle; - S3_STORAGE *s3 = s3FileHandle->storage; + S3FileHandle *s3FileHandle = (S3FileHandle *)fileHandle; + S3Storage *s3 = s3FileHandle->storage; WT_FILE_HANDLE *wtFileHandle = s3FileHandle->wtFileHandle; - /* - * We require exclusive access to the list of file handles when removing file handles. The - * lock_guard will be unlocked automatically once the scope is exited. - */ + + // We require exclusive access to the list of file handles when removing file handles. The + // lock_guard will be unlocked automatically once the scope is exited. { std::lock_guard<std::mutex> lock(s3->fhMutex); s3->fhList.remove(s3FileHandle); } - if (wtFileHandle != NULL) { + if (wtFileHandle != nullptr) { s3->statistics.fhOps++; if ((ret = wtFileHandle->close(wtFileHandle, session)) != 0) s3->log->LogErrorMessage("S3FileClose: close file handle failed."); + else + s3->log->LogDebugMessage("S3FileClose: Successfully closed file handle."); } free(s3FileHandle->iface.name); @@ -276,91 +271,86 @@ S3FileClose(WT_FILE_HANDLE *fileHandle, WT_SESSION *session) return (ret); } -/* - * S3Open -- - * File open for the s3 storage source. - */ +// File open for the s3 storage source. static int -S3Open(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *name, +S3FileOpen(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *name, WT_FS_OPEN_FILE_TYPE fileType, uint32_t flags, WT_FILE_HANDLE **fileHandlePtr) { - S3_FILE_HANDLE *s3FileHandle; - S3_FILE_SYSTEM *fs = (S3_FILE_SYSTEM *)fileSystem; - S3_STORAGE *s3 = fs->storage; - WT_FILE_SYSTEM *wtFileSystem = fs->wtFileSystem; - WT_FILE_HANDLE *wtFileHandle; + S3FileSystem *fs = (S3FileSystem *)fileSystem; + S3Storage *s3 = FS2S3(fileSystem); int ret; - *fileHandlePtr = NULL; + *fileHandlePtr = nullptr; - /* We only support opening the file in read only mode. */ + // We only support opening the file in read only mode. if ((flags & WT_FS_OPEN_READONLY) == 0 || (flags & WT_FS_OPEN_CREATE) != 0) { - s3->log->LogErrorMessage("S3Open: read-only access required."); + s3->log->LogErrorMessage("S3FileOpen: read-only access required."); return (EINVAL); } - /* - * Currently, only data files should be being opened; although this constraint can be relaxed in - * the future. - */ + // Currently, only data files should be being opened; although this constraint can be relaxed in + // the future. if (fileType != WT_FS_OPEN_FILE_TYPE_DATA && fileType != WT_FS_OPEN_FILE_TYPE_REGULAR) { - s3->log->LogErrorMessage("S3Open: only data file and regular types supported."); + s3->log->LogErrorMessage("S3FileOpen: only data file and regular types supported."); return (EINVAL); } - if ((s3FileHandle = (S3_FILE_HANDLE *)calloc(1, sizeof(S3_FILE_HANDLE))) == NULL) { - s3->log->LogErrorMessage("S3Open: unable to allocate memory for file handle."); + S3FileHandle *s3FileHandle; + if ((s3FileHandle = (S3FileHandle *)calloc(1, sizeof(S3FileHandle))) == nullptr) { + s3->log->LogErrorMessage("S3FileOpen: unable to allocate memory for file handle."); return (ENOMEM); } - /* Make a copy from S3 if the file is not in the cache. */ + // Make a copy from S3 if the file is not in the cache. const std::string cachePath = S3Path(fs->cacheDir, name); if (!LocalFileExists(cachePath)) { s3->statistics.getObjectCount++; if ((ret = fs->connection->GetObject(name, cachePath)) != 0) { - s3->log->LogErrorMessage("S3Open: GetObject request to S3 failed."); + s3->log->LogErrorMessage("S3FileOpen: GetObject request to S3 failed."); return (ret); } } - /* Use WiredTiger's native file handle open. */ + // Use WiredTiger's native file handle open. + WT_FILE_SYSTEM *wtFileSystem = fs->wtFileSystem; + WT_FILE_HANDLE *wtFileHandle; ret = wtFileSystem->fs_open_file( wtFileSystem, session, cachePath.c_str(), fileType, flags, &wtFileHandle); if (ret != 0) { - s3->log->LogErrorMessage("S3Open: fs_open_file failed."); + s3->log->LogErrorMessage("S3FileOpen: fs_open_file failed."); return (ret); - } + } else + s3->log->LogDebugMessage("S3FileOpen: fs_open_file succeeded."); s3FileHandle->wtFileHandle = wtFileHandle; s3FileHandle->storage = s3; + // We only define the functions we need since S3 is read-only. WT_FILE_HANDLE *fileHandle = (WT_FILE_HANDLE *)s3FileHandle; fileHandle->close = S3FileClose; - fileHandle->fh_advise = NULL; - fileHandle->fh_extend = NULL; - fileHandle->fh_extend_nolock = NULL; + fileHandle->fh_advise = nullptr; + fileHandle->fh_extend = nullptr; + fileHandle->fh_extend_nolock = nullptr; fileHandle->fh_lock = S3FileLock; - fileHandle->fh_map = NULL; - fileHandle->fh_map_discard = NULL; - fileHandle->fh_map_preload = NULL; - fileHandle->fh_unmap = NULL; + fileHandle->fh_map = nullptr; + fileHandle->fh_map_discard = nullptr; + fileHandle->fh_map_preload = nullptr; + fileHandle->fh_unmap = nullptr; fileHandle->fh_read = S3FileRead; fileHandle->fh_size = S3FileSize; - fileHandle->fh_sync = NULL; - fileHandle->fh_sync_nowait = NULL; - fileHandle->fh_truncate = NULL; - fileHandle->fh_write = NULL; + fileHandle->fh_sync = nullptr; + fileHandle->fh_sync_nowait = nullptr; + fileHandle->fh_truncate = nullptr; + fileHandle->fh_write = nullptr; fileHandle->name = strdup(name); - if (fileHandle->name == NULL) { - s3->log->LogErrorMessage("S3Open: unable to allocate memory for object name."); + if (fileHandle->name == nullptr) { + s3->log->LogErrorMessage("S3FileOpen: unable to allocate memory for object name."); return (ENOMEM); } - /* - * We require exclusive access to the list of file handles when adding file handles to it. The - * lock_guard will be unlocked automatically when the scope is exited. - */ + // We require exclusive access to the list of file handles when adding file handles to it. The + // lock_guard will be unlocked automatically when the scope is exited. { std::lock_guard<std::mutex> lock(s3->fhMutex); s3FileHandle->storage->fhList.push_back(s3FileHandle); @@ -370,157 +360,141 @@ S3Open(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *name, return (0); } -/* - * S3Rename -- - * POSIX rename, not supported for cloud objects. - */ +// POSIX rename, not supported for cloud objects. static int S3Rename(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *from, const char *to, uint32_t flags) { - S3_STORAGE *s3 = FS2S3(file_system); + S3Storage *s3 = FS2S3(file_system); - (void)to; /* unused */ - (void)flags; /* unused */ + UNUSED(to); + UNUSED(flags); s3->log->LogErrorMessage(std::string(from) + ": rename of file not supported"); return (ENOTSUP); } -/* - * S3Remove -- - * POSIX remove, not supported for cloud objects. - */ +// POSIX remove, not supported for cloud objects. static int S3Remove(WT_FILE_SYSTEM *file_system, WT_SESSION *session, const char *name, uint32_t flags) { - S3_STORAGE *s3 = FS2S3(file_system); + S3Storage *s3 = FS2S3(file_system); - (void)flags; /* unused */ + UNUSED(flags); s3->log->LogErrorMessage(std::string(name) + ": remove of file not supported"); return (ENOTSUP); } -/* - * S3Size -- - * Get the size of a file in bytes, by file name. - */ +// Get the size of a file in bytes, by file name. + static int -S3Size(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *name, wt_off_t *sizep) +S3ObjectSize(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *name, wt_off_t *sizep) { - S3_STORAGE *s3 = FS2S3(fileSystem); - size_t objectSize; - bool exist; + // sizep must be initialised before the function returns. *sizep = 0; + + S3Storage *s3 = FS2S3(fileSystem); + S3FileSystem *fs = (S3FileSystem *)fileSystem; + size_t objectSize; + bool exists; int ret; - S3_FILE_SYSTEM *fs = (S3_FILE_SYSTEM *)fileSystem; s3->statistics.objectExistsCount++; - if ((ret = fs->connection->ObjectExists(name, exist, objectSize)) != 0) + if ((ret = fs->connection->ObjectExists(name, exists, objectSize)) != 0) { + s3->log->LogDebugMessage( + "S3ObjectSize: Found S3 object size to be " + std::to_string(objectSize) + " bytes."); return (ret); + } *sizep = objectSize; return (ret); } -/* - * S3FileLock -- - * Lock/unlock a file. - */ +// Lock/unlock a file. static int S3FileLock(WT_FILE_HANDLE *fileHandle, WT_SESSION *session, bool lock) { - /* Locks are always granted. */ - (void)session; /* Unused */ - (void)lock; /* Unused */ + // Locks are always granted. + UNUSED(session); + UNUSED(lock); - ((S3_FILE_HANDLE *)fileHandle)->storage->statistics.fhOps++; return (0); } -/* - * S3FileRead -- - * Read a file using WiredTiger's native file handle read. - */ +// Read a file using WiredTiger's native file handle read. + static int S3FileRead(WT_FILE_HANDLE *fileHandle, WT_SESSION *session, wt_off_t offset, size_t len, void *buf) { - S3_FILE_HANDLE *s3FileHandle = (S3_FILE_HANDLE *)fileHandle; - S3_STORAGE *s3 = s3FileHandle->storage; + S3FileHandle *s3FileHandle = (S3FileHandle *)fileHandle; + S3Storage *s3 = s3FileHandle->storage; WT_FILE_HANDLE *wtFileHandle = s3FileHandle->wtFileHandle; int ret; s3->statistics.fhReadOps++; if ((ret = wtFileHandle->fh_read(wtFileHandle, session, offset, len, buf)) != 0) s3->log->LogErrorMessage("S3FileRead: fh_read failed."); + else + s3->log->LogDebugMessage( + "S3FileRead: fh_read succeeded in reading " + std::to_string(len) + " bytes."); return (ret); } -/* - * S3FileSize -- - * Get the size of a file in bytes, by file handle. - */ +// Get the size of a file in bytes, by file handle. static int S3FileSize(WT_FILE_HANDLE *fileHandle, WT_SESSION *session, wt_off_t *sizep) { - S3_FILE_HANDLE *s3FileHandle = (S3_FILE_HANDLE *)fileHandle; - S3_STORAGE *s3 = s3FileHandle->storage; + S3FileHandle *s3FileHandle = (S3FileHandle *)fileHandle; + S3Storage *s3 = s3FileHandle->storage; WT_FILE_HANDLE *wtFileHandle = s3FileHandle->wtFileHandle; s3->statistics.fhOps++; return (wtFileHandle->fh_size(wtFileHandle, session, sizep)); } -/* - * S3CustomizeFileSystem -- - * Return a customized file system to access the s3 storage source objects. The authToken - * contains the AWS access key ID and the AWS secret key as comma-separated values. - */ +// Return a customized file system to access the s3 storage source objects. The authToken +// contains the AWS access key ID and the AWS secret key as comma-separated values. static int S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, const char *bucket, const char *authToken, const char *config, WT_FILE_SYSTEM **fileSystem) { - S3_FILE_SYSTEM *fs; - WT_FILE_SYSTEM *wtFileSystem; - S3_STORAGE *s3; + S3Storage *s3; int ret; - std::string cacheDir; - s3 = (S3_STORAGE *)storageSource; + s3 = (S3Storage *)storageSource; + + // We need to have a bucket to setup the file system. The bucket is expected to be a name and a + // region, separated by a semi-colon. eg: 'abcd;ap-southeast-2'. - /* - * We need to have a bucket to setup the file system. The bucket is expected to be a name and a - * region, separated by a semi-colon. eg: 'abcd;ap-southeast-2'. - */ - if (bucket == NULL || strlen(bucket) == 0) { + if (bucket == nullptr || strlen(bucket) == 0) { s3->log->LogErrorMessage("S3CustomizeFileSystem: bucket not specified."); return (EINVAL); } int delimiter = std::string(bucket).find(';'); if (delimiter == std::string::npos || delimiter == 0 || delimiter == strlen(bucket) - 1) { s3->log->LogErrorMessage( - "S3CustomizeFileSystem: bucket malformed, " - "should be a name and a region separated by a semi-colon."); + "S3CustomizeFileSystem: improper bucket name, " + "should be a name and a region separated by a semicolon."); return (EINVAL); } const std::string bucketName = std::string(bucket).substr(0, delimiter); const std::string region = std::string(bucket).substr(delimiter + 1); - /* Fail if there is no authentication provided. */ - if (authToken == NULL || strlen(authToken) == 0) { + // Fail if there is no authentication provided. + if (authToken == nullptr || strlen(authToken) == 0) { s3->log->LogErrorMessage("S3CustomizeFileSystem: authToken not specified."); return (EINVAL); } - /* - * An auth token is needed to setup the file system. The token is expected to be an access key - * and a secret key separated by a semi-colon. - */ - if (authToken == NULL || strlen(authToken) == 0) { + // An auth token is needed to setup the file system. The token is expected to be an access key + // and a secret key separated by a semi-colon. + if (authToken == nullptr || strlen(authToken) == 0) { s3->log->LogErrorMessage("S3CustomizeFileSystem: auth token not specified."); return (EINVAL); } delimiter = std::string(authToken).find(';'); if (delimiter == std::string::npos || delimiter == 0 || delimiter == strlen(authToken) - 1) { - s3->log->LogErrorMessage("S3CustomizeFileSystem: authToken malformed."); + s3->log->LogErrorMessage( + "S3CustomizeFileSystem: improper authToken, should be an access key and a secret key " + "separated by a semicolon."); return (EINVAL); } const std::string accessKeyId = std::string(authToken).substr(0, delimiter); @@ -530,11 +504,12 @@ S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, con credentials.SetAWSAccessKeyId(accessKeyId); credentials.SetAWSSecretKey(secretKey); - /* - * Parse configuration string. - */ + s3->log->LogDebugMessage( + "S3CustomizeFileSystem: AWS access key and secret key set successfully."); - /* Get any prefix to be used for the object keys. */ + // Parse configuration string. + + // Get any prefix to be used for the object keys. WT_CONFIG_ITEM objPrefixConf; std::string objPrefix; if ((ret = s3->wtApi->config_get_string( @@ -545,18 +520,17 @@ S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, con return (ret); } - /* Configure the AWS Client configuration. */ + // Configure the AWS Client configuration. Aws::S3Crt::ClientConfiguration awsConfig; awsConfig.partSize = partSize; awsConfig.region = region; awsConfig.throughputTargetGbps = throughputTargetGbps; - /* - * Get the directory to setup the cache, or use the default one. The default cache directory is - * named "cache-<name>", where name is the last component of the bucket name's path. We'll - * create it if it doesn't exist. - */ + // Get the directory to setup the cache, or use the default one. The default cache directory is + // named "cache-<name>", where name is the last component of the bucket name's path. We'll + // create it if it doesn't exist. WT_CONFIG_ITEM cacheDirConf; + std::string cacheDir; std::string cacheStr; if ((ret = s3->wtApi->config_get_string( s3->wtApi, session, config, "cache_directory", &cacheDirConf)) == 0) @@ -566,21 +540,23 @@ S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, con ret = 0; } else { s3->log->LogErrorMessage( - "wiredtiger_extension_init: error parsing config for cache directory."); + "S3CustomizeFileSystem: error parsing config for cache directory."); return (ret); } - /* Fetch the native WT file system. */ + // Fetch the native WT file system. + WT_FILE_SYSTEM *wtFileSystem; if ((ret = s3->wtApi->file_system_get(s3->wtApi, session, &wtFileSystem)) != 0) return (ret); - /* Get a copy of the home and cache directory. */ + // Get a copy of the home and cache directory. const std::string homeDir = session->connection->get_home(session->connection); if ((ret = S3GetDirectory(*s3, homeDir, cacheStr, true, cacheDir)) != 0) return (ret); - /* Create the file system. */ - if ((fs = (S3_FILE_SYSTEM *)calloc(1, sizeof(S3_FILE_SYSTEM))) == NULL) { + // Create the file system. + S3FileSystem *fs; + if ((fs = (S3FileSystem *)calloc(1, sizeof(S3FileSystem))) == nullptr) { s3->log->LogErrorMessage( "S3CustomizeFileSystem: unable to allocate memory for file system."); return (ENOMEM); @@ -590,7 +566,6 @@ S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, con fs->homeDir = homeDir; fs->cacheDir = cacheDir; - /* New can fail; will deal with this later. */ try { fs->connection = new S3Connection(credentials, awsConfig, bucketName, objPrefix); } catch (std::invalid_argument &e) { @@ -601,13 +576,15 @@ S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, con fs->fileSystem.fs_directory_list_single = S3ObjectListSingle; fs->fileSystem.fs_directory_list_free = S3ObjectListFree; fs->fileSystem.terminate = S3FileSystemTerminate; - fs->fileSystem.fs_exist = S3Exist; - fs->fileSystem.fs_open_file = S3Open; + fs->fileSystem.fs_exist = S3FileExists; + fs->fileSystem.fs_open_file = S3FileOpen; fs->fileSystem.fs_remove = S3Remove; fs->fileSystem.fs_rename = S3Rename; - fs->fileSystem.fs_size = S3Size; + fs->fileSystem.fs_size = S3ObjectSize; - /* Add to the list of the active file systems. Lock will be freed when the scope is exited. */ + s3->log->LogDebugMessage("S3CustomizeFileSystem: S3 connection established."); + + // Add to the list of the active file systems. Lock will be freed when the scope is exited. { std::lock_guard<std::mutex> lockGuard(s3->fsListMutex); s3->fsList.push_back(fs); @@ -617,19 +594,16 @@ S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, con return (ret); } -/* - * S3FileSystemTerminate -- - * Discard any resources on termination of the file system. - */ +// Discard any resources on termination of the file system. static int S3FileSystemTerminate(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session) { - S3_FILE_SYSTEM *fs = (S3_FILE_SYSTEM *)fileSystem; - S3_STORAGE *s3 = fs->storage; + S3FileSystem *fs = (S3FileSystem *)fileSystem; + S3Storage *s3 = FS2S3(fileSystem); - UNUSED(session); /* unused */ + UNUSED(session); - /* Remove from the active filesystems list. The lock will be freed when the scope is exited. */ + // Remove from the active filesystems list. The lock will be freed when the scope is exited. { std::lock_guard<std::mutex> lockGuard(s3->fsListMutex); s3->fsList.remove(fs); @@ -640,90 +614,70 @@ S3FileSystemTerminate(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session) return (0); } -/* - * S3ObjectList -- - * Return a list of object names for the given location. - */ +// Return a list of object names for the given location. static int -S3ObjectList(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *directory, - const char *prefix, char ***objectList, uint32_t *count) +S3ObjectListInternal(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *directory, + const char *prefix, char ***objectList, uint32_t *count, bool listSingle) { - S3_FILE_SYSTEM *fs = (S3_FILE_SYSTEM *)fileSystem; - S3_STORAGE *s3 = FS2S3(fileSystem); - + S3FileSystem *fs = (S3FileSystem *)fileSystem; + S3Storage *s3 = FS2S3(fileSystem); std::vector<std::string> objects; std::string completePrefix; - if (directory != NULL) { + *count = 0; + + if (directory != nullptr) { completePrefix += directory; - /* Add a terminating '/' if one doesn't exist. */ + // Add a terminating '/' if one doesn't exist. if (completePrefix.length() > 1 && completePrefix[completePrefix.length() - 1] != '/') completePrefix += '/'; } - if (prefix != NULL) + if (prefix != nullptr) completePrefix += prefix; int ret; s3->statistics.listObjectsCount++; - if ((ret = fs->connection->ListObjects(completePrefix, objects)) != 0) { + + ret = listSingle ? fs->connection->ListObjects(completePrefix, objects, 1, true) : + fs->connection->ListObjects(completePrefix, objects); + + if (ret != 0) { s3->log->LogErrorMessage("S3ObjectList: ListObjects request to S3 failed."); return (ret); } *count = objects.size(); + s3->log->LogDebugMessage("S3ObjectList: ListObjects request to S3 succeeded. Received " + + std::to_string(*count) + " objects."); S3ObjectListAdd(*s3, objectList, objects, *count); return (ret); } -/* - * S3ObjectListSingle -- - * Return a single object name for the given location. - */ +// Return a list of object names for the given location. static int -S3ObjectListSingle(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *directory, +S3ObjectList(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *directory, const char *prefix, char ***objectList, uint32_t *count) { - S3_FILE_SYSTEM *fs = (S3_FILE_SYSTEM *)fileSystem; - S3_STORAGE *s3 = FS2S3(fileSystem); - - std::vector<std::string> objects; - std::string completePrefix; - - if (directory != NULL) { - completePrefix += directory; - /* Add a terminating '/' if one doesn't exist. */ - if (completePrefix.length() > 1 && completePrefix[completePrefix.length() - 1] != '/') - completePrefix += '/'; - } - if (prefix != NULL) - completePrefix += prefix; - - int ret; - s3->statistics.listObjectsCount++; - if ((ret = fs->connection->ListObjects(completePrefix, objects, 1, true)) != 0) { - s3->log->LogErrorMessage("S3ObjectListSingle: ListObjects request to S3 failed."); - return (ret); - } - - *count = objects.size(); - - S3ObjectListAdd(*s3, objectList, objects, *count); + return (S3ObjectListInternal(fileSystem, session, directory, prefix, objectList, count, false)); +} - return (ret); +// Return a single object name for the given location. +static int +S3ObjectListSingle(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *directory, + const char *prefix, char ***objectList, uint32_t *count) +{ + return (S3ObjectListInternal(fileSystem, session, directory, prefix, objectList, count, true)); } -/* - * S3ObjectListFree -- - * Free memory allocated by S3ObjectList. - */ +// Free memory allocated by S3ObjectList. static int S3ObjectListFree(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, char **objectList, uint32_t count) { UNUSED(fileSystem); UNUSED(session); - if (objectList != NULL) { + if (objectList != nullptr) { while (count > 0) free(objectList[--count]); free(objectList); @@ -732,22 +686,19 @@ S3ObjectListFree(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, char **objectL return (0); } -/* - * S3ObjectListAdd -- - * Add objects retrieved from S3 bucket into the object list, and allocate the memory needed. - */ +// Add objects retrieved from S3 bucket into the object list, and allocate the memory needed. static int -S3ObjectListAdd(const S3_STORAGE &s3, char ***objectList, const std::vector<std::string> &objects, +S3ObjectListAdd(const S3Storage &s3, char ***objectList, const std::vector<std::string> &objects, const uint32_t count) { char **entries; - if ((entries = (char **)malloc(sizeof(char *) * count)) == NULL) { + if ((entries = (char **)malloc(sizeof(char *) * count)) == nullptr) { s3.log->LogErrorMessage("S3ObjectListAdd: unable to allocate memory for object list."); return (ENOMEM); } for (int i = 0; i < count; i++) { - if ((entries[i] = strdup(objects[i].c_str())) == NULL) { + if ((entries[i] = strdup(objects[i].c_str())) == nullptr) { s3.log->LogErrorMessage( "S3ObjectListAdd: unable to allocate memory for object string."); return (ENOMEM); @@ -758,15 +709,12 @@ S3ObjectListAdd(const S3_STORAGE &s3, char ***objectList, const std::vector<std: return (0); } -/* - * S3AddReference -- - * Add a reference to the storage source so we can reference count to know when to really - * terminate. - */ +// Add a reference to the storage source so we can reference count to know when to really +// terminate. static int S3AddReference(WT_STORAGE_SOURCE *storageSource) { - S3_STORAGE *s3 = (S3_STORAGE *)storageSource; + S3Storage *s3 = (S3Storage *)storageSource; if (s3->referenceCount == 0 || s3->referenceCount + 1 == 0) { s3->log->LogErrorMessage("S3AddReference: missing reference or overflow."); @@ -777,64 +725,55 @@ S3AddReference(WT_STORAGE_SOURCE *storageSource) return (0); } -/* - * S3Terminate -- - * Discard any resources on termination. - */ +// Discard any resources on termination. + static int S3Terminate(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session) { - S3_STORAGE *s3 = (S3_STORAGE *)storageSource; + S3Storage *s3 = (S3Storage *)storageSource; if (--s3->referenceCount != 0) return (0); - /* - * Is it currently unclear at the moment what the multi-threading will look like in the - * extension. The current implementation is NOT thread-safe, and needs to be addressed in the - * future, as mulitple threads could call terminate leading to a race condition. - */ + // It is currently unclear at the moment what the multi-threading will look like in the + // extension. The current implementation is NOT thread-safe, and needs to be addressed in the + // future, as multiple threads could call terminate leading to a race condition. while (!s3->fhList.empty()) { - S3_FILE_HANDLE *fs = s3->fhList.front(); + S3FileHandle *fs = s3->fhList.front(); S3FileClose((WT_FILE_HANDLE *)fs, session); } - /* - * Terminate any active filesystems. There are no references to the storage source, so it is - * safe to walk the active filesystem list without a lock. The removal from the list happens - * under a lock. Also, removal happens from the front and addition at the end, so we are safe. - */ + + // Terminate any active filesystems. There are no references to the storage source, so it is + // safe to walk the active filesystem list without a lock. The removal from the list happens + // under a lock. Also, removal happens from the front and addition at the end, so we are safe. while (!s3->fsList.empty()) { - S3_FILE_SYSTEM *fs = s3->fsList.front(); + S3FileSystem *fs = s3->fsList.front(); S3FileSystemTerminate(&fs->fileSystem, session); } - /* Log collected statistics on termination. */ - S3ShowStatistics(*s3); + S3LogStatistics(*s3); Aws::Utils::Logging::ShutdownAWSLogging(); Aws::ShutdownAPI(options); + s3->log->LogDebugMessage("S3Terminate: Terminated S3 storage source."); delete (s3); return (0); } -/* - * S3Flush -- - * Flush file to S3 Store using AWS SDK C++ PutObject. - */ +// Flush file to S3 Store using AWS SDK C++ PutObject. static int S3Flush(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, WT_FILE_SYSTEM *fileSystem, const char *source, const char *object, const char *config) { - S3_STORAGE *s3 = (S3_STORAGE *)storageSource; - S3_FILE_SYSTEM *fs = (S3_FILE_SYSTEM *)fileSystem; + S3Storage *s3 = (S3Storage *)storageSource; + S3FileSystem *fs = (S3FileSystem *)fileSystem; WT_FILE_SYSTEM *wtFileSystem = fs->wtFileSystem; - int ret; bool nativeExist; FS2S3(fileSystem)->statistics.putObjectCount++; - /* Confirm that the file exists on the native filesystem. */ + // Confirm that the file exists on the native filesystem. if ((ret = wtFileSystem->fs_exist(wtFileSystem, session, source, &nativeExist)) != 0) { s3->log->LogErrorMessage("S3Flush: Failed to check for the existence of " + std::string(source) + " on the native filesystem."); @@ -845,29 +784,30 @@ S3Flush(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, WT_FILE_SYSTEM *f return (ENOENT); } - /* Upload the object into the bucket. */ + // Upload the object into the bucket. if (ret = (fs->connection->PutObject(object, source)) != 0) s3->log->LogErrorMessage("S3Flush: PutObject request to S3 failed."); + else + s3->log->LogDebugMessage("S3Flush: Uploaded object to S3."); return (ret); } -/* - * S3FlushFinish -- - * Flush local file to cache. - */ +// Flush local file to cache. static int S3FlushFinish(WT_STORAGE_SOURCE *storage, WT_SESSION *session, WT_FILE_SYSTEM *fileSystem, const char *source, const char *object, const char *config) { - S3_STORAGE *s3 = (S3_STORAGE *)storage; - S3_FILE_SYSTEM *fs = (S3_FILE_SYSTEM *)fileSystem; - /* Constructing the pathname for source and cache from file system and local. */ + S3Storage *s3 = (S3Storage *)storage; + S3FileSystem *fs = (S3FileSystem *)fileSystem; + // Constructing the pathname for source and cache from file system and local. std::string srcPath = S3Path(fs->homeDir, source); std::string destPath = S3Path(fs->cacheDir, object); - /* Linking file with the local file. */ - int ret = link(srcPath.c_str(), destPath.c_str()); + // Linking file with the local file. + std::error_code ec; + std::filesystem::create_hard_link(srcPath.c_str(), destPath.c_str(), ec); + int ret = ec.value(); if (ret != 0) { ret = errno; s3->log->LogErrorMessage( @@ -875,21 +815,22 @@ S3FlushFinish(WT_STORAGE_SOURCE *storage, WT_SESSION *session, WT_FILE_SYSTEM *f return (ret); } - /* The file should be read-only. */ - ret = chmod(destPath.c_str(), 0444); + // The file should be read-only. + std::filesystem::permissions(destPath.c_str(), + std::filesystem::perms::owner_read | std::filesystem::perms::group_read | + std::filesystem::perms::others_read, + std::filesystem::perm_options::add, ec); + ret = ec.value(); if (ret != 0) { ret = errno; - s3->log->LogErrorMessage("S3FlushFinish: chmod of " + destPath + " failed"); + s3->log->LogErrorMessage("S3FlushFinish: read permissions of " + destPath + " failed"); } return (ret); } -/* - * S3ShowStatistics -- - * Log collected statistics. - */ +// Log collected statistics. static void -S3ShowStatistics(const S3_STORAGE &s3) +S3LogStatistics(const S3Storage &s3) { s3.log->LogDebugMessage( "S3 list objects count: " + std::to_string(s3.statistics.listObjectsCount)); @@ -904,28 +845,19 @@ S3ShowStatistics(const S3_STORAGE &s3) "File handle read operations: " + std::to_string(s3.statistics.fhReadOps)); } -/* - * wiredtiger_extension_init -- - * A S3 storage source library. - */ +// A S3 storage source library. int wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) { - S3_STORAGE *s3; - S3_FILE_SYSTEM *fs; + S3Storage *s3; WT_CONFIG_ITEM v; - /* No error handling for now. */ - s3 = new S3_STORAGE; - + s3 = new S3Storage; s3->wtApi = connection->get_extension_api(connection); + int ret = s3->wtApi->config_get(s3->wtApi, nullptr, config, "verbose.tiered", &v); - int ret = s3->wtApi->config_get(s3->wtApi, NULL, config, "verbose", &v); - - /* - * Create a logger for the storage source. Verbose level defaults to WT_VERBOSE_ERROR (-3) if it - * is outside the valid range or not found. - */ + // Create a logger for the storage source. Verbose level defaults to WT_VERBOSE_ERROR (-3) if it + // is outside the valid range or not found. s3->verbose = WT_VERBOSE_ERROR; s3->log = Aws::MakeShared<S3LogSystem>("storage", s3->wtApi, s3->verbose); @@ -939,32 +871,33 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) return (ret != 0 ? ret : EINVAL); } - /* Set up statistics. */ + // Set up statistics. s3->statistics = {0}; - /* Initialize the AWS SDK. */ + // Initialize the AWS SDK. Aws::Utils::Logging::InitializeAWSLogging(s3->log); Aws::InitAPI(options); - /* - * Allocate a S3 storage structure, with a WT_STORAGE structure as the first field, allowing us - * to treat references to either type of structure as a reference to the other type. - */ + // Allocate a S3 storage structure, with a WT_STORAGE structure as the first field, allowing us + // to treat references to either type of structure as a reference to the other type. s3->storageSource.ss_customize_file_system = S3CustomizeFileSystem; s3->storageSource.ss_add_reference = S3AddReference; s3->storageSource.terminate = S3Terminate; s3->storageSource.ss_flush = S3Flush; s3->storageSource.ss_flush_finish = S3FlushFinish; - /* - * The first reference is implied by the call to add_storage_source. - */ + // The first reference is implied by the call to add_storage_source. s3->referenceCount = 1; - /* Load the storage */ - if ((ret = connection->add_storage_source(connection, "s3_store", &s3->storageSource, NULL)) != - 0) + // Load the storage + if ((ret = connection->add_storage_source( + connection, "s3_store", &s3->storageSource, nullptr)) != 0) { + s3->log->LogErrorMessage( + "wiredtiger_extension_init: Could not load S3 storage source, shutting down."); + Aws::Utils::Logging::ShutdownAWSLogging(); + Aws::ShutdownAPI(options); delete (s3); + } return (ret); } diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/test/test_s3_connection.cpp b/src/third_party/wiredtiger/ext/storage_sources/s3_store/test/test_s3_connection.cpp index 5440edfd59d..c93fc492c10 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/s3_store/test/test_s3_connection.cpp +++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/test/test_s3_connection.cpp @@ -1,12 +1,39 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ #include <s3_connection.h> #include <fstream> #include <random> -/* Default config settings for the Test environment. */ +// Default config settings for the Test environment. namespace TestDefaults { const Aws::String region = Aws::Region::AP_SOUTHEAST_2; const double throughputTargetGbps = 5; -const uint64_t partSize = 8 * 1024 * 1024; /* 8 MB. */ +const uint64_t partSize = 8 * 1024 * 1024; // 8 MB. static std::string bucketName("s3testext"); // Can be overridden with environment variables. static std::string objPrefix("s3test_artefacts--unit_"); // To be concatenated with a random string. } // namespace TestDefaults @@ -19,7 +46,7 @@ int TestGetObject(const Aws::S3Crt::ClientConfiguration &config); int TestObjectExists(const Aws::S3Crt::ClientConfiguration &config); int TestBadBucket(const Aws::S3Crt::ClientConfiguration &config); -/* Wrapper for unit test functions. */ +// Wrapper for unit test functions. #define TEST(func, config) \ do { \ int __ret; \ @@ -27,11 +54,8 @@ int TestBadBucket(const Aws::S3Crt::ClientConfiguration &config); return (__ret); \ } while (0) -/* - * randomizeTestPrefix -- - * Concatenates a random suffix to the prefix being used for the test object keys. Example of - * generated test prefix: "s3test_artefacts/unit_" 2022-31-01-16-34-10_623843294/" - */ +// Concatenates a random suffix to the prefix being used for the test object keys. Example of +// generated test prefix: "s3test_artefacts/unit_" 2022-31-01-16-34-10_623843294/" static int randomizeTestPrefix() { @@ -43,7 +67,7 @@ randomizeTestPrefix() TestDefaults::objPrefix += timeStr; - /* Create a random device and use it to generate a random seed to initialize the generator. */ + // Create a random device and use it to generate a random seed to initialize the generator. std::random_device myRandomDevice; unsigned seed = myRandomDevice(); std::default_random_engine myRandomEngine(seed); @@ -54,20 +78,17 @@ randomizeTestPrefix() return (TEST_SUCCESS); } -/* - * setupTestDefaults -- - * Override the defaults with the ones specific for this test instance. - */ +// Overrides the defaults with the ones specific for this test instance. static int setupTestDefaults() { - /* Prefer to use the bucket provided through the environment variable. */ + // Prefer to use the bucket provided through the environment variable. const char *envBucket = std::getenv("WT_S3_EXT_BUCKET"); - if (envBucket != NULL) + if (envBucket != nullptr) TestDefaults::bucketName = envBucket; std::cerr << "Bucket to be used for testing: " << TestDefaults::bucketName << std::endl; - /* Append the prefix to be used for object names by a unique string. */ + // Append the prefix to be used for object names by a unique string. if (randomizeTestPrefix() != 0) return (TEST_FAILURE); std::cerr << "Generated prefix: " << TestDefaults::objPrefix << std::endl; @@ -79,7 +100,7 @@ static int CleanupTestListObjects(S3Connection &conn, const int totalObjects, const std::string &prefix, const std::string &fileName) { - /* Delete objects and file at end of test. */ + // Delete objects and file at end of test. int ret = 0; for (int i = 0; i < totalObjects; i++) { if ((ret = conn.DeleteObject(prefix + std::to_string(i) + ".txt")) != 0) @@ -92,50 +113,47 @@ CleanupTestListObjects(S3Connection &conn, const int totalObjects, const std::st return (ret); } -/* - * TestListObjects -- - * Unit test for listing S3 objects under the test bucket. - */ -/* Todo: Remove code duplication in this function. */ +// Lists S3 objects under the test bucket. +// Todo: Remove code duplication in this function. int TestListObjects(const Aws::S3Crt::ClientConfiguration &config) { S3Connection conn(config, TestDefaults::bucketName, TestDefaults::objPrefix); std::vector<std::string> objects; - /* Name of file to insert in the test. */ + // Name of file to insert in the test. const std::string fileName = "test_list_objects.txt"; - /* Total objects to insert in the test. */ + // Total objects to insert in the test. const int32_t totalObjects = 20; - /* Prefix for objects in this test. */ + // Prefix for objects in this test. const std::string prefix = "test_list_objects_"; - /* Parameter for getting single object. */ + // Parameter for getting single object. const bool listSingle = true; - /* Number of objects to access per iteration of AWS. */ + // Number of objects to access per iteration of AWS. int32_t batchSize = 1; - /* Expected number of matches. */ + // Expected number of matches. int32_t expectedResult = 0; int ret; - /* No matching objects. */ + // No matching objects. if ((ret = conn.ListObjects(prefix, objects)) != 0) return (ret); if (objects.size() != expectedResult) return (TEST_FAILURE); - /* No matching objects with listSingle. */ + // No matching objects with listSingle. if ((ret = conn.ListObjects(prefix, objects, batchSize, listSingle)) != 0) return (ret); if (objects.size() != expectedResult) return (TEST_FAILURE); - /* Create file to prepare for test. */ + // Create file to prepare for test. if (!static_cast<bool>(std::ofstream(fileName).put('.'))) { std::cerr << "Error creating file." << std::endl; return (TEST_FAILURE); } - /* Put objects to prepare for test. */ + // Put objects to prepare for test. for (int i = 0; i < totalObjects; i++) { if ((ret = conn.PutObject(prefix + std::to_string(i) + ".txt", fileName)) != 0) { CleanupTestListObjects(conn, i, prefix, fileName); @@ -143,7 +161,7 @@ TestListObjects(const Aws::S3Crt::ClientConfiguration &config) } } - /* List all objects. */ + // List all objects. expectedResult = totalObjects; if ((ret = conn.ListObjects(prefix, objects)) != 0) { CleanupTestListObjects(conn, totalObjects, prefix, fileName); @@ -154,7 +172,7 @@ TestListObjects(const Aws::S3Crt::ClientConfiguration &config) return (TEST_FAILURE); } - /* List single. */ + // List single. objects.clear(); expectedResult = 1; if ((ret = conn.ListObjects(prefix, objects, batchSize, listSingle)) != 0) { @@ -166,7 +184,7 @@ TestListObjects(const Aws::S3Crt::ClientConfiguration &config) return (TEST_FAILURE); } - /* Expected number of matches with test_list_objects_1 prefix. */ + // Expected number of matches with test_list_objects_1 prefix. objects.clear(); expectedResult = 11; if ((ret = conn.ListObjects(prefix + "1", objects)) != 0) { @@ -178,7 +196,7 @@ TestListObjects(const Aws::S3Crt::ClientConfiguration &config) return (TEST_FAILURE); } - /* List with 5 objects per AWS request. */ + // List with 5 objects per AWS request. objects.clear(); batchSize = 5; expectedResult = totalObjects; @@ -191,7 +209,7 @@ TestListObjects(const Aws::S3Crt::ClientConfiguration &config) return (TEST_FAILURE); } - /* ListSingle with 8 objects per AWS request. */ + // ListSingle with 8 objects per AWS request. objects.clear(); expectedResult = 1; if ((ret = conn.ListObjects(prefix, objects, batchSize, listSingle)) != 0) { @@ -203,7 +221,7 @@ TestListObjects(const Aws::S3Crt::ClientConfiguration &config) return (TEST_FAILURE); } - /* List with 8 objects per AWS request. */ + // List with 8 objects per AWS request. objects.clear(); batchSize = 8; expectedResult = totalObjects; @@ -216,7 +234,7 @@ TestListObjects(const Aws::S3Crt::ClientConfiguration &config) return (TEST_FAILURE); } - /* ListSingle with 8 objects per AWS request. */ + // ListSingle with 8 objects per AWS request. objects.clear(); expectedResult = 1; if ((ret = conn.ListObjects(prefix, objects, batchSize, listSingle)) != 0) { @@ -233,10 +251,7 @@ TestListObjects(const Aws::S3Crt::ClientConfiguration &config) return (TEST_SUCCESS); } -/* - * TestGetObject -- - * Unit test to get an object from an S3 Bucket. - */ +// Gets an object from an S3 Bucket. int TestGetObject(const Aws::S3Crt::ClientConfiguration &config) { @@ -246,24 +261,24 @@ TestGetObject(const Aws::S3Crt::ClientConfiguration &config) const std::string objectName = "permanent_object"; const std::string path = "./" + objectName; - /* Create a file and upload to the bucket. */ + // Create a file and upload to the bucket. std::ofstream File(objectName); File << "Test payload"; File.close(); if ((ret = conn.PutObject(objectName, objectName)) != 0) return (ret); - /* Delete the local copy of the file. */ + // Delete the local copy of the file. if (std::remove(path.c_str()) != 0) return (TEST_FAILURE); - /* Download the file from S3 */ + // Download the file from S3 if ((ret = conn.GetObject(objectName, path)) != 0) { std::cerr << "TestGetObject: call to S3Connection:GetObject has failed." << std::endl; return (ret); } - /* The file should now be in the current directory. */ + // The file should now be in the current directory. std::ifstream f(path); if (!f.good()) { std::cerr << "TestGetObject: target " << objectName @@ -271,7 +286,7 @@ TestGetObject(const Aws::S3Crt::ClientConfiguration &config) return (TEST_FAILURE); } - /* Clean up test artifacts. */ + // Clean up test artifacts. if (std::remove(path.c_str()) != 0) return (TEST_FAILURE); @@ -282,10 +297,7 @@ TestGetObject(const Aws::S3Crt::ClientConfiguration &config) return (TEST_SUCCESS); } -/* - * TestObjectExists -- - * Unit test to check if an object exists in an AWS bucket and size of the object is correct. - */ +// Checks if an object exists in an AWS bucket and if the size of the object is correct. int TestObjectExists(const Aws::S3Crt::ClientConfiguration &config) { @@ -297,7 +309,7 @@ TestObjectExists(const Aws::S3Crt::ClientConfiguration &config) const std::string objectName = "test_object"; const std::string fileName = "test_object.txt"; - /* Create a file to upload to the bucket.*/ + // Create a file to upload to the bucket. std::ofstream File(fileName); std::string payload = "Test payload"; File << payload; @@ -327,22 +339,19 @@ TestObjectExists(const Aws::S3Crt::ClientConfiguration &config) return (TEST_SUCCESS); } -/* - * TestBadBucket -- - * Unit test to check if connection to a non-existing bucket fails gracefully. - */ +// Checks if connection to a non-existing bucket fails gracefully. int TestBadBucket(const Aws::S3Crt::ClientConfiguration &config) { int ret = TEST_FAILURE; - /* The connection object instantitation should not succeed. */ + // The connection object instantitation should not succeed. try { S3Connection conn(config, "BadBucket", TestDefaults::objPrefix); (void)conn; std::cerr << "TestBadBucket: Failed to generate exception for the bad bucket." << std::endl; } catch (std::invalid_argument &e) { - /* Make sure we get the expected exception message. */ + // Make sure we get the expected exception message. if (std::string(e.what()).compare("BadBucket : No such bucket.") == 0) ret = 0; else @@ -354,13 +363,13 @@ TestBadBucket(const Aws::S3Crt::ClientConfiguration &config) return (ret); ret = TEST_FAILURE; - /* Also check for the dynamic allocation. */ + // Also check for the dynamic allocation. try { auto conn2 = new S3Connection(config, "BadBucket2", TestDefaults::objPrefix); (void)conn2; std::cerr << "TestBadBucket: Failed to generate exception for the bad bucket." << std::endl; } catch (std::invalid_argument &e) { - /* Make sure we get the expected exception message. */ + // Make sure we get the expected exception message. if (std::string(e.what()).compare("BadBucket2 : No such bucket.") == 0) ret = 0; else @@ -375,24 +384,21 @@ TestBadBucket(const Aws::S3Crt::ClientConfiguration &config) return (TEST_SUCCESS); } -/* - * main -- - * Set up configs and call unit tests. - */ +// Sets up configs and calls unit tests. int main() { - /* Setup the test environment. */ + // Setup the test environment. if (setupTestDefaults() != 0) return (TEST_FAILURE); - /* Set up the config to use the defaults specified. */ + // Set up the config to use the defaults specified. Aws::S3Crt::ClientConfiguration awsConfig; awsConfig.region = TestDefaults::region; awsConfig.throughputTargetGbps = TestDefaults::throughputTargetGbps; awsConfig.partSize = TestDefaults::partSize; - /* Set the SDK options and initialize the API. */ + // Set the SDK options and initialize the API. Aws::SDKOptions options; Aws::InitAPI(options); @@ -401,7 +407,7 @@ main() TEST(TestListObjects, awsConfig); TEST(TestGetObject, awsConfig); - /* Shutdown the API at end of tests. */ + // Shutdown the API at end of tests. Aws::ShutdownAPI(options); return (TEST_SUCCESS); } diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index ed796a51ed0..addef77d1ec 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "07cee370d83fd1c90f4ecf6781331db020960323" + "commit": "7a7d6bf9ab40cd5635ee960fac1b21edf118a007" } |