diff options
6 files changed, 147 insertions, 23 deletions
diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index d3cf93e200e..14ba3fc093e 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -387,6 +387,7 @@ Resize RocksDB Rogerio Runtime +SDK SDKOptions SIMD SLIST diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.cpp b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.cpp new file mode 100644 index 00000000000..4835adabbf5 --- /dev/null +++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.cpp @@ -0,0 +1,67 @@ +#include <aws/core/Aws.h> +#include "s3_log_system.h" +#include <cstdarg> + +S3LogSystem::S3LogSystem(WT_EXTENSION_API *wtApi, uint32_t wtVerbosityLevel) +{ + // If the verbosity level is out of range it will default to AWS SDK Error level. + if (verbosityMapping.find(wtVerbosityLevel) != verbosityMapping.end()) { + awsLogLevel = verbosityMapping.at(wtVerbosityLevel); + } else { + awsLogLevel = Aws::Utils::Logging::LogLevel::Error; + } + this->wtApi = wtApi; + this->wtVerbosityLevel = wtVerbosityLevel; +} + +void +S3LogSystem::Log(Aws::Utils::Logging::LogLevel logLevel, const char *tag, const char *format, ...) +{ + std::stringstream ss; + std::va_list args; + va_list tmpArgs; + va_start(args, format); + char *outputBuff = nullptr; + int requiredLength; + +#ifdef _WIN32 + requiredLength = _vscprintf(formatStr, tmpArgs) + 1; + outputBuff = (char *)malloc(requiredLength); + vsnprintf_s(outputBuff, requiredLength, _TRUNCATE, formatStr, args); +#else + requiredLength = vsnprintf(nullptr, 0, format, tmpArgs) + 1; + outputBuff = (char *)malloc(requiredLength); + vsnprintf(outputBuff, requiredLength, format, args); +#endif + va_end(tmpArgs); + ss << outputBuff << std::endl; + free(outputBuff); + LogAwsMessage(tag, ss.str()); + va_end(args); +} + +void +S3LogSystem::LogStream( + Aws::Utils::Logging::LogLevel logLevel, const char *tag, const Aws::OStringStream &messageStream) +{ + LogAwsMessage(tag, messageStream.rdbuf()->str().c_str()); +} + +void +S3LogSystem::LogAwsMessage(const char *tag, const std::string &message) const +{ + wtApi->err_printf(wtApi, NULL, "%s : %s", tag, message.c_str()); +} + +void +S3LogSystem::LogVerboseMessage(int32_t verbosityLevel, const std::string &message) +{ + if (verbosityLevel <= wtVerbosityLevel) + wtApi->err_printf(wtApi, NULL, "%s", message.c_str()); +} + +void +S3LogSystem::Flush() +{ + return; +} diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.h b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.h new file mode 100644 index 00000000000..36dabc69fc3 --- /dev/null +++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.h @@ -0,0 +1,36 @@ +#include <wiredtiger.h> +#include <wiredtiger_ext.h> + +#include <aws/core/utils/logging/LogSystemInterface.h> +#include <aws/core/utils/logging/LogLevel.h> + +#include <atomic> + +class S3LogSystem : public Aws::Utils::Logging::LogSystemInterface { + + public: + S3LogSystem(WT_EXTENSION_API *wtApi, uint32_t wtVerbosityLevel); + Aws::Utils::Logging::LogLevel + GetLogLevel(void) const override + { + return awsLogLevel; + } + void Log( + Aws::Utils::Logging::LogLevel logLevel, const char *tag, const char *format, ...) override; + void LogStream(Aws::Utils::Logging::LogLevel logLevel, const char *tag, + const Aws::OStringStream &messageStream) override; + void Flush() override; + + private: + void LogAwsMessage(const char *tag, const std::string &message) const; + void LogVerboseMessage(int32_t verbosityLevel, const std::string &message); + std::atomic<Aws::Utils::Logging::LogLevel> awsLogLevel; + WT_EXTENSION_API *wtApi; + int32_t wtVerbosityLevel; +}; +// Mapping the desired WiredTiger extension verbosity level to a rough equivalent AWS +// SDK verbosity level. +static const std::map<int32_t, Aws::Utils::Logging::LogLevel> verbosityMapping = { + {-3, Aws::Utils::Logging::LogLevel::Error}, {-2, Aws::Utils::Logging::LogLevel::Warn}, + {-1, Aws::Utils::Logging::LogLevel::Info}, {0, Aws::Utils::Logging::LogLevel::Info}, + {1, Aws::Utils::Logging::LogLevel::Debug}}; diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp index bab8c5ff818..2b6a756c261 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp +++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp @@ -29,8 +29,11 @@ #include <wiredtiger.h> #include <wiredtiger_ext.h> -#include <aws/core/Aws.h> #include "s3_connection.h" +#include "s3_log_system.h" +#include <aws/core/Aws.h> +#include <aws/core/utils/memory/stl/AWSString.h> +#include <aws/core/utils/logging/AWSLogging.h> #define UNUSED(x) (void)(x) @@ -38,13 +41,15 @@ typedef struct { WT_STORAGE_SOURCE storageSource; /* Must come first */ WT_EXTENSION_API *wtApi; /* Extension API */ + int32_t verbose; } S3_STORAGE; typedef struct { /* Must come first - this is the interface for the file system we are implementing. */ WT_FILE_SYSTEM fileSystem; - S3_STORAGE *s3Storage; - S3Connection *conn; + S3_STORAGE *storage; + S3Connection *connection; + S3LogSystem *log; } S3_FILE_SYSTEM; /* Configuration variables for connecting to S3CrtClient. */ @@ -69,7 +74,9 @@ S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, con const char *authToken, const char *config, WT_FILE_SYSTEM **fileSystem) { S3_FILE_SYSTEM *fs; - int ret; + S3_STORAGE *s3; + WT_CONFIG_ITEM v; + s3 = (S3_STORAGE *)storageSource; /* Mark parameters as unused for now, until implemented. */ UNUSED(session); @@ -77,18 +84,21 @@ S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, con UNUSED(authToken); UNUSED(config); - Aws::S3Crt::ClientConfiguration aws_config; - aws_config.region = region; - aws_config.throughputTargetGbps = throughputTargetGbps; - aws_config.partSize = partSize; + Aws::S3Crt::ClientConfiguration awsConfig; + awsConfig.region = region; + awsConfig.throughputTargetGbps = throughputTargetGbps; + awsConfig.partSize = partSize; + + Aws::Utils::Logging::InitializeAWSLogging( + Aws::MakeShared<S3LogSystem>("storage", s3->wtApi, s3->verbose)); if ((fs = (S3_FILE_SYSTEM *)calloc(1, sizeof(S3_FILE_SYSTEM))) == NULL) return (errno); - fs->s3Storage = (S3_STORAGE *)storageSource; + fs->storage = (S3_STORAGE *)storageSource; /* New can fail; will deal with this later. */ - fs->conn = new S3Connection(aws_config); + fs->connection = new S3Connection(awsConfig); fs->fileSystem.terminate = S3FileSystemTerminate; @@ -96,7 +106,7 @@ S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, con { /* List S3 buckets. */ std::vector<std::string> buckets; - if (fs->conn->ListBuckets(buckets)) { + if (fs->connection->ListBuckets(buckets)) { std::cout << "All buckets under my account:" << std::endl; for (const std::string &bucket : buckets) { std::cout << " * " << bucket << std::endl; @@ -110,7 +120,7 @@ S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, con /* List objects. */ std::vector<std::string> bucketObjects; - if (fs->conn->ListObjects(firstBucket, bucketObjects)) { + if (fs->connection->ListObjects(firstBucket, bucketObjects)) { std::cout << "Objects in bucket '" << firstBucket << "':" << std::endl; if (!bucketObjects.empty()) { for (const auto &object : bucketObjects) { @@ -123,11 +133,11 @@ S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, con } /* Put object. */ - fs->conn->PutObject(firstBucket, "WiredTiger.turtle", "WiredTiger.turtle"); + fs->connection->PutObject(firstBucket, "WiredTiger.turtle", "WiredTiger.turtle"); /* List objects again. */ bucketObjects.clear(); - if (fs->conn->ListObjects(firstBucket, bucketObjects)) { + if (fs->connection->ListObjects(firstBucket, bucketObjects)) { std::cout << "Objects in bucket '" << firstBucket << "':" << std::endl; if (!bucketObjects.empty()) { for (const auto &object : bucketObjects) { @@ -140,11 +150,11 @@ S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, con } /* Delete object. */ - fs->conn->DeleteObject(firstBucket, "WiredTiger.turtle"); + fs->connection->DeleteObject(firstBucket, "WiredTiger.turtle"); /* List objects again. */ bucketObjects.clear(); - if (fs->conn->ListObjects(firstBucket, bucketObjects)) { + if (fs->connection->ListObjects(firstBucket, bucketObjects)) { std::cout << "Objects in bucket '" << firstBucket << "':" << std::endl; if (!bucketObjects.empty()) { for (const auto &object : bucketObjects) { @@ -176,7 +186,7 @@ S3FileSystemTerminate(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session) UNUSED(session); /* unused */ fs = (S3_FILE_SYSTEM *)fileSystem; - delete (fs->conn); + delete (fs->connection); free(fs); return (0); @@ -218,13 +228,25 @@ int wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) { S3_STORAGE *s3; - int ret; + S3_FILE_SYSTEM *fs; + WT_CONFIG_ITEM v; if ((s3 = (S3_STORAGE *)calloc(1, sizeof(S3_STORAGE))) == NULL) return (errno); s3->wtApi = connection->get_extension_api(connection); - UNUSED(config); + + int ret = s3->wtApi->config_get(s3->wtApi, NULL, config, "verbose", &v); + + // If a verbose level is not found, it will set the level to -3 (Error). + if (ret == 0 && v.val >= -3 && v.val <= 1) + s3->verbose = v.val; + else if (ret == WT_NOTFOUND) + s3->verbose = -3; + else { + free(s3); + return (ret != 0 ? ret : EINVAL); + } Aws::InitAPI(options); diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index e3dc839508c..5f20e2810fc 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "781d8746198fdce53f2dbea8485635f8965595eb" + "commit": "4aa9cea2145f2f6fd05f38841798192cea68336e" } diff --git a/src/third_party/wiredtiger/test/suite/test_s3_store01.py b/src/third_party/wiredtiger/test/suite/test_s3_store01.py index 4fda30914a2..c0b78df2136 100644 --- a/src/third_party/wiredtiger/test/suite/test_s3_store01.py +++ b/src/third_party/wiredtiger/test/suite/test_s3_store01.py @@ -35,7 +35,7 @@ class test_s3_store01(wttest.WiredTigerTestCase): # Load the s3 store extension, skip the test if missing. def conn_extensions(self, extlist): extlist.skip_if_missing = True - extlist.extension('storage_sources', 's3_store') + extlist.extension('storage_sources', 's3_store=(config=\"(verbose=-3)\")') def get_s3_storage_source(self): return self.conn.get_storage_source('s3_store') @@ -49,7 +49,5 @@ class test_s3_store01(wttest.WiredTigerTestCase): fs = s3_store.ss_customize_file_system(session, "./objects", "Secret", None) fs.terminate(session) - s3_store.terminate(session) - if __name__ == '__main__': wttest.run() |