Import wiredtiger: 4aa9cea2145f2f6fd05f38841798192cea68336e from branch mongodb-master

ref: 781d874619..4aa9cea214
for: 5.3.0

WT-8662       Integrate S3 storage source extension logging with WiredTiger verbose messaging

6 files changed, 147 insertions, 23 deletions

diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok
index d3cf93e200e..14ba3fc093e 100644
--- a/src/third_party/wiredtiger/dist/s_string.ok
+++ b/src/third_party/wiredtiger/dist/s_string.ok
@@ -387,6 +387,7 @@ Resize
 RocksDB
 Rogerio
 Runtime
+SDK
 SDKOptions
 SIMD
 SLIST
diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.cpp b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.cpp
new file mode 100644
index 00000000000..4835adabbf5
--- /dev/null
+++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.cpp
@@ -0,0 +1,67 @@
+#include <aws/core/Aws.h>
+#include "s3_log_system.h"
+#include <cstdarg>
+
+S3LogSystem::S3LogSystem(WT_EXTENSION_API *wtApi, uint32_t wtVerbosityLevel)
+{
+    // If the verbosity level is out of range it will default to AWS SDK Error level.
+    if (verbosityMapping.find(wtVerbosityLevel) != verbosityMapping.end()) {
+        awsLogLevel = verbosityMapping.at(wtVerbosityLevel);
+    } else {
+        awsLogLevel = Aws::Utils::Logging::LogLevel::Error;
+    }
+    this->wtApi = wtApi;
+    this->wtVerbosityLevel = wtVerbosityLevel;
+}
+
+void
+S3LogSystem::Log(Aws::Utils::Logging::LogLevel logLevel, const char *tag, const char *format, ...)
+{
+    std::stringstream ss;
+    std::va_list args;
+    va_list tmpArgs;
+    va_start(args, format);
+    char *outputBuff = nullptr;
+    int requiredLength;
+
+#ifdef _WIN32
+    requiredLength = _vscprintf(formatStr, tmpArgs) + 1;
+    outputBuff = (char *)malloc(requiredLength);
+    vsnprintf_s(outputBuff, requiredLength, _TRUNCATE, formatStr, args);
+#else
+    requiredLength = vsnprintf(nullptr, 0, format, tmpArgs) + 1;
+    outputBuff = (char *)malloc(requiredLength);
+    vsnprintf(outputBuff, requiredLength, format, args);
+#endif
+    va_end(tmpArgs);
+    ss << outputBuff << std::endl;
+    free(outputBuff);
+    LogAwsMessage(tag, ss.str());
+    va_end(args);
+}
+
+void
+S3LogSystem::LogStream(
+  Aws::Utils::Logging::LogLevel logLevel, const char *tag, const Aws::OStringStream &messageStream)
+{
+    LogAwsMessage(tag, messageStream.rdbuf()->str().c_str());
+}
+
+void
+S3LogSystem::LogAwsMessage(const char *tag, const std::string &message) const
+{
+    wtApi->err_printf(wtApi, NULL, "%s : %s", tag, message.c_str());
+}
+
+void
+S3LogSystem::LogVerboseMessage(int32_t verbosityLevel, const std::string &message)
+{
+    if (verbosityLevel <= wtVerbosityLevel)
+        wtApi->err_printf(wtApi, NULL, "%s", message.c_str());
+}
+
+void
+S3LogSystem::Flush()
+{
+    return;
+}
diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.h b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.h
new file mode 100644
index 00000000000..36dabc69fc3
--- /dev/null
+++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_log_system.h
@@ -0,0 +1,36 @@
+#include <wiredtiger.h>
+#include <wiredtiger_ext.h>
+
+#include <aws/core/utils/logging/LogSystemInterface.h>
+#include <aws/core/utils/logging/LogLevel.h>
+
+#include <atomic>
+
+class S3LogSystem : public Aws::Utils::Logging::LogSystemInterface {
+
+    public:
+    S3LogSystem(WT_EXTENSION_API *wtApi, uint32_t wtVerbosityLevel);
+    Aws::Utils::Logging::LogLevel
+    GetLogLevel(void) const override
+    {
+        return awsLogLevel;
+    }
+    void Log(
+      Aws::Utils::Logging::LogLevel logLevel, const char *tag, const char *format, ...) override;
+    void LogStream(Aws::Utils::Logging::LogLevel logLevel, const char *tag,
+      const Aws::OStringStream &messageStream) override;
+    void Flush() override;
+
+    private:
+    void LogAwsMessage(const char *tag, const std::string &message) const;
+    void LogVerboseMessage(int32_t verbosityLevel, const std::string &message);
+    std::atomic<Aws::Utils::Logging::LogLevel> awsLogLevel;
+    WT_EXTENSION_API *wtApi;
+    int32_t wtVerbosityLevel;
+};
+// Mapping the desired WiredTiger extension verbosity level to a rough equivalent AWS
+// SDK verbosity level.
+static const std::map<int32_t, Aws::Utils::Logging::LogLevel> verbosityMapping = {
+  {-3, Aws::Utils::Logging::LogLevel::Error}, {-2, Aws::Utils::Logging::LogLevel::Warn},
+  {-1, Aws::Utils::Logging::LogLevel::Info}, {0, Aws::Utils::Logging::LogLevel::Info},
+  {1, Aws::Utils::Logging::LogLevel::Debug}};
diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp
index bab8c5ff818..2b6a756c261 100644
--- a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp
+++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp
@@ -29,8 +29,11 @@
 #include <wiredtiger.h>
 #include <wiredtiger_ext.h>
 
-#include <aws/core/Aws.h>
 #include "s3_connection.h"
+#include "s3_log_system.h"
+#include <aws/core/Aws.h>
+#include <aws/core/utils/memory/stl/AWSString.h>
+#include <aws/core/utils/logging/AWSLogging.h>
 
 #define UNUSED(x) (void)(x)
 
@@ -38,13 +41,15 @@
 typedef struct {
     WT_STORAGE_SOURCE storageSource; /* Must come first */
     WT_EXTENSION_API *wtApi;         /* Extension API */
+    int32_t verbose;
 } S3_STORAGE;
 
 typedef struct {
     /* Must come first - this is the interface for the file system we are implementing. */
     WT_FILE_SYSTEM fileSystem;
-    S3_STORAGE *s3Storage;
-    S3Connection *conn;
+    S3_STORAGE *storage;
+    S3Connection *connection;
+    S3LogSystem *log;
 } S3_FILE_SYSTEM;
 
 /* Configuration variables for connecting to S3CrtClient. */
@@ -69,7 +74,9 @@ S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, con
   const char *authToken, const char *config, WT_FILE_SYSTEM **fileSystem)
 {
     S3_FILE_SYSTEM *fs;
-    int ret;
+    S3_STORAGE *s3;
+    WT_CONFIG_ITEM v;
+    s3 = (S3_STORAGE *)storageSource;
 
     /* Mark parameters as unused for now, until implemented. */
     UNUSED(session);
@@ -77,18 +84,21 @@ S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, con
     UNUSED(authToken);
     UNUSED(config);
 
-    Aws::S3Crt::ClientConfiguration aws_config;
-    aws_config.region = region;
-    aws_config.throughputTargetGbps = throughputTargetGbps;
-    aws_config.partSize = partSize;
+    Aws::S3Crt::ClientConfiguration awsConfig;
+    awsConfig.region = region;
+    awsConfig.throughputTargetGbps = throughputTargetGbps;
+    awsConfig.partSize = partSize;
+
+    Aws::Utils::Logging::InitializeAWSLogging(
+      Aws::MakeShared<S3LogSystem>("storage", s3->wtApi, s3->verbose));
 
     if ((fs = (S3_FILE_SYSTEM *)calloc(1, sizeof(S3_FILE_SYSTEM))) == NULL)
         return (errno);
 
-    fs->s3Storage = (S3_STORAGE *)storageSource;
+    fs->storage = (S3_STORAGE *)storageSource;
 
     /* New can fail; will deal with this later. */
-    fs->conn = new S3Connection(aws_config);
+    fs->connection = new S3Connection(awsConfig);
 
     fs->fileSystem.terminate = S3FileSystemTerminate;
 
@@ -96,7 +106,7 @@ S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, con
     {
         /* List S3 buckets. */
         std::vector<std::string> buckets;
-        if (fs->conn->ListBuckets(buckets)) {
+        if (fs->connection->ListBuckets(buckets)) {
             std::cout << "All buckets under my account:" << std::endl;
             for (const std::string &bucket : buckets) {
                 std::cout << "  * " << bucket << std::endl;
@@ -110,7 +120,7 @@ S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, con
 
             /* List objects. */
             std::vector<std::string> bucketObjects;
-            if (fs->conn->ListObjects(firstBucket, bucketObjects)) {
+            if (fs->connection->ListObjects(firstBucket, bucketObjects)) {
                 std::cout << "Objects in bucket '" << firstBucket << "':" << std::endl;
                 if (!bucketObjects.empty()) {
                     for (const auto &object : bucketObjects) {
@@ -123,11 +133,11 @@ S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, con
             }
 
             /* Put object. */
-            fs->conn->PutObject(firstBucket, "WiredTiger.turtle", "WiredTiger.turtle");
+            fs->connection->PutObject(firstBucket, "WiredTiger.turtle", "WiredTiger.turtle");
 
             /* List objects again. */
             bucketObjects.clear();
-            if (fs->conn->ListObjects(firstBucket, bucketObjects)) {
+            if (fs->connection->ListObjects(firstBucket, bucketObjects)) {
                 std::cout << "Objects in bucket '" << firstBucket << "':" << std::endl;
                 if (!bucketObjects.empty()) {
                     for (const auto &object : bucketObjects) {
@@ -140,11 +150,11 @@ S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, con
             }
 
             /* Delete object. */
-            fs->conn->DeleteObject(firstBucket, "WiredTiger.turtle");
+            fs->connection->DeleteObject(firstBucket, "WiredTiger.turtle");
 
             /* List objects again. */
             bucketObjects.clear();
-            if (fs->conn->ListObjects(firstBucket, bucketObjects)) {
+            if (fs->connection->ListObjects(firstBucket, bucketObjects)) {
                 std::cout << "Objects in bucket '" << firstBucket << "':" << std::endl;
                 if (!bucketObjects.empty()) {
                     for (const auto &object : bucketObjects) {
@@ -176,7 +186,7 @@ S3FileSystemTerminate(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session)
     UNUSED(session); /* unused */
 
     fs = (S3_FILE_SYSTEM *)fileSystem;
-    delete (fs->conn);
+    delete (fs->connection);
     free(fs);
 
     return (0);
@@ -218,13 +228,25 @@ int
 wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config)
 {
     S3_STORAGE *s3;
-    int ret;
+    S3_FILE_SYSTEM *fs;
+    WT_CONFIG_ITEM v;
 
     if ((s3 = (S3_STORAGE *)calloc(1, sizeof(S3_STORAGE))) == NULL)
         return (errno);
 
     s3->wtApi = connection->get_extension_api(connection);
-    UNUSED(config);
+
+    int ret = s3->wtApi->config_get(s3->wtApi, NULL, config, "verbose", &v);
+
+    // If a verbose level is not found, it will set the level to -3 (Error).
+    if (ret == 0 && v.val >= -3 && v.val <= 1)
+        s3->verbose = v.val;
+    else if (ret == WT_NOTFOUND)
+        s3->verbose = -3;
+    else {
+        free(s3);
+        return (ret != 0 ? ret : EINVAL);
+    }
 
     Aws::InitAPI(options);
 
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index e3dc839508c..5f20e2810fc 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "781d8746198fdce53f2dbea8485635f8965595eb"
+    "commit": "4aa9cea2145f2f6fd05f38841798192cea68336e"
 }
diff --git a/src/third_party/wiredtiger/test/suite/test_s3_store01.py b/src/third_party/wiredtiger/test/suite/test_s3_store01.py
index 4fda30914a2..c0b78df2136 100644
--- a/src/third_party/wiredtiger/test/suite/test_s3_store01.py
+++ b/src/third_party/wiredtiger/test/suite/test_s3_store01.py
@@ -35,7 +35,7 @@ class test_s3_store01(wttest.WiredTigerTestCase):
     # Load the s3 store extension, skip the test if missing.
     def conn_extensions(self, extlist):
         extlist.skip_if_missing = True
-        extlist.extension('storage_sources', 's3_store')
+        extlist.extension('storage_sources', 's3_store=(config=\"(verbose=-3)\")')
 
     def get_s3_storage_source(self):
         return self.conn.get_storage_source('s3_store')
@@ -49,7 +49,5 @@ class test_s3_store01(wttest.WiredTigerTestCase):
         fs = s3_store.ss_customize_file_system(session, "./objects", "Secret", None)
         fs.terminate(session)
 
-        s3_store.terminate(session)
-
 if __name__ == '__main__':
     wttest.run()