Import wiredtiger: fb7568b423b3a085bc0d592270dfea8d1bbcc687 from branch mongodb-master

ref: 303a8d241d..fb7568b423
for: 5.3.0

WT-8761       Write statistics for the S3 extension

3 files changed, 100 insertions, 40 deletions

diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp
index eed71eb99a1..a2a0842c927 100644
--- a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp
+++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp
@@ -45,6 +45,19 @@
 struct S3_FILE_HANDLE;
 struct S3_FILE_SYSTEM;
 
+/* Statistics to be collected for the S3 storage. */
+struct S3_STATISTICS {
+    /* Operations using AWS SDK. */
+    uint64_t listObjectsCount;  /* Number of S3 list objects requests */
+    uint64_t putObjectCount;    /* Number of S3 put object requests */
+    uint64_t getObjectCount;    /* Number of S3 get object requests */
+    uint64_t objectExistsCount; /* Number of S3 object exists requests */
+
+    /* Operations using WiredTiger's native file handle operations. */
+    uint64_t fhOps;     /* Number of non read/write file handle operations */
+    uint64_t fhReadOps; /* Number of file handle read operations */
+};
+
 /* S3 storage source structure. */
 struct S3_STORAGE {
     WT_STORAGE_SOURCE storageSource; /* Must come first */
@@ -55,8 +68,10 @@ struct S3_STORAGE {
     std::mutex fhMutex;                 /* Protect the file handle list*/
     std::list<S3_FILE_HANDLE *> fhList; /* List of open file handles */
 
-    uint32_t referenceCount; /* Number of references to this storge source */
+    uint32_t referenceCount; /* Number of references to this storage source */
     int32_t verbose;
+
+    S3_STATISTICS statistics;
 };
 
 struct S3_FILE_SYSTEM {
@@ -115,6 +130,8 @@ static int S3ObjectListAdd(
 static int S3ObjectListSingle(
   WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, char ***, uint32_t *);
 static int S3ObjectListFree(WT_FILE_SYSTEM *, WT_SESSION *, char **, uint32_t);
+static void S3ShowStatistics(const S3_STATISTICS &);
+
 static int S3FileClose(WT_FILE_HANDLE *, WT_SESSION *);
 /*
  * S3Path --
@@ -143,16 +160,19 @@ S3Path(const std::string &dir, const std::string &name)
 static int
 S3Exist(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *name, bool *exist)
 {
-    S3_STORAGE *s3;
-    s3 = FS2S3(fileSystem);
     S3_FILE_SYSTEM *fs = (S3_FILE_SYSTEM *)fileSystem;
+    int ret = 0;
 
-    /* It's not in the cache, try the S3 bucket. */
+    /* Check if file exists in the cache. */
     *exist = S3CacheExists(fileSystem, name);
-    if (!*exist)
-        return (fs->connection->ObjectExists(name, *exist));
+    if (*exist)
+        return (ret);
 
-    return (0);
+    /* It's not in the cache, try the S3 bucket. */
+    FS2S3(fileSystem)->statistics.objectExistsCount++;
+    if ((ret = fs->connection->ObjectExists(name, *exist)) != 0)
+        std::cerr << "S3Exist: ObjectExists request to S3 failed." << std::endl;
+    return (ret);
 }
 
 /*
@@ -222,7 +242,6 @@ S3FileClose(WT_FILE_HANDLE *fileHandle, WT_SESSION *session)
     S3_FILE_HANDLE *s3FileHandle = (S3_FILE_HANDLE *)fileHandle;
     S3_STORAGE *storage = s3FileHandle->storage;
     WT_FILE_HANDLE *wtFileHandle = s3FileHandle->wtFileHandle;
-
     /*
      * We require exclusive access to the list of file handles when removing file handles. The
      * lock_guard will be unlocked automatically once the scope is exited.
@@ -232,12 +251,8 @@ S3FileClose(WT_FILE_HANDLE *fileHandle, WT_SESSION *session)
         storage->fhList.remove(s3FileHandle);
     }
     if (wtFileHandle != NULL) {
+        storage->statistics.fhOps++;
         ret = wtFileHandle->close(wtFileHandle, session);
-        if (ret != 0) {
-            free(s3FileHandle->iface.name);
-            free(s3FileHandle);
-            return (ret);
-        }
     }
 
     free(s3FileHandle->iface.name);
@@ -255,9 +270,9 @@ S3Open(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *name,
   WT_FS_OPEN_FILE_TYPE fileType, uint32_t flags, WT_FILE_HANDLE **fileHandlePtr)
 {
     S3_FILE_HANDLE *s3FileHandle;
-    S3_FILE_SYSTEM *s3Fs = (S3_FILE_SYSTEM *)fileSystem;
-    S3_STORAGE *s3 = s3Fs->storage;
-    WT_FILE_SYSTEM *wtFileSystem = s3Fs->wtFileSystem;
+    S3_FILE_SYSTEM *fs = (S3_FILE_SYSTEM *)fileSystem;
+    S3_STORAGE *s3 = fs->storage;
+    WT_FILE_SYSTEM *wtFileSystem = fs->wtFileSystem;
     WT_FILE_HANDLE *wtFileHandle;
     int ret;
 
@@ -282,10 +297,11 @@ S3Open(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *name,
         return (ENOMEM);
 
     /* Make a copy from S3 if the file is not in the cache. */
-    const std::string cachePath = S3Path(s3Fs->cacheDir, name);
+    const std::string cachePath = S3Path(fs->cacheDir, name);
     if (!LocalFileExists(cachePath)) {
-        if ((ret = s3Fs->connection->GetObject(name, cachePath)) != 0) {
-            std::cerr << "ss_open_object: GetObject request to s3 failed" << std::endl;
+        s3->statistics.getObjectCount++;
+        if ((ret = fs->connection->GetObject(name, cachePath)) != 0) {
+            std::cerr << "ss_open_object: GetObject request to S3 failed." << std::endl;
             return (ret);
         }
     }
@@ -294,7 +310,7 @@ S3Open(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *name,
     ret = wtFileSystem->fs_open_file(
       wtFileSystem, session, cachePath.c_str(), fileType, flags, &wtFileHandle);
     if (ret != 0) {
-        std::cerr << "ss_open_objet: open " << name << std::endl;
+        std::cerr << "ss_open_object: fs_open_file failed." << name << std::endl;
         return (ret);
     }
 
@@ -342,10 +358,16 @@ S3Open(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *name,
  *     Read a file using WiredTiger's native file handle read.
  */
 static int
-S3FileRead(WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t offset, size_t len, void *buf)
+S3FileRead(WT_FILE_HANDLE *fileHandle, WT_SESSION *session, wt_off_t offset, size_t len, void *buf)
 {
-    WT_FILE_HANDLE *wtFileHandle = ((S3_FILE_HANDLE *)file_handle)->wtFileHandle;
-    return (wtFileHandle->fh_read(wtFileHandle, session, offset, len, buf));
+    S3_FILE_HANDLE *s3FileHandle = (S3_FILE_HANDLE *)fileHandle;
+    S3_STORAGE *storage = s3FileHandle->storage;
+    WT_FILE_HANDLE *wtFileHandle = s3FileHandle->wtFileHandle;
+    int ret;
+    storage->statistics.fhReadOps++;
+    if ((ret = wtFileHandle->fh_read(wtFileHandle, session, offset, len, buf)) != 0)
+        std::cerr << "S3FileRead: fh_read failed." << std::endl;
+    return (ret);
 }
 
 /*
@@ -477,6 +499,8 @@ S3ObjectList(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *direct
   const char *prefix, char ***objectList, uint32_t *count)
 {
     S3_FILE_SYSTEM *fs = (S3_FILE_SYSTEM *)fileSystem;
+    S3_STORAGE *s3 = FS2S3(fileSystem);
+
     std::vector<std::string> objects;
     std::string completePrefix;
 
@@ -490,11 +514,14 @@ S3ObjectList(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *direct
         completePrefix += prefix;
 
     int ret;
-    if (ret = fs->connection->ListObjects(completePrefix, objects) != 0)
+    s3->statistics.listObjectsCount++;
+    if ((ret = fs->connection->ListObjects(completePrefix, objects)) != 0) {
+        std::cerr << "S3ObjectList: ListObjects request to S3 failed." << std::endl;
         return (ret);
+    }
     *count = objects.size();
 
-    S3ObjectListAdd(fs->storage, objectList, objects, *count);
+    S3ObjectListAdd(s3, objectList, objects, *count);
 
     return (ret);
 }
@@ -508,6 +535,8 @@ S3ObjectListSingle(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *
   const char *prefix, char ***objectList, uint32_t *count)
 {
     S3_FILE_SYSTEM *fs = (S3_FILE_SYSTEM *)fileSystem;
+    S3_STORAGE *s3 = FS2S3(fileSystem);
+
     std::vector<std::string> objects;
     std::string completePrefix;
 
@@ -521,11 +550,15 @@ S3ObjectListSingle(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session, const char *
         completePrefix += prefix;
 
     int ret;
-    if (ret = fs->connection->ListObjects(completePrefix, objects, 1, true) != 0)
+    s3->statistics.listObjectsCount++;
+    if ((ret = fs->connection->ListObjects(completePrefix, objects, 1, true)) != 0) {
+        std::cerr << "S3ObjectListSingle: ListObjects request to S3 failed." << std::endl;
         return (ret);
+    }
+
     *count = objects.size();
 
-    S3ObjectListAdd(fs->storage, objectList, objects, *count);
+    S3ObjectListAdd(s3, objectList, objects, *count);
 
     return (ret);
 }
@@ -617,6 +650,9 @@ S3Terminate(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session)
         S3FileSystemTerminate(&fs->fileSystem, session);
     }
 
+    /* Log collected statistics on termination. */
+    S3ShowStatistics(s3->statistics);
+
     Aws::Utils::Logging::ShutdownAWSLogging();
     Aws::ShutdownAPI(options);
 
@@ -633,7 +669,12 @@ S3Flush(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, WT_FILE_SYSTEM *f
   const char *source, const char *object, const char *config)
 {
     S3_FILE_SYSTEM *fs = (S3_FILE_SYSTEM *)fileSystem;
-    return (fs->connection->PutObject(object, source));
+
+    int ret;
+    FS2S3(fileSystem)->statistics.putObjectCount++;
+    if (ret = (fs->connection->PutObject(object, source)) != 0)
+        std::cerr << "S3Flush: PutObject request to S3 failed." << std::endl;
+    return (ret);
 }
 
 /*
@@ -659,6 +700,22 @@ S3FlushFinish(WT_STORAGE_SOURCE *storage, WT_SESSION *session, WT_FILE_SYSTEM *f
 }
 
 /*
+ * S3ShowStatistics --
+ *     Log collected statistics.
+ */
+static void
+S3ShowStatistics(const S3_STATISTICS &statistics)
+{
+    std::cout << "S3 list objects count: " << statistics.listObjectsCount << std::endl;
+    std::cout << "S3 put object count: " << statistics.putObjectCount << std::endl;
+    std::cout << "S3 get object count: " << statistics.putObjectCount << std::endl;
+    std::cout << "S3 object exists count: " << statistics.objectExistsCount << std::endl;
+
+    std::cout << "Non read/write file handle operations: " << statistics.fhOps << std::endl;
+    std::cout << "File handle read operations: " << statistics.fhReadOps << std::endl;
+}
+
+/*
  * wiredtiger_extension_init --
  *     A S3 storage source library.
  */
@@ -685,6 +742,9 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config)
         return (ret != 0 ? ret : EINVAL);
     }
 
+    /* Set up statistics. */
+    s3->statistics = {0};
+
     /* Create a logger for this storage source, and then initialize the AWS SDK. */
     Aws::Utils::Logging::InitializeAWSLogging(
       Aws::MakeShared<S3LogSystem>("storage", s3->wtApi, s3->verbose));
diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/test/test_s3_connection.cpp b/src/third_party/wiredtiger/ext/storage_sources/s3_store/test/test_s3_connection.cpp
index 1b39f5c44ef..b53783038ed 100644
--- a/src/third_party/wiredtiger/ext/storage_sources/s3_store/test/test_s3_connection.cpp
+++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/test/test_s3_connection.cpp
@@ -81,7 +81,7 @@ CleanupTestListObjects(S3Connection &conn, const int totalObjects, const std::st
     /* Delete objects and file at end of test. */
     int ret = 0;
     for (int i = 0; i < totalObjects; i++) {
-        if (ret = conn.DeleteObject(prefix + std::to_string(i) + ".txt") != 0)
+        if ((ret = conn.DeleteObject(prefix + std::to_string(i) + ".txt")) != 0)
             std::cerr << "Error in CleanupTestListBuckets: failed to remove "
                       << TestDefaults::objPrefix + prefix << std::to_string(i) << ".txt from "
                       << TestDefaults::bucketName << std::endl;
@@ -117,13 +117,13 @@ TestListObjects(const Aws::S3Crt::ClientConfiguration &config)
 
     int ret;
     /* No matching objects. */
-    if (ret = conn.ListObjects(prefix, objects) != 0)
+    if ((ret = conn.ListObjects(prefix, objects)) != 0)
         return (ret);
     if (objects.size() != expectedResult)
         return (TEST_FAILURE);
 
     /* No matching objects with listSingle. */
-    if (ret = conn.ListObjects(prefix, objects, batchSize, listSingle) != 0)
+    if ((ret = conn.ListObjects(prefix, objects, batchSize, listSingle)) != 0)
         return (ret);
     if (objects.size() != expectedResult)
         return (TEST_FAILURE);
@@ -136,7 +136,7 @@ TestListObjects(const Aws::S3Crt::ClientConfiguration &config)
 
     /* Put objects to prepare for test. */
     for (int i = 0; i < totalObjects; i++) {
-        if (ret = conn.PutObject(prefix + std::to_string(i) + ".txt", fileName) != 0) {
+        if ((ret = conn.PutObject(prefix + std::to_string(i) + ".txt", fileName)) != 0) {
             CleanupTestListObjects(conn, i, prefix, fileName);
             return (ret);
         }
@@ -144,7 +144,7 @@ TestListObjects(const Aws::S3Crt::ClientConfiguration &config)
 
     /* List all objects. */
     expectedResult = totalObjects;
-    if (ret = conn.ListObjects(prefix, objects) != 0) {
+    if ((ret = conn.ListObjects(prefix, objects)) != 0) {
         CleanupTestListObjects(conn, totalObjects, prefix, fileName);
         return (ret);
     }
@@ -156,7 +156,7 @@ TestListObjects(const Aws::S3Crt::ClientConfiguration &config)
     /* List single. */
     objects.clear();
     expectedResult = 1;
-    if (ret = conn.ListObjects(prefix, objects, batchSize, listSingle) != 0) {
+    if ((ret = conn.ListObjects(prefix, objects, batchSize, listSingle)) != 0) {
         CleanupTestListObjects(conn, totalObjects, prefix, fileName);
         return (ret);
     }
@@ -168,7 +168,7 @@ TestListObjects(const Aws::S3Crt::ClientConfiguration &config)
     /* Expected number of matches with test_list_objects_1 prefix. */
     objects.clear();
     expectedResult = 11;
-    if (ret = conn.ListObjects(prefix + "1", objects) != 0) {
+    if ((ret = conn.ListObjects(prefix + "1", objects)) != 0) {
         CleanupTestListObjects(conn, totalObjects, prefix, fileName);
         return (ret);
     }
@@ -181,7 +181,7 @@ TestListObjects(const Aws::S3Crt::ClientConfiguration &config)
     objects.clear();
     batchSize = 5;
     expectedResult = totalObjects;
-    if (ret = conn.ListObjects(prefix, objects, batchSize) != 0) {
+    if ((ret = conn.ListObjects(prefix, objects, batchSize)) != 0) {
         CleanupTestListObjects(conn, totalObjects, prefix, fileName);
         return (ret);
     }
@@ -193,7 +193,7 @@ TestListObjects(const Aws::S3Crt::ClientConfiguration &config)
     /* ListSingle with 8 objects per AWS request. */
     objects.clear();
     expectedResult = 1;
-    if (ret = conn.ListObjects(prefix, objects, batchSize, listSingle) != 0) {
+    if ((ret = conn.ListObjects(prefix, objects, batchSize, listSingle)) != 0) {
         CleanupTestListObjects(conn, totalObjects, prefix, fileName);
         return (ret);
     }
@@ -206,7 +206,7 @@ TestListObjects(const Aws::S3Crt::ClientConfiguration &config)
     objects.clear();
     batchSize = 8;
     expectedResult = totalObjects;
-    if (ret = conn.ListObjects(prefix, objects, batchSize) != 0) {
+    if ((ret = conn.ListObjects(prefix, objects, batchSize)) != 0) {
         CleanupTestListObjects(conn, totalObjects, prefix, fileName);
         return (ret);
     }
@@ -218,7 +218,7 @@ TestListObjects(const Aws::S3Crt::ClientConfiguration &config)
     /* ListSingle with 8 objects per AWS request. */
     objects.clear();
     expectedResult = 1;
-    if (ret = conn.ListObjects(prefix, objects, batchSize, listSingle) != 0) {
+    if ((ret = conn.ListObjects(prefix, objects, batchSize, listSingle)) != 0) {
         CleanupTestListObjects(conn, totalObjects, prefix, fileName);
         return (ret);
     }
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 220bc8a08f2..da7acc80af5 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "303a8d241da752b7c46a5ce84e5537fc360a65a8"
+    "commit": "fb7568b423b3a085bc0d592270dfea8d1bbcc687"
 }