Import wiredtiger: cb132beb37743fd2e6b237ef006371b1ac26bf45 from branch mongodb-master

ref: 6c8c5c49ee..cb132beb37
for: 7.0.0-rc0

WT-10513 gcp unit testing for connection class

2 files changed, 253 insertions, 3 deletions

diff --git a/src/third_party/wiredtiger/ext/storage_sources/gcp_store/test/test_gcp_connection.cpp b/src/third_party/wiredtiger/ext/storage_sources/gcp_store/test/test_gcp_connection.cpp
index e303ab35da2..0fef6e7e730 100644
--- a/src/third_party/wiredtiger/ext/storage_sources/gcp_store/test/test_gcp_connection.cpp
+++ b/src/third_party/wiredtiger/ext/storage_sources/gcp_store/test/test_gcp_connection.cpp
@@ -27,8 +27,258 @@
  */
 
 #define CATCH_CONFIG_MAIN
+#include "gcp_connection.h"
+
 #include <catch2/catch.hpp>
 
-#include "gcp_connection.h"
+namespace gcs = google::cloud::storage;
+using namespace gcs;
+
+static std::string
+create_file(const std::string file_name, const std::string payload)
+{
+    std::ofstream file(file_name);
+    file << payload;
+    file.close();
+    return file_name;
+}
+
+static auto
+upload_file(gcs::Client client, std::string bucket_name, const std::string bucket_prefix,
+  const std::string file_name, const std::string object_name)
+{
+    auto metadata = client.UploadFile("./" + file_name, bucket_name, bucket_prefix + object_name);
+
+    return metadata;
+}
+
+static bool
+file_exists_in_bucket(gcs::Client client, std::string bucket_name, const std::string bucket_prefix,
+  const std::string object_name)
+{
+    auto metadata = client.GetObjectMetadata(bucket_name, bucket_prefix + object_name);
+
+    // Metadata ok implies that the file is present.
+    return metadata.ok();
+}
+
+static int
+num_objects_in_bucket(gcs::Client client, std::string bucket_name, const std::string bucket_prefix)
+{
+    auto objects_iterator = client.ListObjects(bucket_name, gcs::Prefix(bucket_prefix));
+    return std::distance(objects_iterator.begin(), objects_iterator.end());
+}
+
+// Concatenates a random suffix to the prefix being used for the test object keys. Example of
+// generated test prefix: "gcptest/unit/2022-31-01-16-34-10/623843294/".
+static std::string
+generate_test_prefix()
+{
+    std::string prefix = "gcptest/unit/";
+    char time_str[100];
+    std::time_t t = std::time(nullptr);
+
+    REQUIRE(std::strftime(time_str, sizeof(time_str), "%F-%H-%M-%S", std::localtime(&t)) != 0);
+
+    prefix += time_str;
+
+    // Create a random device and use it to generate a random seed to initialize the generator.
+    std::random_device my_random_device;
+    unsigned seed = my_random_device();
+    std::default_random_engine my_random_engine(seed);
+
+    prefix += "/" + std::to_string(my_random_engine()) + "/";
+
+    return prefix;
+}
+
+TEST_CASE("Testing class gcpConnection", "gcp-connection")
+{
+    std::string test_bucket_name = "unit_testing_gcp";
+
+    // Set up the test environment.
+    std::string test_bucket_prefix = generate_test_prefix();
+    gcp_connection conn(test_bucket_name, test_bucket_prefix);
+
+    const std::string object_name = "test_object";
+    const std::string file_name = object_name + ".txt";
+    const std::string non_existant_object_name = "test_non_exist";
+    const std::string non_existant_file_name = non_existant_object_name + ".txt";
+    const bool list_single = true;
+    std::vector<std::string> objects;
+
+    gcs::Client client = gcs::Client();
+
+    std::string payload = "Test payload :)";
+    create_file(file_name, payload);
+
+    SECTION("Simple list test", "[gcp-connection]")
+    {
+        // No matching objects. Objects list should be empty.
+        REQUIRE(conn.list_objects(objects, false) == 0);
+        REQUIRE(objects.empty());
+
+        // No matching objects with list_single. Objects list should be empty.
+        REQUIRE(conn.list_objects(objects, list_single) == 0);
+        REQUIRE(objects.empty());
+
+        // Upload 1 file to the bucket and test list_objects function.
+        // List_objects should return an objects list with size 1.
+        REQUIRE(
+          upload_file(client, test_bucket_name, test_bucket_prefix, file_name, object_name).ok());
+        REQUIRE(file_exists_in_bucket(client, test_bucket_name, test_bucket_prefix, object_name));
+        REQUIRE(conn.list_objects(objects, false) == 0);
+        REQUIRE(objects.size() == 1);
+        objects.clear();
+
+        // Delete the object we uploaded and test list_objects function.
+        // List_objects should return an empty objects list.
+        client.DeleteObject(test_bucket_name, test_bucket_prefix + object_name);
+        REQUIRE(conn.list_objects(objects, false) == 0);
+        REQUIRE(objects.empty());
+        objects.clear();
+
+        // Upload multiple files and test list.
+        const int32_t total_objects = 20;
+        for (int i = 0; i < total_objects; i++) {
+            std::string multi_file_name = object_name + std::to_string(i);
+            REQUIRE(
+              upload_file(client, test_bucket_name, test_bucket_prefix, file_name, multi_file_name)
+                .ok());
+        }
+
+        // List objects should return a list with size total_objects.
+        REQUIRE(conn.list_objects(objects, false) == 0);
+        REQUIRE(objects.size() == total_objects);
+        objects.clear();
+
+        // Test if list single correctly returns one object.
+        REQUIRE(conn.list_objects(objects, list_single) == 0);
+        REQUIRE(objects.size() == 1);
+        objects.clear();
+
+        // Delete all object we uploaded.
+        for (int i = 0; i < total_objects; i++) {
+            client.DeleteObject(
+              test_bucket_name, test_bucket_prefix + object_name + std::to_string(i));
+        }
+
+        // Bucket should be cleared.
+        REQUIRE(conn.list_objects(objects, false) == 0);
+        REQUIRE(objects.empty());
+    }
+
+    SECTION("Simple put test", "[gcp-connection]")
+    {
+
+        // Upload a file that does not exist locally - should fail.
+        REQUIRE(conn.put_object(non_existant_object_name, non_existant_file_name) == ENOENT);
+
+        // Check number of files with the given prefix that are currently in the bucket.
+        REQUIRE(num_objects_in_bucket(client, test_bucket_name, test_bucket_prefix) == 0);
+
+        // Upload a test file.
+        REQUIRE(conn.put_object(object_name, "./" + file_name) == 0);
+
+        // Check the bucket contains the uploaded file.
+        REQUIRE(file_exists_in_bucket(client, test_bucket_name, test_bucket_prefix, object_name));
+
+        // Delete the uploaded file.
+        client.DeleteObject(test_bucket_name, test_bucket_prefix + object_name);
+    }
+
+    SECTION("Simple delete test", "[gcp-connection]")
+    {
+
+        // Delete a file that does not exist in the bucket - should fail.
+        REQUIRE(conn.delete_object(non_existant_object_name) == ENOENT);
+
+        // Upload a test file.
+        REQUIRE(
+          upload_file(client, test_bucket_name, test_bucket_prefix, file_name, object_name).ok());
+        REQUIRE(file_exists_in_bucket(client, test_bucket_name, test_bucket_prefix, object_name));
+
+        // Delete the uploaded file.
+        REQUIRE(conn.delete_object(object_name) == 0);
+
+        // Check that the file has been deleted.
+        REQUIRE_FALSE(
+          file_exists_in_bucket(client, test_bucket_name, test_bucket_prefix, object_name));
+    }
+
+    SECTION("Simple object exists test", "[gcp-connection]")
+    {
+        bool exists;
+        size_t size;
+
+        REQUIRE(conn.object_exists(object_name, exists, size) == 0);
+        REQUIRE(exists == false);
+        REQUIRE(size == 0);
+
+        // Upload a test file.
+        REQUIRE(
+          upload_file(client, test_bucket_name, test_bucket_prefix, file_name, object_name).ok());
+        REQUIRE(num_objects_in_bucket(client, test_bucket_name, test_bucket_prefix) == 1);
+
+        // Check the bucket contains the uploaded file.
+        REQUIRE(conn.object_exists(object_name, exists, size) == 0);
+        REQUIRE(exists);
+        REQUIRE(size != 0);
+
+        // Delete the uploaded file.
+        auto dl_metadata = client.DeleteObject(test_bucket_name, test_bucket_prefix + object_name);
+        REQUIRE(dl_metadata.ok());
+
+        // Check that the file has been deleted.
+        REQUIRE_FALSE(
+          file_exists_in_bucket(client, test_bucket_name, test_bucket_prefix, object_name));
+
+        // Check for a file that is not in the bucket.
+        // Object exists should modify the exists variable and set it to false.
+        REQUIRE(conn.object_exists(object_name, exists, size) == 0);
+        REQUIRE(exists == false);
+        REQUIRE(size == 0);
+    }
+
+    SECTION("Read tests", "[gcp-connection]")
+    {
+        // Upload a test file.
+        REQUIRE(
+          upload_file(client, test_bucket_name, test_bucket_prefix, file_name, object_name).ok());
+        REQUIRE(file_exists_in_bucket(client, test_bucket_name, test_bucket_prefix, object_name));
+
+        // Read GCP objects under the test bucket with no offset.
+        char buf[1024];
+
+        REQUIRE(conn.read_object(object_name, 0, payload.length(), buf) == 0);
+        REQUIRE(payload.compare(buf) == 0);
+        memset(buf, 0, 1000);
+
+        // Read GCP objects under the test bucket with offset.
+        const int str_len = payload.length() - payload.find(" ");
+        REQUIRE(conn.read_object(object_name, payload.find(" "), str_len, buf) == 0);
+        REQUIRE(payload.substr(payload.find(" "), str_len).compare(buf) == 0);
+        memset(buf, 0, 1000);
+
+        // Read GCP objects under the test bucket with len > file length.
+        REQUIRE(conn.read_object(object_name, 0, 100000, buf) == EINVAL);
+
+        // Read GCP objects under the test bucket with offset < 0.
+        REQUIRE(conn.read_object(object_name, -5, 15, buf) == EINVAL);
+
+        // Read GCP objects under the test bucket with offset > file length.
+        REQUIRE(conn.read_object(object_name, 1000, 15, buf) == EINVAL);
+    }
 
-TEST_CASE("Testing class gcpConnection", "gcp-connection") {}
+    // Cleanup
+    // List and loop through objects with prefix.
+    for (auto &&object_metadata :
+      client.ListObjects(test_bucket_name, gcs::Prefix(test_bucket_prefix))) {
+        // Delete the test file.
+        if (object_metadata) {
+            auto dl_metadata =
+              client.DeleteObject(test_bucket_name, object_metadata.value().name());
+            REQUIRE(dl_metadata.ok());
+        }
+    }
+}
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index a1a175e4512..a9a85d0abbd 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "6c8c5c49eec2b71b05baa871eed9cf97160874b5"
+    "commit": "cb132beb37743fd2e6b237ef006371b1ac26bf45"
 }