diff options
author | Clarisse Cheah <clarisse.cheah@mongodb.com> | 2023-02-06 00:48:25 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2023-02-06 01:25:32 +0000 |
commit | a312fc233cd5598ecdae1ac5c369f81b0517a620 (patch) | |
tree | faa8140744c88027cbbbeacf23b5e2f8a4211e63 | |
parent | 0df304c2f48536a2acadd05833565e111d28deb9 (diff) | |
download | mongo-a312fc233cd5598ecdae1ac5c369f81b0517a620.tar.gz |
Import wiredtiger: a9a57414fdd7d91ba80bd718d3b9649f4cbdeaa2 from branch mongodb-master
ref: 37a5636ad5..a9a57414fd
for: 6.3.0-rc0
WT-10510 Implement unit testing for connection class for Azure
-rw-r--r-- | src/third_party/wiredtiger/ext/storage_sources/azure_store/test/test_azure_connection.cpp | 198 | ||||
-rw-r--r-- | src/third_party/wiredtiger/import.data | 2 |
2 files changed, 198 insertions, 2 deletions
diff --git a/src/third_party/wiredtiger/ext/storage_sources/azure_store/test/test_azure_connection.cpp b/src/third_party/wiredtiger/ext/storage_sources/azure_store/test/test_azure_connection.cpp index 2d99d909a8a..14da1f3e9c7 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/azure_store/test/test_azure_connection.cpp +++ b/src/third_party/wiredtiger/ext/storage_sources/azure_store/test/test_azure_connection.cpp @@ -30,5 +30,201 @@ #include <catch2/catch.hpp> #include "azure_connection.h" +#include <fstream> +#include <string> -TEST_CASE("testing class azure_connection", "azure_connection") {} +static std::string +create_file(const std::string &object_name, const std::string &payload) +{ + std::ofstream file(object_name + ".txt"); + file << payload; + file.close(); + return object_name + ".txt"; +} + +// Concatenates a random suffix to the prefix being used for the test object keys. Example of +// generated test prefix: "azuretest/unit/2023-31-01-16-34-10/623843294--". +static std::string +randomize_test_prefix() +{ + char time_str[100]; + std::time_t t = std::time(nullptr); + + REQUIRE(std::strftime(time_str, sizeof(time_str), "%F-%H-%M-%S", std::localtime(&t)) != 0); + + std::string obj_prefix("azuretest/unit/"); // To be concatenated with a random string. + + obj_prefix += time_str; + + // Create a random device and use it to generate a random seed to initialize the generator. + std::random_device my_random_device; + unsigned seed = my_random_device(); + std::default_random_engine my_random_engine(seed); + + obj_prefix += '/' + std::to_string(my_random_engine()); + obj_prefix += "--"; + + return obj_prefix; +} + +TEST_CASE("Testing Azure Connection Class", "azure-connection") +{ + auto azure_client = Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString( + std::getenv("AZURE_STORAGE_CONNECTION_STRING"), "myblobcontainer1"); + bool exists = false; + + std::string obj_prefix = randomize_test_prefix(); + + azure_connection conn = azure_connection("myblobcontainer1", obj_prefix); + azure_connection conn_bad = azure_connection("myblobcontainer1", "bad_prefix_"); + + std::vector<std::pair<std::string, std::string>> blob_objects; + + const std::string object_name = "test_object"; + const std::string object_name_2 = "test_object_2"; + const std::string non_exist_object_key = "test_non_exist"; + + // Payloads for blobs uploaded to container. + const std::string payload = "payload"; + const std::string payload_2 = "Testing offset and substring"; + blob_objects.push_back(std::make_pair(object_name, payload)); + blob_objects.push_back(std::make_pair(object_name_2, payload_2)); + + // Add objects into the container. + for (auto pair : blob_objects) { + auto blob_client = azure_client.GetBlockBlobClient(obj_prefix + pair.first); + blob_client.UploadFrom(create_file(pair.first, pair.second)); + } + + SECTION("Check Azure connection constructor.", "[azure-connection]") + { + REQUIRE_THROWS_WITH(azure_connection("Bad_bucket", ""), "Bad_bucket : No such bucket."); + } + + SECTION("Check object exists in Azure.", "[azure-connection]") + { + // Object exists so there should be 1 object. + REQUIRE(conn.object_exists(object_name, exists) == 0); + REQUIRE(exists == true); + + // Object does not exist so there should be 0 objects. + REQUIRE(conn.object_exists(non_exist_object_key, exists) == 0); + REQUIRE(exists == false); + } + + SECTION("List Azure objects under the test bucket.", "[azure-connection]") + { + std::vector<std::string> objects; + + // No matching objects. Object size should be 0. + REQUIRE(conn.list_objects(obj_prefix + non_exist_object_key, objects, false) == 0); + REQUIRE(objects.size() == 0); + + // No matching objects with list single functionality. Object size should be 0. + REQUIRE(conn.list_objects(obj_prefix + non_exist_object_key, objects, true) == 0); + REQUIRE(objects.size() == 0); + + // List all objects. Object size should be 2. + REQUIRE(conn.list_objects(obj_prefix + object_name, objects, false) == 0); + REQUIRE(objects.size() == blob_objects.size()); + objects.clear(); + + // List single. Object size should be 1. + REQUIRE(conn.list_objects(obj_prefix + object_name, objects, true) == 0); + REQUIRE(objects.size() == 1); + + objects.clear(); + } + + SECTION("Test delete functionality for Azure.", "[azure-connection]") + { + auto blob_client = azure_client.GetBlockBlobClient(obj_prefix + object_name + "1"); + std::vector<std::string> container; + + blob_client.UploadFrom(create_file(object_name + "1", payload)); + + // Test that an object can be deleted. + REQUIRE(conn.delete_object(object_name + "1") == 0); + + // Test that removing an object that doesn't exist returns a ENOENT. + REQUIRE(conn.delete_object(non_exist_object_key) == ENOENT); + + auto list_blob_response = azure_client.ListBlobs(); + for (const auto blob_item : list_blob_response.Blobs) { + container.push_back(blob_item.Name); + } + + // Check that the object does not exist in the container. + REQUIRE(std::find(container.begin(), container.end(), obj_prefix + object_name + "1") == + std::end(container)); + } + + SECTION("Check put functionality in Azure.", "[azure-connection]") + { + const std::string path = "./" + create_file(object_name, payload); + std::vector<std::string> container; + + REQUIRE(conn.put_object(object_name + "1", path) == 0); + + auto blob_client = azure_client.GetBlockBlobClient(obj_prefix + object_name + "1"); + + // Test that putting an object that doesn't exist locally returns -1. + REQUIRE(conn.put_object(non_exist_object_key, non_exist_object_key + ".txt") == -1); + + auto list_blob_response = azure_client.ListBlobs(); + for (const auto blob_item : list_blob_response.Blobs) { + container.push_back(blob_item.Name); + } + + // Check that the object exists in the container. + REQUIRE(std::find(container.begin(), container.end(), obj_prefix + object_name + "1") != + std::end(container)); + // Check that when putting an object fails that object is not in the container. + REQUIRE(std::find(container.begin(), container.end(), obj_prefix + non_exist_object_key) == + std::end(container)); + + blob_client.Delete(); + } + + SECTION("Check read functionality in Azure.", "[azure-connection]") + { + char buffer[1024]; + + // Test reading whole file. + REQUIRE(conn.read_object(object_name, 0, payload.length(), buffer) == 0); + REQUIRE(payload.compare(buffer) == 0); + memset(buffer, 0, 1024); + + // Check that read works from the first ' ' to the end. + const int str_len = payload_2.length() - payload_2.find(" "); + REQUIRE(conn.read_object(object_name_2, payload_2.find(" "), str_len, buffer) == 0); + REQUIRE(payload_2.substr(payload_2.find(" ")).compare(buffer) == 0); + memset(buffer, 0, 1024); + + // Test overflow on positive offset but past EOF. + REQUIRE(conn.read_object(object_name, 1, 1000, buffer) == -1); + + // Test overflow on negative offset but past EOF. + REQUIRE(conn.read_object(object_name, -1, 1000, buffer) == -1); + + // Test overflow with negative offset. + REQUIRE(conn.read_object(object_name, -1, 12, buffer) == -1); + + // Tests that reading a existing object but wrong prefix returns a ENOENT. + REQUIRE(conn_bad.read_object(object_name, 0, 1, buffer) == ENOENT); + + // Test that reading a non existent object returns a ENOENT. + REQUIRE(conn.read_object(non_exist_object_key, 0, 1, buffer) == ENOENT); + } + + // Delete the objects we added earlier so we have no objects in the container. + for (auto pair : blob_objects) { + auto blob_client = azure_client.GetBlockBlobClient(obj_prefix + pair.first); + blob_client.Delete(); + } + + // Sanity check that nothing exists. + Azure::Storage::Blobs::ListBlobsOptions blob_parameters; + blob_parameters.Prefix = obj_prefix; + REQUIRE(azure_client.ListBlobs(blob_parameters).Blobs.size() == 0); +} diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 32622a7db88..7f977ec4559 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "37a5636ad5b5a87711161db80cecf2b4e88d3b3c" + "commit": "a9a57414fdd7d91ba80bd718d3b9649f4cbdeaa2" } |