From 32fe9d75405b5d04d1af25bab2fd086ffab81213 Mon Sep 17 00:00:00 2001 From: Luke Chen Date: Wed, 2 Feb 2022 15:00:22 +1100 Subject: Import wiredtiger: aab82d0585230a36d9ca094ae050d8256fa1d7f4 from branch mongodb-master ref: 0d65fabfca..aab82d0585 for: 5.3.0 WT-8750 Change extension to follow C++ style conventions --- src/third_party/wiredtiger/dist/s_clang-format | 2 +- src/third_party/wiredtiger/dist/s_copyright | 1 + src/third_party/wiredtiger/dist/s_style | 2 +- .../storage_sources/s3_store/aws_bucket_conn.cxx | 108 --------- .../ext/storage_sources/s3_store/aws_bucket_conn.h | 28 --- .../ext/storage_sources/s3_store/s3_connection.cpp | 106 +++++++++ .../ext/storage_sources/s3_store/s3_connection.h | 28 +++ .../storage_sources/s3_store/s3_storage_source.cpp | 245 +++++++++++++++++++++ .../ext/storage_sources/s3_store/s3_store.cxx | 245 --------------------- .../s3_store/test/test_aws_bucket_conn.cxx | 61 ----- .../s3_store/test/test_s3_connection.cpp | 61 +++++ src/third_party/wiredtiger/import.data | 2 +- 12 files changed, 444 insertions(+), 445 deletions(-) delete mode 100644 src/third_party/wiredtiger/ext/storage_sources/s3_store/aws_bucket_conn.cxx delete mode 100644 src/third_party/wiredtiger/ext/storage_sources/s3_store/aws_bucket_conn.h create mode 100644 src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.cpp create mode 100644 src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.h create mode 100644 src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp delete mode 100644 src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_store.cxx delete mode 100644 src/third_party/wiredtiger/ext/storage_sources/s3_store/test/test_aws_bucket_conn.cxx create mode 100644 src/third_party/wiredtiger/ext/storage_sources/s3_store/test/test_s3_connection.cpp (limited to 'src/third_party/wiredtiger') diff --git a/src/third_party/wiredtiger/dist/s_clang-format b/src/third_party/wiredtiger/dist/s_clang-format index 0bad455c804..80de76b7c36 100755 --- a/src/third_party/wiredtiger/dist/s_clang-format +++ b/src/third_party/wiredtiger/dist/s_clang-format @@ -42,7 +42,7 @@ fi case $# in 0) # Get all source files that aren't in s_clang-format.list. - search=`find bench examples ext src test -name '*.[ch]' -o -name '*.cxx'` + search=`find bench examples ext src test -name '*.[ch]' -o -name '*.cxx' -o -name '*.cpp'` for f in `cat dist/s_clang-format.list`; do search=`echo "$search" | sed "\#$f#d"` done;; diff --git a/src/third_party/wiredtiger/dist/s_copyright b/src/third_party/wiredtiger/dist/s_copyright index 98d1b45e1b2..2b9d727fb15 100755 --- a/src/third_party/wiredtiger/dist/s_copyright +++ b/src/third_party/wiredtiger/dist/s_copyright @@ -108,6 +108,7 @@ fi # Search for files, skipping some well-known 3rd party directories. (cd .. && find [a-z]* -name '*.[ch]' \ -o -name '*.cxx' \ + -o -name '*.cpp' \ -o -name '*.in' \ -o -name '*.py' \ -o -name '*.swig' | diff --git a/src/third_party/wiredtiger/dist/s_style b/src/third_party/wiredtiger/dist/s_style index 0b94f582332..d76a78bfb68 100755 --- a/src/third_party/wiredtiger/dist/s_style +++ b/src/third_party/wiredtiger/dist/s_style @@ -17,7 +17,7 @@ if [ $# -ne 1 ]; then cd .. find bench examples ext src test \ - -name '*.[chsy]' -o -name '*.in' -o -name '*.dox' -o -name '*.cxx' -o -name '*.py' -o -name '*.cmake' -o -name '*.i' | + -name '*.[chsy]' -o -name '*.in' -o -name '*.dox' -o -name '*.cxx' -o -name '*.cpp' -o -name '*.py' -o -name '*.cmake' -o -name '*.i' | sed -e '/Makefile.in/d' \ -e '/test\/3rdparty/d' \ -e '/checksum\/power8/d' \ diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/aws_bucket_conn.cxx b/src/third_party/wiredtiger/ext/storage_sources/s3_store/aws_bucket_conn.cxx deleted file mode 100644 index eaa82eb137d..00000000000 --- a/src/third_party/wiredtiger/ext/storage_sources/s3_store/aws_bucket_conn.cxx +++ /dev/null @@ -1,108 +0,0 @@ -#include -#include -#include -#include -#include "aws_bucket_conn.h" - -#include -#include -#include -#include - -/* - * list_buckets -- - * Builds a list of buckets from AWS account into a vector. Returns true if success, otherwise - * false. - */ -bool -aws_bucket_conn::list_buckets(std::vector &buckets) const -{ - auto outcome = m_s3_crt_client.ListBuckets(); - if (outcome.IsSuccess()) { - for (const auto &bucket : outcome.GetResult().GetBuckets()) - buckets.push_back(bucket.GetName()); - return true; - } else { - std::cerr << "Error in list_buckets: " << outcome.GetError().GetMessage() << std::endl - << std::endl; - return false; - } -} - -/* - * list_objects -- - * Builds a list of object names from a S3 bucket into a vector. Returns true if success, - * otherwise false. - */ -bool -aws_bucket_conn::list_objects( - const std::string &bucket_name, std::vector &objects) const -{ - Aws::S3Crt::Model::ListObjectsRequest request; - request.WithBucket(bucket_name); - Aws::S3Crt::Model::ListObjectsOutcome outcomes = m_s3_crt_client.ListObjects(request); - - if (outcomes.IsSuccess()) { - for (const auto &object : outcomes.GetResult().GetContents()) - objects.push_back(object.GetKey()); - return true; - } else { - std::cerr << "Error in list_buckets: " << outcomes.GetError().GetMessage() << std::endl; - return false; - } -} - -/* - * put_object -- - * Puts an object into an S3 bucket. Returns true if success, otherwise false. - */ -bool -aws_bucket_conn::put_object( - const std::string &bucket_name, const std::string &object_key, const std::string &file_name) const -{ - Aws::S3Crt::Model::PutObjectRequest request; - request.SetBucket(bucket_name); - request.SetKey(object_key); - - std::shared_ptr input_data = Aws::MakeShared( - "s3-source", file_name.c_str(), std::ios_base::in | std::ios_base::binary); - - request.SetBody(input_data); - - Aws::S3Crt::Model::PutObjectOutcome outcome = m_s3_crt_client.PutObject(request); - - if (outcome.IsSuccess()) { - return true; - } else { - std::cerr << "Error in put_object: " << outcome.GetError().GetMessage() << std::endl; - return false; - } -} - -/* - * delete_object -- - * Deletes an object from S3 bucket. Returns true if success, otherwise false. - */ -bool -aws_bucket_conn::delete_object(const std::string &bucket_name, const std::string &object_key) const -{ - Aws::S3Crt::Model::DeleteObjectRequest request; - request.SetBucket(bucket_name); - request.SetKey(object_key); - - Aws::S3Crt::Model::DeleteObjectOutcome outcome = m_s3_crt_client.DeleteObject(request); - - if (outcome.IsSuccess()) { - return true; - } else { - std::cerr << "Error in delete_object: " << outcome.GetError().GetMessage() << std::endl; - return false; - } -} - -/* - * aws_bucket_conn -- - * Constructor for AWS bucket connection. - */ -aws_bucket_conn::aws_bucket_conn(const Aws::S3Crt::ClientConfiguration &config) - : m_s3_crt_client(config){}; diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/aws_bucket_conn.h b/src/third_party/wiredtiger/ext/storage_sources/s3_store/aws_bucket_conn.h deleted file mode 100644 index fb44a75498f..00000000000 --- a/src/third_party/wiredtiger/ext/storage_sources/s3_store/aws_bucket_conn.h +++ /dev/null @@ -1,28 +0,0 @@ - -#ifndef AWS_BUCKET_CONN -#define AWS_BUCKET_CONN - -#include -#include - -#include -#include - -/* - * Class to represent an active connection to the AWS S3 endpoint. Allows for interaction with S3 - * client. - */ -class aws_bucket_conn { - public: - explicit aws_bucket_conn(const Aws::S3Crt::ClientConfiguration &config); - bool list_buckets(std::vector &buckets) const; - bool list_objects(const std::string &bucket_name, std::vector &objects) const; - bool put_object(const std::string &bucket_name, const std::string &object_key, - const std::string &file_name) const; - bool delete_object(const std::string &bucket_name, const std::string &object_key) const; - ~aws_bucket_conn() = default; - - private: - const Aws::S3Crt::S3CrtClient m_s3_crt_client; -}; -#endif diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.cpp b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.cpp new file mode 100644 index 00000000000..03eeed0cbe7 --- /dev/null +++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.cpp @@ -0,0 +1,106 @@ +#include +#include +#include +#include +#include "s3_connection.h" + +#include +#include +#include +#include + +/* + * ListBuckets -- + * Builds a list of buckets from AWS account into a vector. Returns true if success, otherwise + * false. + */ +bool +S3Connection::ListBuckets(std::vector &buckets) const +{ + auto outcome = m_S3CrtClient.ListBuckets(); + if (outcome.IsSuccess()) { + for (const auto &bucket : outcome.GetResult().GetBuckets()) + buckets.push_back(bucket.GetName()); + return true; + } else { + std::cerr << "Error in ListBuckets: " << outcome.GetError().GetMessage() << std::endl + << std::endl; + return false; + } +} + +/* + * ListObjects -- + * Builds a list of object names from a S3 bucket into a vector. Returns true if success, + * otherwise false. + */ +bool +S3Connection::ListObjects(const std::string &bucketName, std::vector &objects) const +{ + Aws::S3Crt::Model::ListObjectsRequest request; + request.WithBucket(bucketName); + Aws::S3Crt::Model::ListObjectsOutcome outcomes = m_S3CrtClient.ListObjects(request); + + if (outcomes.IsSuccess()) { + for (const auto &object : outcomes.GetResult().GetContents()) + objects.push_back(object.GetKey()); + return true; + } else { + std::cerr << "Error in ListObjects: " << outcomes.GetError().GetMessage() << std::endl; + return false; + } +} + +/* + * PutObject -- + * Puts an object into an S3 bucket. Returns true if success, otherwise false. + */ +bool +S3Connection::PutObject( + const std::string &bucketName, const std::string &objectKey, const std::string &fileName) const +{ + Aws::S3Crt::Model::PutObjectRequest request; + request.SetBucket(bucketName); + request.SetKey(objectKey); + + std::shared_ptr inputData = Aws::MakeShared( + "s3-source", fileName.c_str(), std::ios_base::in | std::ios_base::binary); + + request.SetBody(inputData); + + Aws::S3Crt::Model::PutObjectOutcome outcome = m_S3CrtClient.PutObject(request); + + if (outcome.IsSuccess()) { + return true; + } else { + std::cerr << "Error in PutObject: " << outcome.GetError().GetMessage() << std::endl; + return false; + } +} + +/* + * DeleteObject -- + * Deletes an object from S3 bucket. Returns true if success, otherwise false. + */ +bool +S3Connection::DeleteObject(const std::string &bucketName, const std::string &objectKey) const +{ + Aws::S3Crt::Model::DeleteObjectRequest request; + request.SetBucket(bucketName); + request.SetKey(objectKey); + + Aws::S3Crt::Model::DeleteObjectOutcome outcome = m_S3CrtClient.DeleteObject(request); + + if (outcome.IsSuccess()) { + return true; + } else { + std::cerr << "Error in DeleteObject: " << outcome.GetError().GetMessage() << std::endl; + return false; + } +} + +/* + * S3Connection -- + * Constructor for AWS S3 bucket connection. + */ +S3Connection::S3Connection(const Aws::S3Crt::ClientConfiguration &config) : m_S3CrtClient(config){}; diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.h b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.h new file mode 100644 index 00000000000..dc603c1fd49 --- /dev/null +++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_connection.h @@ -0,0 +1,28 @@ + +#ifndef S3CONNECTION +#define S3CONNECTION + +#include +#include + +#include +#include + +/* + * Class to represent an active connection to the AWS S3 endpoint. Allows for interaction with S3 + * client. + */ +class S3Connection { + public: + explicit S3Connection(const Aws::S3Crt::ClientConfiguration &config); + bool ListBuckets(std::vector &buckets) const; + bool ListObjects(const std::string &bucketName, std::vector &objects) const; + bool PutObject(const std::string &bucketName, const std::string &objectKey, + const std::string &fileName) const; + bool DeleteObject(const std::string &bucketName, const std::string &objectKey) const; + ~S3Connection() = default; + + private: + const Aws::S3Crt::S3CrtClient m_S3CrtClient; +}; +#endif diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp new file mode 100644 index 00000000000..bab8c5ff818 --- /dev/null +++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp @@ -0,0 +1,245 @@ +/*- + * Public Domain 2014-present MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include + +#include +#include "s3_connection.h" + +#define UNUSED(x) (void)(x) + +/* S3 storage source structure. */ +typedef struct { + WT_STORAGE_SOURCE storageSource; /* Must come first */ + WT_EXTENSION_API *wtApi; /* Extension API */ +} S3_STORAGE; + +typedef struct { + /* Must come first - this is the interface for the file system we are implementing. */ + WT_FILE_SYSTEM fileSystem; + S3_STORAGE *s3Storage; + S3Connection *conn; +} S3_FILE_SYSTEM; + +/* Configuration variables for connecting to S3CrtClient. */ +const Aws::String region = Aws::Region::AP_SOUTHEAST_2; +const double throughputTargetGbps = 5; +const uint64_t partSize = 8 * 1024 * 1024; /* 8 MB. */ + +/* Setting SDK options. */ +Aws::SDKOptions options; + +static int S3CustomizeFileSystem( + WT_STORAGE_SOURCE *, WT_SESSION *, const char *, const char *, const char *, WT_FILE_SYSTEM **); +static int S3AddReference(WT_STORAGE_SOURCE *); +static int S3FileSystemTerminate(WT_FILE_SYSTEM *, WT_SESSION *); + +/* + * S3CustomizeFileSystem -- + * Return a customized file system to access the s3 storage source objects. + */ +static int +S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, const char *bucketName, + const char *authToken, const char *config, WT_FILE_SYSTEM **fileSystem) +{ + S3_FILE_SYSTEM *fs; + int ret; + + /* Mark parameters as unused for now, until implemented. */ + UNUSED(session); + UNUSED(bucketName); + UNUSED(authToken); + UNUSED(config); + + Aws::S3Crt::ClientConfiguration aws_config; + aws_config.region = region; + aws_config.throughputTargetGbps = throughputTargetGbps; + aws_config.partSize = partSize; + + if ((fs = (S3_FILE_SYSTEM *)calloc(1, sizeof(S3_FILE_SYSTEM))) == NULL) + return (errno); + + fs->s3Storage = (S3_STORAGE *)storageSource; + + /* New can fail; will deal with this later. */ + fs->conn = new S3Connection(aws_config); + + fs->fileSystem.terminate = S3FileSystemTerminate; + + /* TODO: Move these into tests. Just testing here temporarily to show all functions work. */ + { + /* List S3 buckets. */ + std::vector buckets; + if (fs->conn->ListBuckets(buckets)) { + std::cout << "All buckets under my account:" << std::endl; + for (const std::string &bucket : buckets) { + std::cout << " * " << bucket << std::endl; + } + std::cout << std::endl; + } + + /* Have at least one bucket to use. */ + if (!buckets.empty()) { + const Aws::String firstBucket = buckets.at(0); + + /* List objects. */ + std::vector bucketObjects; + if (fs->conn->ListObjects(firstBucket, bucketObjects)) { + std::cout << "Objects in bucket '" << firstBucket << "':" << std::endl; + if (!bucketObjects.empty()) { + for (const auto &object : bucketObjects) { + std::cout << " * " << object << std::endl; + } + } else { + std::cout << "No objects in bucket." << std::endl; + } + std::cout << std::endl; + } + + /* Put object. */ + fs->conn->PutObject(firstBucket, "WiredTiger.turtle", "WiredTiger.turtle"); + + /* List objects again. */ + bucketObjects.clear(); + if (fs->conn->ListObjects(firstBucket, bucketObjects)) { + std::cout << "Objects in bucket '" << firstBucket << "':" << std::endl; + if (!bucketObjects.empty()) { + for (const auto &object : bucketObjects) { + std::cout << " * " << object << std::endl; + } + } else { + std::cout << "No objects in bucket." << std::endl; + } + std::cout << std::endl; + } + + /* Delete object. */ + fs->conn->DeleteObject(firstBucket, "WiredTiger.turtle"); + + /* List objects again. */ + bucketObjects.clear(); + if (fs->conn->ListObjects(firstBucket, bucketObjects)) { + std::cout << "Objects in bucket '" << firstBucket << "':" << std::endl; + if (!bucketObjects.empty()) { + for (const auto &object : bucketObjects) { + std::cout << " * " << object << std::endl; + } + } else { + std::cout << "No objects in bucket." << std::endl; + } + std::cout << std::endl; + } + } else { + std::cout << "No buckets in AWS account." << std::endl; + } + } + + *fileSystem = &fs->fileSystem; + return 0; +} + +/* + * S3FileSystemTerminate -- + * Discard any resources on termination of the file system. + */ +static int +S3FileSystemTerminate(WT_FILE_SYSTEM *fileSystem, WT_SESSION *session) +{ + S3_FILE_SYSTEM *fs; + + UNUSED(session); /* unused */ + + fs = (S3_FILE_SYSTEM *)fileSystem; + delete (fs->conn); + free(fs); + + return (0); +} + +/* + * S3AddReference -- + * Add a reference to the storage source so we can reference count to know when to really + * terminate. + */ +static int +S3AddReference(WT_STORAGE_SOURCE *storageSource) +{ + UNUSED(storageSource); + return (0); +} + +/* + * S3Terminate -- + * Discard any resources on termination. + */ +static int +S3Terminate(WT_STORAGE_SOURCE *storage, WT_SESSION *session) +{ + S3_STORAGE *s3; + s3 = (S3_STORAGE *)storage; + + Aws::ShutdownAPI(options); + + free(s3); + return (0); +} + +/* + * wiredtiger_extension_init -- + * A S3 storage source library. + */ +int +wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) +{ + S3_STORAGE *s3; + int ret; + + if ((s3 = (S3_STORAGE *)calloc(1, sizeof(S3_STORAGE))) == NULL) + return (errno); + + s3->wtApi = connection->get_extension_api(connection); + UNUSED(config); + + Aws::InitAPI(options); + + /* + * Allocate a S3 storage structure, with a WT_STORAGE structure as the first field, allowing us + * to treat references to either type of structure as a reference to the other type. + */ + s3->storageSource.ss_customize_file_system = S3CustomizeFileSystem; + s3->storageSource.ss_add_reference = S3AddReference; + s3->storageSource.terminate = S3Terminate; + + /* Load the storage */ + if ((ret = connection->add_storage_source(connection, "s3_store", &s3->storageSource, NULL)) != + 0) + free(s3); + + return (ret); +} diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_store.cxx b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_store.cxx deleted file mode 100644 index cee9a787898..00000000000 --- a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_store.cxx +++ /dev/null @@ -1,245 +0,0 @@ -/*- - * Public Domain 2014-present MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -#include -#include - -#include -#include "aws_bucket_conn.h" - -#define UNUSED(x) (void)(x) - -/* S3 storage source structure. */ -typedef struct { - WT_STORAGE_SOURCE storage_source; /* Must come first */ - WT_EXTENSION_API *wt_api; /* Extension API */ -} S3_STORAGE; - -typedef struct { - /* Must come first - this is the interface for the file system we are implementing. */ - WT_FILE_SYSTEM file_system; - S3_STORAGE *s3_storage; - aws_bucket_conn *conn; -} S3_FILE_SYSTEM; - -/* Configuration variables for connecting to S3CrtClient. */ -const Aws::String region = Aws::Region::AP_SOUTHEAST_2; -const double throughput_target_gbps = 5; -const uint64_t part_size = 8 * 1024 * 1024; /* 8 MB. */ - -/* Setting SDK options. */ -Aws::SDKOptions options; - -static int s3_customize_file_system( - WT_STORAGE_SOURCE *, WT_SESSION *, const char *, const char *, const char *, WT_FILE_SYSTEM **); -static int s3_add_reference(WT_STORAGE_SOURCE *); -static int s3_fs_terminate(WT_FILE_SYSTEM *, WT_SESSION *); - -/* - * s3_customize_file_system -- - * Return a customized file system to access the s3 storage source objects. - */ -static int -s3_customize_file_system(WT_STORAGE_SOURCE *storage_source, WT_SESSION *session, - const char *bucket_name, const char *auth_token, const char *config, - WT_FILE_SYSTEM **file_systemp) -{ - S3_FILE_SYSTEM *fs; - int ret; - - /* Mark parameters as unused for now, until implemented. */ - UNUSED(session); - UNUSED(bucket_name); - UNUSED(auth_token); - UNUSED(config); - - Aws::S3Crt::ClientConfiguration aws_config; - aws_config.region = region; - aws_config.throughputTargetGbps = throughput_target_gbps; - aws_config.partSize = part_size; - - if ((fs = (S3_FILE_SYSTEM *)calloc(1, sizeof(S3_FILE_SYSTEM))) == NULL) - return (errno); - - fs->s3_storage = (S3_STORAGE *)storage_source; - - /* New can fail; will deal with this later. */ - fs->conn = new aws_bucket_conn(aws_config); - fs->file_system.terminate = s3_fs_terminate; - - /* TODO: Move these into tests. Just testing here temporarily to show all functions work. */ - { - /* List S3 buckets. */ - std::vector buckets; - if (fs->conn->list_buckets(buckets)) { - std::cout << "All buckets under my account:" << std::endl; - for (const std::string &bucket : buckets) { - std::cout << " * " << bucket << std::endl; - } - std::cout << std::endl; - } - - /* Have at least one bucket to use. */ - if (!buckets.empty()) { - const Aws::String first_bucket = buckets.at(0); - - /* List objects. */ - std::vector bucket_objects; - if (fs->conn->list_objects(first_bucket, bucket_objects)) { - std::cout << "Objects in bucket '" << first_bucket << "':" << std::endl; - if (!bucket_objects.empty()) { - for (const auto &object : bucket_objects) { - std::cout << " * " << object << std::endl; - } - } else { - std::cout << "No objects in bucket." << std::endl; - } - std::cout << std::endl; - } - - /* Put object. */ - fs->conn->put_object(first_bucket, "WiredTiger.turtle", "WiredTiger.turtle"); - - /* List objects again. */ - bucket_objects.clear(); - if (fs->conn->list_objects(first_bucket, bucket_objects)) { - std::cout << "Objects in bucket '" << first_bucket << "':" << std::endl; - if (!bucket_objects.empty()) { - for (const auto &object : bucket_objects) { - std::cout << " * " << object << std::endl; - } - } else { - std::cout << "No objects in bucket." << std::endl; - } - std::cout << std::endl; - } - - /* Delete object. */ - fs->conn->delete_object(first_bucket, "WiredTiger.turtle"); - - /* List objects again. */ - bucket_objects.clear(); - if (fs->conn->list_objects(first_bucket, bucket_objects)) { - std::cout << "Objects in bucket '" << first_bucket << "':" << std::endl; - if (!bucket_objects.empty()) { - for (const auto &object : bucket_objects) { - std::cout << " * " << object << std::endl; - } - } else { - std::cout << "No objects in bucket." << std::endl; - } - std::cout << std::endl; - } - } else { - std::cout << "No buckets in AWS account." << std::endl; - } - } - - *file_systemp = &fs->file_system; - return 0; -} - -/* - * s3_fs_terminate -- - * Discard any resources on termination of the file system. - */ -static int -s3_fs_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *session) -{ - S3_FILE_SYSTEM *s3_fs; - - UNUSED(session); /* unused */ - - s3_fs = (S3_FILE_SYSTEM *)file_system; - delete (s3_fs->conn); - free(s3_fs); - - return (0); -} - -/* - * s3_add_reference -- - * Add a reference to the storage source so we can reference count to know when to really - * terminate. - */ -static int -s3_add_reference(WT_STORAGE_SOURCE *storage_source) -{ - UNUSED(storage_source); - return (0); -} - -/* - * s3_terminate -- - * Discard any resources on termination. - */ -static int -s3_terminate(WT_STORAGE_SOURCE *storage, WT_SESSION *session) -{ - S3_STORAGE *s3; - s3 = (S3_STORAGE *)storage; - - Aws::ShutdownAPI(options); - - free(s3); - return (0); -} - -/* - * wiredtiger_extension_init -- - * A S3 storage source library. - */ -int -wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) -{ - S3_STORAGE *s3; - int ret; - - if ((s3 = (S3_STORAGE *)calloc(1, sizeof(S3_STORAGE))) == NULL) - return (errno); - - s3->wt_api = connection->get_extension_api(connection); - UNUSED(config); - - Aws::InitAPI(options); - - /* - * Allocate a S3 storage structure, with a WT_STORAGE structure as the first field, allowing us - * to treat references to either type of structure as a reference to the other type. - */ - s3->storage_source.ss_customize_file_system = s3_customize_file_system; - s3->storage_source.ss_add_reference = s3_add_reference; - s3->storage_source.terminate = s3_terminate; - - /* Load the storage */ - if ((ret = connection->add_storage_source(connection, "s3_store", &s3->storage_source, NULL)) != - 0) - free(s3); - - return (ret); -} diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/test/test_aws_bucket_conn.cxx b/src/third_party/wiredtiger/ext/storage_sources/s3_store/test/test_aws_bucket_conn.cxx deleted file mode 100644 index 4f7f2ae8b47..00000000000 --- a/src/third_party/wiredtiger/ext/storage_sources/s3_store/test/test_aws_bucket_conn.cxx +++ /dev/null @@ -1,61 +0,0 @@ -#include - -/* Default config settings for the S3CrtClient. */ -namespace test_defaults { -const Aws::String region = Aws::Region::AP_SOUTHEAST_2; -const double throughput_target_gbps = 5; -const uint64_t part_size = 8 * 1024 * 1024; /* 8 MB. */ -} // namespace test_defaults - -int test_list_buckets(const Aws::S3Crt::ClientConfiguration &config); - -/* Wrapper for unit test functions. */ -#define TEST(func, config, expected_output) \ - do { \ - int __ret; \ - if ((__ret = (func(config))) != expected_output) \ - return (__ret); \ - } while (0) - -/* - * test_list_buckets -- - * Example of a unit test to list S3 buckets under the associated AWS account. - */ -int -test_list_buckets(const Aws::S3Crt::ClientConfiguration &config) -{ - aws_bucket_conn conn(config); - std::vector buckets; - if (!conn.list_buckets(buckets)) - return 1; - - std::cout << "All buckets under my account:" << std::endl; - for (const auto &bucket : buckets) - std::cout << " * " << bucket << std::endl; - return 0; -} - -/* - * main -- - * Set up configs and call unit tests. - */ -int -main() -{ - /* Set up the config to use the defaults specified. */ - Aws::S3Crt::ClientConfiguration aws_config; - aws_config.region = test_defaults::region; - aws_config.throughputTargetGbps = test_defaults::throughput_target_gbps; - aws_config.partSize = test_defaults::part_size; - - /* Set the SDK options and initialize the API. */ - Aws::SDKOptions options; - Aws::InitAPI(options); - - int expected_output = 0; - TEST(test_list_buckets, aws_config, expected_output); - - /* Shutdown the API at end of tests. */ - Aws::ShutdownAPI(options); - return 0; -} diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/test/test_s3_connection.cpp b/src/third_party/wiredtiger/ext/storage_sources/s3_store/test/test_s3_connection.cpp new file mode 100644 index 00000000000..1ecbca2c699 --- /dev/null +++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/test/test_s3_connection.cpp @@ -0,0 +1,61 @@ +#include + +/* Default config settings for the S3CrtClient. */ +namespace TestDefaults { +const Aws::String region = Aws::Region::AP_SOUTHEAST_2; +const double throughputTargetGbps = 5; +const uint64_t partSize = 8 * 1024 * 1024; /* 8 MB. */ +} // namespace TestDefaults + +int TestListBuckets(const Aws::S3Crt::ClientConfiguration &config); + +/* Wrapper for unit test functions. */ +#define TEST(func, config, expectedOutput) \ + do { \ + int __ret; \ + if ((__ret = (func(config))) != expectedOutput) \ + return (__ret); \ + } while (0) + +/* + * TestListBuckets -- + * Example of a unit test to list S3 buckets under the associated AWS account. + */ +int +TestListBuckets(const Aws::S3Crt::ClientConfiguration &config) +{ + S3Connection conn(config); + std::vector buckets; + if (!conn.ListBuckets(buckets)) + return 1; + + std::cout << "All buckets under my account:" << std::endl; + for (const auto &bucket : buckets) + std::cout << " * " << bucket << std::endl; + return 0; +} + +/* + * main -- + * Set up configs and call unit tests. + */ +int +main() +{ + /* Set up the config to use the defaults specified. */ + Aws::S3Crt::ClientConfiguration awsConfig; + awsConfig.region = TestDefaults::region; + awsConfig.throughputTargetGbps = TestDefaults::throughputTargetGbps; + awsConfig.partSize = TestDefaults::partSize; + + /* Set the SDK options and initialize the API. */ + Aws::SDKOptions options; + Aws::InitAPI(options); + + int expectedOutput = 0; + TEST(TestListBuckets, awsConfig, expectedOutput); + + /* Shutdown the API at end of tests. */ + Aws::ShutdownAPI(options); + return 0; +} diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 17934318fa2..8b1103fde80 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "0d65fabfca1e3435e8f8298a98d4fc91c56e331c" + "commit": "aab82d0585230a36d9ca094ae050d8256fa1d7f4" } -- cgit v1.2.1