diff options
author | Luke Chen <luke.chen@mongodb.com> | 2022-02-22 13:40:26 +1100 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-02-22 03:06:00 +0000 |
commit | 6ff120d1adda68cd0fd86d35e7c09ac39eb2ebb3 (patch) | |
tree | 05910677c53e3e1a304cba3b5b4160bb4fa14d24 | |
parent | 624df79cbb0ef6e302cb53fa8883b93fe5f565e4 (diff) | |
download | mongo-6ff120d1adda68cd0fd86d35e7c09ac39eb2ebb3.tar.gz |
Import wiredtiger: af9f6d8b5801e8e2ef998f3f6aaabccf6218b343 from branch mongodb-5.3
ref: 9b75c335cb..af9f6d8b58
for: 5.3.0-rc1
WT-8812 Accept bucket location as a configuration parameter
3 files changed, 27 insertions, 10 deletions
diff --git a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp index d2248d9fda6..c579ea53394 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp +++ b/src/third_party/wiredtiger/ext/storage_sources/s3_store/s3_storage_source.cpp @@ -102,7 +102,6 @@ struct S3_FILE_HANDLE { }; /* Configuration variables for connecting to S3CrtClient. */ -const Aws::String region = Aws::Region::AP_SOUTHEAST_2; const double throughputTargetGbps = 5; const uint64_t partSize = 8 * 1024 * 1024; /* 8 MB. */ @@ -442,12 +441,33 @@ S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, con std::string objPrefix; if ((ret = s3->wtApi->config_get_string( s3->wtApi, session, config, "prefix", &objPrefixConf)) == 0) - objPrefix = objPrefixConf.str; + objPrefix = std::string(objPrefixConf.str, objPrefixConf.len); else if (ret != WT_NOTFOUND) { std::cerr << "Error: customize_file_system: config parsing for object prefix"; - return (1); + return (ret); } + /* Configure the AWS Client configuration. */ + Aws::S3Crt::ClientConfiguration awsConfig; + awsConfig.throughputTargetGbps = throughputTargetGbps; + awsConfig.partSize = partSize; + + /* + * Get the AWS region to be used. The allowable values for AWS region are listed here in the AWS + * documentation: http://sdk.amazonaws.com/cpp/api/LATEST/namespace_aws_1_1_region.html + */ + WT_CONFIG_ITEM regionConf; + std::string region; + if ((ret = s3->wtApi->config_get_string(s3->wtApi, session, config, "region", ®ionConf)) == + 0) + awsConfig.region = std::string(regionConf.str, regionConf.len); + else if (ret != WT_NOTFOUND) { + std::cerr << "Error: customize_file_system: config parsing for AWS region"; + return (ret); + } else { + std::cerr << "Error: Region not specified" << std::endl; + return (EINVAL); + } /* * Get the directory to setup the cache, or use the default one. The default cache directory is * named "cache-<name>", where name is the last component of the bucket name's path. We'll @@ -457,7 +477,7 @@ S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, con std::string cacheStr; if ((ret = s3->wtApi->config_get_string( s3->wtApi, session, config, "cache_directory", &cacheDirConf)) == 0) - cacheStr = cacheDirConf.str; + cacheStr = std::string(cacheDirConf.str, cacheDirConf.len); else if (ret == WT_NOTFOUND) { cacheStr = "cache-" + std::string(bucketName); ret = 0; @@ -481,11 +501,6 @@ S3CustomizeFileSystem(WT_STORAGE_SOURCE *storageSource, WT_SESSION *session, con fs->homeDir = homeDir; fs->cacheDir = cacheDir; - Aws::S3Crt::ClientConfiguration awsConfig; - awsConfig.region = region; - awsConfig.throughputTargetGbps = throughputTargetGbps; - awsConfig.partSize = partSize; - /* New can fail; will deal with this later. */ fs->connection = new S3Connection(awsConfig, bucketName, objPrefix); fs->fileSystem.fs_directory_list = S3ObjectList; diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 7b663897537..d48c098fa74 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-5.3", - "commit": "9b75c335cb7a8d9fd3b287d49f8ea30e72ee1983" + "commit": "af9f6d8b5801e8e2ef998f3f6aaabccf6218b343" } diff --git a/src/third_party/wiredtiger/test/suite/test_s3_store01.py b/src/third_party/wiredtiger/test/suite/test_s3_store01.py index d9c55cc54aa..53fb8fdf493 100644 --- a/src/third_party/wiredtiger/test/suite/test_s3_store01.py +++ b/src/third_party/wiredtiger/test/suite/test_s3_store01.py @@ -42,6 +42,8 @@ class test_s3_store01(wttest.WiredTigerTestCase): fs_config = 'prefix=' + prefix + fs_config += ',region=ap-southeast-2' + # Bucket name can be overridden by an environment variable. bucket_name = os.getenv('WT_S3_EXT_BUCKET') if bucket_name is None: |