From 4c40b70635fc5dea877f5bb5f7e98865e43d7b06 Mon Sep 17 00:00:00 2001 From: Will Korteland Date: Wed, 8 Feb 2023 23:24:37 +0000 Subject: Import wiredtiger: cb3b25c3072b5802a30674e600432e073c0a9396 from branch mongodb-master ref: 3861add9b4..cb3b25c307 for: 7.0.0-rc0 WT-10579 Update list_objects to support prefix search --- .../storage_sources/gcp_store/gcp_connection.cpp | 23 +++++++++++----------- .../ext/storage_sources/gcp_store/gcp_connection.h | 5 +++-- src/third_party/wiredtiger/import.data | 2 +- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/third_party/wiredtiger/ext/storage_sources/gcp_store/gcp_connection.cpp b/src/third_party/wiredtiger/ext/storage_sources/gcp_store/gcp_connection.cpp index f4f2852ce7b..5fdd7132209 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/gcp_store/gcp_connection.cpp +++ b/src/third_party/wiredtiger/ext/storage_sources/gcp_store/gcp_connection.cpp @@ -34,7 +34,7 @@ namespace gcs = google::cloud::storage; using namespace gcs; gcp_connection::gcp_connection(const std::string &bucket_name, const std::string &prefix) - : _gcp_client(gcs::Client()), _bucket_name(bucket_name), _object_prefix(prefix) + : _gcp_client(gcs::Client()), _bucket_name(bucket_name), _bucket_prefix(prefix) { // StatusOr either contains a usable BucketMetadata value or a Status object explaining why the // value is not present. The value's validity is checked by StatusOr::ok(). @@ -48,11 +48,12 @@ gcp_connection::gcp_connection(const std::string &bucket_name, const std::string // Builds a list of object names from the bucket. int -gcp_connection::list_objects(std::vector &objects, bool list_single) +gcp_connection::list_objects( + std::string search_prefix, std::vector &objects, bool list_single) { // Fetch the objects from the given bucket that match with the prefix given. for (auto &&object_metadata : - _gcp_client.ListObjects(_bucket_name, gcs::Prefix(_object_prefix))) { + _gcp_client.ListObjects(_bucket_name, gcs::Prefix(_bucket_prefix + search_prefix))) { // Check if the current object is accessible (object exists but the user does not have // permissions to access) if (!object_metadata) { @@ -79,12 +80,12 @@ gcp_connection::put_object(const std::string &object_key, const std::string &fil // Client library automatically computes a hash on the client-side to // verify data integrity. google::cloud::StatusOr metadata = - _gcp_client.UploadFile(file_path, _bucket_name, _object_prefix + object_key); + _gcp_client.UploadFile(file_path, _bucket_name, _bucket_prefix + object_key); // Check if file has been successfully uploaded. if (!metadata) return handle_error(metadata.status(), - "Upload of '" + _object_prefix + object_key + "' failed: " + metadata.status().message()); + "Upload of '" + _bucket_prefix + object_key + "' failed: " + metadata.status().message()); return 0; } @@ -93,11 +94,11 @@ gcp_connection::put_object(const std::string &object_key, const std::string &fil int gcp_connection::delete_object(const std::string &object_key) { - auto status = _gcp_client.DeleteObject(_bucket_name, _object_prefix + object_key); + auto status = _gcp_client.DeleteObject(_bucket_name, _bucket_prefix + object_key); if (!status.ok()) return handle_error( - status, "Delete '" + _object_prefix + object_key + "' failed: " + status.message()); + status, "Delete '" + _bucket_prefix + object_key + "' failed: " + status.message()); return 0; } @@ -133,11 +134,11 @@ gcp_connection::read_object(const std::string &object_key, int64_t offset, size_ } gcs::ObjectReadStream stream = _gcp_client.ReadObject( - _bucket_name, _object_prefix + object_key, gcs::ReadFromOffset(offset)); + _bucket_name, _bucket_prefix + object_key, gcs::ReadFromOffset(offset)); if (stream.bad()) return handle_error(stream.status(), - "Read '" + _object_prefix + object_key + "' failed: " + stream.status().message()); + "Read '" + _bucket_prefix + object_key + "' failed: " + stream.status().message()); std::istreambuf_iterator begin{stream}, end; std::string buffer{begin, end}; @@ -154,7 +155,7 @@ gcp_connection::object_exists(const std::string &object_key, bool &exists, size_ object_size = 0; google::cloud::StatusOr metadata = - _gcp_client.GetObjectMetadata(_bucket_name, _object_prefix + object_key); + _gcp_client.GetObjectMetadata(_bucket_name, _bucket_prefix + object_key); // Check if object exists and is accessible. if (metadata.ok()) { @@ -171,7 +172,7 @@ gcp_connection::object_exists(const std::string &object_key, bool &exists, size_ } return handle_error(metadata.status(), - "Object exists check for '" + _object_prefix + object_key + + "Object exists check for '" + _bucket_prefix + object_key + "' failed: " + metadata.status().message()); } diff --git a/src/third_party/wiredtiger/ext/storage_sources/gcp_store/gcp_connection.h b/src/third_party/wiredtiger/ext/storage_sources/gcp_store/gcp_connection.h index 5815de5cf2a..031cca1073e 100644 --- a/src/third_party/wiredtiger/ext/storage_sources/gcp_store/gcp_connection.h +++ b/src/third_party/wiredtiger/ext/storage_sources/gcp_store/gcp_connection.h @@ -44,7 +44,8 @@ static const std::map toErrno = { class gcp_connection { public: gcp_connection(const std::string &bucket_name, const std::string &prefix); - int list_objects(std::vector &objects, bool list_single); + int list_objects( + std::string search_prefix, std::vector &objects, bool list_single); int put_object(const std::string &object_key, const std::string &file_path); int delete_object(const std::string &object_key); int object_exists(const std::string &object_key, bool &exists, size_t &object_size); @@ -56,5 +57,5 @@ class gcp_connection { private: google::cloud::storage::Client _gcp_client; const std::string _bucket_name; - const std::string _object_prefix; + const std::string _bucket_prefix; }; diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index b1a3644a388..29e0420cd34 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -2,5 +2,5 @@ "vendor": "wiredtiger", "github": "wiredtiger/wiredtiger.git", "branch": "mongodb-master", - "commit": "3861add9b4352d4d290241113414bce8913bcbaa" + "commit": "cb3b25c3072b5802a30674e600432e073c0a9396" } -- cgit v1.2.1