SERVER-36363 Support GET requests in HTTPClient

5 files changed, 99 insertions, 46 deletions

diff --git a/src/mongo/db/free_mon/free_mon_mongod.cpp b/src/mongo/db/free_mon/free_mon_mongod.cpp
index 390c59f0473..112136fe8ca 100644
--- a/src/mongo/db/free_mon/free_mon_mongod.cpp
+++ b/src/mongo/db/free_mon/free_mon_mongod.cpp
@@ -127,14 +127,15 @@ public:
         return _client
             ->postAsync(
                 _executor.get(), exportedExportedFreeMonEndpointURL.getLocked() + "/register", data)
-            .then([](std::vector<uint8_t> blob) {
+            .then([](DataBuilder&& blob) {
 
-                if (blob.empty()) {
+                if (!blob.size()) {
                     uasserted(ErrorCodes::FreeMonHttpTemporaryFailure, "Empty response received");
                 }
 
-                ConstDataRange cdr(reinterpret_cast<char*>(blob.data()), blob.size());
-
+                auto blobSize = blob.size();
+                auto blobData = blob.release();
+                ConstDataRange cdr(blobData.get(), blobSize);
                 auto swDoc = cdr.read<Validated<BSONObj>>();
                 uassertStatusOK(swDoc.getStatus());
 
@@ -155,13 +156,15 @@ public:
         return _client
             ->postAsync(
                 _executor.get(), exportedExportedFreeMonEndpointURL.getLocked() + "/metrics", data)
-            .then([](std::vector<uint8_t> blob) {
+            .then([](DataBuilder&& blob) {
 
-                if (blob.empty()) {
+                if (!blob.size()) {
                     uasserted(ErrorCodes::FreeMonHttpTemporaryFailure, "Empty response received");
                 }
 
-                ConstDataRange cdr(reinterpret_cast<char*>(blob.data()), blob.size());
+                auto blobSize = blob.size();
+                auto blobData = blob.release();
+                ConstDataRange cdr(blobData.get(), blobSize);
 
                 auto swDoc = cdr.read<Validated<BSONObj>>();
                 uassertStatusOK(swDoc.getStatus());
diff --git a/src/mongo/util/net/http_client.cpp b/src/mongo/util/net/http_client.cpp
index ab39a3970f2..6c5bc0f0d2a 100644
--- a/src/mongo/util/net/http_client.cpp
+++ b/src/mongo/util/net/http_client.cpp
@@ -39,11 +39,10 @@
 
 namespace mongo {
 
-Future<std::vector<uint8_t>> HttpClient::postAsync(
-    executor::ThreadPoolTaskExecutor* executor,
-    StringData url,
-    std::shared_ptr<std::vector<std::uint8_t>> data) const {
-    auto pf = makePromiseFuture<std::vector<uint8_t>>();
+Future<DataBuilder> HttpClient::postAsync(executor::ThreadPoolTaskExecutor* executor,
+                                          StringData url,
+                                          std::shared_ptr<std::vector<std::uint8_t>> data) const {
+    auto pf = makePromiseFuture<DataBuilder>();
     std::string urlString(url.toString());
 
     auto status = executor->scheduleWork([
@@ -55,7 +54,26 @@ Future<std::vector<uint8_t>> HttpClient::postAsync(
         ConstDataRange cdr(reinterpret_cast<char*>(data->data()), data->size());
         try {
             auto result = this->post(urlString, cdr);
-            shared_promise.emplaceValue(result);
+            shared_promise.emplaceValue(std::move(result));
+        } catch (...) {
+            shared_promise.setError(exceptionToStatus());
+        }
+    });
+
+    uassertStatusOK(status);
+    return std::move(pf.future);
+}
+
+Future<DataBuilder> HttpClient::getAsync(executor::ThreadPoolTaskExecutor* executor,
+                                         StringData url) const {
+    auto pf = makePromiseFuture<DataBuilder>();
+    std::string urlString(url.toString());
+
+    auto status = executor->scheduleWork([ shared_promise = pf.promise.share(), urlString, this ](
+        const executor::TaskExecutor::CallbackArgs& cbArgs) mutable {
+        try {
+            auto result = this->get(urlString);
+            shared_promise.emplaceValue(std::move(result));
         } catch (...) {
             shared_promise.setError(exceptionToStatus());
         }
diff --git a/src/mongo/util/net/http_client.h b/src/mongo/util/net/http_client.h
index e10c1f94900..69055b394b8 100644
--- a/src/mongo/util/net/http_client.h
+++ b/src/mongo/util/net/http_client.h
@@ -32,6 +32,7 @@
 #include <memory>
 #include <vector>
 
+#include "mongo/base/data_builder.h"
 #include "mongo/base/data_range.h"
 #include "mongo/base/string_data.h"
 #include "mongo/executor/thread_pool_task_executor.h"
@@ -64,14 +65,24 @@ public:
     /**
      * Perform a POST request to specified URL.
      */
-    virtual std::vector<uint8_t> post(const std::string& url, ConstDataRange data) const = 0;
+    virtual DataBuilder post(StringData url, ConstDataRange data) const = 0;
 
     /**
      * Futurized helper for HttpClient::post().
      */
-    Future<std::vector<uint8_t>> postAsync(executor::ThreadPoolTaskExecutor* executor,
-                                           StringData url,
-                                           std::shared_ptr<std::vector<std::uint8_t>> data) const;
+    Future<DataBuilder> postAsync(executor::ThreadPoolTaskExecutor* executor,
+                                  StringData url,
+                                  std::shared_ptr<std::vector<std::uint8_t>> data) const;
+
+    /**
+     * Perform a GET request from the specified URL.
+     */
+    virtual DataBuilder get(StringData url) const = 0;
+
+    /**
+     * Futurized helpr for HttpClient::get().
+     */
+    Future<DataBuilder> getAsync(executor::ThreadPoolTaskExecutor* executor, StringData url) const;
 
     /**
      * Factory method provided by client implementation.
diff --git a/src/mongo/util/net/http_client_curl.cpp b/src/mongo/util/net/http_client_curl.cpp
index 8948a0b262c..45c44049b83 100644
--- a/src/mongo/util/net/http_client_curl.cpp
+++ b/src/mongo/util/net/http_client_curl.cpp
@@ -151,7 +151,6 @@ public:
         curl_easy_setopt(_handle.get(), CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
         curl_easy_setopt(_handle.get(), CURLOPT_NOSIGNAL, 1);
         curl_easy_setopt(_handle.get(), CURLOPT_PROTOCOLS, CURLPROTO_HTTPS);
-        curl_easy_setopt(_handle.get(), CURLOPT_READFUNCTION, ReadMemoryCallback);
 #if LIBCURL_VERSION_NUM > 0x072200
         // Requires >= 7.34.0
         curl_easy_setopt(_handle.get(), CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2);
@@ -181,36 +180,52 @@ public:
         _headers = headers;
     }
 
-    std::vector<uint8_t> post(const std::string& url, ConstDataRange cdr) const final {
+    DataBuilder get(StringData url) const final {
         // Make a local copy of the base handle for this request.
         CurlHandle myHandle(curl_easy_duphandle(_handle.get()));
         uassert(ErrorCodes::InternalError, "Curl initialization failed", myHandle);
 
-        curl_easy_setopt(myHandle.get(), CURLOPT_URL, url.c_str());
-        curl_easy_setopt(myHandle.get(), CURLOPT_POST, 1);
+        return doRequest(myHandle.get(), url);
+    }
 
-        DataBuilder dataBuilder(4096);
-        curl_easy_setopt(myHandle.get(), CURLOPT_WRITEDATA, &dataBuilder);
+    DataBuilder post(StringData url, ConstDataRange cdr) const final {
+        // Make a local copy of the base handle for this request.
+        CurlHandle myHandle(curl_easy_duphandle(_handle.get()));
+        uassert(ErrorCodes::InternalError, "Curl initialization failed", myHandle);
+
+        curl_easy_setopt(myHandle.get(), CURLOPT_POST, 1);
 
         ConstDataRangeCursor cdrc(cdr);
+        curl_easy_setopt(myHandle.get(), CURLOPT_READFUNCTION, ReadMemoryCallback);
         curl_easy_setopt(myHandle.get(), CURLOPT_READDATA, &cdrc);
         curl_easy_setopt(myHandle.get(), CURLOPT_POSTFIELDSIZE, (long)cdrc.length());
 
+        return doRequest(myHandle.get(), url);
+    }
+
+private:
+    DataBuilder doRequest(CURL* handle, StringData url) const {
+        const auto urlString = url.toString();
+        curl_easy_setopt(handle, CURLOPT_URL, urlString.c_str());
+
+        DataBuilder dataBuilder(4096);
+        curl_easy_setopt(handle, CURLOPT_WRITEDATA, &dataBuilder);
+
         curl_slist* chunk = nullptr;
         for (const auto& header : _headers) {
             chunk = curl_slist_append(chunk, header.c_str());
         }
-        curl_easy_setopt(myHandle.get(), CURLOPT_HTTPHEADER, chunk);
+        curl_easy_setopt(handle, CURLOPT_HTTPHEADER, chunk);
         CurlSlist _headers(chunk);
 
-        CURLcode result = curl_easy_perform(myHandle.get());
+        CURLcode result = curl_easy_perform(handle);
         uassert(ErrorCodes::OperationFailed,
                 str::stream() << "Bad HTTP response from API server: "
                               << curl_easy_strerror(result),
                 result == CURLE_OK);
 
         long statusCode;
-        result = curl_easy_getinfo(myHandle.get(), CURLINFO_RESPONSE_CODE, &statusCode);
+        result = curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &statusCode);
         uassert(ErrorCodes::OperationFailed,
                 str::stream() << "Unexpected error retrieving response: "
                               << curl_easy_strerror(result),
@@ -220,8 +235,7 @@ public:
                 str::stream() << "Unexpected http status code from server: " << statusCode,
                 statusCode == 200);
 
-        auto response = dataBuilder.getCursor();
-        return std::vector<uint8_t>(response.data(), response.data() + response.length());
+        return dataBuilder;
     }
 
 private:
diff --git a/src/mongo/util/net/http_client_winhttp.cpp b/src/mongo/util/net/http_client_winhttp.cpp
index 634d4044996..32a27ee0f99 100644
--- a/src/mongo/util/net/http_client_winhttp.cpp
+++ b/src/mongo/util/net/http_client_winhttp.cpp
@@ -151,14 +151,25 @@ public:
         _headers = toNativeString(header.c_str());
     }
 
-    std::vector<uint8_t> post(const std::string& urlString, ConstDataRange cdr) const final {
+    DataBuilder post(StringData url, ConstDataRange cdr) const final {
+        return doRequest(
+            L"POST", url, const_cast<void*>(static_cast<const void*>(cdr.data())), cdr.length());
+    }
+
+    DataBuilder get(StringData url) const final {
+        return doRequest(L"GET", url, nullptr, 0);
+    }
+
+private:
+    DataBuilder doRequest(LPCWSTR method, StringData urlSD, LPVOID data, DWORD data_len) const {
         const auto uassertWithErrno = [](StringData reason, bool ok) {
             const auto msg = errnoWithDescription(GetLastError());
             uassert(ErrorCodes::OperationFailed, str::stream() << reason << ": " << msg, ok);
         };
 
         // Break down URL for handling below.
-        auto url = uassertStatusOK(parseUrl(toNativeString(urlString.c_str())));
+        const auto urlString = toNativeString(urlSD.toString().c_str());
+        auto url = uassertStatusOK(parseUrl(urlString));
         uassert(
             ErrorCodes::BadValue, "URL endpoint must be https://", url.https || _allowInsecureHTTP);
 
@@ -204,7 +215,7 @@ public:
         uassertWithErrno("Failed connecting to remote host", connect);
 
         request = WinHttpOpenRequest(connect,
-                                     L"POST",
+                                     method,
                                      (url.path + url.query).c_str(),
                                      nullptr,
                                      WINHTTP_NO_REFERER,
@@ -222,14 +233,9 @@ public:
             uassertWithErrno("Failed setting authentication credentials", result);
         }
 
-        uassertWithErrno("Failed sending HTTP request",
-                         WinHttpSendRequest(request,
-                                            _headers.c_str(),
-                                            -1L,
-                                            const_cast<void*>(static_cast<const void*>(cdr.data())),
-                                            cdr.length(),
-                                            cdr.length(),
-                                            0));
+        uassertWithErrno(
+            "Failed sending HTTP request",
+            WinHttpSendRequest(request, _headers.c_str(), -1L, data, data_len, data_len, 0));
 
         uassertWithErrno("Failed receiving response from server",
                          WinHttpReceiveResponse(request, nullptr));
@@ -249,9 +255,8 @@ public:
                 str::stream() << "Unexpected http status code from server: " << statusCode,
                 statusCode == 200);
 
-        // Marshal response into vector.
-        std::vector<uint8_t> ret;
-        auto sz = ret.size();
+        std::vector<char> buffer;
+        DataBuilder ret(4096);
         for (;;) {
             DWORD len = 0;
             uassertWithErrno("Failed receiving response data",
@@ -259,12 +264,14 @@ public:
             if (!len) {
                 break;
             }
-            ret.resize(sz + len);
+
+            buffer.resize(len);
             uassertWithErrno("Failed reading response data",
-                             WinHttpReadData(request, ret.data() + sz, len, &len));
-            sz += len;
+                             WinHttpReadData(request, buffer.data(), len, &len));
+
+            ConstDataRange cdr(buffer.data(), len);
+            ret.writeAndAdvance(cdr);
         }
-        ret.resize(sz);
 
         return ret;
     }