SERVER-31061 Add `mongo+srv://` support for URIs.

The shell now supports parsing and handling `mongo+srv://` style URIs,
as part of the DNS Seedlist support.  These URIs require DNS SRV and
TXT record lookups, for extra configuration options.  The shell also
supports a (limited) form of connection-failover -- when initially
connecting to a non-replica-set cluster, the shell will try each
host listed, in order, until a connection can be established.

1 files changed, 103 insertions, 69 deletions

diff --git a/src/mongo/client/mongo_uri.cpp b/src/mongo/client/mongo_uri.cpp
index e2356881290..e7c500d719b 100644
--- a/src/mongo/client/mongo_uri.cpp
+++ b/src/mongo/client/mongo_uri.cpp
@@ -34,24 +34,26 @@
 
 #include <utility>
 
+#include <boost/algorithm/string/case_conv.hpp>
+#include <boost/algorithm/string/classification.hpp>
+#include <boost/algorithm/string/find_iterator.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+
 #include "mongo/base/status_with.h"
 #include "mongo/bson/bsonobjbuilder.h"
 #include "mongo/client/dbclientinterface.h"
 #include "mongo/client/sasl_client_authenticate.h"
 #include "mongo/db/namespace_string.h"
+#include "mongo/util/dns_query.h"
 #include "mongo/util/hex.h"
 #include "mongo/util/mongoutils/str.h"
 
-#include <boost/algorithm/string/case_conv.hpp>
-#include <boost/algorithm/string/classification.hpp>
-#include <boost/algorithm/string/find_iterator.hpp>
-#include <boost/algorithm/string/predicate.hpp>
-
 namespace {
 constexpr std::array<char, 16> hexits{
     '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'};
 const mongo::StringData kURIPrefix{"mongodb://"};
-}
+const mongo::StringData kURISRVPrefix{"mongodb+srv://"};
+}  // namespace
 
 /**
  * RFC 3986 Section 2.1 - Percent Encoding
@@ -94,7 +96,8 @@ namespace mongo {
 namespace {
 
 /**
- * Helper Method for MongoURI::parse() to split a string into exactly 2 pieces by a char delimeter
+ * Helper Method for MongoURI::parse() to split a string into exactly 2 pieces by a char
+ * delimeter.
  */
 std::pair<StringData, StringData> partitionForward(StringData str, const char c) {
     const auto delim = str.find(c);
@@ -105,8 +108,8 @@ std::pair<StringData, StringData> partitionForward(StringData str, const char c)
 }
 
 /**
- * Helper method for MongoURI::parse() to split a string into exactly 2 pieces by a char delimiter
- * searching backward from the end of the string.
+ * Helper method for MongoURI::parse() to split a string into exactly 2 pieces by a char
+ * delimiter searching backward from the end of the string.
  */
 std::pair<StringData, StringData> partitionBackward(StringData str, const char c) {
     const auto delim = str.rfind(c);
@@ -121,17 +124,17 @@ std::pair<StringData, StringData> partitionBackward(StringData str, const char c
  *
  * foo=bar&baz=qux&...
  */
-StatusWith<MongoURI::OptionsMap> parseOptions(StringData options, StringData url) {
+MongoURI::OptionsMap parseOptions(StringData options, StringData url) {
     MongoURI::OptionsMap ret;
     if (options.empty()) {
         return ret;
     }
 
     if (options.find('?') != std::string::npos) {
-        return Status(ErrorCodes::FailedToParse,
-                      str::stream()
-                          << "URI Cannot Contain multiple questions marks for mongodb:// URL: "
-                          << url);
+        throw DBException(ErrorCodes::FailedToParse,
+                          str::stream()
+                              << "URI Cannot Contain multiple questions marks for mongodb:// URL: "
+                              << url);
     }
 
     const auto optionsStr = options.toString();
@@ -141,16 +144,16 @@ StatusWith<MongoURI::OptionsMap> parseOptions(StringData options, StringData url
          ++i) {
         const auto opt = boost::copy_range<std::string>(*i);
         if (opt.empty()) {
-            return Status(ErrorCodes::FailedToParse,
-                          str::stream()
-                              << "Missing a key/value pair in the options for mongodb:// URL: "
-                              << url);
+            throw DBException(ErrorCodes::FailedToParse,
+                              str::stream()
+                                  << "Missing a key/value pair in the options for mongodb:// URL: "
+                                  << url);
         }
 
         const auto kvPair = partitionForward(opt, '=');
         const auto keyRaw = kvPair.first;
         if (keyRaw.empty()) {
-            return Status(
+            throw DBException(
                 ErrorCodes::FailedToParse,
                 str::stream()
                     << "Missing a key for key/value pair in the options for mongodb:// URL: "
@@ -158,7 +161,7 @@ StatusWith<MongoURI::OptionsMap> parseOptions(StringData options, StringData url
         }
         const auto key = uriDecode(keyRaw);
         if (!key.isOK()) {
-            return Status(
+            throw DBException(
                 ErrorCodes::FailedToParse,
                 str::stream() << "Key '" << keyRaw
                               << "' in options cannot properly be URL decoded for mongodb:// URL: "
@@ -166,14 +169,14 @@ StatusWith<MongoURI::OptionsMap> parseOptions(StringData options, StringData url
         }
         const auto valRaw = kvPair.second;
         if (valRaw.empty()) {
-            return Status(ErrorCodes::FailedToParse,
-                          str::stream() << "Missing value for key '" << keyRaw
-                                        << "' in the options for mongodb:// URL: "
-                                        << url);
+            throw DBException(ErrorCodes::FailedToParse,
+                              str::stream() << "Missing value for key '" << keyRaw
+                                            << "' in the options for mongodb:// URL: "
+                                            << url);
         }
         const auto val = uriDecode(valRaw);
         if (!val.isOK()) {
-            return Status(
+            throw DBException(
                 ErrorCodes::FailedToParse,
                 str::stream() << "Value '" << valRaw << "' for key '" << keyRaw
                               << "' in options cannot properly be URL decoded for mongodb:// URL: "
@@ -186,20 +189,38 @@ StatusWith<MongoURI::OptionsMap> parseOptions(StringData options, StringData url
     return ret;
 }
 
+MongoURI::OptionsMap addTXTOptions(MongoURI::OptionsMap options,
+                                   const std::string& host,
+                                   const StringData url,
+                                   const bool isSeedlist) {
+    // If there is no seedlist mode, then don't add any TXT options.
+    if (!isSeedlist)
+        return options;
+
+    // Get all TXT records and parse them as options, adding them to the options set.
+    const auto txtRecords = dns::getTXTRecords(host);
+
+    for (const auto& record : txtRecords) {
+        auto txtOptions = parseOptions(record, url);
+        // Note that, `std::map` and `std::unordered_map` insert does not replace existing
+        // values -- this gives the desired behavior that user-specified values override TXT
+        // record specified values.
+        options.insert(begin(txtOptions), end(txtOptions));
+    }
+
+    return options;
+}
 }  // namespace
 
-StatusWith<MongoURI> MongoURI::parse(const std::string& url) {
+MongoURI MongoURI::parseImpl(const std::string& url) {
     const StringData urlSD(url);
 
     // 1. Validate and remove the scheme prefix mongodb://
-    if (!urlSD.startsWith(kURIPrefix)) {
-        const auto cs_status = ConnectionString::parse(url);
-        if (!cs_status.isOK()) {
-            return cs_status.getStatus();
-        }
-        return MongoURI(cs_status.getValue());
+    const bool isSeedlist = urlSD.startsWith(kURISRVPrefix);
+    if (!(urlSD.startsWith(kURIPrefix) || isSeedlist)) {
+        return MongoURI(uassertStatusOK(ConnectionString::parse(url)));
     }
-    const auto uriWithoutPrefix = urlSD.substr(kURIPrefix.size());
+    const auto uriWithoutPrefix = urlSD.substr(urlSD.find("://") + 3);
 
     // 2. Split the string by the first, unescaped / (if any), yielding:
     // split[0]: User information and host identifers
@@ -211,14 +232,15 @@ StatusWith<MongoURI> MongoURI::parse(const std::string& url) {
     // 2.b Make sure that there are no question marks in the left side of the /
     //     as any options after the ? must still have the / delimeter
     if (databaseAndOptions.empty() && userAndHostInfo.find('?') != std::string::npos) {
-        return Status(
+        throw DBException(
             ErrorCodes::FailedToParse,
             str::stream()
                 << "URI must contain slash delimeter between hosts and options for mongodb:// URL: "
                 << url);
     }
 
-    // 3. Split the user information and host identifiers string by the last, unescaped @, yielding:
+    // 3. Split the user information and host identifiers string by the last, unescaped @,
+    // yielding:
     // split[0]: User information
     // split[1]: Host identifiers;
     const auto userAndHost = partitionBackward(userAndHostInfo, '@');
@@ -237,18 +259,21 @@ StatusWith<MongoURI> MongoURI::parse(const std::string& url) {
     };
 
     if (containsColonOrAt(usernameSD)) {
-        return Status(ErrorCodes::FailedToParse,
-                      str::stream() << "Username must be URL Encoded for mongodb:// URL: " << url);
+        throw DBException(ErrorCodes::FailedToParse,
+                          str::stream() << "Username must be URL Encoded for mongodb:// URL: "
+                                        << url);
     }
+
     if (containsColonOrAt(passwordSD)) {
-        return Status(ErrorCodes::FailedToParse,
-                      str::stream() << "Password must be URL Encoded for mongodb:// URL: " << url);
+        throw DBException(ErrorCodes::FailedToParse,
+                          str::stream() << "Password must be URL Encoded for mongodb:// URL: "
+                                        << url);
     }
 
     // Get the username and make sure it did not fail to decode
     const auto usernameWithStatus = uriDecode(usernameSD);
     if (!usernameWithStatus.isOK()) {
-        return Status(
+        throw DBException(
             ErrorCodes::FailedToParse,
             str::stream() << "Username cannot properly be URL decoded for mongodb:// URL: " << url);
     }
@@ -257,7 +282,7 @@ StatusWith<MongoURI> MongoURI::parse(const std::string& url) {
     // Get the password and make sure it did not fail to decode
     const auto passwordWithStatus = uriDecode(passwordSD);
     if (!passwordWithStatus.isOK())
-        return Status(
+        throw DBException(
             ErrorCodes::FailedToParse,
             str::stream() << "Password cannot properly be URL decoded for mongodb:// URL: " << url);
     const auto password = passwordWithStatus.getValue();
@@ -271,7 +296,7 @@ StatusWith<MongoURI> MongoURI::parse(const std::string& url) {
          ++i) {
         const auto hostWithStatus = uriDecode(boost::copy_range<std::string>(*i));
         if (!hostWithStatus.isOK()) {
-            return Status(
+            throw DBException(
                 ErrorCodes::FailedToParse,
                 str::stream() << "Host cannot properly be URL decoded for mongodb:// URL: " << url);
         }
@@ -282,23 +307,35 @@ StatusWith<MongoURI> MongoURI::parse(const std::string& url) {
         }
 
         if ((host.find('/') != std::string::npos) && !StringData(host).endsWith(".sock")) {
-            return Status(
+            throw DBException(
                 ErrorCodes::FailedToParse,
                 str::stream() << "'" << host << "' in '" << url
                               << "' appears to be a unix socket, but does not end in '.sock'");
         }
 
-        const auto statusHostAndPort = HostAndPort::parse(host);
-        if (!statusHostAndPort.isOK()) {
-            return statusHostAndPort.getStatus();
-        }
-        servers.push_back(statusHostAndPort.getValue());
+        servers.push_back(uassertStatusOK(HostAndPort::parse(host)));
     }
     if (servers.empty()) {
-        return Status(ErrorCodes::FailedToParse, "No server(s) specified");
+        throw DBException(ErrorCodes::FailedToParse, "No server(s) specified");
     }
 
-    // 6. Split the auth database and connection options string by the first, unescaped ?, yielding:
+    const std::string canonicalHost = servers.front().host();
+    // If we're in seedlist mode, lookup the SRV record for `_mongodb._tcp` on the specified
+    // domain name.  Take that list of servers as the new list of servers.
+    if (isSeedlist) {
+        if (servers.size() > 1) {
+            throw DBException(ErrorCodes::FailedToParse,
+                              "Only a single server may be specified with a mongo+srv:// url.");
+        }
+        auto srvEntries = dns::lookupSRVRecords("_mongodb._tcp." + canonicalHost);
+        servers.clear();
+        std::transform(begin(srvEntries), end(srvEntries), back_inserter(servers), [](auto& srv) {
+            return HostAndPort(std::move(srv.host), srv.port);
+        });
+    }
+
+    // 6. Split the auth database and connection options string by the first, unescaped ?,
+    // yielding:
     // split[0] = auth database
     // split[1] = connection options
     const auto dbAndOpts = partitionForward(databaseAndOptions, '?');
@@ -306,10 +343,10 @@ StatusWith<MongoURI> MongoURI::parse(const std::string& url) {
     const auto connectionOptions = dbAndOpts.second;
     const auto databaseWithStatus = uriDecode(databaseSD);
     if (!databaseWithStatus.isOK()) {
-        return Status(ErrorCodes::FailedToParse,
-                      str::stream()
-                          << "Database name cannot properly be URL decoded for mongodb:// URL: "
-                          << url);
+        throw DBException(ErrorCodes::FailedToParse,
+                          str::stream() << "Database name cannot properly be URL "
+                                           "decoded for mongodb:// URL: "
+                                        << url);
     }
     const auto database = databaseWithStatus.getValue();
 
@@ -320,35 +357,32 @@ StatusWith<MongoURI> MongoURI::parse(const std::string& url) {
     if (!database.empty() &&
         !NamespaceString::validDBName(database,
                                       NamespaceString::DollarInDbNameBehavior::Disallow)) {
-        return Status(ErrorCodes::FailedToParse,
-                      str::stream()
-                          << "Database name cannot have reserved characters for mongodb:// URL: "
-                          << url);
+        throw DBException(ErrorCodes::FailedToParse,
+                          str::stream() << "Database name cannot have reserved "
+                                           "characters for mongodb:// URL: "
+                                        << url);
     }
 
     // 8. Validate, split, and URL decode the connection options
-    const auto optsWith = parseOptions(connectionOptions, url);
-    if (!optsWith.isOK()) {
-        return optsWith.getStatus();
-    }
-    const auto options = optsWith.getValue();
+    auto options =
+        addTXTOptions(parseOptions(connectionOptions, url), canonicalHost, url, isSeedlist);
 
     // If a replica set option was specified, store it in the 'setName' field.
     const auto optIter = options.find("replicaSet");
     std::string setName;
-    if (optIter != options.end()) {
+    if (optIter != end(options)) {
         setName = optIter->second;
         invariant(!setName.empty());
     }
 
-    if ((servers.size() > 1) && setName.empty()) {
-        return Status(ErrorCodes::FailedToParse,
-                      "Cannot list multiple servers in URL without 'replicaSet' option");
-    }
-
     ConnectionString cs(
         setName.empty() ? ConnectionString::MASTER : ConnectionString::SET, servers, setName);
     return MongoURI(std::move(cs), username, password, database, std::move(options));
 }
 
+StatusWith<MongoURI> MongoURI::parse(const std::string& url) try {
+    return parseImpl(url);
+} catch (const std::exception&) {
+    return exceptionToStatus();
+}
 }  // namespace mongo