SERVER-19111 Get rid of ShardStatus

This change moves the "shard status" retrieval to common static functions
and gets rid of the ShardStatus function.

14 files changed, 246 insertions, 118 deletions

diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript
index 6fa24a749e5..9b309f769cc 100644
--- a/src/mongo/s/SConscript
+++ b/src/mongo/s/SConscript
@@ -10,27 +10,35 @@ env.SConscript(
     ],
 )
 
-#
-# Schema and backward compatibility code for "config" collections.
-#
+env.Library(
+    target='base',
+    source=[
+        'type_locks.cpp',
+        'type_lockpings.cpp',
+        'type_config_version.cpp',
+        'type_mongos.cpp',
+    ],
+    LIBDEPS=[
+        '$BUILD_DIR/mongo/base/base',
+        '$BUILD_DIR/mongo/bson/bson',
+        '$BUILD_DIR/mongo/db/common',
+    ]
+)
 
-env.Library('base', ['mongo_version_range.cpp',
-                     'type_locks.cpp',
-                     'type_lockpings.cpp',
-                     'type_config_version.cpp',
-                     'type_mongos.cpp'],
-            LIBDEPS=['$BUILD_DIR/mongo/base/base',
-                     '$BUILD_DIR/mongo/bson/bson'])
+env.Library(
+    target='shard_util',
+    source=[
+        'shard_util.cpp',
+    ],
+    LIBDEPS=[
+        'client/sharding_client',
+    ]
+)
 
 env.CppUnitTest('chunk_version_test', 'chunk_version_test.cpp',
                 LIBDEPS=['base',
                          '$BUILD_DIR/mongo/db/common'])
 
-env.CppUnitTest('mongo_version_range_test', 'mongo_version_range_test.cpp', 
-                LIBDEPS=['base',
-                         '$BUILD_DIR/mongo/bson/bson',
-                         '$BUILD_DIR/mongo/db/common'])
-
 env.CppUnitTest('type_config_version_test', 'type_config_version_test.cpp',
                 LIBDEPS=['base',
                          '$BUILD_DIR/mongo/db/common'])
@@ -76,12 +84,12 @@ env.CppUnitTest(
     ],
     LIBDEPS=[
         'catalog/legacy/catalog_manager_legacy',
+        'coreshard',
         'metadata',
+        'mongoscore',
         '$BUILD_DIR/mongo/db/coredb',
-        '$BUILD_DIR/mongo/s/coreshard',
         '$BUILD_DIR/mongo/db/auth/authorization_manager_mock_init',
         '$BUILD_DIR/mongo/dbtests/mocklib',
-        '$BUILD_DIR/mongo/s/mongoscore',
         '$BUILD_DIR/mongo/db/common',
     ])
 
@@ -207,9 +215,9 @@ env.Library(
     ],
     LIBDEPS=[
         '$BUILD_DIR/mongo/bson/bson',
-        '$BUILD_DIR/mongo/s/client/sharding_client',
         'cluster_ops',
         'cluster_write_op_conversion',
+        'client/sharding_client',
     ],
 )
 
@@ -219,10 +227,10 @@ env.CppUnitTest(
         'chunk_manager_targeter_test.cpp',
     ],
     LIBDEPS=[
+        'coreshard',
+        'mongoscore',
         '$BUILD_DIR/mongo/db/coredb',
-        '$BUILD_DIR/mongo/s/coreshard',
         '$BUILD_DIR/mongo/db/auth/authorization_manager_mock_init',
-        '$BUILD_DIR/mongo/s/mongoscore',
     ]
 )
 
@@ -243,8 +251,10 @@ env.Library(
     ],
     LIBDEPS=[
         'base',
+        'catalog/catalog_manager',
         'client/sharding_client',
-        'cluster_ops_impl'
+        'cluster_ops_impl',
+        'shard_util',
     ]
 )
 
diff --git a/src/mongo/s/balancer_policy.cpp b/src/mongo/s/balancer_policy.cpp
index cbfe1c8b9ab..18228e092e6 100644
--- a/src/mongo/s/balancer_policy.cpp
+++ b/src/mongo/s/balancer_policy.cpp
@@ -35,12 +35,16 @@
 
 #include <algorithm>
 
+#include "mongo/client/read_preference.h"
+#include "mongo/client/remote_command_targeter.h"
 #include "mongo/s/catalog/catalog_manager.h"
 #include "mongo/s/catalog/type_shard.h"
 #include "mongo/s/catalog/type_tags.h"
 #include "mongo/s/chunk_manager.h"
+#include "mongo/s/client/shard.h"
 #include "mongo/s/client/shard_registry.h"
 #include "mongo/s/grid.h"
+#include "mongo/s/shard_util.h"
 #include "mongo/util/log.h"
 #include "mongo/util/stringutils.h"
 
@@ -52,6 +56,37 @@ using std::set;
 using std::string;
 using std::vector;
 
+namespace {
+
+/**
+ * Executes the serverStatus command against the specified shard and obtains the version of the
+ * running MongoD service.
+ *
+ * The MongoD version or throws an exception. Known exception codes are:
+ *  ShardNotFound if shard by that id is not available on the registry
+ *  NoSuchKey if the version could not be retrieved
+ */
+std::string retrieveShardMongoDVersion(ShardId shardId, ShardRegistry* shardRegistry) {
+    auto shard = shardRegistry->getShard(shardId);
+    if (!shard) {
+        uassertStatusOK({ErrorCodes::ShardNotFound, "Shard not found"});
+    }
+
+    auto shardHost = uassertStatusOK(
+        shard->getTargeter()->findHost({ReadPreference::PrimaryOnly, TagSet::primaryOnly()}));
+
+    BSONObj serverStatus =
+        uassertStatusOK(shardRegistry->runCommand(shardHost, "admin", BSON("serverStatus" << 1)));
+    BSONElement versionElement = serverStatus["version"];
+    if (versionElement.type() != String) {
+        uassertStatusOK({ErrorCodes::NoSuchKey, "version field not found in serverStatus"});
+    }
+
+    return versionElement.str();
+}
+
+}  // namespace
+
 string TagRange::toString() const {
     return str::stream() << min << " -->> " << max << "  on  " << tag;
 }
@@ -247,26 +282,19 @@ Status DistributionStatus::populateShardInfoMap(ShardInfoMap* shardInfo) {
         }
 
         for (const ShardType& shardData : shards) {
-            std::shared_ptr<Shard> shard = grid.shardRegistry()->getShard(shardData.getName());
-
-            // The shard must still exist in the registry. If it doesn't, which may happen in
-            // the very low proability case that it gets dropped between the call to
-            // getAllShards above and the call to getShard, just don't account for it since
-            // it is missing anyways.
-            if (!shard) {
-                warning() << "Shard [" << shardData.getName() << "] was not found. Skipping.";
-                continue;
-            }
+            std::set<std::string> dummy;
 
-            ShardStatus shardStatus = shard->getStatus();
+            const long long shardSizeBytes = uassertStatusOK(
+                shardutil::retrieveTotalShardSize(shardData.getName(), grid.shardRegistry()));
 
-            std::set<std::string> dummy;
+            const std::string shardMongodVersion =
+                retrieveShardMongoDVersion(shardData.getName(), grid.shardRegistry());
 
             ShardInfo newShardEntry(shardData.getMaxSizeMB(),
-                                    shardStatus.dataSizeBytes() / 1024 / 1024,
+                                    shardSizeBytes / 1024 / 1024,
                                     shardData.getDraining(),
                                     dummy,
-                                    shardStatus.mongoVersion());
+                                    shardMongodVersion);
 
             for (const string& shardTag : shardData.getTags()) {
                 newShardEntry.addTag(shardTag);
diff --git a/src/mongo/s/catalog/SConscript b/src/mongo/s/catalog/SConscript
index 78bad9a62a9..a22f760d478 100644
--- a/src/mongo/s/catalog/SConscript
+++ b/src/mongo/s/catalog/SConscript
@@ -40,7 +40,9 @@ env.Library(
         '$BUILD_DIR/mongo/bson/bson',
         '$BUILD_DIR/mongo/db/common',
         '$BUILD_DIR/mongo/s/batch_write_types',
-        '$BUILD_DIR/mongo/s/catalog/catalog_types'
+        '$BUILD_DIR/mongo/s/catalog/catalog_types',
+        '$BUILD_DIR/mongo/s/client/sharding_client',
+        '$BUILD_DIR/mongo/s/shard_util',
     ]
 )
 
@@ -67,7 +69,7 @@ env.CppUnitTest(
         'type_tags_test.cpp',
     ],
     LIBDEPS=[
-        '$BUILD_DIR/mongo/s/catalog/catalog_types'
+        'catalog_types'
     ]
 )
 
diff --git a/src/mongo/s/catalog/catalog_manager.cpp b/src/mongo/s/catalog/catalog_manager.cpp
index af5783770d5..bc73dd543c5 100644
--- a/src/mongo/s/catalog/catalog_manager.cpp
+++ b/src/mongo/s/catalog/catalog_manager.cpp
@@ -38,6 +38,7 @@
 #include "mongo/s/catalog/type_collection.h"
 #include "mongo/s/catalog/type_database.h"
 #include "mongo/s/client/shard_registry.h"
+#include "mongo/s/shard_util.h"
 #include "mongo/s/write_ops/batched_command_request.h"
 #include "mongo/s/write_ops/batched_command_response.h"
 #include "mongo/s/write_ops/batched_delete_document.h"
@@ -206,28 +207,28 @@ StatusWith<ShardId> CatalogManager::selectShardForNewDatabase(ShardRegistry* sha
         }
     }
 
-    auto bestShard = shardRegistry->getShard(allShardIds[0]);
-    if (!bestShard) {
-        return {ErrorCodes::ShardNotFound, "Candidate shard disappeared"};
-    }
+    ShardId candidateShardId = allShardIds[0];
 
-    ShardStatus bestStatus = bestShard->getStatus();
+    auto candidateSizeStatus = shardutil::retrieveTotalShardSize(candidateShardId, shardRegistry);
+    if (!candidateSizeStatus.isOK()) {
+        return candidateSizeStatus.getStatus();
+    }
 
     for (size_t i = 1; i < allShardIds.size(); i++) {
-        const auto shard = shardRegistry->getShard(allShardIds[i]);
-        if (!shard) {
-            continue;
-        }
+        const ShardId shardId = allShardIds[i];
 
-        const ShardStatus status = shard->getStatus();
+        const auto sizeStatus = shardutil::retrieveTotalShardSize(shardId, shardRegistry);
+        if (!sizeStatus.isOK()) {
+            return sizeStatus.getStatus();
+        }
 
-        if (status < bestStatus) {
-            bestShard = shard;
-            bestStatus = status;
+        if (sizeStatus.getValue() < candidateSizeStatus.getValue()) {
+            candidateSizeStatus = sizeStatus;
+            candidateShardId = shardId;
         }
     }
 
-    return bestShard->getId();
+    return candidateShardId;
 }
 
 }  // namespace mongo
diff --git a/src/mongo/s/catalog/legacy/SConscript b/src/mongo/s/catalog/legacy/SConscript
index d177efdcc9d..25408427142 100644
--- a/src/mongo/s/catalog/legacy/SConscript
+++ b/src/mongo/s/catalog/legacy/SConscript
@@ -18,9 +18,28 @@ env.Library(
         'legacy_dist_lock_pinger.cpp',
     ],
     LIBDEPS=[
+        'mongo_version_range',
         '$BUILD_DIR/mongo/s/catalog/catalog_manager',
         '$BUILD_DIR/mongo/s/catalog/dist_lock_manager',
-        '$BUILD_DIR/mongo/s/client/sharding_client',
     ]
 )
 
+env.Library(
+    target='mongo_version_range',
+    source=[
+        'mongo_version_range.cpp',
+    ],
+    LIBDEPS=[
+        '$BUILD_DIR/mongo/s/base',
+    ]
+)
+
+env.CppUnitTest(
+    target='mongo_version_range_test',
+    source=[
+        'mongo_version_range_test.cpp',
+    ],
+    LIBDEPS=[
+        'mongo_version_range',
+    ]
+)
diff --git a/src/mongo/s/catalog/legacy/config_upgrade.cpp b/src/mongo/s/catalog/legacy/config_upgrade.cpp
index d15694364a8..4c87e65c9ba 100644
--- a/src/mongo/s/catalog/legacy/config_upgrade.cpp
+++ b/src/mongo/s/catalog/legacy/config_upgrade.cpp
@@ -32,18 +32,17 @@
 
 #include "mongo/s/catalog/legacy/config_upgrade.h"
 
-
 #include "mongo/client/connpool.h"
 #include "mongo/client/dbclientcursor.h"
 #include "mongo/client/syncclusterconnection.h"
 #include "mongo/s/catalog/catalog_manager.h"
+#include "mongo/s/catalog/dist_lock_manager.h"
 #include "mongo/s/catalog/legacy/cluster_client_internal.h"
+#include "mongo/s/catalog/legacy/mongo_version_range.h"
 #include "mongo/s/catalog/type_collection.h"
 #include "mongo/s/catalog/type_database.h"
 #include "mongo/s/catalog/type_settings.h"
 #include "mongo/s/catalog/type_shard.h"
-#include "mongo/s/catalog/dist_lock_manager.h"
-#include "mongo/s/mongo_version_range.h"
 #include "mongo/s/type_config_version.h"
 #include "mongo/stdx/functional.h"
 #include "mongo/util/assert_util.h"
diff --git a/src/mongo/s/mongo_version_range.cpp b/src/mongo/s/catalog/legacy/mongo_version_range.cpp
index 7ac4b2c4474..a4ac14d2bb5 100644
--- a/src/mongo/s/mongo_version_range.cpp
+++ b/src/mongo/s/catalog/legacy/mongo_version_range.cpp
@@ -26,7 +26,9 @@
  *    then also delete it in the license file.
  */
 
-#include "mongo/s/mongo_version_range.h"
+#include "mongo/platform/basic.h"
+
+#include "mongo/s/catalog/legacy/mongo_version_range.h"
 
 #include "mongo/util/stringutils.h"
 
diff --git a/src/mongo/s/mongo_version_range.h b/src/mongo/s/catalog/legacy/mongo_version_range.h
index 2e5c443fcc9..2e5c443fcc9 100644
--- a/src/mongo/s/mongo_version_range.h
+++ b/src/mongo/s/catalog/legacy/mongo_version_range.h
diff --git a/src/mongo/s/mongo_version_range_test.cpp b/src/mongo/s/catalog/legacy/mongo_version_range_test.cpp
index 35397b1d486..c97132c05e6 100644
--- a/src/mongo/s/mongo_version_range_test.cpp
+++ b/src/mongo/s/catalog/legacy/mongo_version_range_test.cpp
@@ -26,7 +26,9 @@
  *    then also delete it in the license file.
  */
 
-#include "mongo/s/mongo_version_range.h"
+#include "mongo/platform/basic.h"
+
+#include "mongo/s/catalog/legacy/mongo_version_range.h"
 
 #include <string>
 #include <vector>
diff --git a/src/mongo/s/client/SConscript b/src/mongo/s/client/SConscript
index 8b414c1eecb..d99b3c99995 100644
--- a/src/mongo/s/client/SConscript
+++ b/src/mongo/s/client/SConscript
@@ -15,7 +15,6 @@ env.Library(
         '$BUILD_DIR/mongo/client/fetcher',
         '$BUILD_DIR/mongo/client/remote_command_runner_impl',
         '$BUILD_DIR/mongo/client/remote_command_targeter',
-        '$BUILD_DIR/mongo/s/catalog/catalog_manager',
     ]
 )
 
diff --git a/src/mongo/s/client/shard.cpp b/src/mongo/s/client/shard.cpp
index 5280a4107c5..4bc77e13eb6 100644
--- a/src/mongo/s/client/shard.cpp
+++ b/src/mongo/s/client/shard.cpp
@@ -34,9 +34,8 @@
 
 #include <string>
 
-#include "mongo/client/replica_set_monitor.h"
-#include "mongo/client/read_preference.h"
 #include "mongo/client/remote_command_targeter.h"
+#include "mongo/client/replica_set_monitor.h"
 #include "mongo/db/jsobj.h"
 #include "mongo/s/client/shard_registry.h"
 #include "mongo/s/grid.h"
@@ -59,25 +58,6 @@ ShardPtr Shard::lookupRSName(const string& name) {
     return grid.shardRegistry()->lookupRSName(name);
 }
 
-ShardStatus Shard::getStatus() const {
-    const ReadPreferenceSetting readPref(ReadPreference::PrimaryOnly, TagSet::primaryOnly());
-    auto shardHost = uassertStatusOK(getTargeter()->findHost(readPref));
-
-    // List databases command
-    BSONObj listDatabases = uassertStatusOK(
-        grid.shardRegistry()->runCommand(shardHost, "admin", BSON("listDatabases" << 1)));
-    BSONElement totalSizeElem = listDatabases["totalSize"];
-    uassert(28590, "totalSize field not found in listDatabases", totalSizeElem.isNumber());
-
-    // Server status command
-    BSONObj serverStatus = uassertStatusOK(
-        grid.shardRegistry()->runCommand(shardHost, "admin", BSON("serverStatus" << 1)));
-    BSONElement versionElement = serverStatus["version"];
-    uassert(28599, "version field not found in serverStatus", versionElement.type() == String);
-
-    return ShardStatus(totalSizeElem.numberLong(), versionElement.str());
-}
-
 std::string Shard::toString() const {
     return _id + ":" + _cs.toString();
 }
@@ -90,15 +70,4 @@ void Shard::removeShard(const ShardId& id) {
     grid.shardRegistry()->remove(id);
 }
 
-ShardStatus::ShardStatus(long long dataSizeBytes, const string& mongoVersion)
-    : _dataSizeBytes(dataSizeBytes), _mongoVersion(mongoVersion) {}
-
-std::string ShardStatus::toString() const {
-    return str::stream() << " dataSizeBytes: " << _dataSizeBytes << " version: " << _mongoVersion;
-}
-
-bool ShardStatus::operator<(const ShardStatus& other) const {
-    return dataSizeBytes() < other.dataSizeBytes();
-}
-
 }  // namespace mongo
diff --git a/src/mongo/s/client/shard.h b/src/mongo/s/client/shard.h
index d22e4d3fa83..c04ca1865dd 100644
--- a/src/mongo/s/client/shard.h
+++ b/src/mongo/s/client/shard.h
@@ -40,29 +40,6 @@ class RemoteCommandTargeter;
 
 using ShardId = std::string;
 
-/**
- * Contains runtime information obtained from the shard.
- */
-class ShardStatus {
-public:
-    ShardStatus(long long dataSizeBytes, const std::string& version);
-
-    long long dataSizeBytes() const {
-        return _dataSizeBytes;
-    }
-    const std::string& mongoVersion() const {
-        return _mongoVersion;
-    }
-
-    std::string toString() const;
-
-    bool operator<(const ShardStatus& other) const;
-
-private:
-    long long _dataSizeBytes;
-    std::string _mongoVersion;
-};
-
 class Shard;
 using ShardPtr = std::shared_ptr<Shard>;
 
@@ -96,11 +73,6 @@ public:
     }
 
     /**
-     * Returns metadata and stats for this shard.
-     */
-    ShardStatus getStatus() const;
-
-    /**
      * Returns a string description of this shard entry.
      */
     std::string toString() const;
diff --git a/src/mongo/s/shard_util.cpp b/src/mongo/s/shard_util.cpp
new file mode 100644
index 00000000000..f954bfc9865
--- /dev/null
+++ b/src/mongo/s/shard_util.cpp
@@ -0,0 +1,69 @@
+/**
+ *    Copyright (C) 2015 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/s/shard_util.h"
+
+#include "mongo/base/status_with.h"
+#include "mongo/client/read_preference.h"
+#include "mongo/client/remote_command_targeter.h"
+#include "mongo/s/client/shard_registry.h"
+#include "mongo/util/mongoutils/str.h"
+
+namespace mongo {
+namespace shardutil {
+
+StatusWith<long long> retrieveTotalShardSize(ShardId shardId, ShardRegistry* shardRegistry) {
+    auto shard = shardRegistry->getShard(shardId);
+    if (!shard) {
+        return {ErrorCodes::ShardNotFound, str::stream() << "shard " << shardId << " not found"};
+    }
+
+    auto shardHostStatus =
+        shard->getTargeter()->findHost({ReadPreference::PrimaryOnly, TagSet::primaryOnly()});
+    if (!shardHostStatus.isOK()) {
+        return shardHostStatus.getStatus();
+    }
+
+    auto listDatabasesStatus =
+        shardRegistry->runCommand(shardHostStatus.getValue(), "admin", BSON("listDatabases" << 1));
+    if (!listDatabasesStatus.isOK()) {
+        return listDatabasesStatus.getStatus();
+    }
+
+    BSONElement totalSizeElem = listDatabasesStatus.getValue()["totalSize"];
+    if (!totalSizeElem.isNumber()) {
+        return {ErrorCodes::NoSuchKey, "totalSize field not found in listDatabases"};
+    }
+
+    return totalSizeElem.numberLong();
+}
+
+}  // namespace shardutil
+}  // namespace mongo
diff --git a/src/mongo/s/shard_util.h b/src/mongo/s/shard_util.h
new file mode 100644
index 00000000000..b9d546b75a2
--- /dev/null
+++ b/src/mongo/s/shard_util.h
@@ -0,0 +1,56 @@
+/**
+ *    Copyright (C) 2015 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include <string>
+
+#include "mongo/s/client/shard.h"
+
+namespace mongo {
+
+class ShardRegistry;
+template <typename T>
+class StatusWith;
+
+/**
+ * Set of methods used to introspect the state of individual shards.
+ */
+namespace shardutil {
+/**
+ * Executes the listDatabases command against the specified shard and obtains the total data
+ * size across all databases in bytes (essentially, the totalSize field).
+ *
+ * Returns OK with the total size or an error. Known errors are:
+ *  ShardNotFound if shard by that id is not available on the registry
+ *  NoSuchKey if the total shard size could not be retrieved
+ */
+StatusWith<long long> retrieveTotalShardSize(ShardId shardId, ShardRegistry* shardRegistry);
+};
+
+}  // namespace mongo