summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/mongo/dbtests/chunktests.cpp3
-rw-r--r--src/mongo/dbtests/sharding.cpp3
-rw-r--r--src/mongo/s/balance.cpp22
-rw-r--r--src/mongo/s/balancer_policy.cpp68
-rw-r--r--src/mongo/s/balancer_policy.h9
-rw-r--r--src/mongo/s/chunk.cpp19
-rw-r--r--src/mongo/s/cluster_client_internal.h3
-rw-r--r--src/mongo/s/config.cpp3
-rw-r--r--src/mongo/s/config.h3
-rw-r--r--src/mongo/s/shard.cpp54
-rw-r--r--src/mongo/s/shard.h27
-rw-r--r--src/mongo/s/shard_test.cpp22
-rw-r--r--src/mongo/s/type_shard.cpp4
13 files changed, 151 insertions, 89 deletions
diff --git a/src/mongo/dbtests/chunktests.cpp b/src/mongo/dbtests/chunktests.cpp
index 7b86089185e..e9cb366f199 100644
--- a/src/mongo/dbtests/chunktests.cpp
+++ b/src/mongo/dbtests/chunktests.cpp
@@ -60,8 +60,7 @@ namespace mongo {
Shard shard(name,
name,
0 /* maxSize */,
- false /* draining */,
- BSONArray() /* tags */);
+ false /* draining */);
shards.insert( shard );
ChunkPtr chunk( new Chunk( this, mySplitPoints[ i-1 ], mySplitPoints[ i ],
diff --git a/src/mongo/dbtests/sharding.cpp b/src/mongo/dbtests/sharding.cpp
index 505b34655d7..6a70c2b0a9c 100644
--- a/src/mongo/dbtests/sharding.cpp
+++ b/src/mongo/dbtests/sharding.cpp
@@ -110,8 +110,7 @@ namespace ShardingTests {
_shard = Shard("shard0000",
"$hostFooBar:27017",
0 /* maxSize */,
- false /* draining */,
- BSONArray() /* tags */);
+ false /* draining */);
// Need to run this to ensure the shard is in the global lookup table
Shard::installShard(_shard.getName(), _shard);
diff --git a/src/mongo/s/balance.cpp b/src/mongo/s/balance.cpp
index 6e61ecd58d0..6659c8e3f64 100644
--- a/src/mongo/s/balance.cpp
+++ b/src/mongo/s/balance.cpp
@@ -350,16 +350,19 @@ namespace mongo {
// TODO: skip unresponsive shards and mark information as stale.
//
- vector<Shard> allShards;
- Shard::getAllShards( allShards );
- if ( allShards.size() < 2) {
- LOG(1) << "can't balance without more active shards" << endl;
+ ShardInfoMap shardInfo;
+ Status loadStatus = DistributionStatus::populateShardInfoMap(&shardInfo);
+
+ if (!loadStatus.isOK()) {
+ warning() << "failed to load shard metadata" << causedBy(loadStatus);
return;
}
-
- ShardInfoMap shardInfo;
- DistributionStatus::populateShardInfoMap(allShards, &shardInfo);
+ if (shardInfo.size() < 2) {
+ LOG(1) << "can't balance without more active shards";
+ return;
+ }
+
OCCASIONALLY warnOnMultiVersion( shardInfo );
//
@@ -403,11 +406,10 @@ namespace mongo {
continue;
}
- for ( vector<Shard>::iterator i=allShards.begin(); i!=allShards.end(); ++i ) {
+ for (ShardInfoMap::const_iterator i = shardInfo.begin(); i != shardInfo.end(); ++i) {
// this just makes sure there is an entry in shardToChunksMap for every shard
- Shard s = *i;
OwnedPointerVector<ChunkType>*& chunkList =
- shardToChunksMap.mutableMap()[s.getName()];
+ shardToChunksMap.mutableMap()[i->first];
if (chunkList == NULL) {
chunkList = new OwnedPointerVector<ChunkType>();
diff --git a/src/mongo/s/balancer_policy.cpp b/src/mongo/s/balancer_policy.cpp
index 31596b107e8..58e0c32272a 100644
--- a/src/mongo/s/balancer_policy.cpp
+++ b/src/mongo/s/balancer_policy.cpp
@@ -36,12 +36,12 @@
#include "mongo/s/balancer_policy.h"
#include "mongo/s/chunk.h"
#include "mongo/s/config.h"
+#include "mongo/s/type_shard.h"
#include "mongo/s/type_tags.h"
#include "mongo/util/log.h"
#include "mongo/util/stringutils.h"
#include "mongo/util/text.h"
-
namespace mongo {
string TagRange::toString() const {
@@ -232,30 +232,66 @@ namespace mongo {
}
}
- void DistributionStatus::populateShardInfoMap(const vector<Shard> allShards,
- ShardInfoMap* shardInfo) {
- for (vector<Shard>::const_iterator it = allShards.begin();
- it != allShards.end(); ++it ) {
- const Shard& shard = *it;
- ShardStatus status = shard.getStatus();
- shardInfo->insert(make_pair(shard.getName(),
- ShardInfo(shard.getMaxSizeMB(),
- status.dataSizeBytes() * 1024 * 1024,
- shard.isDraining(),
- shard.tags(),
- status.mongoVersion())));
+ Status DistributionStatus::populateShardInfoMap(ShardInfoMap* shardInfo) {
+ try {
+ ScopedDbConnection conn(configServer.getPrimary().getConnString(), 30);
+
+ auto_ptr<DBClientCursor> cursor(conn->query(ShardType::ConfigNS , Query()));
+ uassert(28597, "Failed to load shard config", cursor.get() != NULL);
+
+ while (cursor->more()) {
+ ShardType shard;
+ std::string errMsg;
+ bool parseOk = shard.parseBSON(cursor->next(), &errMsg);
+
+ if (!parseOk) {
+ return Status(ErrorCodes::UnsupportedFormat,
+ errMsg);
+ }
+
+ std::set<std::string> dummy;
+ ShardInfo newShardEntry(shard.getMaxSize(),
+ Shard::getShardDataSizeBytes(shard.getHost()) /
+ 1024 / 1024,
+ shard.getDraining(),
+ dummy,
+ Shard::getShardMongoVersion(shard.getHost()));
+
+ if (shard.isTagsSet()) {
+ BSONArrayIteratorSorted tagIter(shard.getTags());
+ while (tagIter.more()) {
+ BSONElement tagElement = tagIter.next();
+ if (tagElement.type() != String) {
+ return Status(ErrorCodes::UnsupportedFormat,
+ str::stream() << "shard tags only supports strings, "
+ << "found " << typeName(tagElement.type()));
+ }
+
+ newShardEntry.addTag(tagElement.String());
+ }
+ }
+
+ shardInfo->insert(make_pair(shard.getName(), newShardEntry));
+ }
+
+ conn.done();
+ }
+ catch (const DBException& ex) {
+ return ex.toStatus();
}
+
+ return Status::OK();
}
- void DistributionStatus::populateShardToChunksMap(const vector<Shard>& allShards,
+ void DistributionStatus::populateShardToChunksMap(const ShardInfoMap& allShards,
const ChunkManager& chunkMgr,
ShardToChunksMap* shardToChunksMap) {
// Makes sure there is an entry in shardToChunksMap for every shard.
- for (vector<Shard>::const_iterator it = allShards.begin();
+ for (ShardInfoMap::const_iterator it = allShards.begin();
it != allShards.end(); ++it) {
OwnedPointerVector<ChunkType>*& chunkList =
- (*shardToChunksMap)[it->getName()];
+ (*shardToChunksMap)[it->first];
if (chunkList == NULL) {
chunkList = new OwnedPointerVector<ChunkType>();
diff --git a/src/mongo/s/balancer_policy.h b/src/mongo/s/balancer_policy.h
index 0d29d5956e2..2ae774f2fc7 100644
--- a/src/mongo/s/balancer_policy.h
+++ b/src/mongo/s/balancer_policy.h
@@ -184,13 +184,16 @@ namespace mongo {
/** writes all state to log() */
void dump() const;
- static void populateShardInfoMap(const std::vector<Shard> allShards,
- ShardInfoMap* shardInfo);
+ /**
+ * Retrieves shard metadata information from the config server as well as some stats
+ * from the shards.
+ */
+ static Status populateShardInfoMap(ShardInfoMap* shardInfo);
/**
* Note: jumbo and versions are not set.
*/
- static void populateShardToChunksMap(const std::vector<Shard>& allShards,
+ static void populateShardToChunksMap(const ShardInfoMap& allShards,
const ChunkManager& chunkMgr,
ShardToChunksMap* shardToChunksMap);
diff --git a/src/mongo/s/chunk.cpp b/src/mongo/s/chunk.cpp
index e50a6930dde..61a6769320f 100644
--- a/src/mongo/s/chunk.cpp
+++ b/src/mongo/s/chunk.cpp
@@ -100,21 +100,24 @@ namespace mongo {
*/
static bool tryMoveToOtherShard(const ChunkManager& manager, const ChunkType& chunk) {
// reload sharding metadata before starting migration
- Shard::reloadShardInfo();
ChunkManagerPtr chunkMgr = manager.reload(false /* just reloaded in mulitsplit */);
- vector<Shard> allShards;
- Shard::getAllShards(allShards);
- if (allShards.size() < 2) {
- LOG(0) << "no need to move top chunk since there's only 1 shard" << endl;
+ ShardInfoMap shardInfo;
+ Status loadStatus = DistributionStatus::populateShardInfoMap(&shardInfo);
+
+ if (!loadStatus.isOK()) {
+ warning() << "failed to load shard metadata while trying to moveChunk after "
+ << "auto-splitting" << causedBy(loadStatus);
return false;
}
- ShardInfoMap shardInfo;
- DistributionStatus::populateShardInfoMap(allShards, &shardInfo);
+ if (shardInfo.size() < 2) {
+ LOG(0) << "no need to move top chunk since there's only 1 shard" << endl;
+ return false;
+ }
OwnedPointerMap<string, OwnedPointerVector<ChunkType> > shardToChunkMap;
- DistributionStatus::populateShardToChunksMap(allShards,
+ DistributionStatus::populateShardToChunksMap(shardInfo,
*chunkMgr,
&shardToChunkMap.mutableMap());
diff --git a/src/mongo/s/cluster_client_internal.h b/src/mongo/s/cluster_client_internal.h
index 5bf4658cd19..12c2eccf50d 100644
--- a/src/mongo/s/cluster_client_internal.h
+++ b/src/mongo/s/cluster_client_internal.h
@@ -102,7 +102,8 @@ namespace mongo {
// Helper function which throws on bad GLEs for non-SCC config servers
void _checkGLE(ScopedDbConnection& conn);
- // Helper function which throws for invalid cursor initialization
+ // Helper function which throws for invalid cursor initialization.
+ // Note: cursor ownership will be passed to this function.
DBClientCursor* _safeCursor(std::auto_ptr<DBClientCursor> cursor);
}
diff --git a/src/mongo/s/config.cpp b/src/mongo/s/config.cpp
index bf37bf9244c..9df7a847808 100644
--- a/src/mongo/s/config.cpp
+++ b/src/mongo/s/config.cpp
@@ -839,8 +839,7 @@ namespace mongo {
_primary = Shard(_primary.getName(),
ConnectionString(fullString, ConnectionString::SYNC),
_primary.getMaxSizeMB(),
- _primary.isDraining(),
- _primary.tags());
+ _primary.isDraining());
Shard::installShard(_primary.getName(), _primary);
LOG(1) << " config string : " << fullString << endl;
diff --git a/src/mongo/s/config.h b/src/mongo/s/config.h
index 87e901a520e..4284ecf463e 100644
--- a/src/mongo/s/config.h
+++ b/src/mongo/s/config.h
@@ -108,8 +108,7 @@ namespace mongo {
_primary("config",
"",
0 /* maxSize */,
- false /* draining */,
- BSONArray() /* tags */),
+ false /* draining */),
_shardingEnabled(false),
_lock("DBConfig") ,
_hitConfigServerLock( "DBConfig::_hitConfigServerLock" ) {
diff --git a/src/mongo/s/shard.cpp b/src/mongo/s/shard.cpp
index f67fb4522f8..43bcb57c0a9 100644
--- a/src/mongo/s/shard.cpp
+++ b/src/mongo/s/shard.cpp
@@ -133,8 +133,7 @@ namespace mongo {
ShardPtr shard = boost::make_shared<Shard>(shardData.getName(),
shardData.getHost(),
maxSize,
- isDraining,
- tags);
+ isDraining);
_lookup[shardData.getName()] = shard;
_installHost(shardData.getHost(), shard);
@@ -352,32 +351,23 @@ namespace mongo {
Shard::Shard(const std::string& name,
const std::string& addr,
long long maxSizeMB,
- bool isDraining,
- const BSONArray& tags):
+ bool isDraining):
_name(name),
_addr(addr),
_maxSizeMB(maxSizeMB),
_isDraining(isDraining) {
_setAddr(addr);
-
- BSONArrayIteratorSorted iter(tags);
- while (iter.more()) {
- BSONElement tag = iter.next();
- _tags.insert(tag.String());
- }
}
Shard::Shard(const std::string& name,
const ConnectionString& connStr,
long long maxSizeMB,
- bool isDraining,
- const set<string>& tags):
+ bool isDraining):
_name(name),
_addr(connStr.toString()),
_cs(connStr),
_maxSizeMB(maxSizeMB),
- _isDraining(isDraining),
- _tags(tags) {
+ _isDraining(isDraining) {
}
Shard Shard::findIfExists( const string& shardName ) {
@@ -450,22 +440,46 @@ namespace mongo {
return res;
}
- ShardStatus Shard::getStatus() const {
- BSONObj serverStatus = runCommand("admin", BSON("serverStatus" << 1));
+ string Shard::getShardMongoVersion(const string& shardHost) {
+ ScopedDbConnection conn(shardHost);
+ BSONObj serverStatus;
+ bool ok = conn->runCommand("admin", BSON("serverStatus" << 1), serverStatus);
+ conn.done();
+
+ uassert(28598,
+ str::stream() << "call to serverStatus on " << shardHost
+ << " failed: " << serverStatus,
+ ok);
+
BSONElement versionElement = serverStatus["version"];
uassert(28589, "version field not found in serverStatus",
versionElement.type() == String);
- string version = serverStatus["version"].String();
+ return serverStatus["version"].String();
+ }
+
+ long long Shard::getShardDataSizeBytes(const string& shardHost) {
+ ScopedDbConnection conn(shardHost);
+ BSONObj listDatabases;
+ bool ok = conn->runCommand("admin", BSON("listDatabases" << 1), listDatabases);
+ conn.done();
+
+ uassert(28599,
+ str::stream() << "call to listDatabases on " << shardHost
+ << " failed: " << listDatabases,
+ ok);
- BSONObj listDatabases = runCommand("admin", BSON("listDatabases" << 1));
BSONElement totalSizeElem = listDatabases["totalSize"];
uassert(28590, "totalSize field not found in listDatabases",
totalSizeElem.isNumber());
- long long dataSizeBytes = listDatabases["totalSize"].numberLong();
+ return listDatabases["totalSize"].numberLong();
+ }
- return ShardStatus(*this, dataSizeBytes, version);
+ ShardStatus Shard::getStatus() const {
+ return ShardStatus(*this,
+ getShardDataSizeBytes(getConnString()),
+ getShardMongoVersion(getConnString()));
}
void Shard::reloadShardInfo() {
diff --git a/src/mongo/s/shard.h b/src/mongo/s/shard.h
index a627c0e19dd..7f1f6ffadae 100644
--- a/src/mongo/s/shard.h
+++ b/src/mongo/s/shard.h
@@ -57,14 +57,12 @@ namespace mongo {
Shard(const std::string& name,
const std::string& addr,
long long maxSizeMB,
- bool isDraining,
- const BSONArray& tags);
+ bool isDraining);
Shard(const std::string& name,
const ConnectionString& connStr,
long long maxSizeMB,
- bool isDraining,
- const std::set<std::string>& tags);
+ bool isDraining);
Shard( const std::string& ident ) {
reset( ident );
@@ -75,8 +73,7 @@ namespace mongo {
_addr(other._addr),
_cs(other._cs),
_maxSizeMB(other._maxSizeMB),
- _isDraining(other._isDraining),
- _tags(other._tags) {
+ _isDraining(other._isDraining) {
}
static Shard make( const std::string& ident ) {
@@ -150,8 +147,21 @@ namespace mongo {
}
BSONObj runCommand( const std::string& db , const BSONObj& cmd ) const ;
+ /**
+ * Returns the version string from the shard based from the serverStatus command result.
+ */
+ static std::string getShardMongoVersion(const std::string& shardHost);
+
+ /**
+ * Returns the total data size in bytes the shard is currently using.
+ */
+ static long long getShardDataSizeBytes(const std::string& shardHost);
+
+ /**
+ * Returns metadata and stats for this shard.
+ */
ShardStatus getStatus() const ;
-
+
/**
* mostly for replica set
* retursn true if node is the shard
@@ -159,8 +169,6 @@ namespace mongo {
*/
bool containsNode( const std::string& node ) const;
- const std::set<std::string>& tags() const { return _tags; }
-
static void getAllShards( std::vector<Shard>& all );
static void printShardInfo( std::ostream& out );
static Shard lookupRSName( const std::string& name);
@@ -190,7 +198,6 @@ namespace mongo {
ConnectionString _cs;
long long _maxSizeMB; // in MBytes, 0 is unlimited
bool _isDraining; // shard is currently being removed
- std::set<std::string> _tags;
};
typedef boost::shared_ptr<Shard> ShardPtr;
diff --git a/src/mongo/s/shard_test.cpp b/src/mongo/s/shard_test.cpp
index 0e90f86e408..ff1fccc88d6 100644
--- a/src/mongo/s/shard_test.cpp
+++ b/src/mongo/s/shard_test.cpp
@@ -34,23 +34,23 @@
namespace mongo {
TEST( Shard, EqualityRs ) {
- Shard a("foo", "bar/a,b", 0, false, BSONArray());
- Shard b("foo", "bar/a,b", 0, false, BSONArray());
+ Shard a("foo", "bar/a,b", 0, false);
+ Shard b("foo", "bar/a,b", 0, false);
ASSERT_EQUALS( a, b );
- b = Shard("foo", "bar/b,a", 0, false, BSONArray());
+ b = Shard("foo", "bar/b,a", 0, false);
ASSERT_EQUALS( a, b );
}
TEST( Shard, EqualitySingle ) {
- ASSERT_EQUALS(Shard("foo", "b.foo.com:123", 0, false, BSONArray()),
- Shard("foo", "b.foo.com:123", 0, false, BSONArray()));
- ASSERT_NOT_EQUALS(Shard("foo", "b.foo.com:123", 0, false, BSONArray()),
- Shard("foo", "a.foo.com:123", 0, false, BSONArray()));
- ASSERT_NOT_EQUALS(Shard("foo", "b.foo.com:123", 0, false, BSONArray()),
- Shard("foo", "b.foo.com:124", 0, false, BSONArray()));
- ASSERT_NOT_EQUALS(Shard("foo", "b.foo.com:123", 0, false, BSONArray()),
- Shard("foa", "b.foo.com:123", 0, false, BSONArray()));
+ ASSERT_EQUALS(Shard("foo", "b.foo.com:123", 0, false),
+ Shard("foo", "b.foo.com:123", 0, false));
+ ASSERT_NOT_EQUALS(Shard("foo", "b.foo.com:123", 0, false),
+ Shard("foo", "a.foo.com:123", 0, false));
+ ASSERT_NOT_EQUALS(Shard("foo", "b.foo.com:123", 0, false),
+ Shard("foo", "b.foo.com:124", 0, false));
+ ASSERT_NOT_EQUALS(Shard("foo", "b.foo.com:123", 0, false),
+ Shard("foa", "b.foo.com:123", 0, false));
}
TEST( Shard, EqualitySync ) {
diff --git a/src/mongo/s/type_shard.cpp b/src/mongo/s/type_shard.cpp
index bf24b19b86a..2b9a4012696 100644
--- a/src/mongo/s/type_shard.cpp
+++ b/src/mongo/s/type_shard.cpp
@@ -38,8 +38,8 @@ namespace mongo {
const BSONField<std::string> ShardType::name("_id");
const BSONField<std::string> ShardType::host("host");
- const BSONField<bool> ShardType::draining("draining");
- const BSONField<long long> ShardType::maxSize("maxSize");
+ const BSONField<bool> ShardType::draining("draining", false);
+ const BSONField<long long> ShardType::maxSize("maxSize", 0);
const BSONField<BSONArray> ShardType::tags("tags");
ShardType::ShardType() {
View Lorry Controller Status