diff options
author | Geert Bosch <geert@mongodb.com> | 2018-07-08 20:37:53 -0400 |
---|---|---|
committer | Geert Bosch <geert@mongodb.com> | 2018-09-10 18:36:19 -0400 |
commit | e2be5a4e3684daf2410e4d5e31439c9669a38e6d (patch) | |
tree | 92dc24f4d7b5637ed44c8c58bdc7f4aad0ed0d74 /src/mongo | |
parent | 1070aa3880ac73bc1923b44a372c61c209a35f61 (diff) | |
download | mongo-e2be5a4e3684daf2410e4d5e31439c9669a38e6d.tar.gz |
SERVER-36014 Remove support for IndexVersion V0
Diffstat (limited to 'src/mongo')
-rw-r--r-- | src/mongo/db/catalog/collection_compact.cpp | 36 | ||||
-rw-r--r-- | src/mongo/db/catalog/index_key_validate.cpp | 1 | ||||
-rw-r--r-- | src/mongo/db/catalog/index_key_validate_test.cpp | 66 | ||||
-rw-r--r-- | src/mongo/db/cloner.cpp | 11 | ||||
-rw-r--r-- | src/mongo/db/index/btree_access_method.cpp | 8 | ||||
-rw-r--r-- | src/mongo/db/index/btree_key_generator.cpp | 271 | ||||
-rw-r--r-- | src/mongo/db/index/btree_key_generator.h | 121 | ||||
-rw-r--r-- | src/mongo/db/index/btree_key_generator_test.cpp | 3 | ||||
-rw-r--r-- | src/mongo/db/index/index_descriptor.cpp | 4 | ||||
-rw-r--r-- | src/mongo/db/index/index_descriptor.h | 8 | ||||
-rw-r--r-- | src/mongo/db/index/sort_key_generator.cpp | 2 | ||||
-rw-r--r-- | src/mongo/db/repair_database.cpp | 80 | ||||
-rw-r--r-- | src/mongo/db/storage/kv/kv_engine_test_harness.cpp | 27 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_index_test.cpp | 2 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_standard_index_test.cpp | 2 |
15 files changed, 197 insertions, 445 deletions
diff --git a/src/mongo/db/catalog/collection_compact.cpp b/src/mongo/db/catalog/collection_compact.cpp index fa94ca47f20..20190d1ea27 100644 --- a/src/mongo/db/catalog/collection_compact.cpp +++ b/src/mongo/db/catalog/collection_compact.cpp @@ -46,37 +46,7 @@ namespace mongo { -using std::endl; -using std::vector; - -using IndexVersion = IndexDescriptor::IndexVersion; - namespace { -BSONObj _compactAdjustIndexSpec(const BSONObj& oldSpec) { - BSONObjBuilder bob; - - for (auto&& indexSpecElem : oldSpec) { - auto indexSpecElemFieldName = indexSpecElem.fieldNameStringData(); - if (IndexDescriptor::kIndexVersionFieldName == indexSpecElemFieldName) { - IndexVersion indexVersion = static_cast<IndexVersion>(indexSpecElem.numberInt()); - if (IndexVersion::kV0 == indexVersion) { - // We automatically upgrade v=0 indexes to v=1 indexes. - bob.append(IndexDescriptor::kIndexVersionFieldName, - static_cast<int>(IndexVersion::kV1)); - } else { - bob.append(IndexDescriptor::kIndexVersionFieldName, static_cast<int>(indexVersion)); - } - } else if (IndexDescriptor::kBackgroundFieldName == indexSpecElemFieldName) { - // Create the new index in the foreground. - continue; - } else { - bob.append(indexSpecElem); - } - } - - return bob.obj(); -} - class MyCompactAdaptor : public RecordStoreCompactAdaptor { public: MyCompactAdaptor(Collection* collection, MultiIndexBlock* indexBlock) @@ -144,13 +114,15 @@ StatusWith<CompactStats> CollectionImpl::compact(OperationContext* opCtx, return StatusWith<CompactStats>(ErrorCodes::BadValue, "cannot compact when indexes in progress"); - vector<BSONObj> indexSpecs; + std::vector<BSONObj> indexSpecs; { IndexCatalog::IndexIterator ii(_indexCatalog.getIndexIterator(opCtx, false)); while (ii.more()) { IndexDescriptor* descriptor = ii.next(); - const BSONObj spec = _compactAdjustIndexSpec(descriptor->infoObj()); + // Compact always creates the new index in the foreground. + const BSONObj spec = + descriptor->infoObj().removeField(IndexDescriptor::kBackgroundFieldName); const BSONObj key = spec.getObjectField("key"); const Status keyStatus = index_key_validate::validateKeyPattern(key, descriptor->version()); diff --git a/src/mongo/db/catalog/index_key_validate.cpp b/src/mongo/db/catalog/index_key_validate.cpp index e8ee2893d0e..8a0bcc191c1 100644 --- a/src/mongo/db/catalog/index_key_validate.cpp +++ b/src/mongo/db/catalog/index_key_validate.cpp @@ -133,7 +133,6 @@ Status validateKeyPattern(const BSONObj& key, IndexDescriptor::IndexVersion inde BSONElement keyElement = it.next(); switch (indexVersion) { - case IndexVersion::kV0: case IndexVersion::kV1: { if (keyElement.type() == BSONType::Object || keyElement.type() == BSONType::Array) { return {code, diff --git a/src/mongo/db/catalog/index_key_validate_test.cpp b/src/mongo/db/catalog/index_key_validate_test.cpp index 1d170fbbde7..9fbd4f345d4 100644 --- a/src/mongo/db/catalog/index_key_validate_test.cpp +++ b/src/mongo/db/catalog/index_key_validate_test.cpp @@ -93,12 +93,10 @@ TEST(IndexKeyValidateTest, KeyElementValueOfZeroFailsForV2Indexes) { validateKeyPattern(BSON("x" << -0.0), IndexVersion::kV2)); } -TEST(IndexKeyValidateTest, KeyElementValueOfZeroSucceedsForV0AndV1Indexes) { - for (auto indexVersion : {IndexVersion::kV0, IndexVersion::kV1}) { - ASSERT_OK(validateKeyPattern(BSON("x" << 0), indexVersion)); - ASSERT_OK(validateKeyPattern(BSON("x" << 0.0), indexVersion)); - ASSERT_OK(validateKeyPattern(BSON("x" << -0.0), indexVersion)); - } +TEST(IndexKeyValidateTest, KeyElementValueOfZeroSucceedsForV1Indexes) { + ASSERT_OK(validateKeyPattern(BSON("x" << 0), IndexVersion::kV1)); + ASSERT_OK(validateKeyPattern(BSON("x" << 0.0), IndexVersion::kV1)); + ASSERT_OK(validateKeyPattern(BSON("x" << -0.0), IndexVersion::kV1)); } TEST(IndexKeyValidateTest, KeyElementValueOfNaNFailsForV2Indexes) { @@ -113,15 +111,13 @@ TEST(IndexKeyValidateTest, KeyElementValueOfNaNFailsForV2Indexes) { } } -TEST(IndexKeyValidateTest, KeyElementValueOfNaNSucceedsForV0AndV1Indexes) { +TEST(IndexKeyValidateTest, KeyElementValueOfNaNSucceedsForV1Indexes) { if (std::numeric_limits<double>::has_quiet_NaN) { - for (auto indexVersion : {IndexVersion::kV0, IndexVersion::kV1}) { - double nan = std::numeric_limits<double>::quiet_NaN(); - ASSERT_OK(validateKeyPattern(BSON("x" << nan), indexVersion)); - ASSERT_OK(validateKeyPattern(BSON("a" << nan << "b" - << "2d"), - indexVersion)); - } + double nan = std::numeric_limits<double>::quiet_NaN(); + ASSERT_OK(validateKeyPattern(BSON("x" << nan), IndexVersion::kV1)); + ASSERT_OK(validateKeyPattern(BSON("a" << nan << "b" + << "2d"), + IndexVersion::kV1)); } } @@ -160,16 +156,14 @@ TEST(IndexKeyValidateTest, KeyElementBooleanValueFailsForV2Indexes) { IndexVersion::kV2)); } -TEST(IndexKeyValidateTest, KeyElementBooleanValueSucceedsForV0AndV1Indexes) { - for (auto indexVersion : {IndexVersion::kV0, IndexVersion::kV1}) { - ASSERT_OK(validateKeyPattern(BSON("x" << true), indexVersion)); - ASSERT_OK(validateKeyPattern(BSON("x" << false), indexVersion)); - ASSERT_OK(validateKeyPattern(BSON("a" - << "2dsphere" - << "b" - << true), - indexVersion)); - } +TEST(IndexKeyValidateTest, KeyElementBooleanValueSucceedsForV1Indexes) { + ASSERT_OK(validateKeyPattern(BSON("x" << true), IndexVersion::kV1)); + ASSERT_OK(validateKeyPattern(BSON("x" << false), IndexVersion::kV1)); + ASSERT_OK(validateKeyPattern(BSON("a" + << "2dsphere" + << "b" + << true), + IndexVersion::kV1)); } TEST(IndexKeyValidateTest, KeyElementNullValueFailsForV2Indexes) { @@ -177,10 +171,8 @@ TEST(IndexKeyValidateTest, KeyElementNullValueFailsForV2Indexes) { validateKeyPattern(BSON("x" << BSONNULL), IndexVersion::kV2)); } -TEST(IndexKeyValidateTest, KeyElementNullValueSucceedsForV0AndV1Indexes) { - for (auto indexVersion : {IndexVersion::kV0, IndexVersion::kV1}) { - ASSERT_OK(validateKeyPattern(BSON("x" << BSONNULL), indexVersion)); - } +TEST(IndexKeyValidateTest, KeyElementNullValueSucceedsForV1Indexes) { + ASSERT_OK(validateKeyPattern(BSON("x" << BSONNULL), IndexVersion::kV1)); } TEST(IndexKeyValidateTest, KeyElementUndefinedValueFailsForV2Indexes) { @@ -188,10 +180,8 @@ TEST(IndexKeyValidateTest, KeyElementUndefinedValueFailsForV2Indexes) { validateKeyPattern(BSON("x" << BSONUndefined), IndexVersion::kV2)); } -TEST(IndexKeyValidateTest, KeyElementUndefinedValueSucceedsForV0AndV1Indexes) { - for (auto indexVersion : {IndexVersion::kV0, IndexVersion::kV1}) { - ASSERT_OK(validateKeyPattern(BSON("x" << BSONUndefined), indexVersion)); - } +TEST(IndexKeyValidateTest, KeyElementUndefinedValueSucceedsForV1Indexes) { + ASSERT_OK(validateKeyPattern(BSON("x" << BSONUndefined), IndexVersion::kV1)); } TEST(IndexKeyValidateTest, KeyElementMinKeyValueFailsForV2Indexes) { @@ -199,10 +189,8 @@ TEST(IndexKeyValidateTest, KeyElementMinKeyValueFailsForV2Indexes) { validateKeyPattern(BSON("x" << MINKEY), IndexVersion::kV2)); } -TEST(IndexKeyValidateTest, KeyElementMinKeyValueSucceedsForV0AndV1Indexes) { - for (auto indexVersion : {IndexVersion::kV0, IndexVersion::kV1}) { - ASSERT_OK(validateKeyPattern(BSON("x" << MINKEY), indexVersion)); - } +TEST(IndexKeyValidateTest, KeyElementMinKeyValueSucceedsForV1Indexes) { + ASSERT_OK(validateKeyPattern(BSON("x" << MINKEY), IndexVersion::kV1)); } TEST(IndexKeyValidateTest, KeyElementMaxKeyValueFailsForV2Indexes) { @@ -210,10 +198,8 @@ TEST(IndexKeyValidateTest, KeyElementMaxKeyValueFailsForV2Indexes) { validateKeyPattern(BSON("x" << MAXKEY), IndexVersion::kV2)); } -TEST(IndexKeyValidateTest, KeyElementMaxKeyValueSucceedsForV0AndV1Indexes) { - for (auto indexVersion : {IndexVersion::kV0, IndexVersion::kV1}) { - ASSERT_OK(validateKeyPattern(BSON("x" << MAXKEY), indexVersion)); - } +TEST(IndexKeyValidateTest, KeyElementMaxKeyValueSucceedsForV1Indexes) { + ASSERT_OK(validateKeyPattern(BSON("x" << MAXKEY), IndexVersion::kV1)); } TEST(IndexKeyValidateTest, KeyElementObjectValueFails) { diff --git a/src/mongo/db/cloner.cpp b/src/mongo/db/cloner.cpp index 742b42026ca..7217687e93e 100644 --- a/src/mongo/db/cloner.cpp +++ b/src/mongo/db/cloner.cpp @@ -96,16 +96,7 @@ BSONObj fixIndexSpec(const string& newDbName, BSONObj indexSpec) { for (auto&& indexSpecElem : indexSpec) { auto indexSpecElemFieldName = indexSpecElem.fieldNameStringData(); - if (IndexDescriptor::kIndexVersionFieldName == indexSpecElemFieldName) { - IndexVersion indexVersion = static_cast<IndexVersion>(indexSpecElem.numberInt()); - if (IndexVersion::kV0 == indexVersion) { - // We automatically upgrade v=0 indexes to v=1 indexes. - bob.append(IndexDescriptor::kIndexVersionFieldName, - static_cast<int>(IndexVersion::kV1)); - } else { - bob.append(IndexDescriptor::kIndexVersionFieldName, static_cast<int>(indexVersion)); - } - } else if (IndexDescriptor::kNamespaceFieldName == indexSpecElemFieldName) { + if (IndexDescriptor::kNamespaceFieldName == indexSpecElemFieldName) { uassert(10024, "bad ns field for index during dbcopy", indexSpecElem.type() == String); const char* p = strchr(indexSpecElem.valuestr(), '.'); uassert(10025, "bad ns field for index during dbcopy [2]", p); diff --git a/src/mongo/db/index/btree_access_method.cpp b/src/mongo/db/index/btree_access_method.cpp index ee599f5d23e..b18a86598be 100644 --- a/src/mongo/db/index/btree_access_method.cpp +++ b/src/mongo/db/index/btree_access_method.cpp @@ -54,12 +54,8 @@ BtreeAccessMethod::BtreeAccessMethod(IndexCatalogEntry* btreeState, SortedDataIn fixed.push_back(BSONElement()); } - _keyGenerator = BtreeKeyGenerator::make(_descriptor->version(), - fieldNames, - fixed, - _descriptor->isSparse(), - btreeState->getCollator()); - massert(16745, "Invalid index version for key generation.", _keyGenerator); + _keyGenerator = std::make_unique<BtreeKeyGenerator>( + fieldNames, fixed, _descriptor->isSparse(), btreeState->getCollator()); } void BtreeAccessMethod::doGetKeys(const BSONObj& obj, diff --git a/src/mongo/db/index/btree_key_generator.cpp b/src/mongo/db/index/btree_key_generator.cpp index d1737ddf314..a02ca40c5cc 100644 --- a/src/mongo/db/index/btree_key_generator.cpp +++ b/src/mongo/db/index/btree_key_generator.cpp @@ -56,8 +56,13 @@ const BSONElement undefinedElt = undefinedObj.firstElement(); BtreeKeyGenerator::BtreeKeyGenerator(std::vector<const char*> fieldNames, std::vector<BSONElement> fixed, - bool isSparse) - : _fieldNames(fieldNames), _isSparse(isSparse), _fixed(fixed) { + bool isSparse, + const CollatorInterface* collator) + : _fieldNames(fieldNames), + _isSparse(isSparse), + _fixed(fixed), + _emptyPositionalInfo(fieldNames.size()), + _collator(collator) { BSONObjBuilder nullKeyBuilder; for (size_t i = 0; i < fieldNames.size(); ++i) { nullKeyBuilder.appendNull(""); @@ -65,31 +70,10 @@ BtreeKeyGenerator::BtreeKeyGenerator(std::vector<const char*> fieldNames, _nullKey = nullKeyBuilder.obj(); _isIdIndex = fieldNames.size() == 1 && std::string("_id") == fieldNames[0]; -} - -std::unique_ptr<BtreeKeyGenerator> BtreeKeyGenerator::make(IndexVersion indexVersion, - std::vector<const char*> fieldNames, - std::vector<BSONElement> fixed, - bool isSparse, - const CollatorInterface* collator) { - switch (indexVersion) { - case IndexVersion::kV0: - return stdx::make_unique<BtreeKeyGeneratorV0>(fieldNames, fixed, isSparse); - case IndexVersion::kV1: - case IndexVersion::kV2: - return stdx::make_unique<BtreeKeyGeneratorV1>(fieldNames, fixed, isSparse, collator); - } - return nullptr; -} - -void BtreeKeyGenerator::getKeys(const BSONObj& obj, - BSONObjSet* keys, - MultikeyPaths* multikeyPaths) const { - // '_fieldNames' and '_fixed' are passed by value so that they can be mutated as part of the - // getKeys call. :| - getKeysImpl(_fieldNames, _fixed, obj, keys, multikeyPaths); - if (keys->empty() && !_isSparse) { - keys->insert(_nullKey); + for (const char* fieldName : fieldNames) { + size_t pathLength = FieldRef{fieldName}.numParts(); + invariant(pathLength > 0); + _pathLengths.push_back(pathLength); } } @@ -99,159 +83,10 @@ static void assertParallelArrays(const char* first, const char* second) { uasserted(ErrorCodes::CannotIndexParallelArrays, ss.str()); } -BtreeKeyGeneratorV0::BtreeKeyGeneratorV0(std::vector<const char*> fieldNames, - std::vector<BSONElement> fixed, - bool isSparse) - : BtreeKeyGenerator(fieldNames, fixed, isSparse) {} - -void BtreeKeyGeneratorV0::getKeysImpl(std::vector<const char*> fieldNames, - std::vector<BSONElement> fixed, - const BSONObj& obj, - BSONObjSet* keys, - MultikeyPaths* multikeyPaths) const { - if (_isIdIndex) { - // we special case for speed - BSONElement e = obj["_id"]; - if (e.eoo()) { - keys->insert(_nullKey); - } else { - int size = e.size() + 5 /* bson over head*/ - 3 /* remove _id string */; - BSONObjBuilder b(size); - b.appendAs(e, ""); - keys->insert(b.obj()); - invariant(keys->begin()->objsize() == size); - } - return; - } - - BSONElement arrElt; - unsigned arrIdx = ~0; - unsigned numNotFound = 0; - - for (unsigned i = 0; i < fieldNames.size(); ++i) { - if (*fieldNames[i] == '\0') - continue; - - BSONElement e = dps::extractElementAtPathOrArrayAlongPath(obj, fieldNames[i]); - - if (e.eoo()) { - e = nullElt; // no matching field - numNotFound++; - } - - if (e.type() != Array) - fieldNames[i] = ""; // no matching field or non-array match - - if (*fieldNames[i] == '\0') - // no need for further object expansion (though array expansion still possible) - fixed[i] = e; - - if (e.type() == Array && arrElt.eoo()) { - // we only expand arrays on a single path -- track the path here - arrIdx = i; - arrElt = e; - } - - // enforce single array path here - if (e.type() == Array && e.rawdata() != arrElt.rawdata()) { - assertParallelArrays(e.fieldName(), arrElt.fieldName()); - } - } - - bool allFound = true; // have we found elements for all field names in the key spec? - for (std::vector<const char*>::const_iterator i = fieldNames.begin(); i != fieldNames.end(); - ++i) { - if (**i != '\0') { - allFound = false; - break; - } - } - - if (_isSparse && numNotFound == _fieldNames.size()) { - // we didn't find any fields - // so we're not going to index this document - return; - } - - bool insertArrayNull = false; - - if (allFound) { - if (arrElt.eoo()) { - // no terminal array element to expand - BSONObjBuilder b(_sizeTracker); - for (std::vector<BSONElement>::iterator i = fixed.begin(); i != fixed.end(); ++i) - b.appendAs(*i, ""); - keys->insert(b.obj()); - } else { - // terminal array element to expand, so generate all keys - BSONObjIterator i(arrElt.embeddedObject()); - if (i.more()) { - while (i.more()) { - BSONObjBuilder b(_sizeTracker); - for (unsigned j = 0; j < fixed.size(); ++j) { - if (j == arrIdx) - b.appendAs(i.next(), ""); - else - b.appendAs(fixed[j], ""); - } - keys->insert(b.obj()); - } - } else if (fixed.size() > 1) { - insertArrayNull = true; - } - } - } else { - // nonterminal array element to expand, so recurse - verify(!arrElt.eoo()); - BSONObjIterator i(arrElt.embeddedObject()); - if (i.more()) { - while (i.more()) { - BSONElement e = i.next(); - if (e.type() == Object) { - getKeysImpl(fieldNames, fixed, e.embeddedObject(), keys, multikeyPaths); - } - } - } else { - insertArrayNull = true; - } - } - - if (insertArrayNull) { - // x : [] - need to insert undefined - BSONObjBuilder b(_sizeTracker); - for (unsigned j = 0; j < fixed.size(); ++j) { - if (j == arrIdx) { - b.appendUndefined(""); - } else { - BSONElement e = fixed[j]; - if (e.eoo()) - b.appendNull(""); - else - b.appendAs(e, ""); - } - } - keys->insert(b.obj()); - } -} - -BtreeKeyGeneratorV1::BtreeKeyGeneratorV1(std::vector<const char*> fieldNames, - std::vector<BSONElement> fixed, - bool isSparse, - const CollatorInterface* collator) - : BtreeKeyGenerator(fieldNames, fixed, isSparse), - _emptyPositionalInfo(fieldNames.size()), - _collator(collator) { - for (const char* fieldName : fieldNames) { - size_t pathLength = FieldRef{fieldName}.numParts(); - invariant(pathLength > 0); - _pathLengths.push_back(pathLength); - } -} - -BSONElement BtreeKeyGeneratorV1::extractNextElement(const BSONObj& obj, - const PositionalPathInfo& positionalInfo, - const char** field, - bool* arrayNestedArray) const { +BSONElement BtreeKeyGenerator::_extractNextElement(const BSONObj& obj, + const PositionalPathInfo& positionalInfo, + const char** field, + bool* arrayNestedArray) const { std::string firstField = mongoutils::str::before(*field, '.'); bool haveObjField = !obj.getField(firstField).eoo(); BSONElement arrField = positionalInfo.positionallyIndexedElt; @@ -280,16 +115,16 @@ BSONElement BtreeKeyGeneratorV1::extractNextElement(const BSONObj& obj, return BSONElement(); } -void BtreeKeyGeneratorV1::_getKeysArrEltFixed(std::vector<const char*>* fieldNames, - std::vector<BSONElement>* fixed, - const BSONElement& arrEntry, - BSONObjSet* keys, - unsigned numNotFound, - const BSONElement& arrObjElt, - const std::set<size_t>& arrIdxs, - bool mayExpandArrayUnembedded, - const std::vector<PositionalPathInfo>& positionalInfo, - MultikeyPaths* multikeyPaths) const { +void BtreeKeyGenerator::_getKeysArrEltFixed(std::vector<const char*>* fieldNames, + std::vector<BSONElement>* fixed, + const BSONElement& arrEntry, + BSONObjSet* keys, + unsigned numNotFound, + const BSONElement& arrObjElt, + const std::set<size_t>& arrIdxs, + bool mayExpandArrayUnembedded, + const std::vector<PositionalPathInfo>& positionalInfo, + MultikeyPaths* multikeyPaths) const { // Set up any terminal array values. for (const auto idx : arrIdxs) { if (*(*fieldNames)[idx] == '\0') { @@ -298,20 +133,18 @@ void BtreeKeyGeneratorV1::_getKeysArrEltFixed(std::vector<const char*>* fieldNam } // Recurse. - getKeysImplWithArray(*fieldNames, - *fixed, - arrEntry.type() == Object ? arrEntry.embeddedObject() : BSONObj(), - keys, - numNotFound, - positionalInfo, - multikeyPaths); + _getKeysWithArray(*fieldNames, + *fixed, + arrEntry.type() == Object ? arrEntry.embeddedObject() : BSONObj(), + keys, + numNotFound, + positionalInfo, + multikeyPaths); } -void BtreeKeyGeneratorV1::getKeysImpl(std::vector<const char*> fieldNames, - std::vector<BSONElement> fixed, - const BSONObj& obj, - BSONObjSet* keys, - MultikeyPaths* multikeyPaths) const { +void BtreeKeyGenerator::getKeys(const BSONObj& obj, + BSONObjSet* keys, + MultikeyPaths* multikeyPaths) const { if (_isIdIndex) { // we special case for speed BSONElement e = obj["_id"]; @@ -336,25 +169,27 @@ void BtreeKeyGeneratorV1::getKeysImpl(std::vector<const char*> fieldNames, if (multikeyPaths) { multikeyPaths->resize(1); } - return; + } else { + if (multikeyPaths) { + invariant(multikeyPaths->empty()); + multikeyPaths->resize(_fieldNames.size()); + } + // '_fieldNames' and '_fixed' are passed by value so that their copies can be mutated as + // part of the _getKeysWithArray method. + _getKeysWithArray(_fieldNames, _fixed, obj, keys, 0, _emptyPositionalInfo, multikeyPaths); } - - if (multikeyPaths) { - invariant(multikeyPaths->empty()); - multikeyPaths->resize(fieldNames.size()); + if (keys->empty() && !_isSparse) { + keys->insert(_nullKey); } - getKeysImplWithArray( - std::move(fieldNames), std::move(fixed), obj, keys, 0, _emptyPositionalInfo, multikeyPaths); } -void BtreeKeyGeneratorV1::getKeysImplWithArray( - std::vector<const char*> fieldNames, - std::vector<BSONElement> fixed, - const BSONObj& obj, - BSONObjSet* keys, - unsigned numNotFound, - const std::vector<PositionalPathInfo>& positionalInfo, - MultikeyPaths* multikeyPaths) const { +void BtreeKeyGenerator::_getKeysWithArray(std::vector<const char*> fieldNames, + std::vector<BSONElement> fixed, + const BSONObj& obj, + BSONObjSet* keys, + unsigned numNotFound, + const std::vector<PositionalPathInfo>& positionalInfo, + MultikeyPaths* multikeyPaths) const { BSONElement arrElt; // A set containing the position of any indexed fields in the key pattern that traverse through @@ -392,7 +227,7 @@ void BtreeKeyGeneratorV1::getKeysImplWithArray( bool arrayNestedArray; // Extract element matching fieldName[ i ] from object xor array. BSONElement e = - extractNextElement(obj, positionalInfo[i], &fieldNames[i], &arrayNestedArray); + _extractNextElement(obj, positionalInfo[i], &fieldNames[i], &arrayNestedArray); if (e.eoo()) { // if field not present, set to null diff --git a/src/mongo/db/index/btree_key_generator.h b/src/mongo/db/index/btree_key_generator.h index 56f47aee09a..68d3ae664aa 100644 --- a/src/mongo/db/index/btree_key_generator.h +++ b/src/mongo/db/index/btree_key_generator.h @@ -47,71 +47,35 @@ class CollatorInterface; */ class BtreeKeyGenerator { public: + /** + * Provides a context to generate keys based on names in 'fieldNames'. The 'fixed' argument + * specifies values that have already been identified for their corresponding fields. + */ BtreeKeyGenerator(std::vector<const char*> fieldNames, std::vector<BSONElement> fixed, - bool isSparse); - - virtual ~BtreeKeyGenerator() {} - - static std::unique_ptr<BtreeKeyGenerator> make(IndexDescriptor::IndexVersion indexVersion, - std::vector<const char*> fieldNames, - std::vector<BSONElement> fixed, - bool isSparse, - const CollatorInterface* collator); + bool isSparse, + const CollatorInterface* collator); + /** + * Generates the index keys for the document 'obj', and stores them in the set 'keys'. + * + * If the 'multikeyPaths' pointer is non-null, then it must point to an empty vector. If this + * index type supports tracking path-level multikey information, then this function resizes + * 'multikeyPaths' to have the same number of elements as the index key pattern and fills each + * element with the prefixes of the indexed field that would cause this index to be multikey as + * a result of inserting 'keys'. + */ void getKeys(const BSONObj& obj, BSONObjSet* keys, MultikeyPaths* multikeyPaths) const; -protected: - // These are used by the getKeysImpl(s) below. +private: + // These are used by getKeys below. std::vector<const char*> _fieldNames; bool _isIdIndex; bool _isSparse; - BSONObj _nullKey; // a full key with all fields null + BSONObj _nullKey; // A full key with all fields null. BSONSizeTracker _sizeTracker; -private: - virtual void getKeysImpl(std::vector<const char*> fieldNames, - std::vector<BSONElement> fixed, - const BSONObj& obj, - BSONObjSet* keys, - MultikeyPaths* multikeyPaths) const = 0; - std::vector<BSONElement> _fixed; -}; - -class BtreeKeyGeneratorV0 : public BtreeKeyGenerator { -public: - BtreeKeyGeneratorV0(std::vector<const char*> fieldNames, - std::vector<BSONElement> fixed, - bool isSparse); - - virtual ~BtreeKeyGeneratorV0() {} - -private: - /** - * Generates the index keys for the document 'obj' and stores them in the set 'keys'. - * - * It isn't possible to create a v0 index, so it's unnecessary to track the prefixes of the - * indexed fields that cause the index to be mulitkey. This function therefore ignores its - * 'multikeyPaths' parameter. - */ - void getKeysImpl(std::vector<const char*> fieldNames, - std::vector<BSONElement> fixed, - const BSONObj& obj, - BSONObjSet* keys, - MultikeyPaths* multikeyPaths) const final; -}; - -class BtreeKeyGeneratorV1 : public BtreeKeyGenerator { -public: - BtreeKeyGeneratorV1(std::vector<const char*> fieldNames, - std::vector<BSONElement> fixed, - bool isSparse, - const CollatorInterface* collator); - - virtual ~BtreeKeyGeneratorV1() {} - -private: /** * Stores info regarding traversal of a positional path. A path through a document is * considered positional if this path element names an array element. Generally this means @@ -166,37 +130,18 @@ private: }; /** - * Generates the index keys for the document 'obj' and stores them in the set 'keys'. - * - * @param fieldNames - fields to index, may be postfixes in recursive calls - * @param fixed - values that have already been identified for their index fields - * @param obj - object from which keys should be extracted, based on names in fieldNames - * @param keys - set where index keys are written - * - * If the 'multikeyPaths' pointer is non-null, then it must point to an empty vector. If this - * index type supports tracking path-level multikey information, then this function resizes - * 'multikeyPaths' to have the same number of elements as the index key pattern and fills each - * element with the prefixes of the indexed field that would cause this index to be multikey as - * a result of inserting 'keys'. + * This recursive method does the heavy-lifting for getKeys(). */ - void getKeysImpl(std::vector<const char*> fieldNames, - std::vector<BSONElement> fixed, - const BSONObj& obj, - BSONObjSet* keys, - MultikeyPaths* multikeyPaths) const final; + void _getKeysWithArray(std::vector<const char*> fieldNames, + std::vector<BSONElement> fixed, + const BSONObj& obj, + BSONObjSet* keys, + unsigned numNotFound, + const std::vector<PositionalPathInfo>& positionalInfo, + MultikeyPaths* multikeyPaths) const; /** - * This recursive method does the heavy-lifting for getKeysImpl(). - */ - void getKeysImplWithArray(std::vector<const char*> fieldNames, - std::vector<BSONElement> fixed, - const BSONObj& obj, - BSONObjSet* keys, - unsigned numNotFound, - const std::vector<PositionalPathInfo>& positionalInfo, - MultikeyPaths* multikeyPaths) const; - /** - * A call to getKeysImplWithArray() begins by calling this for each field in the key pattern. It + * A call to _getKeysWithArray() begins by calling this for each field in the key pattern. It * traverses the path '*field' in 'obj' until either reaching the end of the path or an array * element. * @@ -223,20 +168,20 @@ private: * to "b.c". * * extractNextElement() will then be called from a recursive call to - * getKeysImplWithArray() for each array element. For instance, it will get called with + * _getKeysWithArray() for each array element. For instance, it will get called with * 'obj' {b: {c: 98}} and '*field' pointing to "b.c". It will return element 98 and * set '*field' to "". Similarly, it will return elemtn 99 and set '*field' to "" for * the second array element. */ - BSONElement extractNextElement(const BSONObj& obj, - const PositionalPathInfo& positionalInfo, - const char** field, - bool* arrayNestedArray) const; + BSONElement _extractNextElement(const BSONObj& obj, + const PositionalPathInfo& positionalInfo, + const char** field, + bool* arrayNestedArray) const; /** * Sets extracted elements in 'fixed' for field paths that we have traversed to the end. * - * Then calls getKeysImplWithArray() recursively. + * Then calls _getKeysWithArray() recursively. */ void _getKeysArrEltFixed(std::vector<const char*>* fieldNames, std::vector<BSONElement>* fixed, diff --git a/src/mongo/db/index/btree_key_generator_test.cpp b/src/mongo/db/index/btree_key_generator_test.cpp index 27336b80c02..5f82ce91462 100644 --- a/src/mongo/db/index/btree_key_generator_test.cpp +++ b/src/mongo/db/index/btree_key_generator_test.cpp @@ -117,8 +117,7 @@ bool testKeygen(const BSONObj& kp, fixed.push_back(BSONElement()); } - unique_ptr<BtreeKeyGenerator> keyGen( - new BtreeKeyGeneratorV1(fieldNames, fixed, sparse, collator)); + auto keyGen = std::make_unique<BtreeKeyGenerator>(fieldNames, fixed, sparse, collator); // // Step 2: ask 'keyGen' to generate index keys for the object 'obj' and report any prefixes of diff --git a/src/mongo/db/index/index_descriptor.cpp b/src/mongo/db/index/index_descriptor.cpp index 1514acd6f81..58cabd564f7 100644 --- a/src/mongo/db/index/index_descriptor.cpp +++ b/src/mongo/db/index/index_descriptor.cpp @@ -99,8 +99,6 @@ constexpr StringData IndexDescriptor::kWeightsFieldName; bool IndexDescriptor::isIndexVersionSupported(IndexVersion indexVersion) { switch (indexVersion) { - case IndexVersion::kV0: - return false; case IndexVersion::kV1: case IndexVersion::kV2: return true; @@ -117,8 +115,6 @@ Status IndexDescriptor::isIndexVersionAllowedForCreation( const ServerGlobalParams::FeatureCompatibility& featureCompatibility, const BSONObj& indexSpec) { switch (indexVersion) { - case IndexVersion::kV0: - break; case IndexVersion::kV1: case IndexVersion::kV2: return Status::OK(); diff --git a/src/mongo/db/index/index_descriptor.h b/src/mongo/db/index/index_descriptor.h index b54ccb7453c..7e509d9923b 100644 --- a/src/mongo/db/index/index_descriptor.h +++ b/src/mongo/db/index/index_descriptor.h @@ -57,7 +57,7 @@ class OperationContext; */ class IndexDescriptor { public: - enum class IndexVersion { kV0 = 0, kV1 = 1, kV2 = 2 }; + enum class IndexVersion { kV1 = 1, kV2 = 2 }; static constexpr IndexVersion kLatestIndexVersion = IndexVersion::kV2; static constexpr StringData k2dIndexBitsFieldName = "bits"_sd; @@ -106,11 +106,9 @@ public: _cachedEntry(NULL) { _indexNamespace = makeIndexNamespace(_parentNS, _indexName); - _version = IndexVersion::kV0; BSONElement e = _infoObj[IndexDescriptor::kIndexVersionFieldName]; - if (e.isNumber()) { - _version = static_cast<IndexVersion>(e.numberInt()); - } + fassert(50942, e.isNumber()); + _version = static_cast<IndexVersion>(e.numberInt()); } diff --git a/src/mongo/db/index/sort_key_generator.cpp b/src/mongo/db/index/sort_key_generator.cpp index e02fff9111b..35154c290d6 100644 --- a/src/mongo/db/index/sort_key_generator.cpp +++ b/src/mongo/db/index/sort_key_generator.cpp @@ -78,7 +78,7 @@ SortKeyGenerator::SortKeyGenerator(const BSONObj& sortSpec, const CollatorInterf } constexpr bool isSparse = false; - _indexKeyGen = stdx::make_unique<BtreeKeyGeneratorV1>(fieldNames, fixed, isSparse, _collator); + _indexKeyGen = stdx::make_unique<BtreeKeyGenerator>(fieldNames, fixed, isSparse, _collator); } StatusWith<BSONObj> SortKeyGenerator::getSortKey(const BSONObj& obj, diff --git a/src/mongo/db/repair_database.cpp b/src/mongo/db/repair_database.cpp index cad01ba9990..9b64c6eaf3b 100644 --- a/src/mongo/db/repair_database.cpp +++ b/src/mongo/db/repair_database.cpp @@ -59,17 +59,12 @@ namespace mongo { -using std::endl; -using std::string; - -using IndexVersion = IndexDescriptor::IndexVersion; - StatusWith<IndexNameObjs> getIndexNameObjs(OperationContext* opCtx, DatabaseCatalogEntry* dbce, CollectionCatalogEntry* cce, stdx::function<bool(const std::string&)> filter) { IndexNameObjs ret; - std::vector<string>& indexNames = ret.first; + std::vector<std::string>& indexNames = ret.first; std::vector<BSONObj>& indexSpecs = ret.second; { // Fetch all indexes @@ -82,38 +77,21 @@ StatusWith<IndexNameObjs> getIndexNameObjs(OperationContext* opCtx, indexSpecs.reserve(indexNames.size()); - for (size_t i = 0; i < indexNames.size(); i++) { - const string& name = indexNames[i]; + for (const auto& name : indexNames) { BSONObj spec = cce->getIndexSpec(opCtx, name); - - IndexVersion newIndexVersion = IndexVersion::kV0; - { - BSONObjBuilder bob; - - for (auto&& indexSpecElem : spec) { - auto indexSpecElemFieldName = indexSpecElem.fieldNameStringData(); - if (IndexDescriptor::kIndexVersionFieldName == indexSpecElemFieldName) { - IndexVersion indexVersion = - static_cast<IndexVersion>(indexSpecElem.numberInt()); - if (IndexVersion::kV0 == indexVersion) { - // We automatically upgrade v=0 indexes to v=1 indexes. - newIndexVersion = IndexVersion::kV1; - } else { - newIndexVersion = indexVersion; - } - - bob.append(IndexDescriptor::kIndexVersionFieldName, - static_cast<int>(newIndexVersion)); - } else { - bob.append(indexSpecElem); - } - } - - indexSpecs.push_back(bob.obj()); + using IndexVersion = IndexDescriptor::IndexVersion; + IndexVersion indexVersion = IndexVersion::kV1; + if (auto indexVersionElem = spec[IndexDescriptor::kIndexVersionFieldName]) { + auto indexVersionNum = indexVersionElem.numberInt(); + invariant(indexVersionNum == static_cast<int>(IndexVersion::kV1) || + indexVersionNum == static_cast<int>(IndexVersion::kV2)); + indexVersion = static_cast<IndexVersion>(indexVersionNum); } + invariant(spec.isOwned()); + indexSpecs.push_back(spec); const BSONObj key = spec.getObjectField("key"); - const Status keyStatus = index_key_validate::validateKeyPattern(key, newIndexVersion); + const Status keyStatus = index_key_validate::validateKeyPattern(key, indexVersion); if (!keyStatus.isOK()) { return Status( ErrorCodes::CannotCreateIndex, @@ -285,9 +263,9 @@ Status repairDatabase(OperationContext* opCtx, StorageEngine* engine, const std: // We must hold some form of lock here invariant(opCtx->lockState()->isLocked()); - invariant(dbName.find('.') == string::npos); + invariant(dbName.find('.') == std::string::npos); - log() << "repairDatabase " << dbName << endl; + log() << "repairDatabase " << dbName; BackgroundOperation::assertNoBgOpInProgForDb(dbName); @@ -326,6 +304,34 @@ Status repairDatabase(OperationContext* opCtx, StorageEngine* engine, const std: return status; } + DatabaseCatalogEntry* dbce = engine->getDatabaseCatalogEntry(opCtx, dbName); + + std::list<std::string> colls; + dbce->getCollectionNamespaces(&colls); + + for (std::list<std::string>::const_iterator it = colls.begin(); it != colls.end(); ++it) { + // Don't check for interrupt after starting to repair a collection otherwise we can + // leave data in an inconsistent state. Interrupting between collections is ok, however. + opCtx->checkForInterrupt(); + + log() << "Repairing collection " << *it; + + Status status = engine->repairRecordStore(opCtx, *it); + if (!status.isOK()) + return status; + + CollectionCatalogEntry* cce = dbce->getCollectionCatalogEntry(*it); + auto swIndexNameObjs = getIndexNameObjs(opCtx, dbce, cce); + if (!swIndexNameObjs.isOK()) + return swIndexNameObjs.getStatus(); + + status = rebuildIndexesOnCollection(opCtx, dbce, cce, swIndexNameObjs.getValue()); + if (!status.isOK()) + return status; + + engine->flushAllFiles(opCtx, true); + } + return Status::OK(); } -} +} // namespace mongo diff --git a/src/mongo/db/storage/kv/kv_engine_test_harness.cpp b/src/mongo/db/storage/kv/kv_engine_test_harness.cpp index 6b9b671fd37..0e8e92732f8 100644 --- a/src/mongo/db/storage/kv/kv_engine_test_harness.cpp +++ b/src/mongo/db/storage/kv/kv_engine_test_harness.cpp @@ -35,6 +35,7 @@ #include "mongo/db/storage/kv/kv_prefix.h" #include "mongo/db/storage/record_store.h" #include "mongo/db/storage/sorted_data_interface.h" +#include "mongo/unittest/death_test.h" #include "mongo/unittest/unittest.h" #include "mongo/util/assert_util.h" #include "mongo/util/clock_source_mock.h" @@ -144,7 +145,10 @@ TEST(KVEngineTestHarness, SimpleSorted1) { ASSERT(engine); string ident = "abc"; - IndexDescriptor desc(NULL, "", BSON("key" << BSON("a" << 1))); + IndexDescriptor desc(nullptr, + "", + BSON("v" << static_cast<int>(IndexDescriptor::kLatestIndexVersion) << "key" + << BSON("a" << 1))); unique_ptr<SortedDataInterface> sorted; { MyOperationContext opCtx(engine); @@ -490,6 +494,27 @@ TEST(KVCatalogTest, RestartForPrefixes) { } } +DEATH_TEST(KVCatalogTest, TerminateOnNonNumericIndexVersion, "Fatal Assertion 50942") { + unique_ptr<KVHarnessHelper> helper(KVHarnessHelper::create()); + KVEngine* engine = helper->getEngine(); + ASSERT(engine); + + string ident = "abc"; + IndexDescriptor desc(nullptr, + "", + BSON("v" + << "1" + << "key" + << BSON("a" << 1))); + unique_ptr<SortedDataInterface> sorted; + { + MyOperationContext opCtx(engine); + ASSERT_OK(engine->createSortedDataInterface(&opCtx, ident, &desc)); + sorted.reset(engine->getSortedDataInterface(&opCtx, ident, &desc)); + ASSERT(sorted); + } +} + } // namespace std::unique_ptr<KVHarnessHelper> KVHarnessHelper::create() { diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_index_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_index_test.cpp index 53ae3fdc453..af16ebf30ef 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_index_test.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_index_test.cpp @@ -73,6 +73,8 @@ public: BSONObj spec = BSON("key" << BSON("a" << 1) << "name" << "testIndex" + << "v" + << static_cast<int>(IndexDescriptor::kLatestIndexVersion) << "ns" << ns << "unique" diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_standard_index_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_standard_index_test.cpp index 095466d27d6..392a221f17f 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_standard_index_test.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_standard_index_test.cpp @@ -73,6 +73,8 @@ public: BSONObj spec = BSON("key" << BSON("a" << 1) << "name" << "testIndex" + << "v" + << static_cast<int>(IndexDescriptor::kLatestIndexVersion) << "ns" << ns << "unique" |