diff options
author | Henrik Edin <henrik.edin@mongodb.com> | 2018-11-21 10:56:47 -0500 |
---|---|---|
committer | Henrik Edin <henrik.edin@mongodb.com> | 2018-12-04 13:43:46 -0500 |
commit | 279fba2819e281053b19507e06b862b852d4e85f (patch) | |
tree | 68070313e69f04b677ce4e8ae66c0ddf40014860 /src/mongo | |
parent | cf8fbf54354a805f0bdb5bc4282c8081f4802971 (diff) | |
download | mongo-279fba2819e281053b19507e06b862b852d4e85f.tar.gz |
SERVER-38248 Change StringMap implementation to absl::flat_hash_map
Diffstat (limited to 'src/mongo')
-rw-r--r-- | src/mongo/base/string_data.h | 4 | ||||
-rw-r--r-- | src/mongo/db/background.cpp | 20 | ||||
-rw-r--r-- | src/mongo/db/commands.cpp | 7 | ||||
-rw-r--r-- | src/mongo/db/matcher/matcher_type_set.cpp | 10 | ||||
-rw-r--r-- | src/mongo/db/matcher/schema/json_schema_parser.cpp | 140 | ||||
-rw-r--r-- | src/mongo/db/pipeline/variables.cpp | 4 | ||||
-rw-r--r-- | src/mongo/db/repl/sync_tail.cpp | 9 | ||||
-rw-r--r-- | src/mongo/db/repl/sync_tail_test.cpp | 70 | ||||
-rw-r--r-- | src/mongo/db/stats/top.cpp | 4 | ||||
-rw-r--r-- | src/mongo/s/transaction_router_test.cpp | 79 | ||||
-rw-r--r-- | src/mongo/util/hash_table_bm.cpp | 77 | ||||
-rw-r--r-- | src/mongo/util/string_map.h | 96 | ||||
-rw-r--r-- | src/mongo/util/string_map_test.cpp | 4 | ||||
-rw-r--r-- | src/mongo/util/unordered_fast_key_table.h | 436 | ||||
-rw-r--r-- | src/mongo/util/unordered_fast_key_table_internal.h | 198 | ||||
-rw-r--r-- | src/mongo/util/unordered_fast_key_table_traits_helpers.h | 96 |
16 files changed, 205 insertions, 1049 deletions
diff --git a/src/mongo/base/string_data.h b/src/mongo/base/string_data.h index 3f33683e335..213a55564d4 100644 --- a/src/mongo/base/string_data.h +++ b/src/mongo/base/string_data.h @@ -104,6 +104,10 @@ public: */ constexpr friend StringData operator"" _sd(const char* c, std::size_t len); + explicit operator std::string() const { + return toString(); + } + /** * Constructs a StringData with begin and end iterators. begin points to the beginning of the * string. end points to the position past the end of the string. In a null-terminated string, diff --git a/src/mongo/db/background.cpp b/src/mongo/db/background.cpp index 9b9698b53e0..692847b348a 100644 --- a/src/mongo/db/background.cpp +++ b/src/mongo/db/background.cpp @@ -97,18 +97,18 @@ void BgInfo::awaitNoBgOps(stdx::unique_lock<stdx::mutex>& lk) { _noOpsInProg.wait(lk); } -void recordBeginAndInsert(BgInfoMap* bgiMap, StringData key) { - std::shared_ptr<BgInfo>& bgInfo = bgiMap->get(key); +void recordBeginAndInsert(BgInfoMap& bgiMap, StringData key) { + std::shared_ptr<BgInfo>& bgInfo = bgiMap[key]; if (!bgInfo) bgInfo.reset(new BgInfo); bgInfo->recordBegin(); } -void recordEndAndRemove(BgInfoMap* bgiMap, StringData key) { - BgInfoMapIterator iter = bgiMap->find(key); - fassert(17431, iter != bgiMap->end()); +void recordEndAndRemove(BgInfoMap& bgiMap, StringData key) { + BgInfoMapIterator iter = bgiMap.find(key); + fassert(17431, iter != bgiMap.end()); if (0 == iter->second->recordEnd()) { - bgiMap->erase(iter); + bgiMap.erase(iter); } } @@ -169,14 +169,14 @@ void BackgroundOperation::awaitNoBgOpInProgForNs(StringData ns) { BackgroundOperation::BackgroundOperation(StringData ns) : _ns(ns) { stdx::lock_guard<stdx::mutex> lk(m); - recordBeginAndInsert(&dbsInProg, _ns.db()); - recordBeginAndInsert(&nsInProg, _ns.ns()); + recordBeginAndInsert(dbsInProg, _ns.db()); + recordBeginAndInsert(nsInProg, _ns.ns()); } BackgroundOperation::~BackgroundOperation() { stdx::lock_guard<stdx::mutex> lk(m); - recordEndAndRemove(&dbsInProg, _ns.db()); - recordEndAndRemove(&nsInProg, _ns.ns()); + recordEndAndRemove(dbsInProg, _ns.db()); + recordEndAndRemove(nsInProg, _ns.ns()); } void BackgroundOperation::dump(std::ostream& ss) { diff --git a/src/mongo/db/commands.cpp b/src/mongo/db/commands.cpp index cea4b6c06c1..2aa894c4091 100644 --- a/src/mongo/db/commands.cpp +++ b/src/mongo/db/commands.cpp @@ -683,10 +683,9 @@ void CommandRegistry::registerCommand(Command* command, StringData name, StringD if (key.empty()) { continue; } - auto hashedKey = CommandMap::HashedKey(key); - auto iter = _commands.find(hashedKey); - invariant(iter == _commands.end(), str::stream() << "command name collision: " << key); - _commands[hashedKey] = command; + + auto result = _commands.try_emplace(key, command); + invariant(result.second, str::stream() << "command name collision: " << key); } } diff --git a/src/mongo/db/matcher/matcher_type_set.cpp b/src/mongo/db/matcher/matcher_type_set.cpp index 8962a6ea103..532481586ee 100644 --- a/src/mongo/db/matcher/matcher_type_set.cpp +++ b/src/mongo/db/matcher/matcher_type_set.cpp @@ -116,11 +116,11 @@ Status parseSingleType(BSONElement elt, constexpr StringData MatcherTypeSet::kMatchesAllNumbersAlias; const StringMap<BSONType> MatcherTypeSet::kJsonSchemaTypeAliasMap = { - {JSONSchemaParser::kSchemaTypeArray, BSONType::Array}, - {JSONSchemaParser::kSchemaTypeBoolean, BSONType::Bool}, - {JSONSchemaParser::kSchemaTypeNull, BSONType::jstNULL}, - {JSONSchemaParser::kSchemaTypeObject, BSONType::Object}, - {JSONSchemaParser::kSchemaTypeString, BSONType::String}, + {std::string(JSONSchemaParser::kSchemaTypeArray), BSONType::Array}, + {std::string(JSONSchemaParser::kSchemaTypeBoolean), BSONType::Bool}, + {std::string(JSONSchemaParser::kSchemaTypeNull), BSONType::jstNULL}, + {std::string(JSONSchemaParser::kSchemaTypeObject), BSONType::Object}, + {std::string(JSONSchemaParser::kSchemaTypeString), BSONType::String}, }; StatusWith<MatcherTypeSet> MatcherTypeSet::fromStringAliases(std::set<StringData> typeAliases, diff --git a/src/mongo/db/matcher/schema/json_schema_parser.cpp b/src/mongo/db/matcher/schema/json_schema_parser.cpp index 3467855e11d..029e0985790 100644 --- a/src/mongo/db/matcher/schema/json_schema_parser.cpp +++ b/src/mongo/db/matcher/schema/json_schema_parser.cpp @@ -989,13 +989,13 @@ Status parseAdditionalItems(StringData path, return Status::OK(); } -Status parseItemsAndAdditionalItems(StringMap<BSONElement>* keywordMap, +Status parseItemsAndAdditionalItems(StringMap<BSONElement>& keywordMap, StringData path, bool ignoreUnknownKeywords, InternalSchemaTypeExpression* typeExpr, AndMatchExpression* andExpr) { boost::optional<long long> startIndexForAdditionalItems; - if (auto itemsElt = keywordMap->get(kSchemaItemsKeyword)) { + if (auto itemsElt = keywordMap[kSchemaItemsKeyword]) { auto index = parseItems(path, itemsElt, ignoreUnknownKeywords, typeExpr, andExpr); if (!index.isOK()) { return index.getStatus(); @@ -1003,7 +1003,7 @@ Status parseItemsAndAdditionalItems(StringMap<BSONElement>* keywordMap, startIndexForAdditionalItems = index.getValue(); } - if (auto additionalItemsElt = keywordMap->get(kSchemaAdditionalItemsKeyword)) { + if (auto additionalItemsElt = keywordMap[kSchemaAdditionalItemsKeyword]) { return parseAdditionalItems(path, additionalItemsElt, startIndexForAdditionalItems, @@ -1025,11 +1025,11 @@ Status parseItemsAndAdditionalItems(StringMap<BSONElement>* keywordMap, * - not * - enum */ -Status translateLogicalKeywords(StringMap<BSONElement>* keywordMap, +Status translateLogicalKeywords(StringMap<BSONElement>& keywordMap, StringData path, AndMatchExpression* andExpr, bool ignoreUnknownKeywords) { - if (auto allOfElt = keywordMap->get(kSchemaAllOfKeyword)) { + if (auto allOfElt = keywordMap[kSchemaAllOfKeyword]) { auto allOfExpr = parseLogicalKeyword<AndMatchExpression>(path, allOfElt, ignoreUnknownKeywords); if (!allOfExpr.isOK()) { @@ -1038,7 +1038,7 @@ Status translateLogicalKeywords(StringMap<BSONElement>* keywordMap, andExpr->add(allOfExpr.getValue().release()); } - if (auto anyOfElt = keywordMap->get(kSchemaAnyOfKeyword)) { + if (auto anyOfElt = keywordMap[kSchemaAnyOfKeyword]) { auto anyOfExpr = parseLogicalKeyword<OrMatchExpression>(path, anyOfElt, ignoreUnknownKeywords); if (!anyOfExpr.isOK()) { @@ -1047,7 +1047,7 @@ Status translateLogicalKeywords(StringMap<BSONElement>* keywordMap, andExpr->add(anyOfExpr.getValue().release()); } - if (auto oneOfElt = keywordMap->get(kSchemaOneOfKeyword)) { + if (auto oneOfElt = keywordMap[kSchemaOneOfKeyword]) { auto oneOfExpr = parseLogicalKeyword<InternalSchemaXorMatchExpression>( path, oneOfElt, ignoreUnknownKeywords); if (!oneOfExpr.isOK()) { @@ -1056,7 +1056,7 @@ Status translateLogicalKeywords(StringMap<BSONElement>* keywordMap, andExpr->add(oneOfExpr.getValue().release()); } - if (auto notElt = keywordMap->get(kSchemaNotKeyword)) { + if (auto notElt = keywordMap[kSchemaNotKeyword]) { if (notElt.type() != BSONType::Object) { return {ErrorCodes::TypeMismatch, str::stream() << "$jsonSchema keyword '" << kSchemaNotKeyword @@ -1073,7 +1073,7 @@ Status translateLogicalKeywords(StringMap<BSONElement>* keywordMap, andExpr->add(notMatchExpr.release()); } - if (auto enumElt = keywordMap->get(kSchemaEnumKeyword)) { + if (auto enumElt = keywordMap[kSchemaEnumKeyword]) { auto enumExpr = parseEnum(path, enumElt); if (!enumExpr.isOK()) { return enumExpr.getStatus(); @@ -1095,12 +1095,12 @@ Status translateLogicalKeywords(StringMap<BSONElement>* keywordMap, * - items * - additionalItems */ -Status translateArrayKeywords(StringMap<BSONElement>* keywordMap, +Status translateArrayKeywords(StringMap<BSONElement>& keywordMap, StringData path, bool ignoreUnknownKeywords, InternalSchemaTypeExpression* typeExpr, AndMatchExpression* andExpr) { - if (auto minItemsElt = keywordMap->get(kSchemaMinItemsKeyword)) { + if (auto minItemsElt = keywordMap[kSchemaMinItemsKeyword]) { auto minItemsExpr = parseLength<InternalSchemaMinItemsMatchExpression>( path, minItemsElt, typeExpr, BSONType::Array); if (!minItemsExpr.isOK()) { @@ -1109,7 +1109,7 @@ Status translateArrayKeywords(StringMap<BSONElement>* keywordMap, andExpr->add(minItemsExpr.getValue().release()); } - if (auto maxItemsElt = keywordMap->get(kSchemaMaxItemsKeyword)) { + if (auto maxItemsElt = keywordMap[kSchemaMaxItemsKeyword]) { auto maxItemsExpr = parseLength<InternalSchemaMaxItemsMatchExpression>( path, maxItemsElt, typeExpr, BSONType::Array); if (!maxItemsExpr.isOK()) { @@ -1118,7 +1118,7 @@ Status translateArrayKeywords(StringMap<BSONElement>* keywordMap, andExpr->add(maxItemsExpr.getValue().release()); } - if (auto uniqueItemsElt = keywordMap->get(kSchemaUniqueItemsKeyword)) { + if (auto uniqueItemsElt = keywordMap[kSchemaUniqueItemsKeyword]) { auto uniqueItemsExpr = parseUniqueItems(uniqueItemsElt, path, typeExpr); if (!uniqueItemsExpr.isOK()) { return uniqueItemsExpr.getStatus(); @@ -1142,13 +1142,13 @@ Status translateArrayKeywords(StringMap<BSONElement>* keywordMap, * - properties * - required */ -Status translateObjectKeywords(StringMap<BSONElement>* keywordMap, +Status translateObjectKeywords(StringMap<BSONElement>& keywordMap, StringData path, InternalSchemaTypeExpression* typeExpr, AndMatchExpression* andExpr, bool ignoreUnknownKeywords) { boost::container::flat_set<StringData> requiredProperties; - if (auto requiredElt = keywordMap->get(kSchemaRequiredKeyword)) { + if (auto requiredElt = keywordMap[kSchemaRequiredKeyword]) { auto requiredStatus = parseRequired(requiredElt); if (!requiredStatus.isOK()) { return requiredStatus.getStatus(); @@ -1156,7 +1156,7 @@ Status translateObjectKeywords(StringMap<BSONElement>* keywordMap, requiredProperties = std::move(requiredStatus.getValue()); } - if (auto propertiesElt = keywordMap->get(kSchemaPropertiesKeyword)) { + if (auto propertiesElt = keywordMap[kSchemaPropertiesKeyword]) { auto propertiesExpr = parseProperties( path, propertiesElt, typeExpr, requiredProperties, ignoreUnknownKeywords); if (!propertiesExpr.isOK()) { @@ -1166,9 +1166,9 @@ Status translateObjectKeywords(StringMap<BSONElement>* keywordMap, } { - auto propertiesElt = keywordMap->get(kSchemaPropertiesKeyword); - auto patternPropertiesElt = keywordMap->get(kSchemaPatternPropertiesKeyword); - auto additionalPropertiesElt = keywordMap->get(kSchemaAdditionalPropertiesKeyword); + auto propertiesElt = keywordMap[kSchemaPropertiesKeyword]; + auto patternPropertiesElt = keywordMap[kSchemaPatternPropertiesKeyword]; + auto additionalPropertiesElt = keywordMap[kSchemaAdditionalPropertiesKeyword]; if (patternPropertiesElt || additionalPropertiesElt) { auto allowedPropertiesExpr = parseAllowedProperties(path, @@ -1192,7 +1192,7 @@ Status translateObjectKeywords(StringMap<BSONElement>* keywordMap, andExpr->add(requiredExpr.getValue().release()); } - if (auto minPropertiesElt = keywordMap->get(kSchemaMinPropertiesKeyword)) { + if (auto minPropertiesElt = keywordMap[kSchemaMinPropertiesKeyword]) { auto minPropExpr = parseNumProperties<InternalSchemaMinPropertiesMatchExpression>( path, minPropertiesElt, typeExpr); if (!minPropExpr.isOK()) { @@ -1201,7 +1201,7 @@ Status translateObjectKeywords(StringMap<BSONElement>* keywordMap, andExpr->add(minPropExpr.getValue().release()); } - if (auto maxPropertiesElt = keywordMap->get(kSchemaMaxPropertiesKeyword)) { + if (auto maxPropertiesElt = keywordMap[kSchemaMaxPropertiesKeyword]) { auto maxPropExpr = parseNumProperties<InternalSchemaMaxPropertiesMatchExpression>( path, maxPropertiesElt, typeExpr); if (!maxPropExpr.isOK()) { @@ -1210,7 +1210,7 @@ Status translateObjectKeywords(StringMap<BSONElement>* keywordMap, andExpr->add(maxPropExpr.getValue().release()); } - if (auto dependenciesElt = keywordMap->get(kSchemaDependenciesKeyword)) { + if (auto dependenciesElt = keywordMap[kSchemaDependenciesKeyword]) { auto dependenciesExpr = parseDependencies(path, dependenciesElt, ignoreUnknownKeywords); if (!dependenciesExpr.isOK()) { return dependenciesExpr.getStatus(); @@ -1234,12 +1234,12 @@ Status translateObjectKeywords(StringMap<BSONElement>* keywordMap, * - maxLength * - pattern */ -Status translateScalarKeywords(StringMap<BSONElement>* keywordMap, +Status translateScalarKeywords(StringMap<BSONElement>& keywordMap, StringData path, InternalSchemaTypeExpression* typeExpr, AndMatchExpression* andExpr) { // String keywords. - if (auto patternElt = keywordMap->get(kSchemaPatternKeyword)) { + if (auto patternElt = keywordMap[kSchemaPatternKeyword]) { auto patternExpr = parsePattern(path, patternElt, typeExpr); if (!patternExpr.isOK()) { return patternExpr.getStatus(); @@ -1247,7 +1247,7 @@ Status translateScalarKeywords(StringMap<BSONElement>* keywordMap, andExpr->add(patternExpr.getValue().release()); } - if (auto maxLengthElt = keywordMap->get(kSchemaMaxLengthKeyword)) { + if (auto maxLengthElt = keywordMap[kSchemaMaxLengthKeyword]) { auto maxLengthExpr = parseLength<InternalSchemaMaxLengthMatchExpression>( path, maxLengthElt, typeExpr, BSONType::String); if (!maxLengthExpr.isOK()) { @@ -1256,7 +1256,7 @@ Status translateScalarKeywords(StringMap<BSONElement>* keywordMap, andExpr->add(maxLengthExpr.getValue().release()); } - if (auto minLengthElt = keywordMap->get(kSchemaMinLengthKeyword)) { + if (auto minLengthElt = keywordMap[kSchemaMinLengthKeyword]) { auto minLengthExpr = parseLength<InternalSchemaMinLengthMatchExpression>( path, minLengthElt, typeExpr, BSONType::String); if (!minLengthExpr.isOK()) { @@ -1266,7 +1266,7 @@ Status translateScalarKeywords(StringMap<BSONElement>* keywordMap, } // Numeric keywords. - if (auto multipleOfElt = keywordMap->get(kSchemaMultipleOfKeyword)) { + if (auto multipleOfElt = keywordMap[kSchemaMultipleOfKeyword]) { auto multipleOfExpr = parseMultipleOf(path, multipleOfElt, typeExpr); if (!multipleOfExpr.isOK()) { return multipleOfExpr.getStatus(); @@ -1274,9 +1274,9 @@ Status translateScalarKeywords(StringMap<BSONElement>* keywordMap, andExpr->add(multipleOfExpr.getValue().release()); } - if (auto maximumElt = keywordMap->get(kSchemaMaximumKeyword)) { + if (auto maximumElt = keywordMap[kSchemaMaximumKeyword]) { bool isExclusiveMaximum = false; - if (auto exclusiveMaximumElt = keywordMap->get(kSchemaExclusiveMaximumKeyword)) { + if (auto exclusiveMaximumElt = keywordMap[kSchemaExclusiveMaximumKeyword]) { if (!exclusiveMaximumElt.isBoolean()) { return {Status(ErrorCodes::TypeMismatch, str::stream() << "$jsonSchema keyword '" @@ -1291,7 +1291,7 @@ Status translateScalarKeywords(StringMap<BSONElement>* keywordMap, return maxExpr.getStatus(); } andExpr->add(maxExpr.getValue().release()); - } else if (keywordMap->get(kSchemaExclusiveMaximumKeyword)) { + } else if (keywordMap[kSchemaExclusiveMaximumKeyword]) { // If "exclusiveMaximum" is present, "maximum" must also be present. return {ErrorCodes::FailedToParse, str::stream() << "$jsonSchema keyword '" << kSchemaMaximumKeyword @@ -1300,9 +1300,9 @@ Status translateScalarKeywords(StringMap<BSONElement>* keywordMap, << " is present"}; } - if (auto minimumElt = keywordMap->get(kSchemaMinimumKeyword)) { + if (auto minimumElt = keywordMap[kSchemaMinimumKeyword]) { bool isExclusiveMinimum = false; - if (auto exclusiveMinimumElt = keywordMap->get(kSchemaExclusiveMinimumKeyword)) { + if (auto exclusiveMinimumElt = keywordMap[kSchemaExclusiveMinimumKeyword]) { if (!exclusiveMinimumElt.isBoolean()) { return {ErrorCodes::TypeMismatch, str::stream() << "$jsonSchema keyword '" << kSchemaExclusiveMinimumKeyword @@ -1316,7 +1316,7 @@ Status translateScalarKeywords(StringMap<BSONElement>* keywordMap, return minExpr.getStatus(); } andExpr->add(minExpr.getValue().release()); - } else if (keywordMap->get(kSchemaExclusiveMinimumKeyword)) { + } else if (keywordMap[kSchemaExclusiveMinimumKeyword]) { // If "exclusiveMinimum" is present, "minimum" must also be present. return {ErrorCodes::FailedToParse, str::stream() << "$jsonSchema keyword '" << kSchemaMinimumKeyword @@ -1333,8 +1333,8 @@ Status translateScalarKeywords(StringMap<BSONElement>* keywordMap, * - description * - title */ -Status validateMetadataKeywords(StringMap<BSONElement>* keywordMap) { - if (auto descriptionElem = keywordMap->get(kSchemaDescriptionKeyword)) { +Status validateMetadataKeywords(StringMap<BSONElement>& keywordMap) { + if (auto descriptionElem = keywordMap[kSchemaDescriptionKeyword]) { if (descriptionElem.type() != BSONType::String) { return Status(ErrorCodes::TypeMismatch, str::stream() << "$jsonSchema keyword '" << kSchemaDescriptionKeyword @@ -1342,7 +1342,7 @@ Status validateMetadataKeywords(StringMap<BSONElement>* keywordMap) { } } - if (auto titleElem = keywordMap->get(kSchemaTitleKeyword)) { + if (auto titleElem = keywordMap[kSchemaTitleKeyword]) { if (titleElem.type() != BSONType::String) { return Status(ErrorCodes::TypeMismatch, str::stream() << "$jsonSchema keyword '" << kSchemaTitleKeyword @@ -1356,35 +1356,35 @@ StatusWithMatchExpression _parse(StringData path, BSONObj schema, bool ignoreUnk // Map from JSON Schema keyword to the corresponding element from 'schema', or to an empty // BSONElement if the JSON Schema keyword is not specified. StringMap<BSONElement> keywordMap{ - {kSchemaAdditionalItemsKeyword, {}}, - {kSchemaAdditionalPropertiesKeyword, {}}, - {kSchemaAllOfKeyword, {}}, - {kSchemaAnyOfKeyword, {}}, - {kSchemaBsonTypeKeyword, {}}, - {kSchemaDependenciesKeyword, {}}, - {kSchemaDescriptionKeyword, {}}, - {kSchemaEnumKeyword, {}}, - {kSchemaExclusiveMaximumKeyword, {}}, - {kSchemaExclusiveMinimumKeyword, {}}, - {kSchemaItemsKeyword, {}}, - {kSchemaMaxItemsKeyword, {}}, - {kSchemaMaxLengthKeyword, {}}, - {kSchemaMaxPropertiesKeyword, {}}, - {kSchemaMaximumKeyword, {}}, - {kSchemaMinItemsKeyword, {}}, - {kSchemaMinLengthKeyword, {}}, - {kSchemaMinPropertiesKeyword, {}}, - {kSchemaMinimumKeyword, {}}, - {kSchemaMultipleOfKeyword, {}}, - {kSchemaNotKeyword, {}}, - {kSchemaOneOfKeyword, {}}, - {kSchemaPatternKeyword, {}}, - {kSchemaPatternPropertiesKeyword, {}}, - {kSchemaPropertiesKeyword, {}}, - {kSchemaRequiredKeyword, {}}, - {kSchemaTitleKeyword, {}}, - {kSchemaTypeKeyword, {}}, - {kSchemaUniqueItemsKeyword, {}}, + {std::string(kSchemaAdditionalItemsKeyword), {}}, + {std::string(kSchemaAdditionalPropertiesKeyword), {}}, + {std::string(kSchemaAllOfKeyword), {}}, + {std::string(kSchemaAnyOfKeyword), {}}, + {std::string(kSchemaBsonTypeKeyword), {}}, + {std::string(kSchemaDependenciesKeyword), {}}, + {std::string(kSchemaDescriptionKeyword), {}}, + {std::string(kSchemaEnumKeyword), {}}, + {std::string(kSchemaExclusiveMaximumKeyword), {}}, + {std::string(kSchemaExclusiveMinimumKeyword), {}}, + {std::string(kSchemaItemsKeyword), {}}, + {std::string(kSchemaMaxItemsKeyword), {}}, + {std::string(kSchemaMaxLengthKeyword), {}}, + {std::string(kSchemaMaxPropertiesKeyword), {}}, + {std::string(kSchemaMaximumKeyword), {}}, + {std::string(kSchemaMinItemsKeyword), {}}, + {std::string(kSchemaMinLengthKeyword), {}}, + {std::string(kSchemaMinPropertiesKeyword), {}}, + {std::string(kSchemaMinimumKeyword), {}}, + {std::string(kSchemaMultipleOfKeyword), {}}, + {std::string(kSchemaNotKeyword), {}}, + {std::string(kSchemaOneOfKeyword), {}}, + {std::string(kSchemaPatternKeyword), {}}, + {std::string(kSchemaPatternPropertiesKeyword), {}}, + {std::string(kSchemaPropertiesKeyword), {}}, + {std::string(kSchemaRequiredKeyword), {}}, + {std::string(kSchemaTitleKeyword), {}}, + {std::string(kSchemaTypeKeyword), {}}, + {std::string(kSchemaUniqueItemsKeyword), {}}, }; for (auto&& elt : schema) { @@ -1411,7 +1411,7 @@ StatusWithMatchExpression _parse(StringData path, BSONObj schema, bool ignoreUnk keywordMap[elt.fieldNameStringData()] = elt; } - auto metadataStatus = validateMetadataKeywords(&keywordMap); + auto metadataStatus = validateMetadataKeywords(keywordMap); if (!metadataStatus.isOK()) { return metadataStatus; } @@ -1447,25 +1447,25 @@ StatusWithMatchExpression _parse(StringData path, BSONObj schema, bool ignoreUnk auto andExpr = stdx::make_unique<AndMatchExpression>(); auto translationStatus = - translateScalarKeywords(&keywordMap, path, typeExpr.get(), andExpr.get()); + translateScalarKeywords(keywordMap, path, typeExpr.get(), andExpr.get()); if (!translationStatus.isOK()) { return translationStatus; } translationStatus = translateArrayKeywords( - &keywordMap, path, ignoreUnknownKeywords, typeExpr.get(), andExpr.get()); + keywordMap, path, ignoreUnknownKeywords, typeExpr.get(), andExpr.get()); if (!translationStatus.isOK()) { return translationStatus; } translationStatus = translateObjectKeywords( - &keywordMap, path, typeExpr.get(), andExpr.get(), ignoreUnknownKeywords); + keywordMap, path, typeExpr.get(), andExpr.get(), ignoreUnknownKeywords); if (!translationStatus.isOK()) { return translationStatus; } translationStatus = - translateLogicalKeywords(&keywordMap, path, andExpr.get(), ignoreUnknownKeywords); + translateLogicalKeywords(keywordMap, path, andExpr.get(), ignoreUnknownKeywords); if (!translationStatus.isOK()) { return translationStatus; } diff --git a/src/mongo/db/pipeline/variables.cpp b/src/mongo/db/pipeline/variables.cpp index 936f23c2b9c..b97335ab304 100644 --- a/src/mongo/db/pipeline/variables.cpp +++ b/src/mongo/db/pipeline/variables.cpp @@ -38,8 +38,8 @@ namespace mongo { constexpr Variables::Id Variables::kRootId; constexpr Variables::Id Variables::kRemoveId; -const StringMap<Variables::Id> Variables::kBuiltinVarNameToId = {{"ROOT"_sd, kRootId}, - {"REMOVE"_sd, kRemoveId}}; +const StringMap<Variables::Id> Variables::kBuiltinVarNameToId = {{"ROOT", kRootId}, + {"REMOVE", kRemoveId}}; void Variables::uassertValidNameForUserWrite(StringData varName) { // System variables users allowed to write to (currently just one) diff --git a/src/mongo/db/repl/sync_tail.cpp b/src/mongo/db/repl/sync_tail.cpp index 43dda15986c..d261d2c417f 100644 --- a/src/mongo/db/repl/sync_tail.cpp +++ b/src/mongo/db/repl/sync_tail.cpp @@ -503,7 +503,7 @@ public: }; CollectionProperties getCollectionProperties(OperationContext* opCtx, - const StringMapTraits::HashedKey& ns) { + const StringMapHashedKey& ns) { auto it = _cache.find(ns); if (it != _cache.end()) { return it->second; @@ -559,8 +559,11 @@ void fillWriterVectors(OperationContext* opCtx, CachedCollectionProperties collPropertiesCache; for (auto&& op : *ops) { - StringMapTraits::HashedKey hashedNs(op.getNss().ns()); - uint32_t hash = hashedNs.hash(); + auto hashedNs = StringMapHasher().hashed_key(op.getNss().ns()); + // Reduce the hash from 64bit down to 32bit, just to allow combinations with murmur3 later + // on. Bit depth not important, we end up just doing integer modulo with this in the end. + // The hash function should provide entropy in the lower bits as it's used in hash tables. + uint32_t hash = static_cast<uint32_t>(hashedNs.hash()); // We need to track all types of ops, including type 'n' (these are generated from chunk // migrations). diff --git a/src/mongo/db/repl/sync_tail_test.cpp b/src/mongo/db/repl/sync_tail_test.cpp index 8cff42112f2..1a0f2aae759 100644 --- a/src/mongo/db/repl/sync_tail_test.cpp +++ b/src/mongo/db/repl/sync_tail_test.cpp @@ -456,76 +456,6 @@ TEST_F(SyncTailTest, createOplogCollectionOptions())); } -TEST_F(SyncTailTest, MultiApplyAssignsOperationsToWriterThreadsBasedOnNamespaceHash) { - // This test relies on implementation details of how multiApply uses hashing to distribute ops - // to threads. It is possible for this test to fail, even if the implementation of multiApply is - // correct. If it fails, consider adjusting the namespace names (to adjust the hash values) or - // the number of threads in the pool. - NamespaceString nss1("test.t0"); - NamespaceString nss2("test.t1"); - auto writerPool = OplogApplier::makeWriterPool(2); - - stdx::mutex mutex; - std::vector<MultiApplier::Operations> operationsApplied; - auto applyOperationFn = - [&mutex, &operationsApplied](OperationContext* opCtx, - MultiApplier::OperationPtrs* operationsForWriterThreadToApply, - SyncTail* st, - WorkerMultikeyPathInfo*) -> Status { - stdx::lock_guard<stdx::mutex> lock(mutex); - operationsApplied.emplace_back(); - for (auto&& opPtr : *operationsForWriterThreadToApply) { - operationsApplied.back().push_back(*opPtr); - } - return Status::OK(); - }; - - auto op1 = makeInsertDocumentOplogEntry({Timestamp(Seconds(1), 0), 1LL}, nss1, BSON("x" << 1)); - auto op2 = makeInsertDocumentOplogEntry({Timestamp(Seconds(2), 0), 1LL}, nss2, BSON("x" << 2)); - - SyncTail syncTail(nullptr, - getConsistencyMarkers(), - getStorageInterface(), - applyOperationFn, - writerPool.get()); - auto lastOpTime = unittest::assertGet(syncTail.multiApply(_opCtx.get(), {op1, op2})); - ASSERT_EQUALS(op2.getOpTime(), lastOpTime); - - // Each writer thread should be given exactly one operation to apply. - std::vector<OpTime> seen; - { - stdx::lock_guard<stdx::mutex> lock(mutex); - ASSERT_EQUALS(operationsApplied.size(), 2U); - for (auto&& operationsAppliedByThread : operationsApplied) { - ASSERT_EQUALS(1U, operationsAppliedByThread.size()); - const auto& oplogEntry = operationsAppliedByThread.front(); - ASSERT_TRUE(std::find(seen.cbegin(), seen.cend(), oplogEntry.getOpTime()) == - seen.cend()); - ASSERT_TRUE(oplogEntry == op1 || oplogEntry == op2); - seen.push_back(oplogEntry.getOpTime()); - } - } - - // Check ops in oplog. - // Obtain the last 2 entries in the oplog using a reverse collection scan. - stdx::lock_guard<stdx::mutex> lock(mutex); - auto storage = getStorageInterface(); - auto operationsWrittenToOplog = - unittest::assertGet(storage->findDocuments(_opCtx.get(), - NamespaceString::kRsOplogNamespace, - {}, - StorageInterface::ScanDirection::kBackward, - {}, - BoundInclusion::kIncludeStartKeyOnly, - 2U)); - ASSERT_EQUALS(2U, operationsWrittenToOplog.size()); - - auto lastEntry = unittest::assertGet(OplogEntry::parse(operationsWrittenToOplog[0])); - auto secondToLastEntry = unittest::assertGet(OplogEntry::parse(operationsWrittenToOplog[1])); - ASSERT_EQUALS(op1, secondToLastEntry); - ASSERT_EQUALS(op2, lastEntry); -} - TEST_F(SyncTailTest, MultiSyncApplyUsesSyncApplyToApplyOperation) { NamespaceString nss("local." + _agent.getSuiteName() + "_" + _agent.getTestName()); auto op = makeCreateCollectionOplogEntry({Timestamp(Seconds(1), 0), 1LL}, nss); diff --git a/src/mongo/db/stats/top.cpp b/src/mongo/db/stats/top.cpp index 63f1a5b26d6..a4b8e5e348d 100644 --- a/src/mongo/db/stats/top.cpp +++ b/src/mongo/db/stats/top.cpp @@ -84,7 +84,7 @@ void Top::record(OperationContext* opCtx, if (ns[0] == '?') return; - auto hashedNs = UsageMap::HashedKey(ns); + auto hashedNs = UsageMap::hasher().hashed_key(ns); stdx::lock_guard<SimpleMutex> lk(_lock); if ((command || logicalOp == LogicalOp::opQuery) && @@ -202,7 +202,7 @@ void Top::_appendStatsEntry(BSONObjBuilder& b, const char* statsName, const Usag } void Top::appendLatencyStats(StringData ns, bool includeHistograms, BSONObjBuilder* builder) { - auto hashedNs = UsageMap::HashedKey(ns); + auto hashedNs = UsageMap::hasher().hashed_key(ns); stdx::lock_guard<SimpleMutex> lk(_lock); BSONObjBuilder latencyStatsBuilder; _usage[hashedNs].opLatencyHistogram.append(includeHistograms, &latencyStatsBuilder); diff --git a/src/mongo/s/transaction_router_test.cpp b/src/mongo/s/transaction_router_test.cpp index 5037c4f9769..e78ae8707fc 100644 --- a/src/mongo/s/transaction_router_test.cpp +++ b/src/mongo/s/transaction_router_test.cpp @@ -30,6 +30,9 @@ #include "mongo/platform/basic.h" +#include <map> +#include <set> + #include "mongo/client/remote_command_targeter_mock.h" #include "mongo/db/logical_clock.h" #include "mongo/db/repl/read_concern_args.h" @@ -684,11 +687,15 @@ TEST_F(TransactionRouterTest, SendCoordinateCommitForMultipleParticipants) { auto cmdName = request.cmdObj.firstElement().fieldNameStringData(); ASSERT_EQ(cmdName, "coordinateCommitTransaction"); + std::set<std::string> expectedParticipants = {shard1.toString(), shard2.toString()}; auto participantElements = request.cmdObj["participants"].Array(); - ASSERT_EQ(2u, participantElements.size()); + ASSERT_EQ(expectedParticipants.size(), participantElements.size()); - ASSERT_BSONOBJ_EQ(BSON("shardId" << shard1.toString()), participantElements.front().Obj()); - ASSERT_BSONOBJ_EQ(BSON("shardId" << shard2.toString()), participantElements.back().Obj()); + for (const auto& element : participantElements) { + auto shardId = element["shardId"].valuestr(); + ASSERT_EQ(1ull, expectedParticipants.count(shardId)); + expectedParticipants.erase(shardId); + } checkSessionDetails(request.cmdObj, lsid, txnNum, true); @@ -1135,29 +1142,24 @@ TEST_F(TransactionRouterTest, AbortForMultipleParticipants) { auto future = launchAsync([&] { return txnRouter->abortTransaction(operationContext()); }); - onCommandForPoolExecutor([&](const RemoteCommandRequest& request) { - ASSERT_EQ(hostAndPort1, request.target); - ASSERT_EQ("admin", request.dbname); + std::map<HostAndPort, boost::optional<bool>> targets = {{hostAndPort1, true}, + {hostAndPort2, {}}}; - auto cmdName = request.cmdObj.firstElement().fieldNameStringData(); - ASSERT_EQ(cmdName, "abortTransaction"); + while (!targets.empty()) { + onCommandForPoolExecutor([&](const RemoteCommandRequest& request) { + auto target = targets.find(request.target); + ASSERT(target != targets.end()); + ASSERT_EQ("admin", request.dbname); - checkSessionDetails(request.cmdObj, lsid, txnNum, true); + auto cmdName = request.cmdObj.firstElement().fieldNameStringData(); + ASSERT_EQ(cmdName, "abortTransaction"); - return BSON("ok" << 1); - }); + checkSessionDetails(request.cmdObj, lsid, txnNum, target->second); - onCommandForPoolExecutor([&](const RemoteCommandRequest& request) { - ASSERT_EQ(hostAndPort2, request.target); - ASSERT_EQ("admin", request.dbname); - - auto cmdName = request.cmdObj.firstElement().fieldNameStringData(); - ASSERT_EQ(cmdName, "abortTransaction"); - - checkSessionDetails(request.cmdObj, lsid, txnNum, boost::none); - - return BSON("ok" << 1); - }); + targets.erase(request.target); + return BSON("ok" << 1); + }); + } auto response = future.timed_get(kFutureTimeout); ASSERT_FALSE(response.empty()); @@ -1277,29 +1279,24 @@ TEST_F(TransactionRouterTest, ImplicitAbortForMultipleParticipants) { auto future = launchAsync([&] { return txnRouter->implicitlyAbortTransaction(operationContext()); }); - onCommandForPoolExecutor([&](const RemoteCommandRequest& request) { - ASSERT_EQ(hostAndPort1, request.target); - ASSERT_EQ("admin", request.dbname); - - auto cmdName = request.cmdObj.firstElement().fieldNameStringData(); - ASSERT_EQ(cmdName, "abortTransaction"); - - checkSessionDetails(request.cmdObj, lsid, txnNum, true); - - return BSON("ok" << 1); - }); + std::map<HostAndPort, boost::optional<bool>> targets = {{hostAndPort1, true}, + {hostAndPort2, {}}}; - onCommandForPoolExecutor([&](const RemoteCommandRequest& request) { - ASSERT_EQ(hostAndPort2, request.target); - ASSERT_EQ("admin", request.dbname); + while (!targets.empty()) { + onCommandForPoolExecutor([&](const RemoteCommandRequest& request) { + auto target = targets.find(request.target); + ASSERT(target != targets.end()); + ASSERT_EQ("admin", request.dbname); - auto cmdName = request.cmdObj.firstElement().fieldNameStringData(); - ASSERT_EQ(cmdName, "abortTransaction"); + auto cmdName = request.cmdObj.firstElement().fieldNameStringData(); + ASSERT_EQ(cmdName, "abortTransaction"); - checkSessionDetails(request.cmdObj, lsid, txnNum, boost::none); + checkSessionDetails(request.cmdObj, lsid, txnNum, target->second); - return BSON("ok" << 1); - }); + targets.erase(request.target); + return BSON("ok" << 1); + }); + } future.timed_get(kFutureTimeout); } diff --git a/src/mongo/util/hash_table_bm.cpp b/src/mongo/util/hash_table_bm.cpp index 5b86a85d6f3..2c7ec3d0480 100644 --- a/src/mongo/util/hash_table_bm.cpp +++ b/src/mongo/util/hash_table_bm.cpp @@ -31,7 +31,6 @@ #include "mongo/platform/basic.h" #include "mongo/util/string_map.h" -#include "mongo/util/unordered_fast_key_table.h" #include <absl/container/flat_hash_map.h> #include <absl/container/node_hash_map.h> @@ -51,59 +50,9 @@ constexpr uint32_t kMaxContainerSize = 1000000; constexpr uint32_t kDefaultSeed = 34862; constexpr uint32_t kOtherSeed = 76453; -template <typename K_L, typename K_S> -struct UnorderedFastKeyTableAbslTraits { - static uint32_t hash(K_L key) { - return static_cast<uint32_t>(absl::Hash<K_L>{}(key)); - } - - static bool equals(K_L a, K_L b) { - return a == b; - } - - static K_S toStorage(K_L lookup) { - return static_cast<K_S>(lookup); - } - - static K_L toLookup(K_S const& storage) { - return static_cast<K_L>(storage); - } - - class HashedKey { - public: - explicit HashedKey(K_L key) - : _key(std::move(key)), _hash(UnorderedFastKeyTableAbslTraits<K_L, K_S>::hash(_key)) {} - - HashedKey(K_L key, uint32_t hash) : _key(std::move(key)), _hash(hash) {} - - K_L key() const { - return _key; - } - - uint32_t hash() const { - return _hash; - } - - private: - K_L _key; - uint32_t _hash; - }; -}; - using StdUnorderedInt = std::unordered_map<uint32_t, bool>; // NOLINT using StdUnorderedString = std::unordered_map<std::string, bool>; // NOLINT -using MongoUnorderedFastKeyTableInt = - UnorderedFastKeyTable<uint32_t, - uint32_t, - bool, - UnorderedFastKeyTableAbslTraits<uint32_t, uint32_t>>; -using MongoUnorderedFastKeyTableString = - UnorderedFastKeyTable<absl::string_view, - std::string, - bool, - UnorderedFastKeyTableAbslTraits<absl::string_view, std::string>>; - using AbslFlatHashMapInt = absl::flat_hash_map<uint32_t, bool>; using AbslFlatHashMapString = absl::flat_hash_map<std::string, bool>; @@ -111,12 +60,6 @@ using AbslNodeHashMapInt = absl::node_hash_map<uint32_t, bool>; using AbslNodeHashMapString = absl::node_hash_map<std::string, bool>; template <typename> -struct IsUnorderedFastKeyTable : std::false_type {}; - -template <typename K_L, typename K_S, typename V, typename Traits> -struct IsUnorderedFastKeyTable<UnorderedFastKeyTable<K_L, K_S, V, Traits>> : std::true_type {}; - -template <typename> struct IsAbslHashMap : std::false_type {}; template <typename K, typename V, typename Hash, typename Eq, typename Allocator> @@ -135,16 +78,6 @@ struct LookupType { typename Container::key_type>; }; -template <class T> -std::enable_if_t<!IsUnorderedFastKeyTable<T>::value, float> getLoadFactor(const T& container) { - return container.load_factor(); -} - -template <class T> -std::enable_if_t<IsUnorderedFastKeyTable<T>::value, float> getLoadFactor(const T& container) { - return container.empty() ? 0.0f : (float)container.size() / container.capacity(); -} - class BaseGenerator { public: template <typename K> @@ -245,7 +178,7 @@ void LookupTest(benchmark::State& state) { } state.counters["size"] = state.range(0); - state.counters["load_factor"] = getLoadFactor(container); + state.counters["load_factor"] = container.load_factor(); } template <class Container, class LookupKey, class StorageGenerator> @@ -316,42 +249,34 @@ static void Range(benchmark::internal::Benchmark* b) { // Integer key tests BENCHMARK_TEMPLATE(BM_SuccessfulLookup, StdUnorderedInt)->Apply(Range); -BENCHMARK_TEMPLATE(BM_SuccessfulLookup, MongoUnorderedFastKeyTableInt)->Apply(Range); BENCHMARK_TEMPLATE(BM_SuccessfulLookup, AbslFlatHashMapInt)->Apply(Range); BENCHMARK_TEMPLATE(BM_SuccessfulLookup, AbslNodeHashMapInt)->Apply(Range); BENCHMARK_TEMPLATE(BM_UnsuccessfulLookup, StdUnorderedInt)->Apply(Range); -BENCHMARK_TEMPLATE(BM_UnsuccessfulLookup, MongoUnorderedFastKeyTableInt)->Apply(Range); BENCHMARK_TEMPLATE(BM_UnsuccessfulLookup, AbslFlatHashMapInt)->Apply(Range); BENCHMARK_TEMPLATE(BM_UnsuccessfulLookup, AbslNodeHashMapInt)->Apply(Range); BENCHMARK_TEMPLATE(BM_UnsuccessfulLookupSeq, StdUnorderedInt)->Apply(Range); -BENCHMARK_TEMPLATE(BM_UnsuccessfulLookupSeq, MongoUnorderedFastKeyTableInt)->Apply(Range); BENCHMARK_TEMPLATE(BM_UnsuccessfulLookupSeq, AbslFlatHashMapInt)->Apply(Range); BENCHMARK_TEMPLATE(BM_UnsuccessfulLookupSeq, AbslNodeHashMapInt)->Apply(Range); BENCHMARK_TEMPLATE(BM_Insert, StdUnorderedInt)->Apply(Range<1>); -BENCHMARK_TEMPLATE(BM_Insert, MongoUnorderedFastKeyTableInt)->Apply(Range<1>); BENCHMARK_TEMPLATE(BM_Insert, AbslFlatHashMapInt)->Apply(Range<1>); BENCHMARK_TEMPLATE(BM_Insert, AbslNodeHashMapInt)->Apply(Range<1>); // String key tests BENCHMARK_TEMPLATE(BM_SuccessfulLookup, StdUnorderedString)->Apply(Range); -BENCHMARK_TEMPLATE(BM_SuccessfulLookup, MongoUnorderedFastKeyTableString)->Apply(Range); BENCHMARK_TEMPLATE(BM_SuccessfulLookup, AbslFlatHashMapString)->Apply(Range); BENCHMARK_TEMPLATE(BM_SuccessfulLookup, AbslNodeHashMapString)->Apply(Range); BENCHMARK_TEMPLATE(BM_UnsuccessfulLookup, StdUnorderedString)->Apply(Range); -BENCHMARK_TEMPLATE(BM_UnsuccessfulLookup, MongoUnorderedFastKeyTableString)->Apply(Range); BENCHMARK_TEMPLATE(BM_UnsuccessfulLookup, AbslFlatHashMapString)->Apply(Range); BENCHMARK_TEMPLATE(BM_UnsuccessfulLookup, AbslNodeHashMapString)->Apply(Range); BENCHMARK_TEMPLATE(BM_UnsuccessfulLookupSeq, StdUnorderedString)->Apply(Range); -BENCHMARK_TEMPLATE(BM_UnsuccessfulLookupSeq, MongoUnorderedFastKeyTableString)->Apply(Range); BENCHMARK_TEMPLATE(BM_UnsuccessfulLookupSeq, AbslNodeHashMapString)->Apply(Range); BENCHMARK_TEMPLATE(BM_Insert, StdUnorderedString)->Apply(Range<1>); -BENCHMARK_TEMPLATE(BM_Insert, MongoUnorderedFastKeyTableString)->Apply(Range<1>); BENCHMARK_TEMPLATE(BM_Insert, AbslFlatHashMapString)->Apply(Range<1>); BENCHMARK_TEMPLATE(BM_Insert, AbslNodeHashMapString)->Apply(Range<1>); diff --git a/src/mongo/util/string_map.h b/src/mongo/util/string_map.h index 03f341ceeff..9d65713ffdf 100644 --- a/src/mongo/util/string_map.h +++ b/src/mongo/util/string_map.h @@ -32,60 +32,88 @@ #pragma once -#include <third_party/murmurhash3/MurmurHash3.h> +#include <absl/container/flat_hash_map.h> +#include <absl/container/flat_hash_set.h> #include "mongo/base/string_data.h" #include "mongo/util/assert_util.h" -#include "mongo/util/unordered_fast_key_table.h" namespace mongo { -struct StringMapTraits { - static uint32_t hash(StringData a) { - uint32_t hash; - MurmurHash3_x86_32(a.rawData(), a.size(), 0, &hash); - return hash; +// Type that bundles a hashed key with the actual string so hashing can be performed outside of +// insert call by using heterogeneous lookup. +struct StringMapHashedKey { +public: + explicit StringMapHashedKey(StringData sd, std::size_t hash) : _sd(sd), _hash(hash) {} + + explicit operator std::string() const { + return _sd.toString(); + } + + StringData key() const { + return _sd; + } + + std::size_t hash() const { + return _hash; + } + +private: + StringData _sd; + std::size_t _hash; +}; + +// Hasher to support heterogeneous lookup for StringData and string-like elements. +struct StringMapHasher { + // This using directive activates heterogeneous lookup in the hash table + using is_transparent = void; + + std::size_t operator()(StringData sd) const { + // Use the default absl string hasher. + return absl::Hash<absl::string_view>{}(absl::string_view(sd.rawData(), sd.size())); } - static bool equals(StringData a, StringData b) { - return a == b; + std::size_t operator()(const std::string& s) const { + return operator()(StringData(s)); } - static std::string toStorage(StringData s) { - return s.toString(); + std::size_t operator()(const char* s) const { + return operator()(StringData(s)); } - static StringData toLookup(const std::string& s) { - return StringData(s); + std::size_t operator()(StringMapHashedKey key) const { + return key.hash(); } - class HashedKey { - public: - explicit HashedKey(StringData key = "") : _key(key), _hash(StringMapTraits::hash(_key)) {} + StringMapHashedKey hashed_key(StringData sd) { + return StringMapHashedKey(sd, operator()(sd)); + } +}; - HashedKey(StringData key, uint32_t hash) : _key(key), _hash(hash) { - // If you claim to know the hash, it better be correct. - dassert(_hash == StringMapTraits::hash(_key)); - } +struct StringMapEq { + // This using directive activates heterogeneous lookup in the hash table + using is_transparent = void; - const StringData& key() const { - return _key; - } + bool operator()(StringData lhs, StringData rhs) const { + return lhs == rhs; + } - uint32_t hash() const { - return _hash; - } + bool operator()(StringMapHashedKey lhs, StringData rhs) const { + return lhs.key() == rhs; + } - private: - StringData _key; - uint32_t _hash; - }; + bool operator()(StringData lhs, StringMapHashedKey rhs) const { + return lhs == rhs.key(); + } + + bool operator()(StringMapHashedKey lhs, StringMapHashedKey rhs) const { + return lhs.key() == rhs.key(); + } }; template <typename V> -using StringMap = UnorderedFastKeyTable<StringData, // K_L - std::string, // K_S - V, - StringMapTraits>; +using StringMap = absl::flat_hash_map<std::string, V, StringMapHasher, StringMapEq>; + +using StringSet = absl::flat_hash_set<std::string, StringMapHasher, StringMapEq>; } // namespace mongo diff --git a/src/mongo/util/string_map_test.cpp b/src/mongo/util/string_map_test.cpp index a4bffe71a90..8bffcbded1b 100644 --- a/src/mongo/util/string_map_test.cpp +++ b/src/mongo/util/string_map_test.cpp @@ -41,7 +41,7 @@ namespace { using namespace mongo; TEST(StringMapTest, Hash1) { - auto hash = StringMapTraits::hash; + StringSet::hasher hash; ASSERT_EQUALS(hash(""), hash("")); ASSERT_EQUALS(hash("a"), hash("a")); ASSERT_EQUALS(hash("abc"), hash("abc")); @@ -61,7 +61,7 @@ TEST(StringMapTest, Hash1) { ASSERT_FALSE(equals((b), (a))); TEST(StringMapTest, Equals1) { - auto equals = StringMapTraits::equals; + StringSet::key_equal equals; equalsBothWays("", ""); equalsBothWays("a", "a"); diff --git a/src/mongo/util/unordered_fast_key_table.h b/src/mongo/util/unordered_fast_key_table.h deleted file mode 100644 index 7d84da79d5c..00000000000 --- a/src/mongo/util/unordered_fast_key_table.h +++ /dev/null @@ -1,436 +0,0 @@ -// unordered_fast_key_table.h - - -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#pragma once - -#include <iterator> -#include <memory> -#include <type_traits> - -#include "mongo/base/disallow_copying.h" -#include "mongo/util/assert_util.h" - -namespace mongo { - -/** - * A hash map that allows a different type to be used stored (K_S) than is used for lookups (K_L). - * - * Takes a Traits class that must have the following: - * - * static uint32_t hash(K_L); // Computes a 32-bit hash of the key. - * static bool equals(K_L, K_L); // Returns true if the keys are equal. - * static K_S toStorage(K_L); // Converts from K_L to K_S. - * static K_L toLookup(K_S); // Converts from K_S to K_L. - * class HashedKey { - * public: - * explicit HashedKey(K_L key); // Computes hash of key. - * HashedKey(K_L key, uint32_t hash); // Populates with known hash. - * - * const K_L& key() const; - * uint32_t hash() const; // Should be free to call repeatedly. - * }; - */ -template <typename K_L, // key lookup - typename K_S, // key storage - typename V, // value - typename Traits> -class UnorderedFastKeyTable { -public: - // Typedefs for compatibility with std::map. - using value_type = std::pair<const K_S, V>; - using key_type = K_L; - using mapped_type = V; - - using HashedKey = typename Traits::HashedKey; - -private: - class Entry { - public: - Entry() = default; - - Entry(const Entry& other) - : _used(other._used), _everUsed(other.everUsed), _curHash(other._curHash) { - if (other.isUsed()) { - new (&_data) value_type(other.getData()); - } - } - - Entry& operator=(const Entry& other) { - if (this == &other) { - return *this; - } - - if (isUsed()) { - unUse(); - } - - _used = other._used; - _everUsed = other._everUsed; - _curHash = other._curHash; - - if (other.isUsed()) { - new (&_data) value_type(other.getData()); - } - - return *this; - } - - ~Entry() { - if (isUsed()) { - unUse(); - } - } - - template <typename... Args> - void emplaceData(const HashedKey& key, Args&&... args) { - dassert(!isUsed()); - _used = true; - _everUsed = true; - _curHash = key.hash(); - new (&_data) value_type(std::piecewise_construct, - std::forward_as_tuple(Traits::toStorage(key.key())), - std::forward_as_tuple(std::forward<Args>(args)...)); - } - - bool isUsed() const { - return _used; - } - - bool wasEverUsed() const { - return _everUsed; - } - - uint32_t getCurHash() const { - dassert(isUsed()); - return _curHash; - } - - void unUse() { - dassert(isUsed()); - _used = false; - reinterpret_cast<value_type*>(&_data)->~value_type(); - } - - value_type& getData() { - dassert(isUsed()); - return *reinterpret_cast<value_type*>(&_data); - } - - const value_type& getData() const { - dassert(isUsed()); - return *reinterpret_cast<const value_type*>(&_data); - } - - private: - bool _used = false; - bool _everUsed = false; - uint32_t _curHash; - typename std::aligned_storage<sizeof(value_type), - std::alignment_of<value_type>::value>::type _data; - }; - - struct Area { - Area() = default; // TODO constexpr - - Area(unsigned capacity, unsigned maxProbe) - : _hashMask(capacity - 1), - _maxProbe(maxProbe), - _entries(capacity ? new Entry[capacity] : nullptr) { - // Capacity must be a power of two or zero. See the comment on _hashMask for why. - dassert((capacity & (capacity - 1)) == 0); - } - - Area(const Area& other) : Area(other.capacity(), other._maxProbe) { - std::copy(other.begin(), other.end(), begin()); - } - - Area& operator=(const Area& other) { - Area(other).swap(this); - return *this; - } - - int find(const HashedKey& key, int* firstEmpty) const; - - bool transfer(Area* newArea) const; - - void swap(Area* other) { - using std::swap; - swap(_hashMask, other->_hashMask); - swap(_maxProbe, other->_maxProbe); - swap(_entries, other->_entries); - } - - unsigned capacity() const { - return _hashMask + 1; - } - - Entry* begin() { - return _entries.get(); - } - Entry* end() { - return _entries.get() + capacity(); - } - - const Entry* begin() const { - return _entries.get(); - } - const Entry* end() const { - return _entries.get() + capacity(); - } - - // Capacity is always a power of two. This means that the operation (hash % capacity) can be - // preformed by (hash & (capacity - 1)). Since we need the mask more than the capacity we - // store it directly and derive the capacity from it. The default capacity is 0 so the - // default hashMask is -1. - unsigned _hashMask = -1; - unsigned _maxProbe = 0; - std::unique_ptr<Entry[]> _entries = {}; - }; - -public: - UnorderedFastKeyTable() = default; // TODO constexpr - - UnorderedFastKeyTable(std::initializer_list<std::pair<key_type, mapped_type>> entries); - - /** - * @return number of elements in map - */ - size_t size() const { - return _size; - } - - bool empty() const { - return _size == 0; - } - - /* - * @return storage space - */ - size_t capacity() const { - return _area.capacity(); - } - - V& operator[](const HashedKey& key) { - return get(key); - } - V& operator[](const K_L& key) { - return get(key); - } - - V& get(const HashedKey& key); - V& get(const K_L& key) { - return get(HashedKey(key)); - } - - void clear() { - *this = {}; - } - - /** - * @return number of elements removed - */ - size_t erase(const HashedKey& key); - size_t erase(const K_L& key) { - if (empty()) - return 0; // Don't waste time hashing. - return erase(HashedKey(key)); - } - - template <typename AreaPtr, - typename reference = decltype(AreaPtr()->begin()->getData()), - typename pointer = typename std::add_pointer<reference>::type> - class iterator_impl - : public std:: - iterator<std::forward_iterator_tag, value_type, std::ptrdiff_t, pointer, reference> { - friend class UnorderedFastKeyTable; - - public: - iterator_impl() { - _position = -1; - } - iterator_impl(AreaPtr area) { - _area = area; - _position = 0; - _max = _area->capacity() - 1; - _skip(); - } - iterator_impl(AreaPtr area, int pos) { - _area = area; - _position = pos; - _max = pos; - } - - template <typename... Args> - iterator_impl(const iterator_impl<Args...>& other) - : _area(other._area), _position(other._position), _max(other._max) {} - - pointer operator->() const { - return &_area->_entries[_position].getData(); - } - - reference operator*() const { - return _area->_entries[_position].getData(); - } - - iterator_impl& operator++() { - if (_position < 0) - return *this; - _position++; - _skip(); - return *this; - } - - iterator_impl operator++(int) { - iterator_impl before(*this); - operator++(); - return before; - } - - bool operator==(const iterator_impl& other) const { - return _position == other._position; - } - bool operator!=(const iterator_impl& other) const { - return _position != other._position; - } - - private: - void _skip() { - while (true) { - if (_position > _max) { - _position = -1; - break; - } - if (_area->_entries[_position].isUsed()) - break; - ++_position; - } - } - - AreaPtr _area; - int _position; - int _max; // inclusive - }; - - using iterator = iterator_impl<Area*>; - using const_iterator = iterator_impl<const Area*>; - - void erase(const_iterator it); - - /** - * @return either a one-shot iterator with the key, or end() - */ - const_iterator find(const K_L& key) const { - if (empty()) - return end(); // Don't waste time hashing. - return const_iterator(&_area, _area.find(HashedKey(key), nullptr)); - } - - const_iterator find(const HashedKey& key) const { - if (empty()) - return end(); - return const_iterator(&_area, _area.find(key, nullptr)); - } - - iterator find(const K_L& key) { - if (empty()) - return end(); // Don't waste time hashing. - return iterator(&_area, _area.find(HashedKey(key), nullptr)); - } - - iterator find(const HashedKey& key) { - if (empty()) - return end(); - return iterator(&_area, _area.find(key, nullptr)); - } - - size_t count(const K_L& key) const { - if (empty()) - return 0; // Don't waste time hashing. - return _area.find(HashedKey(key), nullptr) != -1; - } - - size_t count(const HashedKey& key) const { - if (empty()) - return 0; - return _area.find(key, nullptr) != -1; - } - - const_iterator begin() const { - return const_iterator(&_area); - } - - const_iterator end() const { - return const_iterator(); - } - - iterator begin() { - return iterator(&_area); - } - - iterator end() { - return iterator(); - } - - const_iterator cbegin() const { - return const_iterator(&_area); - } - - const_iterator cend() const { - return const_iterator(); - } - - template <typename... Args> - std::pair<iterator, bool> try_emplace(const HashedKey& key, Args&&... args); - - template <typename... Args> - std::pair<iterator, bool> try_emplace(const K_L& key, Args&&... args) { - return try_emplace(HashedKey(key), std::forward<Args>(args)...); - } - - void swap(UnorderedFastKeyTable& other) { - _area.swap(&(other._area)); - std::swap(_size, other._size); - } - - friend void swap(UnorderedFastKeyTable& lhs, UnorderedFastKeyTable& rhs) { - return lhs.swap(rhs); - } - -private: - void _grow(); - - size_t _size = 0; - Area _area; -}; -} - -#include "mongo/util/unordered_fast_key_table_internal.h" diff --git a/src/mongo/util/unordered_fast_key_table_internal.h b/src/mongo/util/unordered_fast_key_table_internal.h deleted file mode 100644 index 920169122d1..00000000000 --- a/src/mongo/util/unordered_fast_key_table_internal.h +++ /dev/null @@ -1,198 +0,0 @@ -// unordered_fast_key_table_internal.h - - -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#pragma once - -#include "mongo/util/unordered_fast_key_table.h" - -namespace mongo { - -template <typename K_L, typename K_S, typename V, typename Traits> -inline int UnorderedFastKeyTable<K_L, K_S, V, Traits>::Area::find(const HashedKey& key, - int* firstEmpty) const { - dassert(capacity()); // Caller must special-case empty tables. - dassert(!firstEmpty || *firstEmpty == -1); // Caller must initialize *firstEmpty. - - unsigned probe = 0; - do { - unsigned pos = (key.hash() + probe) & _hashMask; - - if (!_entries[pos].isUsed()) { - // space is empty - if (firstEmpty && *firstEmpty == -1) - *firstEmpty = pos; - if (!_entries[pos].wasEverUsed()) - return -1; - continue; - } - - if (_entries[pos].getCurHash() != key.hash()) { - // space has something else - continue; - } - - if (!Traits::equals(key.key(), Traits::toLookup(_entries[pos].getData().first))) { - // hashes match - // strings are not equals - continue; - } - - // hashes and strings are equal - // yay! - return pos; - } while (++probe < _maxProbe); - return -1; -} - -template <typename K_L, typename K_S, typename V, typename Traits> -inline bool UnorderedFastKeyTable<K_L, K_S, V, Traits>::Area::transfer(Area* newArea) const { - for (auto&& entry : *this) { - if (!entry.isUsed()) - continue; - - int firstEmpty = -1; - int loc = newArea->find( - HashedKey(Traits::toLookup(entry.getData().first), entry.getCurHash()), &firstEmpty); - - verify(loc == -1); - if (firstEmpty < 0) { - return false; - } - - newArea->_entries[firstEmpty] = entry; - } - return true; -} - -template <typename K_L, typename K_S, typename V, typename Traits> -inline UnorderedFastKeyTable<K_L, K_S, V, Traits>::UnorderedFastKeyTable( - std::initializer_list<std::pair<key_type, mapped_type>> entries) - : UnorderedFastKeyTable() { - for (auto&& entry : entries) { - // Only insert the entry if the key is not equivalent to the key of any other element - // already in the table. - auto key = HashedKey(entry.first); - if (find(key) == end()) { - get(key) = entry.second; - } - } -} - -template <typename K_L, typename K_S, typename V, typename Traits> -inline V& UnorderedFastKeyTable<K_L, K_S, V, Traits>::get(const HashedKey& key) { - return try_emplace(key).first->second; -} - -template <typename K_L, typename K_S, typename V, typename Traits> -inline size_t UnorderedFastKeyTable<K_L, K_S, V, Traits>::erase(const HashedKey& key) { - if (_size == 0) - return 0; // Nothing to delete. - - int pos = _area.find(key, nullptr); - - if (pos < 0) - return 0; - - --_size; - _area._entries[pos].unUse(); - return 1; -} - -template <typename K_L, typename K_S, typename V, typename Traits> -void UnorderedFastKeyTable<K_L, K_S, V, Traits>::erase(const_iterator it) { - dassert(it._position >= 0); - dassert(it._area == &_area); - - --_size; - _area._entries[it._position].unUse(); -} - -template <typename K_L, typename K_S, typename V, typename Traits> -template <typename... Args> -inline auto UnorderedFastKeyTable<K_L, K_S, V, Traits>::try_emplace(const HashedKey& key, - Args&&... args) - -> std::pair<iterator, bool> { - if (!_area._entries) { - // This is the first insert ever. Need to allocate initial space. - dassert(_area.capacity() == 0); - _grow(); - } - - for (int numGrowTries = 0; numGrowTries < 5; numGrowTries++) { - int firstEmpty = -1; - int pos = _area.find(key, &firstEmpty); - if (pos >= 0) { - // This is only possible the first pass through the loop, since you're allocating space - // for a new element after that. - dassert(numGrowTries == 0); - return {iterator(&_area, pos), false}; - } - - // key not in map - // need to add - if (firstEmpty >= 0) { - _size++; - _area._entries[firstEmpty].emplaceData(key, std::forward<Args>(args)...); - return {iterator(&_area, firstEmpty), true}; - } - - // no space left in map - _grow(); - } - msgasserted(16471, "UnorderedFastKeyTable couldn't add entry after growing many times"); -} - -template <typename K_L, typename K_S, typename V, typename Traits> -inline void UnorderedFastKeyTable<K_L, K_S, V, Traits>::_grow() { - unsigned capacity = _area.capacity(); - for (int numGrowTries = 0; numGrowTries < 5; numGrowTries++) { - if (capacity == 0) { - const unsigned kDefaultStartingCapacity = 16; - capacity = kDefaultStartingCapacity; - } else { - capacity *= 2; - } - - const double kMaxProbeRatio = 0.05; - unsigned maxProbes = (capacity * kMaxProbeRatio) + 1; // Round up - - Area newArea(capacity, maxProbes); - bool success = _area.transfer(&newArea); - if (!success) { - continue; - } - _area.swap(&newArea); - return; - } - msgasserted(16845, "UnorderedFastKeyTable::_grow couldn't add entry after growing many times"); -} -} diff --git a/src/mongo/util/unordered_fast_key_table_traits_helpers.h b/src/mongo/util/unordered_fast_key_table_traits_helpers.h deleted file mode 100644 index 5eea729f6ed..00000000000 --- a/src/mongo/util/unordered_fast_key_table_traits_helpers.h +++ /dev/null @@ -1,96 +0,0 @@ - -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#pragma once - -#include "mongo/util/unordered_fast_key_table.h" - -namespace mongo { - -template <typename Key, typename Hasher> -struct UnorderedFastKeyTableTraitsFactoryForPtrKey { - struct Traits { - static uint32_t hash(const Key* a) { - return Hasher()(*a); - } - - static bool equals(const Key* a, const Key* b) { - return *a == *b; - } - - static Key toStorage(const Key* s) { - return *s; - } - - static const Key* toLookup(const Key& s) { - return &s; - } - - class HashedKey { - public: - HashedKey() = default; - - explicit HashedKey(const Key* key) : _key(key), _hash(Traits::hash(_key)) {} - - HashedKey(const Key* key, uint32_t hash) : _key(key), _hash(hash) { - // If you claim to know the hash, it better be correct. - dassert(_hash == Traits::hash(_key)); - } - - const Key* key() const { - return _key; - } - - uint32_t hash() const { - return _hash; - } - - private: - const Key* _key = nullptr; - uint32_t _hash = 0; - }; - }; - - template <typename V> - using type = UnorderedFastKeyTable<Key*, Key, V, Traits>; -}; - -/** - * Provides a Hasher which forwards to an instance's .hash() method. This should only be used with - * high quality hashing functions because UnorderedFastKeyMap uses bit masks, rather than % by - * prime, which can provide poor behavior without good overall distribution. - */ -template <typename T> -struct UnorderedFastKeyTableInstanceMethodHasher { - auto operator()(const T& t) const -> decltype(t.hash()) { - return t.hash(); - } -}; -} |