diff options
Diffstat (limited to 'src/mongo/db/storage')
7 files changed, 210 insertions, 12 deletions
diff --git a/src/mongo/db/storage/key_string.cpp b/src/mongo/db/storage/key_string.cpp index 0a93caa0c9a..9d1d88121b2 100644 --- a/src/mongo/db/storage/key_string.cpp +++ b/src/mongo/db/storage/key_string.cpp @@ -2705,6 +2705,15 @@ void Value::serializeWithoutRecordIdLong(BufBuilder& buf) const { buf.appendBuf(_buffer.get() + _ksSize, _buffer.size() - _ksSize); // Serialize TypeBits } +void Value::serializeWithoutRecordIdStr(BufBuilder& buf) const { + dassert(decodeRecordIdStrAtEnd(_buffer.get(), _ksSize).isValid()); + + const int32_t sizeWithoutRecordId = sizeWithoutRecordIdStrAtEnd(_buffer.get(), _ksSize); + buf.appendNum(sizeWithoutRecordId); // Serialize size of KeyString + buf.appendBuf(_buffer.get(), sizeWithoutRecordId); // Serialize KeyString + buf.appendBuf(_buffer.get() + _ksSize, _buffer.size() - _ksSize); // Serialize TypeBits +} + size_t Value::getApproximateSize() const { auto size = sizeof(Value); size += !_buffer.isShared() ? SharedBuffer::kHolderSize + _buffer.size() : 0; diff --git a/src/mongo/db/storage/key_string.h b/src/mongo/db/storage/key_string.h index d8db757213e..f0e1441004c 100644 --- a/src/mongo/db/storage/key_string.h +++ b/src/mongo/db/storage/key_string.h @@ -335,13 +335,21 @@ public: return *this; } + /** + * Compare with another KeyString::Value or Builder. + */ template <class T> int compare(const T& other) const; int compareWithTypeBits(const Value& other) const; + /** + * Compare with another KeyString::Value or Builder, ignoring the RecordId part of both. + */ template <class T> int compareWithoutRecordIdLong(const T& other) const; + template <class T> + int compareWithoutRecordIdStr(const T& other) const; // Returns the size of the stored KeyString. size_t getSize() const { @@ -383,11 +391,12 @@ public: } /** - * Serializes this Value, excluing the RecordId, into a storable format with TypeBits + * Serializes this Value, excluding the RecordId, into a storable format with TypeBits * information. The serialized format takes the following form: * [keystring size][keystring encoding][typebits encoding] */ void serializeWithoutRecordIdLong(BufBuilder& buf) const; + void serializeWithoutRecordIdStr(BufBuilder& buf) const; // Deserialize the Value from a serialized format. static Value deserialize(BufReader& buf, KeyString::Version version) { @@ -616,11 +625,19 @@ public: return _typeBits; } + /** + * Compare with another KeyString::Value or Builder. + */ template <class T> int compare(const T& other) const; + /** + * Compare with another KeyString::Value or Builder, ignoring the RecordId part of both. + */ template <class T> int compareWithoutRecordIdLong(const T& other) const; + template <class T> + int compareWithoutRecordIdStr(const T& other) const; /** * @return a hex encoding of this key @@ -1030,6 +1047,16 @@ int BuilderBase<BufferT>::compareWithoutRecordIdLong(const T& other) const { !other.isEmpty() ? sizeWithoutRecordIdLongAtEnd(other.getBuffer(), other.getSize()) : 0); } +template <class BufferT> +template <class T> +int BuilderBase<BufferT>::compareWithoutRecordIdStr(const T& other) const { + return KeyString::compare( + getBuffer(), + other.getBuffer(), + !isEmpty() ? sizeWithoutRecordIdStrAtEnd(getBuffer(), getSize()) : 0, + !other.isEmpty() ? sizeWithoutRecordIdStrAtEnd(other.getBuffer(), other.getSize()) : 0); +} + template <class T> int Value::compare(const T& other) const { return KeyString::compare(getBuffer(), other.getBuffer(), getSize(), other.getSize()); @@ -1044,6 +1071,15 @@ int Value::compareWithoutRecordIdLong(const T& other) const { !other.isEmpty() ? sizeWithoutRecordIdLongAtEnd(other.getBuffer(), other.getSize()) : 0); } +template <class T> +int Value::compareWithoutRecordIdStr(const T& other) const { + return KeyString::compare( + getBuffer(), + other.getBuffer(), + !isEmpty() ? sizeWithoutRecordIdStrAtEnd(getBuffer(), getSize()) : 0, + !other.isEmpty() ? sizeWithoutRecordIdStrAtEnd(other.getBuffer(), other.getSize()) : 0); +} + /** * Takes key string and key pattern information and uses it to present human-readable information * about an index or collection entry. diff --git a/src/mongo/db/storage/sorted_data_interface_test_keyformat_string.cpp b/src/mongo/db/storage/sorted_data_interface_test_keyformat_string.cpp index 604c4ae0168..b3a75c6f593 100644 --- a/src/mongo/db/storage/sorted_data_interface_test_keyformat_string.cpp +++ b/src/mongo/db/storage/sorted_data_interface_test_keyformat_string.cpp @@ -110,6 +110,87 @@ TEST(SortedDataInterface, KeyFormatStringInsertDuplicates) { } } +TEST(SortedDataInterface, KeyFormatStringUniqueInsertRemoveDuplicates) { + const auto harnessHelper(newSortedDataInterfaceHarnessHelper()); + const std::unique_ptr<SortedDataInterface> sorted(harnessHelper->newSortedDataInterface( + /*unique=*/true, /*partial=*/false, KeyFormat::String)); + const ServiceContext::UniqueOperationContext opCtx(harnessHelper->newOperationContext()); + ASSERT(sorted->isEmpty(opCtx.get())); + + std::string buf1(12, 0); + std::string buf2(12, 1); + std::string buf3(12, 0xff); + + RecordId rid1(buf1.c_str(), 12); + RecordId rid2(buf2.c_str(), 12); + RecordId rid3(buf3.c_str(), 12); + + { + WriteUnitOfWork uow(opCtx.get()); + ASSERT_OK(sorted->insert(opCtx.get(), + makeKeyString(sorted.get(), key1, rid1), + /*dupsAllowed*/ true)); + Status status = sorted->insert(opCtx.get(), + makeKeyString(sorted.get(), key1, rid2), + /*dupsAllowed*/ false); + ASSERT_EQ(ErrorCodes::DuplicateKey, status.code()); + + ASSERT_OK(sorted->insert(opCtx.get(), + makeKeyString(sorted.get(), key1, rid3), + /*dupsAllowed*/ true)); + uow.commit(); + } + + ASSERT_EQUALS(2, sorted->numEntries(opCtx.get())); + + { + WriteUnitOfWork uow(opCtx.get()); + sorted->unindex(opCtx.get(), + makeKeyString(sorted.get(), key1, rid1), + /*dupsAllowed*/ true); + + ASSERT_OK(sorted->insert(opCtx.get(), + makeKeyString(sorted.get(), key2, rid1), + /*dupsAllowed*/ true)); + uow.commit(); + } + + ASSERT_EQUALS(2, sorted->numEntries(opCtx.get())); + + auto ksSeek = makeKeyStringForSeek(sorted.get(), key1, true, true); + { + auto cursor = sorted->newCursor(opCtx.get()); + auto entry = cursor->seek(ksSeek); + ASSERT(entry); + ASSERT_EQ(*entry, IndexKeyEntry(key1, rid3)); + + entry = cursor->next(); + ASSERT(entry); + ASSERT_EQ(*entry, IndexKeyEntry(key2, rid1)); + + entry = cursor->next(); + ASSERT_FALSE(entry); + } + + { + auto cursor = sorted->newCursor(opCtx.get()); + auto entry = cursor->seekForKeyString(ksSeek); + ASSERT(entry); + ASSERT_EQ(entry->loc, rid3); + auto ks1 = makeKeyString(sorted.get(), key1, rid3); + ASSERT_EQ(entry->keyString, ks1); + + entry = cursor->nextKeyString(); + ASSERT(entry); + ASSERT_EQ(entry->loc, rid1); + auto ks2 = makeKeyString(sorted.get(), key2, rid1); + ASSERT_EQ(entry->keyString, ks2); + + entry = cursor->nextKeyString(); + ASSERT_FALSE(entry); + } +} + TEST(SortedDataInterface, KeyFormatStringSetEndPosition) { const auto harnessHelper(newSortedDataInterfaceHarnessHelper()); const std::unique_ptr<SortedDataInterface> sorted(harnessHelper->newSortedDataInterface( @@ -228,6 +309,58 @@ TEST(SortedDataInterface, KeyFormatStringUnindex) { ASSERT_EQUALS(0, sorted->numEntries(opCtx.get())); } +TEST(SortedDataInterface, KeyFormatStringUniqueUnindex) { + const auto harnessHelper(newSortedDataInterfaceHarnessHelper()); + const std::unique_ptr<SortedDataInterface> sorted(harnessHelper->newSortedDataInterface( + /*unique=*/true, /*partial=*/false, KeyFormat::String)); + const ServiceContext::UniqueOperationContext opCtx(harnessHelper->newOperationContext()); + ASSERT(sorted->isEmpty(opCtx.get())); + + std::string buf1(12, 0); + std::string buf2(12, 1); + std::string buf3(12, 0xff); + + RecordId rid1(buf1.c_str(), 12); + RecordId rid2(buf2.c_str(), 12); + RecordId rid3(buf3.c_str(), 12); + + { + WriteUnitOfWork uow(opCtx.get()); + ASSERT_OK(sorted->insert(opCtx.get(), + makeKeyString(sorted.get(), key1, rid1), + /*dupsAllowed*/ false)); + ASSERT_OK(sorted->insert(opCtx.get(), + makeKeyString(sorted.get(), key2, rid2), + /*dupsAllowed*/ false)); + ASSERT_OK(sorted->insert(opCtx.get(), + makeKeyString(sorted.get(), key3, rid3), + /*dupsAllowed*/ false)); + uow.commit(); + } + ASSERT_EQUALS(3, sorted->numEntries(opCtx.get())); + + { + WriteUnitOfWork uow(opCtx.get()); + // Does not exist, does nothing. + sorted->unindex(opCtx.get(), + makeKeyString(sorted.get(), key1, rid3), + /*dupsAllowed*/ false); + + sorted->unindex(opCtx.get(), + makeKeyString(sorted.get(), key1, rid1), + /*dupsAllowed*/ false); + sorted->unindex(opCtx.get(), + makeKeyString(sorted.get(), key2, rid2), + /*dupsAllowed*/ false); + sorted->unindex(opCtx.get(), + makeKeyString(sorted.get(), key3, rid3), + /*dupsAllowed*/ false); + + uow.commit(); + } + ASSERT_EQUALS(0, sorted->numEntries(opCtx.get())); +} + TEST(SortedDataInterface, InsertReservedRecordIdStr) { const auto harnessHelper(newSortedDataInterfaceHarnessHelper()); const std::unique_ptr<SortedDataInterface> sorted(harnessHelper->newSortedDataInterface( diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp index 8ad8a557afd..b3fe456cbb2 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp @@ -654,11 +654,13 @@ public: } Status addKey(const KeyString::Value& newKeyString) override { - dassertRecordIdAtEnd(newKeyString, KeyFormat::Long); + dassertRecordIdAtEnd(newKeyString, _idx->rsKeyFormat()); // Do a duplicate check, but only if dups aren't allowed. if (!_dupsAllowed) { - const int cmp = newKeyString.compareWithoutRecordIdLong(_previousKeyString); + const int cmp = (_idx->_rsKeyFormat == KeyFormat::Long) + ? newKeyString.compareWithoutRecordIdLong(_previousKeyString) + : newKeyString.compareWithoutRecordIdStr(_previousKeyString); if (cmp == 0) { // Duplicate found! auto newKey = KeyString::toBson(newKeyString, _idx->_ordering); @@ -842,6 +844,8 @@ public: dassert(KeyString::decodeDiscriminator( key.getBuffer(), key.getSize(), _idx.getOrdering(), key.getTypeBits()) == KeyString::Discriminator::kInclusive); + // seekExact is only used on the _id index, which only uses the Long format. + invariant(KeyFormat::Long == _idx.rsKeyFormat()); auto ksEntry = [&]() { if (_forward) { @@ -1303,6 +1307,9 @@ private: // Must not throw WriteConflictException, throwing a WriteConflictException will retry the // operation effectively skipping over this key. void _updateIdAndTypeBitsFromValue() { + // Old-format unique index keys always use the Long format. + invariant(_idx.rsKeyFormat() == KeyFormat::Long); + // We assume that cursors can only ever see unique indexes in their "pristine" state, // where no duplicates are possible. The cases where dups are allowed should hold // sufficient locks to ensure that no cursor ever sees them. @@ -1340,6 +1347,9 @@ public: // Must not throw WriteConflictException, throwing a WriteConflictException will retry the // operation effectively skipping over this key. void updateIdAndTypeBits() override { + // _id index keys always use the Long format. + invariant(_idx.rsKeyFormat() == KeyFormat::Long); + WT_CURSOR* c = _cursor->get(); WT_ITEM item; auto ret = c->get_value(c, &item); @@ -1365,10 +1375,10 @@ public: WiredTigerIndexUnique::WiredTigerIndexUnique(OperationContext* ctx, const std::string& uri, StringData ident, + KeyFormat rsKeyFormat, const IndexDescriptor* desc, bool isReadOnly) - : WiredTigerIndex(ctx, uri, ident, KeyFormat::Long, desc, isReadOnly), - _partial(desc->isPartial()) { + : WiredTigerIndex(ctx, uri, ident, rsKeyFormat, desc, isReadOnly), _partial(desc->isPartial()) { // _id indexes must use WiredTigerIdIndex invariant(!isIdIndex()); // All unique indexes should be in the timestamp-safe format version as of version 4.2. @@ -1488,6 +1498,7 @@ Status WiredTigerIdIndex::_insert(OperationContext* opCtx, WT_CURSOR* c, const KeyString::Value& keyString, bool dupsAllowed) { + invariant(KeyFormat::Long == _rsKeyFormat); invariant(!dupsAllowed); const RecordId id = KeyString::decodeRecordIdLongAtEnd(keyString.getBuffer(), keyString.getSize()); @@ -1534,8 +1545,9 @@ Status WiredTigerIndexUnique::_insert(OperationContext* opCtx, if (!dupsAllowed) { // A prefix key is KeyString of index key. It is the component of the index entry that // should be unique. - auto sizeWithoutRecordId = - KeyString::sizeWithoutRecordIdLongAtEnd(keyString.getBuffer(), keyString.getSize()); + auto sizeWithoutRecordId = (_rsKeyFormat == KeyFormat::Long) + ? KeyString::sizeWithoutRecordIdLongAtEnd(keyString.getBuffer(), keyString.getSize()) + : KeyString::sizeWithoutRecordIdStrAtEnd(keyString.getBuffer(), keyString.getSize()); WiredTigerItem prefixKeyItem(keyString.getBuffer(), sizeWithoutRecordId); // First phase inserts the prefix key to prohibit concurrent insertions of same key @@ -1611,6 +1623,7 @@ void WiredTigerIdIndex::_unindex(OperationContext* opCtx, WT_CURSOR* c, const KeyString::Value& keyString, bool dupsAllowed) { + invariant(KeyFormat::Long == _rsKeyFormat); const RecordId id = KeyString::decodeRecordIdLongAtEnd(keyString.getBuffer(), keyString.getSize()); invariant(id.isValid()); @@ -1701,6 +1714,12 @@ void WiredTigerIndexUnique::_unindex(OperationContext* opCtx, return; } + if (KeyFormat::String == _rsKeyFormat) { + // This is a unique index on a clustered collection. These indexes will only have keys + // in the timestamp safe format where the RecordId is appended at the end of the key. + return; + } + // After a rolling upgrade an index can have keys from both timestamp unsafe (old) and // timestamp safe (new) unique indexes. Old format keys just had the index key while new // format key has index key + Record id. WT_NOTFOUND is possible if index key is in old format. diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h index f37605e784f..a109b3ae5f4 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h @@ -201,6 +201,7 @@ public: WiredTigerIndexUnique(OperationContext* ctx, const std::string& uri, StringData ident, + KeyFormat rsKeyFormat, const IndexDescriptor* desc, bool readOnly = false); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index bcc74d25591..b46954184ea 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -1579,12 +1579,12 @@ std::unique_ptr<SortedDataInterface> WiredTigerKVEngine::getSortedDataInterface( invariant(!collOptions.clusteredIndex); return std::make_unique<WiredTigerIdIndex>(opCtx, _uri(ident), ident, desc, _readOnly); } + auto keyFormat = (collOptions.clusteredIndex) ? KeyFormat::String : KeyFormat::Long; if (desc->unique()) { - invariant(!collOptions.clusteredIndex); - return std::make_unique<WiredTigerIndexUnique>(opCtx, _uri(ident), ident, desc, _readOnly); + return std::make_unique<WiredTigerIndexUnique>( + opCtx, _uri(ident), ident, keyFormat, desc, _readOnly); } - auto keyFormat = (collOptions.clusteredIndex) ? KeyFormat::String : KeyFormat::Long; return std::make_unique<WiredTigerIndexStandard>( opCtx, _uri(ident), ident, keyFormat, desc, _readOnly); } diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_standard_index_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_standard_index_test.cpp index 09c654ca822..0ecb7d90806 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_standard_index_test.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_standard_index_test.cpp @@ -130,8 +130,8 @@ public: invariantWTOK(WiredTigerIndex::Create(&opCtx, uri, result.getValue())); if (unique) { - invariant(keyFormat == KeyFormat::Long); - return std::make_unique<WiredTigerIndexUnique>(&opCtx, uri, "" /* ident */, &desc); + return std::make_unique<WiredTigerIndexUnique>( + &opCtx, uri, "" /* ident */, keyFormat, &desc); } return std::make_unique<WiredTigerIndexStandard>( &opCtx, uri, "" /* ident */, keyFormat, &desc); |