diff options
Diffstat (limited to 'db')
44 files changed, 2341 insertions, 3366 deletions
diff --git a/db/autocompact_test.cc b/db/autocompact_test.cc index d20a236..69341e3 100644 --- a/db/autocompact_test.cc +++ b/db/autocompact_test.cc @@ -2,29 +2,24 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -#include "leveldb/db.h" +#include "gtest/gtest.h" #include "db/db_impl.h" #include "leveldb/cache.h" -#include "util/testharness.h" +#include "leveldb/db.h" #include "util/testutil.h" namespace leveldb { -class AutoCompactTest { +class AutoCompactTest : public testing::Test { public: - std::string dbname_; - Cache* tiny_cache_; - Options options_; - DB* db_; - AutoCompactTest() { - dbname_ = test::TmpDir() + "/autocompact_test"; + dbname_ = testing::TempDir() + "autocompact_test"; tiny_cache_ = NewLRUCache(100); options_.block_cache = tiny_cache_; DestroyDB(dbname_, options_); options_.create_if_missing = true; options_.compression = kNoCompression; - ASSERT_OK(DB::Open(options_, dbname_, &db_)); + EXPECT_LEVELDB_OK(DB::Open(options_, dbname_, &db_)); } ~AutoCompactTest() { @@ -35,7 +30,7 @@ class AutoCompactTest { std::string Key(int i) { char buf[100]; - snprintf(buf, sizeof(buf), "key%06d", i); + std::snprintf(buf, sizeof(buf), "key%06d", i); return std::string(buf); } @@ -47,6 +42,12 @@ class AutoCompactTest { } void DoReads(int n); + + private: + std::string dbname_; + Cache* tiny_cache_; + Options options_; + DB* db_; }; static const int kValueSize = 200 * 1024; @@ -61,15 +62,15 @@ void AutoCompactTest::DoReads(int n) { // Fill database for (int i = 0; i < kCount; i++) { - ASSERT_OK(db_->Put(WriteOptions(), Key(i), value)); + ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), Key(i), value)); } - ASSERT_OK(dbi->TEST_CompactMemTable()); + ASSERT_LEVELDB_OK(dbi->TEST_CompactMemTable()); // Delete everything for (int i = 0; i < kCount; i++) { - ASSERT_OK(db_->Delete(WriteOptions(), Key(i))); + ASSERT_LEVELDB_OK(db_->Delete(WriteOptions(), Key(i))); } - ASSERT_OK(dbi->TEST_CompactMemTable()); + ASSERT_LEVELDB_OK(dbi->TEST_CompactMemTable()); // Get initial measurement of the space we will be reading. const int64_t initial_size = Size(Key(0), Key(n)); @@ -81,17 +82,16 @@ void AutoCompactTest::DoReads(int n) { ASSERT_LT(read, 100) << "Taking too long to compact"; Iterator* iter = db_->NewIterator(ReadOptions()); for (iter->SeekToFirst(); - iter->Valid() && iter->key().ToString() < limit_key; - iter->Next()) { + iter->Valid() && iter->key().ToString() < limit_key; iter->Next()) { // Drop data } delete iter; // Wait a little bit to allow any triggered compactions to complete. Env::Default()->SleepForMicroseconds(1000000); uint64_t size = Size(Key(0), Key(n)); - fprintf(stderr, "iter %3d => %7.3f MB [other %7.3f MB]\n", - read+1, size/1048576.0, Size(Key(n), Key(kCount))/1048576.0); - if (size <= initial_size/10) { + std::fprintf(stderr, "iter %3d => %7.3f MB [other %7.3f MB]\n", read + 1, + size / 1048576.0, Size(Key(n), Key(kCount)) / 1048576.0); + if (size <= initial_size / 10) { break; } } @@ -100,19 +100,11 @@ void AutoCompactTest::DoReads(int n) { // is pretty much unchanged. const int64_t final_other_size = Size(Key(n), Key(kCount)); ASSERT_LE(final_other_size, initial_other_size + 1048576); - ASSERT_GE(final_other_size, initial_other_size/5 - 1048576); + ASSERT_GE(final_other_size, initial_other_size / 5 - 1048576); } -TEST(AutoCompactTest, ReadAll) { - DoReads(kCount); -} +TEST_F(AutoCompactTest, ReadAll) { DoReads(kCount); } -TEST(AutoCompactTest, ReadHalf) { - DoReads(kCount/2); -} +TEST_F(AutoCompactTest, ReadHalf) { DoReads(kCount / 2); } } // namespace leveldb - -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); -} diff --git a/db/builder.cc b/db/builder.cc index 5fa405d..e6329e0 100644 --- a/db/builder.cc +++ b/db/builder.cc @@ -4,8 +4,8 @@ #include "db/builder.h" -#include "db/filename.h" #include "db/dbformat.h" +#include "db/filename.h" #include "db/table_cache.h" #include "db/version_edit.h" #include "leveldb/db.h" @@ -14,12 +14,8 @@ namespace leveldb { -Status BuildTable(const std::string& dbname, - Env* env, - const Options& options, - TableCache* table_cache, - Iterator* iter, - FileMetaData* meta) { +Status BuildTable(const std::string& dbname, Env* env, const Options& options, + TableCache* table_cache, Iterator* iter, FileMetaData* meta) { Status s; meta->file_size = 0; iter->SeekToFirst(); @@ -34,11 +30,14 @@ Status BuildTable(const std::string& dbname, TableBuilder* builder = new TableBuilder(options, file); meta->smallest.DecodeFrom(iter->key()); + Slice key; for (; iter->Valid(); iter->Next()) { - Slice key = iter->key(); - meta->largest.DecodeFrom(key); + key = iter->key(); builder->Add(key, iter->value()); } + if (!key.empty()) { + meta->largest.DecodeFrom(key); + } // Finish and check for builder errors s = builder->Finish(); @@ -60,8 +59,7 @@ Status BuildTable(const std::string& dbname, if (s.ok()) { // Verify that the table is usable - Iterator* it = table_cache->NewIterator(ReadOptions(), - meta->number, + Iterator* it = table_cache->NewIterator(ReadOptions(), meta->number, meta->file_size); s = it->status(); delete it; @@ -76,7 +74,7 @@ Status BuildTable(const std::string& dbname, if (s.ok() && meta->file_size > 0) { // Keep it } else { - env->DeleteFile(fname); + env->RemoveFile(fname); } return s; } diff --git a/db/builder.h b/db/builder.h index 0289730..7bd0b80 100644 --- a/db/builder.h +++ b/db/builder.h @@ -22,12 +22,8 @@ class VersionEdit; // *meta will be filled with metadata about the generated table. // If no data is present in *iter, meta->file_size will be set to // zero, and no Table file will be produced. -Status BuildTable(const std::string& dbname, - Env* env, - const Options& options, - TableCache* table_cache, - Iterator* iter, - FileMetaData* meta); +Status BuildTable(const std::string& dbname, Env* env, const Options& options, + TableCache* table_cache, Iterator* iter, FileMetaData* meta); } // namespace leveldb @@ -4,7 +4,11 @@ #include "leveldb/c.h" -#include <stdlib.h> +#include <string.h> + +#include <cstdint> +#include <cstdlib> + #include "leveldb/cache.h" #include "leveldb/comparator.h" #include "leveldb/db.h" @@ -42,69 +46,72 @@ using leveldb::WriteOptions; extern "C" { -struct leveldb_t { DB* rep; }; -struct leveldb_iterator_t { Iterator* rep; }; -struct leveldb_writebatch_t { WriteBatch rep; }; -struct leveldb_snapshot_t { const Snapshot* rep; }; -struct leveldb_readoptions_t { ReadOptions rep; }; -struct leveldb_writeoptions_t { WriteOptions rep; }; -struct leveldb_options_t { Options rep; }; -struct leveldb_cache_t { Cache* rep; }; -struct leveldb_seqfile_t { SequentialFile* rep; }; -struct leveldb_randomfile_t { RandomAccessFile* rep; }; -struct leveldb_writablefile_t { WritableFile* rep; }; -struct leveldb_logger_t { Logger* rep; }; -struct leveldb_filelock_t { FileLock* rep; }; +struct leveldb_t { + DB* rep; +}; +struct leveldb_iterator_t { + Iterator* rep; +}; +struct leveldb_writebatch_t { + WriteBatch rep; +}; +struct leveldb_snapshot_t { + const Snapshot* rep; +}; +struct leveldb_readoptions_t { + ReadOptions rep; +}; +struct leveldb_writeoptions_t { + WriteOptions rep; +}; +struct leveldb_options_t { + Options rep; +}; +struct leveldb_cache_t { + Cache* rep; +}; +struct leveldb_seqfile_t { + SequentialFile* rep; +}; +struct leveldb_randomfile_t { + RandomAccessFile* rep; +}; +struct leveldb_writablefile_t { + WritableFile* rep; +}; +struct leveldb_logger_t { + Logger* rep; +}; +struct leveldb_filelock_t { + FileLock* rep; +}; struct leveldb_comparator_t : public Comparator { - void* state_; - void (*destructor_)(void*); - int (*compare_)( - void*, - const char* a, size_t alen, - const char* b, size_t blen); - const char* (*name_)(void*); - - virtual ~leveldb_comparator_t() { - (*destructor_)(state_); - } + ~leveldb_comparator_t() override { (*destructor_)(state_); } - virtual int Compare(const Slice& a, const Slice& b) const { + int Compare(const Slice& a, const Slice& b) const override { return (*compare_)(state_, a.data(), a.size(), b.data(), b.size()); } - virtual const char* Name() const { - return (*name_)(state_); - } + const char* Name() const override { return (*name_)(state_); } // No-ops since the C binding does not support key shortening methods. - virtual void FindShortestSeparator(std::string*, const Slice&) const { } - virtual void FindShortSuccessor(std::string* key) const { } -}; + void FindShortestSeparator(std::string*, const Slice&) const override {} + void FindShortSuccessor(std::string* key) const override {} -struct leveldb_filterpolicy_t : public FilterPolicy { void* state_; void (*destructor_)(void*); + int (*compare_)(void*, const char* a, size_t alen, const char* b, + size_t blen); const char* (*name_)(void*); - char* (*create_)( - void*, - const char* const* key_array, const size_t* key_length_array, - int num_keys, - size_t* filter_length); - unsigned char (*key_match_)( - void*, - const char* key, size_t length, - const char* filter, size_t filter_length); - - virtual ~leveldb_filterpolicy_t() { - (*destructor_)(state_); - } +}; - virtual const char* Name() const { - return (*name_)(state_); - } +struct leveldb_filterpolicy_t : public FilterPolicy { + ~leveldb_filterpolicy_t() override { (*destructor_)(state_); } + + const char* Name() const override { return (*name_)(state_); } - virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const { + void CreateFilter(const Slice* keys, int n, std::string* dst) const override { std::vector<const char*> key_pointers(n); std::vector<size_t> key_sizes(n); for (int i = 0; i < n; i++) { @@ -114,13 +121,22 @@ struct leveldb_filterpolicy_t : public FilterPolicy { size_t len; char* filter = (*create_)(state_, &key_pointers[0], &key_sizes[0], n, &len); dst->append(filter, len); - free(filter); + std::free(filter); } - virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const { - return (*key_match_)(state_, key.data(), key.size(), - filter.data(), filter.size()); + bool KeyMayMatch(const Slice& key, const Slice& filter) const override { + return (*key_match_)(state_, key.data(), key.size(), filter.data(), + filter.size()); } + + void* state_; + void (*destructor_)(void*); + const char* (*name_)(void*); + char* (*create_)(void*, const char* const* key_array, + const size_t* key_length_array, int num_keys, + size_t* filter_length); + uint8_t (*key_match_)(void*, const char* key, size_t length, + const char* filter, size_t filter_length); }; struct leveldb_env_t { @@ -136,22 +152,21 @@ static bool SaveError(char** errptr, const Status& s) { *errptr = strdup(s.ToString().c_str()); } else { // TODO(sanjay): Merge with existing error? - free(*errptr); + std::free(*errptr); *errptr = strdup(s.ToString().c_str()); } return true; } static char* CopyString(const std::string& str) { - char* result = reinterpret_cast<char*>(malloc(sizeof(char) * str.size())); - memcpy(result, str.data(), sizeof(char) * str.size()); + char* result = + reinterpret_cast<char*>(std::malloc(sizeof(char) * str.size())); + std::memcpy(result, str.data(), sizeof(char) * str.size()); return result; } -leveldb_t* leveldb_open( - const leveldb_options_t* options, - const char* name, - char** errptr) { +leveldb_t* leveldb_open(const leveldb_options_t* options, const char* name, + char** errptr) { DB* db; if (SaveError(errptr, DB::Open(options->rep, std::string(name), &db))) { return nullptr; @@ -166,39 +181,26 @@ void leveldb_close(leveldb_t* db) { delete db; } -void leveldb_put( - leveldb_t* db, - const leveldb_writeoptions_t* options, - const char* key, size_t keylen, - const char* val, size_t vallen, - char** errptr) { +void leveldb_put(leveldb_t* db, const leveldb_writeoptions_t* options, + const char* key, size_t keylen, const char* val, size_t vallen, + char** errptr) { SaveError(errptr, db->rep->Put(options->rep, Slice(key, keylen), Slice(val, vallen))); } -void leveldb_delete( - leveldb_t* db, - const leveldb_writeoptions_t* options, - const char* key, size_t keylen, - char** errptr) { +void leveldb_delete(leveldb_t* db, const leveldb_writeoptions_t* options, + const char* key, size_t keylen, char** errptr) { SaveError(errptr, db->rep->Delete(options->rep, Slice(key, keylen))); } - -void leveldb_write( - leveldb_t* db, - const leveldb_writeoptions_t* options, - leveldb_writebatch_t* batch, - char** errptr) { +void leveldb_write(leveldb_t* db, const leveldb_writeoptions_t* options, + leveldb_writebatch_t* batch, char** errptr) { SaveError(errptr, db->rep->Write(options->rep, &batch->rep)); } -char* leveldb_get( - leveldb_t* db, - const leveldb_readoptions_t* options, - const char* key, size_t keylen, - size_t* vallen, - char** errptr) { +char* leveldb_get(leveldb_t* db, const leveldb_readoptions_t* options, + const char* key, size_t keylen, size_t* vallen, + char** errptr) { char* result = nullptr; std::string tmp; Status s = db->rep->Get(options->rep, Slice(key, keylen), &tmp); @@ -215,30 +217,25 @@ char* leveldb_get( } leveldb_iterator_t* leveldb_create_iterator( - leveldb_t* db, - const leveldb_readoptions_t* options) { + leveldb_t* db, const leveldb_readoptions_t* options) { leveldb_iterator_t* result = new leveldb_iterator_t; result->rep = db->rep->NewIterator(options->rep); return result; } -const leveldb_snapshot_t* leveldb_create_snapshot( - leveldb_t* db) { +const leveldb_snapshot_t* leveldb_create_snapshot(leveldb_t* db) { leveldb_snapshot_t* result = new leveldb_snapshot_t; result->rep = db->rep->GetSnapshot(); return result; } -void leveldb_release_snapshot( - leveldb_t* db, - const leveldb_snapshot_t* snapshot) { +void leveldb_release_snapshot(leveldb_t* db, + const leveldb_snapshot_t* snapshot) { db->rep->ReleaseSnapshot(snapshot->rep); delete snapshot; } -char* leveldb_property_value( - leveldb_t* db, - const char* propname) { +char* leveldb_property_value(leveldb_t* db, const char* propname) { std::string tmp; if (db->rep->GetProperty(Slice(propname), &tmp)) { // We use strdup() since we expect human readable output. @@ -248,12 +245,12 @@ char* leveldb_property_value( } } -void leveldb_approximate_sizes( - leveldb_t* db, - int num_ranges, - const char* const* range_start_key, const size_t* range_start_key_len, - const char* const* range_limit_key, const size_t* range_limit_key_len, - uint64_t* sizes) { +void leveldb_approximate_sizes(leveldb_t* db, int num_ranges, + const char* const* range_start_key, + const size_t* range_start_key_len, + const char* const* range_limit_key, + const size_t* range_limit_key_len, + uint64_t* sizes) { Range* ranges = new Range[num_ranges]; for (int i = 0; i < num_ranges; i++) { ranges[i].start = Slice(range_start_key[i], range_start_key_len[i]); @@ -263,10 +260,9 @@ void leveldb_approximate_sizes( delete[] ranges; } -void leveldb_compact_range( - leveldb_t* db, - const char* start_key, size_t start_key_len, - const char* limit_key, size_t limit_key_len) { +void leveldb_compact_range(leveldb_t* db, const char* start_key, + size_t start_key_len, const char* limit_key, + size_t limit_key_len) { Slice a, b; db->rep->CompactRange( // Pass null Slice if corresponding "const char*" is null @@ -274,17 +270,13 @@ void leveldb_compact_range( (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : nullptr)); } -void leveldb_destroy_db( - const leveldb_options_t* options, - const char* name, - char** errptr) { +void leveldb_destroy_db(const leveldb_options_t* options, const char* name, + char** errptr) { SaveError(errptr, DestroyDB(name, options->rep)); } -void leveldb_repair_db( - const leveldb_options_t* options, - const char* name, - char** errptr) { +void leveldb_repair_db(const leveldb_options_t* options, const char* name, + char** errptr) { SaveError(errptr, RepairDB(name, options->rep)); } @@ -293,7 +285,7 @@ void leveldb_iter_destroy(leveldb_iterator_t* iter) { delete iter; } -unsigned char leveldb_iter_valid(const leveldb_iterator_t* iter) { +uint8_t leveldb_iter_valid(const leveldb_iterator_t* iter) { return iter->rep->Valid(); } @@ -309,13 +301,9 @@ void leveldb_iter_seek(leveldb_iterator_t* iter, const char* k, size_t klen) { iter->rep->Seek(Slice(k, klen)); } -void leveldb_iter_next(leveldb_iterator_t* iter) { - iter->rep->Next(); -} +void leveldb_iter_next(leveldb_iterator_t* iter) { iter->rep->Next(); } -void leveldb_iter_prev(leveldb_iterator_t* iter) { - iter->rep->Prev(); -} +void leveldb_iter_prev(leveldb_iterator_t* iter) { iter->rep->Prev(); } const char* leveldb_iter_key(const leveldb_iterator_t* iter, size_t* klen) { Slice s = iter->rep->key(); @@ -337,41 +325,34 @@ leveldb_writebatch_t* leveldb_writebatch_create() { return new leveldb_writebatch_t; } -void leveldb_writebatch_destroy(leveldb_writebatch_t* b) { - delete b; -} +void leveldb_writebatch_destroy(leveldb_writebatch_t* b) { delete b; } -void leveldb_writebatch_clear(leveldb_writebatch_t* b) { - b->rep.Clear(); -} +void leveldb_writebatch_clear(leveldb_writebatch_t* b) { b->rep.Clear(); } -void leveldb_writebatch_put( - leveldb_writebatch_t* b, - const char* key, size_t klen, - const char* val, size_t vlen) { +void leveldb_writebatch_put(leveldb_writebatch_t* b, const char* key, + size_t klen, const char* val, size_t vlen) { b->rep.Put(Slice(key, klen), Slice(val, vlen)); } -void leveldb_writebatch_delete( - leveldb_writebatch_t* b, - const char* key, size_t klen) { +void leveldb_writebatch_delete(leveldb_writebatch_t* b, const char* key, + size_t klen) { b->rep.Delete(Slice(key, klen)); } -void leveldb_writebatch_iterate( - leveldb_writebatch_t* b, - void* state, - void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen), - void (*deleted)(void*, const char* k, size_t klen)) { +void leveldb_writebatch_iterate(const leveldb_writebatch_t* b, void* state, + void (*put)(void*, const char* k, size_t klen, + const char* v, size_t vlen), + void (*deleted)(void*, const char* k, + size_t klen)) { class H : public WriteBatch::Handler { public: void* state_; void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen); void (*deleted_)(void*, const char* k, size_t klen); - virtual void Put(const Slice& key, const Slice& value) { + void Put(const Slice& key, const Slice& value) override { (*put_)(state_, key.data(), key.size(), value.data(), value.size()); } - virtual void Delete(const Slice& key) { + void Delete(const Slice& key) override { (*deleted_)(state_, key.data(), key.size()); } }; @@ -382,38 +363,34 @@ void leveldb_writebatch_iterate( b->rep.Iterate(&handler); } -leveldb_options_t* leveldb_options_create() { - return new leveldb_options_t; +void leveldb_writebatch_append(leveldb_writebatch_t* destination, + const leveldb_writebatch_t* source) { + destination->rep.Append(source->rep); } -void leveldb_options_destroy(leveldb_options_t* options) { - delete options; -} +leveldb_options_t* leveldb_options_create() { return new leveldb_options_t; } + +void leveldb_options_destroy(leveldb_options_t* options) { delete options; } -void leveldb_options_set_comparator( - leveldb_options_t* opt, - leveldb_comparator_t* cmp) { +void leveldb_options_set_comparator(leveldb_options_t* opt, + leveldb_comparator_t* cmp) { opt->rep.comparator = cmp; } -void leveldb_options_set_filter_policy( - leveldb_options_t* opt, - leveldb_filterpolicy_t* policy) { +void leveldb_options_set_filter_policy(leveldb_options_t* opt, + leveldb_filterpolicy_t* policy) { opt->rep.filter_policy = policy; } -void leveldb_options_set_create_if_missing( - leveldb_options_t* opt, unsigned char v) { +void leveldb_options_set_create_if_missing(leveldb_options_t* opt, uint8_t v) { opt->rep.create_if_missing = v; } -void leveldb_options_set_error_if_exists( - leveldb_options_t* opt, unsigned char v) { +void leveldb_options_set_error_if_exists(leveldb_options_t* opt, uint8_t v) { opt->rep.error_if_exists = v; } -void leveldb_options_set_paranoid_checks( - leveldb_options_t* opt, unsigned char v) { +void leveldb_options_set_paranoid_checks(leveldb_options_t* opt, uint8_t v) { opt->rep.paranoid_checks = v; } @@ -454,12 +431,9 @@ void leveldb_options_set_compression(leveldb_options_t* opt, int t) { } leveldb_comparator_t* leveldb_comparator_create( - void* state, - void (*destructor)(void*), - int (*compare)( - void*, - const char* a, size_t alen, - const char* b, size_t blen), + void* state, void (*destructor)(void*), + int (*compare)(void*, const char* a, size_t alen, const char* b, + size_t blen), const char* (*name)(void*)) { leveldb_comparator_t* result = new leveldb_comparator_t; result->state_ = state; @@ -469,22 +443,15 @@ leveldb_comparator_t* leveldb_comparator_create( return result; } -void leveldb_comparator_destroy(leveldb_comparator_t* cmp) { - delete cmp; -} +void leveldb_comparator_destroy(leveldb_comparator_t* cmp) { delete cmp; } leveldb_filterpolicy_t* leveldb_filterpolicy_create( - void* state, - void (*destructor)(void*), - char* (*create_filter)( - void*, - const char* const* key_array, const size_t* key_length_array, - int num_keys, - size_t* filter_length), - unsigned char (*key_may_match)( - void*, - const char* key, size_t length, - const char* filter, size_t filter_length), + void* state, void (*destructor)(void*), + char* (*create_filter)(void*, const char* const* key_array, + const size_t* key_length_array, int num_keys, + size_t* filter_length), + uint8_t (*key_may_match)(void*, const char* key, size_t length, + const char* filter, size_t filter_length), const char* (*name)(void*)) { leveldb_filterpolicy_t* result = new leveldb_filterpolicy_t; result->state_ = state; @@ -504,7 +471,8 @@ leveldb_filterpolicy_t* leveldb_filterpolicy_create_bloom(int bits_per_key) { // they delegate to a NewBloomFilterPolicy() instead of user // supplied C functions. struct Wrapper : public leveldb_filterpolicy_t { - const FilterPolicy* rep_; + static void DoNothing(void*) {} + ~Wrapper() { delete rep_; } const char* Name() const { return rep_->Name(); } void CreateFilter(const Slice* keys, int n, std::string* dst) const { @@ -513,7 +481,8 @@ leveldb_filterpolicy_t* leveldb_filterpolicy_create_bloom(int bits_per_key) { bool KeyMayMatch(const Slice& key, const Slice& filter) const { return rep_->KeyMayMatch(key, filter); } - static void DoNothing(void*) { } + + const FilterPolicy* rep_; }; Wrapper* wrapper = new Wrapper; wrapper->rep_ = NewBloomFilterPolicy(bits_per_key); @@ -526,24 +495,19 @@ leveldb_readoptions_t* leveldb_readoptions_create() { return new leveldb_readoptions_t; } -void leveldb_readoptions_destroy(leveldb_readoptions_t* opt) { - delete opt; -} +void leveldb_readoptions_destroy(leveldb_readoptions_t* opt) { delete opt; } -void leveldb_readoptions_set_verify_checksums( - leveldb_readoptions_t* opt, - unsigned char v) { +void leveldb_readoptions_set_verify_checksums(leveldb_readoptions_t* opt, + uint8_t v) { opt->rep.verify_checksums = v; } -void leveldb_readoptions_set_fill_cache( - leveldb_readoptions_t* opt, unsigned char v) { +void leveldb_readoptions_set_fill_cache(leveldb_readoptions_t* opt, uint8_t v) { opt->rep.fill_cache = v; } -void leveldb_readoptions_set_snapshot( - leveldb_readoptions_t* opt, - const leveldb_snapshot_t* snap) { +void leveldb_readoptions_set_snapshot(leveldb_readoptions_t* opt, + const leveldb_snapshot_t* snap) { opt->rep.snapshot = (snap ? snap->rep : nullptr); } @@ -551,12 +515,9 @@ leveldb_writeoptions_t* leveldb_writeoptions_create() { return new leveldb_writeoptions_t; } -void leveldb_writeoptions_destroy(leveldb_writeoptions_t* opt) { - delete opt; -} +void leveldb_writeoptions_destroy(leveldb_writeoptions_t* opt) { delete opt; } -void leveldb_writeoptions_set_sync( - leveldb_writeoptions_t* opt, unsigned char v) { +void leveldb_writeoptions_set_sync(leveldb_writeoptions_t* opt, uint8_t v) { opt->rep.sync = v; } @@ -589,22 +550,16 @@ char* leveldb_env_get_test_directory(leveldb_env_t* env) { return nullptr; } - char* buffer = static_cast<char*>(malloc(result.size() + 1)); - memcpy(buffer, result.data(), result.size()); + char* buffer = static_cast<char*>(std::malloc(result.size() + 1)); + std::memcpy(buffer, result.data(), result.size()); buffer[result.size()] = '\0'; return buffer; } -void leveldb_free(void* ptr) { - free(ptr); -} +void leveldb_free(void* ptr) { std::free(ptr); } -int leveldb_major_version() { - return kMajorVersion; -} +int leveldb_major_version() { return kMajorVersion; } -int leveldb_minor_version() { - return kMinorVersion; -} +int leveldb_minor_version() { return kMinorVersion; } } // end extern "C" diff --git a/db/c_test.c b/db/c_test.c index 7284de5..16c77ee 100644 --- a/db/c_test.c +++ b/db/c_test.c @@ -120,7 +120,7 @@ static const char* CmpName(void* arg) { } // Custom filter policy -static unsigned char fake_filter_result = 1; +static uint8_t fake_filter_result = 1; static void FilterDestroy(void* arg) { } static const char* FilterName(void* arg) { return "TestFilter"; @@ -135,10 +135,8 @@ static char* FilterCreate( memcpy(result, "fake", 4); return result; } -unsigned char FilterKeyMatch( - void* arg, - const char* key, size_t length, - const char* filter, size_t filter_length) { +uint8_t FilterKeyMatch(void* arg, const char* key, size_t length, + const char* filter, size_t filter_length) { CheckCondition(filter_length == 4); CheckCondition(memcmp(filter, "fake", 4) == 0); return fake_filter_result; @@ -228,12 +226,18 @@ int main(int argc, char** argv) { leveldb_writebatch_clear(wb); leveldb_writebatch_put(wb, "bar", 3, "b", 1); leveldb_writebatch_put(wb, "box", 3, "c", 1); - leveldb_writebatch_delete(wb, "bar", 3); + + leveldb_writebatch_t* wb2 = leveldb_writebatch_create(); + leveldb_writebatch_delete(wb2, "bar", 3); + leveldb_writebatch_append(wb, wb2); + leveldb_writebatch_destroy(wb2); + leveldb_write(db, woptions, wb, &err); CheckNoError(err); CheckGet(db, roptions, "foo", "hello"); CheckGet(db, roptions, "bar", NULL); CheckGet(db, roptions, "box", "c"); + int pos = 0; leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel); CheckCondition(pos == 3); diff --git a/db/corruption_test.cc b/db/corruption_test.cc index 0b93c24..dc7da76 100644 --- a/db/corruption_test.cc +++ b/db/corruption_test.cc @@ -2,53 +2,42 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -#include "leveldb/db.h" - -#include <errno.h> -#include <fcntl.h> -#include <sys/stat.h> #include <sys/types.h> -#include "leveldb/cache.h" -#include "leveldb/env.h" -#include "leveldb/table.h" -#include "leveldb/write_batch.h" + +#include "gtest/gtest.h" #include "db/db_impl.h" #include "db/filename.h" #include "db/log_format.h" #include "db/version_set.h" +#include "leveldb/cache.h" +#include "leveldb/db.h" +#include "leveldb/table.h" +#include "leveldb/write_batch.h" #include "util/logging.h" -#include "util/testharness.h" #include "util/testutil.h" namespace leveldb { static const int kValueSize = 1000; -class CorruptionTest { +class CorruptionTest : public testing::Test { public: - test::ErrorEnv env_; - std::string dbname_; - Cache* tiny_cache_; - Options options_; - DB* db_; - - CorruptionTest() { - tiny_cache_ = NewLRUCache(100); + CorruptionTest() + : db_(nullptr), + dbname_("/memenv/corruption_test"), + tiny_cache_(NewLRUCache(100)) { options_.env = &env_; options_.block_cache = tiny_cache_; - dbname_ = test::TmpDir() + "/corruption_test"; DestroyDB(dbname_, options_); - db_ = nullptr; options_.create_if_missing = true; Reopen(); options_.create_if_missing = false; } ~CorruptionTest() { - delete db_; - DestroyDB(dbname_, Options()); - delete tiny_cache_; + delete db_; + delete tiny_cache_; } Status TryReopen() { @@ -57,21 +46,19 @@ class CorruptionTest { return DB::Open(options_, dbname_, &db_); } - void Reopen() { - ASSERT_OK(TryReopen()); - } + void Reopen() { ASSERT_LEVELDB_OK(TryReopen()); } void RepairDB() { delete db_; db_ = nullptr; - ASSERT_OK(::leveldb::RepairDB(dbname_, options_)); + ASSERT_LEVELDB_OK(::leveldb::RepairDB(dbname_, options_)); } void Build(int n) { std::string key_space, value_space; WriteBatch batch; for (int i = 0; i < n; i++) { - //if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n); + // if ((i % 100) == 0) std::fprintf(stderr, "@ %d of %d\n", i, n); Slice key = Key(i, &key_space); batch.Clear(); batch.Put(key, Value(i, &value_space)); @@ -81,7 +68,7 @@ class CorruptionTest { if (i == n - 1) { options.sync = true; } - ASSERT_OK(db_->Write(options, &batch)); + ASSERT_LEVELDB_OK(db_->Write(options, &batch)); } } @@ -100,8 +87,7 @@ class CorruptionTest { // Ignore boundary keys. continue; } - if (!ConsumeDecimalNumber(&in, &key) || - !in.empty() || + if (!ConsumeDecimalNumber(&in, &key) || !in.empty() || key < next_expected) { bad_keys++; continue; @@ -116,9 +102,10 @@ class CorruptionTest { } delete iter; - fprintf(stderr, - "expected=%d..%d; got=%d; bad_keys=%d; bad_values=%d; missed=%d\n", - min_expected, max_expected, correct, bad_keys, bad_values, missed); + std::fprintf( + stderr, + "expected=%d..%d; got=%d; bad_keys=%d; bad_values=%d; missed=%d\n", + min_expected, max_expected, correct, bad_keys, bad_values, missed); ASSERT_LE(min_expected, correct); ASSERT_GE(max_expected, correct); } @@ -126,14 +113,13 @@ class CorruptionTest { void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) { // Pick file to corrupt std::vector<std::string> filenames; - ASSERT_OK(env_.GetChildren(dbname_, &filenames)); + ASSERT_LEVELDB_OK(env_.target()->GetChildren(dbname_, &filenames)); uint64_t number; FileType type; std::string fname; int picked_number = -1; for (size_t i = 0; i < filenames.size(); i++) { - if (ParseFileName(filenames[i], &number, &type) && - type == filetype && + if (ParseFileName(filenames[i], &number, &type) && type == filetype && int(number) > picked_number) { // Pick latest file fname = dbname_ + "/" + filenames[i]; picked_number = number; @@ -141,35 +127,32 @@ class CorruptionTest { } ASSERT_TRUE(!fname.empty()) << filetype; - struct stat sbuf; - if (stat(fname.c_str(), &sbuf) != 0) { - const char* msg = strerror(errno); - ASSERT_TRUE(false) << fname << ": " << msg; - } + uint64_t file_size; + ASSERT_LEVELDB_OK(env_.target()->GetFileSize(fname, &file_size)); if (offset < 0) { // Relative to end of file; make it absolute - if (-offset > sbuf.st_size) { + if (-offset > file_size) { offset = 0; } else { - offset = sbuf.st_size + offset; + offset = file_size + offset; } } - if (offset > sbuf.st_size) { - offset = sbuf.st_size; + if (offset > file_size) { + offset = file_size; } - if (offset + bytes_to_corrupt > sbuf.st_size) { - bytes_to_corrupt = sbuf.st_size - offset; + if (offset + bytes_to_corrupt > file_size) { + bytes_to_corrupt = file_size - offset; } // Do it std::string contents; - Status s = ReadFileToString(Env::Default(), fname, &contents); + Status s = ReadFileToString(env_.target(), fname, &contents); ASSERT_TRUE(s.ok()) << s.ToString(); for (int i = 0; i < bytes_to_corrupt; i++) { contents[i + offset] ^= 0x80; } - s = WriteStringToFile(Env::Default(), contents, fname); + s = WriteStringToFile(env_.target(), contents, fname); ASSERT_TRUE(s.ok()) << s.ToString(); } @@ -187,7 +170,7 @@ class CorruptionTest { // Return the ith key Slice Key(int i, std::string* storage) { char buf[100]; - snprintf(buf, sizeof(buf), "%016d", i); + std::snprintf(buf, sizeof(buf), "%016d", i); storage->assign(buf, strlen(buf)); return Slice(*storage); } @@ -197,12 +180,20 @@ class CorruptionTest { Random r(k); return test::RandomString(&r, kValueSize, storage); } + + test::ErrorEnv env_; + Options options_; + DB* db_; + + private: + std::string dbname_; + Cache* tiny_cache_; }; -TEST(CorruptionTest, Recovery) { +TEST_F(CorruptionTest, Recovery) { Build(100); Check(100, 100); - Corrupt(kLogFile, 19, 1); // WriteBatch tag for first record + Corrupt(kLogFile, 19, 1); // WriteBatch tag for first record Corrupt(kLogFile, log::kBlockSize + 1000, 1); // Somewhere in second block Reopen(); @@ -210,13 +201,13 @@ TEST(CorruptionTest, Recovery) { Check(36, 36); } -TEST(CorruptionTest, RecoverWriteError) { +TEST_F(CorruptionTest, RecoverWriteError) { env_.writable_file_error_ = true; Status s = TryReopen(); ASSERT_TRUE(!s.ok()); } -TEST(CorruptionTest, NewFileErrorDuringWrite) { +TEST_F(CorruptionTest, NewFileErrorDuringWrite) { // Do enough writing to force minor compaction env_.writable_file_error_ = true; const int num = 3 + (Options().write_buffer_size / kValueSize); @@ -233,7 +224,7 @@ TEST(CorruptionTest, NewFileErrorDuringWrite) { Reopen(); } -TEST(CorruptionTest, TableFile) { +TEST_F(CorruptionTest, TableFile) { Build(100); DBImpl* dbi = reinterpret_cast<DBImpl*>(db_); dbi->TEST_CompactMemTable(); @@ -244,7 +235,7 @@ TEST(CorruptionTest, TableFile) { Check(90, 99); } -TEST(CorruptionTest, TableFileRepair) { +TEST_F(CorruptionTest, TableFileRepair) { options_.block_size = 2 * kValueSize; // Limit scope of corruption options_.paranoid_checks = true; Reopen(); @@ -260,7 +251,7 @@ TEST(CorruptionTest, TableFileRepair) { Check(95, 99); } -TEST(CorruptionTest, TableFileIndexData) { +TEST_F(CorruptionTest, TableFileIndexData) { Build(10000); // Enough to build multiple Tables DBImpl* dbi = reinterpret_cast<DBImpl*>(db_); dbi->TEST_CompactMemTable(); @@ -270,36 +261,36 @@ TEST(CorruptionTest, TableFileIndexData) { Check(5000, 9999); } -TEST(CorruptionTest, MissingDescriptor) { +TEST_F(CorruptionTest, MissingDescriptor) { Build(1000); RepairDB(); Reopen(); Check(1000, 1000); } -TEST(CorruptionTest, SequenceNumberRecovery) { - ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1")); - ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2")); - ASSERT_OK(db_->Put(WriteOptions(), "foo", "v3")); - ASSERT_OK(db_->Put(WriteOptions(), "foo", "v4")); - ASSERT_OK(db_->Put(WriteOptions(), "foo", "v5")); +TEST_F(CorruptionTest, SequenceNumberRecovery) { + ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v1")); + ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v2")); + ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v3")); + ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v4")); + ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v5")); RepairDB(); Reopen(); std::string v; - ASSERT_OK(db_->Get(ReadOptions(), "foo", &v)); + ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), "foo", &v)); ASSERT_EQ("v5", v); // Write something. If sequence number was not recovered properly, // it will be hidden by an earlier write. - ASSERT_OK(db_->Put(WriteOptions(), "foo", "v6")); - ASSERT_OK(db_->Get(ReadOptions(), "foo", &v)); + ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v6")); + ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), "foo", &v)); ASSERT_EQ("v6", v); Reopen(); - ASSERT_OK(db_->Get(ReadOptions(), "foo", &v)); + ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), "foo", &v)); ASSERT_EQ("v6", v); } -TEST(CorruptionTest, CorruptedDescriptor) { - ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello")); +TEST_F(CorruptionTest, CorruptedDescriptor) { + ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "hello")); DBImpl* dbi = reinterpret_cast<DBImpl*>(db_); dbi->TEST_CompactMemTable(); dbi->TEST_CompactRange(0, nullptr, nullptr); @@ -311,11 +302,11 @@ TEST(CorruptionTest, CorruptedDescriptor) { RepairDB(); Reopen(); std::string v; - ASSERT_OK(db_->Get(ReadOptions(), "foo", &v)); + ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), "foo", &v)); ASSERT_EQ("hello", v); } -TEST(CorruptionTest, CompactionInputError) { +TEST_F(CorruptionTest, CompactionInputError) { Build(10); DBImpl* dbi = reinterpret_cast<DBImpl*>(db_); dbi->TEST_CompactMemTable(); @@ -330,7 +321,7 @@ TEST(CorruptionTest, CompactionInputError) { Check(10000, 10000); } -TEST(CorruptionTest, CompactionInputErrorParanoid) { +TEST_F(CorruptionTest, CompactionInputErrorParanoid) { options_.paranoid_checks = true; options_.write_buffer_size = 512 << 10; Reopen(); @@ -351,24 +342,21 @@ TEST(CorruptionTest, CompactionInputErrorParanoid) { ASSERT_TRUE(!s.ok()) << "write did not fail in corrupted paranoid db"; } -TEST(CorruptionTest, UnrelatedKeys) { +TEST_F(CorruptionTest, UnrelatedKeys) { Build(10); DBImpl* dbi = reinterpret_cast<DBImpl*>(db_); dbi->TEST_CompactMemTable(); Corrupt(kTableFile, 100, 1); std::string tmp1, tmp2; - ASSERT_OK(db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2))); + ASSERT_LEVELDB_OK( + db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2))); std::string v; - ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v)); + ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v)); ASSERT_EQ(Value(1000, &tmp2).ToString(), v); dbi->TEST_CompactMemTable(); - ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v)); + ASSERT_LEVELDB_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v)); ASSERT_EQ(Value(1000, &tmp2).ToString(), v); } } // namespace leveldb - -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); -} diff --git a/db/db_bench.cc b/db/db_bench.cc deleted file mode 100644 index 115cf45..0000000 --- a/db/db_bench.cc +++ /dev/null @@ -1,1016 +0,0 @@ -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -#include <sys/types.h> -#include <stdio.h> -#include <stdlib.h> -#include "leveldb/cache.h" -#include "leveldb/db.h" -#include "leveldb/env.h" -#include "leveldb/filter_policy.h" -#include "leveldb/write_batch.h" -#include "port/port.h" -#include "util/crc32c.h" -#include "util/histogram.h" -#include "util/mutexlock.h" -#include "util/random.h" -#include "util/testutil.h" - -// Comma-separated list of operations to run in the specified order -// Actual benchmarks: -// fillseq -- write N values in sequential key order in async mode -// fillrandom -- write N values in random key order in async mode -// overwrite -- overwrite N values in random key order in async mode -// fillsync -- write N/100 values in random key order in sync mode -// fill100K -- write N/1000 100K values in random order in async mode -// deleteseq -- delete N keys in sequential order -// deleterandom -- delete N keys in random order -// readseq -- read N times sequentially -// readreverse -- read N times in reverse order -// readrandom -- read N times in random order -// readmissing -- read N missing keys in random order -// readhot -- read N times in random order from 1% section of DB -// seekrandom -- N random seeks -// open -- cost of opening a DB -// crc32c -- repeated crc32c of 4K of data -// acquireload -- load N*1000 times -// Meta operations: -// compact -- Compact the entire DB -// stats -- Print DB stats -// sstables -- Print sstable info -// heapprofile -- Dump a heap profile (if supported by this port) -static const char* FLAGS_benchmarks = - "fillseq," - "fillsync," - "fillrandom," - "overwrite," - "readrandom," - "readrandom," // Extra run to allow previous compactions to quiesce - "readseq," - "readreverse," - "compact," - "readrandom," - "readseq," - "readreverse," - "fill100K," - "crc32c," - "snappycomp," - "snappyuncomp," - "acquireload," - ; - -// Number of key/values to place in database -static int FLAGS_num = 1000000; - -// Number of read operations to do. If negative, do FLAGS_num reads. -static int FLAGS_reads = -1; - -// Number of concurrent threads to run. -static int FLAGS_threads = 1; - -// Size of each value -static int FLAGS_value_size = 100; - -// Arrange to generate values that shrink to this fraction of -// their original size after compression -static double FLAGS_compression_ratio = 0.5; - -// Print histogram of operation timings -static bool FLAGS_histogram = false; - -// Number of bytes to buffer in memtable before compacting -// (initialized to default value by "main") -static int FLAGS_write_buffer_size = 0; - -// Number of bytes written to each file. -// (initialized to default value by "main") -static int FLAGS_max_file_size = 0; - -// Approximate size of user data packed per block (before compression. -// (initialized to default value by "main") -static int FLAGS_block_size = 0; - -// Number of bytes to use as a cache of uncompressed data. -// Negative means use default settings. -static int FLAGS_cache_size = -1; - -// Maximum number of files to keep open at the same time (use default if == 0) -static int FLAGS_open_files = 0; - -// Bloom filter bits per key. -// Negative means use default settings. -static int FLAGS_bloom_bits = -1; - -// If true, do not destroy the existing database. If you set this -// flag and also specify a benchmark that wants a fresh database, that -// benchmark will fail. -static bool FLAGS_use_existing_db = false; - -// If true, reuse existing log/MANIFEST files when re-opening a database. -static bool FLAGS_reuse_logs = false; - -// Use the db with the following name. -static const char* FLAGS_db = nullptr; - -namespace leveldb { - -namespace { -leveldb::Env* g_env = nullptr; - -// Helper for quickly generating random data. -class RandomGenerator { - private: - std::string data_; - int pos_; - - public: - RandomGenerator() { - // We use a limited amount of data over and over again and ensure - // that it is larger than the compression window (32KB), and also - // large enough to serve all typical value sizes we want to write. - Random rnd(301); - std::string piece; - while (data_.size() < 1048576) { - // Add a short fragment that is as compressible as specified - // by FLAGS_compression_ratio. - test::CompressibleString(&rnd, FLAGS_compression_ratio, 100, &piece); - data_.append(piece); - } - pos_ = 0; - } - - Slice Generate(size_t len) { - if (pos_ + len > data_.size()) { - pos_ = 0; - assert(len < data_.size()); - } - pos_ += len; - return Slice(data_.data() + pos_ - len, len); - } -}; - -#if defined(__linux) -static Slice TrimSpace(Slice s) { - size_t start = 0; - while (start < s.size() && isspace(s[start])) { - start++; - } - size_t limit = s.size(); - while (limit > start && isspace(s[limit-1])) { - limit--; - } - return Slice(s.data() + start, limit - start); -} -#endif - -static void AppendWithSpace(std::string* str, Slice msg) { - if (msg.empty()) return; - if (!str->empty()) { - str->push_back(' '); - } - str->append(msg.data(), msg.size()); -} - -class Stats { - private: - double start_; - double finish_; - double seconds_; - int done_; - int next_report_; - int64_t bytes_; - double last_op_finish_; - Histogram hist_; - std::string message_; - - public: - Stats() { Start(); } - - void Start() { - next_report_ = 100; - last_op_finish_ = start_; - hist_.Clear(); - done_ = 0; - bytes_ = 0; - seconds_ = 0; - start_ = g_env->NowMicros(); - finish_ = start_; - message_.clear(); - } - - void Merge(const Stats& other) { - hist_.Merge(other.hist_); - done_ += other.done_; - bytes_ += other.bytes_; - seconds_ += other.seconds_; - if (other.start_ < start_) start_ = other.start_; - if (other.finish_ > finish_) finish_ = other.finish_; - - // Just keep the messages from one thread - if (message_.empty()) message_ = other.message_; - } - - void Stop() { - finish_ = g_env->NowMicros(); - seconds_ = (finish_ - start_) * 1e-6; - } - - void AddMessage(Slice msg) { - AppendWithSpace(&message_, msg); - } - - void FinishedSingleOp() { - if (FLAGS_histogram) { - double now = g_env->NowMicros(); - double micros = now - last_op_finish_; - hist_.Add(micros); - if (micros > 20000) { - fprintf(stderr, "long op: %.1f micros%30s\r", micros, ""); - fflush(stderr); - } - last_op_finish_ = now; - } - - done_++; - if (done_ >= next_report_) { - if (next_report_ < 1000) next_report_ += 100; - else if (next_report_ < 5000) next_report_ += 500; - else if (next_report_ < 10000) next_report_ += 1000; - else if (next_report_ < 50000) next_report_ += 5000; - else if (next_report_ < 100000) next_report_ += 10000; - else if (next_report_ < 500000) next_report_ += 50000; - else next_report_ += 100000; - fprintf(stderr, "... finished %d ops%30s\r", done_, ""); - fflush(stderr); - } - } - - void AddBytes(int64_t n) { - bytes_ += n; - } - - void Report(const Slice& name) { - // Pretend at least one op was done in case we are running a benchmark - // that does not call FinishedSingleOp(). - if (done_ < 1) done_ = 1; - - std::string extra; - if (bytes_ > 0) { - // Rate is computed on actual elapsed time, not the sum of per-thread - // elapsed times. - double elapsed = (finish_ - start_) * 1e-6; - char rate[100]; - snprintf(rate, sizeof(rate), "%6.1f MB/s", - (bytes_ / 1048576.0) / elapsed); - extra = rate; - } - AppendWithSpace(&extra, message_); - - fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n", - name.ToString().c_str(), - seconds_ * 1e6 / done_, - (extra.empty() ? "" : " "), - extra.c_str()); - if (FLAGS_histogram) { - fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str()); - } - fflush(stdout); - } -}; - -// State shared by all concurrent executions of the same benchmark. -struct SharedState { - port::Mutex mu; - port::CondVar cv GUARDED_BY(mu); - int total GUARDED_BY(mu); - - // Each thread goes through the following states: - // (1) initializing - // (2) waiting for others to be initialized - // (3) running - // (4) done - - int num_initialized GUARDED_BY(mu); - int num_done GUARDED_BY(mu); - bool start GUARDED_BY(mu); - - SharedState(int total) - : cv(&mu), total(total), num_initialized(0), num_done(0), start(false) { } -}; - -// Per-thread state for concurrent executions of the same benchmark. -struct ThreadState { - int tid; // 0..n-1 when running in n threads - Random rand; // Has different seeds for different threads - Stats stats; - SharedState* shared; - - ThreadState(int index) - : tid(index), - rand(1000 + index) { - } -}; - -} // namespace - -class Benchmark { - private: - Cache* cache_; - const FilterPolicy* filter_policy_; - DB* db_; - int num_; - int value_size_; - int entries_per_batch_; - WriteOptions write_options_; - int reads_; - int heap_counter_; - - void PrintHeader() { - const int kKeySize = 16; - PrintEnvironment(); - fprintf(stdout, "Keys: %d bytes each\n", kKeySize); - fprintf(stdout, "Values: %d bytes each (%d bytes after compression)\n", - FLAGS_value_size, - static_cast<int>(FLAGS_value_size * FLAGS_compression_ratio + 0.5)); - fprintf(stdout, "Entries: %d\n", num_); - fprintf(stdout, "RawSize: %.1f MB (estimated)\n", - ((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_) - / 1048576.0)); - fprintf(stdout, "FileSize: %.1f MB (estimated)\n", - (((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_) - / 1048576.0)); - PrintWarnings(); - fprintf(stdout, "------------------------------------------------\n"); - } - - void PrintWarnings() { -#if defined(__GNUC__) && !defined(__OPTIMIZE__) - fprintf(stdout, - "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n" - ); -#endif -#ifndef NDEBUG - fprintf(stdout, - "WARNING: Assertions are enabled; benchmarks unnecessarily slow\n"); -#endif - - // See if snappy is working by attempting to compress a compressible string - const char text[] = "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy"; - std::string compressed; - if (!port::Snappy_Compress(text, sizeof(text), &compressed)) { - fprintf(stdout, "WARNING: Snappy compression is not enabled\n"); - } else if (compressed.size() >= sizeof(text)) { - fprintf(stdout, "WARNING: Snappy compression is not effective\n"); - } - } - - void PrintEnvironment() { - fprintf(stderr, "LevelDB: version %d.%d\n", - kMajorVersion, kMinorVersion); - -#if defined(__linux) - time_t now = time(nullptr); - fprintf(stderr, "Date: %s", ctime(&now)); // ctime() adds newline - - FILE* cpuinfo = fopen("/proc/cpuinfo", "r"); - if (cpuinfo != nullptr) { - char line[1000]; - int num_cpus = 0; - std::string cpu_type; - std::string cache_size; - while (fgets(line, sizeof(line), cpuinfo) != nullptr) { - const char* sep = strchr(line, ':'); - if (sep == nullptr) { - continue; - } - Slice key = TrimSpace(Slice(line, sep - 1 - line)); - Slice val = TrimSpace(Slice(sep + 1)); - if (key == "model name") { - ++num_cpus; - cpu_type = val.ToString(); - } else if (key == "cache size") { - cache_size = val.ToString(); - } - } - fclose(cpuinfo); - fprintf(stderr, "CPU: %d * %s\n", num_cpus, cpu_type.c_str()); - fprintf(stderr, "CPUCache: %s\n", cache_size.c_str()); - } -#endif - } - - public: - Benchmark() - : cache_(FLAGS_cache_size >= 0 ? NewLRUCache(FLAGS_cache_size) : nullptr), - filter_policy_(FLAGS_bloom_bits >= 0 - ? NewBloomFilterPolicy(FLAGS_bloom_bits) - : nullptr), - db_(nullptr), - num_(FLAGS_num), - value_size_(FLAGS_value_size), - entries_per_batch_(1), - reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads), - heap_counter_(0) { - std::vector<std::string> files; - g_env->GetChildren(FLAGS_db, &files); - for (size_t i = 0; i < files.size(); i++) { - if (Slice(files[i]).starts_with("heap-")) { - g_env->DeleteFile(std::string(FLAGS_db) + "/" + files[i]); - } - } - if (!FLAGS_use_existing_db) { - DestroyDB(FLAGS_db, Options()); - } - } - - ~Benchmark() { - delete db_; - delete cache_; - delete filter_policy_; - } - - void Run() { - PrintHeader(); - Open(); - - const char* benchmarks = FLAGS_benchmarks; - while (benchmarks != nullptr) { - const char* sep = strchr(benchmarks, ','); - Slice name; - if (sep == nullptr) { - name = benchmarks; - benchmarks = nullptr; - } else { - name = Slice(benchmarks, sep - benchmarks); - benchmarks = sep + 1; - } - - // Reset parameters that may be overridden below - num_ = FLAGS_num; - reads_ = (FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads); - value_size_ = FLAGS_value_size; - entries_per_batch_ = 1; - write_options_ = WriteOptions(); - - void (Benchmark::*method)(ThreadState*) = nullptr; - bool fresh_db = false; - int num_threads = FLAGS_threads; - - if (name == Slice("open")) { - method = &Benchmark::OpenBench; - num_ /= 10000; - if (num_ < 1) num_ = 1; - } else if (name == Slice("fillseq")) { - fresh_db = true; - method = &Benchmark::WriteSeq; - } else if (name == Slice("fillbatch")) { - fresh_db = true; - entries_per_batch_ = 1000; - method = &Benchmark::WriteSeq; - } else if (name == Slice("fillrandom")) { - fresh_db = true; - method = &Benchmark::WriteRandom; - } else if (name == Slice("overwrite")) { - fresh_db = false; - method = &Benchmark::WriteRandom; - } else if (name == Slice("fillsync")) { - fresh_db = true; - num_ /= 1000; - write_options_.sync = true; - method = &Benchmark::WriteRandom; - } else if (name == Slice("fill100K")) { - fresh_db = true; - num_ /= 1000; - value_size_ = 100 * 1000; - method = &Benchmark::WriteRandom; - } else if (name == Slice("readseq")) { - method = &Benchmark::ReadSequential; - } else if (name == Slice("readreverse")) { - method = &Benchmark::ReadReverse; - } else if (name == Slice("readrandom")) { - method = &Benchmark::ReadRandom; - } else if (name == Slice("readmissing")) { - method = &Benchmark::ReadMissing; - } else if (name == Slice("seekrandom")) { - method = &Benchmark::SeekRandom; - } else if (name == Slice("readhot")) { - method = &Benchmark::ReadHot; - } else if (name == Slice("readrandomsmall")) { - reads_ /= 1000; - method = &Benchmark::ReadRandom; - } else if (name == Slice("deleteseq")) { - method = &Benchmark::DeleteSeq; - } else if (name == Slice("deleterandom")) { - method = &Benchmark::DeleteRandom; - } else if (name == Slice("readwhilewriting")) { - num_threads++; // Add extra thread for writing - method = &Benchmark::ReadWhileWriting; - } else if (name == Slice("compact")) { - method = &Benchmark::Compact; - } else if (name == Slice("crc32c")) { - method = &Benchmark::Crc32c; - } else if (name == Slice("acquireload")) { - method = &Benchmark::AcquireLoad; - } else if (name == Slice("snappycomp")) { - method = &Benchmark::SnappyCompress; - } else if (name == Slice("snappyuncomp")) { - method = &Benchmark::SnappyUncompress; - } else if (name == Slice("heapprofile")) { - HeapProfile(); - } else if (name == Slice("stats")) { - PrintStats("leveldb.stats"); - } else if (name == Slice("sstables")) { - PrintStats("leveldb.sstables"); - } else { - if (name != Slice()) { // No error message for empty name - fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str()); - } - } - - if (fresh_db) { - if (FLAGS_use_existing_db) { - fprintf(stdout, "%-12s : skipped (--use_existing_db is true)\n", - name.ToString().c_str()); - method = nullptr; - } else { - delete db_; - db_ = nullptr; - DestroyDB(FLAGS_db, Options()); - Open(); - } - } - - if (method != nullptr) { - RunBenchmark(num_threads, name, method); - } - } - } - - private: - struct ThreadArg { - Benchmark* bm; - SharedState* shared; - ThreadState* thread; - void (Benchmark::*method)(ThreadState*); - }; - - static void ThreadBody(void* v) { - ThreadArg* arg = reinterpret_cast<ThreadArg*>(v); - SharedState* shared = arg->shared; - ThreadState* thread = arg->thread; - { - MutexLock l(&shared->mu); - shared->num_initialized++; - if (shared->num_initialized >= shared->total) { - shared->cv.SignalAll(); - } - while (!shared->start) { - shared->cv.Wait(); - } - } - - thread->stats.Start(); - (arg->bm->*(arg->method))(thread); - thread->stats.Stop(); - - { - MutexLock l(&shared->mu); - shared->num_done++; - if (shared->num_done >= shared->total) { - shared->cv.SignalAll(); - } - } - } - - void RunBenchmark(int n, Slice name, - void (Benchmark::*method)(ThreadState*)) { - SharedState shared(n); - - ThreadArg* arg = new ThreadArg[n]; - for (int i = 0; i < n; i++) { - arg[i].bm = this; - arg[i].method = method; - arg[i].shared = &shared; - arg[i].thread = new ThreadState(i); - arg[i].thread->shared = &shared; - g_env->StartThread(ThreadBody, &arg[i]); - } - - shared.mu.Lock(); - while (shared.num_initialized < n) { - shared.cv.Wait(); - } - - shared.start = true; - shared.cv.SignalAll(); - while (shared.num_done < n) { - shared.cv.Wait(); - } - shared.mu.Unlock(); - - for (int i = 1; i < n; i++) { - arg[0].thread->stats.Merge(arg[i].thread->stats); - } - arg[0].thread->stats.Report(name); - - for (int i = 0; i < n; i++) { - delete arg[i].thread; - } - delete[] arg; - } - - void Crc32c(ThreadState* thread) { - // Checksum about 500MB of data total - const int size = 4096; - const char* label = "(4K per op)"; - std::string data(size, 'x'); - int64_t bytes = 0; - uint32_t crc = 0; - while (bytes < 500 * 1048576) { - crc = crc32c::Value(data.data(), size); - thread->stats.FinishedSingleOp(); - bytes += size; - } - // Print so result is not dead - fprintf(stderr, "... crc=0x%x\r", static_cast<unsigned int>(crc)); - - thread->stats.AddBytes(bytes); - thread->stats.AddMessage(label); - } - - void AcquireLoad(ThreadState* thread) { - int dummy; - port::AtomicPointer ap(&dummy); - int count = 0; - void *ptr = nullptr; - thread->stats.AddMessage("(each op is 1000 loads)"); - while (count < 100000) { - for (int i = 0; i < 1000; i++) { - ptr = ap.Acquire_Load(); - } - count++; - thread->stats.FinishedSingleOp(); - } - if (ptr == nullptr) exit(1); // Disable unused variable warning. - } - - void SnappyCompress(ThreadState* thread) { - RandomGenerator gen; - Slice input = gen.Generate(Options().block_size); - int64_t bytes = 0; - int64_t produced = 0; - bool ok = true; - std::string compressed; - while (ok && bytes < 1024 * 1048576) { // Compress 1G - ok = port::Snappy_Compress(input.data(), input.size(), &compressed); - produced += compressed.size(); - bytes += input.size(); - thread->stats.FinishedSingleOp(); - } - - if (!ok) { - thread->stats.AddMessage("(snappy failure)"); - } else { - char buf[100]; - snprintf(buf, sizeof(buf), "(output: %.1f%%)", - (produced * 100.0) / bytes); - thread->stats.AddMessage(buf); - thread->stats.AddBytes(bytes); - } - } - - void SnappyUncompress(ThreadState* thread) { - RandomGenerator gen; - Slice input = gen.Generate(Options().block_size); - std::string compressed; - bool ok = port::Snappy_Compress(input.data(), input.size(), &compressed); - int64_t bytes = 0; - char* uncompressed = new char[input.size()]; - while (ok && bytes < 1024 * 1048576) { // Compress 1G - ok = port::Snappy_Uncompress(compressed.data(), compressed.size(), - uncompressed); - bytes += input.size(); - thread->stats.FinishedSingleOp(); - } - delete[] uncompressed; - - if (!ok) { - thread->stats.AddMessage("(snappy failure)"); - } else { - thread->stats.AddBytes(bytes); - } - } - - void Open() { - assert(db_ == nullptr); - Options options; - options.env = g_env; - options.create_if_missing = !FLAGS_use_existing_db; - options.block_cache = cache_; - options.write_buffer_size = FLAGS_write_buffer_size; - options.max_file_size = FLAGS_max_file_size; - options.block_size = FLAGS_block_size; - options.max_open_files = FLAGS_open_files; - options.filter_policy = filter_policy_; - options.reuse_logs = FLAGS_reuse_logs; - Status s = DB::Open(options, FLAGS_db, &db_); - if (!s.ok()) { - fprintf(stderr, "open error: %s\n", s.ToString().c_str()); - exit(1); - } - } - - void OpenBench(ThreadState* thread) { - for (int i = 0; i < num_; i++) { - delete db_; - Open(); - thread->stats.FinishedSingleOp(); - } - } - - void WriteSeq(ThreadState* thread) { - DoWrite(thread, true); - } - - void WriteRandom(ThreadState* thread) { - DoWrite(thread, false); - } - - void DoWrite(ThreadState* thread, bool seq) { - if (num_ != FLAGS_num) { - char msg[100]; - snprintf(msg, sizeof(msg), "(%d ops)", num_); - thread->stats.AddMessage(msg); - } - - RandomGenerator gen; - WriteBatch batch; - Status s; - int64_t bytes = 0; - for (int i = 0; i < num_; i += entries_per_batch_) { - batch.Clear(); - for (int j = 0; j < entries_per_batch_; j++) { - const int k = seq ? i+j : (thread->rand.Next() % FLAGS_num); - char key[100]; - snprintf(key, sizeof(key), "%016d", k); - batch.Put(key, gen.Generate(value_size_)); - bytes += value_size_ + strlen(key); - thread->stats.FinishedSingleOp(); - } - s = db_->Write(write_options_, &batch); - if (!s.ok()) { - fprintf(stderr, "put error: %s\n", s.ToString().c_str()); - exit(1); - } - } - thread->stats.AddBytes(bytes); - } - - void ReadSequential(ThreadState* thread) { - Iterator* iter = db_->NewIterator(ReadOptions()); - int i = 0; - int64_t bytes = 0; - for (iter->SeekToFirst(); i < reads_ && iter->Valid(); iter->Next()) { - bytes += iter->key().size() + iter->value().size(); - thread->stats.FinishedSingleOp(); - ++i; - } - delete iter; - thread->stats.AddBytes(bytes); - } - - void ReadReverse(ThreadState* thread) { - Iterator* iter = db_->NewIterator(ReadOptions()); - int i = 0; - int64_t bytes = 0; - for (iter->SeekToLast(); i < reads_ && iter->Valid(); iter->Prev()) { - bytes += iter->key().size() + iter->value().size(); - thread->stats.FinishedSingleOp(); - ++i; - } - delete iter; - thread->stats.AddBytes(bytes); - } - - void ReadRandom(ThreadState* thread) { - ReadOptions options; - std::string value; - int found = 0; - for (int i = 0; i < reads_; i++) { - char key[100]; - const int k = thread->rand.Next() % FLAGS_num; - snprintf(key, sizeof(key), "%016d", k); - if (db_->Get(options, key, &value).ok()) { - found++; - } - thread->stats.FinishedSingleOp(); - } - char msg[100]; - snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_); - thread->stats.AddMessage(msg); - } - - void ReadMissing(ThreadState* thread) { - ReadOptions options; - std::string value; - for (int i = 0; i < reads_; i++) { - char key[100]; - const int k = thread->rand.Next() % FLAGS_num; - snprintf(key, sizeof(key), "%016d.", k); - db_->Get(options, key, &value); - thread->stats.FinishedSingleOp(); - } - } - - void ReadHot(ThreadState* thread) { - ReadOptions options; - std::string value; - const int range = (FLAGS_num + 99) / 100; - for (int i = 0; i < reads_; i++) { - char key[100]; - const int k = thread->rand.Next() % range; - snprintf(key, sizeof(key), "%016d", k); - db_->Get(options, key, &value); - thread->stats.FinishedSingleOp(); - } - } - - void SeekRandom(ThreadState* thread) { - ReadOptions options; - int found = 0; - for (int i = 0; i < reads_; i++) { - Iterator* iter = db_->NewIterator(options); - char key[100]; - const int k = thread->rand.Next() % FLAGS_num; - snprintf(key, sizeof(key), "%016d", k); - iter->Seek(key); - if (iter->Valid() && iter->key() == key) found++; - delete iter; - thread->stats.FinishedSingleOp(); - } - char msg[100]; - snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_); - thread->stats.AddMessage(msg); - } - - void DoDelete(ThreadState* thread, bool seq) { - RandomGenerator gen; - WriteBatch batch; - Status s; - for (int i = 0; i < num_; i += entries_per_batch_) { - batch.Clear(); - for (int j = 0; j < entries_per_batch_; j++) { - const int k = seq ? i+j : (thread->rand.Next() % FLAGS_num); - char key[100]; - snprintf(key, sizeof(key), "%016d", k); - batch.Delete(key); - thread->stats.FinishedSingleOp(); - } - s = db_->Write(write_options_, &batch); - if (!s.ok()) { - fprintf(stderr, "del error: %s\n", s.ToString().c_str()); - exit(1); - } - } - } - - void DeleteSeq(ThreadState* thread) { - DoDelete(thread, true); - } - - void DeleteRandom(ThreadState* thread) { - DoDelete(thread, false); - } - - void ReadWhileWriting(ThreadState* thread) { - if (thread->tid > 0) { - ReadRandom(thread); - } else { - // Special thread that keeps writing until other threads are done. - RandomGenerator gen; - while (true) { - { - MutexLock l(&thread->shared->mu); - if (thread->shared->num_done + 1 >= thread->shared->num_initialized) { - // Other threads have finished - break; - } - } - - const int k = thread->rand.Next() % FLAGS_num; - char key[100]; - snprintf(key, sizeof(key), "%016d", k); - Status s = db_->Put(write_options_, key, gen.Generate(value_size_)); - if (!s.ok()) { - fprintf(stderr, "put error: %s\n", s.ToString().c_str()); - exit(1); - } - } - - // Do not count any of the preceding work/delay in stats. - thread->stats.Start(); - } - } - - void Compact(ThreadState* thread) { - db_->CompactRange(nullptr, nullptr); - } - - void PrintStats(const char* key) { - std::string stats; - if (!db_->GetProperty(key, &stats)) { - stats = "(failed)"; - } - fprintf(stdout, "\n%s\n", stats.c_str()); - } - - static void WriteToFile(void* arg, const char* buf, int n) { - reinterpret_cast<WritableFile*>(arg)->Append(Slice(buf, n)); - } - - void HeapProfile() { - char fname[100]; - snprintf(fname, sizeof(fname), "%s/heap-%04d", FLAGS_db, ++heap_counter_); - WritableFile* file; - Status s = g_env->NewWritableFile(fname, &file); - if (!s.ok()) { - fprintf(stderr, "%s\n", s.ToString().c_str()); - return; - } - bool ok = port::GetHeapProfile(WriteToFile, file); - delete file; - if (!ok) { - fprintf(stderr, "heap profiling not supported\n"); - g_env->DeleteFile(fname); - } - } -}; - -} // namespace leveldb - -int main(int argc, char** argv) { - FLAGS_write_buffer_size = leveldb::Options().write_buffer_size; - FLAGS_max_file_size = leveldb::Options().max_file_size; - FLAGS_block_size = leveldb::Options().block_size; - FLAGS_open_files = leveldb::Options().max_open_files; - std::string default_db_path; - - for (int i = 1; i < argc; i++) { - double d; - int n; - char junk; - if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) { - FLAGS_benchmarks = argv[i] + strlen("--benchmarks="); - } else if (sscanf(argv[i], "--compression_ratio=%lf%c", &d, &junk) == 1) { - FLAGS_compression_ratio = d; - } else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 && - (n == 0 || n == 1)) { - FLAGS_histogram = n; - } else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 && - (n == 0 || n == 1)) { - FLAGS_use_existing_db = n; - } else if (sscanf(argv[i], "--reuse_logs=%d%c", &n, &junk) == 1 && - (n == 0 || n == 1)) { - FLAGS_reuse_logs = n; - } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) { - FLAGS_num = n; - } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) { - FLAGS_reads = n; - } else if (sscanf(argv[i], "--threads=%d%c", &n, &junk) == 1) { - FLAGS_threads = n; - } else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) { - FLAGS_value_size = n; - } else if (sscanf(argv[i], "--write_buffer_size=%d%c", &n, &junk) == 1) { - FLAGS_write_buffer_size = n; - } else if (sscanf(argv[i], "--max_file_size=%d%c", &n, &junk) == 1) { - FLAGS_max_file_size = n; - } else if (sscanf(argv[i], "--block_size=%d%c", &n, &junk) == 1) { - FLAGS_block_size = n; - } else if (sscanf(argv[i], "--cache_size=%d%c", &n, &junk) == 1) { - FLAGS_cache_size = n; - } else if (sscanf(argv[i], "--bloom_bits=%d%c", &n, &junk) == 1) { - FLAGS_bloom_bits = n; - } else if (sscanf(argv[i], "--open_files=%d%c", &n, &junk) == 1) { - FLAGS_open_files = n; - } else if (strncmp(argv[i], "--db=", 5) == 0) { - FLAGS_db = argv[i] + 5; - } else { - fprintf(stderr, "Invalid flag '%s'\n", argv[i]); - exit(1); - } - } - - leveldb::g_env = leveldb::Env::Default(); - - // Choose a location for the test database if none given with --db=<path> - if (FLAGS_db == nullptr) { - leveldb::g_env->GetTestDirectory(&default_db_path); - default_db_path += "/dbbench"; - FLAGS_db = default_db_path.c_str(); - } - - leveldb::Benchmark benchmark; - benchmark.Run(); - return 0; -} diff --git a/db/db_impl.cc b/db/db_impl.cc index fefb883..1a4e459 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -4,10 +4,10 @@ #include "db/db_impl.h" -#include <stdint.h> -#include <stdio.h> - #include <algorithm> +#include <atomic> +#include <cstdint> +#include <cstdio> #include <set> #include <string> #include <vector> @@ -41,16 +41,33 @@ const int kNumNonTableCacheFiles = 10; // Information kept for every waiting writer struct DBImpl::Writer { + explicit Writer(port::Mutex* mu) + : batch(nullptr), sync(false), done(false), cv(mu) {} + Status status; WriteBatch* batch; bool sync; bool done; port::CondVar cv; - - explicit Writer(port::Mutex* mu) : cv(mu) { } }; struct DBImpl::CompactionState { + // Files produced by compaction + struct Output { + uint64_t number; + uint64_t file_size; + InternalKey smallest, largest; + }; + + Output* current_output() { return &outputs[outputs.size() - 1]; } + + explicit CompactionState(Compaction* c) + : compaction(c), + smallest_snapshot(0), + outfile(nullptr), + builder(nullptr), + total_bytes(0) {} + Compaction* const compaction; // Sequence numbers < smallest_snapshot are not significant since we @@ -59,12 +76,6 @@ struct DBImpl::CompactionState { // we can drop all entries for the same key with sequence numbers < S. SequenceNumber smallest_snapshot; - // Files produced by compaction - struct Output { - uint64_t number; - uint64_t file_size; - InternalKey smallest, largest; - }; std::vector<Output> outputs; // State kept for output being generated @@ -72,15 +83,6 @@ struct DBImpl::CompactionState { TableBuilder* builder; uint64_t total_bytes; - - Output* current_output() { return &outputs[outputs.size()-1]; } - - explicit CompactionState(Compaction* c) - : compaction(c), - outfile(nullptr), - builder(nullptr), - total_bytes(0) { - } }; // Fix user-supplied options to be reasonable @@ -96,10 +98,10 @@ Options SanitizeOptions(const std::string& dbname, Options result = src; result.comparator = icmp; result.filter_policy = (src.filter_policy != nullptr) ? ipolicy : nullptr; - ClipToRange(&result.max_open_files, 64 + kNumNonTableCacheFiles, 50000); - ClipToRange(&result.write_buffer_size, 64<<10, 1<<30); - ClipToRange(&result.max_file_size, 1<<20, 1<<30); - ClipToRange(&result.block_size, 1<<10, 4<<20); + ClipToRange(&result.max_open_files, 64 + kNumNonTableCacheFiles, 50000); + ClipToRange(&result.write_buffer_size, 64 << 10, 1 << 30); + ClipToRange(&result.max_file_size, 1 << 20, 1 << 30); + ClipToRange(&result.block_size, 1 << 10, 4 << 20); if (result.info_log == nullptr) { // Open a log file in the same directory as the db src.env->CreateDir(dbname); // In case it does not exist @@ -132,10 +134,11 @@ DBImpl::DBImpl(const Options& raw_options, const std::string& dbname) dbname_(dbname), table_cache_(new TableCache(dbname_, options_, TableCacheSize(options_))), db_lock_(nullptr), - shutting_down_(nullptr), + shutting_down_(false), background_work_finished_signal_(&mutex_), mem_(nullptr), imm_(nullptr), + has_imm_(false), logfile_(nullptr), logfile_number_(0), log_(nullptr), @@ -144,14 +147,12 @@ DBImpl::DBImpl(const Options& raw_options, const std::string& dbname) background_compaction_scheduled_(false), manual_compaction_(nullptr), versions_(new VersionSet(dbname_, &options_, table_cache_, - &internal_comparator_)) { - has_imm_.Release_Store(nullptr); -} + &internal_comparator_)) {} DBImpl::~DBImpl() { - // Wait for background work to finish + // Wait for background work to finish. mutex_.Lock(); - shutting_down_.Release_Store(this); // Any non-null value is ok + shutting_down_.store(true, std::memory_order_release); while (background_compaction_scheduled_) { background_work_finished_signal_.Wait(); } @@ -196,6 +197,9 @@ Status DBImpl::NewDB() { new_db.EncodeTo(&record); s = log.AddRecord(record); if (s.ok()) { + s = file->Sync(); + } + if (s.ok()) { s = file->Close(); } } @@ -204,7 +208,7 @@ Status DBImpl::NewDB() { // Make "CURRENT" file that points to the new manifest file. s = SetCurrentFile(env_, dbname_, 1); } else { - env_->DeleteFile(manifest); + env_->RemoveFile(manifest); } return s; } @@ -218,7 +222,7 @@ void DBImpl::MaybeIgnoreError(Status* s) const { } } -void DBImpl::DeleteObsoleteFiles() { +void DBImpl::RemoveObsoleteFiles() { mutex_.AssertHeld(); if (!bg_error_.ok()) { @@ -235,8 +239,9 @@ void DBImpl::DeleteObsoleteFiles() { env_->GetChildren(dbname_, &filenames); // Ignoring errors on purpose uint64_t number; FileType type; - for (size_t i = 0; i < filenames.size(); i++) { - if (ParseFileName(filenames[i], &number, &type)) { + std::vector<std::string> files_to_delete; + for (std::string& filename : filenames) { + if (ParseFileName(filename, &number, &type)) { bool keep = true; switch (type) { case kLogFile: @@ -264,19 +269,27 @@ void DBImpl::DeleteObsoleteFiles() { } if (!keep) { + files_to_delete.push_back(std::move(filename)); if (type == kTableFile) { table_cache_->Evict(number); } - Log(options_.info_log, "Delete type=%d #%lld\n", - static_cast<int>(type), + Log(options_.info_log, "Delete type=%d #%lld\n", static_cast<int>(type), static_cast<unsigned long long>(number)); - env_->DeleteFile(dbname_ + "/" + filenames[i]); } } } + + // While deleting all files unblock other threads. All files being deleted + // have unique names which will not collide with newly created files and + // are therefore safe to delete while allowing other threads to proceed. + mutex_.Unlock(); + for (const std::string& filename : files_to_delete) { + env_->RemoveFile(dbname_ + "/" + filename); + } + mutex_.Lock(); } -Status DBImpl::Recover(VersionEdit* edit, bool *save_manifest) { +Status DBImpl::Recover(VersionEdit* edit, bool* save_manifest) { mutex_.AssertHeld(); // Ignore error from CreateDir since the creation of the DB is @@ -291,6 +304,8 @@ Status DBImpl::Recover(VersionEdit* edit, bool *save_manifest) { if (!env_->FileExists(CurrentFileName(dbname_))) { if (options_.create_if_missing) { + Log(options_.info_log, "Creating DB %s since it was missing.", + dbname_.c_str()); s = NewDB(); if (!s.ok()) { return s; @@ -301,8 +316,8 @@ Status DBImpl::Recover(VersionEdit* edit, bool *save_manifest) { } } else { if (options_.error_if_exists) { - return Status::InvalidArgument( - dbname_, "exists (error_if_exists is true)"); + return Status::InvalidArgument(dbname_, + "exists (error_if_exists is true)"); } } @@ -340,8 +355,8 @@ Status DBImpl::Recover(VersionEdit* edit, bool *save_manifest) { } if (!expected.empty()) { char buf[50]; - snprintf(buf, sizeof(buf), "%d missing files; e.g.", - static_cast<int>(expected.size())); + std::snprintf(buf, sizeof(buf), "%d missing files; e.g.", + static_cast<int>(expected.size())); return Status::Corruption(buf, TableFileName(dbname_, *(expected.begin()))); } @@ -375,10 +390,10 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log, Logger* info_log; const char* fname; Status* status; // null if options_.paranoid_checks==false - virtual void Corruption(size_t bytes, const Status& s) { + void Corruption(size_t bytes, const Status& s) override { Log(info_log, "%s%s: dropping %d bytes; %s", - (this->status == nullptr ? "(ignoring error) " : ""), - fname, static_cast<int>(bytes), s.ToString().c_str()); + (this->status == nullptr ? "(ignoring error) " : ""), fname, + static_cast<int>(bytes), s.ToString().c_str()); if (this->status != nullptr && this->status->ok()) *this->status = s; } }; @@ -404,10 +419,9 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log, // paranoid_checks==false so that corruptions cause entire commits // to be skipped instead of propagating bad information (like overly // large sequence numbers). - log::Reader reader(file, &reporter, true/*checksum*/, - 0/*initial_offset*/); + log::Reader reader(file, &reporter, true /*checksum*/, 0 /*initial_offset*/); Log(options_.info_log, "Recovering log #%llu", - (unsigned long long) log_number); + (unsigned long long)log_number); // Read all the records and add to a memtable std::string scratch; @@ -415,11 +429,10 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log, WriteBatch batch; int compactions = 0; MemTable* mem = nullptr; - while (reader.ReadRecord(&record, &scratch) && - status.ok()) { + while (reader.ReadRecord(&record, &scratch) && status.ok()) { if (record.size() < 12) { - reporter.Corruption( - record.size(), Status::Corruption("log record too small")); + reporter.Corruption(record.size(), + Status::Corruption("log record too small")); continue; } WriteBatchInternal::SetContents(&batch, record); @@ -433,9 +446,8 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log, if (!status.ok()) { break; } - const SequenceNumber last_seq = - WriteBatchInternal::Sequence(&batch) + - WriteBatchInternal::Count(&batch) - 1; + const SequenceNumber last_seq = WriteBatchInternal::Sequence(&batch) + + WriteBatchInternal::Count(&batch) - 1; if (last_seq > *max_sequence) { *max_sequence = last_seq; } @@ -499,7 +511,7 @@ Status DBImpl::WriteLevel0Table(MemTable* mem, VersionEdit* edit, pending_outputs_.insert(meta.number); Iterator* iter = mem->NewIterator(); Log(options_.info_log, "Level-0 table #%llu: started", - (unsigned long long) meta.number); + (unsigned long long)meta.number); Status s; { @@ -509,13 +521,11 @@ Status DBImpl::WriteLevel0Table(MemTable* mem, VersionEdit* edit, } Log(options_.info_log, "Level-0 table #%llu: %lld bytes %s", - (unsigned long long) meta.number, - (unsigned long long) meta.file_size, + (unsigned long long)meta.number, (unsigned long long)meta.file_size, s.ToString().c_str()); delete iter; pending_outputs_.erase(meta.number); - // Note that if file_size is zero, the file has been deleted and // should not be added to the manifest. int level = 0; @@ -525,8 +535,8 @@ Status DBImpl::WriteLevel0Table(MemTable* mem, VersionEdit* edit, if (base != nullptr) { level = base->PickLevelForMemTableOutput(min_user_key, max_user_key); } - edit->AddFile(level, meta.number, meta.file_size, - meta.smallest, meta.largest); + edit->AddFile(level, meta.number, meta.file_size, meta.smallest, + meta.largest); } CompactionStats stats; @@ -547,7 +557,7 @@ void DBImpl::CompactMemTable() { Status s = WriteLevel0Table(imm_, &edit, base); base->Unref(); - if (s.ok() && shutting_down_.Acquire_Load()) { + if (s.ok() && shutting_down_.load(std::memory_order_acquire)) { s = Status::IOError("Deleting DB during memtable compaction"); } @@ -562,8 +572,8 @@ void DBImpl::CompactMemTable() { // Commit to the new state imm_->Unref(); imm_ = nullptr; - has_imm_.Release_Store(nullptr); - DeleteObsoleteFiles(); + has_imm_.store(false, std::memory_order_release); + RemoveObsoleteFiles(); } else { RecordBackgroundError(s); } @@ -610,7 +620,8 @@ void DBImpl::TEST_CompactRange(int level, const Slice* begin, } MutexLock l(&mutex_); - while (!manual.done && !shutting_down_.Acquire_Load() && bg_error_.ok()) { + while (!manual.done && !shutting_down_.load(std::memory_order_acquire) && + bg_error_.ok()) { if (manual_compaction_ == nullptr) { // Idle manual_compaction_ = &manual; MaybeScheduleCompaction(); @@ -652,12 +663,11 @@ void DBImpl::MaybeScheduleCompaction() { mutex_.AssertHeld(); if (background_compaction_scheduled_) { // Already scheduled - } else if (shutting_down_.Acquire_Load()) { + } else if (shutting_down_.load(std::memory_order_acquire)) { // DB is being deleted; no more background compactions } else if (!bg_error_.ok()) { // Already got an error; no more changes - } else if (imm_ == nullptr && - manual_compaction_ == nullptr && + } else if (imm_ == nullptr && manual_compaction_ == nullptr && !versions_->NeedsCompaction()) { // No work to be done } else { @@ -673,7 +683,7 @@ void DBImpl::BGWork(void* db) { void DBImpl::BackgroundCall() { MutexLock l(&mutex_); assert(background_compaction_scheduled_); - if (shutting_down_.Acquire_Load()) { + if (shutting_down_.load(std::memory_order_acquire)) { // No more background work when shutting down. } else if (!bg_error_.ok()) { // No more background work after a background error. @@ -709,8 +719,7 @@ void DBImpl::BackgroundCompaction() { } Log(options_.info_log, "Manual compaction at level-%d from %s .. %s; will stop at %s\n", - m->level, - (m->begin ? m->begin->DebugString().c_str() : "(begin)"), + m->level, (m->begin ? m->begin->DebugString().c_str() : "(begin)"), (m->end ? m->end->DebugString().c_str() : "(end)"), (m->done ? "(end)" : manual_end.DebugString().c_str())); } else { @@ -724,20 +733,18 @@ void DBImpl::BackgroundCompaction() { // Move file to next level assert(c->num_input_files(0) == 1); FileMetaData* f = c->input(0, 0); - c->edit()->DeleteFile(c->level(), f->number); - c->edit()->AddFile(c->level() + 1, f->number, f->file_size, - f->smallest, f->largest); + c->edit()->RemoveFile(c->level(), f->number); + c->edit()->AddFile(c->level() + 1, f->number, f->file_size, f->smallest, + f->largest); status = versions_->LogAndApply(c->edit(), &mutex_); if (!status.ok()) { RecordBackgroundError(status); } VersionSet::LevelSummaryStorage tmp; Log(options_.info_log, "Moved #%lld to level-%d %lld bytes %s: %s\n", - static_cast<unsigned long long>(f->number), - c->level() + 1, + static_cast<unsigned long long>(f->number), c->level() + 1, static_cast<unsigned long long>(f->file_size), - status.ToString().c_str(), - versions_->LevelSummary(&tmp)); + status.ToString().c_str(), versions_->LevelSummary(&tmp)); } else { CompactionState* compact = new CompactionState(c); status = DoCompactionWork(compact); @@ -746,17 +753,16 @@ void DBImpl::BackgroundCompaction() { } CleanupCompaction(compact); c->ReleaseInputs(); - DeleteObsoleteFiles(); + RemoveObsoleteFiles(); } delete c; if (status.ok()) { // Done - } else if (shutting_down_.Acquire_Load()) { + } else if (shutting_down_.load(std::memory_order_acquire)) { // Ignore compaction errors found during shutting down } else { - Log(options_.info_log, - "Compaction error: %s", status.ToString().c_str()); + Log(options_.info_log, "Compaction error: %s", status.ToString().c_str()); } if (is_manual) { @@ -851,31 +857,25 @@ Status DBImpl::FinishCompactionOutputFile(CompactionState* compact, if (s.ok() && current_entries > 0) { // Verify that the table is usable - Iterator* iter = table_cache_->NewIterator(ReadOptions(), - output_number, - current_bytes); + Iterator* iter = + table_cache_->NewIterator(ReadOptions(), output_number, current_bytes); s = iter->status(); delete iter; if (s.ok()) { - Log(options_.info_log, - "Generated table #%llu@%d: %lld keys, %lld bytes", - (unsigned long long) output_number, - compact->compaction->level(), - (unsigned long long) current_entries, - (unsigned long long) current_bytes); + Log(options_.info_log, "Generated table #%llu@%d: %lld keys, %lld bytes", + (unsigned long long)output_number, compact->compaction->level(), + (unsigned long long)current_entries, + (unsigned long long)current_bytes); } } return s; } - Status DBImpl::InstallCompactionResults(CompactionState* compact) { mutex_.AssertHeld(); - Log(options_.info_log, "Compacted %d@%d + %d@%d files => %lld bytes", - compact->compaction->num_input_files(0), - compact->compaction->level(), - compact->compaction->num_input_files(1), - compact->compaction->level() + 1, + Log(options_.info_log, "Compacted %d@%d + %d@%d files => %lld bytes", + compact->compaction->num_input_files(0), compact->compaction->level(), + compact->compaction->num_input_files(1), compact->compaction->level() + 1, static_cast<long long>(compact->total_bytes)); // Add compaction outputs @@ -883,9 +883,8 @@ Status DBImpl::InstallCompactionResults(CompactionState* compact) { const int level = compact->compaction->level(); for (size_t i = 0; i < compact->outputs.size(); i++) { const CompactionState::Output& out = compact->outputs[i]; - compact->compaction->edit()->AddFile( - level + 1, - out.number, out.file_size, out.smallest, out.largest); + compact->compaction->edit()->AddFile(level + 1, out.number, out.file_size, + out.smallest, out.largest); } return versions_->LogAndApply(compact->compaction->edit(), &mutex_); } @@ -894,9 +893,8 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) { const uint64_t start_micros = env_->NowMicros(); int64_t imm_micros = 0; // Micros spent doing imm_ compactions - Log(options_.info_log, "Compacting %d@%d + %d@%d files", - compact->compaction->num_input_files(0), - compact->compaction->level(), + Log(options_.info_log, "Compacting %d@%d + %d@%d files", + compact->compaction->num_input_files(0), compact->compaction->level(), compact->compaction->num_input_files(1), compact->compaction->level() + 1); @@ -909,19 +907,20 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) { compact->smallest_snapshot = snapshots_.oldest()->sequence_number(); } + Iterator* input = versions_->MakeInputIterator(compact->compaction); + // Release mutex while we're actually doing the compaction work mutex_.Unlock(); - Iterator* input = versions_->MakeInputIterator(compact->compaction); input->SeekToFirst(); Status status; ParsedInternalKey ikey; std::string current_user_key; bool has_current_user_key = false; SequenceNumber last_sequence_for_key = kMaxSequenceNumber; - for (; input->Valid() && !shutting_down_.Acquire_Load(); ) { + while (input->Valid() && !shutting_down_.load(std::memory_order_acquire)) { // Prioritize immutable compaction work - if (has_imm_.NoBarrier_Load() != nullptr) { + if (has_imm_.load(std::memory_order_relaxed)) { const uint64_t imm_start = env_->NowMicros(); mutex_.Lock(); if (imm_ != nullptr) { @@ -951,8 +950,8 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) { last_sequence_for_key = kMaxSequenceNumber; } else { if (!has_current_user_key || - user_comparator()->Compare(ikey.user_key, - Slice(current_user_key)) != 0) { + user_comparator()->Compare(ikey.user_key, Slice(current_user_key)) != + 0) { // First occurrence of this user key current_user_key.assign(ikey.user_key.data(), ikey.user_key.size()); has_current_user_key = true; @@ -961,7 +960,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) { if (last_sequence_for_key <= compact->smallest_snapshot) { // Hidden by an newer entry for same user key - drop = true; // (A) + drop = true; // (A) } else if (ikey.type == kTypeDeletion && ikey.sequence <= compact->smallest_snapshot && compact->compaction->IsBaseLevelForKey(ikey.user_key)) { @@ -1014,7 +1013,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) { input->Next(); } - if (status.ok() && shutting_down_.Acquire_Load()) { + if (status.ok() && shutting_down_.load(std::memory_order_acquire)) { status = Status::IOError("Deleting DB during compaction"); } if (status.ok() && compact->builder != nullptr) { @@ -1047,8 +1046,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) { RecordBackgroundError(status); } VersionSet::LevelSummaryStorage tmp; - Log(options_.info_log, - "compacted to: %s", versions_->LevelSummary(&tmp)); + Log(options_.info_log, "compacted to: %s", versions_->LevelSummary(&tmp)); return status; } @@ -1061,7 +1059,7 @@ struct IterState { MemTable* const imm GUARDED_BY(mu); IterState(port::Mutex* mutex, MemTable* mem, MemTable* imm, Version* version) - : mu(mutex), version(version), mem(mem), imm(imm) { } + : mu(mutex), version(version), mem(mem), imm(imm) {} }; static void CleanupIteratorState(void* arg1, void* arg2) { @@ -1114,8 +1112,7 @@ int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes() { return versions_->MaxNextLevelOverlappingBytes(); } -Status DBImpl::Get(const ReadOptions& options, - const Slice& key, +Status DBImpl::Get(const ReadOptions& options, const Slice& key, std::string* value) { Status s; MutexLock l(&mutex_); @@ -1166,12 +1163,12 @@ Iterator* DBImpl::NewIterator(const ReadOptions& options) { SequenceNumber latest_snapshot; uint32_t seed; Iterator* iter = NewInternalIterator(options, &latest_snapshot, &seed); - return NewDBIterator( - this, user_comparator(), iter, - (options.snapshot != nullptr - ? static_cast<const SnapshotImpl*>(options.snapshot)->sequence_number() - : latest_snapshot), - seed); + return NewDBIterator(this, user_comparator(), iter, + (options.snapshot != nullptr + ? static_cast<const SnapshotImpl*>(options.snapshot) + ->sequence_number() + : latest_snapshot), + seed); } void DBImpl::RecordReadSample(Slice key) { @@ -1200,9 +1197,9 @@ Status DBImpl::Delete(const WriteOptions& options, const Slice& key) { return DB::Delete(options, key); } -Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) { +Status DBImpl::Write(const WriteOptions& options, WriteBatch* updates) { Writer w(&mutex_); - w.batch = my_batch; + w.batch = updates; w.sync = options.sync; w.done = false; @@ -1216,13 +1213,13 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) { } // May temporarily unlock and wait. - Status status = MakeRoomForWrite(my_batch == nullptr); + Status status = MakeRoomForWrite(updates == nullptr); uint64_t last_sequence = versions_->LastSequence(); Writer* last_writer = &w; - if (status.ok() && my_batch != nullptr) { // nullptr batch is for compactions - WriteBatch* updates = BuildBatchGroup(&last_writer); - WriteBatchInternal::SetSequence(updates, last_sequence + 1); - last_sequence += WriteBatchInternal::Count(updates); + if (status.ok() && updates != nullptr) { // nullptr batch is for compactions + WriteBatch* write_batch = BuildBatchGroup(&last_writer); + WriteBatchInternal::SetSequence(write_batch, last_sequence + 1); + last_sequence += WriteBatchInternal::Count(write_batch); // Add to log and apply to memtable. We can release the lock // during this phase since &w is currently responsible for logging @@ -1230,7 +1227,7 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) { // into mem_. { mutex_.Unlock(); - status = log_->AddRecord(WriteBatchInternal::Contents(updates)); + status = log_->AddRecord(WriteBatchInternal::Contents(write_batch)); bool sync_error = false; if (status.ok() && options.sync) { status = logfile_->Sync(); @@ -1239,7 +1236,7 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) { } } if (status.ok()) { - status = WriteBatchInternal::InsertInto(updates, mem_); + status = WriteBatchInternal::InsertInto(write_batch, mem_); } mutex_.Lock(); if (sync_error) { @@ -1249,7 +1246,7 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) { RecordBackgroundError(status); } } - if (updates == tmp_batch_) tmp_batch_->Clear(); + if (write_batch == tmp_batch_) tmp_batch_->Clear(); versions_->SetLastSequence(last_sequence); } @@ -1288,8 +1285,8 @@ WriteBatch* DBImpl::BuildBatchGroup(Writer** last_writer) { // original write is small, limit the growth so we do not slow // down the small write too much. size_t max_size = 1 << 20; - if (size <= (128<<10)) { - max_size = size + (128<<10); + if (size <= (128 << 10)) { + max_size = size + (128 << 10); } *last_writer = first; @@ -1335,9 +1332,8 @@ Status DBImpl::MakeRoomForWrite(bool force) { // Yield previous error s = bg_error_; break; - } else if ( - allow_delay && - versions_->NumLevelFiles(0) >= config::kL0_SlowdownWritesTrigger) { + } else if (allow_delay && versions_->NumLevelFiles(0) >= + config::kL0_SlowdownWritesTrigger) { // We are getting close to hitting a hard limit on the number of // L0 files. Rather than delaying a single write by several // seconds when we hit the hard limit, start delaying each @@ -1378,10 +1374,10 @@ Status DBImpl::MakeRoomForWrite(bool force) { logfile_number_ = new_log_number; log_ = new log::Writer(lfile); imm_ = mem_; - has_imm_.Release_Store(imm_); + has_imm_.store(true, std::memory_order_release); mem_ = new MemTable(internal_comparator_); mem_->Ref(); - force = false; // Do not force another compaction if have room + force = false; // Do not force another compaction if have room MaybeScheduleCompaction(); } } @@ -1405,31 +1401,26 @@ bool DBImpl::GetProperty(const Slice& property, std::string* value) { return false; } else { char buf[100]; - snprintf(buf, sizeof(buf), "%d", - versions_->NumLevelFiles(static_cast<int>(level))); + std::snprintf(buf, sizeof(buf), "%d", + versions_->NumLevelFiles(static_cast<int>(level))); *value = buf; return true; } } else if (in == "stats") { char buf[200]; - snprintf(buf, sizeof(buf), - " Compactions\n" - "Level Files Size(MB) Time(sec) Read(MB) Write(MB)\n" - "--------------------------------------------------\n" - ); + std::snprintf(buf, sizeof(buf), + " Compactions\n" + "Level Files Size(MB) Time(sec) Read(MB) Write(MB)\n" + "--------------------------------------------------\n"); value->append(buf); for (int level = 0; level < config::kNumLevels; level++) { int files = versions_->NumLevelFiles(level); if (stats_[level].micros > 0 || files > 0) { - snprintf( - buf, sizeof(buf), - "%3d %8d %8.0f %9.0f %8.0f %9.0f\n", - level, - files, - versions_->NumLevelBytes(level) / 1048576.0, - stats_[level].micros / 1e6, - stats_[level].bytes_read / 1048576.0, - stats_[level].bytes_written / 1048576.0); + std::snprintf(buf, sizeof(buf), "%3d %8d %8.0f %9.0f %8.0f %9.0f\n", + level, files, versions_->NumLevelBytes(level) / 1048576.0, + stats_[level].micros / 1e6, + stats_[level].bytes_read / 1048576.0, + stats_[level].bytes_written / 1048576.0); value->append(buf); } } @@ -1446,8 +1437,8 @@ bool DBImpl::GetProperty(const Slice& property, std::string* value) { total_usage += imm_->ApproximateMemoryUsage(); } char buf[50]; - snprintf(buf, sizeof(buf), "%llu", - static_cast<unsigned long long>(total_usage)); + std::snprintf(buf, sizeof(buf), "%llu", + static_cast<unsigned long long>(total_usage)); value->append(buf); return true; } @@ -1455,16 +1446,11 @@ bool DBImpl::GetProperty(const Slice& property, std::string* value) { return false; } -void DBImpl::GetApproximateSizes( - const Range* range, int n, - uint64_t* sizes) { +void DBImpl::GetApproximateSizes(const Range* range, int n, uint64_t* sizes) { // TODO(opt): better implementation - Version* v; - { - MutexLock l(&mutex_); - versions_->current()->Ref(); - v = versions_->current(); - } + MutexLock l(&mutex_); + Version* v = versions_->current(); + v->Ref(); for (int i = 0; i < n; i++) { // Convert user_key into a corresponding internal key. @@ -1475,10 +1461,7 @@ void DBImpl::GetApproximateSizes( sizes[i] = (limit >= start ? limit - start : 0); } - { - MutexLock l(&mutex_); - v->Unref(); - } + v->Unref(); } // Default implementations of convenience methods that subclasses of DB @@ -1495,10 +1478,9 @@ Status DB::Delete(const WriteOptions& opt, const Slice& key) { return Write(opt, &batch); } -DB::~DB() { } +DB::~DB() = default; -Status DB::Open(const Options& options, const std::string& dbname, - DB** dbptr) { +Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) { *dbptr = nullptr; DBImpl* impl = new DBImpl(options, dbname); @@ -1528,7 +1510,7 @@ Status DB::Open(const Options& options, const std::string& dbname, s = impl->versions_->LogAndApply(&edit, &impl->mutex_); } if (s.ok()) { - impl->DeleteObsoleteFiles(); + impl->RemoveObsoleteFiles(); impl->MaybeScheduleCompaction(); } impl->mutex_.Unlock(); @@ -1541,8 +1523,7 @@ Status DB::Open(const Options& options, const std::string& dbname, return s; } -Snapshot::~Snapshot() { -} +Snapshot::~Snapshot() = default; Status DestroyDB(const std::string& dbname, const Options& options) { Env* env = options.env; @@ -1562,15 +1543,15 @@ Status DestroyDB(const std::string& dbname, const Options& options) { for (size_t i = 0; i < filenames.size(); i++) { if (ParseFileName(filenames[i], &number, &type) && type != kDBLockFile) { // Lock file will be deleted at end - Status del = env->DeleteFile(dbname + "/" + filenames[i]); + Status del = env->RemoveFile(dbname + "/" + filenames[i]); if (result.ok() && !del.ok()) { result = del; } } } env->UnlockFile(lock); // Ignore error since state is already gone - env->DeleteFile(lockname); - env->DeleteDir(dbname); // Ignore error in case dir contains other files + env->RemoveFile(lockname); + env->RemoveDir(dbname); // Ignore error in case dir contains other files } return result; } diff --git a/db/db_impl.h b/db/db_impl.h index 00e800a..c7b0172 100644 --- a/db/db_impl.h +++ b/db/db_impl.h @@ -5,8 +5,11 @@ #ifndef STORAGE_LEVELDB_DB_DB_IMPL_H_ #define STORAGE_LEVELDB_DB_DB_IMPL_H_ +#include <atomic> #include <deque> #include <set> +#include <string> + #include "db/dbformat.h" #include "db/log_writer.h" #include "db/snapshot.h" @@ -26,21 +29,25 @@ class VersionSet; class DBImpl : public DB { public: DBImpl(const Options& options, const std::string& dbname); - virtual ~DBImpl(); + + DBImpl(const DBImpl&) = delete; + DBImpl& operator=(const DBImpl&) = delete; + + ~DBImpl() override; // Implementations of the DB interface - virtual Status Put(const WriteOptions&, const Slice& key, const Slice& value); - virtual Status Delete(const WriteOptions&, const Slice& key); - virtual Status Write(const WriteOptions& options, WriteBatch* updates); - virtual Status Get(const ReadOptions& options, - const Slice& key, - std::string* value); - virtual Iterator* NewIterator(const ReadOptions&); - virtual const Snapshot* GetSnapshot(); - virtual void ReleaseSnapshot(const Snapshot* snapshot); - virtual bool GetProperty(const Slice& property, std::string* value); - virtual void GetApproximateSizes(const Range* range, int n, uint64_t* sizes); - virtual void CompactRange(const Slice* begin, const Slice* end); + Status Put(const WriteOptions&, const Slice& key, + const Slice& value) override; + Status Delete(const WriteOptions&, const Slice& key) override; + Status Write(const WriteOptions& options, WriteBatch* updates) override; + Status Get(const ReadOptions& options, const Slice& key, + std::string* value) override; + Iterator* NewIterator(const ReadOptions&) override; + const Snapshot* GetSnapshot() override; + void ReleaseSnapshot(const Snapshot* snapshot) override; + bool GetProperty(const Slice& property, std::string* value) override; + void GetApproximateSizes(const Range* range, int n, uint64_t* sizes) override; + void CompactRange(const Slice* begin, const Slice* end) override; // Extra methods (for testing) that are not in the public DB interface @@ -69,6 +76,31 @@ class DBImpl : public DB { struct CompactionState; struct Writer; + // Information for a manual compaction + struct ManualCompaction { + int level; + bool done; + const InternalKey* begin; // null means beginning of key range + const InternalKey* end; // null means end of key range + InternalKey tmp_storage; // Used to keep track of compaction progress + }; + + // Per level compaction stats. stats_[level] stores the stats for + // compactions that produced data for the specified "level". + struct CompactionStats { + CompactionStats() : micros(0), bytes_read(0), bytes_written(0) {} + + void Add(const CompactionStats& c) { + this->micros += c.micros; + this->bytes_read += c.bytes_read; + this->bytes_written += c.bytes_written; + } + + int64_t micros; + int64_t bytes_read; + int64_t bytes_written; + }; + Iterator* NewInternalIterator(const ReadOptions&, SequenceNumber* latest_snapshot, uint32_t* seed); @@ -84,7 +116,7 @@ class DBImpl : public DB { void MaybeIgnoreError(Status* s) const; // Delete any unneeded files and stale in-memory entries. - void DeleteObsoleteFiles() EXCLUSIVE_LOCKS_REQUIRED(mutex_); + void RemoveObsoleteFiles() EXCLUSIVE_LOCKS_REQUIRED(mutex_); // Compact the in-memory write buffer to disk. Switches to a new // log-file/memtable and writes a new descriptor iff successful. @@ -119,6 +151,10 @@ class DBImpl : public DB { Status InstallCompactionResults(CompactionState* compact) EXCLUSIVE_LOCKS_REQUIRED(mutex_); + const Comparator* user_comparator() const { + return internal_comparator_.user_comparator(); + } + // Constant after construction Env* const env_; const InternalKeyComparator internal_comparator_; @@ -136,11 +172,11 @@ class DBImpl : public DB { // State below is protected by mutex_ port::Mutex mutex_; - port::AtomicPointer shutting_down_; + std::atomic<bool> shutting_down_; port::CondVar background_work_finished_signal_ GUARDED_BY(mutex_); MemTable* mem_; MemTable* imm_ GUARDED_BY(mutex_); // Memtable being compacted - port::AtomicPointer has_imm_; // So bg thread can detect non-null imm_ + std::atomic<bool> has_imm_; // So bg thread can detect non-null imm_ WritableFile* logfile_; uint64_t logfile_number_ GUARDED_BY(mutex_); log::Writer* log_; @@ -159,45 +195,14 @@ class DBImpl : public DB { // Has a background compaction been scheduled or is running? bool background_compaction_scheduled_ GUARDED_BY(mutex_); - // Information for a manual compaction - struct ManualCompaction { - int level; - bool done; - const InternalKey* begin; // null means beginning of key range - const InternalKey* end; // null means end of key range - InternalKey tmp_storage; // Used to keep track of compaction progress - }; ManualCompaction* manual_compaction_ GUARDED_BY(mutex_); - VersionSet* const versions_; + VersionSet* const versions_ GUARDED_BY(mutex_); // Have we encountered a background error in paranoid mode? Status bg_error_ GUARDED_BY(mutex_); - // Per level compaction stats. stats_[level] stores the stats for - // compactions that produced data for the specified "level". - struct CompactionStats { - int64_t micros; - int64_t bytes_read; - int64_t bytes_written; - - CompactionStats() : micros(0), bytes_read(0), bytes_written(0) { } - - void Add(const CompactionStats& c) { - this->micros += c.micros; - this->bytes_read += c.bytes_read; - this->bytes_written += c.bytes_written; - } - }; CompactionStats stats_[config::kNumLevels] GUARDED_BY(mutex_); - - // No copying allowed - DBImpl(const DBImpl&); - void operator=(const DBImpl&); - - const Comparator* user_comparator() const { - return internal_comparator_.user_comparator(); - } }; // Sanitize db options. The caller should delete result.info_log if diff --git a/db/db_iter.cc b/db/db_iter.cc index 3b2035e..532c2db 100644 --- a/db/db_iter.cc +++ b/db/db_iter.cc @@ -4,9 +4,9 @@ #include "db/db_iter.h" -#include "db/filename.h" #include "db/db_impl.h" #include "db/dbformat.h" +#include "db/filename.h" #include "leveldb/env.h" #include "leveldb/iterator.h" #include "port/port.h" @@ -21,9 +21,9 @@ static void DumpInternalIter(Iterator* iter) { for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { ParsedInternalKey k; if (!ParseInternalKey(iter->key(), &k)) { - fprintf(stderr, "Corrupt '%s'\n", EscapeString(iter->key()).c_str()); + std::fprintf(stderr, "Corrupt '%s'\n", EscapeString(iter->key()).c_str()); } else { - fprintf(stderr, "@ '%s'\n", k.DebugString().c_str()); + std::fprintf(stderr, "@ '%s'\n", k.DebugString().c_str()); } } } @@ -36,17 +36,14 @@ namespace { // combines multiple entries for the same userkey found in the DB // representation into a single entry while accounting for sequence // numbers, deletion markers, overwrites, etc. -class DBIter: public Iterator { +class DBIter : public Iterator { public: // Which direction is the iterator currently moving? // (1) When moving forward, the internal iterator is positioned at // the exact entry that yields this->key(), this->value() // (2) When moving backwards, the internal iterator is positioned // just before all entries whose user key == this->key(). - enum Direction { - kForward, - kReverse - }; + enum Direction { kForward, kReverse }; DBIter(DBImpl* db, const Comparator* cmp, Iterator* iter, SequenceNumber s, uint32_t seed) @@ -57,21 +54,22 @@ class DBIter: public Iterator { direction_(kForward), valid_(false), rnd_(seed), - bytes_counter_(RandomPeriod()) { - } - virtual ~DBIter() { - delete iter_; - } - virtual bool Valid() const { return valid_; } - virtual Slice key() const { + bytes_until_read_sampling_(RandomCompactionPeriod()) {} + + DBIter(const DBIter&) = delete; + DBIter& operator=(const DBIter&) = delete; + + ~DBIter() override { delete iter_; } + bool Valid() const override { return valid_; } + Slice key() const override { assert(valid_); return (direction_ == kForward) ? ExtractUserKey(iter_->key()) : saved_key_; } - virtual Slice value() const { + Slice value() const override { assert(valid_); return (direction_ == kForward) ? iter_->value() : saved_value_; } - virtual Status status() const { + Status status() const override { if (status_.ok()) { return iter_->status(); } else { @@ -79,11 +77,11 @@ class DBIter: public Iterator { } } - virtual void Next(); - virtual void Prev(); - virtual void Seek(const Slice& target); - virtual void SeekToFirst(); - virtual void SeekToLast(); + void Next() override; + void Prev() override; + void Seek(const Slice& target) override; + void SeekToFirst() override; + void SeekToLast() override; private: void FindNextUserEntry(bool skipping, std::string* skip); @@ -103,38 +101,35 @@ class DBIter: public Iterator { } } - // Pick next gap with average value of config::kReadBytesPeriod. - ssize_t RandomPeriod() { - return rnd_.Uniform(2*config::kReadBytesPeriod); + // Picks the number of bytes that can be read until a compaction is scheduled. + size_t RandomCompactionPeriod() { + return rnd_.Uniform(2 * config::kReadBytesPeriod); } DBImpl* db_; const Comparator* const user_comparator_; Iterator* const iter_; SequenceNumber const sequence_; - Status status_; - std::string saved_key_; // == current key when direction_==kReverse - std::string saved_value_; // == current raw value when direction_==kReverse + std::string saved_key_; // == current key when direction_==kReverse + std::string saved_value_; // == current raw value when direction_==kReverse Direction direction_; bool valid_; - Random rnd_; - ssize_t bytes_counter_; - - // No copying allowed - DBIter(const DBIter&); - void operator=(const DBIter&); + size_t bytes_until_read_sampling_; }; inline bool DBIter::ParseKey(ParsedInternalKey* ikey) { Slice k = iter_->key(); - ssize_t n = k.size() + iter_->value().size(); - bytes_counter_ -= n; - while (bytes_counter_ < 0) { - bytes_counter_ += RandomPeriod(); + + size_t bytes_read = k.size() + iter_->value().size(); + while (bytes_until_read_sampling_ < bytes_read) { + bytes_until_read_sampling_ += RandomCompactionPeriod(); db_->RecordReadSample(k); } + assert(bytes_until_read_sampling_ >= bytes_read); + bytes_until_read_sampling_ -= bytes_read; + if (!ParseInternalKey(k, ikey)) { status_ = Status::Corruption("corrupted internal key in DBIter"); return false; @@ -165,6 +160,15 @@ void DBIter::Next() { } else { // Store in saved_key_ the current key so we skip it below. SaveKey(ExtractUserKey(iter_->key()), &saved_key_); + + // iter_ is pointing to current key. We can now safely move to the next to + // avoid checking current key. + iter_->Next(); + if (!iter_->Valid()) { + valid_ = false; + saved_key_.clear(); + return; + } } FindNextUserEntry(true, &saved_key_); @@ -218,8 +222,8 @@ void DBIter::Prev() { ClearSavedValue(); return; } - if (user_comparator_->Compare(ExtractUserKey(iter_->key()), - saved_key_) < 0) { + if (user_comparator_->Compare(ExtractUserKey(iter_->key()), saved_key_) < + 0) { break; } } @@ -275,8 +279,8 @@ void DBIter::Seek(const Slice& target) { direction_ = kForward; ClearSavedValue(); saved_key_.clear(); - AppendInternalKey( - &saved_key_, ParsedInternalKey(target, sequence_, kValueTypeForSeek)); + AppendInternalKey(&saved_key_, + ParsedInternalKey(target, sequence_, kValueTypeForSeek)); iter_->Seek(saved_key_); if (iter_->Valid()) { FindNextUserEntry(false, &saved_key_ /* temporary storage */); @@ -305,12 +309,9 @@ void DBIter::SeekToLast() { } // anonymous namespace -Iterator* NewDBIterator( - DBImpl* db, - const Comparator* user_key_comparator, - Iterator* internal_iter, - SequenceNumber sequence, - uint32_t seed) { +Iterator* NewDBIterator(DBImpl* db, const Comparator* user_key_comparator, + Iterator* internal_iter, SequenceNumber sequence, + uint32_t seed) { return new DBIter(db, user_key_comparator, internal_iter, sequence, seed); } diff --git a/db/db_iter.h b/db/db_iter.h index 262840e..5977fc8 100644 --- a/db/db_iter.h +++ b/db/db_iter.h @@ -5,9 +5,10 @@ #ifndef STORAGE_LEVELDB_DB_DB_ITER_H_ #define STORAGE_LEVELDB_DB_DB_ITER_H_ -#include <stdint.h> -#include "leveldb/db.h" +#include <cstdint> + #include "db/dbformat.h" +#include "leveldb/db.h" namespace leveldb { @@ -16,10 +17,8 @@ class DBImpl; // Return a new iterator that converts internal keys (yielded by // "*internal_iter") that were live at the specified "sequence" number // into appropriate user keys. -Iterator* NewDBIterator(DBImpl* db, - const Comparator* user_key_comparator, - Iterator* internal_iter, - SequenceNumber sequence, +Iterator* NewDBIterator(DBImpl* db, const Comparator* user_key_comparator, + Iterator* internal_iter, SequenceNumber sequence, uint32_t seed); } // namespace leveldb diff --git a/db/db_test.cc b/db/db_test.cc index 878b7d4..9bd6e14 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -3,20 +3,25 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "leveldb/db.h" -#include "leveldb/filter_policy.h" + +#include <atomic> +#include <cinttypes> +#include <string> + +#include "gtest/gtest.h" #include "db/db_impl.h" #include "db/filename.h" #include "db/version_set.h" #include "db/write_batch_internal.h" #include "leveldb/cache.h" #include "leveldb/env.h" +#include "leveldb/filter_policy.h" #include "leveldb/table.h" #include "port/port.h" #include "port/thread_annotations.h" #include "util/hash.h" #include "util/logging.h" #include "util/mutexlock.h" -#include "util/testharness.h" #include "util/testutil.h" namespace leveldb { @@ -28,22 +33,17 @@ static std::string RandomString(Random* rnd, int len) { } static std::string RandomKey(Random* rnd) { - int len = (rnd->OneIn(3) - ? 1 // Short sometimes to encourage collisions - : (rnd->OneIn(100) ? rnd->Skewed(10) : rnd->Uniform(10))); + int len = + (rnd->OneIn(3) ? 1 // Short sometimes to encourage collisions + : (rnd->OneIn(100) ? rnd->Skewed(10) : rnd->Uniform(10))); return test::RandomKey(rnd, len); } namespace { class AtomicCounter { - private: - port::Mutex mu_; - int count_ GUARDED_BY(mu_); public: - AtomicCounter() : count_(0) { } - void Increment() { - IncrementBy(1); - } + AtomicCounter() : count_(0) {} + void Increment() { IncrementBy(1); } void IncrementBy(int count) LOCKS_EXCLUDED(mu_) { MutexLock l(&mu_); count_ += count; @@ -56,12 +56,16 @@ class AtomicCounter { MutexLock l(&mu_); count_ = 0; } + + private: + port::Mutex mu_; + int count_ GUARDED_BY(mu_); }; void DelayMilliseconds(int millis) { Env::Default()->SleepForMicroseconds(millis * 1000); } -} +} // namespace // Test Env to override default Env behavior for testing. class TestEnv : public EnvWrapper { @@ -93,54 +97,51 @@ class TestEnv : public EnvWrapper { bool ignore_dot_files_; }; -// Special Env used to delay background operations +// Special Env used to delay background operations. class SpecialEnv : public EnvWrapper { public: // sstable/log Sync() calls are blocked while this pointer is non-null. - port::AtomicPointer delay_data_sync_; + std::atomic<bool> delay_data_sync_; // sstable/log Sync() calls return an error. - port::AtomicPointer data_sync_error_; + std::atomic<bool> data_sync_error_; // Simulate no-space errors while this pointer is non-null. - port::AtomicPointer no_space_; + std::atomic<bool> no_space_; // Simulate non-writable file system while this pointer is non-null. - port::AtomicPointer non_writable_; + std::atomic<bool> non_writable_; // Force sync of manifest files to fail while this pointer is non-null. - port::AtomicPointer manifest_sync_error_; + std::atomic<bool> manifest_sync_error_; // Force write to manifest files to fail while this pointer is non-null. - port::AtomicPointer manifest_write_error_; + std::atomic<bool> manifest_write_error_; bool count_random_reads_; AtomicCounter random_read_counter_; - explicit SpecialEnv(Env* base) : EnvWrapper(base) { - delay_data_sync_.Release_Store(nullptr); - data_sync_error_.Release_Store(nullptr); - no_space_.Release_Store(nullptr); - non_writable_.Release_Store(nullptr); - count_random_reads_ = false; - manifest_sync_error_.Release_Store(nullptr); - manifest_write_error_.Release_Store(nullptr); - } + explicit SpecialEnv(Env* base) + : EnvWrapper(base), + delay_data_sync_(false), + data_sync_error_(false), + no_space_(false), + non_writable_(false), + manifest_sync_error_(false), + manifest_write_error_(false), + count_random_reads_(false) {} Status NewWritableFile(const std::string& f, WritableFile** r) { class DataFile : public WritableFile { private: - SpecialEnv* env_; - WritableFile* base_; + SpecialEnv* const env_; + WritableFile* const base_; public: - DataFile(SpecialEnv* env, WritableFile* base) - : env_(env), - base_(base) { - } + DataFile(SpecialEnv* env, WritableFile* base) : env_(env), base_(base) {} ~DataFile() { delete base_; } Status Append(const Slice& data) { - if (env_->no_space_.Acquire_Load() != nullptr) { + if (env_->no_space_.load(std::memory_order_acquire)) { // Drop writes on the floor return Status::OK(); } else { @@ -150,10 +151,10 @@ class SpecialEnv : public EnvWrapper { Status Close() { return base_->Close(); } Status Flush() { return base_->Flush(); } Status Sync() { - if (env_->data_sync_error_.Acquire_Load() != nullptr) { + if (env_->data_sync_error_.load(std::memory_order_acquire)) { return Status::IOError("simulated data sync error"); } - while (env_->delay_data_sync_.Acquire_Load() != nullptr) { + while (env_->delay_data_sync_.load(std::memory_order_acquire)) { DelayMilliseconds(100); } return base_->Sync(); @@ -163,11 +164,12 @@ class SpecialEnv : public EnvWrapper { private: SpecialEnv* env_; WritableFile* base_; + public: - ManifestFile(SpecialEnv* env, WritableFile* b) : env_(env), base_(b) { } + ManifestFile(SpecialEnv* env, WritableFile* b) : env_(env), base_(b) {} ~ManifestFile() { delete base_; } Status Append(const Slice& data) { - if (env_->manifest_write_error_.Acquire_Load() != nullptr) { + if (env_->manifest_write_error_.load(std::memory_order_acquire)) { return Status::IOError("simulated writer error"); } else { return base_->Append(data); @@ -176,7 +178,7 @@ class SpecialEnv : public EnvWrapper { Status Close() { return base_->Close(); } Status Flush() { return base_->Flush(); } Status Sync() { - if (env_->manifest_sync_error_.Acquire_Load() != nullptr) { + if (env_->manifest_sync_error_.load(std::memory_order_acquire)) { return Status::IOError("simulated sync error"); } else { return base_->Sync(); @@ -184,7 +186,7 @@ class SpecialEnv : public EnvWrapper { } }; - if (non_writable_.Acquire_Load() != nullptr) { + if (non_writable_.load(std::memory_order_acquire)) { return Status::IOError("simulated write error"); } @@ -205,13 +207,13 @@ class SpecialEnv : public EnvWrapper { private: RandomAccessFile* target_; AtomicCounter* counter_; + public: CountingFile(RandomAccessFile* target, AtomicCounter* counter) - : target_(target), counter_(counter) { - } - virtual ~CountingFile() { delete target_; } - virtual Status Read(uint64_t offset, size_t n, Slice* result, - char* scratch) const { + : target_(target), counter_(counter) {} + ~CountingFile() override { delete target_; } + Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const override { counter_->Increment(); return target_->Read(offset, n, result, scratch); } @@ -225,20 +227,7 @@ class SpecialEnv : public EnvWrapper { } }; -class DBTest { - private: - const FilterPolicy* filter_policy_; - - // Sequence of option configurations to try - enum OptionConfig { - kDefault, - kReuse, - kFilter, - kUncompressed, - kEnd - }; - int option_config_; - +class DBTest : public testing::Test { public: std::string dbname_; SpecialEnv* env_; @@ -246,10 +235,9 @@ class DBTest { Options last_options_; - DBTest() : option_config_(kDefault), - env_(new SpecialEnv(Env::Default())) { + DBTest() : env_(new SpecialEnv(Env::Default())), option_config_(kDefault) { filter_policy_ = NewBloomFilterPolicy(10); - dbname_ = test::TmpDir() + "/db_test"; + dbname_ = testing::TempDir() + "db_test"; DestroyDB(dbname_, Options()); db_ = nullptr; Reopen(); @@ -294,12 +282,10 @@ class DBTest { return options; } - DBImpl* dbfull() { - return reinterpret_cast<DBImpl*>(db_); - } + DBImpl* dbfull() { return reinterpret_cast<DBImpl*>(db_); } void Reopen(Options* options = nullptr) { - ASSERT_OK(TryReopen(options)); + ASSERT_LEVELDB_OK(TryReopen(options)); } void Close() { @@ -311,7 +297,7 @@ class DBTest { delete db_; db_ = nullptr; DestroyDB(dbname_, Options()); - ASSERT_OK(TryReopen(options)); + ASSERT_LEVELDB_OK(TryReopen(options)); } Status TryReopen(Options* options) { @@ -333,9 +319,7 @@ class DBTest { return db_->Put(WriteOptions(), k, v); } - Status Delete(const std::string& k) { - return db_->Delete(WriteOptions(), k); - } + Status Delete(const std::string& k) { return db_->Delete(WriteOptions(), k); } std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) { ReadOptions options; @@ -367,11 +351,11 @@ class DBTest { // Check reverse iteration results are the reverse of forward results size_t matched = 0; for (iter->SeekToLast(); iter->Valid(); iter->Prev()) { - ASSERT_LT(matched, forward.size()); - ASSERT_EQ(IterStatus(iter), forward[forward.size() - matched - 1]); + EXPECT_LT(matched, forward.size()); + EXPECT_EQ(IterStatus(iter), forward[forward.size() - matched - 1]); matched++; } - ASSERT_EQ(matched, forward.size()); + EXPECT_EQ(matched, forward.size()); delete iter; return result; @@ -421,10 +405,9 @@ class DBTest { int NumTableFilesAtLevel(int level) { std::string property; - ASSERT_TRUE( - db_->GetProperty("leveldb.num-files-at-level" + NumberToString(level), - &property)); - return atoi(property.c_str()); + EXPECT_TRUE(db_->GetProperty( + "leveldb.num-files-at-level" + NumberToString(level), &property)); + return std::stoi(property); } int TotalTableFiles() { @@ -442,7 +425,7 @@ class DBTest { for (int level = 0; level < config::kNumLevels; level++) { int f = NumTableFilesAtLevel(level); char buf[100]; - snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f); + std::snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f); result += buf; if (f > 0) { last_non_zero_offset = result.size(); @@ -470,11 +453,12 @@ class DBTest { } // Do n memtable compactions, each of which produces an sstable - // covering the range [small,large]. - void MakeTables(int n, const std::string& small, const std::string& large) { + // covering the range [small_key,large_key]. + void MakeTables(int n, const std::string& small_key, + const std::string& large_key) { for (int i = 0; i < n; i++) { - Put(small, "begin"); - Put(large, "end"); + Put(small_key, "begin"); + Put(large_key, "end"); dbfull()->TEST_CompactMemTable(); } } @@ -486,14 +470,14 @@ class DBTest { } void DumpFileCounts(const char* label) { - fprintf(stderr, "---\n%s:\n", label); - fprintf(stderr, "maxoverlap: %lld\n", - static_cast<long long>( - dbfull()->TEST_MaxNextLevelOverlappingBytes())); + std::fprintf(stderr, "---\n%s:\n", label); + std::fprintf( + stderr, "maxoverlap: %lld\n", + static_cast<long long>(dbfull()->TEST_MaxNextLevelOverlappingBytes())); for (int level = 0; level < config::kNumLevels; level++) { int num = NumTableFilesAtLevel(level); if (num > 0) { - fprintf(stderr, " level %3d : %d files\n", level, num); + std::fprintf(stderr, " level %3d : %d files\n", level, num); } } } @@ -516,12 +500,12 @@ class DBTest { bool DeleteAnSSTFile() { std::vector<std::string> filenames; - ASSERT_OK(env_->GetChildren(dbname_, &filenames)); + EXPECT_LEVELDB_OK(env_->GetChildren(dbname_, &filenames)); uint64_t number; FileType type; for (size_t i = 0; i < filenames.size(); i++) { if (ParseFileName(filenames[i], &number, &type) && type == kTableFile) { - ASSERT_OK(env_->DeleteFile(TableFileName(dbname_, number))); + EXPECT_LEVELDB_OK(env_->RemoveFile(TableFileName(dbname_, number))); return true; } } @@ -531,7 +515,7 @@ class DBTest { // Returns number of files renamed. int RenameLDBToSST() { std::vector<std::string> filenames; - ASSERT_OK(env_->GetChildren(dbname_, &filenames)); + EXPECT_LEVELDB_OK(env_->GetChildren(dbname_, &filenames)); uint64_t number; FileType type; int files_renamed = 0; @@ -539,88 +523,117 @@ class DBTest { if (ParseFileName(filenames[i], &number, &type) && type == kTableFile) { const std::string from = TableFileName(dbname_, number); const std::string to = SSTTableFileName(dbname_, number); - ASSERT_OK(env_->RenameFile(from, to)); + EXPECT_LEVELDB_OK(env_->RenameFile(from, to)); files_renamed++; } } return files_renamed; } + + private: + // Sequence of option configurations to try + enum OptionConfig { kDefault, kReuse, kFilter, kUncompressed, kEnd }; + + const FilterPolicy* filter_policy_; + int option_config_; }; -TEST(DBTest, Empty) { +TEST_F(DBTest, Empty) { do { ASSERT_TRUE(db_ != nullptr); ASSERT_EQ("NOT_FOUND", Get("foo")); } while (ChangeOptions()); } -TEST(DBTest, ReadWrite) { +TEST_F(DBTest, EmptyKey) { + do { + ASSERT_LEVELDB_OK(Put("", "v1")); + ASSERT_EQ("v1", Get("")); + ASSERT_LEVELDB_OK(Put("", "v2")); + ASSERT_EQ("v2", Get("")); + } while (ChangeOptions()); +} + +TEST_F(DBTest, EmptyValue) { do { - ASSERT_OK(Put("foo", "v1")); + ASSERT_LEVELDB_OK(Put("key", "v1")); + ASSERT_EQ("v1", Get("key")); + ASSERT_LEVELDB_OK(Put("key", "")); + ASSERT_EQ("", Get("key")); + ASSERT_LEVELDB_OK(Put("key", "v2")); + ASSERT_EQ("v2", Get("key")); + } while (ChangeOptions()); +} + +TEST_F(DBTest, ReadWrite) { + do { + ASSERT_LEVELDB_OK(Put("foo", "v1")); ASSERT_EQ("v1", Get("foo")); - ASSERT_OK(Put("bar", "v2")); - ASSERT_OK(Put("foo", "v3")); + ASSERT_LEVELDB_OK(Put("bar", "v2")); + ASSERT_LEVELDB_OK(Put("foo", "v3")); ASSERT_EQ("v3", Get("foo")); ASSERT_EQ("v2", Get("bar")); } while (ChangeOptions()); } -TEST(DBTest, PutDeleteGet) { +TEST_F(DBTest, PutDeleteGet) { do { - ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1")); + ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v1")); ASSERT_EQ("v1", Get("foo")); - ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2")); + ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), "foo", "v2")); ASSERT_EQ("v2", Get("foo")); - ASSERT_OK(db_->Delete(WriteOptions(), "foo")); + ASSERT_LEVELDB_OK(db_->Delete(WriteOptions(), "foo")); ASSERT_EQ("NOT_FOUND", Get("foo")); } while (ChangeOptions()); } -TEST(DBTest, GetFromImmutableLayer) { +TEST_F(DBTest, GetFromImmutableLayer) { do { Options options = CurrentOptions(); options.env = env_; options.write_buffer_size = 100000; // Small write buffer Reopen(&options); - ASSERT_OK(Put("foo", "v1")); + ASSERT_LEVELDB_OK(Put("foo", "v1")); ASSERT_EQ("v1", Get("foo")); - env_->delay_data_sync_.Release_Store(env_); // Block sync calls - Put("k1", std::string(100000, 'x')); // Fill memtable - Put("k2", std::string(100000, 'y')); // Trigger compaction + // Block sync calls. + env_->delay_data_sync_.store(true, std::memory_order_release); + Put("k1", std::string(100000, 'x')); // Fill memtable. + Put("k2", std::string(100000, 'y')); // Trigger compaction. ASSERT_EQ("v1", Get("foo")); - env_->delay_data_sync_.Release_Store(nullptr); // Release sync calls + // Release sync calls. + env_->delay_data_sync_.store(false, std::memory_order_release); } while (ChangeOptions()); } -TEST(DBTest, GetFromVersions) { +TEST_F(DBTest, GetFromVersions) { do { - ASSERT_OK(Put("foo", "v1")); + ASSERT_LEVELDB_OK(Put("foo", "v1")); dbfull()->TEST_CompactMemTable(); ASSERT_EQ("v1", Get("foo")); } while (ChangeOptions()); } -TEST(DBTest, GetMemUsage) { +TEST_F(DBTest, GetMemUsage) { do { - ASSERT_OK(Put("foo", "v1")); + ASSERT_LEVELDB_OK(Put("foo", "v1")); std::string val; ASSERT_TRUE(db_->GetProperty("leveldb.approximate-memory-usage", &val)); - int mem_usage = atoi(val.c_str()); + int mem_usage = std::stoi(val); ASSERT_GT(mem_usage, 0); - ASSERT_LT(mem_usage, 5*1024*1024); + ASSERT_LT(mem_usage, 5 * 1024 * 1024); } while (ChangeOptions()); } -TEST(DBTest, GetSnapshot) { +TEST_F(DBTest, GetSnapshot) { do { // Try with both a short key and a long key for (int i = 0; i < 2; i++) { std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x'); - ASSERT_OK(Put(key, "v1")); + ASSERT_LEVELDB_OK(Put(key, "v1")); const Snapshot* s1 = db_->GetSnapshot(); - ASSERT_OK(Put(key, "v2")); + ASSERT_LEVELDB_OK(Put(key, "v2")); ASSERT_EQ("v2", Get(key)); ASSERT_EQ("v1", Get(key, s1)); dbfull()->TEST_CompactMemTable(); @@ -631,16 +644,16 @@ TEST(DBTest, GetSnapshot) { } while (ChangeOptions()); } -TEST(DBTest, GetIdenticalSnapshots) { +TEST_F(DBTest, GetIdenticalSnapshots) { do { // Try with both a short key and a long key for (int i = 0; i < 2; i++) { std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x'); - ASSERT_OK(Put(key, "v1")); + ASSERT_LEVELDB_OK(Put(key, "v1")); const Snapshot* s1 = db_->GetSnapshot(); const Snapshot* s2 = db_->GetSnapshot(); const Snapshot* s3 = db_->GetSnapshot(); - ASSERT_OK(Put(key, "v2")); + ASSERT_LEVELDB_OK(Put(key, "v2")); ASSERT_EQ("v2", Get(key)); ASSERT_EQ("v1", Get(key, s1)); ASSERT_EQ("v1", Get(key, s2)); @@ -656,13 +669,13 @@ TEST(DBTest, GetIdenticalSnapshots) { } while (ChangeOptions()); } -TEST(DBTest, IterateOverEmptySnapshot) { +TEST_F(DBTest, IterateOverEmptySnapshot) { do { const Snapshot* snapshot = db_->GetSnapshot(); ReadOptions read_options; read_options.snapshot = snapshot; - ASSERT_OK(Put("foo", "v1")); - ASSERT_OK(Put("foo", "v2")); + ASSERT_LEVELDB_OK(Put("foo", "v1")); + ASSERT_LEVELDB_OK(Put("foo", "v2")); Iterator* iterator1 = db_->NewIterator(read_options); iterator1->SeekToFirst(); @@ -680,41 +693,41 @@ TEST(DBTest, IterateOverEmptySnapshot) { } while (ChangeOptions()); } -TEST(DBTest, GetLevel0Ordering) { +TEST_F(DBTest, GetLevel0Ordering) { do { // Check that we process level-0 files in correct order. The code // below generates two level-0 files where the earlier one comes // before the later one in the level-0 file list since the earlier // one has a smaller "smallest" key. - ASSERT_OK(Put("bar", "b")); - ASSERT_OK(Put("foo", "v1")); + ASSERT_LEVELDB_OK(Put("bar", "b")); + ASSERT_LEVELDB_OK(Put("foo", "v1")); dbfull()->TEST_CompactMemTable(); - ASSERT_OK(Put("foo", "v2")); + ASSERT_LEVELDB_OK(Put("foo", "v2")); dbfull()->TEST_CompactMemTable(); ASSERT_EQ("v2", Get("foo")); } while (ChangeOptions()); } -TEST(DBTest, GetOrderedByLevels) { +TEST_F(DBTest, GetOrderedByLevels) { do { - ASSERT_OK(Put("foo", "v1")); + ASSERT_LEVELDB_OK(Put("foo", "v1")); Compact("a", "z"); ASSERT_EQ("v1", Get("foo")); - ASSERT_OK(Put("foo", "v2")); + ASSERT_LEVELDB_OK(Put("foo", "v2")); ASSERT_EQ("v2", Get("foo")); dbfull()->TEST_CompactMemTable(); ASSERT_EQ("v2", Get("foo")); } while (ChangeOptions()); } -TEST(DBTest, GetPicksCorrectFile) { +TEST_F(DBTest, GetPicksCorrectFile) { do { // Arrange to have multiple files in a non-level-0 level. - ASSERT_OK(Put("a", "va")); + ASSERT_LEVELDB_OK(Put("a", "va")); Compact("a", "b"); - ASSERT_OK(Put("x", "vx")); + ASSERT_LEVELDB_OK(Put("x", "vx")); Compact("x", "y"); - ASSERT_OK(Put("f", "vf")); + ASSERT_LEVELDB_OK(Put("f", "vf")); Compact("f", "g"); ASSERT_EQ("va", Get("a")); ASSERT_EQ("vf", Get("f")); @@ -722,7 +735,7 @@ TEST(DBTest, GetPicksCorrectFile) { } while (ChangeOptions()); } -TEST(DBTest, GetEncountersEmptyLevel) { +TEST_F(DBTest, GetEncountersEmptyLevel) { do { // Arrange for the following to happen: // * sstable A in level 0 @@ -734,8 +747,7 @@ TEST(DBTest, GetEncountersEmptyLevel) { // Step 1: First place sstables in levels 0 and 2 int compaction_count = 0; - while (NumTableFilesAtLevel(0) == 0 || - NumTableFilesAtLevel(2) == 0) { + while (NumTableFilesAtLevel(0) == 0 || NumTableFilesAtLevel(2) == 0) { ASSERT_LE(compaction_count, 100) << "could not fill levels 0 and 2"; compaction_count++; Put("a", "begin"); @@ -761,7 +773,7 @@ TEST(DBTest, GetEncountersEmptyLevel) { } while (ChangeOptions()); } -TEST(DBTest, IterEmpty) { +TEST_F(DBTest, IterEmpty) { Iterator* iter = db_->NewIterator(ReadOptions()); iter->SeekToFirst(); @@ -776,8 +788,8 @@ TEST(DBTest, IterEmpty) { delete iter; } -TEST(DBTest, IterSingle) { - ASSERT_OK(Put("a", "va")); +TEST_F(DBTest, IterSingle) { + ASSERT_LEVELDB_OK(Put("a", "va")); Iterator* iter = db_->NewIterator(ReadOptions()); iter->SeekToFirst(); @@ -814,10 +826,10 @@ TEST(DBTest, IterSingle) { delete iter; } -TEST(DBTest, IterMulti) { - ASSERT_OK(Put("a", "va")); - ASSERT_OK(Put("b", "vb")); - ASSERT_OK(Put("c", "vc")); +TEST_F(DBTest, IterMulti) { + ASSERT_LEVELDB_OK(Put("a", "va")); + ASSERT_LEVELDB_OK(Put("b", "vb")); + ASSERT_LEVELDB_OK(Put("c", "vc")); Iterator* iter = db_->NewIterator(ReadOptions()); iter->SeekToFirst(); @@ -872,11 +884,11 @@ TEST(DBTest, IterMulti) { ASSERT_EQ(IterStatus(iter), "b->vb"); // Make sure iter stays at snapshot - ASSERT_OK(Put("a", "va2")); - ASSERT_OK(Put("a2", "va3")); - ASSERT_OK(Put("b", "vb2")); - ASSERT_OK(Put("c", "vc2")); - ASSERT_OK(Delete("b")); + ASSERT_LEVELDB_OK(Put("a", "va2")); + ASSERT_LEVELDB_OK(Put("a2", "va3")); + ASSERT_LEVELDB_OK(Put("b", "vb2")); + ASSERT_LEVELDB_OK(Put("c", "vc2")); + ASSERT_LEVELDB_OK(Delete("b")); iter->SeekToFirst(); ASSERT_EQ(IterStatus(iter), "a->va"); iter->Next(); @@ -897,12 +909,12 @@ TEST(DBTest, IterMulti) { delete iter; } -TEST(DBTest, IterSmallAndLargeMix) { - ASSERT_OK(Put("a", "va")); - ASSERT_OK(Put("b", std::string(100000, 'b'))); - ASSERT_OK(Put("c", "vc")); - ASSERT_OK(Put("d", std::string(100000, 'd'))); - ASSERT_OK(Put("e", std::string(100000, 'e'))); +TEST_F(DBTest, IterSmallAndLargeMix) { + ASSERT_LEVELDB_OK(Put("a", "va")); + ASSERT_LEVELDB_OK(Put("b", std::string(100000, 'b'))); + ASSERT_LEVELDB_OK(Put("c", "vc")); + ASSERT_LEVELDB_OK(Put("d", std::string(100000, 'd'))); + ASSERT_LEVELDB_OK(Put("e", std::string(100000, 'e'))); Iterator* iter = db_->NewIterator(ReadOptions()); @@ -935,12 +947,30 @@ TEST(DBTest, IterSmallAndLargeMix) { delete iter; } -TEST(DBTest, IterMultiWithDelete) { +TEST_F(DBTest, IterMultiWithDelete) { + do { + ASSERT_LEVELDB_OK(Put("a", "va")); + ASSERT_LEVELDB_OK(Put("b", "vb")); + ASSERT_LEVELDB_OK(Put("c", "vc")); + ASSERT_LEVELDB_OK(Delete("b")); + ASSERT_EQ("NOT_FOUND", Get("b")); + + Iterator* iter = db_->NewIterator(ReadOptions()); + iter->Seek("c"); + ASSERT_EQ(IterStatus(iter), "c->vc"); + iter->Prev(); + ASSERT_EQ(IterStatus(iter), "a->va"); + delete iter; + } while (ChangeOptions()); +} + +TEST_F(DBTest, IterMultiWithDeleteAndCompaction) { do { - ASSERT_OK(Put("a", "va")); - ASSERT_OK(Put("b", "vb")); - ASSERT_OK(Put("c", "vc")); - ASSERT_OK(Delete("b")); + ASSERT_LEVELDB_OK(Put("b", "vb")); + ASSERT_LEVELDB_OK(Put("c", "vc")); + ASSERT_LEVELDB_OK(Put("a", "va")); + dbfull()->TEST_CompactMemTable(); + ASSERT_LEVELDB_OK(Delete("b")); ASSERT_EQ("NOT_FOUND", Get("b")); Iterator* iter = db_->NewIterator(ReadOptions()); @@ -948,39 +978,41 @@ TEST(DBTest, IterMultiWithDelete) { ASSERT_EQ(IterStatus(iter), "c->vc"); iter->Prev(); ASSERT_EQ(IterStatus(iter), "a->va"); + iter->Seek("b"); + ASSERT_EQ(IterStatus(iter), "c->vc"); delete iter; } while (ChangeOptions()); } -TEST(DBTest, Recover) { +TEST_F(DBTest, Recover) { do { - ASSERT_OK(Put("foo", "v1")); - ASSERT_OK(Put("baz", "v5")); + ASSERT_LEVELDB_OK(Put("foo", "v1")); + ASSERT_LEVELDB_OK(Put("baz", "v5")); Reopen(); ASSERT_EQ("v1", Get("foo")); ASSERT_EQ("v1", Get("foo")); ASSERT_EQ("v5", Get("baz")); - ASSERT_OK(Put("bar", "v2")); - ASSERT_OK(Put("foo", "v3")); + ASSERT_LEVELDB_OK(Put("bar", "v2")); + ASSERT_LEVELDB_OK(Put("foo", "v3")); Reopen(); ASSERT_EQ("v3", Get("foo")); - ASSERT_OK(Put("foo", "v4")); + ASSERT_LEVELDB_OK(Put("foo", "v4")); ASSERT_EQ("v4", Get("foo")); ASSERT_EQ("v2", Get("bar")); ASSERT_EQ("v5", Get("baz")); } while (ChangeOptions()); } -TEST(DBTest, RecoveryWithEmptyLog) { +TEST_F(DBTest, RecoveryWithEmptyLog) { do { - ASSERT_OK(Put("foo", "v1")); - ASSERT_OK(Put("foo", "v2")); + ASSERT_LEVELDB_OK(Put("foo", "v1")); + ASSERT_LEVELDB_OK(Put("foo", "v2")); Reopen(); Reopen(); - ASSERT_OK(Put("foo", "v3")); + ASSERT_LEVELDB_OK(Put("foo", "v3")); Reopen(); ASSERT_EQ("v3", Get("foo")); } while (ChangeOptions()); @@ -988,7 +1020,7 @@ TEST(DBTest, RecoveryWithEmptyLog) { // Check that writes done during a memtable compaction are recovered // if the database is shutdown during the memtable compaction. -TEST(DBTest, RecoverDuringMemtableCompaction) { +TEST_F(DBTest, RecoverDuringMemtableCompaction) { do { Options options = CurrentOptions(); options.env = env_; @@ -996,10 +1028,12 @@ TEST(DBTest, RecoverDuringMemtableCompaction) { Reopen(&options); // Trigger a long memtable compaction and reopen the database during it - ASSERT_OK(Put("foo", "v1")); // Goes to 1st log file - ASSERT_OK(Put("big1", std::string(10000000, 'x'))); // Fills memtable - ASSERT_OK(Put("big2", std::string(1000, 'y'))); // Triggers compaction - ASSERT_OK(Put("bar", "v2")); // Goes to new log file + ASSERT_LEVELDB_OK(Put("foo", "v1")); // Goes to 1st log file + ASSERT_LEVELDB_OK( + Put("big1", std::string(10000000, 'x'))); // Fills memtable + ASSERT_LEVELDB_OK( + Put("big2", std::string(1000, 'y'))); // Triggers compaction + ASSERT_LEVELDB_OK(Put("bar", "v2")); // Goes to new log file Reopen(&options); ASSERT_EQ("v1", Get("foo")); @@ -1011,11 +1045,11 @@ TEST(DBTest, RecoverDuringMemtableCompaction) { static std::string Key(int i) { char buf[100]; - snprintf(buf, sizeof(buf), "key%06d", i); + std::snprintf(buf, sizeof(buf), "key%06d", i); return std::string(buf); } -TEST(DBTest, MinorCompactionsHappen) { +TEST_F(DBTest, MinorCompactionsHappen) { Options options = CurrentOptions(); options.write_buffer_size = 10000; Reopen(&options); @@ -1024,7 +1058,7 @@ TEST(DBTest, MinorCompactionsHappen) { int starting_num_tables = TotalTableFiles(); for (int i = 0; i < N; i++) { - ASSERT_OK(Put(Key(i), Key(i) + std::string(1000, 'v'))); + ASSERT_LEVELDB_OK(Put(Key(i), Key(i) + std::string(1000, 'v'))); } int ending_num_tables = TotalTableFiles(); ASSERT_GT(ending_num_tables, starting_num_tables); @@ -1040,14 +1074,14 @@ TEST(DBTest, MinorCompactionsHappen) { } } -TEST(DBTest, RecoverWithLargeLog) { +TEST_F(DBTest, RecoverWithLargeLog) { { Options options = CurrentOptions(); Reopen(&options); - ASSERT_OK(Put("big1", std::string(200000, '1'))); - ASSERT_OK(Put("big2", std::string(200000, '2'))); - ASSERT_OK(Put("small3", std::string(10, '3'))); - ASSERT_OK(Put("small4", std::string(10, '4'))); + ASSERT_LEVELDB_OK(Put("big1", std::string(200000, '1'))); + ASSERT_LEVELDB_OK(Put("big2", std::string(200000, '2'))); + ASSERT_LEVELDB_OK(Put("small3", std::string(10, '3'))); + ASSERT_LEVELDB_OK(Put("small4", std::string(10, '4'))); ASSERT_EQ(NumTableFilesAtLevel(0), 0); } @@ -1064,9 +1098,9 @@ TEST(DBTest, RecoverWithLargeLog) { ASSERT_GT(NumTableFilesAtLevel(0), 1); } -TEST(DBTest, CompactionsGenerateMultipleFiles) { +TEST_F(DBTest, CompactionsGenerateMultipleFiles) { Options options = CurrentOptions(); - options.write_buffer_size = 100000000; // Large write buffer + options.write_buffer_size = 100000000; // Large write buffer Reopen(&options); Random rnd(301); @@ -1076,7 +1110,7 @@ TEST(DBTest, CompactionsGenerateMultipleFiles) { std::vector<std::string> values; for (int i = 0; i < 80; i++) { values.push_back(RandomString(&rnd, 100000)); - ASSERT_OK(Put(Key(i), values[i])); + ASSERT_LEVELDB_OK(Put(Key(i), values[i])); } // Reopening moves updates to level-0 @@ -1090,7 +1124,7 @@ TEST(DBTest, CompactionsGenerateMultipleFiles) { } } -TEST(DBTest, RepeatedWritesToSameKey) { +TEST_F(DBTest, RepeatedWritesToSameKey) { Options options = CurrentOptions(); options.env = env_; options.write_buffer_size = 100000; // Small write buffer @@ -1105,11 +1139,11 @@ TEST(DBTest, RepeatedWritesToSameKey) { for (int i = 0; i < 5 * kMaxFiles; i++) { Put("key", value); ASSERT_LE(TotalTableFiles(), kMaxFiles); - fprintf(stderr, "after %d: %d files\n", int(i+1), TotalTableFiles()); + std::fprintf(stderr, "after %d: %d files\n", i + 1, TotalTableFiles()); } } -TEST(DBTest, SparseMerge) { +TEST_F(DBTest, SparseMerge) { Options options = CurrentOptions(); options.compression = kNoCompression; Reopen(&options); @@ -1127,7 +1161,7 @@ TEST(DBTest, SparseMerge) { // Write approximately 100MB of "B" values for (int i = 0; i < 100000; i++) { char key[100]; - snprintf(key, sizeof(key), "B%010d", i); + std::snprintf(key, sizeof(key), "B%010d", i); Put(key, value); } Put("C", "vc"); @@ -1135,35 +1169,34 @@ TEST(DBTest, SparseMerge) { dbfull()->TEST_CompactRange(0, nullptr, nullptr); // Make sparse update - Put("A", "va2"); + Put("A", "va2"); Put("B100", "bvalue2"); - Put("C", "vc2"); + Put("C", "vc2"); dbfull()->TEST_CompactMemTable(); // Compactions should not cause us to create a situation where // a file overlaps too much data at the next level. - ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576); + ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20 * 1048576); dbfull()->TEST_CompactRange(0, nullptr, nullptr); - ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576); + ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20 * 1048576); dbfull()->TEST_CompactRange(1, nullptr, nullptr); - ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576); + ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20 * 1048576); } static bool Between(uint64_t val, uint64_t low, uint64_t high) { bool result = (val >= low) && (val <= high); if (!result) { - fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n", - (unsigned long long)(val), - (unsigned long long)(low), - (unsigned long long)(high)); + std::fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n", + (unsigned long long)(val), (unsigned long long)(low), + (unsigned long long)(high)); } return result; } -TEST(DBTest, ApproximateSizes) { +TEST_F(DBTest, ApproximateSizes) { do { Options options = CurrentOptions(); - options.write_buffer_size = 100000000; // Large write buffer + options.write_buffer_size = 100000000; // Large write buffer options.compression = kNoCompression; DestroyAndReopen(); @@ -1178,7 +1211,7 @@ TEST(DBTest, ApproximateSizes) { static const int S2 = 105000; // Allow some expansion from metadata Random rnd(301); for (int i = 0; i < N; i++) { - ASSERT_OK(Put(Key(i), RandomString(&rnd, S1))); + ASSERT_LEVELDB_OK(Put(Key(i), RandomString(&rnd, S1))); } // 0 because GetApproximateSizes() does not account for memtable space @@ -1198,12 +1231,13 @@ TEST(DBTest, ApproximateSizes) { for (int compact_start = 0; compact_start < N; compact_start += 10) { for (int i = 0; i < N; i += 10) { - ASSERT_TRUE(Between(Size("", Key(i)), S1*i, S2*i)); - ASSERT_TRUE(Between(Size("", Key(i)+".suffix"), S1*(i+1), S2*(i+1))); - ASSERT_TRUE(Between(Size(Key(i), Key(i+10)), S1*10, S2*10)); + ASSERT_TRUE(Between(Size("", Key(i)), S1 * i, S2 * i)); + ASSERT_TRUE(Between(Size("", Key(i) + ".suffix"), S1 * (i + 1), + S2 * (i + 1))); + ASSERT_TRUE(Between(Size(Key(i), Key(i + 10)), S1 * 10, S2 * 10)); } - ASSERT_TRUE(Between(Size("", Key(50)), S1*50, S2*50)); - ASSERT_TRUE(Between(Size("", Key(50)+".suffix"), S1*50, S2*50)); + ASSERT_TRUE(Between(Size("", Key(50)), S1 * 50, S2 * 50)); + ASSERT_TRUE(Between(Size("", Key(50) + ".suffix"), S1 * 50, S2 * 50)); std::string cstart_str = Key(compact_start); std::string cend_str = Key(compact_start + 9); @@ -1218,7 +1252,7 @@ TEST(DBTest, ApproximateSizes) { } while (ChangeOptions()); } -TEST(DBTest, ApproximateSizes_MixOfSmallAndLarge) { +TEST_F(DBTest, ApproximateSizes_MixOfSmallAndLarge) { do { Options options = CurrentOptions(); options.compression = kNoCompression; @@ -1226,18 +1260,18 @@ TEST(DBTest, ApproximateSizes_MixOfSmallAndLarge) { Random rnd(301); std::string big1 = RandomString(&rnd, 100000); - ASSERT_OK(Put(Key(0), RandomString(&rnd, 10000))); - ASSERT_OK(Put(Key(1), RandomString(&rnd, 10000))); - ASSERT_OK(Put(Key(2), big1)); - ASSERT_OK(Put(Key(3), RandomString(&rnd, 10000))); - ASSERT_OK(Put(Key(4), big1)); - ASSERT_OK(Put(Key(5), RandomString(&rnd, 10000))); - ASSERT_OK(Put(Key(6), RandomString(&rnd, 300000))); - ASSERT_OK(Put(Key(7), RandomString(&rnd, 10000))); + ASSERT_LEVELDB_OK(Put(Key(0), RandomString(&rnd, 10000))); + ASSERT_LEVELDB_OK(Put(Key(1), RandomString(&rnd, 10000))); + ASSERT_LEVELDB_OK(Put(Key(2), big1)); + ASSERT_LEVELDB_OK(Put(Key(3), RandomString(&rnd, 10000))); + ASSERT_LEVELDB_OK(Put(Key(4), big1)); + ASSERT_LEVELDB_OK(Put(Key(5), RandomString(&rnd, 10000))); + ASSERT_LEVELDB_OK(Put(Key(6), RandomString(&rnd, 300000))); + ASSERT_LEVELDB_OK(Put(Key(7), RandomString(&rnd, 10000))); if (options.reuse_logs) { // Need to force a memtable compaction since recovery does not do so. - ASSERT_OK(dbfull()->TEST_CompactMemTable()); + ASSERT_LEVELDB_OK(dbfull()->TEST_CompactMemTable()); } // Check sizes across recovery by reopening a few times @@ -1261,7 +1295,7 @@ TEST(DBTest, ApproximateSizes_MixOfSmallAndLarge) { } while (ChangeOptions()); } -TEST(DBTest, IteratorPinsRef) { +TEST_F(DBTest, IteratorPinsRef) { Put("foo", "hello"); // Get iterator that will yield the current contents of the DB. @@ -1270,7 +1304,8 @@ TEST(DBTest, IteratorPinsRef) { // Write to force compactions Put("foo", "newvalue1"); for (int i = 0; i < 100; i++) { - ASSERT_OK(Put(Key(i), Key(i) + std::string(100000, 'v'))); // 100K values + ASSERT_LEVELDB_OK( + Put(Key(i), Key(i) + std::string(100000, 'v'))); // 100K values } Put("foo", "newvalue2"); @@ -1283,7 +1318,7 @@ TEST(DBTest, IteratorPinsRef) { delete iter; } -TEST(DBTest, Snapshot) { +TEST_F(DBTest, Snapshot) { do { Put("foo", "v1"); const Snapshot* s1 = db_->GetSnapshot(); @@ -1312,7 +1347,7 @@ TEST(DBTest, Snapshot) { } while (ChangeOptions()); } -TEST(DBTest, HiddenValuesAreRemoved) { +TEST_F(DBTest, HiddenValuesAreRemoved) { do { Random rnd(301); FillLevels("a", "z"); @@ -1322,9 +1357,9 @@ TEST(DBTest, HiddenValuesAreRemoved) { Put("pastfoo", "v"); const Snapshot* snapshot = db_->GetSnapshot(); Put("foo", "tiny"); - Put("pastfoo2", "v2"); // Advance sequence number one more + Put("pastfoo2", "v2"); // Advance sequence number one more - ASSERT_OK(dbfull()->TEST_CompactMemTable()); + ASSERT_LEVELDB_OK(dbfull()->TEST_CompactMemTable()); ASSERT_GT(NumTableFilesAtLevel(0), 0); ASSERT_EQ(big, Get("foo", snapshot)); @@ -1343,71 +1378,72 @@ TEST(DBTest, HiddenValuesAreRemoved) { } while (ChangeOptions()); } -TEST(DBTest, DeletionMarkers1) { +TEST_F(DBTest, DeletionMarkers1) { Put("foo", "v1"); - ASSERT_OK(dbfull()->TEST_CompactMemTable()); + ASSERT_LEVELDB_OK(dbfull()->TEST_CompactMemTable()); const int last = config::kMaxMemCompactLevel; - ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level + ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level // Place a table at level last-1 to prevent merging with preceding mutation Put("a", "begin"); Put("z", "end"); dbfull()->TEST_CompactMemTable(); ASSERT_EQ(NumTableFilesAtLevel(last), 1); - ASSERT_EQ(NumTableFilesAtLevel(last-1), 1); + ASSERT_EQ(NumTableFilesAtLevel(last - 1), 1); Delete("foo"); Put("foo", "v2"); ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]"); - ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2 + ASSERT_LEVELDB_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2 ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]"); Slice z("z"); - dbfull()->TEST_CompactRange(last-2, nullptr, &z); + dbfull()->TEST_CompactRange(last - 2, nullptr, &z); // DEL eliminated, but v1 remains because we aren't compacting that level // (DEL can be eliminated because v2 hides v1). ASSERT_EQ(AllEntriesFor("foo"), "[ v2, v1 ]"); - dbfull()->TEST_CompactRange(last-1, nullptr, nullptr); + dbfull()->TEST_CompactRange(last - 1, nullptr, nullptr); // Merging last-1 w/ last, so we are the base level for "foo", so // DEL is removed. (as is v1). ASSERT_EQ(AllEntriesFor("foo"), "[ v2 ]"); } -TEST(DBTest, DeletionMarkers2) { +TEST_F(DBTest, DeletionMarkers2) { Put("foo", "v1"); - ASSERT_OK(dbfull()->TEST_CompactMemTable()); + ASSERT_LEVELDB_OK(dbfull()->TEST_CompactMemTable()); const int last = config::kMaxMemCompactLevel; - ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level + ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level // Place a table at level last-1 to prevent merging with preceding mutation Put("a", "begin"); Put("z", "end"); dbfull()->TEST_CompactMemTable(); ASSERT_EQ(NumTableFilesAtLevel(last), 1); - ASSERT_EQ(NumTableFilesAtLevel(last-1), 1); + ASSERT_EQ(NumTableFilesAtLevel(last - 1), 1); Delete("foo"); ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]"); - ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2 + ASSERT_LEVELDB_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2 ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]"); - dbfull()->TEST_CompactRange(last-2, nullptr, nullptr); + dbfull()->TEST_CompactRange(last - 2, nullptr, nullptr); // DEL kept: "last" file overlaps ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]"); - dbfull()->TEST_CompactRange(last-1, nullptr, nullptr); + dbfull()->TEST_CompactRange(last - 1, nullptr, nullptr); // Merging last-1 w/ last, so we are the base level for "foo", so // DEL is removed. (as is v1). ASSERT_EQ(AllEntriesFor("foo"), "[ ]"); } -TEST(DBTest, OverlapInLevel0) { +TEST_F(DBTest, OverlapInLevel0) { do { ASSERT_EQ(config::kMaxMemCompactLevel, 2) << "Fix test to match config"; - // Fill levels 1 and 2 to disable the pushing of new memtables to levels > 0. - ASSERT_OK(Put("100", "v100")); - ASSERT_OK(Put("999", "v999")); + // Fill levels 1 and 2 to disable the pushing of new memtables to levels > + // 0. + ASSERT_LEVELDB_OK(Put("100", "v100")); + ASSERT_LEVELDB_OK(Put("999", "v999")); dbfull()->TEST_CompactMemTable(); - ASSERT_OK(Delete("100")); - ASSERT_OK(Delete("999")); + ASSERT_LEVELDB_OK(Delete("100")); + ASSERT_LEVELDB_OK(Delete("999")); dbfull()->TEST_CompactMemTable(); ASSERT_EQ("0,1,1", FilesPerLevel()); @@ -1415,12 +1451,12 @@ TEST(DBTest, OverlapInLevel0) { // files[0] 200 .. 900 // files[1] 300 .. 500 // Note that files are sorted by smallest key. - ASSERT_OK(Put("300", "v300")); - ASSERT_OK(Put("500", "v500")); + ASSERT_LEVELDB_OK(Put("300", "v300")); + ASSERT_LEVELDB_OK(Put("500", "v500")); dbfull()->TEST_CompactMemTable(); - ASSERT_OK(Put("200", "v200")); - ASSERT_OK(Put("600", "v600")); - ASSERT_OK(Put("900", "v900")); + ASSERT_LEVELDB_OK(Put("200", "v200")); + ASSERT_LEVELDB_OK(Put("600", "v600")); + ASSERT_LEVELDB_OK(Put("900", "v900")); dbfull()->TEST_CompactMemTable(); ASSERT_EQ("2,1,1", FilesPerLevel()); @@ -1432,23 +1468,23 @@ TEST(DBTest, OverlapInLevel0) { // Do a memtable compaction. Before bug-fix, the compaction would // not detect the overlap with level-0 files and would incorrectly place // the deletion in a deeper level. - ASSERT_OK(Delete("600")); + ASSERT_LEVELDB_OK(Delete("600")); dbfull()->TEST_CompactMemTable(); ASSERT_EQ("3", FilesPerLevel()); ASSERT_EQ("NOT_FOUND", Get("600")); } while (ChangeOptions()); } -TEST(DBTest, L0_CompactionBug_Issue44_a) { +TEST_F(DBTest, L0_CompactionBug_Issue44_a) { Reopen(); - ASSERT_OK(Put("b", "v")); + ASSERT_LEVELDB_OK(Put("b", "v")); Reopen(); - ASSERT_OK(Delete("b")); - ASSERT_OK(Delete("a")); + ASSERT_LEVELDB_OK(Delete("b")); + ASSERT_LEVELDB_OK(Delete("a")); Reopen(); - ASSERT_OK(Delete("a")); + ASSERT_LEVELDB_OK(Delete("a")); Reopen(); - ASSERT_OK(Put("a", "v")); + ASSERT_LEVELDB_OK(Put("a", "v")); Reopen(); Reopen(); ASSERT_EQ("(a->v)", Contents()); @@ -1456,23 +1492,23 @@ TEST(DBTest, L0_CompactionBug_Issue44_a) { ASSERT_EQ("(a->v)", Contents()); } -TEST(DBTest, L0_CompactionBug_Issue44_b) { +TEST_F(DBTest, L0_CompactionBug_Issue44_b) { Reopen(); - Put("",""); + Put("", ""); Reopen(); Delete("e"); - Put("",""); + Put("", ""); Reopen(); Put("c", "cv"); Reopen(); - Put("",""); + Put("", ""); Reopen(); - Put("",""); + Put("", ""); DelayMilliseconds(1000); // Wait for compaction to finish Reopen(); - Put("d","dv"); + Put("d", "dv"); Reopen(); - Put("",""); + Put("", ""); Reopen(); Delete("d"); Delete("b"); @@ -1482,26 +1518,26 @@ TEST(DBTest, L0_CompactionBug_Issue44_b) { ASSERT_EQ("(->)(c->cv)", Contents()); } -TEST(DBTest, Fflush_Issue474) { +TEST_F(DBTest, Fflush_Issue474) { static const int kNum = 100000; Random rnd(test::RandomSeed()); for (int i = 0; i < kNum; i++) { - fflush(nullptr); - ASSERT_OK(Put(RandomKey(&rnd), RandomString(&rnd, 100))); + std::fflush(nullptr); + ASSERT_LEVELDB_OK(Put(RandomKey(&rnd), RandomString(&rnd, 100))); } } -TEST(DBTest, ComparatorCheck) { +TEST_F(DBTest, ComparatorCheck) { class NewComparator : public Comparator { public: - virtual const char* Name() const { return "leveldb.NewComparator"; } - virtual int Compare(const Slice& a, const Slice& b) const { + const char* Name() const override { return "leveldb.NewComparator"; } + int Compare(const Slice& a, const Slice& b) const override { return BytewiseComparator()->Compare(a, b); } - virtual void FindShortestSeparator(std::string* s, const Slice& l) const { + void FindShortestSeparator(std::string* s, const Slice& l) const override { BytewiseComparator()->FindShortestSeparator(s, l); } - virtual void FindShortSuccessor(std::string* key) const { + void FindShortSuccessor(std::string* key) const override { BytewiseComparator()->FindShortSuccessor(key); } }; @@ -1514,28 +1550,29 @@ TEST(DBTest, ComparatorCheck) { << s.ToString(); } -TEST(DBTest, CustomComparator) { +TEST_F(DBTest, CustomComparator) { class NumberComparator : public Comparator { public: - virtual const char* Name() const { return "test.NumberComparator"; } - virtual int Compare(const Slice& a, const Slice& b) const { + const char* Name() const override { return "test.NumberComparator"; } + int Compare(const Slice& a, const Slice& b) const override { return ToNumber(a) - ToNumber(b); } - virtual void FindShortestSeparator(std::string* s, const Slice& l) const { - ToNumber(*s); // Check format - ToNumber(l); // Check format + void FindShortestSeparator(std::string* s, const Slice& l) const override { + ToNumber(*s); // Check format + ToNumber(l); // Check format } - virtual void FindShortSuccessor(std::string* key) const { - ToNumber(*key); // Check format + void FindShortSuccessor(std::string* key) const override { + ToNumber(*key); // Check format } + private: static int ToNumber(const Slice& x) { // Check that there are no extra characters. - ASSERT_TRUE(x.size() >= 2 && x[0] == '[' && x[x.size()-1] == ']') + EXPECT_TRUE(x.size() >= 2 && x[0] == '[' && x[x.size() - 1] == ']') << EscapeString(x); int val; char ignored; - ASSERT_TRUE(sscanf(x.ToString().c_str(), "[%i]%c", &val, &ignored) == 1) + EXPECT_TRUE(sscanf(x.ToString().c_str(), "[%i]%c", &val, &ignored) == 1) << EscapeString(x); return val; } @@ -1544,11 +1581,11 @@ TEST(DBTest, CustomComparator) { Options new_options = CurrentOptions(); new_options.create_if_missing = true; new_options.comparator = &cmp; - new_options.filter_policy = nullptr; // Cannot use bloom filters + new_options.filter_policy = nullptr; // Cannot use bloom filters new_options.write_buffer_size = 1000; // Compact more often DestroyAndReopen(&new_options); - ASSERT_OK(Put("[10]", "ten")); - ASSERT_OK(Put("[0x14]", "twenty")); + ASSERT_LEVELDB_OK(Put("[10]", "ten")); + ASSERT_LEVELDB_OK(Put("[0x14]", "twenty")); for (int i = 0; i < 2; i++) { ASSERT_EQ("ten", Get("[10]")); ASSERT_EQ("ten", Get("[0xa]")); @@ -1562,14 +1599,14 @@ TEST(DBTest, CustomComparator) { for (int run = 0; run < 2; run++) { for (int i = 0; i < 1000; i++) { char buf[100]; - snprintf(buf, sizeof(buf), "[%d]", i*10); - ASSERT_OK(Put(buf, buf)); + std::snprintf(buf, sizeof(buf), "[%d]", i * 10); + ASSERT_LEVELDB_OK(Put(buf, buf)); } Compact("[0]", "[1000000]"); } } -TEST(DBTest, ManualCompaction) { +TEST_F(DBTest, ManualCompaction) { ASSERT_EQ(config::kMaxMemCompactLevel, 2) << "Need to update this test to match kMaxMemCompactLevel"; @@ -1603,8 +1640,8 @@ TEST(DBTest, ManualCompaction) { ASSERT_EQ("0,0,1", FilesPerLevel()); } -TEST(DBTest, DBOpen_Options) { - std::string dbname = test::TmpDir() + "/db_options_test"; +TEST_F(DBTest, DBOpen_Options) { + std::string dbname = testing::TempDir() + "db_options_test"; DestroyDB(dbname, Options()); // Does not exist, and create_if_missing == false: error @@ -1618,7 +1655,7 @@ TEST(DBTest, DBOpen_Options) { // Does not exist, and create_if_missing == true: OK opts.create_if_missing = true; s = DB::Open(opts, dbname, &db); - ASSERT_OK(s); + ASSERT_LEVELDB_OK(s); ASSERT_TRUE(db != nullptr); delete db; @@ -1635,50 +1672,50 @@ TEST(DBTest, DBOpen_Options) { opts.create_if_missing = true; opts.error_if_exists = false; s = DB::Open(opts, dbname, &db); - ASSERT_OK(s); + ASSERT_LEVELDB_OK(s); ASSERT_TRUE(db != nullptr); delete db; db = nullptr; } -TEST(DBTest, DestroyEmptyDir) { - std::string dbname = test::TmpDir() + "/db_empty_dir"; +TEST_F(DBTest, DestroyEmptyDir) { + std::string dbname = testing::TempDir() + "db_empty_dir"; TestEnv env(Env::Default()); - env.DeleteDir(dbname); + env.RemoveDir(dbname); ASSERT_TRUE(!env.FileExists(dbname)); Options opts; opts.env = &env; - ASSERT_OK(env.CreateDir(dbname)); + ASSERT_LEVELDB_OK(env.CreateDir(dbname)); ASSERT_TRUE(env.FileExists(dbname)); std::vector<std::string> children; - ASSERT_OK(env.GetChildren(dbname, &children)); - // The POSIX env does not filter out '.' and '..' special files. + ASSERT_LEVELDB_OK(env.GetChildren(dbname, &children)); + // The stock Env's do not filter out '.' and '..' special files. ASSERT_EQ(2, children.size()); - ASSERT_OK(DestroyDB(dbname, opts)); + ASSERT_LEVELDB_OK(DestroyDB(dbname, opts)); ASSERT_TRUE(!env.FileExists(dbname)); // Should also be destroyed if Env is filtering out dot files. env.SetIgnoreDotFiles(true); - ASSERT_OK(env.CreateDir(dbname)); + ASSERT_LEVELDB_OK(env.CreateDir(dbname)); ASSERT_TRUE(env.FileExists(dbname)); - ASSERT_OK(env.GetChildren(dbname, &children)); + ASSERT_LEVELDB_OK(env.GetChildren(dbname, &children)); ASSERT_EQ(0, children.size()); - ASSERT_OK(DestroyDB(dbname, opts)); + ASSERT_LEVELDB_OK(DestroyDB(dbname, opts)); ASSERT_TRUE(!env.FileExists(dbname)); } -TEST(DBTest, DestroyOpenDB) { - std::string dbname = test::TmpDir() + "/open_db_dir"; - env_->DeleteDir(dbname); +TEST_F(DBTest, DestroyOpenDB) { + std::string dbname = testing::TempDir() + "open_db_dir"; + env_->RemoveDir(dbname); ASSERT_TRUE(!env_->FileExists(dbname)); Options opts; opts.create_if_missing = true; DB* db = nullptr; - ASSERT_OK(DB::Open(opts, dbname, &db)); + ASSERT_LEVELDB_OK(DB::Open(opts, dbname, &db)); ASSERT_TRUE(db != nullptr); // Must fail to destroy an open db. @@ -1690,68 +1727,70 @@ TEST(DBTest, DestroyOpenDB) { db = nullptr; // Should succeed destroying a closed db. - ASSERT_OK(DestroyDB(dbname, Options())); + ASSERT_LEVELDB_OK(DestroyDB(dbname, Options())); ASSERT_TRUE(!env_->FileExists(dbname)); } -TEST(DBTest, Locking) { +TEST_F(DBTest, Locking) { DB* db2 = nullptr; Status s = DB::Open(CurrentOptions(), dbname_, &db2); ASSERT_TRUE(!s.ok()) << "Locking did not prevent re-opening db"; } // Check that number of files does not grow when we are out of space -TEST(DBTest, NoSpace) { +TEST_F(DBTest, NoSpace) { Options options = CurrentOptions(); options.env = env_; Reopen(&options); - ASSERT_OK(Put("foo", "v1")); + ASSERT_LEVELDB_OK(Put("foo", "v1")); ASSERT_EQ("v1", Get("foo")); Compact("a", "z"); const int num_files = CountFiles(); - env_->no_space_.Release_Store(env_); // Force out-of-space errors + // Force out-of-space errors. + env_->no_space_.store(true, std::memory_order_release); for (int i = 0; i < 10; i++) { - for (int level = 0; level < config::kNumLevels-1; level++) { + for (int level = 0; level < config::kNumLevels - 1; level++) { dbfull()->TEST_CompactRange(level, nullptr, nullptr); } } - env_->no_space_.Release_Store(nullptr); + env_->no_space_.store(false, std::memory_order_release); ASSERT_LT(CountFiles(), num_files + 3); } -TEST(DBTest, NonWritableFileSystem) { +TEST_F(DBTest, NonWritableFileSystem) { Options options = CurrentOptions(); options.write_buffer_size = 1000; options.env = env_; Reopen(&options); - ASSERT_OK(Put("foo", "v1")); - env_->non_writable_.Release_Store(env_); // Force errors for new files + ASSERT_LEVELDB_OK(Put("foo", "v1")); + // Force errors for new files. + env_->non_writable_.store(true, std::memory_order_release); std::string big(100000, 'x'); int errors = 0; for (int i = 0; i < 20; i++) { - fprintf(stderr, "iter %d; errors %d\n", i, errors); + std::fprintf(stderr, "iter %d; errors %d\n", i, errors); if (!Put("foo", big).ok()) { errors++; DelayMilliseconds(100); } } ASSERT_GT(errors, 0); - env_->non_writable_.Release_Store(nullptr); + env_->non_writable_.store(false, std::memory_order_release); } -TEST(DBTest, WriteSyncError) { +TEST_F(DBTest, WriteSyncError) { // Check that log sync errors cause the DB to disallow future writes. // (a) Cause log sync calls to fail Options options = CurrentOptions(); options.env = env_; Reopen(&options); - env_->data_sync_error_.Release_Store(env_); + env_->data_sync_error_.store(true, std::memory_order_release); // (b) Normal write should succeed WriteOptions w; - ASSERT_OK(db_->Put(w, "k1", "v1")); + ASSERT_LEVELDB_OK(db_->Put(w, "k1", "v1")); ASSERT_EQ("v1", Get("k1")); // (c) Do a sync write; should fail @@ -1761,7 +1800,7 @@ TEST(DBTest, WriteSyncError) { ASSERT_EQ("NOT_FOUND", Get("k2")); // (d) make sync behave normally - env_->data_sync_error_.Release_Store(nullptr); + env_->data_sync_error_.store(false, std::memory_order_release); // (e) Do a non-sync write; should fail w.sync = false; @@ -1771,7 +1810,7 @@ TEST(DBTest, WriteSyncError) { ASSERT_EQ("NOT_FOUND", Get("k3")); } -TEST(DBTest, ManifestWriteError) { +TEST_F(DBTest, ManifestWriteError) { // Test for the following problem: // (a) Compaction produces file F // (b) Log record containing F is written to MANIFEST file, but Sync() fails @@ -1781,9 +1820,8 @@ TEST(DBTest, ManifestWriteError) { // We iterate twice. In the second iteration, everything is the // same except the log record never makes it to the MANIFEST file. for (int iter = 0; iter < 2; iter++) { - port::AtomicPointer* error_type = (iter == 0) - ? &env_->manifest_sync_error_ - : &env_->manifest_write_error_; + std::atomic<bool>* error_type = (iter == 0) ? &env_->manifest_sync_error_ + : &env_->manifest_write_error_; // Insert foo=>bar mapping Options options = CurrentOptions(); @@ -1791,29 +1829,29 @@ TEST(DBTest, ManifestWriteError) { options.create_if_missing = true; options.error_if_exists = false; DestroyAndReopen(&options); - ASSERT_OK(Put("foo", "bar")); + ASSERT_LEVELDB_OK(Put("foo", "bar")); ASSERT_EQ("bar", Get("foo")); // Memtable compaction (will succeed) dbfull()->TEST_CompactMemTable(); ASSERT_EQ("bar", Get("foo")); const int last = config::kMaxMemCompactLevel; - ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo=>bar is now in last level + ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo=>bar is now in last level // Merging compaction (will fail) - error_type->Release_Store(env_); + error_type->store(true, std::memory_order_release); dbfull()->TEST_CompactRange(last, nullptr, nullptr); // Should fail ASSERT_EQ("bar", Get("foo")); // Recovery: should not lose data - error_type->Release_Store(nullptr); + error_type->store(false, std::memory_order_release); Reopen(&options); ASSERT_EQ("bar", Get("foo")); } } -TEST(DBTest, MissingSSTFile) { - ASSERT_OK(Put("foo", "bar")); +TEST_F(DBTest, MissingSSTFile) { + ASSERT_LEVELDB_OK(Put("foo", "bar")); ASSERT_EQ("bar", Get("foo")); // Dump the memtable to disk. @@ -1826,12 +1864,11 @@ TEST(DBTest, MissingSSTFile) { options.paranoid_checks = true; Status s = TryReopen(&options); ASSERT_TRUE(!s.ok()); - ASSERT_TRUE(s.ToString().find("issing") != std::string::npos) - << s.ToString(); + ASSERT_TRUE(s.ToString().find("issing") != std::string::npos) << s.ToString(); } -TEST(DBTest, StillReadSST) { - ASSERT_OK(Put("foo", "bar")); +TEST_F(DBTest, StillReadSST) { + ASSERT_LEVELDB_OK(Put("foo", "bar")); ASSERT_EQ("bar", Get("foo")); // Dump the memtable to disk. @@ -1846,18 +1883,18 @@ TEST(DBTest, StillReadSST) { ASSERT_EQ("bar", Get("foo")); } -TEST(DBTest, FilesDeletedAfterCompaction) { - ASSERT_OK(Put("foo", "v2")); +TEST_F(DBTest, FilesDeletedAfterCompaction) { + ASSERT_LEVELDB_OK(Put("foo", "v2")); Compact("a", "z"); const int num_files = CountFiles(); for (int i = 0; i < 10; i++) { - ASSERT_OK(Put("foo", "v2")); + ASSERT_LEVELDB_OK(Put("foo", "v2")); Compact("a", "z"); } ASSERT_EQ(CountFiles(), num_files); } -TEST(DBTest, BloomFilter) { +TEST_F(DBTest, BloomFilter) { env_->count_random_reads_ = true; Options options = CurrentOptions(); options.env = env_; @@ -1868,16 +1905,16 @@ TEST(DBTest, BloomFilter) { // Populate multiple layers const int N = 10000; for (int i = 0; i < N; i++) { - ASSERT_OK(Put(Key(i), Key(i))); + ASSERT_LEVELDB_OK(Put(Key(i), Key(i))); } Compact("a", "z"); for (int i = 0; i < N; i += 100) { - ASSERT_OK(Put(Key(i), Key(i))); + ASSERT_LEVELDB_OK(Put(Key(i), Key(i))); } dbfull()->TEST_CompactMemTable(); // Prevent auto compactions triggered by seeks - env_->delay_data_sync_.Release_Store(env_); + env_->delay_data_sync_.store(true, std::memory_order_release); // Lookup present keys. Should rarely read from small sstable. env_->random_read_counter_.Reset(); @@ -1885,9 +1922,9 @@ TEST(DBTest, BloomFilter) { ASSERT_EQ(Key(i), Get(Key(i))); } int reads = env_->random_read_counter_.Read(); - fprintf(stderr, "%d present => %d reads\n", N, reads); + std::fprintf(stderr, "%d present => %d reads\n", N, reads); ASSERT_GE(reads, N); - ASSERT_LE(reads, N + 2*N/100); + ASSERT_LE(reads, N + 2 * N / 100); // Lookup present keys. Should rarely read from either sstable. env_->random_read_counter_.Reset(); @@ -1895,10 +1932,10 @@ TEST(DBTest, BloomFilter) { ASSERT_EQ("NOT_FOUND", Get(Key(i) + ".missing")); } reads = env_->random_read_counter_.Read(); - fprintf(stderr, "%d missing => %d reads\n", N, reads); - ASSERT_LE(reads, 3*N/100); + std::fprintf(stderr, "%d missing => %d reads\n", N, reads); + ASSERT_LE(reads, 3 * N / 100); - env_->delay_data_sync_.Release_Store(nullptr); + env_->delay_data_sync_.store(false, std::memory_order_release); Close(); delete options.block_cache; delete options.filter_policy; @@ -1913,9 +1950,9 @@ static const int kNumKeys = 1000; struct MTState { DBTest* test; - port::AtomicPointer stop; - port::AtomicPointer counter[kNumThreads]; - port::AtomicPointer thread_done[kNumThreads]; + std::atomic<bool> stop; + std::atomic<int> counter[kNumThreads]; + std::atomic<bool> thread_done[kNumThreads]; }; struct MTThread { @@ -1927,24 +1964,24 @@ static void MTThreadBody(void* arg) { MTThread* t = reinterpret_cast<MTThread*>(arg); int id = t->id; DB* db = t->state->test->db_; - uintptr_t counter = 0; - fprintf(stderr, "... starting thread %d\n", id); + int counter = 0; + std::fprintf(stderr, "... starting thread %d\n", id); Random rnd(1000 + id); std::string value; char valbuf[1500]; - while (t->state->stop.Acquire_Load() == nullptr) { - t->state->counter[id].Release_Store(reinterpret_cast<void*>(counter)); + while (!t->state->stop.load(std::memory_order_acquire)) { + t->state->counter[id].store(counter, std::memory_order_release); int key = rnd.Uniform(kNumKeys); char keybuf[20]; - snprintf(keybuf, sizeof(keybuf), "%016d", key); + std::snprintf(keybuf, sizeof(keybuf), "%016d", key); if (rnd.OneIn(2)) { // Write values of the form <key, my id, counter>. // We add some padding for force compactions. - snprintf(valbuf, sizeof(valbuf), "%d.%d.%-1000d", - key, id, static_cast<int>(counter)); - ASSERT_OK(db->Put(WriteOptions(), Slice(keybuf), Slice(valbuf))); + std::snprintf(valbuf, sizeof(valbuf), "%d.%d.%-1000d", key, id, + static_cast<int>(counter)); + ASSERT_LEVELDB_OK(db->Put(WriteOptions(), Slice(keybuf), Slice(valbuf))); } else { // Read a value and verify that it matches the pattern written above. Status s = db->Get(ReadOptions(), Slice(keybuf), &value); @@ -1952,33 +1989,32 @@ static void MTThreadBody(void* arg) { // Key has not yet been written } else { // Check that the writer thread counter is >= the counter in the value - ASSERT_OK(s); + ASSERT_LEVELDB_OK(s); int k, w, c; ASSERT_EQ(3, sscanf(value.c_str(), "%d.%d.%d", &k, &w, &c)) << value; ASSERT_EQ(k, key); ASSERT_GE(w, 0); ASSERT_LT(w, kNumThreads); - ASSERT_LE(static_cast<uintptr_t>(c), reinterpret_cast<uintptr_t>( - t->state->counter[w].Acquire_Load())); + ASSERT_LE(c, t->state->counter[w].load(std::memory_order_acquire)); } } counter++; } - t->state->thread_done[id].Release_Store(t); - fprintf(stderr, "... stopping thread %d after %d ops\n", id, int(counter)); + t->state->thread_done[id].store(true, std::memory_order_release); + std::fprintf(stderr, "... stopping thread %d after %d ops\n", id, counter); } } // namespace -TEST(DBTest, MultiThreaded) { +TEST_F(DBTest, MultiThreaded) { do { // Initialize state MTState mt; mt.test = this; - mt.stop.Release_Store(0); + mt.stop.store(false, std::memory_order_release); for (int id = 0; id < kNumThreads; id++) { - mt.counter[id].Release_Store(0); - mt.thread_done[id].Release_Store(0); + mt.counter[id].store(false, std::memory_order_release); + mt.thread_done[id].store(false, std::memory_order_release); } // Start threads @@ -1993,9 +2029,9 @@ TEST(DBTest, MultiThreaded) { DelayMilliseconds(kTestSeconds * 1000); // Stop the threads and wait for them to finish - mt.stop.Release_Store(&mt); + mt.stop.store(true, std::memory_order_release); for (int id = 0; id < kNumThreads; id++) { - while (mt.thread_done[id].Acquire_Load() == nullptr) { + while (!mt.thread_done[id].load(std::memory_order_acquire)) { DelayMilliseconds(100); } } @@ -2006,27 +2042,27 @@ namespace { typedef std::map<std::string, std::string> KVMap; } -class ModelDB: public DB { +class ModelDB : public DB { public: class ModelSnapshot : public Snapshot { public: KVMap map_; }; - explicit ModelDB(const Options& options): options_(options) { } - ~ModelDB() { } - virtual Status Put(const WriteOptions& o, const Slice& k, const Slice& v) { + explicit ModelDB(const Options& options) : options_(options) {} + ~ModelDB() override = default; + Status Put(const WriteOptions& o, const Slice& k, const Slice& v) override { return DB::Put(o, k, v); } - virtual Status Delete(const WriteOptions& o, const Slice& key) { + Status Delete(const WriteOptions& o, const Slice& key) override { return DB::Delete(o, key); } - virtual Status Get(const ReadOptions& options, - const Slice& key, std::string* value) { - assert(false); // Not implemented + Status Get(const ReadOptions& options, const Slice& key, + std::string* value) override { + assert(false); // Not implemented return Status::NotFound(key); } - virtual Iterator* NewIterator(const ReadOptions& options) { + Iterator* NewIterator(const ReadOptions& options) override { if (options.snapshot == nullptr) { KVMap* saved = new KVMap; *saved = map_; @@ -2037,68 +2073,65 @@ class ModelDB: public DB { return new ModelIter(snapshot_state, false); } } - virtual const Snapshot* GetSnapshot() { + const Snapshot* GetSnapshot() override { ModelSnapshot* snapshot = new ModelSnapshot; snapshot->map_ = map_; return snapshot; } - virtual void ReleaseSnapshot(const Snapshot* snapshot) { + void ReleaseSnapshot(const Snapshot* snapshot) override { delete reinterpret_cast<const ModelSnapshot*>(snapshot); } - virtual Status Write(const WriteOptions& options, WriteBatch* batch) { + Status Write(const WriteOptions& options, WriteBatch* batch) override { class Handler : public WriteBatch::Handler { public: KVMap* map_; - virtual void Put(const Slice& key, const Slice& value) { + void Put(const Slice& key, const Slice& value) override { (*map_)[key.ToString()] = value.ToString(); } - virtual void Delete(const Slice& key) { - map_->erase(key.ToString()); - } + void Delete(const Slice& key) override { map_->erase(key.ToString()); } }; Handler handler; handler.map_ = &map_; return batch->Iterate(&handler); } - virtual bool GetProperty(const Slice& property, std::string* value) { + bool GetProperty(const Slice& property, std::string* value) override { return false; } - virtual void GetApproximateSizes(const Range* r, int n, uint64_t* sizes) { + void GetApproximateSizes(const Range* r, int n, uint64_t* sizes) override { for (int i = 0; i < n; i++) { sizes[i] = 0; } } - virtual void CompactRange(const Slice* start, const Slice* end) { - } + void CompactRange(const Slice* start, const Slice* end) override {} private: - class ModelIter: public Iterator { + class ModelIter : public Iterator { public: ModelIter(const KVMap* map, bool owned) - : map_(map), owned_(owned), iter_(map_->end()) { - } - ~ModelIter() { + : map_(map), owned_(owned), iter_(map_->end()) {} + ~ModelIter() override { if (owned_) delete map_; } - virtual bool Valid() const { return iter_ != map_->end(); } - virtual void SeekToFirst() { iter_ = map_->begin(); } - virtual void SeekToLast() { + bool Valid() const override { return iter_ != map_->end(); } + void SeekToFirst() override { iter_ = map_->begin(); } + void SeekToLast() override { if (map_->empty()) { iter_ = map_->end(); } else { iter_ = map_->find(map_->rbegin()->first); } } - virtual void Seek(const Slice& k) { + void Seek(const Slice& k) override { iter_ = map_->lower_bound(k.ToString()); } - virtual void Next() { ++iter_; } - virtual void Prev() { --iter_; } - virtual Slice key() const { return iter_->first; } - virtual Slice value() const { return iter_->second; } - virtual Status status() const { return Status::OK(); } + void Next() override { ++iter_; } + void Prev() override { --iter_; } + Slice key() const override { return iter_->first; } + Slice value() const override { return iter_->second; } + Status status() const override { return Status::OK(); } + private: const KVMap* const map_; const bool owned_; // Do we own map_ @@ -2108,9 +2141,7 @@ class ModelDB: public DB { KVMap map_; }; -static bool CompareIterators(int step, - DB* model, - DB* db, +static bool CompareIterators(int step, DB* model, DB* db, const Snapshot* model_snap, const Snapshot* db_snap) { ReadOptions options; @@ -2120,43 +2151,82 @@ static bool CompareIterators(int step, Iterator* dbiter = db->NewIterator(options); bool ok = true; int count = 0; + std::vector<std::string> seek_keys; + // Compare equality of all elements using Next(). Save some of the keys for + // comparing Seek equality. for (miter->SeekToFirst(), dbiter->SeekToFirst(); - ok && miter->Valid() && dbiter->Valid(); - miter->Next(), dbiter->Next()) { + ok && miter->Valid() && dbiter->Valid(); miter->Next(), dbiter->Next()) { count++; if (miter->key().compare(dbiter->key()) != 0) { - fprintf(stderr, "step %d: Key mismatch: '%s' vs. '%s'\n", - step, - EscapeString(miter->key()).c_str(), - EscapeString(dbiter->key()).c_str()); + std::fprintf(stderr, "step %d: Key mismatch: '%s' vs. '%s'\n", step, + EscapeString(miter->key()).c_str(), + EscapeString(dbiter->key()).c_str()); ok = false; break; } if (miter->value().compare(dbiter->value()) != 0) { - fprintf(stderr, "step %d: Value mismatch for key '%s': '%s' vs. '%s'\n", - step, - EscapeString(miter->key()).c_str(), - EscapeString(miter->value()).c_str(), - EscapeString(miter->value()).c_str()); + std::fprintf(stderr, + "step %d: Value mismatch for key '%s': '%s' vs. '%s'\n", + step, EscapeString(miter->key()).c_str(), + EscapeString(miter->value()).c_str(), + EscapeString(miter->value()).c_str()); ok = false; + break; + } + + if (count % 10 == 0) { + seek_keys.push_back(miter->key().ToString()); } } if (ok) { if (miter->Valid() != dbiter->Valid()) { - fprintf(stderr, "step %d: Mismatch at end of iterators: %d vs. %d\n", - step, miter->Valid(), dbiter->Valid()); + std::fprintf(stderr, "step %d: Mismatch at end of iterators: %d vs. %d\n", + step, miter->Valid(), dbiter->Valid()); ok = false; } } - fprintf(stderr, "%d entries compared: ok=%d\n", count, ok); + + if (ok) { + // Validate iterator equality when performing seeks. + for (auto kiter = seek_keys.begin(); ok && kiter != seek_keys.end(); + ++kiter) { + miter->Seek(*kiter); + dbiter->Seek(*kiter); + if (!miter->Valid() || !dbiter->Valid()) { + std::fprintf(stderr, "step %d: Seek iterators invalid: %d vs. %d\n", + step, miter->Valid(), dbiter->Valid()); + ok = false; + } + if (miter->key().compare(dbiter->key()) != 0) { + std::fprintf(stderr, "step %d: Seek key mismatch: '%s' vs. '%s'\n", + step, EscapeString(miter->key()).c_str(), + EscapeString(dbiter->key()).c_str()); + ok = false; + break; + } + + if (miter->value().compare(dbiter->value()) != 0) { + std::fprintf( + stderr, + "step %d: Seek value mismatch for key '%s': '%s' vs. '%s'\n", step, + EscapeString(miter->key()).c_str(), + EscapeString(miter->value()).c_str(), + EscapeString(miter->value()).c_str()); + ok = false; + break; + } + } + } + + std::fprintf(stderr, "%d entries compared: ok=%d\n", count, ok); delete miter; delete dbiter; return ok; } -TEST(DBTest, Randomized) { +TEST_F(DBTest, Randomized) { Random rnd(test::RandomSeed()); do { ModelDB model(CurrentOptions()); @@ -2166,26 +2236,23 @@ TEST(DBTest, Randomized) { std::string k, v; for (int step = 0; step < N; step++) { if (step % 100 == 0) { - fprintf(stderr, "Step %d of %d\n", step, N); + std::fprintf(stderr, "Step %d of %d\n", step, N); } // TODO(sanjay): Test Get() works int p = rnd.Uniform(100); - if (p < 45) { // Put - k = RandomKey(&rnd); - v = RandomString(&rnd, - rnd.OneIn(20) - ? 100 + rnd.Uniform(100) - : rnd.Uniform(8)); - ASSERT_OK(model.Put(WriteOptions(), k, v)); - ASSERT_OK(db_->Put(WriteOptions(), k, v)); - - } else if (p < 90) { // Delete + if (p < 45) { // Put k = RandomKey(&rnd); - ASSERT_OK(model.Delete(WriteOptions(), k)); - ASSERT_OK(db_->Delete(WriteOptions(), k)); + v = RandomString( + &rnd, rnd.OneIn(20) ? 100 + rnd.Uniform(100) : rnd.Uniform(8)); + ASSERT_LEVELDB_OK(model.Put(WriteOptions(), k, v)); + ASSERT_LEVELDB_OK(db_->Put(WriteOptions(), k, v)); + } else if (p < 90) { // Delete + k = RandomKey(&rnd); + ASSERT_LEVELDB_OK(model.Delete(WriteOptions(), k)); + ASSERT_LEVELDB_OK(db_->Delete(WriteOptions(), k)); - } else { // Multi-element batch + } else { // Multi-element batch WriteBatch b; const int num = rnd.Uniform(8); for (int i = 0; i < num; i++) { @@ -2202,8 +2269,8 @@ TEST(DBTest, Randomized) { b.Delete(k); } } - ASSERT_OK(model.Write(WriteOptions(), &b)); - ASSERT_OK(db_->Write(WriteOptions(), &b)); + ASSERT_LEVELDB_OK(model.Write(WriteOptions(), &b)); + ASSERT_LEVELDB_OK(db_->Write(WriteOptions(), &b)); } if ((step % 100) == 0) { @@ -2227,74 +2294,4 @@ TEST(DBTest, Randomized) { } while (ChangeOptions()); } -std::string MakeKey(unsigned int num) { - char buf[30]; - snprintf(buf, sizeof(buf), "%016u", num); - return std::string(buf); -} - -void BM_LogAndApply(int iters, int num_base_files) { - std::string dbname = test::TmpDir() + "/leveldb_test_benchmark"; - DestroyDB(dbname, Options()); - - DB* db = nullptr; - Options opts; - opts.create_if_missing = true; - Status s = DB::Open(opts, dbname, &db); - ASSERT_OK(s); - ASSERT_TRUE(db != nullptr); - - delete db; - db = nullptr; - - Env* env = Env::Default(); - - port::Mutex mu; - MutexLock l(&mu); - - InternalKeyComparator cmp(BytewiseComparator()); - Options options; - VersionSet vset(dbname, &options, nullptr, &cmp); - bool save_manifest; - ASSERT_OK(vset.Recover(&save_manifest)); - VersionEdit vbase; - uint64_t fnum = 1; - for (int i = 0; i < num_base_files; i++) { - InternalKey start(MakeKey(2*fnum), 1, kTypeValue); - InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion); - vbase.AddFile(2, fnum++, 1 /* file size */, start, limit); - } - ASSERT_OK(vset.LogAndApply(&vbase, &mu)); - - uint64_t start_micros = env->NowMicros(); - - for (int i = 0; i < iters; i++) { - VersionEdit vedit; - vedit.DeleteFile(2, fnum); - InternalKey start(MakeKey(2*fnum), 1, kTypeValue); - InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion); - vedit.AddFile(2, fnum++, 1 /* file size */, start, limit); - vset.LogAndApply(&vedit, &mu); - } - uint64_t stop_micros = env->NowMicros(); - unsigned int us = stop_micros - start_micros; - char buf[16]; - snprintf(buf, sizeof(buf), "%d", num_base_files); - fprintf(stderr, - "BM_LogAndApply/%-6s %8d iters : %9u us (%7.0f us / iter)\n", - buf, iters, us, ((float)us) / iters); -} - } // namespace leveldb - -int main(int argc, char** argv) { - if (argc > 1 && std::string(argv[1]) == "--benchmark") { - leveldb::BM_LogAndApply(1000, 1); - leveldb::BM_LogAndApply(1000, 100); - leveldb::BM_LogAndApply(1000, 10000); - leveldb::BM_LogAndApply(100, 100000); - return 0; - } - - return leveldb::test::RunAllTests(); -} diff --git a/db/dbformat.cc b/db/dbformat.cc index 20a7ca4..2a5749f 100644 --- a/db/dbformat.cc +++ b/db/dbformat.cc @@ -2,8 +2,11 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -#include <stdio.h> #include "db/dbformat.h" + +#include <cstdio> +#include <sstream> + #include "port/port.h" #include "util/coding.h" @@ -21,26 +24,20 @@ void AppendInternalKey(std::string* result, const ParsedInternalKey& key) { } std::string ParsedInternalKey::DebugString() const { - char buf[50]; - snprintf(buf, sizeof(buf), "' @ %llu : %d", - (unsigned long long) sequence, - int(type)); - std::string result = "'"; - result += EscapeString(user_key.ToString()); - result += buf; - return result; + std::ostringstream ss; + ss << '\'' << EscapeString(user_key.ToString()) << "' @ " << sequence << " : " + << static_cast<int>(type); + return ss.str(); } std::string InternalKey::DebugString() const { - std::string result; ParsedInternalKey parsed; if (ParseInternalKey(rep_, &parsed)) { - result = parsed.DebugString(); - } else { - result = "(bad)"; - result.append(EscapeString(rep_)); + return parsed.DebugString(); } - return result; + std::ostringstream ss; + ss << "(bad)" << EscapeString(rep_); + return ss.str(); } const char* InternalKeyComparator::Name() const { @@ -65,9 +62,8 @@ int InternalKeyComparator::Compare(const Slice& akey, const Slice& bkey) const { return r; } -void InternalKeyComparator::FindShortestSeparator( - std::string* start, - const Slice& limit) const { +void InternalKeyComparator::FindShortestSeparator(std::string* start, + const Slice& limit) const { // Attempt to shorten the user portion of the key Slice user_start = ExtractUserKey(*start); Slice user_limit = ExtractUserKey(limit); @@ -77,7 +73,8 @@ void InternalKeyComparator::FindShortestSeparator( user_comparator_->Compare(user_start, tmp) < 0) { // User key has become shorter physically, but larger logically. // Tack on the earliest possible number to the shortened user key. - PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek)); + PutFixed64(&tmp, + PackSequenceAndType(kMaxSequenceNumber, kValueTypeForSeek)); assert(this->Compare(*start, tmp) < 0); assert(this->Compare(tmp, limit) < 0); start->swap(tmp); @@ -92,15 +89,14 @@ void InternalKeyComparator::FindShortSuccessor(std::string* key) const { user_comparator_->Compare(user_key, tmp) < 0) { // User key has become shorter physically, but larger logically. // Tack on the earliest possible number to the shortened user key. - PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek)); + PutFixed64(&tmp, + PackSequenceAndType(kMaxSequenceNumber, kValueTypeForSeek)); assert(this->Compare(*key, tmp) < 0); key->swap(tmp); } } -const char* InternalFilterPolicy::Name() const { - return user_policy_->Name(); -} +const char* InternalFilterPolicy::Name() const { return user_policy_->Name(); } void InternalFilterPolicy::CreateFilter(const Slice* keys, int n, std::string* dst) const { @@ -130,7 +126,7 @@ LookupKey::LookupKey(const Slice& user_key, SequenceNumber s) { start_ = dst; dst = EncodeVarint32(dst, usize + 8); kstart_ = dst; - memcpy(dst, user_key.data(), usize); + std::memcpy(dst, user_key.data(), usize); dst += usize; EncodeFixed64(dst, PackSequenceAndType(s, kValueTypeForSeek)); dst += 8; diff --git a/db/dbformat.h b/db/dbformat.h index c4d9575..a1c30ed 100644 --- a/db/dbformat.h +++ b/db/dbformat.h @@ -5,7 +5,10 @@ #ifndef STORAGE_LEVELDB_DB_DBFORMAT_H_ #define STORAGE_LEVELDB_DB_DBFORMAT_H_ -#include <stdio.h> +#include <cstddef> +#include <cstdint> +#include <string> + #include "leveldb/comparator.h" #include "leveldb/db.h" #include "leveldb/filter_policy.h" @@ -48,10 +51,7 @@ class InternalKey; // Value types encoded as the last component of internal keys. // DO NOT CHANGE THESE ENUM VALUES: they are embedded in the on-disk // data structures. -enum ValueType { - kTypeDeletion = 0x0, - kTypeValue = 0x1 -}; +enum ValueType { kTypeDeletion = 0x0, kTypeValue = 0x1 }; // kValueTypeForSeek defines the ValueType that should be passed when // constructing a ParsedInternalKey object for seeking to a particular // sequence number (since we sort sequence numbers in decreasing order @@ -64,17 +64,16 @@ typedef uint64_t SequenceNumber; // We leave eight bits empty at the bottom so a type and sequence# // can be packed together into 64-bits. -static const SequenceNumber kMaxSequenceNumber = - ((0x1ull << 56) - 1); +static const SequenceNumber kMaxSequenceNumber = ((0x1ull << 56) - 1); struct ParsedInternalKey { Slice user_key; SequenceNumber sequence; ValueType type; - ParsedInternalKey() { } // Intentionally left uninitialized (for speed) + ParsedInternalKey() {} // Intentionally left uninitialized (for speed) ParsedInternalKey(const Slice& u, const SequenceNumber& seq, ValueType t) - : user_key(u), sequence(seq), type(t) { } + : user_key(u), sequence(seq), type(t) {} std::string DebugString() const; }; @@ -103,14 +102,14 @@ inline Slice ExtractUserKey(const Slice& internal_key) { class InternalKeyComparator : public Comparator { private: const Comparator* user_comparator_; + public: - explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c) { } - virtual const char* Name() const; - virtual int Compare(const Slice& a, const Slice& b) const; - virtual void FindShortestSeparator( - std::string* start, - const Slice& limit) const; - virtual void FindShortSuccessor(std::string* key) const; + explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c) {} + const char* Name() const override; + int Compare(const Slice& a, const Slice& b) const override; + void FindShortestSeparator(std::string* start, + const Slice& limit) const override; + void FindShortSuccessor(std::string* key) const override; const Comparator* user_comparator() const { return user_comparator_; } @@ -121,11 +120,12 @@ class InternalKeyComparator : public Comparator { class InternalFilterPolicy : public FilterPolicy { private: const FilterPolicy* const user_policy_; + public: - explicit InternalFilterPolicy(const FilterPolicy* p) : user_policy_(p) { } - virtual const char* Name() const; - virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const; - virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const; + explicit InternalFilterPolicy(const FilterPolicy* p) : user_policy_(p) {} + const char* Name() const override; + void CreateFilter(const Slice* keys, int n, std::string* dst) const override; + bool KeyMayMatch(const Slice& key, const Slice& filter) const override; }; // Modules in this directory should keep internal keys wrapped inside @@ -134,13 +134,18 @@ class InternalFilterPolicy : public FilterPolicy { class InternalKey { private: std::string rep_; + public: - InternalKey() { } // Leave rep_ as empty to indicate it is invalid + InternalKey() {} // Leave rep_ as empty to indicate it is invalid InternalKey(const Slice& user_key, SequenceNumber s, ValueType t) { AppendInternalKey(&rep_, ParsedInternalKey(user_key, s, t)); } - void DecodeFrom(const Slice& s) { rep_.assign(s.data(), s.size()); } + bool DecodeFrom(const Slice& s) { + rep_.assign(s.data(), s.size()); + return !rep_.empty(); + } + Slice Encode() const { assert(!rep_.empty()); return rep_; @@ -158,8 +163,8 @@ class InternalKey { std::string DebugString() const; }; -inline int InternalKeyComparator::Compare( - const InternalKey& a, const InternalKey& b) const { +inline int InternalKeyComparator::Compare(const InternalKey& a, + const InternalKey& b) const { return Compare(a.Encode(), b.Encode()); } @@ -168,11 +173,11 @@ inline bool ParseInternalKey(const Slice& internal_key, const size_t n = internal_key.size(); if (n < 8) return false; uint64_t num = DecodeFixed64(internal_key.data() + n - 8); - unsigned char c = num & 0xff; + uint8_t c = num & 0xff; result->sequence = num >> 8; result->type = static_cast<ValueType>(c); result->user_key = Slice(internal_key.data(), n - 8); - return (c <= static_cast<unsigned char>(kTypeValue)); + return (c <= static_cast<uint8_t>(kTypeValue)); } // A helper class useful for DBImpl::Get() @@ -182,6 +187,9 @@ class LookupKey { // the specified sequence number. LookupKey(const Slice& user_key, SequenceNumber sequence); + LookupKey(const LookupKey&) = delete; + LookupKey& operator=(const LookupKey&) = delete; + ~LookupKey(); // Return a key suitable for lookup in a MemTable. @@ -204,11 +212,7 @@ class LookupKey { const char* start_; const char* kstart_; const char* end_; - char space_[200]; // Avoid allocation for short keys - - // No copying allowed - LookupKey(const LookupKey&); - void operator=(const LookupKey&); + char space_[200]; // Avoid allocation for short keys }; inline LookupKey::~LookupKey() { diff --git a/db/dbformat_test.cc b/db/dbformat_test.cc index 5d82f5d..7f3f81a 100644 --- a/db/dbformat_test.cc +++ b/db/dbformat_test.cc @@ -3,13 +3,13 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "db/dbformat.h" + +#include "gtest/gtest.h" #include "util/logging.h" -#include "util/testharness.h" namespace leveldb { -static std::string IKey(const std::string& user_key, - uint64_t seq, +static std::string IKey(const std::string& user_key, uint64_t seq, ValueType vt) { std::string encoded; AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt)); @@ -28,9 +28,7 @@ static std::string ShortSuccessor(const std::string& s) { return result; } -static void TestKey(const std::string& key, - uint64_t seq, - ValueType vt) { +static void TestKey(const std::string& key, uint64_t seq, ValueType vt) { std::string encoded = IKey(key, seq, vt); Slice in(encoded); @@ -44,16 +42,20 @@ static void TestKey(const std::string& key, ASSERT_TRUE(!ParseInternalKey(Slice("bar"), &decoded)); } -class FormatTest { }; - TEST(FormatTest, InternalKey_EncodeDecode) { - const char* keys[] = { "", "k", "hello", "longggggggggggggggggggggg" }; - const uint64_t seq[] = { - 1, 2, 3, - (1ull << 8) - 1, 1ull << 8, (1ull << 8) + 1, - (1ull << 16) - 1, 1ull << 16, (1ull << 16) + 1, - (1ull << 32) - 1, 1ull << 32, (1ull << 32) + 1 - }; + const char* keys[] = {"", "k", "hello", "longggggggggggggggggggggg"}; + const uint64_t seq[] = {1, + 2, + 3, + (1ull << 8) - 1, + 1ull << 8, + (1ull << 8) + 1, + (1ull << 16) - 1, + 1ull << 16, + (1ull << 16) + 1, + (1ull << 32) - 1, + 1ull << 32, + (1ull << 32) + 1}; for (int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) { for (int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) { TestKey(keys[k], seq[s], kTypeValue); @@ -62,40 +64,44 @@ TEST(FormatTest, InternalKey_EncodeDecode) { } } +TEST(FormatTest, InternalKey_DecodeFromEmpty) { + InternalKey internal_key; + + ASSERT_TRUE(!internal_key.DecodeFrom("")); +} + TEST(FormatTest, InternalKeyShortSeparator) { // When user keys are same ASSERT_EQ(IKey("foo", 100, kTypeValue), - Shorten(IKey("foo", 100, kTypeValue), - IKey("foo", 99, kTypeValue))); - ASSERT_EQ(IKey("foo", 100, kTypeValue), - Shorten(IKey("foo", 100, kTypeValue), - IKey("foo", 101, kTypeValue))); - ASSERT_EQ(IKey("foo", 100, kTypeValue), - Shorten(IKey("foo", 100, kTypeValue), - IKey("foo", 100, kTypeValue))); - ASSERT_EQ(IKey("foo", 100, kTypeValue), - Shorten(IKey("foo", 100, kTypeValue), - IKey("foo", 100, kTypeDeletion))); + Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 99, kTypeValue))); + ASSERT_EQ( + IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 101, kTypeValue))); + ASSERT_EQ( + IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeValue))); + ASSERT_EQ( + IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeDeletion))); // When user keys are misordered ASSERT_EQ(IKey("foo", 100, kTypeValue), - Shorten(IKey("foo", 100, kTypeValue), - IKey("bar", 99, kTypeValue))); + Shorten(IKey("foo", 100, kTypeValue), IKey("bar", 99, kTypeValue))); // When user keys are different, but correctly ordered - ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek), - Shorten(IKey("foo", 100, kTypeValue), - IKey("hello", 200, kTypeValue))); + ASSERT_EQ( + IKey("g", kMaxSequenceNumber, kValueTypeForSeek), + Shorten(IKey("foo", 100, kTypeValue), IKey("hello", 200, kTypeValue))); // When start user key is prefix of limit user key - ASSERT_EQ(IKey("foo", 100, kTypeValue), - Shorten(IKey("foo", 100, kTypeValue), - IKey("foobar", 200, kTypeValue))); + ASSERT_EQ( + IKey("foo", 100, kTypeValue), + Shorten(IKey("foo", 100, kTypeValue), IKey("foobar", 200, kTypeValue))); // When limit user key is prefix of start user key - ASSERT_EQ(IKey("foobar", 100, kTypeValue), - Shorten(IKey("foobar", 100, kTypeValue), - IKey("foo", 200, kTypeValue))); + ASSERT_EQ( + IKey("foobar", 100, kTypeValue), + Shorten(IKey("foobar", 100, kTypeValue), IKey("foo", 200, kTypeValue))); } TEST(FormatTest, InternalKeyShortestSuccessor) { @@ -105,8 +111,18 @@ TEST(FormatTest, InternalKeyShortestSuccessor) { ShortSuccessor(IKey("\xff\xff", 100, kTypeValue))); } -} // namespace leveldb +TEST(FormatTest, ParsedInternalKeyDebugString) { + ParsedInternalKey key("The \"key\" in 'single quotes'", 42, kTypeValue); + + ASSERT_EQ("'The \"key\" in 'single quotes'' @ 42 : 1", key.DebugString()); +} + +TEST(FormatTest, InternalKeyDebugString) { + InternalKey key("The \"key\" in 'single quotes'", 42, kTypeValue); + ASSERT_EQ("'The \"key\" in 'single quotes'' @ 42 : 1", key.DebugString()); -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); + InternalKey invalid_key; + ASSERT_EQ("(bad)", invalid_key.DebugString()); } + +} // namespace leveldb diff --git a/db/dumpfile.cc b/db/dumpfile.cc index 941988b..6085475 100644 --- a/db/dumpfile.cc +++ b/db/dumpfile.cc @@ -4,7 +4,7 @@ #include "leveldb/dumpfile.h" -#include <stdio.h> +#include <cstdio> #include "db/dbformat.h" #include "db/filename.h" @@ -38,8 +38,7 @@ bool GuessType(const std::string& fname, FileType* type) { // Notified when log reader encounters corruption. class CorruptionReporter : public log::Reader::Reporter { public: - WritableFile* dst_; - virtual void Corruption(size_t bytes, const Status& status) { + void Corruption(size_t bytes, const Status& status) override { std::string r = "corruption: "; AppendNumberTo(&r, bytes); r += " bytes; "; @@ -47,6 +46,8 @@ class CorruptionReporter : public log::Reader::Reporter { r.push_back('\n'); dst_->Append(r); } + + WritableFile* dst_; }; // Print contents of a log file. (*func)() is called on every record. @@ -73,8 +74,7 @@ Status PrintLogContents(Env* env, const std::string& fname, // Called on every item found in a WriteBatch. class WriteBatchItemPrinter : public WriteBatch::Handler { public: - WritableFile* dst_; - virtual void Put(const Slice& key, const Slice& value) { + void Put(const Slice& key, const Slice& value) override { std::string r = " put '"; AppendEscapedStringTo(&r, key); r += "' '"; @@ -82,14 +82,15 @@ class WriteBatchItemPrinter : public WriteBatch::Handler { r += "'\n"; dst_->Append(r); } - virtual void Delete(const Slice& key) { + void Delete(const Slice& key) override { std::string r = " del '"; AppendEscapedStringTo(&r, key); r += "'\n"; dst_->Append(r); } -}; + WritableFile* dst_; +}; // Called on every log record (each one of which is a WriteBatch) // found in a kLogFile. @@ -216,9 +217,12 @@ Status DumpFile(Env* env, const std::string& fname, WritableFile* dst) { return Status::InvalidArgument(fname + ": unknown file type"); } switch (ftype) { - case kLogFile: return DumpLog(env, fname, dst); - case kDescriptorFile: return DumpDescriptor(env, fname, dst); - case kTableFile: return DumpTable(env, fname, dst); + case kLogFile: + return DumpLog(env, fname, dst); + case kDescriptorFile: + return DumpDescriptor(env, fname, dst); + case kTableFile: + return DumpTable(env, fname, dst); default: break; } diff --git a/db/fault_injection_test.cc b/db/fault_injection_test.cc index 7894999..ef864a4 100644 --- a/db/fault_injection_test.cc +++ b/db/fault_injection_test.cc @@ -9,12 +9,13 @@ #include <map> #include <set> -#include "leveldb/db.h" +#include "gtest/gtest.h" #include "db/db_impl.h" #include "db/filename.h" #include "db/log_format.h" #include "db/version_set.h" #include "leveldb/cache.h" +#include "leveldb/db.h" #include "leveldb/env.h" #include "leveldb/table.h" #include "leveldb/write_batch.h" @@ -22,7 +23,6 @@ #include "port/thread_annotations.h" #include "util/logging.h" #include "util/mutexlock.h" -#include "util/testharness.h" #include "util/testutil.h" namespace leveldb { @@ -56,8 +56,7 @@ Status Truncate(const std::string& filename, uint64_t length) { SequentialFile* orig_file; Status s = env->NewSequentialFile(filename, &orig_file); - if (!s.ok()) - return s; + if (!s.ok()) return s; char* scratch = new char[length]; leveldb::Slice result; @@ -73,7 +72,7 @@ Status Truncate(const std::string& filename, uint64_t length) { if (s.ok()) { s = env->RenameFile(tmp_name, filename); } else { - env->DeleteFile(tmp_name); + env->RemoveFile(tmp_name); } } } @@ -85,15 +84,15 @@ Status Truncate(const std::string& filename, uint64_t length) { struct FileState { std::string filename_; - ssize_t pos_; - ssize_t pos_at_last_sync_; - ssize_t pos_at_last_flush_; + int64_t pos_; + int64_t pos_at_last_sync_; + int64_t pos_at_last_flush_; FileState(const std::string& filename) : filename_(filename), pos_(-1), pos_at_last_sync_(-1), - pos_at_last_flush_(-1) { } + pos_at_last_flush_(-1) {} FileState() : pos_(-1), pos_at_last_sync_(-1), pos_at_last_flush_(-1) {} @@ -108,14 +107,13 @@ struct FileState { // is written to or sync'ed. class TestWritableFile : public WritableFile { public: - TestWritableFile(const FileState& state, - WritableFile* f, + TestWritableFile(const FileState& state, WritableFile* f, FaultInjectionTestEnv* env); - virtual ~TestWritableFile(); - virtual Status Append(const Slice& data); - virtual Status Close(); - virtual Status Flush(); - virtual Status Sync(); + ~TestWritableFile() override; + Status Append(const Slice& data) override; + Status Close() override; + Status Flush() override; + Status Sync() override; private: FileState state_; @@ -130,17 +128,17 @@ class FaultInjectionTestEnv : public EnvWrapper { public: FaultInjectionTestEnv() : EnvWrapper(Env::Default()), filesystem_active_(true) {} - virtual ~FaultInjectionTestEnv() { } - virtual Status NewWritableFile(const std::string& fname, - WritableFile** result); - virtual Status NewAppendableFile(const std::string& fname, - WritableFile** result); - virtual Status DeleteFile(const std::string& f); - virtual Status RenameFile(const std::string& s, const std::string& t); + ~FaultInjectionTestEnv() override = default; + Status NewWritableFile(const std::string& fname, + WritableFile** result) override; + Status NewAppendableFile(const std::string& fname, + WritableFile** result) override; + Status RemoveFile(const std::string& f) override; + Status RenameFile(const std::string& s, const std::string& t) override; void WritableFileClosed(const FileState& state); Status DropUnsyncedFileData(); - Status DeleteFilesCreatedAfterLastDirSync(); + Status RemoveFilesCreatedAfterLastDirSync(); void DirWasSynced(); bool IsFileCreatedSinceLastDirSync(const std::string& filename); void ResetState(); @@ -165,13 +163,9 @@ class FaultInjectionTestEnv : public EnvWrapper { bool filesystem_active_ GUARDED_BY(mutex_); // Record flushes, syncs, writes }; -TestWritableFile::TestWritableFile(const FileState& state, - WritableFile* f, +TestWritableFile::TestWritableFile(const FileState& state, WritableFile* f, FaultInjectionTestEnv* env) - : state_(state), - target_(f), - writable_file_opened_(true), - env_(env) { + : state_(state), target_(f), writable_file_opened_(true), env_(env) { assert(f != nullptr); } @@ -274,10 +268,11 @@ Status FaultInjectionTestEnv::NewAppendableFile(const std::string& fname, Status FaultInjectionTestEnv::DropUnsyncedFileData() { Status s; MutexLock l(&mutex_); - for (std::map<std::string, FileState>::const_iterator it = - db_file_state_.begin(); - s.ok() && it != db_file_state_.end(); ++it) { - const FileState& state = it->second; + for (const auto& kvp : db_file_state_) { + if (!s.ok()) { + break; + } + const FileState& state = kvp.second; if (!state.IsFullySynced()) { s = state.DropUnsyncedData(); } @@ -303,9 +298,9 @@ void FaultInjectionTestEnv::UntrackFile(const std::string& f) { new_files_since_last_dir_sync_.erase(f); } -Status FaultInjectionTestEnv::DeleteFile(const std::string& f) { - Status s = EnvWrapper::DeleteFile(f); - ASSERT_OK(s); +Status FaultInjectionTestEnv::RemoveFile(const std::string& f) { + Status s = EnvWrapper::RemoveFile(f); + EXPECT_LEVELDB_OK(s); if (s.ok()) { UntrackFile(f); } @@ -340,18 +335,20 @@ void FaultInjectionTestEnv::ResetState() { SetFilesystemActive(true); } -Status FaultInjectionTestEnv::DeleteFilesCreatedAfterLastDirSync() { - // Because DeleteFile access this container make a copy to avoid deadlock +Status FaultInjectionTestEnv::RemoveFilesCreatedAfterLastDirSync() { + // Because RemoveFile access this container make a copy to avoid deadlock mutex_.Lock(); std::set<std::string> new_files(new_files_since_last_dir_sync_.begin(), new_files_since_last_dir_sync_.end()); mutex_.Unlock(); - Status s; - std::set<std::string>::const_iterator it; - for (it = new_files.begin(); s.ok() && it != new_files.end(); ++it) { - s = DeleteFile(*it); + Status status; + for (const auto& new_file : new_files) { + Status remove_status = RemoveFile(new_file); + if (!remove_status.ok() && status.ok()) { + status = std::move(remove_status); + } } - return s; + return status; } void FaultInjectionTestEnv::WritableFileClosed(const FileState& state) { @@ -360,11 +357,11 @@ void FaultInjectionTestEnv::WritableFileClosed(const FileState& state) { } Status FileState::DropUnsyncedData() const { - ssize_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_; + int64_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_; return Truncate(filename_, sync_pos); } -class FaultInjectionTest { +class FaultInjectionTest : public testing::Test { public: enum ExpectedVerifResult { VAL_EXPECT_NO_ERROR, VAL_EXPECT_ERROR }; enum ResetMethod { RESET_DROP_UNSYNCED_DATA, RESET_DELETE_UNSYNCED_FILES }; @@ -379,7 +376,7 @@ class FaultInjectionTest { : env_(new FaultInjectionTestEnv), tiny_cache_(NewLRUCache(100)), db_(nullptr) { - dbname_ = test::TmpDir() + "/fault_test"; + dbname_ = testing::TempDir() + "fault_test"; DestroyDB(dbname_, Options()); // Destroy any db from earlier run options_.reuse_logs = true; options_.env = env_; @@ -395,9 +392,7 @@ class FaultInjectionTest { delete env_; } - void ReuseLogs(bool reuse) { - options_.reuse_logs = reuse; - } + void ReuseLogs(bool reuse) { options_.reuse_logs = reuse; } void Build(int start_idx, int num_vals) { std::string key_space, value_space; @@ -407,7 +402,7 @@ class FaultInjectionTest { batch.Clear(); batch.Put(key, Value(i, &value_space)); WriteOptions options; - ASSERT_OK(db_->Write(options, &batch)); + ASSERT_LEVELDB_OK(db_->Write(options, &batch)); } } @@ -429,10 +424,10 @@ class FaultInjectionTest { s = ReadValue(i, &val); if (expected == VAL_EXPECT_NO_ERROR) { if (s.ok()) { - ASSERT_EQ(value_space, val); + EXPECT_EQ(value_space, val); } } else if (s.ok()) { - fprintf(stderr, "Expected an error at %d, but was OK\n", i); + std::fprintf(stderr, "Expected an error at %d, but was OK\n", i); s = Status::IOError(dbname_, "Expected value error:"); } else { s = Status::OK(); // An expected error @@ -444,7 +439,7 @@ class FaultInjectionTest { // Return the ith key Slice Key(int i, std::string* storage) const { char buf[100]; - snprintf(buf, sizeof(buf), "%016d", i); + std::snprintf(buf, sizeof(buf), "%016d", i); storage->assign(buf, strlen(buf)); return Slice(*storage); } @@ -469,9 +464,8 @@ class FaultInjectionTest { void DeleteAllData() { Iterator* iter = db_->NewIterator(ReadOptions()); - WriteOptions options; for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { - ASSERT_OK(db_->Delete(WriteOptions(), iter->key())); + ASSERT_LEVELDB_OK(db_->Delete(WriteOptions(), iter->key())); } delete iter; @@ -480,10 +474,10 @@ class FaultInjectionTest { void ResetDBState(ResetMethod reset_method) { switch (reset_method) { case RESET_DROP_UNSYNCED_DATA: - ASSERT_OK(env_->DropUnsyncedFileData()); + ASSERT_LEVELDB_OK(env_->DropUnsyncedFileData()); break; case RESET_DELETE_UNSYNCED_FILES: - ASSERT_OK(env_->DeleteFilesCreatedAfterLastDirSync()); + ASSERT_LEVELDB_OK(env_->RemoveFilesCreatedAfterLastDirSync()); break; default: assert(false); @@ -498,35 +492,34 @@ class FaultInjectionTest { } void PartialCompactTestReopenWithFault(ResetMethod reset_method, - int num_pre_sync, - int num_post_sync) { + int num_pre_sync, int num_post_sync) { env_->SetFilesystemActive(false); CloseDB(); ResetDBState(reset_method); - ASSERT_OK(OpenDB()); - ASSERT_OK(Verify(0, num_pre_sync, FaultInjectionTest::VAL_EXPECT_NO_ERROR)); - ASSERT_OK(Verify(num_pre_sync, num_post_sync, FaultInjectionTest::VAL_EXPECT_ERROR)); + ASSERT_LEVELDB_OK(OpenDB()); + ASSERT_LEVELDB_OK( + Verify(0, num_pre_sync, FaultInjectionTest::VAL_EXPECT_NO_ERROR)); + ASSERT_LEVELDB_OK(Verify(num_pre_sync, num_post_sync, + FaultInjectionTest::VAL_EXPECT_ERROR)); } - void NoWriteTestPreFault() { - } + void NoWriteTestPreFault() {} void NoWriteTestReopenWithFault(ResetMethod reset_method) { CloseDB(); ResetDBState(reset_method); - ASSERT_OK(OpenDB()); + ASSERT_LEVELDB_OK(OpenDB()); } void DoTest() { Random rnd(0); - ASSERT_OK(OpenDB()); + ASSERT_LEVELDB_OK(OpenDB()); for (size_t idx = 0; idx < kNumIterations; idx++) { int num_pre_sync = rnd.Uniform(kMaxNumValues); int num_post_sync = rnd.Uniform(kMaxNumValues); PartialCompactTestPreFault(num_pre_sync, num_post_sync); - PartialCompactTestReopenWithFault(RESET_DROP_UNSYNCED_DATA, - num_pre_sync, + PartialCompactTestReopenWithFault(RESET_DROP_UNSYNCED_DATA, num_pre_sync, num_post_sync); NoWriteTestPreFault(); @@ -536,8 +529,7 @@ class FaultInjectionTest { // No new files created so we expect all values since no files will be // dropped. PartialCompactTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES, - num_pre_sync + num_post_sync, - 0); + num_pre_sync + num_post_sync, 0); NoWriteTestPreFault(); NoWriteTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES); @@ -545,18 +537,14 @@ class FaultInjectionTest { } }; -TEST(FaultInjectionTest, FaultTestNoLogReuse) { +TEST_F(FaultInjectionTest, FaultTestNoLogReuse) { ReuseLogs(false); DoTest(); } -TEST(FaultInjectionTest, FaultTestWithLogReuse) { +TEST_F(FaultInjectionTest, FaultTestWithLogReuse) { ReuseLogs(true); DoTest(); } } // namespace leveldb - -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); -} diff --git a/db/filename.cc b/db/filename.cc index 6539bbe..e526249 100644 --- a/db/filename.cc +++ b/db/filename.cc @@ -2,9 +2,11 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -#include <ctype.h> -#include <stdio.h> #include "db/filename.h" + +#include <cassert> +#include <cstdio> + #include "db/dbformat.h" #include "leveldb/env.h" #include "util/logging.h" @@ -18,9 +20,8 @@ Status WriteStringToFileSync(Env* env, const Slice& data, static std::string MakeFileName(const std::string& dbname, uint64_t number, const char* suffix) { char buf[100]; - snprintf(buf, sizeof(buf), "/%06llu.%s", - static_cast<unsigned long long>(number), - suffix); + std::snprintf(buf, sizeof(buf), "/%06llu.%s", + static_cast<unsigned long long>(number), suffix); return dbname + buf; } @@ -42,8 +43,8 @@ std::string SSTTableFileName(const std::string& dbname, uint64_t number) { std::string DescriptorFileName(const std::string& dbname, uint64_t number) { assert(number > 0); char buf[100]; - snprintf(buf, sizeof(buf), "/MANIFEST-%06llu", - static_cast<unsigned long long>(number)); + std::snprintf(buf, sizeof(buf), "/MANIFEST-%06llu", + static_cast<unsigned long long>(number)); return dbname + buf; } @@ -51,9 +52,7 @@ std::string CurrentFileName(const std::string& dbname) { return dbname + "/CURRENT"; } -std::string LockFileName(const std::string& dbname) { - return dbname + "/LOCK"; -} +std::string LockFileName(const std::string& dbname) { return dbname + "/LOCK"; } std::string TempFileName(const std::string& dbname, uint64_t number) { assert(number > 0); @@ -69,7 +68,6 @@ std::string OldInfoLogFileName(const std::string& dbname) { return dbname + "/LOG.old"; } - // Owned filenames have the form: // dbname/CURRENT // dbname/LOCK @@ -77,8 +75,7 @@ std::string OldInfoLogFileName(const std::string& dbname) { // dbname/LOG.old // dbname/MANIFEST-[0-9]+ // dbname/[0-9]+.(log|sst|ldb) -bool ParseFileName(const std::string& filename, - uint64_t* number, +bool ParseFileName(const std::string& filename, uint64_t* number, FileType* type) { Slice rest(filename); if (rest == "CURRENT") { @@ -136,7 +133,7 @@ Status SetCurrentFile(Env* env, const std::string& dbname, s = env->RenameFile(tmp, CurrentFileName(dbname)); } if (!s.ok()) { - env->DeleteFile(tmp); + env->RemoveFile(tmp); } return s; } diff --git a/db/filename.h b/db/filename.h index 62cb3ef..563c6d8 100644 --- a/db/filename.h +++ b/db/filename.h @@ -7,8 +7,9 @@ #ifndef STORAGE_LEVELDB_DB_FILENAME_H_ #define STORAGE_LEVELDB_DB_FILENAME_H_ -#include <stdint.h> +#include <cstdint> #include <string> + #include "leveldb/slice.h" #include "leveldb/status.h" #include "port/port.h" @@ -69,8 +70,7 @@ std::string OldInfoLogFileName(const std::string& dbname); // If filename is a leveldb file, store the type of the file in *type. // The number encoded in the filename is stored in *number. If the // filename was successfully parsed, returns true. Else return false. -bool ParseFileName(const std::string& filename, - uint64_t* number, +bool ParseFileName(const std::string& filename, uint64_t* number, FileType* type); // Make the CURRENT file point to the descriptor file with the diff --git a/db/filename_test.cc b/db/filename_test.cc index 0bde538..9ac0111 100644 --- a/db/filename_test.cc +++ b/db/filename_test.cc @@ -4,15 +4,13 @@ #include "db/filename.h" +#include "gtest/gtest.h" #include "db/dbformat.h" #include "port/port.h" #include "util/logging.h" -#include "util/testharness.h" namespace leveldb { -class FileNameTest { }; - TEST(FileNameTest, Parse) { Slice db; FileType type; @@ -24,17 +22,17 @@ TEST(FileNameTest, Parse) { uint64_t number; FileType type; } cases[] = { - { "100.log", 100, kLogFile }, - { "0.log", 0, kLogFile }, - { "0.sst", 0, kTableFile }, - { "0.ldb", 0, kTableFile }, - { "CURRENT", 0, kCurrentFile }, - { "LOCK", 0, kDBLockFile }, - { "MANIFEST-2", 2, kDescriptorFile }, - { "MANIFEST-7", 7, kDescriptorFile }, - { "LOG", 0, kInfoLogFile }, - { "LOG.old", 0, kInfoLogFile }, - { "18446744073709551615.log", 18446744073709551615ull, kLogFile }, + {"100.log", 100, kLogFile}, + {"0.log", 0, kLogFile}, + {"0.sst", 0, kTableFile}, + {"0.ldb", 0, kTableFile}, + {"CURRENT", 0, kCurrentFile}, + {"LOCK", 0, kDBLockFile}, + {"MANIFEST-2", 2, kDescriptorFile}, + {"MANIFEST-7", 7, kDescriptorFile}, + {"LOG", 0, kInfoLogFile}, + {"LOG.old", 0, kInfoLogFile}, + {"18446744073709551615.log", 18446744073709551615ull, kLogFile}, }; for (int i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { std::string f = cases[i].fname; @@ -44,30 +42,28 @@ TEST(FileNameTest, Parse) { } // Errors - static const char* errors[] = { - "", - "foo", - "foo-dx-100.log", - ".log", - "", - "manifest", - "CURREN", - "CURRENTX", - "MANIFES", - "MANIFEST", - "MANIFEST-", - "XMANIFEST-3", - "MANIFEST-3x", - "LOC", - "LOCKx", - "LO", - "LOGx", - "18446744073709551616.log", - "184467440737095516150.log", - "100", - "100.", - "100.lop" - }; + static const char* errors[] = {"", + "foo", + "foo-dx-100.log", + ".log", + "", + "manifest", + "CURREN", + "CURRENTX", + "MANIFES", + "MANIFEST", + "MANIFEST-", + "XMANIFEST-3", + "MANIFEST-3x", + "LOC", + "LOCKx", + "LO", + "LOGx", + "18446744073709551616.log", + "184467440737095516150.log", + "100", + "100.", + "100.lop"}; for (int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) { std::string f = errors[i]; ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f; @@ -129,7 +125,3 @@ TEST(FileNameTest, Construction) { } } // namespace leveldb - -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); -} diff --git a/db/leveldbutil.cc b/db/leveldbutil.cc index 9f4b7dd..95ee897 100644 --- a/db/leveldbutil.cc +++ b/db/leveldbutil.cc @@ -2,7 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -#include <stdio.h> +#include <cstdio> + #include "leveldb/dumpfile.h" #include "leveldb/env.h" #include "leveldb/status.h" @@ -12,13 +13,13 @@ namespace { class StdoutPrinter : public WritableFile { public: - virtual Status Append(const Slice& data) { + Status Append(const Slice& data) override { fwrite(data.data(), 1, data.size(), stdout); return Status::OK(); } - virtual Status Close() { return Status::OK(); } - virtual Status Flush() { return Status::OK(); } - virtual Status Sync() { return Status::OK(); } + Status Close() override { return Status::OK(); } + Status Flush() override { return Status::OK(); } + Status Sync() override { return Status::OK(); } }; bool HandleDumpCommand(Env* env, char** files, int num) { @@ -27,7 +28,7 @@ bool HandleDumpCommand(Env* env, char** files, int num) { for (int i = 0; i < num; i++) { Status s = DumpFile(env, files[i], &printer); if (!s.ok()) { - fprintf(stderr, "%s\n", s.ToString().c_str()); + std::fprintf(stderr, "%s\n", s.ToString().c_str()); ok = false; } } @@ -38,11 +39,10 @@ bool HandleDumpCommand(Env* env, char** files, int num) { } // namespace leveldb static void Usage() { - fprintf( + std::fprintf( stderr, "Usage: leveldbutil command...\n" - " dump files... -- dump contents of specified files\n" - ); + " dump files... -- dump contents of specified files\n"); } int main(int argc, char** argv) { @@ -54,7 +54,7 @@ int main(int argc, char** argv) { } else { std::string command = argv[1]; if (command == "dump") { - ok = leveldb::HandleDumpCommand(env, argv+2, argc-2); + ok = leveldb::HandleDumpCommand(env, argv + 2, argc - 2); } else { Usage(); ok = false; diff --git a/db/log_reader.cc b/db/log_reader.cc index 19c4df6..9880279 100644 --- a/db/log_reader.cc +++ b/db/log_reader.cc @@ -4,7 +4,8 @@ #include "db/log_reader.h" -#include <stdio.h> +#include <cstdio> + #include "leveldb/env.h" #include "util/coding.h" #include "util/crc32c.h" @@ -12,8 +13,7 @@ namespace leveldb { namespace log { -Reader::Reporter::~Reporter() { -} +Reader::Reporter::~Reporter() = default; Reader::Reader(SequentialFile* file, Reporter* reporter, bool checksum, uint64_t initial_offset) @@ -26,12 +26,9 @@ Reader::Reader(SequentialFile* file, Reporter* reporter, bool checksum, last_record_offset_(0), end_of_buffer_offset_(0), initial_offset_(initial_offset), - resyncing_(initial_offset > 0) { -} + resyncing_(initial_offset > 0) {} -Reader::~Reader() { - delete[] backing_store_; -} +Reader::~Reader() { delete[] backing_store_; } bool Reader::SkipToInitialBlock() { const size_t offset_in_block = initial_offset_ % kBlockSize; @@ -163,7 +160,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) { default: { char buf[40]; - snprintf(buf, sizeof(buf), "unknown record type %u", record_type); + std::snprintf(buf, sizeof(buf), "unknown record type %u", record_type); ReportCorruption( (fragment.size() + (in_fragmented_record ? scratch->size() : 0)), buf); @@ -176,9 +173,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) { return false; } -uint64_t Reader::LastRecordOffset() { - return last_record_offset_; -} +uint64_t Reader::LastRecordOffset() { return last_record_offset_; } void Reader::ReportCorruption(uint64_t bytes, const char* reason) { ReportDrop(bytes, Status::Corruption(reason)); diff --git a/db/log_reader.h b/db/log_reader.h index 7dcce8e..ba711f8 100644 --- a/db/log_reader.h +++ b/db/log_reader.h @@ -5,7 +5,7 @@ #ifndef STORAGE_LEVELDB_DB_LOG_READER_H_ #define STORAGE_LEVELDB_DB_LOG_READER_H_ -#include <stdint.h> +#include <cstdint> #include "db/log_format.h" #include "leveldb/slice.h" @@ -24,7 +24,7 @@ class Reader { public: virtual ~Reporter(); - // Some corruption was detected. "size" is the approximate number + // Some corruption was detected. "bytes" is the approximate number // of bytes dropped due to the corruption. virtual void Corruption(size_t bytes, const Status& status) = 0; }; @@ -43,6 +43,9 @@ class Reader { Reader(SequentialFile* file, Reporter* reporter, bool checksum, uint64_t initial_offset); + Reader(const Reader&) = delete; + Reader& operator=(const Reader&) = delete; + ~Reader(); // Read the next record into *record. Returns true if read @@ -58,26 +61,6 @@ class Reader { uint64_t LastRecordOffset(); private: - SequentialFile* const file_; - Reporter* const reporter_; - bool const checksum_; - char* const backing_store_; - Slice buffer_; - bool eof_; // Last Read() indicated EOF by returning < kBlockSize - - // Offset of the last record returned by ReadRecord. - uint64_t last_record_offset_; - // Offset of the first location past the end of buffer_. - uint64_t end_of_buffer_offset_; - - // Offset at which to start looking for the first record to return - uint64_t const initial_offset_; - - // True if we are resynchronizing after a seek (initial_offset_ > 0). In - // particular, a run of kMiddleType and kLastType records can be silently - // skipped in this mode - bool resyncing_; - // Extend record types with the following special values enum { kEof = kMaxRecordType + 1, @@ -102,9 +85,25 @@ class Reader { void ReportCorruption(uint64_t bytes, const char* reason); void ReportDrop(uint64_t bytes, const Status& reason); - // No copying allowed - Reader(const Reader&); - void operator=(const Reader&); + SequentialFile* const file_; + Reporter* const reporter_; + bool const checksum_; + char* const backing_store_; + Slice buffer_; + bool eof_; // Last Read() indicated EOF by returning < kBlockSize + + // Offset of the last record returned by ReadRecord. + uint64_t last_record_offset_; + // Offset of the first location past the end of buffer_. + uint64_t end_of_buffer_offset_; + + // Offset at which to start looking for the first record to return + uint64_t const initial_offset_; + + // True if we are resynchronizing after a seek (initial_offset_ > 0). In + // particular, a run of kMiddleType and kLastType records can be silently + // skipped in this mode + bool resyncing_; }; } // namespace log diff --git a/db/log_test.cc b/db/log_test.cc index 48a5928..d55d4dd 100644 --- a/db/log_test.cc +++ b/db/log_test.cc @@ -2,13 +2,13 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "gtest/gtest.h" #include "db/log_reader.h" #include "db/log_writer.h" #include "leveldb/env.h" #include "util/coding.h" #include "util/crc32c.h" #include "util/random.h" -#include "util/testharness.h" namespace leveldb { namespace log { @@ -27,7 +27,7 @@ static std::string BigString(const std::string& partial_string, size_t n) { // Construct a string from a number static std::string NumberString(int n) { char buf[50]; - snprintf(buf, sizeof(buf), "%d.", n); + std::snprintf(buf, sizeof(buf), "%d.", n); return std::string(buf); } @@ -36,88 +36,13 @@ static std::string RandomSkewedString(int i, Random* rnd) { return BigString(NumberString(i), rnd->Skewed(17)); } -class LogTest { - private: - class StringDest : public WritableFile { - public: - std::string contents_; - - virtual Status Close() { return Status::OK(); } - virtual Status Flush() { return Status::OK(); } - virtual Status Sync() { return Status::OK(); } - virtual Status Append(const Slice& slice) { - contents_.append(slice.data(), slice.size()); - return Status::OK(); - } - }; - - class StringSource : public SequentialFile { - public: - Slice contents_; - bool force_error_; - bool returned_partial_; - StringSource() : force_error_(false), returned_partial_(false) { } - - virtual Status Read(size_t n, Slice* result, char* scratch) { - ASSERT_TRUE(!returned_partial_) << "must not Read() after eof/error"; - - if (force_error_) { - force_error_ = false; - returned_partial_ = true; - return Status::Corruption("read error"); - } - - if (contents_.size() < n) { - n = contents_.size(); - returned_partial_ = true; - } - *result = Slice(contents_.data(), n); - contents_.remove_prefix(n); - return Status::OK(); - } - - virtual Status Skip(uint64_t n) { - if (n > contents_.size()) { - contents_.clear(); - return Status::NotFound("in-memory file skipped past end"); - } - - contents_.remove_prefix(n); - - return Status::OK(); - } - }; - - class ReportCollector : public Reader::Reporter { - public: - size_t dropped_bytes_; - std::string message_; - - ReportCollector() : dropped_bytes_(0) { } - virtual void Corruption(size_t bytes, const Status& status) { - dropped_bytes_ += bytes; - message_.append(status.ToString()); - } - }; - - StringDest dest_; - StringSource source_; - ReportCollector report_; - bool reading_; - Writer* writer_; - Reader* reader_; - - // Record metadata for testing initial offset functionality - static size_t initial_offset_record_sizes_[]; - static uint64_t initial_offset_last_record_offsets_[]; - static int num_initial_offset_records_; - +class LogTest : public testing::Test { public: - LogTest() : reading_(false), - writer_(new Writer(&dest_)), - reader_(new Reader(&source_, &report_, true/*checksum*/, - 0/*initial_offset*/)) { - } + LogTest() + : reading_(false), + writer_(new Writer(&dest_)), + reader_(new Reader(&source_, &report_, true /*checksum*/, + 0 /*initial_offset*/)) {} ~LogTest() { delete writer_; @@ -134,9 +59,7 @@ class LogTest { writer_->AddRecord(Slice(msg)); } - size_t WrittenBytes() const { - return dest_.contents_.size(); - } + size_t WrittenBytes() const { return dest_.contents_.size(); } std::string Read() { if (!reading_) { @@ -166,22 +89,16 @@ class LogTest { void FixChecksum(int header_offset, int len) { // Compute crc of type/len/data - uint32_t crc = crc32c::Value(&dest_.contents_[header_offset+6], 1 + len); + uint32_t crc = crc32c::Value(&dest_.contents_[header_offset + 6], 1 + len); crc = crc32c::Mask(crc); EncodeFixed32(&dest_.contents_[header_offset], crc); } - void ForceError() { - source_.force_error_ = true; - } + void ForceError() { source_.force_error_ = true; } - size_t DroppedBytes() const { - return report_.dropped_bytes_; - } + size_t DroppedBytes() const { return report_.dropped_bytes_; } - std::string ReportMessage() const { - return report_.message_; - } + std::string ReportMessage() const { return report_.message_; } // Returns OK iff recorded error message contains "msg" std::string MatchError(const std::string& msg) const { @@ -202,14 +119,14 @@ class LogTest { void StartReadingAt(uint64_t initial_offset) { delete reader_; - reader_ = new Reader(&source_, &report_, true/*checksum*/, initial_offset); + reader_ = new Reader(&source_, &report_, true /*checksum*/, initial_offset); } void CheckOffsetPastEndReturnsNoRecords(uint64_t offset_past_end) { WriteInitialOffsetLog(); reading_ = true; source_.contents_ = Slice(dest_.contents_); - Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/, + Reader* offset_reader = new Reader(&source_, &report_, true /*checksum*/, WrittenBytes() + offset_past_end); Slice record; std::string scratch; @@ -222,8 +139,8 @@ class LogTest { WriteInitialOffsetLog(); reading_ = true; source_.contents_ = Slice(dest_.contents_); - Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/, - initial_offset); + Reader* offset_reader = + new Reader(&source_, &report_, true /*checksum*/, initial_offset); // Read all records from expected_record_offset through the last one. ASSERT_LT(expected_record_offset, num_initial_offset_records_); @@ -240,38 +157,110 @@ class LogTest { } delete offset_reader; } + + private: + class StringDest : public WritableFile { + public: + Status Close() override { return Status::OK(); } + Status Flush() override { return Status::OK(); } + Status Sync() override { return Status::OK(); } + Status Append(const Slice& slice) override { + contents_.append(slice.data(), slice.size()); + return Status::OK(); + } + + std::string contents_; + }; + + class StringSource : public SequentialFile { + public: + StringSource() : force_error_(false), returned_partial_(false) {} + + Status Read(size_t n, Slice* result, char* scratch) override { + EXPECT_TRUE(!returned_partial_) << "must not Read() after eof/error"; + + if (force_error_) { + force_error_ = false; + returned_partial_ = true; + return Status::Corruption("read error"); + } + + if (contents_.size() < n) { + n = contents_.size(); + returned_partial_ = true; + } + *result = Slice(contents_.data(), n); + contents_.remove_prefix(n); + return Status::OK(); + } + + Status Skip(uint64_t n) override { + if (n > contents_.size()) { + contents_.clear(); + return Status::NotFound("in-memory file skipped past end"); + } + + contents_.remove_prefix(n); + + return Status::OK(); + } + + Slice contents_; + bool force_error_; + bool returned_partial_; + }; + + class ReportCollector : public Reader::Reporter { + public: + ReportCollector() : dropped_bytes_(0) {} + void Corruption(size_t bytes, const Status& status) override { + dropped_bytes_ += bytes; + message_.append(status.ToString()); + } + + size_t dropped_bytes_; + std::string message_; + }; + + // Record metadata for testing initial offset functionality + static size_t initial_offset_record_sizes_[]; + static uint64_t initial_offset_last_record_offsets_[]; + static int num_initial_offset_records_; + + StringDest dest_; + StringSource source_; + ReportCollector report_; + bool reading_; + Writer* writer_; + Reader* reader_; +}; + +size_t LogTest::initial_offset_record_sizes_[] = { + 10000, // Two sizable records in first block + 10000, + 2 * log::kBlockSize - 1000, // Span three blocks + 1, + 13716, // Consume all but two bytes of block 3. + log::kBlockSize - kHeaderSize, // Consume the entirety of block 4. }; -size_t LogTest::initial_offset_record_sizes_[] = - {10000, // Two sizable records in first block - 10000, - 2 * log::kBlockSize - 1000, // Span three blocks - 1, - 13716, // Consume all but two bytes of block 3. - log::kBlockSize - kHeaderSize, // Consume the entirety of block 4. - }; - -uint64_t LogTest::initial_offset_last_record_offsets_[] = - {0, - kHeaderSize + 10000, - 2 * (kHeaderSize + 10000), - 2 * (kHeaderSize + 10000) + - (2 * log::kBlockSize - 1000) + 3 * kHeaderSize, - 2 * (kHeaderSize + 10000) + - (2 * log::kBlockSize - 1000) + 3 * kHeaderSize - + kHeaderSize + 1, - 3 * log::kBlockSize, - }; +uint64_t LogTest::initial_offset_last_record_offsets_[] = { + 0, + kHeaderSize + 10000, + 2 * (kHeaderSize + 10000), + 2 * (kHeaderSize + 10000) + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize, + 2 * (kHeaderSize + 10000) + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize + + kHeaderSize + 1, + 3 * log::kBlockSize, +}; // LogTest::initial_offset_last_record_offsets_ must be defined before this. int LogTest::num_initial_offset_records_ = - sizeof(LogTest::initial_offset_last_record_offsets_)/sizeof(uint64_t); + sizeof(LogTest::initial_offset_last_record_offsets_) / sizeof(uint64_t); -TEST(LogTest, Empty) { - ASSERT_EQ("EOF", Read()); -} +TEST_F(LogTest, Empty) { ASSERT_EQ("EOF", Read()); } -TEST(LogTest, ReadWrite) { +TEST_F(LogTest, ReadWrite) { Write("foo"); Write("bar"); Write(""); @@ -284,7 +273,7 @@ TEST(LogTest, ReadWrite) { ASSERT_EQ("EOF", Read()); // Make sure reads at eof work } -TEST(LogTest, ManyBlocks) { +TEST_F(LogTest, ManyBlocks) { for (int i = 0; i < 100000; i++) { Write(NumberString(i)); } @@ -294,7 +283,7 @@ TEST(LogTest, ManyBlocks) { ASSERT_EQ("EOF", Read()); } -TEST(LogTest, Fragmentation) { +TEST_F(LogTest, Fragmentation) { Write("small"); Write(BigString("medium", 50000)); Write(BigString("large", 100000)); @@ -304,9 +293,9 @@ TEST(LogTest, Fragmentation) { ASSERT_EQ("EOF", Read()); } -TEST(LogTest, MarginalTrailer) { +TEST_F(LogTest, MarginalTrailer) { // Make a trailer that is exactly the same length as an empty record. - const int n = kBlockSize - 2*kHeaderSize; + const int n = kBlockSize - 2 * kHeaderSize; Write(BigString("foo", n)); ASSERT_EQ(kBlockSize - kHeaderSize, WrittenBytes()); Write(""); @@ -317,9 +306,9 @@ TEST(LogTest, MarginalTrailer) { ASSERT_EQ("EOF", Read()); } -TEST(LogTest, MarginalTrailer2) { +TEST_F(LogTest, MarginalTrailer2) { // Make a trailer that is exactly the same length as an empty record. - const int n = kBlockSize - 2*kHeaderSize; + const int n = kBlockSize - 2 * kHeaderSize; Write(BigString("foo", n)); ASSERT_EQ(kBlockSize - kHeaderSize, WrittenBytes()); Write("bar"); @@ -330,8 +319,8 @@ TEST(LogTest, MarginalTrailer2) { ASSERT_EQ("", ReportMessage()); } -TEST(LogTest, ShortTrailer) { - const int n = kBlockSize - 2*kHeaderSize + 4; +TEST_F(LogTest, ShortTrailer) { + const int n = kBlockSize - 2 * kHeaderSize + 4; Write(BigString("foo", n)); ASSERT_EQ(kBlockSize - kHeaderSize + 4, WrittenBytes()); Write(""); @@ -342,15 +331,15 @@ TEST(LogTest, ShortTrailer) { ASSERT_EQ("EOF", Read()); } -TEST(LogTest, AlignedEof) { - const int n = kBlockSize - 2*kHeaderSize + 4; +TEST_F(LogTest, AlignedEof) { + const int n = kBlockSize - 2 * kHeaderSize + 4; Write(BigString("foo", n)); ASSERT_EQ(kBlockSize - kHeaderSize + 4, WrittenBytes()); ASSERT_EQ(BigString("foo", n), Read()); ASSERT_EQ("EOF", Read()); } -TEST(LogTest, OpenForAppend) { +TEST_F(LogTest, OpenForAppend) { Write("hello"); ReopenForAppend(); Write("world"); @@ -359,7 +348,7 @@ TEST(LogTest, OpenForAppend) { ASSERT_EQ("EOF", Read()); } -TEST(LogTest, RandomRead) { +TEST_F(LogTest, RandomRead) { const int N = 500; Random write_rnd(301); for (int i = 0; i < N; i++) { @@ -374,7 +363,7 @@ TEST(LogTest, RandomRead) { // Tests of all the error paths in log_reader.cc follow: -TEST(LogTest, ReadError) { +TEST_F(LogTest, ReadError) { Write("foo"); ForceError(); ASSERT_EQ("EOF", Read()); @@ -382,7 +371,7 @@ TEST(LogTest, ReadError) { ASSERT_EQ("OK", MatchError("read error")); } -TEST(LogTest, BadRecordType) { +TEST_F(LogTest, BadRecordType) { Write("foo"); // Type is stored in header[6] IncrementByte(6, 100); @@ -392,16 +381,16 @@ TEST(LogTest, BadRecordType) { ASSERT_EQ("OK", MatchError("unknown record type")); } -TEST(LogTest, TruncatedTrailingRecordIsIgnored) { +TEST_F(LogTest, TruncatedTrailingRecordIsIgnored) { Write("foo"); - ShrinkSize(4); // Drop all payload as well as a header byte + ShrinkSize(4); // Drop all payload as well as a header byte ASSERT_EQ("EOF", Read()); // Truncated last record is ignored, not treated as an error. ASSERT_EQ(0, DroppedBytes()); ASSERT_EQ("", ReportMessage()); } -TEST(LogTest, BadLength) { +TEST_F(LogTest, BadLength) { const int kPayloadSize = kBlockSize - kHeaderSize; Write(BigString("bar", kPayloadSize)); Write("foo"); @@ -412,7 +401,7 @@ TEST(LogTest, BadLength) { ASSERT_EQ("OK", MatchError("bad record length")); } -TEST(LogTest, BadLengthAtEndIsIgnored) { +TEST_F(LogTest, BadLengthAtEndIsIgnored) { Write("foo"); ShrinkSize(1); ASSERT_EQ("EOF", Read()); @@ -420,7 +409,7 @@ TEST(LogTest, BadLengthAtEndIsIgnored) { ASSERT_EQ("", ReportMessage()); } -TEST(LogTest, ChecksumMismatch) { +TEST_F(LogTest, ChecksumMismatch) { Write("foo"); IncrementByte(0, 10); ASSERT_EQ("EOF", Read()); @@ -428,7 +417,7 @@ TEST(LogTest, ChecksumMismatch) { ASSERT_EQ("OK", MatchError("checksum mismatch")); } -TEST(LogTest, UnexpectedMiddleType) { +TEST_F(LogTest, UnexpectedMiddleType) { Write("foo"); SetByte(6, kMiddleType); FixChecksum(0, 3); @@ -437,7 +426,7 @@ TEST(LogTest, UnexpectedMiddleType) { ASSERT_EQ("OK", MatchError("missing start")); } -TEST(LogTest, UnexpectedLastType) { +TEST_F(LogTest, UnexpectedLastType) { Write("foo"); SetByte(6, kLastType); FixChecksum(0, 3); @@ -446,7 +435,7 @@ TEST(LogTest, UnexpectedLastType) { ASSERT_EQ("OK", MatchError("missing start")); } -TEST(LogTest, UnexpectedFullType) { +TEST_F(LogTest, UnexpectedFullType) { Write("foo"); Write("bar"); SetByte(6, kFirstType); @@ -457,7 +446,7 @@ TEST(LogTest, UnexpectedFullType) { ASSERT_EQ("OK", MatchError("partial record without end")); } -TEST(LogTest, UnexpectedFirstType) { +TEST_F(LogTest, UnexpectedFirstType) { Write("foo"); Write(BigString("bar", 100000)); SetByte(6, kFirstType); @@ -468,7 +457,7 @@ TEST(LogTest, UnexpectedFirstType) { ASSERT_EQ("OK", MatchError("partial record without end")); } -TEST(LogTest, MissingLastIsIgnored) { +TEST_F(LogTest, MissingLastIsIgnored) { Write(BigString("bar", kBlockSize)); // Remove the LAST block, including header. ShrinkSize(14); @@ -477,7 +466,7 @@ TEST(LogTest, MissingLastIsIgnored) { ASSERT_EQ(0, DroppedBytes()); } -TEST(LogTest, PartialLastIsIgnored) { +TEST_F(LogTest, PartialLastIsIgnored) { Write(BigString("bar", kBlockSize)); // Cause a bad record length in the LAST block. ShrinkSize(1); @@ -486,13 +475,13 @@ TEST(LogTest, PartialLastIsIgnored) { ASSERT_EQ(0, DroppedBytes()); } -TEST(LogTest, SkipIntoMultiRecord) { +TEST_F(LogTest, SkipIntoMultiRecord) { // Consider a fragmented record: // first(R1), middle(R1), last(R1), first(R2) // If initial_offset points to a record after first(R1) but before first(R2) // incomplete fragment errors are not actual errors, and must be suppressed // until a new first or full record is encountered. - Write(BigString("foo", 3*kBlockSize)); + Write(BigString("foo", 3 * kBlockSize)); Write("correct"); StartReadingAt(kBlockSize); @@ -502,7 +491,7 @@ TEST(LogTest, SkipIntoMultiRecord) { ASSERT_EQ("EOF", Read()); } -TEST(LogTest, ErrorJoinsRecords) { +TEST_F(LogTest, ErrorJoinsRecords) { // Consider two fragmented records: // first(R1) last(R1) first(R2) last(R2) // where the middle two fragments disappear. We do not want @@ -514,78 +503,56 @@ TEST(LogTest, ErrorJoinsRecords) { Write("correct"); // Wipe the middle block - for (int offset = kBlockSize; offset < 2*kBlockSize; offset++) { + for (int offset = kBlockSize; offset < 2 * kBlockSize; offset++) { SetByte(offset, 'x'); } ASSERT_EQ("correct", Read()); ASSERT_EQ("EOF", Read()); const size_t dropped = DroppedBytes(); - ASSERT_LE(dropped, 2*kBlockSize + 100); - ASSERT_GE(dropped, 2*kBlockSize); + ASSERT_LE(dropped, 2 * kBlockSize + 100); + ASSERT_GE(dropped, 2 * kBlockSize); } -TEST(LogTest, ReadStart) { - CheckInitialOffsetRecord(0, 0); -} +TEST_F(LogTest, ReadStart) { CheckInitialOffsetRecord(0, 0); } -TEST(LogTest, ReadSecondOneOff) { - CheckInitialOffsetRecord(1, 1); -} +TEST_F(LogTest, ReadSecondOneOff) { CheckInitialOffsetRecord(1, 1); } -TEST(LogTest, ReadSecondTenThousand) { - CheckInitialOffsetRecord(10000, 1); -} +TEST_F(LogTest, ReadSecondTenThousand) { CheckInitialOffsetRecord(10000, 1); } -TEST(LogTest, ReadSecondStart) { - CheckInitialOffsetRecord(10007, 1); -} +TEST_F(LogTest, ReadSecondStart) { CheckInitialOffsetRecord(10007, 1); } -TEST(LogTest, ReadThirdOneOff) { - CheckInitialOffsetRecord(10008, 2); -} +TEST_F(LogTest, ReadThirdOneOff) { CheckInitialOffsetRecord(10008, 2); } -TEST(LogTest, ReadThirdStart) { - CheckInitialOffsetRecord(20014, 2); -} +TEST_F(LogTest, ReadThirdStart) { CheckInitialOffsetRecord(20014, 2); } -TEST(LogTest, ReadFourthOneOff) { - CheckInitialOffsetRecord(20015, 3); -} +TEST_F(LogTest, ReadFourthOneOff) { CheckInitialOffsetRecord(20015, 3); } -TEST(LogTest, ReadFourthFirstBlockTrailer) { +TEST_F(LogTest, ReadFourthFirstBlockTrailer) { CheckInitialOffsetRecord(log::kBlockSize - 4, 3); } -TEST(LogTest, ReadFourthMiddleBlock) { +TEST_F(LogTest, ReadFourthMiddleBlock) { CheckInitialOffsetRecord(log::kBlockSize + 1, 3); } -TEST(LogTest, ReadFourthLastBlock) { +TEST_F(LogTest, ReadFourthLastBlock) { CheckInitialOffsetRecord(2 * log::kBlockSize + 1, 3); } -TEST(LogTest, ReadFourthStart) { +TEST_F(LogTest, ReadFourthStart) { CheckInitialOffsetRecord( 2 * (kHeaderSize + 1000) + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize, 3); } -TEST(LogTest, ReadInitialOffsetIntoBlockPadding) { +TEST_F(LogTest, ReadInitialOffsetIntoBlockPadding) { CheckInitialOffsetRecord(3 * log::kBlockSize - 3, 5); } -TEST(LogTest, ReadEnd) { - CheckOffsetPastEndReturnsNoRecords(0); -} +TEST_F(LogTest, ReadEnd) { CheckOffsetPastEndReturnsNoRecords(0); } -TEST(LogTest, ReadPastEnd) { - CheckOffsetPastEndReturnsNoRecords(5); -} +TEST_F(LogTest, ReadPastEnd) { CheckOffsetPastEndReturnsNoRecords(5); } } // namespace log } // namespace leveldb - -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); -} diff --git a/db/log_writer.cc b/db/log_writer.cc index 74a0327..ad66bfb 100644 --- a/db/log_writer.cc +++ b/db/log_writer.cc @@ -4,7 +4,8 @@ #include "db/log_writer.h" -#include <stdint.h> +#include <cstdint> + #include "leveldb/env.h" #include "util/coding.h" #include "util/crc32c.h" @@ -19,9 +20,7 @@ static void InitTypeCrc(uint32_t* type_crc) { } } -Writer::Writer(WritableFile* dest) - : dest_(dest), - block_offset_(0) { +Writer::Writer(WritableFile* dest) : dest_(dest), block_offset_(0) { InitTypeCrc(type_crc_); } @@ -30,8 +29,7 @@ Writer::Writer(WritableFile* dest, uint64_t dest_length) InitTypeCrc(type_crc_); } -Writer::~Writer() { -} +Writer::~Writer() = default; Status Writer::AddRecord(const Slice& slice) { const char* ptr = slice.data(); @@ -49,7 +47,7 @@ Status Writer::AddRecord(const Slice& slice) { // Switch to a new block if (leftover > 0) { // Fill the trailer (literal below relies on kHeaderSize being 7) - assert(kHeaderSize == 7); + static_assert(kHeaderSize == 7, ""); dest_->Append(Slice("\x00\x00\x00\x00\x00\x00", leftover)); } block_offset_ = 0; @@ -81,30 +79,31 @@ Status Writer::AddRecord(const Slice& slice) { return s; } -Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n) { - assert(n <= 0xffff); // Must fit in two bytes - assert(block_offset_ + kHeaderSize + n <= kBlockSize); +Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr, + size_t length) { + assert(length <= 0xffff); // Must fit in two bytes + assert(block_offset_ + kHeaderSize + length <= kBlockSize); // Format the header char buf[kHeaderSize]; - buf[4] = static_cast<char>(n & 0xff); - buf[5] = static_cast<char>(n >> 8); + buf[4] = static_cast<char>(length & 0xff); + buf[5] = static_cast<char>(length >> 8); buf[6] = static_cast<char>(t); // Compute the crc of the record type and the payload. - uint32_t crc = crc32c::Extend(type_crc_[t], ptr, n); - crc = crc32c::Mask(crc); // Adjust for storage + uint32_t crc = crc32c::Extend(type_crc_[t], ptr, length); + crc = crc32c::Mask(crc); // Adjust for storage EncodeFixed32(buf, crc); // Write the header and the payload Status s = dest_->Append(Slice(buf, kHeaderSize)); if (s.ok()) { - s = dest_->Append(Slice(ptr, n)); + s = dest_->Append(Slice(ptr, length)); if (s.ok()) { s = dest_->Flush(); } } - block_offset_ += kHeaderSize + n; + block_offset_ += kHeaderSize + length; return s; } diff --git a/db/log_writer.h b/db/log_writer.h index 9e7cc47..ad36794 100644 --- a/db/log_writer.h +++ b/db/log_writer.h @@ -5,7 +5,8 @@ #ifndef STORAGE_LEVELDB_DB_LOG_WRITER_H_ #define STORAGE_LEVELDB_DB_LOG_WRITER_H_ -#include <stdint.h> +#include <cstdint> + #include "db/log_format.h" #include "leveldb/slice.h" #include "leveldb/status.h" @@ -28,24 +29,23 @@ class Writer { // "*dest" must remain live while this Writer is in use. Writer(WritableFile* dest, uint64_t dest_length); + Writer(const Writer&) = delete; + Writer& operator=(const Writer&) = delete; + ~Writer(); Status AddRecord(const Slice& slice); private: + Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length); + WritableFile* dest_; - int block_offset_; // Current offset in block + int block_offset_; // Current offset in block // crc32c values for all supported record types. These are // pre-computed to reduce the overhead of computing the crc of the // record type stored in the header. uint32_t type_crc_[kMaxRecordType + 1]; - - Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length); - - // No copying allowed - Writer(const Writer&); - void operator=(const Writer&); }; } // namespace log diff --git a/db/memtable.cc b/db/memtable.cc index 8cefcae..4f09340 100644 --- a/db/memtable.cc +++ b/db/memtable.cc @@ -18,20 +18,15 @@ static Slice GetLengthPrefixedSlice(const char* data) { return Slice(p, len); } -MemTable::MemTable(const InternalKeyComparator& cmp) - : comparator_(cmp), - refs_(0), - table_(comparator_, &arena_) { -} +MemTable::MemTable(const InternalKeyComparator& comparator) + : comparator_(comparator), refs_(0), table_(comparator_, &arena_) {} -MemTable::~MemTable() { - assert(refs_ == 0); -} +MemTable::~MemTable() { assert(refs_ == 0); } size_t MemTable::ApproximateMemoryUsage() { return arena_.MemoryUsage(); } -int MemTable::KeyComparator::operator()(const char* aptr, const char* bptr) - const { +int MemTable::KeyComparator::operator()(const char* aptr, + const char* bptr) const { // Internal keys are encoded as length-prefixed strings. Slice a = GetLengthPrefixedSlice(aptr); Slice b = GetLengthPrefixedSlice(bptr); @@ -48,60 +43,58 @@ static const char* EncodeKey(std::string* scratch, const Slice& target) { return scratch->data(); } -class MemTableIterator: public Iterator { +class MemTableIterator : public Iterator { public: - explicit MemTableIterator(MemTable::Table* table) : iter_(table) { } - - virtual bool Valid() const { return iter_.Valid(); } - virtual void Seek(const Slice& k) { iter_.Seek(EncodeKey(&tmp_, k)); } - virtual void SeekToFirst() { iter_.SeekToFirst(); } - virtual void SeekToLast() { iter_.SeekToLast(); } - virtual void Next() { iter_.Next(); } - virtual void Prev() { iter_.Prev(); } - virtual Slice key() const { return GetLengthPrefixedSlice(iter_.key()); } - virtual Slice value() const { + explicit MemTableIterator(MemTable::Table* table) : iter_(table) {} + + MemTableIterator(const MemTableIterator&) = delete; + MemTableIterator& operator=(const MemTableIterator&) = delete; + + ~MemTableIterator() override = default; + + bool Valid() const override { return iter_.Valid(); } + void Seek(const Slice& k) override { iter_.Seek(EncodeKey(&tmp_, k)); } + void SeekToFirst() override { iter_.SeekToFirst(); } + void SeekToLast() override { iter_.SeekToLast(); } + void Next() override { iter_.Next(); } + void Prev() override { iter_.Prev(); } + Slice key() const override { return GetLengthPrefixedSlice(iter_.key()); } + Slice value() const override { Slice key_slice = GetLengthPrefixedSlice(iter_.key()); return GetLengthPrefixedSlice(key_slice.data() + key_slice.size()); } - virtual Status status() const { return Status::OK(); } + Status status() const override { return Status::OK(); } private: MemTable::Table::Iterator iter_; - std::string tmp_; // For passing to EncodeKey - - // No copying allowed - MemTableIterator(const MemTableIterator&); - void operator=(const MemTableIterator&); + std::string tmp_; // For passing to EncodeKey }; -Iterator* MemTable::NewIterator() { - return new MemTableIterator(&table_); -} +Iterator* MemTable::NewIterator() { return new MemTableIterator(&table_); } -void MemTable::Add(SequenceNumber s, ValueType type, - const Slice& key, +void MemTable::Add(SequenceNumber s, ValueType type, const Slice& key, const Slice& value) { // Format of an entry is concatenation of: // key_size : varint32 of internal_key.size() // key bytes : char[internal_key.size()] - // tag : uint64((sequence << 8) | type) + // tag : uint64((sequence << 8) | type) // value_size : varint32 of value.size() // value bytes : char[value.size()] size_t key_size = key.size(); size_t val_size = value.size(); size_t internal_key_size = key_size + 8; - const size_t encoded_len = - VarintLength(internal_key_size) + internal_key_size + - VarintLength(val_size) + val_size; + const size_t encoded_len = VarintLength(internal_key_size) + + internal_key_size + VarintLength(val_size) + + val_size; char* buf = arena_.Allocate(encoded_len); char* p = EncodeVarint32(buf, internal_key_size); - memcpy(p, key.data(), key_size); + std::memcpy(p, key.data(), key_size); p += key_size; EncodeFixed64(p, (s << 8) | type); p += 8; p = EncodeVarint32(p, val_size); - memcpy(p, value.data(), val_size); + std::memcpy(p, value.data(), val_size); assert(p + val_size == buf + encoded_len); table_.Insert(buf); } @@ -122,10 +115,9 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) { // all entries with overly large sequence numbers. const char* entry = iter.key(); uint32_t key_length; - const char* key_ptr = GetVarint32Ptr(entry, entry+5, &key_length); + const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length); if (comparator_.comparator.user_comparator()->Compare( - Slice(key_ptr, key_length - 8), - key.user_key()) == 0) { + Slice(key_ptr, key_length - 8), key.user_key()) == 0) { // Correct user key const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8); switch (static_cast<ValueType>(tag & 0xff)) { diff --git a/db/memtable.h b/db/memtable.h index f2a6736..9d986b1 100644 --- a/db/memtable.h +++ b/db/memtable.h @@ -6,9 +6,10 @@ #define STORAGE_LEVELDB_DB_MEMTABLE_H_ #include <string> -#include "leveldb/db.h" + #include "db/dbformat.h" #include "db/skiplist.h" +#include "leveldb/db.h" #include "util/arena.h" namespace leveldb { @@ -22,6 +23,9 @@ class MemTable { // is zero and the caller must call Ref() at least once. explicit MemTable(const InternalKeyComparator& comparator); + MemTable(const MemTable&) = delete; + MemTable& operator=(const MemTable&) = delete; + // Increase reference count. void Ref() { ++refs_; } @@ -49,8 +53,7 @@ class MemTable { // Add an entry into memtable that maps key to value at the // specified sequence number and with the specified type. // Typically value will be empty if type==kTypeDeletion. - void Add(SequenceNumber seq, ValueType type, - const Slice& key, + void Add(SequenceNumber seq, ValueType type, const Slice& key, const Slice& value); // If memtable contains a value for key, store it in *value and return true. @@ -60,26 +63,23 @@ class MemTable { bool Get(const LookupKey& key, std::string* value, Status* s); private: - ~MemTable(); // Private since only Unref() should be used to delete it + friend class MemTableIterator; + friend class MemTableBackwardIterator; struct KeyComparator { const InternalKeyComparator comparator; - explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) { } + explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) {} int operator()(const char* a, const char* b) const; }; - friend class MemTableIterator; - friend class MemTableBackwardIterator; typedef SkipList<const char*, KeyComparator> Table; + ~MemTable(); // Private since only Unref() should be used to delete it + KeyComparator comparator_; int refs_; Arena arena_; Table table_; - - // No copying allowed - MemTable(const MemTable&); - void operator=(const MemTable&); }; } // namespace leveldb diff --git a/db/recovery_test.cc b/db/recovery_test.cc index c852803..1d9f621 100644 --- a/db/recovery_test.cc +++ b/db/recovery_test.cc @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "gtest/gtest.h" #include "db/db_impl.h" #include "db/filename.h" #include "db/version_set.h" @@ -10,15 +11,14 @@ #include "leveldb/env.h" #include "leveldb/write_batch.h" #include "util/logging.h" -#include "util/testharness.h" #include "util/testutil.h" namespace leveldb { -class RecoveryTest { +class RecoveryTest : public testing::Test { public: RecoveryTest() : env_(Env::Default()), db_(nullptr) { - dbname_ = test::TmpDir() + "/recovery_test"; + dbname_ = testing::TempDir() + "recovery_test"; DestroyDB(dbname_, Options()); Open(); } @@ -63,7 +63,7 @@ class RecoveryTest { } void Open(Options* options = nullptr) { - ASSERT_OK(OpenWithStatus(options)); + ASSERT_LEVELDB_OK(OpenWithStatus(options)); ASSERT_EQ(1, NumLogs()); } @@ -84,37 +84,37 @@ class RecoveryTest { std::string ManifestFileName() { std::string current; - ASSERT_OK(ReadFileToString(env_, CurrentFileName(dbname_), ¤t)); + EXPECT_LEVELDB_OK( + ReadFileToString(env_, CurrentFileName(dbname_), ¤t)); size_t len = current.size(); - if (len > 0 && current[len-1] == '\n') { + if (len > 0 && current[len - 1] == '\n') { current.resize(len - 1); } return dbname_ + "/" + current; } - std::string LogName(uint64_t number) { - return LogFileName(dbname_, number); - } + std::string LogName(uint64_t number) { return LogFileName(dbname_, number); } - size_t DeleteLogFiles() { + size_t RemoveLogFiles() { + // Linux allows unlinking open files, but Windows does not. + // Closing the db allows for file deletion. + Close(); std::vector<uint64_t> logs = GetFiles(kLogFile); for (size_t i = 0; i < logs.size(); i++) { - ASSERT_OK(env_->DeleteFile(LogName(logs[i]))) << LogName(logs[i]); + EXPECT_LEVELDB_OK(env_->RemoveFile(LogName(logs[i]))) << LogName(logs[i]); } return logs.size(); } - void DeleteManifestFile() { - ASSERT_OK(env_->DeleteFile(ManifestFileName())); + void RemoveManifestFile() { + ASSERT_LEVELDB_OK(env_->RemoveFile(ManifestFileName())); } - uint64_t FirstLogFile() { - return GetFiles(kLogFile)[0]; - } + uint64_t FirstLogFile() { return GetFiles(kLogFile)[0]; } std::vector<uint64_t> GetFiles(FileType t) { std::vector<std::string> filenames; - ASSERT_OK(env_->GetChildren(dbname_, &filenames)); + EXPECT_LEVELDB_OK(env_->GetChildren(dbname_, &filenames)); std::vector<uint64_t> result; for (size_t i = 0; i < filenames.size(); i++) { uint64_t number; @@ -126,35 +126,29 @@ class RecoveryTest { return result; } - int NumLogs() { - return GetFiles(kLogFile).size(); - } + int NumLogs() { return GetFiles(kLogFile).size(); } - int NumTables() { - return GetFiles(kTableFile).size(); - } + int NumTables() { return GetFiles(kTableFile).size(); } uint64_t FileSize(const std::string& fname) { uint64_t result; - ASSERT_OK(env_->GetFileSize(fname, &result)) << fname; + EXPECT_LEVELDB_OK(env_->GetFileSize(fname, &result)) << fname; return result; } - void CompactMemTable() { - dbfull()->TEST_CompactMemTable(); - } + void CompactMemTable() { dbfull()->TEST_CompactMemTable(); } // Directly construct a log file that sets key to val. void MakeLogFile(uint64_t lognum, SequenceNumber seq, Slice key, Slice val) { std::string fname = LogFileName(dbname_, lognum); WritableFile* file; - ASSERT_OK(env_->NewWritableFile(fname, &file)); + ASSERT_LEVELDB_OK(env_->NewWritableFile(fname, &file)); log::Writer writer(file); WriteBatch batch; batch.Put(key, val); WriteBatchInternal::SetSequence(&batch, seq); - ASSERT_OK(writer.AddRecord(WriteBatchInternal::Contents(&batch))); - ASSERT_OK(file->Flush()); + ASSERT_LEVELDB_OK(writer.AddRecord(WriteBatchInternal::Contents(&batch))); + ASSERT_LEVELDB_OK(file->Flush()); delete file; } @@ -164,12 +158,13 @@ class RecoveryTest { DB* db_; }; -TEST(RecoveryTest, ManifestReused) { +TEST_F(RecoveryTest, ManifestReused) { if (!CanAppend()) { - fprintf(stderr, "skipping test because env does not support appending\n"); + std::fprintf(stderr, + "skipping test because env does not support appending\n"); return; } - ASSERT_OK(Put("foo", "bar")); + ASSERT_LEVELDB_OK(Put("foo", "bar")); Close(); std::string old_manifest = ManifestFileName(); Open(); @@ -180,12 +175,13 @@ TEST(RecoveryTest, ManifestReused) { ASSERT_EQ("bar", Get("foo")); } -TEST(RecoveryTest, LargeManifestCompacted) { +TEST_F(RecoveryTest, LargeManifestCompacted) { if (!CanAppend()) { - fprintf(stderr, "skipping test because env does not support appending\n"); + std::fprintf(stderr, + "skipping test because env does not support appending\n"); return; } - ASSERT_OK(Put("foo", "bar")); + ASSERT_LEVELDB_OK(Put("foo", "bar")); Close(); std::string old_manifest = ManifestFileName(); @@ -193,10 +189,10 @@ TEST(RecoveryTest, LargeManifestCompacted) { { uint64_t len = FileSize(old_manifest); WritableFile* file; - ASSERT_OK(env()->NewAppendableFile(old_manifest, &file)); - std::string zeroes(3*1048576 - static_cast<size_t>(len), 0); - ASSERT_OK(file->Append(zeroes)); - ASSERT_OK(file->Flush()); + ASSERT_LEVELDB_OK(env()->NewAppendableFile(old_manifest, &file)); + std::string zeroes(3 * 1048576 - static_cast<size_t>(len), 0); + ASSERT_LEVELDB_OK(file->Append(zeroes)); + ASSERT_LEVELDB_OK(file->Flush()); delete file; } @@ -211,22 +207,23 @@ TEST(RecoveryTest, LargeManifestCompacted) { ASSERT_EQ("bar", Get("foo")); } -TEST(RecoveryTest, NoLogFiles) { - ASSERT_OK(Put("foo", "bar")); - ASSERT_EQ(1, DeleteLogFiles()); +TEST_F(RecoveryTest, NoLogFiles) { + ASSERT_LEVELDB_OK(Put("foo", "bar")); + ASSERT_EQ(1, RemoveLogFiles()); Open(); ASSERT_EQ("NOT_FOUND", Get("foo")); Open(); ASSERT_EQ("NOT_FOUND", Get("foo")); } -TEST(RecoveryTest, LogFileReuse) { +TEST_F(RecoveryTest, LogFileReuse) { if (!CanAppend()) { - fprintf(stderr, "skipping test because env does not support appending\n"); + std::fprintf(stderr, + "skipping test because env does not support appending\n"); return; } for (int i = 0; i < 2; i++) { - ASSERT_OK(Put("foo", "bar")); + ASSERT_LEVELDB_OK(Put("foo", "bar")); if (i == 0) { // Compact to ensure current log is empty CompactMemTable(); @@ -250,13 +247,13 @@ TEST(RecoveryTest, LogFileReuse) { } } -TEST(RecoveryTest, MultipleMemTables) { +TEST_F(RecoveryTest, MultipleMemTables) { // Make a large log. const int kNum = 1000; for (int i = 0; i < kNum; i++) { char buf[100]; - snprintf(buf, sizeof(buf), "%050d", i); - ASSERT_OK(Put(buf, buf)); + std::snprintf(buf, sizeof(buf), "%050d", i); + ASSERT_LEVELDB_OK(Put(buf, buf)); } ASSERT_EQ(0, NumTables()); Close(); @@ -267,35 +264,35 @@ TEST(RecoveryTest, MultipleMemTables) { // Force creation of multiple memtables by reducing the write buffer size. Options opt; opt.reuse_logs = true; - opt.write_buffer_size = (kNum*100) / 2; + opt.write_buffer_size = (kNum * 100) / 2; Open(&opt); ASSERT_LE(2, NumTables()); ASSERT_EQ(1, NumLogs()); ASSERT_NE(old_log_file, FirstLogFile()) << "must not reuse log"; for (int i = 0; i < kNum; i++) { char buf[100]; - snprintf(buf, sizeof(buf), "%050d", i); + std::snprintf(buf, sizeof(buf), "%050d", i); ASSERT_EQ(buf, Get(buf)); } } -TEST(RecoveryTest, MultipleLogFiles) { - ASSERT_OK(Put("foo", "bar")); +TEST_F(RecoveryTest, MultipleLogFiles) { + ASSERT_LEVELDB_OK(Put("foo", "bar")); Close(); ASSERT_EQ(1, NumLogs()); // Make a bunch of uncompacted log files. uint64_t old_log = FirstLogFile(); - MakeLogFile(old_log+1, 1000, "hello", "world"); - MakeLogFile(old_log+2, 1001, "hi", "there"); - MakeLogFile(old_log+3, 1002, "foo", "bar2"); + MakeLogFile(old_log + 1, 1000, "hello", "world"); + MakeLogFile(old_log + 2, 1001, "hi", "there"); + MakeLogFile(old_log + 3, 1002, "foo", "bar2"); // Recover and check that all log files were processed. Open(); ASSERT_LE(1, NumTables()); ASSERT_EQ(1, NumLogs()); uint64_t new_log = FirstLogFile(); - ASSERT_LE(old_log+3, new_log); + ASSERT_LE(old_log + 3, new_log); ASSERT_EQ("bar2", Get("foo")); ASSERT_EQ("world", Get("hello")); ASSERT_EQ("there", Get("hi")); @@ -313,7 +310,7 @@ TEST(RecoveryTest, MultipleLogFiles) { // Check that introducing an older log file does not cause it to be re-read. Close(); - MakeLogFile(old_log+1, 2000, "hello", "stale write"); + MakeLogFile(old_log + 1, 2000, "hello", "stale write"); Open(); ASSERT_LE(1, NumTables()); ASSERT_EQ(1, NumLogs()); @@ -325,17 +322,13 @@ TEST(RecoveryTest, MultipleLogFiles) { ASSERT_EQ("there", Get("hi")); } -TEST(RecoveryTest, ManifestMissing) { - ASSERT_OK(Put("foo", "bar")); +TEST_F(RecoveryTest, ManifestMissing) { + ASSERT_LEVELDB_OK(Put("foo", "bar")); Close(); - DeleteManifestFile(); + RemoveManifestFile(); Status status = OpenWithStatus(); ASSERT_TRUE(status.IsCorruption()); } } // namespace leveldb - -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); -} diff --git a/db/repair.cc b/db/repair.cc index df8dcd2..97a27c6 100644 --- a/db/repair.cc +++ b/db/repair.cc @@ -84,9 +84,7 @@ class Repairer { "recovered %d files; %llu bytes. " "Some data may have been lost. " "****", - dbname_.c_str(), - static_cast<int>(tables_.size()), - bytes); + dbname_.c_str(), static_cast<int>(tables_.size()), bytes); } return status; } @@ -97,22 +95,6 @@ class Repairer { SequenceNumber max_sequence; }; - std::string const dbname_; - Env* const env_; - InternalKeyComparator const icmp_; - InternalFilterPolicy const ipolicy_; - Options const options_; - bool owns_info_log_; - bool owns_cache_; - TableCache* table_cache_; - VersionEdit edit_; - - std::vector<std::string> manifests_; - std::vector<uint64_t> table_numbers_; - std::vector<uint64_t> logs_; - std::vector<TableInfo> tables_; - uint64_t next_file_number_; - Status FindFiles() { std::vector<std::string> filenames; Status status = env_->GetChildren(dbname_, &filenames); @@ -152,8 +134,7 @@ class Repairer { Status status = ConvertLogToTable(logs_[i]); if (!status.ok()) { Log(options_.info_log, "Log #%llu: ignoring conversion error: %s", - (unsigned long long) logs_[i], - status.ToString().c_str()); + (unsigned long long)logs_[i], status.ToString().c_str()); } ArchiveFile(logname); } @@ -164,11 +145,10 @@ class Repairer { Env* env; Logger* info_log; uint64_t lognum; - virtual void Corruption(size_t bytes, const Status& s) { + void Corruption(size_t bytes, const Status& s) override { // We print error messages for corruption, but continue repairing. Log(info_log, "Log #%llu: dropping %d bytes; %s", - (unsigned long long) lognum, - static_cast<int>(bytes), + (unsigned long long)lognum, static_cast<int>(bytes), s.ToString().c_str()); } }; @@ -190,8 +170,8 @@ class Repairer { // corruptions cause entire commits to be skipped instead of // propagating bad information (like overly large sequence // numbers). - log::Reader reader(lfile, &reporter, false/*do not checksum*/, - 0/*initial_offset*/); + log::Reader reader(lfile, &reporter, false /*do not checksum*/, + 0 /*initial_offset*/); // Read all the records and add to a memtable std::string scratch; @@ -202,8 +182,8 @@ class Repairer { int counter = 0; while (reader.ReadRecord(&record, &scratch)) { if (record.size() < 12) { - reporter.Corruption( - record.size(), Status::Corruption("log record too small")); + reporter.Corruption(record.size(), + Status::Corruption("log record too small")); continue; } WriteBatchInternal::SetContents(&batch, record); @@ -212,8 +192,7 @@ class Repairer { counter += WriteBatchInternal::Count(&batch); } else { Log(options_.info_log, "Log #%llu: ignoring %s", - (unsigned long long) log, - status.ToString().c_str()); + (unsigned long long)log, status.ToString().c_str()); status = Status::OK(); // Keep going with rest of file } } @@ -234,9 +213,7 @@ class Repairer { } } Log(options_.info_log, "Log #%llu: %d ops saved to Table #%llu %s", - (unsigned long long) log, - counter, - (unsigned long long) meta.number, + (unsigned long long)log, counter, (unsigned long long)meta.number, status.ToString().c_str()); return status; } @@ -272,8 +249,7 @@ class Repairer { ArchiveFile(TableFileName(dbname_, number)); ArchiveFile(SSTTableFileName(dbname_, number)); Log(options_.info_log, "Table #%llu: dropped: %s", - (unsigned long long) t.meta.number, - status.ToString().c_str()); + (unsigned long long)t.meta.number, status.ToString().c_str()); return; } @@ -287,8 +263,7 @@ class Repairer { Slice key = iter->key(); if (!ParseInternalKey(key, &parsed)) { Log(options_.info_log, "Table #%llu: unparsable key %s", - (unsigned long long) t.meta.number, - EscapeString(key).c_str()); + (unsigned long long)t.meta.number, EscapeString(key).c_str()); continue; } @@ -307,9 +282,7 @@ class Repairer { } delete iter; Log(options_.info_log, "Table #%llu: %d entries %s", - (unsigned long long) t.meta.number, - counter, - status.ToString().c_str()); + (unsigned long long)t.meta.number, counter, status.ToString().c_str()); if (status.ok()) { tables_.push_back(t); @@ -363,12 +336,12 @@ class Repairer { s = env_->RenameFile(copy, orig); if (s.ok()) { Log(options_.info_log, "Table #%llu: %d entries repaired", - (unsigned long long) t.meta.number, counter); + (unsigned long long)t.meta.number, counter); tables_.push_back(t); } } if (!s.ok()) { - env_->DeleteFile(copy); + env_->RemoveFile(copy); } } @@ -395,11 +368,12 @@ class Repairer { for (size_t i = 0; i < tables_.size(); i++) { // TODO(opt): separate out into multiple levels const TableInfo& t = tables_[i]; - edit_.AddFile(0, t.meta.number, t.meta.file_size, - t.meta.smallest, t.meta.largest); + edit_.AddFile(0, t.meta.number, t.meta.file_size, t.meta.smallest, + t.meta.largest); } - //fprintf(stderr, "NewDescriptor:\n%s\n", edit_.DebugString().c_str()); + // std::fprintf(stderr, + // "NewDescriptor:\n%s\n", edit_.DebugString().c_str()); { log::Writer log(file); std::string record; @@ -413,7 +387,7 @@ class Repairer { file = nullptr; if (!status.ok()) { - env_->DeleteFile(tmp); + env_->RemoveFile(tmp); } else { // Discard older manifests for (size_t i = 0; i < manifests_.size(); i++) { @@ -425,7 +399,7 @@ class Repairer { if (status.ok()) { status = SetCurrentFile(env_, dbname_, 1); } else { - env_->DeleteFile(tmp); + env_->RemoveFile(tmp); } } return status; @@ -447,9 +421,25 @@ class Repairer { new_file.append("/"); new_file.append((slash == nullptr) ? fname.c_str() : slash + 1); Status s = env_->RenameFile(fname, new_file); - Log(options_.info_log, "Archiving %s: %s\n", - fname.c_str(), s.ToString().c_str()); + Log(options_.info_log, "Archiving %s: %s\n", fname.c_str(), + s.ToString().c_str()); } + + const std::string dbname_; + Env* const env_; + InternalKeyComparator const icmp_; + InternalFilterPolicy const ipolicy_; + const Options options_; + bool owns_info_log_; + bool owns_cache_; + TableCache* table_cache_; + VersionEdit edit_; + + std::vector<std::string> manifests_; + std::vector<uint64_t> table_numbers_; + std::vector<uint64_t> logs_; + std::vector<TableInfo> tables_; + uint64_t next_file_number_; }; } // namespace diff --git a/db/skiplist.h b/db/skiplist.h index b806ce0..f716834 100644 --- a/db/skiplist.h +++ b/db/skiplist.h @@ -27,9 +27,10 @@ // // ... prev vs. next pointer ordering ... -#include <assert.h> -#include <stdlib.h> -#include "port/port.h" +#include <atomic> +#include <cassert> +#include <cstdlib> + #include "util/arena.h" #include "util/random.h" @@ -37,7 +38,7 @@ namespace leveldb { class Arena; -template<typename Key, class Comparator> +template <typename Key, class Comparator> class SkipList { private: struct Node; @@ -48,6 +49,9 @@ class SkipList { // must remain allocated for the lifetime of the skiplist object. explicit SkipList(Comparator cmp, Arena* arena); + SkipList(const SkipList&) = delete; + SkipList& operator=(const SkipList&) = delete; + // Insert key into the list. // REQUIRES: nothing that compares equal to key is currently in the list. void Insert(const Key& key); @@ -97,24 +101,10 @@ class SkipList { private: enum { kMaxHeight = 12 }; - // Immutable after construction - Comparator const compare_; - Arena* const arena_; // Arena used for allocations of nodes - - Node* const head_; - - // Modified only by Insert(). Read racily by readers, but stale - // values are ok. - port::AtomicPointer max_height_; // Height of the entire list - inline int GetMaxHeight() const { - return static_cast<int>( - reinterpret_cast<intptr_t>(max_height_.NoBarrier_Load())); + return max_height_.load(std::memory_order_relaxed); } - // Read/written only by Insert(). - Random rnd_; - Node* NewNode(const Key& key, int height); int RandomHeight(); bool Equal(const Key& a, const Key& b) const { return (compare_(a, b) == 0); } @@ -137,15 +127,24 @@ class SkipList { // Return head_ if list is empty. Node* FindLast() const; - // No copying allowed - SkipList(const SkipList&); - void operator=(const SkipList&); + // Immutable after construction + Comparator const compare_; + Arena* const arena_; // Arena used for allocations of nodes + + Node* const head_; + + // Modified only by Insert(). Read racily by readers, but stale + // values are ok. + std::atomic<int> max_height_; // Height of the entire list + + // Read/written only by Insert(). + Random rnd_; }; // Implementation details follow -template<typename Key, class Comparator> -struct SkipList<Key,Comparator>::Node { - explicit Node(const Key& k) : key(k) { } +template <typename Key, class Comparator> +struct SkipList<Key, Comparator>::Node { + explicit Node(const Key& k) : key(k) {} Key const key; @@ -155,63 +154,63 @@ struct SkipList<Key,Comparator>::Node { assert(n >= 0); // Use an 'acquire load' so that we observe a fully initialized // version of the returned Node. - return reinterpret_cast<Node*>(next_[n].Acquire_Load()); + return next_[n].load(std::memory_order_acquire); } void SetNext(int n, Node* x) { assert(n >= 0); // Use a 'release store' so that anybody who reads through this // pointer observes a fully initialized version of the inserted node. - next_[n].Release_Store(x); + next_[n].store(x, std::memory_order_release); } // No-barrier variants that can be safely used in a few locations. Node* NoBarrier_Next(int n) { assert(n >= 0); - return reinterpret_cast<Node*>(next_[n].NoBarrier_Load()); + return next_[n].load(std::memory_order_relaxed); } void NoBarrier_SetNext(int n, Node* x) { assert(n >= 0); - next_[n].NoBarrier_Store(x); + next_[n].store(x, std::memory_order_relaxed); } private: // Array of length equal to the node height. next_[0] is lowest level link. - port::AtomicPointer next_[1]; + std::atomic<Node*> next_[1]; }; -template<typename Key, class Comparator> -typename SkipList<Key,Comparator>::Node* -SkipList<Key,Comparator>::NewNode(const Key& key, int height) { - char* mem = arena_->AllocateAligned( - sizeof(Node) + sizeof(port::AtomicPointer) * (height - 1)); - return new (mem) Node(key); +template <typename Key, class Comparator> +typename SkipList<Key, Comparator>::Node* SkipList<Key, Comparator>::NewNode( + const Key& key, int height) { + char* const node_memory = arena_->AllocateAligned( + sizeof(Node) + sizeof(std::atomic<Node*>) * (height - 1)); + return new (node_memory) Node(key); } -template<typename Key, class Comparator> -inline SkipList<Key,Comparator>::Iterator::Iterator(const SkipList* list) { +template <typename Key, class Comparator> +inline SkipList<Key, Comparator>::Iterator::Iterator(const SkipList* list) { list_ = list; node_ = nullptr; } -template<typename Key, class Comparator> -inline bool SkipList<Key,Comparator>::Iterator::Valid() const { +template <typename Key, class Comparator> +inline bool SkipList<Key, Comparator>::Iterator::Valid() const { return node_ != nullptr; } -template<typename Key, class Comparator> -inline const Key& SkipList<Key,Comparator>::Iterator::key() const { +template <typename Key, class Comparator> +inline const Key& SkipList<Key, Comparator>::Iterator::key() const { assert(Valid()); return node_->key; } -template<typename Key, class Comparator> -inline void SkipList<Key,Comparator>::Iterator::Next() { +template <typename Key, class Comparator> +inline void SkipList<Key, Comparator>::Iterator::Next() { assert(Valid()); node_ = node_->Next(0); } -template<typename Key, class Comparator> -inline void SkipList<Key,Comparator>::Iterator::Prev() { +template <typename Key, class Comparator> +inline void SkipList<Key, Comparator>::Iterator::Prev() { // Instead of using explicit "prev" links, we just search for the // last node that falls before key. assert(Valid()); @@ -221,30 +220,30 @@ inline void SkipList<Key,Comparator>::Iterator::Prev() { } } -template<typename Key, class Comparator> -inline void SkipList<Key,Comparator>::Iterator::Seek(const Key& target) { +template <typename Key, class Comparator> +inline void SkipList<Key, Comparator>::Iterator::Seek(const Key& target) { node_ = list_->FindGreaterOrEqual(target, nullptr); } -template<typename Key, class Comparator> -inline void SkipList<Key,Comparator>::Iterator::SeekToFirst() { +template <typename Key, class Comparator> +inline void SkipList<Key, Comparator>::Iterator::SeekToFirst() { node_ = list_->head_->Next(0); } -template<typename Key, class Comparator> -inline void SkipList<Key,Comparator>::Iterator::SeekToLast() { +template <typename Key, class Comparator> +inline void SkipList<Key, Comparator>::Iterator::SeekToLast() { node_ = list_->FindLast(); if (node_ == list_->head_) { node_ = nullptr; } } -template<typename Key, class Comparator> -int SkipList<Key,Comparator>::RandomHeight() { +template <typename Key, class Comparator> +int SkipList<Key, Comparator>::RandomHeight() { // Increase height with probability 1 in kBranching static const unsigned int kBranching = 4; int height = 1; - while (height < kMaxHeight && ((rnd_.Next() % kBranching) == 0)) { + while (height < kMaxHeight && rnd_.OneIn(kBranching)) { height++; } assert(height > 0); @@ -252,15 +251,16 @@ int SkipList<Key,Comparator>::RandomHeight() { return height; } -template<typename Key, class Comparator> -bool SkipList<Key,Comparator>::KeyIsAfterNode(const Key& key, Node* n) const { +template <typename Key, class Comparator> +bool SkipList<Key, Comparator>::KeyIsAfterNode(const Key& key, Node* n) const { // null n is considered infinite return (n != nullptr) && (compare_(n->key, key) < 0); } -template<typename Key, class Comparator> -typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindGreaterOrEqual(const Key& key, Node** prev) - const { +template <typename Key, class Comparator> +typename SkipList<Key, Comparator>::Node* +SkipList<Key, Comparator>::FindGreaterOrEqual(const Key& key, + Node** prev) const { Node* x = head_; int level = GetMaxHeight() - 1; while (true) { @@ -280,9 +280,9 @@ typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindGreaterOr } } -template<typename Key, class Comparator> -typename SkipList<Key,Comparator>::Node* -SkipList<Key,Comparator>::FindLessThan(const Key& key) const { +template <typename Key, class Comparator> +typename SkipList<Key, Comparator>::Node* +SkipList<Key, Comparator>::FindLessThan(const Key& key) const { Node* x = head_; int level = GetMaxHeight() - 1; while (true) { @@ -301,8 +301,8 @@ SkipList<Key,Comparator>::FindLessThan(const Key& key) const { } } -template<typename Key, class Comparator> -typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindLast() +template <typename Key, class Comparator> +typename SkipList<Key, Comparator>::Node* SkipList<Key, Comparator>::FindLast() const { Node* x = head_; int level = GetMaxHeight() - 1; @@ -321,20 +321,20 @@ typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindLast() } } -template<typename Key, class Comparator> -SkipList<Key,Comparator>::SkipList(Comparator cmp, Arena* arena) +template <typename Key, class Comparator> +SkipList<Key, Comparator>::SkipList(Comparator cmp, Arena* arena) : compare_(cmp), arena_(arena), head_(NewNode(0 /* any key will do */, kMaxHeight)), - max_height_(reinterpret_cast<void*>(1)), + max_height_(1), rnd_(0xdeadbeef) { for (int i = 0; i < kMaxHeight; i++) { head_->SetNext(i, nullptr); } } -template<typename Key, class Comparator> -void SkipList<Key,Comparator>::Insert(const Key& key) { +template <typename Key, class Comparator> +void SkipList<Key, Comparator>::Insert(const Key& key) { // TODO(opt): We can use a barrier-free variant of FindGreaterOrEqual() // here since Insert() is externally synchronized. Node* prev[kMaxHeight]; @@ -348,8 +348,6 @@ void SkipList<Key,Comparator>::Insert(const Key& key) { for (int i = GetMaxHeight(); i < height; i++) { prev[i] = head_; } - //fprintf(stderr, "Change height from %d to %d\n", max_height_, height); - // It is ok to mutate max_height_ without any synchronization // with concurrent readers. A concurrent reader that observes // the new value of max_height_ will see either the old value of @@ -357,7 +355,7 @@ void SkipList<Key,Comparator>::Insert(const Key& key) { // the loop below. In the former case the reader will // immediately drop to the next level since nullptr sorts after all // keys. In the latter case the reader will use the new node. - max_height_.NoBarrier_Store(reinterpret_cast<void*>(height)); + max_height_.store(height, std::memory_order_relaxed); } x = NewNode(key, height); @@ -369,8 +367,8 @@ void SkipList<Key,Comparator>::Insert(const Key& key) { } } -template<typename Key, class Comparator> -bool SkipList<Key,Comparator>::Contains(const Key& key) const { +template <typename Key, class Comparator> +bool SkipList<Key, Comparator>::Contains(const Key& key) const { Node* x = FindGreaterOrEqual(key, nullptr); if (x != nullptr && Equal(key, x->key)) { return true; diff --git a/db/skiplist_test.cc b/db/skiplist_test.cc index 24e0887..1d355cb 100644 --- a/db/skiplist_test.cc +++ b/db/skiplist_test.cc @@ -3,14 +3,18 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "db/skiplist.h" + +#include <atomic> #include <set> + +#include "gtest/gtest.h" #include "leveldb/env.h" #include "port/port.h" #include "port/thread_annotations.h" #include "util/arena.h" #include "util/hash.h" #include "util/random.h" -#include "util/testharness.h" +#include "util/testutil.h" namespace leveldb { @@ -28,8 +32,6 @@ struct Comparator { } }; -class SkipTest { }; - TEST(SkipTest, Empty) { Arena arena; Comparator cmp; @@ -114,8 +116,7 @@ TEST(SkipTest, InsertAndLookup) { // Compare against model iterator for (std::set<Key>::reverse_iterator model_iter = keys.rbegin(); - model_iter != keys.rend(); - ++model_iter) { + model_iter != keys.rend(); ++model_iter) { ASSERT_TRUE(iter.Valid()); ASSERT_EQ(*model_iter, iter.key()); iter.Prev(); @@ -128,7 +129,7 @@ TEST(SkipTest, InsertAndLookup) { // concurrent readers (with no synchronization other than when a // reader's iterator is created), the reader always observes all the // data that was present in the skip list when the iterator was -// constructor. Because insertions are happening concurrently, we may +// constructed. Because insertions are happening concurrently, we may // also observe new values that were inserted since the iterator was // constructed, but we should never miss any values that were present // at iterator construction time. @@ -150,19 +151,19 @@ TEST(SkipTest, InsertAndLookup) { // been concurrently added since the iterator started. class ConcurrentTest { private: - static const uint32_t K = 4; + static constexpr uint32_t K = 4; static uint64_t key(Key key) { return (key >> 40); } static uint64_t gen(Key key) { return (key >> 8) & 0xffffffffu; } static uint64_t hash(Key key) { return key & 0xff; } static uint64_t HashNumbers(uint64_t k, uint64_t g) { - uint64_t data[2] = { k, g }; + uint64_t data[2] = {k, g}; return Hash(reinterpret_cast<char*>(data), sizeof(data), 0); } static Key MakeKey(uint64_t k, uint64_t g) { - assert(sizeof(Key) == sizeof(uint64_t)); + static_assert(sizeof(Key) == sizeof(uint64_t), ""); assert(k <= K); // We sometimes pass K to seek to the end of the skiplist assert(g <= 0xffffffffu); return ((k << 40) | (g << 8) | (HashNumbers(k, g) & 0xff)); @@ -188,13 +189,11 @@ class ConcurrentTest { // Per-key generation struct State { - port::AtomicPointer generation[K]; - void Set(int k, intptr_t v) { - generation[k].Release_Store(reinterpret_cast<void*>(v)); - } - intptr_t Get(int k) { - return reinterpret_cast<intptr_t>(generation[k].Acquire_Load()); + std::atomic<int> generation[K]; + void Set(int k, int v) { + generation[k].store(v, std::memory_order_release); } + int Get(int k) { return generation[k].load(std::memory_order_acquire); } State() { for (int k = 0; k < K; k++) { @@ -213,7 +212,7 @@ class ConcurrentTest { SkipList<Key, Comparator> list_; public: - ConcurrentTest() : list_(Comparator(), &arena_) { } + ConcurrentTest() : list_(Comparator(), &arena_) {} // REQUIRES: External synchronization void WriteStep(Random* rnd) { @@ -252,11 +251,9 @@ class ConcurrentTest { // Note that generation 0 is never inserted, so it is ok if // <*,0,*> is missing. ASSERT_TRUE((gen(pos) == 0) || - (gen(pos) > static_cast<Key>(initial_state.Get(key(pos)))) - ) << "key: " << key(pos) - << "; gen: " << gen(pos) - << "; initgen: " - << initial_state.Get(key(pos)); + (gen(pos) > static_cast<Key>(initial_state.Get(key(pos))))) + << "key: " << key(pos) << "; gen: " << gen(pos) + << "; initgen: " << initial_state.Get(key(pos)); // Advance to next key in the valid key space if (key(pos) < key(current)) { @@ -283,7 +280,9 @@ class ConcurrentTest { } } }; -const uint32_t ConcurrentTest::K; + +// Needed when building in C++11 mode. +constexpr uint32_t ConcurrentTest::K; // Simple test that does single-threaded testing of the ConcurrentTest // scaffolding. @@ -300,19 +299,12 @@ class TestState { public: ConcurrentTest t_; int seed_; - port::AtomicPointer quit_flag_; + std::atomic<bool> quit_flag_; - enum ReaderState { - STARTING, - RUNNING, - DONE - }; + enum ReaderState { STARTING, RUNNING, DONE }; explicit TestState(int s) - : seed_(s), - quit_flag_(nullptr), - state_(STARTING), - state_cv_(&mu_) {} + : seed_(s), quit_flag_(false), state_(STARTING), state_cv_(&mu_) {} void Wait(ReaderState s) LOCKS_EXCLUDED(mu_) { mu_.Lock(); @@ -340,7 +332,7 @@ static void ConcurrentReader(void* arg) { Random rnd(state->seed_); int64_t reads = 0; state->Change(TestState::RUNNING); - while (!state->quit_flag_.Acquire_Load()) { + while (!state->quit_flag_.load(std::memory_order_acquire)) { state->t_.ReadStep(&rnd); ++reads; } @@ -354,7 +346,7 @@ static void RunConcurrent(int run) { const int kSize = 1000; for (int i = 0; i < N; i++) { if ((i % 100) == 0) { - fprintf(stderr, "Run %d of %d\n", i, N); + std::fprintf(stderr, "Run %d of %d\n", i, N); } TestState state(seed + 1); Env::Default()->Schedule(ConcurrentReader, &state); @@ -362,7 +354,7 @@ static void RunConcurrent(int run) { for (int i = 0; i < kSize; i++) { state.t_.WriteStep(&rnd); } - state.quit_flag_.Release_Store(&state); // Any non-null arg will do + state.quit_flag_.store(true, std::memory_order_release); state.Wait(TestState::DONE); } } @@ -374,7 +366,3 @@ TEST(SkipTest, Concurrent4) { RunConcurrent(4); } TEST(SkipTest, Concurrent5) { RunConcurrent(5); } } // namespace leveldb - -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); -} diff --git a/db/snapshot.h b/db/snapshot.h index c43d9f9..817bb7b 100644 --- a/db/snapshot.h +++ b/db/snapshot.h @@ -25,7 +25,7 @@ class SnapshotImpl : public Snapshot { friend class SnapshotList; // SnapshotImpl is kept in a doubly-linked circular list. The SnapshotList - // implementation operates on the next/previous fields direcly. + // implementation operates on the next/previous fields directly. SnapshotImpl* prev_; SnapshotImpl* next_; @@ -44,8 +44,14 @@ class SnapshotList { } bool empty() const { return head_.next_ == &head_; } - SnapshotImpl* oldest() const { assert(!empty()); return head_.next_; } - SnapshotImpl* newest() const { assert(!empty()); return head_.prev_; } + SnapshotImpl* oldest() const { + assert(!empty()); + return head_.next_; + } + SnapshotImpl* newest() const { + assert(!empty()); + return head_.prev_; + } // Creates a SnapshotImpl and appends it to the end of the list. SnapshotImpl* New(SequenceNumber sequence_number) { diff --git a/db/table_cache.cc b/db/table_cache.cc index 7226d3b..73f05fd 100644 --- a/db/table_cache.cc +++ b/db/table_cache.cc @@ -29,18 +29,14 @@ static void UnrefEntry(void* arg1, void* arg2) { cache->Release(h); } -TableCache::TableCache(const std::string& dbname, - const Options& options, +TableCache::TableCache(const std::string& dbname, const Options& options, int entries) : env_(options.env), dbname_(dbname), options_(options), - cache_(NewLRUCache(entries)) { -} + cache_(NewLRUCache(entries)) {} -TableCache::~TableCache() { - delete cache_; -} +TableCache::~TableCache() { delete cache_; } Status TableCache::FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle** handle) { @@ -80,8 +76,7 @@ Status TableCache::FindTable(uint64_t file_number, uint64_t file_size, } Iterator* TableCache::NewIterator(const ReadOptions& options, - uint64_t file_number, - uint64_t file_size, + uint64_t file_number, uint64_t file_size, Table** tableptr) { if (tableptr != nullptr) { *tableptr = nullptr; @@ -102,17 +97,15 @@ Iterator* TableCache::NewIterator(const ReadOptions& options, return result; } -Status TableCache::Get(const ReadOptions& options, - uint64_t file_number, - uint64_t file_size, - const Slice& k, - void* arg, - void (*saver)(void*, const Slice&, const Slice&)) { +Status TableCache::Get(const ReadOptions& options, uint64_t file_number, + uint64_t file_size, const Slice& k, void* arg, + void (*handle_result)(void*, const Slice&, + const Slice&)) { Cache::Handle* handle = nullptr; Status s = FindTable(file_number, file_size, &handle); if (s.ok()) { Table* t = reinterpret_cast<TableAndFile*>(cache_->Value(handle))->table; - s = t->InternalGet(options, k, arg, saver); + s = t->InternalGet(options, k, arg, handle_result); cache_->Release(handle); } return s; diff --git a/db/table_cache.h b/db/table_cache.h index ae8bee5..db8a123 100644 --- a/db/table_cache.h +++ b/db/table_cache.h @@ -7,8 +7,9 @@ #ifndef STORAGE_LEVELDB_DB_TABLE_CACHE_H_ #define STORAGE_LEVELDB_DB_TABLE_CACHE_H_ +#include <cstdint> #include <string> -#include <stdint.h> + #include "db/dbformat.h" #include "leveldb/cache.h" #include "leveldb/table.h" @@ -21,6 +22,10 @@ class Env; class TableCache { public: TableCache(const std::string& dbname, const Options& options, int entries); + + TableCache(const TableCache&) = delete; + TableCache& operator=(const TableCache&) = delete; + ~TableCache(); // Return an iterator for the specified file number (the corresponding @@ -30,30 +35,25 @@ class TableCache { // underlies the returned iterator. The returned "*tableptr" object is owned // by the cache and should not be deleted, and is valid for as long as the // returned iterator is live. - Iterator* NewIterator(const ReadOptions& options, - uint64_t file_number, - uint64_t file_size, - Table** tableptr = nullptr); + Iterator* NewIterator(const ReadOptions& options, uint64_t file_number, + uint64_t file_size, Table** tableptr = nullptr); // If a seek to internal key "k" in specified file finds an entry, // call (*handle_result)(arg, found_key, found_value). - Status Get(const ReadOptions& options, - uint64_t file_number, - uint64_t file_size, - const Slice& k, - void* arg, + Status Get(const ReadOptions& options, uint64_t file_number, + uint64_t file_size, const Slice& k, void* arg, void (*handle_result)(void*, const Slice&, const Slice&)); // Evict any entry for the specified file number void Evict(uint64_t file_number); private: + Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**); + Env* const env_; const std::string dbname_; const Options& options_; Cache* cache_; - - Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**); }; } // namespace leveldb diff --git a/db/version_edit.cc b/db/version_edit.cc index b7a366d..3e9012f 100644 --- a/db/version_edit.cc +++ b/db/version_edit.cc @@ -12,15 +12,15 @@ namespace leveldb { // Tag numbers for serialized VersionEdit. These numbers are written to // disk and should not be changed. enum Tag { - kComparator = 1, - kLogNumber = 2, - kNextFileNumber = 3, - kLastSequence = 4, - kCompactPointer = 5, - kDeletedFile = 6, - kNewFile = 7, + kComparator = 1, + kLogNumber = 2, + kNextFileNumber = 3, + kLastSequence = 4, + kCompactPointer = 5, + kDeletedFile = 6, + kNewFile = 7, // 8 was used for large value refs - kPrevLogNumber = 9 + kPrevLogNumber = 9 }; void VersionEdit::Clear() { @@ -66,12 +66,10 @@ void VersionEdit::EncodeTo(std::string* dst) const { PutLengthPrefixedSlice(dst, compact_pointers_[i].second.Encode()); } - for (DeletedFileSet::const_iterator iter = deleted_files_.begin(); - iter != deleted_files_.end(); - ++iter) { + for (const auto& deleted_file_kvp : deleted_files_) { PutVarint32(dst, kDeletedFile); - PutVarint32(dst, iter->first); // level - PutVarint64(dst, iter->second); // file number + PutVarint32(dst, deleted_file_kvp.first); // level + PutVarint64(dst, deleted_file_kvp.second); // file number } for (size_t i = 0; i < new_files_.size(); i++) { @@ -88,8 +86,7 @@ void VersionEdit::EncodeTo(std::string* dst) const { static bool GetInternalKey(Slice* input, InternalKey* dst) { Slice str; if (GetLengthPrefixedSlice(input, &str)) { - dst->DecodeFrom(str); - return true; + return dst->DecodeFrom(str); } else { return false; } @@ -97,8 +94,7 @@ static bool GetInternalKey(Slice* input, InternalKey* dst) { static bool GetLevel(Slice* input, int* level) { uint32_t v; - if (GetVarint32(input, &v) && - v < config::kNumLevels) { + if (GetVarint32(input, &v) && v < config::kNumLevels) { *level = v; return true; } else { @@ -163,8 +159,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) { break; case kCompactPointer: - if (GetLevel(&input, &level) && - GetInternalKey(&input, &key)) { + if (GetLevel(&input, &level) && GetInternalKey(&input, &key)) { compact_pointers_.push_back(std::make_pair(level, key)); } else { msg = "compaction pointer"; @@ -172,8 +167,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) { break; case kDeletedFile: - if (GetLevel(&input, &level) && - GetVarint64(&input, &number)) { + if (GetLevel(&input, &level) && GetVarint64(&input, &number)) { deleted_files_.insert(std::make_pair(level, number)); } else { msg = "deleted file"; @@ -181,8 +175,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) { break; case kNewFile: - if (GetLevel(&input, &level) && - GetVarint64(&input, &f.number) && + if (GetLevel(&input, &level) && GetVarint64(&input, &f.number) && GetVarint64(&input, &f.file_size) && GetInternalKey(&input, &f.smallest) && GetInternalKey(&input, &f.largest)) { @@ -238,13 +231,11 @@ std::string VersionEdit::DebugString() const { r.append(" "); r.append(compact_pointers_[i].second.DebugString()); } - for (DeletedFileSet::const_iterator iter = deleted_files_.begin(); - iter != deleted_files_.end(); - ++iter) { - r.append("\n DeleteFile: "); - AppendNumberTo(&r, iter->first); + for (const auto& deleted_files_kvp : deleted_files_) { + r.append("\n RemoveFile: "); + AppendNumberTo(&r, deleted_files_kvp.first); r.append(" "); - AppendNumberTo(&r, iter->second); + AppendNumberTo(&r, deleted_files_kvp.second); } for (size_t i = 0; i < new_files_.size(); i++) { const FileMetaData& f = new_files_[i].second; diff --git a/db/version_edit.h b/db/version_edit.h index eaef77b..137b4b1 100644 --- a/db/version_edit.h +++ b/db/version_edit.h @@ -8,6 +8,7 @@ #include <set> #include <utility> #include <vector> + #include "db/dbformat.h" namespace leveldb { @@ -15,20 +16,20 @@ namespace leveldb { class VersionSet; struct FileMetaData { + FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0) {} + int refs; - int allowed_seeks; // Seeks allowed until compaction + int allowed_seeks; // Seeks allowed until compaction uint64_t number; - uint64_t file_size; // File size in bytes - InternalKey smallest; // Smallest internal key served by table - InternalKey largest; // Largest internal key served by table - - FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0) { } + uint64_t file_size; // File size in bytes + InternalKey smallest; // Smallest internal key served by table + InternalKey largest; // Largest internal key served by table }; class VersionEdit { public: VersionEdit() { Clear(); } - ~VersionEdit() { } + ~VersionEdit() = default; void Clear(); @@ -59,10 +60,8 @@ class VersionEdit { // Add the specified file at the specified number. // REQUIRES: This version has not been saved (see VersionSet::SaveTo) // REQUIRES: "smallest" and "largest" are smallest and largest keys in file - void AddFile(int level, uint64_t file, - uint64_t file_size, - const InternalKey& smallest, - const InternalKey& largest) { + void AddFile(int level, uint64_t file, uint64_t file_size, + const InternalKey& smallest, const InternalKey& largest) { FileMetaData f; f.number = file; f.file_size = file_size; @@ -72,7 +71,7 @@ class VersionEdit { } // Delete the specified "file" from the specified "level". - void DeleteFile(int level, uint64_t file) { + void RemoveFile(int level, uint64_t file) { deleted_files_.insert(std::make_pair(level, file)); } @@ -84,7 +83,7 @@ class VersionEdit { private: friend class VersionSet; - typedef std::set< std::pair<int, uint64_t> > DeletedFileSet; + typedef std::set<std::pair<int, uint64_t>> DeletedFileSet; std::string comparator_; uint64_t log_number_; @@ -97,9 +96,9 @@ class VersionEdit { bool has_next_file_number_; bool has_last_sequence_; - std::vector< std::pair<int, InternalKey> > compact_pointers_; + std::vector<std::pair<int, InternalKey>> compact_pointers_; DeletedFileSet deleted_files_; - std::vector< std::pair<int, FileMetaData> > new_files_; + std::vector<std::pair<int, FileMetaData>> new_files_; }; } // namespace leveldb diff --git a/db/version_edit_test.cc b/db/version_edit_test.cc index 280310b..a108c15 100644 --- a/db/version_edit_test.cc +++ b/db/version_edit_test.cc @@ -3,7 +3,8 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "db/version_edit.h" -#include "util/testharness.h" + +#include "gtest/gtest.h" namespace leveldb { @@ -17,8 +18,6 @@ static void TestEncodeDecode(const VersionEdit& edit) { ASSERT_EQ(encoded, encoded2); } -class VersionEditTest { }; - TEST(VersionEditTest, EncodeDecode) { static const uint64_t kBig = 1ull << 50; @@ -28,7 +27,7 @@ TEST(VersionEditTest, EncodeDecode) { edit.AddFile(3, kBig + 300 + i, kBig + 400 + i, InternalKey("foo", kBig + 500 + i, kTypeValue), InternalKey("zoo", kBig + 600 + i, kTypeDeletion)); - edit.DeleteFile(4, kBig + 700 + i); + edit.RemoveFile(4, kBig + 700 + i); edit.SetCompactPointer(i, InternalKey("x", kBig + 900 + i, kTypeValue)); } @@ -40,7 +39,3 @@ TEST(VersionEditTest, EncodeDecode) { } } // namespace leveldb - -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); -} diff --git a/db/version_set.cc b/db/version_set.cc index c27ccad..4e37bf9 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -5,7 +5,8 @@ #include "db/version_set.h" #include <algorithm> -#include <stdio.h> +#include <cstdio> + #include "db/filename.h" #include "db/log_reader.h" #include "db/log_writer.h" @@ -84,8 +85,7 @@ Version::~Version() { } int FindFile(const InternalKeyComparator& icmp, - const std::vector<FileMetaData*>& files, - const Slice& key) { + const std::vector<FileMetaData*>& files, const Slice& key) { uint32_t left = 0; uint32_t right = files.size(); while (left < right) { @@ -104,26 +104,25 @@ int FindFile(const InternalKeyComparator& icmp, return right; } -static bool AfterFile(const Comparator* ucmp, - const Slice* user_key, const FileMetaData* f) { +static bool AfterFile(const Comparator* ucmp, const Slice* user_key, + const FileMetaData* f) { // null user_key occurs before all keys and is therefore never after *f return (user_key != nullptr && ucmp->Compare(*user_key, f->largest.user_key()) > 0); } -static bool BeforeFile(const Comparator* ucmp, - const Slice* user_key, const FileMetaData* f) { +static bool BeforeFile(const Comparator* ucmp, const Slice* user_key, + const FileMetaData* f) { // null user_key occurs after all keys and is therefore never before *f return (user_key != nullptr && ucmp->Compare(*user_key, f->smallest.user_key()) < 0); } -bool SomeFileOverlapsRange( - const InternalKeyComparator& icmp, - bool disjoint_sorted_files, - const std::vector<FileMetaData*>& files, - const Slice* smallest_user_key, - const Slice* largest_user_key) { +bool SomeFileOverlapsRange(const InternalKeyComparator& icmp, + bool disjoint_sorted_files, + const std::vector<FileMetaData*>& files, + const Slice* smallest_user_key, + const Slice* largest_user_key) { const Comparator* ucmp = icmp.user_comparator(); if (!disjoint_sorted_files) { // Need to check against all files @@ -143,8 +142,9 @@ bool SomeFileOverlapsRange( uint32_t index = 0; if (smallest_user_key != nullptr) { // Find the earliest possible internal key for smallest_user_key - InternalKey small(*smallest_user_key, kMaxSequenceNumber,kValueTypeForSeek); - index = FindFile(icmp, files, small.Encode()); + InternalKey small_key(*smallest_user_key, kMaxSequenceNumber, + kValueTypeForSeek); + index = FindFile(icmp, files, small_key.Encode()); } if (index >= files.size()) { @@ -164,25 +164,21 @@ class Version::LevelFileNumIterator : public Iterator { public: LevelFileNumIterator(const InternalKeyComparator& icmp, const std::vector<FileMetaData*>* flist) - : icmp_(icmp), - flist_(flist), - index_(flist->size()) { // Marks as invalid + : icmp_(icmp), flist_(flist), index_(flist->size()) { // Marks as invalid } - virtual bool Valid() const { - return index_ < flist_->size(); - } - virtual void Seek(const Slice& target) { + bool Valid() const override { return index_ < flist_->size(); } + void Seek(const Slice& target) override { index_ = FindFile(icmp_, *flist_, target); } - virtual void SeekToFirst() { index_ = 0; } - virtual void SeekToLast() { + void SeekToFirst() override { index_ = 0; } + void SeekToLast() override { index_ = flist_->empty() ? 0 : flist_->size() - 1; } - virtual void Next() { + void Next() override { assert(Valid()); index_++; } - virtual void Prev() { + void Prev() override { assert(Valid()); if (index_ == 0) { index_ = flist_->size(); // Marks as invalid @@ -190,17 +186,18 @@ class Version::LevelFileNumIterator : public Iterator { index_--; } } - Slice key() const { + Slice key() const override { assert(Valid()); return (*flist_)[index_]->largest.Encode(); } - Slice value() const { + Slice value() const override { assert(Valid()); EncodeFixed64(value_buf_, (*flist_)[index_]->number); - EncodeFixed64(value_buf_+8, (*flist_)[index_]->file_size); + EncodeFixed64(value_buf_ + 8, (*flist_)[index_]->file_size); return Slice(value_buf_, sizeof(value_buf_)); } - virtual Status status() const { return Status::OK(); } + Status status() const override { return Status::OK(); } + private: const InternalKeyComparator icmp_; const std::vector<FileMetaData*>* const flist_; @@ -210,16 +207,14 @@ class Version::LevelFileNumIterator : public Iterator { mutable char value_buf_[16]; }; -static Iterator* GetFileIterator(void* arg, - const ReadOptions& options, +static Iterator* GetFileIterator(void* arg, const ReadOptions& options, const Slice& file_value) { TableCache* cache = reinterpret_cast<TableCache*>(arg); if (file_value.size() != 16) { return NewErrorIterator( Status::Corruption("FileReader invoked with unexpected value")); } else { - return cache->NewIterator(options, - DecodeFixed64(file_value.data()), + return cache->NewIterator(options, DecodeFixed64(file_value.data()), DecodeFixed64(file_value.data() + 8)); } } @@ -227,17 +222,16 @@ static Iterator* GetFileIterator(void* arg, Iterator* Version::NewConcatenatingIterator(const ReadOptions& options, int level) const { return NewTwoLevelIterator( - new LevelFileNumIterator(vset_->icmp_, &files_[level]), - &GetFileIterator, vset_->table_cache_, options); + new LevelFileNumIterator(vset_->icmp_, &files_[level]), &GetFileIterator, + vset_->table_cache_, options); } void Version::AddIterators(const ReadOptions& options, std::vector<Iterator*>* iters) { // Merge all level zero files together since they may overlap for (size_t i = 0; i < files_[0].size(); i++) { - iters->push_back( - vset_->table_cache_->NewIterator( - options, files_[0][i]->number, files_[0][i]->file_size)); + iters->push_back(vset_->table_cache_->NewIterator( + options, files_[0][i]->number, files_[0][i]->file_size)); } // For levels > 0, we can use a concatenating iterator that sequentially @@ -264,7 +258,7 @@ struct Saver { Slice user_key; std::string* value; }; -} +} // namespace static void SaveValue(void* arg, const Slice& ikey, const Slice& v) { Saver* s = reinterpret_cast<Saver*>(arg); ParsedInternalKey parsed_key; @@ -284,10 +278,8 @@ static bool NewestFirst(FileMetaData* a, FileMetaData* b) { return a->number > b->number; } -void Version::ForEachOverlapping(Slice user_key, Slice internal_key, - void* arg, +void Version::ForEachOverlapping(Slice user_key, Slice internal_key, void* arg, bool (*func)(void*, int, FileMetaData*)) { - // TODO(sanjay): Change Version::Get() to use this function. const Comparator* ucmp = vset_->icmp_.user_comparator(); // Search level-0 in order from newest to oldest. @@ -329,103 +321,82 @@ void Version::ForEachOverlapping(Slice user_key, Slice internal_key, } } -Status Version::Get(const ReadOptions& options, - const LookupKey& k, - std::string* value, - GetStats* stats) { - Slice ikey = k.internal_key(); - Slice user_key = k.user_key(); - const Comparator* ucmp = vset_->icmp_.user_comparator(); - Status s; - +Status Version::Get(const ReadOptions& options, const LookupKey& k, + std::string* value, GetStats* stats) { stats->seek_file = nullptr; stats->seek_file_level = -1; - FileMetaData* last_file_read = nullptr; - int last_file_read_level = -1; - // We can search level-by-level since entries never hop across - // levels. Therefore we are guaranteed that if we find data - // in an smaller level, later levels are irrelevant. - std::vector<FileMetaData*> tmp; - FileMetaData* tmp2; - for (int level = 0; level < config::kNumLevels; level++) { - size_t num_files = files_[level].size(); - if (num_files == 0) continue; + struct State { + Saver saver; + GetStats* stats; + const ReadOptions* options; + Slice ikey; + FileMetaData* last_file_read; + int last_file_read_level; - // Get the list of files to search in this level - FileMetaData* const* files = &files_[level][0]; - if (level == 0) { - // Level-0 files may overlap each other. Find all files that - // overlap user_key and process them in order from newest to oldest. - tmp.reserve(num_files); - for (uint32_t i = 0; i < num_files; i++) { - FileMetaData* f = files[i]; - if (ucmp->Compare(user_key, f->smallest.user_key()) >= 0 && - ucmp->Compare(user_key, f->largest.user_key()) <= 0) { - tmp.push_back(f); - } - } - if (tmp.empty()) continue; + VersionSet* vset; + Status s; + bool found; - std::sort(tmp.begin(), tmp.end(), NewestFirst); - files = &tmp[0]; - num_files = tmp.size(); - } else { - // Binary search to find earliest index whose largest key >= ikey. - uint32_t index = FindFile(vset_->icmp_, files_[level], ikey); - if (index >= num_files) { - files = nullptr; - num_files = 0; - } else { - tmp2 = files[index]; - if (ucmp->Compare(user_key, tmp2->smallest.user_key()) < 0) { - // All of "tmp2" is past any data for user_key - files = nullptr; - num_files = 0; - } else { - files = &tmp2; - num_files = 1; - } - } - } + static bool Match(void* arg, int level, FileMetaData* f) { + State* state = reinterpret_cast<State*>(arg); - for (uint32_t i = 0; i < num_files; ++i) { - if (last_file_read != nullptr && stats->seek_file == nullptr) { + if (state->stats->seek_file == nullptr && + state->last_file_read != nullptr) { // We have had more than one seek for this read. Charge the 1st file. - stats->seek_file = last_file_read; - stats->seek_file_level = last_file_read_level; + state->stats->seek_file = state->last_file_read; + state->stats->seek_file_level = state->last_file_read_level; } - FileMetaData* f = files[i]; - last_file_read = f; - last_file_read_level = level; - - Saver saver; - saver.state = kNotFound; - saver.ucmp = ucmp; - saver.user_key = user_key; - saver.value = value; - s = vset_->table_cache_->Get(options, f->number, f->file_size, - ikey, &saver, SaveValue); - if (!s.ok()) { - return s; + state->last_file_read = f; + state->last_file_read_level = level; + + state->s = state->vset->table_cache_->Get(*state->options, f->number, + f->file_size, state->ikey, + &state->saver, SaveValue); + if (!state->s.ok()) { + state->found = true; + return false; } - switch (saver.state) { + switch (state->saver.state) { case kNotFound: - break; // Keep searching in other files + return true; // Keep searching in other files case kFound: - return s; + state->found = true; + return false; case kDeleted: - s = Status::NotFound(Slice()); // Use empty error message for speed - return s; + return false; case kCorrupt: - s = Status::Corruption("corrupted key for ", user_key); - return s; + state->s = + Status::Corruption("corrupted key for ", state->saver.user_key); + state->found = true; + return false; } + + // Not reached. Added to avoid false compilation warnings of + // "control reaches end of non-void function". + return false; } - } + }; + + State state; + state.found = false; + state.stats = stats; + state.last_file_read = nullptr; + state.last_file_read_level = -1; - return Status::NotFound(Slice()); // Use an empty error message for speed + state.options = &options; + state.ikey = k.internal_key(); + state.vset = vset_; + + state.saver.state = kNotFound; + state.saver.ucmp = vset_->icmp_.user_comparator(); + state.saver.user_key = k.user_key(); + state.saver.value = value; + + ForEachOverlapping(state.saver.user_key, state.ikey, &state, &State::Match); + + return state.found ? state.s : Status::NotFound(Slice()); } bool Version::UpdateStats(const GetStats& stats) { @@ -479,9 +450,7 @@ bool Version::RecordReadSample(Slice internal_key) { return false; } -void Version::Ref() { - ++refs_; -} +void Version::Ref() { ++refs_; } void Version::Unref() { assert(this != &vset_->dummy_versions_); @@ -492,16 +461,14 @@ void Version::Unref() { } } -bool Version::OverlapInLevel(int level, - const Slice* smallest_user_key, +bool Version::OverlapInLevel(int level, const Slice* smallest_user_key, const Slice* largest_user_key) { return SomeFileOverlapsRange(vset_->icmp_, (level > 0), files_[level], smallest_user_key, largest_user_key); } -int Version::PickLevelForMemTableOutput( - const Slice& smallest_user_key, - const Slice& largest_user_key) { +int Version::PickLevelForMemTableOutput(const Slice& smallest_user_key, + const Slice& largest_user_key) { int level = 0; if (!OverlapInLevel(0, &smallest_user_key, &largest_user_key)) { // Push to next level if there is no overlap in next level, @@ -528,11 +495,9 @@ int Version::PickLevelForMemTableOutput( } // Store in "*inputs" all files in "level" that overlap [begin,end] -void Version::GetOverlappingInputs( - int level, - const InternalKey* begin, - const InternalKey* end, - std::vector<FileMetaData*>* inputs) { +void Version::GetOverlappingInputs(int level, const InternalKey* begin, + const InternalKey* end, + std::vector<FileMetaData*>* inputs) { assert(level >= 0); assert(level < config::kNumLevels); inputs->clear(); @@ -544,7 +509,7 @@ void Version::GetOverlappingInputs( user_end = end->user_key(); } const Comparator* user_cmp = vset_->icmp_.user_comparator(); - for (size_t i = 0; i < files_[level].size(); ) { + for (size_t i = 0; i < files_[level].size();) { FileMetaData* f = files_[level][i++]; const Slice file_start = f->smallest.user_key(); const Slice file_limit = f->largest.user_key(); @@ -561,8 +526,8 @@ void Version::GetOverlappingInputs( user_begin = file_start; inputs->clear(); i = 0; - } else if (end != nullptr && user_cmp->Compare(file_limit, - user_end) > 0) { + } else if (end != nullptr && + user_cmp->Compare(file_limit, user_end) > 0) { user_end = file_limit; inputs->clear(); i = 0; @@ -630,9 +595,7 @@ class VersionSet::Builder { public: // Initialize a builder with the files from *base and other info from *vset - Builder(VersionSet* vset, Version* base) - : vset_(vset), - base_(base) { + Builder(VersionSet* vset, Version* base) : vset_(vset), base_(base) { base_->Ref(); BySmallestKey cmp; cmp.internal_comparator = &vset_->icmp_; @@ -646,8 +609,8 @@ class VersionSet::Builder { const FileSet* added = levels_[level].added_files; std::vector<FileMetaData*> to_unref; to_unref.reserve(added->size()); - for (FileSet::const_iterator it = added->begin(); - it != added->end(); ++it) { + for (FileSet::const_iterator it = added->begin(); it != added->end(); + ++it) { to_unref.push_back(*it); } delete added; @@ -663,7 +626,7 @@ class VersionSet::Builder { } // Apply all of the edits in *edit to the current state. - void Apply(VersionEdit* edit) { + void Apply(const VersionEdit* edit) { // Update compaction pointers for (size_t i = 0; i < edit->compact_pointers_.size(); i++) { const int level = edit->compact_pointers_[i].first; @@ -672,12 +635,9 @@ class VersionSet::Builder { } // Delete files - const VersionEdit::DeletedFileSet& del = edit->deleted_files_; - for (VersionEdit::DeletedFileSet::const_iterator iter = del.begin(); - iter != del.end(); - ++iter) { - const int level = iter->first; - const uint64_t number = iter->second; + for (const auto& deleted_file_set_kvp : edit->deleted_files_) { + const int level = deleted_file_set_kvp.first; + const uint64_t number = deleted_file_set_kvp.second; levels_[level].deleted_files.insert(number); } @@ -700,7 +660,7 @@ class VersionSet::Builder { // same as the compaction of 40KB of data. We are a little // conservative and allow approximately one seek for every 16KB // of data before triggering a compaction. - f->allowed_seeks = (f->file_size / 16384); + f->allowed_seeks = static_cast<int>((f->file_size / 16384U)); if (f->allowed_seeks < 100) f->allowed_seeks = 100; levels_[level].deleted_files.erase(f->number); @@ -718,20 +678,17 @@ class VersionSet::Builder { const std::vector<FileMetaData*>& base_files = base_->files_[level]; std::vector<FileMetaData*>::const_iterator base_iter = base_files.begin(); std::vector<FileMetaData*>::const_iterator base_end = base_files.end(); - const FileSet* added = levels_[level].added_files; - v->files_[level].reserve(base_files.size() + added->size()); - for (FileSet::const_iterator added_iter = added->begin(); - added_iter != added->end(); - ++added_iter) { + const FileSet* added_files = levels_[level].added_files; + v->files_[level].reserve(base_files.size() + added_files->size()); + for (const auto& added_file : *added_files) { // Add all smaller files listed in base_ - for (std::vector<FileMetaData*>::const_iterator bpos - = std::upper_bound(base_iter, base_end, *added_iter, cmp); - base_iter != bpos; - ++base_iter) { + for (std::vector<FileMetaData*>::const_iterator bpos = + std::upper_bound(base_iter, base_end, added_file, cmp); + base_iter != bpos; ++base_iter) { MaybeAddFile(v, level, *base_iter); } - MaybeAddFile(v, level, *added_iter); + MaybeAddFile(v, level, added_file); } // Add remaining base files @@ -743,13 +700,13 @@ class VersionSet::Builder { // Make sure there is no overlap in levels > 0 if (level > 0) { for (uint32_t i = 1; i < v->files_[level].size(); i++) { - const InternalKey& prev_end = v->files_[level][i-1]->largest; + const InternalKey& prev_end = v->files_[level][i - 1]->largest; const InternalKey& this_begin = v->files_[level][i]->smallest; if (vset_->icmp_.Compare(prev_end, this_begin) >= 0) { - fprintf(stderr, "overlapping ranges in same level %s vs. %s\n", - prev_end.DebugString().c_str(), - this_begin.DebugString().c_str()); - abort(); + std::fprintf(stderr, "overlapping ranges in same level %s vs. %s\n", + prev_end.DebugString().c_str(), + this_begin.DebugString().c_str()); + std::abort(); } } } @@ -764,7 +721,7 @@ class VersionSet::Builder { std::vector<FileMetaData*>* files = &v->files_[level]; if (level > 0 && !files->empty()) { // Must not overlap - assert(vset_->icmp_.Compare((*files)[files->size()-1]->largest, + assert(vset_->icmp_.Compare((*files)[files->size() - 1]->largest, f->smallest) < 0); } f->refs++; @@ -773,8 +730,7 @@ class VersionSet::Builder { } }; -VersionSet::VersionSet(const std::string& dbname, - const Options* options, +VersionSet::VersionSet(const std::string& dbname, const Options* options, TableCache* table_cache, const InternalKeyComparator* cmp) : env_(options->env), @@ -850,7 +806,6 @@ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu) { // first call to LogAndApply (when opening the database). assert(descriptor_file_ == nullptr); new_manifest_file = DescriptorFileName(dbname_, manifest_file_number_); - edit->SetNextFile(next_file_number_); s = env_->NewWritableFile(new_manifest_file, &descriptor_file_); if (s.ok()) { descriptor_log_ = new log::Writer(descriptor_file_); @@ -896,17 +851,17 @@ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu) { delete descriptor_file_; descriptor_log_ = nullptr; descriptor_file_ = nullptr; - env_->DeleteFile(new_manifest_file); + env_->RemoveFile(new_manifest_file); } } return s; } -Status VersionSet::Recover(bool *save_manifest) { +Status VersionSet::Recover(bool* save_manifest) { struct LogReporter : public log::Reader::Reporter { Status* status; - virtual void Corruption(size_t bytes, const Status& s) { + void Corruption(size_t bytes, const Status& s) override { if (this->status->ok()) *this->status = s; } }; @@ -917,7 +872,7 @@ Status VersionSet::Recover(bool *save_manifest) { if (!s.ok()) { return s; } - if (current.empty() || current[current.size()-1] != '\n') { + if (current.empty() || current[current.size() - 1] != '\n') { return Status::Corruption("CURRENT file does not end with newline"); } current.resize(current.size() - 1); @@ -927,8 +882,8 @@ Status VersionSet::Recover(bool *save_manifest) { s = env_->NewSequentialFile(dscname, &file); if (!s.ok()) { if (s.IsNotFound()) { - return Status::Corruption( - "CURRENT points to a non-existent file", s.ToString()); + return Status::Corruption("CURRENT points to a non-existent file", + s.ToString()); } return s; } @@ -942,14 +897,17 @@ Status VersionSet::Recover(bool *save_manifest) { uint64_t log_number = 0; uint64_t prev_log_number = 0; Builder builder(this, current_); + int read_records = 0; { LogReporter reporter; reporter.status = &s; - log::Reader reader(file, &reporter, true/*checksum*/, 0/*initial_offset*/); + log::Reader reader(file, &reporter, true /*checksum*/, + 0 /*initial_offset*/); Slice record; std::string scratch; while (reader.ReadRecord(&record, &scratch) && s.ok()) { + ++read_records; VersionEdit edit; s = edit.DecodeFrom(record); if (s.ok()) { @@ -1024,6 +982,10 @@ Status VersionSet::Recover(bool *save_manifest) { } else { *save_manifest = true; } + } else { + std::string error = s.ToString(); + Log(options_->info_log, "Error recovering version set with %d records: %s", + read_records, error.c_str()); } return s; @@ -1071,7 +1033,7 @@ void VersionSet::Finalize(Version* v) { int best_level = -1; double best_score = -1; - for (int level = 0; level < config::kNumLevels-1; level++) { + for (int level = 0; level < config::kNumLevels - 1; level++) { double score; if (level == 0) { // We treat level-0 specially by bounding the number of files @@ -1086,7 +1048,7 @@ void VersionSet::Finalize(Version* v) { // setting, or very high compression ratios, or lots of // overwrites/deletions). score = v->files_[level].size() / - static_cast<double>(config::kL0_CompactionTrigger); + static_cast<double>(config::kL0_CompactionTrigger); } else { // Compute the ratio of current size to size limit. const uint64_t level_bytes = TotalFileSize(v->files_[level]); @@ -1142,16 +1104,13 @@ int VersionSet::NumLevelFiles(int level) const { const char* VersionSet::LevelSummary(LevelSummaryStorage* scratch) const { // Update code if kNumLevels changes - assert(config::kNumLevels == 7); - snprintf(scratch->buffer, sizeof(scratch->buffer), - "files[ %d %d %d %d %d %d %d ]", - int(current_->files_[0].size()), - int(current_->files_[1].size()), - int(current_->files_[2].size()), - int(current_->files_[3].size()), - int(current_->files_[4].size()), - int(current_->files_[5].size()), - int(current_->files_[6].size())); + static_assert(config::kNumLevels == 7, ""); + std::snprintf( + scratch->buffer, sizeof(scratch->buffer), "files[ %d %d %d %d %d %d %d ]", + int(current_->files_[0].size()), int(current_->files_[1].size()), + int(current_->files_[2].size()), int(current_->files_[3].size()), + int(current_->files_[4].size()), int(current_->files_[5].size()), + int(current_->files_[6].size())); return scratch->buffer; } @@ -1188,8 +1147,7 @@ uint64_t VersionSet::ApproximateOffsetOf(Version* v, const InternalKey& ikey) { } void VersionSet::AddLiveFiles(std::set<uint64_t>* live) { - for (Version* v = dummy_versions_.next_; - v != &dummy_versions_; + for (Version* v = dummy_versions_.next_; v != &dummy_versions_; v = v->next_) { for (int level = 0; level < config::kNumLevels; level++) { const std::vector<FileMetaData*>& files = v->files_[level]; @@ -1212,7 +1170,7 @@ int64_t VersionSet::MaxNextLevelOverlappingBytes() { for (int level = 1; level < config::kNumLevels - 1; level++) { for (size_t i = 0; i < current_->files_[level].size(); i++) { const FileMetaData* f = current_->files_[level][i]; - current_->GetOverlappingInputs(level+1, &f->smallest, &f->largest, + current_->GetOverlappingInputs(level + 1, &f->smallest, &f->largest, &overlaps); const int64_t sum = TotalFileSize(overlaps); if (sum > result) { @@ -1227,8 +1185,7 @@ int64_t VersionSet::MaxNextLevelOverlappingBytes() { // *smallest, *largest. // REQUIRES: inputs is not empty void VersionSet::GetRange(const std::vector<FileMetaData*>& inputs, - InternalKey* smallest, - InternalKey* largest) { + InternalKey* smallest, InternalKey* largest) { assert(!inputs.empty()); smallest->Clear(); largest->Clear(); @@ -1253,8 +1210,7 @@ void VersionSet::GetRange(const std::vector<FileMetaData*>& inputs, // REQUIRES: inputs is not empty void VersionSet::GetRange2(const std::vector<FileMetaData*>& inputs1, const std::vector<FileMetaData*>& inputs2, - InternalKey* smallest, - InternalKey* largest) { + InternalKey* smallest, InternalKey* largest) { std::vector<FileMetaData*> all = inputs1; all.insert(all.end(), inputs2.begin(), inputs2.end()); GetRange(all, smallest, largest); @@ -1276,8 +1232,8 @@ Iterator* VersionSet::MakeInputIterator(Compaction* c) { if (c->level() + which == 0) { const std::vector<FileMetaData*>& files = c->inputs_[which]; for (size_t i = 0; i < files.size(); i++) { - list[num++] = table_cache_->NewIterator( - options, files[i]->number, files[i]->file_size); + list[num++] = table_cache_->NewIterator(options, files[i]->number, + files[i]->file_size); } } else { // Create concatenating iterator for the files from this level @@ -1304,7 +1260,7 @@ Compaction* VersionSet::PickCompaction() { if (size_compaction) { level = current_->compaction_level_; assert(level >= 0); - assert(level+1 < config::kNumLevels); + assert(level + 1 < config::kNumLevels); c = new Compaction(options_, level); // Pick the first file that comes after compact_pointer_[level] @@ -1347,12 +1303,95 @@ Compaction* VersionSet::PickCompaction() { return c; } +// Finds the largest key in a vector of files. Returns true if files is not +// empty. +bool FindLargestKey(const InternalKeyComparator& icmp, + const std::vector<FileMetaData*>& files, + InternalKey* largest_key) { + if (files.empty()) { + return false; + } + *largest_key = files[0]->largest; + for (size_t i = 1; i < files.size(); ++i) { + FileMetaData* f = files[i]; + if (icmp.Compare(f->largest, *largest_key) > 0) { + *largest_key = f->largest; + } + } + return true; +} + +// Finds minimum file b2=(l2, u2) in level file for which l2 > u1 and +// user_key(l2) = user_key(u1) +FileMetaData* FindSmallestBoundaryFile( + const InternalKeyComparator& icmp, + const std::vector<FileMetaData*>& level_files, + const InternalKey& largest_key) { + const Comparator* user_cmp = icmp.user_comparator(); + FileMetaData* smallest_boundary_file = nullptr; + for (size_t i = 0; i < level_files.size(); ++i) { + FileMetaData* f = level_files[i]; + if (icmp.Compare(f->smallest, largest_key) > 0 && + user_cmp->Compare(f->smallest.user_key(), largest_key.user_key()) == + 0) { + if (smallest_boundary_file == nullptr || + icmp.Compare(f->smallest, smallest_boundary_file->smallest) < 0) { + smallest_boundary_file = f; + } + } + } + return smallest_boundary_file; +} + +// Extracts the largest file b1 from |compaction_files| and then searches for a +// b2 in |level_files| for which user_key(u1) = user_key(l2). If it finds such a +// file b2 (known as a boundary file) it adds it to |compaction_files| and then +// searches again using this new upper bound. +// +// If there are two blocks, b1=(l1, u1) and b2=(l2, u2) and +// user_key(u1) = user_key(l2), and if we compact b1 but not b2 then a +// subsequent get operation will yield an incorrect result because it will +// return the record from b2 in level i rather than from b1 because it searches +// level by level for records matching the supplied user key. +// +// parameters: +// in level_files: List of files to search for boundary files. +// in/out compaction_files: List of files to extend by adding boundary files. +void AddBoundaryInputs(const InternalKeyComparator& icmp, + const std::vector<FileMetaData*>& level_files, + std::vector<FileMetaData*>* compaction_files) { + InternalKey largest_key; + + // Quick return if compaction_files is empty. + if (!FindLargestKey(icmp, *compaction_files, &largest_key)) { + return; + } + + bool continue_searching = true; + while (continue_searching) { + FileMetaData* smallest_boundary_file = + FindSmallestBoundaryFile(icmp, level_files, largest_key); + + // If a boundary file was found advance largest_key, otherwise we're done. + if (smallest_boundary_file != NULL) { + compaction_files->push_back(smallest_boundary_file); + largest_key = smallest_boundary_file->largest; + } else { + continue_searching = false; + } + } +} + void VersionSet::SetupOtherInputs(Compaction* c) { const int level = c->level(); InternalKey smallest, largest; + + AddBoundaryInputs(icmp_, current_->files_[level], &c->inputs_[0]); GetRange(c->inputs_[0], &smallest, &largest); - current_->GetOverlappingInputs(level+1, &smallest, &largest, &c->inputs_[1]); + current_->GetOverlappingInputs(level + 1, &smallest, &largest, + &c->inputs_[1]); + AddBoundaryInputs(icmp_, current_->files_[level + 1], &c->inputs_[1]); // Get entire range covered by compaction InternalKey all_start, all_limit; @@ -1363,6 +1402,7 @@ void VersionSet::SetupOtherInputs(Compaction* c) { if (!c->inputs_[1].empty()) { std::vector<FileMetaData*> expanded0; current_->GetOverlappingInputs(level, &all_start, &all_limit, &expanded0); + AddBoundaryInputs(icmp_, current_->files_[level], &expanded0); const int64_t inputs0_size = TotalFileSize(c->inputs_[0]); const int64_t inputs1_size = TotalFileSize(c->inputs_[1]); const int64_t expanded0_size = TotalFileSize(expanded0); @@ -1372,18 +1412,15 @@ void VersionSet::SetupOtherInputs(Compaction* c) { InternalKey new_start, new_limit; GetRange(expanded0, &new_start, &new_limit); std::vector<FileMetaData*> expanded1; - current_->GetOverlappingInputs(level+1, &new_start, &new_limit, + current_->GetOverlappingInputs(level + 1, &new_start, &new_limit, &expanded1); + AddBoundaryInputs(icmp_, current_->files_[level + 1], &expanded1); if (expanded1.size() == c->inputs_[1].size()) { Log(options_->info_log, "Expanding@%d %d+%d (%ld+%ld bytes) to %d+%d (%ld+%ld bytes)\n", - level, - int(c->inputs_[0].size()), - int(c->inputs_[1].size()), - long(inputs0_size), long(inputs1_size), - int(expanded0.size()), - int(expanded1.size()), - long(expanded0_size), long(inputs1_size)); + level, int(c->inputs_[0].size()), int(c->inputs_[1].size()), + long(inputs0_size), long(inputs1_size), int(expanded0.size()), + int(expanded1.size()), long(expanded0_size), long(inputs1_size)); smallest = new_start; largest = new_limit; c->inputs_[0] = expanded0; @@ -1408,10 +1445,8 @@ void VersionSet::SetupOtherInputs(Compaction* c) { c->edit_.SetCompactPointer(level, largest); } -Compaction* VersionSet::CompactRange( - int level, - const InternalKey* begin, - const InternalKey* end) { +Compaction* VersionSet::CompactRange(int level, const InternalKey* begin, + const InternalKey* end) { std::vector<FileMetaData*> inputs; current_->GetOverlappingInputs(level, begin, end, &inputs); if (inputs.empty()) { @@ -1474,7 +1509,7 @@ bool Compaction::IsTrivialMove() const { void Compaction::AddInputDeletions(VersionEdit* edit) { for (int which = 0; which < 2; which++) { for (size_t i = 0; i < inputs_[which].size(); i++) { - edit->DeleteFile(level_ + which, inputs_[which][i]->number); + edit->RemoveFile(level_ + which, inputs_[which][i]->number); } } } @@ -1484,7 +1519,7 @@ bool Compaction::IsBaseLevelForKey(const Slice& user_key) { const Comparator* user_cmp = input_version_->vset_->icmp_.user_comparator(); for (int lvl = level_ + 2; lvl < config::kNumLevels; lvl++) { const std::vector<FileMetaData*>& files = input_version_->files_[lvl]; - for (; level_ptrs_[lvl] < files.size(); ) { + while (level_ptrs_[lvl] < files.size()) { FileMetaData* f = files[level_ptrs_[lvl]]; if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) { // We've advanced far enough @@ -1505,8 +1540,9 @@ bool Compaction::ShouldStopBefore(const Slice& internal_key) { // Scan to find earliest grandparent file that contains key. const InternalKeyComparator* icmp = &vset->icmp_; while (grandparent_index_ < grandparents_.size() && - icmp->Compare(internal_key, - grandparents_[grandparent_index_]->largest.Encode()) > 0) { + icmp->Compare(internal_key, + grandparents_[grandparent_index_]->largest.Encode()) > + 0) { if (seen_key_) { overlapped_bytes_ += grandparents_[grandparent_index_]->file_size; } diff --git a/db/version_set.h b/db/version_set.h index 77b9895..ea0c925 100644 --- a/db/version_set.h +++ b/db/version_set.h @@ -18,6 +18,7 @@ #include <map> #include <set> #include <vector> + #include "db/dbformat.h" #include "db/version_edit.h" #include "port/port.h" @@ -25,7 +26,9 @@ namespace leveldb { -namespace log { class Writer; } +namespace log { +class Writer; +} class Compaction; class Iterator; @@ -40,8 +43,7 @@ class WritableFile; // Return files.size() if there is no such file. // REQUIRES: "files" contains a sorted list of non-overlapping files. int FindFile(const InternalKeyComparator& icmp, - const std::vector<FileMetaData*>& files, - const Slice& key); + const std::vector<FileMetaData*>& files, const Slice& key); // Returns true iff some file in "files" overlaps the user key range // [*smallest,*largest]. @@ -57,6 +59,11 @@ bool SomeFileOverlapsRange(const InternalKeyComparator& icmp, class Version { public: + struct GetStats { + FileMetaData* seek_file; + int seek_file_level; + }; + // Append to *iters a sequence of iterators that will // yield the contents of this Version when merged together. // REQUIRES: This version has been saved (see VersionSet::SaveTo) @@ -65,10 +72,6 @@ class Version { // Lookup the value for key. If found, store it in *val and // return OK. Else return a non-OK status. Fills *stats. // REQUIRES: lock is not held - struct GetStats { - FileMetaData* seek_file; - int seek_file_level; - }; Status Get(const ReadOptions&, const LookupKey& key, std::string* val, GetStats* stats); @@ -90,16 +93,15 @@ class Version { void GetOverlappingInputs( int level, - const InternalKey* begin, // nullptr means before all keys - const InternalKey* end, // nullptr means after all keys + const InternalKey* begin, // nullptr means before all keys + const InternalKey* end, // nullptr means after all keys std::vector<FileMetaData*>* inputs); // Returns true iff some file in the specified level overlaps // some part of [*smallest_user_key,*largest_user_key]. // smallest_user_key==nullptr represents a key smaller than all the DB's keys. // largest_user_key==nullptr represents a key largest than all the DB's keys. - bool OverlapInLevel(int level, - const Slice* smallest_user_key, + bool OverlapInLevel(int level, const Slice* smallest_user_key, const Slice* largest_user_key); // Return the level at which we should place a new memtable compaction @@ -117,6 +119,22 @@ class Version { friend class VersionSet; class LevelFileNumIterator; + + explicit Version(VersionSet* vset) + : vset_(vset), + next_(this), + prev_(this), + refs_(0), + file_to_compact_(nullptr), + file_to_compact_level_(-1), + compaction_score_(-1), + compaction_level_(-1) {} + + Version(const Version&) = delete; + Version& operator=(const Version&) = delete; + + ~Version(); + Iterator* NewConcatenatingIterator(const ReadOptions&, int level) const; // Call func(arg, level, f) for every file that overlaps user_key in @@ -124,14 +142,13 @@ class Version { // false, makes no more calls. // // REQUIRES: user portion of internal_key == user_key. - void ForEachOverlapping(Slice user_key, Slice internal_key, - void* arg, + void ForEachOverlapping(Slice user_key, Slice internal_key, void* arg, bool (*func)(void*, int, FileMetaData*)); - VersionSet* vset_; // VersionSet to which this Version belongs - Version* next_; // Next version in linked list - Version* prev_; // Previous version in linked list - int refs_; // Number of live refs to this version + VersionSet* vset_; // VersionSet to which this Version belongs + Version* next_; // Next version in linked list + Version* prev_; // Previous version in linked list + int refs_; // Number of live refs to this version // List of files per level std::vector<FileMetaData*> files_[config::kNumLevels]; @@ -145,28 +162,15 @@ class Version { // are initialized by Finalize(). double compaction_score_; int compaction_level_; - - explicit Version(VersionSet* vset) - : vset_(vset), next_(this), prev_(this), refs_(0), - file_to_compact_(nullptr), - file_to_compact_level_(-1), - compaction_score_(-1), - compaction_level_(-1) { - } - - ~Version(); - - // No copying allowed - Version(const Version&); - void operator=(const Version&); }; class VersionSet { public: - VersionSet(const std::string& dbname, - const Options* options, - TableCache* table_cache, - const InternalKeyComparator*); + VersionSet(const std::string& dbname, const Options* options, + TableCache* table_cache, const InternalKeyComparator*); + VersionSet(const VersionSet&) = delete; + VersionSet& operator=(const VersionSet&) = delete; + ~VersionSet(); // Apply *edit to the current version to form a new descriptor that @@ -178,7 +182,7 @@ class VersionSet { EXCLUSIVE_LOCKS_REQUIRED(mu); // Recover the last saved descriptor from persistent storage. - Status Recover(bool *save_manifest); + Status Recover(bool* save_manifest); // Return the current version. Version* current() const { return current_; } @@ -233,10 +237,8 @@ class VersionSet { // the specified level. Returns nullptr if there is nothing in that // level that overlaps the specified range. Caller should delete // the result. - Compaction* CompactRange( - int level, - const InternalKey* begin, - const InternalKey* end); + Compaction* CompactRange(int level, const InternalKey* begin, + const InternalKey* end); // Return the maximum overlapping data (in bytes) at next level for any // file at a level >= 1. @@ -277,14 +279,12 @@ class VersionSet { void Finalize(Version* v); - void GetRange(const std::vector<FileMetaData*>& inputs, - InternalKey* smallest, + void GetRange(const std::vector<FileMetaData*>& inputs, InternalKey* smallest, InternalKey* largest); void GetRange2(const std::vector<FileMetaData*>& inputs1, const std::vector<FileMetaData*>& inputs2, - InternalKey* smallest, - InternalKey* largest); + InternalKey* smallest, InternalKey* largest); void SetupOtherInputs(Compaction* c); @@ -313,10 +313,6 @@ class VersionSet { // Per-level key at which the next compaction at that level should start. // Either an empty string, or a valid InternalKey. std::string compact_pointer_[config::kNumLevels]; - - // No copying allowed - VersionSet(const VersionSet&); - void operator=(const VersionSet&); }; // A Compaction encapsulates information about a compaction. @@ -373,9 +369,9 @@ class Compaction { VersionEdit edit_; // Each compaction reads inputs from "level_" and "level_+1" - std::vector<FileMetaData*> inputs_[2]; // The two sets of inputs + std::vector<FileMetaData*> inputs_[2]; // The two sets of inputs - // State used to check for number of of overlapping grandparent files + // State used to check for number of overlapping grandparent files // (parent == level_ + 1, grandparent == level_ + 2) std::vector<FileMetaData*> grandparents_; size_t grandparent_index_; // Index in grandparent_starts_ diff --git a/db/version_set_test.cc b/db/version_set_test.cc index d21433e..64bb983 100644 --- a/db/version_set_test.cc +++ b/db/version_set_test.cc @@ -3,18 +3,16 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include "db/version_set.h" + +#include "gtest/gtest.h" #include "util/logging.h" -#include "util/testharness.h" #include "util/testutil.h" namespace leveldb { -class FindFileTest { +class FindFileTest : public testing::Test { public: - std::vector<FileMetaData*> files_; - bool disjoint_sorted_files_; - - FindFileTest() : disjoint_sorted_files_(true) { } + FindFileTest() : disjoint_sorted_files_(true) {} ~FindFileTest() { for (int i = 0; i < files_.size(); i++) { @@ -46,17 +44,22 @@ class FindFileTest { (smallest != nullptr ? &s : nullptr), (largest != nullptr ? &l : nullptr)); } + + bool disjoint_sorted_files_; + + private: + std::vector<FileMetaData*> files_; }; -TEST(FindFileTest, Empty) { +TEST_F(FindFileTest, Empty) { ASSERT_EQ(0, Find("foo")); - ASSERT_TRUE(! Overlaps("a", "z")); - ASSERT_TRUE(! Overlaps(nullptr, "z")); - ASSERT_TRUE(! Overlaps("a", nullptr)); - ASSERT_TRUE(! Overlaps(nullptr, nullptr)); + ASSERT_TRUE(!Overlaps("a", "z")); + ASSERT_TRUE(!Overlaps(nullptr, "z")); + ASSERT_TRUE(!Overlaps("a", nullptr)); + ASSERT_TRUE(!Overlaps(nullptr, nullptr)); } -TEST(FindFileTest, Single) { +TEST_F(FindFileTest, Single) { Add("p", "q"); ASSERT_EQ(0, Find("a")); ASSERT_EQ(0, Find("p")); @@ -65,8 +68,8 @@ TEST(FindFileTest, Single) { ASSERT_EQ(1, Find("q1")); ASSERT_EQ(1, Find("z")); - ASSERT_TRUE(! Overlaps("a", "b")); - ASSERT_TRUE(! Overlaps("z1", "z2")); + ASSERT_TRUE(!Overlaps("a", "b")); + ASSERT_TRUE(!Overlaps("z1", "z2")); ASSERT_TRUE(Overlaps("a", "p")); ASSERT_TRUE(Overlaps("a", "q")); ASSERT_TRUE(Overlaps("a", "z")); @@ -78,16 +81,15 @@ TEST(FindFileTest, Single) { ASSERT_TRUE(Overlaps("q", "q")); ASSERT_TRUE(Overlaps("q", "q1")); - ASSERT_TRUE(! Overlaps(nullptr, "j")); - ASSERT_TRUE(! Overlaps("r", nullptr)); + ASSERT_TRUE(!Overlaps(nullptr, "j")); + ASSERT_TRUE(!Overlaps("r", nullptr)); ASSERT_TRUE(Overlaps(nullptr, "p")); ASSERT_TRUE(Overlaps(nullptr, "p1")); ASSERT_TRUE(Overlaps("q", nullptr)); ASSERT_TRUE(Overlaps(nullptr, nullptr)); } - -TEST(FindFileTest, Multiple) { +TEST_F(FindFileTest, Multiple) { Add("150", "200"); Add("200", "250"); Add("300", "350"); @@ -110,10 +112,10 @@ TEST(FindFileTest, Multiple) { ASSERT_EQ(3, Find("450")); ASSERT_EQ(4, Find("451")); - ASSERT_TRUE(! Overlaps("100", "149")); - ASSERT_TRUE(! Overlaps("251", "299")); - ASSERT_TRUE(! Overlaps("451", "500")); - ASSERT_TRUE(! Overlaps("351", "399")); + ASSERT_TRUE(!Overlaps("100", "149")); + ASSERT_TRUE(!Overlaps("251", "299")); + ASSERT_TRUE(!Overlaps("451", "500")); + ASSERT_TRUE(!Overlaps("351", "399")); ASSERT_TRUE(Overlaps("100", "150")); ASSERT_TRUE(Overlaps("100", "200")); @@ -125,13 +127,13 @@ TEST(FindFileTest, Multiple) { ASSERT_TRUE(Overlaps("450", "500")); } -TEST(FindFileTest, MultipleNullBoundaries) { +TEST_F(FindFileTest, MultipleNullBoundaries) { Add("150", "200"); Add("200", "250"); Add("300", "350"); Add("400", "450"); - ASSERT_TRUE(! Overlaps(nullptr, "149")); - ASSERT_TRUE(! Overlaps("451", nullptr)); + ASSERT_TRUE(!Overlaps(nullptr, "149")); + ASSERT_TRUE(!Overlaps("451", nullptr)); ASSERT_TRUE(Overlaps(nullptr, nullptr)); ASSERT_TRUE(Overlaps(nullptr, "150")); ASSERT_TRUE(Overlaps(nullptr, "199")); @@ -145,21 +147,21 @@ TEST(FindFileTest, MultipleNullBoundaries) { ASSERT_TRUE(Overlaps("450", nullptr)); } -TEST(FindFileTest, OverlapSequenceChecks) { +TEST_F(FindFileTest, OverlapSequenceChecks) { Add("200", "200", 5000, 3000); - ASSERT_TRUE(! Overlaps("199", "199")); - ASSERT_TRUE(! Overlaps("201", "300")); + ASSERT_TRUE(!Overlaps("199", "199")); + ASSERT_TRUE(!Overlaps("201", "300")); ASSERT_TRUE(Overlaps("200", "200")); ASSERT_TRUE(Overlaps("190", "200")); ASSERT_TRUE(Overlaps("200", "210")); } -TEST(FindFileTest, OverlappingFiles) { +TEST_F(FindFileTest, OverlappingFiles) { Add("150", "600"); Add("400", "500"); disjoint_sorted_files_ = false; - ASSERT_TRUE(! Overlaps("100", "149")); - ASSERT_TRUE(! Overlaps("601", "700")); + ASSERT_TRUE(!Overlaps("100", "149")); + ASSERT_TRUE(!Overlaps("601", "700")); ASSERT_TRUE(Overlaps("100", "150")); ASSERT_TRUE(Overlaps("100", "200")); ASSERT_TRUE(Overlaps("100", "300")); @@ -172,8 +174,158 @@ TEST(FindFileTest, OverlappingFiles) { ASSERT_TRUE(Overlaps("600", "700")); } -} // namespace leveldb +void AddBoundaryInputs(const InternalKeyComparator& icmp, + const std::vector<FileMetaData*>& level_files, + std::vector<FileMetaData*>* compaction_files); + +class AddBoundaryInputsTest : public testing::Test { + public: + std::vector<FileMetaData*> level_files_; + std::vector<FileMetaData*> compaction_files_; + std::vector<FileMetaData*> all_files_; + InternalKeyComparator icmp_; + + AddBoundaryInputsTest() : icmp_(BytewiseComparator()) {} + + ~AddBoundaryInputsTest() { + for (size_t i = 0; i < all_files_.size(); ++i) { + delete all_files_[i]; + } + all_files_.clear(); + } + + FileMetaData* CreateFileMetaData(uint64_t number, InternalKey smallest, + InternalKey largest) { + FileMetaData* f = new FileMetaData(); + f->number = number; + f->smallest = smallest; + f->largest = largest; + all_files_.push_back(f); + return f; + } +}; + +TEST_F(AddBoundaryInputsTest, TestEmptyFileSets) { + AddBoundaryInputs(icmp_, level_files_, &compaction_files_); + ASSERT_TRUE(compaction_files_.empty()); + ASSERT_TRUE(level_files_.empty()); +} + +TEST_F(AddBoundaryInputsTest, TestEmptyLevelFiles) { + FileMetaData* f1 = + CreateFileMetaData(1, InternalKey("100", 2, kTypeValue), + InternalKey(InternalKey("100", 1, kTypeValue))); + compaction_files_.push_back(f1); -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); + AddBoundaryInputs(icmp_, level_files_, &compaction_files_); + ASSERT_EQ(1, compaction_files_.size()); + ASSERT_EQ(f1, compaction_files_[0]); + ASSERT_TRUE(level_files_.empty()); } + +TEST_F(AddBoundaryInputsTest, TestEmptyCompactionFiles) { + FileMetaData* f1 = + CreateFileMetaData(1, InternalKey("100", 2, kTypeValue), + InternalKey(InternalKey("100", 1, kTypeValue))); + level_files_.push_back(f1); + + AddBoundaryInputs(icmp_, level_files_, &compaction_files_); + ASSERT_TRUE(compaction_files_.empty()); + ASSERT_EQ(1, level_files_.size()); + ASSERT_EQ(f1, level_files_[0]); +} + +TEST_F(AddBoundaryInputsTest, TestNoBoundaryFiles) { + FileMetaData* f1 = + CreateFileMetaData(1, InternalKey("100", 2, kTypeValue), + InternalKey(InternalKey("100", 1, kTypeValue))); + FileMetaData* f2 = + CreateFileMetaData(1, InternalKey("200", 2, kTypeValue), + InternalKey(InternalKey("200", 1, kTypeValue))); + FileMetaData* f3 = + CreateFileMetaData(1, InternalKey("300", 2, kTypeValue), + InternalKey(InternalKey("300", 1, kTypeValue))); + + level_files_.push_back(f3); + level_files_.push_back(f2); + level_files_.push_back(f1); + compaction_files_.push_back(f2); + compaction_files_.push_back(f3); + + AddBoundaryInputs(icmp_, level_files_, &compaction_files_); + ASSERT_EQ(2, compaction_files_.size()); +} + +TEST_F(AddBoundaryInputsTest, TestOneBoundaryFiles) { + FileMetaData* f1 = + CreateFileMetaData(1, InternalKey("100", 3, kTypeValue), + InternalKey(InternalKey("100", 2, kTypeValue))); + FileMetaData* f2 = + CreateFileMetaData(1, InternalKey("100", 1, kTypeValue), + InternalKey(InternalKey("200", 3, kTypeValue))); + FileMetaData* f3 = + CreateFileMetaData(1, InternalKey("300", 2, kTypeValue), + InternalKey(InternalKey("300", 1, kTypeValue))); + + level_files_.push_back(f3); + level_files_.push_back(f2); + level_files_.push_back(f1); + compaction_files_.push_back(f1); + + AddBoundaryInputs(icmp_, level_files_, &compaction_files_); + ASSERT_EQ(2, compaction_files_.size()); + ASSERT_EQ(f1, compaction_files_[0]); + ASSERT_EQ(f2, compaction_files_[1]); +} + +TEST_F(AddBoundaryInputsTest, TestTwoBoundaryFiles) { + FileMetaData* f1 = + CreateFileMetaData(1, InternalKey("100", 6, kTypeValue), + InternalKey(InternalKey("100", 5, kTypeValue))); + FileMetaData* f2 = + CreateFileMetaData(1, InternalKey("100", 2, kTypeValue), + InternalKey(InternalKey("300", 1, kTypeValue))); + FileMetaData* f3 = + CreateFileMetaData(1, InternalKey("100", 4, kTypeValue), + InternalKey(InternalKey("100", 3, kTypeValue))); + + level_files_.push_back(f2); + level_files_.push_back(f3); + level_files_.push_back(f1); + compaction_files_.push_back(f1); + + AddBoundaryInputs(icmp_, level_files_, &compaction_files_); + ASSERT_EQ(3, compaction_files_.size()); + ASSERT_EQ(f1, compaction_files_[0]); + ASSERT_EQ(f3, compaction_files_[1]); + ASSERT_EQ(f2, compaction_files_[2]); +} + +TEST_F(AddBoundaryInputsTest, TestDisjoinFilePointers) { + FileMetaData* f1 = + CreateFileMetaData(1, InternalKey("100", 6, kTypeValue), + InternalKey(InternalKey("100", 5, kTypeValue))); + FileMetaData* f2 = + CreateFileMetaData(1, InternalKey("100", 6, kTypeValue), + InternalKey(InternalKey("100", 5, kTypeValue))); + FileMetaData* f3 = + CreateFileMetaData(1, InternalKey("100", 2, kTypeValue), + InternalKey(InternalKey("300", 1, kTypeValue))); + FileMetaData* f4 = + CreateFileMetaData(1, InternalKey("100", 4, kTypeValue), + InternalKey(InternalKey("100", 3, kTypeValue))); + + level_files_.push_back(f2); + level_files_.push_back(f3); + level_files_.push_back(f4); + + compaction_files_.push_back(f1); + + AddBoundaryInputs(icmp_, level_files_, &compaction_files_); + ASSERT_EQ(3, compaction_files_.size()); + ASSERT_EQ(f1, compaction_files_[0]); + ASSERT_EQ(f4, compaction_files_[1]); + ASSERT_EQ(f3, compaction_files_[2]); +} + +} // namespace leveldb diff --git a/db/write_batch.cc b/db/write_batch.cc index 7f8f3e8..b54313c 100644 --- a/db/write_batch.cc +++ b/db/write_batch.cc @@ -15,10 +15,10 @@ #include "leveldb/write_batch.h" -#include "leveldb/db.h" #include "db/dbformat.h" #include "db/memtable.h" #include "db/write_batch_internal.h" +#include "leveldb/db.h" #include "util/coding.h" namespace leveldb { @@ -26,22 +26,18 @@ namespace leveldb { // WriteBatch header has an 8-byte sequence number followed by a 4-byte count. static const size_t kHeader = 12; -WriteBatch::WriteBatch() { - Clear(); -} +WriteBatch::WriteBatch() { Clear(); } -WriteBatch::~WriteBatch() { } +WriteBatch::~WriteBatch() = default; -WriteBatch::Handler::~Handler() { } +WriteBatch::Handler::~Handler() = default; void WriteBatch::Clear() { rep_.clear(); rep_.resize(kHeader); } -size_t WriteBatch::ApproximateSize() { - return rep_.size(); -} +size_t WriteBatch::ApproximateSize() const { return rep_.size(); } Status WriteBatch::Iterate(Handler* handler) const { Slice input(rep_); @@ -112,25 +108,28 @@ void WriteBatch::Delete(const Slice& key) { PutLengthPrefixedSlice(&rep_, key); } +void WriteBatch::Append(const WriteBatch& source) { + WriteBatchInternal::Append(this, &source); +} + namespace { class MemTableInserter : public WriteBatch::Handler { public: SequenceNumber sequence_; MemTable* mem_; - virtual void Put(const Slice& key, const Slice& value) { + void Put(const Slice& key, const Slice& value) override { mem_->Add(sequence_, kTypeValue, key, value); sequence_++; } - virtual void Delete(const Slice& key) { + void Delete(const Slice& key) override { mem_->Add(sequence_, kTypeDeletion, key, Slice()); sequence_++; } }; } // namespace -Status WriteBatchInternal::InsertInto(const WriteBatch* b, - MemTable* memtable) { +Status WriteBatchInternal::InsertInto(const WriteBatch* b, MemTable* memtable) { MemTableInserter inserter; inserter.sequence_ = WriteBatchInternal::Sequence(b); inserter.mem_ = memtable; diff --git a/db/write_batch_internal.h b/db/write_batch_internal.h index 9448ef7..fce86e3 100644 --- a/db/write_batch_internal.h +++ b/db/write_batch_internal.h @@ -29,13 +29,9 @@ class WriteBatchInternal { // this batch. static void SetSequence(WriteBatch* batch, SequenceNumber seq); - static Slice Contents(const WriteBatch* batch) { - return Slice(batch->rep_); - } + static Slice Contents(const WriteBatch* batch) { return Slice(batch->rep_); } - static size_t ByteSize(const WriteBatch* batch) { - return batch->rep_.size(); - } + static size_t ByteSize(const WriteBatch* batch) { return batch->rep_.size(); } static void SetContents(WriteBatch* batch, const Slice& contents); @@ -46,5 +42,4 @@ class WriteBatchInternal { } // namespace leveldb - #endif // STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_ diff --git a/db/write_batch_test.cc b/db/write_batch_test.cc index 8d38023..1a3ea8f 100644 --- a/db/write_batch_test.cc +++ b/db/write_batch_test.cc @@ -2,13 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. -#include "leveldb/db.h" - +#include "gtest/gtest.h" #include "db/memtable.h" #include "db/write_batch_internal.h" +#include "leveldb/db.h" #include "leveldb/env.h" #include "util/logging.h" -#include "util/testharness.h" namespace leveldb { @@ -22,7 +21,7 @@ static std::string PrintContents(WriteBatch* b) { Iterator* iter = mem->NewIterator(); for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { ParsedInternalKey ikey; - ASSERT_TRUE(ParseInternalKey(iter->key(), &ikey)); + EXPECT_TRUE(ParseInternalKey(iter->key(), &ikey)); switch (ikey.type) { case kTypeValue: state.append("Put("); @@ -52,8 +51,6 @@ static std::string PrintContents(WriteBatch* b) { return state; } -class WriteBatchTest { }; - TEST(WriteBatchTest, Empty) { WriteBatch batch; ASSERT_EQ("", PrintContents(&batch)); @@ -68,10 +65,11 @@ TEST(WriteBatchTest, Multiple) { WriteBatchInternal::SetSequence(&batch, 100); ASSERT_EQ(100, WriteBatchInternal::Sequence(&batch)); ASSERT_EQ(3, WriteBatchInternal::Count(&batch)); - ASSERT_EQ("Put(baz, boo)@102" - "Delete(box)@101" - "Put(foo, bar)@100", - PrintContents(&batch)); + ASSERT_EQ( + "Put(baz, boo)@102" + "Delete(box)@101" + "Put(foo, bar)@100", + PrintContents(&batch)); } TEST(WriteBatchTest, Corruption) { @@ -81,36 +79,37 @@ TEST(WriteBatchTest, Corruption) { WriteBatchInternal::SetSequence(&batch, 200); Slice contents = WriteBatchInternal::Contents(&batch); WriteBatchInternal::SetContents(&batch, - Slice(contents.data(),contents.size()-1)); - ASSERT_EQ("Put(foo, bar)@200" - "ParseError()", - PrintContents(&batch)); + Slice(contents.data(), contents.size() - 1)); + ASSERT_EQ( + "Put(foo, bar)@200" + "ParseError()", + PrintContents(&batch)); } TEST(WriteBatchTest, Append) { WriteBatch b1, b2; WriteBatchInternal::SetSequence(&b1, 200); WriteBatchInternal::SetSequence(&b2, 300); - WriteBatchInternal::Append(&b1, &b2); - ASSERT_EQ("", - PrintContents(&b1)); + b1.Append(b2); + ASSERT_EQ("", PrintContents(&b1)); b2.Put("a", "va"); - WriteBatchInternal::Append(&b1, &b2); - ASSERT_EQ("Put(a, va)@200", - PrintContents(&b1)); + b1.Append(b2); + ASSERT_EQ("Put(a, va)@200", PrintContents(&b1)); b2.Clear(); b2.Put("b", "vb"); - WriteBatchInternal::Append(&b1, &b2); - ASSERT_EQ("Put(a, va)@200" - "Put(b, vb)@201", - PrintContents(&b1)); + b1.Append(b2); + ASSERT_EQ( + "Put(a, va)@200" + "Put(b, vb)@201", + PrintContents(&b1)); b2.Delete("foo"); - WriteBatchInternal::Append(&b1, &b2); - ASSERT_EQ("Put(a, va)@200" - "Put(b, vb)@202" - "Put(b, vb)@201" - "Delete(foo)@203", - PrintContents(&b1)); + b1.Append(b2); + ASSERT_EQ( + "Put(a, va)@200" + "Put(b, vb)@202" + "Put(b, vb)@201" + "Delete(foo)@203", + PrintContents(&b1)); } TEST(WriteBatchTest, ApproximateSize) { @@ -131,7 +130,3 @@ TEST(WriteBatchTest, ApproximateSize) { } } // namespace leveldb - -int main(int argc, char** argv) { - return leveldb::test::RunAllTests(); -} |