summaryrefslogtreecommitdiff
path: root/db
diff options
context:
space:
mode:
authorivan <sjingivan@gmail.com>2019-05-23 14:02:04 +0800
committerGitHub <noreply@github.com>2019-05-23 14:02:04 +0800
commit63d5315e1c224e52da8ec68d118c5b73ba2a63fc (patch)
treec87a7385af799948f1180a643218e6c51c6752e9 /db
parent2883fcd849ca7b479d8a2f4fc929f0b6c7b9e372 (diff)
parentc00e177f3613068eda4bff4abfbd3bd4165a86e8 (diff)
downloadleveldb-63d5315e1c224e52da8ec68d118c5b73ba2a63fc.tar.gz
Merge branch 'master' into master
Diffstat (limited to 'db')
-rw-r--r--db/autocompact_test.cc36
-rw-r--r--db/builder.cc25
-rw-r--r--db/builder.h8
-rw-r--r--db/c.cc389
-rw-r--r--db/c_test.c28
-rw-r--r--db/corruption_test.cc100
-rw-r--r--db/db_bench.cc991
-rw-r--r--db/db_impl.cc679
-rw-r--r--db/db_impl.h154
-rw-r--r--db/db_iter.cc93
-rw-r--r--db/db_iter.h12
-rw-r--r--db/db_test.cc745
-rw-r--r--db/dbformat.cc43
-rw-r--r--db/dbformat.h72
-rw-r--r--db/dbformat_test.cc95
-rw-r--r--db/dumpfile.cc29
-rw-r--r--db/fault_injection_test.cc249
-rw-r--r--db/filename.cc37
-rw-r--r--db/filename.h31
-rw-r--r--db/filename_test.cc86
-rw-r--r--db/leveldbutil.cc (renamed from db/leveldb_main.cc)19
-rw-r--r--db/log_format.h2
-rw-r--r--db/log_reader.cc48
-rw-r--r--db/log_reader.h42
-rw-r--r--db/log_test.cc342
-rw-r--r--db/log_writer.cc38
-rw-r--r--db/log_writer.h20
-rw-r--r--db/memtable.cc70
-rw-r--r--db/memtable.h28
-rw-r--r--db/recovery_test.cc330
-rw-r--r--db/repair.cc95
-rw-r--r--db/skiplist.h182
-rw-r--r--db/skiplist_test.cc65
-rw-r--r--db/snapshot.h78
-rw-r--r--db/table_cache.cc47
-rw-r--r--db/table_cache.h33
-rw-r--r--db/version_edit.cc55
-rw-r--r--db/version_edit.h27
-rw-r--r--db/version_edit_test.cc6
-rw-r--r--db/version_set.cc486
-rw-r--r--db/version_set.h129
-rw-r--r--db/version_set_test.cc243
-rw-r--r--db/write_batch.cc23
-rw-r--r--db/write_batch_internal.h9
-rw-r--r--db/write_batch_test.cc73
45 files changed, 3095 insertions, 3297 deletions
diff --git a/db/autocompact_test.cc b/db/autocompact_test.cc
index d20a236..e6c97a0 100644
--- a/db/autocompact_test.cc
+++ b/db/autocompact_test.cc
@@ -2,9 +2,9 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#include "leveldb/db.h"
#include "db/db_impl.h"
#include "leveldb/cache.h"
+#include "leveldb/db.h"
#include "util/testharness.h"
#include "util/testutil.h"
@@ -12,11 +12,6 @@ namespace leveldb {
class AutoCompactTest {
public:
- std::string dbname_;
- Cache* tiny_cache_;
- Options options_;
- DB* db_;
-
AutoCompactTest() {
dbname_ = test::TmpDir() + "/autocompact_test";
tiny_cache_ = NewLRUCache(100);
@@ -47,6 +42,12 @@ class AutoCompactTest {
}
void DoReads(int n);
+
+ private:
+ std::string dbname_;
+ Cache* tiny_cache_;
+ Options options_;
+ DB* db_;
};
static const int kValueSize = 200 * 1024;
@@ -81,17 +82,16 @@ void AutoCompactTest::DoReads(int n) {
ASSERT_LT(read, 100) << "Taking too long to compact";
Iterator* iter = db_->NewIterator(ReadOptions());
for (iter->SeekToFirst();
- iter->Valid() && iter->key().ToString() < limit_key;
- iter->Next()) {
+ iter->Valid() && iter->key().ToString() < limit_key; iter->Next()) {
// Drop data
}
delete iter;
// Wait a little bit to allow any triggered compactions to complete.
Env::Default()->SleepForMicroseconds(1000000);
uint64_t size = Size(Key(0), Key(n));
- fprintf(stderr, "iter %3d => %7.3f MB [other %7.3f MB]\n",
- read+1, size/1048576.0, Size(Key(n), Key(kCount))/1048576.0);
- if (size <= initial_size/10) {
+ fprintf(stderr, "iter %3d => %7.3f MB [other %7.3f MB]\n", read + 1,
+ size / 1048576.0, Size(Key(n), Key(kCount)) / 1048576.0);
+ if (size <= initial_size / 10) {
break;
}
}
@@ -100,19 +100,13 @@ void AutoCompactTest::DoReads(int n) {
// is pretty much unchanged.
const int64_t final_other_size = Size(Key(n), Key(kCount));
ASSERT_LE(final_other_size, initial_other_size + 1048576);
- ASSERT_GE(final_other_size, initial_other_size/5 - 1048576);
+ ASSERT_GE(final_other_size, initial_other_size / 5 - 1048576);
}
-TEST(AutoCompactTest, ReadAll) {
- DoReads(kCount);
-}
+TEST(AutoCompactTest, ReadAll) { DoReads(kCount); }
-TEST(AutoCompactTest, ReadHalf) {
- DoReads(kCount/2);
-}
+TEST(AutoCompactTest, ReadHalf) { DoReads(kCount / 2); }
} // namespace leveldb
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
+int main(int argc, char** argv) { return leveldb::test::RunAllTests(); }
diff --git a/db/builder.cc b/db/builder.cc
index f419882..9520ee4 100644
--- a/db/builder.cc
+++ b/db/builder.cc
@@ -4,8 +4,8 @@
#include "db/builder.h"
-#include "db/filename.h"
#include "db/dbformat.h"
+#include "db/filename.h"
#include "db/table_cache.h"
#include "db/version_edit.h"
#include "leveldb/db.h"
@@ -14,12 +14,8 @@
namespace leveldb {
-Status BuildTable(const std::string& dbname,
- Env* env,
- const Options& options,
- TableCache* table_cache,
- Iterator* iter,
- FileMetaData* meta) {
+Status BuildTable(const std::string& dbname, Env* env, const Options& options,
+ TableCache* table_cache, Iterator* iter, FileMetaData* meta) {
Status s;
meta->file_size = 0;
iter->SeekToFirst();
@@ -41,14 +37,10 @@ Status BuildTable(const std::string& dbname,
}
// Finish and check for builder errors
+ s = builder->Finish();
if (s.ok()) {
- s = builder->Finish();
- if (s.ok()) {
- meta->file_size = builder->FileSize();
- assert(meta->file_size > 0);
- }
- } else {
- builder->Abandon();
+ meta->file_size = builder->FileSize();
+ assert(meta->file_size > 0);
}
delete builder;
@@ -60,12 +52,11 @@ Status BuildTable(const std::string& dbname,
s = file->Close();
}
delete file;
- file = NULL;
+ file = nullptr;
if (s.ok()) {
// Verify that the table is usable
- Iterator* it = table_cache->NewIterator(ReadOptions(),
- meta->number,
+ Iterator* it = table_cache->NewIterator(ReadOptions(), meta->number,
meta->file_size);
s = it->status();
delete it;
diff --git a/db/builder.h b/db/builder.h
index 62431fc..7bd0b80 100644
--- a/db/builder.h
+++ b/db/builder.h
@@ -22,12 +22,8 @@ class VersionEdit;
// *meta will be filled with metadata about the generated table.
// If no data is present in *iter, meta->file_size will be set to
// zero, and no Table file will be produced.
-extern Status BuildTable(const std::string& dbname,
- Env* env,
- const Options& options,
- TableCache* table_cache,
- Iterator* iter,
- FileMetaData* meta);
+Status BuildTable(const std::string& dbname, Env* env, const Options& options,
+ TableCache* table_cache, Iterator* iter, FileMetaData* meta);
} // namespace leveldb
diff --git a/db/c.cc b/db/c.cc
index 08ff0ad..1f6fd64 100644
--- a/db/c.cc
+++ b/db/c.cc
@@ -5,7 +5,7 @@
#include "leveldb/c.h"
#include <stdlib.h>
-#include <unistd.h>
+
#include "leveldb/cache.h"
#include "leveldb/comparator.h"
#include "leveldb/db.h"
@@ -43,69 +43,72 @@ using leveldb::WriteOptions;
extern "C" {
-struct leveldb_t { DB* rep; };
-struct leveldb_iterator_t { Iterator* rep; };
-struct leveldb_writebatch_t { WriteBatch rep; };
-struct leveldb_snapshot_t { const Snapshot* rep; };
-struct leveldb_readoptions_t { ReadOptions rep; };
-struct leveldb_writeoptions_t { WriteOptions rep; };
-struct leveldb_options_t { Options rep; };
-struct leveldb_cache_t { Cache* rep; };
-struct leveldb_seqfile_t { SequentialFile* rep; };
-struct leveldb_randomfile_t { RandomAccessFile* rep; };
-struct leveldb_writablefile_t { WritableFile* rep; };
-struct leveldb_logger_t { Logger* rep; };
-struct leveldb_filelock_t { FileLock* rep; };
+struct leveldb_t {
+ DB* rep;
+};
+struct leveldb_iterator_t {
+ Iterator* rep;
+};
+struct leveldb_writebatch_t {
+ WriteBatch rep;
+};
+struct leveldb_snapshot_t {
+ const Snapshot* rep;
+};
+struct leveldb_readoptions_t {
+ ReadOptions rep;
+};
+struct leveldb_writeoptions_t {
+ WriteOptions rep;
+};
+struct leveldb_options_t {
+ Options rep;
+};
+struct leveldb_cache_t {
+ Cache* rep;
+};
+struct leveldb_seqfile_t {
+ SequentialFile* rep;
+};
+struct leveldb_randomfile_t {
+ RandomAccessFile* rep;
+};
+struct leveldb_writablefile_t {
+ WritableFile* rep;
+};
+struct leveldb_logger_t {
+ Logger* rep;
+};
+struct leveldb_filelock_t {
+ FileLock* rep;
+};
struct leveldb_comparator_t : public Comparator {
- void* state_;
- void (*destructor_)(void*);
- int (*compare_)(
- void*,
- const char* a, size_t alen,
- const char* b, size_t blen);
- const char* (*name_)(void*);
+ ~leveldb_comparator_t() override { (*destructor_)(state_); }
- virtual ~leveldb_comparator_t() {
- (*destructor_)(state_);
- }
-
- virtual int Compare(const Slice& a, const Slice& b) const {
+ int Compare(const Slice& a, const Slice& b) const override {
return (*compare_)(state_, a.data(), a.size(), b.data(), b.size());
}
- virtual const char* Name() const {
- return (*name_)(state_);
- }
+ const char* Name() const override { return (*name_)(state_); }
// No-ops since the C binding does not support key shortening methods.
- virtual void FindShortestSeparator(std::string*, const Slice&) const { }
- virtual void FindShortSuccessor(std::string* key) const { }
-};
+ void FindShortestSeparator(std::string*, const Slice&) const override {}
+ void FindShortSuccessor(std::string* key) const override {}
-struct leveldb_filterpolicy_t : public FilterPolicy {
void* state_;
void (*destructor_)(void*);
+ int (*compare_)(void*, const char* a, size_t alen, const char* b,
+ size_t blen);
const char* (*name_)(void*);
- char* (*create_)(
- void*,
- const char* const* key_array, const size_t* key_length_array,
- int num_keys,
- size_t* filter_length);
- unsigned char (*key_match_)(
- void*,
- const char* key, size_t length,
- const char* filter, size_t filter_length);
-
- virtual ~leveldb_filterpolicy_t() {
- (*destructor_)(state_);
- }
+};
- virtual const char* Name() const {
- return (*name_)(state_);
- }
+struct leveldb_filterpolicy_t : public FilterPolicy {
+ ~leveldb_filterpolicy_t() override { (*destructor_)(state_); }
- virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const {
+ const char* Name() const override { return (*name_)(state_); }
+
+ void CreateFilter(const Slice* keys, int n, std::string* dst) const override {
std::vector<const char*> key_pointers(n);
std::vector<size_t> key_sizes(n);
for (int i = 0; i < n; i++) {
@@ -118,10 +121,19 @@ struct leveldb_filterpolicy_t : public FilterPolicy {
free(filter);
}
- virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const {
- return (*key_match_)(state_, key.data(), key.size(),
- filter.data(), filter.size());
+ bool KeyMayMatch(const Slice& key, const Slice& filter) const override {
+ return (*key_match_)(state_, key.data(), key.size(), filter.data(),
+ filter.size());
}
+
+ void* state_;
+ void (*destructor_)(void*);
+ const char* (*name_)(void*);
+ char* (*create_)(void*, const char* const* key_array,
+ const size_t* key_length_array, int num_keys,
+ size_t* filter_length);
+ unsigned char (*key_match_)(void*, const char* key, size_t length,
+ const char* filter, size_t filter_length);
};
struct leveldb_env_t {
@@ -130,10 +142,10 @@ struct leveldb_env_t {
};
static bool SaveError(char** errptr, const Status& s) {
- assert(errptr != NULL);
+ assert(errptr != nullptr);
if (s.ok()) {
return false;
- } else if (*errptr == NULL) {
+ } else if (*errptr == nullptr) {
*errptr = strdup(s.ToString().c_str());
} else {
// TODO(sanjay): Merge with existing error?
@@ -149,13 +161,11 @@ static char* CopyString(const std::string& str) {
return result;
}
-leveldb_t* leveldb_open(
- const leveldb_options_t* options,
- const char* name,
- char** errptr) {
+leveldb_t* leveldb_open(const leveldb_options_t* options, const char* name,
+ char** errptr) {
DB* db;
if (SaveError(errptr, DB::Open(options->rep, std::string(name), &db))) {
- return NULL;
+ return nullptr;
}
leveldb_t* result = new leveldb_t;
result->rep = db;
@@ -167,40 +177,27 @@ void leveldb_close(leveldb_t* db) {
delete db;
}
-void leveldb_put(
- leveldb_t* db,
- const leveldb_writeoptions_t* options,
- const char* key, size_t keylen,
- const char* val, size_t vallen,
- char** errptr) {
+void leveldb_put(leveldb_t* db, const leveldb_writeoptions_t* options,
+ const char* key, size_t keylen, const char* val, size_t vallen,
+ char** errptr) {
SaveError(errptr,
db->rep->Put(options->rep, Slice(key, keylen), Slice(val, vallen)));
}
-void leveldb_delete(
- leveldb_t* db,
- const leveldb_writeoptions_t* options,
- const char* key, size_t keylen,
- char** errptr) {
+void leveldb_delete(leveldb_t* db, const leveldb_writeoptions_t* options,
+ const char* key, size_t keylen, char** errptr) {
SaveError(errptr, db->rep->Delete(options->rep, Slice(key, keylen)));
}
-
-void leveldb_write(
- leveldb_t* db,
- const leveldb_writeoptions_t* options,
- leveldb_writebatch_t* batch,
- char** errptr) {
+void leveldb_write(leveldb_t* db, const leveldb_writeoptions_t* options,
+ leveldb_writebatch_t* batch, char** errptr) {
SaveError(errptr, db->rep->Write(options->rep, &batch->rep));
}
-char* leveldb_get(
- leveldb_t* db,
- const leveldb_readoptions_t* options,
- const char* key, size_t keylen,
- size_t* vallen,
- char** errptr) {
- char* result = NULL;
+char* leveldb_get(leveldb_t* db, const leveldb_readoptions_t* options,
+ const char* key, size_t keylen, size_t* vallen,
+ char** errptr) {
+ char* result = nullptr;
std::string tmp;
Status s = db->rep->Get(options->rep, Slice(key, keylen), &tmp);
if (s.ok()) {
@@ -216,45 +213,40 @@ char* leveldb_get(
}
leveldb_iterator_t* leveldb_create_iterator(
- leveldb_t* db,
- const leveldb_readoptions_t* options) {
+ leveldb_t* db, const leveldb_readoptions_t* options) {
leveldb_iterator_t* result = new leveldb_iterator_t;
result->rep = db->rep->NewIterator(options->rep);
return result;
}
-const leveldb_snapshot_t* leveldb_create_snapshot(
- leveldb_t* db) {
+const leveldb_snapshot_t* leveldb_create_snapshot(leveldb_t* db) {
leveldb_snapshot_t* result = new leveldb_snapshot_t;
result->rep = db->rep->GetSnapshot();
return result;
}
-void leveldb_release_snapshot(
- leveldb_t* db,
- const leveldb_snapshot_t* snapshot) {
+void leveldb_release_snapshot(leveldb_t* db,
+ const leveldb_snapshot_t* snapshot) {
db->rep->ReleaseSnapshot(snapshot->rep);
delete snapshot;
}
-char* leveldb_property_value(
- leveldb_t* db,
- const char* propname) {
+char* leveldb_property_value(leveldb_t* db, const char* propname) {
std::string tmp;
if (db->rep->GetProperty(Slice(propname), &tmp)) {
// We use strdup() since we expect human readable output.
return strdup(tmp.c_str());
} else {
- return NULL;
+ return nullptr;
}
}
-void leveldb_approximate_sizes(
- leveldb_t* db,
- int num_ranges,
- const char* const* range_start_key, const size_t* range_start_key_len,
- const char* const* range_limit_key, const size_t* range_limit_key_len,
- uint64_t* sizes) {
+void leveldb_approximate_sizes(leveldb_t* db, int num_ranges,
+ const char* const* range_start_key,
+ const size_t* range_start_key_len,
+ const char* const* range_limit_key,
+ const size_t* range_limit_key_len,
+ uint64_t* sizes) {
Range* ranges = new Range[num_ranges];
for (int i = 0; i < num_ranges; i++) {
ranges[i].start = Slice(range_start_key[i], range_start_key_len[i]);
@@ -264,28 +256,23 @@ void leveldb_approximate_sizes(
delete[] ranges;
}
-void leveldb_compact_range(
- leveldb_t* db,
- const char* start_key, size_t start_key_len,
- const char* limit_key, size_t limit_key_len) {
+void leveldb_compact_range(leveldb_t* db, const char* start_key,
+ size_t start_key_len, const char* limit_key,
+ size_t limit_key_len) {
Slice a, b;
db->rep->CompactRange(
- // Pass NULL Slice if corresponding "const char*" is NULL
- (start_key ? (a = Slice(start_key, start_key_len), &a) : NULL),
- (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : NULL));
+ // Pass null Slice if corresponding "const char*" is null
+ (start_key ? (a = Slice(start_key, start_key_len), &a) : nullptr),
+ (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : nullptr));
}
-void leveldb_destroy_db(
- const leveldb_options_t* options,
- const char* name,
- char** errptr) {
+void leveldb_destroy_db(const leveldb_options_t* options, const char* name,
+ char** errptr) {
SaveError(errptr, DestroyDB(name, options->rep));
}
-void leveldb_repair_db(
- const leveldb_options_t* options,
- const char* name,
- char** errptr) {
+void leveldb_repair_db(const leveldb_options_t* options, const char* name,
+ char** errptr) {
SaveError(errptr, RepairDB(name, options->rep));
}
@@ -310,13 +297,9 @@ void leveldb_iter_seek(leveldb_iterator_t* iter, const char* k, size_t klen) {
iter->rep->Seek(Slice(k, klen));
}
-void leveldb_iter_next(leveldb_iterator_t* iter) {
- iter->rep->Next();
-}
+void leveldb_iter_next(leveldb_iterator_t* iter) { iter->rep->Next(); }
-void leveldb_iter_prev(leveldb_iterator_t* iter) {
- iter->rep->Prev();
-}
+void leveldb_iter_prev(leveldb_iterator_t* iter) { iter->rep->Prev(); }
const char* leveldb_iter_key(const leveldb_iterator_t* iter, size_t* klen) {
Slice s = iter->rep->key();
@@ -338,41 +321,34 @@ leveldb_writebatch_t* leveldb_writebatch_create() {
return new leveldb_writebatch_t;
}
-void leveldb_writebatch_destroy(leveldb_writebatch_t* b) {
- delete b;
-}
+void leveldb_writebatch_destroy(leveldb_writebatch_t* b) { delete b; }
-void leveldb_writebatch_clear(leveldb_writebatch_t* b) {
- b->rep.Clear();
-}
+void leveldb_writebatch_clear(leveldb_writebatch_t* b) { b->rep.Clear(); }
-void leveldb_writebatch_put(
- leveldb_writebatch_t* b,
- const char* key, size_t klen,
- const char* val, size_t vlen) {
+void leveldb_writebatch_put(leveldb_writebatch_t* b, const char* key,
+ size_t klen, const char* val, size_t vlen) {
b->rep.Put(Slice(key, klen), Slice(val, vlen));
}
-void leveldb_writebatch_delete(
- leveldb_writebatch_t* b,
- const char* key, size_t klen) {
+void leveldb_writebatch_delete(leveldb_writebatch_t* b, const char* key,
+ size_t klen) {
b->rep.Delete(Slice(key, klen));
}
-void leveldb_writebatch_iterate(
- leveldb_writebatch_t* b,
- void* state,
- void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
- void (*deleted)(void*, const char* k, size_t klen)) {
+void leveldb_writebatch_iterate(const leveldb_writebatch_t* b, void* state,
+ void (*put)(void*, const char* k, size_t klen,
+ const char* v, size_t vlen),
+ void (*deleted)(void*, const char* k,
+ size_t klen)) {
class H : public WriteBatch::Handler {
public:
void* state_;
void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
void (*deleted_)(void*, const char* k, size_t klen);
- virtual void Put(const Slice& key, const Slice& value) {
+ void Put(const Slice& key, const Slice& value) override {
(*put_)(state_, key.data(), key.size(), value.data(), value.size());
}
- virtual void Delete(const Slice& key) {
+ void Delete(const Slice& key) override {
(*deleted_)(state_, key.data(), key.size());
}
};
@@ -383,47 +359,46 @@ void leveldb_writebatch_iterate(
b->rep.Iterate(&handler);
}
-leveldb_options_t* leveldb_options_create() {
- return new leveldb_options_t;
+void leveldb_writebatch_append(leveldb_writebatch_t* destination,
+ const leveldb_writebatch_t* source) {
+ destination->rep.Append(source->rep);
}
-void leveldb_options_destroy(leveldb_options_t* options) {
- delete options;
-}
+leveldb_options_t* leveldb_options_create() { return new leveldb_options_t; }
+
+void leveldb_options_destroy(leveldb_options_t* options) { delete options; }
-void leveldb_options_set_comparator(
- leveldb_options_t* opt,
- leveldb_comparator_t* cmp) {
+void leveldb_options_set_comparator(leveldb_options_t* opt,
+ leveldb_comparator_t* cmp) {
opt->rep.comparator = cmp;
}
-void leveldb_options_set_filter_policy(
- leveldb_options_t* opt,
- leveldb_filterpolicy_t* policy) {
+void leveldb_options_set_filter_policy(leveldb_options_t* opt,
+ leveldb_filterpolicy_t* policy) {
opt->rep.filter_policy = policy;
}
-void leveldb_options_set_create_if_missing(
- leveldb_options_t* opt, unsigned char v) {
+void leveldb_options_set_create_if_missing(leveldb_options_t* opt,
+ unsigned char v) {
opt->rep.create_if_missing = v;
}
-void leveldb_options_set_error_if_exists(
- leveldb_options_t* opt, unsigned char v) {
+void leveldb_options_set_error_if_exists(leveldb_options_t* opt,
+ unsigned char v) {
opt->rep.error_if_exists = v;
}
-void leveldb_options_set_paranoid_checks(
- leveldb_options_t* opt, unsigned char v) {
+void leveldb_options_set_paranoid_checks(leveldb_options_t* opt,
+ unsigned char v) {
opt->rep.paranoid_checks = v;
}
void leveldb_options_set_env(leveldb_options_t* opt, leveldb_env_t* env) {
- opt->rep.env = (env ? env->rep : NULL);
+ opt->rep.env = (env ? env->rep : nullptr);
}
void leveldb_options_set_info_log(leveldb_options_t* opt, leveldb_logger_t* l) {
- opt->rep.info_log = (l ? l->rep : NULL);
+ opt->rep.info_log = (l ? l->rep : nullptr);
}
void leveldb_options_set_write_buffer_size(leveldb_options_t* opt, size_t s) {
@@ -446,17 +421,18 @@ void leveldb_options_set_block_restart_interval(leveldb_options_t* opt, int n) {
opt->rep.block_restart_interval = n;
}
+void leveldb_options_set_max_file_size(leveldb_options_t* opt, size_t s) {
+ opt->rep.max_file_size = s;
+}
+
void leveldb_options_set_compression(leveldb_options_t* opt, int t) {
opt->rep.compression = static_cast<CompressionType>(t);
}
leveldb_comparator_t* leveldb_comparator_create(
- void* state,
- void (*destructor)(void*),
- int (*compare)(
- void*,
- const char* a, size_t alen,
- const char* b, size_t blen),
+ void* state, void (*destructor)(void*),
+ int (*compare)(void*, const char* a, size_t alen, const char* b,
+ size_t blen),
const char* (*name)(void*)) {
leveldb_comparator_t* result = new leveldb_comparator_t;
result->state_ = state;
@@ -466,22 +442,15 @@ leveldb_comparator_t* leveldb_comparator_create(
return result;
}
-void leveldb_comparator_destroy(leveldb_comparator_t* cmp) {
- delete cmp;
-}
+void leveldb_comparator_destroy(leveldb_comparator_t* cmp) { delete cmp; }
leveldb_filterpolicy_t* leveldb_filterpolicy_create(
- void* state,
- void (*destructor)(void*),
- char* (*create_filter)(
- void*,
- const char* const* key_array, const size_t* key_length_array,
- int num_keys,
- size_t* filter_length),
- unsigned char (*key_may_match)(
- void*,
- const char* key, size_t length,
- const char* filter, size_t filter_length),
+ void* state, void (*destructor)(void*),
+ char* (*create_filter)(void*, const char* const* key_array,
+ const size_t* key_length_array, int num_keys,
+ size_t* filter_length),
+ unsigned char (*key_may_match)(void*, const char* key, size_t length,
+ const char* filter, size_t filter_length),
const char* (*name)(void*)) {
leveldb_filterpolicy_t* result = new leveldb_filterpolicy_t;
result->state_ = state;
@@ -501,7 +470,8 @@ leveldb_filterpolicy_t* leveldb_filterpolicy_create_bloom(int bits_per_key) {
// they delegate to a NewBloomFilterPolicy() instead of user
// supplied C functions.
struct Wrapper : public leveldb_filterpolicy_t {
- const FilterPolicy* rep_;
+ static void DoNothing(void*) {}
+
~Wrapper() { delete rep_; }
const char* Name() const { return rep_->Name(); }
void CreateFilter(const Slice* keys, int n, std::string* dst) const {
@@ -510,11 +480,12 @@ leveldb_filterpolicy_t* leveldb_filterpolicy_create_bloom(int bits_per_key) {
bool KeyMayMatch(const Slice& key, const Slice& filter) const {
return rep_->KeyMayMatch(key, filter);
}
- static void DoNothing(void*) { }
+
+ const FilterPolicy* rep_;
};
Wrapper* wrapper = new Wrapper;
wrapper->rep_ = NewBloomFilterPolicy(bits_per_key);
- wrapper->state_ = NULL;
+ wrapper->state_ = nullptr;
wrapper->destructor_ = &Wrapper::DoNothing;
return wrapper;
}
@@ -523,37 +494,31 @@ leveldb_readoptions_t* leveldb_readoptions_create() {
return new leveldb_readoptions_t;
}
-void leveldb_readoptions_destroy(leveldb_readoptions_t* opt) {
- delete opt;
-}
+void leveldb_readoptions_destroy(leveldb_readoptions_t* opt) { delete opt; }
-void leveldb_readoptions_set_verify_checksums(
- leveldb_readoptions_t* opt,
- unsigned char v) {
+void leveldb_readoptions_set_verify_checksums(leveldb_readoptions_t* opt,
+ unsigned char v) {
opt->rep.verify_checksums = v;
}
-void leveldb_readoptions_set_fill_cache(
- leveldb_readoptions_t* opt, unsigned char v) {
+void leveldb_readoptions_set_fill_cache(leveldb_readoptions_t* opt,
+ unsigned char v) {
opt->rep.fill_cache = v;
}
-void leveldb_readoptions_set_snapshot(
- leveldb_readoptions_t* opt,
- const leveldb_snapshot_t* snap) {
- opt->rep.snapshot = (snap ? snap->rep : NULL);
+void leveldb_readoptions_set_snapshot(leveldb_readoptions_t* opt,
+ const leveldb_snapshot_t* snap) {
+ opt->rep.snapshot = (snap ? snap->rep : nullptr);
}
leveldb_writeoptions_t* leveldb_writeoptions_create() {
return new leveldb_writeoptions_t;
}
-void leveldb_writeoptions_destroy(leveldb_writeoptions_t* opt) {
- delete opt;
-}
+void leveldb_writeoptions_destroy(leveldb_writeoptions_t* opt) { delete opt; }
-void leveldb_writeoptions_set_sync(
- leveldb_writeoptions_t* opt, unsigned char v) {
+void leveldb_writeoptions_set_sync(leveldb_writeoptions_t* opt,
+ unsigned char v) {
opt->rep.sync = v;
}
@@ -580,16 +545,22 @@ void leveldb_env_destroy(leveldb_env_t* env) {
delete env;
}
-void leveldb_free(void* ptr) {
- free(ptr);
-}
+char* leveldb_env_get_test_directory(leveldb_env_t* env) {
+ std::string result;
+ if (!env->rep->GetTestDirectory(&result).ok()) {
+ return nullptr;
+ }
-int leveldb_major_version() {
- return kMajorVersion;
+ char* buffer = static_cast<char*>(malloc(result.size() + 1));
+ memcpy(buffer, result.data(), result.size());
+ buffer[result.size()] = '\0';
+ return buffer;
}
-int leveldb_minor_version() {
- return kMinorVersion;
-}
+void leveldb_free(void* ptr) { free(ptr); }
+
+int leveldb_major_version() { return kMajorVersion; }
+
+int leveldb_minor_version() { return kMinorVersion; }
} // end extern "C"
diff --git a/db/c_test.c b/db/c_test.c
index 7cd5ee0..ae14b99 100644
--- a/db/c_test.c
+++ b/db/c_test.c
@@ -8,24 +8,14 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
-#include <sys/types.h>
-#include <unistd.h>
const char* phase = "";
-static char dbname[200];
static void StartPhase(const char* name) {
fprintf(stderr, "=== Test %s\n", name);
phase = name;
}
-static const char* GetTempDir(void) {
- const char* ret = getenv("TEST_TMPDIR");
- if (ret == NULL || ret[0] == '\0')
- ret = "/tmp";
- return ret;
-}
-
#define CheckNoError(err) \
if ((err) != NULL) { \
fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, (err)); \
@@ -162,21 +152,19 @@ int main(int argc, char** argv) {
leveldb_options_t* options;
leveldb_readoptions_t* roptions;
leveldb_writeoptions_t* woptions;
+ char* dbname;
char* err = NULL;
int run = -1;
CheckCondition(leveldb_major_version() >= 1);
CheckCondition(leveldb_minor_version() >= 1);
- snprintf(dbname, sizeof(dbname),
- "%s/leveldb_c_test-%d",
- GetTempDir(),
- ((int) geteuid()));
-
StartPhase("create_objects");
cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
env = leveldb_create_default_env();
cache = leveldb_cache_create_lru(100000);
+ dbname = leveldb_env_get_test_directory(env);
+ CheckCondition(dbname != NULL);
options = leveldb_options_create();
leveldb_options_set_comparator(options, cmp);
@@ -189,6 +177,7 @@ int main(int argc, char** argv) {
leveldb_options_set_max_open_files(options, 10);
leveldb_options_set_block_size(options, 1024);
leveldb_options_set_block_restart_interval(options, 8);
+ leveldb_options_set_max_file_size(options, 3 << 20);
leveldb_options_set_compression(options, leveldb_no_compression);
roptions = leveldb_readoptions_create();
@@ -239,12 +228,18 @@ int main(int argc, char** argv) {
leveldb_writebatch_clear(wb);
leveldb_writebatch_put(wb, "bar", 3, "b", 1);
leveldb_writebatch_put(wb, "box", 3, "c", 1);
- leveldb_writebatch_delete(wb, "bar", 3);
+
+ leveldb_writebatch_t* wb2 = leveldb_writebatch_create();
+ leveldb_writebatch_delete(wb2, "bar", 3);
+ leveldb_writebatch_append(wb, wb2);
+ leveldb_writebatch_destroy(wb2);
+
leveldb_write(db, woptions, wb, &err);
CheckNoError(err);
CheckGet(db, roptions, "foo", "hello");
CheckGet(db, roptions, "bar", NULL);
CheckGet(db, roptions, "box", "c");
+
int pos = 0;
leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel);
CheckCondition(pos == 3);
@@ -381,6 +376,7 @@ int main(int argc, char** argv) {
leveldb_options_destroy(options);
leveldb_readoptions_destroy(roptions);
leveldb_writeoptions_destroy(woptions);
+ leveldb_free(dbname);
leveldb_cache_destroy(cache);
leveldb_comparator_destroy(cmp);
leveldb_env_destroy(env);
diff --git a/db/corruption_test.cc b/db/corruption_test.cc
index 96afc68..42f5237 100644
--- a/db/corruption_test.cc
+++ b/db/corruption_test.cc
@@ -2,20 +2,16 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#include "leveldb/db.h"
-
-#include <errno.h>
-#include <fcntl.h>
-#include <sys/stat.h>
#include <sys/types.h>
-#include "leveldb/cache.h"
-#include "leveldb/env.h"
-#include "leveldb/table.h"
-#include "leveldb/write_batch.h"
+
#include "db/db_impl.h"
#include "db/filename.h"
#include "db/log_format.h"
#include "db/version_set.h"
+#include "leveldb/cache.h"
+#include "leveldb/db.h"
+#include "leveldb/table.h"
+#include "leveldb/write_batch.h"
#include "util/logging.h"
#include "util/testharness.h"
#include "util/testutil.h"
@@ -26,44 +22,35 @@ static const int kValueSize = 1000;
class CorruptionTest {
public:
- test::ErrorEnv env_;
- std::string dbname_;
- Cache* tiny_cache_;
- Options options_;
- DB* db_;
-
- CorruptionTest() {
- tiny_cache_ = NewLRUCache(100);
+ CorruptionTest()
+ : db_(nullptr),
+ dbname_("/memenv/corruption_test"),
+ tiny_cache_(NewLRUCache(100)) {
options_.env = &env_;
options_.block_cache = tiny_cache_;
- dbname_ = test::TmpDir() + "/db_test";
DestroyDB(dbname_, options_);
- db_ = NULL;
options_.create_if_missing = true;
Reopen();
options_.create_if_missing = false;
}
~CorruptionTest() {
- delete db_;
- DestroyDB(dbname_, Options());
- delete tiny_cache_;
+ delete db_;
+ delete tiny_cache_;
}
Status TryReopen() {
delete db_;
- db_ = NULL;
+ db_ = nullptr;
return DB::Open(options_, dbname_, &db_);
}
- void Reopen() {
- ASSERT_OK(TryReopen());
- }
+ void Reopen() { ASSERT_OK(TryReopen()); }
void RepairDB() {
delete db_;
- db_ = NULL;
+ db_ = nullptr;
ASSERT_OK(::leveldb::RepairDB(dbname_, options_));
}
@@ -71,7 +58,7 @@ class CorruptionTest {
std::string key_space, value_space;
WriteBatch batch;
for (int i = 0; i < n; i++) {
- //if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n);
+ // if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n);
Slice key = Key(i, &key_space);
batch.Clear();
batch.Put(key, Value(i, &value_space));
@@ -100,8 +87,7 @@ class CorruptionTest {
// Ignore boundary keys.
continue;
}
- if (!ConsumeDecimalNumber(&in, &key) ||
- !in.empty() ||
+ if (!ConsumeDecimalNumber(&in, &key) || !in.empty() ||
key < next_expected) {
bad_keys++;
continue;
@@ -126,14 +112,13 @@ class CorruptionTest {
void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) {
// Pick file to corrupt
std::vector<std::string> filenames;
- ASSERT_OK(env_.GetChildren(dbname_, &filenames));
+ ASSERT_OK(env_.target()->GetChildren(dbname_, &filenames));
uint64_t number;
FileType type;
std::string fname;
int picked_number = -1;
for (size_t i = 0; i < filenames.size(); i++) {
- if (ParseFileName(filenames[i], &number, &type) &&
- type == filetype &&
+ if (ParseFileName(filenames[i], &number, &type) && type == filetype &&
int(number) > picked_number) { // Pick latest file
fname = dbname_ + "/" + filenames[i];
picked_number = number;
@@ -141,35 +126,32 @@ class CorruptionTest {
}
ASSERT_TRUE(!fname.empty()) << filetype;
- struct stat sbuf;
- if (stat(fname.c_str(), &sbuf) != 0) {
- const char* msg = strerror(errno);
- ASSERT_TRUE(false) << fname << ": " << msg;
- }
+ uint64_t file_size;
+ ASSERT_OK(env_.target()->GetFileSize(fname, &file_size));
if (offset < 0) {
// Relative to end of file; make it absolute
- if (-offset > sbuf.st_size) {
+ if (-offset > file_size) {
offset = 0;
} else {
- offset = sbuf.st_size + offset;
+ offset = file_size + offset;
}
}
- if (offset > sbuf.st_size) {
- offset = sbuf.st_size;
+ if (offset > file_size) {
+ offset = file_size;
}
- if (offset + bytes_to_corrupt > sbuf.st_size) {
- bytes_to_corrupt = sbuf.st_size - offset;
+ if (offset + bytes_to_corrupt > file_size) {
+ bytes_to_corrupt = file_size - offset;
}
// Do it
std::string contents;
- Status s = ReadFileToString(Env::Default(), fname, &contents);
+ Status s = ReadFileToString(env_.target(), fname, &contents);
ASSERT_TRUE(s.ok()) << s.ToString();
for (int i = 0; i < bytes_to_corrupt; i++) {
contents[i + offset] ^= 0x80;
}
- s = WriteStringToFile(Env::Default(), contents, fname);
+ s = WriteStringToFile(env_.target(), contents, fname);
ASSERT_TRUE(s.ok()) << s.ToString();
}
@@ -197,12 +179,20 @@ class CorruptionTest {
Random r(k);
return test::RandomString(&r, kValueSize, storage);
}
+
+ test::ErrorEnv env_;
+ Options options_;
+ DB* db_;
+
+ private:
+ std::string dbname_;
+ Cache* tiny_cache_;
};
TEST(CorruptionTest, Recovery) {
Build(100);
Check(100, 100);
- Corrupt(kLogFile, 19, 1); // WriteBatch tag for first record
+ Corrupt(kLogFile, 19, 1); // WriteBatch tag for first record
Corrupt(kLogFile, log::kBlockSize + 1000, 1); // Somewhere in second block
Reopen();
@@ -237,8 +227,8 @@ TEST(CorruptionTest, TableFile) {
Build(100);
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
dbi->TEST_CompactMemTable();
- dbi->TEST_CompactRange(0, NULL, NULL);
- dbi->TEST_CompactRange(1, NULL, NULL);
+ dbi->TEST_CompactRange(0, nullptr, nullptr);
+ dbi->TEST_CompactRange(1, nullptr, nullptr);
Corrupt(kTableFile, 100, 1);
Check(90, 99);
@@ -251,8 +241,8 @@ TEST(CorruptionTest, TableFileRepair) {
Build(100);
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
dbi->TEST_CompactMemTable();
- dbi->TEST_CompactRange(0, NULL, NULL);
- dbi->TEST_CompactRange(1, NULL, NULL);
+ dbi->TEST_CompactRange(0, nullptr, nullptr);
+ dbi->TEST_CompactRange(1, nullptr, nullptr);
Corrupt(kTableFile, 100, 1);
RepairDB();
@@ -302,7 +292,7 @@ TEST(CorruptionTest, CorruptedDescriptor) {
ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello"));
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
dbi->TEST_CompactMemTable();
- dbi->TEST_CompactRange(0, NULL, NULL);
+ dbi->TEST_CompactRange(0, nullptr, nullptr);
Corrupt(kDescriptorFile, 0, 1000);
Status s = TryReopen();
@@ -343,7 +333,7 @@ TEST(CorruptionTest, CompactionInputErrorParanoid) {
Corrupt(kTableFile, 100, 1);
env_.SleepForMicroseconds(100000);
}
- dbi->CompactRange(NULL, NULL);
+ dbi->CompactRange(nullptr, nullptr);
// Write must fail because of corrupted table
std::string tmp1, tmp2;
@@ -369,6 +359,4 @@ TEST(CorruptionTest, UnrelatedKeys) {
} // namespace leveldb
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
+int main(int argc, char** argv) { return leveldb::test::RunAllTests(); }
diff --git a/db/db_bench.cc b/db/db_bench.cc
deleted file mode 100644
index e5975bd..0000000
--- a/db/db_bench.cc
+++ /dev/null
@@ -1,991 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <sys/types.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include "db/db_impl.h"
-#include "db/version_set.h"
-#include "leveldb/cache.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-#include "leveldb/write_batch.h"
-#include "port/port.h"
-#include "util/crc32c.h"
-#include "util/histogram.h"
-#include "util/mutexlock.h"
-#include "util/random.h"
-#include "util/testutil.h"
-
-// Comma-separated list of operations to run in the specified order
-// Actual benchmarks:
-// fillseq -- write N values in sequential key order in async mode
-// fillrandom -- write N values in random key order in async mode
-// overwrite -- overwrite N values in random key order in async mode
-// fillsync -- write N/100 values in random key order in sync mode
-// fill100K -- write N/1000 100K values in random order in async mode
-// deleteseq -- delete N keys in sequential order
-// deleterandom -- delete N keys in random order
-// readseq -- read N times sequentially
-// readreverse -- read N times in reverse order
-// readrandom -- read N times in random order
-// readmissing -- read N missing keys in random order
-// readhot -- read N times in random order from 1% section of DB
-// seekrandom -- N random seeks
-// open -- cost of opening a DB
-// crc32c -- repeated crc32c of 4K of data
-// acquireload -- load N*1000 times
-// Meta operations:
-// compact -- Compact the entire DB
-// stats -- Print DB stats
-// sstables -- Print sstable info
-// heapprofile -- Dump a heap profile (if supported by this port)
-static const char* FLAGS_benchmarks =
- "fillseq,"
- "fillsync,"
- "fillrandom,"
- "overwrite,"
- "readrandom,"
- "readrandom," // Extra run to allow previous compactions to quiesce
- "readseq,"
- "readreverse,"
- "compact,"
- "readrandom,"
- "readseq,"
- "readreverse,"
- "fill100K,"
- "crc32c,"
- "snappycomp,"
- "snappyuncomp,"
- "acquireload,"
- ;
-
-// Number of key/values to place in database
-static int FLAGS_num = 1000000;
-
-// Number of read operations to do. If negative, do FLAGS_num reads.
-static int FLAGS_reads = -1;
-
-// Number of concurrent threads to run.
-static int FLAGS_threads = 1;
-
-// Size of each value
-static int FLAGS_value_size = 100;
-
-// Arrange to generate values that shrink to this fraction of
-// their original size after compression
-static double FLAGS_compression_ratio = 0.5;
-
-// Print histogram of operation timings
-static bool FLAGS_histogram = false;
-
-// Number of bytes to buffer in memtable before compacting
-// (initialized to default value by "main")
-static int FLAGS_write_buffer_size = 0;
-
-// Number of bytes to use as a cache of uncompressed data.
-// Negative means use default settings.
-static int FLAGS_cache_size = -1;
-
-// Maximum number of files to keep open at the same time (use default if == 0)
-static int FLAGS_open_files = 0;
-
-// Bloom filter bits per key.
-// Negative means use default settings.
-static int FLAGS_bloom_bits = -1;
-
-// If true, do not destroy the existing database. If you set this
-// flag and also specify a benchmark that wants a fresh database, that
-// benchmark will fail.
-static bool FLAGS_use_existing_db = false;
-
-// Use the db with the following name.
-static const char* FLAGS_db = NULL;
-
-namespace leveldb {
-
-namespace {
-
-// Helper for quickly generating random data.
-class RandomGenerator {
- private:
- std::string data_;
- int pos_;
-
- public:
- RandomGenerator() {
- // We use a limited amount of data over and over again and ensure
- // that it is larger than the compression window (32KB), and also
- // large enough to serve all typical value sizes we want to write.
- Random rnd(301);
- std::string piece;
- while (data_.size() < 1048576) {
- // Add a short fragment that is as compressible as specified
- // by FLAGS_compression_ratio.
- test::CompressibleString(&rnd, FLAGS_compression_ratio, 100, &piece);
- data_.append(piece);
- }
- pos_ = 0;
- }
-
- Slice Generate(size_t len) {
- if (pos_ + len > data_.size()) {
- pos_ = 0;
- assert(len < data_.size());
- }
- pos_ += len;
- return Slice(data_.data() + pos_ - len, len);
- }
-};
-
-static Slice TrimSpace(Slice s) {
- size_t start = 0;
- while (start < s.size() && isspace(s[start])) {
- start++;
- }
- size_t limit = s.size();
- while (limit > start && isspace(s[limit-1])) {
- limit--;
- }
- return Slice(s.data() + start, limit - start);
-}
-
-static void AppendWithSpace(std::string* str, Slice msg) {
- if (msg.empty()) return;
- if (!str->empty()) {
- str->push_back(' ');
- }
- str->append(msg.data(), msg.size());
-}
-
-class Stats {
- private:
- double start_;
- double finish_;
- double seconds_;
- int done_;
- int next_report_;
- int64_t bytes_;
- double last_op_finish_;
- Histogram hist_;
- std::string message_;
-
- public:
- Stats() { Start(); }
-
- void Start() {
- next_report_ = 100;
- last_op_finish_ = start_;
- hist_.Clear();
- done_ = 0;
- bytes_ = 0;
- seconds_ = 0;
- start_ = Env::Default()->NowMicros();
- finish_ = start_;
- message_.clear();
- }
-
- void Merge(const Stats& other) {
- hist_.Merge(other.hist_);
- done_ += other.done_;
- bytes_ += other.bytes_;
- seconds_ += other.seconds_;
- if (other.start_ < start_) start_ = other.start_;
- if (other.finish_ > finish_) finish_ = other.finish_;
-
- // Just keep the messages from one thread
- if (message_.empty()) message_ = other.message_;
- }
-
- void Stop() {
- finish_ = Env::Default()->NowMicros();
- seconds_ = (finish_ - start_) * 1e-6;
- }
-
- void AddMessage(Slice msg) {
- AppendWithSpace(&message_, msg);
- }
-
- void FinishedSingleOp() {
- if (FLAGS_histogram) {
- double now = Env::Default()->NowMicros();
- double micros = now - last_op_finish_;
- hist_.Add(micros);
- if (micros > 20000) {
- fprintf(stderr, "long op: %.1f micros%30s\r", micros, "");
- fflush(stderr);
- }
- last_op_finish_ = now;
- }
-
- done_++;
- if (done_ >= next_report_) {
- if (next_report_ < 1000) next_report_ += 100;
- else if (next_report_ < 5000) next_report_ += 500;
- else if (next_report_ < 10000) next_report_ += 1000;
- else if (next_report_ < 50000) next_report_ += 5000;
- else if (next_report_ < 100000) next_report_ += 10000;
- else if (next_report_ < 500000) next_report_ += 50000;
- else next_report_ += 100000;
- fprintf(stderr, "... finished %d ops%30s\r", done_, "");
- fflush(stderr);
- }
- }
-
- void AddBytes(int64_t n) {
- bytes_ += n;
- }
-
- void Report(const Slice& name) {
- // Pretend at least one op was done in case we are running a benchmark
- // that does not call FinishedSingleOp().
- if (done_ < 1) done_ = 1;
-
- std::string extra;
- if (bytes_ > 0) {
- // Rate is computed on actual elapsed time, not the sum of per-thread
- // elapsed times.
- double elapsed = (finish_ - start_) * 1e-6;
- char rate[100];
- snprintf(rate, sizeof(rate), "%6.1f MB/s",
- (bytes_ / 1048576.0) / elapsed);
- extra = rate;
- }
- AppendWithSpace(&extra, message_);
-
- fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n",
- name.ToString().c_str(),
- seconds_ * 1e6 / done_,
- (extra.empty() ? "" : " "),
- extra.c_str());
- if (FLAGS_histogram) {
- fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str());
- }
- fflush(stdout);
- }
-};
-
-// State shared by all concurrent executions of the same benchmark.
-struct SharedState {
- port::Mutex mu;
- port::CondVar cv;
- int total;
-
- // Each thread goes through the following states:
- // (1) initializing
- // (2) waiting for others to be initialized
- // (3) running
- // (4) done
-
- int num_initialized;
- int num_done;
- bool start;
-
- SharedState() : cv(&mu) { }
-};
-
-// Per-thread state for concurrent executions of the same benchmark.
-struct ThreadState {
- int tid; // 0..n-1 when running in n threads
- Random rand; // Has different seeds for different threads
- Stats stats;
- SharedState* shared;
-
- ThreadState(int index)
- : tid(index),
- rand(1000 + index) {
- }
-};
-
-} // namespace
-
-class Benchmark {
- private:
- Cache* cache_;
- const FilterPolicy* filter_policy_;
- DB* db_;
- int num_;
- int value_size_;
- int entries_per_batch_;
- WriteOptions write_options_;
- int reads_;
- int heap_counter_;
-
- void PrintHeader() {
- const int kKeySize = 16;
- PrintEnvironment();
- fprintf(stdout, "Keys: %d bytes each\n", kKeySize);
- fprintf(stdout, "Values: %d bytes each (%d bytes after compression)\n",
- FLAGS_value_size,
- static_cast<int>(FLAGS_value_size * FLAGS_compression_ratio + 0.5));
- fprintf(stdout, "Entries: %d\n", num_);
- fprintf(stdout, "RawSize: %.1f MB (estimated)\n",
- ((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_)
- / 1048576.0));
- fprintf(stdout, "FileSize: %.1f MB (estimated)\n",
- (((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_)
- / 1048576.0));
- PrintWarnings();
- fprintf(stdout, "------------------------------------------------\n");
- }
-
- void PrintWarnings() {
-#if defined(__GNUC__) && !defined(__OPTIMIZE__)
- fprintf(stdout,
- "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n"
- );
-#endif
-#ifndef NDEBUG
- fprintf(stdout,
- "WARNING: Assertions are enabled; benchmarks unnecessarily slow\n");
-#endif
-
- // See if snappy is working by attempting to compress a compressible string
- const char text[] = "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy";
- std::string compressed;
- if (!port::Snappy_Compress(text, sizeof(text), &compressed)) {
- fprintf(stdout, "WARNING: Snappy compression is not enabled\n");
- } else if (compressed.size() >= sizeof(text)) {
- fprintf(stdout, "WARNING: Snappy compression is not effective\n");
- }
- }
-
- void PrintEnvironment() {
- fprintf(stderr, "LevelDB: version %d.%d\n",
- kMajorVersion, kMinorVersion);
-
-#if defined(__linux)
- time_t now = time(NULL);
- fprintf(stderr, "Date: %s", ctime(&now)); // ctime() adds newline
-
- FILE* cpuinfo = fopen("/proc/cpuinfo", "r");
- if (cpuinfo != NULL) {
- char line[1000];
- int num_cpus = 0;
- std::string cpu_type;
- std::string cache_size;
- while (fgets(line, sizeof(line), cpuinfo) != NULL) {
- const char* sep = strchr(line, ':');
- if (sep == NULL) {
- continue;
- }
- Slice key = TrimSpace(Slice(line, sep - 1 - line));
- Slice val = TrimSpace(Slice(sep + 1));
- if (key == "model name") {
- ++num_cpus;
- cpu_type = val.ToString();
- } else if (key == "cache size") {
- cache_size = val.ToString();
- }
- }
- fclose(cpuinfo);
- fprintf(stderr, "CPU: %d * %s\n", num_cpus, cpu_type.c_str());
- fprintf(stderr, "CPUCache: %s\n", cache_size.c_str());
- }
-#endif
- }
-
- public:
- Benchmark()
- : cache_(FLAGS_cache_size >= 0 ? NewLRUCache(FLAGS_cache_size) : NULL),
- filter_policy_(FLAGS_bloom_bits >= 0
- ? NewBloomFilterPolicy(FLAGS_bloom_bits)
- : NULL),
- db_(NULL),
- num_(FLAGS_num),
- value_size_(FLAGS_value_size),
- entries_per_batch_(1),
- reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
- heap_counter_(0) {
- std::vector<std::string> files;
- Env::Default()->GetChildren(FLAGS_db, &files);
- for (size_t i = 0; i < files.size(); i++) {
- if (Slice(files[i]).starts_with("heap-")) {
- Env::Default()->DeleteFile(std::string(FLAGS_db) + "/" + files[i]);
- }
- }
- if (!FLAGS_use_existing_db) {
- DestroyDB(FLAGS_db, Options());
- }
- }
-
- ~Benchmark() {
- delete db_;
- delete cache_;
- delete filter_policy_;
- }
-
- void Run() {
- PrintHeader();
- Open();
-
- const char* benchmarks = FLAGS_benchmarks;
- while (benchmarks != NULL) {
- const char* sep = strchr(benchmarks, ',');
- Slice name;
- if (sep == NULL) {
- name = benchmarks;
- benchmarks = NULL;
- } else {
- name = Slice(benchmarks, sep - benchmarks);
- benchmarks = sep + 1;
- }
-
- // Reset parameters that may be overridden below
- num_ = FLAGS_num;
- reads_ = (FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads);
- value_size_ = FLAGS_value_size;
- entries_per_batch_ = 1;
- write_options_ = WriteOptions();
-
- void (Benchmark::*method)(ThreadState*) = NULL;
- bool fresh_db = false;
- int num_threads = FLAGS_threads;
-
- if (name == Slice("open")) {
- method = &Benchmark::OpenBench;
- num_ /= 10000;
- if (num_ < 1) num_ = 1;
- } else if (name == Slice("fillseq")) {
- fresh_db = true;
- method = &Benchmark::WriteSeq;
- } else if (name == Slice("fillbatch")) {
- fresh_db = true;
- entries_per_batch_ = 1000;
- method = &Benchmark::WriteSeq;
- } else if (name == Slice("fillrandom")) {
- fresh_db = true;
- method = &Benchmark::WriteRandom;
- } else if (name == Slice("overwrite")) {
- fresh_db = false;
- method = &Benchmark::WriteRandom;
- } else if (name == Slice("fillsync")) {
- fresh_db = true;
- num_ /= 1000;
- write_options_.sync = true;
- method = &Benchmark::WriteRandom;
- } else if (name == Slice("fill100K")) {
- fresh_db = true;
- num_ /= 1000;
- value_size_ = 100 * 1000;
- method = &Benchmark::WriteRandom;
- } else if (name == Slice("readseq")) {
- method = &Benchmark::ReadSequential;
- } else if (name == Slice("readreverse")) {
- method = &Benchmark::ReadReverse;
- } else if (name == Slice("readrandom")) {
- method = &Benchmark::ReadRandom;
- } else if (name == Slice("readmissing")) {
- method = &Benchmark::ReadMissing;
- } else if (name == Slice("seekrandom")) {
- method = &Benchmark::SeekRandom;
- } else if (name == Slice("readhot")) {
- method = &Benchmark::ReadHot;
- } else if (name == Slice("readrandomsmall")) {
- reads_ /= 1000;
- method = &Benchmark::ReadRandom;
- } else if (name == Slice("deleteseq")) {
- method = &Benchmark::DeleteSeq;
- } else if (name == Slice("deleterandom")) {
- method = &Benchmark::DeleteRandom;
- } else if (name == Slice("readwhilewriting")) {
- num_threads++; // Add extra thread for writing
- method = &Benchmark::ReadWhileWriting;
- } else if (name == Slice("compact")) {
- method = &Benchmark::Compact;
- } else if (name == Slice("crc32c")) {
- method = &Benchmark::Crc32c;
- } else if (name == Slice("acquireload")) {
- method = &Benchmark::AcquireLoad;
- } else if (name == Slice("snappycomp")) {
- method = &Benchmark::SnappyCompress;
- } else if (name == Slice("snappyuncomp")) {
- method = &Benchmark::SnappyUncompress;
- } else if (name == Slice("heapprofile")) {
- HeapProfile();
- } else if (name == Slice("stats")) {
- PrintStats("leveldb.stats");
- } else if (name == Slice("sstables")) {
- PrintStats("leveldb.sstables");
- } else {
- if (name != Slice()) { // No error message for empty name
- fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str());
- }
- }
-
- if (fresh_db) {
- if (FLAGS_use_existing_db) {
- fprintf(stdout, "%-12s : skipped (--use_existing_db is true)\n",
- name.ToString().c_str());
- method = NULL;
- } else {
- delete db_;
- db_ = NULL;
- DestroyDB(FLAGS_db, Options());
- Open();
- }
- }
-
- if (method != NULL) {
- RunBenchmark(num_threads, name, method);
- }
- }
- }
-
- private:
- struct ThreadArg {
- Benchmark* bm;
- SharedState* shared;
- ThreadState* thread;
- void (Benchmark::*method)(ThreadState*);
- };
-
- static void ThreadBody(void* v) {
- ThreadArg* arg = reinterpret_cast<ThreadArg*>(v);
- SharedState* shared = arg->shared;
- ThreadState* thread = arg->thread;
- {
- MutexLock l(&shared->mu);
- shared->num_initialized++;
- if (shared->num_initialized >= shared->total) {
- shared->cv.SignalAll();
- }
- while (!shared->start) {
- shared->cv.Wait();
- }
- }
-
- thread->stats.Start();
- (arg->bm->*(arg->method))(thread);
- thread->stats.Stop();
-
- {
- MutexLock l(&shared->mu);
- shared->num_done++;
- if (shared->num_done >= shared->total) {
- shared->cv.SignalAll();
- }
- }
- }
-
- void RunBenchmark(int n, Slice name,
- void (Benchmark::*method)(ThreadState*)) {
- SharedState shared;
- shared.total = n;
- shared.num_initialized = 0;
- shared.num_done = 0;
- shared.start = false;
-
- ThreadArg* arg = new ThreadArg[n];
- for (int i = 0; i < n; i++) {
- arg[i].bm = this;
- arg[i].method = method;
- arg[i].shared = &shared;
- arg[i].thread = new ThreadState(i);
- arg[i].thread->shared = &shared;
- Env::Default()->StartThread(ThreadBody, &arg[i]);
- }
-
- shared.mu.Lock();
- while (shared.num_initialized < n) {
- shared.cv.Wait();
- }
-
- shared.start = true;
- shared.cv.SignalAll();
- while (shared.num_done < n) {
- shared.cv.Wait();
- }
- shared.mu.Unlock();
-
- for (int i = 1; i < n; i++) {
- arg[0].thread->stats.Merge(arg[i].thread->stats);
- }
- arg[0].thread->stats.Report(name);
-
- for (int i = 0; i < n; i++) {
- delete arg[i].thread;
- }
- delete[] arg;
- }
-
- void Crc32c(ThreadState* thread) {
- // Checksum about 500MB of data total
- const int size = 4096;
- const char* label = "(4K per op)";
- std::string data(size, 'x');
- int64_t bytes = 0;
- uint32_t crc = 0;
- while (bytes < 500 * 1048576) {
- crc = crc32c::Value(data.data(), size);
- thread->stats.FinishedSingleOp();
- bytes += size;
- }
- // Print so result is not dead
- fprintf(stderr, "... crc=0x%x\r", static_cast<unsigned int>(crc));
-
- thread->stats.AddBytes(bytes);
- thread->stats.AddMessage(label);
- }
-
- void AcquireLoad(ThreadState* thread) {
- int dummy;
- port::AtomicPointer ap(&dummy);
- int count = 0;
- void *ptr = NULL;
- thread->stats.AddMessage("(each op is 1000 loads)");
- while (count < 100000) {
- for (int i = 0; i < 1000; i++) {
- ptr = ap.Acquire_Load();
- }
- count++;
- thread->stats.FinishedSingleOp();
- }
- if (ptr == NULL) exit(1); // Disable unused variable warning.
- }
-
- void SnappyCompress(ThreadState* thread) {
- RandomGenerator gen;
- Slice input = gen.Generate(Options().block_size);
- int64_t bytes = 0;
- int64_t produced = 0;
- bool ok = true;
- std::string compressed;
- while (ok && bytes < 1024 * 1048576) { // Compress 1G
- ok = port::Snappy_Compress(input.data(), input.size(), &compressed);
- produced += compressed.size();
- bytes += input.size();
- thread->stats.FinishedSingleOp();
- }
-
- if (!ok) {
- thread->stats.AddMessage("(snappy failure)");
- } else {
- char buf[100];
- snprintf(buf, sizeof(buf), "(output: %.1f%%)",
- (produced * 100.0) / bytes);
- thread->stats.AddMessage(buf);
- thread->stats.AddBytes(bytes);
- }
- }
-
- void SnappyUncompress(ThreadState* thread) {
- RandomGenerator gen;
- Slice input = gen.Generate(Options().block_size);
- std::string compressed;
- bool ok = port::Snappy_Compress(input.data(), input.size(), &compressed);
- int64_t bytes = 0;
- char* uncompressed = new char[input.size()];
- while (ok && bytes < 1024 * 1048576) { // Compress 1G
- ok = port::Snappy_Uncompress(compressed.data(), compressed.size(),
- uncompressed);
- bytes += input.size();
- thread->stats.FinishedSingleOp();
- }
- delete[] uncompressed;
-
- if (!ok) {
- thread->stats.AddMessage("(snappy failure)");
- } else {
- thread->stats.AddBytes(bytes);
- }
- }
-
- void Open() {
- assert(db_ == NULL);
- Options options;
- options.create_if_missing = !FLAGS_use_existing_db;
- options.block_cache = cache_;
- options.write_buffer_size = FLAGS_write_buffer_size;
- options.max_open_files = FLAGS_open_files;
- options.filter_policy = filter_policy_;
- Status s = DB::Open(options, FLAGS_db, &db_);
- if (!s.ok()) {
- fprintf(stderr, "open error: %s\n", s.ToString().c_str());
- exit(1);
- }
- }
-
- void OpenBench(ThreadState* thread) {
- for (int i = 0; i < num_; i++) {
- delete db_;
- Open();
- thread->stats.FinishedSingleOp();
- }
- }
-
- void WriteSeq(ThreadState* thread) {
- DoWrite(thread, true);
- }
-
- void WriteRandom(ThreadState* thread) {
- DoWrite(thread, false);
- }
-
- void DoWrite(ThreadState* thread, bool seq) {
- if (num_ != FLAGS_num) {
- char msg[100];
- snprintf(msg, sizeof(msg), "(%d ops)", num_);
- thread->stats.AddMessage(msg);
- }
-
- RandomGenerator gen;
- WriteBatch batch;
- Status s;
- int64_t bytes = 0;
- for (int i = 0; i < num_; i += entries_per_batch_) {
- batch.Clear();
- for (int j = 0; j < entries_per_batch_; j++) {
- const int k = seq ? i+j : (thread->rand.Next() % FLAGS_num);
- char key[100];
- snprintf(key, sizeof(key), "%016d", k);
- batch.Put(key, gen.Generate(value_size_));
- bytes += value_size_ + strlen(key);
- thread->stats.FinishedSingleOp();
- }
- s = db_->Write(write_options_, &batch);
- if (!s.ok()) {
- fprintf(stderr, "put error: %s\n", s.ToString().c_str());
- exit(1);
- }
- }
- thread->stats.AddBytes(bytes);
- }
-
- void ReadSequential(ThreadState* thread) {
- Iterator* iter = db_->NewIterator(ReadOptions());
- int i = 0;
- int64_t bytes = 0;
- for (iter->SeekToFirst(); i < reads_ && iter->Valid(); iter->Next()) {
- bytes += iter->key().size() + iter->value().size();
- thread->stats.FinishedSingleOp();
- ++i;
- }
- delete iter;
- thread->stats.AddBytes(bytes);
- }
-
- void ReadReverse(ThreadState* thread) {
- Iterator* iter = db_->NewIterator(ReadOptions());
- int i = 0;
- int64_t bytes = 0;
- for (iter->SeekToLast(); i < reads_ && iter->Valid(); iter->Prev()) {
- bytes += iter->key().size() + iter->value().size();
- thread->stats.FinishedSingleOp();
- ++i;
- }
- delete iter;
- thread->stats.AddBytes(bytes);
- }
-
- void ReadRandom(ThreadState* thread) {
- ReadOptions options;
- std::string value;
- int found = 0;
- for (int i = 0; i < reads_; i++) {
- char key[100];
- const int k = thread->rand.Next() % FLAGS_num;
- snprintf(key, sizeof(key), "%016d", k);
- if (db_->Get(options, key, &value).ok()) {
- found++;
- }
- thread->stats.FinishedSingleOp();
- }
- char msg[100];
- snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_);
- thread->stats.AddMessage(msg);
- }
-
- void ReadMissing(ThreadState* thread) {
- ReadOptions options;
- std::string value;
- for (int i = 0; i < reads_; i++) {
- char key[100];
- const int k = thread->rand.Next() % FLAGS_num;
- snprintf(key, sizeof(key), "%016d.", k);
- db_->Get(options, key, &value);
- thread->stats.FinishedSingleOp();
- }
- }
-
- void ReadHot(ThreadState* thread) {
- ReadOptions options;
- std::string value;
- const int range = (FLAGS_num + 99) / 100;
- for (int i = 0; i < reads_; i++) {
- char key[100];
- const int k = thread->rand.Next() % range;
- snprintf(key, sizeof(key), "%016d", k);
- db_->Get(options, key, &value);
- thread->stats.FinishedSingleOp();
- }
- }
-
- void SeekRandom(ThreadState* thread) {
- ReadOptions options;
- int found = 0;
- for (int i = 0; i < reads_; i++) {
- Iterator* iter = db_->NewIterator(options);
- char key[100];
- const int k = thread->rand.Next() % FLAGS_num;
- snprintf(key, sizeof(key), "%016d", k);
- iter->Seek(key);
- if (iter->Valid() && iter->key() == key) found++;
- delete iter;
- thread->stats.FinishedSingleOp();
- }
- char msg[100];
- snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_);
- thread->stats.AddMessage(msg);
- }
-
- void DoDelete(ThreadState* thread, bool seq) {
- RandomGenerator gen;
- WriteBatch batch;
- Status s;
- for (int i = 0; i < num_; i += entries_per_batch_) {
- batch.Clear();
- for (int j = 0; j < entries_per_batch_; j++) {
- const int k = seq ? i+j : (thread->rand.Next() % FLAGS_num);
- char key[100];
- snprintf(key, sizeof(key), "%016d", k);
- batch.Delete(key);
- thread->stats.FinishedSingleOp();
- }
- s = db_->Write(write_options_, &batch);
- if (!s.ok()) {
- fprintf(stderr, "del error: %s\n", s.ToString().c_str());
- exit(1);
- }
- }
- }
-
- void DeleteSeq(ThreadState* thread) {
- DoDelete(thread, true);
- }
-
- void DeleteRandom(ThreadState* thread) {
- DoDelete(thread, false);
- }
-
- void ReadWhileWriting(ThreadState* thread) {
- if (thread->tid > 0) {
- ReadRandom(thread);
- } else {
- // Special thread that keeps writing until other threads are done.
- RandomGenerator gen;
- while (true) {
- {
- MutexLock l(&thread->shared->mu);
- if (thread->shared->num_done + 1 >= thread->shared->num_initialized) {
- // Other threads have finished
- break;
- }
- }
-
- const int k = thread->rand.Next() % FLAGS_num;
- char key[100];
- snprintf(key, sizeof(key), "%016d", k);
- Status s = db_->Put(write_options_, key, gen.Generate(value_size_));
- if (!s.ok()) {
- fprintf(stderr, "put error: %s\n", s.ToString().c_str());
- exit(1);
- }
- }
-
- // Do not count any of the preceding work/delay in stats.
- thread->stats.Start();
- }
- }
-
- void Compact(ThreadState* thread) {
- db_->CompactRange(NULL, NULL);
- }
-
- void PrintStats(const char* key) {
- std::string stats;
- if (!db_->GetProperty(key, &stats)) {
- stats = "(failed)";
- }
- fprintf(stdout, "\n%s\n", stats.c_str());
- }
-
- static void WriteToFile(void* arg, const char* buf, int n) {
- reinterpret_cast<WritableFile*>(arg)->Append(Slice(buf, n));
- }
-
- void HeapProfile() {
- char fname[100];
- snprintf(fname, sizeof(fname), "%s/heap-%04d", FLAGS_db, ++heap_counter_);
- WritableFile* file;
- Status s = Env::Default()->NewWritableFile(fname, &file);
- if (!s.ok()) {
- fprintf(stderr, "%s\n", s.ToString().c_str());
- return;
- }
- bool ok = port::GetHeapProfile(WriteToFile, file);
- delete file;
- if (!ok) {
- fprintf(stderr, "heap profiling not supported\n");
- Env::Default()->DeleteFile(fname);
- }
- }
-};
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- FLAGS_write_buffer_size = leveldb::Options().write_buffer_size;
- FLAGS_open_files = leveldb::Options().max_open_files;
- std::string default_db_path;
-
- for (int i = 1; i < argc; i++) {
- double d;
- int n;
- char junk;
- if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) {
- FLAGS_benchmarks = argv[i] + strlen("--benchmarks=");
- } else if (sscanf(argv[i], "--compression_ratio=%lf%c", &d, &junk) == 1) {
- FLAGS_compression_ratio = d;
- } else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 &&
- (n == 0 || n == 1)) {
- FLAGS_histogram = n;
- } else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 &&
- (n == 0 || n == 1)) {
- FLAGS_use_existing_db = n;
- } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) {
- FLAGS_num = n;
- } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) {
- FLAGS_reads = n;
- } else if (sscanf(argv[i], "--threads=%d%c", &n, &junk) == 1) {
- FLAGS_threads = n;
- } else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) {
- FLAGS_value_size = n;
- } else if (sscanf(argv[i], "--write_buffer_size=%d%c", &n, &junk) == 1) {
- FLAGS_write_buffer_size = n;
- } else if (sscanf(argv[i], "--cache_size=%d%c", &n, &junk) == 1) {
- FLAGS_cache_size = n;
- } else if (sscanf(argv[i], "--bloom_bits=%d%c", &n, &junk) == 1) {
- FLAGS_bloom_bits = n;
- } else if (sscanf(argv[i], "--open_files=%d%c", &n, &junk) == 1) {
- FLAGS_open_files = n;
- } else if (strncmp(argv[i], "--db=", 5) == 0) {
- FLAGS_db = argv[i] + 5;
- } else {
- fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
- exit(1);
- }
- }
-
- // Choose a location for the test database if none given with --db=<path>
- if (FLAGS_db == NULL) {
- leveldb::Env::Default()->GetTestDirectory(&default_db_path);
- default_db_path += "/dbbench";
- FLAGS_db = default_db_path.c_str();
- }
-
- leveldb::Benchmark benchmark;
- benchmark.Run();
- return 0;
-}
diff --git a/db/db_impl.cc b/db/db_impl.cc
index cc1cea4..6e8fe61 100644
--- a/db/db_impl.cc
+++ b/db/db_impl.cc
@@ -4,12 +4,15 @@
#include "db/db_impl.h"
+#include <stdint.h>
+#include <stdio.h>
+
#include <algorithm>
+#include <atomic>
#include <set>
#include <string>
-#include <stdint.h>
-#include <stdio.h>
#include <vector>
+
#include "db/builder.h"
#include "db/db_iter.h"
#include "db/dbformat.h"
@@ -39,16 +42,33 @@ const int kNumNonTableCacheFiles = 10;
// Information kept for every waiting writer
struct DBImpl::Writer {
+ explicit Writer(port::Mutex* mu)
+ : batch(nullptr), sync(false), done(false), cv(mu) {}
+
Status status;
WriteBatch* batch;
bool sync;
bool done;
port::CondVar cv;
-
- explicit Writer(port::Mutex* mu) : cv(mu) { }
};
struct DBImpl::CompactionState {
+ // Files produced by compaction
+ struct Output {
+ uint64_t number;
+ uint64_t file_size;
+ InternalKey smallest, largest;
+ };
+
+ Output* current_output() { return &outputs[outputs.size() - 1]; }
+
+ explicit CompactionState(Compaction* c)
+ : compaction(c),
+ smallest_snapshot(0),
+ outfile(nullptr),
+ builder(nullptr),
+ total_bytes(0) {}
+
Compaction* const compaction;
// Sequence numbers < smallest_snapshot are not significant since we
@@ -57,12 +77,6 @@ struct DBImpl::CompactionState {
// we can drop all entries for the same key with sequence numbers < S.
SequenceNumber smallest_snapshot;
- // Files produced by compaction
- struct Output {
- uint64_t number;
- uint64_t file_size;
- InternalKey smallest, largest;
- };
std::vector<Output> outputs;
// State kept for output being generated
@@ -70,19 +84,10 @@ struct DBImpl::CompactionState {
TableBuilder* builder;
uint64_t total_bytes;
-
- Output* current_output() { return &outputs[outputs.size()-1]; }
-
- explicit CompactionState(Compaction* c)
- : compaction(c),
- outfile(NULL),
- builder(NULL),
- total_bytes(0) {
- }
};
// Fix user-supplied options to be reasonable
-template <class T,class V>
+template <class T, class V>
static void ClipToRange(T* ptr, V minvalue, V maxvalue) {
if (static_cast<V>(*ptr) > maxvalue) *ptr = maxvalue;
if (static_cast<V>(*ptr) < minvalue) *ptr = minvalue;
@@ -93,26 +98,32 @@ Options SanitizeOptions(const std::string& dbname,
const Options& src) {
Options result = src;
result.comparator = icmp;
- result.filter_policy = (src.filter_policy != NULL) ? ipolicy : NULL;
- ClipToRange(&result.max_open_files, 64 + kNumNonTableCacheFiles, 50000);
- ClipToRange(&result.write_buffer_size, 64<<10, 1<<30);
- ClipToRange(&result.block_size, 1<<10, 4<<20);
- if (result.info_log == NULL) {
+ result.filter_policy = (src.filter_policy != nullptr) ? ipolicy : nullptr;
+ ClipToRange(&result.max_open_files, 64 + kNumNonTableCacheFiles, 50000);
+ ClipToRange(&result.write_buffer_size, 64 << 10, 1 << 30);
+ ClipToRange(&result.max_file_size, 1 << 20, 1 << 30);
+ ClipToRange(&result.block_size, 1 << 10, 4 << 20);
+ if (result.info_log == nullptr) {
// Open a log file in the same directory as the db
src.env->CreateDir(dbname); // In case it does not exist
src.env->RenameFile(InfoLogFileName(dbname), OldInfoLogFileName(dbname));
Status s = src.env->NewLogger(InfoLogFileName(dbname), &result.info_log);
if (!s.ok()) {
// No place suitable for logging
- result.info_log = NULL;
+ result.info_log = nullptr;
}
}
- if (result.block_cache == NULL) {
+ if (result.block_cache == nullptr) {
result.block_cache = NewLRUCache(8 << 20);
}
return result;
}
+static int TableCacheSize(const Options& sanitized_options) {
+ // Reserve ten files or so for other uses and give the rest to TableCache.
+ return sanitized_options.max_open_files - kNumNonTableCacheFiles;
+}
+
DBImpl::DBImpl(const Options& raw_options, const std::string& dbname)
: env_(raw_options.env),
internal_comparator_(raw_options.comparator),
@@ -122,45 +133,39 @@ DBImpl::DBImpl(const Options& raw_options, const std::string& dbname)
owns_info_log_(options_.info_log != raw_options.info_log),
owns_cache_(options_.block_cache != raw_options.block_cache),
dbname_(dbname),
- db_lock_(NULL),
- shutting_down_(NULL),
- bg_cv_(&mutex_),
- mem_(new MemTable(internal_comparator_)),
- imm_(NULL),
- logfile_(NULL),
+ table_cache_(new TableCache(dbname_, options_, TableCacheSize(options_))),
+ db_lock_(nullptr),
+ shutting_down_(false),
+ background_work_finished_signal_(&mutex_),
+ mem_(nullptr),
+ imm_(nullptr),
+ has_imm_(false),
+ logfile_(nullptr),
logfile_number_(0),
- log_(NULL),
+ log_(nullptr),
seed_(0),
tmp_batch_(new WriteBatch),
- bg_compaction_scheduled_(false),
- manual_compaction_(NULL) {
- mem_->Ref();
- has_imm_.Release_Store(NULL);
-
- // Reserve ten files or so for other uses and give the rest to TableCache.
- const int table_cache_size = options_.max_open_files - kNumNonTableCacheFiles;
- table_cache_ = new TableCache(dbname_, &options_, table_cache_size);
-
- versions_ = new VersionSet(dbname_, &options_, table_cache_,
- &internal_comparator_);
-}
+ background_compaction_scheduled_(false),
+ manual_compaction_(nullptr),
+ versions_(new VersionSet(dbname_, &options_, table_cache_,
+ &internal_comparator_)) {}
DBImpl::~DBImpl() {
- // Wait for background work to finish
+ // Wait for background work to finish.
mutex_.Lock();
- shutting_down_.Release_Store(this); // Any non-NULL value is ok
- while (bg_compaction_scheduled_) {
- bg_cv_.Wait();
+ shutting_down_.store(true, std::memory_order_release);
+ while (background_compaction_scheduled_) {
+ background_work_finished_signal_.Wait();
}
mutex_.Unlock();
- if (db_lock_ != NULL) {
+ if (db_lock_ != nullptr) {
env_->UnlockFile(db_lock_);
}
delete versions_;
- if (mem_ != NULL) mem_->Unref();
- if (imm_ != NULL) imm_->Unref();
+ if (mem_ != nullptr) mem_->Unref();
+ if (imm_ != nullptr) imm_->Unref();
delete tmp_batch_;
delete log_;
delete logfile_;
@@ -237,7 +242,6 @@ void DBImpl::DeleteObsoleteFiles() {
// Unlock while deleting obsolete files
mutex_.Unlock();
-
uint64_t number;
FileType type;
for (size_t i = 0; i < filenames.size(); i++) {
@@ -272,8 +276,7 @@ void DBImpl::DeleteObsoleteFiles() {
if (type == kTableFile) {
table_cache_->Evict(number);
}
- Log(options_.info_log, "Delete type=%d #%lld\n",
- int(type),
+ Log(options_.info_log, "Delete type=%d #%lld\n", static_cast<int>(type),
static_cast<unsigned long long>(number));
env_->DeleteFile(dbname_ + "/" + filenames[i]);
}
@@ -282,14 +285,14 @@ void DBImpl::DeleteObsoleteFiles() {
mutex_.Lock();
}
-Status DBImpl::Recover(VersionEdit* edit) {
+Status DBImpl::Recover(VersionEdit* edit, bool* save_manifest) {
mutex_.AssertHeld();
// Ignore error from CreateDir since the creation of the DB is
// committed only when the descriptor is created, and this directory
// may already exist from a previous failed creation attempt.
env_->CreateDir(dbname_);
- assert(db_lock_ == NULL);
+ assert(db_lock_ == nullptr);
Status s = env_->LockFile(LockFileName(dbname_), &db_lock_);
if (!s.ok()) {
return s;
@@ -307,82 +310,85 @@ Status DBImpl::Recover(VersionEdit* edit) {
}
} else {
if (options_.error_if_exists) {
- return Status::InvalidArgument(
- dbname_, "exists (error_if_exists is true)");
+ return Status::InvalidArgument(dbname_,
+ "exists (error_if_exists is true)");
}
}
- s = versions_->Recover();
- if (s.ok()) {
- SequenceNumber max_sequence(0);
-
- // Recover from all newer log files than the ones named in the
- // descriptor (new log files may have been added by the previous
- // incarnation without registering them in the descriptor).
- //
- // Note that PrevLogNumber() is no longer used, but we pay
- // attention to it in case we are recovering a database
- // produced by an older version of leveldb.
- const uint64_t min_log = versions_->LogNumber();
- const uint64_t prev_log = versions_->PrevLogNumber();
- std::vector<std::string> filenames;
- s = env_->GetChildren(dbname_, &filenames);
+ s = versions_->Recover(save_manifest);
+ if (!s.ok()) {
+ return s;
+ }
+ SequenceNumber max_sequence(0);
+
+ // Recover from all newer log files than the ones named in the
+ // descriptor (new log files may have been added by the previous
+ // incarnation without registering them in the descriptor).
+ //
+ // Note that PrevLogNumber() is no longer used, but we pay
+ // attention to it in case we are recovering a database
+ // produced by an older version of leveldb.
+ const uint64_t min_log = versions_->LogNumber();
+ const uint64_t prev_log = versions_->PrevLogNumber();
+ std::vector<std::string> filenames;
+ s = env_->GetChildren(dbname_, &filenames);
+ if (!s.ok()) {
+ return s;
+ }
+ std::set<uint64_t> expected;
+ versions_->AddLiveFiles(&expected);
+ uint64_t number;
+ FileType type;
+ std::vector<uint64_t> logs;
+ for (size_t i = 0; i < filenames.size(); i++) {
+ if (ParseFileName(filenames[i], &number, &type)) {
+ expected.erase(number);
+ if (type == kLogFile && ((number >= min_log) || (number == prev_log)))
+ logs.push_back(number);
+ }
+ }
+ if (!expected.empty()) {
+ char buf[50];
+ snprintf(buf, sizeof(buf), "%d missing files; e.g.",
+ static_cast<int>(expected.size()));
+ return Status::Corruption(buf, TableFileName(dbname_, *(expected.begin())));
+ }
+
+ // Recover in the order in which the logs were generated
+ std::sort(logs.begin(), logs.end());
+ for (size_t i = 0; i < logs.size(); i++) {
+ s = RecoverLogFile(logs[i], (i == logs.size() - 1), save_manifest, edit,
+ &max_sequence);
if (!s.ok()) {
return s;
}
- std::set<uint64_t> expected;
- versions_->AddLiveFiles(&expected);
- uint64_t number;
- FileType type;
- std::vector<uint64_t> logs;
- for (size_t i = 0; i < filenames.size(); i++) {
- if (ParseFileName(filenames[i], &number, &type)) {
- expected.erase(number);
- if (type == kLogFile && ((number >= min_log) || (number == prev_log)))
- logs.push_back(number);
- }
- }
- if (!expected.empty()) {
- char buf[50];
- snprintf(buf, sizeof(buf), "%d missing files; e.g.",
- static_cast<int>(expected.size()));
- return Status::Corruption(buf, TableFileName(dbname_, *(expected.begin())));
- }
-
- // Recover in the order in which the logs were generated
- std::sort(logs.begin(), logs.end());
- for (size_t i = 0; i < logs.size(); i++) {
- s = RecoverLogFile(logs[i], edit, &max_sequence);
- // The previous incarnation may not have written any MANIFEST
- // records after allocating this log number. So we manually
- // update the file number allocation counter in VersionSet.
- versions_->MarkFileNumberUsed(logs[i]);
- }
+ // The previous incarnation may not have written any MANIFEST
+ // records after allocating this log number. So we manually
+ // update the file number allocation counter in VersionSet.
+ versions_->MarkFileNumberUsed(logs[i]);
+ }
- if (s.ok()) {
- if (versions_->LastSequence() < max_sequence) {
- versions_->SetLastSequence(max_sequence);
- }
- }
+ if (versions_->LastSequence() < max_sequence) {
+ versions_->SetLastSequence(max_sequence);
}
- return s;
+ return Status::OK();
}
-Status DBImpl::RecoverLogFile(uint64_t log_number,
- VersionEdit* edit,
+Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log,
+ bool* save_manifest, VersionEdit* edit,
SequenceNumber* max_sequence) {
struct LogReporter : public log::Reader::Reporter {
Env* env;
Logger* info_log;
const char* fname;
- Status* status; // NULL if options_.paranoid_checks==false
- virtual void Corruption(size_t bytes, const Status& s) {
+ Status* status; // null if options_.paranoid_checks==false
+ void Corruption(size_t bytes, const Status& s) override {
Log(info_log, "%s%s: dropping %d bytes; %s",
- (this->status == NULL ? "(ignoring error) " : ""),
- fname, static_cast<int>(bytes), s.ToString().c_str());
- if (this->status != NULL && this->status->ok()) *this->status = s;
+ (this->status == nullptr ? "(ignoring error) " : ""), fname,
+ static_cast<int>(bytes), s.ToString().c_str());
+ if (this->status != nullptr && this->status->ok()) *this->status = s;
}
};
@@ -402,31 +408,30 @@ Status DBImpl::RecoverLogFile(uint64_t log_number,
reporter.env = env_;
reporter.info_log = options_.info_log;
reporter.fname = fname.c_str();
- reporter.status = (options_.paranoid_checks ? &status : NULL);
+ reporter.status = (options_.paranoid_checks ? &status : nullptr);
// We intentionally make log::Reader do checksumming even if
// paranoid_checks==false so that corruptions cause entire commits
// to be skipped instead of propagating bad information (like overly
// large sequence numbers).
- log::Reader reader(file, &reporter, true/*checksum*/,
- 0/*initial_offset*/);
+ log::Reader reader(file, &reporter, true /*checksum*/, 0 /*initial_offset*/);
Log(options_.info_log, "Recovering log #%llu",
- (unsigned long long) log_number);
+ (unsigned long long)log_number);
// Read all the records and add to a memtable
std::string scratch;
Slice record;
WriteBatch batch;
- MemTable* mem = NULL;
- while (reader.ReadRecord(&record, &scratch) &&
- status.ok()) {
+ int compactions = 0;
+ MemTable* mem = nullptr;
+ while (reader.ReadRecord(&record, &scratch) && status.ok()) {
if (record.size() < 12) {
- reporter.Corruption(
- record.size(), Status::Corruption("log record too small"));
+ reporter.Corruption(record.size(),
+ Status::Corruption("log record too small"));
continue;
}
WriteBatchInternal::SetContents(&batch, record);
- if (mem == NULL) {
+ if (mem == nullptr) {
mem = new MemTable(internal_comparator_);
mem->Ref();
}
@@ -435,33 +440,59 @@ Status DBImpl::RecoverLogFile(uint64_t log_number,
if (!status.ok()) {
break;
}
- const SequenceNumber last_seq =
- WriteBatchInternal::Sequence(&batch) +
- WriteBatchInternal::Count(&batch) - 1;
+ const SequenceNumber last_seq = WriteBatchInternal::Sequence(&batch) +
+ WriteBatchInternal::Count(&batch) - 1;
if (last_seq > *max_sequence) {
*max_sequence = last_seq;
}
if (mem->ApproximateMemoryUsage() > options_.write_buffer_size) {
- status = WriteLevel0Table(mem, edit, NULL);
+ compactions++;
+ *save_manifest = true;
+ status = WriteLevel0Table(mem, edit, nullptr);
+ mem->Unref();
+ mem = nullptr;
if (!status.ok()) {
// Reflect errors immediately so that conditions like full
// file-systems cause the DB::Open() to fail.
break;
}
- mem->Unref();
- mem = NULL;
}
}
- if (status.ok() && mem != NULL) {
- status = WriteLevel0Table(mem, edit, NULL);
- // Reflect errors immediately so that conditions like full
- // file-systems cause the DB::Open() to fail.
+ delete file;
+
+ // See if we should keep reusing the last log file.
+ if (status.ok() && options_.reuse_logs && last_log && compactions == 0) {
+ assert(logfile_ == nullptr);
+ assert(log_ == nullptr);
+ assert(mem_ == nullptr);
+ uint64_t lfile_size;
+ if (env_->GetFileSize(fname, &lfile_size).ok() &&
+ env_->NewAppendableFile(fname, &logfile_).ok()) {
+ Log(options_.info_log, "Reusing old log %s \n", fname.c_str());
+ log_ = new log::Writer(logfile_, lfile_size);
+ logfile_number_ = log_number;
+ if (mem != nullptr) {
+ mem_ = mem;
+ mem = nullptr;
+ } else {
+ // mem can be nullptr if lognum exists but was empty.
+ mem_ = new MemTable(internal_comparator_);
+ mem_->Ref();
+ }
+ }
+ }
+
+ if (mem != nullptr) {
+ // mem did not get reused; compact it.
+ if (status.ok()) {
+ *save_manifest = true;
+ status = WriteLevel0Table(mem, edit, nullptr);
+ }
+ mem->Unref();
}
- if (mem != NULL) mem->Unref();
- delete file;
return status;
}
@@ -474,7 +505,7 @@ Status DBImpl::WriteLevel0Table(MemTable* mem, VersionEdit* edit,
pending_outputs_.insert(meta.number);
Iterator* iter = mem->NewIterator();
Log(options_.info_log, "Level-0 table #%llu: started",
- (unsigned long long) meta.number);
+ (unsigned long long)meta.number);
Status s;
{
@@ -484,24 +515,22 @@ Status DBImpl::WriteLevel0Table(MemTable* mem, VersionEdit* edit,
}
Log(options_.info_log, "Level-0 table #%llu: %lld bytes %s",
- (unsigned long long) meta.number,
- (unsigned long long) meta.file_size,
+ (unsigned long long)meta.number, (unsigned long long)meta.file_size,
s.ToString().c_str());
delete iter;
pending_outputs_.erase(meta.number);
-
// Note that if file_size is zero, the file has been deleted and
// should not be added to the manifest.
int level = 0;
if (s.ok() && meta.file_size > 0) {
const Slice min_user_key = meta.smallest.user_key();
const Slice max_user_key = meta.largest.user_key();
- if (base != NULL) {
+ if (base != nullptr) {
level = base->PickLevelForMemTableOutput(min_user_key, max_user_key);
}
- edit->AddFile(level, meta.number, meta.file_size,
- meta.smallest, meta.largest);
+ edit->AddFile(level, meta.number, meta.file_size, meta.smallest,
+ meta.largest);
}
CompactionStats stats;
@@ -513,7 +542,7 @@ Status DBImpl::WriteLevel0Table(MemTable* mem, VersionEdit* edit,
void DBImpl::CompactMemTable() {
mutex_.AssertHeld();
- assert(imm_ != NULL);
+ assert(imm_ != nullptr);
// Save the contents of the memtable as a new Table
VersionEdit edit;
@@ -522,7 +551,7 @@ void DBImpl::CompactMemTable() {
Status s = WriteLevel0Table(imm_, &edit, base);
base->Unref();
- if (s.ok() && shutting_down_.Acquire_Load()) {
+ if (s.ok() && shutting_down_.load(std::memory_order_acquire)) {
s = Status::IOError("Deleting DB during memtable compaction");
}
@@ -536,8 +565,8 @@ void DBImpl::CompactMemTable() {
if (s.ok()) {
// Commit to the new state
imm_->Unref();
- imm_ = NULL;
- has_imm_.Release_Store(NULL);
+ imm_ = nullptr;
+ has_imm_.store(false, std::memory_order_release);
DeleteObsoleteFiles();
} else {
RecordBackgroundError(s);
@@ -555,13 +584,14 @@ void DBImpl::CompactRange(const Slice* begin, const Slice* end) {
}
}
}
- TEST_CompactMemTable(); // TODO(sanjay): Skip if memtable does not overlap
+ TEST_CompactMemTable(); // TODO(sanjay): Skip if memtable does not overlap
for (int level = 0; level < max_level_with_files; level++) {
TEST_CompactRange(level, begin, end);
}
}
-void DBImpl::TEST_CompactRange(int level, const Slice* begin,const Slice* end) {
+void DBImpl::TEST_CompactRange(int level, const Slice* begin,
+ const Slice* end) {
assert(level >= 0);
assert(level + 1 < config::kNumLevels);
@@ -570,44 +600,45 @@ void DBImpl::TEST_CompactRange(int level, const Slice* begin,const Slice* end) {
ManualCompaction manual;
manual.level = level;
manual.done = false;
- if (begin == NULL) {
- manual.begin = NULL;
+ if (begin == nullptr) {
+ manual.begin = nullptr;
} else {
begin_storage = InternalKey(*begin, kMaxSequenceNumber, kValueTypeForSeek);
manual.begin = &begin_storage;
}
- if (end == NULL) {
- manual.end = NULL;
+ if (end == nullptr) {
+ manual.end = nullptr;
} else {
end_storage = InternalKey(*end, 0, static_cast<ValueType>(0));
manual.end = &end_storage;
}
MutexLock l(&mutex_);
- while (!manual.done && !shutting_down_.Acquire_Load() && bg_error_.ok()) {
- if (manual_compaction_ == NULL) { // Idle
+ while (!manual.done && !shutting_down_.load(std::memory_order_acquire) &&
+ bg_error_.ok()) {
+ if (manual_compaction_ == nullptr) { // Idle
manual_compaction_ = &manual;
MaybeScheduleCompaction();
} else { // Running either my compaction or another compaction.
- bg_cv_.Wait();
+ background_work_finished_signal_.Wait();
}
}
if (manual_compaction_ == &manual) {
// Cancel my manual compaction since we aborted early for some reason.
- manual_compaction_ = NULL;
+ manual_compaction_ = nullptr;
}
}
Status DBImpl::TEST_CompactMemTable() {
- // NULL batch means just wait for earlier writes to be done
- Status s = Write(WriteOptions(), NULL);
+ // nullptr batch means just wait for earlier writes to be done
+ Status s = Write(WriteOptions(), nullptr);
if (s.ok()) {
// Wait until the compaction completes
MutexLock l(&mutex_);
- while (imm_ != NULL && bg_error_.ok()) {
- bg_cv_.Wait();
+ while (imm_ != nullptr && bg_error_.ok()) {
+ background_work_finished_signal_.Wait();
}
- if (imm_ != NULL) {
+ if (imm_ != nullptr) {
s = bg_error_;
}
}
@@ -618,24 +649,23 @@ void DBImpl::RecordBackgroundError(const Status& s) {
mutex_.AssertHeld();
if (bg_error_.ok()) {
bg_error_ = s;
- bg_cv_.SignalAll();
+ background_work_finished_signal_.SignalAll();
}
}
void DBImpl::MaybeScheduleCompaction() {
mutex_.AssertHeld();
- if (bg_compaction_scheduled_) {
+ if (background_compaction_scheduled_) {
// Already scheduled
- } else if (shutting_down_.Acquire_Load()) {
+ } else if (shutting_down_.load(std::memory_order_acquire)) {
// DB is being deleted; no more background compactions
} else if (!bg_error_.ok()) {
// Already got an error; no more changes
- } else if (imm_ == NULL &&
- manual_compaction_ == NULL &&
+ } else if (imm_ == nullptr && manual_compaction_ == nullptr &&
!versions_->NeedsCompaction()) {
// No work to be done
} else {
- bg_compaction_scheduled_ = true;
+ background_compaction_scheduled_ = true;
env_->Schedule(&DBImpl::BGWork, this);
}
}
@@ -646,8 +676,8 @@ void DBImpl::BGWork(void* db) {
void DBImpl::BackgroundCall() {
MutexLock l(&mutex_);
- assert(bg_compaction_scheduled_);
- if (shutting_down_.Acquire_Load()) {
+ assert(background_compaction_scheduled_);
+ if (shutting_down_.load(std::memory_order_acquire)) {
// No more background work when shutting down.
} else if (!bg_error_.ok()) {
// No more background work after a background error.
@@ -655,36 +685,35 @@ void DBImpl::BackgroundCall() {
BackgroundCompaction();
}
- bg_compaction_scheduled_ = false;
+ background_compaction_scheduled_ = false;
// Previous compaction may have produced too many files in a level,
// so reschedule another compaction if needed.
MaybeScheduleCompaction();
- bg_cv_.SignalAll();
+ background_work_finished_signal_.SignalAll();
}
void DBImpl::BackgroundCompaction() {
mutex_.AssertHeld();
- if (imm_ != NULL) {
+ if (imm_ != nullptr) {
CompactMemTable();
return;
}
Compaction* c;
- bool is_manual = (manual_compaction_ != NULL);
+ bool is_manual = (manual_compaction_ != nullptr);
InternalKey manual_end;
if (is_manual) {
ManualCompaction* m = manual_compaction_;
c = versions_->CompactRange(m->level, m->begin, m->end);
- m->done = (c == NULL);
- if (c != NULL) {
+ m->done = (c == nullptr);
+ if (c != nullptr) {
manual_end = c->input(0, c->num_input_files(0) - 1)->largest;
}
Log(options_.info_log,
"Manual compaction at level-%d from %s .. %s; will stop at %s\n",
- m->level,
- (m->begin ? m->begin->DebugString().c_str() : "(begin)"),
+ m->level, (m->begin ? m->begin->DebugString().c_str() : "(begin)"),
(m->end ? m->end->DebugString().c_str() : "(end)"),
(m->done ? "(end)" : manual_end.DebugString().c_str()));
} else {
@@ -692,26 +721,24 @@ void DBImpl::BackgroundCompaction() {
}
Status status;
- if (c == NULL) {
+ if (c == nullptr) {
// Nothing to do
} else if (!is_manual && c->IsTrivialMove()) {
// Move file to next level
assert(c->num_input_files(0) == 1);
FileMetaData* f = c->input(0, 0);
c->edit()->DeleteFile(c->level(), f->number);
- c->edit()->AddFile(c->level() + 1, f->number, f->file_size,
- f->smallest, f->largest);
+ c->edit()->AddFile(c->level() + 1, f->number, f->file_size, f->smallest,
+ f->largest);
status = versions_->LogAndApply(c->edit(), &mutex_);
if (!status.ok()) {
RecordBackgroundError(status);
}
VersionSet::LevelSummaryStorage tmp;
Log(options_.info_log, "Moved #%lld to level-%d %lld bytes %s: %s\n",
- static_cast<unsigned long long>(f->number),
- c->level() + 1,
+ static_cast<unsigned long long>(f->number), c->level() + 1,
static_cast<unsigned long long>(f->file_size),
- status.ToString().c_str(),
- versions_->LevelSummary(&tmp));
+ status.ToString().c_str(), versions_->LevelSummary(&tmp));
} else {
CompactionState* compact = new CompactionState(c);
status = DoCompactionWork(compact);
@@ -726,11 +753,10 @@ void DBImpl::BackgroundCompaction() {
if (status.ok()) {
// Done
- } else if (shutting_down_.Acquire_Load()) {
+ } else if (shutting_down_.load(std::memory_order_acquire)) {
// Ignore compaction errors found during shutting down
} else {
- Log(options_.info_log,
- "Compaction error: %s", status.ToString().c_str());
+ Log(options_.info_log, "Compaction error: %s", status.ToString().c_str());
}
if (is_manual) {
@@ -744,18 +770,18 @@ void DBImpl::BackgroundCompaction() {
m->tmp_storage = manual_end;
m->begin = &m->tmp_storage;
}
- manual_compaction_ = NULL;
+ manual_compaction_ = nullptr;
}
}
void DBImpl::CleanupCompaction(CompactionState* compact) {
mutex_.AssertHeld();
- if (compact->builder != NULL) {
+ if (compact->builder != nullptr) {
// May happen if we get a shutdown call in the middle of compaction
compact->builder->Abandon();
delete compact->builder;
} else {
- assert(compact->outfile == NULL);
+ assert(compact->outfile == nullptr);
}
delete compact->outfile;
for (size_t i = 0; i < compact->outputs.size(); i++) {
@@ -766,8 +792,8 @@ void DBImpl::CleanupCompaction(CompactionState* compact) {
}
Status DBImpl::OpenCompactionOutputFile(CompactionState* compact) {
- assert(compact != NULL);
- assert(compact->builder == NULL);
+ assert(compact != nullptr);
+ assert(compact->builder == nullptr);
uint64_t file_number;
{
mutex_.Lock();
@@ -792,9 +818,9 @@ Status DBImpl::OpenCompactionOutputFile(CompactionState* compact) {
Status DBImpl::FinishCompactionOutputFile(CompactionState* compact,
Iterator* input) {
- assert(compact != NULL);
- assert(compact->outfile != NULL);
- assert(compact->builder != NULL);
+ assert(compact != nullptr);
+ assert(compact->outfile != nullptr);
+ assert(compact->builder != nullptr);
const uint64_t output_number = compact->current_output()->number;
assert(output_number != 0);
@@ -811,7 +837,7 @@ Status DBImpl::FinishCompactionOutputFile(CompactionState* compact,
compact->current_output()->file_size = current_bytes;
compact->total_bytes += current_bytes;
delete compact->builder;
- compact->builder = NULL;
+ compact->builder = nullptr;
// Finish and check for file errors
if (s.ok()) {
@@ -821,34 +847,29 @@ Status DBImpl::FinishCompactionOutputFile(CompactionState* compact,
s = compact->outfile->Close();
}
delete compact->outfile;
- compact->outfile = NULL;
+ compact->outfile = nullptr;
if (s.ok() && current_entries > 0) {
// Verify that the table is usable
- Iterator* iter = table_cache_->NewIterator(ReadOptions(),
- output_number,
- current_bytes);
+ Iterator* iter =
+ table_cache_->NewIterator(ReadOptions(), output_number, current_bytes);
s = iter->status();
delete iter;
if (s.ok()) {
- Log(options_.info_log,
- "Generated table #%llu: %lld keys, %lld bytes",
- (unsigned long long) output_number,
- (unsigned long long) current_entries,
- (unsigned long long) current_bytes);
+ Log(options_.info_log, "Generated table #%llu@%d: %lld keys, %lld bytes",
+ (unsigned long long)output_number, compact->compaction->level(),
+ (unsigned long long)current_entries,
+ (unsigned long long)current_bytes);
}
}
return s;
}
-
Status DBImpl::InstallCompactionResults(CompactionState* compact) {
mutex_.AssertHeld();
- Log(options_.info_log, "Compacted %d@%d + %d@%d files => %lld bytes",
- compact->compaction->num_input_files(0),
- compact->compaction->level(),
- compact->compaction->num_input_files(1),
- compact->compaction->level() + 1,
+ Log(options_.info_log, "Compacted %d@%d + %d@%d files => %lld bytes",
+ compact->compaction->num_input_files(0), compact->compaction->level(),
+ compact->compaction->num_input_files(1), compact->compaction->level() + 1,
static_cast<long long>(compact->total_bytes));
// Add compaction outputs
@@ -856,9 +877,8 @@ Status DBImpl::InstallCompactionResults(CompactionState* compact) {
const int level = compact->compaction->level();
for (size_t i = 0; i < compact->outputs.size(); i++) {
const CompactionState::Output& out = compact->outputs[i];
- compact->compaction->edit()->AddFile(
- level + 1,
- out.number, out.file_size, out.smallest, out.largest);
+ compact->compaction->edit()->AddFile(level + 1, out.number, out.file_size,
+ out.smallest, out.largest);
}
return versions_->LogAndApply(compact->compaction->edit(), &mutex_);
}
@@ -867,39 +887,40 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
const uint64_t start_micros = env_->NowMicros();
int64_t imm_micros = 0; // Micros spent doing imm_ compactions
- Log(options_.info_log, "Compacting %d@%d + %d@%d files",
- compact->compaction->num_input_files(0),
- compact->compaction->level(),
+ Log(options_.info_log, "Compacting %d@%d + %d@%d files",
+ compact->compaction->num_input_files(0), compact->compaction->level(),
compact->compaction->num_input_files(1),
compact->compaction->level() + 1);
assert(versions_->NumLevelFiles(compact->compaction->level()) > 0);
- assert(compact->builder == NULL);
- assert(compact->outfile == NULL);
+ assert(compact->builder == nullptr);
+ assert(compact->outfile == nullptr);
if (snapshots_.empty()) {
compact->smallest_snapshot = versions_->LastSequence();
} else {
- compact->smallest_snapshot = snapshots_.oldest()->number_;
+ compact->smallest_snapshot = snapshots_.oldest()->sequence_number();
}
+ Iterator* input = versions_->MakeInputIterator(compact->compaction);
+
// Release mutex while we're actually doing the compaction work
mutex_.Unlock();
- Iterator* input = versions_->MakeInputIterator(compact->compaction);
input->SeekToFirst();
Status status;
ParsedInternalKey ikey;
std::string current_user_key;
bool has_current_user_key = false;
SequenceNumber last_sequence_for_key = kMaxSequenceNumber;
- for (; input->Valid() && !shutting_down_.Acquire_Load(); ) {
+ while (input->Valid() && !shutting_down_.load(std::memory_order_acquire)) {
// Prioritize immutable compaction work
- if (has_imm_.NoBarrier_Load() != NULL) {
+ if (has_imm_.load(std::memory_order_relaxed)) {
const uint64_t imm_start = env_->NowMicros();
mutex_.Lock();
- if (imm_ != NULL) {
+ if (imm_ != nullptr) {
CompactMemTable();
- bg_cv_.SignalAll(); // Wakeup MakeRoomForWrite() if necessary
+ // Wake up MakeRoomForWrite() if necessary.
+ background_work_finished_signal_.SignalAll();
}
mutex_.Unlock();
imm_micros += (env_->NowMicros() - imm_start);
@@ -907,7 +928,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
Slice key = input->key();
if (compact->compaction->ShouldStopBefore(key) &&
- compact->builder != NULL) {
+ compact->builder != nullptr) {
status = FinishCompactionOutputFile(compact, input);
if (!status.ok()) {
break;
@@ -923,8 +944,8 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
last_sequence_for_key = kMaxSequenceNumber;
} else {
if (!has_current_user_key ||
- user_comparator()->Compare(ikey.user_key,
- Slice(current_user_key)) != 0) {
+ user_comparator()->Compare(ikey.user_key, Slice(current_user_key)) !=
+ 0) {
// First occurrence of this user key
current_user_key.assign(ikey.user_key.data(), ikey.user_key.size());
has_current_user_key = true;
@@ -933,7 +954,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
if (last_sequence_for_key <= compact->smallest_snapshot) {
// Hidden by an newer entry for same user key
- drop = true; // (A)
+ drop = true; // (A)
} else if (ikey.type == kTypeDeletion &&
ikey.sequence <= compact->smallest_snapshot &&
compact->compaction->IsBaseLevelForKey(ikey.user_key)) {
@@ -961,7 +982,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
if (!drop) {
// Open output file if necessary
- if (compact->builder == NULL) {
+ if (compact->builder == nullptr) {
status = OpenCompactionOutputFile(compact);
if (!status.ok()) {
break;
@@ -986,17 +1007,17 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
input->Next();
}
- if (status.ok() && shutting_down_.Acquire_Load()) {
+ if (status.ok() && shutting_down_.load(std::memory_order_acquire)) {
status = Status::IOError("Deleting DB during compaction");
}
- if (status.ok() && compact->builder != NULL) {
+ if (status.ok() && compact->builder != nullptr) {
status = FinishCompactionOutputFile(compact, input);
}
if (status.ok()) {
status = input->status();
}
delete input;
- input = NULL;
+ input = nullptr;
CompactionStats stats;
stats.micros = env_->NowMicros() - start_micros - imm_micros;
@@ -1019,34 +1040,37 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
RecordBackgroundError(status);
}
VersionSet::LevelSummaryStorage tmp;
- Log(options_.info_log,
- "compacted to: %s", versions_->LevelSummary(&tmp));
+ Log(options_.info_log, "compacted to: %s", versions_->LevelSummary(&tmp));
return status;
}
namespace {
+
struct IterState {
- port::Mutex* mu;
- Version* version;
- MemTable* mem;
- MemTable* imm;
+ port::Mutex* const mu;
+ Version* const version GUARDED_BY(mu);
+ MemTable* const mem GUARDED_BY(mu);
+ MemTable* const imm GUARDED_BY(mu);
+
+ IterState(port::Mutex* mutex, MemTable* mem, MemTable* imm, Version* version)
+ : mu(mutex), version(version), mem(mem), imm(imm) {}
};
static void CleanupIteratorState(void* arg1, void* arg2) {
IterState* state = reinterpret_cast<IterState*>(arg1);
state->mu->Lock();
state->mem->Unref();
- if (state->imm != NULL) state->imm->Unref();
+ if (state->imm != nullptr) state->imm->Unref();
state->version->Unref();
state->mu->Unlock();
delete state;
}
-} // namespace
+
+} // anonymous namespace
Iterator* DBImpl::NewInternalIterator(const ReadOptions& options,
SequenceNumber* latest_snapshot,
uint32_t* seed) {
- IterState* cleanup = new IterState;
mutex_.Lock();
*latest_snapshot = versions_->LastSequence();
@@ -1054,7 +1078,7 @@ Iterator* DBImpl::NewInternalIterator(const ReadOptions& options,
std::vector<Iterator*> list;
list.push_back(mem_->NewIterator());
mem_->Ref();
- if (imm_ != NULL) {
+ if (imm_ != nullptr) {
list.push_back(imm_->NewIterator());
imm_->Ref();
}
@@ -1063,11 +1087,8 @@ Iterator* DBImpl::NewInternalIterator(const ReadOptions& options,
NewMergingIterator(&internal_comparator_, &list[0], list.size());
versions_->current()->Ref();
- cleanup->mu = &mutex_;
- cleanup->mem = mem_;
- cleanup->imm = imm_;
- cleanup->version = versions_->current();
- internal_iter->RegisterCleanup(CleanupIteratorState, cleanup, NULL);
+ IterState* cleanup = new IterState(&mutex_, mem_, imm_, versions_->current());
+ internal_iter->RegisterCleanup(CleanupIteratorState, cleanup, nullptr);
*seed = ++seed_;
mutex_.Unlock();
@@ -1085,14 +1106,14 @@ int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes() {
return versions_->MaxNextLevelOverlappingBytes();
}
-Status DBImpl::Get(const ReadOptions& options,
- const Slice& key,
+Status DBImpl::Get(const ReadOptions& options, const Slice& key,
std::string* value) {
Status s;
MutexLock l(&mutex_);
SequenceNumber snapshot;
- if (options.snapshot != NULL) {
- snapshot = reinterpret_cast<const SnapshotImpl*>(options.snapshot)->number_;
+ if (options.snapshot != nullptr) {
+ snapshot =
+ static_cast<const SnapshotImpl*>(options.snapshot)->sequence_number();
} else {
snapshot = versions_->LastSequence();
}
@@ -1101,7 +1122,7 @@ Status DBImpl::Get(const ReadOptions& options,
MemTable* imm = imm_;
Version* current = versions_->current();
mem->Ref();
- if (imm != NULL) imm->Ref();
+ if (imm != nullptr) imm->Ref();
current->Ref();
bool have_stat_update = false;
@@ -1114,7 +1135,7 @@ Status DBImpl::Get(const ReadOptions& options,
LookupKey lkey(key, snapshot);
if (mem->Get(lkey, value, &s)) {
// Done
- } else if (imm != NULL && imm->Get(lkey, value, &s)) {
+ } else if (imm != nullptr && imm->Get(lkey, value, &s)) {
// Done
} else {
s = current->Get(options, lkey, value, &stats);
@@ -1127,7 +1148,7 @@ Status DBImpl::Get(const ReadOptions& options,
MaybeScheduleCompaction();
}
mem->Unref();
- if (imm != NULL) imm->Unref();
+ if (imm != nullptr) imm->Unref();
current->Unref();
return s;
}
@@ -1136,12 +1157,12 @@ Iterator* DBImpl::NewIterator(const ReadOptions& options) {
SequenceNumber latest_snapshot;
uint32_t seed;
Iterator* iter = NewInternalIterator(options, &latest_snapshot, &seed);
- return NewDBIterator(
- this, user_comparator(), iter,
- (options.snapshot != NULL
- ? reinterpret_cast<const SnapshotImpl*>(options.snapshot)->number_
- : latest_snapshot),
- seed);
+ return NewDBIterator(this, user_comparator(), iter,
+ (options.snapshot != nullptr
+ ? static_cast<const SnapshotImpl*>(options.snapshot)
+ ->sequence_number()
+ : latest_snapshot),
+ seed);
}
void DBImpl::RecordReadSample(Slice key) {
@@ -1156,9 +1177,9 @@ const Snapshot* DBImpl::GetSnapshot() {
return snapshots_.New(versions_->LastSequence());
}
-void DBImpl::ReleaseSnapshot(const Snapshot* s) {
+void DBImpl::ReleaseSnapshot(const Snapshot* snapshot) {
MutexLock l(&mutex_);
- snapshots_.Delete(reinterpret_cast<const SnapshotImpl*>(s));
+ snapshots_.Delete(static_cast<const SnapshotImpl*>(snapshot));
}
// Convenience methods
@@ -1170,9 +1191,9 @@ Status DBImpl::Delete(const WriteOptions& options, const Slice& key) {
return DB::Delete(options, key);
}
-Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) {
+Status DBImpl::Write(const WriteOptions& options, WriteBatch* updates) {
Writer w(&mutex_);
- w.batch = my_batch;
+ w.batch = updates;
w.sync = options.sync;
w.done = false;
@@ -1186,10 +1207,10 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) {
}
// May temporarily unlock and wait.
- Status status = MakeRoomForWrite(my_batch == NULL);
+ Status status = MakeRoomForWrite(updates == nullptr);
uint64_t last_sequence = versions_->LastSequence();
Writer* last_writer = &w;
- if (status.ok() && my_batch != NULL) { // NULL batch is for compactions
+ if (status.ok() && updates != nullptr) { // nullptr batch is for compactions
WriteBatch* updates = BuildBatchGroup(&last_writer);
WriteBatchInternal::SetSequence(updates, last_sequence + 1);
last_sequence += WriteBatchInternal::Count(updates);
@@ -1244,12 +1265,13 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) {
}
// REQUIRES: Writer list must be non-empty
-// REQUIRES: First writer must have a non-NULL batch
+// REQUIRES: First writer must have a non-null batch
WriteBatch* DBImpl::BuildBatchGroup(Writer** last_writer) {
+ mutex_.AssertHeld();
assert(!writers_.empty());
Writer* first = writers_.front();
WriteBatch* result = first->batch;
- assert(result != NULL);
+ assert(result != nullptr);
size_t size = WriteBatchInternal::ByteSize(first->batch);
@@ -1257,8 +1279,8 @@ WriteBatch* DBImpl::BuildBatchGroup(Writer** last_writer) {
// original write is small, limit the growth so we do not slow
// down the small write too much.
size_t max_size = 1 << 20;
- if (size <= (128<<10)) {
- max_size = size + (128<<10);
+ if (size <= (128 << 10)) {
+ max_size = size + (128 << 10);
}
*last_writer = first;
@@ -1271,7 +1293,7 @@ WriteBatch* DBImpl::BuildBatchGroup(Writer** last_writer) {
break;
}
- if (w->batch != NULL) {
+ if (w->batch != nullptr) {
size += WriteBatchInternal::ByteSize(w->batch);
if (size > max_size) {
// Do not make batch too big
@@ -1304,9 +1326,8 @@ Status DBImpl::MakeRoomForWrite(bool force) {
// Yield previous error
s = bg_error_;
break;
- } else if (
- allow_delay &&
- versions_->NumLevelFiles(0) >= config::kL0_SlowdownWritesTrigger) {
+ } else if (allow_delay && versions_->NumLevelFiles(0) >=
+ config::kL0_SlowdownWritesTrigger) {
// We are getting close to hitting a hard limit on the number of
// L0 files. Rather than delaying a single write by several
// seconds when we hit the hard limit, start delaying each
@@ -1321,20 +1342,20 @@ Status DBImpl::MakeRoomForWrite(bool force) {
(mem_->ApproximateMemoryUsage() <= options_.write_buffer_size)) {
// There is room in current memtable
break;
- } else if (imm_ != NULL) {
+ } else if (imm_ != nullptr) {
// We have filled up the current memtable, but the previous
// one is still being compacted, so we wait.
Log(options_.info_log, "Current memtable full; waiting...\n");
- bg_cv_.Wait();
+ background_work_finished_signal_.Wait();
} else if (versions_->NumLevelFiles(0) >= config::kL0_StopWritesTrigger) {
// There are too many level-0 files.
Log(options_.info_log, "Too many L0 files; waiting...\n");
- bg_cv_.Wait();
+ background_work_finished_signal_.Wait();
} else {
// Attempt to switch to a new memtable and trigger compaction of old
assert(versions_->PrevLogNumber() == 0);
uint64_t new_log_number = versions_->NewFileNumber();
- WritableFile* lfile = NULL;
+ WritableFile* lfile = nullptr;
s = env_->NewWritableFile(LogFileName(dbname_, new_log_number), &lfile);
if (!s.ok()) {
// Avoid chewing through file number space in a tight loop.
@@ -1347,10 +1368,10 @@ Status DBImpl::MakeRoomForWrite(bool force) {
logfile_number_ = new_log_number;
log_ = new log::Writer(lfile);
imm_ = mem_;
- has_imm_.Release_Store(imm_);
+ has_imm_.store(true, std::memory_order_release);
mem_ = new MemTable(internal_comparator_);
mem_->Ref();
- force = false; // Do not force another compaction if have room
+ force = false; // Do not force another compaction if have room
MaybeScheduleCompaction();
}
}
@@ -1384,21 +1405,16 @@ bool DBImpl::GetProperty(const Slice& property, std::string* value) {
snprintf(buf, sizeof(buf),
" Compactions\n"
"Level Files Size(MB) Time(sec) Read(MB) Write(MB)\n"
- "--------------------------------------------------\n"
- );
+ "--------------------------------------------------\n");
value->append(buf);
for (int level = 0; level < config::kNumLevels; level++) {
int files = versions_->NumLevelFiles(level);
if (stats_[level].micros > 0 || files > 0) {
- snprintf(
- buf, sizeof(buf),
- "%3d %8d %8.0f %9.0f %8.0f %9.0f\n",
- level,
- files,
- versions_->NumLevelBytes(level) / 1048576.0,
- stats_[level].micros / 1e6,
- stats_[level].bytes_read / 1048576.0,
- stats_[level].bytes_written / 1048576.0);
+ snprintf(buf, sizeof(buf), "%3d %8d %8.0f %9.0f %8.0f %9.0f\n", level,
+ files, versions_->NumLevelBytes(level) / 1048576.0,
+ stats_[level].micros / 1e6,
+ stats_[level].bytes_read / 1048576.0,
+ stats_[level].bytes_written / 1048576.0);
value->append(buf);
}
}
@@ -1406,21 +1422,29 @@ bool DBImpl::GetProperty(const Slice& property, std::string* value) {
} else if (in == "sstables") {
*value = versions_->current()->DebugString();
return true;
+ } else if (in == "approximate-memory-usage") {
+ size_t total_usage = options_.block_cache->TotalCharge();
+ if (mem_) {
+ total_usage += mem_->ApproximateMemoryUsage();
+ }
+ if (imm_) {
+ total_usage += imm_->ApproximateMemoryUsage();
+ }
+ char buf[50];
+ snprintf(buf, sizeof(buf), "%llu",
+ static_cast<unsigned long long>(total_usage));
+ value->append(buf);
+ return true;
}
return false;
}
-void DBImpl::GetApproximateSizes(
- const Range* range, int n,
- uint64_t* sizes) {
+void DBImpl::GetApproximateSizes(const Range* range, int n, uint64_t* sizes) {
// TODO(opt): better implementation
- Version* v;
- {
- MutexLock l(&mutex_);
- versions_->current()->Ref();
- v = versions_->current();
- }
+ MutexLock l(&mutex_);
+ Version* v = versions_->current();
+ v->Ref();
for (int i = 0; i < n; i++) {
// Convert user_key into a corresponding internal key.
@@ -1431,10 +1455,7 @@ void DBImpl::GetApproximateSizes(
sizes[i] = (limit >= start ? limit - start : 0);
}
- {
- MutexLock l(&mutex_);
- v->Unref();
- }
+ v->Unref();
}
// Default implementations of convenience methods that subclasses of DB
@@ -1451,17 +1472,19 @@ Status DB::Delete(const WriteOptions& opt, const Slice& key) {
return Write(opt, &batch);
}
-DB::~DB() { }
+DB::~DB() = default;
-Status DB::Open(const Options& options, const std::string& dbname,
- DB** dbptr) {
- *dbptr = NULL;
+Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) {
+ *dbptr = nullptr;
DBImpl* impl = new DBImpl(options, dbname);
impl->mutex_.Lock();
VersionEdit edit;
- Status s = impl->Recover(&edit); // Handles create_if_missing, error_if_exists
- if (s.ok()) {
+ // Recover handles create_if_missing, error_if_exists
+ bool save_manifest = false;
+ Status s = impl->Recover(&edit, &save_manifest);
+ if (s.ok() && impl->mem_ == nullptr) {
+ // Create new log and a corresponding memtable.
uint64_t new_log_number = impl->versions_->NewFileNumber();
WritableFile* lfile;
s = options.env->NewWritableFile(LogFileName(dbname, new_log_number),
@@ -1471,15 +1494,22 @@ Status DB::Open(const Options& options, const std::string& dbname,
impl->logfile_ = lfile;
impl->logfile_number_ = new_log_number;
impl->log_ = new log::Writer(lfile);
- s = impl->versions_->LogAndApply(&edit, &impl->mutex_);
- }
- if (s.ok()) {
- impl->DeleteObsoleteFiles();
- impl->MaybeScheduleCompaction();
+ impl->mem_ = new MemTable(impl->internal_comparator_);
+ impl->mem_->Ref();
}
}
+ if (s.ok() && save_manifest) {
+ edit.SetPrevLogNumber(0); // No older logs needed after recovery.
+ edit.SetLogNumber(impl->logfile_number_);
+ s = impl->versions_->LogAndApply(&edit, &impl->mutex_);
+ }
+ if (s.ok()) {
+ impl->DeleteObsoleteFiles();
+ impl->MaybeScheduleCompaction();
+ }
impl->mutex_.Unlock();
if (s.ok()) {
+ assert(impl->mem_ != nullptr);
*dbptr = impl;
} else {
delete impl;
@@ -1487,21 +1517,20 @@ Status DB::Open(const Options& options, const std::string& dbname,
return s;
}
-Snapshot::~Snapshot() {
-}
+Snapshot::~Snapshot() = default;
Status DestroyDB(const std::string& dbname, const Options& options) {
Env* env = options.env;
std::vector<std::string> filenames;
- // Ignore error in case directory does not exist
- env->GetChildren(dbname, &filenames);
- if (filenames.empty()) {
+ Status result = env->GetChildren(dbname, &filenames);
+ if (!result.ok()) {
+ // Ignore error in case directory does not exist
return Status::OK();
}
FileLock* lock;
const std::string lockname = LockFileName(dbname);
- Status result = env->LockFile(lockname, &lock);
+ result = env->LockFile(lockname, &lock);
if (result.ok()) {
uint64_t number;
FileType type;
diff --git a/db/db_impl.h b/db/db_impl.h
index cfc9981..685735c 100644
--- a/db/db_impl.h
+++ b/db/db_impl.h
@@ -5,8 +5,11 @@
#ifndef STORAGE_LEVELDB_DB_DB_IMPL_H_
#define STORAGE_LEVELDB_DB_DB_IMPL_H_
+#include <atomic>
#include <deque>
#include <set>
+#include <string>
+
#include "db/dbformat.h"
#include "db/log_writer.h"
#include "db/snapshot.h"
@@ -26,21 +29,25 @@ class VersionSet;
class DBImpl : public DB {
public:
DBImpl(const Options& options, const std::string& dbname);
- virtual ~DBImpl();
+
+ DBImpl(const DBImpl&) = delete;
+ DBImpl& operator=(const DBImpl&) = delete;
+
+ ~DBImpl() override;
// Implementations of the DB interface
- virtual Status Put(const WriteOptions&, const Slice& key, const Slice& value);
- virtual Status Delete(const WriteOptions&, const Slice& key);
- virtual Status Write(const WriteOptions& options, WriteBatch* updates);
- virtual Status Get(const ReadOptions& options,
- const Slice& key,
- std::string* value);
- virtual Iterator* NewIterator(const ReadOptions&);
- virtual const Snapshot* GetSnapshot();
- virtual void ReleaseSnapshot(const Snapshot* snapshot);
- virtual bool GetProperty(const Slice& property, std::string* value);
- virtual void GetApproximateSizes(const Range* range, int n, uint64_t* sizes);
- virtual void CompactRange(const Slice* begin, const Slice* end);
+ Status Put(const WriteOptions&, const Slice& key,
+ const Slice& value) override;
+ Status Delete(const WriteOptions&, const Slice& key) override;
+ Status Write(const WriteOptions& options, WriteBatch* updates) override;
+ Status Get(const ReadOptions& options, const Slice& key,
+ std::string* value) override;
+ Iterator* NewIterator(const ReadOptions&) override;
+ const Snapshot* GetSnapshot() override;
+ void ReleaseSnapshot(const Snapshot* snapshot) override;
+ bool GetProperty(const Slice& property, std::string* value) override;
+ void GetApproximateSizes(const Range* range, int n, uint64_t* sizes) override;
+ void CompactRange(const Slice* begin, const Slice* end) override;
// Extra methods (for testing) that are not in the public DB interface
@@ -69,6 +76,31 @@ class DBImpl : public DB {
struct CompactionState;
struct Writer;
+ // Information for a manual compaction
+ struct ManualCompaction {
+ int level;
+ bool done;
+ const InternalKey* begin; // null means beginning of key range
+ const InternalKey* end; // null means end of key range
+ InternalKey tmp_storage; // Used to keep track of compaction progress
+ };
+
+ // Per level compaction stats. stats_[level] stores the stats for
+ // compactions that produced data for the specified "level".
+ struct CompactionStats {
+ CompactionStats() : micros(0), bytes_read(0), bytes_written(0) {}
+
+ void Add(const CompactionStats& c) {
+ this->micros += c.micros;
+ this->bytes_read += c.bytes_read;
+ this->bytes_written += c.bytes_written;
+ }
+
+ int64_t micros;
+ int64_t bytes_read;
+ int64_t bytes_written;
+ };
+
Iterator* NewInternalIterator(const ReadOptions&,
SequenceNumber* latest_snapshot,
uint32_t* seed);
@@ -78,21 +110,21 @@ class DBImpl : public DB {
// Recover the descriptor from persistent storage. May do a significant
// amount of work to recover recently logged updates. Any changes to
// be made to the descriptor are added to *edit.
- Status Recover(VersionEdit* edit) EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+ Status Recover(VersionEdit* edit, bool* save_manifest)
+ EXCLUSIVE_LOCKS_REQUIRED(mutex_);
void MaybeIgnoreError(Status* s) const;
// Delete any unneeded files and stale in-memory entries.
- void DeleteObsoleteFiles();
+ void DeleteObsoleteFiles() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
// Compact the in-memory write buffer to disk. Switches to a new
// log-file/memtable and writes a new descriptor iff successful.
// Errors are recorded in bg_error_.
void CompactMemTable() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
- Status RecoverLogFile(uint64_t log_number,
- VersionEdit* edit,
- SequenceNumber* max_sequence)
+ Status RecoverLogFile(uint64_t log_number, bool last_log, bool* save_manifest,
+ VersionEdit* edit, SequenceNumber* max_sequence)
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
Status WriteLevel0Table(MemTable* mem, VersionEdit* edit, Version* base)
@@ -100,14 +132,15 @@ class DBImpl : public DB {
Status MakeRoomForWrite(bool force /* compact even if there is room? */)
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
- WriteBatch* BuildBatchGroup(Writer** last_writer);
+ WriteBatch* BuildBatchGroup(Writer** last_writer)
+ EXCLUSIVE_LOCKS_REQUIRED(mutex_);
void RecordBackgroundError(const Status& s);
void MaybeScheduleCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
static void BGWork(void* db);
void BackgroundCall();
- void BackgroundCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+ void BackgroundCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
void CleanupCompaction(CompactionState* compact)
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
Status DoCompactionWork(CompactionState* compact)
@@ -118,93 +151,66 @@ class DBImpl : public DB {
Status InstallCompactionResults(CompactionState* compact)
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+ const Comparator* user_comparator() const {
+ return internal_comparator_.user_comparator();
+ }
+
// Constant after construction
Env* const env_;
const InternalKeyComparator internal_comparator_;
const InternalFilterPolicy internal_filter_policy_;
const Options options_; // options_.comparator == &internal_comparator_
- bool owns_info_log_;
- bool owns_cache_;
+ const bool owns_info_log_;
+ const bool owns_cache_;
const std::string dbname_;
// table_cache_ provides its own synchronization
- TableCache* table_cache_;
+ TableCache* const table_cache_;
- // Lock over the persistent DB state. Non-NULL iff successfully acquired.
+ // Lock over the persistent DB state. Non-null iff successfully acquired.
FileLock* db_lock_;
// State below is protected by mutex_
port::Mutex mutex_;
- port::AtomicPointer shutting_down_;
- port::CondVar bg_cv_; // Signalled when background work finishes
+ std::atomic<bool> shutting_down_;
+ port::CondVar background_work_finished_signal_ GUARDED_BY(mutex_);
MemTable* mem_;
- MemTable* imm_; // Memtable being compacted
- port::AtomicPointer has_imm_; // So bg thread can detect non-NULL imm_
+ MemTable* imm_ GUARDED_BY(mutex_); // Memtable being compacted
+ std::atomic<bool> has_imm_; // So bg thread can detect non-null imm_
WritableFile* logfile_;
- uint64_t logfile_number_;
+ uint64_t logfile_number_ GUARDED_BY(mutex_);
log::Writer* log_;
- uint32_t seed_; // For sampling.
+ uint32_t seed_ GUARDED_BY(mutex_); // For sampling.
// Queue of writers.
- std::deque<Writer*> writers_;
- WriteBatch* tmp_batch_;
+ std::deque<Writer*> writers_ GUARDED_BY(mutex_);
+ WriteBatch* tmp_batch_ GUARDED_BY(mutex_);
- SnapshotList snapshots_;
+ SnapshotList snapshots_ GUARDED_BY(mutex_);
// Set of table files to protect from deletion because they are
// part of ongoing compactions.
- std::set<uint64_t> pending_outputs_;
+ std::set<uint64_t> pending_outputs_ GUARDED_BY(mutex_);
// Has a background compaction been scheduled or is running?
- bool bg_compaction_scheduled_;
+ bool background_compaction_scheduled_ GUARDED_BY(mutex_);
- // Information for a manual compaction
- struct ManualCompaction {
- int level;
- bool done;
- const InternalKey* begin; // NULL means beginning of key range
- const InternalKey* end; // NULL means end of key range
- InternalKey tmp_storage; // Used to keep track of compaction progress
- };
- ManualCompaction* manual_compaction_;
+ ManualCompaction* manual_compaction_ GUARDED_BY(mutex_);
- VersionSet* versions_;
+ VersionSet* const versions_ GUARDED_BY(mutex_);
// Have we encountered a background error in paranoid mode?
- Status bg_error_;
+ Status bg_error_ GUARDED_BY(mutex_);
- // Per level compaction stats. stats_[level] stores the stats for
- // compactions that produced data for the specified "level".
- struct CompactionStats {
- int64_t micros;
- int64_t bytes_read;
- int64_t bytes_written;
-
- CompactionStats() : micros(0), bytes_read(0), bytes_written(0) { }
-
- void Add(const CompactionStats& c) {
- this->micros += c.micros;
- this->bytes_read += c.bytes_read;
- this->bytes_written += c.bytes_written;
- }
- };
- CompactionStats stats_[config::kNumLevels];
-
- // No copying allowed
- DBImpl(const DBImpl&);
- void operator=(const DBImpl&);
-
- const Comparator* user_comparator() const {
- return internal_comparator_.user_comparator();
- }
+ CompactionStats stats_[config::kNumLevels] GUARDED_BY(mutex_);
};
// Sanitize db options. The caller should delete result.info_log if
// it is not equal to src.info_log.
-extern Options SanitizeOptions(const std::string& db,
- const InternalKeyComparator* icmp,
- const InternalFilterPolicy* ipolicy,
- const Options& src);
+Options SanitizeOptions(const std::string& db,
+ const InternalKeyComparator* icmp,
+ const InternalFilterPolicy* ipolicy,
+ const Options& src);
} // namespace leveldb
diff --git a/db/db_iter.cc b/db/db_iter.cc
index 3b2035e..98715a9 100644
--- a/db/db_iter.cc
+++ b/db/db_iter.cc
@@ -4,9 +4,9 @@
#include "db/db_iter.h"
-#include "db/filename.h"
#include "db/db_impl.h"
#include "db/dbformat.h"
+#include "db/filename.h"
#include "leveldb/env.h"
#include "leveldb/iterator.h"
#include "port/port.h"
@@ -36,17 +36,14 @@ namespace {
// combines multiple entries for the same userkey found in the DB
// representation into a single entry while accounting for sequence
// numbers, deletion markers, overwrites, etc.
-class DBIter: public Iterator {
+class DBIter : public Iterator {
public:
// Which direction is the iterator currently moving?
// (1) When moving forward, the internal iterator is positioned at
// the exact entry that yields this->key(), this->value()
// (2) When moving backwards, the internal iterator is positioned
// just before all entries whose user key == this->key().
- enum Direction {
- kForward,
- kReverse
- };
+ enum Direction { kForward, kReverse };
DBIter(DBImpl* db, const Comparator* cmp, Iterator* iter, SequenceNumber s,
uint32_t seed)
@@ -57,21 +54,22 @@ class DBIter: public Iterator {
direction_(kForward),
valid_(false),
rnd_(seed),
- bytes_counter_(RandomPeriod()) {
- }
- virtual ~DBIter() {
- delete iter_;
- }
- virtual bool Valid() const { return valid_; }
- virtual Slice key() const {
+ bytes_until_read_sampling_(RandomCompactionPeriod()) {}
+
+ DBIter(const DBIter&) = delete;
+ DBIter& operator=(const DBIter&) = delete;
+
+ ~DBIter() override { delete iter_; }
+ bool Valid() const override { return valid_; }
+ Slice key() const override {
assert(valid_);
return (direction_ == kForward) ? ExtractUserKey(iter_->key()) : saved_key_;
}
- virtual Slice value() const {
+ Slice value() const override {
assert(valid_);
return (direction_ == kForward) ? iter_->value() : saved_value_;
}
- virtual Status status() const {
+ Status status() const override {
if (status_.ok()) {
return iter_->status();
} else {
@@ -79,11 +77,11 @@ class DBIter: public Iterator {
}
}
- virtual void Next();
- virtual void Prev();
- virtual void Seek(const Slice& target);
- virtual void SeekToFirst();
- virtual void SeekToLast();
+ void Next() override;
+ void Prev() override;
+ void Seek(const Slice& target) override;
+ void SeekToFirst() override;
+ void SeekToLast() override;
private:
void FindNextUserEntry(bool skipping, std::string* skip);
@@ -103,38 +101,35 @@ class DBIter: public Iterator {
}
}
- // Pick next gap with average value of config::kReadBytesPeriod.
- ssize_t RandomPeriod() {
- return rnd_.Uniform(2*config::kReadBytesPeriod);
+ // Picks the number of bytes that can be read until a compaction is scheduled.
+ size_t RandomCompactionPeriod() {
+ return rnd_.Uniform(2 * config::kReadBytesPeriod);
}
DBImpl* db_;
const Comparator* const user_comparator_;
Iterator* const iter_;
SequenceNumber const sequence_;
-
Status status_;
- std::string saved_key_; // == current key when direction_==kReverse
- std::string saved_value_; // == current raw value when direction_==kReverse
+ std::string saved_key_; // == current key when direction_==kReverse
+ std::string saved_value_; // == current raw value when direction_==kReverse
Direction direction_;
bool valid_;
-
Random rnd_;
- ssize_t bytes_counter_;
-
- // No copying allowed
- DBIter(const DBIter&);
- void operator=(const DBIter&);
+ size_t bytes_until_read_sampling_;
};
inline bool DBIter::ParseKey(ParsedInternalKey* ikey) {
Slice k = iter_->key();
- ssize_t n = k.size() + iter_->value().size();
- bytes_counter_ -= n;
- while (bytes_counter_ < 0) {
- bytes_counter_ += RandomPeriod();
+
+ size_t bytes_read = k.size() + iter_->value().size();
+ while (bytes_until_read_sampling_ < bytes_read) {
+ bytes_until_read_sampling_ += RandomCompactionPeriod();
db_->RecordReadSample(k);
}
+ assert(bytes_until_read_sampling_ >= bytes_read);
+ bytes_until_read_sampling_ -= bytes_read;
+
if (!ParseInternalKey(k, ikey)) {
status_ = Status::Corruption("corrupted internal key in DBIter");
return false;
@@ -165,6 +160,15 @@ void DBIter::Next() {
} else {
// Store in saved_key_ the current key so we skip it below.
SaveKey(ExtractUserKey(iter_->key()), &saved_key_);
+
+ // iter_ is pointing to current key. We can now safely move to the next to
+ // avoid checking current key.
+ iter_->Next();
+ if (!iter_->Valid()) {
+ valid_ = false;
+ saved_key_.clear();
+ return;
+ }
}
FindNextUserEntry(true, &saved_key_);
@@ -218,8 +222,8 @@ void DBIter::Prev() {
ClearSavedValue();
return;
}
- if (user_comparator_->Compare(ExtractUserKey(iter_->key()),
- saved_key_) < 0) {
+ if (user_comparator_->Compare(ExtractUserKey(iter_->key()), saved_key_) <
+ 0) {
break;
}
}
@@ -275,8 +279,8 @@ void DBIter::Seek(const Slice& target) {
direction_ = kForward;
ClearSavedValue();
saved_key_.clear();
- AppendInternalKey(
- &saved_key_, ParsedInternalKey(target, sequence_, kValueTypeForSeek));
+ AppendInternalKey(&saved_key_,
+ ParsedInternalKey(target, sequence_, kValueTypeForSeek));
iter_->Seek(saved_key_);
if (iter_->Valid()) {
FindNextUserEntry(false, &saved_key_ /* temporary storage */);
@@ -305,12 +309,9 @@ void DBIter::SeekToLast() {
} // anonymous namespace
-Iterator* NewDBIterator(
- DBImpl* db,
- const Comparator* user_key_comparator,
- Iterator* internal_iter,
- SequenceNumber sequence,
- uint32_t seed) {
+Iterator* NewDBIterator(DBImpl* db, const Comparator* user_key_comparator,
+ Iterator* internal_iter, SequenceNumber sequence,
+ uint32_t seed) {
return new DBIter(db, user_key_comparator, internal_iter, sequence, seed);
}
diff --git a/db/db_iter.h b/db/db_iter.h
index 04927e9..fd93e91 100644
--- a/db/db_iter.h
+++ b/db/db_iter.h
@@ -6,8 +6,9 @@
#define STORAGE_LEVELDB_DB_DB_ITER_H_
#include <stdint.h>
-#include "leveldb/db.h"
+
#include "db/dbformat.h"
+#include "leveldb/db.h"
namespace leveldb {
@@ -16,12 +17,9 @@ class DBImpl;
// Return a new iterator that converts internal keys (yielded by
// "*internal_iter") that were live at the specified "sequence" number
// into appropriate user keys.
-extern Iterator* NewDBIterator(
- DBImpl* db,
- const Comparator* user_key_comparator,
- Iterator* internal_iter,
- SequenceNumber sequence,
- uint32_t seed);
+Iterator* NewDBIterator(DBImpl* db, const Comparator* user_key_comparator,
+ Iterator* internal_iter, SequenceNumber sequence,
+ uint32_t seed);
} // namespace leveldb
diff --git a/db/db_test.cc b/db/db_test.cc
index 0fed913..9a8faf1 100644
--- a/db/db_test.cc
+++ b/db/db_test.cc
@@ -3,14 +3,20 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "leveldb/db.h"
-#include "leveldb/filter_policy.h"
+
+#include <atomic>
+#include <string>
+
#include "db/db_impl.h"
#include "db/filename.h"
#include "db/version_set.h"
#include "db/write_batch_internal.h"
#include "leveldb/cache.h"
#include "leveldb/env.h"
+#include "leveldb/filter_policy.h"
#include "leveldb/table.h"
+#include "port/port.h"
+#include "port/thread_annotations.h"
#include "util/hash.h"
#include "util/logging.h"
#include "util/mutexlock.h"
@@ -25,83 +31,116 @@ static std::string RandomString(Random* rnd, int len) {
return r;
}
+static std::string RandomKey(Random* rnd) {
+ int len =
+ (rnd->OneIn(3) ? 1 // Short sometimes to encourage collisions
+ : (rnd->OneIn(100) ? rnd->Skewed(10) : rnd->Uniform(10)));
+ return test::RandomKey(rnd, len);
+}
+
namespace {
class AtomicCounter {
- private:
- port::Mutex mu_;
- int count_;
public:
- AtomicCounter() : count_(0) { }
- void Increment() {
- IncrementBy(1);
- }
- void IncrementBy(int count) {
+ AtomicCounter() : count_(0) {}
+ void Increment() { IncrementBy(1); }
+ void IncrementBy(int count) LOCKS_EXCLUDED(mu_) {
MutexLock l(&mu_);
count_ += count;
}
- int Read() {
+ int Read() LOCKS_EXCLUDED(mu_) {
MutexLock l(&mu_);
return count_;
}
- void Reset() {
+ void Reset() LOCKS_EXCLUDED(mu_) {
MutexLock l(&mu_);
count_ = 0;
}
+
+ private:
+ port::Mutex mu_;
+ int count_ GUARDED_BY(mu_);
};
void DelayMilliseconds(int millis) {
Env::Default()->SleepForMicroseconds(millis * 1000);
}
-}
+} // namespace
-// Special Env used to delay background operations
+// Test Env to override default Env behavior for testing.
+class TestEnv : public EnvWrapper {
+ public:
+ explicit TestEnv(Env* base) : EnvWrapper(base), ignore_dot_files_(false) {}
+
+ void SetIgnoreDotFiles(bool ignored) { ignore_dot_files_ = ignored; }
+
+ Status GetChildren(const std::string& dir,
+ std::vector<std::string>* result) override {
+ Status s = target()->GetChildren(dir, result);
+ if (!s.ok() || !ignore_dot_files_) {
+ return s;
+ }
+
+ std::vector<std::string>::iterator it = result->begin();
+ while (it != result->end()) {
+ if ((*it == ".") || (*it == "..")) {
+ it = result->erase(it);
+ } else {
+ ++it;
+ }
+ }
+
+ return s;
+ }
+
+ private:
+ bool ignore_dot_files_;
+};
+
+// Special Env used to delay background operations.
class SpecialEnv : public EnvWrapper {
public:
- // sstable/log Sync() calls are blocked while this pointer is non-NULL.
- port::AtomicPointer delay_data_sync_;
+ // sstable/log Sync() calls are blocked while this pointer is non-null.
+ std::atomic<bool> delay_data_sync_;
// sstable/log Sync() calls return an error.
- port::AtomicPointer data_sync_error_;
+ std::atomic<bool> data_sync_error_;
- // Simulate no-space errors while this pointer is non-NULL.
- port::AtomicPointer no_space_;
+ // Simulate no-space errors while this pointer is non-null.
+ std::atomic<bool> no_space_;
- // Simulate non-writable file system while this pointer is non-NULL
- port::AtomicPointer non_writable_;
+ // Simulate non-writable file system while this pointer is non-null.
+ std::atomic<bool> non_writable_;
- // Force sync of manifest files to fail while this pointer is non-NULL
- port::AtomicPointer manifest_sync_error_;
+ // Force sync of manifest files to fail while this pointer is non-null.
+ std::atomic<bool> manifest_sync_error_;
- // Force write to manifest files to fail while this pointer is non-NULL
- port::AtomicPointer manifest_write_error_;
+ // Force write to manifest files to fail while this pointer is non-null.
+ std::atomic<bool> manifest_write_error_;
bool count_random_reads_;
AtomicCounter random_read_counter_;
- explicit SpecialEnv(Env* base) : EnvWrapper(base) {
- delay_data_sync_.Release_Store(NULL);
- data_sync_error_.Release_Store(NULL);
- no_space_.Release_Store(NULL);
- non_writable_.Release_Store(NULL);
- count_random_reads_ = false;
- manifest_sync_error_.Release_Store(NULL);
- manifest_write_error_.Release_Store(NULL);
- }
+ explicit SpecialEnv(Env* base)
+ : EnvWrapper(base),
+ delay_data_sync_(false),
+ data_sync_error_(false),
+ no_space_(false),
+ non_writable_(false),
+ manifest_sync_error_(false),
+ manifest_write_error_(false),
+ count_random_reads_(false) {}
Status NewWritableFile(const std::string& f, WritableFile** r) {
class DataFile : public WritableFile {
private:
- SpecialEnv* env_;
- WritableFile* base_;
+ SpecialEnv* const env_;
+ WritableFile* const base_;
public:
- DataFile(SpecialEnv* env, WritableFile* base)
- : env_(env),
- base_(base) {
- }
+ DataFile(SpecialEnv* env, WritableFile* base) : env_(env), base_(base) {}
~DataFile() { delete base_; }
Status Append(const Slice& data) {
- if (env_->no_space_.Acquire_Load() != NULL) {
+ if (env_->no_space_.load(std::memory_order_acquire)) {
// Drop writes on the floor
return Status::OK();
} else {
@@ -111,10 +150,10 @@ class SpecialEnv : public EnvWrapper {
Status Close() { return base_->Close(); }
Status Flush() { return base_->Flush(); }
Status Sync() {
- if (env_->data_sync_error_.Acquire_Load() != NULL) {
+ if (env_->data_sync_error_.load(std::memory_order_acquire)) {
return Status::IOError("simulated data sync error");
}
- while (env_->delay_data_sync_.Acquire_Load() != NULL) {
+ while (env_->delay_data_sync_.load(std::memory_order_acquire)) {
DelayMilliseconds(100);
}
return base_->Sync();
@@ -124,11 +163,12 @@ class SpecialEnv : public EnvWrapper {
private:
SpecialEnv* env_;
WritableFile* base_;
+
public:
- ManifestFile(SpecialEnv* env, WritableFile* b) : env_(env), base_(b) { }
+ ManifestFile(SpecialEnv* env, WritableFile* b) : env_(env), base_(b) {}
~ManifestFile() { delete base_; }
Status Append(const Slice& data) {
- if (env_->manifest_write_error_.Acquire_Load() != NULL) {
+ if (env_->manifest_write_error_.load(std::memory_order_acquire)) {
return Status::IOError("simulated writer error");
} else {
return base_->Append(data);
@@ -137,7 +177,7 @@ class SpecialEnv : public EnvWrapper {
Status Close() { return base_->Close(); }
Status Flush() { return base_->Flush(); }
Status Sync() {
- if (env_->manifest_sync_error_.Acquire_Load() != NULL) {
+ if (env_->manifest_sync_error_.load(std::memory_order_acquire)) {
return Status::IOError("simulated sync error");
} else {
return base_->Sync();
@@ -145,16 +185,16 @@ class SpecialEnv : public EnvWrapper {
}
};
- if (non_writable_.Acquire_Load() != NULL) {
+ if (non_writable_.load(std::memory_order_acquire)) {
return Status::IOError("simulated write error");
}
Status s = target()->NewWritableFile(f, r);
if (s.ok()) {
- if (strstr(f.c_str(), ".ldb") != NULL ||
- strstr(f.c_str(), ".log") != NULL) {
+ if (strstr(f.c_str(), ".ldb") != nullptr ||
+ strstr(f.c_str(), ".log") != nullptr) {
*r = new DataFile(this, *r);
- } else if (strstr(f.c_str(), "MANIFEST") != NULL) {
+ } else if (strstr(f.c_str(), "MANIFEST") != nullptr) {
*r = new ManifestFile(this, *r);
}
}
@@ -166,13 +206,13 @@ class SpecialEnv : public EnvWrapper {
private:
RandomAccessFile* target_;
AtomicCounter* counter_;
+
public:
CountingFile(RandomAccessFile* target, AtomicCounter* counter)
- : target_(target), counter_(counter) {
- }
- virtual ~CountingFile() { delete target_; }
- virtual Status Read(uint64_t offset, size_t n, Slice* result,
- char* scratch) const {
+ : target_(target), counter_(counter) {}
+ ~CountingFile() override { delete target_; }
+ Status Read(uint64_t offset, size_t n, Slice* result,
+ char* scratch) const override {
counter_->Increment();
return target_->Read(offset, n, result, scratch);
}
@@ -187,18 +227,6 @@ class SpecialEnv : public EnvWrapper {
};
class DBTest {
- private:
- const FilterPolicy* filter_policy_;
-
- // Sequence of option configurations to try
- enum OptionConfig {
- kDefault,
- kFilter,
- kUncompressed,
- kEnd
- };
- int option_config_;
-
public:
std::string dbname_;
SpecialEnv* env_;
@@ -206,12 +234,11 @@ class DBTest {
Options last_options_;
- DBTest() : option_config_(kDefault),
- env_(new SpecialEnv(Env::Default())) {
+ DBTest() : env_(new SpecialEnv(Env::Default())), option_config_(kDefault) {
filter_policy_ = NewBloomFilterPolicy(10);
dbname_ = test::TmpDir() + "/db_test";
DestroyDB(dbname_, Options());
- db_ = NULL;
+ db_ = nullptr;
Reopen();
}
@@ -237,7 +264,11 @@ class DBTest {
// Return the current option configuration.
Options CurrentOptions() {
Options options;
+ options.reuse_logs = false;
switch (option_config_) {
+ case kReuse:
+ options.reuse_logs = true;
+ break;
case kFilter:
options.filter_policy = filter_policy_;
break;
@@ -250,31 +281,27 @@ class DBTest {
return options;
}
- DBImpl* dbfull() {
- return reinterpret_cast<DBImpl*>(db_);
- }
+ DBImpl* dbfull() { return reinterpret_cast<DBImpl*>(db_); }
- void Reopen(Options* options = NULL) {
- ASSERT_OK(TryReopen(options));
- }
+ void Reopen(Options* options = nullptr) { ASSERT_OK(TryReopen(options)); }
void Close() {
delete db_;
- db_ = NULL;
+ db_ = nullptr;
}
- void DestroyAndReopen(Options* options = NULL) {
+ void DestroyAndReopen(Options* options = nullptr) {
delete db_;
- db_ = NULL;
+ db_ = nullptr;
DestroyDB(dbname_, Options());
ASSERT_OK(TryReopen(options));
}
Status TryReopen(Options* options) {
delete db_;
- db_ = NULL;
+ db_ = nullptr;
Options opts;
- if (options != NULL) {
+ if (options != nullptr) {
opts = *options;
} else {
opts = CurrentOptions();
@@ -289,11 +316,9 @@ class DBTest {
return db_->Put(WriteOptions(), k, v);
}
- Status Delete(const std::string& k) {
- return db_->Delete(WriteOptions(), k);
- }
+ Status Delete(const std::string& k) { return db_->Delete(WriteOptions(), k); }
- std::string Get(const std::string& k, const Snapshot* snapshot = NULL) {
+ std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) {
ReadOptions options;
options.snapshot = snapshot;
std::string result;
@@ -377,10 +402,9 @@ class DBTest {
int NumTableFilesAtLevel(int level) {
std::string property;
- ASSERT_TRUE(
- db_->GetProperty("leveldb.num-files-at-level" + NumberToString(level),
- &property));
- return atoi(property.c_str());
+ ASSERT_TRUE(db_->GetProperty(
+ "leveldb.num-files-at-level" + NumberToString(level), &property));
+ return std::stoi(property);
}
int TotalTableFiles() {
@@ -426,11 +450,12 @@ class DBTest {
}
// Do n memtable compactions, each of which produces an sstable
- // covering the range [small,large].
- void MakeTables(int n, const std::string& small, const std::string& large) {
+ // covering the range [small_key,large_key].
+ void MakeTables(int n, const std::string& small_key,
+ const std::string& large_key) {
for (int i = 0; i < n; i++) {
- Put(small, "begin");
- Put(large, "end");
+ Put(small_key, "begin");
+ Put(large_key, "end");
dbfull()->TEST_CompactMemTable();
}
}
@@ -443,9 +468,9 @@ class DBTest {
void DumpFileCounts(const char* label) {
fprintf(stderr, "---\n%s:\n", label);
- fprintf(stderr, "maxoverlap: %lld\n",
- static_cast<long long>(
- dbfull()->TEST_MaxNextLevelOverlappingBytes()));
+ fprintf(
+ stderr, "maxoverlap: %lld\n",
+ static_cast<long long>(dbfull()->TEST_MaxNextLevelOverlappingBytes()));
for (int level = 0; level < config::kNumLevels; level++) {
int num = NumTableFilesAtLevel(level);
if (num > 0) {
@@ -501,15 +526,42 @@ class DBTest {
}
return files_renamed;
}
+
+ private:
+ // Sequence of option configurations to try
+ enum OptionConfig { kDefault, kReuse, kFilter, kUncompressed, kEnd };
+
+ const FilterPolicy* filter_policy_;
+ int option_config_;
};
TEST(DBTest, Empty) {
do {
- ASSERT_TRUE(db_ != NULL);
+ ASSERT_TRUE(db_ != nullptr);
ASSERT_EQ("NOT_FOUND", Get("foo"));
} while (ChangeOptions());
}
+TEST(DBTest, EmptyKey) {
+ do {
+ ASSERT_OK(Put("", "v1"));
+ ASSERT_EQ("v1", Get(""));
+ ASSERT_OK(Put("", "v2"));
+ ASSERT_EQ("v2", Get(""));
+ } while (ChangeOptions());
+}
+
+TEST(DBTest, EmptyValue) {
+ do {
+ ASSERT_OK(Put("key", "v1"));
+ ASSERT_EQ("v1", Get("key"));
+ ASSERT_OK(Put("key", ""));
+ ASSERT_EQ("", Get("key"));
+ ASSERT_OK(Put("key", "v2"));
+ ASSERT_EQ("v2", Get("key"));
+ } while (ChangeOptions());
+}
+
TEST(DBTest, ReadWrite) {
do {
ASSERT_OK(Put("foo", "v1"));
@@ -542,11 +594,13 @@ TEST(DBTest, GetFromImmutableLayer) {
ASSERT_OK(Put("foo", "v1"));
ASSERT_EQ("v1", Get("foo"));
- env_->delay_data_sync_.Release_Store(env_); // Block sync calls
- Put("k1", std::string(100000, 'x')); // Fill memtable
- Put("k2", std::string(100000, 'y')); // Trigger compaction
+ // Block sync calls.
+ env_->delay_data_sync_.store(true, std::memory_order_release);
+ Put("k1", std::string(100000, 'x')); // Fill memtable.
+ Put("k2", std::string(100000, 'y')); // Trigger compaction.
ASSERT_EQ("v1", Get("foo"));
- env_->delay_data_sync_.Release_Store(NULL); // Release sync calls
+ // Release sync calls.
+ env_->delay_data_sync_.store(false, std::memory_order_release);
} while (ChangeOptions());
}
@@ -558,6 +612,17 @@ TEST(DBTest, GetFromVersions) {
} while (ChangeOptions());
}
+TEST(DBTest, GetMemUsage) {
+ do {
+ ASSERT_OK(Put("foo", "v1"));
+ std::string val;
+ ASSERT_TRUE(db_->GetProperty("leveldb.approximate-memory-usage", &val));
+ int mem_usage = std::stoi(val);
+ ASSERT_GT(mem_usage, 0);
+ ASSERT_LT(mem_usage, 5 * 1024 * 1024);
+ } while (ChangeOptions());
+}
+
TEST(DBTest, GetSnapshot) {
do {
// Try with both a short key and a long key
@@ -576,6 +641,55 @@ TEST(DBTest, GetSnapshot) {
} while (ChangeOptions());
}
+TEST(DBTest, GetIdenticalSnapshots) {
+ do {
+ // Try with both a short key and a long key
+ for (int i = 0; i < 2; i++) {
+ std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x');
+ ASSERT_OK(Put(key, "v1"));
+ const Snapshot* s1 = db_->GetSnapshot();
+ const Snapshot* s2 = db_->GetSnapshot();
+ const Snapshot* s3 = db_->GetSnapshot();
+ ASSERT_OK(Put(key, "v2"));
+ ASSERT_EQ("v2", Get(key));
+ ASSERT_EQ("v1", Get(key, s1));
+ ASSERT_EQ("v1", Get(key, s2));
+ ASSERT_EQ("v1", Get(key, s3));
+ db_->ReleaseSnapshot(s1);
+ dbfull()->TEST_CompactMemTable();
+ ASSERT_EQ("v2", Get(key));
+ ASSERT_EQ("v1", Get(key, s2));
+ db_->ReleaseSnapshot(s2);
+ ASSERT_EQ("v1", Get(key, s3));
+ db_->ReleaseSnapshot(s3);
+ }
+ } while (ChangeOptions());
+}
+
+TEST(DBTest, IterateOverEmptySnapshot) {
+ do {
+ const Snapshot* snapshot = db_->GetSnapshot();
+ ReadOptions read_options;
+ read_options.snapshot = snapshot;
+ ASSERT_OK(Put("foo", "v1"));
+ ASSERT_OK(Put("foo", "v2"));
+
+ Iterator* iterator1 = db_->NewIterator(read_options);
+ iterator1->SeekToFirst();
+ ASSERT_TRUE(!iterator1->Valid());
+ delete iterator1;
+
+ dbfull()->TEST_CompactMemTable();
+
+ Iterator* iterator2 = db_->NewIterator(read_options);
+ iterator2->SeekToFirst();
+ ASSERT_TRUE(!iterator2->Valid());
+ delete iterator2;
+
+ db_->ReleaseSnapshot(snapshot);
+ } while (ChangeOptions());
+}
+
TEST(DBTest, GetLevel0Ordering) {
do {
// Check that we process level-0 files in correct order. The code
@@ -630,8 +744,7 @@ TEST(DBTest, GetEncountersEmptyLevel) {
// Step 1: First place sstables in levels 0 and 2
int compaction_count = 0;
- while (NumTableFilesAtLevel(0) == 0 ||
- NumTableFilesAtLevel(2) == 0) {
+ while (NumTableFilesAtLevel(0) == 0 || NumTableFilesAtLevel(2) == 0) {
ASSERT_LE(compaction_count, 100) << "could not fill levels 0 and 2";
compaction_count++;
Put("a", "begin");
@@ -640,7 +753,7 @@ TEST(DBTest, GetEncountersEmptyLevel) {
}
// Step 2: clear level 1 if necessary.
- dbfull()->TEST_CompactRange(1, NULL, NULL);
+ dbfull()->TEST_CompactRange(1, nullptr, nullptr);
ASSERT_EQ(NumTableFilesAtLevel(0), 1);
ASSERT_EQ(NumTableFilesAtLevel(1), 0);
ASSERT_EQ(NumTableFilesAtLevel(2), 1);
@@ -768,10 +881,10 @@ TEST(DBTest, IterMulti) {
ASSERT_EQ(IterStatus(iter), "b->vb");
// Make sure iter stays at snapshot
- ASSERT_OK(Put("a", "va2"));
+ ASSERT_OK(Put("a", "va2"));
ASSERT_OK(Put("a2", "va3"));
- ASSERT_OK(Put("b", "vb2"));
- ASSERT_OK(Put("c", "vc2"));
+ ASSERT_OK(Put("b", "vb2"));
+ ASSERT_OK(Put("c", "vc2"));
ASSERT_OK(Delete("b"));
iter->SeekToFirst();
ASSERT_EQ(IterStatus(iter), "a->va");
@@ -962,7 +1075,7 @@ TEST(DBTest, RecoverWithLargeLog) {
TEST(DBTest, CompactionsGenerateMultipleFiles) {
Options options = CurrentOptions();
- options.write_buffer_size = 100000000; // Large write buffer
+ options.write_buffer_size = 100000000; // Large write buffer
Reopen(&options);
Random rnd(301);
@@ -977,7 +1090,7 @@ TEST(DBTest, CompactionsGenerateMultipleFiles) {
// Reopening moves updates to level-0
Reopen(&options);
- dbfull()->TEST_CompactRange(0, NULL, NULL);
+ dbfull()->TEST_CompactRange(0, nullptr, nullptr);
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
ASSERT_GT(NumTableFilesAtLevel(1), 1);
@@ -1001,7 +1114,7 @@ TEST(DBTest, RepeatedWritesToSameKey) {
for (int i = 0; i < 5 * kMaxFiles; i++) {
Put("key", value);
ASSERT_LE(TotalTableFiles(), kMaxFiles);
- fprintf(stderr, "after %d: %d files\n", int(i+1), TotalTableFiles());
+ fprintf(stderr, "after %d: %d files\n", i + 1, TotalTableFiles());
}
}
@@ -1028,29 +1141,28 @@ TEST(DBTest, SparseMerge) {
}
Put("C", "vc");
dbfull()->TEST_CompactMemTable();
- dbfull()->TEST_CompactRange(0, NULL, NULL);
+ dbfull()->TEST_CompactRange(0, nullptr, nullptr);
// Make sparse update
- Put("A", "va2");
+ Put("A", "va2");
Put("B100", "bvalue2");
- Put("C", "vc2");
+ Put("C", "vc2");
dbfull()->TEST_CompactMemTable();
// Compactions should not cause us to create a situation where
// a file overlaps too much data at the next level.
- ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
- dbfull()->TEST_CompactRange(0, NULL, NULL);
- ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
- dbfull()->TEST_CompactRange(1, NULL, NULL);
- ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
+ ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20 * 1048576);
+ dbfull()->TEST_CompactRange(0, nullptr, nullptr);
+ ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20 * 1048576);
+ dbfull()->TEST_CompactRange(1, nullptr, nullptr);
+ ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20 * 1048576);
}
static bool Between(uint64_t val, uint64_t low, uint64_t high) {
bool result = (val >= low) && (val <= high);
if (!result) {
fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n",
- (unsigned long long)(val),
- (unsigned long long)(low),
+ (unsigned long long)(val), (unsigned long long)(low),
(unsigned long long)(high));
}
return result;
@@ -1059,7 +1171,7 @@ static bool Between(uint64_t val, uint64_t low, uint64_t high) {
TEST(DBTest, ApproximateSizes) {
do {
Options options = CurrentOptions();
- options.write_buffer_size = 100000000; // Large write buffer
+ options.write_buffer_size = 100000000; // Large write buffer
options.compression = kNoCompression;
DestroyAndReopen();
@@ -1080,18 +1192,27 @@ TEST(DBTest, ApproximateSizes) {
// 0 because GetApproximateSizes() does not account for memtable space
ASSERT_TRUE(Between(Size("", Key(50)), 0, 0));
+ if (options.reuse_logs) {
+ // Recovery will reuse memtable, and GetApproximateSizes() does not
+ // account for memtable usage;
+ Reopen(&options);
+ ASSERT_TRUE(Between(Size("", Key(50)), 0, 0));
+ continue;
+ }
+
// Check sizes across recovery by reopening a few times
for (int run = 0; run < 3; run++) {
Reopen(&options);
for (int compact_start = 0; compact_start < N; compact_start += 10) {
for (int i = 0; i < N; i += 10) {
- ASSERT_TRUE(Between(Size("", Key(i)), S1*i, S2*i));
- ASSERT_TRUE(Between(Size("", Key(i)+".suffix"), S1*(i+1), S2*(i+1)));
- ASSERT_TRUE(Between(Size(Key(i), Key(i+10)), S1*10, S2*10));
+ ASSERT_TRUE(Between(Size("", Key(i)), S1 * i, S2 * i));
+ ASSERT_TRUE(Between(Size("", Key(i) + ".suffix"), S1 * (i + 1),
+ S2 * (i + 1)));
+ ASSERT_TRUE(Between(Size(Key(i), Key(i + 10)), S1 * 10, S2 * 10));
}
- ASSERT_TRUE(Between(Size("", Key(50)), S1*50, S2*50));
- ASSERT_TRUE(Between(Size("", Key(50)+".suffix"), S1*50, S2*50));
+ ASSERT_TRUE(Between(Size("", Key(50)), S1 * 50, S2 * 50));
+ ASSERT_TRUE(Between(Size("", Key(50) + ".suffix"), S1 * 50, S2 * 50));
std::string cstart_str = Key(compact_start);
std::string cend_str = Key(compact_start + 9);
@@ -1123,6 +1244,11 @@ TEST(DBTest, ApproximateSizes_MixOfSmallAndLarge) {
ASSERT_OK(Put(Key(6), RandomString(&rnd, 300000)));
ASSERT_OK(Put(Key(7), RandomString(&rnd, 10000)));
+ if (options.reuse_logs) {
+ // Need to force a memtable compaction since recovery does not do so.
+ ASSERT_OK(dbfull()->TEST_CompactMemTable());
+ }
+
// Check sizes across recovery by reopening a few times
for (int run = 0; run < 3; run++) {
Reopen(&options);
@@ -1139,7 +1265,7 @@ TEST(DBTest, ApproximateSizes_MixOfSmallAndLarge) {
ASSERT_TRUE(Between(Size(Key(3), Key(5)), 110000, 111000));
- dbfull()->TEST_CompactRange(0, NULL, NULL);
+ dbfull()->TEST_CompactRange(0, nullptr, nullptr);
}
} while (ChangeOptions());
}
@@ -1153,7 +1279,7 @@ TEST(DBTest, IteratorPinsRef) {
// Write to force compactions
Put("foo", "newvalue1");
for (int i = 0; i < 100; i++) {
- ASSERT_OK(Put(Key(i), Key(i) + std::string(100000, 'v'))); // 100K values
+ ASSERT_OK(Put(Key(i), Key(i) + std::string(100000, 'v'))); // 100K values
}
Put("foo", "newvalue2");
@@ -1205,7 +1331,7 @@ TEST(DBTest, HiddenValuesAreRemoved) {
Put("pastfoo", "v");
const Snapshot* snapshot = db_->GetSnapshot();
Put("foo", "tiny");
- Put("pastfoo2", "v2"); // Advance sequence number one more
+ Put("pastfoo2", "v2"); // Advance sequence number one more
ASSERT_OK(dbfull()->TEST_CompactMemTable());
ASSERT_GT(NumTableFilesAtLevel(0), 0);
@@ -1215,11 +1341,11 @@ TEST(DBTest, HiddenValuesAreRemoved) {
db_->ReleaseSnapshot(snapshot);
ASSERT_EQ(AllEntriesFor("foo"), "[ tiny, " + big + " ]");
Slice x("x");
- dbfull()->TEST_CompactRange(0, NULL, &x);
+ dbfull()->TEST_CompactRange(0, nullptr, &x);
ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]");
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
ASSERT_GE(NumTableFilesAtLevel(1), 1);
- dbfull()->TEST_CompactRange(1, NULL, &x);
+ dbfull()->TEST_CompactRange(1, nullptr, &x);
ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]");
ASSERT_TRUE(Between(Size("", "pastfoo"), 0, 1000));
@@ -1230,14 +1356,14 @@ TEST(DBTest, DeletionMarkers1) {
Put("foo", "v1");
ASSERT_OK(dbfull()->TEST_CompactMemTable());
const int last = config::kMaxMemCompactLevel;
- ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level
+ ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level
// Place a table at level last-1 to prevent merging with preceding mutation
Put("a", "begin");
Put("z", "end");
dbfull()->TEST_CompactMemTable();
ASSERT_EQ(NumTableFilesAtLevel(last), 1);
- ASSERT_EQ(NumTableFilesAtLevel(last-1), 1);
+ ASSERT_EQ(NumTableFilesAtLevel(last - 1), 1);
Delete("foo");
Put("foo", "v2");
@@ -1245,11 +1371,11 @@ TEST(DBTest, DeletionMarkers1) {
ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2
ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]");
Slice z("z");
- dbfull()->TEST_CompactRange(last-2, NULL, &z);
+ dbfull()->TEST_CompactRange(last - 2, nullptr, &z);
// DEL eliminated, but v1 remains because we aren't compacting that level
// (DEL can be eliminated because v2 hides v1).
ASSERT_EQ(AllEntriesFor("foo"), "[ v2, v1 ]");
- dbfull()->TEST_CompactRange(last-1, NULL, NULL);
+ dbfull()->TEST_CompactRange(last - 1, nullptr, nullptr);
// Merging last-1 w/ last, so we are the base level for "foo", so
// DEL is removed. (as is v1).
ASSERT_EQ(AllEntriesFor("foo"), "[ v2 ]");
@@ -1259,23 +1385,23 @@ TEST(DBTest, DeletionMarkers2) {
Put("foo", "v1");
ASSERT_OK(dbfull()->TEST_CompactMemTable());
const int last = config::kMaxMemCompactLevel;
- ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level
+ ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level
// Place a table at level last-1 to prevent merging with preceding mutation
Put("a", "begin");
Put("z", "end");
dbfull()->TEST_CompactMemTable();
ASSERT_EQ(NumTableFilesAtLevel(last), 1);
- ASSERT_EQ(NumTableFilesAtLevel(last-1), 1);
+ ASSERT_EQ(NumTableFilesAtLevel(last - 1), 1);
Delete("foo");
ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2
ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
- dbfull()->TEST_CompactRange(last-2, NULL, NULL);
+ dbfull()->TEST_CompactRange(last - 2, nullptr, nullptr);
// DEL kept: "last" file overlaps
ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
- dbfull()->TEST_CompactRange(last-1, NULL, NULL);
+ dbfull()->TEST_CompactRange(last - 1, nullptr, nullptr);
// Merging last-1 w/ last, so we are the base level for "foo", so
// DEL is removed. (as is v1).
ASSERT_EQ(AllEntriesFor("foo"), "[ ]");
@@ -1285,7 +1411,8 @@ TEST(DBTest, OverlapInLevel0) {
do {
ASSERT_EQ(config::kMaxMemCompactLevel, 2) << "Fix test to match config";
- // Fill levels 1 and 2 to disable the pushing of new memtables to levels > 0.
+ // Fill levels 1 and 2 to disable the pushing of new memtables to levels >
+ // 0.
ASSERT_OK(Put("100", "v100"));
ASSERT_OK(Put("999", "v999"));
dbfull()->TEST_CompactMemTable();
@@ -1308,8 +1435,8 @@ TEST(DBTest, OverlapInLevel0) {
ASSERT_EQ("2,1,1", FilesPerLevel());
// Compact away the placeholder files we created initially
- dbfull()->TEST_CompactRange(1, NULL, NULL);
- dbfull()->TEST_CompactRange(2, NULL, NULL);
+ dbfull()->TEST_CompactRange(1, nullptr, nullptr);
+ dbfull()->TEST_CompactRange(2, nullptr, nullptr);
ASSERT_EQ("2", FilesPerLevel());
// Do a memtable compaction. Before bug-fix, the compaction would
@@ -1341,21 +1468,21 @@ TEST(DBTest, L0_CompactionBug_Issue44_a) {
TEST(DBTest, L0_CompactionBug_Issue44_b) {
Reopen();
- Put("","");
+ Put("", "");
Reopen();
Delete("e");
- Put("","");
+ Put("", "");
Reopen();
Put("c", "cv");
Reopen();
- Put("","");
+ Put("", "");
Reopen();
- Put("","");
+ Put("", "");
DelayMilliseconds(1000); // Wait for compaction to finish
Reopen();
- Put("d","dv");
+ Put("d", "dv");
Reopen();
- Put("","");
+ Put("", "");
Reopen();
Delete("d");
Delete("b");
@@ -1365,17 +1492,26 @@ TEST(DBTest, L0_CompactionBug_Issue44_b) {
ASSERT_EQ("(->)(c->cv)", Contents());
}
+TEST(DBTest, Fflush_Issue474) {
+ static const int kNum = 100000;
+ Random rnd(test::RandomSeed());
+ for (int i = 0; i < kNum; i++) {
+ fflush(nullptr);
+ ASSERT_OK(Put(RandomKey(&rnd), RandomString(&rnd, 100)));
+ }
+}
+
TEST(DBTest, ComparatorCheck) {
class NewComparator : public Comparator {
public:
- virtual const char* Name() const { return "leveldb.NewComparator"; }
- virtual int Compare(const Slice& a, const Slice& b) const {
+ const char* Name() const override { return "leveldb.NewComparator"; }
+ int Compare(const Slice& a, const Slice& b) const override {
return BytewiseComparator()->Compare(a, b);
}
- virtual void FindShortestSeparator(std::string* s, const Slice& l) const {
+ void FindShortestSeparator(std::string* s, const Slice& l) const override {
BytewiseComparator()->FindShortestSeparator(s, l);
}
- virtual void FindShortSuccessor(std::string* key) const {
+ void FindShortSuccessor(std::string* key) const override {
BytewiseComparator()->FindShortSuccessor(key);
}
};
@@ -1391,21 +1527,22 @@ TEST(DBTest, ComparatorCheck) {
TEST(DBTest, CustomComparator) {
class NumberComparator : public Comparator {
public:
- virtual const char* Name() const { return "test.NumberComparator"; }
- virtual int Compare(const Slice& a, const Slice& b) const {
+ const char* Name() const override { return "test.NumberComparator"; }
+ int Compare(const Slice& a, const Slice& b) const override {
return ToNumber(a) - ToNumber(b);
}
- virtual void FindShortestSeparator(std::string* s, const Slice& l) const {
- ToNumber(*s); // Check format
- ToNumber(l); // Check format
+ void FindShortestSeparator(std::string* s, const Slice& l) const override {
+ ToNumber(*s); // Check format
+ ToNumber(l); // Check format
}
- virtual void FindShortSuccessor(std::string* key) const {
- ToNumber(*key); // Check format
+ void FindShortSuccessor(std::string* key) const override {
+ ToNumber(*key); // Check format
}
+
private:
static int ToNumber(const Slice& x) {
// Check that there are no extra characters.
- ASSERT_TRUE(x.size() >= 2 && x[0] == '[' && x[x.size()-1] == ']')
+ ASSERT_TRUE(x.size() >= 2 && x[0] == '[' && x[x.size() - 1] == ']')
<< EscapeString(x);
int val;
char ignored;
@@ -1418,7 +1555,7 @@ TEST(DBTest, CustomComparator) {
Options new_options = CurrentOptions();
new_options.create_if_missing = true;
new_options.comparator = &cmp;
- new_options.filter_policy = NULL; // Cannot use bloom filters
+ new_options.filter_policy = nullptr; // Cannot use bloom filters
new_options.write_buffer_size = 1000; // Compact more often
DestroyAndReopen(&new_options);
ASSERT_OK(Put("[10]", "ten"));
@@ -1436,7 +1573,7 @@ TEST(DBTest, CustomComparator) {
for (int run = 0; run < 2; run++) {
for (int i = 0; i < 1000; i++) {
char buf[100];
- snprintf(buf, sizeof(buf), "[%d]", i*10);
+ snprintf(buf, sizeof(buf), "[%d]", i * 10);
ASSERT_OK(Put(buf, buf));
}
Compact("[0]", "[1000000]");
@@ -1473,7 +1610,7 @@ TEST(DBTest, ManualCompaction) {
// Compact all
MakeTables(1, "a", "z");
ASSERT_EQ("0,1,2", FilesPerLevel());
- db_->CompactRange(NULL, NULL);
+ db_->CompactRange(nullptr, nullptr);
ASSERT_EQ("0,0,1", FilesPerLevel());
}
@@ -1482,42 +1619,94 @@ TEST(DBTest, DBOpen_Options) {
DestroyDB(dbname, Options());
// Does not exist, and create_if_missing == false: error
- DB* db = NULL;
+ DB* db = nullptr;
Options opts;
opts.create_if_missing = false;
Status s = DB::Open(opts, dbname, &db);
- ASSERT_TRUE(strstr(s.ToString().c_str(), "does not exist") != NULL);
- ASSERT_TRUE(db == NULL);
+ ASSERT_TRUE(strstr(s.ToString().c_str(), "does not exist") != nullptr);
+ ASSERT_TRUE(db == nullptr);
// Does not exist, and create_if_missing == true: OK
opts.create_if_missing = true;
s = DB::Open(opts, dbname, &db);
ASSERT_OK(s);
- ASSERT_TRUE(db != NULL);
+ ASSERT_TRUE(db != nullptr);
delete db;
- db = NULL;
+ db = nullptr;
// Does exist, and error_if_exists == true: error
opts.create_if_missing = false;
opts.error_if_exists = true;
s = DB::Open(opts, dbname, &db);
- ASSERT_TRUE(strstr(s.ToString().c_str(), "exists") != NULL);
- ASSERT_TRUE(db == NULL);
+ ASSERT_TRUE(strstr(s.ToString().c_str(), "exists") != nullptr);
+ ASSERT_TRUE(db == nullptr);
// Does exist, and error_if_exists == false: OK
opts.create_if_missing = true;
opts.error_if_exists = false;
s = DB::Open(opts, dbname, &db);
ASSERT_OK(s);
- ASSERT_TRUE(db != NULL);
+ ASSERT_TRUE(db != nullptr);
delete db;
- db = NULL;
+ db = nullptr;
+}
+
+TEST(DBTest, DestroyEmptyDir) {
+ std::string dbname = test::TmpDir() + "/db_empty_dir";
+ TestEnv env(Env::Default());
+ env.DeleteDir(dbname);
+ ASSERT_TRUE(!env.FileExists(dbname));
+
+ Options opts;
+ opts.env = &env;
+
+ ASSERT_OK(env.CreateDir(dbname));
+ ASSERT_TRUE(env.FileExists(dbname));
+ std::vector<std::string> children;
+ ASSERT_OK(env.GetChildren(dbname, &children));
+ // The stock Env's do not filter out '.' and '..' special files.
+ ASSERT_EQ(2, children.size());
+ ASSERT_OK(DestroyDB(dbname, opts));
+ ASSERT_TRUE(!env.FileExists(dbname));
+
+ // Should also be destroyed if Env is filtering out dot files.
+ env.SetIgnoreDotFiles(true);
+ ASSERT_OK(env.CreateDir(dbname));
+ ASSERT_TRUE(env.FileExists(dbname));
+ ASSERT_OK(env.GetChildren(dbname, &children));
+ ASSERT_EQ(0, children.size());
+ ASSERT_OK(DestroyDB(dbname, opts));
+ ASSERT_TRUE(!env.FileExists(dbname));
+}
+
+TEST(DBTest, DestroyOpenDB) {
+ std::string dbname = test::TmpDir() + "/open_db_dir";
+ env_->DeleteDir(dbname);
+ ASSERT_TRUE(!env_->FileExists(dbname));
+
+ Options opts;
+ opts.create_if_missing = true;
+ DB* db = nullptr;
+ ASSERT_OK(DB::Open(opts, dbname, &db));
+ ASSERT_TRUE(db != nullptr);
+
+ // Must fail to destroy an open db.
+ ASSERT_TRUE(env_->FileExists(dbname));
+ ASSERT_TRUE(!DestroyDB(dbname, Options()).ok());
+ ASSERT_TRUE(env_->FileExists(dbname));
+
+ delete db;
+ db = nullptr;
+
+ // Should succeed destroying a closed db.
+ ASSERT_OK(DestroyDB(dbname, Options()));
+ ASSERT_TRUE(!env_->FileExists(dbname));
}
TEST(DBTest, Locking) {
- DB* db2 = NULL;
+ DB* db2 = nullptr;
Status s = DB::Open(CurrentOptions(), dbname_, &db2);
ASSERT_TRUE(!s.ok()) << "Locking did not prevent re-opening db";
}
@@ -1532,13 +1721,14 @@ TEST(DBTest, NoSpace) {
ASSERT_EQ("v1", Get("foo"));
Compact("a", "z");
const int num_files = CountFiles();
- env_->no_space_.Release_Store(env_); // Force out-of-space errors
+ // Force out-of-space errors.
+ env_->no_space_.store(true, std::memory_order_release);
for (int i = 0; i < 10; i++) {
- for (int level = 0; level < config::kNumLevels-1; level++) {
- dbfull()->TEST_CompactRange(level, NULL, NULL);
+ for (int level = 0; level < config::kNumLevels - 1; level++) {
+ dbfull()->TEST_CompactRange(level, nullptr, nullptr);
}
}
- env_->no_space_.Release_Store(NULL);
+ env_->no_space_.store(false, std::memory_order_release);
ASSERT_LT(CountFiles(), num_files + 3);
}
@@ -1548,7 +1738,8 @@ TEST(DBTest, NonWritableFileSystem) {
options.env = env_;
Reopen(&options);
ASSERT_OK(Put("foo", "v1"));
- env_->non_writable_.Release_Store(env_); // Force errors for new files
+ // Force errors for new files.
+ env_->non_writable_.store(true, std::memory_order_release);
std::string big(100000, 'x');
int errors = 0;
for (int i = 0; i < 20; i++) {
@@ -1559,7 +1750,7 @@ TEST(DBTest, NonWritableFileSystem) {
}
}
ASSERT_GT(errors, 0);
- env_->non_writable_.Release_Store(NULL);
+ env_->non_writable_.store(false, std::memory_order_release);
}
TEST(DBTest, WriteSyncError) {
@@ -1569,7 +1760,7 @@ TEST(DBTest, WriteSyncError) {
Options options = CurrentOptions();
options.env = env_;
Reopen(&options);
- env_->data_sync_error_.Release_Store(env_);
+ env_->data_sync_error_.store(true, std::memory_order_release);
// (b) Normal write should succeed
WriteOptions w;
@@ -1583,7 +1774,7 @@ TEST(DBTest, WriteSyncError) {
ASSERT_EQ("NOT_FOUND", Get("k2"));
// (d) make sync behave normally
- env_->data_sync_error_.Release_Store(NULL);
+ env_->data_sync_error_.store(false, std::memory_order_release);
// (e) Do a non-sync write; should fail
w.sync = false;
@@ -1603,9 +1794,8 @@ TEST(DBTest, ManifestWriteError) {
// We iterate twice. In the second iteration, everything is the
// same except the log record never makes it to the MANIFEST file.
for (int iter = 0; iter < 2; iter++) {
- port::AtomicPointer* error_type = (iter == 0)
- ? &env_->manifest_sync_error_
- : &env_->manifest_write_error_;
+ std::atomic<bool>* error_type = (iter == 0) ? &env_->manifest_sync_error_
+ : &env_->manifest_write_error_;
// Insert foo=>bar mapping
Options options = CurrentOptions();
@@ -1620,15 +1810,15 @@ TEST(DBTest, ManifestWriteError) {
dbfull()->TEST_CompactMemTable();
ASSERT_EQ("bar", Get("foo"));
const int last = config::kMaxMemCompactLevel;
- ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo=>bar is now in last level
+ ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo=>bar is now in last level
// Merging compaction (will fail)
- error_type->Release_Store(env_);
- dbfull()->TEST_CompactRange(last, NULL, NULL); // Should fail
+ error_type->store(true, std::memory_order_release);
+ dbfull()->TEST_CompactRange(last, nullptr, nullptr); // Should fail
ASSERT_EQ("bar", Get("foo"));
// Recovery: should not lose data
- error_type->Release_Store(NULL);
+ error_type->store(false, std::memory_order_release);
Reopen(&options);
ASSERT_EQ("bar", Get("foo"));
}
@@ -1648,8 +1838,7 @@ TEST(DBTest, MissingSSTFile) {
options.paranoid_checks = true;
Status s = TryReopen(&options);
ASSERT_TRUE(!s.ok());
- ASSERT_TRUE(s.ToString().find("issing") != std::string::npos)
- << s.ToString();
+ ASSERT_TRUE(s.ToString().find("issing") != std::string::npos) << s.ToString();
}
TEST(DBTest, StillReadSST) {
@@ -1699,7 +1888,7 @@ TEST(DBTest, BloomFilter) {
dbfull()->TEST_CompactMemTable();
// Prevent auto compactions triggered by seeks
- env_->delay_data_sync_.Release_Store(env_);
+ env_->delay_data_sync_.store(true, std::memory_order_release);
// Lookup present keys. Should rarely read from small sstable.
env_->random_read_counter_.Reset();
@@ -1709,7 +1898,7 @@ TEST(DBTest, BloomFilter) {
int reads = env_->random_read_counter_.Read();
fprintf(stderr, "%d present => %d reads\n", N, reads);
ASSERT_GE(reads, N);
- ASSERT_LE(reads, N + 2*N/100);
+ ASSERT_LE(reads, N + 2 * N / 100);
// Lookup present keys. Should rarely read from either sstable.
env_->random_read_counter_.Reset();
@@ -1718,9 +1907,9 @@ TEST(DBTest, BloomFilter) {
}
reads = env_->random_read_counter_.Read();
fprintf(stderr, "%d missing => %d reads\n", N, reads);
- ASSERT_LE(reads, 3*N/100);
+ ASSERT_LE(reads, 3 * N / 100);
- env_->delay_data_sync_.Release_Store(NULL);
+ env_->delay_data_sync_.store(false, std::memory_order_release);
Close();
delete options.block_cache;
delete options.filter_policy;
@@ -1735,9 +1924,9 @@ static const int kNumKeys = 1000;
struct MTState {
DBTest* test;
- port::AtomicPointer stop;
- port::AtomicPointer counter[kNumThreads];
- port::AtomicPointer thread_done[kNumThreads];
+ std::atomic<bool> stop;
+ std::atomic<int> counter[kNumThreads];
+ std::atomic<bool> thread_done[kNumThreads];
};
struct MTThread {
@@ -1749,13 +1938,13 @@ static void MTThreadBody(void* arg) {
MTThread* t = reinterpret_cast<MTThread*>(arg);
int id = t->id;
DB* db = t->state->test->db_;
- uintptr_t counter = 0;
+ int counter = 0;
fprintf(stderr, "... starting thread %d\n", id);
Random rnd(1000 + id);
std::string value;
char valbuf[1500];
- while (t->state->stop.Acquire_Load() == NULL) {
- t->state->counter[id].Release_Store(reinterpret_cast<void*>(counter));
+ while (!t->state->stop.load(std::memory_order_acquire)) {
+ t->state->counter[id].store(counter, std::memory_order_release);
int key = rnd.Uniform(kNumKeys);
char keybuf[20];
@@ -1764,8 +1953,8 @@ static void MTThreadBody(void* arg) {
if (rnd.OneIn(2)) {
// Write values of the form <key, my id, counter>.
// We add some padding for force compactions.
- snprintf(valbuf, sizeof(valbuf), "%d.%d.%-1000d",
- key, id, static_cast<int>(counter));
+ snprintf(valbuf, sizeof(valbuf), "%d.%d.%-1000d", key, id,
+ static_cast<int>(counter));
ASSERT_OK(db->Put(WriteOptions(), Slice(keybuf), Slice(valbuf)));
} else {
// Read a value and verify that it matches the pattern written above.
@@ -1780,14 +1969,13 @@ static void MTThreadBody(void* arg) {
ASSERT_EQ(k, key);
ASSERT_GE(w, 0);
ASSERT_LT(w, kNumThreads);
- ASSERT_LE(static_cast<uintptr_t>(c), reinterpret_cast<uintptr_t>(
- t->state->counter[w].Acquire_Load()));
+ ASSERT_LE(c, t->state->counter[w].load(std::memory_order_acquire));
}
}
counter++;
}
- t->state->thread_done[id].Release_Store(t);
- fprintf(stderr, "... stopping thread %d after %d ops\n", id, int(counter));
+ t->state->thread_done[id].store(true, std::memory_order_release);
+ fprintf(stderr, "... stopping thread %d after %d ops\n", id, counter);
}
} // namespace
@@ -1797,10 +1985,10 @@ TEST(DBTest, MultiThreaded) {
// Initialize state
MTState mt;
mt.test = this;
- mt.stop.Release_Store(0);
+ mt.stop.store(false, std::memory_order_release);
for (int id = 0; id < kNumThreads; id++) {
- mt.counter[id].Release_Store(0);
- mt.thread_done[id].Release_Store(0);
+ mt.counter[id].store(false, std::memory_order_release);
+ mt.thread_done[id].store(false, std::memory_order_release);
}
// Start threads
@@ -1815,9 +2003,9 @@ TEST(DBTest, MultiThreaded) {
DelayMilliseconds(kTestSeconds * 1000);
// Stop the threads and wait for them to finish
- mt.stop.Release_Store(&mt);
+ mt.stop.store(true, std::memory_order_release);
for (int id = 0; id < kNumThreads; id++) {
- while (mt.thread_done[id].Acquire_Load() == NULL) {
+ while (!mt.thread_done[id].load(std::memory_order_acquire)) {
DelayMilliseconds(100);
}
}
@@ -1828,28 +2016,28 @@ namespace {
typedef std::map<std::string, std::string> KVMap;
}
-class ModelDB: public DB {
+class ModelDB : public DB {
public:
class ModelSnapshot : public Snapshot {
public:
KVMap map_;
};
- explicit ModelDB(const Options& options): options_(options) { }
- ~ModelDB() { }
- virtual Status Put(const WriteOptions& o, const Slice& k, const Slice& v) {
+ explicit ModelDB(const Options& options) : options_(options) {}
+ ~ModelDB() override = default;
+ Status Put(const WriteOptions& o, const Slice& k, const Slice& v) override {
return DB::Put(o, k, v);
}
- virtual Status Delete(const WriteOptions& o, const Slice& key) {
+ Status Delete(const WriteOptions& o, const Slice& key) override {
return DB::Delete(o, key);
}
- virtual Status Get(const ReadOptions& options,
- const Slice& key, std::string* value) {
- assert(false); // Not implemented
+ Status Get(const ReadOptions& options, const Slice& key,
+ std::string* value) override {
+ assert(false); // Not implemented
return Status::NotFound(key);
}
- virtual Iterator* NewIterator(const ReadOptions& options) {
- if (options.snapshot == NULL) {
+ Iterator* NewIterator(const ReadOptions& options) override {
+ if (options.snapshot == nullptr) {
KVMap* saved = new KVMap;
*saved = map_;
return new ModelIter(saved, true);
@@ -1859,68 +2047,65 @@ class ModelDB: public DB {
return new ModelIter(snapshot_state, false);
}
}
- virtual const Snapshot* GetSnapshot() {
+ const Snapshot* GetSnapshot() override {
ModelSnapshot* snapshot = new ModelSnapshot;
snapshot->map_ = map_;
return snapshot;
}
- virtual void ReleaseSnapshot(const Snapshot* snapshot) {
+ void ReleaseSnapshot(const Snapshot* snapshot) override {
delete reinterpret_cast<const ModelSnapshot*>(snapshot);
}
- virtual Status Write(const WriteOptions& options, WriteBatch* batch) {
+ Status Write(const WriteOptions& options, WriteBatch* batch) override {
class Handler : public WriteBatch::Handler {
public:
KVMap* map_;
- virtual void Put(const Slice& key, const Slice& value) {
+ void Put(const Slice& key, const Slice& value) override {
(*map_)[key.ToString()] = value.ToString();
}
- virtual void Delete(const Slice& key) {
- map_->erase(key.ToString());
- }
+ void Delete(const Slice& key) override { map_->erase(key.ToString()); }
};
Handler handler;
handler.map_ = &map_;
return batch->Iterate(&handler);
}
- virtual bool GetProperty(const Slice& property, std::string* value) {
+ bool GetProperty(const Slice& property, std::string* value) override {
return false;
}
- virtual void GetApproximateSizes(const Range* r, int n, uint64_t* sizes) {
+ void GetApproximateSizes(const Range* r, int n, uint64_t* sizes) override {
for (int i = 0; i < n; i++) {
sizes[i] = 0;
}
}
- virtual void CompactRange(const Slice* start, const Slice* end) {
- }
+ void CompactRange(const Slice* start, const Slice* end) override {}
private:
- class ModelIter: public Iterator {
+ class ModelIter : public Iterator {
public:
ModelIter(const KVMap* map, bool owned)
- : map_(map), owned_(owned), iter_(map_->end()) {
- }
- ~ModelIter() {
+ : map_(map), owned_(owned), iter_(map_->end()) {}
+ ~ModelIter() override {
if (owned_) delete map_;
}
- virtual bool Valid() const { return iter_ != map_->end(); }
- virtual void SeekToFirst() { iter_ = map_->begin(); }
- virtual void SeekToLast() {
+ bool Valid() const override { return iter_ != map_->end(); }
+ void SeekToFirst() override { iter_ = map_->begin(); }
+ void SeekToLast() override {
if (map_->empty()) {
iter_ = map_->end();
} else {
iter_ = map_->find(map_->rbegin()->first);
}
}
- virtual void Seek(const Slice& k) {
+ void Seek(const Slice& k) override {
iter_ = map_->lower_bound(k.ToString());
}
- virtual void Next() { ++iter_; }
- virtual void Prev() { --iter_; }
- virtual Slice key() const { return iter_->first; }
- virtual Slice value() const { return iter_->second; }
- virtual Status status() const { return Status::OK(); }
+ void Next() override { ++iter_; }
+ void Prev() override { --iter_; }
+ Slice key() const override { return iter_->first; }
+ Slice value() const override { return iter_->second; }
+ Status status() const override { return Status::OK(); }
+
private:
const KVMap* const map_;
const bool owned_; // Do we own map_
@@ -1930,16 +2115,7 @@ class ModelDB: public DB {
KVMap map_;
};
-static std::string RandomKey(Random* rnd) {
- int len = (rnd->OneIn(3)
- ? 1 // Short sometimes to encourage collisions
- : (rnd->OneIn(100) ? rnd->Skewed(10) : rnd->Uniform(10)));
- return test::RandomKey(rnd, len);
-}
-
-static bool CompareIterators(int step,
- DB* model,
- DB* db,
+static bool CompareIterators(int step, DB* model, DB* db,
const Snapshot* model_snap,
const Snapshot* db_snap) {
ReadOptions options;
@@ -1950,12 +2126,10 @@ static bool CompareIterators(int step,
bool ok = true;
int count = 0;
for (miter->SeekToFirst(), dbiter->SeekToFirst();
- ok && miter->Valid() && dbiter->Valid();
- miter->Next(), dbiter->Next()) {
+ ok && miter->Valid() && dbiter->Valid(); miter->Next(), dbiter->Next()) {
count++;
if (miter->key().compare(dbiter->key()) != 0) {
- fprintf(stderr, "step %d: Key mismatch: '%s' vs. '%s'\n",
- step,
+ fprintf(stderr, "step %d: Key mismatch: '%s' vs. '%s'\n", step,
EscapeString(miter->key()).c_str(),
EscapeString(dbiter->key()).c_str());
ok = false;
@@ -1964,8 +2138,7 @@ static bool CompareIterators(int step,
if (miter->value().compare(dbiter->value()) != 0) {
fprintf(stderr, "step %d: Value mismatch for key '%s': '%s' vs. '%s'\n",
- step,
- EscapeString(miter->key()).c_str(),
+ step, EscapeString(miter->key()).c_str(),
EscapeString(miter->value()).c_str(),
EscapeString(miter->value()).c_str());
ok = false;
@@ -1990,8 +2163,8 @@ TEST(DBTest, Randomized) {
do {
ModelDB model(CurrentOptions());
const int N = 10000;
- const Snapshot* model_snap = NULL;
- const Snapshot* db_snap = NULL;
+ const Snapshot* model_snap = nullptr;
+ const Snapshot* db_snap = nullptr;
std::string k, v;
for (int step = 0; step < N; step++) {
if (step % 100 == 0) {
@@ -1999,22 +2172,19 @@ TEST(DBTest, Randomized) {
}
// TODO(sanjay): Test Get() works
int p = rnd.Uniform(100);
- if (p < 45) { // Put
+ if (p < 45) { // Put
k = RandomKey(&rnd);
- v = RandomString(&rnd,
- rnd.OneIn(20)
- ? 100 + rnd.Uniform(100)
- : rnd.Uniform(8));
+ v = RandomString(
+ &rnd, rnd.OneIn(20) ? 100 + rnd.Uniform(100) : rnd.Uniform(8));
ASSERT_OK(model.Put(WriteOptions(), k, v));
ASSERT_OK(db_->Put(WriteOptions(), k, v));
- } else if (p < 90) { // Delete
+ } else if (p < 90) { // Delete
k = RandomKey(&rnd);
ASSERT_OK(model.Delete(WriteOptions(), k));
ASSERT_OK(db_->Delete(WriteOptions(), k));
-
- } else { // Multi-element batch
+ } else { // Multi-element batch
WriteBatch b;
const int num = rnd.Uniform(8);
for (int i = 0; i < num; i++) {
@@ -2036,23 +2206,23 @@ TEST(DBTest, Randomized) {
}
if ((step % 100) == 0) {
- ASSERT_TRUE(CompareIterators(step, &model, db_, NULL, NULL));
+ ASSERT_TRUE(CompareIterators(step, &model, db_, nullptr, nullptr));
ASSERT_TRUE(CompareIterators(step, &model, db_, model_snap, db_snap));
// Save a snapshot from each DB this time that we'll use next
// time we compare things, to make sure the current state is
// preserved with the snapshot
- if (model_snap != NULL) model.ReleaseSnapshot(model_snap);
- if (db_snap != NULL) db_->ReleaseSnapshot(db_snap);
+ if (model_snap != nullptr) model.ReleaseSnapshot(model_snap);
+ if (db_snap != nullptr) db_->ReleaseSnapshot(db_snap);
Reopen();
- ASSERT_TRUE(CompareIterators(step, &model, db_, NULL, NULL));
+ ASSERT_TRUE(CompareIterators(step, &model, db_, nullptr, nullptr));
model_snap = model.GetSnapshot();
db_snap = db_->GetSnapshot();
}
}
- if (model_snap != NULL) model.ReleaseSnapshot(model_snap);
- if (db_snap != NULL) db_->ReleaseSnapshot(db_snap);
+ if (model_snap != nullptr) model.ReleaseSnapshot(model_snap);
+ if (db_snap != nullptr) db_->ReleaseSnapshot(db_snap);
} while (ChangeOptions());
}
@@ -2066,15 +2236,15 @@ void BM_LogAndApply(int iters, int num_base_files) {
std::string dbname = test::TmpDir() + "/leveldb_test_benchmark";
DestroyDB(dbname, Options());
- DB* db = NULL;
+ DB* db = nullptr;
Options opts;
opts.create_if_missing = true;
Status s = DB::Open(opts, dbname, &db);
ASSERT_OK(s);
- ASSERT_TRUE(db != NULL);
+ ASSERT_TRUE(db != nullptr);
delete db;
- db = NULL;
+ db = nullptr;
Env* env = Env::Default();
@@ -2083,13 +2253,14 @@ void BM_LogAndApply(int iters, int num_base_files) {
InternalKeyComparator cmp(BytewiseComparator());
Options options;
- VersionSet vset(dbname, &options, NULL, &cmp);
- ASSERT_OK(vset.Recover());
+ VersionSet vset(dbname, &options, nullptr, &cmp);
+ bool save_manifest;
+ ASSERT_OK(vset.Recover(&save_manifest));
VersionEdit vbase;
uint64_t fnum = 1;
for (int i = 0; i < num_base_files; i++) {
- InternalKey start(MakeKey(2*fnum), 1, kTypeValue);
- InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion);
+ InternalKey start(MakeKey(2 * fnum), 1, kTypeValue);
+ InternalKey limit(MakeKey(2 * fnum + 1), 1, kTypeDeletion);
vbase.AddFile(2, fnum++, 1 /* file size */, start, limit);
}
ASSERT_OK(vset.LogAndApply(&vbase, &mu));
@@ -2099,8 +2270,8 @@ void BM_LogAndApply(int iters, int num_base_files) {
for (int i = 0; i < iters; i++) {
VersionEdit vedit;
vedit.DeleteFile(2, fnum);
- InternalKey start(MakeKey(2*fnum), 1, kTypeValue);
- InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion);
+ InternalKey start(MakeKey(2 * fnum), 1, kTypeValue);
+ InternalKey limit(MakeKey(2 * fnum + 1), 1, kTypeDeletion);
vedit.AddFile(2, fnum++, 1 /* file size */, start, limit);
vset.LogAndApply(&vedit, &mu);
}
@@ -2109,8 +2280,8 @@ void BM_LogAndApply(int iters, int num_base_files) {
char buf[16];
snprintf(buf, sizeof(buf), "%d", num_base_files);
fprintf(stderr,
- "BM_LogAndApply/%-6s %8d iters : %9u us (%7.0f us / iter)\n",
- buf, iters, us, ((float)us) / iters);
+ "BM_LogAndApply/%-6s %8d iters : %9u us (%7.0f us / iter)\n", buf,
+ iters, us, ((float)us) / iters);
}
} // namespace leveldb
diff --git a/db/dbformat.cc b/db/dbformat.cc
index 20a7ca4..459eddf 100644
--- a/db/dbformat.cc
+++ b/db/dbformat.cc
@@ -2,8 +2,12 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#include <stdio.h>
#include "db/dbformat.h"
+
+#include <stdio.h>
+
+#include <sstream>
+
#include "port/port.h"
#include "util/coding.h"
@@ -21,26 +25,20 @@ void AppendInternalKey(std::string* result, const ParsedInternalKey& key) {
}
std::string ParsedInternalKey::DebugString() const {
- char buf[50];
- snprintf(buf, sizeof(buf), "' @ %llu : %d",
- (unsigned long long) sequence,
- int(type));
- std::string result = "'";
- result += EscapeString(user_key.ToString());
- result += buf;
- return result;
+ std::ostringstream ss;
+ ss << '\'' << EscapeString(user_key.ToString()) << "' @ " << sequence << " : "
+ << static_cast<int>(type);
+ return ss.str();
}
std::string InternalKey::DebugString() const {
- std::string result;
ParsedInternalKey parsed;
if (ParseInternalKey(rep_, &parsed)) {
- result = parsed.DebugString();
- } else {
- result = "(bad)";
- result.append(EscapeString(rep_));
+ return parsed.DebugString();
}
- return result;
+ std::ostringstream ss;
+ ss << "(bad)" << EscapeString(rep_);
+ return ss.str();
}
const char* InternalKeyComparator::Name() const {
@@ -65,9 +63,8 @@ int InternalKeyComparator::Compare(const Slice& akey, const Slice& bkey) const {
return r;
}
-void InternalKeyComparator::FindShortestSeparator(
- std::string* start,
- const Slice& limit) const {
+void InternalKeyComparator::FindShortestSeparator(std::string* start,
+ const Slice& limit) const {
// Attempt to shorten the user portion of the key
Slice user_start = ExtractUserKey(*start);
Slice user_limit = ExtractUserKey(limit);
@@ -77,7 +74,8 @@ void InternalKeyComparator::FindShortestSeparator(
user_comparator_->Compare(user_start, tmp) < 0) {
// User key has become shorter physically, but larger logically.
// Tack on the earliest possible number to the shortened user key.
- PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek));
+ PutFixed64(&tmp,
+ PackSequenceAndType(kMaxSequenceNumber, kValueTypeForSeek));
assert(this->Compare(*start, tmp) < 0);
assert(this->Compare(tmp, limit) < 0);
start->swap(tmp);
@@ -92,15 +90,14 @@ void InternalKeyComparator::FindShortSuccessor(std::string* key) const {
user_comparator_->Compare(user_key, tmp) < 0) {
// User key has become shorter physically, but larger logically.
// Tack on the earliest possible number to the shortened user key.
- PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek));
+ PutFixed64(&tmp,
+ PackSequenceAndType(kMaxSequenceNumber, kValueTypeForSeek));
assert(this->Compare(*key, tmp) < 0);
key->swap(tmp);
}
}
-const char* InternalFilterPolicy::Name() const {
- return user_policy_->Name();
-}
+const char* InternalFilterPolicy::Name() const { return user_policy_->Name(); }
void InternalFilterPolicy::CreateFilter(const Slice* keys, int n,
std::string* dst) const {
diff --git a/db/dbformat.h b/db/dbformat.h
index ea897b1..abcb489 100644
--- a/db/dbformat.h
+++ b/db/dbformat.h
@@ -6,6 +6,7 @@
#define STORAGE_LEVELDB_DB_DBFORMAT_H_
#include <stdio.h>
+
#include "leveldb/comparator.h"
#include "leveldb/db.h"
#include "leveldb/filter_policy.h"
@@ -48,10 +49,7 @@ class InternalKey;
// Value types encoded as the last component of internal keys.
// DO NOT CHANGE THESE ENUM VALUES: they are embedded in the on-disk
// data structures.
-enum ValueType {
- kTypeDeletion = 0x0,
- kTypeValue = 0x1
-};
+enum ValueType { kTypeDeletion = 0x0, kTypeValue = 0x1 };
// kValueTypeForSeek defines the ValueType that should be passed when
// constructing a ParsedInternalKey object for seeking to a particular
// sequence number (since we sort sequence numbers in decreasing order
@@ -64,17 +62,16 @@ typedef uint64_t SequenceNumber;
// We leave eight bits empty at the bottom so a type and sequence#
// can be packed together into 64-bits.
-static const SequenceNumber kMaxSequenceNumber =
- ((0x1ull << 56) - 1);
+static const SequenceNumber kMaxSequenceNumber = ((0x1ull << 56) - 1);
struct ParsedInternalKey {
Slice user_key;
SequenceNumber sequence;
ValueType type;
- ParsedInternalKey() { } // Intentionally left uninitialized (for speed)
+ ParsedInternalKey() {} // Intentionally left uninitialized (for speed)
ParsedInternalKey(const Slice& u, const SequenceNumber& seq, ValueType t)
- : user_key(u), sequence(seq), type(t) { }
+ : user_key(u), sequence(seq), type(t) {}
std::string DebugString() const;
};
@@ -84,15 +81,13 @@ inline size_t InternalKeyEncodingLength(const ParsedInternalKey& key) {
}
// Append the serialization of "key" to *result.
-extern void AppendInternalKey(std::string* result,
- const ParsedInternalKey& key);
+void AppendInternalKey(std::string* result, const ParsedInternalKey& key);
// Attempt to parse an internal key from "internal_key". On success,
// stores the parsed data in "*result", and returns true.
//
// On error, returns false, leaves "*result" in an undefined state.
-extern bool ParseInternalKey(const Slice& internal_key,
- ParsedInternalKey* result);
+bool ParseInternalKey(const Slice& internal_key, ParsedInternalKey* result);
// Returns the user key portion of an internal key.
inline Slice ExtractUserKey(const Slice& internal_key) {
@@ -100,27 +95,19 @@ inline Slice ExtractUserKey(const Slice& internal_key) {
return Slice(internal_key.data(), internal_key.size() - 8);
}
-inline ValueType ExtractValueType(const Slice& internal_key) {
- assert(internal_key.size() >= 8);
- const size_t n = internal_key.size();
- uint64_t num = DecodeFixed64(internal_key.data() + n - 8);
- unsigned char c = num & 0xff;
- return static_cast<ValueType>(c);
-}
-
// A comparator for internal keys that uses a specified comparator for
// the user key portion and breaks ties by decreasing sequence number.
class InternalKeyComparator : public Comparator {
private:
const Comparator* user_comparator_;
+
public:
- explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c) { }
- virtual const char* Name() const;
- virtual int Compare(const Slice& a, const Slice& b) const;
- virtual void FindShortestSeparator(
- std::string* start,
- const Slice& limit) const;
- virtual void FindShortSuccessor(std::string* key) const;
+ explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c) {}
+ const char* Name() const override;
+ int Compare(const Slice& a, const Slice& b) const override;
+ void FindShortestSeparator(std::string* start,
+ const Slice& limit) const override;
+ void FindShortSuccessor(std::string* key) const override;
const Comparator* user_comparator() const { return user_comparator_; }
@@ -131,11 +118,12 @@ class InternalKeyComparator : public Comparator {
class InternalFilterPolicy : public FilterPolicy {
private:
const FilterPolicy* const user_policy_;
+
public:
- explicit InternalFilterPolicy(const FilterPolicy* p) : user_policy_(p) { }
- virtual const char* Name() const;
- virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const;
- virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const;
+ explicit InternalFilterPolicy(const FilterPolicy* p) : user_policy_(p) {}
+ const char* Name() const override;
+ void CreateFilter(const Slice* keys, int n, std::string* dst) const override;
+ bool KeyMayMatch(const Slice& key, const Slice& filter) const override;
};
// Modules in this directory should keep internal keys wrapped inside
@@ -144,13 +132,18 @@ class InternalFilterPolicy : public FilterPolicy {
class InternalKey {
private:
std::string rep_;
+
public:
- InternalKey() { } // Leave rep_ as empty to indicate it is invalid
+ InternalKey() {} // Leave rep_ as empty to indicate it is invalid
InternalKey(const Slice& user_key, SequenceNumber s, ValueType t) {
AppendInternalKey(&rep_, ParsedInternalKey(user_key, s, t));
}
- void DecodeFrom(const Slice& s) { rep_.assign(s.data(), s.size()); }
+ bool DecodeFrom(const Slice& s) {
+ rep_.assign(s.data(), s.size());
+ return !rep_.empty();
+ }
+
Slice Encode() const {
assert(!rep_.empty());
return rep_;
@@ -168,8 +161,8 @@ class InternalKey {
std::string DebugString() const;
};
-inline int InternalKeyComparator::Compare(
- const InternalKey& a, const InternalKey& b) const {
+inline int InternalKeyComparator::Compare(const InternalKey& a,
+ const InternalKey& b) const {
return Compare(a.Encode(), b.Encode());
}
@@ -192,6 +185,9 @@ class LookupKey {
// the specified sequence number.
LookupKey(const Slice& user_key, SequenceNumber sequence);
+ LookupKey(const LookupKey&) = delete;
+ LookupKey& operator=(const LookupKey&) = delete;
+
~LookupKey();
// Return a key suitable for lookup in a MemTable.
@@ -214,11 +210,7 @@ class LookupKey {
const char* start_;
const char* kstart_;
const char* end_;
- char space_[200]; // Avoid allocation for short keys
-
- // No copying allowed
- LookupKey(const LookupKey&);
- void operator=(const LookupKey&);
+ char space_[200]; // Avoid allocation for short keys
};
inline LookupKey::~LookupKey() {
diff --git a/db/dbformat_test.cc b/db/dbformat_test.cc
index 5d82f5d..1209369 100644
--- a/db/dbformat_test.cc
+++ b/db/dbformat_test.cc
@@ -8,8 +8,7 @@
namespace leveldb {
-static std::string IKey(const std::string& user_key,
- uint64_t seq,
+static std::string IKey(const std::string& user_key, uint64_t seq,
ValueType vt) {
std::string encoded;
AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt));
@@ -28,9 +27,7 @@ static std::string ShortSuccessor(const std::string& s) {
return result;
}
-static void TestKey(const std::string& key,
- uint64_t seq,
- ValueType vt) {
+static void TestKey(const std::string& key, uint64_t seq, ValueType vt) {
std::string encoded = IKey(key, seq, vt);
Slice in(encoded);
@@ -44,16 +41,22 @@ static void TestKey(const std::string& key,
ASSERT_TRUE(!ParseInternalKey(Slice("bar"), &decoded));
}
-class FormatTest { };
+class FormatTest {};
TEST(FormatTest, InternalKey_EncodeDecode) {
- const char* keys[] = { "", "k", "hello", "longggggggggggggggggggggg" };
- const uint64_t seq[] = {
- 1, 2, 3,
- (1ull << 8) - 1, 1ull << 8, (1ull << 8) + 1,
- (1ull << 16) - 1, 1ull << 16, (1ull << 16) + 1,
- (1ull << 32) - 1, 1ull << 32, (1ull << 32) + 1
- };
+ const char* keys[] = {"", "k", "hello", "longggggggggggggggggggggg"};
+ const uint64_t seq[] = {1,
+ 2,
+ 3,
+ (1ull << 8) - 1,
+ 1ull << 8,
+ (1ull << 8) + 1,
+ (1ull << 16) - 1,
+ 1ull << 16,
+ (1ull << 16) + 1,
+ (1ull << 32) - 1,
+ 1ull << 32,
+ (1ull << 32) + 1};
for (int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) {
for (int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) {
TestKey(keys[k], seq[s], kTypeValue);
@@ -62,40 +65,44 @@ TEST(FormatTest, InternalKey_EncodeDecode) {
}
}
+TEST(FormatTest, InternalKey_DecodeFromEmpty) {
+ InternalKey internal_key;
+
+ ASSERT_TRUE(!internal_key.DecodeFrom(""));
+}
+
TEST(FormatTest, InternalKeyShortSeparator) {
// When user keys are same
ASSERT_EQ(IKey("foo", 100, kTypeValue),
- Shorten(IKey("foo", 100, kTypeValue),
- IKey("foo", 99, kTypeValue)));
- ASSERT_EQ(IKey("foo", 100, kTypeValue),
- Shorten(IKey("foo", 100, kTypeValue),
- IKey("foo", 101, kTypeValue)));
- ASSERT_EQ(IKey("foo", 100, kTypeValue),
- Shorten(IKey("foo", 100, kTypeValue),
- IKey("foo", 100, kTypeValue)));
- ASSERT_EQ(IKey("foo", 100, kTypeValue),
- Shorten(IKey("foo", 100, kTypeValue),
- IKey("foo", 100, kTypeDeletion)));
+ Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 99, kTypeValue)));
+ ASSERT_EQ(
+ IKey("foo", 100, kTypeValue),
+ Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 101, kTypeValue)));
+ ASSERT_EQ(
+ IKey("foo", 100, kTypeValue),
+ Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeValue)));
+ ASSERT_EQ(
+ IKey("foo", 100, kTypeValue),
+ Shorten(IKey("foo", 100, kTypeValue), IKey("foo", 100, kTypeDeletion)));
// When user keys are misordered
ASSERT_EQ(IKey("foo", 100, kTypeValue),
- Shorten(IKey("foo", 100, kTypeValue),
- IKey("bar", 99, kTypeValue)));
+ Shorten(IKey("foo", 100, kTypeValue), IKey("bar", 99, kTypeValue)));
// When user keys are different, but correctly ordered
- ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek),
- Shorten(IKey("foo", 100, kTypeValue),
- IKey("hello", 200, kTypeValue)));
+ ASSERT_EQ(
+ IKey("g", kMaxSequenceNumber, kValueTypeForSeek),
+ Shorten(IKey("foo", 100, kTypeValue), IKey("hello", 200, kTypeValue)));
// When start user key is prefix of limit user key
- ASSERT_EQ(IKey("foo", 100, kTypeValue),
- Shorten(IKey("foo", 100, kTypeValue),
- IKey("foobar", 200, kTypeValue)));
+ ASSERT_EQ(
+ IKey("foo", 100, kTypeValue),
+ Shorten(IKey("foo", 100, kTypeValue), IKey("foobar", 200, kTypeValue)));
// When limit user key is prefix of start user key
- ASSERT_EQ(IKey("foobar", 100, kTypeValue),
- Shorten(IKey("foobar", 100, kTypeValue),
- IKey("foo", 200, kTypeValue)));
+ ASSERT_EQ(
+ IKey("foobar", 100, kTypeValue),
+ Shorten(IKey("foobar", 100, kTypeValue), IKey("foo", 200, kTypeValue)));
}
TEST(FormatTest, InternalKeyShortestSuccessor) {
@@ -105,8 +112,20 @@ TEST(FormatTest, InternalKeyShortestSuccessor) {
ShortSuccessor(IKey("\xff\xff", 100, kTypeValue)));
}
-} // namespace leveldb
+TEST(FormatTest, ParsedInternalKeyDebugString) {
+ ParsedInternalKey key("The \"key\" in 'single quotes'", 42, kTypeValue);
+
+ ASSERT_EQ("'The \"key\" in 'single quotes'' @ 42 : 1", key.DebugString());
+}
+
+TEST(FormatTest, InternalKeyDebugString) {
+ InternalKey key("The \"key\" in 'single quotes'", 42, kTypeValue);
+ ASSERT_EQ("'The \"key\" in 'single quotes'' @ 42 : 1", key.DebugString());
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
+ InternalKey invalid_key;
+ ASSERT_EQ("(bad)", invalid_key.DebugString());
}
+
+} // namespace leveldb
+
+int main(int argc, char** argv) { return leveldb::test::RunAllTests(); }
diff --git a/db/dumpfile.cc b/db/dumpfile.cc
index 61c47c2..77d5900 100644
--- a/db/dumpfile.cc
+++ b/db/dumpfile.cc
@@ -2,7 +2,10 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
+#include "leveldb/dumpfile.h"
+
#include <stdio.h>
+
#include "db/dbformat.h"
#include "db/filename.h"
#include "db/log_reader.h"
@@ -35,8 +38,7 @@ bool GuessType(const std::string& fname, FileType* type) {
// Notified when log reader encounters corruption.
class CorruptionReporter : public log::Reader::Reporter {
public:
- WritableFile* dst_;
- virtual void Corruption(size_t bytes, const Status& status) {
+ void Corruption(size_t bytes, const Status& status) override {
std::string r = "corruption: ";
AppendNumberTo(&r, bytes);
r += " bytes; ";
@@ -44,6 +46,8 @@ class CorruptionReporter : public log::Reader::Reporter {
r.push_back('\n');
dst_->Append(r);
}
+
+ WritableFile* dst_;
};
// Print contents of a log file. (*func)() is called on every record.
@@ -70,8 +74,7 @@ Status PrintLogContents(Env* env, const std::string& fname,
// Called on every item found in a WriteBatch.
class WriteBatchItemPrinter : public WriteBatch::Handler {
public:
- WritableFile* dst_;
- virtual void Put(const Slice& key, const Slice& value) {
+ void Put(const Slice& key, const Slice& value) override {
std::string r = " put '";
AppendEscapedStringTo(&r, key);
r += "' '";
@@ -79,14 +82,15 @@ class WriteBatchItemPrinter : public WriteBatch::Handler {
r += "'\n";
dst_->Append(r);
}
- virtual void Delete(const Slice& key) {
+ void Delete(const Slice& key) override {
std::string r = " del '";
AppendEscapedStringTo(&r, key);
r += "'\n";
dst_->Append(r);
}
-};
+ WritableFile* dst_;
+};
// Called on every log record (each one of which is a WriteBatch)
// found in a kLogFile.
@@ -142,8 +146,8 @@ Status DumpDescriptor(Env* env, const std::string& fname, WritableFile* dst) {
Status DumpTable(Env* env, const std::string& fname, WritableFile* dst) {
uint64_t file_size;
- RandomAccessFile* file = NULL;
- Table* table = NULL;
+ RandomAccessFile* file = nullptr;
+ Table* table = nullptr;
Status s = env->GetFileSize(fname, &file_size);
if (s.ok()) {
s = env->NewRandomAccessFile(fname, &file);
@@ -213,9 +217,12 @@ Status DumpFile(Env* env, const std::string& fname, WritableFile* dst) {
return Status::InvalidArgument(fname + ": unknown file type");
}
switch (ftype) {
- case kLogFile: return DumpLog(env, fname, dst);
- case kDescriptorFile: return DumpDescriptor(env, fname, dst);
- case kTableFile: return DumpTable(env, fname, dst);
+ case kLogFile:
+ return DumpLog(env, fname, dst);
+ case kDescriptorFile:
+ return DumpDescriptor(env, fname, dst);
+ case kTableFile:
+ return DumpTable(env, fname, dst);
default:
break;
}
diff --git a/db/fault_injection_test.cc b/db/fault_injection_test.cc
index 8e3e603..5b31bb8 100644
--- a/db/fault_injection_test.cc
+++ b/db/fault_injection_test.cc
@@ -6,18 +6,20 @@
// the last "sync". It then checks for data loss errors by purposely dropping
// file data (or entire files) not protected by a "sync".
-#include "leveldb/db.h"
-
#include <map>
#include <set>
+
#include "db/db_impl.h"
#include "db/filename.h"
#include "db/log_format.h"
#include "db/version_set.h"
#include "leveldb/cache.h"
+#include "leveldb/db.h"
#include "leveldb/env.h"
#include "leveldb/table.h"
#include "leveldb/write_batch.h"
+#include "port/port.h"
+#include "port/thread_annotations.h"
#include "util/logging.h"
#include "util/mutexlock.h"
#include "util/testharness.h"
@@ -34,7 +36,7 @@ class FaultInjectionTestEnv;
namespace {
// Assume a filename, and not a directory name like "/foo/bar/"
-static std::string GetDirName(const std::string filename) {
+static std::string GetDirName(const std::string& filename) {
size_t found = filename.find_last_of("/\\");
if (found == std::string::npos) {
return "";
@@ -54,8 +56,7 @@ Status Truncate(const std::string& filename, uint64_t length) {
SequentialFile* orig_file;
Status s = env->NewSequentialFile(filename, &orig_file);
- if (!s.ok())
- return s;
+ if (!s.ok()) return s;
char* scratch = new char[length];
leveldb::Slice result;
@@ -83,15 +84,15 @@ Status Truncate(const std::string& filename, uint64_t length) {
struct FileState {
std::string filename_;
- ssize_t pos_;
- ssize_t pos_at_last_sync_;
- ssize_t pos_at_last_flush_;
+ int64_t pos_;
+ int64_t pos_at_last_sync_;
+ int64_t pos_at_last_flush_;
FileState(const std::string& filename)
: filename_(filename),
pos_(-1),
pos_at_last_sync_(-1),
- pos_at_last_flush_(-1) { }
+ pos_at_last_flush_(-1) {}
FileState() : pos_(-1), pos_at_last_sync_(-1), pos_at_last_flush_(-1) {}
@@ -106,14 +107,13 @@ struct FileState {
// is written to or sync'ed.
class TestWritableFile : public WritableFile {
public:
- TestWritableFile(const std::string& fname,
- WritableFile* f,
+ TestWritableFile(const FileState& state, WritableFile* f,
FaultInjectionTestEnv* env);
- virtual ~TestWritableFile();
- virtual Status Append(const Slice& data);
- virtual Status Close();
- virtual Status Flush();
- virtual Status Sync();
+ ~TestWritableFile() override;
+ Status Append(const Slice& data) override;
+ Status Close() override;
+ Status Flush() override;
+ Status Sync() override;
private:
FileState state_;
@@ -126,12 +126,15 @@ class TestWritableFile : public WritableFile {
class FaultInjectionTestEnv : public EnvWrapper {
public:
- FaultInjectionTestEnv() : EnvWrapper(Env::Default()), filesystem_active_(true) {}
- virtual ~FaultInjectionTestEnv() { }
- virtual Status NewWritableFile(const std::string& fname,
- WritableFile** result);
- virtual Status DeleteFile(const std::string& f);
- virtual Status RenameFile(const std::string& s, const std::string& t);
+ FaultInjectionTestEnv()
+ : EnvWrapper(Env::Default()), filesystem_active_(true) {}
+ ~FaultInjectionTestEnv() override = default;
+ Status NewWritableFile(const std::string& fname,
+ WritableFile** result) override;
+ Status NewAppendableFile(const std::string& fname,
+ WritableFile** result) override;
+ Status DeleteFile(const std::string& f) override;
+ Status RenameFile(const std::string& s, const std::string& t) override;
void WritableFileClosed(const FileState& state);
Status DropUnsyncedFileData();
@@ -144,25 +147,26 @@ class FaultInjectionTestEnv : public EnvWrapper {
// system reset. Setting to inactive will freeze our saved filesystem state so
// that it will stop being recorded. It can then be reset back to the state at
// the time of the reset.
- bool IsFilesystemActive() const { return filesystem_active_; }
- void SetFilesystemActive(bool active) { filesystem_active_ = active; }
+ bool IsFilesystemActive() LOCKS_EXCLUDED(mutex_) {
+ MutexLock l(&mutex_);
+ return filesystem_active_;
+ }
+ void SetFilesystemActive(bool active) LOCKS_EXCLUDED(mutex_) {
+ MutexLock l(&mutex_);
+ filesystem_active_ = active;
+ }
private:
port::Mutex mutex_;
- std::map<std::string, FileState> db_file_state_;
- std::set<std::string> new_files_since_last_dir_sync_;
- bool filesystem_active_; // Record flushes, syncs, writes
+ std::map<std::string, FileState> db_file_state_ GUARDED_BY(mutex_);
+ std::set<std::string> new_files_since_last_dir_sync_ GUARDED_BY(mutex_);
+ bool filesystem_active_ GUARDED_BY(mutex_); // Record flushes, syncs, writes
};
-TestWritableFile::TestWritableFile(const std::string& fname,
- WritableFile* f,
+TestWritableFile::TestWritableFile(const FileState& state, WritableFile* f,
FaultInjectionTestEnv* env)
- : state_(fname),
- target_(f),
- writable_file_opened_(true),
- env_(env) {
- assert(f != NULL);
- state_.pos_ = 0;
+ : state_(state), target_(f), writable_file_opened_(true), env_(env) {
+ assert(f != nullptr);
}
TestWritableFile::~TestWritableFile() {
@@ -228,9 +232,12 @@ Status FaultInjectionTestEnv::NewWritableFile(const std::string& fname,
WritableFile* actual_writable_file;
Status s = target()->NewWritableFile(fname, &actual_writable_file);
if (s.ok()) {
- *result = new TestWritableFile(fname, actual_writable_file, this);
- // WritableFile doesn't append to files, so if the same file is opened again
- // then it will be truncated - so forget our saved state.
+ FileState state(fname);
+ state.pos_ = 0;
+ *result = new TestWritableFile(state, actual_writable_file, this);
+ // NewWritableFile doesn't append to files, so if the same file is
+ // opened again then it will be truncated - so forget our saved
+ // state.
UntrackFile(fname);
MutexLock l(&mutex_);
new_files_since_last_dir_sync_.insert(fname);
@@ -238,13 +245,34 @@ Status FaultInjectionTestEnv::NewWritableFile(const std::string& fname,
return s;
}
+Status FaultInjectionTestEnv::NewAppendableFile(const std::string& fname,
+ WritableFile** result) {
+ WritableFile* actual_writable_file;
+ Status s = target()->NewAppendableFile(fname, &actual_writable_file);
+ if (s.ok()) {
+ FileState state(fname);
+ state.pos_ = 0;
+ {
+ MutexLock l(&mutex_);
+ if (db_file_state_.count(fname) == 0) {
+ new_files_since_last_dir_sync_.insert(fname);
+ } else {
+ state = db_file_state_[fname];
+ }
+ }
+ *result = new TestWritableFile(state, actual_writable_file, this);
+ }
+ return s;
+}
+
Status FaultInjectionTestEnv::DropUnsyncedFileData() {
Status s;
MutexLock l(&mutex_);
- for (std::map<std::string, FileState>::const_iterator it =
- db_file_state_.begin();
- s.ok() && it != db_file_state_.end(); ++it) {
- const FileState& state = it->second;
+ for (const auto& kvp : db_file_state_) {
+ if (!s.ok()) {
+ break;
+ }
+ const FileState& state = kvp.second;
if (!state.IsFullySynced()) {
s = state.DropUnsyncedData();
}
@@ -301,9 +329,9 @@ Status FaultInjectionTestEnv::RenameFile(const std::string& s,
}
void FaultInjectionTestEnv::ResetState() {
- MutexLock l(&mutex_);
- db_file_state_.clear();
- new_files_since_last_dir_sync_.clear();
+ // Since we are not destroying the database, the existing files
+ // should keep their recorded synced/flushed state. Therefore
+ // we do not reset db_file_state_ and new_files_since_last_dir_sync_.
SetFilesystemActive(true);
}
@@ -313,12 +341,14 @@ Status FaultInjectionTestEnv::DeleteFilesCreatedAfterLastDirSync() {
std::set<std::string> new_files(new_files_since_last_dir_sync_.begin(),
new_files_since_last_dir_sync_.end());
mutex_.Unlock();
- Status s;
- std::set<std::string>::const_iterator it;
- for (it = new_files.begin(); s.ok() && it != new_files.end(); ++it) {
- s = DeleteFile(*it);
+ Status status;
+ for (const auto& new_file : new_files) {
+ Status delete_status = DeleteFile(new_file);
+ if (!delete_status.ok() && status.ok()) {
+ status = std::move(delete_status);
+ }
}
- return s;
+ return status;
}
void FaultInjectionTestEnv::WritableFileClosed(const FileState& state) {
@@ -327,7 +357,7 @@ void FaultInjectionTestEnv::WritableFileClosed(const FileState& state) {
}
Status FileState::DropUnsyncedData() const {
- ssize_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_;
+ int64_t sync_pos = pos_at_last_sync_ == -1 ? 0 : pos_at_last_sync_;
return Truncate(filename_, sync_pos);
}
@@ -342,53 +372,28 @@ class FaultInjectionTest {
Options options_;
DB* db_;
- FaultInjectionTest() : env_(NULL), tiny_cache_(NULL), db_(NULL) { NewDB(); }
-
- ~FaultInjectionTest() { ASSERT_OK(TearDown()); }
-
- Status NewDB() {
- assert(db_ == NULL);
- assert(tiny_cache_ == NULL);
- assert(env_ == NULL);
-
- env_ = new FaultInjectionTestEnv();
-
- options_ = Options();
+ FaultInjectionTest()
+ : env_(new FaultInjectionTestEnv),
+ tiny_cache_(NewLRUCache(100)),
+ db_(nullptr) {
+ dbname_ = test::TmpDir() + "/fault_test";
+ DestroyDB(dbname_, Options()); // Destroy any db from earlier run
+ options_.reuse_logs = true;
options_.env = env_;
options_.paranoid_checks = true;
-
- tiny_cache_ = NewLRUCache(100);
options_.block_cache = tiny_cache_;
- dbname_ = test::TmpDir() + "/fault_test";
-
options_.create_if_missing = true;
- Status s = OpenDB();
- options_.create_if_missing = false;
- return s;
}
- Status SetUp() {
- Status s = TearDown();
- if (s.ok()) {
- s = NewDB();
- }
- return s;
- }
-
- Status TearDown() {
+ ~FaultInjectionTest() {
CloseDB();
-
- Status s = DestroyDB(dbname_, Options());
-
+ DestroyDB(dbname_, Options());
delete tiny_cache_;
- tiny_cache_ = NULL;
-
delete env_;
- env_ = NULL;
-
- return s;
}
+ void ReuseLogs(bool reuse) { options_.reuse_logs = reuse; }
+
void Build(int start_idx, int num_vals) {
std::string key_space, value_space;
WriteBatch batch;
@@ -447,19 +452,18 @@ class FaultInjectionTest {
Status OpenDB() {
delete db_;
- db_ = NULL;
+ db_ = nullptr;
env_->ResetState();
return DB::Open(options_, dbname_, &db_);
}
void CloseDB() {
delete db_;
- db_ = NULL;
+ db_ = nullptr;
}
void DeleteAllData() {
Iterator* iter = db_->NewIterator(ReadOptions());
- WriteOptions options;
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
ASSERT_OK(db_->Delete(WriteOptions(), iter->key()));
}
@@ -483,60 +487,65 @@ class FaultInjectionTest {
void PartialCompactTestPreFault(int num_pre_sync, int num_post_sync) {
DeleteAllData();
Build(0, num_pre_sync);
- db_->CompactRange(NULL, NULL);
+ db_->CompactRange(nullptr, nullptr);
Build(num_pre_sync, num_post_sync);
}
void PartialCompactTestReopenWithFault(ResetMethod reset_method,
- int num_pre_sync,
- int num_post_sync) {
+ int num_pre_sync, int num_post_sync) {
env_->SetFilesystemActive(false);
CloseDB();
ResetDBState(reset_method);
ASSERT_OK(OpenDB());
ASSERT_OK(Verify(0, num_pre_sync, FaultInjectionTest::VAL_EXPECT_NO_ERROR));
- ASSERT_OK(Verify(num_pre_sync, num_post_sync, FaultInjectionTest::VAL_EXPECT_ERROR));
+ ASSERT_OK(Verify(num_pre_sync, num_post_sync,
+ FaultInjectionTest::VAL_EXPECT_ERROR));
}
- void NoWriteTestPreFault() {
- }
+ void NoWriteTestPreFault() {}
void NoWriteTestReopenWithFault(ResetMethod reset_method) {
CloseDB();
ResetDBState(reset_method);
ASSERT_OK(OpenDB());
}
-};
-TEST(FaultInjectionTest, FaultTest) {
- Random rnd(0);
- ASSERT_OK(SetUp());
- for (size_t idx = 0; idx < kNumIterations; idx++) {
- int num_pre_sync = rnd.Uniform(kMaxNumValues);
- int num_post_sync = rnd.Uniform(kMaxNumValues);
+ void DoTest() {
+ Random rnd(0);
+ ASSERT_OK(OpenDB());
+ for (size_t idx = 0; idx < kNumIterations; idx++) {
+ int num_pre_sync = rnd.Uniform(kMaxNumValues);
+ int num_post_sync = rnd.Uniform(kMaxNumValues);
- PartialCompactTestPreFault(num_pre_sync, num_post_sync);
- PartialCompactTestReopenWithFault(RESET_DROP_UNSYNCED_DATA,
- num_pre_sync,
- num_post_sync);
+ PartialCompactTestPreFault(num_pre_sync, num_post_sync);
+ PartialCompactTestReopenWithFault(RESET_DROP_UNSYNCED_DATA, num_pre_sync,
+ num_post_sync);
- NoWriteTestPreFault();
- NoWriteTestReopenWithFault(RESET_DROP_UNSYNCED_DATA);
+ NoWriteTestPreFault();
+ NoWriteTestReopenWithFault(RESET_DROP_UNSYNCED_DATA);
- PartialCompactTestPreFault(num_pre_sync, num_post_sync);
- // No new files created so we expect all values since no files will be
- // dropped.
- PartialCompactTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES,
- num_pre_sync + num_post_sync,
- 0);
+ PartialCompactTestPreFault(num_pre_sync, num_post_sync);
+ // No new files created so we expect all values since no files will be
+ // dropped.
+ PartialCompactTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES,
+ num_pre_sync + num_post_sync, 0);
- NoWriteTestPreFault();
- NoWriteTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES);
+ NoWriteTestPreFault();
+ NoWriteTestReopenWithFault(RESET_DELETE_UNSYNCED_FILES);
+ }
}
+};
+
+TEST(FaultInjectionTest, FaultTestNoLogReuse) {
+ ReuseLogs(false);
+ DoTest();
+}
+
+TEST(FaultInjectionTest, FaultTestWithLogReuse) {
+ ReuseLogs(true);
+ DoTest();
}
} // namespace leveldb
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
+int main(int argc, char** argv) { return leveldb::test::RunAllTests(); }
diff --git a/db/filename.cc b/db/filename.cc
index da32946..85de45c 100644
--- a/db/filename.cc
+++ b/db/filename.cc
@@ -2,9 +2,11 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
+#include "db/filename.h"
+
#include <ctype.h>
#include <stdio.h>
-#include "db/filename.h"
+
#include "db/dbformat.h"
#include "leveldb/env.h"
#include "util/logging.h"
@@ -12,31 +14,30 @@
namespace leveldb {
// A utility routine: write "data" to the named file and Sync() it.
-extern Status WriteStringToFileSync(Env* env, const Slice& data,
- const std::string& fname);
+Status WriteStringToFileSync(Env* env, const Slice& data,
+ const std::string& fname);
-static std::string MakeFileName(const std::string& name, uint64_t number,
+static std::string MakeFileName(const std::string& dbname, uint64_t number,
const char* suffix) {
char buf[100];
snprintf(buf, sizeof(buf), "/%06llu.%s",
- static_cast<unsigned long long>(number),
- suffix);
- return name + buf;
+ static_cast<unsigned long long>(number), suffix);
+ return dbname + buf;
}
-std::string LogFileName(const std::string& name, uint64_t number) {
+std::string LogFileName(const std::string& dbname, uint64_t number) {
assert(number > 0);
- return MakeFileName(name, number, "log");
+ return MakeFileName(dbname, number, "log");
}
-std::string TableFileName(const std::string& name, uint64_t number) {
+std::string TableFileName(const std::string& dbname, uint64_t number) {
assert(number > 0);
- return MakeFileName(name, number, "ldb");
+ return MakeFileName(dbname, number, "ldb");
}
-std::string SSTTableFileName(const std::string& name, uint64_t number) {
+std::string SSTTableFileName(const std::string& dbname, uint64_t number) {
assert(number > 0);
- return MakeFileName(name, number, "sst");
+ return MakeFileName(dbname, number, "sst");
}
std::string DescriptorFileName(const std::string& dbname, uint64_t number) {
@@ -51,9 +52,7 @@ std::string CurrentFileName(const std::string& dbname) {
return dbname + "/CURRENT";
}
-std::string LockFileName(const std::string& dbname) {
- return dbname + "/LOCK";
-}
+std::string LockFileName(const std::string& dbname) { return dbname + "/LOCK"; }
std::string TempFileName(const std::string& dbname, uint64_t number) {
assert(number > 0);
@@ -69,7 +68,6 @@ std::string OldInfoLogFileName(const std::string& dbname) {
return dbname + "/LOG.old";
}
-
// Owned filenames have the form:
// dbname/CURRENT
// dbname/LOCK
@@ -77,10 +75,9 @@ std::string OldInfoLogFileName(const std::string& dbname) {
// dbname/LOG.old
// dbname/MANIFEST-[0-9]+
// dbname/[0-9]+.(log|sst|ldb)
-bool ParseFileName(const std::string& fname,
- uint64_t* number,
+bool ParseFileName(const std::string& filename, uint64_t* number,
FileType* type) {
- Slice rest(fname);
+ Slice rest(filename);
if (rest == "CURRENT") {
*number = 0;
*type = kCurrentFile;
diff --git a/db/filename.h b/db/filename.h
index 87a7526..524e813 100644
--- a/db/filename.h
+++ b/db/filename.h
@@ -8,7 +8,9 @@
#define STORAGE_LEVELDB_DB_FILENAME_H_
#include <stdint.h>
+
#include <string>
+
#include "leveldb/slice.h"
#include "leveldb/status.h"
#include "port/port.h"
@@ -30,55 +32,52 @@ enum FileType {
// Return the name of the log file with the specified number
// in the db named by "dbname". The result will be prefixed with
// "dbname".
-extern std::string LogFileName(const std::string& dbname, uint64_t number);
+std::string LogFileName(const std::string& dbname, uint64_t number);
// Return the name of the sstable with the specified number
// in the db named by "dbname". The result will be prefixed with
// "dbname".
-extern std::string TableFileName(const std::string& dbname, uint64_t number);
+std::string TableFileName(const std::string& dbname, uint64_t number);
// Return the legacy file name for an sstable with the specified number
// in the db named by "dbname". The result will be prefixed with
// "dbname".
-extern std::string SSTTableFileName(const std::string& dbname, uint64_t number);
+std::string SSTTableFileName(const std::string& dbname, uint64_t number);
// Return the name of the descriptor file for the db named by
// "dbname" and the specified incarnation number. The result will be
// prefixed with "dbname".
-extern std::string DescriptorFileName(const std::string& dbname,
- uint64_t number);
+std::string DescriptorFileName(const std::string& dbname, uint64_t number);
// Return the name of the current file. This file contains the name
// of the current manifest file. The result will be prefixed with
// "dbname".
-extern std::string CurrentFileName(const std::string& dbname);
+std::string CurrentFileName(const std::string& dbname);
// Return the name of the lock file for the db named by
// "dbname". The result will be prefixed with "dbname".
-extern std::string LockFileName(const std::string& dbname);
+std::string LockFileName(const std::string& dbname);
// Return the name of a temporary file owned by the db named "dbname".
// The result will be prefixed with "dbname".
-extern std::string TempFileName(const std::string& dbname, uint64_t number);
+std::string TempFileName(const std::string& dbname, uint64_t number);
// Return the name of the info log file for "dbname".
-extern std::string InfoLogFileName(const std::string& dbname);
+std::string InfoLogFileName(const std::string& dbname);
// Return the name of the old info log file for "dbname".
-extern std::string OldInfoLogFileName(const std::string& dbname);
+std::string OldInfoLogFileName(const std::string& dbname);
// If filename is a leveldb file, store the type of the file in *type.
// The number encoded in the filename is stored in *number. If the
// filename was successfully parsed, returns true. Else return false.
-extern bool ParseFileName(const std::string& filename,
- uint64_t* number,
- FileType* type);
+bool ParseFileName(const std::string& filename, uint64_t* number,
+ FileType* type);
// Make the CURRENT file point to the descriptor file with the
// specified number.
-extern Status SetCurrentFile(Env* env, const std::string& dbname,
- uint64_t descriptor_number);
-
+Status SetCurrentFile(Env* env, const std::string& dbname,
+ uint64_t descriptor_number);
} // namespace leveldb
diff --git a/db/filename_test.cc b/db/filename_test.cc
index a32556d..952f320 100644
--- a/db/filename_test.cc
+++ b/db/filename_test.cc
@@ -11,7 +11,7 @@
namespace leveldb {
-class FileNameTest { };
+class FileNameTest {};
TEST(FileNameTest, Parse) {
Slice db;
@@ -24,17 +24,17 @@ TEST(FileNameTest, Parse) {
uint64_t number;
FileType type;
} cases[] = {
- { "100.log", 100, kLogFile },
- { "0.log", 0, kLogFile },
- { "0.sst", 0, kTableFile },
- { "0.ldb", 0, kTableFile },
- { "CURRENT", 0, kCurrentFile },
- { "LOCK", 0, kDBLockFile },
- { "MANIFEST-2", 2, kDescriptorFile },
- { "MANIFEST-7", 7, kDescriptorFile },
- { "LOG", 0, kInfoLogFile },
- { "LOG.old", 0, kInfoLogFile },
- { "18446744073709551615.log", 18446744073709551615ull, kLogFile },
+ {"100.log", 100, kLogFile},
+ {"0.log", 0, kLogFile},
+ {"0.sst", 0, kTableFile},
+ {"0.ldb", 0, kTableFile},
+ {"CURRENT", 0, kCurrentFile},
+ {"LOCK", 0, kDBLockFile},
+ {"MANIFEST-2", 2, kDescriptorFile},
+ {"MANIFEST-7", 7, kDescriptorFile},
+ {"LOG", 0, kInfoLogFile},
+ {"LOG.old", 0, kInfoLogFile},
+ {"18446744073709551615.log", 18446744073709551615ull, kLogFile},
};
for (int i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
std::string f = cases[i].fname;
@@ -44,30 +44,28 @@ TEST(FileNameTest, Parse) {
}
// Errors
- static const char* errors[] = {
- "",
- "foo",
- "foo-dx-100.log",
- ".log",
- "",
- "manifest",
- "CURREN",
- "CURRENTX",
- "MANIFES",
- "MANIFEST",
- "MANIFEST-",
- "XMANIFEST-3",
- "MANIFEST-3x",
- "LOC",
- "LOCKx",
- "LO",
- "LOGx",
- "18446744073709551616.log",
- "184467440737095516150.log",
- "100",
- "100.",
- "100.lop"
- };
+ static const char* errors[] = {"",
+ "foo",
+ "foo-dx-100.log",
+ ".log",
+ "",
+ "manifest",
+ "CURREN",
+ "CURRENTX",
+ "MANIFES",
+ "MANIFEST",
+ "MANIFEST-",
+ "XMANIFEST-3",
+ "MANIFEST-3x",
+ "LOC",
+ "LOCKx",
+ "LO",
+ "LOGx",
+ "18446744073709551616.log",
+ "184467440737095516150.log",
+ "100",
+ "100.",
+ "100.lop"};
for (int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) {
std::string f = errors[i];
ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f;
@@ -114,10 +112,20 @@ TEST(FileNameTest, Construction) {
ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type));
ASSERT_EQ(999, number);
ASSERT_EQ(kTempFile, type);
+
+ fname = InfoLogFileName("foo");
+ ASSERT_EQ("foo/", std::string(fname.data(), 4));
+ ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type));
+ ASSERT_EQ(0, number);
+ ASSERT_EQ(kInfoLogFile, type);
+
+ fname = OldInfoLogFileName("foo");
+ ASSERT_EQ("foo/", std::string(fname.data(), 4));
+ ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type));
+ ASSERT_EQ(0, number);
+ ASSERT_EQ(kInfoLogFile, type);
}
} // namespace leveldb
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
+int main(int argc, char** argv) { return leveldb::test::RunAllTests(); }
diff --git a/db/leveldb_main.cc b/db/leveldbutil.cc
index 9f4b7dd..55cdcc5 100644
--- a/db/leveldb_main.cc
+++ b/db/leveldbutil.cc
@@ -3,6 +3,7 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include <stdio.h>
+
#include "leveldb/dumpfile.h"
#include "leveldb/env.h"
#include "leveldb/status.h"
@@ -12,13 +13,13 @@ namespace {
class StdoutPrinter : public WritableFile {
public:
- virtual Status Append(const Slice& data) {
+ Status Append(const Slice& data) override {
fwrite(data.data(), 1, data.size(), stdout);
return Status::OK();
}
- virtual Status Close() { return Status::OK(); }
- virtual Status Flush() { return Status::OK(); }
- virtual Status Sync() { return Status::OK(); }
+ Status Close() override { return Status::OK(); }
+ Status Flush() override { return Status::OK(); }
+ Status Sync() override { return Status::OK(); }
};
bool HandleDumpCommand(Env* env, char** files, int num) {
@@ -38,11 +39,9 @@ bool HandleDumpCommand(Env* env, char** files, int num) {
} // namespace leveldb
static void Usage() {
- fprintf(
- stderr,
- "Usage: leveldbutil command...\n"
- " dump files... -- dump contents of specified files\n"
- );
+ fprintf(stderr,
+ "Usage: leveldbutil command...\n"
+ " dump files... -- dump contents of specified files\n");
}
int main(int argc, char** argv) {
@@ -54,7 +53,7 @@ int main(int argc, char** argv) {
} else {
std::string command = argv[1];
if (command == "dump") {
- ok = leveldb::HandleDumpCommand(env, argv+2, argc-2);
+ ok = leveldb::HandleDumpCommand(env, argv + 2, argc - 2);
} else {
Usage();
ok = false;
diff --git a/db/log_format.h b/db/log_format.h
index a8c06ef..356e69f 100644
--- a/db/log_format.h
+++ b/db/log_format.h
@@ -3,7 +3,7 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors.
//
// Log format information shared by reader and writer.
-// See ../doc/log_format.txt for more detail.
+// See ../doc/log_format.md for more detail.
#ifndef STORAGE_LEVELDB_DB_LOG_FORMAT_H_
#define STORAGE_LEVELDB_DB_LOG_FORMAT_H_
diff --git a/db/log_reader.cc b/db/log_reader.cc
index e44b66c..b770fee 100644
--- a/db/log_reader.cc
+++ b/db/log_reader.cc
@@ -5,6 +5,7 @@
#include "db/log_reader.h"
#include <stdio.h>
+
#include "leveldb/env.h"
#include "util/coding.h"
#include "util/crc32c.h"
@@ -12,8 +13,7 @@
namespace leveldb {
namespace log {
-Reader::Reporter::~Reporter() {
-}
+Reader::Reporter::~Reporter() = default;
Reader::Reader(SequentialFile* file, Reporter* reporter, bool checksum,
uint64_t initial_offset)
@@ -25,20 +25,17 @@ Reader::Reader(SequentialFile* file, Reporter* reporter, bool checksum,
eof_(false),
last_record_offset_(0),
end_of_buffer_offset_(0),
- initial_offset_(initial_offset) {
-}
+ initial_offset_(initial_offset),
+ resyncing_(initial_offset > 0) {}
-Reader::~Reader() {
- delete[] backing_store_;
-}
+Reader::~Reader() { delete[] backing_store_; }
bool Reader::SkipToInitialBlock() {
- size_t offset_in_block = initial_offset_ % kBlockSize;
+ const size_t offset_in_block = initial_offset_ % kBlockSize;
uint64_t block_start_location = initial_offset_ - offset_in_block;
// Don't search a block if we'd be in the trailer
if (offset_in_block > kBlockSize - 6) {
- offset_in_block = 0;
block_start_location += kBlockSize;
}
@@ -72,8 +69,25 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) {
Slice fragment;
while (true) {
- uint64_t physical_record_offset = end_of_buffer_offset_ - buffer_.size();
const unsigned int record_type = ReadPhysicalRecord(&fragment);
+
+ // ReadPhysicalRecord may have only had an empty trailer remaining in its
+ // internal buffer. Calculate the offset of the next physical record now
+ // that it has returned, properly accounting for its header size.
+ uint64_t physical_record_offset =
+ end_of_buffer_offset_ - buffer_.size() - kHeaderSize - fragment.size();
+
+ if (resyncing_) {
+ if (record_type == kMiddleType) {
+ continue;
+ } else if (record_type == kLastType) {
+ resyncing_ = false;
+ continue;
+ } else {
+ resyncing_ = false;
+ }
+ }
+
switch (record_type) {
case kFullType:
if (in_fragmented_record) {
@@ -81,9 +95,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) {
// it could emit an empty kFirstType record at the tail end
// of a block followed by a kFullType or kFirstType record
// at the beginning of the next block.
- if (scratch->empty()) {
- in_fragmented_record = false;
- } else {
+ if (!scratch->empty()) {
ReportCorruption(scratch->size(), "partial record without end(1)");
}
}
@@ -99,9 +111,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) {
// it could emit an empty kFirstType record at the tail end
// of a block followed by a kFullType or kFirstType record
// at the beginning of the next block.
- if (scratch->empty()) {
- in_fragmented_record = false;
- } else {
+ if (!scratch->empty()) {
ReportCorruption(scratch->size(), "partial record without end(2)");
}
}
@@ -163,16 +173,14 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) {
return false;
}
-uint64_t Reader::LastRecordOffset() {
- return last_record_offset_;
-}
+uint64_t Reader::LastRecordOffset() { return last_record_offset_; }
void Reader::ReportCorruption(uint64_t bytes, const char* reason) {
ReportDrop(bytes, Status::Corruption(reason));
}
void Reader::ReportDrop(uint64_t bytes, const Status& reason) {
- if (reporter_ != NULL &&
+ if (reporter_ != nullptr &&
end_of_buffer_offset_ - buffer_.size() - bytes >= initial_offset_) {
reporter_->Corruption(static_cast<size_t>(bytes), reason);
}
diff --git a/db/log_reader.h b/db/log_reader.h
index 6aff791..001da89 100644
--- a/db/log_reader.h
+++ b/db/log_reader.h
@@ -32,7 +32,7 @@ class Reader {
// Create a reader that will return log records from "*file".
// "*file" must remain live while this Reader is in use.
//
- // If "reporter" is non-NULL, it is notified whenever some data is
+ // If "reporter" is non-null, it is notified whenever some data is
// dropped due to a detected corruption. "*reporter" must remain
// live while this Reader is in use.
//
@@ -43,6 +43,9 @@ class Reader {
Reader(SequentialFile* file, Reporter* reporter, bool checksum,
uint64_t initial_offset);
+ Reader(const Reader&) = delete;
+ Reader& operator=(const Reader&) = delete;
+
~Reader();
// Read the next record into *record. Returns true if read
@@ -58,21 +61,6 @@ class Reader {
uint64_t LastRecordOffset();
private:
- SequentialFile* const file_;
- Reporter* const reporter_;
- bool const checksum_;
- char* const backing_store_;
- Slice buffer_;
- bool eof_; // Last Read() indicated EOF by returning < kBlockSize
-
- // Offset of the last record returned by ReadRecord.
- uint64_t last_record_offset_;
- // Offset of the first location past the end of buffer_.
- uint64_t end_of_buffer_offset_;
-
- // Offset at which to start looking for the first record to return
- uint64_t const initial_offset_;
-
// Extend record types with the following special values
enum {
kEof = kMaxRecordType + 1,
@@ -97,9 +85,25 @@ class Reader {
void ReportCorruption(uint64_t bytes, const char* reason);
void ReportDrop(uint64_t bytes, const Status& reason);
- // No copying allowed
- Reader(const Reader&);
- void operator=(const Reader&);
+ SequentialFile* const file_;
+ Reporter* const reporter_;
+ bool const checksum_;
+ char* const backing_store_;
+ Slice buffer_;
+ bool eof_; // Last Read() indicated EOF by returning < kBlockSize
+
+ // Offset of the last record returned by ReadRecord.
+ uint64_t last_record_offset_;
+ // Offset of the first location past the end of buffer_.
+ uint64_t end_of_buffer_offset_;
+
+ // Offset at which to start looking for the first record to return
+ uint64_t const initial_offset_;
+
+ // True if we are resynchronizing after a seek (initial_offset_ > 0). In
+ // particular, a run of kMiddleType and kLastType records can be silently
+ // skipped in this mode
+ bool resyncing_;
};
} // namespace log
diff --git a/db/log_test.cc b/db/log_test.cc
index dcf0562..0e31648 100644
--- a/db/log_test.cc
+++ b/db/log_test.cc
@@ -37,95 +37,29 @@ static std::string RandomSkewedString(int i, Random* rnd) {
}
class LogTest {
- private:
- class StringDest : public WritableFile {
- public:
- std::string contents_;
-
- virtual Status Close() { return Status::OK(); }
- virtual Status Flush() { return Status::OK(); }
- virtual Status Sync() { return Status::OK(); }
- virtual Status Append(const Slice& slice) {
- contents_.append(slice.data(), slice.size());
- return Status::OK();
- }
- };
-
- class StringSource : public SequentialFile {
- public:
- Slice contents_;
- bool force_error_;
- bool returned_partial_;
- StringSource() : force_error_(false), returned_partial_(false) { }
-
- virtual Status Read(size_t n, Slice* result, char* scratch) {
- ASSERT_TRUE(!returned_partial_) << "must not Read() after eof/error";
-
- if (force_error_) {
- force_error_ = false;
- returned_partial_ = true;
- return Status::Corruption("read error");
- }
-
- if (contents_.size() < n) {
- n = contents_.size();
- returned_partial_ = true;
- }
- *result = Slice(contents_.data(), n);
- contents_.remove_prefix(n);
- return Status::OK();
- }
-
- virtual Status Skip(uint64_t n) {
- if (n > contents_.size()) {
- contents_.clear();
- return Status::NotFound("in-memory file skipepd past end");
- }
-
- contents_.remove_prefix(n);
-
- return Status::OK();
- }
- };
-
- class ReportCollector : public Reader::Reporter {
- public:
- size_t dropped_bytes_;
- std::string message_;
-
- ReportCollector() : dropped_bytes_(0) { }
- virtual void Corruption(size_t bytes, const Status& status) {
- dropped_bytes_ += bytes;
- message_.append(status.ToString());
- }
- };
-
- StringDest dest_;
- StringSource source_;
- ReportCollector report_;
- bool reading_;
- Writer writer_;
- Reader reader_;
-
- // Record metadata for testing initial offset functionality
- static size_t initial_offset_record_sizes_[];
- static uint64_t initial_offset_last_record_offsets_[];
-
public:
- LogTest() : reading_(false),
- writer_(&dest_),
- reader_(&source_, &report_, true/*checksum*/,
- 0/*initial_offset*/) {
+ LogTest()
+ : reading_(false),
+ writer_(new Writer(&dest_)),
+ reader_(new Reader(&source_, &report_, true /*checksum*/,
+ 0 /*initial_offset*/)) {}
+
+ ~LogTest() {
+ delete writer_;
+ delete reader_;
+ }
+
+ void ReopenForAppend() {
+ delete writer_;
+ writer_ = new Writer(&dest_, dest_.contents_.size());
}
void Write(const std::string& msg) {
ASSERT_TRUE(!reading_) << "Write() after starting to read";
- writer_.AddRecord(Slice(msg));
+ writer_->AddRecord(Slice(msg));
}
- size_t WrittenBytes() const {
- return dest_.contents_.size();
- }
+ size_t WrittenBytes() const { return dest_.contents_.size(); }
std::string Read() {
if (!reading_) {
@@ -134,7 +68,7 @@ class LogTest {
}
std::string scratch;
Slice record;
- if (reader_.ReadRecord(&record, &scratch)) {
+ if (reader_->ReadRecord(&record, &scratch)) {
return record.ToString();
} else {
return "EOF";
@@ -155,22 +89,16 @@ class LogTest {
void FixChecksum(int header_offset, int len) {
// Compute crc of type/len/data
- uint32_t crc = crc32c::Value(&dest_.contents_[header_offset+6], 1 + len);
+ uint32_t crc = crc32c::Value(&dest_.contents_[header_offset + 6], 1 + len);
crc = crc32c::Mask(crc);
EncodeFixed32(&dest_.contents_[header_offset], crc);
}
- void ForceError() {
- source_.force_error_ = true;
- }
+ void ForceError() { source_.force_error_ = true; }
- size_t DroppedBytes() const {
- return report_.dropped_bytes_;
- }
+ size_t DroppedBytes() const { return report_.dropped_bytes_; }
- std::string ReportMessage() const {
- return report_.message_;
- }
+ std::string ReportMessage() const { return report_.message_; }
// Returns OK iff recorded error message contains "msg"
std::string MatchError(const std::string& msg) const {
@@ -182,18 +110,23 @@ class LogTest {
}
void WriteInitialOffsetLog() {
- for (int i = 0; i < 4; i++) {
+ for (int i = 0; i < num_initial_offset_records_; i++) {
std::string record(initial_offset_record_sizes_[i],
static_cast<char>('a' + i));
Write(record);
}
}
+ void StartReadingAt(uint64_t initial_offset) {
+ delete reader_;
+ reader_ = new Reader(&source_, &report_, true /*checksum*/, initial_offset);
+ }
+
void CheckOffsetPastEndReturnsNoRecords(uint64_t offset_past_end) {
WriteInitialOffsetLog();
reading_ = true;
source_.contents_ = Slice(dest_.contents_);
- Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/,
+ Reader* offset_reader = new Reader(&source_, &report_, true /*checksum*/,
WrittenBytes() + offset_past_end);
Slice record;
std::string scratch;
@@ -206,38 +139,126 @@ class LogTest {
WriteInitialOffsetLog();
reading_ = true;
source_.contents_ = Slice(dest_.contents_);
- Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/,
- initial_offset);
- Slice record;
- std::string scratch;
- ASSERT_TRUE(offset_reader->ReadRecord(&record, &scratch));
- ASSERT_EQ(initial_offset_record_sizes_[expected_record_offset],
- record.size());
- ASSERT_EQ(initial_offset_last_record_offsets_[expected_record_offset],
- offset_reader->LastRecordOffset());
- ASSERT_EQ((char)('a' + expected_record_offset), record.data()[0]);
+ Reader* offset_reader =
+ new Reader(&source_, &report_, true /*checksum*/, initial_offset);
+
+ // Read all records from expected_record_offset through the last one.
+ ASSERT_LT(expected_record_offset, num_initial_offset_records_);
+ for (; expected_record_offset < num_initial_offset_records_;
+ ++expected_record_offset) {
+ Slice record;
+ std::string scratch;
+ ASSERT_TRUE(offset_reader->ReadRecord(&record, &scratch));
+ ASSERT_EQ(initial_offset_record_sizes_[expected_record_offset],
+ record.size());
+ ASSERT_EQ(initial_offset_last_record_offsets_[expected_record_offset],
+ offset_reader->LastRecordOffset());
+ ASSERT_EQ((char)('a' + expected_record_offset), record.data()[0]);
+ }
delete offset_reader;
}
+ private:
+ class StringDest : public WritableFile {
+ public:
+ Status Close() override { return Status::OK(); }
+ Status Flush() override { return Status::OK(); }
+ Status Sync() override { return Status::OK(); }
+ Status Append(const Slice& slice) override {
+ contents_.append(slice.data(), slice.size());
+ return Status::OK();
+ }
+
+ std::string contents_;
+ };
+
+ class StringSource : public SequentialFile {
+ public:
+ StringSource() : force_error_(false), returned_partial_(false) {}
+
+ Status Read(size_t n, Slice* result, char* scratch) override {
+ ASSERT_TRUE(!returned_partial_) << "must not Read() after eof/error";
+
+ if (force_error_) {
+ force_error_ = false;
+ returned_partial_ = true;
+ return Status::Corruption("read error");
+ }
+
+ if (contents_.size() < n) {
+ n = contents_.size();
+ returned_partial_ = true;
+ }
+ *result = Slice(contents_.data(), n);
+ contents_.remove_prefix(n);
+ return Status::OK();
+ }
+
+ Status Skip(uint64_t n) override {
+ if (n > contents_.size()) {
+ contents_.clear();
+ return Status::NotFound("in-memory file skipped past end");
+ }
+
+ contents_.remove_prefix(n);
+
+ return Status::OK();
+ }
+
+ Slice contents_;
+ bool force_error_;
+ bool returned_partial_;
+ };
+
+ class ReportCollector : public Reader::Reporter {
+ public:
+ ReportCollector() : dropped_bytes_(0) {}
+ void Corruption(size_t bytes, const Status& status) override {
+ dropped_bytes_ += bytes;
+ message_.append(status.ToString());
+ }
+
+ size_t dropped_bytes_;
+ std::string message_;
+ };
+
+ // Record metadata for testing initial offset functionality
+ static size_t initial_offset_record_sizes_[];
+ static uint64_t initial_offset_last_record_offsets_[];
+ static int num_initial_offset_records_;
+
+ StringDest dest_;
+ StringSource source_;
+ ReportCollector report_;
+ bool reading_;
+ Writer* writer_;
+ Reader* reader_;
};
-size_t LogTest::initial_offset_record_sizes_[] =
- {10000, // Two sizable records in first block
- 10000,
- 2 * log::kBlockSize - 1000, // Span three blocks
- 1};
+size_t LogTest::initial_offset_record_sizes_[] = {
+ 10000, // Two sizable records in first block
+ 10000,
+ 2 * log::kBlockSize - 1000, // Span three blocks
+ 1,
+ 13716, // Consume all but two bytes of block 3.
+ log::kBlockSize - kHeaderSize, // Consume the entirety of block 4.
+};
-uint64_t LogTest::initial_offset_last_record_offsets_[] =
- {0,
- kHeaderSize + 10000,
- 2 * (kHeaderSize + 10000),
- 2 * (kHeaderSize + 10000) +
- (2 * log::kBlockSize - 1000) + 3 * kHeaderSize};
+uint64_t LogTest::initial_offset_last_record_offsets_[] = {
+ 0,
+ kHeaderSize + 10000,
+ 2 * (kHeaderSize + 10000),
+ 2 * (kHeaderSize + 10000) + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize,
+ 2 * (kHeaderSize + 10000) + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize +
+ kHeaderSize + 1,
+ 3 * log::kBlockSize,
+};
+// LogTest::initial_offset_last_record_offsets_ must be defined before this.
+int LogTest::num_initial_offset_records_ =
+ sizeof(LogTest::initial_offset_last_record_offsets_) / sizeof(uint64_t);
-TEST(LogTest, Empty) {
- ASSERT_EQ("EOF", Read());
-}
+TEST(LogTest, Empty) { ASSERT_EQ("EOF", Read()); }
TEST(LogTest, ReadWrite) {
Write("foo");
@@ -274,7 +295,7 @@ TEST(LogTest, Fragmentation) {
TEST(LogTest, MarginalTrailer) {
// Make a trailer that is exactly the same length as an empty record.
- const int n = kBlockSize - 2*kHeaderSize;
+ const int n = kBlockSize - 2 * kHeaderSize;
Write(BigString("foo", n));
ASSERT_EQ(kBlockSize - kHeaderSize, WrittenBytes());
Write("");
@@ -287,7 +308,7 @@ TEST(LogTest, MarginalTrailer) {
TEST(LogTest, MarginalTrailer2) {
// Make a trailer that is exactly the same length as an empty record.
- const int n = kBlockSize - 2*kHeaderSize;
+ const int n = kBlockSize - 2 * kHeaderSize;
Write(BigString("foo", n));
ASSERT_EQ(kBlockSize - kHeaderSize, WrittenBytes());
Write("bar");
@@ -299,7 +320,7 @@ TEST(LogTest, MarginalTrailer2) {
}
TEST(LogTest, ShortTrailer) {
- const int n = kBlockSize - 2*kHeaderSize + 4;
+ const int n = kBlockSize - 2 * kHeaderSize + 4;
Write(BigString("foo", n));
ASSERT_EQ(kBlockSize - kHeaderSize + 4, WrittenBytes());
Write("");
@@ -311,13 +332,22 @@ TEST(LogTest, ShortTrailer) {
}
TEST(LogTest, AlignedEof) {
- const int n = kBlockSize - 2*kHeaderSize + 4;
+ const int n = kBlockSize - 2 * kHeaderSize + 4;
Write(BigString("foo", n));
ASSERT_EQ(kBlockSize - kHeaderSize + 4, WrittenBytes());
ASSERT_EQ(BigString("foo", n), Read());
ASSERT_EQ("EOF", Read());
}
+TEST(LogTest, OpenForAppend) {
+ Write("hello");
+ ReopenForAppend();
+ Write("world");
+ ASSERT_EQ("hello", Read());
+ ASSERT_EQ("world", Read());
+ ASSERT_EQ("EOF", Read());
+}
+
TEST(LogTest, RandomRead) {
const int N = 500;
Random write_rnd(301);
@@ -353,7 +383,7 @@ TEST(LogTest, BadRecordType) {
TEST(LogTest, TruncatedTrailingRecordIsIgnored) {
Write("foo");
- ShrinkSize(4); // Drop all payload as well as a header byte
+ ShrinkSize(4); // Drop all payload as well as a header byte
ASSERT_EQ("EOF", Read());
// Truncated last record is ignored, not treated as an error.
ASSERT_EQ(0, DroppedBytes());
@@ -445,6 +475,22 @@ TEST(LogTest, PartialLastIsIgnored) {
ASSERT_EQ(0, DroppedBytes());
}
+TEST(LogTest, SkipIntoMultiRecord) {
+ // Consider a fragmented record:
+ // first(R1), middle(R1), last(R1), first(R2)
+ // If initial_offset points to a record after first(R1) but before first(R2)
+ // incomplete fragment errors are not actual errors, and must be suppressed
+ // until a new first or full record is encountered.
+ Write(BigString("foo", 3 * kBlockSize));
+ Write("correct");
+ StartReadingAt(kBlockSize);
+
+ ASSERT_EQ("correct", Read());
+ ASSERT_EQ("", ReportMessage());
+ ASSERT_EQ(0, DroppedBytes());
+ ASSERT_EQ("EOF", Read());
+}
+
TEST(LogTest, ErrorJoinsRecords) {
// Consider two fragmented records:
// first(R1) last(R1) first(R2) last(R2)
@@ -457,44 +503,30 @@ TEST(LogTest, ErrorJoinsRecords) {
Write("correct");
// Wipe the middle block
- for (int offset = kBlockSize; offset < 2*kBlockSize; offset++) {
+ for (int offset = kBlockSize; offset < 2 * kBlockSize; offset++) {
SetByte(offset, 'x');
}
ASSERT_EQ("correct", Read());
ASSERT_EQ("EOF", Read());
const size_t dropped = DroppedBytes();
- ASSERT_LE(dropped, 2*kBlockSize + 100);
- ASSERT_GE(dropped, 2*kBlockSize);
+ ASSERT_LE(dropped, 2 * kBlockSize + 100);
+ ASSERT_GE(dropped, 2 * kBlockSize);
}
-TEST(LogTest, ReadStart) {
- CheckInitialOffsetRecord(0, 0);
-}
+TEST(LogTest, ReadStart) { CheckInitialOffsetRecord(0, 0); }
-TEST(LogTest, ReadSecondOneOff) {
- CheckInitialOffsetRecord(1, 1);
-}
+TEST(LogTest, ReadSecondOneOff) { CheckInitialOffsetRecord(1, 1); }
-TEST(LogTest, ReadSecondTenThousand) {
- CheckInitialOffsetRecord(10000, 1);
-}
+TEST(LogTest, ReadSecondTenThousand) { CheckInitialOffsetRecord(10000, 1); }
-TEST(LogTest, ReadSecondStart) {
- CheckInitialOffsetRecord(10007, 1);
-}
+TEST(LogTest, ReadSecondStart) { CheckInitialOffsetRecord(10007, 1); }
-TEST(LogTest, ReadThirdOneOff) {
- CheckInitialOffsetRecord(10008, 2);
-}
+TEST(LogTest, ReadThirdOneOff) { CheckInitialOffsetRecord(10008, 2); }
-TEST(LogTest, ReadThirdStart) {
- CheckInitialOffsetRecord(20014, 2);
-}
+TEST(LogTest, ReadThirdStart) { CheckInitialOffsetRecord(20014, 2); }
-TEST(LogTest, ReadFourthOneOff) {
- CheckInitialOffsetRecord(20015, 3);
-}
+TEST(LogTest, ReadFourthOneOff) { CheckInitialOffsetRecord(20015, 3); }
TEST(LogTest, ReadFourthFirstBlockTrailer) {
CheckInitialOffsetRecord(log::kBlockSize - 4, 3);
@@ -514,17 +546,15 @@ TEST(LogTest, ReadFourthStart) {
3);
}
-TEST(LogTest, ReadEnd) {
- CheckOffsetPastEndReturnsNoRecords(0);
+TEST(LogTest, ReadInitialOffsetIntoBlockPadding) {
+ CheckInitialOffsetRecord(3 * log::kBlockSize - 3, 5);
}
-TEST(LogTest, ReadPastEnd) {
- CheckOffsetPastEndReturnsNoRecords(5);
-}
+TEST(LogTest, ReadEnd) { CheckOffsetPastEndReturnsNoRecords(0); }
+
+TEST(LogTest, ReadPastEnd) { CheckOffsetPastEndReturnsNoRecords(5); }
} // namespace log
} // namespace leveldb
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
+int main(int argc, char** argv) { return leveldb::test::RunAllTests(); }
diff --git a/db/log_writer.cc b/db/log_writer.cc
index 2da99ac..bfb16fb 100644
--- a/db/log_writer.cc
+++ b/db/log_writer.cc
@@ -5,6 +5,7 @@
#include "db/log_writer.h"
#include <stdint.h>
+
#include "leveldb/env.h"
#include "util/coding.h"
#include "util/crc32c.h"
@@ -12,18 +13,24 @@
namespace leveldb {
namespace log {
-Writer::Writer(WritableFile* dest)
- : dest_(dest),
- block_offset_(0) {
+static void InitTypeCrc(uint32_t* type_crc) {
for (int i = 0; i <= kMaxRecordType; i++) {
char t = static_cast<char>(i);
- type_crc_[i] = crc32c::Value(&t, 1);
+ type_crc[i] = crc32c::Value(&t, 1);
}
}
-Writer::~Writer() {
+Writer::Writer(WritableFile* dest) : dest_(dest), block_offset_(0) {
+ InitTypeCrc(type_crc_);
+}
+
+Writer::Writer(WritableFile* dest, uint64_t dest_length)
+ : dest_(dest), block_offset_(dest_length % kBlockSize) {
+ InitTypeCrc(type_crc_);
}
+Writer::~Writer() = default;
+
Status Writer::AddRecord(const Slice& slice) {
const char* ptr = slice.data();
size_t left = slice.size();
@@ -40,7 +47,7 @@ Status Writer::AddRecord(const Slice& slice) {
// Switch to a new block
if (leftover > 0) {
// Fill the trailer (literal below relies on kHeaderSize being 7)
- assert(kHeaderSize == 7);
+ static_assert(kHeaderSize == 7, "");
dest_->Append(Slice("\x00\x00\x00\x00\x00\x00", leftover));
}
block_offset_ = 0;
@@ -72,30 +79,31 @@ Status Writer::AddRecord(const Slice& slice) {
return s;
}
-Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n) {
- assert(n <= 0xffff); // Must fit in two bytes
- assert(block_offset_ + kHeaderSize + n <= kBlockSize);
+Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr,
+ size_t length) {
+ assert(length <= 0xffff); // Must fit in two bytes
+ assert(block_offset_ + kHeaderSize + length <= kBlockSize);
// Format the header
char buf[kHeaderSize];
- buf[4] = static_cast<char>(n & 0xff);
- buf[5] = static_cast<char>(n >> 8);
+ buf[4] = static_cast<char>(length & 0xff);
+ buf[5] = static_cast<char>(length >> 8);
buf[6] = static_cast<char>(t);
// Compute the crc of the record type and the payload.
- uint32_t crc = crc32c::Extend(type_crc_[t], ptr, n);
- crc = crc32c::Mask(crc); // Adjust for storage
+ uint32_t crc = crc32c::Extend(type_crc_[t], ptr, length);
+ crc = crc32c::Mask(crc); // Adjust for storage
EncodeFixed32(buf, crc);
// Write the header and the payload
Status s = dest_->Append(Slice(buf, kHeaderSize));
if (s.ok()) {
- s = dest_->Append(Slice(ptr, n));
+ s = dest_->Append(Slice(ptr, length));
if (s.ok()) {
s = dest_->Flush();
}
}
- block_offset_ += kHeaderSize + n;
+ block_offset_ += kHeaderSize + length;
return s;
}
diff --git a/db/log_writer.h b/db/log_writer.h
index a3a954d..c0a2114 100644
--- a/db/log_writer.h
+++ b/db/log_writer.h
@@ -6,6 +6,7 @@
#define STORAGE_LEVELDB_DB_LOG_WRITER_H_
#include <stdint.h>
+
#include "db/log_format.h"
#include "leveldb/slice.h"
#include "leveldb/status.h"
@@ -22,24 +23,29 @@ class Writer {
// "*dest" must be initially empty.
// "*dest" must remain live while this Writer is in use.
explicit Writer(WritableFile* dest);
+
+ // Create a writer that will append data to "*dest".
+ // "*dest" must have initial length "dest_length".
+ // "*dest" must remain live while this Writer is in use.
+ Writer(WritableFile* dest, uint64_t dest_length);
+
+ Writer(const Writer&) = delete;
+ Writer& operator=(const Writer&) = delete;
+
~Writer();
Status AddRecord(const Slice& slice);
private:
+ Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length);
+
WritableFile* dest_;
- int block_offset_; // Current offset in block
+ int block_offset_; // Current offset in block
// crc32c values for all supported record types. These are
// pre-computed to reduce the overhead of computing the crc of the
// record type stored in the header.
uint32_t type_crc_[kMaxRecordType + 1];
-
- Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length);
-
- // No copying allowed
- Writer(const Writer&);
- void operator=(const Writer&);
};
} // namespace log
diff --git a/db/memtable.cc b/db/memtable.cc
index bfec0a7..00931d4 100644
--- a/db/memtable.cc
+++ b/db/memtable.cc
@@ -18,20 +18,15 @@ static Slice GetLengthPrefixedSlice(const char* data) {
return Slice(p, len);
}
-MemTable::MemTable(const InternalKeyComparator& cmp)
- : comparator_(cmp),
- refs_(0),
- table_(comparator_, &arena_) {
-}
+MemTable::MemTable(const InternalKeyComparator& comparator)
+ : comparator_(comparator), refs_(0), table_(comparator_, &arena_) {}
-MemTable::~MemTable() {
- assert(refs_ == 0);
-}
+MemTable::~MemTable() { assert(refs_ == 0); }
size_t MemTable::ApproximateMemoryUsage() { return arena_.MemoryUsage(); }
-int MemTable::KeyComparator::operator()(const char* aptr, const char* bptr)
- const {
+int MemTable::KeyComparator::operator()(const char* aptr,
+ const char* bptr) const {
// Internal keys are encoded as length-prefixed strings.
Slice a = GetLengthPrefixedSlice(aptr);
Slice b = GetLengthPrefixedSlice(bptr);
@@ -48,39 +43,37 @@ static const char* EncodeKey(std::string* scratch, const Slice& target) {
return scratch->data();
}
-class MemTableIterator: public Iterator {
+class MemTableIterator : public Iterator {
public:
- explicit MemTableIterator(MemTable::Table* table) : iter_(table) { }
-
- virtual bool Valid() const { return iter_.Valid(); }
- virtual void Seek(const Slice& k) { iter_.Seek(EncodeKey(&tmp_, k)); }
- virtual void SeekToFirst() { iter_.SeekToFirst(); }
- virtual void SeekToLast() { iter_.SeekToLast(); }
- virtual void Next() { iter_.Next(); }
- virtual void Prev() { iter_.Prev(); }
- virtual Slice key() const { return GetLengthPrefixedSlice(iter_.key()); }
- virtual Slice value() const {
+ explicit MemTableIterator(MemTable::Table* table) : iter_(table) {}
+
+ MemTableIterator(const MemTableIterator&) = delete;
+ MemTableIterator& operator=(const MemTableIterator&) = delete;
+
+ ~MemTableIterator() override = default;
+
+ bool Valid() const override { return iter_.Valid(); }
+ void Seek(const Slice& k) override { iter_.Seek(EncodeKey(&tmp_, k)); }
+ void SeekToFirst() override { iter_.SeekToFirst(); }
+ void SeekToLast() override { iter_.SeekToLast(); }
+ void Next() override { iter_.Next(); }
+ void Prev() override { iter_.Prev(); }
+ Slice key() const override { return GetLengthPrefixedSlice(iter_.key()); }
+ Slice value() const override {
Slice key_slice = GetLengthPrefixedSlice(iter_.key());
return GetLengthPrefixedSlice(key_slice.data() + key_slice.size());
}
- virtual Status status() const { return Status::OK(); }
+ Status status() const override { return Status::OK(); }
private:
MemTable::Table::Iterator iter_;
- std::string tmp_; // For passing to EncodeKey
-
- // No copying allowed
- MemTableIterator(const MemTableIterator&);
- void operator=(const MemTableIterator&);
+ std::string tmp_; // For passing to EncodeKey
};
-Iterator* MemTable::NewIterator() {
- return new MemTableIterator(&table_);
-}
+Iterator* MemTable::NewIterator() { return new MemTableIterator(&table_); }
-void MemTable::Add(SequenceNumber s, ValueType type,
- const Slice& key,
+void MemTable::Add(SequenceNumber s, ValueType type, const Slice& key,
const Slice& value) {
// Format of an entry is concatenation of:
// key_size : varint32 of internal_key.size()
@@ -90,9 +83,9 @@ void MemTable::Add(SequenceNumber s, ValueType type,
size_t key_size = key.size();
size_t val_size = value.size();
size_t internal_key_size = key_size + 8;
- const size_t encoded_len =
- VarintLength(internal_key_size) + internal_key_size +
- VarintLength(val_size) + val_size;
+ const size_t encoded_len = VarintLength(internal_key_size) +
+ internal_key_size + VarintLength(val_size) +
+ val_size;
char* buf = arena_.Allocate(encoded_len);
char* p = EncodeVarint32(buf, internal_key_size);
memcpy(p, key.data(), key_size);
@@ -101,7 +94,7 @@ void MemTable::Add(SequenceNumber s, ValueType type,
p += 8;
p = EncodeVarint32(p, val_size);
memcpy(p, value.data(), val_size);
- assert((p + val_size) - buf == encoded_len);
+ assert(p + val_size == buf + encoded_len);
table_.Insert(buf);
}
@@ -121,10 +114,9 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) {
// all entries with overly large sequence numbers.
const char* entry = iter.key();
uint32_t key_length;
- const char* key_ptr = GetVarint32Ptr(entry, entry+5, &key_length);
+ const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length);
if (comparator_.comparator.user_comparator()->Compare(
- Slice(key_ptr, key_length - 8),
- key.user_key()) == 0) {
+ Slice(key_ptr, key_length - 8), key.user_key()) == 0) {
// Correct user key
const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8);
switch (static_cast<ValueType>(tag & 0xff)) {
diff --git a/db/memtable.h b/db/memtable.h
index 92e90bb..9d986b1 100644
--- a/db/memtable.h
+++ b/db/memtable.h
@@ -6,15 +6,15 @@
#define STORAGE_LEVELDB_DB_MEMTABLE_H_
#include <string>
-#include "leveldb/db.h"
+
#include "db/dbformat.h"
#include "db/skiplist.h"
+#include "leveldb/db.h"
#include "util/arena.h"
namespace leveldb {
class InternalKeyComparator;
-class Mutex;
class MemTableIterator;
class MemTable {
@@ -23,6 +23,9 @@ class MemTable {
// is zero and the caller must call Ref() at least once.
explicit MemTable(const InternalKeyComparator& comparator);
+ MemTable(const MemTable&) = delete;
+ MemTable& operator=(const MemTable&) = delete;
+
// Increase reference count.
void Ref() { ++refs_; }
@@ -36,10 +39,7 @@ class MemTable {
}
// Returns an estimate of the number of bytes of data in use by this
- // data structure.
- //
- // REQUIRES: external synchronization to prevent simultaneous
- // operations on the same MemTable.
+ // data structure. It is safe to call when MemTable is being modified.
size_t ApproximateMemoryUsage();
// Return an iterator that yields the contents of the memtable.
@@ -53,8 +53,7 @@ class MemTable {
// Add an entry into memtable that maps key to value at the
// specified sequence number and with the specified type.
// Typically value will be empty if type==kTypeDeletion.
- void Add(SequenceNumber seq, ValueType type,
- const Slice& key,
+ void Add(SequenceNumber seq, ValueType type, const Slice& key,
const Slice& value);
// If memtable contains a value for key, store it in *value and return true.
@@ -64,26 +63,23 @@ class MemTable {
bool Get(const LookupKey& key, std::string* value, Status* s);
private:
- ~MemTable(); // Private since only Unref() should be used to delete it
+ friend class MemTableIterator;
+ friend class MemTableBackwardIterator;
struct KeyComparator {
const InternalKeyComparator comparator;
- explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) { }
+ explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) {}
int operator()(const char* a, const char* b) const;
};
- friend class MemTableIterator;
- friend class MemTableBackwardIterator;
typedef SkipList<const char*, KeyComparator> Table;
+ ~MemTable(); // Private since only Unref() should be used to delete it
+
KeyComparator comparator_;
int refs_;
Arena arena_;
Table table_;
-
- // No copying allowed
- MemTable(const MemTable&);
- void operator=(const MemTable&);
};
} // namespace leveldb
diff --git a/db/recovery_test.cc b/db/recovery_test.cc
new file mode 100644
index 0000000..547a959
--- /dev/null
+++ b/db/recovery_test.cc
@@ -0,0 +1,330 @@
+// Copyright (c) 2014 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/db_impl.h"
+#include "db/filename.h"
+#include "db/version_set.h"
+#include "db/write_batch_internal.h"
+#include "leveldb/db.h"
+#include "leveldb/env.h"
+#include "leveldb/write_batch.h"
+#include "util/logging.h"
+#include "util/testharness.h"
+#include "util/testutil.h"
+
+namespace leveldb {
+
+class RecoveryTest {
+ public:
+ RecoveryTest() : env_(Env::Default()), db_(nullptr) {
+ dbname_ = test::TmpDir() + "/recovery_test";
+ DestroyDB(dbname_, Options());
+ Open();
+ }
+
+ ~RecoveryTest() {
+ Close();
+ DestroyDB(dbname_, Options());
+ }
+
+ DBImpl* dbfull() const { return reinterpret_cast<DBImpl*>(db_); }
+ Env* env() const { return env_; }
+
+ bool CanAppend() {
+ WritableFile* tmp;
+ Status s = env_->NewAppendableFile(CurrentFileName(dbname_), &tmp);
+ delete tmp;
+ if (s.IsNotSupportedError()) {
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+ void Close() {
+ delete db_;
+ db_ = nullptr;
+ }
+
+ Status OpenWithStatus(Options* options = nullptr) {
+ Close();
+ Options opts;
+ if (options != nullptr) {
+ opts = *options;
+ } else {
+ opts.reuse_logs = true; // TODO(sanjay): test both ways
+ opts.create_if_missing = true;
+ }
+ if (opts.env == nullptr) {
+ opts.env = env_;
+ }
+ return DB::Open(opts, dbname_, &db_);
+ }
+
+ void Open(Options* options = nullptr) {
+ ASSERT_OK(OpenWithStatus(options));
+ ASSERT_EQ(1, NumLogs());
+ }
+
+ Status Put(const std::string& k, const std::string& v) {
+ return db_->Put(WriteOptions(), k, v);
+ }
+
+ std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) {
+ std::string result;
+ Status s = db_->Get(ReadOptions(), k, &result);
+ if (s.IsNotFound()) {
+ result = "NOT_FOUND";
+ } else if (!s.ok()) {
+ result = s.ToString();
+ }
+ return result;
+ }
+
+ std::string ManifestFileName() {
+ std::string current;
+ ASSERT_OK(ReadFileToString(env_, CurrentFileName(dbname_), &current));
+ size_t len = current.size();
+ if (len > 0 && current[len - 1] == '\n') {
+ current.resize(len - 1);
+ }
+ return dbname_ + "/" + current;
+ }
+
+ std::string LogName(uint64_t number) { return LogFileName(dbname_, number); }
+
+ size_t DeleteLogFiles() {
+ // Linux allows unlinking open files, but Windows does not.
+ // Closing the db allows for file deletion.
+ Close();
+ std::vector<uint64_t> logs = GetFiles(kLogFile);
+ for (size_t i = 0; i < logs.size(); i++) {
+ ASSERT_OK(env_->DeleteFile(LogName(logs[i]))) << LogName(logs[i]);
+ }
+ return logs.size();
+ }
+
+ void DeleteManifestFile() { ASSERT_OK(env_->DeleteFile(ManifestFileName())); }
+
+ uint64_t FirstLogFile() { return GetFiles(kLogFile)[0]; }
+
+ std::vector<uint64_t> GetFiles(FileType t) {
+ std::vector<std::string> filenames;
+ ASSERT_OK(env_->GetChildren(dbname_, &filenames));
+ std::vector<uint64_t> result;
+ for (size_t i = 0; i < filenames.size(); i++) {
+ uint64_t number;
+ FileType type;
+ if (ParseFileName(filenames[i], &number, &type) && type == t) {
+ result.push_back(number);
+ }
+ }
+ return result;
+ }
+
+ int NumLogs() { return GetFiles(kLogFile).size(); }
+
+ int NumTables() { return GetFiles(kTableFile).size(); }
+
+ uint64_t FileSize(const std::string& fname) {
+ uint64_t result;
+ ASSERT_OK(env_->GetFileSize(fname, &result)) << fname;
+ return result;
+ }
+
+ void CompactMemTable() { dbfull()->TEST_CompactMemTable(); }
+
+ // Directly construct a log file that sets key to val.
+ void MakeLogFile(uint64_t lognum, SequenceNumber seq, Slice key, Slice val) {
+ std::string fname = LogFileName(dbname_, lognum);
+ WritableFile* file;
+ ASSERT_OK(env_->NewWritableFile(fname, &file));
+ log::Writer writer(file);
+ WriteBatch batch;
+ batch.Put(key, val);
+ WriteBatchInternal::SetSequence(&batch, seq);
+ ASSERT_OK(writer.AddRecord(WriteBatchInternal::Contents(&batch)));
+ ASSERT_OK(file->Flush());
+ delete file;
+ }
+
+ private:
+ std::string dbname_;
+ Env* env_;
+ DB* db_;
+};
+
+TEST(RecoveryTest, ManifestReused) {
+ if (!CanAppend()) {
+ fprintf(stderr, "skipping test because env does not support appending\n");
+ return;
+ }
+ ASSERT_OK(Put("foo", "bar"));
+ Close();
+ std::string old_manifest = ManifestFileName();
+ Open();
+ ASSERT_EQ(old_manifest, ManifestFileName());
+ ASSERT_EQ("bar", Get("foo"));
+ Open();
+ ASSERT_EQ(old_manifest, ManifestFileName());
+ ASSERT_EQ("bar", Get("foo"));
+}
+
+TEST(RecoveryTest, LargeManifestCompacted) {
+ if (!CanAppend()) {
+ fprintf(stderr, "skipping test because env does not support appending\n");
+ return;
+ }
+ ASSERT_OK(Put("foo", "bar"));
+ Close();
+ std::string old_manifest = ManifestFileName();
+
+ // Pad with zeroes to make manifest file very big.
+ {
+ uint64_t len = FileSize(old_manifest);
+ WritableFile* file;
+ ASSERT_OK(env()->NewAppendableFile(old_manifest, &file));
+ std::string zeroes(3 * 1048576 - static_cast<size_t>(len), 0);
+ ASSERT_OK(file->Append(zeroes));
+ ASSERT_OK(file->Flush());
+ delete file;
+ }
+
+ Open();
+ std::string new_manifest = ManifestFileName();
+ ASSERT_NE(old_manifest, new_manifest);
+ ASSERT_GT(10000, FileSize(new_manifest));
+ ASSERT_EQ("bar", Get("foo"));
+
+ Open();
+ ASSERT_EQ(new_manifest, ManifestFileName());
+ ASSERT_EQ("bar", Get("foo"));
+}
+
+TEST(RecoveryTest, NoLogFiles) {
+ ASSERT_OK(Put("foo", "bar"));
+ ASSERT_EQ(1, DeleteLogFiles());
+ Open();
+ ASSERT_EQ("NOT_FOUND", Get("foo"));
+ Open();
+ ASSERT_EQ("NOT_FOUND", Get("foo"));
+}
+
+TEST(RecoveryTest, LogFileReuse) {
+ if (!CanAppend()) {
+ fprintf(stderr, "skipping test because env does not support appending\n");
+ return;
+ }
+ for (int i = 0; i < 2; i++) {
+ ASSERT_OK(Put("foo", "bar"));
+ if (i == 0) {
+ // Compact to ensure current log is empty
+ CompactMemTable();
+ }
+ Close();
+ ASSERT_EQ(1, NumLogs());
+ uint64_t number = FirstLogFile();
+ if (i == 0) {
+ ASSERT_EQ(0, FileSize(LogName(number)));
+ } else {
+ ASSERT_LT(0, FileSize(LogName(number)));
+ }
+ Open();
+ ASSERT_EQ(1, NumLogs());
+ ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file";
+ ASSERT_EQ("bar", Get("foo"));
+ Open();
+ ASSERT_EQ(1, NumLogs());
+ ASSERT_EQ(number, FirstLogFile()) << "did not reuse log file";
+ ASSERT_EQ("bar", Get("foo"));
+ }
+}
+
+TEST(RecoveryTest, MultipleMemTables) {
+ // Make a large log.
+ const int kNum = 1000;
+ for (int i = 0; i < kNum; i++) {
+ char buf[100];
+ snprintf(buf, sizeof(buf), "%050d", i);
+ ASSERT_OK(Put(buf, buf));
+ }
+ ASSERT_EQ(0, NumTables());
+ Close();
+ ASSERT_EQ(0, NumTables());
+ ASSERT_EQ(1, NumLogs());
+ uint64_t old_log_file = FirstLogFile();
+
+ // Force creation of multiple memtables by reducing the write buffer size.
+ Options opt;
+ opt.reuse_logs = true;
+ opt.write_buffer_size = (kNum * 100) / 2;
+ Open(&opt);
+ ASSERT_LE(2, NumTables());
+ ASSERT_EQ(1, NumLogs());
+ ASSERT_NE(old_log_file, FirstLogFile()) << "must not reuse log";
+ for (int i = 0; i < kNum; i++) {
+ char buf[100];
+ snprintf(buf, sizeof(buf), "%050d", i);
+ ASSERT_EQ(buf, Get(buf));
+ }
+}
+
+TEST(RecoveryTest, MultipleLogFiles) {
+ ASSERT_OK(Put("foo", "bar"));
+ Close();
+ ASSERT_EQ(1, NumLogs());
+
+ // Make a bunch of uncompacted log files.
+ uint64_t old_log = FirstLogFile();
+ MakeLogFile(old_log + 1, 1000, "hello", "world");
+ MakeLogFile(old_log + 2, 1001, "hi", "there");
+ MakeLogFile(old_log + 3, 1002, "foo", "bar2");
+
+ // Recover and check that all log files were processed.
+ Open();
+ ASSERT_LE(1, NumTables());
+ ASSERT_EQ(1, NumLogs());
+ uint64_t new_log = FirstLogFile();
+ ASSERT_LE(old_log + 3, new_log);
+ ASSERT_EQ("bar2", Get("foo"));
+ ASSERT_EQ("world", Get("hello"));
+ ASSERT_EQ("there", Get("hi"));
+
+ // Test that previous recovery produced recoverable state.
+ Open();
+ ASSERT_LE(1, NumTables());
+ ASSERT_EQ(1, NumLogs());
+ if (CanAppend()) {
+ ASSERT_EQ(new_log, FirstLogFile());
+ }
+ ASSERT_EQ("bar2", Get("foo"));
+ ASSERT_EQ("world", Get("hello"));
+ ASSERT_EQ("there", Get("hi"));
+
+ // Check that introducing an older log file does not cause it to be re-read.
+ Close();
+ MakeLogFile(old_log + 1, 2000, "hello", "stale write");
+ Open();
+ ASSERT_LE(1, NumTables());
+ ASSERT_EQ(1, NumLogs());
+ if (CanAppend()) {
+ ASSERT_EQ(new_log, FirstLogFile());
+ }
+ ASSERT_EQ("bar2", Get("foo"));
+ ASSERT_EQ("world", Get("hello"));
+ ASSERT_EQ("there", Get("hi"));
+}
+
+TEST(RecoveryTest, ManifestMissing) {
+ ASSERT_OK(Put("foo", "bar"));
+ Close();
+ DeleteManifestFile();
+
+ Status status = OpenWithStatus();
+ ASSERT_TRUE(status.IsCorruption());
+}
+
+} // namespace leveldb
+
+int main(int argc, char** argv) { return leveldb::test::RunAllTests(); }
diff --git a/db/repair.cc b/db/repair.cc
index 4cd4bb0..d9d12ba 100644
--- a/db/repair.cc
+++ b/db/repair.cc
@@ -54,7 +54,7 @@ class Repairer {
owns_cache_(options_.block_cache != options.block_cache),
next_file_number_(1) {
// TableCache can be small since we expect each table to be opened once.
- table_cache_ = new TableCache(dbname_, &options_, 10);
+ table_cache_ = new TableCache(dbname_, options_, 10);
}
~Repairer() {
@@ -84,9 +84,7 @@ class Repairer {
"recovered %d files; %llu bytes. "
"Some data may have been lost. "
"****",
- dbname_.c_str(),
- static_cast<int>(tables_.size()),
- bytes);
+ dbname_.c_str(), static_cast<int>(tables_.size()), bytes);
}
return status;
}
@@ -97,22 +95,6 @@ class Repairer {
SequenceNumber max_sequence;
};
- std::string const dbname_;
- Env* const env_;
- InternalKeyComparator const icmp_;
- InternalFilterPolicy const ipolicy_;
- Options const options_;
- bool owns_info_log_;
- bool owns_cache_;
- TableCache* table_cache_;
- VersionEdit edit_;
-
- std::vector<std::string> manifests_;
- std::vector<uint64_t> table_numbers_;
- std::vector<uint64_t> logs_;
- std::vector<TableInfo> tables_;
- uint64_t next_file_number_;
-
Status FindFiles() {
std::vector<std::string> filenames;
Status status = env_->GetChildren(dbname_, &filenames);
@@ -152,8 +134,7 @@ class Repairer {
Status status = ConvertLogToTable(logs_[i]);
if (!status.ok()) {
Log(options_.info_log, "Log #%llu: ignoring conversion error: %s",
- (unsigned long long) logs_[i],
- status.ToString().c_str());
+ (unsigned long long)logs_[i], status.ToString().c_str());
}
ArchiveFile(logname);
}
@@ -164,11 +145,10 @@ class Repairer {
Env* env;
Logger* info_log;
uint64_t lognum;
- virtual void Corruption(size_t bytes, const Status& s) {
+ void Corruption(size_t bytes, const Status& s) override {
// We print error messages for corruption, but continue repairing.
Log(info_log, "Log #%llu: dropping %d bytes; %s",
- (unsigned long long) lognum,
- static_cast<int>(bytes),
+ (unsigned long long)lognum, static_cast<int>(bytes),
s.ToString().c_str());
}
};
@@ -190,8 +170,8 @@ class Repairer {
// corruptions cause entire commits to be skipped instead of
// propagating bad information (like overly large sequence
// numbers).
- log::Reader reader(lfile, &reporter, false/*do not checksum*/,
- 0/*initial_offset*/);
+ log::Reader reader(lfile, &reporter, false /*do not checksum*/,
+ 0 /*initial_offset*/);
// Read all the records and add to a memtable
std::string scratch;
@@ -202,8 +182,8 @@ class Repairer {
int counter = 0;
while (reader.ReadRecord(&record, &scratch)) {
if (record.size() < 12) {
- reporter.Corruption(
- record.size(), Status::Corruption("log record too small"));
+ reporter.Corruption(record.size(),
+ Status::Corruption("log record too small"));
continue;
}
WriteBatchInternal::SetContents(&batch, record);
@@ -212,8 +192,7 @@ class Repairer {
counter += WriteBatchInternal::Count(&batch);
} else {
Log(options_.info_log, "Log #%llu: ignoring %s",
- (unsigned long long) log,
- status.ToString().c_str());
+ (unsigned long long)log, status.ToString().c_str());
status = Status::OK(); // Keep going with rest of file
}
}
@@ -227,16 +206,14 @@ class Repairer {
status = BuildTable(dbname_, env_, options_, table_cache_, iter, &meta);
delete iter;
mem->Unref();
- mem = NULL;
+ mem = nullptr;
if (status.ok()) {
if (meta.file_size > 0) {
table_numbers_.push_back(meta.number);
}
}
Log(options_.info_log, "Log #%llu: %d ops saved to Table #%llu %s",
- (unsigned long long) log,
- counter,
- (unsigned long long) meta.number,
+ (unsigned long long)log, counter, (unsigned long long)meta.number,
status.ToString().c_str());
return status;
}
@@ -272,8 +249,7 @@ class Repairer {
ArchiveFile(TableFileName(dbname_, number));
ArchiveFile(SSTTableFileName(dbname_, number));
Log(options_.info_log, "Table #%llu: dropped: %s",
- (unsigned long long) t.meta.number,
- status.ToString().c_str());
+ (unsigned long long)t.meta.number, status.ToString().c_str());
return;
}
@@ -287,8 +263,7 @@ class Repairer {
Slice key = iter->key();
if (!ParseInternalKey(key, &parsed)) {
Log(options_.info_log, "Table #%llu: unparsable key %s",
- (unsigned long long) t.meta.number,
- EscapeString(key).c_str());
+ (unsigned long long)t.meta.number, EscapeString(key).c_str());
continue;
}
@@ -307,9 +282,7 @@ class Repairer {
}
delete iter;
Log(options_.info_log, "Table #%llu: %d entries %s",
- (unsigned long long) t.meta.number,
- counter,
- status.ToString().c_str());
+ (unsigned long long)t.meta.number, counter, status.ToString().c_str());
if (status.ok()) {
tables_.push_back(t);
@@ -350,20 +323,20 @@ class Repairer {
}
}
delete builder;
- builder = NULL;
+ builder = nullptr;
if (s.ok()) {
s = file->Close();
}
delete file;
- file = NULL;
+ file = nullptr;
if (counter > 0 && s.ok()) {
std::string orig = TableFileName(dbname_, t.meta.number);
s = env_->RenameFile(copy, orig);
if (s.ok()) {
Log(options_.info_log, "Table #%llu: %d entries repaired",
- (unsigned long long) t.meta.number, counter);
+ (unsigned long long)t.meta.number, counter);
tables_.push_back(t);
}
}
@@ -395,11 +368,11 @@ class Repairer {
for (size_t i = 0; i < tables_.size(); i++) {
// TODO(opt): separate out into multiple levels
const TableInfo& t = tables_[i];
- edit_.AddFile(0, t.meta.number, t.meta.file_size,
- t.meta.smallest, t.meta.largest);
+ edit_.AddFile(0, t.meta.number, t.meta.file_size, t.meta.smallest,
+ t.meta.largest);
}
- //fprintf(stderr, "NewDescriptor:\n%s\n", edit_.DebugString().c_str());
+ // fprintf(stderr, "NewDescriptor:\n%s\n", edit_.DebugString().c_str());
{
log::Writer log(file);
std::string record;
@@ -410,7 +383,7 @@ class Repairer {
status = file->Close();
}
delete file;
- file = NULL;
+ file = nullptr;
if (!status.ok()) {
env_->DeleteFile(tmp);
@@ -438,18 +411,34 @@ class Repairer {
// dir/lost/foo
const char* slash = strrchr(fname.c_str(), '/');
std::string new_dir;
- if (slash != NULL) {
+ if (slash != nullptr) {
new_dir.assign(fname.data(), slash - fname.data());
}
new_dir.append("/lost");
env_->CreateDir(new_dir); // Ignore error
std::string new_file = new_dir;
new_file.append("/");
- new_file.append((slash == NULL) ? fname.c_str() : slash + 1);
+ new_file.append((slash == nullptr) ? fname.c_str() : slash + 1);
Status s = env_->RenameFile(fname, new_file);
- Log(options_.info_log, "Archiving %s: %s\n",
- fname.c_str(), s.ToString().c_str());
+ Log(options_.info_log, "Archiving %s: %s\n", fname.c_str(),
+ s.ToString().c_str());
}
+
+ const std::string dbname_;
+ Env* const env_;
+ InternalKeyComparator const icmp_;
+ InternalFilterPolicy const ipolicy_;
+ const Options options_;
+ bool owns_info_log_;
+ bool owns_cache_;
+ TableCache* table_cache_;
+ VersionEdit edit_;
+
+ std::vector<std::string> manifests_;
+ std::vector<uint64_t> table_numbers_;
+ std::vector<uint64_t> logs_;
+ std::vector<TableInfo> tables_;
+ uint64_t next_file_number_;
};
} // namespace
diff --git a/db/skiplist.h b/db/skiplist.h
index 8bd7776..a59b45b 100644
--- a/db/skiplist.h
+++ b/db/skiplist.h
@@ -27,9 +27,10 @@
//
// ... prev vs. next pointer ordering ...
-#include <assert.h>
-#include <stdlib.h>
-#include "port/port.h"
+#include <atomic>
+#include <cassert>
+#include <cstdlib>
+
#include "util/arena.h"
#include "util/random.h"
@@ -37,7 +38,7 @@ namespace leveldb {
class Arena;
-template<typename Key, class Comparator>
+template <typename Key, class Comparator>
class SkipList {
private:
struct Node;
@@ -48,6 +49,9 @@ class SkipList {
// must remain allocated for the lifetime of the skiplist object.
explicit SkipList(Comparator cmp, Arena* arena);
+ SkipList(const SkipList&) = delete;
+ SkipList& operator=(const SkipList&) = delete;
+
// Insert key into the list.
// REQUIRES: nothing that compares equal to key is currently in the list.
void Insert(const Key& key);
@@ -97,24 +101,10 @@ class SkipList {
private:
enum { kMaxHeight = 12 };
- // Immutable after construction
- Comparator const compare_;
- Arena* const arena_; // Arena used for allocations of nodes
-
- Node* const head_;
-
- // Modified only by Insert(). Read racily by readers, but stale
- // values are ok.
- port::AtomicPointer max_height_; // Height of the entire list
-
inline int GetMaxHeight() const {
- return static_cast<int>(
- reinterpret_cast<intptr_t>(max_height_.NoBarrier_Load()));
+ return max_height_.load(std::memory_order_relaxed);
}
- // Read/written only by Insert().
- Random rnd_;
-
Node* NewNode(const Key& key, int height);
int RandomHeight();
bool Equal(const Key& a, const Key& b) const { return (compare_(a, b) == 0); }
@@ -123,9 +113,9 @@ class SkipList {
bool KeyIsAfterNode(const Key& key, Node* n) const;
// Return the earliest node that comes at or after key.
- // Return NULL if there is no such node.
+ // Return nullptr if there is no such node.
//
- // If prev is non-NULL, fills prev[level] with pointer to previous
+ // If prev is non-null, fills prev[level] with pointer to previous
// node at "level" for every level in [0..max_height_-1].
Node* FindGreaterOrEqual(const Key& key, Node** prev) const;
@@ -137,15 +127,24 @@ class SkipList {
// Return head_ if list is empty.
Node* FindLast() const;
- // No copying allowed
- SkipList(const SkipList&);
- void operator=(const SkipList&);
+ // Immutable after construction
+ Comparator const compare_;
+ Arena* const arena_; // Arena used for allocations of nodes
+
+ Node* const head_;
+
+ // Modified only by Insert(). Read racily by readers, but stale
+ // values are ok.
+ std::atomic<int> max_height_; // Height of the entire list
+
+ // Read/written only by Insert().
+ Random rnd_;
};
// Implementation details follow
-template<typename Key, class Comparator>
-struct SkipList<Key,Comparator>::Node {
- explicit Node(const Key& k) : key(k) { }
+template <typename Key, class Comparator>
+struct SkipList<Key, Comparator>::Node {
+ explicit Node(const Key& k) : key(k) {}
Key const key;
@@ -155,92 +154,92 @@ struct SkipList<Key,Comparator>::Node {
assert(n >= 0);
// Use an 'acquire load' so that we observe a fully initialized
// version of the returned Node.
- return reinterpret_cast<Node*>(next_[n].Acquire_Load());
+ return next_[n].load(std::memory_order_acquire);
}
void SetNext(int n, Node* x) {
assert(n >= 0);
// Use a 'release store' so that anybody who reads through this
// pointer observes a fully initialized version of the inserted node.
- next_[n].Release_Store(x);
+ next_[n].store(x, std::memory_order_release);
}
// No-barrier variants that can be safely used in a few locations.
Node* NoBarrier_Next(int n) {
assert(n >= 0);
- return reinterpret_cast<Node*>(next_[n].NoBarrier_Load());
+ return next_[n].load(std::memory_order_relaxed);
}
void NoBarrier_SetNext(int n, Node* x) {
assert(n >= 0);
- next_[n].NoBarrier_Store(x);
+ next_[n].store(x, std::memory_order_relaxed);
}
private:
// Array of length equal to the node height. next_[0] is lowest level link.
- port::AtomicPointer next_[1];
+ std::atomic<Node*> next_[1];
};
-template<typename Key, class Comparator>
-typename SkipList<Key,Comparator>::Node*
-SkipList<Key,Comparator>::NewNode(const Key& key, int height) {
- char* mem = arena_->AllocateAligned(
- sizeof(Node) + sizeof(port::AtomicPointer) * (height - 1));
- return new (mem) Node(key);
+template <typename Key, class Comparator>
+typename SkipList<Key, Comparator>::Node* SkipList<Key, Comparator>::NewNode(
+ const Key& key, int height) {
+ char* const node_memory = arena_->AllocateAligned(
+ sizeof(Node) + sizeof(std::atomic<Node*>) * (height - 1));
+ return new (node_memory) Node(key);
}
-template<typename Key, class Comparator>
-inline SkipList<Key,Comparator>::Iterator::Iterator(const SkipList* list) {
+template <typename Key, class Comparator>
+inline SkipList<Key, Comparator>::Iterator::Iterator(const SkipList* list) {
list_ = list;
- node_ = NULL;
+ node_ = nullptr;
}
-template<typename Key, class Comparator>
-inline bool SkipList<Key,Comparator>::Iterator::Valid() const {
- return node_ != NULL;
+template <typename Key, class Comparator>
+inline bool SkipList<Key, Comparator>::Iterator::Valid() const {
+ return node_ != nullptr;
}
-template<typename Key, class Comparator>
-inline const Key& SkipList<Key,Comparator>::Iterator::key() const {
+template <typename Key, class Comparator>
+inline const Key& SkipList<Key, Comparator>::Iterator::key() const {
assert(Valid());
return node_->key;
}
-template<typename Key, class Comparator>
-inline void SkipList<Key,Comparator>::Iterator::Next() {
+template <typename Key, class Comparator>
+inline void SkipList<Key, Comparator>::Iterator::Next() {
assert(Valid());
node_ = node_->Next(0);
}
-template<typename Key, class Comparator>
-inline void SkipList<Key,Comparator>::Iterator::Prev() {
+template <typename Key, class Comparator>
+inline void SkipList<Key, Comparator>::Iterator::Prev() {
// Instead of using explicit "prev" links, we just search for the
// last node that falls before key.
assert(Valid());
node_ = list_->FindLessThan(node_->key);
if (node_ == list_->head_) {
- node_ = NULL;
+ node_ = nullptr;
}
}
-template<typename Key, class Comparator>
-inline void SkipList<Key,Comparator>::Iterator::Seek(const Key& target) {
- node_ = list_->FindGreaterOrEqual(target, NULL);
+template <typename Key, class Comparator>
+inline void SkipList<Key, Comparator>::Iterator::Seek(const Key& target) {
+ node_ = list_->FindGreaterOrEqual(target, nullptr);
}
-template<typename Key, class Comparator>
-inline void SkipList<Key,Comparator>::Iterator::SeekToFirst() {
+template <typename Key, class Comparator>
+inline void SkipList<Key, Comparator>::Iterator::SeekToFirst() {
node_ = list_->head_->Next(0);
}
-template<typename Key, class Comparator>
-inline void SkipList<Key,Comparator>::Iterator::SeekToLast() {
+template <typename Key, class Comparator>
+inline void SkipList<Key, Comparator>::Iterator::SeekToLast() {
node_ = list_->FindLast();
if (node_ == list_->head_) {
- node_ = NULL;
+ node_ = nullptr;
}
}
-template<typename Key, class Comparator>
-int SkipList<Key,Comparator>::RandomHeight() {
+template <typename Key, class Comparator>
+int SkipList<Key, Comparator>::RandomHeight() {
// Increase height with probability 1 in kBranching
static const unsigned int kBranching = 4;
int height = 1;
@@ -252,15 +251,16 @@ int SkipList<Key,Comparator>::RandomHeight() {
return height;
}
-template<typename Key, class Comparator>
-bool SkipList<Key,Comparator>::KeyIsAfterNode(const Key& key, Node* n) const {
- // NULL n is considered infinite
- return (n != NULL) && (compare_(n->key, key) < 0);
+template <typename Key, class Comparator>
+bool SkipList<Key, Comparator>::KeyIsAfterNode(const Key& key, Node* n) const {
+ // null n is considered infinite
+ return (n != nullptr) && (compare_(n->key, key) < 0);
}
-template<typename Key, class Comparator>
-typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindGreaterOrEqual(const Key& key, Node** prev)
- const {
+template <typename Key, class Comparator>
+typename SkipList<Key, Comparator>::Node*
+SkipList<Key, Comparator>::FindGreaterOrEqual(const Key& key,
+ Node** prev) const {
Node* x = head_;
int level = GetMaxHeight() - 1;
while (true) {
@@ -269,7 +269,7 @@ typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindGreaterOr
// Keep searching in this list
x = next;
} else {
- if (prev != NULL) prev[level] = x;
+ if (prev != nullptr) prev[level] = x;
if (level == 0) {
return next;
} else {
@@ -280,15 +280,15 @@ typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindGreaterOr
}
}
-template<typename Key, class Comparator>
-typename SkipList<Key,Comparator>::Node*
-SkipList<Key,Comparator>::FindLessThan(const Key& key) const {
+template <typename Key, class Comparator>
+typename SkipList<Key, Comparator>::Node*
+SkipList<Key, Comparator>::FindLessThan(const Key& key) const {
Node* x = head_;
int level = GetMaxHeight() - 1;
while (true) {
assert(x == head_ || compare_(x->key, key) < 0);
Node* next = x->Next(level);
- if (next == NULL || compare_(next->key, key) >= 0) {
+ if (next == nullptr || compare_(next->key, key) >= 0) {
if (level == 0) {
return x;
} else {
@@ -301,14 +301,14 @@ SkipList<Key,Comparator>::FindLessThan(const Key& key) const {
}
}
-template<typename Key, class Comparator>
-typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindLast()
+template <typename Key, class Comparator>
+typename SkipList<Key, Comparator>::Node* SkipList<Key, Comparator>::FindLast()
const {
Node* x = head_;
int level = GetMaxHeight() - 1;
while (true) {
Node* next = x->Next(level);
- if (next == NULL) {
+ if (next == nullptr) {
if (level == 0) {
return x;
} else {
@@ -321,43 +321,41 @@ typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindLast()
}
}
-template<typename Key, class Comparator>
-SkipList<Key,Comparator>::SkipList(Comparator cmp, Arena* arena)
+template <typename Key, class Comparator>
+SkipList<Key, Comparator>::SkipList(Comparator cmp, Arena* arena)
: compare_(cmp),
arena_(arena),
head_(NewNode(0 /* any key will do */, kMaxHeight)),
- max_height_(reinterpret_cast<void*>(1)),
+ max_height_(1),
rnd_(0xdeadbeef) {
for (int i = 0; i < kMaxHeight; i++) {
- head_->SetNext(i, NULL);
+ head_->SetNext(i, nullptr);
}
}
-template<typename Key, class Comparator>
-void SkipList<Key,Comparator>::Insert(const Key& key) {
+template <typename Key, class Comparator>
+void SkipList<Key, Comparator>::Insert(const Key& key) {
// TODO(opt): We can use a barrier-free variant of FindGreaterOrEqual()
// here since Insert() is externally synchronized.
Node* prev[kMaxHeight];
Node* x = FindGreaterOrEqual(key, prev);
// Our data structure does not allow duplicate insertion
- assert(x == NULL || !Equal(key, x->key));
+ assert(x == nullptr || !Equal(key, x->key));
int height = RandomHeight();
if (height > GetMaxHeight()) {
for (int i = GetMaxHeight(); i < height; i++) {
prev[i] = head_;
}
- //fprintf(stderr, "Change height from %d to %d\n", max_height_, height);
-
// It is ok to mutate max_height_ without any synchronization
// with concurrent readers. A concurrent reader that observes
// the new value of max_height_ will see either the old value of
- // new level pointers from head_ (NULL), or a new value set in
+ // new level pointers from head_ (nullptr), or a new value set in
// the loop below. In the former case the reader will
- // immediately drop to the next level since NULL sorts after all
+ // immediately drop to the next level since nullptr sorts after all
// keys. In the latter case the reader will use the new node.
- max_height_.NoBarrier_Store(reinterpret_cast<void*>(height));
+ max_height_.store(height, std::memory_order_relaxed);
}
x = NewNode(key, height);
@@ -369,10 +367,10 @@ void SkipList<Key,Comparator>::Insert(const Key& key) {
}
}
-template<typename Key, class Comparator>
-bool SkipList<Key,Comparator>::Contains(const Key& key) const {
- Node* x = FindGreaterOrEqual(key, NULL);
- if (x != NULL && Equal(key, x->key)) {
+template <typename Key, class Comparator>
+bool SkipList<Key, Comparator>::Contains(const Key& key) const {
+ Node* x = FindGreaterOrEqual(key, nullptr);
+ if (x != nullptr && Equal(key, x->key)) {
return true;
} else {
return false;
diff --git a/db/skiplist_test.cc b/db/skiplist_test.cc
index c78f4b4..9fa2d96 100644
--- a/db/skiplist_test.cc
+++ b/db/skiplist_test.cc
@@ -3,8 +3,13 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "db/skiplist.h"
+
+#include <atomic>
#include <set>
+
#include "leveldb/env.h"
+#include "port/port.h"
+#include "port/thread_annotations.h"
#include "util/arena.h"
#include "util/hash.h"
#include "util/random.h"
@@ -26,7 +31,7 @@ struct Comparator {
}
};
-class SkipTest { };
+class SkipTest {};
TEST(SkipTest, Empty) {
Arena arena;
@@ -112,8 +117,7 @@ TEST(SkipTest, InsertAndLookup) {
// Compare against model iterator
for (std::set<Key>::reverse_iterator model_iter = keys.rbegin();
- model_iter != keys.rend();
- ++model_iter) {
+ model_iter != keys.rend(); ++model_iter) {
ASSERT_TRUE(iter.Valid());
ASSERT_EQ(*model_iter, iter.key());
iter.Prev();
@@ -126,7 +130,7 @@ TEST(SkipTest, InsertAndLookup) {
// concurrent readers (with no synchronization other than when a
// reader's iterator is created), the reader always observes all the
// data that was present in the skip list when the iterator was
-// constructor. Because insertions are happening concurrently, we may
+// constructed. Because insertions are happening concurrently, we may
// also observe new values that were inserted since the iterator was
// constructed, but we should never miss any values that were present
// at iterator construction time.
@@ -155,12 +159,12 @@ class ConcurrentTest {
static uint64_t hash(Key key) { return key & 0xff; }
static uint64_t HashNumbers(uint64_t k, uint64_t g) {
- uint64_t data[2] = { k, g };
+ uint64_t data[2] = {k, g};
return Hash(reinterpret_cast<char*>(data), sizeof(data), 0);
}
static Key MakeKey(uint64_t k, uint64_t g) {
- assert(sizeof(Key) == sizeof(uint64_t));
+ static_assert(sizeof(Key) == sizeof(uint64_t), "");
assert(k <= K); // We sometimes pass K to seek to the end of the skiplist
assert(g <= 0xffffffffu);
return ((k << 40) | (g << 8) | (HashNumbers(k, g) & 0xff));
@@ -186,13 +190,11 @@ class ConcurrentTest {
// Per-key generation
struct State {
- port::AtomicPointer generation[K];
- void Set(int k, intptr_t v) {
- generation[k].Release_Store(reinterpret_cast<void*>(v));
- }
- intptr_t Get(int k) {
- return reinterpret_cast<intptr_t>(generation[k].Acquire_Load());
+ std::atomic<int> generation[K];
+ void Set(int k, int v) {
+ generation[k].store(v, std::memory_order_release);
}
+ int Get(int k) { return generation[k].load(std::memory_order_acquire); }
State() {
for (int k = 0; k < K; k++) {
@@ -211,7 +213,7 @@ class ConcurrentTest {
SkipList<Key, Comparator> list_;
public:
- ConcurrentTest() : list_(Comparator(), &arena_) { }
+ ConcurrentTest() : list_(Comparator(), &arena_) {}
// REQUIRES: External synchronization
void WriteStep(Random* rnd) {
@@ -250,11 +252,9 @@ class ConcurrentTest {
// Note that generation 0 is never inserted, so it is ok if
// <*,0,*> is missing.
ASSERT_TRUE((gen(pos) == 0) ||
- (gen(pos) > initial_state.Get(key(pos)))
- ) << "key: " << key(pos)
- << "; gen: " << gen(pos)
- << "; initgen: "
- << initial_state.Get(key(pos));
+ (gen(pos) > static_cast<Key>(initial_state.Get(key(pos)))))
+ << "key: " << key(pos) << "; gen: " << gen(pos)
+ << "; initgen: " << initial_state.Get(key(pos));
// Advance to next key in the valid key space
if (key(pos) < key(current)) {
@@ -298,21 +298,14 @@ class TestState {
public:
ConcurrentTest t_;
int seed_;
- port::AtomicPointer quit_flag_;
+ std::atomic<bool> quit_flag_;
- enum ReaderState {
- STARTING,
- RUNNING,
- DONE
- };
+ enum ReaderState { STARTING, RUNNING, DONE };
explicit TestState(int s)
- : seed_(s),
- quit_flag_(NULL),
- state_(STARTING),
- state_cv_(&mu_) {}
+ : seed_(s), quit_flag_(false), state_(STARTING), state_cv_(&mu_) {}
- void Wait(ReaderState s) {
+ void Wait(ReaderState s) LOCKS_EXCLUDED(mu_) {
mu_.Lock();
while (state_ != s) {
state_cv_.Wait();
@@ -320,7 +313,7 @@ class TestState {
mu_.Unlock();
}
- void Change(ReaderState s) {
+ void Change(ReaderState s) LOCKS_EXCLUDED(mu_) {
mu_.Lock();
state_ = s;
state_cv_.Signal();
@@ -329,8 +322,8 @@ class TestState {
private:
port::Mutex mu_;
- ReaderState state_;
- port::CondVar state_cv_;
+ ReaderState state_ GUARDED_BY(mu_);
+ port::CondVar state_cv_ GUARDED_BY(mu_);
};
static void ConcurrentReader(void* arg) {
@@ -338,7 +331,7 @@ static void ConcurrentReader(void* arg) {
Random rnd(state->seed_);
int64_t reads = 0;
state->Change(TestState::RUNNING);
- while (!state->quit_flag_.Acquire_Load()) {
+ while (!state->quit_flag_.load(std::memory_order_acquire)) {
state->t_.ReadStep(&rnd);
++reads;
}
@@ -360,7 +353,7 @@ static void RunConcurrent(int run) {
for (int i = 0; i < kSize; i++) {
state.t_.WriteStep(&rnd);
}
- state.quit_flag_.Release_Store(&state); // Any non-NULL arg will do
+ state.quit_flag_.store(true, std::memory_order_release);
state.Wait(TestState::DONE);
}
}
@@ -373,6 +366,4 @@ TEST(SkipTest, Concurrent5) { RunConcurrent(5); }
} // namespace leveldb
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
+int main(int argc, char** argv) { return leveldb::test::RunAllTests(); }
diff --git a/db/snapshot.h b/db/snapshot.h
index 6ed413c..9f1d664 100644
--- a/db/snapshot.h
+++ b/db/snapshot.h
@@ -16,50 +16,78 @@ class SnapshotList;
// Each SnapshotImpl corresponds to a particular sequence number.
class SnapshotImpl : public Snapshot {
public:
- SequenceNumber number_; // const after creation
+ SnapshotImpl(SequenceNumber sequence_number)
+ : sequence_number_(sequence_number) {}
+
+ SequenceNumber sequence_number() const { return sequence_number_; }
private:
friend class SnapshotList;
- // SnapshotImpl is kept in a doubly-linked circular list
+ // SnapshotImpl is kept in a doubly-linked circular list. The SnapshotList
+ // implementation operates on the next/previous fields direcly.
SnapshotImpl* prev_;
SnapshotImpl* next_;
- SnapshotList* list_; // just for sanity checks
+ const SequenceNumber sequence_number_;
+
+#if !defined(NDEBUG)
+ SnapshotList* list_ = nullptr;
+#endif // !defined(NDEBUG)
};
class SnapshotList {
public:
- SnapshotList() {
- list_.prev_ = &list_;
- list_.next_ = &list_;
+ SnapshotList() : head_(0) {
+ head_.prev_ = &head_;
+ head_.next_ = &head_;
+ }
+
+ bool empty() const { return head_.next_ == &head_; }
+ SnapshotImpl* oldest() const {
+ assert(!empty());
+ return head_.next_;
}
+ SnapshotImpl* newest() const {
+ assert(!empty());
+ return head_.prev_;
+ }
+
+ // Creates a SnapshotImpl and appends it to the end of the list.
+ SnapshotImpl* New(SequenceNumber sequence_number) {
+ assert(empty() || newest()->sequence_number_ <= sequence_number);
+
+ SnapshotImpl* snapshot = new SnapshotImpl(sequence_number);
- bool empty() const { return list_.next_ == &list_; }
- SnapshotImpl* oldest() const { assert(!empty()); return list_.next_; }
- SnapshotImpl* newest() const { assert(!empty()); return list_.prev_; }
-
- const SnapshotImpl* New(SequenceNumber seq) {
- SnapshotImpl* s = new SnapshotImpl;
- s->number_ = seq;
- s->list_ = this;
- s->next_ = &list_;
- s->prev_ = list_.prev_;
- s->prev_->next_ = s;
- s->next_->prev_ = s;
- return s;
+#if !defined(NDEBUG)
+ snapshot->list_ = this;
+#endif // !defined(NDEBUG)
+ snapshot->next_ = &head_;
+ snapshot->prev_ = head_.prev_;
+ snapshot->prev_->next_ = snapshot;
+ snapshot->next_->prev_ = snapshot;
+ return snapshot;
}
- void Delete(const SnapshotImpl* s) {
- assert(s->list_ == this);
- s->prev_->next_ = s->next_;
- s->next_->prev_ = s->prev_;
- delete s;
+ // Removes a SnapshotImpl from this list.
+ //
+ // The snapshot must have been created by calling New() on this list.
+ //
+ // The snapshot pointer should not be const, because its memory is
+ // deallocated. However, that would force us to change DB::ReleaseSnapshot(),
+ // which is in the API, and currently takes a const Snapshot.
+ void Delete(const SnapshotImpl* snapshot) {
+#if !defined(NDEBUG)
+ assert(snapshot->list_ == this);
+#endif // !defined(NDEBUG)
+ snapshot->prev_->next_ = snapshot->next_;
+ snapshot->next_->prev_ = snapshot->prev_;
+ delete snapshot;
}
private:
// Dummy head of doubly-linked list of snapshots
- SnapshotImpl list_;
+ SnapshotImpl head_;
};
} // namespace leveldb
diff --git a/db/table_cache.cc b/db/table_cache.cc
index e3d82cd..73f05fd 100644
--- a/db/table_cache.cc
+++ b/db/table_cache.cc
@@ -29,18 +29,14 @@ static void UnrefEntry(void* arg1, void* arg2) {
cache->Release(h);
}
-TableCache::TableCache(const std::string& dbname,
- const Options* options,
+TableCache::TableCache(const std::string& dbname, const Options& options,
int entries)
- : env_(options->env),
+ : env_(options.env),
dbname_(dbname),
options_(options),
- cache_(NewLRUCache(entries)) {
-}
+ cache_(NewLRUCache(entries)) {}
-TableCache::~TableCache() {
- delete cache_;
-}
+TableCache::~TableCache() { delete cache_; }
Status TableCache::FindTable(uint64_t file_number, uint64_t file_size,
Cache::Handle** handle) {
@@ -49,10 +45,10 @@ Status TableCache::FindTable(uint64_t file_number, uint64_t file_size,
EncodeFixed64(buf, file_number);
Slice key(buf, sizeof(buf));
*handle = cache_->Lookup(key);
- if (*handle == NULL) {
+ if (*handle == nullptr) {
std::string fname = TableFileName(dbname_, file_number);
- RandomAccessFile* file = NULL;
- Table* table = NULL;
+ RandomAccessFile* file = nullptr;
+ Table* table = nullptr;
s = env_->NewRandomAccessFile(fname, &file);
if (!s.ok()) {
std::string old_fname = SSTTableFileName(dbname_, file_number);
@@ -61,11 +57,11 @@ Status TableCache::FindTable(uint64_t file_number, uint64_t file_size,
}
}
if (s.ok()) {
- s = Table::Open(*options_, file, file_size, &table);
+ s = Table::Open(options_, file, file_size, &table);
}
if (!s.ok()) {
- assert(table == NULL);
+ assert(table == nullptr);
delete file;
// We do not cache error results so that if the error is transient,
// or somebody repairs the file, we recover automatically.
@@ -80,14 +76,13 @@ Status TableCache::FindTable(uint64_t file_number, uint64_t file_size,
}
Iterator* TableCache::NewIterator(const ReadOptions& options,
- uint64_t file_number,
- uint64_t file_size,
+ uint64_t file_number, uint64_t file_size,
Table** tableptr) {
- if (tableptr != NULL) {
- *tableptr = NULL;
+ if (tableptr != nullptr) {
+ *tableptr = nullptr;
}
- Cache::Handle* handle = NULL;
+ Cache::Handle* handle = nullptr;
Status s = FindTable(file_number, file_size, &handle);
if (!s.ok()) {
return NewErrorIterator(s);
@@ -96,23 +91,21 @@ Iterator* TableCache::NewIterator(const ReadOptions& options,
Table* table = reinterpret_cast<TableAndFile*>(cache_->Value(handle))->table;
Iterator* result = table->NewIterator(options);
result->RegisterCleanup(&UnrefEntry, cache_, handle);
- if (tableptr != NULL) {
+ if (tableptr != nullptr) {
*tableptr = table;
}
return result;
}
-Status TableCache::Get(const ReadOptions& options,
- uint64_t file_number,
- uint64_t file_size,
- const Slice& k,
- void* arg,
- void (*saver)(void*, const Slice&, const Slice&)) {
- Cache::Handle* handle = NULL;
+Status TableCache::Get(const ReadOptions& options, uint64_t file_number,
+ uint64_t file_size, const Slice& k, void* arg,
+ void (*handle_result)(void*, const Slice&,
+ const Slice&)) {
+ Cache::Handle* handle = nullptr;
Status s = FindTable(file_number, file_size, &handle);
if (s.ok()) {
Table* t = reinterpret_cast<TableAndFile*>(cache_->Value(handle))->table;
- s = t->InternalGet(options, k, arg, saver);
+ s = t->InternalGet(options, k, arg, handle_result);
cache_->Release(handle);
}
return s;
diff --git a/db/table_cache.h b/db/table_cache.h
index 8cf4aaf..93069c8 100644
--- a/db/table_cache.h
+++ b/db/table_cache.h
@@ -7,8 +7,10 @@
#ifndef STORAGE_LEVELDB_DB_TABLE_CACHE_H_
#define STORAGE_LEVELDB_DB_TABLE_CACHE_H_
-#include <string>
#include <stdint.h>
+
+#include <string>
+
#include "db/dbformat.h"
#include "leveldb/cache.h"
#include "leveldb/table.h"
@@ -20,40 +22,35 @@ class Env;
class TableCache {
public:
- TableCache(const std::string& dbname, const Options* options, int entries);
+ TableCache(const std::string& dbname, const Options& options, int entries);
~TableCache();
// Return an iterator for the specified file number (the corresponding
// file length must be exactly "file_size" bytes). If "tableptr" is
- // non-NULL, also sets "*tableptr" to point to the Table object
- // underlying the returned iterator, or NULL if no Table object underlies
- // the returned iterator. The returned "*tableptr" object is owned by
- // the cache and should not be deleted, and is valid for as long as the
+ // non-null, also sets "*tableptr" to point to the Table object
+ // underlying the returned iterator, or to nullptr if no Table object
+ // underlies the returned iterator. The returned "*tableptr" object is owned
+ // by the cache and should not be deleted, and is valid for as long as the
// returned iterator is live.
- Iterator* NewIterator(const ReadOptions& options,
- uint64_t file_number,
- uint64_t file_size,
- Table** tableptr = NULL);
+ Iterator* NewIterator(const ReadOptions& options, uint64_t file_number,
+ uint64_t file_size, Table** tableptr = nullptr);
// If a seek to internal key "k" in specified file finds an entry,
// call (*handle_result)(arg, found_key, found_value).
- Status Get(const ReadOptions& options,
- uint64_t file_number,
- uint64_t file_size,
- const Slice& k,
- void* arg,
+ Status Get(const ReadOptions& options, uint64_t file_number,
+ uint64_t file_size, const Slice& k, void* arg,
void (*handle_result)(void*, const Slice&, const Slice&));
// Evict any entry for the specified file number
void Evict(uint64_t file_number);
private:
+ Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**);
+
Env* const env_;
const std::string dbname_;
- const Options* options_;
+ const Options& options_;
Cache* cache_;
-
- Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**);
};
} // namespace leveldb
diff --git a/db/version_edit.cc b/db/version_edit.cc
index f10a2d5..cd770ef 100644
--- a/db/version_edit.cc
+++ b/db/version_edit.cc
@@ -12,15 +12,15 @@ namespace leveldb {
// Tag numbers for serialized VersionEdit. These numbers are written to
// disk and should not be changed.
enum Tag {
- kComparator = 1,
- kLogNumber = 2,
- kNextFileNumber = 3,
- kLastSequence = 4,
- kCompactPointer = 5,
- kDeletedFile = 6,
- kNewFile = 7,
+ kComparator = 1,
+ kLogNumber = 2,
+ kNextFileNumber = 3,
+ kLastSequence = 4,
+ kCompactPointer = 5,
+ kDeletedFile = 6,
+ kNewFile = 7,
// 8 was used for large value refs
- kPrevLogNumber = 9
+ kPrevLogNumber = 9
};
void VersionEdit::Clear() {
@@ -66,12 +66,10 @@ void VersionEdit::EncodeTo(std::string* dst) const {
PutLengthPrefixedSlice(dst, compact_pointers_[i].second.Encode());
}
- for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
- iter != deleted_files_.end();
- ++iter) {
+ for (const auto& deleted_file_kvp : deleted_files_) {
PutVarint32(dst, kDeletedFile);
- PutVarint32(dst, iter->first); // level
- PutVarint64(dst, iter->second); // file number
+ PutVarint32(dst, deleted_file_kvp.first); // level
+ PutVarint64(dst, deleted_file_kvp.second); // file number
}
for (size_t i = 0; i < new_files_.size(); i++) {
@@ -88,8 +86,7 @@ void VersionEdit::EncodeTo(std::string* dst) const {
static bool GetInternalKey(Slice* input, InternalKey* dst) {
Slice str;
if (GetLengthPrefixedSlice(input, &str)) {
- dst->DecodeFrom(str);
- return true;
+ return dst->DecodeFrom(str);
} else {
return false;
}
@@ -97,8 +94,7 @@ static bool GetInternalKey(Slice* input, InternalKey* dst) {
static bool GetLevel(Slice* input, int* level) {
uint32_t v;
- if (GetVarint32(input, &v) &&
- v < config::kNumLevels) {
+ if (GetVarint32(input, &v) && v < config::kNumLevels) {
*level = v;
return true;
} else {
@@ -109,7 +105,7 @@ static bool GetLevel(Slice* input, int* level) {
Status VersionEdit::DecodeFrom(const Slice& src) {
Clear();
Slice input = src;
- const char* msg = NULL;
+ const char* msg = nullptr;
uint32_t tag;
// Temporary storage for parsing
@@ -119,7 +115,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
Slice str;
InternalKey key;
- while (msg == NULL && GetVarint32(&input, &tag)) {
+ while (msg == nullptr && GetVarint32(&input, &tag)) {
switch (tag) {
case kComparator:
if (GetLengthPrefixedSlice(&input, &str)) {
@@ -163,8 +159,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
break;
case kCompactPointer:
- if (GetLevel(&input, &level) &&
- GetInternalKey(&input, &key)) {
+ if (GetLevel(&input, &level) && GetInternalKey(&input, &key)) {
compact_pointers_.push_back(std::make_pair(level, key));
} else {
msg = "compaction pointer";
@@ -172,8 +167,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
break;
case kDeletedFile:
- if (GetLevel(&input, &level) &&
- GetVarint64(&input, &number)) {
+ if (GetLevel(&input, &level) && GetVarint64(&input, &number)) {
deleted_files_.insert(std::make_pair(level, number));
} else {
msg = "deleted file";
@@ -181,8 +175,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
break;
case kNewFile:
- if (GetLevel(&input, &level) &&
- GetVarint64(&input, &f.number) &&
+ if (GetLevel(&input, &level) && GetVarint64(&input, &f.number) &&
GetVarint64(&input, &f.file_size) &&
GetInternalKey(&input, &f.smallest) &&
GetInternalKey(&input, &f.largest)) {
@@ -198,12 +191,12 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
}
}
- if (msg == NULL && !input.empty()) {
+ if (msg == nullptr && !input.empty()) {
msg = "invalid tag";
}
Status result;
- if (msg != NULL) {
+ if (msg != nullptr) {
result = Status::Corruption("VersionEdit", msg);
}
return result;
@@ -238,13 +231,11 @@ std::string VersionEdit::DebugString() const {
r.append(" ");
r.append(compact_pointers_[i].second.DebugString());
}
- for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
- iter != deleted_files_.end();
- ++iter) {
+ for (const auto& deleted_files_kvp : deleted_files_) {
r.append("\n DeleteFile: ");
- AppendNumberTo(&r, iter->first);
+ AppendNumberTo(&r, deleted_files_kvp.first);
r.append(" ");
- AppendNumberTo(&r, iter->second);
+ AppendNumberTo(&r, deleted_files_kvp.second);
}
for (size_t i = 0; i < new_files_.size(); i++) {
const FileMetaData& f = new_files_[i].second;
diff --git a/db/version_edit.h b/db/version_edit.h
index eaef77b..86b2f22 100644
--- a/db/version_edit.h
+++ b/db/version_edit.h
@@ -8,6 +8,7 @@
#include <set>
#include <utility>
#include <vector>
+
#include "db/dbformat.h"
namespace leveldb {
@@ -15,20 +16,20 @@ namespace leveldb {
class VersionSet;
struct FileMetaData {
+ FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0) {}
+
int refs;
- int allowed_seeks; // Seeks allowed until compaction
+ int allowed_seeks; // Seeks allowed until compaction
uint64_t number;
- uint64_t file_size; // File size in bytes
- InternalKey smallest; // Smallest internal key served by table
- InternalKey largest; // Largest internal key served by table
-
- FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0) { }
+ uint64_t file_size; // File size in bytes
+ InternalKey smallest; // Smallest internal key served by table
+ InternalKey largest; // Largest internal key served by table
};
class VersionEdit {
public:
VersionEdit() { Clear(); }
- ~VersionEdit() { }
+ ~VersionEdit() = default;
void Clear();
@@ -59,10 +60,8 @@ class VersionEdit {
// Add the specified file at the specified number.
// REQUIRES: This version has not been saved (see VersionSet::SaveTo)
// REQUIRES: "smallest" and "largest" are smallest and largest keys in file
- void AddFile(int level, uint64_t file,
- uint64_t file_size,
- const InternalKey& smallest,
- const InternalKey& largest) {
+ void AddFile(int level, uint64_t file, uint64_t file_size,
+ const InternalKey& smallest, const InternalKey& largest) {
FileMetaData f;
f.number = file;
f.file_size = file_size;
@@ -84,7 +83,7 @@ class VersionEdit {
private:
friend class VersionSet;
- typedef std::set< std::pair<int, uint64_t> > DeletedFileSet;
+ typedef std::set<std::pair<int, uint64_t> > DeletedFileSet;
std::string comparator_;
uint64_t log_number_;
@@ -97,9 +96,9 @@ class VersionEdit {
bool has_next_file_number_;
bool has_last_sequence_;
- std::vector< std::pair<int, InternalKey> > compact_pointers_;
+ std::vector<std::pair<int, InternalKey> > compact_pointers_;
DeletedFileSet deleted_files_;
- std::vector< std::pair<int, FileMetaData> > new_files_;
+ std::vector<std::pair<int, FileMetaData> > new_files_;
};
} // namespace leveldb
diff --git a/db/version_edit_test.cc b/db/version_edit_test.cc
index 280310b..0b7cda8 100644
--- a/db/version_edit_test.cc
+++ b/db/version_edit_test.cc
@@ -17,7 +17,7 @@ static void TestEncodeDecode(const VersionEdit& edit) {
ASSERT_EQ(encoded, encoded2);
}
-class VersionEditTest { };
+class VersionEditTest {};
TEST(VersionEditTest, EncodeDecode) {
static const uint64_t kBig = 1ull << 50;
@@ -41,6 +41,4 @@ TEST(VersionEditTest, EncodeDecode) {
} // namespace leveldb
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
+int main(int argc, char** argv) { return leveldb::test::RunAllTests(); }
diff --git a/db/version_set.cc b/db/version_set.cc
index aa83df5..b62a2d0 100644
--- a/db/version_set.cc
+++ b/db/version_set.cc
@@ -4,8 +4,10 @@
#include "db/version_set.h"
-#include <algorithm>
#include <stdio.h>
+
+#include <algorithm>
+
#include "db/filename.h"
#include "db/log_reader.h"
#include "db/log_writer.h"
@@ -20,21 +22,29 @@
namespace leveldb {
-static const int kTargetFileSize = 2 * 1048576;
+static size_t TargetFileSize(const Options* options) {
+ return options->max_file_size;
+}
// Maximum bytes of overlaps in grandparent (i.e., level+2) before we
// stop building a single file in a level->level+1 compaction.
-static const int64_t kMaxGrandParentOverlapBytes = 10 * kTargetFileSize;
+static int64_t MaxGrandParentOverlapBytes(const Options* options) {
+ return 10 * TargetFileSize(options);
+}
// Maximum number of bytes in all compacted files. We avoid expanding
// the lower level file set of a compaction if it would make the
// total compaction cover more than this many bytes.
-static const int64_t kExpandedCompactionByteSizeLimit = 25 * kTargetFileSize;
+static int64_t ExpandedCompactionByteSizeLimit(const Options* options) {
+ return 25 * TargetFileSize(options);
+}
-static double MaxBytesForLevel(int level) {
+static double MaxBytesForLevel(const Options* options, int level) {
// Note: the result for level zero is not really used since we set
// the level-0 compaction threshold based on number of files.
- double result = 10 * 1048576.0; // Result for both level-0 and level-1
+
+ // Result for both level-0 and level-1
+ double result = 10. * 1048576.0;
while (level > 1) {
result *= 10;
level--;
@@ -42,8 +52,9 @@ static double MaxBytesForLevel(int level) {
return result;
}
-static uint64_t MaxFileSizeForLevel(int level) {
- return kTargetFileSize; // We could vary per level to reduce number of files?
+static uint64_t MaxFileSizeForLevel(const Options* options, int level) {
+ // We could vary per level to reduce number of files?
+ return TargetFileSize(options);
}
static int64_t TotalFileSize(const std::vector<FileMetaData*>& files) {
@@ -75,8 +86,7 @@ Version::~Version() {
}
int FindFile(const InternalKeyComparator& icmp,
- const std::vector<FileMetaData*>& files,
- const Slice& key) {
+ const std::vector<FileMetaData*>& files, const Slice& key) {
uint32_t left = 0;
uint32_t right = files.size();
while (left < right) {
@@ -95,26 +105,25 @@ int FindFile(const InternalKeyComparator& icmp,
return right;
}
-static bool AfterFile(const Comparator* ucmp,
- const Slice* user_key, const FileMetaData* f) {
- // NULL user_key occurs before all keys and is therefore never after *f
- return (user_key != NULL &&
+static bool AfterFile(const Comparator* ucmp, const Slice* user_key,
+ const FileMetaData* f) {
+ // null user_key occurs before all keys and is therefore never after *f
+ return (user_key != nullptr &&
ucmp->Compare(*user_key, f->largest.user_key()) > 0);
}
-static bool BeforeFile(const Comparator* ucmp,
- const Slice* user_key, const FileMetaData* f) {
- // NULL user_key occurs after all keys and is therefore never before *f
- return (user_key != NULL &&
+static bool BeforeFile(const Comparator* ucmp, const Slice* user_key,
+ const FileMetaData* f) {
+ // null user_key occurs after all keys and is therefore never before *f
+ return (user_key != nullptr &&
ucmp->Compare(*user_key, f->smallest.user_key()) < 0);
}
-bool SomeFileOverlapsRange(
- const InternalKeyComparator& icmp,
- bool disjoint_sorted_files,
- const std::vector<FileMetaData*>& files,
- const Slice* smallest_user_key,
- const Slice* largest_user_key) {
+bool SomeFileOverlapsRange(const InternalKeyComparator& icmp,
+ bool disjoint_sorted_files,
+ const std::vector<FileMetaData*>& files,
+ const Slice* smallest_user_key,
+ const Slice* largest_user_key) {
const Comparator* ucmp = icmp.user_comparator();
if (!disjoint_sorted_files) {
// Need to check against all files
@@ -132,10 +141,11 @@ bool SomeFileOverlapsRange(
// Binary search over file list
uint32_t index = 0;
- if (smallest_user_key != NULL) {
+ if (smallest_user_key != nullptr) {
// Find the earliest possible internal key for smallest_user_key
- InternalKey small(*smallest_user_key, kMaxSequenceNumber,kValueTypeForSeek);
- index = FindFile(icmp, files, small.Encode());
+ InternalKey small_key(*smallest_user_key, kMaxSequenceNumber,
+ kValueTypeForSeek);
+ index = FindFile(icmp, files, small_key.Encode());
}
if (index >= files.size()) {
@@ -155,25 +165,21 @@ class Version::LevelFileNumIterator : public Iterator {
public:
LevelFileNumIterator(const InternalKeyComparator& icmp,
const std::vector<FileMetaData*>* flist)
- : icmp_(icmp),
- flist_(flist),
- index_(flist->size()) { // Marks as invalid
- }
- virtual bool Valid() const {
- return index_ < flist_->size();
+ : icmp_(icmp), flist_(flist), index_(flist->size()) { // Marks as invalid
}
- virtual void Seek(const Slice& target) {
+ bool Valid() const override { return index_ < flist_->size(); }
+ void Seek(const Slice& target) override {
index_ = FindFile(icmp_, *flist_, target);
}
- virtual void SeekToFirst() { index_ = 0; }
- virtual void SeekToLast() {
+ void SeekToFirst() override { index_ = 0; }
+ void SeekToLast() override {
index_ = flist_->empty() ? 0 : flist_->size() - 1;
}
- virtual void Next() {
+ void Next() override {
assert(Valid());
index_++;
}
- virtual void Prev() {
+ void Prev() override {
assert(Valid());
if (index_ == 0) {
index_ = flist_->size(); // Marks as invalid
@@ -181,17 +187,18 @@ class Version::LevelFileNumIterator : public Iterator {
index_--;
}
}
- Slice key() const {
+ Slice key() const override {
assert(Valid());
return (*flist_)[index_]->largest.Encode();
}
- Slice value() const {
+ Slice value() const override {
assert(Valid());
EncodeFixed64(value_buf_, (*flist_)[index_]->number);
- EncodeFixed64(value_buf_+8, (*flist_)[index_]->file_size);
+ EncodeFixed64(value_buf_ + 8, (*flist_)[index_]->file_size);
return Slice(value_buf_, sizeof(value_buf_));
}
- virtual Status status() const { return Status::OK(); }
+ Status status() const override { return Status::OK(); }
+
private:
const InternalKeyComparator icmp_;
const std::vector<FileMetaData*>* const flist_;
@@ -201,16 +208,14 @@ class Version::LevelFileNumIterator : public Iterator {
mutable char value_buf_[16];
};
-static Iterator* GetFileIterator(void* arg,
- const ReadOptions& options,
+static Iterator* GetFileIterator(void* arg, const ReadOptions& options,
const Slice& file_value) {
TableCache* cache = reinterpret_cast<TableCache*>(arg);
if (file_value.size() != 16) {
return NewErrorIterator(
Status::Corruption("FileReader invoked with unexpected value"));
} else {
- return cache->NewIterator(options,
- DecodeFixed64(file_value.data()),
+ return cache->NewIterator(options, DecodeFixed64(file_value.data()),
DecodeFixed64(file_value.data() + 8));
}
}
@@ -218,17 +223,16 @@ static Iterator* GetFileIterator(void* arg,
Iterator* Version::NewConcatenatingIterator(const ReadOptions& options,
int level) const {
return NewTwoLevelIterator(
- new LevelFileNumIterator(vset_->icmp_, &files_[level]),
- &GetFileIterator, vset_->table_cache_, options);
+ new LevelFileNumIterator(vset_->icmp_, &files_[level]), &GetFileIterator,
+ vset_->table_cache_, options);
}
void Version::AddIterators(const ReadOptions& options,
std::vector<Iterator*>* iters) {
// Merge all level zero files together since they may overlap
for (size_t i = 0; i < files_[0].size(); i++) {
- iters->push_back(
- vset_->table_cache_->NewIterator(
- options, files_[0][i]->number, files_[0][i]->file_size));
+ iters->push_back(vset_->table_cache_->NewIterator(
+ options, files_[0][i]->number, files_[0][i]->file_size));
}
// For levels > 0, we can use a concatenating iterator that sequentially
@@ -255,7 +259,7 @@ struct Saver {
Slice user_key;
std::string* value;
};
-}
+} // namespace
static void SaveValue(void* arg, const Slice& ikey, const Slice& v) {
Saver* s = reinterpret_cast<Saver*>(arg);
ParsedInternalKey parsed_key;
@@ -275,8 +279,7 @@ static bool NewestFirst(FileMetaData* a, FileMetaData* b) {
return a->number > b->number;
}
-void Version::ForEachOverlapping(Slice user_key, Slice internal_key,
- void* arg,
+void Version::ForEachOverlapping(Slice user_key, Slice internal_key, void* arg,
bool (*func)(void*, int, FileMetaData*)) {
// TODO(sanjay): Change Version::Get() to use this function.
const Comparator* ucmp = vset_->icmp_.user_comparator();
@@ -320,23 +323,21 @@ void Version::ForEachOverlapping(Slice user_key, Slice internal_key,
}
}
-Status Version::Get(const ReadOptions& options,
- const LookupKey& k,
- std::string* value,
- GetStats* stats) {
+Status Version::Get(const ReadOptions& options, const LookupKey& k,
+ std::string* value, GetStats* stats) {
Slice ikey = k.internal_key();
Slice user_key = k.user_key();
const Comparator* ucmp = vset_->icmp_.user_comparator();
Status s;
- stats->seek_file = NULL;
+ stats->seek_file = nullptr;
stats->seek_file_level = -1;
- FileMetaData* last_file_read = NULL;
+ FileMetaData* last_file_read = nullptr;
int last_file_read_level = -1;
// We can search level-by-level since entries never hop across
// levels. Therefore we are guaranteed that if we find data
- // in an smaller level, later levels are irrelevant.
+ // in a smaller level, later levels are irrelevant.
std::vector<FileMetaData*> tmp;
FileMetaData* tmp2;
for (int level = 0; level < config::kNumLevels; level++) {
@@ -365,13 +366,13 @@ Status Version::Get(const ReadOptions& options,
// Binary search to find earliest index whose largest key >= ikey.
uint32_t index = FindFile(vset_->icmp_, files_[level], ikey);
if (index >= num_files) {
- files = NULL;
+ files = nullptr;
num_files = 0;
} else {
tmp2 = files[index];
if (ucmp->Compare(user_key, tmp2->smallest.user_key()) < 0) {
// All of "tmp2" is past any data for user_key
- files = NULL;
+ files = nullptr;
num_files = 0;
} else {
files = &tmp2;
@@ -381,7 +382,7 @@ Status Version::Get(const ReadOptions& options,
}
for (uint32_t i = 0; i < num_files; ++i) {
- if (last_file_read != NULL && stats->seek_file == NULL) {
+ if (last_file_read != nullptr && stats->seek_file == nullptr) {
// We have had more than one seek for this read. Charge the 1st file.
stats->seek_file = last_file_read;
stats->seek_file_level = last_file_read_level;
@@ -396,14 +397,14 @@ Status Version::Get(const ReadOptions& options,
saver.ucmp = ucmp;
saver.user_key = user_key;
saver.value = value;
- s = vset_->table_cache_->Get(options, f->number, f->file_size,
- ikey, &saver, SaveValue);
+ s = vset_->table_cache_->Get(options, f->number, f->file_size, ikey,
+ &saver, SaveValue);
if (!s.ok()) {
return s;
}
switch (saver.state) {
case kNotFound:
- break; // Keep searching in other files
+ break; // Keep searching in other files
case kFound:
return s;
case kDeleted:
@@ -421,9 +422,9 @@ Status Version::Get(const ReadOptions& options,
bool Version::UpdateStats(const GetStats& stats) {
FileMetaData* f = stats.seek_file;
- if (f != NULL) {
+ if (f != nullptr) {
f->allowed_seeks--;
- if (f->allowed_seeks <= 0 && file_to_compact_ == NULL) {
+ if (f->allowed_seeks <= 0 && file_to_compact_ == nullptr) {
file_to_compact_ = f;
file_to_compact_level_ = stats.seek_file_level;
return true;
@@ -470,9 +471,7 @@ bool Version::RecordReadSample(Slice internal_key) {
return false;
}
-void Version::Ref() {
- ++refs_;
-}
+void Version::Ref() { ++refs_; }
void Version::Unref() {
assert(this != &vset_->dummy_versions_);
@@ -483,16 +482,14 @@ void Version::Unref() {
}
}
-bool Version::OverlapInLevel(int level,
- const Slice* smallest_user_key,
+bool Version::OverlapInLevel(int level, const Slice* smallest_user_key,
const Slice* largest_user_key) {
return SomeFileOverlapsRange(vset_->icmp_, (level > 0), files_[level],
smallest_user_key, largest_user_key);
}
-int Version::PickLevelForMemTableOutput(
- const Slice& smallest_user_key,
- const Slice& largest_user_key) {
+int Version::PickLevelForMemTableOutput(const Slice& smallest_user_key,
+ const Slice& largest_user_key) {
int level = 0;
if (!OverlapInLevel(0, &smallest_user_key, &largest_user_key)) {
// Push to next level if there is no overlap in next level,
@@ -508,7 +505,7 @@ int Version::PickLevelForMemTableOutput(
// Check that file does not overlap too many grandparent bytes.
GetOverlappingInputs(level + 2, &start, &limit, &overlaps);
const int64_t sum = TotalFileSize(overlaps);
- if (sum > kMaxGrandParentOverlapBytes) {
+ if (sum > MaxGrandParentOverlapBytes(vset_->options_)) {
break;
}
}
@@ -519,40 +516,39 @@ int Version::PickLevelForMemTableOutput(
}
// Store in "*inputs" all files in "level" that overlap [begin,end]
-void Version::GetOverlappingInputs(
- int level,
- const InternalKey* begin,
- const InternalKey* end,
- std::vector<FileMetaData*>* inputs) {
+void Version::GetOverlappingInputs(int level, const InternalKey* begin,
+ const InternalKey* end,
+ std::vector<FileMetaData*>* inputs) {
assert(level >= 0);
assert(level < config::kNumLevels);
inputs->clear();
Slice user_begin, user_end;
- if (begin != NULL) {
+ if (begin != nullptr) {
user_begin = begin->user_key();
}
- if (end != NULL) {
+ if (end != nullptr) {
user_end = end->user_key();
}
const Comparator* user_cmp = vset_->icmp_.user_comparator();
- for (size_t i = 0; i < files_[level].size(); ) {
+ for (size_t i = 0; i < files_[level].size();) {
FileMetaData* f = files_[level][i++];
const Slice file_start = f->smallest.user_key();
const Slice file_limit = f->largest.user_key();
- if (begin != NULL && user_cmp->Compare(file_limit, user_begin) < 0) {
+ if (begin != nullptr && user_cmp->Compare(file_limit, user_begin) < 0) {
// "f" is completely before specified range; skip it
- } else if (end != NULL && user_cmp->Compare(file_start, user_end) > 0) {
+ } else if (end != nullptr && user_cmp->Compare(file_start, user_end) > 0) {
// "f" is completely after specified range; skip it
} else {
inputs->push_back(f);
if (level == 0) {
// Level-0 files may overlap each other. So check if the newly
// added file has expanded the range. If so, restart search.
- if (begin != NULL && user_cmp->Compare(file_start, user_begin) < 0) {
+ if (begin != nullptr && user_cmp->Compare(file_start, user_begin) < 0) {
user_begin = file_start;
inputs->clear();
i = 0;
- } else if (end != NULL && user_cmp->Compare(file_limit, user_end) > 0) {
+ } else if (end != nullptr &&
+ user_cmp->Compare(file_limit, user_end) > 0) {
user_end = file_limit;
inputs->clear();
i = 0;
@@ -620,9 +616,7 @@ class VersionSet::Builder {
public:
// Initialize a builder with the files from *base and other info from *vset
- Builder(VersionSet* vset, Version* base)
- : vset_(vset),
- base_(base) {
+ Builder(VersionSet* vset, Version* base) : vset_(vset), base_(base) {
base_->Ref();
BySmallestKey cmp;
cmp.internal_comparator = &vset_->icmp_;
@@ -636,8 +630,8 @@ class VersionSet::Builder {
const FileSet* added = levels_[level].added_files;
std::vector<FileMetaData*> to_unref;
to_unref.reserve(added->size());
- for (FileSet::const_iterator it = added->begin();
- it != added->end(); ++it) {
+ for (FileSet::const_iterator it = added->begin(); it != added->end();
+ ++it) {
to_unref.push_back(*it);
}
delete added;
@@ -662,12 +656,9 @@ class VersionSet::Builder {
}
// Delete files
- const VersionEdit::DeletedFileSet& del = edit->deleted_files_;
- for (VersionEdit::DeletedFileSet::const_iterator iter = del.begin();
- iter != del.end();
- ++iter) {
- const int level = iter->first;
- const uint64_t number = iter->second;
+ for (const auto& deleted_file_set_kvp : edit->deleted_files_) {
+ const int level = deleted_file_set_kvp.first;
+ const uint64_t number = deleted_file_set_kvp.second;
levels_[level].deleted_files.insert(number);
}
@@ -690,7 +681,7 @@ class VersionSet::Builder {
// same as the compaction of 40KB of data. We are a little
// conservative and allow approximately one seek for every 16KB
// of data before triggering a compaction.
- f->allowed_seeks = (f->file_size / 16384);
+ f->allowed_seeks = static_cast<int>((f->file_size / 16384U));
if (f->allowed_seeks < 100) f->allowed_seeks = 100;
levels_[level].deleted_files.erase(f->number);
@@ -708,20 +699,17 @@ class VersionSet::Builder {
const std::vector<FileMetaData*>& base_files = base_->files_[level];
std::vector<FileMetaData*>::const_iterator base_iter = base_files.begin();
std::vector<FileMetaData*>::const_iterator base_end = base_files.end();
- const FileSet* added = levels_[level].added_files;
- v->files_[level].reserve(base_files.size() + added->size());
- for (FileSet::const_iterator added_iter = added->begin();
- added_iter != added->end();
- ++added_iter) {
+ const FileSet* added_files = levels_[level].added_files;
+ v->files_[level].reserve(base_files.size() + added_files->size());
+ for (const auto& added_file : *added_files) {
// Add all smaller files listed in base_
- for (std::vector<FileMetaData*>::const_iterator bpos
- = std::upper_bound(base_iter, base_end, *added_iter, cmp);
- base_iter != bpos;
- ++base_iter) {
+ for (std::vector<FileMetaData*>::const_iterator bpos =
+ std::upper_bound(base_iter, base_end, added_file, cmp);
+ base_iter != bpos; ++base_iter) {
MaybeAddFile(v, level, *base_iter);
}
- MaybeAddFile(v, level, *added_iter);
+ MaybeAddFile(v, level, added_file);
}
// Add remaining base files
@@ -733,7 +721,7 @@ class VersionSet::Builder {
// Make sure there is no overlap in levels > 0
if (level > 0) {
for (uint32_t i = 1; i < v->files_[level].size(); i++) {
- const InternalKey& prev_end = v->files_[level][i-1]->largest;
+ const InternalKey& prev_end = v->files_[level][i - 1]->largest;
const InternalKey& this_begin = v->files_[level][i]->smallest;
if (vset_->icmp_.Compare(prev_end, this_begin) >= 0) {
fprintf(stderr, "overlapping ranges in same level %s vs. %s\n",
@@ -754,7 +742,7 @@ class VersionSet::Builder {
std::vector<FileMetaData*>* files = &v->files_[level];
if (level > 0 && !files->empty()) {
// Must not overlap
- assert(vset_->icmp_.Compare((*files)[files->size()-1]->largest,
+ assert(vset_->icmp_.Compare((*files)[files->size() - 1]->largest,
f->smallest) < 0);
}
f->refs++;
@@ -763,8 +751,7 @@ class VersionSet::Builder {
}
};
-VersionSet::VersionSet(const std::string& dbname,
- const Options* options,
+VersionSet::VersionSet(const std::string& dbname, const Options* options,
TableCache* table_cache,
const InternalKeyComparator* cmp)
: env_(options->env),
@@ -777,10 +764,10 @@ VersionSet::VersionSet(const std::string& dbname,
last_sequence_(0),
log_number_(0),
prev_log_number_(0),
- descriptor_file_(NULL),
- descriptor_log_(NULL),
+ descriptor_file_(nullptr),
+ descriptor_log_(nullptr),
dummy_versions_(this),
- current_(NULL) {
+ current_(nullptr) {
AppendVersion(new Version(this));
}
@@ -795,7 +782,7 @@ void VersionSet::AppendVersion(Version* v) {
// Make "v" current
assert(v->refs_ == 0);
assert(v != current_);
- if (current_ != NULL) {
+ if (current_ != nullptr) {
current_->Unref();
}
current_ = v;
@@ -835,10 +822,10 @@ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu) {
// a temporary file that contains a snapshot of the current version.
std::string new_manifest_file;
Status s;
- if (descriptor_log_ == NULL) {
+ if (descriptor_log_ == nullptr) {
// No reason to unlock *mu here since we only hit this path in the
// first call to LogAndApply (when opening the database).
- assert(descriptor_file_ == NULL);
+ assert(descriptor_file_ == nullptr);
new_manifest_file = DescriptorFileName(dbname_, manifest_file_number_);
edit->SetNextFile(next_file_number_);
s = env_->NewWritableFile(new_manifest_file, &descriptor_file_);
@@ -884,8 +871,8 @@ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu) {
if (!new_manifest_file.empty()) {
delete descriptor_log_;
delete descriptor_file_;
- descriptor_log_ = NULL;
- descriptor_file_ = NULL;
+ descriptor_log_ = nullptr;
+ descriptor_file_ = nullptr;
env_->DeleteFile(new_manifest_file);
}
}
@@ -893,10 +880,10 @@ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu) {
return s;
}
-Status VersionSet::Recover() {
+Status VersionSet::Recover(bool* save_manifest) {
struct LogReporter : public log::Reader::Reporter {
Status* status;
- virtual void Corruption(size_t bytes, const Status& s) {
+ void Corruption(size_t bytes, const Status& s) override {
if (this->status->ok()) *this->status = s;
}
};
@@ -907,7 +894,7 @@ Status VersionSet::Recover() {
if (!s.ok()) {
return s;
}
- if (current.empty() || current[current.size()-1] != '\n') {
+ if (current.empty() || current[current.size() - 1] != '\n') {
return Status::Corruption("CURRENT file does not end with newline");
}
current.resize(current.size() - 1);
@@ -916,6 +903,10 @@ Status VersionSet::Recover() {
SequentialFile* file;
s = env_->NewSequentialFile(dscname, &file);
if (!s.ok()) {
+ if (s.IsNotFound()) {
+ return Status::Corruption("CURRENT points to a non-existent file",
+ s.ToString());
+ }
return s;
}
@@ -932,7 +923,8 @@ Status VersionSet::Recover() {
{
LogReporter reporter;
reporter.status = &s;
- log::Reader reader(file, &reporter, true/*checksum*/, 0/*initial_offset*/);
+ log::Reader reader(file, &reporter, true /*checksum*/,
+ 0 /*initial_offset*/);
Slice record;
std::string scratch;
while (reader.ReadRecord(&record, &scratch) && s.ok()) {
@@ -973,7 +965,7 @@ Status VersionSet::Recover() {
}
}
delete file;
- file = NULL;
+ file = nullptr;
if (s.ok()) {
if (!have_next_file) {
@@ -1003,11 +995,49 @@ Status VersionSet::Recover() {
last_sequence_ = last_sequence;
log_number_ = log_number;
prev_log_number_ = prev_log_number;
+
+ // See if we can reuse the existing MANIFEST file.
+ if (ReuseManifest(dscname, current)) {
+ // No need to save new manifest
+ } else {
+ *save_manifest = true;
+ }
}
return s;
}
+bool VersionSet::ReuseManifest(const std::string& dscname,
+ const std::string& dscbase) {
+ if (!options_->reuse_logs) {
+ return false;
+ }
+ FileType manifest_type;
+ uint64_t manifest_number;
+ uint64_t manifest_size;
+ if (!ParseFileName(dscbase, &manifest_number, &manifest_type) ||
+ manifest_type != kDescriptorFile ||
+ !env_->GetFileSize(dscname, &manifest_size).ok() ||
+ // Make new compacted MANIFEST if old one is too big
+ manifest_size >= TargetFileSize(options_)) {
+ return false;
+ }
+
+ assert(descriptor_file_ == nullptr);
+ assert(descriptor_log_ == nullptr);
+ Status r = env_->NewAppendableFile(dscname, &descriptor_file_);
+ if (!r.ok()) {
+ Log(options_->info_log, "Reuse MANIFEST: %s\n", r.ToString().c_str());
+ assert(descriptor_file_ == nullptr);
+ return false;
+ }
+
+ Log(options_->info_log, "Reusing MANIFEST %s\n", dscname.c_str());
+ descriptor_log_ = new log::Writer(descriptor_file_, manifest_size);
+ manifest_file_number_ = manifest_number;
+ return true;
+}
+
void VersionSet::MarkFileNumberUsed(uint64_t number) {
if (next_file_number_ <= number) {
next_file_number_ = number + 1;
@@ -1019,7 +1049,7 @@ void VersionSet::Finalize(Version* v) {
int best_level = -1;
double best_score = -1;
- for (int level = 0; level < config::kNumLevels-1; level++) {
+ for (int level = 0; level < config::kNumLevels - 1; level++) {
double score;
if (level == 0) {
// We treat level-0 specially by bounding the number of files
@@ -1034,11 +1064,12 @@ void VersionSet::Finalize(Version* v) {
// setting, or very high compression ratios, or lots of
// overwrites/deletions).
score = v->files_[level].size() /
- static_cast<double>(config::kL0_CompactionTrigger);
+ static_cast<double>(config::kL0_CompactionTrigger);
} else {
// Compute the ratio of current size to size limit.
const uint64_t level_bytes = TotalFileSize(v->files_[level]);
- score = static_cast<double>(level_bytes) / MaxBytesForLevel(level);
+ score =
+ static_cast<double>(level_bytes) / MaxBytesForLevel(options_, level);
}
if (score > best_score) {
@@ -1089,16 +1120,12 @@ int VersionSet::NumLevelFiles(int level) const {
const char* VersionSet::LevelSummary(LevelSummaryStorage* scratch) const {
// Update code if kNumLevels changes
- assert(config::kNumLevels == 7);
+ static_assert(config::kNumLevels == 7, "");
snprintf(scratch->buffer, sizeof(scratch->buffer),
- "files[ %d %d %d %d %d %d %d ]",
- int(current_->files_[0].size()),
- int(current_->files_[1].size()),
- int(current_->files_[2].size()),
- int(current_->files_[3].size()),
- int(current_->files_[4].size()),
- int(current_->files_[5].size()),
- int(current_->files_[6].size()));
+ "files[ %d %d %d %d %d %d %d ]", int(current_->files_[0].size()),
+ int(current_->files_[1].size()), int(current_->files_[2].size()),
+ int(current_->files_[3].size()), int(current_->files_[4].size()),
+ int(current_->files_[5].size()), int(current_->files_[6].size()));
return scratch->buffer;
}
@@ -1124,7 +1151,7 @@ uint64_t VersionSet::ApproximateOffsetOf(Version* v, const InternalKey& ikey) {
Table* tableptr;
Iterator* iter = table_cache_->NewIterator(
ReadOptions(), files[i]->number, files[i]->file_size, &tableptr);
- if (tableptr != NULL) {
+ if (tableptr != nullptr) {
result += tableptr->ApproximateOffsetOf(ikey.Encode());
}
delete iter;
@@ -1135,8 +1162,7 @@ uint64_t VersionSet::ApproximateOffsetOf(Version* v, const InternalKey& ikey) {
}
void VersionSet::AddLiveFiles(std::set<uint64_t>* live) {
- for (Version* v = dummy_versions_.next_;
- v != &dummy_versions_;
+ for (Version* v = dummy_versions_.next_; v != &dummy_versions_;
v = v->next_) {
for (int level = 0; level < config::kNumLevels; level++) {
const std::vector<FileMetaData*>& files = v->files_[level];
@@ -1159,7 +1185,7 @@ int64_t VersionSet::MaxNextLevelOverlappingBytes() {
for (int level = 1; level < config::kNumLevels - 1; level++) {
for (size_t i = 0; i < current_->files_[level].size(); i++) {
const FileMetaData* f = current_->files_[level][i];
- current_->GetOverlappingInputs(level+1, &f->smallest, &f->largest,
+ current_->GetOverlappingInputs(level + 1, &f->smallest, &f->largest,
&overlaps);
const int64_t sum = TotalFileSize(overlaps);
if (sum > result) {
@@ -1174,8 +1200,7 @@ int64_t VersionSet::MaxNextLevelOverlappingBytes() {
// *smallest, *largest.
// REQUIRES: inputs is not empty
void VersionSet::GetRange(const std::vector<FileMetaData*>& inputs,
- InternalKey* smallest,
- InternalKey* largest) {
+ InternalKey* smallest, InternalKey* largest) {
assert(!inputs.empty());
smallest->Clear();
largest->Clear();
@@ -1200,8 +1225,7 @@ void VersionSet::GetRange(const std::vector<FileMetaData*>& inputs,
// REQUIRES: inputs is not empty
void VersionSet::GetRange2(const std::vector<FileMetaData*>& inputs1,
const std::vector<FileMetaData*>& inputs2,
- InternalKey* smallest,
- InternalKey* largest) {
+ InternalKey* smallest, InternalKey* largest) {
std::vector<FileMetaData*> all = inputs1;
all.insert(all.end(), inputs2.begin(), inputs2.end());
GetRange(all, smallest, largest);
@@ -1223,8 +1247,8 @@ Iterator* VersionSet::MakeInputIterator(Compaction* c) {
if (c->level() + which == 0) {
const std::vector<FileMetaData*>& files = c->inputs_[which];
for (size_t i = 0; i < files.size(); i++) {
- list[num++] = table_cache_->NewIterator(
- options, files[i]->number, files[i]->file_size);
+ list[num++] = table_cache_->NewIterator(options, files[i]->number,
+ files[i]->file_size);
}
} else {
// Create concatenating iterator for the files from this level
@@ -1247,12 +1271,12 @@ Compaction* VersionSet::PickCompaction() {
// We prefer compactions triggered by too much data in a level over
// the compactions triggered by seeks.
const bool size_compaction = (current_->compaction_score_ >= 1);
- const bool seek_compaction = (current_->file_to_compact_ != NULL);
+ const bool seek_compaction = (current_->file_to_compact_ != nullptr);
if (size_compaction) {
level = current_->compaction_level_;
assert(level >= 0);
- assert(level+1 < config::kNumLevels);
- c = new Compaction(level);
+ assert(level + 1 < config::kNumLevels);
+ c = new Compaction(options_, level);
// Pick the first file that comes after compact_pointer_[level]
for (size_t i = 0; i < current_->files_[level].size(); i++) {
@@ -1269,10 +1293,10 @@ Compaction* VersionSet::PickCompaction() {
}
} else if (seek_compaction) {
level = current_->file_to_compact_level_;
- c = new Compaction(level);
+ c = new Compaction(options_, level);
c->inputs_[0].push_back(current_->file_to_compact_);
} else {
- return NULL;
+ return nullptr;
}
c->input_version_ = current_;
@@ -1294,12 +1318,94 @@ Compaction* VersionSet::PickCompaction() {
return c;
}
+// Finds the largest key in a vector of files. Returns true if files it not
+// empty.
+bool FindLargestKey(const InternalKeyComparator& icmp,
+ const std::vector<FileMetaData*>& files,
+ InternalKey* largest_key) {
+ if (files.empty()) {
+ return false;
+ }
+ *largest_key = files[0]->largest;
+ for (size_t i = 1; i < files.size(); ++i) {
+ FileMetaData* f = files[i];
+ if (icmp.Compare(f->largest, *largest_key) > 0) {
+ *largest_key = f->largest;
+ }
+ }
+ return true;
+}
+
+// Finds minimum file b2=(l2, u2) in level file for which l2 > u1 and
+// user_key(l2) = user_key(u1)
+FileMetaData* FindSmallestBoundaryFile(
+ const InternalKeyComparator& icmp,
+ const std::vector<FileMetaData*>& level_files,
+ const InternalKey& largest_key) {
+ const Comparator* user_cmp = icmp.user_comparator();
+ FileMetaData* smallest_boundary_file = nullptr;
+ for (size_t i = 0; i < level_files.size(); ++i) {
+ FileMetaData* f = level_files[i];
+ if (icmp.Compare(f->smallest, largest_key) > 0 &&
+ user_cmp->Compare(f->smallest.user_key(), largest_key.user_key()) ==
+ 0) {
+ if (smallest_boundary_file == nullptr ||
+ icmp.Compare(f->smallest, smallest_boundary_file->smallest) < 0) {
+ smallest_boundary_file = f;
+ }
+ }
+ }
+ return smallest_boundary_file;
+}
+
+// Extracts the largest file b1 from |compaction_files| and then searches for a
+// b2 in |level_files| for which user_key(u1) = user_key(l2). If it finds such a
+// file b2 (known as a boundary file) it adds it to |compaction_files| and then
+// searches again using this new upper bound.
+//
+// If there are two blocks, b1=(l1, u1) and b2=(l2, u2) and
+// user_key(u1) = user_key(l2), and if we compact b1 but not b2 then a
+// subsequent get operation will yield an incorrect result because it will
+// return the record from b2 in level i rather than from b1 because it searches
+// level by level for records matching the supplied user key.
+//
+// parameters:
+// in level_files: List of files to search for boundary files.
+// in/out compaction_files: List of files to extend by adding boundary files.
+void AddBoundaryInputs(const InternalKeyComparator& icmp,
+ const std::vector<FileMetaData*>& level_files,
+ std::vector<FileMetaData*>* compaction_files) {
+ InternalKey largest_key;
+
+ // Quick return if compaction_files is empty.
+ if (!FindLargestKey(icmp, *compaction_files, &largest_key)) {
+ return;
+ }
+
+ bool continue_searching = true;
+ while (continue_searching) {
+ FileMetaData* smallest_boundary_file =
+ FindSmallestBoundaryFile(icmp, level_files, largest_key);
+
+ // If a boundary file was found advance largest_key, otherwise we're done.
+ if (smallest_boundary_file != NULL) {
+ compaction_files->push_back(smallest_boundary_file);
+ largest_key = smallest_boundary_file->largest;
+ } else {
+ continue_searching = false;
+ }
+ }
+}
+
void VersionSet::SetupOtherInputs(Compaction* c) {
const int level = c->level();
InternalKey smallest, largest;
+
+ AddBoundaryInputs(icmp_, current_->files_[level], &c->inputs_[0]);
GetRange(c->inputs_[0], &smallest, &largest);
- current_->GetOverlappingInputs(level+1, &smallest, &largest, &c->inputs_[1]);
+ current_->GetOverlappingInputs(level + 1, &smallest, &largest,
+ &c->inputs_[1]);
// Get entire range covered by compaction
InternalKey all_start, all_limit;
@@ -1310,26 +1416,24 @@ void VersionSet::SetupOtherInputs(Compaction* c) {
if (!c->inputs_[1].empty()) {
std::vector<FileMetaData*> expanded0;
current_->GetOverlappingInputs(level, &all_start, &all_limit, &expanded0);
+ AddBoundaryInputs(icmp_, current_->files_[level], &expanded0);
const int64_t inputs0_size = TotalFileSize(c->inputs_[0]);
const int64_t inputs1_size = TotalFileSize(c->inputs_[1]);
const int64_t expanded0_size = TotalFileSize(expanded0);
if (expanded0.size() > c->inputs_[0].size() &&
- inputs1_size + expanded0_size < kExpandedCompactionByteSizeLimit) {
+ inputs1_size + expanded0_size <
+ ExpandedCompactionByteSizeLimit(options_)) {
InternalKey new_start, new_limit;
GetRange(expanded0, &new_start, &new_limit);
std::vector<FileMetaData*> expanded1;
- current_->GetOverlappingInputs(level+1, &new_start, &new_limit,
+ current_->GetOverlappingInputs(level + 1, &new_start, &new_limit,
&expanded1);
if (expanded1.size() == c->inputs_[1].size()) {
Log(options_->info_log,
"Expanding@%d %d+%d (%ld+%ld bytes) to %d+%d (%ld+%ld bytes)\n",
- level,
- int(c->inputs_[0].size()),
- int(c->inputs_[1].size()),
- long(inputs0_size), long(inputs1_size),
- int(expanded0.size()),
- int(expanded1.size()),
- long(expanded0_size), long(inputs1_size));
+ level, int(c->inputs_[0].size()), int(c->inputs_[1].size()),
+ long(inputs0_size), long(inputs1_size), int(expanded0.size()),
+ int(expanded1.size()), long(expanded0_size), long(inputs1_size));
smallest = new_start;
largest = new_limit;
c->inputs_[0] = expanded0;
@@ -1346,13 +1450,6 @@ void VersionSet::SetupOtherInputs(Compaction* c) {
&c->grandparents_);
}
- if (false) {
- Log(options_->info_log, "Compacting %d '%s' .. '%s'",
- level,
- smallest.DebugString().c_str(),
- largest.DebugString().c_str());
- }
-
// Update the place where we will do the next compaction for this level.
// We update this immediately instead of waiting for the VersionEdit
// to be applied so that if the compaction fails, we will try a different
@@ -1361,14 +1458,12 @@ void VersionSet::SetupOtherInputs(Compaction* c) {
c->edit_.SetCompactPointer(level, largest);
}
-Compaction* VersionSet::CompactRange(
- int level,
- const InternalKey* begin,
- const InternalKey* end) {
+Compaction* VersionSet::CompactRange(int level, const InternalKey* begin,
+ const InternalKey* end) {
std::vector<FileMetaData*> inputs;
current_->GetOverlappingInputs(level, begin, end, &inputs);
if (inputs.empty()) {
- return NULL;
+ return nullptr;
}
// Avoid compacting too much in one shot in case the range is large.
@@ -1376,7 +1471,7 @@ Compaction* VersionSet::CompactRange(
// and we must not pick one file and drop another older file if the
// two files overlap.
if (level > 0) {
- const uint64_t limit = MaxFileSizeForLevel(level);
+ const uint64_t limit = MaxFileSizeForLevel(options_, level);
uint64_t total = 0;
for (size_t i = 0; i < inputs.size(); i++) {
uint64_t s = inputs[i]->file_size;
@@ -1388,7 +1483,7 @@ Compaction* VersionSet::CompactRange(
}
}
- Compaction* c = new Compaction(level);
+ Compaction* c = new Compaction(options_, level);
c->input_version_ = current_;
c->input_version_->Ref();
c->inputs_[0] = inputs;
@@ -1396,10 +1491,10 @@ Compaction* VersionSet::CompactRange(
return c;
}
-Compaction::Compaction(int level)
+Compaction::Compaction(const Options* options, int level)
: level_(level),
- max_output_file_size_(MaxFileSizeForLevel(level)),
- input_version_(NULL),
+ max_output_file_size_(MaxFileSizeForLevel(options, level)),
+ input_version_(nullptr),
grandparent_index_(0),
seen_key_(false),
overlapped_bytes_(0) {
@@ -1409,18 +1504,19 @@ Compaction::Compaction(int level)
}
Compaction::~Compaction() {
- if (input_version_ != NULL) {
+ if (input_version_ != nullptr) {
input_version_->Unref();
}
}
bool Compaction::IsTrivialMove() const {
+ const VersionSet* vset = input_version_->vset_;
// Avoid a move if there is lots of overlapping grandparent data.
// Otherwise, the move could create a parent file that will require
// a very expensive merge later on.
- return (num_input_files(0) == 1 &&
- num_input_files(1) == 0 &&
- TotalFileSize(grandparents_) <= kMaxGrandParentOverlapBytes);
+ return (num_input_files(0) == 1 && num_input_files(1) == 0 &&
+ TotalFileSize(grandparents_) <=
+ MaxGrandParentOverlapBytes(vset->options_));
}
void Compaction::AddInputDeletions(VersionEdit* edit) {
@@ -1436,7 +1532,7 @@ bool Compaction::IsBaseLevelForKey(const Slice& user_key) {
const Comparator* user_cmp = input_version_->vset_->icmp_.user_comparator();
for (int lvl = level_ + 2; lvl < config::kNumLevels; lvl++) {
const std::vector<FileMetaData*>& files = input_version_->files_[lvl];
- for (; level_ptrs_[lvl] < files.size(); ) {
+ while (level_ptrs_[lvl] < files.size()) {
FileMetaData* f = files[level_ptrs_[lvl]];
if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
// We've advanced far enough
@@ -1453,11 +1549,13 @@ bool Compaction::IsBaseLevelForKey(const Slice& user_key) {
}
bool Compaction::ShouldStopBefore(const Slice& internal_key) {
+ const VersionSet* vset = input_version_->vset_;
// Scan to find earliest grandparent file that contains key.
- const InternalKeyComparator* icmp = &input_version_->vset_->icmp_;
+ const InternalKeyComparator* icmp = &vset->icmp_;
while (grandparent_index_ < grandparents_.size() &&
- icmp->Compare(internal_key,
- grandparents_[grandparent_index_]->largest.Encode()) > 0) {
+ icmp->Compare(internal_key,
+ grandparents_[grandparent_index_]->largest.Encode()) >
+ 0) {
if (seen_key_) {
overlapped_bytes_ += grandparents_[grandparent_index_]->file_size;
}
@@ -1465,7 +1563,7 @@ bool Compaction::ShouldStopBefore(const Slice& internal_key) {
}
seen_key_ = true;
- if (overlapped_bytes_ > kMaxGrandParentOverlapBytes) {
+ if (overlapped_bytes_ > MaxGrandParentOverlapBytes(vset->options_)) {
// Too much overlap for current output; start new output
overlapped_bytes_ = 0;
return true;
@@ -1475,9 +1573,9 @@ bool Compaction::ShouldStopBefore(const Slice& internal_key) {
}
void Compaction::ReleaseInputs() {
- if (input_version_ != NULL) {
+ if (input_version_ != nullptr) {
input_version_->Unref();
- input_version_ = NULL;
+ input_version_ = nullptr;
}
}
diff --git a/db/version_set.h b/db/version_set.h
index 8dc14b8..69f3d70 100644
--- a/db/version_set.h
+++ b/db/version_set.h
@@ -18,6 +18,7 @@
#include <map>
#include <set>
#include <vector>
+
#include "db/dbformat.h"
#include "db/version_edit.h"
#include "port/port.h"
@@ -25,7 +26,9 @@
namespace leveldb {
-namespace log { class Writer; }
+namespace log {
+class Writer;
+}
class Compaction;
class Iterator;
@@ -39,30 +42,23 @@ class WritableFile;
// Return the smallest index i such that files[i]->largest >= key.
// Return files.size() if there is no such file.
// REQUIRES: "files" contains a sorted list of non-overlapping files.
-extern int FindFile(const InternalKeyComparator& icmp,
- const std::vector<FileMetaData*>& files,
- const Slice& key);
+int FindFile(const InternalKeyComparator& icmp,
+ const std::vector<FileMetaData*>& files, const Slice& key);
// Returns true iff some file in "files" overlaps the user key range
// [*smallest,*largest].
-// smallest==NULL represents a key smaller than all keys in the DB.
-// largest==NULL represents a key largest than all keys in the DB.
+// smallest==nullptr represents a key smaller than all keys in the DB.
+// largest==nullptr represents a key largest than all keys in the DB.
// REQUIRES: If disjoint_sorted_files, files[] contains disjoint ranges
// in sorted order.
-extern bool SomeFileOverlapsRange(
- const InternalKeyComparator& icmp,
- bool disjoint_sorted_files,
- const std::vector<FileMetaData*>& files,
- const Slice* smallest_user_key,
- const Slice* largest_user_key);
+bool SomeFileOverlapsRange(const InternalKeyComparator& icmp,
+ bool disjoint_sorted_files,
+ const std::vector<FileMetaData*>& files,
+ const Slice* smallest_user_key,
+ const Slice* largest_user_key);
class Version {
public:
- // Append to *iters a sequence of iterators that will
- // yield the contents of this Version when merged together.
- // REQUIRES: This version has been saved (see VersionSet::SaveTo)
- void AddIterators(const ReadOptions&, std::vector<Iterator*>* iters);
-
// Lookup the value for key. If found, store it in *val and
// return OK. Else return a non-OK status. Fills *stats.
// REQUIRES: lock is not held
@@ -70,6 +66,12 @@ class Version {
FileMetaData* seek_file;
int seek_file_level;
};
+
+ // Append to *iters a sequence of iterators that will
+ // yield the contents of this Version when merged together.
+ // REQUIRES: This version has been saved (see VersionSet::SaveTo)
+ void AddIterators(const ReadOptions&, std::vector<Iterator*>* iters);
+
Status Get(const ReadOptions&, const LookupKey& key, std::string* val,
GetStats* stats);
@@ -91,16 +93,15 @@ class Version {
void GetOverlappingInputs(
int level,
- const InternalKey* begin, // NULL means before all keys
- const InternalKey* end, // NULL means after all keys
+ const InternalKey* begin, // nullptr means before all keys
+ const InternalKey* end, // nullptr means after all keys
std::vector<FileMetaData*>* inputs);
// Returns true iff some file in the specified level overlaps
// some part of [*smallest_user_key,*largest_user_key].
- // smallest_user_key==NULL represents a key smaller than all keys in the DB.
- // largest_user_key==NULL represents a key largest than all keys in the DB.
- bool OverlapInLevel(int level,
- const Slice* smallest_user_key,
+ // smallest_user_key==nullptr represents a key smaller than all the DB's keys.
+ // largest_user_key==nullptr represents a key largest than all the DB's keys.
+ bool OverlapInLevel(int level, const Slice* smallest_user_key,
const Slice* largest_user_key);
// Return the level at which we should place a new memtable compaction
@@ -118,6 +119,22 @@ class Version {
friend class VersionSet;
class LevelFileNumIterator;
+
+ explicit Version(VersionSet* vset)
+ : vset_(vset),
+ next_(this),
+ prev_(this),
+ refs_(0),
+ file_to_compact_(nullptr),
+ file_to_compact_level_(-1),
+ compaction_score_(-1),
+ compaction_level_(-1) {}
+
+ Version(const Version&) = delete;
+ Version& operator=(const Version&) = delete;
+
+ ~Version();
+
Iterator* NewConcatenatingIterator(const ReadOptions&, int level) const;
// Call func(arg, level, f) for every file that overlaps user_key in
@@ -125,14 +142,13 @@ class Version {
// false, makes no more calls.
//
// REQUIRES: user portion of internal_key == user_key.
- void ForEachOverlapping(Slice user_key, Slice internal_key,
- void* arg,
+ void ForEachOverlapping(Slice user_key, Slice internal_key, void* arg,
bool (*func)(void*, int, FileMetaData*));
- VersionSet* vset_; // VersionSet to which this Version belongs
- Version* next_; // Next version in linked list
- Version* prev_; // Previous version in linked list
- int refs_; // Number of live refs to this version
+ VersionSet* vset_; // VersionSet to which this Version belongs
+ Version* next_; // Next version in linked list
+ Version* prev_; // Previous version in linked list
+ int refs_; // Number of live refs to this version
// List of files per level
std::vector<FileMetaData*> files_[config::kNumLevels];
@@ -146,28 +162,15 @@ class Version {
// are initialized by Finalize().
double compaction_score_;
int compaction_level_;
-
- explicit Version(VersionSet* vset)
- : vset_(vset), next_(this), prev_(this), refs_(0),
- file_to_compact_(NULL),
- file_to_compact_level_(-1),
- compaction_score_(-1),
- compaction_level_(-1) {
- }
-
- ~Version();
-
- // No copying allowed
- Version(const Version&);
- void operator=(const Version&);
};
class VersionSet {
public:
- VersionSet(const std::string& dbname,
- const Options* options,
- TableCache* table_cache,
- const InternalKeyComparator*);
+ VersionSet(const std::string& dbname, const Options* options,
+ TableCache* table_cache, const InternalKeyComparator*);
+ VersionSet(const VersionSet&) = delete;
+ VersionSet& operator=(const VersionSet&) = delete;
+
~VersionSet();
// Apply *edit to the current version to form a new descriptor that
@@ -179,7 +182,7 @@ class VersionSet {
EXCLUSIVE_LOCKS_REQUIRED(mu);
// Recover the last saved descriptor from persistent storage.
- Status Recover();
+ Status Recover(bool* save_manifest);
// Return the current version.
Version* current() const { return current_; }
@@ -225,19 +228,17 @@ class VersionSet {
uint64_t PrevLogNumber() const { return prev_log_number_; }
// Pick level and inputs for a new compaction.
- // Returns NULL if there is no compaction to be done.
+ // Returns nullptr if there is no compaction to be done.
// Otherwise returns a pointer to a heap-allocated object that
// describes the compaction. Caller should delete the result.
Compaction* PickCompaction();
// Return a compaction object for compacting the range [begin,end] in
- // the specified level. Returns NULL if there is nothing in that
+ // the specified level. Returns nullptr if there is nothing in that
// level that overlaps the specified range. Caller should delete
// the result.
- Compaction* CompactRange(
- int level,
- const InternalKey* begin,
- const InternalKey* end);
+ Compaction* CompactRange(int level, const InternalKey* begin,
+ const InternalKey* end);
// Return the maximum overlapping data (in bytes) at next level for any
// file at a level >= 1.
@@ -250,7 +251,7 @@ class VersionSet {
// Returns true iff some level needs a compaction.
bool NeedsCompaction() const {
Version* v = current_;
- return (v->compaction_score_ >= 1) || (v->file_to_compact_ != NULL);
+ return (v->compaction_score_ >= 1) || (v->file_to_compact_ != nullptr);
}
// Add all files listed in any live version to *live.
@@ -274,16 +275,16 @@ class VersionSet {
friend class Compaction;
friend class Version;
+ bool ReuseManifest(const std::string& dscname, const std::string& dscbase);
+
void Finalize(Version* v);
- void GetRange(const std::vector<FileMetaData*>& inputs,
- InternalKey* smallest,
+ void GetRange(const std::vector<FileMetaData*>& inputs, InternalKey* smallest,
InternalKey* largest);
void GetRange2(const std::vector<FileMetaData*>& inputs1,
const std::vector<FileMetaData*>& inputs2,
- InternalKey* smallest,
- InternalKey* largest);
+ InternalKey* smallest, InternalKey* largest);
void SetupOtherInputs(Compaction* c);
@@ -312,10 +313,6 @@ class VersionSet {
// Per-level key at which the next compaction at that level should start.
// Either an empty string, or a valid InternalKey.
std::string compact_pointer_[config::kNumLevels];
-
- // No copying allowed
- VersionSet(const VersionSet&);
- void operator=(const VersionSet&);
};
// A Compaction encapsulates information about a compaction.
@@ -364,7 +361,7 @@ class Compaction {
friend class Version;
friend class VersionSet;
- explicit Compaction(int level);
+ Compaction(const Options* options, int level);
int level_;
uint64_t max_output_file_size_;
@@ -372,9 +369,9 @@ class Compaction {
VersionEdit edit_;
// Each compaction reads inputs from "level_" and "level_+1"
- std::vector<FileMetaData*> inputs_[2]; // The two sets of inputs
+ std::vector<FileMetaData*> inputs_[2]; // The two sets of inputs
- // State used to check for number of of overlapping grandparent files
+ // State used to check for number of overlapping grandparent files
// (parent == level_ + 1, grandparent == level_ + 2)
std::vector<FileMetaData*> grandparents_;
size_t grandparent_index_; // Index in grandparent_starts_
diff --git a/db/version_set_test.cc b/db/version_set_test.cc
index 501e34d..c1056a1 100644
--- a/db/version_set_test.cc
+++ b/db/version_set_test.cc
@@ -11,10 +11,7 @@ namespace leveldb {
class FindFileTest {
public:
- std::vector<FileMetaData*> files_;
- bool disjoint_sorted_files_;
-
- FindFileTest() : disjoint_sorted_files_(true) { }
+ FindFileTest() : disjoint_sorted_files_(true) {}
~FindFileTest() {
for (int i = 0; i < files_.size(); i++) {
@@ -40,20 +37,25 @@ class FindFileTest {
bool Overlaps(const char* smallest, const char* largest) {
InternalKeyComparator cmp(BytewiseComparator());
- Slice s(smallest != NULL ? smallest : "");
- Slice l(largest != NULL ? largest : "");
+ Slice s(smallest != nullptr ? smallest : "");
+ Slice l(largest != nullptr ? largest : "");
return SomeFileOverlapsRange(cmp, disjoint_sorted_files_, files_,
- (smallest != NULL ? &s : NULL),
- (largest != NULL ? &l : NULL));
+ (smallest != nullptr ? &s : nullptr),
+ (largest != nullptr ? &l : nullptr));
}
+
+ bool disjoint_sorted_files_;
+
+ private:
+ std::vector<FileMetaData*> files_;
};
TEST(FindFileTest, Empty) {
ASSERT_EQ(0, Find("foo"));
- ASSERT_TRUE(! Overlaps("a", "z"));
- ASSERT_TRUE(! Overlaps(NULL, "z"));
- ASSERT_TRUE(! Overlaps("a", NULL));
- ASSERT_TRUE(! Overlaps(NULL, NULL));
+ ASSERT_TRUE(!Overlaps("a", "z"));
+ ASSERT_TRUE(!Overlaps(nullptr, "z"));
+ ASSERT_TRUE(!Overlaps("a", nullptr));
+ ASSERT_TRUE(!Overlaps(nullptr, nullptr));
}
TEST(FindFileTest, Single) {
@@ -65,8 +67,8 @@ TEST(FindFileTest, Single) {
ASSERT_EQ(1, Find("q1"));
ASSERT_EQ(1, Find("z"));
- ASSERT_TRUE(! Overlaps("a", "b"));
- ASSERT_TRUE(! Overlaps("z1", "z2"));
+ ASSERT_TRUE(!Overlaps("a", "b"));
+ ASSERT_TRUE(!Overlaps("z1", "z2"));
ASSERT_TRUE(Overlaps("a", "p"));
ASSERT_TRUE(Overlaps("a", "q"));
ASSERT_TRUE(Overlaps("a", "z"));
@@ -78,15 +80,14 @@ TEST(FindFileTest, Single) {
ASSERT_TRUE(Overlaps("q", "q"));
ASSERT_TRUE(Overlaps("q", "q1"));
- ASSERT_TRUE(! Overlaps(NULL, "j"));
- ASSERT_TRUE(! Overlaps("r", NULL));
- ASSERT_TRUE(Overlaps(NULL, "p"));
- ASSERT_TRUE(Overlaps(NULL, "p1"));
- ASSERT_TRUE(Overlaps("q", NULL));
- ASSERT_TRUE(Overlaps(NULL, NULL));
+ ASSERT_TRUE(!Overlaps(nullptr, "j"));
+ ASSERT_TRUE(!Overlaps("r", nullptr));
+ ASSERT_TRUE(Overlaps(nullptr, "p"));
+ ASSERT_TRUE(Overlaps(nullptr, "p1"));
+ ASSERT_TRUE(Overlaps("q", nullptr));
+ ASSERT_TRUE(Overlaps(nullptr, nullptr));
}
-
TEST(FindFileTest, Multiple) {
Add("150", "200");
Add("200", "250");
@@ -110,10 +111,10 @@ TEST(FindFileTest, Multiple) {
ASSERT_EQ(3, Find("450"));
ASSERT_EQ(4, Find("451"));
- ASSERT_TRUE(! Overlaps("100", "149"));
- ASSERT_TRUE(! Overlaps("251", "299"));
- ASSERT_TRUE(! Overlaps("451", "500"));
- ASSERT_TRUE(! Overlaps("351", "399"));
+ ASSERT_TRUE(!Overlaps("100", "149"));
+ ASSERT_TRUE(!Overlaps("251", "299"));
+ ASSERT_TRUE(!Overlaps("451", "500"));
+ ASSERT_TRUE(!Overlaps("351", "399"));
ASSERT_TRUE(Overlaps("100", "150"));
ASSERT_TRUE(Overlaps("100", "200"));
@@ -130,25 +131,25 @@ TEST(FindFileTest, MultipleNullBoundaries) {
Add("200", "250");
Add("300", "350");
Add("400", "450");
- ASSERT_TRUE(! Overlaps(NULL, "149"));
- ASSERT_TRUE(! Overlaps("451", NULL));
- ASSERT_TRUE(Overlaps(NULL, NULL));
- ASSERT_TRUE(Overlaps(NULL, "150"));
- ASSERT_TRUE(Overlaps(NULL, "199"));
- ASSERT_TRUE(Overlaps(NULL, "200"));
- ASSERT_TRUE(Overlaps(NULL, "201"));
- ASSERT_TRUE(Overlaps(NULL, "400"));
- ASSERT_TRUE(Overlaps(NULL, "800"));
- ASSERT_TRUE(Overlaps("100", NULL));
- ASSERT_TRUE(Overlaps("200", NULL));
- ASSERT_TRUE(Overlaps("449", NULL));
- ASSERT_TRUE(Overlaps("450", NULL));
+ ASSERT_TRUE(!Overlaps(nullptr, "149"));
+ ASSERT_TRUE(!Overlaps("451", nullptr));
+ ASSERT_TRUE(Overlaps(nullptr, nullptr));
+ ASSERT_TRUE(Overlaps(nullptr, "150"));
+ ASSERT_TRUE(Overlaps(nullptr, "199"));
+ ASSERT_TRUE(Overlaps(nullptr, "200"));
+ ASSERT_TRUE(Overlaps(nullptr, "201"));
+ ASSERT_TRUE(Overlaps(nullptr, "400"));
+ ASSERT_TRUE(Overlaps(nullptr, "800"));
+ ASSERT_TRUE(Overlaps("100", nullptr));
+ ASSERT_TRUE(Overlaps("200", nullptr));
+ ASSERT_TRUE(Overlaps("449", nullptr));
+ ASSERT_TRUE(Overlaps("450", nullptr));
}
TEST(FindFileTest, OverlapSequenceChecks) {
Add("200", "200", 5000, 3000);
- ASSERT_TRUE(! Overlaps("199", "199"));
- ASSERT_TRUE(! Overlaps("201", "300"));
+ ASSERT_TRUE(!Overlaps("199", "199"));
+ ASSERT_TRUE(!Overlaps("201", "300"));
ASSERT_TRUE(Overlaps("200", "200"));
ASSERT_TRUE(Overlaps("190", "200"));
ASSERT_TRUE(Overlaps("200", "210"));
@@ -158,8 +159,8 @@ TEST(FindFileTest, OverlappingFiles) {
Add("150", "600");
Add("400", "500");
disjoint_sorted_files_ = false;
- ASSERT_TRUE(! Overlaps("100", "149"));
- ASSERT_TRUE(! Overlaps("601", "700"));
+ ASSERT_TRUE(!Overlaps("100", "149"));
+ ASSERT_TRUE(!Overlaps("601", "700"));
ASSERT_TRUE(Overlaps("100", "150"));
ASSERT_TRUE(Overlaps("100", "200"));
ASSERT_TRUE(Overlaps("100", "300"));
@@ -172,8 +173,160 @@ TEST(FindFileTest, OverlappingFiles) {
ASSERT_TRUE(Overlaps("600", "700"));
}
-} // namespace leveldb
+void AddBoundaryInputs(const InternalKeyComparator& icmp,
+ const std::vector<FileMetaData*>& level_files,
+ std::vector<FileMetaData*>* compaction_files);
+
+class AddBoundaryInputsTest {
+ public:
+ std::vector<FileMetaData*> level_files_;
+ std::vector<FileMetaData*> compaction_files_;
+ std::vector<FileMetaData*> all_files_;
+ InternalKeyComparator icmp_;
+
+ AddBoundaryInputsTest() : icmp_(BytewiseComparator()) {}
+
+ ~AddBoundaryInputsTest() {
+ for (size_t i = 0; i < all_files_.size(); ++i) {
+ delete all_files_[i];
+ }
+ all_files_.clear();
+ }
+
+ FileMetaData* CreateFileMetaData(uint64_t number, InternalKey smallest,
+ InternalKey largest) {
+ FileMetaData* f = new FileMetaData();
+ f->number = number;
+ f->smallest = smallest;
+ f->largest = largest;
+ all_files_.push_back(f);
+ return f;
+ }
+};
+
+TEST(AddBoundaryInputsTest, TestEmptyFileSets) {
+ AddBoundaryInputs(icmp_, level_files_, &compaction_files_);
+ ASSERT_TRUE(compaction_files_.empty());
+ ASSERT_TRUE(level_files_.empty());
+}
+
+TEST(AddBoundaryInputsTest, TestEmptyLevelFiles) {
+ FileMetaData* f1 =
+ CreateFileMetaData(1, InternalKey("100", 2, kTypeValue),
+ InternalKey(InternalKey("100", 1, kTypeValue)));
+ compaction_files_.push_back(f1);
+
+ AddBoundaryInputs(icmp_, level_files_, &compaction_files_);
+ ASSERT_EQ(1, compaction_files_.size());
+ ASSERT_EQ(f1, compaction_files_[0]);
+ ASSERT_TRUE(level_files_.empty());
+}
+
+TEST(AddBoundaryInputsTest, TestEmptyCompactionFiles) {
+ FileMetaData* f1 =
+ CreateFileMetaData(1, InternalKey("100", 2, kTypeValue),
+ InternalKey(InternalKey("100", 1, kTypeValue)));
+ level_files_.push_back(f1);
+
+ AddBoundaryInputs(icmp_, level_files_, &compaction_files_);
+ ASSERT_TRUE(compaction_files_.empty());
+ ASSERT_EQ(1, level_files_.size());
+ ASSERT_EQ(f1, level_files_[0]);
+}
+
+TEST(AddBoundaryInputsTest, TestNoBoundaryFiles) {
+ FileMetaData* f1 =
+ CreateFileMetaData(1, InternalKey("100", 2, kTypeValue),
+ InternalKey(InternalKey("100", 1, kTypeValue)));
+ FileMetaData* f2 =
+ CreateFileMetaData(1, InternalKey("200", 2, kTypeValue),
+ InternalKey(InternalKey("200", 1, kTypeValue)));
+ FileMetaData* f3 =
+ CreateFileMetaData(1, InternalKey("300", 2, kTypeValue),
+ InternalKey(InternalKey("300", 1, kTypeValue)));
+
+ level_files_.push_back(f3);
+ level_files_.push_back(f2);
+ level_files_.push_back(f1);
+ compaction_files_.push_back(f2);
+ compaction_files_.push_back(f3);
+
+ AddBoundaryInputs(icmp_, level_files_, &compaction_files_);
+ ASSERT_EQ(2, compaction_files_.size());
+}
+
+TEST(AddBoundaryInputsTest, TestOneBoundaryFiles) {
+ FileMetaData* f1 =
+ CreateFileMetaData(1, InternalKey("100", 3, kTypeValue),
+ InternalKey(InternalKey("100", 2, kTypeValue)));
+ FileMetaData* f2 =
+ CreateFileMetaData(1, InternalKey("100", 1, kTypeValue),
+ InternalKey(InternalKey("200", 3, kTypeValue)));
+ FileMetaData* f3 =
+ CreateFileMetaData(1, InternalKey("300", 2, kTypeValue),
+ InternalKey(InternalKey("300", 1, kTypeValue)));
+
+ level_files_.push_back(f3);
+ level_files_.push_back(f2);
+ level_files_.push_back(f1);
+ compaction_files_.push_back(f1);
+
+ AddBoundaryInputs(icmp_, level_files_, &compaction_files_);
+ ASSERT_EQ(2, compaction_files_.size());
+ ASSERT_EQ(f1, compaction_files_[0]);
+ ASSERT_EQ(f2, compaction_files_[1]);
+}
+
+TEST(AddBoundaryInputsTest, TestTwoBoundaryFiles) {
+ FileMetaData* f1 =
+ CreateFileMetaData(1, InternalKey("100", 6, kTypeValue),
+ InternalKey(InternalKey("100", 5, kTypeValue)));
+ FileMetaData* f2 =
+ CreateFileMetaData(1, InternalKey("100", 2, kTypeValue),
+ InternalKey(InternalKey("300", 1, kTypeValue)));
+ FileMetaData* f3 =
+ CreateFileMetaData(1, InternalKey("100", 4, kTypeValue),
+ InternalKey(InternalKey("100", 3, kTypeValue)));
+
+ level_files_.push_back(f2);
+ level_files_.push_back(f3);
+ level_files_.push_back(f1);
+ compaction_files_.push_back(f1);
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
+ AddBoundaryInputs(icmp_, level_files_, &compaction_files_);
+ ASSERT_EQ(3, compaction_files_.size());
+ ASSERT_EQ(f1, compaction_files_[0]);
+ ASSERT_EQ(f3, compaction_files_[1]);
+ ASSERT_EQ(f2, compaction_files_[2]);
}
+
+TEST(AddBoundaryInputsTest, TestDisjoinFilePointers) {
+ FileMetaData* f1 =
+ CreateFileMetaData(1, InternalKey("100", 6, kTypeValue),
+ InternalKey(InternalKey("100", 5, kTypeValue)));
+ FileMetaData* f2 =
+ CreateFileMetaData(1, InternalKey("100", 6, kTypeValue),
+ InternalKey(InternalKey("100", 5, kTypeValue)));
+ FileMetaData* f3 =
+ CreateFileMetaData(1, InternalKey("100", 2, kTypeValue),
+ InternalKey(InternalKey("300", 1, kTypeValue)));
+ FileMetaData* f4 =
+ CreateFileMetaData(1, InternalKey("100", 4, kTypeValue),
+ InternalKey(InternalKey("100", 3, kTypeValue)));
+
+ level_files_.push_back(f2);
+ level_files_.push_back(f3);
+ level_files_.push_back(f4);
+
+ compaction_files_.push_back(f1);
+
+ AddBoundaryInputs(icmp_, level_files_, &compaction_files_);
+ ASSERT_EQ(3, compaction_files_.size());
+ ASSERT_EQ(f1, compaction_files_[0]);
+ ASSERT_EQ(f4, compaction_files_[1]);
+ ASSERT_EQ(f3, compaction_files_[2]);
+}
+
+} // namespace leveldb
+
+int main(int argc, char** argv) { return leveldb::test::RunAllTests(); }
diff --git a/db/write_batch.cc b/db/write_batch.cc
index 33f4a42..b54313c 100644
--- a/db/write_batch.cc
+++ b/db/write_batch.cc
@@ -15,10 +15,10 @@
#include "leveldb/write_batch.h"
-#include "leveldb/db.h"
#include "db/dbformat.h"
#include "db/memtable.h"
#include "db/write_batch_internal.h"
+#include "leveldb/db.h"
#include "util/coding.h"
namespace leveldb {
@@ -26,19 +26,19 @@ namespace leveldb {
// WriteBatch header has an 8-byte sequence number followed by a 4-byte count.
static const size_t kHeader = 12;
-WriteBatch::WriteBatch() {
- Clear();
-}
+WriteBatch::WriteBatch() { Clear(); }
-WriteBatch::~WriteBatch() { }
+WriteBatch::~WriteBatch() = default;
-WriteBatch::Handler::~Handler() { }
+WriteBatch::Handler::~Handler() = default;
void WriteBatch::Clear() {
rep_.clear();
rep_.resize(kHeader);
}
+size_t WriteBatch::ApproximateSize() const { return rep_.size(); }
+
Status WriteBatch::Iterate(Handler* handler) const {
Slice input(rep_);
if (input.size() < kHeader) {
@@ -108,25 +108,28 @@ void WriteBatch::Delete(const Slice& key) {
PutLengthPrefixedSlice(&rep_, key);
}
+void WriteBatch::Append(const WriteBatch& source) {
+ WriteBatchInternal::Append(this, &source);
+}
+
namespace {
class MemTableInserter : public WriteBatch::Handler {
public:
SequenceNumber sequence_;
MemTable* mem_;
- virtual void Put(const Slice& key, const Slice& value) {
+ void Put(const Slice& key, const Slice& value) override {
mem_->Add(sequence_, kTypeValue, key, value);
sequence_++;
}
- virtual void Delete(const Slice& key) {
+ void Delete(const Slice& key) override {
mem_->Add(sequence_, kTypeDeletion, key, Slice());
sequence_++;
}
};
} // namespace
-Status WriteBatchInternal::InsertInto(const WriteBatch* b,
- MemTable* memtable) {
+Status WriteBatchInternal::InsertInto(const WriteBatch* b, MemTable* memtable) {
MemTableInserter inserter;
inserter.sequence_ = WriteBatchInternal::Sequence(b);
inserter.mem_ = memtable;
diff --git a/db/write_batch_internal.h b/db/write_batch_internal.h
index 9448ef7..fce86e3 100644
--- a/db/write_batch_internal.h
+++ b/db/write_batch_internal.h
@@ -29,13 +29,9 @@ class WriteBatchInternal {
// this batch.
static void SetSequence(WriteBatch* batch, SequenceNumber seq);
- static Slice Contents(const WriteBatch* batch) {
- return Slice(batch->rep_);
- }
+ static Slice Contents(const WriteBatch* batch) { return Slice(batch->rep_); }
- static size_t ByteSize(const WriteBatch* batch) {
- return batch->rep_.size();
- }
+ static size_t ByteSize(const WriteBatch* batch) { return batch->rep_.size(); }
static void SetContents(WriteBatch* batch, const Slice& contents);
@@ -46,5 +42,4 @@ class WriteBatchInternal {
} // namespace leveldb
-
#endif // STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
diff --git a/db/write_batch_test.cc b/db/write_batch_test.cc
index 9064e3d..c32317f 100644
--- a/db/write_batch_test.cc
+++ b/db/write_batch_test.cc
@@ -52,7 +52,7 @@ static std::string PrintContents(WriteBatch* b) {
return state;
}
-class WriteBatchTest { };
+class WriteBatchTest {};
TEST(WriteBatchTest, Empty) {
WriteBatch batch;
@@ -68,10 +68,11 @@ TEST(WriteBatchTest, Multiple) {
WriteBatchInternal::SetSequence(&batch, 100);
ASSERT_EQ(100, WriteBatchInternal::Sequence(&batch));
ASSERT_EQ(3, WriteBatchInternal::Count(&batch));
- ASSERT_EQ("Put(baz, boo)@102"
- "Delete(box)@101"
- "Put(foo, bar)@100",
- PrintContents(&batch));
+ ASSERT_EQ(
+ "Put(baz, boo)@102"
+ "Delete(box)@101"
+ "Put(foo, bar)@100",
+ PrintContents(&batch));
}
TEST(WriteBatchTest, Corruption) {
@@ -81,40 +82,56 @@ TEST(WriteBatchTest, Corruption) {
WriteBatchInternal::SetSequence(&batch, 200);
Slice contents = WriteBatchInternal::Contents(&batch);
WriteBatchInternal::SetContents(&batch,
- Slice(contents.data(),contents.size()-1));
- ASSERT_EQ("Put(foo, bar)@200"
- "ParseError()",
- PrintContents(&batch));
+ Slice(contents.data(), contents.size() - 1));
+ ASSERT_EQ(
+ "Put(foo, bar)@200"
+ "ParseError()",
+ PrintContents(&batch));
}
TEST(WriteBatchTest, Append) {
WriteBatch b1, b2;
WriteBatchInternal::SetSequence(&b1, 200);
WriteBatchInternal::SetSequence(&b2, 300);
- WriteBatchInternal::Append(&b1, &b2);
- ASSERT_EQ("",
- PrintContents(&b1));
+ b1.Append(b2);
+ ASSERT_EQ("", PrintContents(&b1));
b2.Put("a", "va");
- WriteBatchInternal::Append(&b1, &b2);
- ASSERT_EQ("Put(a, va)@200",
- PrintContents(&b1));
+ b1.Append(b2);
+ ASSERT_EQ("Put(a, va)@200", PrintContents(&b1));
b2.Clear();
b2.Put("b", "vb");
- WriteBatchInternal::Append(&b1, &b2);
- ASSERT_EQ("Put(a, va)@200"
- "Put(b, vb)@201",
- PrintContents(&b1));
+ b1.Append(b2);
+ ASSERT_EQ(
+ "Put(a, va)@200"
+ "Put(b, vb)@201",
+ PrintContents(&b1));
b2.Delete("foo");
- WriteBatchInternal::Append(&b1, &b2);
- ASSERT_EQ("Put(a, va)@200"
- "Put(b, vb)@202"
- "Put(b, vb)@201"
- "Delete(foo)@203",
- PrintContents(&b1));
+ b1.Append(b2);
+ ASSERT_EQ(
+ "Put(a, va)@200"
+ "Put(b, vb)@202"
+ "Put(b, vb)@201"
+ "Delete(foo)@203",
+ PrintContents(&b1));
}
-} // namespace leveldb
+TEST(WriteBatchTest, ApproximateSize) {
+ WriteBatch batch;
+ size_t empty_size = batch.ApproximateSize();
+
+ batch.Put(Slice("foo"), Slice("bar"));
+ size_t one_key_size = batch.ApproximateSize();
+ ASSERT_LT(empty_size, one_key_size);
+
+ batch.Put(Slice("baz"), Slice("boo"));
+ size_t two_keys_size = batch.ApproximateSize();
+ ASSERT_LT(one_key_size, two_keys_size);
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
+ batch.Delete(Slice("box"));
+ size_t post_delete_size = batch.ApproximateSize();
+ ASSERT_LT(two_keys_size, post_delete_size);
}
+
+} // namespace leveldb
+
+int main(int argc, char** argv) { return leveldb::test::RunAllTests(); }