diff options
author | David Grogan <dgrogan@chromium.org> | 2012-10-12 11:53:12 -0700 |
---|---|---|
committer | David Grogan <dgrogan@chromium.org> | 2012-10-12 11:53:12 -0700 |
commit | 946e5b5a4ce7980917b22a408f090a4e86c3fa44 (patch) | |
tree | b5e1a48797824a460af5ce61784671f7205f1a7d /util | |
parent | dd0d562b4d4fbd07db6a44f9e221f8d368fee8e4 (diff) | |
download | leveldb-946e5b5a4ce7980917b22a408f090a4e86c3fa44.tar.gz |
Update to leveldb 1.6v1.6
Highlights
----------
Mmap at most 1000 files on Posix to improve performance for large databases.
Support for more architectures (thanks to Alexander K.)
Building and porting
--------------------
HP/UX support (issue 126)
AtomicPointer for ia64 (issue 123)
Sparc v9 support (issue 124)
Atomic ops for powerpc
Use -fno-builtin-memcmp only when using g++
Simplify IOS build rules (issue 114)
Use CXXFLAGS instead of CFLAGS when invoking C++ compiler (issue 118)
Fix snappy shared library problem (issue 94)
Fix shared library installation path regression
Endian-ness detection tweak for FreeBSD
Bug fixes
---------
Stop ignoring FLAGS_open_files in db_bench
Make bloom test behavior agnostic to endian-ness
Performance
-----------
Limit number of mmapped files to 1000 to improve perf for large dbs
Do not delay for 1 second on shutdown path (issue 125)
Misc
----
Make InMemoryEnv return a no-op logger
C binding now has a wrapper for free (issue 117)
Add thread-safety annotations
Added an in-process lock table (issue 120)
Make RandomAccessFile and SequentialFile non-copyable
Diffstat (limited to 'util')
-rw-r--r-- | util/bloom_test.cc | 5 | ||||
-rw-r--r-- | util/coding.cc | 40 | ||||
-rw-r--r-- | util/env_posix.cc | 101 | ||||
-rw-r--r-- | util/mutexlock.h | 8 |
4 files changed, 123 insertions, 31 deletions
diff --git a/util/bloom_test.cc b/util/bloom_test.cc index 4a6ea1b..0bf8e8d 100644 --- a/util/bloom_test.cc +++ b/util/bloom_test.cc @@ -4,6 +4,7 @@ #include "leveldb/filter_policy.h" +#include "util/coding.h" #include "util/logging.h" #include "util/testharness.h" #include "util/testutil.h" @@ -13,8 +14,8 @@ namespace leveldb { static const int kVerbose = 1; static Slice Key(int i, char* buffer) { - memcpy(buffer, &i, sizeof(i)); - return Slice(buffer, sizeof(i)); + EncodeFixed32(buffer, i); + return Slice(buffer, sizeof(uint32_t)); } class BloomTest { diff --git a/util/coding.cc b/util/coding.cc index dbd7a65..21e3186 100644 --- a/util/coding.cc +++ b/util/coding.cc @@ -7,29 +7,29 @@ namespace leveldb { void EncodeFixed32(char* buf, uint32_t value) { -#if __BYTE_ORDER == __LITTLE_ENDIAN - memcpy(buf, &value, sizeof(value)); -#else - buf[0] = value & 0xff; - buf[1] = (value >> 8) & 0xff; - buf[2] = (value >> 16) & 0xff; - buf[3] = (value >> 24) & 0xff; -#endif + if (port::kLittleEndian) { + memcpy(buf, &value, sizeof(value)); + } else { + buf[0] = value & 0xff; + buf[1] = (value >> 8) & 0xff; + buf[2] = (value >> 16) & 0xff; + buf[3] = (value >> 24) & 0xff; + } } void EncodeFixed64(char* buf, uint64_t value) { -#if __BYTE_ORDER == __LITTLE_ENDIAN - memcpy(buf, &value, sizeof(value)); -#else - buf[0] = value & 0xff; - buf[1] = (value >> 8) & 0xff; - buf[2] = (value >> 16) & 0xff; - buf[3] = (value >> 24) & 0xff; - buf[4] = (value >> 32) & 0xff; - buf[5] = (value >> 40) & 0xff; - buf[6] = (value >> 48) & 0xff; - buf[7] = (value >> 56) & 0xff; -#endif + if (port::kLittleEndian) { + memcpy(buf, &value, sizeof(value)); + } else { + buf[0] = value & 0xff; + buf[1] = (value >> 8) & 0xff; + buf[2] = (value >> 16) & 0xff; + buf[3] = (value >> 24) & 0xff; + buf[4] = (value >> 32) & 0xff; + buf[5] = (value >> 40) & 0xff; + buf[6] = (value >> 48) & 0xff; + buf[7] = (value >> 56) & 0xff; + } } void PutFixed32(std::string* dst, uint32_t value) { diff --git a/util/env_posix.cc b/util/env_posix.cc index cb1f6fc..78e09c9 100644 --- a/util/env_posix.cc +++ b/util/env_posix.cc @@ -3,6 +3,7 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #include <deque> +#include <set> #include <dirent.h> #include <errno.h> #include <fcntl.h> @@ -23,6 +24,7 @@ #include "leveldb/slice.h" #include "port/port.h" #include "util/logging.h" +#include "util/mutexlock.h" #include "util/posix_logger.h" namespace leveldb { @@ -90,18 +92,75 @@ class PosixRandomAccessFile: public RandomAccessFile { } }; +// Helper class to limit mmap file usage so that we do not end up +// running out virtual memory or running into kernel performance +// problems for very large databases. +class MmapLimiter { + public: + // Up to 1000 mmaps for 64-bit binaries; none for smaller pointer sizes. + MmapLimiter() { + SetAllowed(sizeof(void*) >= 8 ? 1000 : 0); + } + + // If another mmap slot is available, acquire it and return true. + // Else return false. + bool Acquire() { + if (GetAllowed() <= 0) { + return false; + } + MutexLock l(&mu_); + intptr_t x = GetAllowed(); + if (x <= 0) { + return false; + } else { + SetAllowed(x - 1); + return true; + } + } + + // Release a slot acquired by a previous call to Acquire() that returned true. + void Release() { + MutexLock l(&mu_); + SetAllowed(GetAllowed() + 1); + } + + private: + port::Mutex mu_; + port::AtomicPointer allowed_; + + intptr_t GetAllowed() const { + return reinterpret_cast<intptr_t>(allowed_.Acquire_Load()); + } + + // REQUIRES: mu_ must be held + void SetAllowed(intptr_t v) { + allowed_.Release_Store(reinterpret_cast<void*>(v)); + } + + MmapLimiter(const MmapLimiter&); + void operator=(const MmapLimiter&); +}; + // mmap() based random-access class PosixMmapReadableFile: public RandomAccessFile { private: std::string filename_; void* mmapped_region_; size_t length_; + MmapLimiter* limiter_; public: // base[0,length-1] contains the mmapped contents of the file. - PosixMmapReadableFile(const std::string& fname, void* base, size_t length) - : filename_(fname), mmapped_region_(base), length_(length) { } - virtual ~PosixMmapReadableFile() { munmap(mmapped_region_, length_); } + PosixMmapReadableFile(const std::string& fname, void* base, size_t length, + MmapLimiter* limiter) + : filename_(fname), mmapped_region_(base), length_(length), + limiter_(limiter) { + } + + virtual ~PosixMmapReadableFile() { + munmap(mmapped_region_, length_); + limiter_->Release(); + } virtual Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const { @@ -300,6 +359,25 @@ static int LockOrUnlock(int fd, bool lock) { class PosixFileLock : public FileLock { public: int fd_; + std::string name_; +}; + +// Set of locked files. We keep a separate set instead of just +// relying on fcntrl(F_SETLK) since fcntl(F_SETLK) does not provide +// any protection against multiple uses from the same process. +class PosixLockTable { + private: + port::Mutex mu_; + std::set<std::string> locked_files_; + public: + bool Insert(const std::string& fname) { + MutexLock l(&mu_); + return locked_files_.insert(fname).second; + } + void Remove(const std::string& fname) { + MutexLock l(&mu_); + locked_files_.erase(fname); + } }; class PosixEnv : public Env { @@ -329,19 +407,21 @@ class PosixEnv : public Env { int fd = open(fname.c_str(), O_RDONLY); if (fd < 0) { s = IOError(fname, errno); - } else if (sizeof(void*) >= 8) { - // Use mmap when virtual address-space is plentiful. + } else if (mmap_limit_.Acquire()) { uint64_t size; s = GetFileSize(fname, &size); if (s.ok()) { void* base = mmap(NULL, size, PROT_READ, MAP_SHARED, fd, 0); if (base != MAP_FAILED) { - *result = new PosixMmapReadableFile(fname, base, size); + *result = new PosixMmapReadableFile(fname, base, size, &mmap_limit_); } else { s = IOError(fname, errno); } } close(fd); + if (!s.ok()) { + mmap_limit_.Release(); + } } else { *result = new PosixRandomAccessFile(fname, fd); } @@ -430,12 +510,17 @@ class PosixEnv : public Env { int fd = open(fname.c_str(), O_RDWR | O_CREAT, 0644); if (fd < 0) { result = IOError(fname, errno); + } else if (!locks_.Insert(fname)) { + close(fd); + result = Status::IOError("lock " + fname, "already held by process"); } else if (LockOrUnlock(fd, true) == -1) { result = IOError("lock " + fname, errno); close(fd); + locks_.Remove(fname); } else { PosixFileLock* my_lock = new PosixFileLock; my_lock->fd_ = fd; + my_lock->name_ = fname; *lock = my_lock; } return result; @@ -447,6 +532,7 @@ class PosixEnv : public Env { if (LockOrUnlock(my_lock->fd_, false) == -1) { result = IOError("unlock", errno); } + locks_.Remove(my_lock->name_); close(my_lock->fd_); delete my_lock; return result; @@ -523,6 +609,9 @@ class PosixEnv : public Env { struct BGItem { void* arg; void (*function)(void*); }; typedef std::deque<BGItem> BGQueue; BGQueue queue_; + + PosixLockTable locks_; + MmapLimiter mmap_limit_; }; PosixEnv::PosixEnv() : page_size_(getpagesize()), diff --git a/util/mutexlock.h b/util/mutexlock.h index c3f3306..1ff5a9e 100644 --- a/util/mutexlock.h +++ b/util/mutexlock.h @@ -6,6 +6,7 @@ #define STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_ #include "port/port.h" +#include "port/thread_annotations.h" namespace leveldb { @@ -19,12 +20,13 @@ namespace leveldb { // ... some complex code, possibly with multiple return paths ... // } -class MutexLock { +class SCOPED_LOCKABLE MutexLock { public: - explicit MutexLock(port::Mutex *mu) : mu_(mu) { + explicit MutexLock(port::Mutex *mu) EXCLUSIVE_LOCK_FUNCTION(mu) + : mu_(mu) { this->mu_->Lock(); } - ~MutexLock() { this->mu_->Unlock(); } + ~MutexLock() UNLOCK_FUNCTION() { this->mu_->Unlock(); } private: port::Mutex *const mu_; |