diff options
| -rw-r--r-- | .appveyor.yml | 38 | ||||
| -rw-r--r-- | CMakeLists.txt | 31 | ||||
| -rw-r--r-- | README.md | 33 | ||||
| -rw-r--r-- | db/corruption_test.cc | 26 | ||||
| -rw-r--r-- | db/db_test.cc | 11 | ||||
| -rw-r--r-- | db/recovery_test.cc | 3 | ||||
| -rw-r--r-- | db/version_set.cc | 6 | ||||
| -rw-r--r-- | include/leveldb/env.h | 30 | ||||
| -rw-r--r-- | port/atomic_pointer.h | 4 | ||||
| -rw-r--r-- | port/port.h | 2 | ||||
| -rw-r--r-- | util/env_windows.cc | 742 | ||||
| -rw-r--r-- | util/env_windows_test.cc | 63 | ||||
| -rw-r--r-- | util/env_windows_test_helper.h | 30 | ||||
| -rw-r--r-- | util/windows_logger.h | 124 |
14 files changed, 1120 insertions, 23 deletions
diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 0000000..78aeaf1 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,38 @@ +# Build matrix / environment variables are explained on: +# https://www.appveyor.com/docs/appveyor-yml/ +# This file can be validated on: https://ci.appveyor.com/tools/validate-yaml + +version: "{build}" + +environment: + matrix: + # AppVeyor currently has no custom job name feature. + # http://help.appveyor.com/discussions/questions/1623-can-i-provide-a-friendly-name-for-jobs + - JOB: Visual Studio 2017 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + CMAKE_GENERATOR: Visual Studio 15 2017 + +platform: + - x86 + - x64 + +configuration: + - RelWithDebInfo + - Debug + +build: + verbosity: minimal + +build_script: + - git submodule update --init --recursive + - mkdir build + - cd build + - if "%platform%"=="x64" set CMAKE_GENERATOR=%CMAKE_GENERATOR% Win64 + - cmake --version + - cmake .. -G "%CMAKE_GENERATOR%" + -DCMAKE_CONFIGURATION_TYPES="%CONFIGURATION%" + - cmake --build . --config "%CONFIGURATION%" + - cd .. + +test_script: + - cd build ; ctest --verbose ; cd .. diff --git a/CMakeLists.txt b/CMakeLists.txt index f6a7c0a..1eaf48e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,6 +15,14 @@ set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) +if (WIN32) + set(LEVELDB_PLATFORM_NAME LEVELDB_PLATFORM_WINDOWS) + # TODO(cmumford): Make UNICODE configurable for Windows. + add_definitions(-D_UNICODE -DUNICODE) +else (WIN32) + set(LEVELDB_PLATFORM_NAME LEVELDB_PLATFORM_POSIX) +endif (WIN32) + option(LEVELDB_BUILD_TESTS "Build LevelDB's unit tests" ON) option(LEVELDB_BUILD_BENCHMARKS "Build LevelDB's benchmarks" ON) option(LEVELDB_INSTALL "Install LevelDB's header and library" ON) @@ -179,12 +187,19 @@ target_sources(leveldb "${LEVELDB_PUBLIC_INCLUDE_DIR}/write_batch.h" ) -# POSIX code is specified separately so we can leave it out in the future. +if (WIN32) +target_sources(leveldb + PRIVATE + "${PROJECT_SOURCE_DIR}/util/env_windows.cc" + "${PROJECT_SOURCE_DIR}/util/windows_logger.h" +) +else (WIN32) target_sources(leveldb PRIVATE "${PROJECT_SOURCE_DIR}/util/env_posix.cc" "${PROJECT_SOURCE_DIR}/util/posix_logger.h" ) +endif (WIN32) # MemEnv is not part of the interface and could be pulled to a separate library. target_sources(leveldb @@ -203,7 +218,7 @@ target_compile_definitions(leveldb # Used by include/export.h when building shared libraries. LEVELDB_COMPILE_LIBRARY # Used by port/port.h. - LEVELDB_PLATFORM_POSIX=1 + ${LEVELDB_PLATFORM_NAME}=1 ) if (NOT HAVE_CXX17_HAS_INCLUDE) target_compile_definitions(leveldb @@ -265,7 +280,7 @@ if(LEVELDB_BUILD_TESTS) target_link_libraries("${test_target_name}" leveldb) target_compile_definitions("${test_target_name}" PRIVATE - LEVELDB_PLATFORM_POSIX=1 + ${LEVELDB_PLATFORM_NAME}=1 ) if (NOT HAVE_CXX17_HAS_INCLUDE) target_compile_definitions("${test_target_name}" @@ -314,8 +329,12 @@ if(LEVELDB_BUILD_TESTS) leveldb_test("${PROJECT_SOURCE_DIR}/util/logging_test.cc") # TODO(costan): This test also uses - # "${PROJECT_SOURCE_DIR}/util/env_posix_test_helper.h" - leveldb_test("${PROJECT_SOURCE_DIR}/util/env_posix_test.cc") + # "${PROJECT_SOURCE_DIR}/util/env_{posix|windows}_test_helper.h" + if (WIN32) + leveldb_test("${PROJECT_SOURCE_DIR}/util/env_windows_test.cc") + else (WIN32) + leveldb_test("${PROJECT_SOURCE_DIR}/util/env_posix_test.cc") + endif (WIN32) endif(NOT BUILD_SHARED_LIBS) endif(LEVELDB_BUILD_TESTS) @@ -339,7 +358,7 @@ if(LEVELDB_BUILD_BENCHMARKS) target_link_libraries("${bench_target_name}" leveldb) target_compile_definitions("${bench_target_name}" PRIVATE - LEVELDB_PLATFORM_POSIX=1 + ${LEVELDB_PLATFORM_NAME}=1 ) if (NOT HAVE_CXX17_HAS_INCLUDE) target_compile_definitions("${bench_target_name}" @@ -30,6 +30,8 @@ Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com) This project supports [CMake](https://cmake.org/) out of the box. +### Build for POSIX + Quick start: ```bash @@ -37,6 +39,29 @@ mkdir -p build && cd build cmake -DCMAKE_BUILD_TYPE=Release .. && cmake --build . ``` +### Building for Windows + +First generate the Visual Studio 2017 project/solution files: + +```bash +mkdir -p build +cd build +cmake -G "Visual Studio 15" .. +``` +The default default will build for x86. For 64-bit run: + +```bash +cmake -G "Visual Studio 15 Win64" .. +``` + +To compile the Windows solution from the command-line: + +```bash +devenv /build Debug leveldb.sln +``` + +or open leveldb.sln in Visual Studio and build from within. + Please see the CMake documentation and `CMakeLists.txt` for more advanced usage. # Contributing to the leveldb Project @@ -48,10 +73,10 @@ will be considered. Contribution requirements: -1. **POSIX only**. We _generally_ will only accept changes that are both - compiled, and tested on a POSIX platform - usually Linux. Very small - changes will sometimes be accepted, but consider that more of an - exception than the rule. +1. **Tested platforms only**. We _generally_ will only accept changes for + platforms that are compiled and tested. This means POSIX (for Linux and + macOS) or Windows. Very small changes will sometimes be accepted, but + consider that more of an exception than the rule. 2. **Stable API**. We strive very hard to maintain a stable API. Changes that require changes for projects using leveldb _might_ be rejected without diff --git a/db/corruption_test.cc b/db/corruption_test.cc index 0b93c24..98aaf8c 100644 --- a/db/corruption_test.cc +++ b/db/corruption_test.cc @@ -20,6 +20,10 @@ #include "util/testharness.h" #include "util/testutil.h" +#if defined(LEVELDB_PLATFORM_WINDOWS) +#include "util/env_windows_test_helper.h" +#endif // defined(LEVELDB_PLATFORM_WINDOWS) + namespace leveldb { static const int kValueSize = 1000; @@ -32,6 +36,17 @@ class CorruptionTest { Options options_; DB* db_; +#if defined(LEVELDB_PLATFORM_WINDOWS) + static void SetFileLimits(int mmap_limit) { + EnvWindowsTestHelper::SetReadOnlyMMapLimit(mmap_limit); + } + + // TODO(cmumford): Modify corruption_test to use MemEnv and remove. + static void RelaxFilePermissions() { + EnvWindowsTestHelper::RelaxFilePermissions(); + } +#endif // defined(LEVELDB_PLATFORM_WINDOWS) + CorruptionTest() { tiny_cache_ = NewLRUCache(100); options_.env = &env_; @@ -370,5 +385,16 @@ TEST(CorruptionTest, UnrelatedKeys) { } // namespace leveldb int main(int argc, char** argv) { +#if defined(LEVELDB_PLATFORM_WINDOWS) + // When Windows maps the contents of a file into memory, even if read/write, + // subsequent attempts to open that file for write access will fail. Forcing + // all RandomAccessFile instances to use base file I/O (e.g. ReadFile) + // allows these tests to open files in order to corrupt their contents. + leveldb::CorruptionTest::SetFileLimits(0); + + // Allow this test to write to (and corrupt) files which are normally + // open for exclusive read access. + leveldb::CorruptionTest::RelaxFilePermissions(); +#endif // defined(LEVELDB_PLATFORM_WINDOWS) return leveldb::test::RunAllTests(); } diff --git a/db/db_test.cc b/db/db_test.cc index 878b7d4..894ed23 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -470,11 +470,12 @@ class DBTest { } // Do n memtable compactions, each of which produces an sstable - // covering the range [small,large]. - void MakeTables(int n, const std::string& small, const std::string& large) { + // covering the range [small_key,large_key]. + void MakeTables(int n, const std::string& small_key, + const std::string& large_key) { for (int i = 0; i < n; i++) { - Put(small, "begin"); - Put(large, "end"); + Put(small_key, "begin"); + Put(large_key, "end"); dbfull()->TEST_CompactMemTable(); } } @@ -1655,7 +1656,7 @@ TEST(DBTest, DestroyEmptyDir) { ASSERT_TRUE(env.FileExists(dbname)); std::vector<std::string> children; ASSERT_OK(env.GetChildren(dbname, &children)); - // The POSIX env does not filter out '.' and '..' special files. + // The stock Env's do not filter out '.' and '..' special files. ASSERT_EQ(2, children.size()); ASSERT_OK(DestroyDB(dbname, opts)); ASSERT_TRUE(!env.FileExists(dbname)); diff --git a/db/recovery_test.cc b/db/recovery_test.cc index c852803..87bd53c 100644 --- a/db/recovery_test.cc +++ b/db/recovery_test.cc @@ -97,6 +97,9 @@ class RecoveryTest { } size_t DeleteLogFiles() { + // Linux allows unlinking open files, but Windows does not. + // Closing the db allows for file deletion. + Close(); std::vector<uint64_t> logs = GetFiles(kLogFile); for (size_t i = 0; i < logs.size(); i++) { ASSERT_OK(env_->DeleteFile(LogName(logs[i]))) << LogName(logs[i]); diff --git a/db/version_set.cc b/db/version_set.cc index c27ccad..ae06089 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -143,8 +143,8 @@ bool SomeFileOverlapsRange( uint32_t index = 0; if (smallest_user_key != nullptr) { // Find the earliest possible internal key for smallest_user_key - InternalKey small(*smallest_user_key, kMaxSequenceNumber,kValueTypeForSeek); - index = FindFile(icmp, files, small.Encode()); + InternalKey small_key(*smallest_user_key, kMaxSequenceNumber,kValueTypeForSeek); + index = FindFile(icmp, files, small_key.Encode()); } if (index >= files.size()) { @@ -700,7 +700,7 @@ class VersionSet::Builder { // same as the compaction of 40KB of data. We are a little // conservative and allow approximately one seek for every 16KB // of data before triggering a compaction. - f->allowed_seeks = (f->file_size / 16384); + f->allowed_seeks = static_cast<int>((f->file_size / 16384U)); if (f->allowed_seeks < 100) f->allowed_seeks = 100; levels_[level].deleted_files.erase(f->number); diff --git a/include/leveldb/env.h b/include/leveldb/env.h index 59e2a6f..946ea98 100644 --- a/include/leveldb/env.h +++ b/include/leveldb/env.h @@ -20,6 +20,27 @@ #include "leveldb/export.h" #include "leveldb/status.h" +#if defined(_WIN32) +// The leveldb::Env class below contains a DeleteFile method. +// At the same time, <windows.h>, a fairly popular header +// file for Windows applications, defines a DeleteFile macro. +// +// Without any intervention on our part, the result of this +// unfortunate coincidence is that the name of the +// leveldb::Env::DeleteFile method seen by the compiler depends on +// whether <windows.h> was included before or after the LevelDB +// headers. +// +// To avoid headaches, we undefined DeleteFile (if defined) and +// redefine it at the bottom of this file. This way <windows.h> +// can be included before this file (or not at all) and the +// exported method will always be leveldb::Env::DeleteFile. +#if defined(DeleteFile) +#undef DeleteFile +#define LEVELDB_DELETEFILE_UNDEFINED +#endif // defined(DeleteFile) +#endif // defined(_WIN32) + namespace leveldb { class FileLock; @@ -356,4 +377,13 @@ class LEVELDB_EXPORT EnvWrapper : public Env { } // namespace leveldb +// Redefine DeleteFile if necessary. +#if defined(_WIN32) && defined(LEVELDB_DELETEFILE_UNDEFINED) +#if defined(UNICODE) +#define DeleteFile DeleteFileW +#else +#define DeleteFile DeleteFileA +#endif // defined(UNICODE) +#endif // defined(_WIN32) && defined(LEVELDB_DELETEFILE_UNDEFINED) + #endif // STORAGE_LEVELDB_INCLUDE_ENV_H_ diff --git a/port/atomic_pointer.h b/port/atomic_pointer.h index bb4e183..d906f63 100644 --- a/port/atomic_pointer.h +++ b/port/atomic_pointer.h @@ -22,10 +22,6 @@ #include <atomic> -#ifdef OS_WIN -#include <windows.h> -#endif - #if defined(_M_X64) || defined(__x86_64__) #define ARCH_CPU_X86_FAMILY 1 #elif defined(_M_IX86) || defined(__i386__) || defined(__i386) diff --git a/port/port.h b/port/port.h index 0975fed..b2210a7 100644 --- a/port/port.h +++ b/port/port.h @@ -10,7 +10,7 @@ // Include the appropriate platform specific file below. If you are // porting to a new platform, see "port_example.h" for documentation // of what the new port_<platform>.h file must provide. -#if defined(LEVELDB_PLATFORM_POSIX) +#if defined(LEVELDB_PLATFORM_POSIX) || defined(LEVELDB_PLATFORM_WINDOWS) # include "port/port_stdcxx.h" #elif defined(LEVELDB_PLATFORM_CHROMIUM) # include "port/port_chromium.h" diff --git a/util/env_windows.cc b/util/env_windows.cc new file mode 100644 index 0000000..03da266 --- /dev/null +++ b/util/env_windows.cc @@ -0,0 +1,742 @@ +// Copyright (c) 2018 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +// Prevent Windows headers from defining min/max macros and instead +// use STL. +#define NOMINMAX +#include <windows.h> + +#include <algorithm> +#include <chrono> +#include <condition_variable> +#include <deque> +#include <memory> +#include <mutex> +#include <sstream> +#include <string> +#include <vector> + +#include "leveldb/env.h" +#include "leveldb/slice.h" +#include "port/port.h" +#include "port/thread_annotations.h" +#include "util/env_windows_test_helper.h" +#include "util/logging.h" +#include "util/mutexlock.h" +#include "util/windows_logger.h" + +#if defined(DeleteFile) +#undef DeleteFile +#endif // defined(DeleteFile) + +namespace leveldb { + +namespace { + +constexpr const size_t kWritableFileBufferSize = 65536; + +// Up to 1000 mmaps for 64-bit binaries; none for 32-bit. +constexpr int kDefaultMmapLimit = sizeof(void*) >= 8 ? 1000 : 0; + +// Modified by EnvWindowsTestHelper::SetReadOnlyMMapLimit(). +int g_mmap_limit = kDefaultMmapLimit; + +// Relax some file access permissions for testing. +bool g_relax_permissions = false; + +std::string GetWindowsErrorMessage(DWORD error_code) { + std::string message; + char* error_text = nullptr; + // Use MBCS version of FormatMessage to match return value. + size_t error_text_size = ::FormatMessageA( + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_IGNORE_INSERTS, + nullptr, error_code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + reinterpret_cast<char*>(&error_text), 0, nullptr); + if (!error_text) { + return message; + } + message.assign(error_text, error_text_size); + ::LocalFree(error_text); + return message; +} + +Status WindowsError(const std::string& context, DWORD error_code) { + if (error_code == ERROR_FILE_NOT_FOUND || error_code == ERROR_PATH_NOT_FOUND) + return Status::NotFound(context, GetWindowsErrorMessage(error_code)); + return Status::IOError(context, GetWindowsErrorMessage(error_code)); +} + +class ScopedHandle { + public: + ScopedHandle(HANDLE handle) : handle_(handle) {} + ScopedHandle(ScopedHandle&& other) noexcept : handle_(other.Release()) {} + ~ScopedHandle() { Close(); } + + ScopedHandle& operator=(ScopedHandle&& rhs) noexcept { + if (this != &rhs) handle_ = rhs.Release(); + return *this; + } + + bool Close() { + if (!is_valid()) { + return true; + } + HANDLE h = handle_; + handle_ = INVALID_HANDLE_VALUE; + return ::CloseHandle(h); + } + + bool is_valid() const { + return handle_ != INVALID_HANDLE_VALUE && handle_ != nullptr; + } + + HANDLE get() const { return handle_; } + + HANDLE Release() { + HANDLE h = handle_; + handle_ = INVALID_HANDLE_VALUE; + return h; + } + + private: + HANDLE handle_; +}; + +// Helper class to limit resource usage to avoid exhaustion. +// Currently used to limit mmap file usage so that we do not end +// up running out virtual memory, or running into kernel performance +// problems for very large databases. +class Limiter { + public: + // Limit maximum number of resources to |n|. + Limiter(intptr_t n) { SetAllowed(n); } + + // If another resource is available, acquire it and return true. + // Else return false. + bool Acquire() LOCKS_EXCLUDED(mu_) { + if (GetAllowed() <= 0) { + return false; + } + MutexLock l(&mu_); + intptr_t x = GetAllowed(); + if (x <= 0) { + return false; + } else { + SetAllowed(x - 1); + return true; + } + } + + // Release a resource acquired by a previous call to Acquire() that returned + // true. + void Release() LOCKS_EXCLUDED(mu_) { + MutexLock l(&mu_); + SetAllowed(GetAllowed() + 1); + } + + private: + port::Mutex mu_; + port::AtomicPointer allowed_; + + intptr_t GetAllowed() const { + return reinterpret_cast<intptr_t>(allowed_.Acquire_Load()); + } + + void SetAllowed(intptr_t v) EXCLUSIVE_LOCKS_REQUIRED(mu_) { + allowed_.Release_Store(reinterpret_cast<void*>(v)); + } + + Limiter(const Limiter&); + void operator=(const Limiter&); +}; + +class WindowsSequentialFile : public SequentialFile { + public: + WindowsSequentialFile(std::string fname, ScopedHandle file) + : filename_(fname), file_(std::move(file)) {} + ~WindowsSequentialFile() override {} + + Status Read(size_t n, Slice* result, char* scratch) override { + Status s; + DWORD bytes_read; + // DWORD is 32-bit, but size_t could technically be larger. However leveldb + // files are limited to leveldb::Options::max_file_size which is clamped to + // 1<<30 or 1 GiB. + assert(n <= std::numeric_limits<DWORD>::max()); + if (!::ReadFile(file_.get(), scratch, static_cast<DWORD>(n), &bytes_read, + nullptr)) { + s = WindowsError(filename_, ::GetLastError()); + } else { + *result = Slice(scratch, bytes_read); + } + return s; + } + + Status Skip(uint64_t n) override { + LARGE_INTEGER distance; + distance.QuadPart = n; + if (!::SetFilePointerEx(file_.get(), distance, nullptr, FILE_CURRENT)) { + return WindowsError(filename_, ::GetLastError()); + } + return Status::OK(); + } + + private: + std::string filename_; + ScopedHandle file_; +}; + +class WindowsRandomAccessFile : public RandomAccessFile { + public: + WindowsRandomAccessFile(std::string fname, ScopedHandle handle) + : filename_(fname), handle_(std::move(handle)) {} + + ~WindowsRandomAccessFile() override = default; + + Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const override { + DWORD bytes_read = 0; + OVERLAPPED overlapped = {0}; + + overlapped.OffsetHigh = static_cast<DWORD>(offset >> 32); + overlapped.Offset = static_cast<DWORD>(offset); + if (!::ReadFile(handle_.get(), scratch, static_cast<DWORD>(n), &bytes_read, + &overlapped)) { + DWORD error_code = ::GetLastError(); + if (error_code != ERROR_HANDLE_EOF) { + *result = Slice(scratch, 0); + return Status::IOError(filename_, GetWindowsErrorMessage(error_code)); + } + } + + *result = Slice(scratch, bytes_read); + return Status::OK(); + } + + private: + std::string filename_; + ScopedHandle handle_; +}; + +class WindowsMmapReadableFile : public RandomAccessFile { + public: + // base[0,length-1] contains the mmapped contents of the file. + WindowsMmapReadableFile(std::string fname, void* base, size_t length, + Limiter* limiter) + : filename_(std::move(fname)), + mmapped_region_(base), + length_(length), + limiter_(limiter) {} + + ~WindowsMmapReadableFile() override { + ::UnmapViewOfFile(mmapped_region_); + limiter_->Release(); + } + + Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const override { + Status s; + if (offset + n > length_) { + *result = Slice(); + s = WindowsError(filename_, ERROR_INVALID_PARAMETER); + } else { + *result = Slice(reinterpret_cast<char*>(mmapped_region_) + offset, n); + } + return s; + } + + private: + std::string filename_; + void* mmapped_region_; + size_t length_; + Limiter* limiter_; +}; + +class WindowsWritableFile : public WritableFile { + public: + WindowsWritableFile(std::string fname, ScopedHandle handle) + : filename_(std::move(fname)), handle_(std::move(handle)), pos_(0) {} + + ~WindowsWritableFile() override = default; + + Status Append(const Slice& data) override { + size_t n = data.size(); + const char* p = data.data(); + + // Fit as much as possible into buffer. + size_t copy = std::min(n, kWritableFileBufferSize - pos_); + memcpy(buf_ + pos_, p, copy); + p += copy; + n -= copy; + pos_ += copy; + if (n == 0) { + return Status::OK(); + } + + // Can't fit in buffer, so need to do at least one write. + Status s = FlushBuffered(); + if (!s.ok()) { + return s; + } + + // Small writes go to buffer, large writes are written directly. + if (n < kWritableFileBufferSize) { + memcpy(buf_, p, n); + pos_ = n; + return Status::OK(); + } + return WriteRaw(p, n); + } + + Status Close() override { + Status result = FlushBuffered(); + if (!handle_.Close() && result.ok()) { + result = WindowsError(filename_, ::GetLastError()); + } + return result; + } + + Status Flush() override { return FlushBuffered(); } + + Status Sync() override { + // On Windows no need to sync parent directory. It's metadata will be + // updated via the creation of the new file, without an explicit sync. + return FlushBuffered(); + } + + private: + Status FlushBuffered() { + Status s = WriteRaw(buf_, pos_); + pos_ = 0; + return s; + } + + Status WriteRaw(const char* p, size_t n) { + DWORD bytes_written; + if (!::WriteFile(handle_.get(), p, static_cast<DWORD>(n), &bytes_written, + nullptr)) { + return Status::IOError(filename_, + GetWindowsErrorMessage(::GetLastError())); + } + return Status::OK(); + } + + // buf_[0, pos_-1] contains data to be written to handle_. + const std::string filename_; + ScopedHandle handle_; + char buf_[kWritableFileBufferSize]; + size_t pos_; +}; + +// Lock or unlock the entire file as specified by |lock|. Returns true +// when successful, false upon failure. Caller should call ::GetLastError() +// to determine cause of failure +bool LockOrUnlock(HANDLE handle, bool lock) { + if (lock) { + return ::LockFile(handle, + /*dwFileOffsetLow=*/0, /*dwFileOffsetHigh=*/0, + /*nNumberOfBytesToLockLow=*/MAXDWORD, + /*nNumberOfBytesToLockHigh=*/MAXDWORD); + } else { + return ::UnlockFile(handle, + /*dwFileOffsetLow=*/0, /*dwFileOffsetHigh=*/0, + /*nNumberOfBytesToLockLow=*/MAXDWORD, + /*nNumberOfBytesToLockHigh=*/MAXDWORD); + } +} + +class WindowsFileLock : public FileLock { + public: + WindowsFileLock(ScopedHandle handle, std::string name) + : handle_(std::move(handle)), name_(std::move(name)) {} + + ScopedHandle& handle() { return handle_; } + const std::string& name() const { return name_; } + + private: + ScopedHandle handle_; + std::string name_; +}; + +class WindowsEnv : public Env { + public: + WindowsEnv(); + ~WindowsEnv() override { + static char msg[] = "Destroying Env::Default()\n"; + fwrite(msg, 1, sizeof(msg), stderr); + abort(); + } + + Status NewSequentialFile(const std::string& fname, + SequentialFile** result) override { + *result = nullptr; + DWORD desired_access = GENERIC_READ; + DWORD share_mode = FILE_SHARE_READ; + if (g_relax_permissions) { + desired_access |= GENERIC_WRITE; + share_mode |= FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE; + } + ScopedHandle handle = + ::CreateFileA(fname.c_str(), desired_access, share_mode, nullptr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (!handle.is_valid()) { + return WindowsError(fname, ::GetLastError()); + } + *result = new WindowsSequentialFile(fname, std::move(handle)); + return Status::OK(); + } + + Status NewRandomAccessFile(const std::string& fname, + RandomAccessFile** result) override { + *result = nullptr; + DWORD desired_access = GENERIC_READ; + DWORD share_mode = FILE_SHARE_READ; + if (g_relax_permissions) { + // desired_access |= GENERIC_WRITE; + share_mode |= FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE; + } + DWORD file_flags = FILE_ATTRIBUTE_READONLY; + + ScopedHandle handle = + ::CreateFileA(fname.c_str(), desired_access, share_mode, nullptr, + OPEN_EXISTING, file_flags, nullptr); + if (!handle.is_valid()) { + return WindowsError(fname, ::GetLastError()); + } + if (!mmap_limiter_.Acquire()) { + *result = new WindowsRandomAccessFile(fname, std::move(handle)); + return Status::OK(); + } + + LARGE_INTEGER file_size; + if (!::GetFileSizeEx(handle.get(), &file_size)) { + return WindowsError(fname, ::GetLastError()); + } + + ScopedHandle mapping = + ::CreateFileMappingA(handle.get(), + /*security attributes=*/nullptr, PAGE_READONLY, + /*dwMaximumSizeHigh=*/0, + /*dwMaximumSizeLow=*/0, nullptr); + if (mapping.is_valid()) { + void* base = MapViewOfFile(mapping.get(), FILE_MAP_READ, 0, 0, 0); + if (base) { + *result = new WindowsMmapReadableFile( + fname, base, static_cast<size_t>(file_size.QuadPart), + &mmap_limiter_); + return Status::OK(); + } + } + Status s = WindowsError(fname, ::GetLastError()); + + if (!s.ok()) { + mmap_limiter_.Release(); + } + return s; + } + + Status NewWritableFile(const std::string& fname, + WritableFile** result) override { + DWORD desired_access = GENERIC_WRITE; + DWORD share_mode = 0; + if (g_relax_permissions) { + desired_access |= GENERIC_READ; + share_mode |= FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE; + } + + ScopedHandle handle = + ::CreateFileA(fname.c_str(), desired_access, share_mode, nullptr, + CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); + if (!handle.is_valid()) { + *result = nullptr; + return WindowsError(fname, ::GetLastError()); + } + + *result = new WindowsWritableFile(fname, std::move(handle)); + return Status::OK(); + } + + Status NewAppendableFile(const std::string& fname, + WritableFile** result) override { + ScopedHandle handle = + ::CreateFileA(fname.c_str(), FILE_APPEND_DATA, 0, nullptr, OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (!handle.is_valid()) { + *result = nullptr; + return WindowsError(fname, ::GetLastError()); + } + + *result = new WindowsWritableFile(fname, std::move(handle)); + return Status::OK(); + } + + bool FileExists(const std::string& fname) override { + return GetFileAttributesA(fname.c_str()) != INVALID_FILE_ATTRIBUTES; + } + + Status GetChildren(const std::string& dir, + std::vector<std::string>* result) override { + const std::string find_pattern = dir + "\\*"; + WIN32_FIND_DATAA find_data; + HANDLE dir_handle = ::FindFirstFileA(find_pattern.c_str(), &find_data); + if (dir_handle == INVALID_HANDLE_VALUE) { + DWORD last_error = ::GetLastError(); + if (last_error == ERROR_FILE_NOT_FOUND) { + return Status::OK(); + } + return WindowsError(dir, last_error); + } + do { + char base_name[_MAX_FNAME]; + char ext[_MAX_EXT]; + + if (!_splitpath_s(find_data.cFileName, nullptr, 0, nullptr, 0, base_name, + ARRAYSIZE(base_name), ext, ARRAYSIZE(ext))) { + result->emplace_back(std::string(base_name) + ext); + } + } while (::FindNextFileA(dir_handle, &find_data)); + DWORD last_error = ::GetLastError(); + ::FindClose(dir_handle); + if (last_error != ERROR_NO_MORE_FILES) { + return WindowsError(dir, last_error); + } + return Status::OK(); + } + + Status DeleteFile(const std::string& fname) override { + if (!::DeleteFileA(fname.c_str())) { + return WindowsError(fname, ::GetLastError()); + } + return Status::OK(); + } + + Status CreateDir(const std::string& name) override { + if (!::CreateDirectoryA(name.c_str(), nullptr)) { + return WindowsError(name, ::GetLastError()); + } + return Status::OK(); + } + + Status DeleteDir(const std::string& name) override { + if (!::RemoveDirectoryA(name.c_str())) { + return WindowsError(name, ::GetLastError()); + } + return Status::OK(); + } + + Status GetFileSize(const std::string& fname, uint64_t* size) override { + WIN32_FILE_ATTRIBUTE_DATA attrs; + if (!::GetFileAttributesExA(fname.c_str(), GetFileExInfoStandard, &attrs)) { + return WindowsError(fname, ::GetLastError()); + } + ULARGE_INTEGER file_size; + file_size.HighPart = attrs.nFileSizeHigh; + file_size.LowPart = attrs.nFileSizeLow; + *size = file_size.QuadPart; + return Status::OK(); + } + + Status RenameFile(const std::string& src, + const std::string& target) override { + // Try a simple move first. It will only succeed when |to_path| doesn't + // already exist. + if (::MoveFileA(src.c_str(), target.c_str())) { + return Status::OK(); + } + DWORD move_error = ::GetLastError(); + + // Try the full-blown replace if the move fails, as ReplaceFile will only + // succeed when |to_path| does exist. When writing to a network share, we + // may not be able to change the ACLs. Ignore ACL errors then + // (REPLACEFILE_IGNORE_MERGE_ERRORS). + if (::ReplaceFileA(target.c_str(), src.c_str(), nullptr, + REPLACEFILE_IGNORE_MERGE_ERRORS, nullptr, nullptr)) { + return Status::OK(); + } + DWORD replace_error = ::GetLastError(); + // In the case of FILE_ERROR_NOT_FOUND from ReplaceFile, it is likely + // that |to_path| does not exist. In this case, the more relevant error + // comes from the call to MoveFile. + if (replace_error == ERROR_FILE_NOT_FOUND || + replace_error == ERROR_PATH_NOT_FOUND) { + return WindowsError(src, move_error); + } else { + return WindowsError(src, replace_error); + } + } + + Status LockFile(const std::string& fname, FileLock** lock) override { + *lock = nullptr; + Status result; + ScopedHandle handle = ::CreateFileA( + fname.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, + /*lpSecurityAttributes=*/nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, + nullptr); + if (!handle.is_valid()) { + result = WindowsError(fname, ::GetLastError()); + } else if (!LockOrUnlock(handle.get(), true)) { + result = WindowsError("lock " + fname, ::GetLastError()); + } else { + *lock = new WindowsFileLock(std::move(handle), std::move(fname)); + } + return result; + } + + Status UnlockFile(FileLock* lock) override { + std::unique_ptr<WindowsFileLock> my_lock( + reinterpret_cast<WindowsFileLock*>(lock)); + Status result; + if (!LockOrUnlock(my_lock->handle().get(), false)) { + result = WindowsError("unlock", ::GetLastError()); + } + return result; + } + + void Schedule(void (*function)(void*), void* arg) override; + + void StartThread(void (*function)(void* arg), void* arg) override { + std::thread t(function, arg); + t.detach(); + } + + Status GetTestDirectory(std::string* result) override { + const char* env = getenv("TEST_TMPDIR"); + if (env && env[0] != '\0') { + *result = env; + return Status::OK(); + } + + char tmp_path[MAX_PATH]; + if (!GetTempPathA(ARRAYSIZE(tmp_path), tmp_path)) { + return WindowsError("GetTempPath", ::GetLastError()); + } + std::stringstream ss; + ss << tmp_path << "leveldbtest-" << std::this_thread::get_id(); + *result = ss.str(); + + // Directory may already exist + CreateDir(*result); + return Status::OK(); + } + + Status NewLogger(const std::string& fname, Logger** result) override { + ScopedHandle handle = + ::CreateFileA(fname.c_str(), GENERIC_WRITE, FILE_SHARE_READ, nullptr, + CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); + if (!handle.is_valid()) { + return WindowsError("NewLogger", ::GetLastError()); + } + *result = new WindowsLogger(handle.Release()); + return Status::OK(); + } + + uint64_t NowMicros() override { + // GetSystemTimeAsFileTime typically has a resolution of 10-20 msec. + // TODO(cmumford): Switch to GetSystemTimePreciseAsFileTime which is + // available in Windows 8 and later. + FILETIME ft; + ::GetSystemTimeAsFileTime(&ft); + // Each tick represents a 100-nanosecond intervals since January 1, 1601 + // (UTC). + uint64_t num_ticks = + (static_cast<uint64_t>(ft.dwHighDateTime) << 32) + ft.dwLowDateTime; + return num_ticks / 10; + } + + void SleepForMicroseconds(int micros) override { + std::this_thread::sleep_for(std::chrono::microseconds(micros)); + } + + private: + // BGThread() is the body of the background thread + void BGThread(); + + std::mutex mu_; + std::condition_variable bgsignal_; + bool started_bgthread_; + + // Entry per Schedule() call + struct BGItem { + void* arg; + void (*function)(void*); + }; + typedef std::deque<BGItem> BGQueue; + BGQueue queue_; + + Limiter mmap_limiter_; +}; + +// Return the maximum number of concurrent mmaps. +int MaxMmaps() { + if (g_mmap_limit >= 0) { + return g_mmap_limit; + } + // Up to 1000 mmaps for 64-bit binaries; none for smaller pointer sizes. + g_mmap_limit = sizeof(void*) >= 8 ? 1000 : 0; + return g_mmap_limit; +} + +WindowsEnv::WindowsEnv() + : started_bgthread_(false), mmap_limiter_(MaxMmaps()) {} + +void WindowsEnv::Schedule(void (*function)(void*), void* arg) { + std::lock_guard<std::mutex> guard(mu_); + + // Start background thread if necessary + if (!started_bgthread_) { + started_bgthread_ = true; + std::thread t(&WindowsEnv::BGThread, this); + t.detach(); + } + + // If the queue is currently empty, the background thread may currently be + // waiting. + if (queue_.empty()) { + bgsignal_.notify_one(); + } + + // Add to priority queue + queue_.push_back(BGItem()); + queue_.back().function = function; + queue_.back().arg = arg; +} + +void WindowsEnv::BGThread() { + while (true) { + // Wait until there is an item that is ready to run + std::unique_lock<std::mutex> lk(mu_); + bgsignal_.wait(lk, [this] { return !queue_.empty(); }); + + void (*function)(void*) = queue_.front().function; + void* arg = queue_.front().arg; + queue_.pop_front(); + + lk.unlock(); + (*function)(arg); + } +} + +} // namespace + +static std::once_flag once; +static Env* default_env; +static void InitDefaultEnv() { default_env = new WindowsEnv(); } + +void EnvWindowsTestHelper::SetReadOnlyMMapLimit(int limit) { + assert(default_env == nullptr); + g_mmap_limit = limit; +} + +void EnvWindowsTestHelper::RelaxFilePermissions() { + assert(default_env == nullptr); + g_relax_permissions = true; +} + +Env* Env::Default() { + std::call_once(once, InitDefaultEnv); + return default_env; +} + +} // namespace leveldb diff --git a/util/env_windows_test.cc b/util/env_windows_test.cc new file mode 100644 index 0000000..4451b9e --- /dev/null +++ b/util/env_windows_test.cc @@ -0,0 +1,63 @@ +// Copyright (c) 2018 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "leveldb/env.h" + +#include "port/port.h" +#include "util/env_windows_test_helper.h" +#include "util/testharness.h" + +namespace leveldb { + +static const int kMMapLimit = 4; + +class EnvWindowsTest { + public: + Env* env_; + EnvWindowsTest() : env_(Env::Default()) {} + + static void SetFileLimits(int mmap_limit) { + EnvWindowsTestHelper::SetReadOnlyMMapLimit(mmap_limit); + } +}; + +TEST(EnvWindowsTest, TestOpenOnRead) { + // Write some test data to a single file that will be opened |n| times. + std::string test_dir; + ASSERT_OK(env_->GetTestDirectory(&test_dir)); + std::string test_file = test_dir + "/open_on_read.txt"; + + FILE* f = fopen(test_file.c_str(), "w"); + ASSERT_TRUE(f != nullptr); + const char kFileData[] = "abcdefghijklmnopqrstuvwxyz"; + fputs(kFileData, f); + fclose(f); + + // Open test file some number above the sum of the two limits to force + // leveldb::WindowsEnv to switch from mapping the file into memory + // to basic file reading. + const int kNumFiles = kMMapLimit + 5; + leveldb::RandomAccessFile* files[kNumFiles] = {0}; + for (int i = 0; i < kNumFiles; i++) { + ASSERT_OK(env_->NewRandomAccessFile(test_file, &files[i])); + } + char scratch; + Slice read_result; + for (int i = 0; i < kNumFiles; i++) { + ASSERT_OK(files[i]->Read(i, 1, &read_result, &scratch)); + ASSERT_EQ(kFileData[i], read_result[0]); + } + for (int i = 0; i < kNumFiles; i++) { + delete files[i]; + } + ASSERT_OK(env_->DeleteFile(test_file)); +} + +} // namespace leveldb + +int main(int argc, char** argv) { + // All tests currently run with the same read-only file limits. + leveldb::EnvWindowsTest::SetFileLimits(leveldb::kMMapLimit); + return leveldb::test::RunAllTests(); +} diff --git a/util/env_windows_test_helper.h b/util/env_windows_test_helper.h new file mode 100644 index 0000000..5ffbe44 --- /dev/null +++ b/util/env_windows_test_helper.h @@ -0,0 +1,30 @@ +// Copyright 2018 (c) The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef STORAGE_LEVELDB_UTIL_ENV_WINDOWS_TEST_HELPER_H_ +#define STORAGE_LEVELDB_UTIL_ENV_WINDOWS_TEST_HELPER_H_ + +namespace leveldb { + +class EnvWindowsTest; + +// A helper for the Windows Env to facilitate testing. +class EnvWindowsTestHelper { + private: + friend class CorruptionTest; + friend class EnvWindowsTest; + + // Set the maximum number of read-only files that will be mapped via mmap. + // Must be called before creating an Env. + static void SetReadOnlyMMapLimit(int limit); + + // Relax file permissions for tests. This results in most files being opened + // with read-write permissions. This is helpful for corruption tests that + // need to corrupt the database files for open databases. + static void RelaxFilePermissions(); +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_UTIL_ENV_WINDOWS_TEST_HELPER_H_ diff --git a/util/windows_logger.h b/util/windows_logger.h new file mode 100644 index 0000000..b2a2cae --- /dev/null +++ b/util/windows_logger.h @@ -0,0 +1,124 @@ +// Copyright (c) 2018 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Logger implementation for the Windows platform. + +#ifndef STORAGE_LEVELDB_UTIL_WINDOWS_LOGGER_H_ +#define STORAGE_LEVELDB_UTIL_WINDOWS_LOGGER_H_ + +#include <stdio.h> + +#include <cassert> +#include <cstdarg> +#include <ctime> +#include <sstream> +#include <thread> + +#include "leveldb/env.h" + +namespace leveldb { + +class WindowsLogger final : public Logger { + public: + WindowsLogger(HANDLE handle) : handle_(handle) { + assert(handle != INVALID_HANDLE_VALUE); + } + + ~WindowsLogger() override { ::CloseHandle(handle_); } + + void Logv(const char* format, va_list arguments) override { + // Record the time as close to the Logv() call as possible. + SYSTEMTIME now_components; + ::GetLocalTime(&now_components); + + // Record the thread ID. + constexpr const int kMaxThreadIdSize = 32; + std::ostringstream thread_stream; + thread_stream << std::this_thread::get_id(); + std::string thread_id = thread_stream.str(); + if (thread_id.size() > kMaxThreadIdSize) { + thread_id.resize(kMaxThreadIdSize); + } + + // We first attempt to print into a stack-allocated buffer. If this attempt + // fails, we make a second attempt with a dynamically allocated buffer. + constexpr const int kStackBufferSize = 512; + char stack_buffer[kStackBufferSize]; + static_assert(sizeof(stack_buffer) == static_cast<size_t>(kStackBufferSize), + "sizeof(char) is expected to be 1 in C++"); + + int dynamic_buffer_size = 0; // Computed in the first iteration. + for (int iteration = 0; iteration < 2; ++iteration) { + const int buffer_size = + (iteration == 0) ? kStackBufferSize : dynamic_buffer_size; + char* const buffer = + (iteration == 0) ? stack_buffer : new char[dynamic_buffer_size]; + + // Print the header into the buffer. + // TODO(costan): Sync this logger with another logger. + int buffer_offset = snprintf( + buffer, buffer_size, "%04d/%02d/%02d-%02d:%02d:%02d.%06d %s ", + now_components.wYear, now_components.wMonth, now_components.wDay, + now_components.wHour, now_components.wMinute, now_components.wSecond, + static_cast<int>(now_components.wMilliseconds * 1000), + std::stoull(thread_id)); + + // The header can be at most 28 characters (10 date + 15 time + + // 3 spacing) plus the thread ID, which should fit comfortably into the + // static buffer. + assert(buffer_offset <= 28 + kMaxThreadIdSize); + static_assert(28 + kMaxThreadIdSize < kStackBufferSize, + "stack-allocated buffer may not fit the message header"); + assert(buffer_offset < buffer_size); + + // Print the message into the buffer. + std::va_list arguments_copy; + va_copy(arguments_copy, arguments); + buffer_offset += std::vsnprintf(buffer + buffer_offset, + buffer_size - buffer_offset, format, + arguments_copy); + va_end(arguments_copy); + + // The code below may append a newline at the end of the buffer, which + // requires an extra character. + if (buffer_offset >= buffer_size - 1) { + // The message did not fit into the buffer. + if (iteration == 0) { + // Re-run the loop and use a dynamically-allocated buffer. The buffer + // will be large enough for the log message, an extra newline and a + // null terminator. + dynamic_buffer_size = buffer_offset + 2; + continue; + } + + // The dynamically-allocated buffer was incorrectly sized. This should + // not happen, assuming a correct implementation of (v)snprintf. Fail + // in tests, recover by truncating the log message in production. + assert(false); + buffer_offset = buffer_size - 1; + } + + // Add a newline if necessary. + if (buffer[buffer_offset - 1] != '\n') { + buffer[buffer_offset] = '\n'; + ++buffer_offset; + } + + assert(buffer_offset <= buffer_size); + ::WriteFile(handle_, buffer, buffer_offset, nullptr, nullptr); + + if (iteration != 0) { + delete[] buffer; + } + break; + } + } + + private: + HANDLE handle_; +}; + +} // namespace leveldb + +#endif // STORAGE_LEVELDB_UTIL_WINDOWS_LOGGER_H_ |