Limit the number of read-only files the POSIX Env will have open.

Background compaction can create an unbounded number of
leveldb::RandomAccessFile instances. On 64-bit systems mmap is used and
file descriptors are only used beyond a certain number of mmap's.
32-bit systems to not use mmap at all. leveldb::RandomAccessFile does not
observe Options.max_open_files so compaction could exhaust the file
descriptor limit.

This change uses getrlimit to determine the maximum number of open
files and limits RandomAccessFile to approximately 20% of that value.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=143505556

1 files changed, 42 insertions, 0 deletions

diff --git a/util/env_test.cc b/util/env_test.cc
index b72cb44..0cb1577 100644
--- a/util/env_test.cc
+++ b/util/env_test.cc
@@ -6,10 +6,13 @@
 
 #include "port/port.h"
 #include "util/testharness.h"
+#include "util/env_posix_test_helper.h"
 
 namespace leveldb {
 
 static const int kDelayMicros = 100000;
+static const int kReadOnlyFileLimit = 4;
+static const int kMMapLimit = 4;
 
 class EnvPosixTest {
  private:
@@ -19,6 +22,11 @@ class EnvPosixTest {
  public:
   Env* env_;
   EnvPosixTest() : env_(Env::Default()) { }
+
+  static void SetFileLimits(int read_only_file_limit, int mmap_limit) {
+    EnvPosixTestHelper::SetReadOnlyFDLimit(read_only_file_limit);
+    EnvPosixTestHelper::SetReadOnlyMMapLimit(mmap_limit);
+  }
 };
 
 static void SetBool(void* ptr) {
@@ -97,8 +105,42 @@ TEST(EnvPosixTest, StartThread) {
   ASSERT_EQ(state.val, 3);
 }
 
+TEST(EnvPosixTest, TestOpenOnRead) {
+  // Write some test data to a single file that will be opened |n| times.
+  std::string test_dir;
+  ASSERT_OK(Env::Default()->GetTestDirectory(&test_dir));
+  std::string test_file = test_dir + "/open_on_read.txt";
+
+  FILE* f = fopen(test_file.c_str(), "w");
+  ASSERT_TRUE(f != NULL);
+  const char kFileData[] = "abcdefghijklmnopqrstuvwxyz";
+  fputs(kFileData, f);
+  fclose(f);
+
+  // Open test file some number above the sum of the two limits to force
+  // open-on-read behavior of POSIX Env leveldb::RandomAccessFile.
+  const int kNumFiles = kReadOnlyFileLimit + kMMapLimit + 5;
+  leveldb::RandomAccessFile* files[kNumFiles] = {0};
+  for (int i = 0; i < kNumFiles; i++) {
+    ASSERT_OK(Env::Default()->NewRandomAccessFile(test_file, &files[i]));
+  }
+  char scratch;
+  Slice read_result;
+  for (int i = 0; i < kNumFiles; i++) {
+    ASSERT_OK(files[i]->Read(i, 1, &read_result, &scratch));
+    ASSERT_EQ(kFileData[i], read_result[0]);
+  }
+  for (int i = 0; i < kNumFiles; i++) {
+    delete files[i];
+  }
+  ASSERT_OK(Env::Default()->DeleteFile(test_file));
+}
+
 }  // namespace leveldb
 
 int main(int argc, char** argv) {
+  // All tests currently run with the same read-only file limits.
+  leveldb::EnvPosixTest::SetFileLimits(leveldb::kReadOnlyFileLimit,
+                                       leveldb::kMMapLimit);
   return leveldb::test::RunAllTests();
 }