Set max_open_files based on ulimit

Summary: We should never set max_open_files to be bigger than the system's ulimit. Otherwise we will get "Too many open files" errors. See an example in this Travis run: https://travis-ci.org/facebook/rocksdb/jobs/79591566

Test Plan:
make check

I will also verify that max_max_open_files is reasonable.

Reviewers: anthony, kradhakrishnan, IslamAbdelRahman, sdong

Reviewed By: sdong

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D46551

db/db_impl.cc

@@ -123,7 +123,11 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src) {
 
   // result.max_open_files means an "infinite" open files.
   if (result.max_open_files != -1) {
-    ClipToRange(&result.max_open_files, 20, 1000000);
+    int max_max_open_files = port::GetMaxOpenFiles();
+    if (max_max_open_files == -1) {
+      max_max_open_files = 1000000;
+    }
+    ClipToRange(&result.max_open_files, 20, max_max_open_files);
   }
 
   if (result.info_log == nullptr) {

include/rocksdb/options.h

@@ -815,7 +815,7 @@ struct DBOptions {
   // files opened are always kept open. You can estimate number of files based
   // on target_file_size_base and target_file_size_multiplier for level-based
   // compaction. For universal-style compaction, you can usually set it to -1.
-  // Default: 5000
+  // Default: 5000 or ulimit value of max open files (whichever is smaller)
   int max_open_files;
 
   // If max_open_files is -1, DB will open all files on DB::Open(). You can

port/port_posix.cc

@@ -15,6 +15,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <sys/time.h>
+#include <sys/resource.h>
 #include <unistd.h>
 #include <cstdlib>
 #include "util/logging.h"
@@ -141,5 +142,20 @@ void Crash(const std::string& srcfile, int srcline) {
   kill(getpid(), SIGTERM);
 }
 
+int GetMaxOpenFiles() {
+#if defined(RLIMIT_NOFILE)
+  struct rlimit no_files_limit;
+  if (getrlimit(RLIMIT_NOFILE, &no_files_limit) != 0) {
+    return -1;
+  }
+  // protect against overflow
+  if (no_files_limit.rlim_cur >= std::numeric_limits<int>::max()) {
+    return std::numeric_limits<int>::max();
+  }
+  return static_cast<int>(no_files_limit.rlim_cur);
+#endif
+  return -1;
+}
+
 }  // namespace port
 }  // namespace rocksdb

port/port_posix.h

@@ -153,6 +153,9 @@ extern void InitOnce(OnceType* once, void (*initializer)());
 #define PREFETCH(addr, rw, locality) __builtin_prefetch(addr, rw, locality)
 
 extern void Crash(const std::string& srcfile, int srcline);
+
+extern int GetMaxOpenFiles();
+
 } // namespace port
 } // namespace rocksdb
 

port/win/port_win.cc

@@ -236,6 +236,7 @@ void Crash(const std::string& srcfile, int srcline) {
   abort();
 }
 
+int GetMaxOpenFiles() { return -1; }
 
 }  // namespace port
 }  // namespace rocksdb

port/win/port_win.h

@@ -227,6 +227,7 @@ inline void* pthread_getspecific(pthread_key_t key) {
 // feel space with zeros in case the file is extended.
 int truncate(const char* path, int64_t length);
 void Crash(const std::string& srcfile, int srcline);
+extern int GetMaxOpenFiles();
 
 }  // namespace port