diff --git a/.gitignore b/.gitignore index e88ccfc00..6364dfdc4 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,7 @@ rocksdb_undump db_test2 trace_analyzer trace_analyzer_test +.DS_Store java/out java/target diff --git a/TARGETS b/TARGETS index a635ed5ac..0cdd3b162 100644 --- a/TARGETS +++ b/TARGETS @@ -30,6 +30,7 @@ ROCKSDB_COMPILER_FLAGS = [ "-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX", "-DROCKSDB_BACKTRACE", "-Wnarrowing", + "-DROCKSDB_NO_DYNAMIC_EXTENSION", ] ROCKSDB_EXTERNAL_DEPS = [ diff --git a/buckifier/targets_cfg.py b/buckifier/targets_cfg.py index 730b5ebf9..79648bb6a 100644 --- a/buckifier/targets_cfg.py +++ b/buckifier/targets_cfg.py @@ -35,6 +35,7 @@ ROCKSDB_COMPILER_FLAGS = [ "-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX", "-DROCKSDB_BACKTRACE", "-Wnarrowing", + "-DROCKSDB_NO_DYNAMIC_EXTENSION", ] ROCKSDB_EXTERNAL_DEPS = [ diff --git a/build_tools/build_detect_platform b/build_tools/build_detect_platform index 7f454bcca..5d42faa30 100755 --- a/build_tools/build_detect_platform +++ b/build_tools/build_detect_platform @@ -602,6 +602,19 @@ EOF fi fi +if [ "$FBCODE_BUILD" != "true" -a "$PLATFORM" = OS_LINUX ]; then + $CXX $COMMON_FLAGS $PLATFORM_SHARED_CFLAGS -x c++ -c - -o test_dl.o 2>/dev/null </dev/null + if [ "$?" = 0 ]; then + EXEC_LDFLAGS+="-ldl" + rm -f test_dl.o + fi + fi +fi + PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS" PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS" diff --git a/env/env_posix.cc b/env/env_posix.cc index 7eb5b7c14..f1a0907c9 100644 --- a/env/env_posix.cc +++ b/env/env_posix.cc @@ -7,8 +7,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors #include +#ifndef ROCKSDB_NO_DYNAMIC_EXTENSION +#include +#endif #include #include + #if defined(OS_LINUX) #include #endif @@ -69,6 +73,17 @@ #endif namespace rocksdb { +#if defined(OS_WIN) +static const std::string kSharedLibExt = ".dll"; +static const char kPathSeparator = ';'; +#else +static const char kPathSeparator = ':'; +#if defined(OS_MACOSX) +static const std::string kSharedLibExt = ".dylib"; +#else +static const std::string kSharedLibExt = ".so"; +#endif +#endif namespace { @@ -115,6 +130,32 @@ int cloexec_flags(int flags, const EnvOptions* options) { return flags; } +#ifndef ROCKSDB_NO_DYNAMIC_EXTENSION +class PosixDynamicLibrary : public DynamicLibrary { + public: + PosixDynamicLibrary(const std::string& name, void* handle) + : name_(name), handle_(handle) {} + ~PosixDynamicLibrary() override { dlclose(handle_); } + + Status LoadSymbol(const std::string& sym_name, FunctionPtr* func) override { + char* err = dlerror(); // Clear any old error + *func = (FunctionPtr)dlsym(handle_, sym_name.c_str()); + if (*func != nullptr) { + return Status::OK(); + } else { + err = dlerror(); + return Status::NotFound("Error finding symbol: " + sym_name, err); + } + } + + const char* Name() const override { return name_.c_str(); } + + private: + std::string name_; + void* handle_; +}; +#endif // !ROCKSDB_NO_DYNAMIC_EXTENSION + class PosixEnv : public Env { public: PosixEnv(); @@ -729,6 +770,64 @@ class PosixEnv : public Env { return result; } +#ifndef ROCKSDB_NO_DYNAMIC_EXTENSION + /** + * Loads the named library into the result. + * If the input name is empty, the current executable is loaded + * On *nix systems, a "lib" prefix is added to the name if one is not supplied + * Comparably, the appropriate shared library extension is added to the name + * if not supplied. If search_path is not specified, the shared library will + * be loaded using the default path (LD_LIBRARY_PATH) If search_path is + * specified, the shared library will be searched for in the directories + * provided by the search path + */ + Status LoadLibrary(const std::string& name, const std::string& path, + std::shared_ptr* result) override { + Status status; + assert(result != nullptr); + if (name.empty()) { + void* hndl = dlopen(NULL, RTLD_NOW); + if (hndl != nullptr) { + result->reset(new PosixDynamicLibrary(name, hndl)); + return Status::OK(); + } + } else { + std::string library_name = name; + if (library_name.find(kSharedLibExt) == std::string::npos) { + library_name = library_name + kSharedLibExt; + } +#if !defined(OS_WIN) + if (library_name.find('/') == std::string::npos && + library_name.compare(0, 3, "lib") != 0) { + library_name = "lib" + library_name; + } +#endif + if (path.empty()) { + void* hndl = dlopen(library_name.c_str(), RTLD_NOW); + if (hndl != nullptr) { + result->reset(new PosixDynamicLibrary(library_name, hndl)); + return Status::OK(); + } + } else { + std::string local_path; + std::stringstream ss(path); + while (getline(ss, local_path, kPathSeparator)) { + if (!path.empty()) { + std::string full_name = local_path + "/" + library_name; + void* hndl = dlopen(full_name.c_str(), RTLD_NOW); + if (hndl != nullptr) { + result->reset(new PosixDynamicLibrary(full_name, hndl)); + return Status::OK(); + } + } + } + } + } + return Status::IOError( + IOErrorMsg("Failed to open shared library: xs", name), dlerror()); + } +#endif // !ROCKSDB_NO_DYNAMIC_EXTENSION + void Schedule(void (*function)(void* arg1), void* arg, Priority pri = LOW, void* tag = nullptr, void (*unschedFunction)(void* arg) = nullptr) override; diff --git a/env/env_test.cc b/env/env_test.cc index e8cb9b245..30d5b5282 100644 --- a/env/env_test.cc +++ b/env/env_test.cc @@ -247,6 +247,52 @@ TEST_F(EnvPosixTest, MemoryMappedFileBuffer) { ASSERT_EQ(expected_data, actual_data); } +#ifndef ROCKSDB_NO_DYNAMIC_EXTENSION +TEST_F(EnvPosixTest, LoadRocksDBLibrary) { + std::shared_ptr library; + std::function function; + Status status = env_->LoadLibrary("no-such-library", "", &library); + ASSERT_NOK(status); + ASSERT_EQ(nullptr, library.get()); + status = env_->LoadLibrary("rocksdb", "", &library); + if (status.ok()) { // If we have can find a rocksdb shared library + ASSERT_NE(nullptr, library.get()); + ASSERT_OK(library->LoadFunction("rocksdb_create_default_env", + &function)); // from C definition + ASSERT_NE(nullptr, function); + ASSERT_NOK(library->LoadFunction("no-such-method", &function)); + ASSERT_EQ(nullptr, function); + ASSERT_OK(env_->LoadLibrary(library->Name(), "", &library)); + } else { + ASSERT_EQ(nullptr, library.get()); + } +} +#endif // !ROCKSDB_NO_DYNAMIC_EXTENSION + +#if !defined(OS_WIN) && !defined(ROCKSDB_NO_DYNAMIC_EXTENSION) +TEST_F(EnvPosixTest, LoadRocksDBLibraryWithSearchPath) { + std::shared_ptr library; + std::function function; + ASSERT_NOK(env_->LoadLibrary("no-such-library", "/tmp", &library)); + ASSERT_EQ(nullptr, library.get()); + ASSERT_NOK(env_->LoadLibrary("dl", "/tmp", &library)); + ASSERT_EQ(nullptr, library.get()); + Status status = env_->LoadLibrary("rocksdb", "/tmp:./", &library); + if (status.ok()) { + ASSERT_NE(nullptr, library.get()); + ASSERT_OK(env_->LoadLibrary(library->Name(), "", &library)); + } + char buff[1024]; + std::string cwd = getcwd(buff, sizeof(buff)); + + status = env_->LoadLibrary("rocksdb", "/tmp:" + cwd, &library); + if (status.ok()) { + ASSERT_NE(nullptr, library.get()); + ASSERT_OK(env_->LoadLibrary(library->Name(), "", &library)); + } +} +#endif // !OS_WIN && !ROCKSDB_NO_DYNAMIC_EXTENSION + TEST_P(EnvPosixTestWithParam, UnSchedule) { std::atomic called(false); env_->SetBackgroundThreads(1, Env::LOW); diff --git a/include/rocksdb/env.h b/include/rocksdb/env.h index 8f6bd6072..a8fe2fb78 100644 --- a/include/rocksdb/env.h +++ b/include/rocksdb/env.h @@ -41,6 +41,7 @@ namespace rocksdb { +class DynamicLibrary; class FileLock; class Logger; class RandomAccessFile; @@ -338,6 +339,18 @@ class Env { // REQUIRES: lock has not already been unlocked. virtual Status UnlockFile(FileLock* lock) = 0; + // Opens `lib_name` as a dynamic library. + // If the 'search_path' is specified, breaks the path into its components + // based on the appropriate platform separator (";" or ";") and looks for the + // library in those directories. If 'search path is not specified, uses the + // default library path search mechanism (such as LD_LIBRARY_PATH). On + // success, stores a dynamic library in `*result`. + virtual Status LoadLibrary(const std::string& /*lib_name*/, + const std::string& /*search_path */, + std::shared_ptr* /*result*/) { + return Status::NotSupported("LoadLibrary is not implemented in this Env"); + } + // Priority for scheduling job in thread pool enum Priority { BOTTOM, LOW, HIGH, USER, TOTAL }; @@ -978,6 +991,29 @@ class FileLock { void operator=(const FileLock&); }; +class DynamicLibrary { + public: + typedef void* (*FunctionPtr)(); + virtual ~DynamicLibrary() {} + + /** Returns the name of the dynamic library */ + virtual const char* Name() const = 0; + + /** + * Loads the symbol for sym_name from the library and updates the input + * function. Returns the loaded symbol + */ + template + Status LoadFunction(const std::string& sym_name, std::function* function) { + FunctionPtr ptr; + Status s = LoadSymbol(sym_name, &ptr); + *function = reinterpret_cast(ptr); + return s; + } + /** Loads and returns the symbol for sym_name from the library */ + virtual Status LoadSymbol(const std::string& sym_name, FunctionPtr* func) = 0; +}; + extern void LogFlush(const std::shared_ptr& info_log); extern void Log(const InfoLogLevel log_level, @@ -1168,6 +1204,12 @@ class EnvWrapper : public Env { Status UnlockFile(FileLock* l) override { return target_->UnlockFile(l); } + Status LoadLibrary(const std::string& lib_name, + const std::string& search_path, + std::shared_ptr* result) override { + return target_->LoadLibrary(lib_name, search_path, result); + } + void Schedule(void (*f)(void* arg), void* a, Priority pri, void* tag = nullptr, void (*u)(void* arg) = nullptr) override { return target_->Schedule(f, a, pri, tag, u);