Plugin Registry (#7949)

Summary:
Added a Plugin class to the ObjectRegistry.  Enabled compile-time and program-time addition of plugins to the Registry.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7949

Reviewed By: mrambacher

Differential Revision: D33517674

Pulled By: pdillinger

fbshipit-source-id: c3e3270aab76a489bfa9e85d78cdfca951912557
main
mrambacher 2 years ago committed by Facebook GitHub Bot
parent f241d082b6
commit b7db7eae26
  1. 77
      CMakeLists.txt
  2. 16
      Makefile
  3. 9
      include/rocksdb/utilities/object_registry.h
  4. 13
      util/build_version.cc.in
  5. 47
      utilities/object_registry.cc
  6. 64
      utilities/object_registry_test.cc

@ -182,26 +182,6 @@ else()
endif()
endif()
string(TIMESTAMP TS "%Y-%m-%d %H:%M:%S" UTC)
set(BUILD_DATE "${TS}" CACHE STRING "the time we first built rocksdb")
find_package(Git)
if(GIT_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git")
execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_SHA COMMAND "${GIT_EXECUTABLE}" rev-parse HEAD )
execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" RESULT_VARIABLE GIT_MOD COMMAND "${GIT_EXECUTABLE}" diff-index HEAD --quiet)
execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_DATE COMMAND "${GIT_EXECUTABLE}" log -1 --date=format:"%Y-%m-%d %T" --format="%ad")
execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_TAG RESULT_VARIABLE rv COMMAND "${GIT_EXECUTABLE}" symbolic-ref -q --short HEAD OUTPUT_STRIP_TRAILING_WHITESPACE)
if (rv AND NOT rv EQUAL 0)
execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_TAG COMMAND "${GIT_EXECUTABLE}" describe --tags --exact-match OUTPUT_STRIP_TRAILING_WHITESPACE)
endif()
else()
set(GIT_SHA 0)
set(GIT_MOD 1)
endif()
string(REGEX REPLACE "[^0-9a-fA-F]+" "" GIT_SHA "${GIT_SHA}")
string(REGEX REPLACE "[^0-9: /-]+" "" GIT_DATE "${GIT_DATE}")
option(WITH_MD_LIBRARY "build with MD" ON)
if(WIN32 AND MSVC)
if(WITH_MD_LIBRARY)
@ -211,9 +191,6 @@ if(WIN32 AND MSVC)
endif()
endif()
set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc)
configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY)
if(MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W4 /wd4127 /wd4800 /wd4996 /wd4351 /wd4100 /wd4204 /wd4324")
@ -1019,6 +996,60 @@ else()
set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT})
endif()
set(ROCKSDB_PLUGIN_EXTERNS "")
set(ROCKSDB_PLUGIN_BUILTINS "")
message(STATUS "ROCKSDB PLUGINS TO BUILD ${ROCKSDB_PLUGINS}")
list(APPEND PLUGINS ${ROCKSDB_PLUGINS})
foreach(PLUGIN IN LISTS PLUGINS)
set(PLUGIN_ROOT "${CMAKE_SOURCE_DIR}/plugin/${PLUGIN}/")
message("including rocksb plugin ${PLUGIN_ROOT}")
set(PLUGINMKFILE "${PLUGIN_ROOT}${PLUGIN}.mk")
if (NOT EXISTS ${PLUGINMKFILE})
message(FATAL_ERROR "Missing plugin makefile: ${PLUGINMKFILE}")
endif()
file(READ ${PLUGINMKFILE} PLUGINMK)
string(REGEX MATCH "SOURCES = ([^\n]*)" FOO ${PLUGINMK})
set(MK_SOURCES ${CMAKE_MATCH_1})
separate_arguments(MK_SOURCES)
foreach(MK_FILE IN LISTS MK_SOURCES)
list(APPEND SOURCES "${PLUGIN_ROOT}${MK_FILE}")
endforeach()
string(REGEX MATCH "_FUNC = ([^\n]*)" FOO ${PLUGINMK})
if (NOT ${CMAKE_MATCH_1} STREQUAL "")
string(APPEND ROCKSDB_PLUGIN_BUILTINS "{\"${PLUGIN}\", " ${CMAKE_MATCH_1} "},")
string(APPEND ROCKSDB_PLUGIN_EXTERNS "int " ${CMAKE_MATCH_1} "(ROCKSDB_NAMESPACE::ObjectLibrary&, const std::string&); ")
endif()
string(REGEX MATCH "_LIBS = ([^\n]*)" FOO ${PLUGINMK})
if (NOT ${CMAKE_MATCH_1} STREQUAL "")
list(APPEND THIRDPARTY_LIBS "${CMAKE_MATCH_1}")
endif()
message("THIRDPARTY_LIBS=${THIRDPARTY_LIBS}")
#TODO: We need to set any compile/link-time flags and add any link libraries
endforeach()
string(TIMESTAMP TS "%Y-%m-%d %H:%M:%S" UTC)
set(BUILD_DATE "${TS}" CACHE STRING "the time we first built rocksdb")
find_package(Git)
if(GIT_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git")
execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_SHA COMMAND "${GIT_EXECUTABLE}" rev-parse HEAD )
execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" RESULT_VARIABLE GIT_MOD COMMAND "${GIT_EXECUTABLE}" diff-index HEAD --quiet)
execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_DATE COMMAND "${GIT_EXECUTABLE}" log -1 --date=format:"%Y-%m-%d %T" --format="%ad")
execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_TAG RESULT_VARIABLE rv COMMAND "${GIT_EXECUTABLE}" symbolic-ref -q --short HEAD OUTPUT_STRIP_TRAILING_WHITESPACE)
if (rv AND NOT rv EQUAL 0)
execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_TAG COMMAND "${GIT_EXECUTABLE}" describe --tags --exact-match OUTPUT_STRIP_TRAILING_WHITESPACE)
endif()
else()
set(GIT_SHA 0)
set(GIT_MOD 1)
endif()
string(REGEX REPLACE "[^0-9a-fA-F]+" "" GIT_SHA "${GIT_SHA}")
string(REGEX REPLACE "[^0-9: /-]+" "" GIT_DATE "${GIT_DATE}")
set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc)
configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY)
add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES} ${BUILD_VERSION_CC})
target_link_libraries(${ROCKSDB_STATIC_LIB} PRIVATE
${THIRDPARTY_LIBS} ${SYSTEM_LIBS})

@ -232,14 +232,20 @@ include make_config.mk
ROCKSDB_PLUGIN_MKS = $(foreach plugin, $(ROCKSDB_PLUGINS), plugin/$(plugin)/*.mk)
include $(ROCKSDB_PLUGIN_MKS)
ROCKSDB_PLUGIN_SOURCES = $(foreach plugin, $(ROCKSDB_PLUGINS), $(foreach source, $($(plugin)_SOURCES), plugin/$(plugin)/$(source)))
ROCKSDB_PLUGIN_HEADERS = $(foreach plugin, $(ROCKSDB_PLUGINS), $(foreach header, $($(plugin)_HEADERS), plugin/$(plugin)/$(header)))
ROCKSDB_PLUGIN_PROTO =ROCKSDB_NAMESPACE::ObjectLibrary\&, const std::string\&
ROCKSDB_PLUGIN_SOURCES = $(foreach p, $(ROCKSDB_PLUGINS), $(foreach source, $($(p)_SOURCES), plugin/$(p)/$(source)))
ROCKSDB_PLUGIN_HEADERS = $(foreach p, $(ROCKSDB_PLUGINS), $(foreach header, $($(p)_HEADERS), plugin/$(p)/$(header)))
ROCKSDB_PLUGIN_LIBS = $(foreach p, $(ROCKSDB_PLUGINS), $(foreach lib, $($(p)_LIBS), -l$(lib)))
ROCKSDB_PLUGIN_W_FUNCS = $(foreach p, $(ROCKSDB_PLUGINS), $(if $($(p)_FUNC), $(p)))
ROCKSDB_PLUGIN_EXTERNS = $(foreach p, $(ROCKSDB_PLUGIN_W_FUNCS), int $($(p)_FUNC)($(ROCKSDB_PLUGIN_PROTO));)
ROCKSDB_PLUGIN_BUILTINS = $(foreach p, $(ROCKSDB_PLUGIN_W_FUNCS), {\"$(p)\"\, $($(p)_FUNC)}\,)
ROCKSDB_PLUGIN_LDFLAGS = $(foreach plugin, $(ROCKSDB_PLUGINS), $($(plugin)_LDFLAGS))
ROCKSDB_PLUGIN_PKGCONFIG_REQUIRES = $(foreach plugin, $(ROCKSDB_PLUGINS), $($(plugin)_PKGCONFIG_REQUIRES))
CXXFLAGS += $(foreach plugin, $(ROCKSDB_PLUGINS), $($(plugin)_CXXFLAGS))
PLATFORM_LDFLAGS += $(ROCKSDB_PLUGIN_LDFLAGS)
# Patch up the link flags for JNI from the plugins
ROCKSDB_PLUGIN_LDFLAGS = $(foreach plugin, $(ROCKSDB_PLUGINS), $($(plugin)_LDFLAGS))
PLATFORM_LDFLAGS += $(ROCKSDB_PLUGIN_LDFLAGS)
JAVA_LDFLAGS += $(ROCKSDB_PLUGIN_LDFLAGS)
JAVA_STATIC_LDFLAGS += $(ROCKSDB_PLUGIN_LDFLAGS)
@ -728,7 +734,7 @@ else
git_mod := $(shell git diff-index HEAD --quiet 2>/dev/null; echo $$?)
git_date := $(shell git log -1 --date=format:"%Y-%m-%d %T" --format="%ad" 2>/dev/null)
endif
gen_build_version = sed -e s/@GIT_SHA@/$(git_sha)/ -e s:@GIT_TAG@:"$(git_tag)": -e s/@GIT_MOD@/"$(git_mod)"/ -e s/@BUILD_DATE@/"$(build_date)"/ -e s/@GIT_DATE@/"$(git_date)"/ util/build_version.cc.in
gen_build_version = sed -e s/@GIT_SHA@/$(git_sha)/ -e s:@GIT_TAG@:"$(git_tag)": -e s/@GIT_MOD@/"$(git_mod)"/ -e s/@BUILD_DATE@/"$(build_date)"/ -e s/@GIT_DATE@/"$(git_date)"/ -e s/@ROCKSDB_PLUGIN_BUILTINS@/'$(ROCKSDB_PLUGIN_BUILTINS)'/ -e s/@ROCKSDB_PLUGIN_EXTERNS@/"$(ROCKSDB_PLUGIN_EXTERNS)"/ util/build_version.cc.in
# Record the version of the source that we are compiling.
# We keep a record of the git revision in this file. It is then built

@ -280,9 +280,7 @@ class ObjectRegistry {
static std::shared_ptr<ObjectRegistry> Default();
explicit ObjectRegistry(const std::shared_ptr<ObjectRegistry>& parent)
: parent_(parent) {}
explicit ObjectRegistry(const std::shared_ptr<ObjectLibrary>& library) {
libraries_.push_back(library);
}
explicit ObjectRegistry(const std::shared_ptr<ObjectLibrary>& library);
std::shared_ptr<ObjectLibrary> AddLibrary(const std::string& id) {
auto library = std::make_shared<ObjectLibrary>(id);
@ -502,6 +500,9 @@ class ObjectRegistry {
// Dump the contents of the registry to the logger
void Dump(Logger* logger) const;
// Invokes the input function to retrieve the properties for this plugin.
int RegisterPlugin(const std::string& name, const RegistrarFunc& func);
private:
static std::string ToManagedObjectKey(const std::string& type,
const std::string& id) {
@ -548,6 +549,8 @@ class ObjectRegistry {
// The libraries are searched in reverse order (back to front) when
// searching for entries.
std::vector<std::shared_ptr<ObjectLibrary>> libraries_;
std::vector<std::string> plugins_;
static std::unordered_map<std::string, RegistrarFunc> builtins_;
std::map<std::string, std::weak_ptr<Customizable>> managed_objects_;
std::shared_ptr<ObjectRegistry> parent_;
mutable std::mutex objects_mutex_; // Mutex for managed objects

@ -3,6 +3,7 @@
#include <memory>
#include "rocksdb/version.h"
#include "rocksdb/utilities/object_registry.h"
#include "util/string_util.h"
// The build script may replace these values with real values based
@ -20,6 +21,16 @@ static const std::string rocksdb_build_date = "rocksdb_build_date:@GIT_DATE@";
static const std::string rocksdb_build_date = "rocksdb_build_date:@BUILD_DATE@";
#endif
#ifndef ROCKSDB_LITE
extern "C" {
@ROCKSDB_PLUGIN_EXTERNS@
} // extern "C"
std::unordered_map<std::string, ROCKSDB_NAMESPACE::RegistrarFunc> ROCKSDB_NAMESPACE::ObjectRegistry::builtins_ = {
@ROCKSDB_PLUGIN_BUILTINS@
};
#endif //ROCKSDB_LITE
namespace ROCKSDB_NAMESPACE {
static void AddProperty(std::unordered_map<std::string, std::string> *props, const std::string& name) {
size_t colon = name.find(":");
@ -52,7 +63,7 @@ std::string GetRocksVersionAsString(bool with_patch) {
return version + "." + ToString(ROCKSDB_PATCH);
} else {
return version;
}
}
}
std::string GetRocksBuildInfoAsString(const std::string& program, bool verbose) {

@ -159,16 +159,18 @@ size_t ObjectLibrary::GetFactoryCount(size_t *types) const {
void ObjectLibrary::Dump(Logger *logger) const {
std::unique_lock<std::mutex> lock(mu_);
for (const auto &iter : factories_) {
ROCKS_LOG_HEADER(logger, " Registered factories for type[%s] ",
iter.first.c_str());
bool printed_one = false;
for (const auto &e : iter.second) {
ROCKS_LOG_HEADER(logger, "%c %s", (printed_one) ? ',' : ':', e->Name());
printed_one = true;
if (logger != nullptr && !factories_.empty()) {
ROCKS_LOG_HEADER(logger, " Registered Library: %s\n", id_.c_str());
for (const auto &iter : factories_) {
ROCKS_LOG_HEADER(logger, " Registered factories for type[%s] ",
iter.first.c_str());
bool printed_one = false;
for (const auto &e : iter.second) {
ROCKS_LOG_HEADER(logger, "%c %s", (printed_one) ? ',' : ':', e->Name());
printed_one = true;
}
}
}
ROCKS_LOG_HEADER(logger, "\n");
}
// Returns the Default singleton instance of the ObjectLibrary
@ -181,6 +183,13 @@ std::shared_ptr<ObjectLibrary> &ObjectLibrary::Default() {
return instance;
}
ObjectRegistry::ObjectRegistry(const std::shared_ptr<ObjectLibrary> &library) {
libraries_.push_back(library);
for (const auto &b : builtins_) {
RegisterPlugin(b.first, b.second);
}
}
std::shared_ptr<ObjectRegistry> ObjectRegistry::Default() {
// Use avoid destruction here so the default ObjectRegistry will not be
// statically destroyed and long-lived.
@ -268,8 +277,18 @@ Status ObjectRegistry::ListManagedObjects(
}
void ObjectRegistry::Dump(Logger *logger) const {
{
if (logger != nullptr) {
std::unique_lock<std::mutex> lock(library_mutex_);
if (!plugins_.empty()) {
ROCKS_LOG_HEADER(logger, " Registered Plugins:");
bool printed_one = false;
for (const auto &plugin : plugins_) {
ROCKS_LOG_HEADER(logger, "%s%s", (printed_one) ? ", " : " ",
plugin.c_str());
printed_one = true;
}
ROCKS_LOG_HEADER(logger, "\n");
}
for (auto iter = libraries_.crbegin(); iter != libraries_.crend(); ++iter) {
iter->get()->Dump(logger);
}
@ -279,5 +298,15 @@ void ObjectRegistry::Dump(Logger *logger) const {
}
}
int ObjectRegistry::RegisterPlugin(const std::string &name,
const RegistrarFunc &func) {
if (!name.empty() && func != nullptr) {
plugins_.push_back(name);
return AddLibrary(name)->Register(func, name);
} else {
return -1;
}
}
#endif // ROCKSDB_LITE
} // namespace ROCKSDB_NAMESPACE

@ -7,6 +7,7 @@
#include "rocksdb/utilities/object_registry.h"
#include "rocksdb/convenience.h"
#include "rocksdb/customizable.h"
#include "test_util/testharness.h"
@ -117,23 +118,25 @@ TEST_F(ObjRegistryTest, LocalRegistry) {
ASSERT_NE(env, nullptr);
}
TEST_F(ObjRegistryTest, CheckShared) {
std::shared_ptr<Env> shared;
std::shared_ptr<ObjectRegistry> registry = ObjectRegistry::NewInstance();
std::shared_ptr<ObjectLibrary> library =
std::make_shared<ObjectLibrary>("shared");
registry->AddLibrary(library);
library->AddFactory<Env>(
static int RegisterTestUnguarded(ObjectLibrary& library,
const std::string& /*arg*/) {
library.AddFactory<Env>(
"unguarded",
[](const std::string& /*uri*/, std::unique_ptr<Env>* /*guard */,
std::string* /* errmsg */) { return Env::Default(); });
library->AddFactory<Env>(
library.AddFactory<Env>(
"guarded", [](const std::string& uri, std::unique_ptr<Env>* guard,
std::string* /* errmsg */) {
guard->reset(new WrappedEnv(Env::Default(), uri));
return guard->get();
});
return 2;
}
TEST_F(ObjRegistryTest, CheckShared) {
std::shared_ptr<Env> shared;
std::shared_ptr<ObjectRegistry> registry = ObjectRegistry::NewInstance();
registry->AddLibrary("shared", RegisterTestUnguarded, "");
ASSERT_OK(registry->NewSharedObject<Env>("guarded", &shared));
ASSERT_NE(shared, nullptr);
@ -145,20 +148,7 @@ TEST_F(ObjRegistryTest, CheckShared) {
TEST_F(ObjRegistryTest, CheckStatic) {
Env* env = nullptr;
std::shared_ptr<ObjectRegistry> registry = ObjectRegistry::NewInstance();
std::shared_ptr<ObjectLibrary> library =
std::make_shared<ObjectLibrary>("static");
registry->AddLibrary(library);
library->AddFactory<Env>(
"unguarded",
[](const std::string& /*uri*/, std::unique_ptr<Env>* /*guard */,
std::string* /* errmsg */) { return Env::Default(); });
library->AddFactory<Env>(
"guarded", [](const std::string& uri, std::unique_ptr<Env>* guard,
std::string* /* errmsg */) {
guard->reset(new WrappedEnv(Env::Default(), uri));
return guard->get();
});
registry->AddLibrary("static", RegisterTestUnguarded, "");
ASSERT_NOK(registry->NewStaticObject<Env>("guarded", &env));
ASSERT_EQ(env, nullptr);
@ -170,20 +160,7 @@ TEST_F(ObjRegistryTest, CheckStatic) {
TEST_F(ObjRegistryTest, CheckUnique) {
std::unique_ptr<Env> unique;
std::shared_ptr<ObjectRegistry> registry = ObjectRegistry::NewInstance();
std::shared_ptr<ObjectLibrary> library =
std::make_shared<ObjectLibrary>("unique");
registry->AddLibrary(library);
library->AddFactory<Env>(
"unguarded",
[](const std::string& /*uri*/, std::unique_ptr<Env>* /*guard */,
std::string* /* errmsg */) { return Env::Default(); });
library->AddFactory<Env>(
"guarded", [](const std::string& uri, std::unique_ptr<Env>* guard,
std::string* /* errmsg */) {
guard->reset(new WrappedEnv(Env::Default(), uri));
return guard->get();
});
registry->AddLibrary("unique", RegisterTestUnguarded, "");
ASSERT_OK(registry->NewUniqueObject<Env>("guarded", &unique));
ASSERT_NE(unique, nullptr);
@ -268,6 +245,7 @@ TEST_F(ObjRegistryTest, TestRegistryParents) {
ASSERT_NOK(child->NewUniqueObject<Env>("cousin", &guard));
ASSERT_NOK(uncle->NewUniqueObject<Env>("cousin", &guard));
}
class MyCustomizable : public Customizable {
public:
static const char* Type() { return "MyCustomizable"; }
@ -493,6 +471,18 @@ TEST_F(ObjRegistryTest, TestGetOrCreateManagedObject) {
ASSERT_EQ(2, obj.use_count());
}
TEST_F(ObjRegistryTest, RegisterPlugin) {
std::shared_ptr<ObjectRegistry> registry = ObjectRegistry::NewInstance();
std::unique_ptr<Env> guard;
Env* env = nullptr;
ASSERT_NOK(registry->NewObject<Env>("unguarded", &env, &guard));
ASSERT_EQ(registry->RegisterPlugin("Missing", nullptr), -1);
ASSERT_EQ(registry->RegisterPlugin("", RegisterTestUnguarded), -1);
ASSERT_GT(registry->RegisterPlugin("Valid", RegisterTestUnguarded), 0);
ASSERT_OK(registry->NewObject<Env>("unguarded", &env, &guard));
ASSERT_NE(env, nullptr);
}
class PatternEntryTest : public testing::Test {};
TEST_F(PatternEntryTest, TestSimpleEntry) {

Loading…
Cancel
Save