From cfcf045accbc5d682a02f4acb1192a7f54f05f1f Mon Sep 17 00:00:00 2001 From: Mark Rambacher Date: Tue, 23 Jul 2019 17:08:26 -0700 Subject: [PATCH] =?UTF-8?q?The=20ObjectRegistry=20class=20replaces=20the?= =?UTF-8?q?=20Registrar=20and=20NewCustomObjects.=E2=80=A6=20(#5293)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: The ObjectRegistry class replaces the Registrar and NewCustomObjects. Objects are registered with the registry by Type (the class must implement the static const char *Type() method). This change is necessary for a few reasons: - By having a class (rather than static template instances), the class can be passed between compilation units, meaning that objects could be registered and shared from a dynamic library with an executable. - By having a class with instances, different units could have different objects registered. This could be useful if, for example, one Option allowed for a dynamic library and one did not. When combined with some other PRs (being able to load shared libraries, a Configurable interface to configure objects to/from string), this code will allow objects in external shared libraries to be added to a RocksDB image at run-time, rather than requiring every new extension to be built into the main library and called explicitly by every program. Test plan (on riversand963's devserver) ``` $COMPILE_WITH_ASAN=1 make -j32 all && sleep 1 && make check ``` All tests pass. Pull Request resolved: https://github.com/facebook/rocksdb/pull/5293 Differential Revision: D16363396 Pulled By: riversand963 fbshipit-source-id: fbe4acb615bfc11103eef40a0b288845791c0180 --- CMakeLists.txt | 3 +- HISTORY.md | 1 + TARGETS | 26 ++- env/env.cc | 15 ++ env/env_basic_test.cc | 4 +- include/rocksdb/comparator.h | 1 + include/rocksdb/env.h | 5 + include/rocksdb/merge_operator.h | 1 + include/rocksdb/statistics.h | 2 +- include/rocksdb/utilities/object_registry.h | 225 +++++++++++++++----- options/options_helper.cc | 26 +-- options/options_test.cc | 33 +-- src.mk | 1 + tools/block_cache_trace_analyzer.cc | 2 +- tools/db_bench_tool.cc | 17 +- tools/ldb_cmd.cc | 11 +- utilities/object_registry.cc | 87 ++++++++ utilities/object_registry_test.cc | 137 ++++++++++-- 18 files changed, 465 insertions(+), 132 deletions(-) create mode 100644 utilities/object_registry.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 0bd731149..086975f3e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -520,7 +520,7 @@ set(SOURCES db/flush_job.cc db/flush_scheduler.cc db/forward_iterator.cc - db/import_column_family_job.cc + db/import_column_family_job.cc db/internal_stats.cc db/logs_with_prep_tracker.cc db/log_reader.cc @@ -681,6 +681,7 @@ set(SOURCES utilities/merge_operators/string_append/stringappend.cc utilities/merge_operators/string_append/stringappend2.cc utilities/merge_operators/uint64add.cc + utilities/object_registry.cc utilities/option_change_migration/option_change_migration.cc utilities/options/options_util.cc utilities/persistent_cache/block_cache_tier.cc diff --git a/HISTORY.md b/HISTORY.md index d452a68a3..592053410 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -20,6 +20,7 @@ * Overload GetAllKeyVersions() to support non-default column family. * Added new APIs ExportColumnFamily() and CreateColumnFamilyWithImport() to support export and import of a Column Family. https://github.com/facebook/rocksdb/issues/3469 * ldb sometimes uses a string-append merge operator if no merge operator is passed in. This is to allow users to print keys from a DB with a merge operator. +* Replaces old Registra with ObjectRegistry to allow user to create custom object from string, also add LoadEnv() to Env. ### New Features * Add an option `snap_refresh_nanos` (default to 0) to periodically refresh the snapshot list in compaction jobs. Assign to 0 to disable the feature. diff --git a/TARGETS b/TARGETS index 122da8b54..ba6f96c0b 100644 --- a/TARGETS +++ b/TARGETS @@ -276,6 +276,7 @@ cpp_library( "utilities/merge_operators/string_append/stringappend.cc", "utilities/merge_operators/string_append/stringappend2.cc", "utilities/merge_operators/uint64add.cc", + "utilities/object_registry.cc", "utilities/option_change_migration/option_change_migration.cc", "utilities/options/options_util.cc", "utilities/persistent_cache/block_cache_tier.cc", @@ -371,11 +372,6 @@ ROCKS_TESTS = [ "logging/auto_roll_logger_test.cc", "serial", ], - [ - "env_logger_test", - "logging/env_logger_test.cc", - "serial", - ], [ "autovector_test", "util/autovector_test.cc", @@ -422,13 +418,13 @@ ROCKS_TESTS = [ "serial", ], [ - "cache_test", - "cache/cache_test.cc", + "cache_simulator_test", + "utilities/simulator_cache/cache_simulator_test.cc", "serial", ], [ - "cache_simulator_test", - "utilities/simulator_cache/cache_simulator_test.cc", + "cache_test", + "cache/cache_test.cc", "serial", ], [ @@ -554,7 +550,7 @@ ROCKS_TESTS = [ [ "db_bloom_filter_test", "db/db_bloom_filter_test.cc", - "parallel", + "serial", ], [ "db_compaction_filter_test", @@ -711,6 +707,11 @@ ROCKS_TESTS = [ "env/env_basic_test.cc", "serial", ], + [ + "env_logger_test", + "logging/env_logger_test.cc", + "serial", + ], [ "env_test", "env/env_test.cc", @@ -796,6 +797,11 @@ ROCKS_TESTS = [ "monitoring/histogram_test.cc", "serial", ], + [ + "import_column_family_test", + "db/import_column_family_test.cc", + "parallel", + ], [ "inlineskiplist_test", "memtable/inlineskiplist_test.cc", diff --git a/env/env.cc b/env/env.cc index 87b6b35c1..4c222cfc1 100644 --- a/env/env.cc +++ b/env/env.cc @@ -16,6 +16,7 @@ #include "port/port.h" #include "port/sys_time.h" #include "rocksdb/options.h" +#include "rocksdb/utilities/object_registry.h" #include "util/autovector.h" namespace rocksdb { @@ -28,6 +29,20 @@ Status Env::NewLogger(const std::string& fname, return NewEnvLogger(fname, this, result); } +Status Env::LoadEnv(const std::string& value, Env** result) { + Env* env = *result; + Status s; +#ifndef ROCKSDB_LITE + s = ObjectRegistry::NewInstance()->NewStaticObject(value, &env); +#else + s = Status::NotSupported("Cannot load environment in LITE mode: ", value); +#endif + if (s.ok()) { + *result = env; + } + return s; +} + std::string Env::PriorityToString(Env::Priority priority) { switch (priority) { case Env::Priority::BOTTOM: diff --git a/env/env_basic_test.cc b/env/env_basic_test.cc index f306edbd6..c955bdb71 100644 --- a/env/env_basic_test.cc +++ b/env/env_basic_test.cc @@ -11,7 +11,6 @@ #include "env/mock_env.h" #include "rocksdb/env.h" -#include "rocksdb/utilities/object_registry.h" #include "test_util/testharness.h" namespace rocksdb { @@ -104,13 +103,12 @@ namespace { // ValuesIn() will skip running tests when given an empty collection. std::vector GetCustomEnvs() { static Env* custom_env; - static std::unique_ptr custom_env_guard; static bool init = false; if (!init) { init = true; const char* uri = getenv("TEST_ENV_URI"); if (uri != nullptr) { - custom_env = NewCustomObject(uri, &custom_env_guard); + Env::LoadEnv(uri, &custom_env); } } diff --git a/include/rocksdb/comparator.h b/include/rocksdb/comparator.h index 9f262367d..e30a9d014 100644 --- a/include/rocksdb/comparator.h +++ b/include/rocksdb/comparator.h @@ -35,6 +35,7 @@ class Comparator { virtual ~Comparator() {} + static const char* Type() { return "Comparator"; } // Three-way comparison. Returns value: // < 0 iff "a" < "b", // == 0 iff "a" == "b", diff --git a/include/rocksdb/env.h b/include/rocksdb/env.h index 126f25747..398a7ff51 100644 --- a/include/rocksdb/env.h +++ b/include/rocksdb/env.h @@ -144,6 +144,11 @@ class Env { virtual ~Env(); + static const char* Type() { return "Environment"; } + + // Loads the environment specified by the input value into the result + static Status LoadEnv(const std::string& value, Env** result); + // Return a default environment suitable for the current operating // system. Sophisticated users may wish to provide their own Env // implementation instead of relying on this default environment. diff --git a/include/rocksdb/merge_operator.h b/include/rocksdb/merge_operator.h index d8ddcc6a0..36f47e254 100644 --- a/include/rocksdb/merge_operator.h +++ b/include/rocksdb/merge_operator.h @@ -46,6 +46,7 @@ class Logger; class MergeOperator { public: virtual ~MergeOperator() {} + static const char* Type() { return "MergeOperator"; } // Gives the client a way to express the read -> modify -> write semantics // key: (IN) The key that's associated with this merge operation. diff --git a/include/rocksdb/statistics.h b/include/rocksdb/statistics.h index 653b460cb..a8d01e034 100644 --- a/include/rocksdb/statistics.h +++ b/include/rocksdb/statistics.h @@ -480,7 +480,7 @@ enum StatsLevel : uint8_t { class Statistics { public: virtual ~Statistics() {} - + static const char* Type() { return "Statistics"; } virtual uint64_t getTickerCount(uint32_t tickerType) const = 0; virtual void histogramData(uint32_t type, HistogramData* const data) const = 0; diff --git a/include/rocksdb/utilities/object_registry.h b/include/rocksdb/utilities/object_registry.h index 86a51b92e..d1516079a 100644 --- a/include/rocksdb/utilities/object_registry.h +++ b/include/rocksdb/utilities/object_registry.h @@ -11,80 +11,195 @@ #include #include #include +#include #include - -#include "rocksdb/env.h" +#include "rocksdb/status.h" namespace rocksdb { - -// Creates a new T using the factory function that was registered with a pattern -// that matches the provided "target" string according to std::regex_match. -// -// If no registered functions match, returns nullptr. If multiple functions -// match, the factory function used is unspecified. -// -// Populates res_guard with result pointer if caller is granted ownership. -template -T* NewCustomObject(const std::string& target, std::unique_ptr* res_guard); - +class Logger; // Returns a new T when called with a string. Populates the std::unique_ptr // argument if granting ownership to caller. template -using FactoryFunc = std::function*)>; - -// To register a factory function for a type T, initialize a Registrar object -// with static storage duration. For example: -// -// static Registrar hdfs_reg("hdfs://.*", &CreateHdfsEnv); -// -// Then, calling NewCustomObject("hdfs://some_path", ...) will match the -// regex provided above, so it returns the result of invoking CreateHdfsEnv. -template -class Registrar { +using FactoryFunc = + std::function*, std::string*)>; + +class ObjectLibrary { public: - explicit Registrar(std::string pattern, FactoryFunc factory); -}; + // Base class for an Entry in the Registry. + class Entry { + public: + virtual ~Entry() {} + Entry(const std::string& name) : name_(std::move(name)) {} + + // Checks to see if the target matches this entry + virtual bool matches(const std::string& target) const { + return name_ == target; + } + const std::string& Name() const { return name_; } + + private: + const std::string name_; // The name of the Entry + }; // End class Entry + + // An Entry containing a FactoryFunc for creating new Objects + template + class FactoryEntry : public Entry { + public: + FactoryEntry(const std::string& name, FactoryFunc f) + : Entry(name), pattern_(std::move(name)), factory_(std::move(f)) {} + ~FactoryEntry() override {} + bool matches(const std::string& target) const override { + return std::regex_match(target, pattern_); + } + // Creates a new T object. + T* NewFactoryObject(const std::string& target, std::unique_ptr* guard, + std::string* msg) const { + return factory_(target, guard, msg); + } -// Implementation details follow. + private: + std::regex pattern_; // The pattern for this entry + FactoryFunc factory_; + }; // End class FactoryEntry + public: + // Finds the entry matching the input name and type + const Entry* FindEntry(const std::string& type, + const std::string& name) const; + void Dump(Logger* logger) const; + + // Registers the factory with the library for the pattern. + // If the pattern matches, the factory may be used to create a new object. + template + const FactoryFunc& Register(const std::string& pattern, + const FactoryFunc& factory) { + std::unique_ptr entry(new FactoryEntry(pattern, factory)); + AddEntry(T::Type(), entry); + return factory; + } + // Returns the default ObjectLibrary + static std::shared_ptr& Default(); -namespace internal { + private: + // Adds the input entry to the list for the given type + void AddEntry(const std::string& type, std::unique_ptr& entry); -template -struct RegistryEntry { - std::regex pattern; - FactoryFunc factory; + // ** FactoryFunctions for this loader, organized by type + std::unordered_map>> entries_; }; -template -struct Registry { - static Registry* Get() { - static Registry instance; - return &instance; +// The ObjectRegistry is used to register objects that can be created by a +// name/pattern at run-time where the specific implementation of the object may +// not be known in advance. +class ObjectRegistry { + public: + static std::shared_ptr NewInstance(); + + ObjectRegistry(); + + void AddLibrary(const std::shared_ptr& library) { + libraries_.emplace_back(library); } - std::vector> entries; - private: - Registry() = default; -}; + // Creates a new T using the factory function that was registered with a + // pattern that matches the provided "target" string according to + // std::regex_match. + // + // If no registered functions match, returns nullptr. If multiple functions + // match, the factory function used is unspecified. + // + // Populates res_guard with result pointer if caller is granted ownership. + template + T* NewObject(const std::string& target, std::unique_ptr* guard, + std::string* errmsg) { + guard->reset(); + const auto* basic = FindEntry(T::Type(), target); + if (basic != nullptr) { + const auto* factory = + static_cast*>(basic); + return factory->NewFactoryObject(target, guard, errmsg); + } else { + *errmsg = std::string("Could not load ") + T::Type(); + return nullptr; + } + } + + // Creates a new unique T using the input factory functions. + // Returns OK if a new unique T was successfully created + // Returns NotFound if the type/target could not be created + // Returns InvalidArgument if the factory return an unguarded object + // (meaning it cannot be managed by a unique ptr) + template + Status NewUniqueObject(const std::string& target, + std::unique_ptr* result) { + std::string errmsg; + T* ptr = NewObject(target, result, &errmsg); + if (ptr == nullptr) { + return Status::NotFound(errmsg, target); + } else if (*result) { + return Status::OK(); + } else { + return Status::InvalidArgument(std::string("Cannot make a unique ") + + T::Type() + " from unguarded one ", + target); + } + } -} // namespace internal + // Creates a new shared T using the input factory functions. + // Returns OK if a new shared T was successfully created + // Returns NotFound if the type/target could not be created + // Returns InvalidArgument if the factory return an unguarded object + // (meaning it cannot be managed by a shared ptr) + template + Status NewSharedObject(const std::string& target, + std::shared_ptr* result) { + std::string errmsg; + std::unique_ptr guard; + T* ptr = NewObject(target, &guard, &errmsg); + if (ptr == nullptr) { + return Status::NotFound(errmsg, target); + } else if (guard) { + result->reset(guard.release()); + return Status::OK(); + } else { + return Status::InvalidArgument(std::string("Cannot make a shared ") + + T::Type() + " from unguarded one ", + target); + } + } -template -T* NewCustomObject(const std::string& target, std::unique_ptr* res_guard) { - res_guard->reset(); - for (const auto& entry : internal::Registry::Get()->entries) { - if (std::regex_match(target, entry.pattern)) { - return entry.factory(target, res_guard); + // Creates a new static T using the input factory functions. + // Returns OK if a new static T was successfully created + // Returns NotFound if the type/target could not be created + // Returns InvalidArgument if the factory return a guarded object + // (meaning it is managed by a unique ptr) + template + Status NewStaticObject(const std::string& target, T** result) { + std::string errmsg; + std::unique_ptr guard; + T* ptr = NewObject(target, &guard, &errmsg); + if (ptr == nullptr) { + return Status::NotFound(errmsg, target); + } else if (guard.get()) { + return Status::InvalidArgument(std::string("Cannot make a static ") + + T::Type() + " from a guarded one ", + target); + } else { + *result = ptr; + return Status::OK(); } } - return nullptr; -} -template -Registrar::Registrar(std::string pattern, FactoryFunc factory) { - internal::Registry::Get()->entries.emplace_back(internal::RegistryEntry{ - std::regex(std::move(pattern)), std::move(factory)}); -} + // Dump the contents of the registry to the logger + void Dump(Logger* logger) const; + + private: + const ObjectLibrary::Entry* FindEntry(const std::string& type, + const std::string& name) const; + // The set of libraries to search for factories for this registry. + // The libraries are searched in reverse order (back to front) when + // searching for entries. + std::vector> libraries_; +}; } // namespace rocksdb #endif // ROCKSDB_LITE diff --git a/options/options_helper.cc b/options/options_helper.cc index 922ece3a8..5733ceed4 100644 --- a/options/options_helper.cc +++ b/options/options_helper.cc @@ -1045,21 +1045,21 @@ Status ParseColumnFamilyOption(const std::string& name, } else { if (name == kNameComparator) { // Try to get comparator from object registry first. - std::unique_ptr comp_guard; - const Comparator* comp = - NewCustomObject(value, &comp_guard); // Only support static comparator for now. - if (comp != nullptr && !comp_guard) { - new_options->comparator = comp; + Status status = ObjectRegistry::NewInstance()->NewStaticObject( + value, &new_options->comparator); + if (status.ok()) { + return status; } } else if (name == kNameMergeOperator) { // Try to get merge operator from object registry first. - std::unique_ptr> mo_guard; - std::shared_ptr* mo = - NewCustomObject>(value, &mo_guard); + std::shared_ptr mo; + Status status = + ObjectRegistry::NewInstance()->NewSharedObject( + value, &new_options->merge_operator); // Only support static comparator for now. - if (mo != nullptr) { - new_options->merge_operator = *mo; + if (status.ok()) { + return status; } } @@ -1191,10 +1191,10 @@ Status ParseDBOption(const std::string& name, NewGenericRateLimiter(static_cast(ParseUint64(value)))); } else if (name == kNameEnv) { // Currently `Env` can be deserialized from object registry only. - std::unique_ptr env_guard; - Env* env = NewCustomObject(value, &env_guard); + Env* env = new_options->env; + Status status = Env::LoadEnv(value, &env); // Only support static env for now. - if (env != nullptr && !env_guard) { + if (status.ok()) { new_options->env = env; } } else { diff --git a/options/options_test.cc b/options/options_test.cc index 823a9c1e0..05ea766f6 100644 --- a/options/options_test.cc +++ b/options/options_test.cc @@ -341,11 +341,11 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) { // Comparator from object registry std::string kCompName = "reverse_comp"; - static Registrar test_reg_a( - kCompName, [](const std::string& /*name*/, - std::unique_ptr* /*comparator_guard*/) { - return ReverseBytewiseComparator(); - }); + ObjectLibrary::Default()->Register( + kCompName, + [](const std::string& /*name*/, + std::unique_ptr* /*guard*/, + std::string* /* errmsg */) { return ReverseBytewiseComparator(); }); ASSERT_OK(GetColumnFamilyOptionsFromString( base_cf_opt, "comparator=" + kCompName + ";", &new_cf_opt)); @@ -354,13 +354,12 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) { // MergeOperator from object registry std::unique_ptr bxo(new BytesXOROperator()); std::string kMoName = bxo->Name(); - static Registrar> test_reg_b( - kMoName, [](const std::string& /*name*/, - std::unique_ptr>* - merge_operator_guard) { - merge_operator_guard->reset( - new std::shared_ptr(new BytesXOROperator())); - return merge_operator_guard->get(); + ObjectLibrary::Default()->Register( + kMoName, + [](const std::string& /*name*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new BytesXOROperator()); + return guard->get(); }); ASSERT_OK(GetColumnFamilyOptionsFromString( @@ -770,9 +769,10 @@ TEST_F(OptionsTest, GetOptionsFromStringTest) { explicit CustomEnv(Env* _target) : EnvWrapper(_target) {} }; - static Registrar test_reg_env( + ObjectLibrary::Default()->Register( kCustomEnvName, - [](const std::string& /*name*/, std::unique_ptr* /*env_guard*/) { + [](const std::string& /*name*/, std::unique_ptr* /*env_guard*/, + std::string* /* errmsg */) { static CustomEnv env(Env::Default()); return &env; }); @@ -813,8 +813,9 @@ TEST_F(OptionsTest, GetOptionsFromStringTest) { ASSERT_EQ(new_options.create_if_missing, true); ASSERT_EQ(new_options.max_open_files, 1); ASSERT_TRUE(new_options.rate_limiter.get() != nullptr); - std::unique_ptr env_guard; - ASSERT_EQ(NewCustomObject(kCustomEnvName, &env_guard), new_options.env); + Env* newEnv = new_options.env; + ASSERT_OK(Env::LoadEnv(kCustomEnvName, &newEnv)); + ASSERT_EQ(newEnv, new_options.env); } TEST_F(OptionsTest, DBOptionsSerialization) { diff --git a/src.mk b/src.mk index 0f04fc739..3462a6a58 100644 --- a/src.mk +++ b/src.mk @@ -195,6 +195,7 @@ LIB_SOURCES = \ utilities/merge_operators/string_append/stringappend2.cc \ utilities/merge_operators/uint64add.cc \ utilities/merge_operators/bytesxor.cc \ + utilities/object_registry.cc \ utilities/option_change_migration/option_change_migration.cc \ utilities/options/options_util.cc \ utilities/persistent_cache/block_cache_tier.cc \ diff --git a/tools/block_cache_trace_analyzer.cc b/tools/block_cache_trace_analyzer.cc index 08143ebcf..761395a66 100644 --- a/tools/block_cache_trace_analyzer.cc +++ b/tools/block_cache_trace_analyzer.cc @@ -1637,7 +1637,7 @@ void BlockCacheTraceAnalyzer::PrintAccessCountStats(bool user_access_only, } fprintf(stdout, "Bottom %" PRIu32 " access count. Access count=%" PRIu64 - " nblocks=%" PRIu64 " %s\n", + " nblocks=%" ROCKSDB_PRIszt " %s\n", bottom_k, naccess_it->first, naccess_it->second.size(), statistics.c_str()); } diff --git a/tools/db_bench_tool.cc b/tools/db_bench_tool.cc index 39f9eebc7..f6a9d9458 100644 --- a/tools/db_bench_tool.cc +++ b/tools/db_bench_tool.cc @@ -3049,8 +3049,9 @@ class Benchmark { std::shared_ptr timestamp_emulator_; std::unique_ptr secondary_update_thread_; std::atomic secondary_update_stopped_{0}; +#ifndef ROCKSDB_LITE uint64_t secondary_db_updates_ = 0; - +#endif // ROCKSDB_LITE struct ThreadArg { Benchmark* bm; SharedState* shared; @@ -6366,13 +6367,12 @@ int db_bench_tool(int argc, char** argv) { exit(1); } if (!FLAGS_statistics_string.empty()) { - std::unique_ptr custom_stats_guard; - dbstats.reset(NewCustomObject(FLAGS_statistics_string, - &custom_stats_guard)); - custom_stats_guard.release(); + Status s = ObjectRegistry::NewInstance()->NewSharedObject( + FLAGS_statistics_string, &dbstats); if (dbstats == nullptr) { - fprintf(stderr, "No Statistics registered matching string: %s\n", - FLAGS_statistics_string.c_str()); + fprintf(stderr, + "No Statistics registered matching string: %s status=%s\n", + FLAGS_statistics_string.c_str(), s.ToString().c_str()); exit(1); } } @@ -6400,12 +6400,11 @@ int db_bench_tool(int argc, char** argv) { StringToCompressionType(FLAGS_compression_type.c_str()); #ifndef ROCKSDB_LITE - std::unique_ptr custom_env_guard; if (!FLAGS_hdfs.empty() && !FLAGS_env_uri.empty()) { fprintf(stderr, "Cannot provide both --hdfs and --env_uri.\n"); exit(1); } else if (!FLAGS_env_uri.empty()) { - FLAGS_env = NewCustomObject(FLAGS_env_uri, &custom_env_guard); + Status s = Env::LoadEnv(FLAGS_env_uri, &FLAGS_env); if (FLAGS_env == nullptr) { fprintf(stderr, "No Env registered for URI: %s\n", FLAGS_env_uri.c_str()); exit(1); diff --git a/tools/ldb_cmd.cc b/tools/ldb_cmd.cc index 338f09fb9..86dfcc54e 100644 --- a/tools/ldb_cmd.cc +++ b/tools/ldb_cmd.cc @@ -20,7 +20,6 @@ #include "rocksdb/utilities/backupable_db.h" #include "rocksdb/utilities/checkpoint.h" #include "rocksdb/utilities/debug.h" -#include "rocksdb/utilities/object_registry.h" #include "rocksdb/utilities/options_util.h" #include "rocksdb/write_batch.h" #include "rocksdb/write_buffer_manager.h" @@ -2854,8 +2853,9 @@ void BackupCommand::DoCommand() { return; } printf("open db OK\n"); - std::unique_ptr custom_env_guard; - Env* custom_env = NewCustomObject(backup_env_uri_, &custom_env_guard); + Env* custom_env = nullptr; + Env::LoadEnv(backup_env_uri_, &custom_env); + BackupableDBOptions backup_options = BackupableDBOptions(backup_dir_, custom_env); backup_options.info_log = logger_.get(); @@ -2889,8 +2889,9 @@ void RestoreCommand::Help(std::string& ret) { } void RestoreCommand::DoCommand() { - std::unique_ptr custom_env_guard; - Env* custom_env = NewCustomObject(backup_env_uri_, &custom_env_guard); + Env* custom_env = nullptr; + Env::LoadEnv(backup_env_uri_, &custom_env); + std::unique_ptr restore_engine; Status status; { diff --git a/utilities/object_registry.cc b/utilities/object_registry.cc new file mode 100644 index 000000000..3706e791e --- /dev/null +++ b/utilities/object_registry.cc @@ -0,0 +1,87 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include "rocksdb/utilities/object_registry.h" + +#include "logging/logging.h" +#include "rocksdb/env.h" + +namespace rocksdb { +#ifndef ROCKSDB_LITE +// Looks through the "type" factories for one that matches "name". +// If found, returns the pointer to the Entry matching this name. +// Otherwise, nullptr is returned +const ObjectLibrary::Entry *ObjectLibrary::FindEntry( + const std::string &type, const std::string &name) const { + auto entries = entries_.find(type); + if (entries != entries_.end()) { + for (const auto &entry : entries->second) { + if (entry->matches(name)) { + return entry.get(); + } + } + } + return nullptr; +} + +void ObjectLibrary::AddEntry(const std::string &type, + std::unique_ptr &entry) { + auto &entries = entries_[type]; + entries.emplace_back(std::move(entry)); +} + +void ObjectLibrary::Dump(Logger *logger) const { + for (const auto &iter : entries_) { + ROCKS_LOG_HEADER(logger, " Registered factories for type[%s] ", + iter.first.c_str()); + bool printed_one = false; + for (const auto &e : iter.second) { + ROCKS_LOG_HEADER(logger, "%c %s", (printed_one) ? ',' : ':', + e->Name().c_str()); + printed_one = true; + } + } + ROCKS_LOG_HEADER(logger, "\n"); +} + +// Returns the Default singleton instance of the ObjectLibrary +// This instance will contain most of the "standard" registered objects +std::shared_ptr &ObjectLibrary::Default() { + static std::shared_ptr instance = + std::make_shared(); + return instance; +} + +std::shared_ptr ObjectRegistry::NewInstance() { + std::shared_ptr instance = std::make_shared(); + return instance; +} + +ObjectRegistry::ObjectRegistry() { + libraries_.push_back(ObjectLibrary::Default()); +} + +// Searches (from back to front) the libraries looking for the +// an entry that matches this pattern. +// Returns the entry if it is found, and nullptr otherwise +const ObjectLibrary::Entry *ObjectRegistry::FindEntry( + const std::string &type, const std::string &name) const { + for (auto iter = libraries_.crbegin(); iter != libraries_.crend(); ++iter) { + const auto *entry = iter->get()->FindEntry(type, name); + if (entry != nullptr) { + return entry; + } + } + return nullptr; +} + +void ObjectRegistry::Dump(Logger *logger) const { + for (auto iter = libraries_.crbegin(); iter != libraries_.crend(); ++iter) { + iter->get()->Dump(logger); + } +} + +#endif // ROCKSDB_LITE +} // namespace rocksdb diff --git a/utilities/object_registry_test.cc b/utilities/object_registry_test.cc index cc7c38d8a..826931845 100644 --- a/utilities/object_registry_test.cc +++ b/utilities/object_registry_test.cc @@ -17,44 +17,145 @@ class EnvRegistryTest : public testing::Test { int EnvRegistryTest::num_a = 0; int EnvRegistryTest::num_b = 0; +static FactoryFunc test_reg_a = ObjectLibrary::Default()->Register( + "a://.*", + [](const std::string& /*uri*/, std::unique_ptr* /*env_guard*/, + std::string* /* errmsg */) { + ++EnvRegistryTest::num_a; + return Env::Default(); + }); -static Registrar test_reg_a("a://.*", - [](const std::string& /*uri*/, - std::unique_ptr* /*env_guard*/) { - ++EnvRegistryTest::num_a; - return Env::Default(); - }); - -static Registrar test_reg_b("b://.*", [](const std::string& /*uri*/, - std::unique_ptr* env_guard) { - ++EnvRegistryTest::num_b; - // Env::Default() is a singleton so we can't grant ownership directly to the - // caller - we must wrap it first. - env_guard->reset(new EnvWrapper(Env::Default())); - return env_guard->get(); -}); +static FactoryFunc test_reg_b = ObjectLibrary::Default()->Register( + "b://.*", [](const std::string& /*uri*/, std::unique_ptr* env_guard, + std::string* /* errmsg */) { + ++EnvRegistryTest::num_b; + // Env::Default() is a singleton so we can't grant ownership directly to + // the caller - we must wrap it first. + env_guard->reset(new EnvWrapper(Env::Default())); + return env_guard->get(); + }); TEST_F(EnvRegistryTest, Basics) { + std::string msg; std::unique_ptr env_guard; - auto res = NewCustomObject("a://test", &env_guard); + auto registry = ObjectRegistry::NewInstance(); + auto res = registry->NewObject("a://test", &env_guard, &msg); ASSERT_NE(res, nullptr); ASSERT_EQ(env_guard, nullptr); ASSERT_EQ(1, num_a); ASSERT_EQ(0, num_b); - res = NewCustomObject("b://test", &env_guard); + res = registry->NewObject("b://test", &env_guard, &msg); ASSERT_NE(res, nullptr); ASSERT_NE(env_guard, nullptr); ASSERT_EQ(1, num_a); ASSERT_EQ(1, num_b); - res = NewCustomObject("c://test", &env_guard); + res = registry->NewObject("c://test", &env_guard, &msg); ASSERT_EQ(res, nullptr); ASSERT_EQ(env_guard, nullptr); ASSERT_EQ(1, num_a); ASSERT_EQ(1, num_b); } +TEST_F(EnvRegistryTest, LocalRegistry) { + std::string msg; + std::unique_ptr guard; + auto registry = ObjectRegistry::NewInstance(); + std::shared_ptr library = std::make_shared(); + registry->AddLibrary(library); + library->Register( + "test-local", + [](const std::string& /*uri*/, std::unique_ptr* /*guard */, + std::string* /* errmsg */) { return Env::Default(); }); + + ObjectLibrary::Default()->Register( + "test-global", + [](const std::string& /*uri*/, std::unique_ptr* /*guard */, + std::string* /* errmsg */) { return Env::Default(); }); + + ASSERT_EQ( + ObjectRegistry::NewInstance()->NewObject("test-local", &guard, &msg), + nullptr); + ASSERT_NE( + ObjectRegistry::NewInstance()->NewObject("test-global", &guard, &msg), + nullptr); + ASSERT_NE(registry->NewObject("test-local", &guard, &msg), nullptr); + ASSERT_NE(registry->NewObject("test-global", &guard, &msg), nullptr); +} + +TEST_F(EnvRegistryTest, CheckShared) { + std::shared_ptr shared; + std::shared_ptr registry = ObjectRegistry::NewInstance(); + std::shared_ptr library = std::make_shared(); + registry->AddLibrary(library); + library->Register( + "unguarded", + [](const std::string& /*uri*/, std::unique_ptr* /*guard */, + std::string* /* errmsg */) { return Env::Default(); }); + + library->Register( + "guarded", [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new EnvWrapper(Env::Default())); + return guard->get(); + }); + + ASSERT_OK(registry->NewSharedObject("guarded", &shared)); + ASSERT_NE(shared, nullptr); + shared.reset(); + ASSERT_NOK(registry->NewSharedObject("unguarded", &shared)); + ASSERT_EQ(shared, nullptr); +} + +TEST_F(EnvRegistryTest, CheckStatic) { + Env* env = nullptr; + std::shared_ptr registry = ObjectRegistry::NewInstance(); + std::shared_ptr library = std::make_shared(); + registry->AddLibrary(library); + library->Register( + "unguarded", + [](const std::string& /*uri*/, std::unique_ptr* /*guard */, + std::string* /* errmsg */) { return Env::Default(); }); + + library->Register( + "guarded", [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new EnvWrapper(Env::Default())); + return guard->get(); + }); + + ASSERT_NOK(registry->NewStaticObject("guarded", &env)); + ASSERT_EQ(env, nullptr); + env = nullptr; + ASSERT_OK(registry->NewStaticObject("unguarded", &env)); + ASSERT_NE(env, nullptr); +} + +TEST_F(EnvRegistryTest, CheckUnique) { + std::unique_ptr unique; + std::shared_ptr registry = ObjectRegistry::NewInstance(); + std::shared_ptr library = std::make_shared(); + registry->AddLibrary(library); + library->Register( + "unguarded", + [](const std::string& /*uri*/, std::unique_ptr* /*guard */, + std::string* /* errmsg */) { return Env::Default(); }); + + library->Register( + "guarded", [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new EnvWrapper(Env::Default())); + return guard->get(); + }); + + ASSERT_OK(registry->NewUniqueObject("guarded", &unique)); + ASSERT_NE(unique, nullptr); + unique.reset(); + ASSERT_NOK(registry->NewUniqueObject("unguarded", &unique)); + ASSERT_EQ(unique, nullptr); +} + } // namespace rocksdb int main(int argc, char** argv) {