From aa0367aabbb2ee891a4f7674351d8b10875670fa Mon Sep 17 00:00:00 2001 From: sdong Date: Tue, 9 Jul 2019 12:46:01 -0700 Subject: [PATCH] Allow ldb to open DB as secondary (#5537) Summary: Right now ldb can open running DB through read-only DB. However, it might leave info logs files to the read-only DB directory. Add an option to open the DB as secondary to avoid it. Pull Request resolved: https://github.com/facebook/rocksdb/pull/5537 Test Plan: Run ./ldb scan --max_keys=10 --db=/tmp/rocksdbtest-2491/dbbench --secondary_path=/tmp --no_value --hex and ./ldb get 0x00000000000000103030303030303030 --hex --db=/tmp/rocksdbtest-2491/dbbench --secondary_path=/tmp against a normal db_bench run and observe the output changes. Also observe that no new info logs files are created under /tmp/rocksdbtest-2491/dbbench. Run without --secondary_path and observe that new info logs created under /tmp/rocksdbtest-2491/dbbench. Differential Revision: D16113886 fbshipit-source-id: 4e09dec47c2528f6ca08a9e7a7894ba2d9daebbb --- HISTORY.md | 1 + include/rocksdb/utilities/ldb_cmd.h | 5 +++++ tools/ldb_cmd.cc | 29 +++++++++++++++++++++++++---- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index d7eb51160..099c9f37e 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -22,6 +22,7 @@ * Add an option `unordered_write` which trades snapshot guarantees with higher write throughput. When used with WRITE_PREPARED transactions with two_write_queues=true, it offers higher throughput with however no compromise on guarantees. * Allow DBImplSecondary to remove memtables with obsolete data after replaying MANIFEST and WAL. * Add an option `failed_move_fall_back_to_copy` (default is true) for external SST ingestion. When `move_files` is true and hard link fails, ingestion falls back to copy if `failed_move_fall_back_to_copy` is true. Otherwise, ingestion reports an error. +* Add argument `--secondary_path` to ldb to open the database as the secondary instance. This would keep the original DB intact. ### Performance Improvements * Reduce binary search when iterator reseek into the same data block. diff --git a/include/rocksdb/utilities/ldb_cmd.h b/include/rocksdb/utilities/ldb_cmd.h index 57ab88a34..e7000742d 100644 --- a/include/rocksdb/utilities/ldb_cmd.h +++ b/include/rocksdb/utilities/ldb_cmd.h @@ -31,6 +31,7 @@ class LDBCommand { // Command-line arguments static const std::string ARG_DB; static const std::string ARG_PATH; + static const std::string ARG_SECONDARY_PATH; static const std::string ARG_HEX; static const std::string ARG_KEY_HEX; static const std::string ARG_VALUE_HEX; @@ -128,6 +129,10 @@ class LDBCommand { protected: LDBCommandExecuteResult exec_state_; std::string db_path_; + // If empty, open DB as primary. If non-empty, open the DB as secondary + // with this secondary path. When running against a database opened by + // another process, ldb wll leave the source directory completely intact. + std::string secondary_path_; std::string column_family_name_; DB* db_; DBWithTTL* db_ttl_; diff --git a/tools/ldb_cmd.cc b/tools/ldb_cmd.cc index fba32d9d6..8f4258cf3 100644 --- a/tools/ldb_cmd.cc +++ b/tools/ldb_cmd.cc @@ -47,6 +47,7 @@ namespace rocksdb { const std::string LDBCommand::ARG_DB = "db"; const std::string LDBCommand::ARG_PATH = "path"; +const std::string LDBCommand::ARG_SECONDARY_PATH = "secondary_path"; const std::string LDBCommand::ARG_HEX = "hex"; const std::string LDBCommand::ARG_KEY_HEX = "key_hex"; const std::string LDBCommand::ARG_VALUE_HEX = "value_hex"; @@ -321,6 +322,12 @@ LDBCommand::LDBCommand(const std::map& options, column_family_name_ = kDefaultColumnFamilyName; } + itr = options.find(ARG_SECONDARY_PATH); + secondary_path_ = ""; + if (itr != options.end()) { + secondary_path_ = itr->second; + } + is_key_hex_ = IsKeyHex(options, flags); is_value_hex_ = IsValueHex(options, flags); is_db_ttl_ = IsFlagPresent(flags, ARG_TTL); @@ -360,6 +367,10 @@ void LDBCommand::OpenDB() { exec_state_ = LDBCommandExecuteResult::Failed( "ldb doesn't support TTL DB with multiple column families"); } + if (!secondary_path_.empty()) { + exec_state_ = LDBCommandExecuteResult::Failed( + "Open as secondary is not supported for TTL DB yet."); + } if (is_read_only_) { st = DBWithTTL::Open(options_, db_path_, &db_ttl_, 0, true); } else { @@ -382,7 +393,7 @@ void LDBCommand::OpenDB() { } } } - if (is_read_only_) { + if (is_read_only_ && secondary_path_.empty()) { if (column_families_.empty()) { st = DB::OpenForReadOnly(options_, db_path_, &db_); } else { @@ -391,10 +402,19 @@ void LDBCommand::OpenDB() { } } else { if (column_families_.empty()) { - st = DB::Open(options_, db_path_, &db_); + if (secondary_path_.empty()) { + st = DB::Open(options_, db_path_, &db_); + } else { + st = DB::OpenAsSecondary(options_, db_path_, secondary_path_, &db_); + } } else { - st = DB::Open(options_, db_path_, column_families_, &handles_opened, - &db_); + if (secondary_path_.empty()) { + st = DB::Open(options_, db_path_, column_families_, &handles_opened, + &db_); + } else { + st = DB::OpenAsSecondary(options_, db_path_, secondary_path_, + column_families_, &handles_opened, &db_); + } } } } @@ -452,6 +472,7 @@ ColumnFamilyHandle* LDBCommand::GetCfHandle() { std::vector LDBCommand::BuildCmdLineOptions( std::vector options) { std::vector ret = {ARG_DB, + ARG_SECONDARY_PATH, ARG_BLOOM_BITS, ARG_BLOCK_SIZE, ARG_AUTO_COMPACTION,