From f307479ba6979d5ea5d2175321aaa41ce4c18536 Mon Sep 17 00:00:00 2001 From: Yanqin Jin Date: Fri, 7 Dec 2018 17:03:49 -0800 Subject: [PATCH] Enable checkpoint of read-only db (#4681) Summary: 1. DBImplReadOnly::GetLiveFiles should not return NotSupported. Instead, it should call DBImpl::GetLiveFiles(flush_memtable=false). 2. In DBImp::Recover, we should also recover the OPTIONS file name and/or number so that an immediate subsequent GetLiveFiles will get the correct OPTIONS name. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4681 Differential Revision: D13069205 Pulled By: riversand963 fbshipit-source-id: 3e6a0174307d06db5a01feb099b306cea1f7f88a --- HISTORY.md | 1 + db/compacted_db_impl.h | 9 +-- db/db_impl_open.cc | 22 ++++++++ db/db_impl_readonly.h | 9 +-- utilities/checkpoint/checkpoint_test.cc | 73 +++++++++++++++++++++++++ 5 files changed, 106 insertions(+), 8 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index ecd63516f..0505514e9 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,7 @@ # Rocksdb Change Log ## Unreleased ### New Features +* Enabled checkpoint on readonly db (DBImplReadOnly). ### Public API Change * Transaction::GetForUpdate is extended with a do_validate parameter with default value of true. If false it skips validating the snapshot before doing the read. Similarly ::Merge, ::Put, ::Delete, and ::SingleDelete are extended with assume_tracked with default value of false. If true it indicates that call is assumed to be after a ::GetForUpdate. diff --git a/db/compacted_db_impl.h b/db/compacted_db_impl.h index 736002e1e..5c574b4b9 100644 --- a/db/compacted_db_impl.h +++ b/db/compacted_db_impl.h @@ -67,10 +67,11 @@ class CompactedDBImpl : public DBImpl { virtual Status EnableFileDeletions(bool /*force*/) override { return Status::NotSupported("Not supported in compacted db mode."); } - virtual Status GetLiveFiles(std::vector&, - uint64_t* /*manifest_file_size*/, - bool /*flush_memtable*/ = true) override { - return Status::NotSupported("Not supported in compacted db mode."); + virtual Status GetLiveFiles(std::vector& ret, + uint64_t* manifest_file_size, + bool /*flush_memtable*/) override { + return DBImpl::GetLiveFiles(ret, manifest_file_size, + false /* flush_memtable */); } using DBImpl::Flush; virtual Status Flush(const FlushOptions& /*options*/, diff --git a/db/db_impl_open.cc b/db/db_impl_open.cc index 24e649973..e3cc79713 100644 --- a/db/db_impl_open.cc +++ b/db/db_impl_open.cc @@ -479,6 +479,28 @@ Status DBImpl::Recover( } } + if (read_only) { + // If we are opening as read-only, we need to update options_file_number_ + // to reflect the most recent OPTIONS file. It does not matter for regular + // read-write db instance because options_file_number_ will later be + // updated to versions_->NewFileNumber() in RenameTempFileToOptionsFile. + std::vector file_names; + if (s.ok()) { + s = env_->GetChildren(GetName(), &file_names); + } + if (s.ok()) { + uint64_t number = 0; + uint64_t options_file_number = 0; + FileType type; + for (const auto& fname : file_names) { + if (ParseFileName(fname, &number, &type) && type == kOptionsFile) { + options_file_number = std::max(number, options_file_number); + } + } + versions_->options_file_number_ = options_file_number; + } + } + return s; } diff --git a/db/db_impl_readonly.h b/db/db_impl_readonly.h index 6ebe1bce7..2d77dbac0 100644 --- a/db/db_impl_readonly.h +++ b/db/db_impl_readonly.h @@ -89,10 +89,11 @@ class DBImplReadOnly : public DBImpl { virtual Status EnableFileDeletions(bool /*force*/) override { return Status::NotSupported("Not supported operation in read only mode."); } - virtual Status GetLiveFiles(std::vector&, - uint64_t* /*manifest_file_size*/, - bool /*flush_memtable*/ = true) override { - return Status::NotSupported("Not supported operation in read only mode."); + virtual Status GetLiveFiles(std::vector& ret, + uint64_t* manifest_file_size, + bool /*flush_memtable*/) override { + return DBImpl::GetLiveFiles(ret, manifest_file_size, + false /* flush_memtable */); } using DBImpl::Flush; diff --git a/utilities/checkpoint/checkpoint_test.cc b/utilities/checkpoint/checkpoint_test.cc index 62c78faa8..b8436ccf5 100644 --- a/utilities/checkpoint/checkpoint_test.cc +++ b/utilities/checkpoint/checkpoint_test.cc @@ -164,6 +164,16 @@ class CheckpointTest : public testing::Test { return DB::OpenForReadOnly(options, dbname_, &db_); } + Status ReadOnlyReopenWithColumnFamilies(const std::vector& cfs, + const Options& options) { + std::vector column_families; + for (const auto& cf : cfs) { + column_families.emplace_back(cf, options); + } + return DB::OpenForReadOnly(options, dbname_, column_families, &handles_, + &db_); + } + Status TryReopen(const Options& options) { Close(); last_options_ = options; @@ -612,6 +622,69 @@ TEST_F(CheckpointTest, CheckpointWithUnsyncedDataDropped) { db_ = nullptr; } +TEST_F(CheckpointTest, CheckpointReadOnlyDB) { + ASSERT_OK(Put("foo", "foo_value")); + ASSERT_OK(Flush()); + Close(); + Options options = CurrentOptions(); + ASSERT_OK(ReadOnlyReopen(options)); + Checkpoint* checkpoint = nullptr; + ASSERT_OK(Checkpoint::Create(db_, &checkpoint)); + ASSERT_OK(checkpoint->CreateCheckpoint(snapshot_name_)); + delete checkpoint; + checkpoint = nullptr; + Close(); + DB* snapshot_db = nullptr; + ASSERT_OK(DB::Open(options, snapshot_name_, &snapshot_db)); + ReadOptions read_opts; + std::string get_result; + ASSERT_OK(snapshot_db->Get(read_opts, "foo", &get_result)); + ASSERT_EQ("foo_value", get_result); + delete snapshot_db; +} + +TEST_F(CheckpointTest, CheckpointReadOnlyDBWithMultipleColumnFamilies) { + Options options = CurrentOptions(); + CreateAndReopenWithCF({"pikachu", "eevee"}, options); + for (int i = 0; i != 3; ++i) { + ASSERT_OK(Put(i, "foo", "foo_value")); + ASSERT_OK(Flush(i)); + } + Close(); + Status s = ReadOnlyReopenWithColumnFamilies( + {kDefaultColumnFamilyName, "pikachu", "eevee"}, options); + ASSERT_OK(s); + Checkpoint* checkpoint = nullptr; + ASSERT_OK(Checkpoint::Create(db_, &checkpoint)); + ASSERT_OK(checkpoint->CreateCheckpoint(snapshot_name_)); + delete checkpoint; + checkpoint = nullptr; + Close(); + + std::vector column_families{ + {kDefaultColumnFamilyName, options}, + {"pikachu", options}, + {"eevee", options}}; + DB* snapshot_db = nullptr; + std::vector snapshot_handles; + s = DB::Open(options, snapshot_name_, column_families, &snapshot_handles, + &snapshot_db); + ASSERT_OK(s); + ReadOptions read_opts; + for (int i = 0; i != 3; ++i) { + std::string get_result; + s = snapshot_db->Get(read_opts, snapshot_handles[i], "foo", &get_result); + ASSERT_OK(s); + ASSERT_EQ("foo_value", get_result); + } + + for (auto snapshot_h : snapshot_handles) { + delete snapshot_h; + } + snapshot_handles.clear(); + delete snapshot_db; +} + } // namespace rocksdb int main(int argc, char** argv) {