From 854d2363616c38d0a1724e5e50a72c2d95a4b5c7 Mon Sep 17 00:00:00 2001 From: Mayank Agarwal Date: Thu, 3 Oct 2013 14:38:32 -0700 Subject: [PATCH] Add backward compatible option in GetLiveFiles to choose whether to not Flush first Summary: As explained in comments in GetLiveFiles in db.h, this option will cause flush to be skipped in GetLiveFiles because some use-cases use GetSortedWalFiles after GetLiveFiles to generate more complete snapshots. Using GetSortedWalFiles after GetLiveFiles allows us to not Flush in GetLiveFiles first because wals have everything. Note: file deletions will be disabled before calling GLF or GSWF so live logs will not move to archive logs or get delted. Note: Manifest file is truncated to a proper value in GLF, so it will always reply from the proper wal files on a restart Test Plan: make Reviewers: dhruba, haobo Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D13257 --- db/db_filesnapshot.cc | 17 ++++++++++------- db/db_impl.h | 3 ++- db/db_impl_readonly.h | 3 ++- db/db_test.cc | 3 ++- include/rocksdb/db.h | 9 ++++++++- include/utilities/stackable_db.h | 6 +++--- utilities/ttl/db_ttl.cc | 5 +++-- utilities/ttl/db_ttl.h | 3 ++- 8 files changed, 32 insertions(+), 17 deletions(-) diff --git a/db/db_filesnapshot.cc b/db/db_filesnapshot.cc index f574dda3c..e517c17c7 100644 --- a/db/db_filesnapshot.cc +++ b/db/db_filesnapshot.cc @@ -30,16 +30,19 @@ Status DBImpl::EnableFileDeletions() { } Status DBImpl::GetLiveFiles(std::vector& ret, - uint64_t* manifest_file_size) { + uint64_t* manifest_file_size, + bool flush_memtable) { *manifest_file_size = 0; - // flush all dirty data to disk. - Status status = Flush(FlushOptions()); - if (!status.ok()) { - Log(options_.info_log, "Cannot Flush data %s\n", - status.ToString().c_str()); - return status; + if (flush_memtable) { + // flush all dirty data to disk. + Status status = Flush(FlushOptions()); + if (!status.ok()) { + Log(options_.info_log, "Cannot Flush data %s\n", + status.ToString().c_str()); + return status; + } } MutexLock l(&mutex_); diff --git a/db/db_impl.h b/db/db_impl.h index c4509c1da..2ffaf6a40 100644 --- a/db/db_impl.h +++ b/db/db_impl.h @@ -72,7 +72,8 @@ class DBImpl : public DB { virtual Status DisableFileDeletions(); virtual Status EnableFileDeletions(); virtual Status GetLiveFiles(std::vector&, - uint64_t* manifest_file_size); + uint64_t* manifest_file_size, + bool flush_memtable = true); virtual Status GetSortedWalFiles(VectorLogPtr& files); virtual Status DeleteWalFiles(const VectorLogPtr& files); virtual SequenceNumber GetLatestSequenceNumber(); diff --git a/db/db_impl_readonly.h b/db/db_impl_readonly.h index b3ade0b6b..d65876988 100644 --- a/db/db_impl_readonly.h +++ b/db/db_impl_readonly.h @@ -60,7 +60,8 @@ public: return Status::NotSupported("Not supported operation in read only mode."); } virtual Status GetLiveFiles(std::vector&, - uint64_t* manifest_file_size) { + uint64_t* manifest_file_size, + bool flush_memtable = true) { return Status::NotSupported("Not supported operation in read only mode."); } virtual Status Flush(const FlushOptions& options) { diff --git a/db/db_test.cc b/db/db_test.cc index 34a2e444d..8c0dea31a 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -3869,7 +3869,8 @@ class ModelDB: public DB { virtual Status EnableFileDeletions() { return Status::OK(); } - virtual Status GetLiveFiles(std::vector&, uint64_t* size) { + virtual Status GetLiveFiles(std::vector&, uint64_t* size, + bool flush_memtable = true) { return Status::OK(); } diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index 98025955f..16bc4d48d 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -244,8 +244,15 @@ class DB { // manifest file is returned in manifest_file_size. The manifest file is an // ever growing file, but only the portion specified by manifest_file_size is // valid for this snapshot. + // Setting flush_memtable to true does Flush before recording the live files. + // Setting flush_memtable to false is useful when we don't want to wait for + // flush which may have to wait for compaction to complete taking an + // indeterminate time. But this will have to use GetSortedWalFiles after + // GetLiveFiles to compensate for memtables missed in this snapshot due to the + // absence of Flush, by WAL files to recover the database consistently later virtual Status GetLiveFiles(std::vector&, - uint64_t* manifest_file_size) = 0; + uint64_t* manifest_file_size, + bool flush_memtable = true) = 0; // Retrieve the sorted list of all wal files with earliest file first virtual Status GetSortedWalFiles(VectorLogPtr& files) = 0; diff --git a/include/utilities/stackable_db.h b/include/utilities/stackable_db.h index 739207b9c..c92d30dcb 100644 --- a/include/utilities/stackable_db.h +++ b/include/utilities/stackable_db.h @@ -133,9 +133,9 @@ class StackableDB : public DB { return sdb_->EnableFileDeletions(); } - virtual Status GetLiveFiles(std::vector& vec, uint64_t* mfs) - override { - return sdb_->GetLiveFiles(vec, mfs); + virtual Status GetLiveFiles(std::vector& vec, uint64_t* mfs, + bool flush_memtable = true) override { + return sdb_->GetLiveFiles(vec, mfs, flush_memtable); } virtual SequenceNumber GetLatestSequenceNumber() override { diff --git a/utilities/ttl/db_ttl.cc b/utilities/ttl/db_ttl.cc index eb39267f8..14c4ab47c 100644 --- a/utilities/ttl/db_ttl.cc +++ b/utilities/ttl/db_ttl.cc @@ -267,8 +267,9 @@ Status DBWithTTL::EnableFileDeletions() { return db_->EnableFileDeletions(); } -Status DBWithTTL::GetLiveFiles(std::vector& vec, uint64_t* mfs) { - return db_->GetLiveFiles(vec, mfs); +Status DBWithTTL::GetLiveFiles(std::vector& vec, uint64_t* mfs, + bool flush_memtable) { + return db_->GetLiveFiles(vec, mfs, flush_memtable); } SequenceNumber DBWithTTL::GetLatestSequenceNumber() { diff --git a/utilities/ttl/db_ttl.h b/utilities/ttl/db_ttl.h index f11f220cd..63aeec622 100644 --- a/utilities/ttl/db_ttl.h +++ b/utilities/ttl/db_ttl.h @@ -75,7 +75,8 @@ class DBWithTTL : public StackableDB { virtual Status EnableFileDeletions(); - virtual Status GetLiveFiles(std::vector& vec, uint64_t* mfs); + virtual Status GetLiveFiles(std::vector& vec, uint64_t* mfs, + bool flush_memtable = true); virtual Status GetSortedWalFiles(VectorLogPtr& files);