From e840213d6eaee8e9f70fb99725a1ea6fe0547824 Mon Sep 17 00:00:00 2001 From: Vitaliy Liptchinsky Date: Fri, 20 Jan 2017 09:26:07 -0800 Subject: [PATCH] Change DB::GetApproximateSizes for more flexibility needed for MyRocks Summary: Added an option to GetApproximateSizes to exclude file stats, as MyRocks has those counted exactly and we need only stats from memtables. Closes https://github.com/facebook/rocksdb/pull/1787 Differential Revision: D4441111 Pulled By: IslamAbdelRahman fbshipit-source-id: c11f4c3 --- HISTORY.md | 1 + db/db_impl.cc | 10 ++++-- db/db_impl.h | 3 +- db/db_test.cc | 29 +++++++++------- include/rocksdb/db.h | 44 ++++++++++++++++++++---- include/rocksdb/utilities/stackable_db.h | 5 +-- 6 files changed, 67 insertions(+), 25 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 81f5080bd..ac5c19155 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -4,6 +4,7 @@ * Support dynamically change `delete_obsolete_files_period_micros` option via SetDBOptions(). * Added EventListener::OnExternalFileIngested which will be called when IngestExternalFile() add a file successfully. * BackupEngine::Open and BackupEngineReadOnly::Open now always return error statuses matching those of the backup Env. +* Added new overloaded function GetApproximateSizes that allows to specify if memtable stats should be computed only without computing SST files' stats approximations. ### Bug Fixes * Fix the bug that if 2PC is enabled, checkpoints may loss some recent transactions. diff --git a/db/db_impl.cc b/db/db_impl.cc index e8e83626b..19165e95c 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -5579,7 +5579,9 @@ ColumnFamilyHandle* DBImpl::GetColumnFamilyHandleUnlocked( void DBImpl::GetApproximateSizes(ColumnFamilyHandle* column_family, const Range* range, int n, uint64_t* sizes, - bool include_memtable) { + uint8_t include_flags) { + assert(include_flags & DB::SizeApproximationFlags::INCLUDE_FILES || + include_flags & DB::SizeApproximationFlags::INCLUDE_MEMTABLES); Version* v; auto cfh = reinterpret_cast(column_family); auto cfd = cfh->cfd(); @@ -5590,8 +5592,10 @@ void DBImpl::GetApproximateSizes(ColumnFamilyHandle* column_family, // Convert user_key into a corresponding internal key. InternalKey k1(range[i].start, kMaxSequenceNumber, kValueTypeForSeek); InternalKey k2(range[i].limit, kMaxSequenceNumber, kValueTypeForSeek); - sizes[i] = versions_->ApproximateSize(v, k1.Encode(), k2.Encode()); - if (include_memtable) { + if (include_flags & DB::SizeApproximationFlags::INCLUDE_FILES) { + sizes[i] = versions_->ApproximateSize(v, k1.Encode(), k2.Encode()); + } + if (include_flags & DB::SizeApproximationFlags::INCLUDE_MEMTABLES) { sizes[i] += sv->mem->ApproximateSize(k1.Encode(), k2.Encode()); sizes[i] += sv->imm->ApproximateSize(k1.Encode(), k2.Encode()); } diff --git a/db/db_impl.h b/db/db_impl.h index ddc6799d6..f27c358d8 100644 --- a/db/db_impl.h +++ b/db/db_impl.h @@ -138,7 +138,8 @@ class DBImpl : public DB { using DB::GetApproximateSizes; virtual void GetApproximateSizes(ColumnFamilyHandle* column_family, const Range* range, int n, uint64_t* sizes, - bool include_memtable = false) override; + uint8_t include_flags + = INCLUDE_FILES) override; using DB::CompactRange; virtual Status CompactRange(const CompactRangeOptions& options, ColumnFamilyHandle* column_family, diff --git a/db/db_test.cc b/db/db_test.cc index d9fa51c33..36c85cb14 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -1407,17 +1407,19 @@ TEST_F(DBTest, ApproximateSizesMemTable) { std::string start = Key(50); std::string end = Key(60); Range r(start, end); - db_->GetApproximateSizes(&r, 1, &size, true); + uint8_t include_both = DB::SizeApproximationFlags::INCLUDE_FILES | + DB::SizeApproximationFlags::INCLUDE_MEMTABLES; + db_->GetApproximateSizes(&r, 1, &size, include_both); ASSERT_GT(size, 6000); ASSERT_LT(size, 204800); // Zero if not including mem table - db_->GetApproximateSizes(&r, 1, &size, false); + db_->GetApproximateSizes(&r, 1, &size); ASSERT_EQ(size, 0); start = Key(500); end = Key(600); r = Range(start, end); - db_->GetApproximateSizes(&r, 1, &size, true); + db_->GetApproximateSizes(&r, 1, &size, include_both); ASSERT_EQ(size, 0); for (int i = 0; i < N; i++) { @@ -1427,13 +1429,13 @@ TEST_F(DBTest, ApproximateSizesMemTable) { start = Key(500); end = Key(600); r = Range(start, end); - db_->GetApproximateSizes(&r, 1, &size, true); + db_->GetApproximateSizes(&r, 1, &size, include_both); ASSERT_EQ(size, 0); start = Key(100); end = Key(1020); r = Range(start, end); - db_->GetApproximateSizes(&r, 1, &size, true); + db_->GetApproximateSizes(&r, 1, &size, include_both); ASSERT_GT(size, 6000); options.max_write_buffer_number = 8; @@ -1456,28 +1458,28 @@ TEST_F(DBTest, ApproximateSizesMemTable) { start = Key(100); end = Key(300); r = Range(start, end); - db_->GetApproximateSizes(&r, 1, &size, true); + db_->GetApproximateSizes(&r, 1, &size, include_both); ASSERT_EQ(size, 0); start = Key(1050); end = Key(1080); r = Range(start, end); - db_->GetApproximateSizes(&r, 1, &size, true); + db_->GetApproximateSizes(&r, 1, &size, include_both); ASSERT_GT(size, 6000); start = Key(2100); end = Key(2300); r = Range(start, end); - db_->GetApproximateSizes(&r, 1, &size, true); + db_->GetApproximateSizes(&r, 1, &size, include_both); ASSERT_EQ(size, 0); start = Key(1050); end = Key(1080); r = Range(start, end); uint64_t size_with_mt, size_without_mt; - db_->GetApproximateSizes(&r, 1, &size_with_mt, true); + db_->GetApproximateSizes(&r, 1, &size_with_mt, include_both); ASSERT_GT(size_with_mt, 6000); - db_->GetApproximateSizes(&r, 1, &size_without_mt, false); + db_->GetApproximateSizes(&r, 1, &size_without_mt); ASSERT_EQ(size_without_mt, 0); Flush(); @@ -1489,8 +1491,8 @@ TEST_F(DBTest, ApproximateSizesMemTable) { start = Key(1050); end = Key(1080); r = Range(start, end); - db_->GetApproximateSizes(&r, 1, &size_with_mt, true); - db_->GetApproximateSizes(&r, 1, &size_without_mt, false); + db_->GetApproximateSizes(&r, 1, &size_with_mt, include_both); + db_->GetApproximateSizes(&r, 1, &size_without_mt); ASSERT_GT(size_with_mt, size_without_mt); ASSERT_GT(size_without_mt, 6000); } @@ -2813,7 +2815,8 @@ class ModelDB : public DB { using DB::GetApproximateSizes; virtual void GetApproximateSizes(ColumnFamilyHandle* column_family, const Range* range, int n, uint64_t* sizes, - bool include_memtable) override { + uint8_t include_flags + = INCLUDE_FILES) override { for (int i = 0; i < n; i++) { sizes[i] = 0; } diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index 2a0304e47..07dab606f 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -586,6 +586,14 @@ class DB { virtual bool GetAggregatedIntProperty(const Slice& property, uint64_t* value) = 0; + // Flags for DB::GetSizeApproximation that specify whether memtable + // stats should be included, or file stats approximation or both + enum SizeApproximationFlags : uint8_t { + NONE = 0, + INCLUDE_MEMTABLES = 1, + INCLUDE_FILES = 1 << 1 + }; + // For each i in [0,n-1], store in "sizes[i]", the approximate // file system space used by keys in "[range[i].start .. range[i].limit)". // @@ -593,16 +601,40 @@ class DB { // if the user data compresses by a factor of ten, the returned // sizes will be one-tenth the size of the corresponding user data size. // - // If include_memtable is set to true, then the result will also - // include those recently written data in the mem-tables if - // the mem-table type supports it. + // If include_flags defines whether the returned size should include + // the recently written data in the mem-tables (if + // the mem-table type supports it), data serialized to disk, or both. + // include_flags should be of type DB::SizeApproximationFlags virtual void GetApproximateSizes(ColumnFamilyHandle* column_family, const Range* range, int n, uint64_t* sizes, - bool include_memtable = false) = 0; + uint8_t include_flags + = INCLUDE_FILES) = 0; virtual void GetApproximateSizes(const Range* range, int n, uint64_t* sizes, - bool include_memtable = false) { + uint8_t include_flags + = INCLUDE_FILES) { GetApproximateSizes(DefaultColumnFamily(), range, n, sizes, - include_memtable); + include_flags); + } + + // Deprecated versions of GetApproximateSizes + ROCKSDB_DEPRECATED_FUNC virtual void GetApproximateSizes( + const Range* range, int n, uint64_t* sizes, + bool include_memtable) { + uint8_t include_flags = SizeApproximationFlags::INCLUDE_FILES; + if (include_memtable) { + include_flags |= SizeApproximationFlags::INCLUDE_MEMTABLES; + } + GetApproximateSizes(DefaultColumnFamily(), range, n, sizes, include_flags); + } + ROCKSDB_DEPRECATED_FUNC virtual void GetApproximateSizes( + ColumnFamilyHandle* column_family, + const Range* range, int n, uint64_t* sizes, + bool include_memtable) { + uint8_t include_flags = SizeApproximationFlags::INCLUDE_FILES; + if (include_memtable) { + include_flags |= SizeApproximationFlags::INCLUDE_MEMTABLES; + } + GetApproximateSizes(column_family, range, n, sizes, include_flags); } // Compact the underlying storage for the key range [*begin,*end]. diff --git a/include/rocksdb/utilities/stackable_db.h b/include/rocksdb/utilities/stackable_db.h index a712d1848..e0b2ead6e 100644 --- a/include/rocksdb/utilities/stackable_db.h +++ b/include/rocksdb/utilities/stackable_db.h @@ -161,9 +161,10 @@ class StackableDB : public DB { using DB::GetApproximateSizes; virtual void GetApproximateSizes(ColumnFamilyHandle* column_family, const Range* r, int n, uint64_t* sizes, - bool include_memtable = false) override { + uint8_t include_flags + = INCLUDE_FILES) override { return db_->GetApproximateSizes(column_family, r, n, sizes, - include_memtable); + include_flags); } using DB::CompactRange;