From 1aaa898cf13d435e53e434db713f17afddaf28ce Mon Sep 17 00:00:00 2001 From: Vitaliy Liptchinsky Date: Mon, 6 Feb 2017 14:42:38 -0800 Subject: [PATCH] Adding GetApproximateMemTableStats method Summary: Added method that returns approx num of entries as well as size for memtables. Closes https://github.com/facebook/rocksdb/pull/1841 Differential Revision: D4511990 Pulled By: VitaliyLi fbshipit-source-id: 9a4576e --- HISTORY.md | 1 + db/db_impl.cc | 26 +++++++++- db/db_impl.h | 5 ++ db/db_test.cc | 61 ++++++++++++++++++++++++ db/memtable.cc | 10 ++-- db/memtable.h | 8 +++- db/memtable_list.cc | 12 +++-- db/memtable_list.h | 3 +- include/rocksdb/db.h | 12 +++++ include/rocksdb/utilities/stackable_db.h | 8 ++++ 10 files changed, 132 insertions(+), 14 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index adc8363f7..8c68168b6 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -7,6 +7,7 @@ * Added new overloaded function GetApproximateSizes that allows to specify if memtable stats should be computed only without computing SST files' stats approximations. * NewLRUCache() will determine number of shard bits automatically based on capacity, if the user doesn't pass one. This also impacts the default block cache when the user doesn't explict provide one. * Change the default of delayed slowdown value to 16MB/s and further increase the L0 stop condition to 36 files. +* Added new function GetApproximateMemTableStats that approximates both number of records and size of memtables. ### Bug Fixes * Fix the bug that if 2PC is enabled, checkpoints may loss some recent transactions. diff --git a/db/db_impl.cc b/db/db_impl.cc index 8b0911c05..09d75d841 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -5558,6 +5558,28 @@ ColumnFamilyHandle* DBImpl::GetColumnFamilyHandle(uint32_t column_family_id) { return cf_memtables->GetColumnFamilyHandle(); } +void DBImpl::GetApproximateMemTableStats(ColumnFamilyHandle* column_family, + const Range& range, + uint64_t* const count, + uint64_t* const size) { + ColumnFamilyHandleImpl* cfh = + reinterpret_cast(column_family); + ColumnFamilyData* cfd = cfh->cfd(); + SuperVersion* sv = GetAndRefSuperVersion(cfd); + + // Convert user_key into a corresponding internal key. + InternalKey k1(range.start, kMaxSequenceNumber, kValueTypeForSeek); + InternalKey k2(range.limit, kMaxSequenceNumber, kValueTypeForSeek); + MemTable::MemTableStats memStats = + sv->mem->ApproximateStats(k1.Encode(), k2.Encode()); + MemTable::MemTableStats immStats = + sv->imm->ApproximateStats(k1.Encode(), k2.Encode()); + *count = memStats.count + immStats.count; + *size = memStats.size + immStats.size; + + ReturnAndCleanupSuperVersion(cfd, sv); +} + void DBImpl::GetApproximateSizes(ColumnFamilyHandle* column_family, const Range* range, int n, uint64_t* sizes, uint8_t include_flags) { @@ -5578,8 +5600,8 @@ void DBImpl::GetApproximateSizes(ColumnFamilyHandle* column_family, sizes[i] += versions_->ApproximateSize(v, k1.Encode(), k2.Encode()); } if (include_flags & DB::SizeApproximationFlags::INCLUDE_MEMTABLES) { - sizes[i] += sv->mem->ApproximateSize(k1.Encode(), k2.Encode()); - sizes[i] += sv->imm->ApproximateSize(k1.Encode(), k2.Encode()); + sizes[i] += sv->mem->ApproximateStats(k1.Encode(), k2.Encode()).size; + sizes[i] += sv->imm->ApproximateStats(k1.Encode(), k2.Encode()).size; } } diff --git a/db/db_impl.h b/db/db_impl.h index ef258d645..79fd29260 100644 --- a/db/db_impl.h +++ b/db/db_impl.h @@ -140,6 +140,11 @@ class DBImpl : public DB { const Range* range, int n, uint64_t* sizes, uint8_t include_flags = INCLUDE_FILES) override; + using DB::GetApproximateMemTableStats; + virtual void GetApproximateMemTableStats(ColumnFamilyHandle* column_family, + const Range& range, + uint64_t* const count, + uint64_t* const size) override; using DB::CompactRange; virtual Status CompactRange(const CompactRangeOptions& options, ColumnFamilyHandle* column_family, diff --git a/db/db_test.cc b/db/db_test.cc index 36c85cb14..1dd1e3ea1 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -1497,6 +1497,59 @@ TEST_F(DBTest, ApproximateSizesMemTable) { ASSERT_GT(size_without_mt, 6000); } +TEST_F(DBTest, GetApproximateMemTableStats) { + Options options = CurrentOptions(); + options.write_buffer_size = 100000000; + options.compression = kNoCompression; + options.create_if_missing = true; + DestroyAndReopen(options); + + const int N = 128; + Random rnd(301); + for (int i = 0; i < N; i++) { + ASSERT_OK(Put(Key(i), RandomString(&rnd, 1024))); + } + + uint64_t count; + uint64_t size; + + std::string start = Key(50); + std::string end = Key(60); + Range r(start, end); + db_->GetApproximateMemTableStats(r, &count, &size); + ASSERT_GT(count, 0); + ASSERT_LE(count, N); + ASSERT_GT(size, 6000); + ASSERT_LT(size, 204800); + + start = Key(500); + end = Key(600); + r = Range(start, end); + db_->GetApproximateMemTableStats(r, &count, &size); + ASSERT_EQ(count, 0); + ASSERT_EQ(size, 0); + + Flush(); + + start = Key(50); + end = Key(60); + r = Range(start, end); + db_->GetApproximateMemTableStats(r, &count, &size); + ASSERT_EQ(count, 0); + ASSERT_EQ(size, 0); + + for (int i = 0; i < N; i++) { + ASSERT_OK(Put(Key(1000 + i), RandomString(&rnd, 1024))); + } + + start = Key(100); + end = Key(1020); + r = Range(start, end); + db_->GetApproximateMemTableStats(r, &count, &size); + ASSERT_GT(count, 20); + ASSERT_GT(size, 6000); +} + TEST_F(DBTest, ApproximateSizes) { do { Options options = CurrentOptions(); @@ -2821,6 +2874,14 @@ class ModelDB : public DB { sizes[i] = 0; } } + using DB::GetApproximateMemTableStats; + virtual void GetApproximateMemTableStats(ColumnFamilyHandle* column_family, + const Range& range, + uint64_t* const count, + uint64_t* const size) override { + *count = 0; + *size = 0; + } using DB::CompactRange; virtual Status CompactRange(const CompactRangeOptions& options, ColumnFamilyHandle* column_family, diff --git a/db/memtable.cc b/db/memtable.cc index 776c8d8e7..a8ece01ce 100644 --- a/db/memtable.cc +++ b/db/memtable.cc @@ -390,16 +390,16 @@ port::RWMutex* MemTable::GetLock(const Slice& key) { return &locks_[hash(key) % locks_.size()]; } -uint64_t MemTable::ApproximateSize(const Slice& start_ikey, - const Slice& end_ikey) { +MemTable::MemTableStats MemTable::ApproximateStats(const Slice& start_ikey, + const Slice& end_ikey) { uint64_t entry_count = table_->ApproximateNumEntries(start_ikey, end_ikey); entry_count += range_del_table_->ApproximateNumEntries(start_ikey, end_ikey); if (entry_count == 0) { - return 0; + return {0, 0}; } uint64_t n = num_entries_.load(std::memory_order_relaxed); if (n == 0) { - return 0; + return {0, 0}; } if (entry_count > n) { // (range_del_)table_->ApproximateNumEntries() is just an estimate so it can @@ -408,7 +408,7 @@ uint64_t MemTable::ApproximateSize(const Slice& start_ikey, entry_count = n; } uint64_t data_size = data_size_.load(std::memory_order_relaxed); - return entry_count * (data_size / n); + return {entry_count * (data_size / n), entry_count}; } void MemTable::Add(SequenceNumber s, ValueType type, diff --git a/db/memtable.h b/db/memtable.h index a6ad3f243..12d7be61d 100644 --- a/db/memtable.h +++ b/db/memtable.h @@ -326,7 +326,13 @@ class MemTable { return table_->IsSnapshotSupported() && !moptions_.inplace_update_support; } - uint64_t ApproximateSize(const Slice& start_ikey, const Slice& end_ikey); + struct MemTableStats { + uint64_t size; + uint64_t count; + }; + + MemTableStats ApproximateStats(const Slice& start_ikey, + const Slice& end_ikey); // Get the lock associated for the key port::RWMutex* GetLock(const Slice& key); diff --git a/db/memtable_list.cc b/db/memtable_list.cc index 1398d7e7d..a14b591b3 100644 --- a/db/memtable_list.cc +++ b/db/memtable_list.cc @@ -190,13 +190,15 @@ uint64_t MemTableListVersion::GetTotalNumEntries() const { return total_num; } -uint64_t MemTableListVersion::ApproximateSize(const Slice& start_ikey, - const Slice& end_ikey) { - uint64_t total_size = 0; +MemTable::MemTableStats MemTableListVersion::ApproximateStats( + const Slice& start_ikey, const Slice& end_ikey) { + MemTable::MemTableStats total_stats = {0, 0}; for (auto& m : memlist_) { - total_size += m->ApproximateSize(start_ikey, end_ikey); + auto mStats = m->ApproximateStats(start_ikey, end_ikey); + total_stats.size += mStats.size; + total_stats.count += mStats.count; } - return total_size; + return total_stats; } uint64_t MemTableListVersion::GetTotalNumDeletes() const { diff --git a/db/memtable_list.h b/db/memtable_list.h index 97438d754..7c969825a 100644 --- a/db/memtable_list.h +++ b/db/memtable_list.h @@ -95,7 +95,8 @@ class MemTableListVersion { uint64_t GetTotalNumDeletes() const; - uint64_t ApproximateSize(const Slice& start_ikey, const Slice& end_ikey); + MemTable::MemTableStats ApproximateStats(const Slice& start_ikey, + const Slice& end_ikey); // Returns the value of MemTable::GetEarliestSequenceNumber() on the most // recent MemTable in this list or kMaxSequenceNumber if the list is empty. diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index b345fba38..cdbb8abf1 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -616,6 +616,18 @@ class DB { include_flags); } + // The method is similar to GetApproximateSizes, except it + // returns approximate number of records in memtables. + virtual void GetApproximateMemTableStats(ColumnFamilyHandle* column_family, + const Range& range, + uint64_t* const count, + uint64_t* const size) = 0; + virtual void GetApproximateMemTableStats(const Range& range, + uint64_t* const count, + uint64_t* const size) { + GetApproximateMemTableStats(DefaultColumnFamily(), range, count, size); + } + // Deprecated versions of GetApproximateSizes ROCKSDB_DEPRECATED_FUNC virtual void GetApproximateSizes( const Range* range, int n, uint64_t* sizes, diff --git a/include/rocksdb/utilities/stackable_db.h b/include/rocksdb/utilities/stackable_db.h index e0b2ead6e..eca800c09 100644 --- a/include/rocksdb/utilities/stackable_db.h +++ b/include/rocksdb/utilities/stackable_db.h @@ -167,6 +167,14 @@ class StackableDB : public DB { include_flags); } + using DB::GetApproximateMemTableStats; + virtual void GetApproximateMemTableStats(ColumnFamilyHandle* column_family, + const Range& range, + uint64_t* const count, + uint64_t* const size) override { + return db_->GetApproximateMemTableStats(column_family, range, count, size); + } + using DB::CompactRange; virtual Status CompactRange(const CompactRangeOptions& options, ColumnFamilyHandle* column_family,