Adding GetApproximateMemTableStats method

Summary:
Added method that returns approx num of entries as well as size for memtables.
Closes https://github.com/facebook/rocksdb/pull/1841

Differential Revision: D4511990

Pulled By: VitaliyLi

fbshipit-source-id: 9a4576e
main
Vitaliy Liptchinsky 8 years ago committed by Facebook Github Bot
parent 9fc23c55f2
commit 1aaa898cf1
  1. 1
      HISTORY.md
  2. 26
      db/db_impl.cc
  3. 5
      db/db_impl.h
  4. 61
      db/db_test.cc
  5. 10
      db/memtable.cc
  6. 8
      db/memtable.h
  7. 12
      db/memtable_list.cc
  8. 3
      db/memtable_list.h
  9. 12
      include/rocksdb/db.h
  10. 8
      include/rocksdb/utilities/stackable_db.h

@ -7,6 +7,7 @@
* Added new overloaded function GetApproximateSizes that allows to specify if memtable stats should be computed only without computing SST files' stats approximations.
* NewLRUCache() will determine number of shard bits automatically based on capacity, if the user doesn't pass one. This also impacts the default block cache when the user doesn't explict provide one.
* Change the default of delayed slowdown value to 16MB/s and further increase the L0 stop condition to 36 files.
* Added new function GetApproximateMemTableStats that approximates both number of records and size of memtables.
### Bug Fixes
* Fix the bug that if 2PC is enabled, checkpoints may loss some recent transactions.

@ -5558,6 +5558,28 @@ ColumnFamilyHandle* DBImpl::GetColumnFamilyHandle(uint32_t column_family_id) {
return cf_memtables->GetColumnFamilyHandle();
}
void DBImpl::GetApproximateMemTableStats(ColumnFamilyHandle* column_family,
const Range& range,
uint64_t* const count,
uint64_t* const size) {
ColumnFamilyHandleImpl* cfh =
reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
ColumnFamilyData* cfd = cfh->cfd();
SuperVersion* sv = GetAndRefSuperVersion(cfd);
// Convert user_key into a corresponding internal key.
InternalKey k1(range.start, kMaxSequenceNumber, kValueTypeForSeek);
InternalKey k2(range.limit, kMaxSequenceNumber, kValueTypeForSeek);
MemTable::MemTableStats memStats =
sv->mem->ApproximateStats(k1.Encode(), k2.Encode());
MemTable::MemTableStats immStats =
sv->imm->ApproximateStats(k1.Encode(), k2.Encode());
*count = memStats.count + immStats.count;
*size = memStats.size + immStats.size;
ReturnAndCleanupSuperVersion(cfd, sv);
}
void DBImpl::GetApproximateSizes(ColumnFamilyHandle* column_family,
const Range* range, int n, uint64_t* sizes,
uint8_t include_flags) {
@ -5578,8 +5600,8 @@ void DBImpl::GetApproximateSizes(ColumnFamilyHandle* column_family,
sizes[i] += versions_->ApproximateSize(v, k1.Encode(), k2.Encode());
}
if (include_flags & DB::SizeApproximationFlags::INCLUDE_MEMTABLES) {
sizes[i] += sv->mem->ApproximateSize(k1.Encode(), k2.Encode());
sizes[i] += sv->imm->ApproximateSize(k1.Encode(), k2.Encode());
sizes[i] += sv->mem->ApproximateStats(k1.Encode(), k2.Encode()).size;
sizes[i] += sv->imm->ApproximateStats(k1.Encode(), k2.Encode()).size;
}
}

@ -140,6 +140,11 @@ class DBImpl : public DB {
const Range* range, int n, uint64_t* sizes,
uint8_t include_flags
= INCLUDE_FILES) override;
using DB::GetApproximateMemTableStats;
virtual void GetApproximateMemTableStats(ColumnFamilyHandle* column_family,
const Range& range,
uint64_t* const count,
uint64_t* const size) override;
using DB::CompactRange;
virtual Status CompactRange(const CompactRangeOptions& options,
ColumnFamilyHandle* column_family,

@ -1497,6 +1497,59 @@ TEST_F(DBTest, ApproximateSizesMemTable) {
ASSERT_GT(size_without_mt, 6000);
}
TEST_F(DBTest, GetApproximateMemTableStats) {
Options options = CurrentOptions();
options.write_buffer_size = 100000000;
options.compression = kNoCompression;
options.create_if_missing = true;
DestroyAndReopen(options);
const int N = 128;
Random rnd(301);
for (int i = 0; i < N; i++) {
ASSERT_OK(Put(Key(i), RandomString(&rnd, 1024)));
}
uint64_t count;
uint64_t size;
std::string start = Key(50);
std::string end = Key(60);
Range r(start, end);
db_->GetApproximateMemTableStats(r, &count, &size);
ASSERT_GT(count, 0);
ASSERT_LE(count, N);
ASSERT_GT(size, 6000);
ASSERT_LT(size, 204800);
start = Key(500);
end = Key(600);
r = Range(start, end);
db_->GetApproximateMemTableStats(r, &count, &size);
ASSERT_EQ(count, 0);
ASSERT_EQ(size, 0);
Flush();
start = Key(50);
end = Key(60);
r = Range(start, end);
db_->GetApproximateMemTableStats(r, &count, &size);
ASSERT_EQ(count, 0);
ASSERT_EQ(size, 0);
for (int i = 0; i < N; i++) {
ASSERT_OK(Put(Key(1000 + i), RandomString(&rnd, 1024)));
}
start = Key(100);
end = Key(1020);
r = Range(start, end);
db_->GetApproximateMemTableStats(r, &count, &size);
ASSERT_GT(count, 20);
ASSERT_GT(size, 6000);
}
TEST_F(DBTest, ApproximateSizes) {
do {
Options options = CurrentOptions();
@ -2821,6 +2874,14 @@ class ModelDB : public DB {
sizes[i] = 0;
}
}
using DB::GetApproximateMemTableStats;
virtual void GetApproximateMemTableStats(ColumnFamilyHandle* column_family,
const Range& range,
uint64_t* const count,
uint64_t* const size) override {
*count = 0;
*size = 0;
}
using DB::CompactRange;
virtual Status CompactRange(const CompactRangeOptions& options,
ColumnFamilyHandle* column_family,

@ -390,16 +390,16 @@ port::RWMutex* MemTable::GetLock(const Slice& key) {
return &locks_[hash(key) % locks_.size()];
}
uint64_t MemTable::ApproximateSize(const Slice& start_ikey,
const Slice& end_ikey) {
MemTable::MemTableStats MemTable::ApproximateStats(const Slice& start_ikey,
const Slice& end_ikey) {
uint64_t entry_count = table_->ApproximateNumEntries(start_ikey, end_ikey);
entry_count += range_del_table_->ApproximateNumEntries(start_ikey, end_ikey);
if (entry_count == 0) {
return 0;
return {0, 0};
}
uint64_t n = num_entries_.load(std::memory_order_relaxed);
if (n == 0) {
return 0;
return {0, 0};
}
if (entry_count > n) {
// (range_del_)table_->ApproximateNumEntries() is just an estimate so it can
@ -408,7 +408,7 @@ uint64_t MemTable::ApproximateSize(const Slice& start_ikey,
entry_count = n;
}
uint64_t data_size = data_size_.load(std::memory_order_relaxed);
return entry_count * (data_size / n);
return {entry_count * (data_size / n), entry_count};
}
void MemTable::Add(SequenceNumber s, ValueType type,

@ -326,7 +326,13 @@ class MemTable {
return table_->IsSnapshotSupported() && !moptions_.inplace_update_support;
}
uint64_t ApproximateSize(const Slice& start_ikey, const Slice& end_ikey);
struct MemTableStats {
uint64_t size;
uint64_t count;
};
MemTableStats ApproximateStats(const Slice& start_ikey,
const Slice& end_ikey);
// Get the lock associated for the key
port::RWMutex* GetLock(const Slice& key);

@ -190,13 +190,15 @@ uint64_t MemTableListVersion::GetTotalNumEntries() const {
return total_num;
}
uint64_t MemTableListVersion::ApproximateSize(const Slice& start_ikey,
const Slice& end_ikey) {
uint64_t total_size = 0;
MemTable::MemTableStats MemTableListVersion::ApproximateStats(
const Slice& start_ikey, const Slice& end_ikey) {
MemTable::MemTableStats total_stats = {0, 0};
for (auto& m : memlist_) {
total_size += m->ApproximateSize(start_ikey, end_ikey);
auto mStats = m->ApproximateStats(start_ikey, end_ikey);
total_stats.size += mStats.size;
total_stats.count += mStats.count;
}
return total_size;
return total_stats;
}
uint64_t MemTableListVersion::GetTotalNumDeletes() const {

@ -95,7 +95,8 @@ class MemTableListVersion {
uint64_t GetTotalNumDeletes() const;
uint64_t ApproximateSize(const Slice& start_ikey, const Slice& end_ikey);
MemTable::MemTableStats ApproximateStats(const Slice& start_ikey,
const Slice& end_ikey);
// Returns the value of MemTable::GetEarliestSequenceNumber() on the most
// recent MemTable in this list or kMaxSequenceNumber if the list is empty.

@ -616,6 +616,18 @@ class DB {
include_flags);
}
// The method is similar to GetApproximateSizes, except it
// returns approximate number of records in memtables.
virtual void GetApproximateMemTableStats(ColumnFamilyHandle* column_family,
const Range& range,
uint64_t* const count,
uint64_t* const size) = 0;
virtual void GetApproximateMemTableStats(const Range& range,
uint64_t* const count,
uint64_t* const size) {
GetApproximateMemTableStats(DefaultColumnFamily(), range, count, size);
}
// Deprecated versions of GetApproximateSizes
ROCKSDB_DEPRECATED_FUNC virtual void GetApproximateSizes(
const Range* range, int n, uint64_t* sizes,

@ -167,6 +167,14 @@ class StackableDB : public DB {
include_flags);
}
using DB::GetApproximateMemTableStats;
virtual void GetApproximateMemTableStats(ColumnFamilyHandle* column_family,
const Range& range,
uint64_t* const count,
uint64_t* const size) override {
return db_->GetApproximateMemTableStats(column_family, range, count, size);
}
using DB::CompactRange;
virtual Status CompactRange(const CompactRangeOptions& options,
ColumnFamilyHandle* column_family,

Loading…
Cancel
Save