NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Change DB::GetApproximateSizes for more flexibility needed for MyRocks

Browse Source 
Summary:
Added an option to GetApproximateSizes to exclude file stats, as MyRocks has those counted exactly and we need only stats from memtables.
Closes https://github.com/facebook/rocksdb/pull/1787

Differential Revision: D4441111

Pulled By: IslamAbdelRahman

fbshipit-source-id: c11f4c3
main
Vitaliy Liptchinsky 8 years ago committed by Facebook Github Bot
parent 9239103cd4
commit e840213d6e
6 changed files with 67 additions and 25 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 1
      HISTORY.md
  2. 10
      db/db_impl.cc
  3. 3
      db/db_impl.h
  4. 29
      db/db_test.cc
  5. 44
      include/rocksdb/db.h
  6. 5
      include/rocksdb/utilities/stackable_db.h

1
HISTORY.md
Unescape View File

@ -4,6 +4,7 @@
* Support dynamically change `delete_obsolete_files_period_micros` option via SetDBOptions().
* Added EventListener::OnExternalFileIngested which will be called when IngestExternalFile() add a file successfully.
* BackupEngine::Open and BackupEngineReadOnly::Open now always return error statuses matching those of the backup Env.
* Added new overloaded function GetApproximateSizes that allows to specify if memtable stats should be computed only without computing SST files' stats approximations.
### Bug Fixes
* Fix the bug that if 2PC is enabled, checkpoints may loss some recent transactions.

10
db/db_impl.cc
Unescape View File

@ -5579,7 +5579,9 @@ ColumnFamilyHandle* DBImpl::GetColumnFamilyHandleUnlocked(
void DBImpl::GetApproximateSizes(ColumnFamilyHandle* column_family,
const Range* range, int n, uint64_t* sizes,
bool include_memtable) {
uint8_t include_flags) {
assert(include_flags & DB::SizeApproximationFlags::INCLUDE_FILES ||
include_flags & DB::SizeApproximationFlags::INCLUDE_MEMTABLES);
Version* v;
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
auto cfd = cfh->cfd();
@ -5590,8 +5592,10 @@ void DBImpl::GetApproximateSizes(ColumnFamilyHandle* column_family,
// Convert user_key into a corresponding internal key.
InternalKey k1(range[i].start, kMaxSequenceNumber, kValueTypeForSeek);
InternalKey k2(range[i].limit, kMaxSequenceNumber, kValueTypeForSeek);
sizes[i] = versions_->ApproximateSize(v, k1.Encode(), k2.Encode());
if (include_memtable) {
if (include_flags & DB::SizeApproximationFlags::INCLUDE_FILES) {
sizes[i] = versions_->ApproximateSize(v, k1.Encode(), k2.Encode());
}
if (include_flags & DB::SizeApproximationFlags::INCLUDE_MEMTABLES) {
sizes[i] += sv->mem->ApproximateSize(k1.Encode(), k2.Encode());
sizes[i] += sv->imm->ApproximateSize(k1.Encode(), k2.Encode());
}

3
db/db_impl.h
Unescape View File

@ -138,7 +138,8 @@ class DBImpl : public DB {
using DB::GetApproximateSizes;
virtual void GetApproximateSizes(ColumnFamilyHandle* column_family,
const Range* range, int n, uint64_t* sizes,
bool include_memtable = false) override;
uint8_t include_flags
= INCLUDE_FILES) override;
using DB::CompactRange;
virtual Status CompactRange(const CompactRangeOptions& options,
ColumnFamilyHandle* column_family,

29
db/db_test.cc
Unescape View File

@ -1407,17 +1407,19 @@ TEST_F(DBTest, ApproximateSizesMemTable) {
std::string start = Key(50);
std::string end = Key(60);
Range r(start, end);
db_->GetApproximateSizes(&r, 1, &size, true);
uint8_t include_both = DB::SizeApproximationFlags::INCLUDE_FILES |
DB::SizeApproximationFlags::INCLUDE_MEMTABLES;
db_->GetApproximateSizes(&r, 1, &size, include_both);
ASSERT_GT(size, 6000);
ASSERT_LT(size, 204800);
// Zero if not including mem table
db_->GetApproximateSizes(&r, 1, &size, false);
db_->GetApproximateSizes(&r, 1, &size);
ASSERT_EQ(size, 0);
start = Key(500);
end = Key(600);
r = Range(start, end);
db_->GetApproximateSizes(&r, 1, &size, true);
db_->GetApproximateSizes(&r, 1, &size, include_both);
ASSERT_EQ(size, 0);
for (int i = 0; i < N; i++) {
@ -1427,13 +1429,13 @@ TEST_F(DBTest, ApproximateSizesMemTable) {
start = Key(500);
end = Key(600);
r = Range(start, end);
db_->GetApproximateSizes(&r, 1, &size, true);
db_->GetApproximateSizes(&r, 1, &size, include_both);
ASSERT_EQ(size, 0);
start = Key(100);
end = Key(1020);
r = Range(start, end);
db_->GetApproximateSizes(&r, 1, &size, true);
db_->GetApproximateSizes(&r, 1, &size, include_both);
ASSERT_GT(size, 6000);
options.max_write_buffer_number = 8;
@ -1456,28 +1458,28 @@ TEST_F(DBTest, ApproximateSizesMemTable) {
start = Key(100);
end = Key(300);
r = Range(start, end);
db_->GetApproximateSizes(&r, 1, &size, true);
db_->GetApproximateSizes(&r, 1, &size, include_both);
ASSERT_EQ(size, 0);
start = Key(1050);
end = Key(1080);
r = Range(start, end);
db_->GetApproximateSizes(&r, 1, &size, true);
db_->GetApproximateSizes(&r, 1, &size, include_both);
ASSERT_GT(size, 6000);
start = Key(2100);
end = Key(2300);
r = Range(start, end);
db_->GetApproximateSizes(&r, 1, &size, true);
db_->GetApproximateSizes(&r, 1, &size, include_both);
ASSERT_EQ(size, 0);
start = Key(1050);
end = Key(1080);
r = Range(start, end);
uint64_t size_with_mt, size_without_mt;
db_->GetApproximateSizes(&r, 1, &size_with_mt, true);
db_->GetApproximateSizes(&r, 1, &size_with_mt, include_both);
ASSERT_GT(size_with_mt, 6000);
db_->GetApproximateSizes(&r, 1, &size_without_mt, false);
db_->GetApproximateSizes(&r, 1, &size_without_mt);
ASSERT_EQ(size_without_mt, 0);
Flush();
@ -1489,8 +1491,8 @@ TEST_F(DBTest, ApproximateSizesMemTable) {
start = Key(1050);
end = Key(1080);
r = Range(start, end);
db_->GetApproximateSizes(&r, 1, &size_with_mt, true);
db_->GetApproximateSizes(&r, 1, &size_without_mt, false);
db_->GetApproximateSizes(&r, 1, &size_with_mt, include_both);
db_->GetApproximateSizes(&r, 1, &size_without_mt);
ASSERT_GT(size_with_mt, size_without_mt);
ASSERT_GT(size_without_mt, 6000);
}
@ -2813,7 +2815,8 @@ class ModelDB : public DB {
using DB::GetApproximateSizes;
virtual void GetApproximateSizes(ColumnFamilyHandle* column_family,
const Range* range, int n, uint64_t* sizes,
bool include_memtable) override {
uint8_t include_flags
= INCLUDE_FILES) override {
for (int i = 0; i < n; i++) {
sizes[i] = 0;
}

44
include/rocksdb/db.h
Unescape View File

@ -586,6 +586,14 @@ class DB {
virtual bool GetAggregatedIntProperty(const Slice& property,
uint64_t* value) = 0;
// Flags for DB::GetSizeApproximation that specify whether memtable
// stats should be included, or file stats approximation or both
enum SizeApproximationFlags : uint8_t {
NONE = 0,
INCLUDE_MEMTABLES = 1,
INCLUDE_FILES = 1 << 1
};
// For each i in [0,n-1], store in "sizes[i]", the approximate
// file system space used by keys in "[range[i].start .. range[i].limit)".
//
@ -593,16 +601,40 @@ class DB {
// if the user data compresses by a factor of ten, the returned
// sizes will be one-tenth the size of the corresponding user data size.
//
// If include_memtable is set to true, then the result will also
// include those recently written data in the mem-tables if
// the mem-table type supports it.
// If include_flags defines whether the returned size should include
// the recently written data in the mem-tables (if
// the mem-table type supports it), data serialized to disk, or both.
// include_flags should be of type DB::SizeApproximationFlags
virtual void GetApproximateSizes(ColumnFamilyHandle* column_family,
const Range* range, int n, uint64_t* sizes,
bool include_memtable = false) = 0;
uint8_t include_flags
= INCLUDE_FILES) = 0;
virtual void GetApproximateSizes(const Range* range, int n, uint64_t* sizes,
bool include_memtable = false) {
uint8_t include_flags
= INCLUDE_FILES) {
GetApproximateSizes(DefaultColumnFamily(), range, n, sizes,
include_memtable);
include_flags);
}
// Deprecated versions of GetApproximateSizes
ROCKSDB_DEPRECATED_FUNC virtual void GetApproximateSizes(
const Range* range, int n, uint64_t* sizes,
bool include_memtable) {
uint8_t include_flags = SizeApproximationFlags::INCLUDE_FILES;
if (include_memtable) {
include_flags |= SizeApproximationFlags::INCLUDE_MEMTABLES;
}
GetApproximateSizes(DefaultColumnFamily(), range, n, sizes, include_flags);
}
ROCKSDB_DEPRECATED_FUNC virtual void GetApproximateSizes(
ColumnFamilyHandle* column_family,
const Range* range, int n, uint64_t* sizes,
bool include_memtable) {
uint8_t include_flags = SizeApproximationFlags::INCLUDE_FILES;
if (include_memtable) {
include_flags |= SizeApproximationFlags::INCLUDE_MEMTABLES;
}
GetApproximateSizes(column_family, range, n, sizes, include_flags);
}
// Compact the underlying storage for the key range [*begin,*end].

5
include/rocksdb/utilities/stackable_db.h
Unescape View File

@ -161,9 +161,10 @@ class StackableDB : public DB {
using DB::GetApproximateSizes;
virtual void GetApproximateSizes(ColumnFamilyHandle* column_family,
const Range* r, int n, uint64_t* sizes,
bool include_memtable = false) override {
uint8_t include_flags
= INCLUDE_FILES) override {
return db_->GetApproximateSizes(column_family, r, n, sizes,
include_memtable);
include_flags);
}
using DB::CompactRange;

Write Preview
Loading…
Cancel
Save