NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add property "rocksdb.obsolete-sst-files-size" (#11533)

Browse Source 
Summary:
See "unreleased_history/new_features/obsolete_sst_files_size.md" for description

Pull Request resolved: https://github.com/facebook/rocksdb/pull/11533

Test Plan: updated unit test

Reviewed By: jowlyzhang

Differential Revision: D46703152

Pulled By: ajkr

fbshipit-source-id: ea5e31cd6293eccc154130c13e66b5271f57c102
oxigraph-main
Andrew Kryczka 2 years ago committed by Facebook GitHub Bot
parent e178f9e477
commit cac3240cbf
9 changed files with 70 additions and 4 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      db/db_impl/db_impl.h
  2. 5
      db/db_impl/db_impl_files.cc
  3. 25
      db/db_properties_test.cc
  4. 12
      db/internal_stats.cc
  5. 2
      db/internal_stats.h
  6. 10
      db/version_set.cc
  7. 3
      db/version_set.h
  8. 14
      include/rocksdb/db.h
  9. 1
      unreleased_history/new_features/obsolete_sst_files_size.md

2
db/db_impl/db_impl.h
Unescape View File

@ -811,6 +811,8 @@ class DBImpl : public DB {
// being deleted.
uint64_t MinObsoleteSstNumberToKeep();
uint64_t GetObsoleteSstFilesSize();
// Returns the list of live files in 'live' and the list
// of all files in the filesystem in 'candidate_files'.
// If force == false and the last call was less than

5
db/db_impl/db_impl_files.cc
Unescape View File

@ -35,6 +35,11 @@ uint64_t DBImpl::MinObsoleteSstNumberToKeep() {
return std::numeric_limits<uint64_t>::max();
}
uint64_t DBImpl::GetObsoleteSstFilesSize() {
mutex_.AssertHeld();
return versions_->GetObsoleteSstFilesSize();
}
Status DBImpl::DisableFileDeletions() {
Status s;
int my_disable_delete_obsolete_files;

25
db/db_properties_test.cc
Unescape View File

@ -1715,14 +1715,35 @@ TEST_F(DBPropertiesTest, SstFilesSize) {
ASSERT_OK(Delete("key" + std::to_string(i)));
}
ASSERT_OK(Flush());
uint64_t sst_size;
bool ok = db_->GetIntProperty(DB::Properties::kTotalSstFilesSize, &sst_size);
ASSERT_TRUE(ok);
ASSERT_TRUE(
db_->GetIntProperty(DB::Properties::kTotalSstFilesSize, &sst_size));
ASSERT_GT(sst_size, 0);
listener->size_before_compaction = sst_size;
uint64_t obsolete_sst_size;
ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kObsoleteSstFilesSize,
&obsolete_sst_size));
ASSERT_EQ(obsolete_sst_size, 0);
// Hold files from being deleted so we can test property for size of obsolete
// SST files.
ASSERT_OK(db_->DisableFileDeletions());
// Compact to clean all keys and trigger listener.
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
ASSERT_TRUE(listener->callback_triggered);
ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kObsoleteSstFilesSize,
&obsolete_sst_size));
ASSERT_EQ(obsolete_sst_size, sst_size);
// Let the obsolete files be deleted.
ASSERT_OK(db_->EnableFileDeletions());
ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kObsoleteSstFilesSize,
&obsolete_sst_size));
ASSERT_EQ(obsolete_sst_size, 0);
}
TEST_F(DBPropertiesTest, MinObsoleteSstNumberToKeep) {

12
db/internal_stats.cc
Unescape View File

@ -291,6 +291,7 @@ static const std::string min_obsolete_sst_number_to_keep_str =
static const std::string base_level_str = "base-level";
static const std::string total_sst_files_size = "total-sst-files-size";
static const std::string live_sst_files_size = "live-sst-files-size";
static const std::string obsolete_sst_files_size = "obsolete-sst-files-size";
static const std::string live_sst_files_size_at_temperature =
"live-sst-files-size-at-temperature";
static const std::string estimate_pending_comp_bytes =
@ -394,6 +395,8 @@ const std::string DB::Properties::kTotalSstFilesSize =
rocksdb_prefix + total_sst_files_size;
const std::string DB::Properties::kLiveSstFilesSize =
rocksdb_prefix + live_sst_files_size;
const std::string DB::Properties::kObsoleteSstFilesSize =
rocksdb_prefix + obsolete_sst_files_size;
const std::string DB::Properties::kBaseLevel = rocksdb_prefix + base_level_str;
const std::string DB::Properties::kEstimatePendingCompactionBytes =
rocksdb_prefix + estimate_pending_comp_bytes;
@ -565,6 +568,9 @@ const UnorderedMap<std::string, DBPropertyInfo>
{DB::Properties::kLiveSstFilesSizeAtTemperature,
{false, &InternalStats::HandleLiveSstFilesSizeAtTemperature, nullptr,
nullptr, nullptr}},
{DB::Properties::kObsoleteSstFilesSize,
{false, nullptr, &InternalStats::HandleObsoleteSstFilesSize, nullptr,
nullptr}},
{DB::Properties::kEstimatePendingCompactionBytes,
{false, nullptr, &InternalStats::HandleEstimatePendingCompactionBytes,
nullptr, nullptr}},
@ -1395,6 +1401,12 @@ bool InternalStats::HandleLiveSstFilesSize(uint64_t* value, DBImpl* /*db*/,
return true;
}
bool InternalStats::HandleObsoleteSstFilesSize(uint64_t* value, DBImpl* db,
Version* /*version*/) {
*value = db->GetObsoleteSstFilesSize();
return true;
}
bool InternalStats::HandleEstimatePendingCompactionBytes(uint64_t* value,
DBImpl* /*db*/,
Version* /*version*/) {

2
db/internal_stats.h
Unescape View File

@ -819,6 +819,8 @@ class InternalStats {
bool HandleBaseLevel(uint64_t* value, DBImpl* db, Version* version);
bool HandleTotalSstFilesSize(uint64_t* value, DBImpl* db, Version* version);
bool HandleLiveSstFilesSize(uint64_t* value, DBImpl* db, Version* version);
bool HandleObsoleteSstFilesSize(uint64_t* value, DBImpl* db,
Version* version);
bool HandleEstimatePendingCompactionBytes(uint64_t* value, DBImpl* db,
Version* version);
bool HandleEstimateTableReadersMem(uint64_t* value, DBImpl* db,

10
db/version_set.cc
Unescape View File

@ -7003,6 +7003,16 @@ void VersionSet::GetObsoleteFiles(std::vector<ObsoleteFileInfo>* files,
obsolete_manifests_.swap(*manifest_filenames);
}
uint64_t VersionSet::GetObsoleteSstFilesSize() const {
uint64_t ret = 0;
for (auto& f : obsolete_files_) {
if (f.metadata != nullptr) {
ret += f.metadata->fd.GetFileSize();
}
}
return ret;
}
ColumnFamilyData* VersionSet::CreateColumnFamily(
const ColumnFamilyOptions& cf_options, const ReadOptions& read_options,
const VersionEdit* edit) {

3
db/version_set.h
Unescape View File

@ -1466,6 +1466,9 @@ class VersionSet {
std::vector<std::string>* manifest_filenames,
uint64_t min_pending_output);
// REQUIRES: DB mutex held
uint64_t GetObsoleteSstFilesSize() const;
ColumnFamilySet* GetColumnFamilySet() { return column_family_set_.get(); }
const UnorderedMap<uint32_t, size_t>& GetRunningColumnFamiliesTimestampSize()

14
include/rocksdb/db.h
Unescape View File

@ -1097,14 +1097,23 @@ class DB {
static const std::string kMinObsoleteSstNumberToKeep;
// "rocksdb.total-sst-files-size" - returns total size (bytes) of all SST
// files.
// files belonging to any of the CF's versions.
// WARNING: may slow down online queries if there are too many files.
static const std::string kTotalSstFilesSize;
// "rocksdb.live-sst-files-size" - returns total size (bytes) of all SST
// files belong to the latest LSM tree.
// files belong to the CF's current version.
static const std::string kLiveSstFilesSize;
// "rocksdb.obsolete-sst-files-size" - returns total size (bytes) of all
// SST files that became obsolete but have not yet been deleted or
// scheduled for deletion. SST files can end up in this state when
// using `DisableFileDeletions()`, for example.
//
// N.B. Unlike the other "*SstFilesSize" properties, this property
// includes SST files that originated in any of the DB's CFs.
static const std::string kObsoleteSstFilesSize;
// "rocksdb.live_sst_files_size_at_temperature" - returns total size (bytes)
// of SST files at all certain file temperature
static const std::string kLiveSstFilesSizeAtTemperature;
@ -1238,6 +1247,7 @@ class DB {
// "rocksdb.min-obsolete-sst-number-to-keep"
// "rocksdb.total-sst-files-size"
// "rocksdb.live-sst-files-size"
// "rocksdb.obsolete-sst-files-size"
// "rocksdb.base-level"
// "rocksdb.estimate-pending-compaction-bytes"
// "rocksdb.num-running-compactions"

1
unreleased_history/new_features/obsolete_sst_files_size.md
Unescape View File

@ -0,0 +1 @@
Added new property "rocksdb.obsolete-sst-files-size-property" that reports the size of SST files that have become obsolete but have not yet been deleted or scheduled for deletion
Write Preview
Loading…
Cancel
Save