NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add property `LiveSstFilesSizeAtTemperature` for tiered storage (#8644)

Browse Source 
Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/8644

Reviewed By: siying, zhichao-cao

Differential Revision: D30236535

Pulled By: jay-zhuang

fbshipit-source-id: 1758d1c46d83a5087560fb63d53a016bf999da81
main
Jay Zhuang 3 years ago committed by Facebook GitHub Bot
parent e51be2c5a1
commit c55460c734
5 changed files with 116 additions and 0 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 1
      HISTORY.md
  2. 81
      db/db_test2.cc
  3. 29
      db/internal_stats.cc
  4. 1
      db/internal_stats.h
  5. 4
      include/rocksdb/db.h

1
HISTORY.md
Unescape View File

@ -16,6 +16,7 @@
* Add CompactionOptionsFIFO.age_for_warm, which allows RocksDB to move old files to warm tier in FIFO compactions. Note that file temperature is still an experimental feature. * Add CompactionOptionsFIFO.age_for_warm, which allows RocksDB to move old files to warm tier in FIFO compactions. Note that file temperature is still an experimental feature.
* Add a comment to suggest btrfs user to disable file preallocation by setting `options.allow_fallocate=false`. * Add a comment to suggest btrfs user to disable file preallocation by setting `options.allow_fallocate=false`.
* Fast forward option in Trace replay changed to double type to allow replaying at a lower speed, by settings the value between 0 and 1. This option can be set via `ReplayOptions` in `Replayer::Replay()`, or via `--trace_replay_fast_forward` in db_bench. * Fast forward option in Trace replay changed to double type to allow replaying at a lower speed, by settings the value between 0 and 1. This option can be set via `ReplayOptions` in `Replayer::Replay()`, or via `--trace_replay_fast_forward` in db_bench.
* Add property `LiveSstFilesSizeAtTemperature` to retrieve sst file size at different temperature.
## Public API change ## Public API change
* Added APIs to decode and replay trace file via Replayer class. Added `DB::NewDefaultReplayer()` to create a default Replayer instance. Added `TraceReader::Reset()` to restart reading a trace file. Created trace_record.h and utilities/replayer.h files to access decoded Trace records and replay them. * Added APIs to decode and replay trace file via Replayer class. Added `DB::NewDefaultReplayer()` to create a default Replayer instance. Added `TraceReader::Reset()` to restart reading a trace file. Created trace_record.h and utilities/replayer.h files to access decoded Trace records and replay them.

81
db/db_test2.cc
Unescape View File

@ -27,6 +27,19 @@ namespace ROCKSDB_NAMESPACE {
class DBTest2 : public DBTestBase { class DBTest2 : public DBTestBase {
public: public:
DBTest2() : DBTestBase("db_test2", /*env_do_fsync=*/true) {} DBTest2() : DBTestBase("db_test2", /*env_do_fsync=*/true) {}
protected:
#ifndef ROCKSDB_LITE
uint64_t GetSstSizeHelper(Temperature temperature) {
std::string prop;
bool s =
dbfull()->GetProperty(DB::Properties::kLiveSstFilesSizeAtTemperature +
ToString(static_cast<uint8_t>(temperature)),
&prop);
assert(s);
return static_cast<uint64_t>(std::atoi(prop.c_str()));
}
#endif // ROCKSDB_LITE
}; };
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
@ -6127,6 +6140,13 @@ TEST_F(DBTest2, BottommostTemperature) {
options.level0_file_num_compaction_trigger = 2; options.level0_file_num_compaction_trigger = 2;
Reopen(options); Reopen(options);
auto size = GetSstSizeHelper(Temperature::kUnknown);
ASSERT_EQ(size, 0);
size = GetSstSizeHelper(Temperature::kWarm);
ASSERT_EQ(size, 0);
size = GetSstSizeHelper(Temperature::kHot);
ASSERT_EQ(size, 0);
ASSERT_OK(Put("foo", "bar")); ASSERT_OK(Put("foo", "bar"));
ASSERT_OK(Put("bar", "bar")); ASSERT_OK(Put("bar", "bar"));
ASSERT_OK(Flush()); ASSERT_OK(Flush());
@ -6139,6 +6159,10 @@ TEST_F(DBTest2, BottommostTemperature) {
db_->GetColumnFamilyMetaData(&metadata); db_->GetColumnFamilyMetaData(&metadata);
ASSERT_EQ(1, metadata.file_count); ASSERT_EQ(1, metadata.file_count);
ASSERT_EQ(Temperature::kWarm, metadata.levels[1].files[0].temperature); ASSERT_EQ(Temperature::kWarm, metadata.levels[1].files[0].temperature);
size = GetSstSizeHelper(Temperature::kUnknown);
ASSERT_EQ(size, 0);
size = GetSstSizeHelper(Temperature::kWarm);
ASSERT_GT(size, 0);
// non-bottommost file still has unknown temperature // non-bottommost file still has unknown temperature
ASSERT_OK(Put("foo", "bar")); ASSERT_OK(Put("foo", "bar"));
@ -6147,6 +6171,10 @@ TEST_F(DBTest2, BottommostTemperature) {
db_->GetColumnFamilyMetaData(&metadata); db_->GetColumnFamilyMetaData(&metadata);
ASSERT_EQ(2, metadata.file_count); ASSERT_EQ(2, metadata.file_count);
ASSERT_EQ(Temperature::kUnknown, metadata.levels[0].files[0].temperature); ASSERT_EQ(Temperature::kUnknown, metadata.levels[0].files[0].temperature);
size = GetSstSizeHelper(Temperature::kUnknown);
ASSERT_GT(size, 0);
size = GetSstSizeHelper(Temperature::kWarm);
ASSERT_GT(size, 0);
// reopen and check the information is persisted // reopen and check the information is persisted
Reopen(options); Reopen(options);
@ -6154,6 +6182,21 @@ TEST_F(DBTest2, BottommostTemperature) {
ASSERT_EQ(2, metadata.file_count); ASSERT_EQ(2, metadata.file_count);
ASSERT_EQ(Temperature::kUnknown, metadata.levels[0].files[0].temperature); ASSERT_EQ(Temperature::kUnknown, metadata.levels[0].files[0].temperature);
ASSERT_EQ(Temperature::kWarm, metadata.levels[1].files[0].temperature); ASSERT_EQ(Temperature::kWarm, metadata.levels[1].files[0].temperature);
size = GetSstSizeHelper(Temperature::kUnknown);
ASSERT_GT(size, 0);
size = GetSstSizeHelper(Temperature::kWarm);
ASSERT_GT(size, 0);
// check other non-exist temperatures
size = GetSstSizeHelper(Temperature::kHot);
ASSERT_EQ(size, 0);
size = GetSstSizeHelper(Temperature::kCold);
ASSERT_EQ(size, 0);
std::string prop;
ASSERT_TRUE(dbfull()->GetProperty(
DB::Properties::kLiveSstFilesSizeAtTemperature + std::to_string(22),
&prop));
ASSERT_EQ(std::atoi(prop.c_str()), 0);
} }
TEST_F(DBTest2, BottommostTemperatureUniversal) { TEST_F(DBTest2, BottommostTemperatureUniversal) {
@ -6167,6 +6210,13 @@ TEST_F(DBTest2, BottommostTemperatureUniversal) {
DestroyAndReopen(options); DestroyAndReopen(options);
auto size = GetSstSizeHelper(Temperature::kUnknown);
ASSERT_EQ(size, 0);
size = GetSstSizeHelper(Temperature::kWarm);
ASSERT_EQ(size, 0);
size = GetSstSizeHelper(Temperature::kHot);
ASSERT_EQ(size, 0);
for (int i = 0; i < kTriggerNum; i++) { for (int i = 0; i < kTriggerNum; i++) {
ASSERT_OK(Put("foo", "bar")); ASSERT_OK(Put("foo", "bar"));
ASSERT_OK(Put("bar", "bar")); ASSERT_OK(Put("bar", "bar"));
@ -6179,6 +6229,10 @@ TEST_F(DBTest2, BottommostTemperatureUniversal) {
ASSERT_EQ(1, metadata.file_count); ASSERT_EQ(1, metadata.file_count);
ASSERT_EQ(Temperature::kUnknown, ASSERT_EQ(Temperature::kUnknown,
metadata.levels[kBottommostLevel].files[0].temperature); metadata.levels[kBottommostLevel].files[0].temperature);
size = GetSstSizeHelper(Temperature::kUnknown);
ASSERT_GT(size, 0);
size = GetSstSizeHelper(Temperature::kWarm);
ASSERT_EQ(size, 0);
ASSERT_OK(Put("foo", "bar")); ASSERT_OK(Put("foo", "bar"));
ASSERT_OK(Put("bar", "bar")); ASSERT_OK(Put("bar", "bar"));
@ -6187,6 +6241,10 @@ TEST_F(DBTest2, BottommostTemperatureUniversal) {
db_->GetColumnFamilyMetaData(&metadata); db_->GetColumnFamilyMetaData(&metadata);
ASSERT_EQ(2, metadata.file_count); ASSERT_EQ(2, metadata.file_count);
ASSERT_EQ(Temperature::kUnknown, metadata.levels[0].files[0].temperature); ASSERT_EQ(Temperature::kUnknown, metadata.levels[0].files[0].temperature);
size = GetSstSizeHelper(Temperature::kUnknown);
ASSERT_GT(size, 0);
size = GetSstSizeHelper(Temperature::kWarm);
ASSERT_EQ(size, 0);
// Update bottommost temperature // Update bottommost temperature
options.bottommost_temperature = Temperature::kWarm; options.bottommost_temperature = Temperature::kWarm;
@ -6195,6 +6253,10 @@ TEST_F(DBTest2, BottommostTemperatureUniversal) {
// Should not impact existing ones // Should not impact existing ones
ASSERT_EQ(Temperature::kUnknown, ASSERT_EQ(Temperature::kUnknown,
metadata.levels[kBottommostLevel].files[0].temperature); metadata.levels[kBottommostLevel].files[0].temperature);
size = GetSstSizeHelper(Temperature::kUnknown);
ASSERT_GT(size, 0);
size = GetSstSizeHelper(Temperature::kWarm);
ASSERT_EQ(size, 0);
// new generated file should have the new settings // new generated file should have the new settings
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
@ -6202,6 +6264,10 @@ TEST_F(DBTest2, BottommostTemperatureUniversal) {
ASSERT_EQ(1, metadata.file_count); ASSERT_EQ(1, metadata.file_count);
ASSERT_EQ(Temperature::kWarm, ASSERT_EQ(Temperature::kWarm,
metadata.levels[kBottommostLevel].files[0].temperature); metadata.levels[kBottommostLevel].files[0].temperature);
size = GetSstSizeHelper(Temperature::kUnknown);
ASSERT_EQ(size, 0);
size = GetSstSizeHelper(Temperature::kWarm);
ASSERT_GT(size, 0);
// non-bottommost file still has unknown temperature // non-bottommost file still has unknown temperature
ASSERT_OK(Put("foo", "bar")); ASSERT_OK(Put("foo", "bar"));
@ -6211,6 +6277,21 @@ TEST_F(DBTest2, BottommostTemperatureUniversal) {
db_->GetColumnFamilyMetaData(&metadata); db_->GetColumnFamilyMetaData(&metadata);
ASSERT_EQ(2, metadata.file_count); ASSERT_EQ(2, metadata.file_count);
ASSERT_EQ(Temperature::kUnknown, metadata.levels[0].files[0].temperature); ASSERT_EQ(Temperature::kUnknown, metadata.levels[0].files[0].temperature);
size = GetSstSizeHelper(Temperature::kUnknown);
ASSERT_GT(size, 0);
size = GetSstSizeHelper(Temperature::kWarm);
ASSERT_GT(size, 0);
// check other non-exist temperatures
size = GetSstSizeHelper(Temperature::kHot);
ASSERT_EQ(size, 0);
size = GetSstSizeHelper(Temperature::kCold);
ASSERT_EQ(size, 0);
std::string prop;
ASSERT_TRUE(dbfull()->GetProperty(
DB::Properties::kLiveSstFilesSizeAtTemperature + std::to_string(22),
&prop));
ASSERT_EQ(std::atoi(prop.c_str()), 0);
} }
#endif // ROCKSDB_LITE #endif // ROCKSDB_LITE

29
db/internal_stats.cc
Unescape View File

@ -262,6 +262,8 @@ static const std::string min_obsolete_sst_number_to_keep_str =
static const std::string base_level_str = "base-level"; static const std::string base_level_str = "base-level";
static const std::string total_sst_files_size = "total-sst-files-size"; static const std::string total_sst_files_size = "total-sst-files-size";
static const std::string live_sst_files_size = "live-sst-files-size"; static const std::string live_sst_files_size = "live-sst-files-size";
static const std::string live_sst_files_size_at_temperature =
"live-sst-files-size-at-temperature";
static const std::string estimate_pending_comp_bytes = static const std::string estimate_pending_comp_bytes =
"estimate-pending-compaction-bytes"; "estimate-pending-compaction-bytes";
static const std::string aggregated_table_properties = static const std::string aggregated_table_properties =
@ -369,6 +371,8 @@ const std::string DB::Properties::kBlockCachePinnedUsage =
rocksdb_prefix + block_cache_pinned_usage; rocksdb_prefix + block_cache_pinned_usage;
const std::string DB::Properties::kOptionsStatistics = const std::string DB::Properties::kOptionsStatistics =
rocksdb_prefix + options_statistics; rocksdb_prefix + options_statistics;
const std::string DB::Properties::kLiveSstFilesSizeAtTemperature =
rocksdb_prefix + live_sst_files_size_at_temperature;
const std::unordered_map<std::string, DBPropertyInfo> const std::unordered_map<std::string, DBPropertyInfo>
InternalStats::ppt_name_to_info = { InternalStats::ppt_name_to_info = {
@ -482,6 +486,9 @@ const std::unordered_map<std::string, DBPropertyInfo>
{DB::Properties::kLiveSstFilesSize, {DB::Properties::kLiveSstFilesSize,
{false, nullptr, &InternalStats::HandleLiveSstFilesSize, nullptr, {false, nullptr, &InternalStats::HandleLiveSstFilesSize, nullptr,
nullptr}}, nullptr}},
{DB::Properties::kLiveSstFilesSizeAtTemperature,
{true, &InternalStats::HandleLiveSstFilesSizeAtTemperature, nullptr,
nullptr, nullptr}},
{DB::Properties::kEstimatePendingCompactionBytes, {DB::Properties::kEstimatePendingCompactionBytes,
{false, nullptr, &InternalStats::HandleEstimatePendingCompactionBytes, {false, nullptr, &InternalStats::HandleEstimatePendingCompactionBytes,
nullptr, nullptr}}, nullptr, nullptr}},
@ -682,6 +689,28 @@ bool InternalStats::HandleBlockCacheEntryStatsMap(
return true; return true;
} }
bool InternalStats::HandleLiveSstFilesSizeAtTemperature(std::string* value,
Slice suffix) {
uint64_t temperature;
bool ok = ConsumeDecimalNumber(&suffix, &temperature) && suffix.empty();
if (!ok) {
return false;
}
uint64_t size = 0;
const auto* vstorage = cfd_->current()->storage_info();
for (int level = 0; level < vstorage->num_levels(); level++) {
for (const auto& file_meta : vstorage->LevelFiles(level)) {
if (static_cast<uint8_t>(file_meta->temperature) == temperature) {
size += file_meta->fd.GetFileSize();
}
}
}
*value = ToString(size);
return true;
}
const DBPropertyInfo* GetPropertyInfo(const Slice& property) { const DBPropertyInfo* GetPropertyInfo(const Slice& property) {
std::string ppt_name = GetPropertyNameAndArg(property).first.ToString(); std::string ppt_name = GetPropertyNameAndArg(property).first.ToString();
auto ppt_info_iter = InternalStats::ppt_name_to_info.find(ppt_name); auto ppt_info_iter = InternalStats::ppt_name_to_info.find(ppt_name);

1
db/internal_stats.h
Unescape View File

@ -675,6 +675,7 @@ class InternalStats {
bool HandleBlockCacheEntryStats(std::string* value, Slice suffix); bool HandleBlockCacheEntryStats(std::string* value, Slice suffix);
bool HandleBlockCacheEntryStatsMap(std::map<std::string, std::string>* values, bool HandleBlockCacheEntryStatsMap(std::map<std::string, std::string>* values,
Slice suffix); Slice suffix);
bool HandleLiveSstFilesSizeAtTemperature(std::string* value, Slice suffix);
// Total number of background errors encountered. Every time a flush task // Total number of background errors encountered. Every time a flush task
// or compaction task fails, this counter is incremented. The failure can // or compaction task fails, this counter is incremented. The failure can
// be caused by any possible reason, including file system errors, out of // be caused by any possible reason, including file system errors, out of

4
include/rocksdb/db.h
Unescape View File

@ -914,6 +914,10 @@ class DB {
// files belong to the latest LSM tree. // files belong to the latest LSM tree.
static const std::string kLiveSstFilesSize; static const std::string kLiveSstFilesSize;
// "rocksdb.live_sst_files_size_at_temperature" - returns total size (bytes)
// of SST files at all certain file temperature
static const std::string kLiveSstFilesSizeAtTemperature;
// "rocksdb.base-level" - returns number of level to which L0 data will be // "rocksdb.base-level" - returns number of level to which L0 data will be
// compacted. // compacted.
static const std::string kBaseLevel; static const std::string kBaseLevel;

Write Preview
Loading…
Cancel
Save