Introduce a global StatsDumpScheduler for stats dumping (#7223)
Summary: Have a global StatsDumpScheduler for all DB instance stats dumping, including `DumpStats()` and `PersistStats()`. Before this, there're 2 dedicate threads for every DB instance, one for DumpStats() one for PersistStats(), which could create lots of threads if there're hundreds DB instances. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7223 Reviewed By: riversand963 Differential Revision: D23056737 Pulled By: jay-zhuang fbshipit-source-id: 0faa2311142a73433ebb3317361db7cbf43faebamain
Jay Zhuang
4 years ago
committed by
Facebook GitHub Bot
parent
d758273ceb
commit
69760b4d05
@ -0,0 +1,103 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
#include "monitoring/stats_dump_scheduler.h" |
||||||
|
|
||||||
|
#include "db/db_impl/db_impl.h" |
||||||
|
#include "util/cast_util.h" |
||||||
|
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
namespace ROCKSDB_NAMESPACE { |
||||||
|
|
||||||
|
StatsDumpScheduler::StatsDumpScheduler(Env* env) { |
||||||
|
timer = std::unique_ptr<Timer>(new Timer(env)); |
||||||
|
} |
||||||
|
|
||||||
|
void StatsDumpScheduler::Register(DBImpl* dbi, |
||||||
|
unsigned int stats_dump_period_sec, |
||||||
|
unsigned int stats_persist_period_sec) { |
||||||
|
static std::atomic<uint64_t> initial_delay(0); |
||||||
|
if (stats_dump_period_sec > 0) { |
||||||
|
timer->Start(); |
||||||
|
timer->Add([dbi]() { dbi->DumpStats(); }, GetTaskName(dbi, "dump_st"), |
||||||
|
initial_delay.fetch_add(1) % |
||||||
|
static_cast<uint64_t>(stats_dump_period_sec) * |
||||||
|
kMicrosInSecond, |
||||||
|
static_cast<uint64_t>(stats_dump_period_sec) * kMicrosInSecond); |
||||||
|
} |
||||||
|
if (stats_persist_period_sec > 0) { |
||||||
|
timer->Start(); |
||||||
|
timer->Add( |
||||||
|
[dbi]() { dbi->PersistStats(); }, GetTaskName(dbi, "pst_st"), |
||||||
|
initial_delay.fetch_add(1) % |
||||||
|
static_cast<uint64_t>(stats_persist_period_sec) * kMicrosInSecond, |
||||||
|
static_cast<uint64_t>(stats_persist_period_sec) * kMicrosInSecond); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
void StatsDumpScheduler::Unregister(DBImpl* dbi) { |
||||||
|
timer->Cancel(GetTaskName(dbi, "dump_st")); |
||||||
|
timer->Cancel(GetTaskName(dbi, "pst_st")); |
||||||
|
if (!timer->HasPendingTask()) { |
||||||
|
timer->Shutdown(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
StatsDumpScheduler* StatsDumpScheduler::Default() { |
||||||
|
// Always use the default Env for the scheduler, as we only use the NowMicros
|
||||||
|
// which is the same for all env.
|
||||||
|
// The Env could only be overridden in test.
|
||||||
|
static StatsDumpScheduler scheduler(Env::Default()); |
||||||
|
return &scheduler; |
||||||
|
} |
||||||
|
|
||||||
|
std::string StatsDumpScheduler::GetTaskName(DBImpl* dbi, |
||||||
|
const std::string& func_name) { |
||||||
|
std::string db_session_id; |
||||||
|
dbi->GetDbSessionId(db_session_id); |
||||||
|
return db_session_id + ":" + func_name; |
||||||
|
} |
||||||
|
|
||||||
|
#ifndef NDEBUG |
||||||
|
|
||||||
|
// Get the static scheduler. For a new env, it needs to re-create the internal
|
||||||
|
// timer, so only re-create it when there's no running task. Otherwise, return
|
||||||
|
// the existing scheduler. Which means if the unittest needs to update MockEnv,
|
||||||
|
// Close all db instances and then re-open them.
|
||||||
|
StatsDumpTestScheduler* StatsDumpTestScheduler::Default(Env* env) { |
||||||
|
static StatsDumpTestScheduler scheduler(env); |
||||||
|
static port::Mutex mutex; |
||||||
|
{ |
||||||
|
MutexLock l(&mutex); |
||||||
|
if (scheduler.timer.get() != nullptr && |
||||||
|
scheduler.timer->TEST_GetPendingTaskNum() == 0) { |
||||||
|
scheduler.timer->Shutdown(); |
||||||
|
scheduler.timer.reset(new Timer(env)); |
||||||
|
} |
||||||
|
} |
||||||
|
return &scheduler; |
||||||
|
} |
||||||
|
|
||||||
|
void StatsDumpTestScheduler::TEST_WaitForRun( |
||||||
|
std::function<void()> callback) const { |
||||||
|
if (timer != nullptr) { |
||||||
|
timer->TEST_WaitForRun(callback); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
size_t StatsDumpTestScheduler::TEST_GetValidTaskNum() const { |
||||||
|
if (timer != nullptr) { |
||||||
|
return timer->TEST_GetPendingTaskNum(); |
||||||
|
} |
||||||
|
return 0; |
||||||
|
} |
||||||
|
|
||||||
|
StatsDumpTestScheduler::StatsDumpTestScheduler(Env* env) |
||||||
|
: StatsDumpScheduler(env) {} |
||||||
|
|
||||||
|
#endif // !NDEBUG
|
||||||
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
@ -0,0 +1,64 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
#include "db/db_impl/db_impl.h" |
||||||
|
#include "util/timer.h" |
||||||
|
|
||||||
|
namespace ROCKSDB_NAMESPACE { |
||||||
|
|
||||||
|
// StatsDumpScheduler is a singleton object, which is scheduling/running
|
||||||
|
// DumpStats() and PersistStats() for all DB instances. All DB instances uses
|
||||||
|
// the same object from `Default()`.
|
||||||
|
// Internally, it uses a single threaded timer object to run the stats dump
|
||||||
|
// functions. Timer thread won't be started if there's no function needs to run,
|
||||||
|
// for example, option.stats_dump_period_sec and option.stats_persist_period_sec
|
||||||
|
// are set to 0.
|
||||||
|
class StatsDumpScheduler { |
||||||
|
public: |
||||||
|
static StatsDumpScheduler* Default(); |
||||||
|
|
||||||
|
StatsDumpScheduler() = delete; |
||||||
|
StatsDumpScheduler(const StatsDumpScheduler&) = delete; |
||||||
|
StatsDumpScheduler(StatsDumpScheduler&&) = delete; |
||||||
|
StatsDumpScheduler& operator=(const StatsDumpScheduler&) = delete; |
||||||
|
StatsDumpScheduler& operator=(StatsDumpScheduler&&) = delete; |
||||||
|
|
||||||
|
void Register(DBImpl* dbi, unsigned int stats_dump_period_sec, |
||||||
|
unsigned int stats_persist_period_sec); |
||||||
|
|
||||||
|
void Unregister(DBImpl* dbi); |
||||||
|
|
||||||
|
protected: |
||||||
|
std::unique_ptr<Timer> timer; |
||||||
|
|
||||||
|
explicit StatsDumpScheduler(Env* env); |
||||||
|
|
||||||
|
private: |
||||||
|
std::string GetTaskName(DBImpl* dbi, const std::string& func_name); |
||||||
|
}; |
||||||
|
|
||||||
|
#ifndef NDEBUG |
||||||
|
// StatsDumpTestScheduler is for unittest, which can specify the Env like
|
||||||
|
// SafeMockTimeEnv. It also contains functions for unittest.
|
||||||
|
class StatsDumpTestScheduler : public StatsDumpScheduler { |
||||||
|
public: |
||||||
|
static StatsDumpTestScheduler* Default(Env* env); |
||||||
|
|
||||||
|
void TEST_WaitForRun(std::function<void()> callback) const; |
||||||
|
|
||||||
|
size_t TEST_GetValidTaskNum() const; |
||||||
|
|
||||||
|
private: |
||||||
|
explicit StatsDumpTestScheduler(Env* env); |
||||||
|
}; |
||||||
|
#endif // !NDEBUG
|
||||||
|
|
||||||
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
@ -0,0 +1,225 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
#include "monitoring/stats_dump_scheduler.h" |
||||||
|
|
||||||
|
#include "db/db_test_util.h" |
||||||
|
|
||||||
|
namespace ROCKSDB_NAMESPACE { |
||||||
|
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
class StatsDumpSchedulerTest : public DBTestBase { |
||||||
|
public: |
||||||
|
StatsDumpSchedulerTest() |
||||||
|
: DBTestBase("/stats_dump_scheduler_test"), |
||||||
|
mock_env_(new SafeMockTimeEnv(Env::Default())) {} |
||||||
|
|
||||||
|
protected: |
||||||
|
std::unique_ptr<SafeMockTimeEnv> mock_env_; |
||||||
|
|
||||||
|
void SetUp() override { |
||||||
|
SyncPoint::GetInstance()->SetCallBack( |
||||||
|
"DBImpl::StartStatsDumpScheduler:Init", [&](void* arg) { |
||||||
|
auto* stats_dump_scheduler_ptr = |
||||||
|
reinterpret_cast<StatsDumpScheduler**>(arg); |
||||||
|
*stats_dump_scheduler_ptr = |
||||||
|
StatsDumpTestScheduler::Default(mock_env_.get()); |
||||||
|
}); |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
TEST_F(StatsDumpSchedulerTest, Basic) { |
||||||
|
constexpr int kPeriodSec = 5; |
||||||
|
Close(); |
||||||
|
Options options; |
||||||
|
options.stats_dump_period_sec = kPeriodSec; |
||||||
|
options.stats_persist_period_sec = kPeriodSec; |
||||||
|
options.create_if_missing = true; |
||||||
|
int mock_time_sec = 0; |
||||||
|
mock_env_->set_current_time(mock_time_sec); |
||||||
|
options.env = mock_env_.get(); |
||||||
|
|
||||||
|
int dump_st_counter = 0; |
||||||
|
SyncPoint::GetInstance()->SetCallBack("DBImpl::DumpStats:StartRunning", |
||||||
|
[&](void*) { dump_st_counter++; }); |
||||||
|
|
||||||
|
int pst_st_counter = 0; |
||||||
|
SyncPoint::GetInstance()->SetCallBack("DBImpl::PersistStats:StartRunning", |
||||||
|
[&](void*) { pst_st_counter++; }); |
||||||
|
SyncPoint::GetInstance()->EnableProcessing(); |
||||||
|
|
||||||
|
Reopen(options); |
||||||
|
|
||||||
|
ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_dump_period_sec); |
||||||
|
ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_persist_period_sec); |
||||||
|
|
||||||
|
mock_time_sec += kPeriodSec - 1; |
||||||
|
dbfull()->TEST_WaitForStatsDumpRun( |
||||||
|
[&] { mock_env_->set_current_time(mock_time_sec); }); |
||||||
|
|
||||||
|
auto scheduler = dbfull()->TEST_GetStatsDumpScheduler(); |
||||||
|
ASSERT_NE(nullptr, scheduler); |
||||||
|
ASSERT_EQ(2, scheduler->TEST_GetValidTaskNum()); |
||||||
|
|
||||||
|
ASSERT_EQ(1, dump_st_counter); |
||||||
|
ASSERT_EQ(1, pst_st_counter); |
||||||
|
|
||||||
|
mock_time_sec += kPeriodSec; |
||||||
|
dbfull()->TEST_WaitForStatsDumpRun( |
||||||
|
[&] { mock_env_->set_current_time(mock_time_sec); }); |
||||||
|
|
||||||
|
ASSERT_EQ(2, dump_st_counter); |
||||||
|
ASSERT_EQ(2, pst_st_counter); |
||||||
|
|
||||||
|
mock_time_sec += kPeriodSec; |
||||||
|
dbfull()->TEST_WaitForStatsDumpRun( |
||||||
|
[&] { mock_env_->set_current_time(mock_time_sec); }); |
||||||
|
|
||||||
|
ASSERT_EQ(3, dump_st_counter); |
||||||
|
ASSERT_EQ(3, pst_st_counter); |
||||||
|
|
||||||
|
// Disable scheduler with SetOption
|
||||||
|
ASSERT_OK(dbfull()->SetDBOptions( |
||||||
|
{{"stats_dump_period_sec", "0"}, {"stats_persist_period_sec", "0"}})); |
||||||
|
ASSERT_EQ(0u, dbfull()->GetDBOptions().stats_dump_period_sec); |
||||||
|
ASSERT_EQ(0u, dbfull()->GetDBOptions().stats_persist_period_sec); |
||||||
|
|
||||||
|
scheduler = dbfull()->TEST_GetStatsDumpScheduler(); |
||||||
|
ASSERT_EQ(0u, scheduler->TEST_GetValidTaskNum()); |
||||||
|
|
||||||
|
// Re-enable one task
|
||||||
|
ASSERT_OK(dbfull()->SetDBOptions({{"stats_dump_period_sec", "5"}})); |
||||||
|
ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_dump_period_sec); |
||||||
|
ASSERT_EQ(0u, dbfull()->GetDBOptions().stats_persist_period_sec); |
||||||
|
|
||||||
|
scheduler = dbfull()->TEST_GetStatsDumpScheduler(); |
||||||
|
ASSERT_NE(nullptr, scheduler); |
||||||
|
ASSERT_EQ(1, scheduler->TEST_GetValidTaskNum()); |
||||||
|
|
||||||
|
dump_st_counter = 0; |
||||||
|
mock_time_sec += kPeriodSec; |
||||||
|
dbfull()->TEST_WaitForStatsDumpRun( |
||||||
|
[&] { mock_env_->set_current_time(mock_time_sec); }); |
||||||
|
ASSERT_EQ(1, dump_st_counter); |
||||||
|
|
||||||
|
Close(); |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(StatsDumpSchedulerTest, MultiInstances) { |
||||||
|
constexpr int kPeriodSec = 5; |
||||||
|
const int kInstanceNum = 10; |
||||||
|
|
||||||
|
Close(); |
||||||
|
Options options; |
||||||
|
options.stats_dump_period_sec = kPeriodSec; |
||||||
|
options.stats_persist_period_sec = kPeriodSec; |
||||||
|
options.create_if_missing = true; |
||||||
|
int mock_time_sec = 0; |
||||||
|
mock_env_->set_current_time(mock_time_sec); |
||||||
|
options.env = mock_env_.get(); |
||||||
|
|
||||||
|
int dump_st_counter = 0; |
||||||
|
SyncPoint::GetInstance()->SetCallBack("DBImpl::DumpStats:2", |
||||||
|
[&](void*) { dump_st_counter++; }); |
||||||
|
|
||||||
|
int pst_st_counter = 0; |
||||||
|
SyncPoint::GetInstance()->SetCallBack("DBImpl::PersistStats:StartRunning", |
||||||
|
[&](void*) { pst_st_counter++; }); |
||||||
|
SyncPoint::GetInstance()->EnableProcessing(); |
||||||
|
|
||||||
|
auto dbs = std::vector<DB*>(kInstanceNum); |
||||||
|
for (int i = 0; i < kInstanceNum; i++) { |
||||||
|
ASSERT_OK( |
||||||
|
DB::Open(options, test::PerThreadDBPath(std::to_string(i)), &(dbs[i]))); |
||||||
|
} |
||||||
|
|
||||||
|
auto dbi = static_cast_with_check<DBImpl>(dbs[kInstanceNum - 1]); |
||||||
|
auto scheduler = dbi->TEST_GetStatsDumpScheduler(); |
||||||
|
ASSERT_EQ(kInstanceNum * 2, scheduler->TEST_GetValidTaskNum()); |
||||||
|
|
||||||
|
int expected_run = kInstanceNum; |
||||||
|
mock_time_sec += kPeriodSec - 1; |
||||||
|
dbi->TEST_WaitForStatsDumpRun( |
||||||
|
[&] { mock_env_->set_current_time(mock_time_sec); }); |
||||||
|
ASSERT_EQ(expected_run, dump_st_counter); |
||||||
|
ASSERT_EQ(expected_run, pst_st_counter); |
||||||
|
|
||||||
|
expected_run += kInstanceNum; |
||||||
|
mock_time_sec += kPeriodSec; |
||||||
|
dbi->TEST_WaitForStatsDumpRun( |
||||||
|
[&] { mock_env_->set_current_time(mock_time_sec); }); |
||||||
|
ASSERT_EQ(expected_run, dump_st_counter); |
||||||
|
ASSERT_EQ(expected_run, pst_st_counter); |
||||||
|
|
||||||
|
expected_run += kInstanceNum; |
||||||
|
mock_time_sec += kPeriodSec; |
||||||
|
dbi->TEST_WaitForStatsDumpRun( |
||||||
|
[&] { mock_env_->set_current_time(mock_time_sec); }); |
||||||
|
ASSERT_EQ(expected_run, dump_st_counter); |
||||||
|
ASSERT_EQ(expected_run, pst_st_counter); |
||||||
|
|
||||||
|
int half = kInstanceNum / 2; |
||||||
|
for (int i = 0; i < half; i++) { |
||||||
|
delete dbs[i]; |
||||||
|
} |
||||||
|
|
||||||
|
expected_run += (kInstanceNum - half) * 2; |
||||||
|
|
||||||
|
mock_time_sec += kPeriodSec; |
||||||
|
dbi->TEST_WaitForStatsDumpRun( |
||||||
|
[&] { mock_env_->set_current_time(mock_time_sec); }); |
||||||
|
mock_time_sec += kPeriodSec; |
||||||
|
dbi->TEST_WaitForStatsDumpRun( |
||||||
|
[&] { mock_env_->set_current_time(mock_time_sec); }); |
||||||
|
ASSERT_EQ(expected_run, dump_st_counter); |
||||||
|
ASSERT_EQ(expected_run, pst_st_counter); |
||||||
|
|
||||||
|
for (int i = half; i < kInstanceNum; i++) { |
||||||
|
dbs[i]->Close(); |
||||||
|
delete dbs[i]; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(StatsDumpSchedulerTest, MultiEnv) { |
||||||
|
constexpr int kDumpPeriodSec = 5; |
||||||
|
constexpr int kPersistPeriodSec = 10; |
||||||
|
Close(); |
||||||
|
Options options1; |
||||||
|
options1.stats_dump_period_sec = kDumpPeriodSec; |
||||||
|
options1.stats_persist_period_sec = kPersistPeriodSec; |
||||||
|
options1.create_if_missing = true; |
||||||
|
mock_env_->set_current_time(0); |
||||||
|
options1.env = mock_env_.get(); |
||||||
|
|
||||||
|
Reopen(options1); |
||||||
|
|
||||||
|
std::unique_ptr<MockTimeEnv> mock_env2(new MockTimeEnv(Env::Default())); |
||||||
|
Options options2; |
||||||
|
options2.stats_dump_period_sec = kDumpPeriodSec; |
||||||
|
options2.stats_persist_period_sec = kPersistPeriodSec; |
||||||
|
options2.create_if_missing = true; |
||||||
|
mock_env2->set_current_time(0); |
||||||
|
options1.env = mock_env2.get(); |
||||||
|
|
||||||
|
std::string dbname = test::PerThreadDBPath("multi_env_test"); |
||||||
|
DB* db; |
||||||
|
ASSERT_OK(DB::Open(options2, dbname, &db)); |
||||||
|
DBImpl* dbi = static_cast_with_check<DBImpl>(db); |
||||||
|
|
||||||
|
ASSERT_EQ(dbi->TEST_GetStatsDumpScheduler(), |
||||||
|
dbfull()->TEST_GetStatsDumpScheduler()); |
||||||
|
|
||||||
|
db->Close(); |
||||||
|
delete db; |
||||||
|
Close(); |
||||||
|
} |
||||||
|
#endif // !ROCKSDB_LITE
|
||||||
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
|
||||||
|
int main(int argc, char** argv) { |
||||||
|
::testing::InitGoogleTest(&argc, argv); |
||||||
|
|
||||||
|
return RUN_ALL_TESTS(); |
||||||
|
} |
||||||
Loading…
Reference in new issue