Summary: We occasionally get write stalls (>1s Write() calls) on HDD under read load. The following timers explain almost all of the stalls: - perf_context.db_mutex_lock_nanos - perf_context.db_condition_wait_nanos - iostats_context.open_time - iostats_context.allocate_time - iostats_context.write_time - iostats_context.range_sync_time - iostats_context.logger_time In my experiments each of these occasionally takes >1s on write path under some workload. There are rare cases when Write() takes long but none of these takes long. Test Plan: Added code to our application to write the listed timings to log for slow writes. They usually add up to almost exactly the time Write() call took. Reviewers: rven, yhchiang, sdong Reviewed By: sdong Subscribers: march, dhruba, tnovak Differential Revision: https://reviews.facebook.net/D39177main
Mike Kolupaev
10 years ago
parent
4266d4fd90
commit
ec7a944360
@ -0,0 +1,30 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#ifndef INCLUDE_ROCKSDB_PERF_LEVEL_H_ |
||||||
|
#define INCLUDE_ROCKSDB_PERF_LEVEL_H_ |
||||||
|
|
||||||
|
#include <stdint.h> |
||||||
|
#include <string> |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
// How much perf stats to collect. Affects perf_context and iostats_context.
|
||||||
|
|
||||||
|
enum PerfLevel { |
||||||
|
kDisable = 0, // disable perf stats
|
||||||
|
kEnableCount = 1, // enable only count stats
|
||||||
|
kEnableTime = 2 // enable time stats too
|
||||||
|
}; |
||||||
|
|
||||||
|
// set the perf stats level for current thread
|
||||||
|
void SetPerfLevel(PerfLevel level); |
||||||
|
|
||||||
|
// get current perf stats level for current thread
|
||||||
|
PerfLevel GetPerfLevel(); |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
|
||||||
|
#endif // INCLUDE_ROCKSDB_PERF_LEVEL_H_
|
||||||
@ -0,0 +1,26 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include <sstream> |
||||||
|
#include "util/perf_level_imp.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
#if defined(IOS_CROSS_COMPILE) |
||||||
|
PerfLevel perf_level = kEnableCount; |
||||||
|
#else |
||||||
|
__thread PerfLevel perf_level = kEnableCount; |
||||||
|
#endif |
||||||
|
|
||||||
|
void SetPerfLevel(PerfLevel level) { |
||||||
|
perf_level = level; |
||||||
|
} |
||||||
|
|
||||||
|
PerfLevel GetPerfLevel() { |
||||||
|
return perf_level; |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,17 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
#pragma once |
||||||
|
#include "rocksdb/perf_level.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
#if defined(IOS_CROSS_COMPILE) |
||||||
|
extern PerfLevel perf_level; |
||||||
|
#else |
||||||
|
extern __thread PerfLevel perf_level; |
||||||
|
#endif |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,54 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
#pragma once |
||||||
|
#include "rocksdb/env.h" |
||||||
|
#include "util/perf_level_imp.h" |
||||||
|
#include "util/stop_watch.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class PerfStepTimer { |
||||||
|
public: |
||||||
|
PerfStepTimer(uint64_t* metric) |
||||||
|
: enabled_(perf_level >= PerfLevel::kEnableTime), |
||||||
|
env_(enabled_ ? Env::Default() : nullptr), |
||||||
|
start_(0), |
||||||
|
metric_(metric) { |
||||||
|
} |
||||||
|
|
||||||
|
~PerfStepTimer() { |
||||||
|
Stop(); |
||||||
|
} |
||||||
|
|
||||||
|
void Start() { |
||||||
|
if (enabled_) { |
||||||
|
start_ = env_->NowNanos(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
void Measure() { |
||||||
|
if (start_) { |
||||||
|
uint64_t now = env_->NowNanos(); |
||||||
|
*metric_ += now - start_; |
||||||
|
start_ = now; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
void Stop() { |
||||||
|
if (start_) { |
||||||
|
*metric_ += env_->NowNanos() - start_; |
||||||
|
start_ = 0; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private: |
||||||
|
const bool enabled_; |
||||||
|
Env* const env_; |
||||||
|
uint64_t start_; |
||||||
|
uint64_t* metric_; |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
Loading…
Reference in new issue