Summary: We occasionally get write stalls (>1s Write() calls) on HDD under read load. The following timers explain almost all of the stalls: - perf_context.db_mutex_lock_nanos - perf_context.db_condition_wait_nanos - iostats_context.open_time - iostats_context.allocate_time - iostats_context.write_time - iostats_context.range_sync_time - iostats_context.logger_time In my experiments each of these occasionally takes >1s on write path under some workload. There are rare cases when Write() takes long but none of these takes long. Test Plan: Added code to our application to write the listed timings to log for slow writes. They usually add up to almost exactly the time Write() call took. Reviewers: rven, yhchiang, sdong Reviewed By: sdong Subscribers: march, dhruba, tnovak Differential Revision: https://reviews.facebook.net/D39177main
Mike Kolupaev
10 years ago
parent
4266d4fd90
commit
ec7a944360
@ -0,0 +1,30 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#ifndef INCLUDE_ROCKSDB_PERF_LEVEL_H_ |
||||
#define INCLUDE_ROCKSDB_PERF_LEVEL_H_ |
||||
|
||||
#include <stdint.h> |
||||
#include <string> |
||||
|
||||
namespace rocksdb { |
||||
|
||||
// How much perf stats to collect. Affects perf_context and iostats_context.
|
||||
|
||||
enum PerfLevel { |
||||
kDisable = 0, // disable perf stats
|
||||
kEnableCount = 1, // enable only count stats
|
||||
kEnableTime = 2 // enable time stats too
|
||||
}; |
||||
|
||||
// set the perf stats level for current thread
|
||||
void SetPerfLevel(PerfLevel level); |
||||
|
||||
// get current perf stats level for current thread
|
||||
PerfLevel GetPerfLevel(); |
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
#endif // INCLUDE_ROCKSDB_PERF_LEVEL_H_
|
||||
@ -0,0 +1,26 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
|
||||
#include <sstream> |
||||
#include "util/perf_level_imp.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
#if defined(IOS_CROSS_COMPILE) |
||||
PerfLevel perf_level = kEnableCount; |
||||
#else |
||||
__thread PerfLevel perf_level = kEnableCount; |
||||
#endif |
||||
|
||||
void SetPerfLevel(PerfLevel level) { |
||||
perf_level = level; |
||||
} |
||||
|
||||
PerfLevel GetPerfLevel() { |
||||
return perf_level; |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,17 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
#pragma once |
||||
#include "rocksdb/perf_level.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
#if defined(IOS_CROSS_COMPILE) |
||||
extern PerfLevel perf_level; |
||||
#else |
||||
extern __thread PerfLevel perf_level; |
||||
#endif |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,54 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
#pragma once |
||||
#include "rocksdb/env.h" |
||||
#include "util/perf_level_imp.h" |
||||
#include "util/stop_watch.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class PerfStepTimer { |
||||
public: |
||||
PerfStepTimer(uint64_t* metric) |
||||
: enabled_(perf_level >= PerfLevel::kEnableTime), |
||||
env_(enabled_ ? Env::Default() : nullptr), |
||||
start_(0), |
||||
metric_(metric) { |
||||
} |
||||
|
||||
~PerfStepTimer() { |
||||
Stop(); |
||||
} |
||||
|
||||
void Start() { |
||||
if (enabled_) { |
||||
start_ = env_->NowNanos(); |
||||
} |
||||
} |
||||
|
||||
void Measure() { |
||||
if (start_) { |
||||
uint64_t now = env_->NowNanos(); |
||||
*metric_ += now - start_; |
||||
start_ = now; |
||||
} |
||||
} |
||||
|
||||
void Stop() { |
||||
if (start_) { |
||||
*metric_ += env_->NowNanos() - start_; |
||||
start_ = 0; |
||||
} |
||||
} |
||||
|
||||
private: |
||||
const bool enabled_; |
||||
Env* const env_; |
||||
uint64_t start_; |
||||
uint64_t* metric_; |
||||
}; |
||||
|
||||
} // namespace rocksdb
|
||||
Loading…
Reference in new issue