Print memory allocation counters

Summary:
Introduced option to dump malloc statistics using new option flag.
    Added new command line option to db_bench tool to enable this
    funtionality.
    Also extended build to support environments with/without jemalloc.

Test Plan:
1) Build rocksdb using `make` command. Launch the following command
    `./db_bench --benchmarks=fillrandom --dump_malloc_stats=true
    --num=10000000` end verified that jemalloc dump is present in LOG file.
    2) Build rocksdb using `DISABLE_JEMALLOC=1  make db_bench -j32` and ran
    the same db_bench tool and found the following message in LOG file:
    "Please compile with jemalloc to enable malloc dump".
    3) Also built rocksdb using `make` command on MacOS to verify behavior
    in non-FB environment.
    Also to debug build configuration change temporary changed
    AM_DEFAULT_VERBOSITY = 1 in Makefile to see compiler and build
    tools output. For case 1) -DROCKSDB_JEMALLOC was present in compiler
    command line. For both 2) and 3) this flag was not present.

Reviewers: sdong

Reviewed By: sdong

Subscribers: andrewkr, dhruba

Differential Revision: https://reviews.facebook.net/D57321
main
Sergey Makarenko 9 years ago
parent eb73980853
commit 1c80dfab24
  1. 5
      Makefile
  2. 9
      build_tools/build_detect_platform
  3. 38
      db/db_impl.cc
  4. 5
      include/rocksdb/options.h
  5. 3
      tools/db_bench_tool.cc
  6. 6
      util/options.cc
  7. 3
      util/options_helper.h
  8. 3
      util/options_settable_test.cc

@ -182,8 +182,11 @@ ifdef COMPILE_WITH_UBSAN
PLATFORM_CXXFLAGS += -fsanitize=undefined
endif
ifndef DISABLE_JEMALLOC
ifdef JEMALLOC
PLATFORM_CXXFLAGS += "-DROCKSDB_JEMALLOC"
PLATFORM_CCFLAGS += "-DROCKSDB_JEMALLOC"
endif
EXEC_LDFLAGS := $(JEMALLOC_LIB) $(EXEC_LDFLAGS)
PLATFORM_CXXFLAGS += $(JEMALLOC_INCLUDE)
PLATFORM_CCFLAGS += $(JEMALLOC_INCLUDE)

@ -299,13 +299,14 @@ EOF
# Test whether jemalloc is available
if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null -ljemalloc \
2>/dev/null; then
2>/dev/null; then
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ljemalloc"
JAVA_LDFLAGS="$JAVA_LDFLAGS -ljemalloc"
JEMALLOC=1
else
# jemalloc is not available. Let's try tcmalloc
if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null \
-ltcmalloc 2>/dev/null; then
-ltcmalloc 2>/dev/null; then
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc"
JAVA_LDFLAGS="$JAVA_LDFLAGS -ltcmalloc"
fi
@ -452,4 +453,6 @@ echo "ROCKSDB_PATCH=$ROCKSDB_PATCH" >> "$OUTPUT"
echo "CLANG_SCAN_BUILD=$CLANG_SCAN_BUILD" >> "$OUTPUT"
echo "CLANG_ANALYZER=$CLANG_ANALYZER" >> "$OUTPUT"
echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT"
if test -z "$JEMALLOC"; then
echo "JEMALLOC=1" >> "$OUTPUT"
fi

@ -12,12 +12,14 @@
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS
#endif
#include <inttypes.h>
#include <stdint.h>
#ifdef OS_SOLARIS
#include <alloca.h>
#endif
#ifdef ROCKSDB_JEMALLOC
#include "jemalloc/jemalloc.h"
#endif
#include <algorithm>
#include <climits>
@ -534,6 +536,37 @@ void DBImpl::PrintStatistics() {
}
}
#ifdef ROCKSDB_JEMALLOC
typedef struct {
char* cur;
char* end;
} MallocStatus;
static void GetJemallocStatus(void* mstat_arg, const char* status) {
MallocStatus* mstat = reinterpret_cast<MallocStatus*>(mstat_arg);
size_t status_len = status ? strlen(status) : 0;
size_t buf_size = (size_t)(mstat->end - mstat->cur);
if (!status_len || status_len > buf_size) {
return;
}
snprintf(mstat->cur, buf_size, status);
mstat->cur += status_len;
}
#endif // ROCKSDB_JEMALLOC
static void DumpMallocStats(std::string* stats) {
#ifdef ROCKSDB_JEMALLOC
MallocStatus mstat;
const uint kMallocStatusLen = 1000000;
std::unique_ptr<char> buf{new char[kMallocStatusLen + 1]};
mstat.cur = buf.get();
mstat.end = buf.get() + kMallocStatusLen;
malloc_stats_print(GetJemallocStatus, &mstat, "");
stats->append(buf.get());
#endif // ROCKSDB_JEMALLOC
}
void DBImpl::MaybeDumpStats() {
if (db_options_.stats_dump_period_sec == 0) return;
@ -566,6 +599,9 @@ void DBImpl::MaybeDumpStats() {
default_cf_internal_stats_->GetStringProperty(
*db_property_info, DB::Properties::kDBStats, &stats);
}
if (db_options_.dump_malloc_stats) {
DumpMallocStats(&stats);
}
Log(InfoLogLevel::WARN_LEVEL,
db_options_.info_log, "------- DUMPING STATS -------");
Log(InfoLogLevel::WARN_LEVEL,

@ -1305,6 +1305,11 @@ struct DBOptions {
//
// DEFAULT: false
bool fail_if_options_file_error;
// If true, then print malloc stats together with rocksdb.stats
// when printing to LOG.
// DEFAULT: false
bool dump_malloc_stats;
};
// Options to control the behavior of a database (passed to DB::Open)

@ -753,6 +753,7 @@ DEFINE_bool(enable_io_prio, false, "Lower the background flush/compaction "
DEFINE_bool(identity_as_first_hash, false, "the first hash function of cuckoo "
"table becomes an identity function. This is only valid when key "
"is 8 bytes");
DEFINE_bool(dump_malloc_stats, true, "Dump malloc stats in LOG ");
enum RepFactory {
kSkipList,
@ -2662,6 +2663,8 @@ class Benchmark {
if (FLAGS_min_level_to_compress >= 0) {
options.compression_per_level.clear();
}
options.dump_malloc_stats = FLAGS_dump_malloc_stats;
}
void OpenDb(const Options& options, const std::string& db_name,

@ -272,7 +272,8 @@ DBOptions::DBOptions()
#ifndef ROCKSDB_LITE
wal_filter(nullptr),
#endif // ROCKSDB_LITE
fail_if_options_file_error(false) {
fail_if_options_file_error(false),
dump_malloc_stats(false) {
}
DBOptions::DBOptions(const Options& options)
@ -341,7 +342,8 @@ DBOptions::DBOptions(const Options& options)
#ifndef ROCKSDB_LITE
wal_filter(options.wal_filter),
#endif // ROCKSDB_LITE
fail_if_options_file_error(options.fail_if_options_file_error) {
fail_if_options_file_error(options.fail_if_options_file_error),
dump_malloc_stats(options.dump_malloc_stats) {
}
static const char* const access_hints[] = {

@ -313,6 +313,9 @@ static std::unordered_map<std::string, OptionTypeInfo> db_options_type_info = {
OptionType::kAccessHint, OptionVerificationType::kNormal}},
{"info_log_level",
{offsetof(struct DBOptions, info_log_level), OptionType::kInfoLogLevel,
OptionVerificationType::kNormal}},
{"dump_malloc_stats",
{offsetof(struct DBOptions, dump_malloc_stats), OptionType::kBoolean,
OptionVerificationType::kNormal}}};
static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {

@ -278,7 +278,8 @@ TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) {
"write_thread_slow_yield_usec=5;"
"write_thread_max_yield_usec=1000;"
"access_hint_on_compaction_start=NONE;"
"info_log_level=DEBUG_LEVEL;",
"info_log_level=DEBUG_LEVEL;"
"dump_malloc_stats=false;",
new_options));
ASSERT_EQ(unset_bytes_base, NumUnsetBytes(new_options_ptr, sizeof(DBOptions),

Loading…
Cancel
Save