From 0dc3040d5463bf11436e3a0c8a8d975ebd134145 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Wed, 31 May 2017 22:41:44 -0700 Subject: [PATCH] db: avoid `#include`ing malloc and jemalloc simultaneously Summary: This fixes a compilation failure on Linux when the system libc is not glibc. jemalloc's configure script incorrectly assumes that glibc is always used on Linux systems, producing glibc-style signatures; when the system libc is e.g. musl, the following error is observed: ``` [ 0%] Building CXX object CMakeFiles/rocksdb.dir/db/db_impl.cc.o In file included from /go/src/github.com/cockroachdb/cockroach/c-deps/rocksdb.src/table/block.h:19:0, from /go/src/github.com/cockroachdb/cockroach/c-deps/rocksdb.src/db/db_impl.cc:77: /x-tools/x86_64-unknown-linux-musl/x86_64-unknown-linux-musl/sysroot/usr/include/malloc.h:19:8: error: declaration of 'size_t malloc_usable_size(void*)' has a different exception specifier size_t malloc_usable_size(void *); ^~~~~~~~~~~~~~~~~~ In file included from /go/src/github.com/cockroachdb/cockroach/c-deps/rocksdb.src/db/db_impl.cc:20:0: /go/native/x86_64-unknown-linux-musl/jemalloc/include/jemalloc/jemalloc.h:78:33: note: from previous declaration 'size_t malloc_usable_size(void*) throw ()' # define je_malloc_usable_size malloc_usable_size ^ /go/native/x86_64-unknown-linux-musl/jemalloc/include/jemalloc/jemalloc.h:239:41: note: in expansion of macro 'je_malloc_usable_size' JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_malloc_usable_size( ^~~~~~~~~~~~~~~~~~~~~ CMakeFiles/rocksdb.dir/build.make:350: recipe for target 'CMakeFiles/rocksdb.dir/db/db_impl.cc.o' failed ``` This works around the issue by rearranging the sources such that jemalloc's headers are never in the same scope as the system's malloc header. The jemalloc issue has been reported as well, see: https://github.com/jemalloc/jemalloc/issues/778. cc tschottdorf Closes https://github.com/facebook/rocksdb/pull/2188 Differential Revision: D5163048 Pulled By: siying fbshipit-source-id: c553125458892def175c1be5682b0330d80b2a0d --- CMakeLists.txt | 1 + TARGETS | 1 + db/db_impl.cc | 37 +-------------------------------- db/malloc_stats.cc | 52 ++++++++++++++++++++++++++++++++++++++++++++++ db/malloc_stats.h | 22 ++++++++++++++++++++ src.mk | 1 + 6 files changed, 78 insertions(+), 36 deletions(-) create mode 100644 db/malloc_stats.cc create mode 100644 db/malloc_stats.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 561a5e807..15784fa1a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -358,6 +358,7 @@ set(SOURCES db/internal_stats.cc db/log_reader.cc db/log_writer.cc + db/malloc_stats.cc db/managed_iterator.cc db/memtable.cc db/memtable_list.cc diff --git a/TARGETS b/TARGETS index 4f5ee1c01..7a5c8f0eb 100644 --- a/TARGETS +++ b/TARGETS @@ -83,6 +83,7 @@ cpp_library( "db/internal_stats.cc", "db/log_reader.cc", "db/log_writer.cc", + "db/malloc_stats.cc", "db/managed_iterator.cc", "db/memtable.cc", "db/memtable_list.cc", diff --git a/db/db_impl.cc b/db/db_impl.cc index 765e9d29c..e660c1c93 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -18,9 +18,6 @@ #ifdef OS_SOLARIS #include #endif -#ifdef ROCKSDB_JEMALLOC -#include "jemalloc/jemalloc.h" -#endif #include #include @@ -46,6 +43,7 @@ #include "db/job_context.h" #include "db/log_reader.h" #include "db/log_writer.h" +#include "db/malloc_stats.h" #include "db/managed_iterator.h" #include "db/memtable.h" #include "db/memtable_list.h" @@ -370,39 +368,6 @@ void DBImpl::PrintStatistics() { } } -#ifndef ROCKSDB_LITE -#ifdef ROCKSDB_JEMALLOC -typedef struct { - char* cur; - char* end; -} MallocStatus; - -static void GetJemallocStatus(void* mstat_arg, const char* status) { - MallocStatus* mstat = reinterpret_cast(mstat_arg); - size_t status_len = status ? strlen(status) : 0; - size_t buf_size = (size_t)(mstat->end - mstat->cur); - if (!status_len || status_len > buf_size) { - return; - } - - snprintf(mstat->cur, buf_size, "%s", status); - mstat->cur += status_len; -} -#endif // ROCKSDB_JEMALLOC - -static void DumpMallocStats(std::string* stats) { -#ifdef ROCKSDB_JEMALLOC - MallocStatus mstat; - const unsigned int kMallocStatusLen = 1000000; - std::unique_ptr buf{new char[kMallocStatusLen + 1]}; - mstat.cur = buf.get(); - mstat.end = buf.get() + kMallocStatusLen; - je_malloc_stats_print(GetJemallocStatus, &mstat, ""); - stats->append(buf.get()); -#endif // ROCKSDB_JEMALLOC -} -#endif // !ROCKSDB_LITE - void DBImpl::MaybeDumpStats() { mutex_.Lock(); unsigned int stats_dump_period_sec = diff --git a/db/malloc_stats.cc b/db/malloc_stats.cc new file mode 100644 index 000000000..dbd2ddb7d --- /dev/null +++ b/db/malloc_stats.cc @@ -0,0 +1,52 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "db/malloc_stats.h" + +#ifndef ROCKSDB_LITE +#include +#include + +namespace rocksdb { + +#ifdef ROCKSDB_JEMALLOC +#include "jemalloc/jemalloc.h" + +typedef struct { + char* cur; + char* end; +} MallocStatus; + +static void GetJemallocStatus(void* mstat_arg, const char* status) { + MallocStatus* mstat = reinterpret_cast(mstat_arg); + size_t status_len = status ? strlen(status) : 0; + size_t buf_size = (size_t)(mstat->end - mstat->cur); + if (!status_len || status_len > buf_size) { + return; + } + + snprintf(mstat->cur, buf_size, "%s", status); + mstat->cur += status_len; +} +#endif // ROCKSDB_JEMALLOC + +void DumpMallocStats(std::string* stats) { +#ifdef ROCKSDB_JEMALLOC + MallocStatus mstat; + const unsigned int kMallocStatusLen = 1000000; + std::unique_ptr buf{new char[kMallocStatusLen + 1]}; + mstat.cur = buf.get(); + mstat.end = buf.get() + kMallocStatusLen; + je_malloc_stats_print(GetJemallocStatus, &mstat, ""); + stats->append(buf.get()); +#endif // ROCKSDB_JEMALLOC +} + +} +#endif // !ROCKSDB_LITE diff --git a/db/malloc_stats.h b/db/malloc_stats.h new file mode 100644 index 000000000..212d447e2 --- /dev/null +++ b/db/malloc_stats.h @@ -0,0 +1,22 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#pragma once + +#ifndef ROCKSDB_LITE + +#include + +namespace rocksdb { + +void DumpMallocStats(std::string*); + +} + +#endif // !ROCKSDB_LITE diff --git a/src.mk b/src.mk index c5645080a..b4ee808f2 100644 --- a/src.mk +++ b/src.mk @@ -35,6 +35,7 @@ LIB_SOURCES = \ db/internal_stats.cc \ db/log_reader.cc \ db/log_writer.cc \ + db/malloc_stats.cc \ db/managed_iterator.cc \ db/memtable.cc \ db/memtable_list.cc \