From a1068c91a1bd00bb3b209cac63ff93bb8780df27 Mon Sep 17 00:00:00 2001 From: Igor Canadi Date: Thu, 8 May 2014 17:25:13 -0700 Subject: [PATCH] Make RocksDB work with newer gflags Summary: Newer gflags switched from `google` namespace to `gflags` namespace. See: https://github.com/facebook/rocksdb/issues/139 and https://github.com/facebook/rocksdb/issues/102 Unfortunately, they don't define any macro with their namespace, so we need to actually try to compile gflags with two different namespace to figure out which one is the correct one. Test Plan: works in fbcode environemnt. I'll also try in ubutnu with newer gflags Reviewers: dhruba Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D18537 --- build_tools/build_detect_platform | 15 ++++++++- build_tools/fbcode.gcc471.sh | 2 +- build_tools/fbcode.gcc481.sh | 2 +- db/db_bench.cc | 49 +++++++++++++++++------------ db/prefix_test.cc | 14 ++++++++- tools/db_repl_stress.cc | 25 +++++++++++---- tools/db_stress.cc | 51 +++++++++++++++++++------------ 7 files changed, 108 insertions(+), 50 deletions(-) diff --git a/build_tools/build_detect_platform b/build_tools/build_detect_platform index ec37a999d..dc4925cfa 100755 --- a/build_tools/build_detect_platform +++ b/build_tools/build_detect_platform @@ -219,12 +219,25 @@ EOF # Test whether gflags library is installed # http://code.google.com/p/gflags/ + # check if the namespace is gflags $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null < + using namespace gflags; int main() {} EOF if [ "$?" = 0 ]; then - COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS" + COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=gflags" + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags" + fi + + # check if namespace is google + $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null < + using namespace google; + int main() {} +EOF + if [ "$?" = 0 ]; then + COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=google" PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags" fi diff --git a/build_tools/fbcode.gcc471.sh b/build_tools/fbcode.gcc471.sh index 929405715..c971cda5b 100644 --- a/build_tools/fbcode.gcc471.sh +++ b/build_tools/fbcode.gcc471.sh @@ -55,7 +55,7 @@ CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/gold -m64 CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC" CFLAGS+=" $LIBGCC_INCLUDE $GLIBC_INCLUDE" CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_ATOMIC_PRESENT -DROCKSDB_FALLOCATE_PRESENT" -CFLAGS+=" -DSNAPPY -DGFLAGS -DZLIB -DBZIP2" +CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2" EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a" EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/350336c/lib/libunwind.a" diff --git a/build_tools/fbcode.gcc481.sh b/build_tools/fbcode.gcc481.sh index d02596f05..c1399099c 100644 --- a/build_tools/fbcode.gcc481.sh +++ b/build_tools/fbcode.gcc481.sh @@ -66,7 +66,7 @@ RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/gold -m64 -mtune=generic" CFLAGS+=" $LIBGCC_INCLUDE $GLIBC_INCLUDE" CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_ATOMIC_PRESENT -DROCKSDB_FALLOCATE_PRESENT" -CFLAGS+=" -DSNAPPY -DGFLAGS -DZLIB -DBZIP2 -DLZ4" +CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4" EXEC_LDFLAGS="-Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so" EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/675d945/lib/libunwind.a" diff --git a/db/db_bench.cc b/db/db_bench.cc index 2e8da9e48..115443010 100644 --- a/db/db_bench.cc +++ b/db/db_bench.cc @@ -8,6 +8,15 @@ // found in the LICENSE file. See the AUTHORS file for names of contributors. #define __STDC_FORMAT_MACROS + +#ifndef GFLAGS +#include +int main() { + fprintf(stderr, "Please install gflags to run rocksdb tools\n"); + return 1; +} +#else + #include #include #include @@ -40,6 +49,9 @@ #include "hdfs/env_hdfs.h" #include "utilities/merge_operators.h" +using GFLAGS::ParseCommandLineFlags; +using GFLAGS::RegisterFlagValidator; +using GFLAGS::SetUsageMessage; DEFINE_string(benchmarks, "fillseq," @@ -376,8 +388,7 @@ static bool ValidateCompressionLevel(const char* flagname, int32_t value) { } static const bool FLAGS_compression_level_dummy __attribute__((unused)) = - google::RegisterFlagValidator(&FLAGS_compression_level, - &ValidateCompressionLevel); + RegisterFlagValidator(&FLAGS_compression_level, &ValidateCompressionLevel); DEFINE_int32(min_level_to_compress, -1, "If non-negative, compression starts" " from this level. Levels with number < min_level_to_compress are" @@ -529,33 +540,29 @@ DEFINE_string(merge_operator, "", "The merge operator to use with the database." " utilities/merge_operators.h"); static const bool FLAGS_soft_rate_limit_dummy __attribute__((unused)) = - google::RegisterFlagValidator(&FLAGS_soft_rate_limit, - &ValidateRateLimit); + RegisterFlagValidator(&FLAGS_soft_rate_limit, &ValidateRateLimit); static const bool FLAGS_hard_rate_limit_dummy __attribute__((unused)) = - google::RegisterFlagValidator(&FLAGS_hard_rate_limit, &ValidateRateLimit); + RegisterFlagValidator(&FLAGS_hard_rate_limit, &ValidateRateLimit); static const bool FLAGS_prefix_size_dummy __attribute__((unused)) = - google::RegisterFlagValidator(&FLAGS_prefix_size, &ValidatePrefixSize); + RegisterFlagValidator(&FLAGS_prefix_size, &ValidatePrefixSize); static const bool FLAGS_key_size_dummy __attribute__((unused)) = - google::RegisterFlagValidator(&FLAGS_key_size, &ValidateKeySize); + RegisterFlagValidator(&FLAGS_key_size, &ValidateKeySize); static const bool FLAGS_cache_numshardbits_dummy __attribute__((unused)) = - google::RegisterFlagValidator(&FLAGS_cache_numshardbits, - &ValidateCacheNumshardbits); + RegisterFlagValidator(&FLAGS_cache_numshardbits, + &ValidateCacheNumshardbits); static const bool FLAGS_readwritepercent_dummy __attribute__((unused)) = - google::RegisterFlagValidator(&FLAGS_readwritepercent, - &ValidateInt32Percent); + RegisterFlagValidator(&FLAGS_readwritepercent, &ValidateInt32Percent); static const bool FLAGS_deletepercent_dummy __attribute__((unused)) = - google::RegisterFlagValidator(&FLAGS_deletepercent, - &ValidateInt32Percent); -static const bool - FLAGS_table_cache_numshardbits_dummy __attribute__((unused)) = - google::RegisterFlagValidator(&FLAGS_table_cache_numshardbits, - &ValidateTableCacheNumshardbits); + RegisterFlagValidator(&FLAGS_deletepercent, &ValidateInt32Percent); +static const bool FLAGS_table_cache_numshardbits_dummy __attribute__((unused)) = + RegisterFlagValidator(&FLAGS_table_cache_numshardbits, + &ValidateTableCacheNumshardbits); namespace rocksdb { @@ -2561,9 +2568,9 @@ class Benchmark { int main(int argc, char** argv) { rocksdb::port::InstallStackTraceHandler(); - google::SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) + - " [OPTIONS]..."); - google::ParseCommandLineFlags(&argc, &argv, true); + SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) + + " [OPTIONS]..."); + ParseCommandLineFlags(&argc, &argv, true); FLAGS_compaction_style_e = (rocksdb::CompactionStyle) FLAGS_compaction_style; if (FLAGS_statistics) { @@ -2614,3 +2621,5 @@ int main(int argc, char** argv) { benchmark.Run(); return 0; } + +#endif // GFLAGS diff --git a/db/prefix_test.cc b/db/prefix_test.cc index 3a88fc8ce..64a4d0617 100644 --- a/db/prefix_test.cc +++ b/db/prefix_test.cc @@ -3,6 +3,14 @@ // LICENSE file in the root directory of this source tree. An additional grant // of patent rights can be found in the PATENTS file in the same directory. +#ifndef GFLAGS +#include +int main() { + fprintf(stderr, "Please install gflags to run rocksdb tools\n"); + return 1; +} +#else + #include #include #include @@ -17,6 +25,8 @@ #include "util/stop_watch.h" #include "util/testharness.h" +using GFLAGS::ParseCommandLineFlags; + DEFINE_bool(trigger_deadlock, false, "issue delete in range scan to trigger PrefixHashMap deadlock"); DEFINE_uint64(bucket_count, 100000, "number of buckets"); @@ -479,9 +489,11 @@ TEST(PrefixTest, DynamicPrefixIterator) { } int main(int argc, char** argv) { - google::ParseCommandLineFlags(&argc, &argv, true); + ParseCommandLineFlags(&argc, &argv, true); std::cout << kDbName << "\n"; rocksdb::test::RunAllTests(); return 0; } + +#endif // GFLAGS diff --git a/tools/db_repl_stress.cc b/tools/db_repl_stress.cc index 27cb6d5ab..5970bb684 100644 --- a/tools/db_repl_stress.cc +++ b/tools/db_repl_stress.cc @@ -2,7 +2,15 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. An additional grant // of patent rights can be found in the PATENTS file in the same directory. -// + +#ifndef GFLAGS +#include +int main() { + fprintf(stderr, "Please install gflags to run rocksdb tools\n"); + return 1; +} +#else + #include #include @@ -13,7 +21,6 @@ #include "port/atomic_pointer.h" #include "util/testutil.h" - // Run a thread to perform Put's. // Another thread uses GetUpdatesSince API to keep getting the updates. // options : @@ -22,6 +29,9 @@ using namespace rocksdb; +using GFLAGS::ParseCommandLineFlags; +using GFLAGS::SetUsageMessage; + struct DataPumpThread { size_t no_records; DB* db; // Assumption DB is Open'ed already. @@ -87,10 +97,11 @@ DEFINE_uint64(wal_size_limit_MB, 10, "the wal size limit for the run" "(in MB)"); int main(int argc, const char** argv) { - google::SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) + - " --num_inserts= --wal_ttl_seconds=" + - " --wal_size_limit_MB="); - google::ParseCommandLineFlags(&argc, const_cast(&argv), true); + SetUsageMessage( + std::string("\nUSAGE:\n") + std::string(argv[0]) + + " --num_inserts= --wal_ttl_seconds=" + + " --wal_size_limit_MB="); + ParseCommandLineFlags(&argc, const_cast(&argv), true); Env* env = Env::Default(); std::string default_db_path; @@ -132,3 +143,5 @@ int main(int argc, const char** argv) { fprintf(stderr, "Successful!\n"); exit(0); } + +#endif // GFLAGS diff --git a/tools/db_stress.cc b/tools/db_stress.cc index c774171d8..e9d759f32 100644 --- a/tools/db_stress.cc +++ b/tools/db_stress.cc @@ -20,6 +20,14 @@ // NOTE that if FLAGS_test_batches_snapshots is set, the test will have // different behavior. See comment of the flag for details. +#ifndef GFLAGS +#include +int main() { + fprintf(stderr, "Please install gflags to run rocksdb tools\n"); + return 1; +} +#else + #include #include #include @@ -45,8 +53,11 @@ #include "hdfs/env_hdfs.h" #include "utilities/merge_operators.h" -static const long KB = 1024; +using GFLAGS::ParseCommandLineFlags; +using GFLAGS::RegisterFlagValidator; +using GFLAGS::SetUsageMessage; +static const long KB = 1024; static bool ValidateUint32Range(const char* flagname, uint64_t value) { if (value > std::numeric_limits::max()) { @@ -61,7 +72,7 @@ static bool ValidateUint32Range(const char* flagname, uint64_t value) { DEFINE_uint64(seed, 2341234, "Seed for PRNG"); static const bool FLAGS_seed_dummy __attribute__((unused)) = - google::RegisterFlagValidator(&FLAGS_seed, &ValidateUint32Range); + RegisterFlagValidator(&FLAGS_seed, &ValidateUint32Range); DEFINE_int64(max_key, 1 * KB* KB, "Max number of key/values to place in database"); @@ -185,7 +196,7 @@ static bool ValidateInt32Positive(const char* flagname, int32_t value) { } DEFINE_int32(reopen, 10, "Number of times database reopens"); static const bool FLAGS_reopen_dummy __attribute__((unused)) = - google::RegisterFlagValidator(&FLAGS_reopen, &ValidateInt32Positive); + RegisterFlagValidator(&FLAGS_reopen, &ValidateInt32Positive); DEFINE_int32(bloom_bits, 10, "Bloom filter bits per key. " "Negative means use default settings."); @@ -213,8 +224,7 @@ DEFINE_int32(kill_random_test, 0, "If non-zero, kill at various points in source code with " "probability 1/this"); static const bool FLAGS_kill_random_test_dummy __attribute__((unused)) = - google::RegisterFlagValidator(&FLAGS_kill_random_test, - &ValidateInt32Positive); + RegisterFlagValidator(&FLAGS_kill_random_test, &ValidateInt32Positive); extern int rocksdb_kill_odds; DEFINE_bool(disable_wal, false, "If true, do not write WAL for write."); @@ -241,32 +251,32 @@ static bool ValidateInt32Percent(const char* flagname, int32_t value) { DEFINE_int32(readpercent, 10, "Ratio of reads to total workload (expressed as a percentage)"); static const bool FLAGS_readpercent_dummy __attribute__((unused)) = - google::RegisterFlagValidator(&FLAGS_readpercent, &ValidateInt32Percent); + RegisterFlagValidator(&FLAGS_readpercent, &ValidateInt32Percent); DEFINE_int32(prefixpercent, 20, "Ratio of prefix iterators to total workload (expressed as a" " percentage)"); static const bool FLAGS_prefixpercent_dummy __attribute__((unused)) = - google::RegisterFlagValidator(&FLAGS_prefixpercent, &ValidateInt32Percent); + RegisterFlagValidator(&FLAGS_prefixpercent, &ValidateInt32Percent); DEFINE_int32(writepercent, 45, " Ratio of deletes to total workload (expressed as a percentage)"); static const bool FLAGS_writepercent_dummy __attribute__((unused)) = - google::RegisterFlagValidator(&FLAGS_writepercent, &ValidateInt32Percent); + RegisterFlagValidator(&FLAGS_writepercent, &ValidateInt32Percent); DEFINE_int32(delpercent, 15, "Ratio of deletes to total workload (expressed as a percentage)"); static const bool FLAGS_delpercent_dummy __attribute__((unused)) = - google::RegisterFlagValidator(&FLAGS_delpercent, &ValidateInt32Percent); + RegisterFlagValidator(&FLAGS_delpercent, &ValidateInt32Percent); DEFINE_int32(iterpercent, 10, "Ratio of iterations to total workload" " (expressed as a percentage)"); static const bool FLAGS_iterpercent_dummy __attribute__((unused)) = - google::RegisterFlagValidator(&FLAGS_iterpercent, &ValidateInt32Percent); + RegisterFlagValidator(&FLAGS_iterpercent, &ValidateInt32Percent); DEFINE_uint64(num_iterations, 10, "Number of iterations per MultiIterate run"); static const bool FLAGS_num_iterations_dummy __attribute__((unused)) = - google::RegisterFlagValidator(&FLAGS_num_iterations, &ValidateUint32Range); + RegisterFlagValidator(&FLAGS_num_iterations, &ValidateUint32Range); DEFINE_bool(disable_seek_compaction, false, "Option to disable compation triggered by read."); @@ -304,19 +314,18 @@ static rocksdb::Env* FLAGS_env = rocksdb::Env::Default(); DEFINE_uint64(ops_per_thread, 1200000, "Number of operations per thread."); static const bool FLAGS_ops_per_thread_dummy __attribute__((unused)) = - google::RegisterFlagValidator(&FLAGS_ops_per_thread, &ValidateUint32Range); + RegisterFlagValidator(&FLAGS_ops_per_thread, &ValidateUint32Range); DEFINE_uint64(log2_keys_per_lock, 2, "Log2 of number of keys per lock"); static const bool FLAGS_log2_keys_per_lock_dummy __attribute__((unused)) = - google::RegisterFlagValidator(&FLAGS_log2_keys_per_lock, - &ValidateUint32Range); + RegisterFlagValidator(&FLAGS_log2_keys_per_lock, &ValidateUint32Range); DEFINE_int32(purge_redundant_percent, 50, "Percentage of times we want to purge redundant keys in memory " "before flushing"); static const bool FLAGS_purge_redundant_percent_dummy __attribute__((unused)) = - google::RegisterFlagValidator(&FLAGS_purge_redundant_percent, - &ValidateInt32Percent); + RegisterFlagValidator(&FLAGS_purge_redundant_percent, + &ValidateInt32Percent); DEFINE_bool(filter_deletes, false, "On true, deletes use KeyMayExist to drop" " the delete if key not present"); @@ -356,7 +365,7 @@ static bool ValidatePrefixSize(const char* flagname, int32_t value) { } DEFINE_int32(prefix_size, 7, "Control the prefix size for HashSkipListRep"); static const bool FLAGS_prefix_size_dummy = - google::RegisterFlagValidator(&FLAGS_prefix_size, &ValidatePrefixSize); + RegisterFlagValidator(&FLAGS_prefix_size, &ValidatePrefixSize); DEFINE_bool(use_merge, false, "On true, replaces all writes with a Merge " "that behaves like a Put"); @@ -1666,9 +1675,9 @@ class StressTest { } // namespace rocksdb int main(int argc, char** argv) { - google::SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) + - " [OPTIONS]..."); - google::ParseCommandLineFlags(&argc, &argv, true); + SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) + + " [OPTIONS]..."); + ParseCommandLineFlags(&argc, &argv, true); if (FLAGS_statistics) { dbstats = rocksdb::CreateDBStatistics(); @@ -1730,3 +1739,5 @@ int main(int argc, char** argv) { return 1; } } + +#endif // GFLAGS