NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use Libraries in the RocksDB Makefile Build (#6660)

Browse Source 
Summary:
Change the linking of tests/tools to be against a library rather than a list of objects.  This change substantially reduces the size of the objects produced.

peterd clean repo size: 264M
Before this change, with make all: 40G
After this change, with make all: 28G
With make LIB_MODE=shared all: 7.0G

The list of TESTS was changed from being hard-coded to generated from the test sources variable.  Note that there are some test sources that are not built as tests (though the set of tests is identical to the previous version).

Added OBJ_DIR option to Makefile to allow objects to be placed in an alternative location.  By default, OBJ_DIR is the same as before ("./").

This change is a precursor to being able to build/run the tests/tools linked against static libraries.  Additionally, it should be possible to clean up and merge some of the rules for building tests and the like if so desired.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6660

Reviewed By: riversand963

Differential Revision: D22244463

Pulled By: pdillinger

fbshipit-source-id: db9c6341d81ed62c2270374f4ede02fb9604c754
main
mrambacher 4 years ago committed by Facebook GitHub Bot
parent 6a243b3ade
commit 80f71b5863
6 changed files with 478 additions and 636 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 918
      Makefile
  2. 67
      buckifier/buckify_rocksdb.py
  3. 36
      build_tools/build_detect_platform
  4. 4
      db/external_sst_file_test.cc
  5. 3
      db_stress_tool/db_stress_tool.cc
  6. 86
      src.mk

918
Makefile
View File

File diff suppressed because it is too large Load Diff

67
buckifier/buckify_rocksdb.py
Unescape View File

@ -48,8 +48,8 @@ def parse_src_mk(repo_path):
if '=' in line:
current_src = line.split('=')[0].strip()
src_files[current_src] = []
elif '.cc' in line:
src_path = line.split('.cc')[0].strip() + '.cc'
elif '.c' in line:
src_path = line.split('\\')[0].strip()
src_files[current_src].append(src_path)
return src_files
@ -69,27 +69,11 @@ def get_cc_files(repo_path):
return cc_files
# Get tests from Makefile
def get_tests(repo_path):
# Get parallel tests from Makefile
def get_parallel_tests(repo_path):
Makefile = repo_path + "/Makefile"
# Dictionary TEST_NAME => IS_PARALLEL
tests = {}
found_tests = False
for line in open(Makefile):
line = line.strip()
if line.startswith("TESTS ="):
found_tests = True
elif found_tests:
if line.endswith("\\"):
# remove the trailing \
line = line[:-1]
line = line.strip()
tests[line] = False
else:
# we consumed all the tests
break
s = set({})
found_parallel_tests = False
for line in open(Makefile):
@ -101,12 +85,12 @@ def get_tests(repo_path):
# remove the trailing \
line = line[:-1]
line = line.strip()
tests[line] = True
s.add(line)
else:
# we consumed all the parallel tests
break
return tests
return s
# Parse extra dependencies passed by user from command line
def get_dependencies():
@ -139,10 +123,10 @@ def generate_targets(repo_path, deps_map):
src_mk = parse_src_mk(repo_path)
# get all .cc files
cc_files = get_cc_files(repo_path)
# get tests from Makefile
tests = get_tests(repo_path)
# get parallel tests from Makefile
parallel_tests = get_parallel_tests(repo_path)
if src_mk is None or cc_files is None or tests is None:
if src_mk is None or cc_files is None or parallel_tests is None:
return False
TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path)
@ -178,32 +162,35 @@ def generate_targets(repo_path, deps_map):
+ ["test_util/testutil.cc"])
print("Extra dependencies:\n{0}".format(json.dumps(deps_map)))
# test for every test we found in the Makefile
# Dictionary test executable name -> relative source file path
test_source_map = {}
print(src_mk)
test_main_sources = src_mk.get("TEST_MAIN_SOURCES", []) + \
src_mk.get("TEST_MAIN_SOURCES_C", [])
for test_src in test_main_sources:
test = test_src.split('.c')[0].strip().split('/')[-1].strip()
test_source_map[test] = test_src
print("" + test + " " + test_src)
for target_alias, deps in deps_map.items():
for test in sorted(tests):
match_src = [src for src in cc_files if ("/%s.c" % test) in src]
if len(match_src) == 0:
print(ColorString.warning("Cannot find .cc file for %s" % test))
continue
elif len(match_src) > 1:
print(ColorString.warning("Found more than one .cc for %s" % test))
print(match_src)
for test, test_src in sorted(test_source_map.items()):
if len(test) == 0:
print(ColorString.warning("Failed to get test name for %s" % test_src))
continue
assert(len(match_src) == 1)
is_parallel = tests[test]
test_target_name = \
test if not target_alias else test + "_" + target_alias
TARGETS.register_test(
test_target_name,
match_src[0],
is_parallel,
test_src,
test in parallel_tests,
json.dumps(deps['extra_deps']),
json.dumps(deps['extra_compiler_flags']))
if test in _EXPORTED_TEST_LIBS:
test_library = "%s_lib" % test_target_name
TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"])
TARGETS.add_library(test_library, [test_src], [":rocksdb_test_lib"])
TARGETS.flush_tests()
print(ColorString.info("Generated TARGETS Summary:"))

36
build_tools/build_detect_platform
Unescape View File

@ -151,7 +151,7 @@ case "$TARGET_OS" in
else
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
fi
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt -ldl"
if test $ROCKSDB_USE_IO_URING; then
# check for liburing
$CXX $CFLAGS -x c++ - -luring -o /dev/null 2>/dev/null <<EOF
@ -296,24 +296,32 @@ EOF
# Test whether gflags library is installed
# http://gflags.github.io/gflags/
# check if the namespace is gflags
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
if $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
#include <gflags/gflags.h>
using namespace GFLAGS_NAMESPACE;
int main() {}
EOF
if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
else
# check if namespace is google
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
then
COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
# check if namespace is gflags
elif $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
#include <gflags/gflags.h>
using namespace gflags;
int main() {}
EOF
then
COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1 -DGFLAGS_NAMESPACE=gflags"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
# check if namespace is google
elif $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
#include <gflags/gflags.h>
using namespace google;
int main() {}
EOF
if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=google"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
fi
then
COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1 -DGFLAGS_NAMESPACE=google"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
fi
fi
@ -441,7 +449,7 @@ EOF
COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_MALLOC_USABLE_SIZE"
fi
fi
if ! test $ROCKSDB_DISABLE_MEMKIND; then
# Test whether memkind library is installed
$CXX $CFLAGS $COMMON_FLAGS -lmemkind -x c++ - -o /dev/null 2>/dev/null <<EOF
@ -457,7 +465,7 @@ EOF
JAVA_LDFLAGS="$JAVA_LDFLAGS -lmemkind"
fi
fi
if ! test $ROCKSDB_DISABLE_PTHREAD_MUTEX_ADAPTIVE_NP; then
# Test whether PTHREAD_MUTEX_ADAPTIVE_NP mutex type is available
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF

4
db/external_sst_file_test.cc
Unescape View File

@ -2854,7 +2854,7 @@ TEST_P(ExternalSSTFileTest, DeltaEncodingWhileGlobalSeqnoPresent) {
auto snap = dbfull()->GetSnapshot();
std::string fname = sst_files_dir_ + "test_file";
rocksdb::SstFileWriter writer(EnvOptions(), options);
ROCKSDB_NAMESPACE::SstFileWriter writer(EnvOptions(), options);
ASSERT_OK(writer.Open(fname));
std::string key1 = "ab";
std::string key2 = "ab";
@ -2904,7 +2904,7 @@ TEST_P(ExternalSSTFileTest,
ASSERT_OK(Put(key0, value));
std::string fname = sst_files_dir_ + "test_file";
rocksdb::SstFileWriter writer(EnvOptions(), options);
ROCKSDB_NAMESPACE::SstFileWriter writer(EnvOptions(), options);
ASSERT_OK(writer.Open(fname));
// key0 is a dummy to ensure the turnaround point (key1) comes from Prev

3
db_stress_tool/db_stress_tool.cc
Unescape View File

@ -45,10 +45,11 @@ int db_stress_tool(int argc, char** argv) {
SanitizeDoubleParam(&FLAGS_memtable_prefix_bloom_size_ratio);
SanitizeDoubleParam(&FLAGS_max_bytes_for_level_multiplier);
#ifndef NDEBUG
if (FLAGS_mock_direct_io) {
test::SetupSyncPointsToMockDirectIO();
}
#endif
if (FLAGS_statistics) {
dbstats = ROCKSDB_NAMESPACE::CreateDBStatistics();
if (FLAGS_test_secondary) {

86
src.mk
Unescape View File

@ -15,13 +15,13 @@ LIB_SOURCES = \
db/c.cc \
db/column_family.cc \
db/compacted_db_impl.cc \
db/compaction/compaction.cc \
db/compaction/compaction.cc \
db/compaction/compaction_iterator.cc \
db/compaction/compaction_job.cc \
db/compaction/compaction_picker.cc \
db/compaction/compaction_picker_fifo.cc \
db/compaction/compaction_picker_level.cc \
db/compaction/compaction_picker_universal.cc \
db/compaction/compaction_picker_universal.cc \
db/convenience.cc \
db/db_filesnapshot.cc \
db/db_impl/db_impl.cc \
@ -36,7 +36,7 @@ LIB_SOURCES = \
db/db_info_dumper.cc \
db/db_iter.cc \
db/dbformat.cc \
db/error_handler.cc \
db/error_handler.cc \
db/event_helpers.cc \
db/experimental.cc \
db/external_sst_file_ingestion_job.cc \
@ -77,7 +77,7 @@ LIB_SOURCES = \
env/env_hdfs.cc \
env/env_posix.cc \
env/file_system.cc \
env/fs_posix.cc \
env/fs_posix.cc \
env/io_posix.cc \
env/mock_env.cc \
file/delete_scheduler.cc \
@ -150,7 +150,7 @@ LIB_SOURCES = \
table/block_based/partitioned_index_reader.cc \
table/block_based/reader_common.cc \
table/block_based/uncompression_dict_reader.cc \
table/block_fetcher.cc \
table/block_fetcher.cc \
table/cuckoo/cuckoo_table_builder.cc \
table/cuckoo/cuckoo_table_factory.cc \
table/cuckoo/cuckoo_table_reader.cc \
@ -191,7 +191,7 @@ LIB_SOURCES = \
util/random.cc \
util/rate_limiter.cc \
util/slice.cc \
util/file_checksum_helper.cc \
util/file_checksum_helper.cc \
util/status.cc \
util/string_util.cc \
util/thread_local.cc \
@ -216,7 +216,7 @@ LIB_SOURCES = \
utilities/memory/memory_util.cc \
utilities/merge_operators/max.cc \
utilities/merge_operators/put.cc \
utilities/merge_operators/sortlist.cc \
utilities/merge_operators/sortlist.cc \
utilities/merge_operators/string_append/stringappend.cc \
utilities/merge_operators/string_append/stringappend2.cc \
utilities/merge_operators/uint64add.cc \
@ -277,7 +277,7 @@ ANALYZER_LIB_SOURCES = \
MOCK_LIB_SOURCES = \
table/mock_table.cc \
test_util/fault_injection_test_fs.cc \
test_util/fault_injection_test_fs.cc \
test_util/fault_injection_test_env.cc
BENCH_LIB_SOURCES = \
@ -307,10 +307,32 @@ FOLLY_SOURCES = \
third-party/folly/folly/synchronization/ParkingLot.cpp \
third-party/folly/folly/synchronization/WaitOptions.cpp \
MAIN_SOURCES = \
TOOLS_MAIN_SOURCES = \
db_stress_tool/db_stress.cc \
tools/blob_dump.cc \
tools/block_cache_analyzer/block_cache_trace_analyzer_tool.cc \
tools/db_repl_stress.cc \
tools/db_sanity_test.cc \
tools/ldb.cc \
tools/sst_dump.cc \
tools/write_stress.cc \
tools/dump/rocksdb_dump.cc \
tools/dump/rocksdb_undump.cc \
tools/trace_analyzer.cc \
BENCH_MAIN_SOURCES = \
cache/cache_bench.cc \
db/range_del_aggregator_bench.cc \
memtable/memtablerep_bench.cc \
table/table_reader_bench.cc \
tools/db_bench.cc \
util/filter_bench.cc \
utilities/persistent_cache/persistent_cache_bench.cc \
#util/log_write_bench.cc \
TEST_MAIN_SOURCES = \
cache/cache_test.cc \
db_stress_tool/db_stress.cc \
cache/lru_cache_test.cc \
db/blob/blob_file_addition_test.cc \
db/blob/blob_file_garbage_test.cc \
db/blob/db_blob_index_test.cc \
@ -324,7 +346,7 @@ MAIN_SOURCES = \
db/corruption_test.cc \
db/cuckoo_table_db_test.cc \
db/db_basic_test.cc \
db/db_with_timestamp_basic_test.cc \
db/db_with_timestamp_basic_test.cc \
db/db_block_cache_test.cc \
db/db_bloom_filter_test.cc \
db/db_compaction_filter_test.cc \
@ -332,6 +354,7 @@ MAIN_SOURCES = \
db/db_dynamic_level_test.cc \
db/db_encryption_test.cc \
db/db_flush_test.cc \
db/import_column_family_test.cc \
db/db_inplace_update_test.cc \
db/db_io_failure_test.cc \
db/db_iter_test.cc \
@ -340,7 +363,7 @@ MAIN_SOURCES = \
db/db_log_iter_test.cc \
db/db_memtable_test.cc \
db/db_merge_operator_test.cc \
db/db_merge_operand_test.cc \
db/db_merge_operand_test.cc \
db/db_options_test.cc \
db/db_properties_test.cc \
db/db_range_del_test.cc \
@ -358,8 +381,7 @@ MAIN_SOURCES = \
db/db_write_test.cc \
db/dbformat_test.cc \
db/deletefile_test.cc \
db/env_timed_test.cc \
db/error_handler_fs_test.cc \
db/error_handler_fs_test.cc \
db/external_sst_file_basic_test.cc \
db/external_sst_file_test.cc \
db/fault_injection_test.cc \
@ -367,29 +389,21 @@ MAIN_SOURCES = \
db/file_reader_writer_test.cc \
db/filename_test.cc \
db/flush_job_test.cc \
db/hash_table_test.cc \
db/hash_test.cc \
db/heap_test.cc \
db/listener_test.cc \
db/log_test.cc \
db/lru_cache_test.cc \
db/manual_compaction_test.cc \
db/memtable_list_test.cc \
db/merge_helper_test.cc \
db/merge_test.cc \
db/obsolete_files_test.cc \
db/options_settable_test.cc \
db/obsolete_files_test.cc \
db/options_file_test.cc \
db/perf_context_test.cc \
db/persistent_cache_test.cc \
db/plain_table_db_test.cc \
db/prefix_test.cc \
db/repair_test.cc \
db/range_del_aggregator_test.cc \
db/range_del_aggregator_bench.cc \
db/range_tombstone_fragmenter_test.cc \
db/table_properties_collector_test.cc \
db/util_merge_operators_test.cc \
db/version_builder_test.cc \
db/version_edit_test.cc \
db/version_set_test.cc \
@ -401,6 +415,7 @@ MAIN_SOURCES = \
env/env_test.cc \
env/io_posix_test.cc \
env/mock_env_test.cc \
file/delete_scheduler_test.cc \
file/random_access_file_reader_test.cc \
logging/auto_roll_logger_test.cc \
logging/env_logger_test.cc \
@ -408,13 +423,13 @@ MAIN_SOURCES = \
memory/arena_test.cc \
memory/memkind_kmem_allocator_test.cc \
memtable/inlineskiplist_test.cc \
memtable/memtablerep_bench.cc \
memtable/skiplist_test.cc \
memtable/write_buffer_manager_test.cc \
monitoring/histogram_test.cc \
monitoring/iostats_context_test.cc \
monitoring/statistics_test.cc \
monitoring/stats_history_test.cc \
options/options_settable_test.cc \
options/options_test.cc \
table/block_based/block_based_filter_block_test.cc \
table/block_based/block_based_table_reader_test.cc \
@ -427,19 +442,14 @@ MAIN_SOURCES = \
table/cuckoo/cuckoo_table_reader_test.cc \
table/merger_test.cc \
table/sst_file_reader_test.cc \
table/table_reader_bench.cc \
table/table_test.cc \
table/block_fetcher_test.cc \
third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc \
test_util/testutil_test.cc \
tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc \
tools/block_cache_analyzer/block_cache_trace_analyzer_tool.cc \
tools/db_bench.cc \
tools/db_bench_tool_test.cc \
tools/db_sanity_test.cc \
tools/ldb_cmd_test.cc \
tools/reduce_levels_test.cc \
tools/sst_dump_test.cc \
tools/trace_analyzer_test.cc \
tools/trace_analyzer_test.cc \
trace_replay/block_cache_tracer_test.cc \
trace_replay/io_tracer_test.cc \
util/autovector_test.cc \
@ -449,9 +459,11 @@ MAIN_SOURCES = \
util/defer_test.cc \
util/dynamic_bloom_test.cc \
util/filelock_test.cc \
util/log_write_bench.cc \
util/rate_limiter_test.cc \
util/file_reader_writer_test.cc \
util/hash_test.cc \
util/heap_test.cc \
util/random_test.cc \
util/rate_limiter_test.cc \
util/repeatable_thread_test.cc \
util/slice_test.cc \
util/slice_transform_test.cc \
@ -467,11 +479,14 @@ MAIN_SOURCES = \
utilities/cassandra/cassandra_row_merge_test.cc \
utilities/cassandra/cassandra_serialize_test.cc \
utilities/checkpoint/checkpoint_test.cc \
utilities/env_timed_test.cc \
utilities/memory/memory_test.cc \
utilities/merge_operators/string_append/stringappend_test.cc \
utilities/object_registry_test.cc \
utilities/option_change_migration/option_change_migration_test.cc \
utilities/options/options_util_test.cc \
utilities/persistent_cache/hash_table_test.cc \
utilities/persistent_cache/persistent_cache_test.cc \
utilities/simulator_cache/cache_simulator_test.cc \
utilities/simulator_cache/sim_cache_test.cc \
utilities/table_properties_collectors/compact_on_deletion_collector_test.cc \
@ -481,8 +496,13 @@ MAIN_SOURCES = \
utilities/transactions/write_prepared_transaction_test.cc \
utilities/transactions/write_unprepared_transaction_test.cc \
utilities/ttl/ttl_test.cc \
utilities/util_merge_operators_test.cc \
utilities/write_batch_with_index/write_batch_with_index_test.cc \
TEST_MAIN_SOURCES_C = \
db/c_test.c \
JNI_NATIVE_SOURCES = \
java/rocksjni/backupenginejni.cc \
java/rocksjni/backupablejni.cc \

Write Preview
Loading…
Cancel
Save