NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use Libraries in the RocksDB Makefile Build (#6660)

Browse Source 
Summary:
Change the linking of tests/tools to be against a library rather than a list of objects.  This change substantially reduces the size of the objects produced.

peterd clean repo size: 264M
Before this change, with make all: 40G
After this change, with make all: 28G
With make LIB_MODE=shared all: 7.0G

The list of TESTS was changed from being hard-coded to generated from the test sources variable.  Note that there are some test sources that are not built as tests (though the set of tests is identical to the previous version).

Added OBJ_DIR option to Makefile to allow objects to be placed in an alternative location.  By default, OBJ_DIR is the same as before ("./").

This change is a precursor to being able to build/run the tests/tools linked against static libraries.  Additionally, it should be possible to clean up and merge some of the rules for building tests and the like if so desired.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6660

Reviewed By: riversand963

Differential Revision: D22244463

Pulled By: pdillinger

fbshipit-source-id: db9c6341d81ed62c2270374f4ede02fb9604c754
main
mrambacher 5 years ago committed by Facebook GitHub Bot
parent 6a243b3ade
commit 80f71b5863
6 changed files with 478 additions and 636 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 904
      Makefile
  2. 67
      buckifier/buckify_rocksdb.py
  3. 24
      build_tools/build_detect_platform
  4. 4
      db/external_sst_file_test.cc
  5. 3
      db_stress_tool/db_stress_tool.cc
  6. 60
      src.mk

904
Makefile
View File

File diff suppressed because it is too large Load Diff

67
buckifier/buckify_rocksdb.py
Unescape View File

@ -48,8 +48,8 @@ def parse_src_mk(repo_path):
if '=' in line: if '=' in line:
current_src = line.split('=')[0].strip() current_src = line.split('=')[0].strip()
src_files[current_src] = [] src_files[current_src] = []
elif '.cc' in line: elif '.c' in line:
src_path = line.split('.cc')[0].strip() + '.cc' src_path = line.split('\\')[0].strip()
src_files[current_src].append(src_path) src_files[current_src].append(src_path)
return src_files return src_files
@ -69,27 +69,11 @@ def get_cc_files(repo_path):
return cc_files return cc_files
# Get tests from Makefile # Get parallel tests from Makefile
def get_tests(repo_path): def get_parallel_tests(repo_path):
Makefile = repo_path + "/Makefile" Makefile = repo_path + "/Makefile"
# Dictionary TEST_NAME => IS_PARALLEL s = set({})
tests = {}
found_tests = False
for line in open(Makefile):
line = line.strip()
if line.startswith("TESTS ="):
found_tests = True
elif found_tests:
if line.endswith("\\"):
# remove the trailing \
line = line[:-1]
line = line.strip()
tests[line] = False
else:
# we consumed all the tests
break
found_parallel_tests = False found_parallel_tests = False
for line in open(Makefile): for line in open(Makefile):
@ -101,12 +85,12 @@ def get_tests(repo_path):
# remove the trailing \ # remove the trailing \
line = line[:-1] line = line[:-1]
line = line.strip() line = line.strip()
tests[line] = True s.add(line)
else: else:
# we consumed all the parallel tests # we consumed all the parallel tests
break break
return tests return s
# Parse extra dependencies passed by user from command line # Parse extra dependencies passed by user from command line
def get_dependencies(): def get_dependencies():
@ -139,10 +123,10 @@ def generate_targets(repo_path, deps_map):
src_mk = parse_src_mk(repo_path) src_mk = parse_src_mk(repo_path)
# get all .cc files # get all .cc files
cc_files = get_cc_files(repo_path) cc_files = get_cc_files(repo_path)
# get tests from Makefile # get parallel tests from Makefile
tests = get_tests(repo_path) parallel_tests = get_parallel_tests(repo_path)
if src_mk is None or cc_files is None or tests is None: if src_mk is None or cc_files is None or parallel_tests is None:
return False return False
TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path) TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path)
@ -178,32 +162,35 @@ def generate_targets(repo_path, deps_map):
+ ["test_util/testutil.cc"]) + ["test_util/testutil.cc"])
print("Extra dependencies:\n{0}".format(json.dumps(deps_map))) print("Extra dependencies:\n{0}".format(json.dumps(deps_map)))
# test for every test we found in the Makefile
# Dictionary test executable name -> relative source file path
test_source_map = {}
print(src_mk)
test_main_sources = src_mk.get("TEST_MAIN_SOURCES", []) + \
src_mk.get("TEST_MAIN_SOURCES_C", [])
for test_src in test_main_sources:
test = test_src.split('.c')[0].strip().split('/')[-1].strip()
test_source_map[test] = test_src
print("" + test + " " + test_src)
for target_alias, deps in deps_map.items(): for target_alias, deps in deps_map.items():
for test in sorted(tests): for test, test_src in sorted(test_source_map.items()):
match_src = [src for src in cc_files if ("/%s.c" % test) in src] if len(test) == 0:
if len(match_src) == 0: print(ColorString.warning("Failed to get test name for %s" % test_src))
print(ColorString.warning("Cannot find .cc file for %s" % test))
continue
elif len(match_src) > 1:
print(ColorString.warning("Found more than one .cc for %s" % test))
print(match_src)
continue continue
assert(len(match_src) == 1)
is_parallel = tests[test]
test_target_name = \ test_target_name = \
test if not target_alias else test + "_" + target_alias test if not target_alias else test + "_" + target_alias
TARGETS.register_test( TARGETS.register_test(
test_target_name, test_target_name,
match_src[0], test_src,
is_parallel, test in parallel_tests,
json.dumps(deps['extra_deps']), json.dumps(deps['extra_deps']),
json.dumps(deps['extra_compiler_flags'])) json.dumps(deps['extra_compiler_flags']))
if test in _EXPORTED_TEST_LIBS: if test in _EXPORTED_TEST_LIBS:
test_library = "%s_lib" % test_target_name test_library = "%s_lib" % test_target_name
TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"]) TARGETS.add_library(test_library, [test_src], [":rocksdb_test_lib"])
TARGETS.flush_tests() TARGETS.flush_tests()
print(ColorString.info("Generated TARGETS Summary:")) print(ColorString.info("Generated TARGETS Summary:"))

24
build_tools/build_detect_platform
Unescape View File

@ -151,7 +151,7 @@ case "$TARGET_OS" in
else else
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic" PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
fi fi
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt" PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt -ldl"
if test $ROCKSDB_USE_IO_URING; then if test $ROCKSDB_USE_IO_URING; then
# check for liburing # check for liburing
$CXX $CFLAGS -x c++ - -luring -o /dev/null 2>/dev/null <<EOF $CXX $CFLAGS -x c++ - -luring -o /dev/null 2>/dev/null <<EOF
@ -296,26 +296,34 @@ EOF
# Test whether gflags library is installed # Test whether gflags library is installed
# http://gflags.github.io/gflags/ # http://gflags.github.io/gflags/
# check if the namespace is gflags # check if the namespace is gflags
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF if $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
#include <gflags/gflags.h> #include <gflags/gflags.h>
using namespace GFLAGS_NAMESPACE;
int main() {} int main() {}
EOF EOF
if [ "$?" = 0 ]; then then
COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1" COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags" PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
else # check if namespace is gflags
elif $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
#include <gflags/gflags.h>
using namespace gflags;
int main() {}
EOF
then
COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1 -DGFLAGS_NAMESPACE=gflags"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
# check if namespace is google # check if namespace is google
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF elif $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
#include <gflags/gflags.h> #include <gflags/gflags.h>
using namespace google; using namespace google;
int main() {} int main() {}
EOF EOF
if [ "$?" = 0 ]; then then
COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=google" COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1 -DGFLAGS_NAMESPACE=google"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags" PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
fi fi
fi fi
fi
if ! test $ROCKSDB_DISABLE_ZLIB; then if ! test $ROCKSDB_DISABLE_ZLIB; then
# Test whether zlib library is installed # Test whether zlib library is installed

4
db/external_sst_file_test.cc
Unescape View File

@ -2854,7 +2854,7 @@ TEST_P(ExternalSSTFileTest, DeltaEncodingWhileGlobalSeqnoPresent) {
auto snap = dbfull()->GetSnapshot(); auto snap = dbfull()->GetSnapshot();
std::string fname = sst_files_dir_ + "test_file"; std::string fname = sst_files_dir_ + "test_file";
rocksdb::SstFileWriter writer(EnvOptions(), options); ROCKSDB_NAMESPACE::SstFileWriter writer(EnvOptions(), options);
ASSERT_OK(writer.Open(fname)); ASSERT_OK(writer.Open(fname));
std::string key1 = "ab"; std::string key1 = "ab";
std::string key2 = "ab"; std::string key2 = "ab";
@ -2904,7 +2904,7 @@ TEST_P(ExternalSSTFileTest,
ASSERT_OK(Put(key0, value)); ASSERT_OK(Put(key0, value));
std::string fname = sst_files_dir_ + "test_file"; std::string fname = sst_files_dir_ + "test_file";
rocksdb::SstFileWriter writer(EnvOptions(), options); ROCKSDB_NAMESPACE::SstFileWriter writer(EnvOptions(), options);
ASSERT_OK(writer.Open(fname)); ASSERT_OK(writer.Open(fname));
// key0 is a dummy to ensure the turnaround point (key1) comes from Prev // key0 is a dummy to ensure the turnaround point (key1) comes from Prev

3
db_stress_tool/db_stress_tool.cc
Unescape View File

@ -45,10 +45,11 @@ int db_stress_tool(int argc, char** argv) {
SanitizeDoubleParam(&FLAGS_memtable_prefix_bloom_size_ratio); SanitizeDoubleParam(&FLAGS_memtable_prefix_bloom_size_ratio);
SanitizeDoubleParam(&FLAGS_max_bytes_for_level_multiplier); SanitizeDoubleParam(&FLAGS_max_bytes_for_level_multiplier);
#ifndef NDEBUG
if (FLAGS_mock_direct_io) { if (FLAGS_mock_direct_io) {
test::SetupSyncPointsToMockDirectIO(); test::SetupSyncPointsToMockDirectIO();
} }
#endif
if (FLAGS_statistics) { if (FLAGS_statistics) {
dbstats = ROCKSDB_NAMESPACE::CreateDBStatistics(); dbstats = ROCKSDB_NAMESPACE::CreateDBStatistics();
if (FLAGS_test_secondary) { if (FLAGS_test_secondary) {

60
src.mk
Unescape View File

@ -307,10 +307,32 @@ FOLLY_SOURCES = \
third-party/folly/folly/synchronization/ParkingLot.cpp \ third-party/folly/folly/synchronization/ParkingLot.cpp \
third-party/folly/folly/synchronization/WaitOptions.cpp \ third-party/folly/folly/synchronization/WaitOptions.cpp \
MAIN_SOURCES = \ TOOLS_MAIN_SOURCES = \
db_stress_tool/db_stress.cc \
tools/blob_dump.cc \
tools/block_cache_analyzer/block_cache_trace_analyzer_tool.cc \
tools/db_repl_stress.cc \
tools/db_sanity_test.cc \
tools/ldb.cc \
tools/sst_dump.cc \
tools/write_stress.cc \
tools/dump/rocksdb_dump.cc \
tools/dump/rocksdb_undump.cc \
tools/trace_analyzer.cc \
BENCH_MAIN_SOURCES = \
cache/cache_bench.cc \ cache/cache_bench.cc \
db/range_del_aggregator_bench.cc \
memtable/memtablerep_bench.cc \
table/table_reader_bench.cc \
tools/db_bench.cc \
util/filter_bench.cc \
utilities/persistent_cache/persistent_cache_bench.cc \
#util/log_write_bench.cc \
TEST_MAIN_SOURCES = \
cache/cache_test.cc \ cache/cache_test.cc \
db_stress_tool/db_stress.cc \ cache/lru_cache_test.cc \
db/blob/blob_file_addition_test.cc \ db/blob/blob_file_addition_test.cc \
db/blob/blob_file_garbage_test.cc \ db/blob/blob_file_garbage_test.cc \
db/blob/db_blob_index_test.cc \ db/blob/db_blob_index_test.cc \
@ -332,6 +354,7 @@ MAIN_SOURCES = \
db/db_dynamic_level_test.cc \ db/db_dynamic_level_test.cc \
db/db_encryption_test.cc \ db/db_encryption_test.cc \
db/db_flush_test.cc \ db/db_flush_test.cc \
db/import_column_family_test.cc \
db/db_inplace_update_test.cc \ db/db_inplace_update_test.cc \
db/db_io_failure_test.cc \ db/db_io_failure_test.cc \
db/db_iter_test.cc \ db/db_iter_test.cc \
@ -358,7 +381,6 @@ MAIN_SOURCES = \
db/db_write_test.cc \ db/db_write_test.cc \
db/dbformat_test.cc \ db/dbformat_test.cc \
db/deletefile_test.cc \ db/deletefile_test.cc \
db/env_timed_test.cc \
db/error_handler_fs_test.cc \ db/error_handler_fs_test.cc \
db/external_sst_file_basic_test.cc \ db/external_sst_file_basic_test.cc \
db/external_sst_file_test.cc \ db/external_sst_file_test.cc \
@ -367,29 +389,21 @@ MAIN_SOURCES = \
db/file_reader_writer_test.cc \ db/file_reader_writer_test.cc \
db/filename_test.cc \ db/filename_test.cc \
db/flush_job_test.cc \ db/flush_job_test.cc \
db/hash_table_test.cc \
db/hash_test.cc \
db/heap_test.cc \
db/listener_test.cc \ db/listener_test.cc \
db/log_test.cc \ db/log_test.cc \
db/lru_cache_test.cc \
db/manual_compaction_test.cc \ db/manual_compaction_test.cc \
db/memtable_list_test.cc \ db/memtable_list_test.cc \
db/merge_helper_test.cc \ db/merge_helper_test.cc \
db/merge_test.cc \ db/merge_test.cc \
db/obsolete_files_test.cc \ db/obsolete_files_test.cc \
db/options_settable_test.cc \
db/options_file_test.cc \ db/options_file_test.cc \
db/perf_context_test.cc \ db/perf_context_test.cc \
db/persistent_cache_test.cc \
db/plain_table_db_test.cc \ db/plain_table_db_test.cc \
db/prefix_test.cc \ db/prefix_test.cc \
db/repair_test.cc \ db/repair_test.cc \
db/range_del_aggregator_test.cc \ db/range_del_aggregator_test.cc \
db/range_del_aggregator_bench.cc \
db/range_tombstone_fragmenter_test.cc \ db/range_tombstone_fragmenter_test.cc \
db/table_properties_collector_test.cc \ db/table_properties_collector_test.cc \
db/util_merge_operators_test.cc \
db/version_builder_test.cc \ db/version_builder_test.cc \
db/version_edit_test.cc \ db/version_edit_test.cc \
db/version_set_test.cc \ db/version_set_test.cc \
@ -401,6 +415,7 @@ MAIN_SOURCES = \
env/env_test.cc \ env/env_test.cc \
env/io_posix_test.cc \ env/io_posix_test.cc \
env/mock_env_test.cc \ env/mock_env_test.cc \
file/delete_scheduler_test.cc \
file/random_access_file_reader_test.cc \ file/random_access_file_reader_test.cc \
logging/auto_roll_logger_test.cc \ logging/auto_roll_logger_test.cc \
logging/env_logger_test.cc \ logging/env_logger_test.cc \
@ -408,13 +423,13 @@ MAIN_SOURCES = \
memory/arena_test.cc \ memory/arena_test.cc \
memory/memkind_kmem_allocator_test.cc \ memory/memkind_kmem_allocator_test.cc \
memtable/inlineskiplist_test.cc \ memtable/inlineskiplist_test.cc \
memtable/memtablerep_bench.cc \
memtable/skiplist_test.cc \ memtable/skiplist_test.cc \
memtable/write_buffer_manager_test.cc \ memtable/write_buffer_manager_test.cc \
monitoring/histogram_test.cc \ monitoring/histogram_test.cc \
monitoring/iostats_context_test.cc \ monitoring/iostats_context_test.cc \
monitoring/statistics_test.cc \ monitoring/statistics_test.cc \
monitoring/stats_history_test.cc \ monitoring/stats_history_test.cc \
options/options_settable_test.cc \
options/options_test.cc \ options/options_test.cc \
table/block_based/block_based_filter_block_test.cc \ table/block_based/block_based_filter_block_test.cc \
table/block_based/block_based_table_reader_test.cc \ table/block_based/block_based_table_reader_test.cc \
@ -427,15 +442,10 @@ MAIN_SOURCES = \
table/cuckoo/cuckoo_table_reader_test.cc \ table/cuckoo/cuckoo_table_reader_test.cc \
table/merger_test.cc \ table/merger_test.cc \
table/sst_file_reader_test.cc \ table/sst_file_reader_test.cc \
table/table_reader_bench.cc \
table/table_test.cc \ table/table_test.cc \
table/block_fetcher_test.cc \ table/block_fetcher_test.cc \
third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc \ test_util/testutil_test.cc \
tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc \ tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc \
tools/block_cache_analyzer/block_cache_trace_analyzer_tool.cc \
tools/db_bench.cc \
tools/db_bench_tool_test.cc \
tools/db_sanity_test.cc \
tools/ldb_cmd_test.cc \ tools/ldb_cmd_test.cc \
tools/reduce_levels_test.cc \ tools/reduce_levels_test.cc \
tools/sst_dump_test.cc \ tools/sst_dump_test.cc \
@ -449,9 +459,11 @@ MAIN_SOURCES = \
util/defer_test.cc \ util/defer_test.cc \
util/dynamic_bloom_test.cc \ util/dynamic_bloom_test.cc \
util/filelock_test.cc \ util/filelock_test.cc \
util/log_write_bench.cc \ util/file_reader_writer_test.cc \
util/rate_limiter_test.cc \ util/hash_test.cc \
util/heap_test.cc \
util/random_test.cc \ util/random_test.cc \
util/rate_limiter_test.cc \
util/repeatable_thread_test.cc \ util/repeatable_thread_test.cc \
util/slice_test.cc \ util/slice_test.cc \
util/slice_transform_test.cc \ util/slice_transform_test.cc \
@ -467,11 +479,14 @@ MAIN_SOURCES = \
utilities/cassandra/cassandra_row_merge_test.cc \ utilities/cassandra/cassandra_row_merge_test.cc \
utilities/cassandra/cassandra_serialize_test.cc \ utilities/cassandra/cassandra_serialize_test.cc \
utilities/checkpoint/checkpoint_test.cc \ utilities/checkpoint/checkpoint_test.cc \
utilities/env_timed_test.cc \
utilities/memory/memory_test.cc \ utilities/memory/memory_test.cc \
utilities/merge_operators/string_append/stringappend_test.cc \ utilities/merge_operators/string_append/stringappend_test.cc \
utilities/object_registry_test.cc \ utilities/object_registry_test.cc \
utilities/option_change_migration/option_change_migration_test.cc \ utilities/option_change_migration/option_change_migration_test.cc \
utilities/options/options_util_test.cc \ utilities/options/options_util_test.cc \
utilities/persistent_cache/hash_table_test.cc \
utilities/persistent_cache/persistent_cache_test.cc \
utilities/simulator_cache/cache_simulator_test.cc \ utilities/simulator_cache/cache_simulator_test.cc \
utilities/simulator_cache/sim_cache_test.cc \ utilities/simulator_cache/sim_cache_test.cc \
utilities/table_properties_collectors/compact_on_deletion_collector_test.cc \ utilities/table_properties_collectors/compact_on_deletion_collector_test.cc \
@ -481,8 +496,13 @@ MAIN_SOURCES = \
utilities/transactions/write_prepared_transaction_test.cc \ utilities/transactions/write_prepared_transaction_test.cc \
utilities/transactions/write_unprepared_transaction_test.cc \ utilities/transactions/write_unprepared_transaction_test.cc \
utilities/ttl/ttl_test.cc \ utilities/ttl/ttl_test.cc \
utilities/util_merge_operators_test.cc \
utilities/write_batch_with_index/write_batch_with_index_test.cc \ utilities/write_batch_with_index/write_batch_with_index_test.cc \
TEST_MAIN_SOURCES_C = \
db/c_test.c \
JNI_NATIVE_SOURCES = \ JNI_NATIVE_SOURCES = \
java/rocksjni/backupenginejni.cc \ java/rocksjni/backupenginejni.cc \
java/rocksjni/backupablejni.cc \ java/rocksjni/backupablejni.cc \

Write Preview
Loading…
Cancel
Save