From ebc647de8723288a093c9f9699a4e729e16aef03 Mon Sep 17 00:00:00 2001 From: Jim Meyering Date: Fri, 20 Feb 2015 17:42:16 -0800 Subject: [PATCH] build: fix missing dependency problems Summary: Any time one would modify a dependent of any *test*.cc file, "make" would fail to rebuild the affected test binaries, e.g., db_test. That was due to the fact that we deliberately excluded those test-related files from the definition of SOURCES and only $(SOURCES) was used to create the automatically-generated .d dependency files. The fix is to generate a .d file for every source file. * src.mk: New file. Defines LIB_SOURCES, MOCK_SOURCES and TEST_BENCH_SOURCES. * Makefile: Include src.mk. Reflect s/SOURCES/LIB_SOURCES/ renaming. * build_tools/build_detect_platform: Remove the code that was used to generate SOURCES= and MOCK_SOURCES= definitions in make_config.mk. Those lists of files are now hard-coded in src.mk. Hard-coding this list of sources is desirable, because without that, one risks including stray .cc files in a build. Not reproducible. Test Plan: Touch a file used by db_test's dependent .o files and ensure that they are all recompiled. Before, none would be: $ touch db/db_impl.h && make db_test CC db/db_test.o CC db/column_family.o CC db/db_filesnapshot.o CC db/db_impl.o CC db/db_impl_debug.o CC db/db_impl_readonly.o CC db/forward_iterator.o CC db/internal_stats.o CC db/managed_iterator.o CC db/repair.o CC db/write_batch.o CC utilities/compacted_db/compacted_db_impl.o CC utilities/ttl/db_ttl_impl.o CC util/ldb_cmd.o CC util/ldb_tool.o CC util/sst_dump_tool.o CC util/xfunc.o CCLD db_test Reviewers: ljin, igor.sugak, igor, rven, sdong Reviewed By: sdong Subscribers: yhchiang, adamretter, fyrz, dhruba Differential Revision: https://reviews.facebook.net/D33849 --- Makefile | 10 +- build_tools/build_detect_platform | 25 ---- src.mk | 225 ++++++++++++++++++++++++++++++ 3 files changed, 230 insertions(+), 30 deletions(-) create mode 100644 src.mk diff --git a/Makefile b/Makefile index 9b2467034..bc4d7de63 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,7 @@ OPT += -DNDEBUG endif #----------------------------------------------- +include src.mk AM_DEFAULT_VERBOSITY = 0 @@ -157,7 +158,7 @@ util/build_version.cc: $(AM_V_at)if test -f $@; then \ cmp -s $@.tmp $@ && : || mv -f $@.tmp $@; else mv -f $@.tmp $@; fi -LIBOBJECTS = $(SOURCES:.cc=.o) +LIBOBJECTS = $(LIB_SOURCES:.cc=.o) MOCKOBJECTS = $(MOCK_SOURCES:.cc=.o) TESTUTIL = ./util/testutil.o @@ -291,7 +292,7 @@ $(SHARED3): $(SHARED4) endif $(SHARED4): - $(CXX) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SOURCES) $(LDFLAGS) -o $@ + $(CXX) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(LIB_SOURCES) $(LDFLAGS) -o $@ endif # PLATFORM_SHARED_EXT @@ -775,13 +776,12 @@ endif # The .d file indicates .cc file's dependencies on .h files. We generate such # dependency by g++'s -MM option, whose output is a make dependency rule. -# The sed command makes sure the "target" file in the generated .d file has -# the correct path prefix. %.d: %.cc @$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) \ -MM -MT'$@' -MT'$(<:.cc=.o)' "$<" -o '$@' -DEPFILES = $(SOURCES:.cc=.d) +all_sources = $(LIB_SOURCES) $(TEST_BENCH_SOURCES) $(MOCK_SOURCES) +DEPFILES = $(all_sources:.cc=.d) depend: $(DEPFILES) diff --git a/build_tools/build_detect_platform b/build_tools/build_detect_platform index 3062759dd..9052e77bb 100755 --- a/build_tools/build_detect_platform +++ b/build_tools/build_detect_platform @@ -169,31 +169,6 @@ esac JAVA_LDFLAGS="$PLATFORM_LDFLAGS" -# We want to make a list of all cc files in $DIRS, -# except for the test and benchmark files. By default, find will output a list -# of all files matching either rule, so we need to append -print to make the -# prune take effect. -DIRS="util db table utilities" - -set -f # temporarily disable globbing so that our patterns arent expanded -PRUNE_TEST="-name *test*.cc -prune" -PRUNE_BENCH="-name *bench*.cc -prune" -PRUNE_MOCK="-name *mock*.cc -prune" -PORTABLE_FILES=`cd "$ROCKSDB_ROOT"; find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o $PRUNE_MOCK -o -name '*.cc' -print | sort | tr "\n" " "` -MOCK_SOURCES=`cd "$ROCKSDB_ROOT"; find $DIRS -name '*mock*.cc' -print | grep -v "test" | sort | tr "\n" " "` -set +f # re-enable globbing - -# If the generated util/build_version.cc is not on the list, add it: -case $PORTABLE_FILES in - *util/build_version.cc*) ;; - *) PORTABLE_FILES="$PORTABLE_FILES util/build_version.cc" ;; -esac - -# The sources consist of the portable files, plus the platform-specific port -# file. -echo "SOURCES=$PORTABLE_FILES $GENERIC_PORT_FILES $PORT_FILES" >> "$OUTPUT" -echo "MOCK_SOURCES=$MOCK_SOURCES" >> "$OUTPUT" - if [ "$CROSS_COMPILE" = "true" -o "$FBCODE_BUILD" = "true" ]; then # Cross-compiling; do not try any compilation tests. # Also don't need any compilation tests if compiling on fbcode diff --git a/src.mk b/src.mk new file mode 100644 index 000000000..7e4ac67c6 --- /dev/null +++ b/src.mk @@ -0,0 +1,225 @@ +# These are the sources from which librocksdb.a is built: +LIB_SOURCES = \ + db/builder.cc \ + db/c.cc \ + db/column_family.cc \ + db/compaction.cc \ + db/compaction_job.cc \ + db/compaction_picker.cc \ + db/db_filesnapshot.cc \ + db/dbformat.cc \ + db/db_impl.cc \ + db/db_impl_debug.cc \ + db/db_impl_readonly.cc \ + db/db_iter.cc \ + db/file_indexer.cc \ + db/filename.cc \ + db/flush_job.cc \ + db/flush_scheduler.cc \ + db/forward_iterator.cc \ + db/internal_stats.cc \ + db/log_reader.cc \ + db/log_writer.cc \ + db/managed_iterator.cc \ + db/memtable_allocator.cc \ + db/memtable.cc \ + db/memtable_list.cc \ + db/merge_helper.cc \ + db/merge_operator.cc \ + db/repair.cc \ + db/table_cache.cc \ + db/table_properties_collector.cc \ + db/transaction_log_impl.cc \ + db/version_builder.cc \ + db/version_edit.cc \ + db/version_set.cc \ + db/wal_manager.cc \ + db/write_batch.cc \ + db/write_controller.cc \ + db/write_thread.cc \ + port/stack_trace.cc \ + port/port_posix.cc \ + table/adaptive_table_factory.cc \ + table/block_based_filter_block.cc \ + table/block_based_table_builder.cc \ + table/block_based_table_factory.cc \ + table/block_based_table_reader.cc \ + table/block_builder.cc \ + table/block.cc \ + table/block_hash_index.cc \ + table/block_prefix_index.cc \ + table/bloom_block.cc \ + table/cuckoo_table_builder.cc \ + table/cuckoo_table_factory.cc \ + table/cuckoo_table_reader.cc \ + table/flush_block_policy.cc \ + table/format.cc \ + table/full_filter_block.cc \ + table/get_context.cc \ + table/iterator.cc \ + table/merger.cc \ + table/meta_blocks.cc \ + table/plain_table_builder.cc \ + table/plain_table_factory.cc \ + table/plain_table_index.cc \ + table/plain_table_key_coding.cc \ + table/plain_table_reader.cc \ + table/table_properties.cc \ + table/two_level_iterator.cc \ + util/arena.cc \ + util/auto_roll_logger.cc \ + util/bloom.cc \ + util/build_version.cc \ + util/cache.cc \ + util/coding.cc \ + util/comparator.cc \ + util/crc32c.cc \ + util/db_info_dumper.cc \ + util/dynamic_bloom.cc \ + util/env.cc \ + util/env_hdfs.cc \ + util/env_posix.cc \ + util/file_util.cc \ + util/filter_policy.cc \ + util/hash.cc \ + util/hash_cuckoo_rep.cc \ + util/hash_linklist_rep.cc \ + util/hash_skiplist_rep.cc \ + util/histogram.cc \ + util/instrumented_mutex.cc \ + util/iostats_context.cc \ + utilities/backupable/backupable_db.cc \ + utilities/checkpoint/checkpoint.cc \ + utilities/compacted_db/compacted_db_impl.cc \ + utilities/document/document_db.cc \ + utilities/document/json_document.cc \ + utilities/geodb/geodb_impl.cc \ + utilities/leveldb_options/leveldb_options.cc \ + utilities/merge_operators/put.cc \ + utilities/merge_operators/string_append/stringappend2.cc \ + utilities/merge_operators/string_append/stringappend.cc \ + utilities/merge_operators/uint64add.cc \ + utilities/redis/redis_lists.cc \ + utilities/spatialdb/spatial_db.cc \ + utilities/ttl/db_ttl_impl.cc \ + utilities/write_batch_with_index/write_batch_with_index.cc \ + util/ldb_cmd.cc \ + util/ldb_tool.cc \ + util/log_buffer.cc \ + util/logging.cc \ + util/memenv.cc \ + util/murmurhash.cc \ + util/mutable_cf_options.cc \ + util/options_builder.cc \ + util/options.cc \ + util/options_helper.cc \ + util/perf_context.cc \ + util/rate_limiter.cc \ + util/skiplistrep.cc \ + util/slice.cc \ + util/sst_dump_tool.cc \ + util/statistics.cc \ + util/status.cc \ + util/string_util.cc \ + util/sync_point.cc \ + util/thread_local.cc \ + util/thread_status_impl.cc \ + util/thread_status_updater.cc \ + util/thread_status_updater_debug.cc \ + util/thread_status_util.cc \ + util/thread_status_util_debug.cc \ + util/vectorrep.cc \ + util/xfunc.cc \ + util/xxhash.cc \ + +MOCK_SOURCES = \ + table/mock_table.cc \ + util/mock_env.cc + +TEST_BENCH_SOURCES = \ + db/column_family_test.cc \ + db/compaction_job_test.cc \ + db/compaction_picker_test.cc \ + db/comparator_db_test.cc \ + db/corruption_test.cc \ + db/cuckoo_table_db_test.cc \ + db/db_bench.cc \ + db/dbformat_test.cc \ + db/db_iter_test.cc \ + db/db_test.cc \ + db/deletefile_test.cc \ + db/fault_injection_test.cc \ + db/file_indexer_test.cc \ + db/filename_test.cc \ + db/flush_job_test.cc \ + db/listener_test.cc \ + db/log_and_apply_bench.cc \ + db/log_test.cc \ + db/memtablerep_bench.cc \ + db/merge_test.cc \ + db/perf_context_test.cc \ + db/plain_table_db_test.cc \ + db/prefix_test.cc \ + db/skiplist_test.cc \ + db/table_properties_collector_test.cc \ + db/version_builder_test.cc \ + db/version_edit_test.cc \ + db/version_set_test.cc \ + db/wal_manager_test.cc \ + db/write_batch_test.cc \ + db/write_controller_test.cc \ + table/block_based_filter_block_test.cc \ + table/block_hash_index_test.cc \ + table/block_test.cc \ + table/cuckoo_table_builder_test.cc \ + table/cuckoo_table_reader_test.cc \ + table/full_filter_block_test.cc \ + table/merger_test.cc \ + table/table_reader_bench.cc \ + table/table_test.cc \ + tools/db_sanity_test.cc \ + tools/reduce_levels_test.cc \ + util/arena_test.cc \ + util/auto_roll_logger_test.cc \ + util/autovector_test.cc \ + util/benchharness.cc \ + util/benchharness_test.cc \ + util/bloom_test.cc \ + util/cache_bench.cc \ + util/cache_test.cc \ + util/coding_test.cc \ + util/crc32c_test.cc \ + util/dynamic_bloom_test.cc \ + util/env_test.cc \ + util/filelock_test.cc \ + util/histogram_test.cc \ + utilities/backupable/backupable_db_test.cc \ + utilities/document/document_db_test.cc \ + utilities/document/json_document_test.cc \ + utilities/geodb/geodb_test.cc \ + utilities/merge_operators/string_append/stringappend_test.cc \ + utilities/redis/redis_lists_test.cc \ + utilities/spatialdb/spatial_db_test.cc \ + utilities/ttl/ttl_test.cc \ + utilities/write_batch_with_index/write_batch_with_index_test.cc \ + util/log_write_bench.cc \ + util/manual_compaction_test.cc \ + util/memenv_test.cc \ + util/mock_env_test.cc \ + util/options_test.cc \ + util/rate_limiter_test.cc \ + util/signal_test.cc \ + util/slice_transform_test.cc \ + util/sst_dump_test.cc \ + util/testharness.cc \ + util/testutil.cc \ + util/thread_list_test.cc \ + util/thread_local_test.cc + +# Currently, we do not generate dependencies for +# java/rocksjni/write_batch_test.cc, because its dependent, +# java/include/org_rocksdb_WriteBatch.h is generated. +# TODO/FIXME: fix the above. Otherwise, the current rules would fail: +# java/rocksjni/write_batch_test.cc:13:44: fatal error: include/org_rocksdb_WriteBatch.h: No such file or directory +# #include "include/org_rocksdb_WriteBatch.h" +# ^