Use Libraries in the RocksDB Makefile Build (#6660)

Summary:
Change the linking of tests/tools to be against a library rather than a list of objects.  This change substantially reduces the size of the objects produced.

peterd clean repo size: 264M
Before this change, with make all: 40G
After this change, with make all: 28G
With make LIB_MODE=shared all: 7.0G

The list of TESTS was changed from being hard-coded to generated from the test sources variable.  Note that there are some test sources that are not built as tests (though the set of tests is identical to the previous version).

Added OBJ_DIR option to Makefile to allow objects to be placed in an alternative location.  By default, OBJ_DIR is the same as before ("./").

This change is a precursor to being able to build/run the tests/tools linked against static libraries.  Additionally, it should be possible to clean up and merge some of the rules for building tests and the like if so desired.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6660

Reviewed By: riversand963

Differential Revision: D22244463

Pulled By: pdillinger

fbshipit-source-id: db9c6341d81ed62c2270374f4ede02fb9604c754

buckifier/buckify_rocksdb.py

@@ -48,8 +48,8 @@ def parse_src_mk(repo_path):
         if '=' in line:
             current_src = line.split('=')[0].strip()
             src_files[current_src] = []
-        elif '.cc' in line:
+        elif '.c' in line:
-            src_path = line.split('.cc')[0].strip() + '.cc'
+            src_path = line.split('\\')[0].strip()
             src_files[current_src].append(src_path)
     return src_files
 
@@ -69,27 +69,11 @@ def get_cc_files(repo_path):
     return cc_files
 
 
-# Get tests from Makefile
+# Get parallel tests from Makefile
-def get_tests(repo_path):
+def get_parallel_tests(repo_path):
     Makefile = repo_path + "/Makefile"
 
-    # Dictionary TEST_NAME => IS_PARALLEL
+    s = set({})
-    tests = {}
-
-    found_tests = False
-    for line in open(Makefile):
-        line = line.strip()
-        if line.startswith("TESTS ="):
-            found_tests = True
-        elif found_tests:
-            if line.endswith("\\"):
-                # remove the trailing \
-                line = line[:-1]
-                line = line.strip()
-                tests[line] = False
-            else:
-                # we consumed all the tests
-                break
 
     found_parallel_tests = False
     for line in open(Makefile):
@@ -101,12 +85,12 @@ def get_tests(repo_path):
                 # remove the trailing \
                 line = line[:-1]
                 line = line.strip()
-                tests[line] = True
+                s.add(line)
             else:
                 # we consumed all the parallel tests
                 break
 
-    return tests
+    return s
 
 # Parse extra dependencies passed by user from command line
 def get_dependencies():
@@ -139,10 +123,10 @@ def generate_targets(repo_path, deps_map):
     src_mk = parse_src_mk(repo_path)
     # get all .cc files
     cc_files = get_cc_files(repo_path)
-    # get tests from Makefile
+    # get parallel tests from Makefile
-    tests = get_tests(repo_path)
+    parallel_tests = get_parallel_tests(repo_path)
 
-    if src_mk is None or cc_files is None or tests is None:
+    if src_mk is None or cc_files is None or parallel_tests is None:
         return False
 
     TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path)
@@ -178,32 +162,35 @@ def generate_targets(repo_path, deps_map):
         + ["test_util/testutil.cc"])
 
     print("Extra dependencies:\n{0}".format(json.dumps(deps_map)))
-    # test for every test we found in the Makefile
+
+    # Dictionary test executable name -> relative source file path
+    test_source_map = {}
+    print(src_mk)
+    test_main_sources = src_mk.get("TEST_MAIN_SOURCES", []) + \
+        src_mk.get("TEST_MAIN_SOURCES_C", [])
+    for test_src in test_main_sources:
+        test = test_src.split('.c')[0].strip().split('/')[-1].strip()
+        test_source_map[test] = test_src
+        print("" + test + " " + test_src)
+
     for target_alias, deps in deps_map.items():
-        for test in sorted(tests):
+        for test, test_src in sorted(test_source_map.items()):
-            match_src = [src for src in cc_files if ("/%s.c" % test) in src]
+            if len(test) == 0:
-            if len(match_src) == 0:
+                print(ColorString.warning("Failed to get test name for %s" % test_src))
-                print(ColorString.warning("Cannot find .cc file for %s" % test))
-                continue
-            elif len(match_src) > 1:
-                print(ColorString.warning("Found more than one .cc for %s" % test))
-                print(match_src)
                 continue
 
-            assert(len(match_src) == 1)
-            is_parallel = tests[test]
             test_target_name = \
                 test if not target_alias else test + "_" + target_alias
             TARGETS.register_test(
                 test_target_name,
-                match_src[0],
+                test_src,
-                is_parallel,
+                test in parallel_tests,
                 json.dumps(deps['extra_deps']),
                 json.dumps(deps['extra_compiler_flags']))
 
             if test in _EXPORTED_TEST_LIBS:
                 test_library = "%s_lib" % test_target_name
-                TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"])
+                TARGETS.add_library(test_library, [test_src], [":rocksdb_test_lib"])
     TARGETS.flush_tests()
 
     print(ColorString.info("Generated TARGETS Summary:"))

build_tools/build_detect_platform

@@ -151,7 +151,7 @@ case "$TARGET_OS" in
         else
             PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
         fi
-        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt -ldl"
         if test $ROCKSDB_USE_IO_URING; then
             # check for liburing
             $CXX $CFLAGS -x c++ - -luring -o /dev/null 2>/dev/null  <<EOF
@@ -296,26 +296,34 @@ EOF
         # Test whether gflags library is installed
         # http://gflags.github.io/gflags/
         # check if the namespace is gflags
-        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
+        if $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
           #include <gflags/gflags.h>
+          using namespace GFLAGS_NAMESPACE;
           int main() {}
 EOF
-        if [ "$?" = 0 ]; then
+        then
           COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1"
           PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
-        else
+        # check if namespace is gflags
+        elif $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
+            #include <gflags/gflags.h>
+            using namespace gflags;
+            int main() {}
+EOF
+        then
+          COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1 -DGFLAGS_NAMESPACE=gflags"
+          PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
         # check if namespace is google
-          $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
+        elif $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
             #include <gflags/gflags.h>
             using namespace google;
             int main() {}
 EOF
-          if [ "$?" = 0 ]; then
+        then
-              COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=google"
+          COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1 -DGFLAGS_NAMESPACE=google"
           PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
         fi
     fi
-    fi
 
     if ! test $ROCKSDB_DISABLE_ZLIB; then
         # Test whether zlib library is installed

db/external_sst_file_test.cc

@@ -2854,7 +2854,7 @@ TEST_P(ExternalSSTFileTest, DeltaEncodingWhileGlobalSeqnoPresent) {
   auto snap = dbfull()->GetSnapshot();
 
   std::string fname = sst_files_dir_ + "test_file";
-  rocksdb::SstFileWriter writer(EnvOptions(), options);
+  ROCKSDB_NAMESPACE::SstFileWriter writer(EnvOptions(), options);
   ASSERT_OK(writer.Open(fname));
   std::string key1 = "ab";
   std::string key2 = "ab";
@@ -2904,7 +2904,7 @@ TEST_P(ExternalSSTFileTest,
   ASSERT_OK(Put(key0, value));
 
   std::string fname = sst_files_dir_ + "test_file";
-  rocksdb::SstFileWriter writer(EnvOptions(), options);
+  ROCKSDB_NAMESPACE::SstFileWriter writer(EnvOptions(), options);
   ASSERT_OK(writer.Open(fname));
 
   // key0 is a dummy to ensure the turnaround point (key1) comes from Prev

db_stress_tool/db_stress_tool.cc

@@ -45,10 +45,11 @@ int db_stress_tool(int argc, char** argv) {
   SanitizeDoubleParam(&FLAGS_memtable_prefix_bloom_size_ratio);
   SanitizeDoubleParam(&FLAGS_max_bytes_for_level_multiplier);
 
+#ifndef NDEBUG
   if (FLAGS_mock_direct_io) {
     test::SetupSyncPointsToMockDirectIO();
   }
-
+#endif
   if (FLAGS_statistics) {
     dbstats = ROCKSDB_NAMESPACE::CreateDBStatistics();
     if (FLAGS_test_secondary) {

src.mk

@@ -307,10 +307,32 @@ FOLLY_SOURCES = \
   third-party/folly/folly/synchronization/ParkingLot.cpp                       \
   third-party/folly/folly/synchronization/WaitOptions.cpp                      \
 
-MAIN_SOURCES =                                                          \
+TOOLS_MAIN_SOURCES =                                                    \
+  db_stress_tool/db_stress.cc                                           \
+  tools/blob_dump.cc                                                    \
+  tools/block_cache_analyzer/block_cache_trace_analyzer_tool.cc         \
+  tools/db_repl_stress.cc                                               \
+  tools/db_sanity_test.cc                                               \
+  tools/ldb.cc                                                          \
+  tools/sst_dump.cc                                                     \
+  tools/write_stress.cc                                                 \
+  tools/dump/rocksdb_dump.cc                                            \
+  tools/dump/rocksdb_undump.cc                                          \
+  tools/trace_analyzer.cc                                               \
+
+BENCH_MAIN_SOURCES =                                                    \
   cache/cache_bench.cc                                                  \
+  db/range_del_aggregator_bench.cc                                      \
+  memtable/memtablerep_bench.cc                                         \
+  table/table_reader_bench.cc                                           \
+  tools/db_bench.cc                                                     \
+  util/filter_bench.cc                                                  \
+  utilities/persistent_cache/persistent_cache_bench.cc                  \
+  #util/log_write_bench.cc                                               \
+
+TEST_MAIN_SOURCES =                                                     \
   cache/cache_test.cc                                                   \
-  db_stress_tool/db_stress.cc                                           \
+  cache/lru_cache_test.cc                                               \
   db/blob/blob_file_addition_test.cc                                    \
   db/blob/blob_file_garbage_test.cc                                     \
   db/blob/db_blob_index_test.cc                                         \
@@ -332,6 +354,7 @@ MAIN_SOURCES =                                                          \
   db/db_dynamic_level_test.cc                                           \
   db/db_encryption_test.cc                                              \
   db/db_flush_test.cc                                                   \
+  db/import_column_family_test.cc                                       \
   db/db_inplace_update_test.cc                                          \
   db/db_io_failure_test.cc                                              \
   db/db_iter_test.cc                                                    \
@@ -358,7 +381,6 @@ MAIN_SOURCES =                                                          \
   db/db_write_test.cc                                                   \
   db/dbformat_test.cc                                                   \
   db/deletefile_test.cc                                                 \
-  db/env_timed_test.cc                                                  \
   db/error_handler_fs_test.cc                                           \
   db/external_sst_file_basic_test.cc                                    \
   db/external_sst_file_test.cc                                          \
@@ -367,29 +389,21 @@ MAIN_SOURCES =                                                          \
   db/file_reader_writer_test.cc                                         \
   db/filename_test.cc                                                   \
   db/flush_job_test.cc                                                  \
-  db/hash_table_test.cc                                                 \
-  db/hash_test.cc                                                       \
-  db/heap_test.cc                                                       \
   db/listener_test.cc                                                   \
   db/log_test.cc                                                        \
-  db/lru_cache_test.cc                                                  \
   db/manual_compaction_test.cc                                          \
   db/memtable_list_test.cc                                              \
   db/merge_helper_test.cc                                               \
   db/merge_test.cc                                                      \
   db/obsolete_files_test.cc                                             \
-  db/options_settable_test.cc                                           \
   db/options_file_test.cc                                               \
   db/perf_context_test.cc                                               \
-  db/persistent_cache_test.cc                                           \
   db/plain_table_db_test.cc                                             \
   db/prefix_test.cc                                                     \
   db/repair_test.cc                                                     \
   db/range_del_aggregator_test.cc                                       \
-  db/range_del_aggregator_bench.cc                                      \
   db/range_tombstone_fragmenter_test.cc                                 \
   db/table_properties_collector_test.cc                                 \
-  db/util_merge_operators_test.cc                                       \
   db/version_builder_test.cc                                            \
   db/version_edit_test.cc                                               \
   db/version_set_test.cc                                                \
@@ -401,6 +415,7 @@ MAIN_SOURCES =                                                          \
   env/env_test.cc                                                       \
   env/io_posix_test.cc                                                  \
   env/mock_env_test.cc                                                  \
+  file/delete_scheduler_test.cc                                         \
   file/random_access_file_reader_test.cc                                \
   logging/auto_roll_logger_test.cc                                      \
   logging/env_logger_test.cc                                            \
@@ -408,13 +423,13 @@ MAIN_SOURCES =                                                          \
   memory/arena_test.cc                                                  \
   memory/memkind_kmem_allocator_test.cc                                 \
   memtable/inlineskiplist_test.cc                                       \
-  memtable/memtablerep_bench.cc                                         \
   memtable/skiplist_test.cc                                             \
   memtable/write_buffer_manager_test.cc                                 \
   monitoring/histogram_test.cc                                          \
   monitoring/iostats_context_test.cc                                    \
   monitoring/statistics_test.cc                                         \
   monitoring/stats_history_test.cc                                      \
+  options/options_settable_test.cc                                      \
   options/options_test.cc                                               \
   table/block_based/block_based_filter_block_test.cc                    \
   table/block_based/block_based_table_reader_test.cc                    \
@@ -427,15 +442,10 @@ MAIN_SOURCES =                                                          \
   table/cuckoo/cuckoo_table_reader_test.cc                              \
   table/merger_test.cc                                                  \
   table/sst_file_reader_test.cc                                         \
-  table/table_reader_bench.cc                                           \
   table/table_test.cc                                                   \
   table/block_fetcher_test.cc                                           \
-  third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc                  \
+  test_util/testutil_test.cc                                            \
   tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc         \
-  tools/block_cache_analyzer/block_cache_trace_analyzer_tool.cc         \
-  tools/db_bench.cc                                                     \
-  tools/db_bench_tool_test.cc                                           \
-  tools/db_sanity_test.cc                                               \
   tools/ldb_cmd_test.cc                                                 \
   tools/reduce_levels_test.cc                                           \
   tools/sst_dump_test.cc                                                \
@@ -449,9 +459,11 @@ MAIN_SOURCES =                                                          \
   util/defer_test.cc                                                    \
   util/dynamic_bloom_test.cc                                            \
   util/filelock_test.cc                                                 \
-  util/log_write_bench.cc                                               \
+  util/file_reader_writer_test.cc                                       \
-  util/rate_limiter_test.cc                                             \
+  util/hash_test.cc                                                     \
+  util/heap_test.cc                                                     \
   util/random_test.cc                                                   \
+  util/rate_limiter_test.cc                                             \
   util/repeatable_thread_test.cc                                        \
   util/slice_test.cc                                                    \
   util/slice_transform_test.cc                                          \
@@ -467,11 +479,14 @@ MAIN_SOURCES =                                                          \
   utilities/cassandra/cassandra_row_merge_test.cc                       \
   utilities/cassandra/cassandra_serialize_test.cc                       \
   utilities/checkpoint/checkpoint_test.cc                               \
+  utilities/env_timed_test.cc                                           \
   utilities/memory/memory_test.cc                                       \
   utilities/merge_operators/string_append/stringappend_test.cc          \
   utilities/object_registry_test.cc                                     \
   utilities/option_change_migration/option_change_migration_test.cc     \
   utilities/options/options_util_test.cc                                \
+  utilities/persistent_cache/hash_table_test.cc                         \
+  utilities/persistent_cache/persistent_cache_test.cc                   \
   utilities/simulator_cache/cache_simulator_test.cc                     \
   utilities/simulator_cache/sim_cache_test.cc                           \
   utilities/table_properties_collectors/compact_on_deletion_collector_test.cc  \
@@ -481,8 +496,13 @@ MAIN_SOURCES =                                                          \
   utilities/transactions/write_prepared_transaction_test.cc             \
   utilities/transactions/write_unprepared_transaction_test.cc           \
   utilities/ttl/ttl_test.cc                                             \
+  utilities/util_merge_operators_test.cc                                \
   utilities/write_batch_with_index/write_batch_with_index_test.cc       \
 
+TEST_MAIN_SOURCES_C = \
+  db/c_test.c                                                           \
+
+
 JNI_NATIVE_SOURCES =                                          \
   java/rocksjni/backupenginejni.cc                            \
   java/rocksjni/backupablejni.cc                              \