From aaafcb80ab305321783c9b9e5f701f11ee12038a Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Wed, 20 May 2020 11:35:28 -0700 Subject: [PATCH] Use in-repo gtest in buck build (#6858) Summary: ... so that we have freedom to upgrade it (see https://github.com/facebook/rocksdb/issues/6808). As a side benefit, gtest will no longer be linked into main library in buck build. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6858 Test Plan: fb internal build & link Reviewed By: riversand963 Differential Revision: D21652061 Pulled By: pdillinger fbshipit-source-id: 6018104af944debde576b5beda6c134e737acedb --- Makefile | 3 ++- TARGETS | 20 ++++++++++++++++++-- buckifier/buckify_rocksdb.py | 23 +++++++++++++++++++++-- buckifier/check_buck_targets.sh | 1 + buckifier/targets_builder.py | 10 ++++++++-- buckifier/targets_cfg.py | 9 +++++---- src.mk | 2 +- 7 files changed, 56 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 8653dc29c..a192365a1 100644 --- a/Makefile +++ b/Makefile @@ -327,7 +327,7 @@ endif export GTEST_THROW_ON_FAILURE=1 export GTEST_HAS_EXCEPTIONS=1 -GTEST_DIR = ./third-party/gtest-1.8.1/fused-src +GTEST_DIR = third-party/gtest-1.8.1/fused-src # AIX: pre-defined system headers are surrounded by an extern "C" block ifeq ($(PLATFORM), OS_AIX) PLATFORM_CCFLAGS += -I$(GTEST_DIR) @@ -2219,6 +2219,7 @@ endif # Source files dependencies detection # --------------------------------------------------------------------------- +# FIXME: nothing checks that entries in MAIN_SOURCES actually exist all_sources = $(LIB_SOURCES) $(MAIN_SOURCES) $(MOCK_LIB_SOURCES) $(TOOL_LIB_SOURCES) $(BENCH_LIB_SOURCES) $(TEST_LIB_SOURCES) $(ANALYZER_LIB_SOURCES) $(STRESS_LIB_SOURCES) DEPFILES = $(all_sources:.cc=.cc.d) diff --git a/TARGETS b/TARGETS index 66f94789f..5df9ae0e9 100644 --- a/TARGETS +++ b/TARGETS @@ -26,7 +26,6 @@ ROCKSDB_EXTERNAL_DEPS = [ ("lz4", None, "lz4"), ("zstd", None), ("tbb", None), - ("googletest", None, "gtest"), ] ROCKSDB_OS_DEPS = [ @@ -79,6 +78,7 @@ ROCKSDB_PREPROCESSOR_FLAGS = [ # Directories with files for #include "-I" + REPO_PATH + "include/", "-I" + REPO_PATH, + "-I" + REPO_PATH + "third-party/gtest-1.8.1/fused-src/", ] ROCKSDB_ARCH_PREPROCESSOR_FLAGS = { @@ -398,7 +398,10 @@ cpp_library( os_deps = ROCKSDB_OS_DEPS, os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS, preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS, - deps = [":rocksdb_lib"], + deps = [ + ":rocksdb_lib", + ":rocksdb_third_party_gtest", + ], external_deps = ROCKSDB_EXTERNAL_DEPS, ) @@ -446,6 +449,19 @@ cpp_library( external_deps = ROCKSDB_EXTERNAL_DEPS, ) +cpp_library( + name = "rocksdb_third_party_gtest", + srcs = ["third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc"], + headers = ["third-party/gtest-1.8.1/fused-src/gtest/gtest.h"], + arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS, + compiler_flags = ROCKSDB_COMPILER_FLAGS, + os_deps = ROCKSDB_OS_DEPS, + os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS, + preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS, + deps = [], + external_deps = ROCKSDB_EXTERNAL_DEPS, +) + cpp_library( name = "env_basic_test_lib", srcs = ["env/env_basic_test.cc"], diff --git a/buckifier/buckify_rocksdb.py b/buckifier/buckify_rocksdb.py index 66b75e0bd..6c13f6ebf 100644 --- a/buckifier/buckify_rocksdb.py +++ b/buckifier/buckify_rocksdb.py @@ -109,6 +109,15 @@ def get_tests(repo_path): return tests +# Get gtest dir from Makefile +def get_gtest_dir(repo_path): + for line in open(repo_path + "/Makefile"): + if line.strip().startswith("GTEST_DIR ="): + return line.split("=")[1].strip() + # if not found + exit_with_error("Unable to find GTEST_DIR in Makefile") + + # Parse extra dependencies passed by user from command line def get_dependencies(): deps_map = { @@ -142,11 +151,14 @@ def generate_targets(repo_path, deps_map): cc_files = get_cc_files(repo_path) # get tests from Makefile tests = get_tests(repo_path) + # get gtest dir + gtest_dir = get_gtest_dir(repo_path) + "/" if src_mk is None or cc_files is None or tests is None: return False - TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path) + TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path, gtest_dir) + # rocksdb_lib TARGETS.add_library( "rocksdb_lib", @@ -159,7 +171,7 @@ def generate_targets(repo_path, deps_map): src_mk.get("TEST_LIB_SOURCES", []) + src_mk.get("EXP_LIB_SOURCES", []) + src_mk.get("ANALYZER_LIB_SOURCES", []), - [":rocksdb_lib"]) + [":rocksdb_lib", ":rocksdb_third_party_gtest"]) # rocksdb_tools_lib TARGETS.add_library( "rocksdb_tools_lib", @@ -173,6 +185,12 @@ def generate_targets(repo_path, deps_map): src_mk.get("ANALYZER_LIB_SOURCES", []) + src_mk.get('STRESS_LIB_SOURCES', []) + ["test_util/testutil.cc"]) + # rocksdb_third_party_gtest + TARGETS.add_library( + "rocksdb_third_party_gtest", + [gtest_dir + "gtest/gtest-all.cc"], + [], + [gtest_dir + "gtest/gtest.h"]) print("Extra dependencies:\n{0}".format(json.dumps(deps_map))) # test for every test we found in the Makefile @@ -219,6 +237,7 @@ def get_rocksdb_path(): return rocksdb_path + def exit_with_error(msg): print(ColorString.error(msg)) sys.exit(1) diff --git a/buckifier/check_buck_targets.sh b/buckifier/check_buck_targets.sh index ff97e9571..ccf77f91d 100755 --- a/buckifier/check_buck_targets.sh +++ b/buckifier/check_buck_targets.sh @@ -26,6 +26,7 @@ then else echo "Please run 'python buckifier/buckify_rocksdb.py' to update TARGETS file." echo "Do not manually update TARGETS file." + python --version mv TARGETS.bkp TARGETS exit 1 fi diff --git a/buckifier/targets_builder.py b/buckifier/targets_builder.py index 7904963f7..dab331ce7 100644 --- a/buckifier/targets_builder.py +++ b/buckifier/targets_builder.py @@ -25,10 +25,12 @@ def pretty_list(lst, indent=8): class TARGETSBuilder(object): - def __init__(self, path): + def __init__(self, path, gtest_dir): self.path = path self.targets_file = open(path, 'w') - self.targets_file.write(targets_cfg.rocksdb_target_header) + header = targets_cfg.rocksdb_target_header_template.format( + gtest_dir=gtest_dir) + self.targets_file.write(header) self.total_lib = 0 self.total_bin = 0 self.total_test = 0 @@ -42,6 +44,8 @@ class TARGETSBuilder(object): if headers is None: headers_attr_prefix = "auto_" headers = "AutoHeaders.RECURSIVE_GLOB" + else: + headers = "[" + pretty_list(headers) + "]" self.targets_file.write(targets_cfg.library_template.format( name=name, srcs=pretty_list(srcs), @@ -55,6 +59,8 @@ class TARGETSBuilder(object): if headers is None: headers_attr_prefix = "auto_" headers = "AutoHeaders.RECURSIVE_GLOB" + else: + headers = "[" + pretty_list(headers) + "]" self.targets_file.write(targets_cfg.rocksdb_library_template.format( name=name, srcs=pretty_list(srcs), diff --git a/buckifier/targets_cfg.py b/buckifier/targets_cfg.py index 60ff10f80..62ef6f6c7 100644 --- a/buckifier/targets_cfg.py +++ b/buckifier/targets_cfg.py @@ -4,7 +4,8 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -rocksdb_target_header = """# This file \100generated by `python buckifier/buckify_rocksdb.py` +rocksdb_target_header_template = \ + """# This file \100generated by `python buckifier/buckify_rocksdb.py` # --> DO NOT EDIT MANUALLY <-- # This file is a Facebook-specific integration for buck builds, so can # only be validated by Facebook employees. @@ -32,7 +33,6 @@ ROCKSDB_EXTERNAL_DEPS = [ ("lz4", None, "lz4"), ("zstd", None), ("tbb", None), - ("googletest", None, "gtest"), ] ROCKSDB_OS_DEPS = [ @@ -85,13 +85,14 @@ ROCKSDB_PREPROCESSOR_FLAGS = [ # Directories with files for #include "-I" + REPO_PATH + "include/", "-I" + REPO_PATH, + "-I" + REPO_PATH + "{gtest_dir}", ] -ROCKSDB_ARCH_PREPROCESSOR_FLAGS = { +ROCKSDB_ARCH_PREPROCESSOR_FLAGS = {{ "x86_64": [ "-DHAVE_PCLMUL", ], -} +}} build_mode = read_config("fbcode", "build_mode") diff --git a/src.mk b/src.mk index cb0c5883e..c5f77dfcb 100644 --- a/src.mk +++ b/src.mk @@ -428,7 +428,7 @@ MAIN_SOURCES = \ table/table_reader_bench.cc \ table/table_test.cc \ table/block_fetcher_test.cc \ - third-party/gtest-1.7.0/fused-src/gtest/gtest-all.cc \ + third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc \ tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc \ tools/block_cache_analyzer/block_cache_trace_analyzer_tool.cc \ tools/db_bench.cc \