Change buckifier to support parameterized dependencies (#5648)

Summary:
Users may desire to specify extra dependencies via buck. This PR allows users to pass additional dependencies as a JSON object so that the buckifier script can generate TARGETS file with desired extra dependencies.

Test plan (on dev server)
```
$python buckifier/buckify_rocksdb.py '{"fake": {"extra_deps": [":test_dep", "//fakes/module:mock1"], "extra_compiler_flags": ["-DROCKSDB_LITE", "-Os"]}}'
Generating TARGETS
Extra dependencies:
{'': {'extra_compiler_flags': [], 'extra_deps': []}, 'test_dep1': {'extra_compiler_flags': ['-O2', '-DROCKSDB_LITE'], 'extra_deps': [':fake', '//dep1/mock']}}
Generated TARGETS Summary:
- 5 libs
- 0 binarys
- 296 tests
```
Verify the TARGETS file.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5648

Differential Revision: D16565043

Pulled By: riversand963

fbshipit-source-id: a6ef02274174fcf159692d7b846e828454d01e89
main
Yanqin Jin 5 years ago committed by Facebook Github Bot
parent d1c9ede195
commit 30edf1874c
  1. 302
      TARGETS
  2. 97
      buckifier/buckify_rocksdb.py
  3. 11
      buckifier/targets_builder.py
  4. 8
      buckifier/targets_cfg.py
  5. 8
      defs.bzl

File diff suppressed because it is too large Load Diff

@ -4,12 +4,31 @@ from __future__ import division
from __future__ import print_function from __future__ import print_function
from __future__ import unicode_literals from __future__ import unicode_literals
from targets_builder import TARGETSBuilder from targets_builder import TARGETSBuilder
import json
import os import os
import fnmatch import fnmatch
import sys import sys
from util import ColorString from util import ColorString
# This script generates TARGETS file for Buck.
# Buck is a build tool specifying dependencies among different build targets.
# User can pass extra dependencies as a JSON object via command line, and this
# script can include these dependencies in the generate TARGETS file.
# Usage:
# $python buckifier/buckify_rocksdb.py
# (This generates a TARGET file without user-specified dependency for unit
# tests.)
# $python buckifier/buckify_rocksdb.py \
# '{"fake": { \
# "extra_deps": [":test_dep", "//fakes/module:mock1"], \
# "extra_compiler_flags": ["-DROCKSDB_LITE", "-Os"], \
# } \
# }'
# (Generated TARGETS file has test_dep and mock1 as dependencies for RocksDB
# unit tests, and will use the extra_compiler_flags to compile the unit test
# source.)
# tests to export as libraries for inclusion in other projects # tests to export as libraries for inclusion in other projects
_EXPORTED_TEST_LIBS = ["env_basic_test"] _EXPORTED_TEST_LIBS = ["env_basic_test"]
@ -86,8 +105,38 @@ def get_tests(repo_path):
return tests return tests
# Parse extra dependencies passed by user from command line
def get_dependencies():
deps_map = {
''.encode('ascii'): {
'extra_deps'.encode('ascii'): [],
'extra_compiler_flags'.encode('ascii'): []
}
}
if len(sys.argv) < 2:
return deps_map
def encode_dict(data):
rv = {}
for k, v in data.items():
if isinstance(k, unicode):
k = k.encode('ascii')
if isinstance(v, unicode):
v = v.encode('ascii')
elif isinstance(v, list):
v = [x.encode('ascii') for x in v]
elif isinstance(v, dict):
v = encode_dict(v)
rv[k] = v
return rv
extra_deps = json.loads(sys.argv[1], object_hook=encode_dict)
for target_alias, deps in extra_deps.items():
deps_map[target_alias] = deps
return deps_map
# Prepare TARGETS file for buck # Prepare TARGETS file for buck
def generate_targets(repo_path): def generate_targets(repo_path, deps_map):
print(ColorString.info("Generating TARGETS")) print(ColorString.info("Generating TARGETS"))
# parsed src.mk file # parsed src.mk file
src_mk = parse_src_mk(repo_path) src_mk = parse_src_mk(repo_path)
@ -121,24 +170,33 @@ def generate_targets(repo_path):
["test_util/testutil.cc"], ["test_util/testutil.cc"],
[":rocksdb_lib"]) [":rocksdb_lib"])
print("Extra dependencies:\n{0}".format(str(deps_map)))
# test for every test we found in the Makefile # test for every test we found in the Makefile
for test in sorted(tests): for target_alias, deps in deps_map.items():
match_src = [src for src in cc_files if ("/%s.c" % test) in src] for test in sorted(tests):
if len(match_src) == 0: match_src = [src for src in cc_files if ("/%s.c" % test) in src]
print(ColorString.warning("Cannot find .cc file for %s" % test)) if len(match_src) == 0:
continue print(ColorString.warning("Cannot find .cc file for %s" % test))
elif len(match_src) > 1: continue
print(ColorString.warning("Found more than one .cc for %s" % test)) elif len(match_src) > 1:
print(match_src) print(ColorString.warning("Found more than one .cc for %s" % test))
continue print(match_src)
continue
assert(len(match_src) == 1)
is_parallel = tests[test] assert(len(match_src) == 1)
TARGETS.register_test(test, match_src[0], is_parallel) is_parallel = tests[test]
test_target_name = \
if test in _EXPORTED_TEST_LIBS: test if not target_alias else test + "_" + target_alias
test_library = "%s_lib" % test TARGETS.register_test(
TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"]) test_target_name,
match_src[0],
is_parallel,
deps['extra_deps'],
deps['extra_compiler_flags'])
if test in _EXPORTED_TEST_LIBS:
test_library = "%s_lib" % test_target_name
TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"])
TARGETS.flush_tests() TARGETS.flush_tests()
print(ColorString.info("Generated TARGETS Summary:")) print(ColorString.info("Generated TARGETS Summary:"))
@ -163,8 +221,9 @@ def exit_with_error(msg):
def main(): def main():
deps_map = get_dependencies()
# Generate TARGETS file for buck # Generate TARGETS file for buck
ok = generate_targets(get_rocksdb_path()) ok = generate_targets(get_rocksdb_path(), deps_map)
if not ok: if not ok:
exit_with_error("Failed to generate TARGETS files") exit_with_error("Failed to generate TARGETS files")

@ -51,14 +51,21 @@ class TARGETSBuilder:
pretty_list(deps))) pretty_list(deps)))
self.total_bin = self.total_bin + 1 self.total_bin = self.total_bin + 1
def register_test(self, test_name, src, is_parallel): def register_test(self,
test_name,
src,
is_parallel,
extra_deps,
extra_compiler_flags):
exec_mode = "serial" exec_mode = "serial"
if is_parallel: if is_parallel:
exec_mode = "parallel" exec_mode = "parallel"
self.tests_cfg += targets_cfg.test_cfg_template % ( self.tests_cfg += targets_cfg.test_cfg_template % (
test_name, test_name,
str(src), str(src),
str(exec_mode)) str(exec_mode),
extra_deps,
extra_compiler_flags)
self.total_test = self.total_test + 1 self.total_test = self.total_test + 1

@ -140,11 +140,13 @@ test_cfg_template = """ [
"%s", "%s",
"%s", "%s",
"%s", "%s",
%s,
%s,
], ],
""" """
unittests_template = """ unittests_template = """
# [test_name, test_src, test_type] # [test_name, test_src, test_type, extra_deps, extra_compiler_flags]
ROCKS_TESTS = [ ROCKS_TESTS = [
%s] %s]
@ -153,6 +155,8 @@ ROCKS_TESTS = [
# will not be included. # will not be included.
[ [
test_binary( test_binary(
extra_compiler_flags = extra_compiler_flags,
extra_deps = extra_deps,
parallelism = parallelism, parallelism = parallelism,
rocksdb_arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS, rocksdb_arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
rocksdb_compiler_flags = ROCKSDB_COMPILER_FLAGS, rocksdb_compiler_flags = ROCKSDB_COMPILER_FLAGS,
@ -163,7 +167,7 @@ ROCKS_TESTS = [
test_cc = test_cc, test_cc = test_cc,
test_name = test_name, test_name = test_name,
) )
for test_name, test_cc, parallelism in ROCKS_TESTS for test_name, test_cc, parallelism, extra_deps, extra_compiler_flags in ROCKS_TESTS
if not is_opt_mode if not is_opt_mode
] ]
""" """

@ -12,7 +12,9 @@ def test_binary(
rocksdb_compiler_flags, rocksdb_compiler_flags,
rocksdb_preprocessor_flags, rocksdb_preprocessor_flags,
rocksdb_external_deps, rocksdb_external_deps,
rocksdb_os_deps): rocksdb_os_deps,
extra_deps,
extra_compiler_flags):
TEST_RUNNER = native.package_name() + "/buckifier/rocks_test_runner.sh" TEST_RUNNER = native.package_name() + "/buckifier/rocks_test_runner.sh"
ttype = "gtest" if parallelism == "parallel" else "simple" ttype = "gtest" if parallelism == "parallel" else "simple"
@ -23,9 +25,9 @@ def test_binary(
srcs = [test_cc], srcs = [test_cc],
arch_preprocessor_flags = rocksdb_arch_preprocessor_flags, arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
os_preprocessor_flags = rocksdb_os_preprocessor_flags, os_preprocessor_flags = rocksdb_os_preprocessor_flags,
compiler_flags = rocksdb_compiler_flags, compiler_flags = rocksdb_compiler_flags + extra_compiler_flags,
preprocessor_flags = rocksdb_preprocessor_flags, preprocessor_flags = rocksdb_preprocessor_flags,
deps = [":rocksdb_test_lib"], deps = [":rocksdb_test_lib"] + extra_deps,
os_deps = rocksdb_os_deps, os_deps = rocksdb_os_deps,
external_deps = rocksdb_external_deps, external_deps = rocksdb_external_deps,
) )

Loading…
Cancel
Save