update buckifier, add support for microbenchmarks (#9598)

Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/9598

Reviewed By: jay-zhuang, hodgesds

Differential Revision: D34130191

fbshipit-source-id: e5413f7d6af70a66940022d153b64a3383eccff1
main
Patrick Somaru 3 years ago committed by Facebook GitHub Bot
parent 2fbc672732
commit f066b5cecb
  1. 2749
      TARGETS
  2. 53
      buckifier/buckify_rocksdb.py
  3. 89
      buckifier/targets_builder.py
  4. 218
      buckifier/targets_cfg.py
  5. 56
      defs.bzl
  6. 2
      microbench/ribbon_bench.cc
  7. 1
      src.mk

2749
TARGETS

File diff suppressed because it is too large Load Diff

@ -163,9 +163,8 @@ def generate_targets(repo_path, deps_map):
src_mk.get("EXP_LIB_SOURCES", []) + src_mk.get("EXP_LIB_SOURCES", []) +
src_mk.get("ANALYZER_LIB_SOURCES", []), src_mk.get("ANALYZER_LIB_SOURCES", []),
[":rocksdb_lib"], [":rocksdb_lib"],
extra_external_deps=""" + [ extra_test_libs=True
("googletest", None, "gtest"), )
]""")
# rocksdb_tools_lib # rocksdb_tools_lib
TARGETS.add_library( TARGETS.add_library(
"rocksdb_tools_lib", "rocksdb_tools_lib",
@ -184,12 +183,19 @@ def generate_targets(repo_path, deps_map):
src_mk.get("ANALYZER_LIB_SOURCES", []) src_mk.get("ANALYZER_LIB_SOURCES", [])
+ src_mk.get('STRESS_LIB_SOURCES', []) + src_mk.get('STRESS_LIB_SOURCES', [])
+ ["test_util/testutil.cc"]) + ["test_util/testutil.cc"])
# bench binaries
for src in src_mk.get("MICROBENCH_SOURCES", []):
name = src.rsplit('/',1)[1].split('.')[0] if '/' in src else src.split('.')[0]
TARGETS.add_binary(
name,
[src],
[],
extra_bench_libs=True
)
print("Extra dependencies:\n{0}".format(json.dumps(deps_map))) print("Extra dependencies:\n{0}".format(json.dumps(deps_map)))
# Dictionary test executable name -> relative source file path # Dictionary test executable name -> relative source file path
test_source_map = {} test_source_map = {}
print(src_mk)
# c_test.c is added through TARGETS.add_c_test(). If there # c_test.c is added through TARGETS.add_c_test(). If there
# are more than one .c test file, we need to extend # are more than one .c test file, we need to extend
@ -200,6 +206,23 @@ def generate_targets(repo_path, deps_map):
return False return False
TARGETS.add_c_test() TARGETS.add_c_test()
try:
with open(f"{repo_path}/buckifier/bench.json") as json_file:
fast_fancy_bench_config_list = json.load(json_file)
for config_dict in fast_fancy_bench_config_list:
TARGETS.add_fancy_bench_config(config_dict['name'],config_dict['benchmarks'], False, config_dict['expected_runtime'])
with open(f"{repo_path}/buckifier/bench-slow.json") as json_file:
slow_fancy_bench_config_list = json.load(json_file)
for config_dict in slow_fancy_bench_config_list:
TARGETS.add_fancy_bench_config(config_dict['name']+"_slow",config_dict['benchmarks'], True, config_dict['expected_runtime'])
except (FileNotFoundError, KeyError):
print(ColorString.warning("Failed to process bench config jsons"))
pass
TARGETS.add_test_header()
for test_src in src_mk.get("TEST_MAIN_SOURCES", []): for test_src in src_mk.get("TEST_MAIN_SOURCES", []):
test = test_src.split('.c')[0].strip().split('/')[-1].strip() test = test_src.split('.c')[0].strip().split('/')[-1].strip()
test_source_map[test] = test_src test_source_map[test] = test_src
@ -213,17 +236,21 @@ def generate_targets(repo_path, deps_map):
test_target_name = \ test_target_name = \
test if not target_alias else test + "_" + target_alias test if not target_alias else test + "_" + target_alias
TARGETS.register_test(
test_target_name,
test_src,
test not in non_parallel_tests,
json.dumps(deps['extra_deps']),
json.dumps(deps['extra_compiler_flags']))
if test in _EXPORTED_TEST_LIBS: if test in _EXPORTED_TEST_LIBS:
test_library = "%s_lib" % test_target_name test_library = "%s_lib" % test_target_name
TARGETS.add_library(test_library, [test_src], [":rocksdb_test_lib"]) TARGETS.add_library(test_library, [test_src], deps=[":rocksdb_test_lib"], extra_test_libs=True)
TARGETS.flush_tests() TARGETS.register_test(
test_target_name,
test_src,
deps = json.dumps(deps['extra_deps'] + [':'+test_library]),
extra_compiler_flags = json.dumps(deps['extra_compiler_flags']))
else:
TARGETS.register_test(
test_target_name,
test_src,
deps = json.dumps(deps['extra_deps'] + [":rocksdb_test_lib"] ),
extra_compiler_flags = json.dumps(deps['extra_compiler_flags']))
print(ColorString.info("Generated TARGETS Summary:")) print(ColorString.info("Generated TARGETS Summary:"))
print(ColorString.info("- %d libs" % TARGETS.total_lib)) print(ColorString.info("- %d libs" % TARGETS.total_lib))

@ -10,6 +10,7 @@ except ImportError:
from __builtin__ import object from __builtin__ import object
from __builtin__ import str from __builtin__ import str
import targets_cfg import targets_cfg
import pprint
def pretty_list(lst, indent=8): def pretty_list(lst, indent=8):
if lst is None or len(lst) == 0: if lst is None or len(lst) == 0:
@ -40,85 +41,71 @@ class TARGETSBuilder(object):
self.targets_file.close() self.targets_file.close()
def add_library(self, name, srcs, deps=None, headers=None, def add_library(self, name, srcs, deps=None, headers=None,
extra_external_deps="", link_whole=False): extra_external_deps="", link_whole=False,
headers_attr_prefix = "" external_dependencies=None, extra_test_libs=False):
if headers is None: if headers is not None:
headers_attr_prefix = "auto_"
headers = "AutoHeaders.RECURSIVE_GLOB"
else:
headers = "[" + pretty_list(headers) + "]" headers = "[" + pretty_list(headers) + "]"
self.targets_file.write(targets_cfg.library_template.format( self.targets_file.write(targets_cfg.library_template.format(
name=name, name=name,
srcs=pretty_list(srcs), srcs=pretty_list(srcs),
headers_attr_prefix=headers_attr_prefix,
headers=headers, headers=headers,
deps=pretty_list(deps), deps=pretty_list(deps),
extra_external_deps=extra_external_deps, extra_external_deps=extra_external_deps,
link_whole=link_whole).encode("utf-8")) link_whole=link_whole,
external_dependencies=pretty_list(external_dependencies),
extra_test_libs=extra_test_libs
).encode("utf-8"))
self.total_lib = self.total_lib + 1 self.total_lib = self.total_lib + 1
def add_rocksdb_library(self, name, srcs, headers=None): def add_rocksdb_library(self, name, srcs, headers=None,
headers_attr_prefix = "" external_dependencies=None):
if headers is None: if headers is not None:
headers_attr_prefix = "auto_"
headers = "AutoHeaders.RECURSIVE_GLOB"
else:
headers = "[" + pretty_list(headers) + "]" headers = "[" + pretty_list(headers) + "]"
self.targets_file.write(targets_cfg.rocksdb_library_template.format( self.targets_file.write(targets_cfg.rocksdb_library_template.format(
name=name, name=name,
srcs=pretty_list(srcs), srcs=pretty_list(srcs),
headers_attr_prefix=headers_attr_prefix, headers=headers,
headers=headers).encode("utf-8")) external_dependencies=pretty_list(external_dependencies)
).encode("utf-8")
)
self.total_lib = self.total_lib + 1 self.total_lib = self.total_lib + 1
def add_binary(self, name, srcs, deps=None): def add_binary(self, name, srcs, deps=None, extra_preprocessor_flags=None,extra_bench_libs=False):
self.targets_file.write(targets_cfg.binary_template.format( self.targets_file.write(targets_cfg.binary_template.format(
name=name, name=name,
srcs=pretty_list(srcs), srcs=pretty_list(srcs),
deps=pretty_list(deps)).encode("utf-8")) deps=pretty_list(deps),
extra_preprocessor_flags=pretty_list(extra_preprocessor_flags),
extra_bench_libs=extra_bench_libs,
).encode("utf-8"))
self.total_bin = self.total_bin + 1 self.total_bin = self.total_bin + 1
def add_c_test(self): def add_c_test(self):
self.targets_file.write(b""" self.targets_file.write(b"""
cpp_binary( add_c_test_wrapper()
name = "c_test_bin", """)
srcs = ["db/c_test.c"],
arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
compiler_flags = ROCKSDB_COMPILER_FLAGS,
include_paths = ROCKSDB_INCLUDE_PATHS,
os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
deps = [":rocksdb_test_lib"],
) if not is_opt_mode else None
custom_unittest( def add_test_header(self):
name = "c_test", self.targets_file.write(b"""
command = [ # Generate a test rule for each entry in ROCKS_TESTS
native.package_name() + "/buckifier/rocks_test_runner.sh", # Do not build the tests in opt mode, since SyncPoint and other test code
"$(location :{})".format("c_test_bin"), # will not be included.
],
type = "simple",
) if not is_opt_mode else None
""") """)
def add_fancy_bench_config(self, name, bench_config, slow, expected_runtime):
self.targets_file.write(targets_cfg.fancy_bench_template.format(
name=name,
bench_config=pprint.pformat(bench_config),
slow=slow,
expected_runtime=expected_runtime,
).encode("utf-8"))
def register_test(self, def register_test(self,
test_name, test_name,
src, src,
is_parallel, deps,
extra_deps,
extra_compiler_flags): extra_compiler_flags):
exec_mode = "serial"
if is_parallel:
exec_mode = "parallel"
self.tests_cfg += targets_cfg.test_cfg_template % (
test_name,
str(src),
str(exec_mode),
extra_deps,
extra_compiler_flags)
self.targets_file.write(targets_cfg.unittests_template.format(test_name=test_name,test_cc=str(src),deps=deps,
extra_compiler_flags=extra_compiler_flags).encode("utf-8"))
self.total_test = self.total_test + 1 self.total_test = self.total_test + 1
def flush_tests(self):
self.targets_file.write(targets_cfg.unittests_template.format(tests=self.tests_cfg).encode("utf-8"))
self.tests_cfg = ""

@ -12,223 +12,35 @@ rocksdb_target_header_template = \
# only be validated by Facebook employees. # only be validated by Facebook employees.
# #
# @noautodeps @nocodemods # @noautodeps @nocodemods
load("//rocks/buckifier:defs.bzl", "cpp_library_wrapper","rocks_cpp_library_wrapper","cpp_binary_wrapper","cpp_unittest_wrapper","fancy_bench_wrapper","add_c_test_wrapper")
load("@fbcode_macros//build_defs:auto_headers.bzl", "AutoHeaders")
load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library")
load(":defs.bzl", "test_binary")
REPO_PATH = package_name() + "/"
ROCKSDB_COMPILER_FLAGS_0 = [
"-fno-builtin-memcmp",
# Allow offsetof to work on non-standard layout types. Some compiler could
# completely reject our usage of offsetof, but we will solve that when it
# happens.
"-Wno-invalid-offsetof",
# Added missing flags from output of build_detect_platform
"-Wnarrowing",
"-DROCKSDB_NO_DYNAMIC_EXTENSION",
]
ROCKSDB_EXTERNAL_DEPS = [
("bzip2", None, "bz2"),
("snappy", None, "snappy"),
("zlib", None, "z"),
("gflags", None, "gflags"),
("lz4", None, "lz4"),
("zstd", None, "zstd"),
]
ROCKSDB_OS_DEPS_0 = [
(
"linux",
[
"third-party//numa:numa",
"third-party//liburing:uring",
"third-party//tbb:tbb",
],
),
(
"macos",
["third-party//tbb:tbb"],
),
]
ROCKSDB_OS_PREPROCESSOR_FLAGS_0 = [
(
"linux",
[
"-DOS_LINUX",
"-DROCKSDB_FALLOCATE_PRESENT",
"-DROCKSDB_MALLOC_USABLE_SIZE",
"-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX",
"-DROCKSDB_RANGESYNC_PRESENT",
"-DROCKSDB_SCHED_GETCPU_PRESENT",
"-DROCKSDB_IOURING_PRESENT",
"-DHAVE_SSE42",
"-DLIBURING",
"-DNUMA",
"-DROCKSDB_PLATFORM_POSIX",
"-DROCKSDB_LIB_IO_POSIX",
"-DTBB",
],
),
(
"macos",
[
"-DOS_MACOSX",
"-DROCKSDB_PLATFORM_POSIX",
"-DROCKSDB_LIB_IO_POSIX",
"-DTBB",
],
),
(
"windows",
[
"-DOS_WIN",
"-DWIN32",
"-D_MBCS",
"-DWIN64",
"-DNOMINMAX",
],
),
]
ROCKSDB_PREPROCESSOR_FLAGS = [
"-DROCKSDB_SUPPORT_THREAD_LOCAL",
# Flags to enable libs we include
"-DSNAPPY",
"-DZLIB",
"-DBZIP2",
"-DLZ4",
"-DZSTD",
"-DZSTD_STATIC_LINKING_ONLY",
"-DGFLAGS=gflags",
# Added missing flags from output of build_detect_platform
"-DROCKSDB_BACKTRACE",
]
# Directories with files for #include
ROCKSDB_INCLUDE_PATHS = [
"",
"include",
]
ROCKSDB_ARCH_PREPROCESSOR_FLAGS = {{
"x86_64": [
"-DHAVE_PCLMUL",
],
}}
build_mode = read_config("fbcode", "build_mode")
is_opt_mode = build_mode.startswith("opt")
# -DNDEBUG is added by default in opt mode in fbcode. But adding it twice
# doesn't harm and avoid forgetting to add it.
ROCKSDB_COMPILER_FLAGS = ROCKSDB_COMPILER_FLAGS_0 + (["-DNDEBUG"] if is_opt_mode else [])
sanitizer = read_config("fbcode", "sanitizer")
# Do not enable jemalloc if sanitizer presents. RocksDB will further detect
# whether the binary is linked with jemalloc at runtime.
ROCKSDB_OS_PREPROCESSOR_FLAGS = ROCKSDB_OS_PREPROCESSOR_FLAGS_0 + ([(
"linux",
["-DROCKSDB_JEMALLOC"],
)] if sanitizer == "" else [])
ROCKSDB_OS_DEPS = ROCKSDB_OS_DEPS_0 + ([(
"linux",
["third-party//jemalloc:headers"],
)] if sanitizer == "" else [])
ROCKSDB_LIB_DEPS = [
":rocksdb_lib",
":rocksdb_test_lib",
] if not is_opt_mode else [":rocksdb_lib"]
""" """
library_template = """ library_template = """
cpp_library( cpp_library_wrapper(name="{name}", srcs=[{srcs}], deps=[{deps}], headers={headers}, link_whole={link_whole}, extra_test_libs={extra_test_libs})
name = "{name}",
srcs = [{srcs}],
{headers_attr_prefix}headers = {headers},
arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
compiler_flags = ROCKSDB_COMPILER_FLAGS,
include_paths = ROCKSDB_INCLUDE_PATHS,
link_whole = {link_whole},
os_deps = ROCKSDB_OS_DEPS,
os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
exported_deps = [{deps}],
exported_external_deps = ROCKSDB_EXTERNAL_DEPS{extra_external_deps},
)
""" """
rocksdb_library_template = """ rocksdb_library_template = """
cpp_library( rocks_cpp_library_wrapper(name="{name}", srcs=[{srcs}], headers={headers})
name = "{name}",
srcs = [{srcs}],
{headers_attr_prefix}headers = {headers},
arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
compiler_flags = ROCKSDB_COMPILER_FLAGS,
include_paths = ROCKSDB_INCLUDE_PATHS,
os_deps = ROCKSDB_OS_DEPS,
os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
exported_deps = ROCKSDB_LIB_DEPS,
exported_external_deps = ROCKSDB_EXTERNAL_DEPS,
)
""" """
binary_template = """ binary_template = """
cpp_binary( cpp_binary_wrapper(name="{name}", srcs=[{srcs}], deps=[{deps}], extra_preprocessor_flags=[{extra_preprocessor_flags}], extra_bench_libs={extra_bench_libs})
name = "{name}",
srcs = [{srcs}],
arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
compiler_flags = ROCKSDB_COMPILER_FLAGS,
preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
include_paths = ROCKSDB_INCLUDE_PATHS,
deps = [{deps}],
external_deps = ROCKSDB_EXTERNAL_DEPS,
)
""" """
test_cfg_template = """ [ unittests_template = """
"%s", cpp_unittest_wrapper(name="{test_name}",
"%s", srcs=["{test_cc}"],
"%s", deps={deps},
%s, extra_compiler_flags={extra_compiler_flags})
%s,
],
""" """
unittests_template = """ fancy_bench_template = """
# [test_name, test_src, test_type, extra_deps, extra_compiler_flags] fancy_bench_wrapper(suite_name="{name}", binary_to_bench_to_metric_list_map={bench_config}, slow={slow}, expected_runtime={expected_runtime})
ROCKS_TESTS = [
{tests}]
# Generate a test rule for each entry in ROCKS_TESTS
# Do not build the tests in opt mode, since SyncPoint and other test code
# will not be included.
[
cpp_unittest(
name = test_name,
srcs = [test_cc],
arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
compiler_flags = ROCKSDB_COMPILER_FLAGS + extra_compiler_flags,
include_paths = ROCKSDB_INCLUDE_PATHS,
os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
deps = [":rocksdb_test_lib"] + extra_deps,
external_deps = ROCKSDB_EXTERNAL_DEPS + [
("googletest", None, "gtest"),
],
)
for test_name, test_cc, parallelism, extra_deps, extra_compiler_flags in ROCKS_TESTS
if not is_opt_mode
]
""" """

@ -1,56 +0,0 @@
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
#
# defs.bzl - Definitions for Facebook-specific buck build integration
# in TARGETS
load("@fbcode_macros//build_defs:coverage.bzl", "coverage")
load("@fbcode_macros//build_defs:cpp_binary.bzl", "cpp_binary")
load("@fbcode_macros//build_defs:custom_unittest.bzl", "custom_unittest")
def test_binary(
test_name,
test_cc,
parallelism,
rocksdb_arch_preprocessor_flags,
rocksdb_os_preprocessor_flags,
rocksdb_compiler_flags,
rocksdb_preprocessor_flags,
rocksdb_external_deps,
rocksdb_os_deps,
extra_deps,
extra_compiler_flags):
TEST_RUNNER = native.package_name() + "/buckifier/rocks_test_runner.sh"
ttype = "gtest" if parallelism == "parallel" else "simple"
test_bin = test_name + "_bin"
cpp_binary(
name = test_bin,
srcs = [test_cc],
arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
os_preprocessor_flags = rocksdb_os_preprocessor_flags,
compiler_flags = rocksdb_compiler_flags + extra_compiler_flags,
preprocessor_flags = rocksdb_preprocessor_flags,
deps = [":rocksdb_test_lib"] + extra_deps,
os_deps = rocksdb_os_deps,
external_deps = rocksdb_external_deps,
)
binary_path = "$(location :{})".format(test_bin)
base_path = native.package_name()
tags = []
if coverage.is_coverage_enabled(base_path):
# This tag instructs testpilot to use
# the lower-memory coverage runner
# (e.g. it tells testpilot that the binary
# is actually instrumented with coverage info)
tags = ["coverage"]
custom_unittest(
name = test_name,
command = [TEST_RUNNER, binary_path],
type = ttype,
env = {"BUCK_BASE_BINARY": binary_path},
tags = tags,
)

@ -143,7 +143,7 @@ static void FilterQueryNegative(benchmark::State &state) {
fp_cnt++; fp_cnt++;
} }
} }
state.counters["FP %"] = state.counters["fp_pct"] =
benchmark::Counter(fp_cnt * 100, benchmark::Counter::kAvgIterations); benchmark::Counter(fp_cnt * 100, benchmark::Counter::kAvgIterations);
} }
BENCHMARK(FilterQueryNegative)->Apply(CustomArguments); BENCHMARK(FilterQueryNegative)->Apply(CustomArguments);

@ -590,6 +590,7 @@ TEST_MAIN_SOURCES_C = \
MICROBENCH_SOURCES = \ MICROBENCH_SOURCES = \
microbench/ribbon_bench.cc \ microbench/ribbon_bench.cc \
microbench/db_basic_bench.cc \
JNI_NATIVE_SOURCES = \ JNI_NATIVE_SOURCES = \
java/rocksjni/backupenginejni.cc \ java/rocksjni/backupenginejni.cc \

Loading…
Cancel
Save