From a30b75cdcf85e188c380a47ed4d69bcc4880b818 Mon Sep 17 00:00:00 2001 From: Islam AbdelRahman Date: Tue, 4 Apr 2017 16:09:31 -0700 Subject: [PATCH] Add buckifier script to github repo Summary: Add buckifier script and TARGETS file to github repo Closes https://github.com/facebook/rocksdb/pull/2083 Differential Revision: D4825822 Pulled By: IslamAbdelRahman fbshipit-source-id: 205f4a7 --- .gitignore | 2 +- TARGETS | 465 +++++++++++++++++++++++++++++++++ buckifier/buckify_rocksdb.py | 172 ++++++++++++ buckifier/rocks_test_runner.sh | 4 + buckifier/targets_builder.py | 65 +++++ buckifier/targets_cfg.py | 100 +++++++ buckifier/util.py | 107 ++++++++ 7 files changed, 914 insertions(+), 1 deletion(-) create mode 100644 TARGETS create mode 100644 buckifier/buckify_rocksdb.py create mode 100755 buckifier/rocks_test_runner.sh create mode 100644 buckifier/targets_builder.py create mode 100644 buckifier/targets_cfg.py create mode 100644 buckifier/util.py diff --git a/.gitignore b/.gitignore index a6fc78568..ee43456ce 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -TARGETS make_config.mk *.a @@ -71,3 +70,4 @@ db_logs/ tp2/ fbcode/ fbcode +buckifier/*.pyc diff --git a/TARGETS b/TARGETS new file mode 100644 index 000000000..053845fd8 --- /dev/null +++ b/TARGETS @@ -0,0 +1,465 @@ +REPO_PATH = "internal_repo_rocksdb/repo/" +BUCK_BINS = "buck-out/gen/" + REPO_PATH +TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh" +rocksdb_compiler_flags = [ + "-msse", + "-msse4.2", + "-fno-builtin-memcmp", + "-DROCKSDB_PLATFORM_POSIX", + "-DROCKSDB_LIB_IO_POSIX", + "-DROCKSDB_FALLOCATE_PRESENT", + "-DROCKSDB_MALLOC_USABLE_SIZE", + "-DOS_LINUX", + # Flags to enable libs we include + "-DSNAPPY", + "-DZLIB", + "-DBZIP2", + "-DLZ4", + "-DZSTD", + "-DGFLAGS=gflags", + "-DNUMA", + "-DTBB", + # Needed to compile in fbcode + "-Wno-expansion-to-defined", +] + +rocksdb_external_deps = [ + ('bzip2', None, 'bz2'), + ('snappy', None, "snappy"), + ('zlib', None, 'z'), + ('gflags', None, 'gflags'), + ('lz4', None, 'lz4'), + ('zstd', None), + ('tbb', None), + ("numa", "2.0.8", "numa"), + ("googletest", None, "gtest"), +] + +rocksdb_preprocessor_flags = [ + # Directories with files for #include + "-I" + REPO_PATH + "include/", + "-I" + REPO_PATH, +] + +cpp_library( + name = "rocksdb_lib", + headers = AutoHeaders.RECURSIVE_GLOB, + srcs = [ + "db/auto_roll_logger.cc", + "db/builder.cc", + "db/c.cc", + "db/column_family.cc", + "db/compacted_db_impl.cc", + "db/compaction.cc", + "db/compaction_iterator.cc", + "db/compaction_job.cc", + "db/compaction_picker.cc", + "db/convenience.cc", + "db/range_del_aggregator.cc", + "db/db_filesnapshot.cc", + "db/dbformat.cc", + "db/db_impl.cc", + "db/db_impl_debug.cc", + "db/db_impl_readonly.cc", + "db/db_impl_experimental.cc", + "db/db_info_dumper.cc", + "db/db_iter.cc", + "db/external_sst_file_ingestion_job.cc", + "db/experimental.cc", + "db/event_helpers.cc", + "db/file_indexer.cc", + "db/filename.cc", + "db/flush_job.cc", + "db/flush_scheduler.cc", + "db/forward_iterator.cc", + "db/internal_stats.cc", + "db/log_reader.cc", + "db/log_writer.cc", + "db/managed_iterator.cc", + "db/memtable_allocator.cc", + "db/memtable.cc", + "db/memtable_list.cc", + "db/merge_helper.cc", + "db/merge_operator.cc", + "db/repair.cc", + "db/snapshot_impl.cc", + "db/table_cache.cc", + "db/table_properties_collector.cc", + "db/transaction_log_impl.cc", + "db/version_builder.cc", + "db/version_edit.cc", + "db/version_set.cc", + "db/wal_manager.cc", + "db/write_batch.cc", + "db/write_batch_base.cc", + "db/write_controller.cc", + "db/write_thread.cc", + "memtable/hash_cuckoo_rep.cc", + "memtable/hash_linklist_rep.cc", + "memtable/hash_skiplist_rep.cc", + "memtable/skiplistrep.cc", + "memtable/vectorrep.cc", + "port/stack_trace.cc", + "port/port_posix.cc", + "table/adaptive_table_factory.cc", + "table/block_based_filter_block.cc", + "table/block_based_table_builder.cc", + "table/block_based_table_factory.cc", + "table/block_based_table_reader.cc", + "table/block_builder.cc", + "table/block.cc", + "table/block_prefix_index.cc", + "table/bloom_block.cc", + "table/cuckoo_table_builder.cc", + "table/cuckoo_table_factory.cc", + "table/cuckoo_table_reader.cc", + "table/flush_block_policy.cc", + "table/format.cc", + "table/full_filter_block.cc", + "table/get_context.cc", + "table/index_builder.cc", + "table/iterator.cc", + "table/merging_iterator.cc", + "table/meta_blocks.cc", + "table/sst_file_writer.cc", + "table/partitioned_filter_block.cc", + "table/plain_table_builder.cc", + "table/plain_table_factory.cc", + "table/plain_table_index.cc", + "table/plain_table_key_coding.cc", + "table/plain_table_reader.cc", + "table/persistent_cache_helper.cc", + "table/table_properties.cc", + "table/two_level_iterator.cc", + "tools/dump/db_dump_tool.cc", + "util/arena.cc", + "util/bloom.cc", + "util/build_version.cc", + "util/cf_options.cc", + "util/clock_cache.cc", + "util/coding.cc", + "util/comparator.cc", + "util/compaction_job_stats_impl.cc", + "util/concurrent_arena.cc", + "util/crc32c.cc", + "util/db_options.cc", + "util/delete_scheduler.cc", + "util/dynamic_bloom.cc", + "util/env.cc", + "util/env_chroot.cc", + "util/env_hdfs.cc", + "util/env_posix.cc", + "util/event_logger.cc", + "util/file_util.cc", + "util/file_reader_writer.cc", + "util/filter_policy.cc", + "util/hash.cc", + "util/histogram.cc", + "util/histogram_windowing.cc", + "util/instrumented_mutex.cc", + "util/iostats_context.cc", + "util/io_posix.cc", + "util/log_buffer.cc", + "util/logging.cc", + "util/lru_cache.cc", + "util/memenv.cc", + "util/murmurhash.cc", + "util/options.cc", + "util/options_helper.cc", + "util/options_parser.cc", + "util/options_sanity_check.cc", + "util/perf_context.cc", + "util/perf_level.cc", + "util/random.cc", + "util/rate_limiter.cc", + "util/sharded_cache.cc", + "util/slice.cc", + "util/sst_file_manager_impl.cc", + "util/statistics.cc", + "util/status.cc", + "util/status_message.cc", + "util/string_util.cc", + "util/sync_point.cc", + "util/thread_local.cc", + "util/thread_status_impl.cc", + "util/thread_status_updater.cc", + "util/thread_status_updater_debug.cc", + "util/thread_status_util.cc", + "util/thread_status_util_debug.cc", + "util/threadpool_imp.cc", + "util/transaction_test_util.cc", + "util/xxhash.cc", + "utilities/backupable/backupable_db.cc", + "utilities/blob_db/blob_db.cc", + "utilities/convenience/info_log_finder.cc", + "utilities/checkpoint/checkpoint.cc", + "utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc", + "utilities/document/document_db.cc", + "utilities/document/json_document_builder.cc", + "utilities/document/json_document.cc", + "utilities/env_mirror.cc", + "utilities/geodb/geodb_impl.cc", + "utilities/leveldb_options/leveldb_options.cc", + "utilities/lua/rocks_lua_compaction_filter.cc", + "utilities/memory/memory_util.cc", + "utilities/merge_operators/put.cc", + "utilities/merge_operators/max.cc", + "utilities/merge_operators/string_append/stringappend2.cc", + "utilities/merge_operators/string_append/stringappend.cc", + "utilities/merge_operators/uint64add.cc", + "utilities/option_change_migration/option_change_migration.cc", + "utilities/options/options_util.cc", + "utilities/persistent_cache/persistent_cache_tier.cc", + "utilities/persistent_cache/volatile_tier_impl.cc", + "utilities/persistent_cache/block_cache_tier_file.cc", + "utilities/persistent_cache/block_cache_tier_metadata.cc", + "utilities/persistent_cache/block_cache_tier.cc", + "utilities/redis/redis_lists.cc", + "utilities/simulator_cache/sim_cache.cc", + "utilities/spatialdb/spatial_db.cc", + "utilities/table_properties_collectors/compact_on_deletion_collector.cc", + "utilities/transactions/optimistic_transaction_impl.cc", + "utilities/transactions/optimistic_transaction_db_impl.cc", + "utilities/transactions/transaction_base.cc", + "utilities/transactions/transaction_db_impl.cc", + "utilities/transactions/transaction_db_mutex_impl.cc", + "utilities/transactions/transaction_lock_mgr.cc", + "utilities/transactions/transaction_impl.cc", + "utilities/transactions/transaction_util.cc", + "utilities/ttl/db_ttl_impl.cc", + "utilities/date_tiered/date_tiered_db_impl.cc", + "utilities/write_batch_with_index/write_batch_with_index.cc", + "utilities/write_batch_with_index/write_batch_with_index_internal.cc", + "tools/ldb_cmd.cc", + "tools/ldb_tool.cc", + "tools/sst_dump_tool.cc", + ], + deps = [], + preprocessor_flags = rocksdb_preprocessor_flags, + compiler_flags = rocksdb_compiler_flags, + external_deps = rocksdb_external_deps, +) + +cpp_library( + name = "rocksdb_test_lib", + headers = AutoHeaders.RECURSIVE_GLOB, + srcs = [ + "table/mock_table.cc", + "util/mock_env.cc", + "util/fault_injection_test_env.cc", + "util/testharness.cc", + "util/testutil.cc", + "db/db_test_util.cc", + "utilities/col_buf_encoder.cc", + "utilities/col_buf_decoder.cc", + "utilities/column_aware_encoding_util.cc", + ], + deps = [":rocksdb_lib"], + preprocessor_flags = rocksdb_preprocessor_flags, + compiler_flags = rocksdb_compiler_flags, + external_deps = rocksdb_external_deps, +) + +cpp_library( + name = "rocksdb_tools_lib", + headers = AutoHeaders.RECURSIVE_GLOB, + srcs = [ + "tools/db_bench_tool.cc", + "util/testutil.cc", + ], + deps = [":rocksdb_lib"], + preprocessor_flags = rocksdb_preprocessor_flags, + compiler_flags = rocksdb_compiler_flags, + external_deps = rocksdb_external_deps, +) + +cpp_library( + name = "env_basic_test_lib", + headers = AutoHeaders.RECURSIVE_GLOB, + srcs = ["util/env_basic_test.cc"], + deps = [":rocksdb_test_lib"], + preprocessor_flags = rocksdb_preprocessor_flags, + compiler_flags = rocksdb_compiler_flags, + external_deps = rocksdb_external_deps, +) + +# [test_name, test_src, test_type] +ROCKS_TESTS = [['merger_test', 'table/merger_test.cc', 'serial'], + ['cache_test', 'util/cache_test.cc', 'serial'], + ['options_file_test', 'db/options_file_test.cc', 'serial'], + ['compaction_picker_test', 'db/compaction_picker_test.cc', 'serial'], + ['corruption_test', 'db/corruption_test.cc', 'serial'], + ['thread_list_test', 'util/thread_list_test.cc', 'serial'], + ['table_properties_collector_test', + 'db/table_properties_collector_test.cc', + 'serial'], + ['document_db_test', 'utilities/document/document_db_test.cc', 'serial'], + ['event_logger_test', 'util/event_logger_test.cc', 'serial'], + ['coding_test', 'util/coding_test.cc', 'serial'], + ['statistics_test', 'util/statistics_test.cc', 'serial'], + ['options_settable_test', 'util/options_settable_test.cc', 'serial'], + ['sst_dump_test', 'tools/sst_dump_test.cc', 'serial'], + ['column_aware_encoding_test', + 'utilities/column_aware_encoding_test.cc', + 'serial'], + ['db_iterator_test', 'db/db_iterator_test.cc', 'serial'], + ['db_sst_test', 'db/db_sst_test.cc', 'parallel'], + ['geodb_test', 'utilities/geodb/geodb_test.cc', 'serial'], + ['listener_test', 'db/listener_test.cc', 'serial'], + ['write_callback_test', 'db/write_callback_test.cc', 'serial'], + ['version_set_test', 'db/version_set_test.cc', 'serial'], + ['full_filter_block_test', 'table/full_filter_block_test.cc', 'serial'], + ['cleanable_test', 'table/cleanable_test.cc', 'serial'], + ['checkpoint_test', 'utilities/checkpoint/checkpoint_test.cc', 'serial'], + ['compact_files_test', 'db/compact_files_test.cc', 'serial'], + ['db_options_test', 'db/db_options_test.cc', 'serial'], + ['object_registry_test', 'utilities/object_registry_test.cc', 'serial'], + ['blob_db_test', 'utilities/blob_db/blob_db_test.cc', 'serial'], + ['auto_roll_logger_test', 'db/auto_roll_logger_test.cc', 'serial'], + ['dbformat_test', 'db/dbformat_test.cc', 'serial'], + ['write_batch_with_index_test', + 'utilities/write_batch_with_index/write_batch_with_index_test.cc', + 'serial'], + ['json_document_test', 'utilities/document/json_document_test.cc', 'serial'], + ['file_reader_writer_test', 'util/file_reader_writer_test.cc', 'serial'], + ['repair_test', 'db/repair_test.cc', 'serial'], + ['persistent_cache_test', + 'utilities/persistent_cache/persistent_cache_test.cc', + 'parallel'], + ['db_bloom_filter_test', 'db/db_bloom_filter_test.cc', 'serial'], + ['external_sst_file_basic_test', + 'db/external_sst_file_basic_test.cc', + 'serial'], + ['options_test', 'util/options_test.cc', 'serial'], + ['perf_context_test', 'db/perf_context_test.cc', 'serial'], + ['db_block_cache_test', 'db/db_block_cache_test.cc', 'serial'], + ['heap_test', 'util/heap_test.cc', 'serial'], + ['db_test2', 'db/db_test2.cc', 'serial'], + ['filelock_test', 'util/filelock_test.cc', 'serial'], + ['write_controller_test', 'db/write_controller_test.cc', 'serial'], + ['compaction_iterator_test', 'db/compaction_iterator_test.cc', 'serial'], + ['spatial_db_test', 'utilities/spatialdb/spatial_db_test.cc', 'serial'], + ['c_test', 'db/c_test.c', 'serial'], + ['range_del_aggregator_test', 'db/range_del_aggregator_test.cc', 'serial'], + ['date_tiered_test', 'utilities/date_tiered/date_tiered_test.cc', 'serial'], + ['ldb_cmd_test', 'tools/ldb_cmd_test.cc', 'serial'], + ['db_test', 'db/db_test.cc', 'parallel'], + ['block_based_filter_block_test', + 'table/block_based_filter_block_test.cc', + 'serial'], + ['merge_test', 'db/merge_test.cc', 'serial'], + ['bloom_test', 'util/bloom_test.cc', 'serial'], + ['block_test', 'table/block_test.cc', 'serial'], + ['cuckoo_table_builder_test', 'table/cuckoo_table_builder_test.cc', 'serial'], + ['backupable_db_test', + 'utilities/backupable/backupable_db_test.cc', + 'parallel'], + ['db_flush_test', 'db/db_flush_test.cc', 'serial'], + ['filename_test', 'db/filename_test.cc', 'serial'], + ['cuckoo_table_reader_test', 'table/cuckoo_table_reader_test.cc', 'serial'], + ['slice_transform_test', 'util/slice_transform_test.cc', 'serial'], + ['cuckoo_table_db_test', 'db/cuckoo_table_db_test.cc', 'serial'], + ['inlineskiplist_test', 'db/inlineskiplist_test.cc', 'parallel'], + ['optimistic_transaction_test', + 'utilities/transactions/optimistic_transaction_test.cc', + 'serial'], + ['hash_table_test', + 'utilities/persistent_cache/hash_table_test.cc', + 'serial'], + ['db_dynamic_level_test', 'db/db_dynamic_level_test.cc', 'serial'], + ['option_change_migration_test', + 'utilities/option_change_migration/option_change_migration_test.cc', + 'serial'], + ['db_inplace_update_test', 'db/db_inplace_update_test.cc', 'serial'], + ['autovector_test', 'util/autovector_test.cc', 'serial'], + ['db_iter_test', 'db/db_iter_test.cc', 'serial'], + ['flush_job_test', 'db/flush_job_test.cc', 'serial'], + ['wal_manager_test', 'db/wal_manager_test.cc', 'serial'], + ['write_batch_test', 'db/write_batch_test.cc', 'serial'], + ['crc32c_test', 'util/crc32c_test.cc', 'serial'], + ['rate_limiter_test', 'util/rate_limiter_test.cc', 'serial'], + ['external_sst_file_test', 'db/external_sst_file_test.cc', 'parallel'], + ['compaction_job_test', 'db/compaction_job_test.cc', 'serial'], + ['mock_env_test', 'util/mock_env_test.cc', 'serial'], + ['db_table_properties_test', 'db/db_table_properties_test.cc', 'serial'], + ['db_compaction_test', 'db/db_compaction_test.cc', 'parallel'], + ['arena_test', 'util/arena_test.cc', 'serial'], + ['stringappend_test', + 'utilities/merge_operators/string_append/stringappend_test.cc', + 'serial'], + ['reduce_levels_test', 'tools/reduce_levels_test.cc', 'serial'], + ['prefix_test', 'db/prefix_test.cc', 'serial'], + ['ttl_test', 'utilities/ttl/ttl_test.cc', 'serial'], + ['merge_helper_test', 'db/merge_helper_test.cc', 'serial'], + ['file_indexer_test', 'db/file_indexer_test.cc', 'serial'], + ['memory_test', 'utilities/memory/memory_test.cc', 'serial'], + ['log_test', 'db/log_test.cc', 'serial'], + ['deletefile_test', 'db/deletefile_test.cc', 'serial'], + ['partitioned_filter_block_test', + 'table/partitioned_filter_block_test.cc', + 'serial'], + ['comparator_db_test', 'db/comparator_db_test.cc', 'serial'], + ['compaction_job_stats_test', 'db/compaction_job_stats_test.cc', 'serial'], + ['thread_local_test', 'util/thread_local_test.cc', 'serial'], + ['version_builder_test', 'db/version_builder_test.cc', 'serial'], + ['db_range_del_test', 'db/db_range_del_test.cc', 'serial'], + ['table_test', 'table/table_test.cc', 'parallel'], + ['db_tailing_iter_test', 'db/db_tailing_iter_test.cc', 'serial'], + ['db_compaction_filter_test', 'db/db_compaction_filter_test.cc', 'parallel'], + ['options_util_test', 'utilities/options/options_util_test.cc', 'serial'], + ['dynamic_bloom_test', 'util/dynamic_bloom_test.cc', 'serial'], + ['db_basic_test', 'db/db_basic_test.cc', 'serial'], + ['db_merge_operator_test', 'db/db_merge_operator_test.cc', 'serial'], + ['manual_compaction_test', 'db/manual_compaction_test.cc', 'parallel'], + ['delete_scheduler_test', 'util/delete_scheduler_test.cc', 'serial'], + ['transaction_test', 'utilities/transactions/transaction_test.cc', 'serial'], + ['db_io_failure_test', 'db/db_io_failure_test.cc', 'serial'], + ['db_log_iter_test', 'db/db_log_iter_test.cc', 'serial'], + ['compact_on_deletion_collector_test', + 'utilities/table_properties_collectors/compact_on_deletion_collector_test.cc', + 'serial'], + ['env_test', 'util/env_test.cc', 'serial'], + ['db_wal_test', 'db/db_wal_test.cc', 'parallel'], + ['sim_cache_test', 'utilities/simulator_cache/sim_cache_test.cc', 'serial'], + ['db_memtable_test', 'db/db_memtable_test.cc', 'serial'], + ['db_universal_compaction_test', + 'db/db_universal_compaction_test.cc', + 'parallel'], + ['histogram_test', 'util/histogram_test.cc', 'serial'], + ['util_merge_operators_test', + 'utilities/util_merge_operators_test.cc', + 'serial'], + ['fault_injection_test', 'db/fault_injection_test.cc', 'parallel'], + ['env_basic_test', 'util/env_basic_test.cc', 'serial'], + ['iostats_context_test', 'util/iostats_context_test.cc', 'serial'], + ['memtable_list_test', 'db/memtable_list_test.cc', 'serial'], + ['column_family_test', 'db/column_family_test.cc', 'serial'], + ['db_properties_test', 'db/db_properties_test.cc', 'serial'], + ['version_edit_test', 'db/version_edit_test.cc', 'serial'], + ['skiplist_test', 'db/skiplist_test.cc', 'serial'], + ['lru_cache_test', 'util/lru_cache_test.cc', 'serial'], + ['plain_table_db_test', 'db/plain_table_db_test.cc', 'serial']] + + +# Generate a test rule for each entry in ROCKS_TESTS +for test_cfg in ROCKS_TESTS: + test_name = test_cfg[0] + test_cc = test_cfg[1] + ttype = "gtest" if test_cfg[2] == "parallel" else "simple" + test_bin = test_name + "_bin" + + cpp_binary ( + name = test_bin, + srcs = [test_cc], + deps = [":rocksdb_test_lib"], + preprocessor_flags = rocksdb_preprocessor_flags, + compiler_flags = rocksdb_compiler_flags, + external_deps = rocksdb_external_deps, + ) + + custom_unittest( + name = test_name, + type = ttype, + deps = [":" + test_bin], + command = [TEST_RUNNER, BUCK_BINS + test_bin] + ) diff --git a/buckifier/buckify_rocksdb.py b/buckifier/buckify_rocksdb.py new file mode 100644 index 000000000..d31ffe346 --- /dev/null +++ b/buckifier/buckify_rocksdb.py @@ -0,0 +1,172 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +from targets_builder import TARGETSBuilder +from optparse import OptionParser +import os +import fnmatch +import sys +import tempfile + +from util import ColorString +import util + +# tests to export as libraries for inclusion in other projects +_EXPORTED_TEST_LIBS = ["env_basic_test"] + +# Parse src.mk files as a Dictionary of +# VAR_NAME => list of files +def parse_src_mk(repo_path): + src_mk = repo_path + "/src.mk" + src_files = {} + for line in open(src_mk): + line = line.strip() + if len(line) == 0 or line[0] == '#': + continue + if '=' in line: + current_src = line.split('=')[0].strip() + src_files[current_src] = [] + elif '.cc' in line: + src_path = line.split('.cc')[0].strip() + '.cc' + src_files[current_src].append(src_path) + return src_files + + +# get all .cc / .c files +def get_cc_files(repo_path): + cc_files = [] + for root, dirnames, filenames in os.walk(repo_path): + root = root[(len(repo_path) + 1):] + if "java" in root: + # Skip java + continue + for filename in fnmatch.filter(filenames, '*.cc'): + cc_files.append(os.path.join(root, filename)) + for filename in fnmatch.filter(filenames, '*.c'): + cc_files.append(os.path.join(root, filename)) + return cc_files + + +# Get tests from Makefile +def get_tests(repo_path): + Makefile = repo_path + "/Makefile" + + # Dictionary TEST_NAME => IS_PARALLEL + tests = {} + + found_tests = False + for line in open(Makefile): + line = line.strip() + if line.startswith("TESTS ="): + found_tests = True + elif found_tests: + if line.endswith("\\"): + # remove the trailing \ + line = line[:-1] + line = line.strip() + tests[line] = False + else: + # we consumed all the tests + break + + found_parallel_tests = False + for line in open(Makefile): + line = line.strip() + if line.startswith("PARALLEL_TEST ="): + found_parallel_tests = True + elif found_parallel_tests: + if line.endswith("\\"): + # remove the trailing \ + line = line[:-1] + line = line.strip() + tests[line] = True + else: + # we consumed all the parallel tests + break + + return tests + + +# Prepare TARGETS file for buck +def generate_targets(repo_path): + print(ColorString.info("Generating TARGETS")) + # parsed src.mk file + src_mk = parse_src_mk(repo_path) + # get all .cc files + cc_files = get_cc_files(repo_path) + # get tests from Makefile + tests = get_tests(repo_path) + + if src_mk is None or cc_files is None or tests is None: + return False + + TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path) + # rocksdb_lib + TARGETS.add_library( + "rocksdb_lib", + src_mk["LIB_SOURCES"] + + src_mk["TOOL_LIB_SOURCES"]) + # rocksdb_test_lib + TARGETS.add_library( + "rocksdb_test_lib", + src_mk.get("MOCK_LIB_SOURCES", []) + + src_mk.get("TEST_LIB_SOURCES", []) + + src_mk.get("EXP_LIB_SOURCES", []), + [":rocksdb_lib"]) + # rocksdb_tools_lib + TARGETS.add_library( + "rocksdb_tools_lib", + src_mk.get("BENCH_LIB_SOURCES", []) + + ["util/testutil.cc"], + [":rocksdb_lib"]) + + # test for every test we found in the Makefile + for test in tests: + match_src = [src for src in cc_files if ("/%s.c" % test) in src] + if len(match_src) == 0: + print(ColorString.warning("Cannot find .cc file for %s" % test)) + continue + elif len(match_src) > 1: + print(ColorString.warning("Found more than one .cc for %s" % test)) + print(match_src) + continue + + assert(len(match_src) == 1) + is_parallel = tests[test] + TARGETS.register_test(test, match_src[0], is_parallel) + + if test in _EXPORTED_TEST_LIBS: + test_library = "%s_lib" % test + TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"]) + TARGETS.flush_tests() + + print(ColorString.info("Generated TARGETS Summary:")) + print(ColorString.info("- %d libs" % TARGETS.total_lib)) + print(ColorString.info("- %d binarys" % TARGETS.total_bin)) + print(ColorString.info("- %d tests" % TARGETS.total_test)) + return True + + +def get_rocksdb_path(): + # rocksdb = {script_dir}/.. + script_dir = os.path.dirname(sys.argv[0]) + script_dir = os.path.abspath(script_dir) + rocksdb_path = os.path.abspath( + os.path.join(script_dir, "../")) + + return rocksdb_path + +def exit_with_error(msg): + print(ColorString.error(msg)) + sys.exit(1) + + +def main(): + # Generate TARGETS file for buck + ok = generate_targets(get_rocksdb_path()) + if not ok: + exit_with_error("Failed to generate TARGETS files") + +if __name__ == "__main__": + main() diff --git a/buckifier/rocks_test_runner.sh b/buckifier/rocks_test_runner.sh new file mode 100755 index 000000000..2ee216934 --- /dev/null +++ b/buckifier/rocks_test_runner.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# Create a tmp directory for the test to use +TEST_DIR=$(mktemp -d /dev/shm/fbcode_rocksdb_XXXXXXX) +TEST_TMPDIR="$TEST_DIR" $@ && rm -rf "$TEST_DIR" diff --git a/buckifier/targets_builder.py b/buckifier/targets_builder.py new file mode 100644 index 000000000..7d47d2d1f --- /dev/null +++ b/buckifier/targets_builder.py @@ -0,0 +1,65 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +import targets_cfg +import pprint + +# TODO(tec): replace this with PrettyPrinter +def pretty_list(lst, indent=6): + if lst is None or len(lst) == 0: + return "" + + if len(lst) == 1: + return "\"%s\"" % lst[0] + + separator = "\",\n%s\"" % (" " * indent) + res = separator.join(lst) + res = "\n" + (" " * indent) + "\"" + res + "\",\n" + (" " * (indent - 2)) + return res + + +class TARGETSBuilder: + def __init__(self, path): + self.path = path + self.targets_file = open(path, 'w') + self.targets_file.write(targets_cfg.rocksdb_target_header) + self.total_lib = 0 + self.total_bin = 0 + self.total_test = 0 + self.tests_cfg = [] + + def __del__(self): + self.targets_file.close() + + def add_library(self, name, srcs, deps=None, headers=None): + if headers is None: + headers = "AutoHeaders.RECURSIVE_GLOB" + self.targets_file.write(targets_cfg.library_template % ( + name, + headers, + pretty_list(srcs), + pretty_list(deps))) + self.total_lib = self.total_lib + 1 + + def add_binary(self, name, srcs, deps=None): + self.targets_file.write(targets_cfg.binary_template % ( + name, + pretty_list(srcs), + pretty_list(deps))) + self.total_bin = self.total_bin + 1 + + def register_test(self, test_name, src, is_parallel): + exec_mode = "serial" + if is_parallel: + exec_mode = "parallel" + self.tests_cfg.append([test_name, str(src), str(exec_mode)]) + + self.total_test = self.total_test + 1 + + def flush_tests(self): + self.targets_file.write(targets_cfg.unittests_template % ( + pprint.PrettyPrinter().pformat(self.tests_cfg) + )) + + self.tests_cfg = [] diff --git a/buckifier/targets_cfg.py b/buckifier/targets_cfg.py new file mode 100644 index 000000000..dcbcf3449 --- /dev/null +++ b/buckifier/targets_cfg.py @@ -0,0 +1,100 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +rocksdb_target_header = """REPO_PATH = "internal_repo_rocksdb/repo/" +BUCK_BINS = "buck-out/gen/" + REPO_PATH +TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh" +rocksdb_compiler_flags = [ + "-msse", + "-msse4.2", + "-fno-builtin-memcmp", + "-DROCKSDB_PLATFORM_POSIX", + "-DROCKSDB_LIB_IO_POSIX", + "-DROCKSDB_FALLOCATE_PRESENT", + "-DROCKSDB_MALLOC_USABLE_SIZE", + "-DOS_LINUX", + # Flags to enable libs we include + "-DSNAPPY", + "-DZLIB", + "-DBZIP2", + "-DLZ4", + "-DZSTD", + "-DGFLAGS=gflags", + "-DNUMA", + "-DTBB", + # Needed to compile in fbcode + "-Wno-expansion-to-defined", +] + +rocksdb_external_deps = [ + ('bzip2', None, 'bz2'), + ('snappy', None, "snappy"), + ('zlib', None, 'z'), + ('gflags', None, 'gflags'), + ('lz4', None, 'lz4'), + ('zstd', None), + ('tbb', None), + ("numa", "2.0.8", "numa"), + ("googletest", None, "gtest"), +] + +rocksdb_preprocessor_flags = [ + # Directories with files for #include + "-I" + REPO_PATH + "include/", + "-I" + REPO_PATH, +] +""" + + +library_template = """ +cpp_library( + name = "%s", + headers = %s, + srcs = [%s], + deps = [%s], + preprocessor_flags = rocksdb_preprocessor_flags, + compiler_flags = rocksdb_compiler_flags, + external_deps = rocksdb_external_deps, +) +""" + +binary_template = """ +cpp_binary( + name = "%s", + srcs = [%s], + deps = [%s], + preprocessor_flags = rocksdb_preprocessor_flags, + compiler_flags = rocksdb_compiler_flags, + external_deps = rocksdb_external_deps, +) +""" + +unittests_template = """ +# [test_name, test_src, test_type] +ROCKS_TESTS = %s + + +# Generate a test rule for each entry in ROCKS_TESTS +for test_cfg in ROCKS_TESTS: + test_name = test_cfg[0] + test_cc = test_cfg[1] + ttype = "gtest" if test_cfg[2] == "parallel" else "simple" + test_bin = test_name + "_bin" + + cpp_binary ( + name = test_bin, + srcs = [test_cc], + deps = [":rocksdb_test_lib"], + preprocessor_flags = rocksdb_preprocessor_flags, + compiler_flags = rocksdb_compiler_flags, + external_deps = rocksdb_external_deps, + ) + + custom_unittest( + name = test_name, + type = ttype, + deps = [":" + test_bin], + command = [TEST_RUNNER, BUCK_BINS + test_bin] + ) +""" diff --git a/buckifier/util.py b/buckifier/util.py new file mode 100644 index 000000000..350b7335c --- /dev/null +++ b/buckifier/util.py @@ -0,0 +1,107 @@ +""" +This module keeps commonly used components. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +import subprocess +import os +import time + +class ColorString: + """ Generate colorful strings on terminal """ + HEADER = '\033[95m' + BLUE = '\033[94m' + GREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + + @staticmethod + def _make_color_str(text, color): + return "".join([color, text.encode('utf-8'), ColorString.ENDC]) + + @staticmethod + def ok(text): + if ColorString.is_disabled: + return text + return ColorString._make_color_str(text, ColorString.GREEN) + + @staticmethod + def info(text): + if ColorString.is_disabled: + return text + return ColorString._make_color_str(text, ColorString.BLUE) + + @staticmethod + def header(text): + if ColorString.is_disabled: + return text + return ColorString._make_color_str(text, ColorString.HEADER) + + @staticmethod + def error(text): + if ColorString.is_disabled: + return text + return ColorString._make_color_str(text, ColorString.FAIL) + + @staticmethod + def warning(text): + if ColorString.is_disabled: + return text + return ColorString._make_color_str(text, ColorString.WARNING) + + is_disabled = False + + +def run_shell_command(shell_cmd, cmd_dir=None): + """ Run a single shell command. + @returns a tuple of shell command return code, stdout, stderr """ + + if cmd_dir is not None and not os.path.exists(cmd_dir): + run_shell_command("mkdir -p %s" % cmd_dir) + + start = time.time() + print("\t>>> Running: " + shell_cmd) + p = subprocess.Popen(shell_cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=cmd_dir) + stdout, stderr = p.communicate() + end = time.time() + + # Report time if we spent more than 5 minutes executing a command + execution_time = end - start + if execution_time > (60 * 5): + mins = (execution_time / 60) + secs = (execution_time % 60) + print("\t>time spent: %d minutes %d seconds" % (mins, secs)) + + + return p.returncode, stdout, stderr + + +def run_shell_commands(shell_cmds, cmd_dir=None, verbose=False): + """ Execute a sequence of shell commands, which is equivalent to + running `cmd1 && cmd2 && cmd3` + @returns boolean indication if all commands succeeds. + """ + + if cmd_dir: + print("\t=== Set current working directory => %s" % cmd_dir) + + for shell_cmd in shell_cmds: + ret_code, stdout, stderr = run_shell_command(shell_cmd, cmd_dir) + if stdout: + if verbose or ret_code != 0: + print(ColorString.info("stdout: \n"), stdout) + if stderr: + # contents in stderr is not necessarily to be error messages. + if verbose or ret_code != 0: + print(ColorString.error("stderr: \n"), stderr) + if ret_code != 0: + return False + + return True