Summary: Add buckifier script and TARGETS file to github repo Closes https://github.com/facebook/rocksdb/pull/2083 Differential Revision: D4825822 Pulled By: IslamAbdelRahman fbshipit-source-id: 205f4a7main
Islam AbdelRahman
8 years ago
committed by
Facebook Github Bot
parent
3450ac8c1b
commit
a30b75cdcf
@ -0,0 +1,465 @@ |
|||||||
|
REPO_PATH = "internal_repo_rocksdb/repo/" |
||||||
|
BUCK_BINS = "buck-out/gen/" + REPO_PATH |
||||||
|
TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh" |
||||||
|
rocksdb_compiler_flags = [ |
||||||
|
"-msse", |
||||||
|
"-msse4.2", |
||||||
|
"-fno-builtin-memcmp", |
||||||
|
"-DROCKSDB_PLATFORM_POSIX", |
||||||
|
"-DROCKSDB_LIB_IO_POSIX", |
||||||
|
"-DROCKSDB_FALLOCATE_PRESENT", |
||||||
|
"-DROCKSDB_MALLOC_USABLE_SIZE", |
||||||
|
"-DOS_LINUX", |
||||||
|
# Flags to enable libs we include |
||||||
|
"-DSNAPPY", |
||||||
|
"-DZLIB", |
||||||
|
"-DBZIP2", |
||||||
|
"-DLZ4", |
||||||
|
"-DZSTD", |
||||||
|
"-DGFLAGS=gflags", |
||||||
|
"-DNUMA", |
||||||
|
"-DTBB", |
||||||
|
# Needed to compile in fbcode |
||||||
|
"-Wno-expansion-to-defined", |
||||||
|
] |
||||||
|
|
||||||
|
rocksdb_external_deps = [ |
||||||
|
('bzip2', None, 'bz2'), |
||||||
|
('snappy', None, "snappy"), |
||||||
|
('zlib', None, 'z'), |
||||||
|
('gflags', None, 'gflags'), |
||||||
|
('lz4', None, 'lz4'), |
||||||
|
('zstd', None), |
||||||
|
('tbb', None), |
||||||
|
("numa", "2.0.8", "numa"), |
||||||
|
("googletest", None, "gtest"), |
||||||
|
] |
||||||
|
|
||||||
|
rocksdb_preprocessor_flags = [ |
||||||
|
# Directories with files for #include |
||||||
|
"-I" + REPO_PATH + "include/", |
||||||
|
"-I" + REPO_PATH, |
||||||
|
] |
||||||
|
|
||||||
|
cpp_library( |
||||||
|
name = "rocksdb_lib", |
||||||
|
headers = AutoHeaders.RECURSIVE_GLOB, |
||||||
|
srcs = [ |
||||||
|
"db/auto_roll_logger.cc", |
||||||
|
"db/builder.cc", |
||||||
|
"db/c.cc", |
||||||
|
"db/column_family.cc", |
||||||
|
"db/compacted_db_impl.cc", |
||||||
|
"db/compaction.cc", |
||||||
|
"db/compaction_iterator.cc", |
||||||
|
"db/compaction_job.cc", |
||||||
|
"db/compaction_picker.cc", |
||||||
|
"db/convenience.cc", |
||||||
|
"db/range_del_aggregator.cc", |
||||||
|
"db/db_filesnapshot.cc", |
||||||
|
"db/dbformat.cc", |
||||||
|
"db/db_impl.cc", |
||||||
|
"db/db_impl_debug.cc", |
||||||
|
"db/db_impl_readonly.cc", |
||||||
|
"db/db_impl_experimental.cc", |
||||||
|
"db/db_info_dumper.cc", |
||||||
|
"db/db_iter.cc", |
||||||
|
"db/external_sst_file_ingestion_job.cc", |
||||||
|
"db/experimental.cc", |
||||||
|
"db/event_helpers.cc", |
||||||
|
"db/file_indexer.cc", |
||||||
|
"db/filename.cc", |
||||||
|
"db/flush_job.cc", |
||||||
|
"db/flush_scheduler.cc", |
||||||
|
"db/forward_iterator.cc", |
||||||
|
"db/internal_stats.cc", |
||||||
|
"db/log_reader.cc", |
||||||
|
"db/log_writer.cc", |
||||||
|
"db/managed_iterator.cc", |
||||||
|
"db/memtable_allocator.cc", |
||||||
|
"db/memtable.cc", |
||||||
|
"db/memtable_list.cc", |
||||||
|
"db/merge_helper.cc", |
||||||
|
"db/merge_operator.cc", |
||||||
|
"db/repair.cc", |
||||||
|
"db/snapshot_impl.cc", |
||||||
|
"db/table_cache.cc", |
||||||
|
"db/table_properties_collector.cc", |
||||||
|
"db/transaction_log_impl.cc", |
||||||
|
"db/version_builder.cc", |
||||||
|
"db/version_edit.cc", |
||||||
|
"db/version_set.cc", |
||||||
|
"db/wal_manager.cc", |
||||||
|
"db/write_batch.cc", |
||||||
|
"db/write_batch_base.cc", |
||||||
|
"db/write_controller.cc", |
||||||
|
"db/write_thread.cc", |
||||||
|
"memtable/hash_cuckoo_rep.cc", |
||||||
|
"memtable/hash_linklist_rep.cc", |
||||||
|
"memtable/hash_skiplist_rep.cc", |
||||||
|
"memtable/skiplistrep.cc", |
||||||
|
"memtable/vectorrep.cc", |
||||||
|
"port/stack_trace.cc", |
||||||
|
"port/port_posix.cc", |
||||||
|
"table/adaptive_table_factory.cc", |
||||||
|
"table/block_based_filter_block.cc", |
||||||
|
"table/block_based_table_builder.cc", |
||||||
|
"table/block_based_table_factory.cc", |
||||||
|
"table/block_based_table_reader.cc", |
||||||
|
"table/block_builder.cc", |
||||||
|
"table/block.cc", |
||||||
|
"table/block_prefix_index.cc", |
||||||
|
"table/bloom_block.cc", |
||||||
|
"table/cuckoo_table_builder.cc", |
||||||
|
"table/cuckoo_table_factory.cc", |
||||||
|
"table/cuckoo_table_reader.cc", |
||||||
|
"table/flush_block_policy.cc", |
||||||
|
"table/format.cc", |
||||||
|
"table/full_filter_block.cc", |
||||||
|
"table/get_context.cc", |
||||||
|
"table/index_builder.cc", |
||||||
|
"table/iterator.cc", |
||||||
|
"table/merging_iterator.cc", |
||||||
|
"table/meta_blocks.cc", |
||||||
|
"table/sst_file_writer.cc", |
||||||
|
"table/partitioned_filter_block.cc", |
||||||
|
"table/plain_table_builder.cc", |
||||||
|
"table/plain_table_factory.cc", |
||||||
|
"table/plain_table_index.cc", |
||||||
|
"table/plain_table_key_coding.cc", |
||||||
|
"table/plain_table_reader.cc", |
||||||
|
"table/persistent_cache_helper.cc", |
||||||
|
"table/table_properties.cc", |
||||||
|
"table/two_level_iterator.cc", |
||||||
|
"tools/dump/db_dump_tool.cc", |
||||||
|
"util/arena.cc", |
||||||
|
"util/bloom.cc", |
||||||
|
"util/build_version.cc", |
||||||
|
"util/cf_options.cc", |
||||||
|
"util/clock_cache.cc", |
||||||
|
"util/coding.cc", |
||||||
|
"util/comparator.cc", |
||||||
|
"util/compaction_job_stats_impl.cc", |
||||||
|
"util/concurrent_arena.cc", |
||||||
|
"util/crc32c.cc", |
||||||
|
"util/db_options.cc", |
||||||
|
"util/delete_scheduler.cc", |
||||||
|
"util/dynamic_bloom.cc", |
||||||
|
"util/env.cc", |
||||||
|
"util/env_chroot.cc", |
||||||
|
"util/env_hdfs.cc", |
||||||
|
"util/env_posix.cc", |
||||||
|
"util/event_logger.cc", |
||||||
|
"util/file_util.cc", |
||||||
|
"util/file_reader_writer.cc", |
||||||
|
"util/filter_policy.cc", |
||||||
|
"util/hash.cc", |
||||||
|
"util/histogram.cc", |
||||||
|
"util/histogram_windowing.cc", |
||||||
|
"util/instrumented_mutex.cc", |
||||||
|
"util/iostats_context.cc", |
||||||
|
"util/io_posix.cc", |
||||||
|
"util/log_buffer.cc", |
||||||
|
"util/logging.cc", |
||||||
|
"util/lru_cache.cc", |
||||||
|
"util/memenv.cc", |
||||||
|
"util/murmurhash.cc", |
||||||
|
"util/options.cc", |
||||||
|
"util/options_helper.cc", |
||||||
|
"util/options_parser.cc", |
||||||
|
"util/options_sanity_check.cc", |
||||||
|
"util/perf_context.cc", |
||||||
|
"util/perf_level.cc", |
||||||
|
"util/random.cc", |
||||||
|
"util/rate_limiter.cc", |
||||||
|
"util/sharded_cache.cc", |
||||||
|
"util/slice.cc", |
||||||
|
"util/sst_file_manager_impl.cc", |
||||||
|
"util/statistics.cc", |
||||||
|
"util/status.cc", |
||||||
|
"util/status_message.cc", |
||||||
|
"util/string_util.cc", |
||||||
|
"util/sync_point.cc", |
||||||
|
"util/thread_local.cc", |
||||||
|
"util/thread_status_impl.cc", |
||||||
|
"util/thread_status_updater.cc", |
||||||
|
"util/thread_status_updater_debug.cc", |
||||||
|
"util/thread_status_util.cc", |
||||||
|
"util/thread_status_util_debug.cc", |
||||||
|
"util/threadpool_imp.cc", |
||||||
|
"util/transaction_test_util.cc", |
||||||
|
"util/xxhash.cc", |
||||||
|
"utilities/backupable/backupable_db.cc", |
||||||
|
"utilities/blob_db/blob_db.cc", |
||||||
|
"utilities/convenience/info_log_finder.cc", |
||||||
|
"utilities/checkpoint/checkpoint.cc", |
||||||
|
"utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc", |
||||||
|
"utilities/document/document_db.cc", |
||||||
|
"utilities/document/json_document_builder.cc", |
||||||
|
"utilities/document/json_document.cc", |
||||||
|
"utilities/env_mirror.cc", |
||||||
|
"utilities/geodb/geodb_impl.cc", |
||||||
|
"utilities/leveldb_options/leveldb_options.cc", |
||||||
|
"utilities/lua/rocks_lua_compaction_filter.cc", |
||||||
|
"utilities/memory/memory_util.cc", |
||||||
|
"utilities/merge_operators/put.cc", |
||||||
|
"utilities/merge_operators/max.cc", |
||||||
|
"utilities/merge_operators/string_append/stringappend2.cc", |
||||||
|
"utilities/merge_operators/string_append/stringappend.cc", |
||||||
|
"utilities/merge_operators/uint64add.cc", |
||||||
|
"utilities/option_change_migration/option_change_migration.cc", |
||||||
|
"utilities/options/options_util.cc", |
||||||
|
"utilities/persistent_cache/persistent_cache_tier.cc", |
||||||
|
"utilities/persistent_cache/volatile_tier_impl.cc", |
||||||
|
"utilities/persistent_cache/block_cache_tier_file.cc", |
||||||
|
"utilities/persistent_cache/block_cache_tier_metadata.cc", |
||||||
|
"utilities/persistent_cache/block_cache_tier.cc", |
||||||
|
"utilities/redis/redis_lists.cc", |
||||||
|
"utilities/simulator_cache/sim_cache.cc", |
||||||
|
"utilities/spatialdb/spatial_db.cc", |
||||||
|
"utilities/table_properties_collectors/compact_on_deletion_collector.cc", |
||||||
|
"utilities/transactions/optimistic_transaction_impl.cc", |
||||||
|
"utilities/transactions/optimistic_transaction_db_impl.cc", |
||||||
|
"utilities/transactions/transaction_base.cc", |
||||||
|
"utilities/transactions/transaction_db_impl.cc", |
||||||
|
"utilities/transactions/transaction_db_mutex_impl.cc", |
||||||
|
"utilities/transactions/transaction_lock_mgr.cc", |
||||||
|
"utilities/transactions/transaction_impl.cc", |
||||||
|
"utilities/transactions/transaction_util.cc", |
||||||
|
"utilities/ttl/db_ttl_impl.cc", |
||||||
|
"utilities/date_tiered/date_tiered_db_impl.cc", |
||||||
|
"utilities/write_batch_with_index/write_batch_with_index.cc", |
||||||
|
"utilities/write_batch_with_index/write_batch_with_index_internal.cc", |
||||||
|
"tools/ldb_cmd.cc", |
||||||
|
"tools/ldb_tool.cc", |
||||||
|
"tools/sst_dump_tool.cc", |
||||||
|
], |
||||||
|
deps = [], |
||||||
|
preprocessor_flags = rocksdb_preprocessor_flags, |
||||||
|
compiler_flags = rocksdb_compiler_flags, |
||||||
|
external_deps = rocksdb_external_deps, |
||||||
|
) |
||||||
|
|
||||||
|
cpp_library( |
||||||
|
name = "rocksdb_test_lib", |
||||||
|
headers = AutoHeaders.RECURSIVE_GLOB, |
||||||
|
srcs = [ |
||||||
|
"table/mock_table.cc", |
||||||
|
"util/mock_env.cc", |
||||||
|
"util/fault_injection_test_env.cc", |
||||||
|
"util/testharness.cc", |
||||||
|
"util/testutil.cc", |
||||||
|
"db/db_test_util.cc", |
||||||
|
"utilities/col_buf_encoder.cc", |
||||||
|
"utilities/col_buf_decoder.cc", |
||||||
|
"utilities/column_aware_encoding_util.cc", |
||||||
|
], |
||||||
|
deps = [":rocksdb_lib"], |
||||||
|
preprocessor_flags = rocksdb_preprocessor_flags, |
||||||
|
compiler_flags = rocksdb_compiler_flags, |
||||||
|
external_deps = rocksdb_external_deps, |
||||||
|
) |
||||||
|
|
||||||
|
cpp_library( |
||||||
|
name = "rocksdb_tools_lib", |
||||||
|
headers = AutoHeaders.RECURSIVE_GLOB, |
||||||
|
srcs = [ |
||||||
|
"tools/db_bench_tool.cc", |
||||||
|
"util/testutil.cc", |
||||||
|
], |
||||||
|
deps = [":rocksdb_lib"], |
||||||
|
preprocessor_flags = rocksdb_preprocessor_flags, |
||||||
|
compiler_flags = rocksdb_compiler_flags, |
||||||
|
external_deps = rocksdb_external_deps, |
||||||
|
) |
||||||
|
|
||||||
|
cpp_library( |
||||||
|
name = "env_basic_test_lib", |
||||||
|
headers = AutoHeaders.RECURSIVE_GLOB, |
||||||
|
srcs = ["util/env_basic_test.cc"], |
||||||
|
deps = [":rocksdb_test_lib"], |
||||||
|
preprocessor_flags = rocksdb_preprocessor_flags, |
||||||
|
compiler_flags = rocksdb_compiler_flags, |
||||||
|
external_deps = rocksdb_external_deps, |
||||||
|
) |
||||||
|
|
||||||
|
# [test_name, test_src, test_type] |
||||||
|
ROCKS_TESTS = [['merger_test', 'table/merger_test.cc', 'serial'], |
||||||
|
['cache_test', 'util/cache_test.cc', 'serial'], |
||||||
|
['options_file_test', 'db/options_file_test.cc', 'serial'], |
||||||
|
['compaction_picker_test', 'db/compaction_picker_test.cc', 'serial'], |
||||||
|
['corruption_test', 'db/corruption_test.cc', 'serial'], |
||||||
|
['thread_list_test', 'util/thread_list_test.cc', 'serial'], |
||||||
|
['table_properties_collector_test', |
||||||
|
'db/table_properties_collector_test.cc', |
||||||
|
'serial'], |
||||||
|
['document_db_test', 'utilities/document/document_db_test.cc', 'serial'], |
||||||
|
['event_logger_test', 'util/event_logger_test.cc', 'serial'], |
||||||
|
['coding_test', 'util/coding_test.cc', 'serial'], |
||||||
|
['statistics_test', 'util/statistics_test.cc', 'serial'], |
||||||
|
['options_settable_test', 'util/options_settable_test.cc', 'serial'], |
||||||
|
['sst_dump_test', 'tools/sst_dump_test.cc', 'serial'], |
||||||
|
['column_aware_encoding_test', |
||||||
|
'utilities/column_aware_encoding_test.cc', |
||||||
|
'serial'], |
||||||
|
['db_iterator_test', 'db/db_iterator_test.cc', 'serial'], |
||||||
|
['db_sst_test', 'db/db_sst_test.cc', 'parallel'], |
||||||
|
['geodb_test', 'utilities/geodb/geodb_test.cc', 'serial'], |
||||||
|
['listener_test', 'db/listener_test.cc', 'serial'], |
||||||
|
['write_callback_test', 'db/write_callback_test.cc', 'serial'], |
||||||
|
['version_set_test', 'db/version_set_test.cc', 'serial'], |
||||||
|
['full_filter_block_test', 'table/full_filter_block_test.cc', 'serial'], |
||||||
|
['cleanable_test', 'table/cleanable_test.cc', 'serial'], |
||||||
|
['checkpoint_test', 'utilities/checkpoint/checkpoint_test.cc', 'serial'], |
||||||
|
['compact_files_test', 'db/compact_files_test.cc', 'serial'], |
||||||
|
['db_options_test', 'db/db_options_test.cc', 'serial'], |
||||||
|
['object_registry_test', 'utilities/object_registry_test.cc', 'serial'], |
||||||
|
['blob_db_test', 'utilities/blob_db/blob_db_test.cc', 'serial'], |
||||||
|
['auto_roll_logger_test', 'db/auto_roll_logger_test.cc', 'serial'], |
||||||
|
['dbformat_test', 'db/dbformat_test.cc', 'serial'], |
||||||
|
['write_batch_with_index_test', |
||||||
|
'utilities/write_batch_with_index/write_batch_with_index_test.cc', |
||||||
|
'serial'], |
||||||
|
['json_document_test', 'utilities/document/json_document_test.cc', 'serial'], |
||||||
|
['file_reader_writer_test', 'util/file_reader_writer_test.cc', 'serial'], |
||||||
|
['repair_test', 'db/repair_test.cc', 'serial'], |
||||||
|
['persistent_cache_test', |
||||||
|
'utilities/persistent_cache/persistent_cache_test.cc', |
||||||
|
'parallel'], |
||||||
|
['db_bloom_filter_test', 'db/db_bloom_filter_test.cc', 'serial'], |
||||||
|
['external_sst_file_basic_test', |
||||||
|
'db/external_sst_file_basic_test.cc', |
||||||
|
'serial'], |
||||||
|
['options_test', 'util/options_test.cc', 'serial'], |
||||||
|
['perf_context_test', 'db/perf_context_test.cc', 'serial'], |
||||||
|
['db_block_cache_test', 'db/db_block_cache_test.cc', 'serial'], |
||||||
|
['heap_test', 'util/heap_test.cc', 'serial'], |
||||||
|
['db_test2', 'db/db_test2.cc', 'serial'], |
||||||
|
['filelock_test', 'util/filelock_test.cc', 'serial'], |
||||||
|
['write_controller_test', 'db/write_controller_test.cc', 'serial'], |
||||||
|
['compaction_iterator_test', 'db/compaction_iterator_test.cc', 'serial'], |
||||||
|
['spatial_db_test', 'utilities/spatialdb/spatial_db_test.cc', 'serial'], |
||||||
|
['c_test', 'db/c_test.c', 'serial'], |
||||||
|
['range_del_aggregator_test', 'db/range_del_aggregator_test.cc', 'serial'], |
||||||
|
['date_tiered_test', 'utilities/date_tiered/date_tiered_test.cc', 'serial'], |
||||||
|
['ldb_cmd_test', 'tools/ldb_cmd_test.cc', 'serial'], |
||||||
|
['db_test', 'db/db_test.cc', 'parallel'], |
||||||
|
['block_based_filter_block_test', |
||||||
|
'table/block_based_filter_block_test.cc', |
||||||
|
'serial'], |
||||||
|
['merge_test', 'db/merge_test.cc', 'serial'], |
||||||
|
['bloom_test', 'util/bloom_test.cc', 'serial'], |
||||||
|
['block_test', 'table/block_test.cc', 'serial'], |
||||||
|
['cuckoo_table_builder_test', 'table/cuckoo_table_builder_test.cc', 'serial'], |
||||||
|
['backupable_db_test', |
||||||
|
'utilities/backupable/backupable_db_test.cc', |
||||||
|
'parallel'], |
||||||
|
['db_flush_test', 'db/db_flush_test.cc', 'serial'], |
||||||
|
['filename_test', 'db/filename_test.cc', 'serial'], |
||||||
|
['cuckoo_table_reader_test', 'table/cuckoo_table_reader_test.cc', 'serial'], |
||||||
|
['slice_transform_test', 'util/slice_transform_test.cc', 'serial'], |
||||||
|
['cuckoo_table_db_test', 'db/cuckoo_table_db_test.cc', 'serial'], |
||||||
|
['inlineskiplist_test', 'db/inlineskiplist_test.cc', 'parallel'], |
||||||
|
['optimistic_transaction_test', |
||||||
|
'utilities/transactions/optimistic_transaction_test.cc', |
||||||
|
'serial'], |
||||||
|
['hash_table_test', |
||||||
|
'utilities/persistent_cache/hash_table_test.cc', |
||||||
|
'serial'], |
||||||
|
['db_dynamic_level_test', 'db/db_dynamic_level_test.cc', 'serial'], |
||||||
|
['option_change_migration_test', |
||||||
|
'utilities/option_change_migration/option_change_migration_test.cc', |
||||||
|
'serial'], |
||||||
|
['db_inplace_update_test', 'db/db_inplace_update_test.cc', 'serial'], |
||||||
|
['autovector_test', 'util/autovector_test.cc', 'serial'], |
||||||
|
['db_iter_test', 'db/db_iter_test.cc', 'serial'], |
||||||
|
['flush_job_test', 'db/flush_job_test.cc', 'serial'], |
||||||
|
['wal_manager_test', 'db/wal_manager_test.cc', 'serial'], |
||||||
|
['write_batch_test', 'db/write_batch_test.cc', 'serial'], |
||||||
|
['crc32c_test', 'util/crc32c_test.cc', 'serial'], |
||||||
|
['rate_limiter_test', 'util/rate_limiter_test.cc', 'serial'], |
||||||
|
['external_sst_file_test', 'db/external_sst_file_test.cc', 'parallel'], |
||||||
|
['compaction_job_test', 'db/compaction_job_test.cc', 'serial'], |
||||||
|
['mock_env_test', 'util/mock_env_test.cc', 'serial'], |
||||||
|
['db_table_properties_test', 'db/db_table_properties_test.cc', 'serial'], |
||||||
|
['db_compaction_test', 'db/db_compaction_test.cc', 'parallel'], |
||||||
|
['arena_test', 'util/arena_test.cc', 'serial'], |
||||||
|
['stringappend_test', |
||||||
|
'utilities/merge_operators/string_append/stringappend_test.cc', |
||||||
|
'serial'], |
||||||
|
['reduce_levels_test', 'tools/reduce_levels_test.cc', 'serial'], |
||||||
|
['prefix_test', 'db/prefix_test.cc', 'serial'], |
||||||
|
['ttl_test', 'utilities/ttl/ttl_test.cc', 'serial'], |
||||||
|
['merge_helper_test', 'db/merge_helper_test.cc', 'serial'], |
||||||
|
['file_indexer_test', 'db/file_indexer_test.cc', 'serial'], |
||||||
|
['memory_test', 'utilities/memory/memory_test.cc', 'serial'], |
||||||
|
['log_test', 'db/log_test.cc', 'serial'], |
||||||
|
['deletefile_test', 'db/deletefile_test.cc', 'serial'], |
||||||
|
['partitioned_filter_block_test', |
||||||
|
'table/partitioned_filter_block_test.cc', |
||||||
|
'serial'], |
||||||
|
['comparator_db_test', 'db/comparator_db_test.cc', 'serial'], |
||||||
|
['compaction_job_stats_test', 'db/compaction_job_stats_test.cc', 'serial'], |
||||||
|
['thread_local_test', 'util/thread_local_test.cc', 'serial'], |
||||||
|
['version_builder_test', 'db/version_builder_test.cc', 'serial'], |
||||||
|
['db_range_del_test', 'db/db_range_del_test.cc', 'serial'], |
||||||
|
['table_test', 'table/table_test.cc', 'parallel'], |
||||||
|
['db_tailing_iter_test', 'db/db_tailing_iter_test.cc', 'serial'], |
||||||
|
['db_compaction_filter_test', 'db/db_compaction_filter_test.cc', 'parallel'], |
||||||
|
['options_util_test', 'utilities/options/options_util_test.cc', 'serial'], |
||||||
|
['dynamic_bloom_test', 'util/dynamic_bloom_test.cc', 'serial'], |
||||||
|
['db_basic_test', 'db/db_basic_test.cc', 'serial'], |
||||||
|
['db_merge_operator_test', 'db/db_merge_operator_test.cc', 'serial'], |
||||||
|
['manual_compaction_test', 'db/manual_compaction_test.cc', 'parallel'], |
||||||
|
['delete_scheduler_test', 'util/delete_scheduler_test.cc', 'serial'], |
||||||
|
['transaction_test', 'utilities/transactions/transaction_test.cc', 'serial'], |
||||||
|
['db_io_failure_test', 'db/db_io_failure_test.cc', 'serial'], |
||||||
|
['db_log_iter_test', 'db/db_log_iter_test.cc', 'serial'], |
||||||
|
['compact_on_deletion_collector_test', |
||||||
|
'utilities/table_properties_collectors/compact_on_deletion_collector_test.cc', |
||||||
|
'serial'], |
||||||
|
['env_test', 'util/env_test.cc', 'serial'], |
||||||
|
['db_wal_test', 'db/db_wal_test.cc', 'parallel'], |
||||||
|
['sim_cache_test', 'utilities/simulator_cache/sim_cache_test.cc', 'serial'], |
||||||
|
['db_memtable_test', 'db/db_memtable_test.cc', 'serial'], |
||||||
|
['db_universal_compaction_test', |
||||||
|
'db/db_universal_compaction_test.cc', |
||||||
|
'parallel'], |
||||||
|
['histogram_test', 'util/histogram_test.cc', 'serial'], |
||||||
|
['util_merge_operators_test', |
||||||
|
'utilities/util_merge_operators_test.cc', |
||||||
|
'serial'], |
||||||
|
['fault_injection_test', 'db/fault_injection_test.cc', 'parallel'], |
||||||
|
['env_basic_test', 'util/env_basic_test.cc', 'serial'], |
||||||
|
['iostats_context_test', 'util/iostats_context_test.cc', 'serial'], |
||||||
|
['memtable_list_test', 'db/memtable_list_test.cc', 'serial'], |
||||||
|
['column_family_test', 'db/column_family_test.cc', 'serial'], |
||||||
|
['db_properties_test', 'db/db_properties_test.cc', 'serial'], |
||||||
|
['version_edit_test', 'db/version_edit_test.cc', 'serial'], |
||||||
|
['skiplist_test', 'db/skiplist_test.cc', 'serial'], |
||||||
|
['lru_cache_test', 'util/lru_cache_test.cc', 'serial'], |
||||||
|
['plain_table_db_test', 'db/plain_table_db_test.cc', 'serial']] |
||||||
|
|
||||||
|
|
||||||
|
# Generate a test rule for each entry in ROCKS_TESTS |
||||||
|
for test_cfg in ROCKS_TESTS: |
||||||
|
test_name = test_cfg[0] |
||||||
|
test_cc = test_cfg[1] |
||||||
|
ttype = "gtest" if test_cfg[2] == "parallel" else "simple" |
||||||
|
test_bin = test_name + "_bin" |
||||||
|
|
||||||
|
cpp_binary ( |
||||||
|
name = test_bin, |
||||||
|
srcs = [test_cc], |
||||||
|
deps = [":rocksdb_test_lib"], |
||||||
|
preprocessor_flags = rocksdb_preprocessor_flags, |
||||||
|
compiler_flags = rocksdb_compiler_flags, |
||||||
|
external_deps = rocksdb_external_deps, |
||||||
|
) |
||||||
|
|
||||||
|
custom_unittest( |
||||||
|
name = test_name, |
||||||
|
type = ttype, |
||||||
|
deps = [":" + test_bin], |
||||||
|
command = [TEST_RUNNER, BUCK_BINS + test_bin] |
||||||
|
) |
||||||
@ -0,0 +1,172 @@ |
|||||||
|
from __future__ import absolute_import |
||||||
|
from __future__ import division |
||||||
|
from __future__ import print_function |
||||||
|
from __future__ import unicode_literals |
||||||
|
from targets_builder import TARGETSBuilder |
||||||
|
from optparse import OptionParser |
||||||
|
import os |
||||||
|
import fnmatch |
||||||
|
import sys |
||||||
|
import tempfile |
||||||
|
|
||||||
|
from util import ColorString |
||||||
|
import util |
||||||
|
|
||||||
|
# tests to export as libraries for inclusion in other projects |
||||||
|
_EXPORTED_TEST_LIBS = ["env_basic_test"] |
||||||
|
|
||||||
|
# Parse src.mk files as a Dictionary of |
||||||
|
# VAR_NAME => list of files |
||||||
|
def parse_src_mk(repo_path): |
||||||
|
src_mk = repo_path + "/src.mk" |
||||||
|
src_files = {} |
||||||
|
for line in open(src_mk): |
||||||
|
line = line.strip() |
||||||
|
if len(line) == 0 or line[0] == '#': |
||||||
|
continue |
||||||
|
if '=' in line: |
||||||
|
current_src = line.split('=')[0].strip() |
||||||
|
src_files[current_src] = [] |
||||||
|
elif '.cc' in line: |
||||||
|
src_path = line.split('.cc')[0].strip() + '.cc' |
||||||
|
src_files[current_src].append(src_path) |
||||||
|
return src_files |
||||||
|
|
||||||
|
|
||||||
|
# get all .cc / .c files |
||||||
|
def get_cc_files(repo_path): |
||||||
|
cc_files = [] |
||||||
|
for root, dirnames, filenames in os.walk(repo_path): |
||||||
|
root = root[(len(repo_path) + 1):] |
||||||
|
if "java" in root: |
||||||
|
# Skip java |
||||||
|
continue |
||||||
|
for filename in fnmatch.filter(filenames, '*.cc'): |
||||||
|
cc_files.append(os.path.join(root, filename)) |
||||||
|
for filename in fnmatch.filter(filenames, '*.c'): |
||||||
|
cc_files.append(os.path.join(root, filename)) |
||||||
|
return cc_files |
||||||
|
|
||||||
|
|
||||||
|
# Get tests from Makefile |
||||||
|
def get_tests(repo_path): |
||||||
|
Makefile = repo_path + "/Makefile" |
||||||
|
|
||||||
|
# Dictionary TEST_NAME => IS_PARALLEL |
||||||
|
tests = {} |
||||||
|
|
||||||
|
found_tests = False |
||||||
|
for line in open(Makefile): |
||||||
|
line = line.strip() |
||||||
|
if line.startswith("TESTS ="): |
||||||
|
found_tests = True |
||||||
|
elif found_tests: |
||||||
|
if line.endswith("\\"): |
||||||
|
# remove the trailing \ |
||||||
|
line = line[:-1] |
||||||
|
line = line.strip() |
||||||
|
tests[line] = False |
||||||
|
else: |
||||||
|
# we consumed all the tests |
||||||
|
break |
||||||
|
|
||||||
|
found_parallel_tests = False |
||||||
|
for line in open(Makefile): |
||||||
|
line = line.strip() |
||||||
|
if line.startswith("PARALLEL_TEST ="): |
||||||
|
found_parallel_tests = True |
||||||
|
elif found_parallel_tests: |
||||||
|
if line.endswith("\\"): |
||||||
|
# remove the trailing \ |
||||||
|
line = line[:-1] |
||||||
|
line = line.strip() |
||||||
|
tests[line] = True |
||||||
|
else: |
||||||
|
# we consumed all the parallel tests |
||||||
|
break |
||||||
|
|
||||||
|
return tests |
||||||
|
|
||||||
|
|
||||||
|
# Prepare TARGETS file for buck |
||||||
|
def generate_targets(repo_path): |
||||||
|
print(ColorString.info("Generating TARGETS")) |
||||||
|
# parsed src.mk file |
||||||
|
src_mk = parse_src_mk(repo_path) |
||||||
|
# get all .cc files |
||||||
|
cc_files = get_cc_files(repo_path) |
||||||
|
# get tests from Makefile |
||||||
|
tests = get_tests(repo_path) |
||||||
|
|
||||||
|
if src_mk is None or cc_files is None or tests is None: |
||||||
|
return False |
||||||
|
|
||||||
|
TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path) |
||||||
|
# rocksdb_lib |
||||||
|
TARGETS.add_library( |
||||||
|
"rocksdb_lib", |
||||||
|
src_mk["LIB_SOURCES"] + |
||||||
|
src_mk["TOOL_LIB_SOURCES"]) |
||||||
|
# rocksdb_test_lib |
||||||
|
TARGETS.add_library( |
||||||
|
"rocksdb_test_lib", |
||||||
|
src_mk.get("MOCK_LIB_SOURCES", []) + |
||||||
|
src_mk.get("TEST_LIB_SOURCES", []) + |
||||||
|
src_mk.get("EXP_LIB_SOURCES", []), |
||||||
|
[":rocksdb_lib"]) |
||||||
|
# rocksdb_tools_lib |
||||||
|
TARGETS.add_library( |
||||||
|
"rocksdb_tools_lib", |
||||||
|
src_mk.get("BENCH_LIB_SOURCES", []) + |
||||||
|
["util/testutil.cc"], |
||||||
|
[":rocksdb_lib"]) |
||||||
|
|
||||||
|
# test for every test we found in the Makefile |
||||||
|
for test in tests: |
||||||
|
match_src = [src for src in cc_files if ("/%s.c" % test) in src] |
||||||
|
if len(match_src) == 0: |
||||||
|
print(ColorString.warning("Cannot find .cc file for %s" % test)) |
||||||
|
continue |
||||||
|
elif len(match_src) > 1: |
||||||
|
print(ColorString.warning("Found more than one .cc for %s" % test)) |
||||||
|
print(match_src) |
||||||
|
continue |
||||||
|
|
||||||
|
assert(len(match_src) == 1) |
||||||
|
is_parallel = tests[test] |
||||||
|
TARGETS.register_test(test, match_src[0], is_parallel) |
||||||
|
|
||||||
|
if test in _EXPORTED_TEST_LIBS: |
||||||
|
test_library = "%s_lib" % test |
||||||
|
TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"]) |
||||||
|
TARGETS.flush_tests() |
||||||
|
|
||||||
|
print(ColorString.info("Generated TARGETS Summary:")) |
||||||
|
print(ColorString.info("- %d libs" % TARGETS.total_lib)) |
||||||
|
print(ColorString.info("- %d binarys" % TARGETS.total_bin)) |
||||||
|
print(ColorString.info("- %d tests" % TARGETS.total_test)) |
||||||
|
return True |
||||||
|
|
||||||
|
|
||||||
|
def get_rocksdb_path(): |
||||||
|
# rocksdb = {script_dir}/.. |
||||||
|
script_dir = os.path.dirname(sys.argv[0]) |
||||||
|
script_dir = os.path.abspath(script_dir) |
||||||
|
rocksdb_path = os.path.abspath( |
||||||
|
os.path.join(script_dir, "../")) |
||||||
|
|
||||||
|
return rocksdb_path |
||||||
|
|
||||||
|
def exit_with_error(msg): |
||||||
|
print(ColorString.error(msg)) |
||||||
|
sys.exit(1) |
||||||
|
|
||||||
|
|
||||||
|
def main(): |
||||||
|
# Generate TARGETS file for buck |
||||||
|
ok = generate_targets(get_rocksdb_path()) |
||||||
|
if not ok: |
||||||
|
exit_with_error("Failed to generate TARGETS files") |
||||||
|
|
||||||
|
if __name__ == "__main__": |
||||||
|
main() |
||||||
@ -0,0 +1,4 @@ |
|||||||
|
#!/bin/bash |
||||||
|
# Create a tmp directory for the test to use |
||||||
|
TEST_DIR=$(mktemp -d /dev/shm/fbcode_rocksdb_XXXXXXX) |
||||||
|
TEST_TMPDIR="$TEST_DIR" $@ && rm -rf "$TEST_DIR" |
||||||
@ -0,0 +1,65 @@ |
|||||||
|
from __future__ import absolute_import |
||||||
|
from __future__ import division |
||||||
|
from __future__ import print_function |
||||||
|
from __future__ import unicode_literals |
||||||
|
import targets_cfg |
||||||
|
import pprint |
||||||
|
|
||||||
|
# TODO(tec): replace this with PrettyPrinter |
||||||
|
def pretty_list(lst, indent=6): |
||||||
|
if lst is None or len(lst) == 0: |
||||||
|
return "" |
||||||
|
|
||||||
|
if len(lst) == 1: |
||||||
|
return "\"%s\"" % lst[0] |
||||||
|
|
||||||
|
separator = "\",\n%s\"" % (" " * indent) |
||||||
|
res = separator.join(lst) |
||||||
|
res = "\n" + (" " * indent) + "\"" + res + "\",\n" + (" " * (indent - 2)) |
||||||
|
return res |
||||||
|
|
||||||
|
|
||||||
|
class TARGETSBuilder: |
||||||
|
def __init__(self, path): |
||||||
|
self.path = path |
||||||
|
self.targets_file = open(path, 'w') |
||||||
|
self.targets_file.write(targets_cfg.rocksdb_target_header) |
||||||
|
self.total_lib = 0 |
||||||
|
self.total_bin = 0 |
||||||
|
self.total_test = 0 |
||||||
|
self.tests_cfg = [] |
||||||
|
|
||||||
|
def __del__(self): |
||||||
|
self.targets_file.close() |
||||||
|
|
||||||
|
def add_library(self, name, srcs, deps=None, headers=None): |
||||||
|
if headers is None: |
||||||
|
headers = "AutoHeaders.RECURSIVE_GLOB" |
||||||
|
self.targets_file.write(targets_cfg.library_template % ( |
||||||
|
name, |
||||||
|
headers, |
||||||
|
pretty_list(srcs), |
||||||
|
pretty_list(deps))) |
||||||
|
self.total_lib = self.total_lib + 1 |
||||||
|
|
||||||
|
def add_binary(self, name, srcs, deps=None): |
||||||
|
self.targets_file.write(targets_cfg.binary_template % ( |
||||||
|
name, |
||||||
|
pretty_list(srcs), |
||||||
|
pretty_list(deps))) |
||||||
|
self.total_bin = self.total_bin + 1 |
||||||
|
|
||||||
|
def register_test(self, test_name, src, is_parallel): |
||||||
|
exec_mode = "serial" |
||||||
|
if is_parallel: |
||||||
|
exec_mode = "parallel" |
||||||
|
self.tests_cfg.append([test_name, str(src), str(exec_mode)]) |
||||||
|
|
||||||
|
self.total_test = self.total_test + 1 |
||||||
|
|
||||||
|
def flush_tests(self): |
||||||
|
self.targets_file.write(targets_cfg.unittests_template % ( |
||||||
|
pprint.PrettyPrinter().pformat(self.tests_cfg) |
||||||
|
)) |
||||||
|
|
||||||
|
self.tests_cfg = [] |
||||||
@ -0,0 +1,100 @@ |
|||||||
|
from __future__ import absolute_import |
||||||
|
from __future__ import division |
||||||
|
from __future__ import print_function |
||||||
|
from __future__ import unicode_literals |
||||||
|
rocksdb_target_header = """REPO_PATH = "internal_repo_rocksdb/repo/" |
||||||
|
BUCK_BINS = "buck-out/gen/" + REPO_PATH |
||||||
|
TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh" |
||||||
|
rocksdb_compiler_flags = [ |
||||||
|
"-msse", |
||||||
|
"-msse4.2", |
||||||
|
"-fno-builtin-memcmp", |
||||||
|
"-DROCKSDB_PLATFORM_POSIX", |
||||||
|
"-DROCKSDB_LIB_IO_POSIX", |
||||||
|
"-DROCKSDB_FALLOCATE_PRESENT", |
||||||
|
"-DROCKSDB_MALLOC_USABLE_SIZE", |
||||||
|
"-DOS_LINUX", |
||||||
|
# Flags to enable libs we include |
||||||
|
"-DSNAPPY", |
||||||
|
"-DZLIB", |
||||||
|
"-DBZIP2", |
||||||
|
"-DLZ4", |
||||||
|
"-DZSTD", |
||||||
|
"-DGFLAGS=gflags", |
||||||
|
"-DNUMA", |
||||||
|
"-DTBB", |
||||||
|
# Needed to compile in fbcode |
||||||
|
"-Wno-expansion-to-defined", |
||||||
|
] |
||||||
|
|
||||||
|
rocksdb_external_deps = [ |
||||||
|
('bzip2', None, 'bz2'), |
||||||
|
('snappy', None, "snappy"), |
||||||
|
('zlib', None, 'z'), |
||||||
|
('gflags', None, 'gflags'), |
||||||
|
('lz4', None, 'lz4'), |
||||||
|
('zstd', None), |
||||||
|
('tbb', None), |
||||||
|
("numa", "2.0.8", "numa"), |
||||||
|
("googletest", None, "gtest"), |
||||||
|
] |
||||||
|
|
||||||
|
rocksdb_preprocessor_flags = [ |
||||||
|
# Directories with files for #include |
||||||
|
"-I" + REPO_PATH + "include/", |
||||||
|
"-I" + REPO_PATH, |
||||||
|
] |
||||||
|
""" |
||||||
|
|
||||||
|
|
||||||
|
library_template = """ |
||||||
|
cpp_library( |
||||||
|
name = "%s", |
||||||
|
headers = %s, |
||||||
|
srcs = [%s], |
||||||
|
deps = [%s], |
||||||
|
preprocessor_flags = rocksdb_preprocessor_flags, |
||||||
|
compiler_flags = rocksdb_compiler_flags, |
||||||
|
external_deps = rocksdb_external_deps, |
||||||
|
) |
||||||
|
""" |
||||||
|
|
||||||
|
binary_template = """ |
||||||
|
cpp_binary( |
||||||
|
name = "%s", |
||||||
|
srcs = [%s], |
||||||
|
deps = [%s], |
||||||
|
preprocessor_flags = rocksdb_preprocessor_flags, |
||||||
|
compiler_flags = rocksdb_compiler_flags, |
||||||
|
external_deps = rocksdb_external_deps, |
||||||
|
) |
||||||
|
""" |
||||||
|
|
||||||
|
unittests_template = """ |
||||||
|
# [test_name, test_src, test_type] |
||||||
|
ROCKS_TESTS = %s |
||||||
|
|
||||||
|
|
||||||
|
# Generate a test rule for each entry in ROCKS_TESTS |
||||||
|
for test_cfg in ROCKS_TESTS: |
||||||
|
test_name = test_cfg[0] |
||||||
|
test_cc = test_cfg[1] |
||||||
|
ttype = "gtest" if test_cfg[2] == "parallel" else "simple" |
||||||
|
test_bin = test_name + "_bin" |
||||||
|
|
||||||
|
cpp_binary ( |
||||||
|
name = test_bin, |
||||||
|
srcs = [test_cc], |
||||||
|
deps = [":rocksdb_test_lib"], |
||||||
|
preprocessor_flags = rocksdb_preprocessor_flags, |
||||||
|
compiler_flags = rocksdb_compiler_flags, |
||||||
|
external_deps = rocksdb_external_deps, |
||||||
|
) |
||||||
|
|
||||||
|
custom_unittest( |
||||||
|
name = test_name, |
||||||
|
type = ttype, |
||||||
|
deps = [":" + test_bin], |
||||||
|
command = [TEST_RUNNER, BUCK_BINS + test_bin] |
||||||
|
) |
||||||
|
""" |
||||||
@ -0,0 +1,107 @@ |
|||||||
|
""" |
||||||
|
This module keeps commonly used components. |
||||||
|
""" |
||||||
|
from __future__ import absolute_import |
||||||
|
from __future__ import division |
||||||
|
from __future__ import print_function |
||||||
|
from __future__ import unicode_literals |
||||||
|
import subprocess |
||||||
|
import os |
||||||
|
import time |
||||||
|
|
||||||
|
class ColorString: |
||||||
|
""" Generate colorful strings on terminal """ |
||||||
|
HEADER = '\033[95m' |
||||||
|
BLUE = '\033[94m' |
||||||
|
GREEN = '\033[92m' |
||||||
|
WARNING = '\033[93m' |
||||||
|
FAIL = '\033[91m' |
||||||
|
ENDC = '\033[0m' |
||||||
|
|
||||||
|
@staticmethod |
||||||
|
def _make_color_str(text, color): |
||||||
|
return "".join([color, text.encode('utf-8'), ColorString.ENDC]) |
||||||
|
|
||||||
|
@staticmethod |
||||||
|
def ok(text): |
||||||
|
if ColorString.is_disabled: |
||||||
|
return text |
||||||
|
return ColorString._make_color_str(text, ColorString.GREEN) |
||||||
|
|
||||||
|
@staticmethod |
||||||
|
def info(text): |
||||||
|
if ColorString.is_disabled: |
||||||
|
return text |
||||||
|
return ColorString._make_color_str(text, ColorString.BLUE) |
||||||
|
|
||||||
|
@staticmethod |
||||||
|
def header(text): |
||||||
|
if ColorString.is_disabled: |
||||||
|
return text |
||||||
|
return ColorString._make_color_str(text, ColorString.HEADER) |
||||||
|
|
||||||
|
@staticmethod |
||||||
|
def error(text): |
||||||
|
if ColorString.is_disabled: |
||||||
|
return text |
||||||
|
return ColorString._make_color_str(text, ColorString.FAIL) |
||||||
|
|
||||||
|
@staticmethod |
||||||
|
def warning(text): |
||||||
|
if ColorString.is_disabled: |
||||||
|
return text |
||||||
|
return ColorString._make_color_str(text, ColorString.WARNING) |
||||||
|
|
||||||
|
is_disabled = False |
||||||
|
|
||||||
|
|
||||||
|
def run_shell_command(shell_cmd, cmd_dir=None): |
||||||
|
""" Run a single shell command. |
||||||
|
@returns a tuple of shell command return code, stdout, stderr """ |
||||||
|
|
||||||
|
if cmd_dir is not None and not os.path.exists(cmd_dir): |
||||||
|
run_shell_command("mkdir -p %s" % cmd_dir) |
||||||
|
|
||||||
|
start = time.time() |
||||||
|
print("\t>>> Running: " + shell_cmd) |
||||||
|
p = subprocess.Popen(shell_cmd, |
||||||
|
shell=True, |
||||||
|
stdout=subprocess.PIPE, |
||||||
|
stderr=subprocess.PIPE, |
||||||
|
cwd=cmd_dir) |
||||||
|
stdout, stderr = p.communicate() |
||||||
|
end = time.time() |
||||||
|
|
||||||
|
# Report time if we spent more than 5 minutes executing a command |
||||||
|
execution_time = end - start |
||||||
|
if execution_time > (60 * 5): |
||||||
|
mins = (execution_time / 60) |
||||||
|
secs = (execution_time % 60) |
||||||
|
print("\t>time spent: %d minutes %d seconds" % (mins, secs)) |
||||||
|
|
||||||
|
|
||||||
|
return p.returncode, stdout, stderr |
||||||
|
|
||||||
|
|
||||||
|
def run_shell_commands(shell_cmds, cmd_dir=None, verbose=False): |
||||||
|
""" Execute a sequence of shell commands, which is equivalent to |
||||||
|
running `cmd1 && cmd2 && cmd3` |
||||||
|
@returns boolean indication if all commands succeeds. |
||||||
|
""" |
||||||
|
|
||||||
|
if cmd_dir: |
||||||
|
print("\t=== Set current working directory => %s" % cmd_dir) |
||||||
|
|
||||||
|
for shell_cmd in shell_cmds: |
||||||
|
ret_code, stdout, stderr = run_shell_command(shell_cmd, cmd_dir) |
||||||
|
if stdout: |
||||||
|
if verbose or ret_code != 0: |
||||||
|
print(ColorString.info("stdout: \n"), stdout) |
||||||
|
if stderr: |
||||||
|
# contents in stderr is not necessarily to be error messages. |
||||||
|
if verbose or ret_code != 0: |
||||||
|
print(ColorString.error("stderr: \n"), stderr) |
||||||
|
if ret_code != 0: |
||||||
|
return False |
||||||
|
|
||||||
|
return True |
||||||
Loading…
Reference in new issue