|
|
|
|
load("@fbcode_macros//build_defs:auto_headers.bzl", "AutoHeaders")
|
|
|
|
|
load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library")
|
|
|
|
|
load(":defs.bzl", "test_binary")
|
|
|
|
|
|
|
|
|
|
REPO_PATH = package_name() + "/"
|
|
|
|
|
|
|
|
|
|
ROCKSDB_COMPILER_FLAGS = [
|
|
|
|
|
"-fno-builtin-memcmp",
|
|
|
|
|
# Needed to compile in fbcode
|
|
|
|
|
"-Wno-expansion-to-defined",
|
|
|
|
|
# Added missing flags from output of build_detect_platform
|
|
|
|
|
"-Wnarrowing",
|
|
|
|
|
"-DROCKSDB_NO_DYNAMIC_EXTENSION",
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
ROCKSDB_EXTERNAL_DEPS = [
|
|
|
|
|
("bzip2", None, "bz2"),
|
|
|
|
|
("snappy", None, "snappy"),
|
|
|
|
|
("zlib", None, "z"),
|
|
|
|
|
("gflags", None, "gflags"),
|
|
|
|
|
("lz4", None, "lz4"),
|
|
|
|
|
("zstd", None),
|
|
|
|
|
("tbb", None),
|
|
|
|
|
("googletest", None, "gtest"),
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
ROCKSDB_OS_DEPS = [
|
|
|
|
|
(
|
|
|
|
|
"linux",
|
|
|
|
|
["third-party//numa:numa"],
|
|
|
|
|
),
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
ROCKSDB_OS_PREPROCESSOR_FLAGS = [
|
|
|
|
|
(
|
|
|
|
|
"linux",
|
|
|
|
|
[
|
|
|
|
|
"-DOS_LINUX",
|
|
|
|
|
"-DROCKSDB_FALLOCATE_PRESENT",
|
|
|
|
|
"-DROCKSDB_MALLOC_USABLE_SIZE",
|
|
|
|
|
"-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX",
|
|
|
|
|
"-DROCKSDB_RANGESYNC_PRESENT",
|
|
|
|
|
"-DROCKSDB_SCHED_GETCPU_PRESENT",
|
|
|
|
|
"-DHAVE_SSE42",
|
|
|
|
|
"-DNUMA",
|
|
|
|
|
],
|
|
|
|
|
),
|
|
|
|
|
(
|
|
|
|
|
"macos",
|
|
|
|
|
["-DOS_MACOSX"],
|
|
|
|
|
),
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
ROCKSDB_PREPROCESSOR_FLAGS = [
|
|
|
|
|
"-DROCKSDB_PLATFORM_POSIX",
|
|
|
|
|
"-DROCKSDB_LIB_IO_POSIX",
|
|
|
|
|
"-DROCKSDB_SUPPORT_THREAD_LOCAL",
|
|
|
|
|
|
|
|
|
|
# Flags to enable libs we include
|
|
|
|
|
"-DSNAPPY",
|
|
|
|
|
"-DZLIB",
|
|
|
|
|
"-DBZIP2",
|
|
|
|
|
"-DLZ4",
|
|
|
|
|
"-DZSTD",
|
|
|
|
|
"-DZSTD_STATIC_LINKING_ONLY",
|
|
|
|
|
"-DGFLAGS=gflags",
|
|
|
|
|
"-DTBB",
|
|
|
|
|
|
|
|
|
|
# Added missing flags from output of build_detect_platform
|
|
|
|
|
"-DROCKSDB_BACKTRACE",
|
|
|
|
|
|
|
|
|
|
# Directories with files for #include
|
|
|
|
|
"-I" + REPO_PATH + "include/",
|
|
|
|
|
"-I" + REPO_PATH,
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
ROCKSDB_ARCH_PREPROCESSOR_FLAGS = {
|
|
|
|
|
"x86_64": [
|
|
|
|
|
"-DHAVE_PCLMUL",
|
|
|
|
|
],
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
build_mode = read_config("fbcode", "build_mode")
|
|
|
|
|
|
|
|
|
|
is_opt_mode = build_mode.startswith("opt")
|
|
|
|
|
|
|
|
|
|
# -DNDEBUG is added by default in opt mode in fbcode. But adding it twice
|
|
|
|
|
# doesn't harm and avoid forgetting to add it.
|
|
|
|
|
ROCKSDB_COMPILER_FLAGS += (["-DNDEBUG"] if is_opt_mode else [])
|
|
|
|
|
|
|
|
|
|
sanitizer = read_config("fbcode", "sanitizer")
|
|
|
|
|
|
|
|
|
|
# Do not enable jemalloc if sanitizer presents. RocksDB will further detect
|
|
|
|
|
# whether the binary is linked with jemalloc at runtime.
|
|
|
|
|
ROCKSDB_OS_PREPROCESSOR_FLAGS += ([(
|
|
|
|
|
"linux",
|
|
|
|
|
["-DROCKSDB_JEMALLOC"],
|
|
|
|
|
)] if sanitizer == "" else [])
|
|
|
|
|
|
|
|
|
|
ROCKSDB_OS_DEPS += ([(
|
|
|
|
|
"linux",
|
|
|
|
|
["third-party//jemalloc:headers"],
|
|
|
|
|
)] if sanitizer == "" else [])
|
|
|
|
|
|
|
|
|
|
cpp_library(
|
|
|
|
|
name = "rocksdb_lib",
|
|
|
|
|
srcs = [
|
|
|
|
|
"cache/clock_cache.cc",
|
|
|
|
|
"cache/lru_cache.cc",
|
|
|
|
|
"cache/sharded_cache.cc",
|
|
|
|
|
"db/builder.cc",
|
|
|
|
|
"db/c.cc",
|
|
|
|
|
"db/column_family.cc",
|
|
|
|
|
"db/compacted_db_impl.cc",
|
|
|
|
|
"db/compaction/compaction.cc",
|
|
|
|
|
"db/compaction/compaction_iterator.cc",
|
|
|
|
|
"db/compaction/compaction_job.cc",
|
|
|
|
|
"db/compaction/compaction_picker.cc",
|
|
|
|
|
"db/compaction/compaction_picker_fifo.cc",
|
|
|
|
|
"db/compaction/compaction_picker_level.cc",
|
|
|
|
|
"db/compaction/compaction_picker_universal.cc",
|
|
|
|
|
"db/convenience.cc",
|
|
|
|
|
"db/db_filesnapshot.cc",
|
|
|
|
|
"db/db_impl/db_impl.cc",
|
|
|
|
|
"db/db_impl/db_impl_compaction_flush.cc",
|
|
|
|
|
"db/db_impl/db_impl_debug.cc",
|
|
|
|
|
"db/db_impl/db_impl_experimental.cc",
|
|
|
|
|
"db/db_impl/db_impl_files.cc",
|
|
|
|
|
"db/db_impl/db_impl_open.cc",
|
|
|
|
|
"db/db_impl/db_impl_readonly.cc",
|
|
|
|
|
"db/db_impl/db_impl_secondary.cc",
|
|
|
|
|
"db/db_impl/db_impl_write.cc",
|
|
|
|
|
"db/db_info_dumper.cc",
|
|
|
|
|
"db/db_iter.cc",
|
|
|
|
|
"db/dbformat.cc",
|
|
|
|
|
"db/error_handler.cc",
|
|
|
|
|
"db/event_helpers.cc",
|
|
|
|
|
"db/experimental.cc",
|
|
|
|
|
"db/external_sst_file_ingestion_job.cc",
|
|
|
|
|
"db/file_indexer.cc",
|
|
|
|
|
"db/flush_job.cc",
|
|
|
|
|
"db/flush_scheduler.cc",
|
|
|
|
|
"db/forward_iterator.cc",
|
Export Import sst files (#5495)
Summary:
Refresh of the earlier change here - https://github.com/facebook/rocksdb/issues/5135
This is a review request for code change needed for - https://github.com/facebook/rocksdb/issues/3469
"Add support for taking snapshot of a column family and creating column family from a given CF snapshot"
We have an implementation for this that we have been testing internally. We have two new APIs that together provide this functionality.
(1) ExportColumnFamily() - This API is modelled after CreateCheckpoint() as below.
// Exports all live SST files of a specified Column Family onto export_dir,
// returning SST files information in metadata.
// - SST files will be created as hard links when the directory specified
// is in the same partition as the db directory, copied otherwise.
// - export_dir should not already exist and will be created by this API.
// - Always triggers a flush.
virtual Status ExportColumnFamily(ColumnFamilyHandle* handle,
const std::string& export_dir,
ExportImportFilesMetaData** metadata);
Internally, the API will DisableFileDeletions(), GetColumnFamilyMetaData(), Parse through
metadata, creating links/copies of all the sst files, EnableFileDeletions() and complete the call by
returning the list of file metadata.
(2) CreateColumnFamilyWithImport() - This API is modeled after IngestExternalFile(), but invoked only during a CF creation as below.
// CreateColumnFamilyWithImport() will create a new column family with
// column_family_name and import external SST files specified in metadata into
// this column family.
// (1) External SST files can be created using SstFileWriter.
// (2) External SST files can be exported from a particular column family in
// an existing DB.
// Option in import_options specifies whether the external files are copied or
// moved (default is copy). When option specifies copy, managing files at
// external_file_path is caller's responsibility. When option specifies a
// move, the call ensures that the specified files at external_file_path are
// deleted on successful return and files are not modified on any error
// return.
// On error return, column family handle returned will be nullptr.
// ColumnFamily will be present on successful return and will not be present
// on error return. ColumnFamily may be present on any crash during this call.
virtual Status CreateColumnFamilyWithImport(
const ColumnFamilyOptions& options, const std::string& column_family_name,
const ImportColumnFamilyOptions& import_options,
const ExportImportFilesMetaData& metadata,
ColumnFamilyHandle** handle);
Internally, this API creates a new CF, parses all the sst files and adds it to the specified column family, at the same level and with same sequence number as in the metadata. Also performs safety checks with respect to overlaps between the sst files being imported.
If incoming sequence number is higher than current local sequence number, local sequence
number is updated to reflect this.
Note, as the sst files is are being moved across Column Families, Column Family name in sst file
will no longer match the actual column family on destination DB. The API does not modify Column
Family name or id in the sst files being imported.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5495
Differential Revision: D16018881
fbshipit-source-id: 9ae2251025d5916d35a9fc4ea4d6707f6be16ff9
5 years ago
|
|
|
"db/import_column_family_job.cc",
|
|
|
|
|
"db/internal_stats.cc",
|
|
|
|
|
"db/log_reader.cc",
|
|
|
|
|
"db/log_writer.cc",
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
7 years ago
|
|
|
"db/logs_with_prep_tracker.cc",
|
|
|
|
|
"db/malloc_stats.cc",
|
|
|
|
|
"db/memtable.cc",
|
|
|
|
|
"db/memtable_list.cc",
|
|
|
|
|
"db/merge_helper.cc",
|
|
|
|
|
"db/merge_operator.cc",
|
|
|
|
|
"db/range_del_aggregator.cc",
|
Use only "local" range tombstones during Get (#4449)
Summary:
Previously, range tombstones were accumulated from every level, which
was necessary if a range tombstone in a higher level covered a key in a lower
level. However, RangeDelAggregator::AddTombstones's complexity is based on
the number of tombstones that are currently stored in it, which is wasteful in
the Get case, where we only need to know the highest sequence number of range
tombstones that cover the key from higher levels, and compute the highest covering
sequence number at the current level. This change introduces this optimization, and
removes the use of RangeDelAggregator from the Get path.
In the benchmark results, the following command was used to initialize the database:
```
./db_bench -db=/dev/shm/5k-rts -use_existing_db=false -benchmarks=filluniquerandom -write_buffer_size=1048576 -compression_type=lz4 -target_file_size_base=1048576 -max_bytes_for_level_base=4194304 -value_size=112 -key_size=16 -block_size=4096 -level_compaction_dynamic_level_bytes=true -num=5000000 -max_background_jobs=12 -benchmark_write_rate_limit=20971520 -range_tombstone_width=100 -writes_per_range_tombstone=100 -max_num_range_tombstones=50000 -bloom_bits=8
```
...and the following command was used to measure read throughput:
```
./db_bench -db=/dev/shm/5k-rts/ -use_existing_db=true -benchmarks=readrandom -disable_auto_compactions=true -num=5000000 -reads=100000 -threads=32
```
The filluniquerandom command was only run once, and the resulting database was used
to measure read performance before and after the PR. Both binaries were compiled with
`DEBUG_LEVEL=0`.
Readrandom results before PR:
```
readrandom : 4.544 micros/op 220090 ops/sec; 16.9 MB/s (63103 of 100000 found)
```
Readrandom results after PR:
```
readrandom : 11.147 micros/op 89707 ops/sec; 6.9 MB/s (63103 of 100000 found)
```
So it's actually slower right now, but this PR paves the way for future optimizations (see #4493).
----
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4449
Differential Revision: D10370575
Pulled By: abhimadan
fbshipit-source-id: 9a2e152be1ef36969055c0e9eb4beb0d96c11f4d
6 years ago
|
|
|
"db/range_tombstone_fragmenter.cc",
|
|
|
|
|
"db/repair.cc",
|
|
|
|
|
"db/snapshot_impl.cc",
|
|
|
|
|
"db/table_cache.cc",
|
|
|
|
|
"db/table_properties_collector.cc",
|
|
|
|
|
"db/transaction_log_impl.cc",
|
|
|
|
|
"db/version_builder.cc",
|
|
|
|
|
"db/version_edit.cc",
|
|
|
|
|
"db/version_set.cc",
|
|
|
|
|
"db/wal_manager.cc",
|
|
|
|
|
"db/write_batch.cc",
|
|
|
|
|
"db/write_batch_base.cc",
|
|
|
|
|
"db/write_controller.cc",
|
|
|
|
|
"db/write_thread.cc",
|
|
|
|
|
"env/env.cc",
|
|
|
|
|
"env/env_chroot.cc",
|
|
|
|
|
"env/env_encryption.cc",
|
|
|
|
|
"env/env_hdfs.cc",
|
|
|
|
|
"env/env_posix.cc",
|
|
|
|
|
"env/io_posix.cc",
|
|
|
|
|
"env/mock_env.cc",
|
|
|
|
|
"file/delete_scheduler.cc",
|
|
|
|
|
"file/file_util.cc",
|
|
|
|
|
"file/filename.cc",
|
|
|
|
|
"file/sst_file_manager_impl.cc",
|
|
|
|
|
"logging/auto_roll_logger.cc",
|
|
|
|
|
"logging/event_logger.cc",
|
|
|
|
|
"logging/log_buffer.cc",
|
|
|
|
|
"memory/arena.cc",
|
|
|
|
|
"memory/concurrent_arena.cc",
|
|
|
|
|
"memory/jemalloc_nodump_allocator.cc",
|
|
|
|
|
"memtable/alloc_tracker.cc",
|
|
|
|
|
"memtable/hash_linklist_rep.cc",
|
|
|
|
|
"memtable/hash_skiplist_rep.cc",
|
|
|
|
|
"memtable/skiplistrep.cc",
|
|
|
|
|
"memtable/vectorrep.cc",
|
|
|
|
|
"memtable/write_buffer_manager.cc",
|
|
|
|
|
"monitoring/histogram.cc",
|
|
|
|
|
"monitoring/histogram_windowing.cc",
|
|
|
|
|
"monitoring/in_memory_stats_history.cc",
|
|
|
|
|
"monitoring/instrumented_mutex.cc",
|
|
|
|
|
"monitoring/iostats_context.cc",
|
|
|
|
|
"monitoring/perf_context.cc",
|
|
|
|
|
"monitoring/perf_level.cc",
|
|
|
|
|
"monitoring/persistent_stats_history.cc",
|
|
|
|
|
"monitoring/statistics.cc",
|
|
|
|
|
"monitoring/thread_status_impl.cc",
|
|
|
|
|
"monitoring/thread_status_updater.cc",
|
|
|
|
|
"monitoring/thread_status_updater_debug.cc",
|
|
|
|
|
"monitoring/thread_status_util.cc",
|
|
|
|
|
"monitoring/thread_status_util_debug.cc",
|
|
|
|
|
"options/cf_options.cc",
|
|
|
|
|
"options/db_options.cc",
|
|
|
|
|
"options/options.cc",
|
|
|
|
|
"options/options_helper.cc",
|
|
|
|
|
"options/options_parser.cc",
|
|
|
|
|
"options/options_sanity_check.cc",
|
|
|
|
|
"port/port_posix.cc",
|
|
|
|
|
"port/stack_trace.cc",
|
|
|
|
|
"table/adaptive/adaptive_table_factory.cc",
|
|
|
|
|
"table/block_based/block.cc",
|
|
|
|
|
"table/block_based/block_based_filter_block.cc",
|
|
|
|
|
"table/block_based/block_based_table_builder.cc",
|
|
|
|
|
"table/block_based/block_based_table_factory.cc",
|
|
|
|
|
"table/block_based/block_based_table_reader.cc",
|
|
|
|
|
"table/block_based/block_builder.cc",
|
|
|
|
|
"table/block_based/block_prefix_index.cc",
|
|
|
|
|
"table/block_based/data_block_footer.cc",
|
|
|
|
|
"table/block_based/data_block_hash_index.cc",
|
Move the filter readers out of the block cache (#5504)
Summary:
Currently, when the block cache is used for the filter block, it is not
really the block itself that is stored in the cache but a FilterBlockReader
object. Since this object is not pure data (it has, for instance, pointers that
might dangle, including in one case a back pointer to the TableReader), it's not
really sharable. To avoid the issues around this, the current code erases the
cache entries when the TableReader is closed (which, BTW, is not sufficient
since a concurrent TableReader might have picked up the object in the meantime).
Instead of doing this, the patch moves the FilterBlockReader out of the cache
altogether, and decouples the filter reader object from the filter block.
In particular, instead of the TableReader owning, or caching/pinning the
FilterBlockReader (based on the customer's settings), with the change the
TableReader unconditionally owns the FilterBlockReader, which in turn
owns/caches/pins the filter block. This change also enables us to reuse the code
paths historically used for data blocks for filters as well.
Note:
Eviction statistics for filter blocks are temporarily broken. We plan to fix this in a
separate phase.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5504
Test Plan: make asan_check
Differential Revision: D16036974
Pulled By: ltamasi
fbshipit-source-id: 770f543c5fb4ed126fd1e04bfd3809cf4ff9c091
5 years ago
|
|
|
"table/block_based/filter_block_reader_common.cc",
|
|
|
|
|
"table/block_based/flush_block_policy.cc",
|
|
|
|
|
"table/block_based/full_filter_block.cc",
|
|
|
|
|
"table/block_based/index_builder.cc",
|
|
|
|
|
"table/block_based/partitioned_filter_block.cc",
|
|
|
|
|
"table/block_based/uncompression_dict_reader.cc",
|
|
|
|
|
"table/block_fetcher.cc",
|
|
|
|
|
"table/bloom_block.cc",
|
|
|
|
|
"table/cuckoo/cuckoo_table_builder.cc",
|
|
|
|
|
"table/cuckoo/cuckoo_table_factory.cc",
|
|
|
|
|
"table/cuckoo/cuckoo_table_reader.cc",
|
|
|
|
|
"table/format.cc",
|
|
|
|
|
"table/get_context.cc",
|
|
|
|
|
"table/iterator.cc",
|
|
|
|
|
"table/merging_iterator.cc",
|
|
|
|
|
"table/meta_blocks.cc",
|
|
|
|
|
"table/persistent_cache_helper.cc",
|
|
|
|
|
"table/plain/plain_table_builder.cc",
|
|
|
|
|
"table/plain/plain_table_factory.cc",
|
|
|
|
|
"table/plain/plain_table_index.cc",
|
|
|
|
|
"table/plain/plain_table_key_coding.cc",
|
|
|
|
|
"table/plain/plain_table_reader.cc",
|
|
|
|
|
"table/sst_file_reader.cc",
|
|
|
|
|
"table/sst_file_writer.cc",
|
|
|
|
|
"table/table_properties.cc",
|
|
|
|
|
"table/two_level_iterator.cc",
|
|
|
|
|
"test_util/sync_point.cc",
|
|
|
|
|
"test_util/sync_point_impl.cc",
|
|
|
|
|
"test_util/transaction_test_util.cc",
|
|
|
|
|
"tools/dump/db_dump_tool.cc",
|
|
|
|
|
"tools/ldb_cmd.cc",
|
|
|
|
|
"tools/ldb_tool.cc",
|
|
|
|
|
"tools/sst_dump_tool.cc",
|
|
|
|
|
"trace_replay/block_cache_tracer.cc",
|
|
|
|
|
"trace_replay/trace_replay.cc",
|
|
|
|
|
"util/bloom.cc",
|
|
|
|
|
"util/build_version.cc",
|
|
|
|
|
"util/coding.cc",
|
|
|
|
|
"util/compaction_job_stats_impl.cc",
|
|
|
|
|
"util/comparator.cc",
|
|
|
|
|
"util/compression_context_cache.cc",
|
Concurrent task limiter for compaction thread control (#4332)
Summary:
The PR is targeting to resolve the issue of:
https://github.com/facebook/rocksdb/issues/3972#issue-330771918
We have a rocksdb created with leveled-compaction with multiple column families (CFs), some of CFs are using HDD to store big and less frequently accessed data and others are using SSD.
When there are continuously write traffics going on to all CFs, the compaction thread pool is mostly occupied by those slow HDD compactions, which blocks fully utilize SSD bandwidth.
Since atomic write and transaction is needed across CFs, so splitting it to multiple rocksdb instance is not an option for us.
With the compaction thread control, we got 30%+ HDD write throughput gain, and also a lot smooth SSD write since less write stall happening.
ConcurrentTaskLimiter can be shared with multi-CFs across rocksdb instances, so the feature does not only work for multi-CFs scenarios, but also for multi-rocksdbs scenarios, who need disk IO resource control per tenant.
The usage is straight forward:
e.g.:
//
// Enable compaction thread limiter thru ColumnFamilyOptions
//
std::shared_ptr<ConcurrentTaskLimiter> ctl(NewConcurrentTaskLimiter("foo_limiter", 4));
Options options;
ColumnFamilyOptions cf_opt(options);
cf_opt.compaction_thread_limiter = ctl;
...
//
// Compaction thread limiter can be tuned or disabled on-the-fly
//
ctl->SetMaxOutstandingTask(12); // enlarge to 12 tasks
...
ctl->ResetMaxOutstandingTask(); // disable (bypass) thread limiter
ctl->SetMaxOutstandingTask(-1); // Same as above
...
ctl->SetMaxOutstandingTask(0); // full throttle (0 task)
//
// Sharing compaction thread limiter among CFs (to resolve multiple storage perf issue)
//
std::shared_ptr<ConcurrentTaskLimiter> ctl_ssd(NewConcurrentTaskLimiter("ssd_limiter", 8));
std::shared_ptr<ConcurrentTaskLimiter> ctl_hdd(NewConcurrentTaskLimiter("hdd_limiter", 4));
Options options;
ColumnFamilyOptions cf_opt_ssd1(options);
ColumnFamilyOptions cf_opt_ssd2(options);
ColumnFamilyOptions cf_opt_hdd1(options);
ColumnFamilyOptions cf_opt_hdd2(options);
ColumnFamilyOptions cf_opt_hdd3(options);
// SSD CFs
cf_opt_ssd1.compaction_thread_limiter = ctl_ssd;
cf_opt_ssd2.compaction_thread_limiter = ctl_ssd;
// HDD CFs
cf_opt_hdd1.compaction_thread_limiter = ctl_hdd;
cf_opt_hdd2.compaction_thread_limiter = ctl_hdd;
cf_opt_hdd3.compaction_thread_limiter = ctl_hdd;
...
//
// The limiter is disabled by default (or set to nullptr explicitly)
//
Options options;
ColumnFamilyOptions cf_opt(options);
cf_opt.compaction_thread_limiter = nullptr;
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4332
Differential Revision: D13226590
Pulled By: siying
fbshipit-source-id: 14307aec55b8bd59c8223d04aa6db3c03d1b0c1d
6 years ago
|
|
|
"util/concurrent_task_limiter_impl.cc",
|
|
|
|
|
"util/crc32c.cc",
|
|
|
|
|
"util/dynamic_bloom.cc",
|
|
|
|
|
"util/file_reader_writer.cc",
|
|
|
|
|
"util/filter_policy.cc",
|
|
|
|
|
"util/hash.cc",
|
|
|
|
|
"util/murmurhash.cc",
|
|
|
|
|
"util/random.cc",
|
|
|
|
|
"util/rate_limiter.cc",
|
|
|
|
|
"util/slice.cc",
|
|
|
|
|
"util/status.cc",
|
|
|
|
|
"util/string_util.cc",
|
|
|
|
|
"util/thread_local.cc",
|
|
|
|
|
"util/threadpool_imp.cc",
|
|
|
|
|
"util/xxhash.cc",
|
|
|
|
|
"utilities/backupable/backupable_db.cc",
|
|
|
|
|
"utilities/blob_db/blob_compaction_filter.cc",
|
|
|
|
|
"utilities/blob_db/blob_db.cc",
|
|
|
|
|
"utilities/blob_db/blob_db_impl.cc",
|
|
|
|
|
"utilities/blob_db/blob_db_impl_filesnapshot.cc",
|
|
|
|
|
"utilities/blob_db/blob_dump_tool.cc",
|
|
|
|
|
"utilities/blob_db/blob_file.cc",
|
|
|
|
|
"utilities/blob_db/blob_log_format.cc",
|
|
|
|
|
"utilities/blob_db/blob_log_reader.cc",
|
|
|
|
|
"utilities/blob_db/blob_log_writer.cc",
|
|
|
|
|
"utilities/cassandra/cassandra_compaction_filter.cc",
|
|
|
|
|
"utilities/cassandra/format.cc",
|
|
|
|
|
"utilities/cassandra/merge_operator.cc",
|
|
|
|
|
"utilities/checkpoint/checkpoint_impl.cc",
|
|
|
|
|
"utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc",
|
|
|
|
|
"utilities/convenience/info_log_finder.cc",
|
|
|
|
|
"utilities/debug.cc",
|
|
|
|
|
"utilities/env_mirror.cc",
|
|
|
|
|
"utilities/env_timed.cc",
|
|
|
|
|
"utilities/leveldb_options/leveldb_options.cc",
|
|
|
|
|
"utilities/memory/memory_util.cc",
|
|
|
|
|
"utilities/merge_operators/bytesxor.cc",
|
|
|
|
|
"utilities/merge_operators/max.cc",
|
|
|
|
|
"utilities/merge_operators/put.cc",
|
|
|
|
|
"utilities/merge_operators/string_append/stringappend.cc",
|
|
|
|
|
"utilities/merge_operators/string_append/stringappend2.cc",
|
|
|
|
|
"utilities/merge_operators/uint64add.cc",
|
|
|
|
|
"utilities/object_registry.cc",
|
|
|
|
|
"utilities/option_change_migration/option_change_migration.cc",
|
|
|
|
|
"utilities/options/options_util.cc",
|
|
|
|
|
"utilities/persistent_cache/block_cache_tier.cc",
|
|
|
|
|
"utilities/persistent_cache/block_cache_tier_file.cc",
|
|
|
|
|
"utilities/persistent_cache/block_cache_tier_metadata.cc",
|
|
|
|
|
"utilities/persistent_cache/persistent_cache_tier.cc",
|
|
|
|
|
"utilities/persistent_cache/volatile_tier_impl.cc",
|
|
|
|
|
"utilities/simulator_cache/cache_simulator.cc",
|
|
|
|
|
"utilities/simulator_cache/sim_cache.cc",
|
|
|
|
|
"utilities/table_properties_collectors/compact_on_deletion_collector.cc",
|
|
|
|
|
"utilities/trace/file_trace_reader_writer.cc",
|
|
|
|
|
"utilities/transactions/optimistic_transaction.cc",
|
|
|
|
|
"utilities/transactions/optimistic_transaction_db_impl.cc",
|
|
|
|
|
"utilities/transactions/pessimistic_transaction.cc",
|
|
|
|
|
"utilities/transactions/pessimistic_transaction_db.cc",
|
|
|
|
|
"utilities/transactions/snapshot_checker.cc",
|
|
|
|
|
"utilities/transactions/transaction_base.cc",
|
|
|
|
|
"utilities/transactions/transaction_db_mutex_impl.cc",
|
|
|
|
|
"utilities/transactions/transaction_lock_mgr.cc",
|
|
|
|
|
"utilities/transactions/transaction_util.cc",
|
|
|
|
|
"utilities/transactions/write_prepared_txn.cc",
|
|
|
|
|
"utilities/transactions/write_prepared_txn_db.cc",
|
|
|
|
|
"utilities/transactions/write_unprepared_txn.cc",
|
|
|
|
|
"utilities/transactions/write_unprepared_txn_db.cc",
|
|
|
|
|
"utilities/ttl/db_ttl_impl.cc",
|
|
|
|
|
"utilities/write_batch_with_index/write_batch_with_index.cc",
|
|
|
|
|
"utilities/write_batch_with_index/write_batch_with_index_internal.cc",
|
|
|
|
|
],
|
|
|
|
|
auto_headers = AutoHeaders.RECURSIVE_GLOB,
|
|
|
|
|
arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
|
|
|
|
|
compiler_flags = ROCKSDB_COMPILER_FLAGS,
|
|
|
|
|
os_deps = ROCKSDB_OS_DEPS,
|
|
|
|
|
os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
|
|
|
|
|
preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
|
|
|
|
|
deps = [],
|
|
|
|
|
external_deps = ROCKSDB_EXTERNAL_DEPS,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
cpp_library(
|
|
|
|
|
name = "rocksdb_test_lib",
|
|
|
|
|
srcs = [
|
|
|
|
|
"db/db_test_util.cc",
|
|
|
|
|
"table/mock_table.cc",
|
|
|
|
|
"test_util/fault_injection_test_env.cc",
|
|
|
|
|
"test_util/testharness.cc",
|
|
|
|
|
"test_util/testutil.cc",
|
Block cache simulator: Add pysim to simulate caches using reinforcement learning. (#5610)
Summary:
This PR implements cache eviction using reinforcement learning. It includes two implementations:
1. An implementation of Thompson Sampling for the Bernoulli Bandit [1].
2. An implementation of LinUCB with disjoint linear models [2].
The idea is that a cache uses multiple eviction policies, e.g., MRU, LRU, and LFU. The cache learns which eviction policy is the best and uses it upon a cache miss.
Thompson Sampling is contextless and does not include any features.
LinUCB includes features such as level, block type, caller, column family id to decide which eviction policy to use.
[1] Daniel J. Russo, Benjamin Van Roy, Abbas Kazerouni, Ian Osband, and Zheng Wen. 2018. A Tutorial on Thompson Sampling. Found. Trends Mach. Learn. 11, 1 (July 2018), 1-96. DOI: https://doi.org/10.1561/2200000070
[2] Lihong Li, Wei Chu, John Langford, and Robert E. Schapire. 2010. A contextual-bandit approach to personalized news article recommendation. In Proceedings of the 19th international conference on World wide web (WWW '10). ACM, New York, NY, USA, 661-670. DOI=http://dx.doi.org/10.1145/1772690.1772758
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5610
Differential Revision: D16435067
Pulled By: HaoyuHuang
fbshipit-source-id: 6549239ae14115c01cb1e70548af9e46d8dc21bb
5 years ago
|
|
|
"tools/block_cache_analyzer/block_cache_trace_analyzer.cc",
|
|
|
|
|
"tools/trace_analyzer_tool.cc",
|
|
|
|
|
"utilities/cassandra/test_utils.cc",
|
|
|
|
|
],
|
|
|
|
|
auto_headers = AutoHeaders.RECURSIVE_GLOB,
|
|
|
|
|
arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
|
|
|
|
|
compiler_flags = ROCKSDB_COMPILER_FLAGS,
|
|
|
|
|
os_deps = ROCKSDB_OS_DEPS,
|
|
|
|
|
os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
|
|
|
|
|
preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
|
|
|
|
|
deps = [":rocksdb_lib"],
|
|
|
|
|
external_deps = ROCKSDB_EXTERNAL_DEPS,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
cpp_library(
|
|
|
|
|
name = "rocksdb_tools_lib",
|
|
|
|
|
srcs = [
|
|
|
|
|
"test_util/testutil.cc",
|
Block cache simulator: Add pysim to simulate caches using reinforcement learning. (#5610)
Summary:
This PR implements cache eviction using reinforcement learning. It includes two implementations:
1. An implementation of Thompson Sampling for the Bernoulli Bandit [1].
2. An implementation of LinUCB with disjoint linear models [2].
The idea is that a cache uses multiple eviction policies, e.g., MRU, LRU, and LFU. The cache learns which eviction policy is the best and uses it upon a cache miss.
Thompson Sampling is contextless and does not include any features.
LinUCB includes features such as level, block type, caller, column family id to decide which eviction policy to use.
[1] Daniel J. Russo, Benjamin Van Roy, Abbas Kazerouni, Ian Osband, and Zheng Wen. 2018. A Tutorial on Thompson Sampling. Found. Trends Mach. Learn. 11, 1 (July 2018), 1-96. DOI: https://doi.org/10.1561/2200000070
[2] Lihong Li, Wei Chu, John Langford, and Robert E. Schapire. 2010. A contextual-bandit approach to personalized news article recommendation. In Proceedings of the 19th international conference on World wide web (WWW '10). ACM, New York, NY, USA, 661-670. DOI=http://dx.doi.org/10.1145/1772690.1772758
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5610
Differential Revision: D16435067
Pulled By: HaoyuHuang
fbshipit-source-id: 6549239ae14115c01cb1e70548af9e46d8dc21bb
5 years ago
|
|
|
"tools/block_cache_analyzer/block_cache_trace_analyzer.cc",
|
|
|
|
|
"tools/db_bench_tool.cc",
|
|
|
|
|
"tools/trace_analyzer_tool.cc",
|
|
|
|
|
],
|
|
|
|
|
auto_headers = AutoHeaders.RECURSIVE_GLOB,
|
|
|
|
|
arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
|
|
|
|
|
compiler_flags = ROCKSDB_COMPILER_FLAGS,
|
|
|
|
|
os_deps = ROCKSDB_OS_DEPS,
|
|
|
|
|
os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
|
|
|
|
|
preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
|
|
|
|
|
deps = [":rocksdb_lib"],
|
|
|
|
|
external_deps = ROCKSDB_EXTERNAL_DEPS,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
cpp_library(
|
|
|
|
|
name = "env_basic_test_lib",
|
|
|
|
|
srcs = ["env/env_basic_test.cc"],
|
|
|
|
|
auto_headers = AutoHeaders.RECURSIVE_GLOB,
|
|
|
|
|
arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
|
|
|
|
|
compiler_flags = ROCKSDB_COMPILER_FLAGS,
|
|
|
|
|
os_deps = ROCKSDB_OS_DEPS,
|
|
|
|
|
os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
|
|
|
|
|
preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
|
|
|
|
|
deps = [":rocksdb_test_lib"],
|
|
|
|
|
external_deps = ROCKSDB_EXTERNAL_DEPS,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
# [test_name, test_src, test_type, extra_deps, extra_compiler_flags]
|
|
|
|
|
ROCKS_TESTS = [
|
|
|
|
|
[
|
|
|
|
|
"arena_test",
|
|
|
|
|
"memory/arena_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"auto_roll_logger_test",
|
|
|
|
|
"logging/auto_roll_logger_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"autovector_test",
|
|
|
|
|
"util/autovector_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"backupable_db_test",
|
|
|
|
|
"utilities/backupable/backupable_db_test.cc",
|
|
|
|
|
"parallel",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"blob_db_test",
|
|
|
|
|
"utilities/blob_db/blob_db_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"block_based_filter_block_test",
|
|
|
|
|
"table/block_based/block_based_filter_block_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"block_cache_trace_analyzer_test",
|
Block cache simulator: Add pysim to simulate caches using reinforcement learning. (#5610)
Summary:
This PR implements cache eviction using reinforcement learning. It includes two implementations:
1. An implementation of Thompson Sampling for the Bernoulli Bandit [1].
2. An implementation of LinUCB with disjoint linear models [2].
The idea is that a cache uses multiple eviction policies, e.g., MRU, LRU, and LFU. The cache learns which eviction policy is the best and uses it upon a cache miss.
Thompson Sampling is contextless and does not include any features.
LinUCB includes features such as level, block type, caller, column family id to decide which eviction policy to use.
[1] Daniel J. Russo, Benjamin Van Roy, Abbas Kazerouni, Ian Osband, and Zheng Wen. 2018. A Tutorial on Thompson Sampling. Found. Trends Mach. Learn. 11, 1 (July 2018), 1-96. DOI: https://doi.org/10.1561/2200000070
[2] Lihong Li, Wei Chu, John Langford, and Robert E. Schapire. 2010. A contextual-bandit approach to personalized news article recommendation. In Proceedings of the 19th international conference on World wide web (WWW '10). ACM, New York, NY, USA, 661-670. DOI=http://dx.doi.org/10.1145/1772690.1772758
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5610
Differential Revision: D16435067
Pulled By: HaoyuHuang
fbshipit-source-id: 6549239ae14115c01cb1e70548af9e46d8dc21bb
5 years ago
|
|
|
"tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"block_cache_tracer_test",
|
|
|
|
|
"trace_replay/block_cache_tracer_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"block_test",
|
|
|
|
|
"table/block_based/block_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"bloom_test",
|
|
|
|
|
"util/bloom_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"c_test",
|
|
|
|
|
"db/c_test.c",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"cache_simulator_test",
|
|
|
|
|
"utilities/simulator_cache/cache_simulator_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"cache_test",
|
|
|
|
|
"cache/cache_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"cassandra_format_test",
|
|
|
|
|
"utilities/cassandra/cassandra_format_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"cassandra_functional_test",
|
|
|
|
|
"utilities/cassandra/cassandra_functional_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"cassandra_row_merge_test",
|
|
|
|
|
"utilities/cassandra/cassandra_row_merge_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"cassandra_serialize_test",
|
|
|
|
|
"utilities/cassandra/cassandra_serialize_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"checkpoint_test",
|
|
|
|
|
"utilities/checkpoint/checkpoint_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"cleanable_test",
|
|
|
|
|
"table/cleanable_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"coding_test",
|
|
|
|
|
"util/coding_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"column_family_test",
|
|
|
|
|
"db/column_family_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"compact_files_test",
|
|
|
|
|
"db/compact_files_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"compact_on_deletion_collector_test",
|
|
|
|
|
"utilities/table_properties_collectors/compact_on_deletion_collector_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"compaction_iterator_test",
|
|
|
|
|
"db/compaction/compaction_iterator_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"compaction_job_stats_test",
|
|
|
|
|
"db/compaction/compaction_job_stats_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"compaction_job_test",
|
|
|
|
|
"db/compaction/compaction_job_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"compaction_picker_test",
|
|
|
|
|
"db/compaction/compaction_picker_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"comparator_db_test",
|
|
|
|
|
"db/comparator_db_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"corruption_test",
|
|
|
|
|
"db/corruption_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"crc32c_test",
|
|
|
|
|
"util/crc32c_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"cuckoo_table_builder_test",
|
|
|
|
|
"table/cuckoo/cuckoo_table_builder_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"cuckoo_table_db_test",
|
|
|
|
|
"db/cuckoo_table_db_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"cuckoo_table_reader_test",
|
|
|
|
|
"table/cuckoo/cuckoo_table_reader_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"data_block_hash_index_test",
|
|
|
|
|
"table/block_based/data_block_hash_index_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_basic_test",
|
|
|
|
|
"db/db_basic_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_blob_index_test",
|
|
|
|
|
"db/db_blob_index_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_block_cache_test",
|
|
|
|
|
"db/db_block_cache_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_bloom_filter_test",
|
|
|
|
|
"db/db_bloom_filter_test.cc",
|
|
|
|
|
"parallel",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_compaction_filter_test",
|
|
|
|
|
"db/db_compaction_filter_test.cc",
|
|
|
|
|
"parallel",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_compaction_test",
|
|
|
|
|
"db/db_compaction_test.cc",
|
|
|
|
|
"parallel",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_dynamic_level_test",
|
|
|
|
|
"db/db_dynamic_level_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_encryption_test",
|
|
|
|
|
"db/db_encryption_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_flush_test",
|
|
|
|
|
"db/db_flush_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_inplace_update_test",
|
|
|
|
|
"db/db_inplace_update_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_io_failure_test",
|
|
|
|
|
"db/db_io_failure_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_iter_stress_test",
|
|
|
|
|
"db/db_iter_stress_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
Change and clarify the relationship between Valid(), status() and Seek*() for all iterators. Also fix some bugs
Summary:
Before this PR, Iterator/InternalIterator may simultaneously have non-ok status() and Valid() = true. That state means that the last operation failed, but the iterator is nevertheless positioned on some unspecified record. Likely intended uses of that are:
* If some sst files are corrupted, a normal iterator can be used to read the data from files that are not corrupted.
* When using read_tier = kBlockCacheTier, read the data that's in block cache, skipping over the data that is not.
However, this behavior wasn't documented well (and until recently the wiki on github had misleading incorrect information). In the code there's a lot of confusion about the relationship between status() and Valid(), and about whether Seek()/SeekToLast()/etc reset the status or not. There were a number of bugs caused by this confusion, both inside rocksdb and in the code that uses rocksdb (including ours).
This PR changes the convention to:
* If status() is not ok, Valid() always returns false.
* Any seek operation resets status. (Before the PR, it depended on iterator type and on particular error.)
This does sacrifice the two use cases listed above, but siying said it's ok.
Overview of the changes:
* A commit that adds missing status checks in MergingIterator. This fixes a bug that actually affects us, and we need it fixed. `DBIteratorTest.NonBlockingIterationBugRepro` explains the scenario.
* Changes to lots of iterator types to make all of them conform to the new convention. Some bug fixes along the way. By far the biggest changes are in DBIter, which is a big messy piece of code; I tried to make it less big and messy but mostly failed.
* A stress-test for DBIter, to gain some confidence that I didn't break it. It does a few million random operations on the iterator, while occasionally modifying the underlying data (like ForwardIterator does) and occasionally returning non-ok status from internal iterator.
To find the iterator types that needed changes I searched for "public .*Iterator" in the code. Here's an overview of all 27 iterator types:
Iterators that didn't need changes:
* status() is always ok(), or Valid() is always false: MemTableIterator, ModelIter, TestIterator, KVIter (2 classes with this name anonymous namespaces), LoggingForwardVectorIterator, VectorIterator, MockTableIterator, EmptyIterator, EmptyInternalIterator.
* Thin wrappers that always pass through Valid() and status(): ArenaWrappedDBIter, TtlIterator, InternalIteratorFromIterator.
Iterators with changes (see inline comments for details):
* DBIter - an overhaul:
- It used to silently skip corrupted keys (`FindParseableKey()`), which seems dangerous. This PR makes it just stop immediately after encountering a corrupted key, just like it would for other kinds of corruption. Let me know if there was actually some deeper meaning in this behavior and I should put it back.
- It had a few code paths silently discarding subiterator's status. The stress test caught a few.
- The backwards iteration code path was expecting the internal iterator's set of keys to be immutable. It's probably always true in practice at the moment, since ForwardIterator doesn't support backwards iteration, but this PR fixes it anyway. See added DBIteratorTest.ReverseToForwardBug for an example.
- Some parts of backwards iteration code path even did things like `assert(iter_->Valid())` after a seek, which is never a safe assumption.
- It used to not reset status on seek for some types of errors.
- Some simplifications and better comments.
- Some things got more complicated from the added error handling. I'm open to ideas for how to make it nicer.
* MergingIterator - check status after every operation on every subiterator, and in some places assert that valid subiterators have ok status.
* ForwardIterator - changed to the new convention, also slightly simplified.
* ForwardLevelIterator - fixed some bugs and simplified.
* LevelIterator - simplified.
* TwoLevelIterator - changed to the new convention. Also fixed a bug that would make SeekForPrev() sometimes silently ignore errors from first_level_iter_.
* BlockBasedTableIterator - minor changes.
* BlockIter - replaced `SetStatus()` with `Invalidate()` to make sure non-ok BlockIter is always invalid.
* PlainTableIterator - some seeks used to not reset status.
* CuckooTableIterator - tiny code cleanup.
* ManagedIterator - fixed some bugs.
* BaseDeltaIterator - changed to the new convention and fixed a bug.
* BlobDBIterator - seeks used to not reset status.
* KeyConvertingIterator - some small change.
Closes https://github.com/facebook/rocksdb/pull/3810
Differential Revision: D7888019
Pulled By: al13n321
fbshipit-source-id: 4aaf6d3421c545d16722a815b2fa2e7912bc851d
7 years ago
|
|
|
[
|
|
|
|
|
"db_iter_test",
|
|
|
|
|
"db/db_iter_test.cc",
|
Change and clarify the relationship between Valid(), status() and Seek*() for all iterators. Also fix some bugs
Summary:
Before this PR, Iterator/InternalIterator may simultaneously have non-ok status() and Valid() = true. That state means that the last operation failed, but the iterator is nevertheless positioned on some unspecified record. Likely intended uses of that are:
* If some sst files are corrupted, a normal iterator can be used to read the data from files that are not corrupted.
* When using read_tier = kBlockCacheTier, read the data that's in block cache, skipping over the data that is not.
However, this behavior wasn't documented well (and until recently the wiki on github had misleading incorrect information). In the code there's a lot of confusion about the relationship between status() and Valid(), and about whether Seek()/SeekToLast()/etc reset the status or not. There were a number of bugs caused by this confusion, both inside rocksdb and in the code that uses rocksdb (including ours).
This PR changes the convention to:
* If status() is not ok, Valid() always returns false.
* Any seek operation resets status. (Before the PR, it depended on iterator type and on particular error.)
This does sacrifice the two use cases listed above, but siying said it's ok.
Overview of the changes:
* A commit that adds missing status checks in MergingIterator. This fixes a bug that actually affects us, and we need it fixed. `DBIteratorTest.NonBlockingIterationBugRepro` explains the scenario.
* Changes to lots of iterator types to make all of them conform to the new convention. Some bug fixes along the way. By far the biggest changes are in DBIter, which is a big messy piece of code; I tried to make it less big and messy but mostly failed.
* A stress-test for DBIter, to gain some confidence that I didn't break it. It does a few million random operations on the iterator, while occasionally modifying the underlying data (like ForwardIterator does) and occasionally returning non-ok status from internal iterator.
To find the iterator types that needed changes I searched for "public .*Iterator" in the code. Here's an overview of all 27 iterator types:
Iterators that didn't need changes:
* status() is always ok(), or Valid() is always false: MemTableIterator, ModelIter, TestIterator, KVIter (2 classes with this name anonymous namespaces), LoggingForwardVectorIterator, VectorIterator, MockTableIterator, EmptyIterator, EmptyInternalIterator.
* Thin wrappers that always pass through Valid() and status(): ArenaWrappedDBIter, TtlIterator, InternalIteratorFromIterator.
Iterators with changes (see inline comments for details):
* DBIter - an overhaul:
- It used to silently skip corrupted keys (`FindParseableKey()`), which seems dangerous. This PR makes it just stop immediately after encountering a corrupted key, just like it would for other kinds of corruption. Let me know if there was actually some deeper meaning in this behavior and I should put it back.
- It had a few code paths silently discarding subiterator's status. The stress test caught a few.
- The backwards iteration code path was expecting the internal iterator's set of keys to be immutable. It's probably always true in practice at the moment, since ForwardIterator doesn't support backwards iteration, but this PR fixes it anyway. See added DBIteratorTest.ReverseToForwardBug for an example.
- Some parts of backwards iteration code path even did things like `assert(iter_->Valid())` after a seek, which is never a safe assumption.
- It used to not reset status on seek for some types of errors.
- Some simplifications and better comments.
- Some things got more complicated from the added error handling. I'm open to ideas for how to make it nicer.
* MergingIterator - check status after every operation on every subiterator, and in some places assert that valid subiterators have ok status.
* ForwardIterator - changed to the new convention, also slightly simplified.
* ForwardLevelIterator - fixed some bugs and simplified.
* LevelIterator - simplified.
* TwoLevelIterator - changed to the new convention. Also fixed a bug that would make SeekForPrev() sometimes silently ignore errors from first_level_iter_.
* BlockBasedTableIterator - minor changes.
* BlockIter - replaced `SetStatus()` with `Invalidate()` to make sure non-ok BlockIter is always invalid.
* PlainTableIterator - some seeks used to not reset status.
* CuckooTableIterator - tiny code cleanup.
* ManagedIterator - fixed some bugs.
* BaseDeltaIterator - changed to the new convention and fixed a bug.
* BlobDBIterator - seeks used to not reset status.
* KeyConvertingIterator - some small change.
Closes https://github.com/facebook/rocksdb/pull/3810
Differential Revision: D7888019
Pulled By: al13n321
fbshipit-source-id: 4aaf6d3421c545d16722a815b2fa2e7912bc851d
7 years ago
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
Change and clarify the relationship between Valid(), status() and Seek*() for all iterators. Also fix some bugs
Summary:
Before this PR, Iterator/InternalIterator may simultaneously have non-ok status() and Valid() = true. That state means that the last operation failed, but the iterator is nevertheless positioned on some unspecified record. Likely intended uses of that are:
* If some sst files are corrupted, a normal iterator can be used to read the data from files that are not corrupted.
* When using read_tier = kBlockCacheTier, read the data that's in block cache, skipping over the data that is not.
However, this behavior wasn't documented well (and until recently the wiki on github had misleading incorrect information). In the code there's a lot of confusion about the relationship between status() and Valid(), and about whether Seek()/SeekToLast()/etc reset the status or not. There were a number of bugs caused by this confusion, both inside rocksdb and in the code that uses rocksdb (including ours).
This PR changes the convention to:
* If status() is not ok, Valid() always returns false.
* Any seek operation resets status. (Before the PR, it depended on iterator type and on particular error.)
This does sacrifice the two use cases listed above, but siying said it's ok.
Overview of the changes:
* A commit that adds missing status checks in MergingIterator. This fixes a bug that actually affects us, and we need it fixed. `DBIteratorTest.NonBlockingIterationBugRepro` explains the scenario.
* Changes to lots of iterator types to make all of them conform to the new convention. Some bug fixes along the way. By far the biggest changes are in DBIter, which is a big messy piece of code; I tried to make it less big and messy but mostly failed.
* A stress-test for DBIter, to gain some confidence that I didn't break it. It does a few million random operations on the iterator, while occasionally modifying the underlying data (like ForwardIterator does) and occasionally returning non-ok status from internal iterator.
To find the iterator types that needed changes I searched for "public .*Iterator" in the code. Here's an overview of all 27 iterator types:
Iterators that didn't need changes:
* status() is always ok(), or Valid() is always false: MemTableIterator, ModelIter, TestIterator, KVIter (2 classes with this name anonymous namespaces), LoggingForwardVectorIterator, VectorIterator, MockTableIterator, EmptyIterator, EmptyInternalIterator.
* Thin wrappers that always pass through Valid() and status(): ArenaWrappedDBIter, TtlIterator, InternalIteratorFromIterator.
Iterators with changes (see inline comments for details):
* DBIter - an overhaul:
- It used to silently skip corrupted keys (`FindParseableKey()`), which seems dangerous. This PR makes it just stop immediately after encountering a corrupted key, just like it would for other kinds of corruption. Let me know if there was actually some deeper meaning in this behavior and I should put it back.
- It had a few code paths silently discarding subiterator's status. The stress test caught a few.
- The backwards iteration code path was expecting the internal iterator's set of keys to be immutable. It's probably always true in practice at the moment, since ForwardIterator doesn't support backwards iteration, but this PR fixes it anyway. See added DBIteratorTest.ReverseToForwardBug for an example.
- Some parts of backwards iteration code path even did things like `assert(iter_->Valid())` after a seek, which is never a safe assumption.
- It used to not reset status on seek for some types of errors.
- Some simplifications and better comments.
- Some things got more complicated from the added error handling. I'm open to ideas for how to make it nicer.
* MergingIterator - check status after every operation on every subiterator, and in some places assert that valid subiterators have ok status.
* ForwardIterator - changed to the new convention, also slightly simplified.
* ForwardLevelIterator - fixed some bugs and simplified.
* LevelIterator - simplified.
* TwoLevelIterator - changed to the new convention. Also fixed a bug that would make SeekForPrev() sometimes silently ignore errors from first_level_iter_.
* BlockBasedTableIterator - minor changes.
* BlockIter - replaced `SetStatus()` with `Invalidate()` to make sure non-ok BlockIter is always invalid.
* PlainTableIterator - some seeks used to not reset status.
* CuckooTableIterator - tiny code cleanup.
* ManagedIterator - fixed some bugs.
* BaseDeltaIterator - changed to the new convention and fixed a bug.
* BlobDBIterator - seeks used to not reset status.
* KeyConvertingIterator - some small change.
Closes https://github.com/facebook/rocksdb/pull/3810
Differential Revision: D7888019
Pulled By: al13n321
fbshipit-source-id: 4aaf6d3421c545d16722a815b2fa2e7912bc851d
7 years ago
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_iterator_test",
|
|
|
|
|
"db/db_iterator_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_log_iter_test",
|
|
|
|
|
"db/db_log_iter_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_memtable_test",
|
|
|
|
|
"db/db_memtable_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_merge_operator_test",
|
|
|
|
|
"db/db_merge_operator_test.cc",
|
|
|
|
|
"parallel",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_options_test",
|
|
|
|
|
"db/db_options_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_properties_test",
|
|
|
|
|
"db/db_properties_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_range_del_test",
|
|
|
|
|
"db/db_range_del_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_secondary_test",
|
|
|
|
|
"db/db_impl/db_secondary_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_sst_test",
|
|
|
|
|
"db/db_sst_test.cc",
|
|
|
|
|
"parallel",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_statistics_test",
|
|
|
|
|
"db/db_statistics_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_table_properties_test",
|
|
|
|
|
"db/db_table_properties_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_tailing_iter_test",
|
|
|
|
|
"db/db_tailing_iter_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_test",
|
|
|
|
|
"db/db_test.cc",
|
|
|
|
|
"parallel",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_test2",
|
|
|
|
|
"db/db_test2.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_universal_compaction_test",
|
|
|
|
|
"db/db_universal_compaction_test.cc",
|
|
|
|
|
"parallel",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_wal_test",
|
|
|
|
|
"db/db_wal_test.cc",
|
|
|
|
|
"parallel",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"db_write_test",
|
|
|
|
|
"db/db_write_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"dbformat_test",
|
|
|
|
|
"db/dbformat_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"delete_scheduler_test",
|
|
|
|
|
"file/delete_scheduler_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"deletefile_test",
|
|
|
|
|
"db/deletefile_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"dynamic_bloom_test",
|
|
|
|
|
"util/dynamic_bloom_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"env_basic_test",
|
|
|
|
|
"env/env_basic_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"env_logger_test",
|
|
|
|
|
"logging/env_logger_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"env_test",
|
|
|
|
|
"env/env_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"env_timed_test",
|
|
|
|
|
"utilities/env_timed_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"error_handler_test",
|
|
|
|
|
"db/error_handler_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"event_logger_test",
|
|
|
|
|
"logging/event_logger_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"external_sst_file_basic_test",
|
|
|
|
|
"db/external_sst_file_basic_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"external_sst_file_test",
|
|
|
|
|
"db/external_sst_file_test.cc",
|
|
|
|
|
"parallel",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"fault_injection_test",
|
|
|
|
|
"db/fault_injection_test.cc",
|
|
|
|
|
"parallel",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"file_indexer_test",
|
|
|
|
|
"db/file_indexer_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"file_reader_writer_test",
|
|
|
|
|
"util/file_reader_writer_test.cc",
|
|
|
|
|
"parallel",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"filelock_test",
|
|
|
|
|
"util/filelock_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"filename_test",
|
|
|
|
|
"db/filename_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"flush_job_test",
|
|
|
|
|
"db/flush_job_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"full_filter_block_test",
|
|
|
|
|
"table/block_based/full_filter_block_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"hash_table_test",
|
|
|
|
|
"utilities/persistent_cache/hash_table_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"hash_test",
|
|
|
|
|
"util/hash_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"heap_test",
|
|
|
|
|
"util/heap_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"histogram_test",
|
|
|
|
|
"monitoring/histogram_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"import_column_family_test",
|
|
|
|
|
"db/import_column_family_test.cc",
|
|
|
|
|
"parallel",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"inlineskiplist_test",
|
|
|
|
|
"memtable/inlineskiplist_test.cc",
|
|
|
|
|
"parallel",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"iostats_context_test",
|
|
|
|
|
"monitoring/iostats_context_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"ldb_cmd_test",
|
|
|
|
|
"tools/ldb_cmd_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"listener_test",
|
|
|
|
|
"db/listener_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"log_test",
|
|
|
|
|
"db/log_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"lru_cache_test",
|
|
|
|
|
"cache/lru_cache_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"manual_compaction_test",
|
|
|
|
|
"db/manual_compaction_test.cc",
|
|
|
|
|
"parallel",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"memory_test",
|
|
|
|
|
"utilities/memory/memory_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"memtable_list_test",
|
|
|
|
|
"db/memtable_list_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"merge_helper_test",
|
|
|
|
|
"db/merge_helper_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"merge_test",
|
|
|
|
|
"db/merge_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"merger_test",
|
|
|
|
|
"table/merger_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"mock_env_test",
|
|
|
|
|
"env/mock_env_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"object_registry_test",
|
|
|
|
|
"utilities/object_registry_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
7 years ago
|
|
|
[
|
|
|
|
|
"obsolete_files_test",
|
|
|
|
|
"db/obsolete_files_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
Skip deleted WALs during recovery
Summary:
This patch record min log number to keep to the manifest while flushing SST files to ignore them and any WAL older than them during recovery. This is to avoid scenarios when we have a gap between the WAL files are fed to the recovery procedure. The gap could happen by for example out-of-order WAL deletion. Such gap could cause problems in 2PC recovery where the prepared and commit entry are placed into two separate WAL and gap in the WALs could result into not processing the WAL with the commit entry and hence breaking the 2PC recovery logic.
Before the commit, for 2PC case, we determined which log number to keep in FindObsoleteFiles(). We looked at the earliest logs with outstanding prepare entries, or prepare entries whose respective commit or abort are in memtable. With the commit, the same calculation is done while we apply the SST flush. Just before installing the flush file, we precompute the earliest log file to keep after the flush finishes using the same logic (but skipping the memtables just flushed), record this information to the manifest entry for this new flushed SST file. This pre-computed value is also remembered in memory, and will later be used to determine whether a log file can be deleted. This value is unlikely to change until next flush because the commit entry will stay in memtable. (In WritePrepared, we could have removed the older log files as soon as all prepared entries are committed. It's not yet done anyway. Even if we do it, the only thing we loss with this new approach is earlier log deletion between two flushes, which does not guarantee to happen anyway because the obsolete file clean-up function is only executed after flush or compaction)
This min log number to keep is stored in the manifest using the safely-ignore customized field of AddFile entry, in order to guarantee that the DB generated using newer release can be opened by previous releases no older than 4.2.
Closes https://github.com/facebook/rocksdb/pull/3765
Differential Revision: D7747618
Pulled By: siying
fbshipit-source-id: d00c92105b4f83852e9754a1b70d6b64cb590729
7 years ago
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"optimistic_transaction_test",
|
|
|
|
|
"utilities/transactions/optimistic_transaction_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"option_change_migration_test",
|
|
|
|
|
"utilities/option_change_migration/option_change_migration_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"options_file_test",
|
|
|
|
|
"db/options_file_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"options_settable_test",
|
|
|
|
|
"options/options_settable_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"options_test",
|
|
|
|
|
"options/options_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"options_util_test",
|
|
|
|
|
"utilities/options/options_util_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"partitioned_filter_block_test",
|
|
|
|
|
"table/block_based/partitioned_filter_block_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"perf_context_test",
|
|
|
|
|
"db/perf_context_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"persistent_cache_test",
|
|
|
|
|
"utilities/persistent_cache/persistent_cache_test.cc",
|
|
|
|
|
"parallel",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"plain_table_db_test",
|
|
|
|
|
"db/plain_table_db_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"prefix_test",
|
|
|
|
|
"db/prefix_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"range_del_aggregator_test",
|
|
|
|
|
"db/range_del_aggregator_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"range_tombstone_fragmenter_test",
|
|
|
|
|
"db/range_tombstone_fragmenter_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"rate_limiter_test",
|
|
|
|
|
"util/rate_limiter_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"reduce_levels_test",
|
|
|
|
|
"tools/reduce_levels_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"repair_test",
|
|
|
|
|
"db/repair_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"repeatable_thread_test",
|
|
|
|
|
"util/repeatable_thread_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"sim_cache_test",
|
|
|
|
|
"utilities/simulator_cache/sim_cache_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"skiplist_test",
|
|
|
|
|
"memtable/skiplist_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"slice_transform_test",
|
|
|
|
|
"util/slice_transform_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"sst_dump_test",
|
|
|
|
|
"tools/sst_dump_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"sst_file_reader_test",
|
|
|
|
|
"table/sst_file_reader_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"statistics_test",
|
|
|
|
|
"monitoring/statistics_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"stats_history_test",
|
|
|
|
|
"monitoring/stats_history_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"stringappend_test",
|
|
|
|
|
"utilities/merge_operators/string_append/stringappend_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"table_properties_collector_test",
|
|
|
|
|
"db/table_properties_collector_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"table_test",
|
|
|
|
|
"table/table_test.cc",
|
|
|
|
|
"parallel",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"thread_list_test",
|
|
|
|
|
"util/thread_list_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"thread_local_test",
|
|
|
|
|
"util/thread_local_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"timer_queue_test",
|
|
|
|
|
"util/timer_queue_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"trace_analyzer_test",
|
|
|
|
|
"tools/trace_analyzer_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"transaction_test",
|
|
|
|
|
"utilities/transactions/transaction_test.cc",
|
|
|
|
|
"parallel",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"ttl_test",
|
|
|
|
|
"utilities/ttl/ttl_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"util_merge_operators_test",
|
|
|
|
|
"utilities/util_merge_operators_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"version_builder_test",
|
|
|
|
|
"db/version_builder_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"version_edit_test",
|
|
|
|
|
"db/version_edit_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"version_set_test",
|
|
|
|
|
"db/version_set_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"wal_manager_test",
|
|
|
|
|
"db/wal_manager_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"write_batch_test",
|
|
|
|
|
"db/write_batch_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"write_batch_with_index_test",
|
|
|
|
|
"utilities/write_batch_with_index/write_batch_with_index_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"write_buffer_manager_test",
|
|
|
|
|
"memtable/write_buffer_manager_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"write_callback_test",
|
|
|
|
|
"db/write_callback_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"write_controller_test",
|
|
|
|
|
"db/write_controller_test.cc",
|
|
|
|
|
"serial",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"write_prepared_transaction_test",
|
|
|
|
|
"utilities/transactions/write_prepared_transaction_test.cc",
|
|
|
|
|
"parallel",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
[
|
|
|
|
|
"write_unprepared_transaction_test",
|
|
|
|
|
"utilities/transactions/write_unprepared_transaction_test.cc",
|
|
|
|
|
"parallel",
|
|
|
|
|
[],
|
|
|
|
|
[],
|
|
|
|
|
],
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
# Generate a test rule for each entry in ROCKS_TESTS
|
|
|
|
|
# Do not build the tests in opt mode, since SyncPoint and other test code
|
|
|
|
|
# will not be included.
|
|
|
|
|
[
|
|
|
|
|
test_binary(
|
|
|
|
|
extra_compiler_flags = extra_compiler_flags,
|
|
|
|
|
extra_deps = extra_deps,
|
|
|
|
|
parallelism = parallelism,
|
|
|
|
|
rocksdb_arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
|
|
|
|
|
rocksdb_compiler_flags = ROCKSDB_COMPILER_FLAGS,
|
|
|
|
|
rocksdb_external_deps = ROCKSDB_EXTERNAL_DEPS,
|
|
|
|
|
rocksdb_os_deps = ROCKSDB_OS_DEPS,
|
|
|
|
|
rocksdb_os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
|
|
|
|
|
rocksdb_preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
|
|
|
|
|
test_cc = test_cc,
|
|
|
|
|
test_name = test_name,
|
|
|
|
|
)
|
|
|
|
|
for test_name, test_cc, parallelism, extra_deps, extra_compiler_flags in ROCKS_TESTS
|
|
|
|
|
if not is_opt_mode
|
|
|
|
|
]
|
