|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
|
|
#include "options/options_helper.h"
|
|
|
|
|
|
|
|
#include <cassert>
|
|
|
|
#include <cctype>
|
|
|
|
#include <cstdlib>
|
Implement XXH3 block checksum type (#9069)
Summary:
XXH3 - latest hash function that is extremely fast on large
data, easily faster than crc32c on most any x86_64 hardware. In
integrating this hash function, I have handled the compression type byte
in a non-standard way to avoid using the streaming API (extra data
movement and active code size because of hash function complexity). This
approach got a thumbs-up from Yann Collet.
Existing functionality change:
* reject bad ChecksumType in options with InvalidArgument
This change split off from https://github.com/facebook/rocksdb/issues/9058 because context-aware checksum is
likely to be handled through different configuration than ChecksumType.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9069
Test Plan:
tests updated, and substantially expanded. Unit tests now check
that we don't accidentally change the values generated by the checksum
algorithms ("schema test") and that we properly handle
invalid/unrecognized checksum types in options or in file footer.
DBTestBase::ChangeOptions (etc.) updated from two to one configuration
changing from default CRC32c ChecksumType. The point of this test code
is to detect possible interactions among features, and the likelihood of
some bad interaction being detected by including configurations other
than XXH3 and CRC32c--and then not detected by stress/crash test--is
extremely low.
Stress/crash test also updated (manual run long enough to see it accepts
new checksum type). db_bench also updated for microbenchmarking
checksums.
### Performance microbenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
./db_bench -benchmarks=crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3
crc32c : 0.200 micros/op 5005220 ops/sec; 19551.6 MB/s (4096 per op)
xxhash : 0.807 micros/op 1238408 ops/sec; 4837.5 MB/s (4096 per op)
xxhash64 : 0.421 micros/op 2376514 ops/sec; 9283.3 MB/s (4096 per op)
xxh3 : 0.171 micros/op 5858391 ops/sec; 22884.3 MB/s (4096 per op)
crc32c : 0.206 micros/op 4859566 ops/sec; 18982.7 MB/s (4096 per op)
xxhash : 0.793 micros/op 1260850 ops/sec; 4925.2 MB/s (4096 per op)
xxhash64 : 0.410 micros/op 2439182 ops/sec; 9528.1 MB/s (4096 per op)
xxh3 : 0.161 micros/op 6202872 ops/sec; 24230.0 MB/s (4096 per op)
crc32c : 0.203 micros/op 4924686 ops/sec; 19237.1 MB/s (4096 per op)
xxhash : 0.839 micros/op 1192388 ops/sec; 4657.8 MB/s (4096 per op)
xxhash64 : 0.424 micros/op 2357391 ops/sec; 9208.6 MB/s (4096 per op)
xxh3 : 0.162 micros/op 6182678 ops/sec; 24151.1 MB/s (4096 per op)
As you can see, especially once warmed up, xxh3 is fastest.
### Performance macrobenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
Test
for I in `seq 1 50`; do for CHK in 0 1 2 3 4; do TEST_TMPDIR=/dev/shm/rocksdb$CHK ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=30000000 -checksum_type=$CHK 2>&1 | grep 'micros/op' | tee -a results-$CHK & done; wait; done
Results (ops/sec)
for FILE in results*; do echo -n "$FILE "; awk '{ s += $5; c++; } END { print 1.0 * s / c; }' < $FILE; done
results-0 252118 # kNoChecksum
results-1 251588 # kCRC32c
results-2 251863 # kxxHash
results-3 252016 # kxxHash64
results-4 252038 # kXXH3
Reviewed By: mrambacher
Differential Revision: D31905249
Pulled By: pdillinger
fbshipit-source-id: cb9b998ebe2523fc7c400eedf62124a78bf4b4d1
3 years ago
|
|
|
#include <set>
|
|
|
|
#include <unordered_set>
|
|
|
|
#include <vector>
|
|
|
|
|
|
|
|
#include "options/cf_options.h"
|
|
|
|
#include "options/db_options.h"
|
|
|
|
#include "rocksdb/cache.h"
|
|
|
|
#include "rocksdb/compaction_filter.h"
|
|
|
|
#include "rocksdb/convenience.h"
|
|
|
|
#include "rocksdb/filter_policy.h"
|
|
|
|
#include "rocksdb/flush_block_policy.h"
|
|
|
|
#include "rocksdb/memtablerep.h"
|
|
|
|
#include "rocksdb/merge_operator.h"
|
|
|
|
#include "rocksdb/options.h"
|
|
|
|
#include "rocksdb/rate_limiter.h"
|
|
|
|
#include "rocksdb/slice_transform.h"
|
|
|
|
#include "rocksdb/table.h"
|
|
|
|
#include "rocksdb/utilities/object_registry.h"
|
|
|
|
#include "rocksdb/utilities/options_type.h"
|
|
|
|
#include "util/string_util.h"
|
|
|
|
|
|
|
|
namespace ROCKSDB_NAMESPACE {
|
|
|
|
ConfigOptions::ConfigOptions()
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
: registry(ObjectRegistry::NewInstance())
|
|
|
|
#endif
|
|
|
|
{
|
|
|
|
env = Env::Default();
|
|
|
|
}
|
|
|
|
|
|
|
|
ConfigOptions::ConfigOptions(const DBOptions& db_opts) : env(db_opts.env) {
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
registry = ObjectRegistry::NewInstance();
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
|
|
|
Status ValidateOptions(const DBOptions& db_opts,
|
|
|
|
const ColumnFamilyOptions& cf_opts) {
|
|
|
|
Status s;
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
auto db_cfg = DBOptionsAsConfigurable(db_opts);
|
|
|
|
auto cf_cfg = CFOptionsAsConfigurable(cf_opts);
|
|
|
|
s = db_cfg->ValidateOptions(db_opts, cf_opts);
|
|
|
|
if (s.ok()) s = cf_cfg->ValidateOptions(db_opts, cf_opts);
|
|
|
|
#else
|
|
|
|
s = cf_opts.table_factory->ValidateOptions(db_opts, cf_opts);
|
|
|
|
#endif
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
DBOptions BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
|
|
|
|
const MutableDBOptions& mutable_db_options) {
|
|
|
|
DBOptions options;
|
|
|
|
|
|
|
|
options.create_if_missing = immutable_db_options.create_if_missing;
|
|
|
|
options.create_missing_column_families =
|
|
|
|
immutable_db_options.create_missing_column_families;
|
|
|
|
options.error_if_exists = immutable_db_options.error_if_exists;
|
|
|
|
options.paranoid_checks = immutable_db_options.paranoid_checks;
|
|
|
|
options.flush_verify_memtable_count =
|
|
|
|
immutable_db_options.flush_verify_memtable_count;
|
|
|
|
options.track_and_verify_wals_in_manifest =
|
|
|
|
immutable_db_options.track_and_verify_wals_in_manifest;
|
|
|
|
options.env = immutable_db_options.env;
|
|
|
|
options.rate_limiter = immutable_db_options.rate_limiter;
|
|
|
|
options.sst_file_manager = immutable_db_options.sst_file_manager;
|
|
|
|
options.info_log = immutable_db_options.info_log;
|
|
|
|
options.info_log_level = immutable_db_options.info_log_level;
|
|
|
|
options.max_open_files = mutable_db_options.max_open_files;
|
|
|
|
options.max_file_opening_threads =
|
|
|
|
immutable_db_options.max_file_opening_threads;
|
|
|
|
options.max_total_wal_size = mutable_db_options.max_total_wal_size;
|
|
|
|
options.statistics = immutable_db_options.statistics;
|
|
|
|
options.use_fsync = immutable_db_options.use_fsync;
|
|
|
|
options.db_paths = immutable_db_options.db_paths;
|
|
|
|
options.db_log_dir = immutable_db_options.db_log_dir;
|
|
|
|
options.wal_dir = immutable_db_options.wal_dir;
|
|
|
|
options.delete_obsolete_files_period_micros =
|
|
|
|
mutable_db_options.delete_obsolete_files_period_micros;
|
|
|
|
options.max_background_jobs = mutable_db_options.max_background_jobs;
|
|
|
|
options.max_background_compactions =
|
|
|
|
mutable_db_options.max_background_compactions;
|
|
|
|
options.bytes_per_sync = mutable_db_options.bytes_per_sync;
|
|
|
|
options.wal_bytes_per_sync = mutable_db_options.wal_bytes_per_sync;
|
Optionally wait on bytes_per_sync to smooth I/O (#5183)
Summary:
The existing implementation does not guarantee bytes reach disk every `bytes_per_sync` when writing SST files, or every `wal_bytes_per_sync` when writing WALs. This can cause confusing behavior for users who enable this feature to avoid large syncs during flush and compaction, but then end up hitting them anyways.
My understanding of the existing behavior is we used `sync_file_range` with `SYNC_FILE_RANGE_WRITE` to submit ranges for async writeback, such that we could continue processing the next range of bytes while that I/O is happening. I believe we can preserve that benefit while also limiting how far the processing can get ahead of the I/O, which prevents huge syncs from happening when the file finishes.
Consider this `sync_file_range` usage: `sync_file_range(fd_, 0, static_cast<off_t>(offset + nbytes), SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WRITE)`. Expanding the range to start at 0 and adding the `SYNC_FILE_RANGE_WAIT_BEFORE` flag causes any pending writeback (like from a previous call to `sync_file_range`) to finish before it proceeds to submit the latest `nbytes` for writeback. The latest `nbytes` are still written back asynchronously, unless processing exceeds I/O speed, in which case the following `sync_file_range` will need to wait on it.
There is a second change in this PR to use `fdatasync` when `sync_file_range` is unavailable (determined statically) or has some known problem with the underlying filesystem (determined dynamically).
The above two changes only apply when the user enables a new option, `strict_bytes_per_sync`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5183
Differential Revision: D14953553
Pulled By: siying
fbshipit-source-id: 445c3862e019fb7b470f9c7f314fc231b62706e9
6 years ago
|
|
|
options.strict_bytes_per_sync = mutable_db_options.strict_bytes_per_sync;
|
|
|
|
options.max_subcompactions = mutable_db_options.max_subcompactions;
|
|
|
|
options.max_background_flushes = mutable_db_options.max_background_flushes;
|
|
|
|
options.max_log_file_size = immutable_db_options.max_log_file_size;
|
|
|
|
options.log_file_time_to_roll = immutable_db_options.log_file_time_to_roll;
|
|
|
|
options.keep_log_file_num = immutable_db_options.keep_log_file_num;
|
|
|
|
options.recycle_log_file_num = immutable_db_options.recycle_log_file_num;
|
|
|
|
options.max_manifest_file_size = immutable_db_options.max_manifest_file_size;
|
|
|
|
options.table_cache_numshardbits =
|
|
|
|
immutable_db_options.table_cache_numshardbits;
|
|
|
|
options.WAL_ttl_seconds = immutable_db_options.WAL_ttl_seconds;
|
|
|
|
options.WAL_size_limit_MB = immutable_db_options.WAL_size_limit_MB;
|
|
|
|
options.manifest_preallocation_size =
|
|
|
|
immutable_db_options.manifest_preallocation_size;
|
|
|
|
options.allow_mmap_reads = immutable_db_options.allow_mmap_reads;
|
|
|
|
options.allow_mmap_writes = immutable_db_options.allow_mmap_writes;
|
|
|
|
options.use_direct_reads = immutable_db_options.use_direct_reads;
|
|
|
|
options.use_direct_io_for_flush_and_compaction =
|
|
|
|
immutable_db_options.use_direct_io_for_flush_and_compaction;
|
|
|
|
options.allow_fallocate = immutable_db_options.allow_fallocate;
|
|
|
|
options.is_fd_close_on_exec = immutable_db_options.is_fd_close_on_exec;
|
|
|
|
options.stats_dump_period_sec = mutable_db_options.stats_dump_period_sec;
|
|
|
|
options.stats_persist_period_sec =
|
|
|
|
mutable_db_options.stats_persist_period_sec;
|
|
|
|
options.persist_stats_to_disk = immutable_db_options.persist_stats_to_disk;
|
|
|
|
options.stats_history_buffer_size =
|
|
|
|
mutable_db_options.stats_history_buffer_size;
|
|
|
|
options.advise_random_on_open = immutable_db_options.advise_random_on_open;
|
|
|
|
options.db_write_buffer_size = immutable_db_options.db_write_buffer_size;
|
|
|
|
options.write_buffer_manager = immutable_db_options.write_buffer_manager;
|
|
|
|
options.access_hint_on_compaction_start =
|
|
|
|
immutable_db_options.access_hint_on_compaction_start;
|
|
|
|
options.compaction_readahead_size =
|
|
|
|
mutable_db_options.compaction_readahead_size;
|
|
|
|
options.random_access_max_buffer_size =
|
|
|
|
immutable_db_options.random_access_max_buffer_size;
|
|
|
|
options.writable_file_max_buffer_size =
|
|
|
|
mutable_db_options.writable_file_max_buffer_size;
|
|
|
|
options.use_adaptive_mutex = immutable_db_options.use_adaptive_mutex;
|
|
|
|
options.listeners = immutable_db_options.listeners;
|
|
|
|
options.enable_thread_tracking = immutable_db_options.enable_thread_tracking;
|
|
|
|
options.delayed_write_rate = mutable_db_options.delayed_write_rate;
|
|
|
|
options.enable_pipelined_write = immutable_db_options.enable_pipelined_write;
|
|
|
|
options.unordered_write = immutable_db_options.unordered_write;
|
|
|
|
options.allow_concurrent_memtable_write =
|
|
|
|
immutable_db_options.allow_concurrent_memtable_write;
|
|
|
|
options.enable_write_thread_adaptive_yield =
|
|
|
|
immutable_db_options.enable_write_thread_adaptive_yield;
|
|
|
|
options.max_write_batch_group_size_bytes =
|
|
|
|
immutable_db_options.max_write_batch_group_size_bytes;
|
|
|
|
options.write_thread_max_yield_usec =
|
|
|
|
immutable_db_options.write_thread_max_yield_usec;
|
|
|
|
options.write_thread_slow_yield_usec =
|
|
|
|
immutable_db_options.write_thread_slow_yield_usec;
|
|
|
|
options.skip_stats_update_on_db_open =
|
|
|
|
immutable_db_options.skip_stats_update_on_db_open;
|
Add an option to prevent DB::Open() from querying sizes of all sst files (#6353)
Summary:
When paranoid_checks is on, DBImpl::CheckConsistency() iterates over all sst files and calls Env::GetFileSize() for each of them. As far as I could understand, this is pretty arbitrary and doesn't affect correctness - if filesystem doesn't corrupt fsynced files, the file sizes will always match; if it does, it may as well corrupt contents as well as sizes, and rocksdb doesn't check contents on open.
If there are thousands of sst files, getting all their sizes takes a while. If, on top of that, Env is overridden to use some remote storage instead of local filesystem, it can be *really* slow and overload the remote storage service. This PR adds an option to not do GetFileSize(); instead it does GetChildren() for parent directory to check that all the expected sst files are at least present, but doesn't check their sizes.
We can't just disable paranoid_checks instead because paranoid_checks do a few other important things: make the DB read-only on write errors, print error messages on read errors, etc.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6353
Test Plan: ran the added sanity check unit test. Will try it out in a LogDevice test cluster where the GetFileSize() calls are causing a lot of trouble.
Differential Revision: D19656425
Pulled By: al13n321
fbshipit-source-id: c2c421b367633033760d1f56747bad206d1fbf82
5 years ago
|
|
|
options.skip_checking_sst_file_sizes_on_db_open =
|
|
|
|
immutable_db_options.skip_checking_sst_file_sizes_on_db_open;
|
|
|
|
options.wal_recovery_mode = immutable_db_options.wal_recovery_mode;
|
|
|
|
options.allow_2pc = immutable_db_options.allow_2pc;
|
|
|
|
options.row_cache = immutable_db_options.row_cache;
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
options.wal_filter = immutable_db_options.wal_filter;
|
|
|
|
#endif // ROCKSDB_LITE
|
|
|
|
options.fail_if_options_file_error =
|
|
|
|
immutable_db_options.fail_if_options_file_error;
|
|
|
|
options.dump_malloc_stats = immutable_db_options.dump_malloc_stats;
|
|
|
|
options.avoid_flush_during_recovery =
|
|
|
|
immutable_db_options.avoid_flush_during_recovery;
|
|
|
|
options.avoid_flush_during_shutdown =
|
|
|
|
mutable_db_options.avoid_flush_during_shutdown;
|
|
|
|
options.allow_ingest_behind = immutable_db_options.allow_ingest_behind;
|
|
|
|
options.two_write_queues = immutable_db_options.two_write_queues;
|
|
|
|
options.manual_wal_flush = immutable_db_options.manual_wal_flush;
|
|
|
|
options.wal_compression = immutable_db_options.wal_compression;
|
|
|
|
options.atomic_flush = immutable_db_options.atomic_flush;
|
|
|
|
options.avoid_unnecessary_blocking_io =
|
|
|
|
immutable_db_options.avoid_unnecessary_blocking_io;
|
|
|
|
options.log_readahead_size = immutable_db_options.log_readahead_size;
|
|
|
|
options.file_checksum_gen_factory =
|
|
|
|
immutable_db_options.file_checksum_gen_factory;
|
|
|
|
options.best_efforts_recovery = immutable_db_options.best_efforts_recovery;
|
|
|
|
options.max_bgerror_resume_count =
|
|
|
|
immutable_db_options.max_bgerror_resume_count;
|
|
|
|
options.bgerror_resume_retry_interval =
|
|
|
|
immutable_db_options.bgerror_resume_retry_interval;
|
|
|
|
options.db_host_id = immutable_db_options.db_host_id;
|
|
|
|
options.allow_data_in_errors = immutable_db_options.allow_data_in_errors;
|
|
|
|
options.checksum_handoff_file_types =
|
|
|
|
immutable_db_options.checksum_handoff_file_types;
|
|
|
|
options.lowest_used_cache_tier = immutable_db_options.lowest_used_cache_tier;
|
|
|
|
return options;
|
|
|
|
}
|
|
|
|
|
|
|
|
ColumnFamilyOptions BuildColumnFamilyOptions(
|
|
|
|
const ColumnFamilyOptions& options,
|
|
|
|
const MutableCFOptions& mutable_cf_options) {
|
|
|
|
ColumnFamilyOptions cf_opts(options);
|
|
|
|
UpdateColumnFamilyOptions(mutable_cf_options, &cf_opts);
|
|
|
|
// TODO(yhchiang): find some way to handle the following derived options
|
|
|
|
// * max_file_size
|
|
|
|
return cf_opts;
|
|
|
|
}
|
|
|
|
|
|
|
|
void UpdateColumnFamilyOptions(const MutableCFOptions& moptions,
|
|
|
|
ColumnFamilyOptions* cf_opts) {
|
|
|
|
// Memtable related options
|
|
|
|
cf_opts->write_buffer_size = moptions.write_buffer_size;
|
|
|
|
cf_opts->max_write_buffer_number = moptions.max_write_buffer_number;
|
|
|
|
cf_opts->arena_block_size = moptions.arena_block_size;
|
|
|
|
cf_opts->memtable_prefix_bloom_size_ratio =
|
|
|
|
moptions.memtable_prefix_bloom_size_ratio;
|
|
|
|
cf_opts->memtable_whole_key_filtering = moptions.memtable_whole_key_filtering;
|
|
|
|
cf_opts->memtable_huge_page_size = moptions.memtable_huge_page_size;
|
|
|
|
cf_opts->max_successive_merges = moptions.max_successive_merges;
|
|
|
|
cf_opts->inplace_update_num_locks = moptions.inplace_update_num_locks;
|
|
|
|
cf_opts->prefix_extractor = moptions.prefix_extractor;
|
|
|
|
|
|
|
|
// Compaction related options
|
|
|
|
cf_opts->disable_auto_compactions = moptions.disable_auto_compactions;
|
|
|
|
cf_opts->soft_pending_compaction_bytes_limit =
|
|
|
|
moptions.soft_pending_compaction_bytes_limit;
|
|
|
|
cf_opts->hard_pending_compaction_bytes_limit =
|
|
|
|
moptions.hard_pending_compaction_bytes_limit;
|
|
|
|
cf_opts->level0_file_num_compaction_trigger =
|
|
|
|
moptions.level0_file_num_compaction_trigger;
|
|
|
|
cf_opts->level0_slowdown_writes_trigger =
|
|
|
|
moptions.level0_slowdown_writes_trigger;
|
|
|
|
cf_opts->level0_stop_writes_trigger = moptions.level0_stop_writes_trigger;
|
|
|
|
cf_opts->max_compaction_bytes = moptions.max_compaction_bytes;
|
|
|
|
cf_opts->target_file_size_base = moptions.target_file_size_base;
|
|
|
|
cf_opts->target_file_size_multiplier = moptions.target_file_size_multiplier;
|
|
|
|
cf_opts->max_bytes_for_level_base = moptions.max_bytes_for_level_base;
|
|
|
|
cf_opts->max_bytes_for_level_multiplier =
|
|
|
|
moptions.max_bytes_for_level_multiplier;
|
|
|
|
cf_opts->ttl = moptions.ttl;
|
|
|
|
cf_opts->periodic_compaction_seconds = moptions.periodic_compaction_seconds;
|
|
|
|
|
|
|
|
cf_opts->max_bytes_for_level_multiplier_additional.clear();
|
|
|
|
for (auto value : moptions.max_bytes_for_level_multiplier_additional) {
|
|
|
|
cf_opts->max_bytes_for_level_multiplier_additional.emplace_back(value);
|
|
|
|
}
|
|
|
|
|
|
|
|
cf_opts->compaction_options_fifo = moptions.compaction_options_fifo;
|
|
|
|
cf_opts->compaction_options_universal = moptions.compaction_options_universal;
|
|
|
|
|
|
|
|
// Blob file related options
|
|
|
|
cf_opts->enable_blob_files = moptions.enable_blob_files;
|
|
|
|
cf_opts->min_blob_size = moptions.min_blob_size;
|
|
|
|
cf_opts->blob_file_size = moptions.blob_file_size;
|
|
|
|
cf_opts->blob_compression_type = moptions.blob_compression_type;
|
|
|
|
cf_opts->enable_blob_garbage_collection =
|
|
|
|
moptions.enable_blob_garbage_collection;
|
|
|
|
cf_opts->blob_garbage_collection_age_cutoff =
|
|
|
|
moptions.blob_garbage_collection_age_cutoff;
|
Make it possible to force the garbage collection of the oldest blob files (#8994)
Summary:
The current BlobDB garbage collection logic works by relocating the valid
blobs from the oldest blob files as they are encountered during compaction,
and cleaning up blob files once they contain nothing but garbage. However,
with sufficiently skewed workloads, it is theoretically possible to end up in a
situation when few or no compactions get scheduled for the SST files that contain
references to the oldest blob files, which can lead to increased space amp due
to the lack of GC.
In order to efficiently handle such workloads, the patch adds a new BlobDB
configuration option called `blob_garbage_collection_force_threshold`,
which signals to BlobDB to schedule targeted compactions for the SST files
that keep alive the oldest batch of blob files if the overall ratio of garbage in
the given blob files meets the threshold *and* all the given blob files are
eligible for GC based on `blob_garbage_collection_age_cutoff`. (For example,
if the new option is set to 0.9, targeted compactions will get scheduled if the
sum of garbage bytes meets or exceeds 90% of the sum of total bytes in the
oldest blob files, assuming all affected blob files are below the age-based cutoff.)
The net result of these targeted compactions is that the valid blobs in the oldest
blob files are relocated and the oldest blob files themselves cleaned up (since
*all* SST files that rely on them get compacted away).
These targeted compactions are similar to periodic compactions in the sense
that they force certain SST files that otherwise would not get picked up to undergo
compaction and also in the sense that instead of merging files from multiple levels,
they target a single file. (Note: such compactions might still include neighboring files
from the same level due to the need of having a "clean cut" boundary but they never
include any files from any other level.)
This functionality is currently only supported with the leveled compaction style
and is inactive by default (since the default value is set to 1.0, i.e. 100%).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8994
Test Plan: Ran `make check` and tested using `db_bench` and the stress/crash tests.
Reviewed By: riversand963
Differential Revision: D31489850
Pulled By: ltamasi
fbshipit-source-id: 44057d511726a0e2a03c5d9313d7511b3f0c4eab
3 years ago
|
|
|
cf_opts->blob_garbage_collection_force_threshold =
|
|
|
|
moptions.blob_garbage_collection_force_threshold;
|
|
|
|
cf_opts->blob_compaction_readahead_size =
|
|
|
|
moptions.blob_compaction_readahead_size;
|
|
|
|
|
|
|
|
// Misc options
|
|
|
|
cf_opts->max_sequential_skip_in_iterations =
|
|
|
|
moptions.max_sequential_skip_in_iterations;
|
|
|
|
cf_opts->check_flush_compaction_key_order =
|
|
|
|
moptions.check_flush_compaction_key_order;
|
|
|
|
cf_opts->paranoid_file_checks = moptions.paranoid_file_checks;
|
|
|
|
cf_opts->report_bg_io_stats = moptions.report_bg_io_stats;
|
|
|
|
cf_opts->compression = moptions.compression;
|
|
|
|
cf_opts->compression_opts = moptions.compression_opts;
|
|
|
|
cf_opts->bottommost_compression = moptions.bottommost_compression;
|
|
|
|
cf_opts->bottommost_compression_opts = moptions.bottommost_compression_opts;
|
|
|
|
cf_opts->sample_for_compression = moptions.sample_for_compression;
|
|
|
|
cf_opts->bottommost_temperature = moptions.bottommost_temperature;
|
|
|
|
}
|
|
|
|
|
|
|
|
void UpdateColumnFamilyOptions(const ImmutableCFOptions& ioptions,
|
|
|
|
ColumnFamilyOptions* cf_opts) {
|
|
|
|
cf_opts->compaction_style = ioptions.compaction_style;
|
|
|
|
cf_opts->compaction_pri = ioptions.compaction_pri;
|
|
|
|
cf_opts->comparator = ioptions.user_comparator;
|
|
|
|
cf_opts->merge_operator = ioptions.merge_operator;
|
|
|
|
cf_opts->compaction_filter = ioptions.compaction_filter;
|
|
|
|
cf_opts->compaction_filter_factory = ioptions.compaction_filter_factory;
|
|
|
|
cf_opts->min_write_buffer_number_to_merge =
|
|
|
|
ioptions.min_write_buffer_number_to_merge;
|
|
|
|
cf_opts->max_write_buffer_number_to_maintain =
|
|
|
|
ioptions.max_write_buffer_number_to_maintain;
|
|
|
|
cf_opts->max_write_buffer_size_to_maintain =
|
|
|
|
ioptions.max_write_buffer_size_to_maintain;
|
|
|
|
cf_opts->inplace_update_support = ioptions.inplace_update_support;
|
|
|
|
cf_opts->inplace_callback = ioptions.inplace_callback;
|
|
|
|
cf_opts->memtable_factory = ioptions.memtable_factory;
|
|
|
|
cf_opts->table_factory = ioptions.table_factory;
|
|
|
|
cf_opts->table_properties_collector_factories =
|
|
|
|
ioptions.table_properties_collector_factories;
|
|
|
|
cf_opts->bloom_locality = ioptions.bloom_locality;
|
|
|
|
cf_opts->compression_per_level = ioptions.compression_per_level;
|
|
|
|
cf_opts->level_compaction_dynamic_level_bytes =
|
|
|
|
ioptions.level_compaction_dynamic_level_bytes;
|
|
|
|
cf_opts->num_levels = ioptions.num_levels;
|
|
|
|
cf_opts->optimize_filters_for_hits = ioptions.optimize_filters_for_hits;
|
|
|
|
cf_opts->force_consistency_checks = ioptions.force_consistency_checks;
|
|
|
|
cf_opts->memtable_insert_with_hint_prefix_extractor =
|
|
|
|
ioptions.memtable_insert_with_hint_prefix_extractor;
|
|
|
|
cf_opts->cf_paths = ioptions.cf_paths;
|
|
|
|
cf_opts->compaction_thread_limiter = ioptions.compaction_thread_limiter;
|
|
|
|
cf_opts->sst_partitioner_factory = ioptions.sst_partitioner_factory;
|
|
|
|
|
|
|
|
// TODO(yhchiang): find some way to handle the following derived options
|
|
|
|
// * max_file_size
|
|
|
|
}
|
|
|
|
|
|
|
|
std::map<CompactionStyle, std::string>
|
|
|
|
OptionsHelper::compaction_style_to_string = {
|
|
|
|
{kCompactionStyleLevel, "kCompactionStyleLevel"},
|
|
|
|
{kCompactionStyleUniversal, "kCompactionStyleUniversal"},
|
|
|
|
{kCompactionStyleFIFO, "kCompactionStyleFIFO"},
|
|
|
|
{kCompactionStyleNone, "kCompactionStyleNone"}};
|
|
|
|
|
|
|
|
std::map<CompactionPri, std::string> OptionsHelper::compaction_pri_to_string = {
|
|
|
|
{kByCompensatedSize, "kByCompensatedSize"},
|
|
|
|
{kOldestLargestSeqFirst, "kOldestLargestSeqFirst"},
|
|
|
|
{kOldestSmallestSeqFirst, "kOldestSmallestSeqFirst"},
|
|
|
|
{kMinOverlappingRatio, "kMinOverlappingRatio"}};
|
|
|
|
|
|
|
|
std::map<CompactionStopStyle, std::string>
|
|
|
|
OptionsHelper::compaction_stop_style_to_string = {
|
|
|
|
{kCompactionStopStyleSimilarSize, "kCompactionStopStyleSimilarSize"},
|
|
|
|
{kCompactionStopStyleTotalSize, "kCompactionStopStyleTotalSize"}};
|
|
|
|
|
|
|
|
std::unordered_map<std::string, ChecksumType>
|
|
|
|
OptionsHelper::checksum_type_string_map = {{"kNoChecksum", kNoChecksum},
|
|
|
|
{"kCRC32c", kCRC32c},
|
|
|
|
{"kxxHash", kxxHash},
|
Implement XXH3 block checksum type (#9069)
Summary:
XXH3 - latest hash function that is extremely fast on large
data, easily faster than crc32c on most any x86_64 hardware. In
integrating this hash function, I have handled the compression type byte
in a non-standard way to avoid using the streaming API (extra data
movement and active code size because of hash function complexity). This
approach got a thumbs-up from Yann Collet.
Existing functionality change:
* reject bad ChecksumType in options with InvalidArgument
This change split off from https://github.com/facebook/rocksdb/issues/9058 because context-aware checksum is
likely to be handled through different configuration than ChecksumType.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9069
Test Plan:
tests updated, and substantially expanded. Unit tests now check
that we don't accidentally change the values generated by the checksum
algorithms ("schema test") and that we properly handle
invalid/unrecognized checksum types in options or in file footer.
DBTestBase::ChangeOptions (etc.) updated from two to one configuration
changing from default CRC32c ChecksumType. The point of this test code
is to detect possible interactions among features, and the likelihood of
some bad interaction being detected by including configurations other
than XXH3 and CRC32c--and then not detected by stress/crash test--is
extremely low.
Stress/crash test also updated (manual run long enough to see it accepts
new checksum type). db_bench also updated for microbenchmarking
checksums.
### Performance microbenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
./db_bench -benchmarks=crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3
crc32c : 0.200 micros/op 5005220 ops/sec; 19551.6 MB/s (4096 per op)
xxhash : 0.807 micros/op 1238408 ops/sec; 4837.5 MB/s (4096 per op)
xxhash64 : 0.421 micros/op 2376514 ops/sec; 9283.3 MB/s (4096 per op)
xxh3 : 0.171 micros/op 5858391 ops/sec; 22884.3 MB/s (4096 per op)
crc32c : 0.206 micros/op 4859566 ops/sec; 18982.7 MB/s (4096 per op)
xxhash : 0.793 micros/op 1260850 ops/sec; 4925.2 MB/s (4096 per op)
xxhash64 : 0.410 micros/op 2439182 ops/sec; 9528.1 MB/s (4096 per op)
xxh3 : 0.161 micros/op 6202872 ops/sec; 24230.0 MB/s (4096 per op)
crc32c : 0.203 micros/op 4924686 ops/sec; 19237.1 MB/s (4096 per op)
xxhash : 0.839 micros/op 1192388 ops/sec; 4657.8 MB/s (4096 per op)
xxhash64 : 0.424 micros/op 2357391 ops/sec; 9208.6 MB/s (4096 per op)
xxh3 : 0.162 micros/op 6182678 ops/sec; 24151.1 MB/s (4096 per op)
As you can see, especially once warmed up, xxh3 is fastest.
### Performance macrobenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
Test
for I in `seq 1 50`; do for CHK in 0 1 2 3 4; do TEST_TMPDIR=/dev/shm/rocksdb$CHK ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=30000000 -checksum_type=$CHK 2>&1 | grep 'micros/op' | tee -a results-$CHK & done; wait; done
Results (ops/sec)
for FILE in results*; do echo -n "$FILE "; awk '{ s += $5; c++; } END { print 1.0 * s / c; }' < $FILE; done
results-0 252118 # kNoChecksum
results-1 251588 # kCRC32c
results-2 251863 # kxxHash
results-3 252016 # kxxHash64
results-4 252038 # kXXH3
Reviewed By: mrambacher
Differential Revision: D31905249
Pulled By: pdillinger
fbshipit-source-id: cb9b998ebe2523fc7c400eedf62124a78bf4b4d1
3 years ago
|
|
|
{"kxxHash64", kxxHash64},
|
|
|
|
{"kXXH3", kXXH3}};
|
|
|
|
|
|
|
|
std::unordered_map<std::string, CompressionType>
|
|
|
|
OptionsHelper::compression_type_string_map = {
|
|
|
|
{"kNoCompression", kNoCompression},
|
|
|
|
{"kSnappyCompression", kSnappyCompression},
|
|
|
|
{"kZlibCompression", kZlibCompression},
|
|
|
|
{"kBZip2Compression", kBZip2Compression},
|
|
|
|
{"kLZ4Compression", kLZ4Compression},
|
|
|
|
{"kLZ4HCCompression", kLZ4HCCompression},
|
|
|
|
{"kXpressCompression", kXpressCompression},
|
|
|
|
{"kZSTD", kZSTD},
|
|
|
|
{"kZSTDNotFinalCompression", kZSTDNotFinalCompression},
|
|
|
|
{"kDisableCompressionOption", kDisableCompressionOption}};
|
|
|
|
|
|
|
|
std::vector<CompressionType> GetSupportedCompressions() {
|
Implement XXH3 block checksum type (#9069)
Summary:
XXH3 - latest hash function that is extremely fast on large
data, easily faster than crc32c on most any x86_64 hardware. In
integrating this hash function, I have handled the compression type byte
in a non-standard way to avoid using the streaming API (extra data
movement and active code size because of hash function complexity). This
approach got a thumbs-up from Yann Collet.
Existing functionality change:
* reject bad ChecksumType in options with InvalidArgument
This change split off from https://github.com/facebook/rocksdb/issues/9058 because context-aware checksum is
likely to be handled through different configuration than ChecksumType.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9069
Test Plan:
tests updated, and substantially expanded. Unit tests now check
that we don't accidentally change the values generated by the checksum
algorithms ("schema test") and that we properly handle
invalid/unrecognized checksum types in options or in file footer.
DBTestBase::ChangeOptions (etc.) updated from two to one configuration
changing from default CRC32c ChecksumType. The point of this test code
is to detect possible interactions among features, and the likelihood of
some bad interaction being detected by including configurations other
than XXH3 and CRC32c--and then not detected by stress/crash test--is
extremely low.
Stress/crash test also updated (manual run long enough to see it accepts
new checksum type). db_bench also updated for microbenchmarking
checksums.
### Performance microbenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
./db_bench -benchmarks=crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3
crc32c : 0.200 micros/op 5005220 ops/sec; 19551.6 MB/s (4096 per op)
xxhash : 0.807 micros/op 1238408 ops/sec; 4837.5 MB/s (4096 per op)
xxhash64 : 0.421 micros/op 2376514 ops/sec; 9283.3 MB/s (4096 per op)
xxh3 : 0.171 micros/op 5858391 ops/sec; 22884.3 MB/s (4096 per op)
crc32c : 0.206 micros/op 4859566 ops/sec; 18982.7 MB/s (4096 per op)
xxhash : 0.793 micros/op 1260850 ops/sec; 4925.2 MB/s (4096 per op)
xxhash64 : 0.410 micros/op 2439182 ops/sec; 9528.1 MB/s (4096 per op)
xxh3 : 0.161 micros/op 6202872 ops/sec; 24230.0 MB/s (4096 per op)
crc32c : 0.203 micros/op 4924686 ops/sec; 19237.1 MB/s (4096 per op)
xxhash : 0.839 micros/op 1192388 ops/sec; 4657.8 MB/s (4096 per op)
xxhash64 : 0.424 micros/op 2357391 ops/sec; 9208.6 MB/s (4096 per op)
xxh3 : 0.162 micros/op 6182678 ops/sec; 24151.1 MB/s (4096 per op)
As you can see, especially once warmed up, xxh3 is fastest.
### Performance macrobenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
Test
for I in `seq 1 50`; do for CHK in 0 1 2 3 4; do TEST_TMPDIR=/dev/shm/rocksdb$CHK ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=30000000 -checksum_type=$CHK 2>&1 | grep 'micros/op' | tee -a results-$CHK & done; wait; done
Results (ops/sec)
for FILE in results*; do echo -n "$FILE "; awk '{ s += $5; c++; } END { print 1.0 * s / c; }' < $FILE; done
results-0 252118 # kNoChecksum
results-1 251588 # kCRC32c
results-2 251863 # kxxHash
results-3 252016 # kxxHash64
results-4 252038 # kXXH3
Reviewed By: mrambacher
Differential Revision: D31905249
Pulled By: pdillinger
fbshipit-source-id: cb9b998ebe2523fc7c400eedf62124a78bf4b4d1
3 years ago
|
|
|
// std::set internally to deduplicate potential name aliases
|
|
|
|
std::set<CompressionType> supported_compressions;
|
|
|
|
for (const auto& comp_to_name : OptionsHelper::compression_type_string_map) {
|
|
|
|
CompressionType t = comp_to_name.second;
|
|
|
|
if (t != kDisableCompressionOption && CompressionTypeSupported(t)) {
|
Implement XXH3 block checksum type (#9069)
Summary:
XXH3 - latest hash function that is extremely fast on large
data, easily faster than crc32c on most any x86_64 hardware. In
integrating this hash function, I have handled the compression type byte
in a non-standard way to avoid using the streaming API (extra data
movement and active code size because of hash function complexity). This
approach got a thumbs-up from Yann Collet.
Existing functionality change:
* reject bad ChecksumType in options with InvalidArgument
This change split off from https://github.com/facebook/rocksdb/issues/9058 because context-aware checksum is
likely to be handled through different configuration than ChecksumType.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9069
Test Plan:
tests updated, and substantially expanded. Unit tests now check
that we don't accidentally change the values generated by the checksum
algorithms ("schema test") and that we properly handle
invalid/unrecognized checksum types in options or in file footer.
DBTestBase::ChangeOptions (etc.) updated from two to one configuration
changing from default CRC32c ChecksumType. The point of this test code
is to detect possible interactions among features, and the likelihood of
some bad interaction being detected by including configurations other
than XXH3 and CRC32c--and then not detected by stress/crash test--is
extremely low.
Stress/crash test also updated (manual run long enough to see it accepts
new checksum type). db_bench also updated for microbenchmarking
checksums.
### Performance microbenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
./db_bench -benchmarks=crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3
crc32c : 0.200 micros/op 5005220 ops/sec; 19551.6 MB/s (4096 per op)
xxhash : 0.807 micros/op 1238408 ops/sec; 4837.5 MB/s (4096 per op)
xxhash64 : 0.421 micros/op 2376514 ops/sec; 9283.3 MB/s (4096 per op)
xxh3 : 0.171 micros/op 5858391 ops/sec; 22884.3 MB/s (4096 per op)
crc32c : 0.206 micros/op 4859566 ops/sec; 18982.7 MB/s (4096 per op)
xxhash : 0.793 micros/op 1260850 ops/sec; 4925.2 MB/s (4096 per op)
xxhash64 : 0.410 micros/op 2439182 ops/sec; 9528.1 MB/s (4096 per op)
xxh3 : 0.161 micros/op 6202872 ops/sec; 24230.0 MB/s (4096 per op)
crc32c : 0.203 micros/op 4924686 ops/sec; 19237.1 MB/s (4096 per op)
xxhash : 0.839 micros/op 1192388 ops/sec; 4657.8 MB/s (4096 per op)
xxhash64 : 0.424 micros/op 2357391 ops/sec; 9208.6 MB/s (4096 per op)
xxh3 : 0.162 micros/op 6182678 ops/sec; 24151.1 MB/s (4096 per op)
As you can see, especially once warmed up, xxh3 is fastest.
### Performance macrobenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
Test
for I in `seq 1 50`; do for CHK in 0 1 2 3 4; do TEST_TMPDIR=/dev/shm/rocksdb$CHK ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=30000000 -checksum_type=$CHK 2>&1 | grep 'micros/op' | tee -a results-$CHK & done; wait; done
Results (ops/sec)
for FILE in results*; do echo -n "$FILE "; awk '{ s += $5; c++; } END { print 1.0 * s / c; }' < $FILE; done
results-0 252118 # kNoChecksum
results-1 251588 # kCRC32c
results-2 251863 # kxxHash
results-3 252016 # kxxHash64
results-4 252038 # kXXH3
Reviewed By: mrambacher
Differential Revision: D31905249
Pulled By: pdillinger
fbshipit-source-id: cb9b998ebe2523fc7c400eedf62124a78bf4b4d1
3 years ago
|
|
|
supported_compressions.insert(t);
|
|
|
|
}
|
|
|
|
}
|
Implement XXH3 block checksum type (#9069)
Summary:
XXH3 - latest hash function that is extremely fast on large
data, easily faster than crc32c on most any x86_64 hardware. In
integrating this hash function, I have handled the compression type byte
in a non-standard way to avoid using the streaming API (extra data
movement and active code size because of hash function complexity). This
approach got a thumbs-up from Yann Collet.
Existing functionality change:
* reject bad ChecksumType in options with InvalidArgument
This change split off from https://github.com/facebook/rocksdb/issues/9058 because context-aware checksum is
likely to be handled through different configuration than ChecksumType.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9069
Test Plan:
tests updated, and substantially expanded. Unit tests now check
that we don't accidentally change the values generated by the checksum
algorithms ("schema test") and that we properly handle
invalid/unrecognized checksum types in options or in file footer.
DBTestBase::ChangeOptions (etc.) updated from two to one configuration
changing from default CRC32c ChecksumType. The point of this test code
is to detect possible interactions among features, and the likelihood of
some bad interaction being detected by including configurations other
than XXH3 and CRC32c--and then not detected by stress/crash test--is
extremely low.
Stress/crash test also updated (manual run long enough to see it accepts
new checksum type). db_bench also updated for microbenchmarking
checksums.
### Performance microbenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
./db_bench -benchmarks=crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3
crc32c : 0.200 micros/op 5005220 ops/sec; 19551.6 MB/s (4096 per op)
xxhash : 0.807 micros/op 1238408 ops/sec; 4837.5 MB/s (4096 per op)
xxhash64 : 0.421 micros/op 2376514 ops/sec; 9283.3 MB/s (4096 per op)
xxh3 : 0.171 micros/op 5858391 ops/sec; 22884.3 MB/s (4096 per op)
crc32c : 0.206 micros/op 4859566 ops/sec; 18982.7 MB/s (4096 per op)
xxhash : 0.793 micros/op 1260850 ops/sec; 4925.2 MB/s (4096 per op)
xxhash64 : 0.410 micros/op 2439182 ops/sec; 9528.1 MB/s (4096 per op)
xxh3 : 0.161 micros/op 6202872 ops/sec; 24230.0 MB/s (4096 per op)
crc32c : 0.203 micros/op 4924686 ops/sec; 19237.1 MB/s (4096 per op)
xxhash : 0.839 micros/op 1192388 ops/sec; 4657.8 MB/s (4096 per op)
xxhash64 : 0.424 micros/op 2357391 ops/sec; 9208.6 MB/s (4096 per op)
xxh3 : 0.162 micros/op 6182678 ops/sec; 24151.1 MB/s (4096 per op)
As you can see, especially once warmed up, xxh3 is fastest.
### Performance macrobenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
Test
for I in `seq 1 50`; do for CHK in 0 1 2 3 4; do TEST_TMPDIR=/dev/shm/rocksdb$CHK ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=30000000 -checksum_type=$CHK 2>&1 | grep 'micros/op' | tee -a results-$CHK & done; wait; done
Results (ops/sec)
for FILE in results*; do echo -n "$FILE "; awk '{ s += $5; c++; } END { print 1.0 * s / c; }' < $FILE; done
results-0 252118 # kNoChecksum
results-1 251588 # kCRC32c
results-2 251863 # kxxHash
results-3 252016 # kxxHash64
results-4 252038 # kXXH3
Reviewed By: mrambacher
Differential Revision: D31905249
Pulled By: pdillinger
fbshipit-source-id: cb9b998ebe2523fc7c400eedf62124a78bf4b4d1
3 years ago
|
|
|
return std::vector<CompressionType>(supported_compressions.begin(),
|
|
|
|
supported_compressions.end());
|
|
|
|
}
|
|
|
|
|
|
|
|
std::vector<CompressionType> GetSupportedDictCompressions() {
|
Implement XXH3 block checksum type (#9069)
Summary:
XXH3 - latest hash function that is extremely fast on large
data, easily faster than crc32c on most any x86_64 hardware. In
integrating this hash function, I have handled the compression type byte
in a non-standard way to avoid using the streaming API (extra data
movement and active code size because of hash function complexity). This
approach got a thumbs-up from Yann Collet.
Existing functionality change:
* reject bad ChecksumType in options with InvalidArgument
This change split off from https://github.com/facebook/rocksdb/issues/9058 because context-aware checksum is
likely to be handled through different configuration than ChecksumType.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9069
Test Plan:
tests updated, and substantially expanded. Unit tests now check
that we don't accidentally change the values generated by the checksum
algorithms ("schema test") and that we properly handle
invalid/unrecognized checksum types in options or in file footer.
DBTestBase::ChangeOptions (etc.) updated from two to one configuration
changing from default CRC32c ChecksumType. The point of this test code
is to detect possible interactions among features, and the likelihood of
some bad interaction being detected by including configurations other
than XXH3 and CRC32c--and then not detected by stress/crash test--is
extremely low.
Stress/crash test also updated (manual run long enough to see it accepts
new checksum type). db_bench also updated for microbenchmarking
checksums.
### Performance microbenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
./db_bench -benchmarks=crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3
crc32c : 0.200 micros/op 5005220 ops/sec; 19551.6 MB/s (4096 per op)
xxhash : 0.807 micros/op 1238408 ops/sec; 4837.5 MB/s (4096 per op)
xxhash64 : 0.421 micros/op 2376514 ops/sec; 9283.3 MB/s (4096 per op)
xxh3 : 0.171 micros/op 5858391 ops/sec; 22884.3 MB/s (4096 per op)
crc32c : 0.206 micros/op 4859566 ops/sec; 18982.7 MB/s (4096 per op)
xxhash : 0.793 micros/op 1260850 ops/sec; 4925.2 MB/s (4096 per op)
xxhash64 : 0.410 micros/op 2439182 ops/sec; 9528.1 MB/s (4096 per op)
xxh3 : 0.161 micros/op 6202872 ops/sec; 24230.0 MB/s (4096 per op)
crc32c : 0.203 micros/op 4924686 ops/sec; 19237.1 MB/s (4096 per op)
xxhash : 0.839 micros/op 1192388 ops/sec; 4657.8 MB/s (4096 per op)
xxhash64 : 0.424 micros/op 2357391 ops/sec; 9208.6 MB/s (4096 per op)
xxh3 : 0.162 micros/op 6182678 ops/sec; 24151.1 MB/s (4096 per op)
As you can see, especially once warmed up, xxh3 is fastest.
### Performance macrobenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
Test
for I in `seq 1 50`; do for CHK in 0 1 2 3 4; do TEST_TMPDIR=/dev/shm/rocksdb$CHK ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=30000000 -checksum_type=$CHK 2>&1 | grep 'micros/op' | tee -a results-$CHK & done; wait; done
Results (ops/sec)
for FILE in results*; do echo -n "$FILE "; awk '{ s += $5; c++; } END { print 1.0 * s / c; }' < $FILE; done
results-0 252118 # kNoChecksum
results-1 251588 # kCRC32c
results-2 251863 # kxxHash
results-3 252016 # kxxHash64
results-4 252038 # kXXH3
Reviewed By: mrambacher
Differential Revision: D31905249
Pulled By: pdillinger
fbshipit-source-id: cb9b998ebe2523fc7c400eedf62124a78bf4b4d1
3 years ago
|
|
|
std::set<CompressionType> dict_compression_types;
|
|
|
|
for (const auto& comp_to_name : OptionsHelper::compression_type_string_map) {
|
|
|
|
CompressionType t = comp_to_name.second;
|
|
|
|
if (t != kDisableCompressionOption && DictCompressionTypeSupported(t)) {
|
Implement XXH3 block checksum type (#9069)
Summary:
XXH3 - latest hash function that is extremely fast on large
data, easily faster than crc32c on most any x86_64 hardware. In
integrating this hash function, I have handled the compression type byte
in a non-standard way to avoid using the streaming API (extra data
movement and active code size because of hash function complexity). This
approach got a thumbs-up from Yann Collet.
Existing functionality change:
* reject bad ChecksumType in options with InvalidArgument
This change split off from https://github.com/facebook/rocksdb/issues/9058 because context-aware checksum is
likely to be handled through different configuration than ChecksumType.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9069
Test Plan:
tests updated, and substantially expanded. Unit tests now check
that we don't accidentally change the values generated by the checksum
algorithms ("schema test") and that we properly handle
invalid/unrecognized checksum types in options or in file footer.
DBTestBase::ChangeOptions (etc.) updated from two to one configuration
changing from default CRC32c ChecksumType. The point of this test code
is to detect possible interactions among features, and the likelihood of
some bad interaction being detected by including configurations other
than XXH3 and CRC32c--and then not detected by stress/crash test--is
extremely low.
Stress/crash test also updated (manual run long enough to see it accepts
new checksum type). db_bench also updated for microbenchmarking
checksums.
### Performance microbenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
./db_bench -benchmarks=crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3
crc32c : 0.200 micros/op 5005220 ops/sec; 19551.6 MB/s (4096 per op)
xxhash : 0.807 micros/op 1238408 ops/sec; 4837.5 MB/s (4096 per op)
xxhash64 : 0.421 micros/op 2376514 ops/sec; 9283.3 MB/s (4096 per op)
xxh3 : 0.171 micros/op 5858391 ops/sec; 22884.3 MB/s (4096 per op)
crc32c : 0.206 micros/op 4859566 ops/sec; 18982.7 MB/s (4096 per op)
xxhash : 0.793 micros/op 1260850 ops/sec; 4925.2 MB/s (4096 per op)
xxhash64 : 0.410 micros/op 2439182 ops/sec; 9528.1 MB/s (4096 per op)
xxh3 : 0.161 micros/op 6202872 ops/sec; 24230.0 MB/s (4096 per op)
crc32c : 0.203 micros/op 4924686 ops/sec; 19237.1 MB/s (4096 per op)
xxhash : 0.839 micros/op 1192388 ops/sec; 4657.8 MB/s (4096 per op)
xxhash64 : 0.424 micros/op 2357391 ops/sec; 9208.6 MB/s (4096 per op)
xxh3 : 0.162 micros/op 6182678 ops/sec; 24151.1 MB/s (4096 per op)
As you can see, especially once warmed up, xxh3 is fastest.
### Performance macrobenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
Test
for I in `seq 1 50`; do for CHK in 0 1 2 3 4; do TEST_TMPDIR=/dev/shm/rocksdb$CHK ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=30000000 -checksum_type=$CHK 2>&1 | grep 'micros/op' | tee -a results-$CHK & done; wait; done
Results (ops/sec)
for FILE in results*; do echo -n "$FILE "; awk '{ s += $5; c++; } END { print 1.0 * s / c; }' < $FILE; done
results-0 252118 # kNoChecksum
results-1 251588 # kCRC32c
results-2 251863 # kxxHash
results-3 252016 # kxxHash64
results-4 252038 # kXXH3
Reviewed By: mrambacher
Differential Revision: D31905249
Pulled By: pdillinger
fbshipit-source-id: cb9b998ebe2523fc7c400eedf62124a78bf4b4d1
3 years ago
|
|
|
dict_compression_types.insert(t);
|
|
|
|
}
|
|
|
|
}
|
Implement XXH3 block checksum type (#9069)
Summary:
XXH3 - latest hash function that is extremely fast on large
data, easily faster than crc32c on most any x86_64 hardware. In
integrating this hash function, I have handled the compression type byte
in a non-standard way to avoid using the streaming API (extra data
movement and active code size because of hash function complexity). This
approach got a thumbs-up from Yann Collet.
Existing functionality change:
* reject bad ChecksumType in options with InvalidArgument
This change split off from https://github.com/facebook/rocksdb/issues/9058 because context-aware checksum is
likely to be handled through different configuration than ChecksumType.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9069
Test Plan:
tests updated, and substantially expanded. Unit tests now check
that we don't accidentally change the values generated by the checksum
algorithms ("schema test") and that we properly handle
invalid/unrecognized checksum types in options or in file footer.
DBTestBase::ChangeOptions (etc.) updated from two to one configuration
changing from default CRC32c ChecksumType. The point of this test code
is to detect possible interactions among features, and the likelihood of
some bad interaction being detected by including configurations other
than XXH3 and CRC32c--and then not detected by stress/crash test--is
extremely low.
Stress/crash test also updated (manual run long enough to see it accepts
new checksum type). db_bench also updated for microbenchmarking
checksums.
### Performance microbenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
./db_bench -benchmarks=crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3,crc32c,xxhash,xxhash64,xxh3
crc32c : 0.200 micros/op 5005220 ops/sec; 19551.6 MB/s (4096 per op)
xxhash : 0.807 micros/op 1238408 ops/sec; 4837.5 MB/s (4096 per op)
xxhash64 : 0.421 micros/op 2376514 ops/sec; 9283.3 MB/s (4096 per op)
xxh3 : 0.171 micros/op 5858391 ops/sec; 22884.3 MB/s (4096 per op)
crc32c : 0.206 micros/op 4859566 ops/sec; 18982.7 MB/s (4096 per op)
xxhash : 0.793 micros/op 1260850 ops/sec; 4925.2 MB/s (4096 per op)
xxhash64 : 0.410 micros/op 2439182 ops/sec; 9528.1 MB/s (4096 per op)
xxh3 : 0.161 micros/op 6202872 ops/sec; 24230.0 MB/s (4096 per op)
crc32c : 0.203 micros/op 4924686 ops/sec; 19237.1 MB/s (4096 per op)
xxhash : 0.839 micros/op 1192388 ops/sec; 4657.8 MB/s (4096 per op)
xxhash64 : 0.424 micros/op 2357391 ops/sec; 9208.6 MB/s (4096 per op)
xxh3 : 0.162 micros/op 6182678 ops/sec; 24151.1 MB/s (4096 per op)
As you can see, especially once warmed up, xxh3 is fastest.
### Performance macrobenchmark (PORTABLE=0 DEBUG_LEVEL=0, Broadwell processor)
Test
for I in `seq 1 50`; do for CHK in 0 1 2 3 4; do TEST_TMPDIR=/dev/shm/rocksdb$CHK ./db_bench -benchmarks=fillseq -memtablerep=vector -allow_concurrent_memtable_write=false -num=30000000 -checksum_type=$CHK 2>&1 | grep 'micros/op' | tee -a results-$CHK & done; wait; done
Results (ops/sec)
for FILE in results*; do echo -n "$FILE "; awk '{ s += $5; c++; } END { print 1.0 * s / c; }' < $FILE; done
results-0 252118 # kNoChecksum
results-1 251588 # kCRC32c
results-2 251863 # kxxHash
results-3 252016 # kxxHash64
results-4 252038 # kXXH3
Reviewed By: mrambacher
Differential Revision: D31905249
Pulled By: pdillinger
fbshipit-source-id: cb9b998ebe2523fc7c400eedf62124a78bf4b4d1
3 years ago
|
|
|
return std::vector<CompressionType>(dict_compression_types.begin(),
|
|
|
|
dict_compression_types.end());
|
|
|
|
}
|
|
|
|
|
|
|
|
std::vector<ChecksumType> GetSupportedChecksums() {
|
|
|
|
std::set<ChecksumType> checksum_types;
|
|
|
|
for (const auto& e : OptionsHelper::checksum_type_string_map) {
|
|
|
|
checksum_types.insert(e.second);
|
|
|
|
}
|
|
|
|
return std::vector<ChecksumType>(checksum_types.begin(),
|
|
|
|
checksum_types.end());
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
static bool ParseOptionHelper(void* opt_address, const OptionType& opt_type,
|
|
|
|
const std::string& value) {
|
|
|
|
switch (opt_type) {
|
|
|
|
case OptionType::kBoolean:
|
|
|
|
*static_cast<bool*>(opt_address) = ParseBoolean("", value);
|
|
|
|
break;
|
|
|
|
case OptionType::kInt:
|
|
|
|
*static_cast<int*>(opt_address) = ParseInt(value);
|
|
|
|
break;
|
|
|
|
case OptionType::kInt32T:
|
|
|
|
*static_cast<int32_t*>(opt_address) = ParseInt32(value);
|
|
|
|
break;
|
|
|
|
case OptionType::kInt64T:
|
|
|
|
PutUnaligned(static_cast<int64_t*>(opt_address), ParseInt64(value));
|
|
|
|
break;
|
|
|
|
case OptionType::kUInt:
|
|
|
|
*static_cast<unsigned int*>(opt_address) = ParseUint32(value);
|
|
|
|
break;
|
|
|
|
case OptionType::kUInt8T:
|
|
|
|
*static_cast<uint8_t*>(opt_address) = ParseUint8(value);
|
|
|
|
break;
|
|
|
|
case OptionType::kUInt32T:
|
|
|
|
*static_cast<uint32_t*>(opt_address) = ParseUint32(value);
|
|
|
|
break;
|
|
|
|
case OptionType::kUInt64T:
|
|
|
|
PutUnaligned(static_cast<uint64_t*>(opt_address), ParseUint64(value));
|
|
|
|
break;
|
|
|
|
case OptionType::kSizeT:
|
|
|
|
PutUnaligned(static_cast<size_t*>(opt_address), ParseSizeT(value));
|
|
|
|
break;
|
|
|
|
case OptionType::kString:
|
|
|
|
*static_cast<std::string*>(opt_address) = value;
|
|
|
|
break;
|
|
|
|
case OptionType::kDouble:
|
|
|
|
*static_cast<double*>(opt_address) = ParseDouble(value);
|
|
|
|
break;
|
|
|
|
case OptionType::kCompactionStyle:
|
|
|
|
return ParseEnum<CompactionStyle>(
|
|
|
|
compaction_style_string_map, value,
|
|
|
|
static_cast<CompactionStyle*>(opt_address));
|
|
|
|
case OptionType::kCompactionPri:
|
|
|
|
return ParseEnum<CompactionPri>(compaction_pri_string_map, value,
|
|
|
|
static_cast<CompactionPri*>(opt_address));
|
|
|
|
case OptionType::kCompressionType:
|
|
|
|
return ParseEnum<CompressionType>(
|
|
|
|
compression_type_string_map, value,
|
|
|
|
static_cast<CompressionType*>(opt_address));
|
|
|
|
case OptionType::kChecksumType:
|
|
|
|
return ParseEnum<ChecksumType>(checksum_type_string_map, value,
|
|
|
|
static_cast<ChecksumType*>(opt_address));
|
|
|
|
case OptionType::kEncodingType:
|
|
|
|
return ParseEnum<EncodingType>(encoding_type_string_map, value,
|
|
|
|
static_cast<EncodingType*>(opt_address));
|
|
|
|
case OptionType::kCompactionStopStyle:
|
|
|
|
return ParseEnum<CompactionStopStyle>(
|
|
|
|
compaction_stop_style_string_map, value,
|
|
|
|
static_cast<CompactionStopStyle*>(opt_address));
|
|
|
|
case OptionType::kEncodedString: {
|
|
|
|
std::string* output_addr = static_cast<std::string*>(opt_address);
|
|
|
|
(Slice(value)).DecodeHex(output_addr);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case OptionType::kTemperature: {
|
|
|
|
return ParseEnum<Temperature>(temperature_string_map, value,
|
|
|
|
static_cast<Temperature*>(opt_address));
|
|
|
|
}
|
|
|
|
default:
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool SerializeSingleOptionHelper(const void* opt_address,
|
|
|
|
const OptionType opt_type,
|
|
|
|
std::string* value) {
|
|
|
|
assert(value);
|
|
|
|
switch (opt_type) {
|
|
|
|
case OptionType::kBoolean:
|
|
|
|
*value = *(static_cast<const bool*>(opt_address)) ? "true" : "false";
|
|
|
|
break;
|
|
|
|
case OptionType::kInt:
|
|
|
|
*value = ToString(*(static_cast<const int*>(opt_address)));
|
|
|
|
break;
|
|
|
|
case OptionType::kInt32T:
|
|
|
|
*value = ToString(*(static_cast<const int32_t*>(opt_address)));
|
|
|
|
break;
|
|
|
|
case OptionType::kInt64T:
|
|
|
|
{
|
|
|
|
int64_t v;
|
|
|
|
GetUnaligned(static_cast<const int64_t*>(opt_address), &v);
|
|
|
|
*value = ToString(v);
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case OptionType::kUInt:
|
|
|
|
*value = ToString(*(static_cast<const unsigned int*>(opt_address)));
|
|
|
|
break;
|
|
|
|
case OptionType::kUInt8T:
|
|
|
|
*value = ToString(*(static_cast<const uint8_t*>(opt_address)));
|
|
|
|
break;
|
|
|
|
case OptionType::kUInt32T:
|
|
|
|
*value = ToString(*(static_cast<const uint32_t*>(opt_address)));
|
|
|
|
break;
|
|
|
|
case OptionType::kUInt64T:
|
|
|
|
{
|
|
|
|
uint64_t v;
|
|
|
|
GetUnaligned(static_cast<const uint64_t*>(opt_address), &v);
|
|
|
|
*value = ToString(v);
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case OptionType::kSizeT:
|
|
|
|
{
|
|
|
|
size_t v;
|
|
|
|
GetUnaligned(static_cast<const size_t*>(opt_address), &v);
|
|
|
|
*value = ToString(v);
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case OptionType::kDouble:
|
|
|
|
*value = ToString(*(static_cast<const double*>(opt_address)));
|
|
|
|
break;
|
|
|
|
case OptionType::kString:
|
|
|
|
*value =
|
|
|
|
EscapeOptionString(*(static_cast<const std::string*>(opt_address)));
|
|
|
|
break;
|
|
|
|
case OptionType::kCompactionStyle:
|
|
|
|
return SerializeEnum<CompactionStyle>(
|
|
|
|
compaction_style_string_map,
|
|
|
|
*(static_cast<const CompactionStyle*>(opt_address)), value);
|
|
|
|
case OptionType::kCompactionPri:
|
|
|
|
return SerializeEnum<CompactionPri>(
|
|
|
|
compaction_pri_string_map,
|
|
|
|
*(static_cast<const CompactionPri*>(opt_address)), value);
|
|
|
|
case OptionType::kCompressionType:
|
|
|
|
return SerializeEnum<CompressionType>(
|
|
|
|
compression_type_string_map,
|
|
|
|
*(static_cast<const CompressionType*>(opt_address)), value);
|
|
|
|
break;
|
|
|
|
case OptionType::kChecksumType:
|
|
|
|
return SerializeEnum<ChecksumType>(
|
|
|
|
checksum_type_string_map,
|
|
|
|
*static_cast<const ChecksumType*>(opt_address), value);
|
|
|
|
case OptionType::kEncodingType:
|
|
|
|
return SerializeEnum<EncodingType>(
|
|
|
|
encoding_type_string_map,
|
|
|
|
*static_cast<const EncodingType*>(opt_address), value);
|
|
|
|
case OptionType::kCompactionStopStyle:
|
|
|
|
return SerializeEnum<CompactionStopStyle>(
|
|
|
|
compaction_stop_style_string_map,
|
|
|
|
*static_cast<const CompactionStopStyle*>(opt_address), value);
|
|
|
|
case OptionType::kEncodedString: {
|
|
|
|
const auto* ptr = static_cast<const std::string*>(opt_address);
|
|
|
|
*value = (Slice(*ptr)).ToString(true);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case OptionType::kTemperature: {
|
|
|
|
return SerializeEnum<Temperature>(
|
|
|
|
temperature_string_map, *static_cast<const Temperature*>(opt_address),
|
|
|
|
value);
|
|
|
|
}
|
|
|
|
default:
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
template <typename T>
|
|
|
|
Status ConfigureFromMap(
|
|
|
|
const ConfigOptions& config_options,
|
|
|
|
const std::unordered_map<std::string, std::string>& opt_map,
|
|
|
|
const std::string& option_name, Configurable* config, T* new_opts) {
|
|
|
|
Status s = config->ConfigureFromMap(config_options, opt_map);
|
|
|
|
if (s.ok()) {
|
|
|
|
*new_opts = *(config->GetOptions<T>(option_name));
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
Status StringToMap(const std::string& opts_str,
|
|
|
|
std::unordered_map<std::string, std::string>* opts_map) {
|
|
|
|
assert(opts_map);
|
|
|
|
// Example:
|
|
|
|
// opts_str = "write_buffer_size=1024;max_write_buffer_number=2;"
|
|
|
|
// "nested_opt={opt1=1;opt2=2};max_bytes_for_level_base=100"
|
|
|
|
size_t pos = 0;
|
|
|
|
std::string opts = trim(opts_str);
|
|
|
|
// If the input string starts and ends with "{...}", strip off the brackets
|
|
|
|
while (opts.size() > 2 && opts[0] == '{' && opts[opts.size() - 1] == '}') {
|
|
|
|
opts = trim(opts.substr(1, opts.size() - 2));
|
|
|
|
}
|
|
|
|
|
|
|
|
while (pos < opts.size()) {
|
|
|
|
size_t eq_pos = opts.find_first_of("={};", pos);
|
|
|
|
if (eq_pos == std::string::npos) {
|
|
|
|
return Status::InvalidArgument("Mismatched key value pair, '=' expected");
|
|
|
|
} else if (opts[eq_pos] != '=') {
|
|
|
|
return Status::InvalidArgument("Unexpected char in key");
|
|
|
|
}
|
|
|
|
|
|
|
|
std::string key = trim(opts.substr(pos, eq_pos - pos));
|
|
|
|
if (key.empty()) {
|
|
|
|
return Status::InvalidArgument("Empty key found");
|
|
|
|
}
|
|
|
|
|
|
|
|
std::string value;
|
|
|
|
Status s = OptionTypeInfo::NextToken(opts, ';', eq_pos + 1, &pos, &value);
|
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
} else {
|
|
|
|
(*opts_map)[key] = value;
|
|
|
|
if (pos == std::string::npos) {
|
|
|
|
break;
|
|
|
|
} else {
|
|
|
|
pos++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
Status GetStringFromDBOptions(std::string* opt_string,
|
|
|
|
const DBOptions& db_options,
|
|
|
|
const std::string& delimiter) {
|
|
|
|
ConfigOptions config_options(db_options);
|
|
|
|
config_options.delimiter = delimiter;
|
|
|
|
return GetStringFromDBOptions(config_options, db_options, opt_string);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status GetStringFromDBOptions(const ConfigOptions& config_options,
|
|
|
|
const DBOptions& db_options,
|
|
|
|
std::string* opt_string) {
|
|
|
|
assert(opt_string);
|
|
|
|
opt_string->clear();
|
|
|
|
auto config = DBOptionsAsConfigurable(db_options);
|
|
|
|
return config->GetOptionString(config_options, opt_string);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
9 years ago
|
|
|
Status GetStringFromColumnFamilyOptions(std::string* opt_string,
|
|
|
|
const ColumnFamilyOptions& cf_options,
|
|
|
|
const std::string& delimiter) {
|
|
|
|
ConfigOptions config_options;
|
|
|
|
config_options.delimiter = delimiter;
|
|
|
|
return GetStringFromColumnFamilyOptions(config_options, cf_options,
|
|
|
|
opt_string);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status GetStringFromColumnFamilyOptions(const ConfigOptions& config_options,
|
|
|
|
const ColumnFamilyOptions& cf_options,
|
|
|
|
std::string* opt_string) {
|
|
|
|
const auto config = CFOptionsAsConfigurable(cf_options);
|
|
|
|
return config->GetOptionString(config_options, opt_string);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status GetStringFromCompressionType(std::string* compression_str,
|
|
|
|
CompressionType compression_type) {
|
|
|
|
bool ok = SerializeEnum<CompressionType>(compression_type_string_map,
|
|
|
|
compression_type, compression_str);
|
|
|
|
if (ok) {
|
|
|
|
return Status::OK();
|
|
|
|
} else {
|
|
|
|
return Status::InvalidArgument("Invalid compression types");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Status GetColumnFamilyOptionsFromMap(
|
|
|
|
const ColumnFamilyOptions& base_options,
|
|
|
|
const std::unordered_map<std::string, std::string>& opts_map,
|
|
|
|
ColumnFamilyOptions* new_options, bool input_strings_escaped,
|
|
|
|
bool ignore_unknown_options) {
|
|
|
|
ConfigOptions config_options;
|
|
|
|
config_options.ignore_unknown_options = ignore_unknown_options;
|
|
|
|
config_options.input_strings_escaped = input_strings_escaped;
|
|
|
|
return GetColumnFamilyOptionsFromMap(config_options, base_options, opts_map,
|
|
|
|
new_options);
|
Add OptionsUtil::LoadOptionsFromFile() API
Summary:
This patch adds OptionsUtil::LoadOptionsFromFile() and
OptionsUtil::LoadLatestOptionsFromDB(), which allow developers
to construct DBOptions and ColumnFamilyOptions from a RocksDB
options file. Note that most pointer-typed options such as
merge_operator will not be constructed.
With this API, developers no longer need to remember all the
options in order to reopen an existing rocksdb instance like
the following:
DBOptions db_options;
std::vector<std::string> cf_names;
std::vector<ColumnFamilyOptions> cf_opts;
// Load primitive-typed options from an existing DB
OptionsUtil::LoadLatestOptionsFromDB(
dbname, &db_options, &cf_names, &cf_opts);
// Initialize necessary pointer-typed options
cf_opts[0].merge_operator.reset(new MyMergeOperator());
...
// Construct the vector of ColumnFamilyDescriptor
std::vector<ColumnFamilyDescriptor> cf_descs;
for (size_t i = 0; i < cf_opts.size(); ++i) {
cf_descs.emplace_back(cf_names[i], cf_opts[i]);
}
// Open the DB
DB* db = nullptr;
std::vector<ColumnFamilyHandle*> cf_handles;
auto s = DB::Open(db_options, dbname, cf_descs,
&handles, &db);
Test Plan:
Augment existing tests in column_family_test
options_test
db_test
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D49095
9 years ago
|
|
|
}
|
|
|
|
|
|
|
|
Status GetColumnFamilyOptionsFromMap(
|
|
|
|
const ConfigOptions& config_options,
|
Add OptionsUtil::LoadOptionsFromFile() API
Summary:
This patch adds OptionsUtil::LoadOptionsFromFile() and
OptionsUtil::LoadLatestOptionsFromDB(), which allow developers
to construct DBOptions and ColumnFamilyOptions from a RocksDB
options file. Note that most pointer-typed options such as
merge_operator will not be constructed.
With this API, developers no longer need to remember all the
options in order to reopen an existing rocksdb instance like
the following:
DBOptions db_options;
std::vector<std::string> cf_names;
std::vector<ColumnFamilyOptions> cf_opts;
// Load primitive-typed options from an existing DB
OptionsUtil::LoadLatestOptionsFromDB(
dbname, &db_options, &cf_names, &cf_opts);
// Initialize necessary pointer-typed options
cf_opts[0].merge_operator.reset(new MyMergeOperator());
...
// Construct the vector of ColumnFamilyDescriptor
std::vector<ColumnFamilyDescriptor> cf_descs;
for (size_t i = 0; i < cf_opts.size(); ++i) {
cf_descs.emplace_back(cf_names[i], cf_opts[i]);
}
// Open the DB
DB* db = nullptr;
std::vector<ColumnFamilyHandle*> cf_handles;
auto s = DB::Open(db_options, dbname, cf_descs,
&handles, &db);
Test Plan:
Augment existing tests in column_family_test
options_test
db_test
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D49095
9 years ago
|
|
|
const ColumnFamilyOptions& base_options,
|
|
|
|
const std::unordered_map<std::string, std::string>& opts_map,
|
|
|
|
ColumnFamilyOptions* new_options) {
|
|
|
|
assert(new_options);
|
|
|
|
|
|
|
|
*new_options = base_options;
|
|
|
|
|
|
|
|
const auto config = CFOptionsAsConfigurable(base_options);
|
|
|
|
Status s = ConfigureFromMap<ColumnFamilyOptions>(
|
|
|
|
config_options, opts_map, OptionsHelper::kCFOptionsName, config.get(),
|
|
|
|
new_options);
|
|
|
|
// Translate any errors (NotFound, NotSupported, to InvalidArgument
|
|
|
|
if (s.ok() || s.IsInvalidArgument()) {
|
|
|
|
return s;
|
|
|
|
} else {
|
|
|
|
return Status::InvalidArgument(s.getState());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Status GetColumnFamilyOptionsFromString(
|
|
|
|
const ColumnFamilyOptions& base_options,
|
|
|
|
const std::string& opts_str,
|
|
|
|
ColumnFamilyOptions* new_options) {
|
|
|
|
ConfigOptions config_options;
|
|
|
|
config_options.input_strings_escaped = false;
|
|
|
|
config_options.ignore_unknown_options = false;
|
|
|
|
return GetColumnFamilyOptionsFromString(config_options, base_options,
|
|
|
|
opts_str, new_options);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status GetColumnFamilyOptionsFromString(const ConfigOptions& config_options,
|
|
|
|
const ColumnFamilyOptions& base_options,
|
|
|
|
const std::string& opts_str,
|
|
|
|
ColumnFamilyOptions* new_options) {
|
|
|
|
std::unordered_map<std::string, std::string> opts_map;
|
|
|
|
Status s = StringToMap(opts_str, &opts_map);
|
|
|
|
if (!s.ok()) {
|
|
|
|
*new_options = base_options;
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
return GetColumnFamilyOptionsFromMap(config_options, base_options, opts_map,
|
|
|
|
new_options);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status GetDBOptionsFromMap(
|
|
|
|
const DBOptions& base_options,
|
|
|
|
const std::unordered_map<std::string, std::string>& opts_map,
|
|
|
|
DBOptions* new_options, bool input_strings_escaped,
|
|
|
|
bool ignore_unknown_options) {
|
|
|
|
ConfigOptions config_options(base_options);
|
|
|
|
config_options.input_strings_escaped = input_strings_escaped;
|
|
|
|
config_options.ignore_unknown_options = ignore_unknown_options;
|
|
|
|
return GetDBOptionsFromMap(config_options, base_options, opts_map,
|
|
|
|
new_options);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status GetDBOptionsFromMap(
|
|
|
|
const ConfigOptions& config_options, const DBOptions& base_options,
|
|
|
|
const std::unordered_map<std::string, std::string>& opts_map,
|
|
|
|
DBOptions* new_options) {
|
|
|
|
assert(new_options);
|
|
|
|
*new_options = base_options;
|
|
|
|
auto config = DBOptionsAsConfigurable(base_options);
|
|
|
|
Status s = ConfigureFromMap<DBOptions>(config_options, opts_map,
|
|
|
|
OptionsHelper::kDBOptionsName,
|
|
|
|
config.get(), new_options);
|
|
|
|
// Translate any errors (NotFound, NotSupported, to InvalidArgument
|
|
|
|
if (s.ok() || s.IsInvalidArgument()) {
|
|
|
|
return s;
|
|
|
|
} else {
|
|
|
|
return Status::InvalidArgument(s.getState());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Status GetDBOptionsFromString(const DBOptions& base_options,
|
|
|
|
const std::string& opts_str,
|
|
|
|
DBOptions* new_options) {
|
|
|
|
ConfigOptions config_options(base_options);
|
|
|
|
config_options.input_strings_escaped = false;
|
|
|
|
config_options.ignore_unknown_options = false;
|
|
|
|
|
|
|
|
return GetDBOptionsFromString(config_options, base_options, opts_str,
|
|
|
|
new_options);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status GetDBOptionsFromString(const ConfigOptions& config_options,
|
|
|
|
const DBOptions& base_options,
|
|
|
|
const std::string& opts_str,
|
|
|
|
DBOptions* new_options) {
|
|
|
|
std::unordered_map<std::string, std::string> opts_map;
|
|
|
|
Status s = StringToMap(opts_str, &opts_map);
|
|
|
|
if (!s.ok()) {
|
|
|
|
*new_options = base_options;
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
return GetDBOptionsFromMap(config_options, base_options, opts_map,
|
|
|
|
new_options);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status GetOptionsFromString(const Options& base_options,
|
|
|
|
const std::string& opts_str, Options* new_options) {
|
|
|
|
ConfigOptions config_options(base_options);
|
|
|
|
config_options.input_strings_escaped = false;
|
|
|
|
config_options.ignore_unknown_options = false;
|
|
|
|
|
|
|
|
return GetOptionsFromString(config_options, base_options, opts_str,
|
|
|
|
new_options);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status GetOptionsFromString(const ConfigOptions& config_options,
|
|
|
|
const Options& base_options,
|
|
|
|
const std::string& opts_str, Options* new_options) {
|
|
|
|
ColumnFamilyOptions new_cf_options;
|
|
|
|
std::unordered_map<std::string, std::string> unused_opts;
|
|
|
|
std::unordered_map<std::string, std::string> opts_map;
|
|
|
|
|
|
|
|
assert(new_options);
|
|
|
|
*new_options = base_options;
|
|
|
|
Status s = StringToMap(opts_str, &opts_map);
|
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
auto config = DBOptionsAsConfigurable(base_options);
|
|
|
|
s = config->ConfigureFromMap(config_options, opts_map, &unused_opts);
|
|
|
|
|
|
|
|
if (s.ok()) {
|
|
|
|
DBOptions* new_db_options =
|
|
|
|
config->GetOptions<DBOptions>(OptionsHelper::kDBOptionsName);
|
|
|
|
if (!unused_opts.empty()) {
|
|
|
|
s = GetColumnFamilyOptionsFromMap(config_options, base_options,
|
|
|
|
unused_opts, &new_cf_options);
|
|
|
|
if (s.ok()) {
|
|
|
|
*new_options = Options(*new_db_options, new_cf_options);
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
*new_options = Options(*new_db_options, base_options);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Translate any errors (NotFound, NotSupported, to InvalidArgument
|
|
|
|
if (s.ok() || s.IsInvalidArgument()) {
|
|
|
|
return s;
|
|
|
|
} else {
|
|
|
|
return Status::InvalidArgument(s.getState());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
std::unordered_map<std::string, EncodingType>
|
|
|
|
OptionsHelper::encoding_type_string_map = {{"kPlain", kPlain},
|
|
|
|
{"kPrefix", kPrefix}};
|
|
|
|
|
|
|
|
std::unordered_map<std::string, CompactionStyle>
|
|
|
|
OptionsHelper::compaction_style_string_map = {
|
|
|
|
{"kCompactionStyleLevel", kCompactionStyleLevel},
|
|
|
|
{"kCompactionStyleUniversal", kCompactionStyleUniversal},
|
|
|
|
{"kCompactionStyleFIFO", kCompactionStyleFIFO},
|
|
|
|
{"kCompactionStyleNone", kCompactionStyleNone}};
|
|
|
|
|
|
|
|
std::unordered_map<std::string, CompactionPri>
|
|
|
|
OptionsHelper::compaction_pri_string_map = {
|
|
|
|
{"kByCompensatedSize", kByCompensatedSize},
|
|
|
|
{"kOldestLargestSeqFirst", kOldestLargestSeqFirst},
|
|
|
|
{"kOldestSmallestSeqFirst", kOldestSmallestSeqFirst},
|
|
|
|
{"kMinOverlappingRatio", kMinOverlappingRatio}};
|
|
|
|
|
|
|
|
std::unordered_map<std::string, CompactionStopStyle>
|
|
|
|
OptionsHelper::compaction_stop_style_string_map = {
|
|
|
|
{"kCompactionStopStyleSimilarSize", kCompactionStopStyleSimilarSize},
|
|
|
|
{"kCompactionStopStyleTotalSize", kCompactionStopStyleTotalSize}};
|
|
|
|
|
|
|
|
std::unordered_map<std::string, Temperature>
|
|
|
|
OptionsHelper::temperature_string_map = {
|
|
|
|
{"kUnknown", Temperature::kUnknown},
|
|
|
|
{"kHot", Temperature::kHot},
|
|
|
|
{"kWarm", Temperature::kWarm},
|
|
|
|
{"kCold", Temperature::kCold}};
|
|
|
|
|
|
|
|
Status OptionTypeInfo::NextToken(const std::string& opts, char delimiter,
|
|
|
|
size_t pos, size_t* end, std::string* token) {
|
|
|
|
while (pos < opts.size() && isspace(opts[pos])) {
|
|
|
|
++pos;
|
|
|
|
}
|
|
|
|
// Empty value at the end
|
|
|
|
if (pos >= opts.size()) {
|
|
|
|
*token = "";
|
|
|
|
*end = std::string::npos;
|
|
|
|
return Status::OK();
|
|
|
|
} else if (opts[pos] == '{') {
|
|
|
|
int count = 1;
|
|
|
|
size_t brace_pos = pos + 1;
|
|
|
|
while (brace_pos < opts.size()) {
|
|
|
|
if (opts[brace_pos] == '{') {
|
|
|
|
++count;
|
|
|
|
} else if (opts[brace_pos] == '}') {
|
|
|
|
--count;
|
|
|
|
if (count == 0) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
++brace_pos;
|
|
|
|
}
|
|
|
|
// found the matching closing brace
|
|
|
|
if (count == 0) {
|
|
|
|
*token = trim(opts.substr(pos + 1, brace_pos - pos - 1));
|
|
|
|
// skip all whitespace and move to the next delimiter
|
|
|
|
// brace_pos points to the next position after the matching '}'
|
|
|
|
pos = brace_pos + 1;
|
|
|
|
while (pos < opts.size() && isspace(opts[pos])) {
|
|
|
|
++pos;
|
|
|
|
}
|
|
|
|
if (pos < opts.size() && opts[pos] != delimiter) {
|
|
|
|
return Status::InvalidArgument("Unexpected chars after nested options");
|
|
|
|
}
|
|
|
|
*end = pos;
|
|
|
|
} else {
|
|
|
|
return Status::InvalidArgument(
|
|
|
|
"Mismatched curly braces for nested options");
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
*end = opts.find(delimiter, pos);
|
|
|
|
if (*end == std::string::npos) {
|
|
|
|
// It either ends with a trailing semi-colon or the last key-value pair
|
|
|
|
*token = trim(opts.substr(pos));
|
|
|
|
} else {
|
|
|
|
*token = trim(opts.substr(pos, *end - pos));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
Status OptionTypeInfo::Parse(const ConfigOptions& config_options,
|
|
|
|
const std::string& opt_name,
|
|
|
|
const std::string& value, void* opt_ptr) const {
|
|
|
|
if (IsDeprecated()) {
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
try {
|
|
|
|
void* opt_addr = static_cast<char*>(opt_ptr) + offset_;
|
|
|
|
const std::string& opt_value = config_options.input_strings_escaped
|
|
|
|
? UnescapeOptionString(value)
|
|
|
|
: value;
|
|
|
|
|
|
|
|
if (opt_addr == nullptr) {
|
|
|
|
return Status::NotFound("Could not find option", opt_name);
|
|
|
|
} else if (parse_func_ != nullptr) {
|
|
|
|
ConfigOptions copy = config_options;
|
|
|
|
copy.invoke_prepare_options = false;
|
|
|
|
return parse_func_(copy, opt_name, opt_value, opt_addr);
|
|
|
|
} else if (ParseOptionHelper(opt_addr, type_, opt_value)) {
|
|
|
|
return Status::OK();
|
|
|
|
} else if (IsConfigurable()) {
|
|
|
|
// The option is <config>.<name>
|
|
|
|
Configurable* config = AsRawPointer<Configurable>(opt_ptr);
|
|
|
|
if (opt_value.empty()) {
|
|
|
|
return Status::OK();
|
|
|
|
} else if (config == nullptr) {
|
|
|
|
return Status::NotFound("Could not find configurable: ", opt_name);
|
|
|
|
} else {
|
|
|
|
ConfigOptions copy = config_options;
|
|
|
|
copy.ignore_unknown_options = false;
|
|
|
|
copy.invoke_prepare_options = false;
|
|
|
|
if (opt_value.find("=") != std::string::npos) {
|
|
|
|
return config->ConfigureFromString(copy, opt_value);
|
|
|
|
} else {
|
|
|
|
return config->ConfigureOption(copy, opt_name, opt_value);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else if (IsByName()) {
|
|
|
|
return Status::NotSupported("Deserializing the option " + opt_name +
|
|
|
|
" is not supported");
|
|
|
|
} else {
|
|
|
|
return Status::InvalidArgument("Error parsing:", opt_name);
|
|
|
|
}
|
|
|
|
} catch (std::exception& e) {
|
|
|
|
return Status::InvalidArgument("Error parsing " + opt_name + ":" +
|
|
|
|
std::string(e.what()));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Status OptionTypeInfo::ParseType(
|
|
|
|
const ConfigOptions& config_options, const std::string& opts_str,
|
|
|
|
const std::unordered_map<std::string, OptionTypeInfo>& type_map,
|
|
|
|
void* opt_addr, std::unordered_map<std::string, std::string>* unused) {
|
|
|
|
std::unordered_map<std::string, std::string> opts_map;
|
|
|
|
Status status = StringToMap(opts_str, &opts_map);
|
|
|
|
if (!status.ok()) {
|
|
|
|
return status;
|
|
|
|
} else {
|
|
|
|
return ParseType(config_options, opts_map, type_map, opt_addr, unused);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Status OptionTypeInfo::ParseType(
|
|
|
|
const ConfigOptions& config_options,
|
|
|
|
const std::unordered_map<std::string, std::string>& opts_map,
|
|
|
|
const std::unordered_map<std::string, OptionTypeInfo>& type_map,
|
|
|
|
void* opt_addr, std::unordered_map<std::string, std::string>* unused) {
|
|
|
|
for (const auto& opts_iter : opts_map) {
|
|
|
|
std::string opt_name;
|
|
|
|
const auto* opt_info = Find(opts_iter.first, type_map, &opt_name);
|
|
|
|
if (opt_info != nullptr) {
|
|
|
|
Status status =
|
|
|
|
opt_info->Parse(config_options, opt_name, opts_iter.second, opt_addr);
|
|
|
|
if (!status.ok()) {
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
} else if (unused != nullptr) {
|
|
|
|
(*unused)[opts_iter.first] = opts_iter.second;
|
|
|
|
} else if (!config_options.ignore_unknown_options) {
|
|
|
|
return Status::NotFound("Unrecognized option", opts_iter.first);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
Status OptionTypeInfo::ParseStruct(
|
|
|
|
const ConfigOptions& config_options, const std::string& struct_name,
|
|
|
|
const std::unordered_map<std::string, OptionTypeInfo>* struct_map,
|
|
|
|
const std::string& opt_name, const std::string& opt_value, void* opt_addr) {
|
|
|
|
assert(struct_map);
|
|
|
|
Status status;
|
|
|
|
if (opt_name == struct_name || EndsWith(opt_name, "." + struct_name)) {
|
|
|
|
// This option represents the entire struct
|
|
|
|
std::unordered_map<std::string, std::string> unused;
|
|
|
|
status =
|
|
|
|
ParseType(config_options, opt_value, *struct_map, opt_addr, &unused);
|
|
|
|
if (status.ok() && !unused.empty()) {
|
|
|
|
status = Status::InvalidArgument(
|
|
|
|
"Unrecognized option", struct_name + "." + unused.begin()->first);
|
|
|
|
}
|
|
|
|
} else if (StartsWith(opt_name, struct_name + ".")) {
|
|
|
|
// This option represents a nested field in the struct (e.g, struct.field)
|
|
|
|
std::string elem_name;
|
|
|
|
const auto opt_info =
|
|
|
|
Find(opt_name.substr(struct_name.size() + 1), *struct_map, &elem_name);
|
|
|
|
if (opt_info != nullptr) {
|
|
|
|
status = opt_info->Parse(config_options, elem_name, opt_value, opt_addr);
|
|
|
|
} else {
|
|
|
|
status = Status::InvalidArgument("Unrecognized option", opt_name);
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// This option represents a field in the struct (e.g. field)
|
|
|
|
std::string elem_name;
|
|
|
|
const auto opt_info = Find(opt_name, *struct_map, &elem_name);
|
|
|
|
if (opt_info != nullptr) {
|
|
|
|
status = opt_info->Parse(config_options, elem_name, opt_value, opt_addr);
|
|
|
|
} else {
|
|
|
|
status = Status::InvalidArgument("Unrecognized option",
|
|
|
|
struct_name + "." + opt_name);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
|
|
|
|
Status OptionTypeInfo::Serialize(const ConfigOptions& config_options,
|
|
|
|
const std::string& opt_name,
|
|
|
|
const void* const opt_ptr,
|
|
|
|
std::string* opt_value) const {
|
|
|
|
// If the option is no longer used in rocksdb and marked as deprecated,
|
|
|
|
// we skip it in the serialization.
|
|
|
|
const void* opt_addr = static_cast<const char*>(opt_ptr) + offset_;
|
|
|
|
if (opt_addr == nullptr || IsDeprecated()) {
|
|
|
|
return Status::OK();
|
|
|
|
} else if (IsEnabled(OptionTypeFlags::kDontSerialize)) {
|
|
|
|
return Status::NotSupported("Cannot serialize option: ", opt_name);
|
|
|
|
} else if (serialize_func_ != nullptr) {
|
|
|
|
return serialize_func_(config_options, opt_name, opt_addr, opt_value);
|
|
|
|
} else if (IsCustomizable()) {
|
|
|
|
const Customizable* custom = AsRawPointer<Customizable>(opt_ptr);
|
|
|
|
opt_value->clear();
|
|
|
|
if (custom == nullptr) {
|
|
|
|
// We do not have a custom object to serialize.
|
|
|
|
// If the option is not mutable and we are doing only mutable options,
|
|
|
|
// we return an empty string (which will cause the option not to be
|
|
|
|
// printed). Otherwise, we return the "nullptr" string, which will result
|
|
|
|
// in "option=nullptr" being printed.
|
|
|
|
if (IsMutable() || !config_options.mutable_options_only) {
|
|
|
|
*opt_value = kNullptrString;
|
|
|
|
} else {
|
|
|
|
*opt_value = "";
|
|
|
|
}
|
|
|
|
} else if (IsEnabled(OptionTypeFlags::kStringNameOnly) &&
|
|
|
|
!config_options.IsDetailed()) {
|
|
|
|
if (!config_options.mutable_options_only || IsMutable()) {
|
|
|
|
*opt_value = custom->GetId();
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
ConfigOptions embedded = config_options;
|
|
|
|
embedded.delimiter = ";";
|
|
|
|
// If this option is mutable, everything inside it should be considered
|
|
|
|
// mutable
|
|
|
|
if (IsMutable()) {
|
|
|
|
embedded.mutable_options_only = false;
|
|
|
|
}
|
|
|
|
std::string value = custom->ToString(embedded);
|
|
|
|
if (!embedded.mutable_options_only ||
|
|
|
|
value.find("=") != std::string::npos) {
|
|
|
|
*opt_value = value;
|
|
|
|
} else {
|
|
|
|
*opt_value = "";
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return Status::OK();
|
|
|
|
} else if (IsConfigurable()) {
|
|
|
|
const Configurable* config = AsRawPointer<Configurable>(opt_ptr);
|
|
|
|
if (config != nullptr) {
|
|
|
|
ConfigOptions embedded = config_options;
|
|
|
|
embedded.delimiter = ";";
|
|
|
|
*opt_value = config->ToString(embedded);
|
|
|
|
}
|
|
|
|
return Status::OK();
|
|
|
|
} else if (config_options.mutable_options_only && !IsMutable()) {
|
|
|
|
return Status::OK();
|
|
|
|
} else if (SerializeSingleOptionHelper(opt_addr, type_, opt_value)) {
|
|
|
|
return Status::OK();
|
|
|
|
} else {
|
|
|
|
return Status::InvalidArgument("Cannot serialize option: ", opt_name);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Status OptionTypeInfo::SerializeType(
|
|
|
|
const ConfigOptions& config_options,
|
|
|
|
const std::unordered_map<std::string, OptionTypeInfo>& type_map,
|
|
|
|
const void* opt_addr, std::string* result) {
|
|
|
|
Status status;
|
|
|
|
for (const auto& iter : type_map) {
|
|
|
|
std::string single;
|
|
|
|
const auto& opt_info = iter.second;
|
|
|
|
if (opt_info.ShouldSerialize()) {
|
|
|
|
status =
|
|
|
|
opt_info.Serialize(config_options, iter.first, opt_addr, &single);
|
|
|
|
if (!status.ok()) {
|
|
|
|
return status;
|
|
|
|
} else {
|
|
|
|
result->append(iter.first + "=" + single + config_options.delimiter);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
|
|
|
|
Status OptionTypeInfo::SerializeStruct(
|
|
|
|
const ConfigOptions& config_options, const std::string& struct_name,
|
|
|
|
const std::unordered_map<std::string, OptionTypeInfo>* struct_map,
|
|
|
|
const std::string& opt_name, const void* opt_addr, std::string* value) {
|
|
|
|
assert(struct_map);
|
|
|
|
Status status;
|
|
|
|
if (EndsWith(opt_name, struct_name)) {
|
|
|
|
// We are going to write the struct as "{ prop1=value1; prop2=value2;}.
|
|
|
|
// Set the delimiter to ";" so that the everything will be on one line.
|
|
|
|
ConfigOptions embedded = config_options;
|
|
|
|
embedded.delimiter = ";";
|
|
|
|
|
|
|
|
// This option represents the entire struct
|
|
|
|
std::string result;
|
|
|
|
status = SerializeType(embedded, *struct_map, opt_addr, &result);
|
|
|
|
if (!status.ok()) {
|
|
|
|
return status;
|
|
|
|
} else {
|
|
|
|
*value = "{" + result + "}";
|
|
|
|
}
|
|
|
|
} else if (StartsWith(opt_name, struct_name + ".")) {
|
|
|
|
// This option represents a nested field in the struct (e.g, struct.field)
|
|
|
|
std::string elem_name;
|
|
|
|
const auto opt_info =
|
|
|
|
Find(opt_name.substr(struct_name.size() + 1), *struct_map, &elem_name);
|
|
|
|
if (opt_info != nullptr) {
|
|
|
|
status = opt_info->Serialize(config_options, elem_name, opt_addr, value);
|
|
|
|
} else {
|
|
|
|
status = Status::InvalidArgument("Unrecognized option", opt_name);
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// This option represents a field in the struct (e.g. field)
|
|
|
|
std::string elem_name;
|
|
|
|
const auto opt_info = Find(opt_name, *struct_map, &elem_name);
|
|
|
|
if (opt_info == nullptr) {
|
|
|
|
status = Status::InvalidArgument("Unrecognized option", opt_name);
|
|
|
|
} else if (opt_info->ShouldSerialize()) {
|
|
|
|
status = opt_info->Serialize(config_options, opt_name + "." + elem_name,
|
|
|
|
opt_addr, value);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
|
|
|
|
template <typename T>
|
|
|
|
bool IsOptionEqual(const void* offset1, const void* offset2) {
|
|
|
|
return (*static_cast<const T*>(offset1) == *static_cast<const T*>(offset2));
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool AreEqualDoubles(const double a, const double b) {
|
|
|
|
return (fabs(a - b) < 0.00001);
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool AreOptionsEqual(OptionType type, const void* this_offset,
|
|
|
|
const void* that_offset) {
|
|
|
|
switch (type) {
|
|
|
|
case OptionType::kBoolean:
|
|
|
|
return IsOptionEqual<bool>(this_offset, that_offset);
|
|
|
|
case OptionType::kInt:
|
|
|
|
return IsOptionEqual<int>(this_offset, that_offset);
|
|
|
|
case OptionType::kUInt:
|
|
|
|
return IsOptionEqual<unsigned int>(this_offset, that_offset);
|
|
|
|
case OptionType::kInt32T:
|
|
|
|
return IsOptionEqual<int32_t>(this_offset, that_offset);
|
|
|
|
case OptionType::kInt64T: {
|
|
|
|
int64_t v1, v2;
|
|
|
|
GetUnaligned(static_cast<const int64_t*>(this_offset), &v1);
|
|
|
|
GetUnaligned(static_cast<const int64_t*>(that_offset), &v2);
|
|
|
|
return (v1 == v2);
|
|
|
|
}
|
|
|
|
case OptionType::kUInt8T:
|
|
|
|
return IsOptionEqual<uint8_t>(this_offset, that_offset);
|
|
|
|
case OptionType::kUInt32T:
|
|
|
|
return IsOptionEqual<uint32_t>(this_offset, that_offset);
|
|
|
|
case OptionType::kUInt64T: {
|
|
|
|
uint64_t v1, v2;
|
|
|
|
GetUnaligned(static_cast<const uint64_t*>(this_offset), &v1);
|
|
|
|
GetUnaligned(static_cast<const uint64_t*>(that_offset), &v2);
|
|
|
|
return (v1 == v2);
|
|
|
|
}
|
|
|
|
case OptionType::kSizeT: {
|
|
|
|
size_t v1, v2;
|
|
|
|
GetUnaligned(static_cast<const size_t*>(this_offset), &v1);
|
|
|
|
GetUnaligned(static_cast<const size_t*>(that_offset), &v2);
|
|
|
|
return (v1 == v2);
|
|
|
|
}
|
|
|
|
case OptionType::kString:
|
|
|
|
return IsOptionEqual<std::string>(this_offset, that_offset);
|
|
|
|
case OptionType::kDouble:
|
|
|
|
return AreEqualDoubles(*static_cast<const double*>(this_offset),
|
|
|
|
*static_cast<const double*>(that_offset));
|
|
|
|
case OptionType::kCompactionStyle:
|
|
|
|
return IsOptionEqual<CompactionStyle>(this_offset, that_offset);
|
|
|
|
case OptionType::kCompactionStopStyle:
|
|
|
|
return IsOptionEqual<CompactionStopStyle>(this_offset, that_offset);
|
|
|
|
case OptionType::kCompactionPri:
|
|
|
|
return IsOptionEqual<CompactionPri>(this_offset, that_offset);
|
|
|
|
case OptionType::kCompressionType:
|
|
|
|
return IsOptionEqual<CompressionType>(this_offset, that_offset);
|
|
|
|
case OptionType::kChecksumType:
|
|
|
|
return IsOptionEqual<ChecksumType>(this_offset, that_offset);
|
|
|
|
case OptionType::kEncodingType:
|
|
|
|
return IsOptionEqual<EncodingType>(this_offset, that_offset);
|
|
|
|
case OptionType::kEncodedString:
|
|
|
|
return IsOptionEqual<std::string>(this_offset, that_offset);
|
|
|
|
case OptionType::kTemperature:
|
|
|
|
return IsOptionEqual<Temperature>(this_offset, that_offset);
|
|
|
|
default:
|
|
|
|
return false;
|
|
|
|
} // End switch
|
|
|
|
}
|
|
|
|
|
|
|
|
bool OptionTypeInfo::AreEqual(const ConfigOptions& config_options,
|
|
|
|
const std::string& opt_name,
|
|
|
|
const void* const this_ptr,
|
|
|
|
const void* const that_ptr,
|
|
|
|
std::string* mismatch) const {
|
|
|
|
auto level = GetSanityLevel();
|
|
|
|
if (!config_options.IsCheckEnabled(level)) {
|
|
|
|
return true; // If the sanity level is not being checked, skip it
|
|
|
|
}
|
|
|
|
const void* this_addr = static_cast<const char*>(this_ptr) + offset_;
|
|
|
|
const void* that_addr = static_cast<const char*>(that_ptr) + offset_;
|
|
|
|
if (this_addr == nullptr || that_addr == nullptr) {
|
|
|
|
if (this_addr == that_addr) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
} else if (equals_func_ != nullptr) {
|
|
|
|
if (equals_func_(config_options, opt_name, this_addr, that_addr,
|
|
|
|
mismatch)) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
} else if (AreOptionsEqual(type_, this_addr, that_addr)) {
|
|
|
|
return true;
|
|
|
|
} else if (IsConfigurable()) {
|
|
|
|
const auto* this_config = AsRawPointer<Configurable>(this_ptr);
|
|
|
|
const auto* that_config = AsRawPointer<Configurable>(that_ptr);
|
|
|
|
if (this_config == that_config) {
|
|
|
|
return true;
|
|
|
|
} else if (this_config != nullptr && that_config != nullptr) {
|
|
|
|
std::string bad_name;
|
|
|
|
bool matches;
|
|
|
|
if (level < config_options.sanity_level) {
|
|
|
|
ConfigOptions copy = config_options;
|
|
|
|
copy.sanity_level = level;
|
|
|
|
matches = this_config->AreEquivalent(copy, that_config, &bad_name);
|
|
|
|
} else {
|
|
|
|
matches =
|
|
|
|
this_config->AreEquivalent(config_options, that_config, &bad_name);
|
|
|
|
}
|
|
|
|
if (!matches) {
|
|
|
|
*mismatch = opt_name + "." + bad_name;
|
|
|
|
}
|
|
|
|
return matches;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (mismatch->empty()) {
|
|
|
|
*mismatch = opt_name;
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool OptionTypeInfo::TypesAreEqual(
|
|
|
|
const ConfigOptions& config_options,
|
|
|
|
const std::unordered_map<std::string, OptionTypeInfo>& type_map,
|
|
|
|
const void* this_addr, const void* that_addr, std::string* mismatch) {
|
|
|
|
for (const auto& iter : type_map) {
|
|
|
|
const auto& opt_info = iter.second;
|
|
|
|
if (!opt_info.AreEqual(config_options, iter.first, this_addr, that_addr,
|
|
|
|
mismatch)) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool OptionTypeInfo::StructsAreEqual(
|
|
|
|
const ConfigOptions& config_options, const std::string& struct_name,
|
|
|
|
const std::unordered_map<std::string, OptionTypeInfo>* struct_map,
|
|
|
|
const std::string& opt_name, const void* this_addr, const void* that_addr,
|
|
|
|
std::string* mismatch) {
|
|
|
|
assert(struct_map);
|
|
|
|
bool matches = true;
|
|
|
|
std::string result;
|
|
|
|
if (EndsWith(opt_name, struct_name)) {
|
|
|
|
// This option represents the entire struct
|
|
|
|
matches = TypesAreEqual(config_options, *struct_map, this_addr, that_addr,
|
|
|
|
&result);
|
|
|
|
if (!matches) {
|
|
|
|
*mismatch = struct_name + "." + result;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
} else if (StartsWith(opt_name, struct_name + ".")) {
|
|
|
|
// This option represents a nested field in the struct (e.g, struct.field)
|
|
|
|
std::string elem_name;
|
|
|
|
const auto opt_info =
|
|
|
|
Find(opt_name.substr(struct_name.size() + 1), *struct_map, &elem_name);
|
|
|
|
assert(opt_info);
|
|
|
|
if (opt_info == nullptr) {
|
|
|
|
*mismatch = opt_name;
|
|
|
|
matches = false;
|
|
|
|
} else if (!opt_info->AreEqual(config_options, elem_name, this_addr,
|
|
|
|
that_addr, &result)) {
|
|
|
|
matches = false;
|
|
|
|
*mismatch = struct_name + "." + result;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// This option represents a field in the struct (e.g. field)
|
|
|
|
std::string elem_name;
|
|
|
|
const auto opt_info = Find(opt_name, *struct_map, &elem_name);
|
|
|
|
assert(opt_info);
|
|
|
|
if (opt_info == nullptr) {
|
|
|
|
*mismatch = struct_name + "." + opt_name;
|
|
|
|
matches = false;
|
|
|
|
} else if (!opt_info->AreEqual(config_options, elem_name, this_addr,
|
|
|
|
that_addr, &result)) {
|
|
|
|
matches = false;
|
|
|
|
*mismatch = struct_name + "." + result;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return matches;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool MatchesOptionsTypeFromMap(
|
|
|
|
const ConfigOptions& config_options,
|
|
|
|
const std::unordered_map<std::string, OptionTypeInfo>& type_map,
|
|
|
|
const void* const this_ptr, const void* const that_ptr,
|
|
|
|
std::string* mismatch) {
|
|
|
|
for (auto& pair : type_map) {
|
|
|
|
// We skip checking deprecated variables as they might
|
|
|
|
// contain random values since they might not be initialized
|
|
|
|
if (config_options.IsCheckEnabled(pair.second.GetSanityLevel())) {
|
|
|
|
if (!pair.second.AreEqual(config_options, pair.first, this_ptr, that_ptr,
|
|
|
|
mismatch) &&
|
|
|
|
!pair.second.AreEqualByName(config_options, pair.first, this_ptr,
|
|
|
|
that_ptr)) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool OptionTypeInfo::AreEqualByName(const ConfigOptions& config_options,
|
|
|
|
const std::string& opt_name,
|
|
|
|
const void* const this_ptr,
|
|
|
|
const void* const that_ptr) const {
|
|
|
|
if (IsByName()) {
|
|
|
|
std::string that_value;
|
|
|
|
if (Serialize(config_options, opt_name, that_ptr, &that_value).ok()) {
|
|
|
|
return AreEqualByName(config_options, opt_name, this_ptr, that_value);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool OptionTypeInfo::AreEqualByName(const ConfigOptions& config_options,
|
|
|
|
const std::string& opt_name,
|
|
|
|
const void* const opt_ptr,
|
|
|
|
const std::string& that_value) const {
|
|
|
|
std::string this_value;
|
|
|
|
if (!IsByName()) {
|
|
|
|
return false;
|
|
|
|
} else if (!Serialize(config_options, opt_name, opt_ptr, &this_value).ok()) {
|
|
|
|
return false;
|
|
|
|
} else if (IsEnabled(OptionVerificationType::kByNameAllowFromNull)) {
|
|
|
|
if (that_value == kNullptrString) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
} else if (IsEnabled(OptionVerificationType::kByNameAllowNull)) {
|
|
|
|
if (that_value == kNullptrString) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return (this_value == that_value);
|
|
|
|
}
|
|
|
|
|
|
|
|
const OptionTypeInfo* OptionTypeInfo::Find(
|
|
|
|
const std::string& opt_name,
|
|
|
|
const std::unordered_map<std::string, OptionTypeInfo>& opt_map,
|
|
|
|
std::string* elem_name) {
|
|
|
|
const auto iter = opt_map.find(opt_name); // Look up the value in the map
|
|
|
|
if (iter != opt_map.end()) { // Found the option in the map
|
|
|
|
*elem_name = opt_name; // Return the name
|
|
|
|
return &(iter->second); // Return the contents of the iterator
|
|
|
|
} else {
|
|
|
|
auto idx = opt_name.find("."); // Look for a separator
|
|
|
|
if (idx > 0 && idx != std::string::npos) { // We found a separator
|
|
|
|
auto siter =
|
|
|
|
opt_map.find(opt_name.substr(0, idx)); // Look for the short name
|
|
|
|
if (siter != opt_map.end()) { // We found the short name
|
|
|
|
if (siter->second.IsStruct() || // If the object is a struct
|
|
|
|
siter->second.IsConfigurable()) { // or a Configurable
|
|
|
|
*elem_name = opt_name.substr(idx + 1); // Return the rest
|
|
|
|
return &(siter->second); // Return the contents of the iterator
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nullptr;
|
|
|
|
}
|
RocksDB Options file format and its serialization / deserialization.
Summary:
This patch defines the format of RocksDB options file, which
follows the INI file format, and implements functions for its
serialization and deserialization. An example RocksDB options
file can be found in examples/rocksdb_option_file_example.ini.
A typical RocksDB options file has three sections, which are
Version, DBOptions, and more than one CFOptions. The RocksDB
options file in general follows the basic INI file format
with the following extensions / modifications:
* Escaped characters
We escaped the following characters:
- \n -- line feed - new line
- \r -- carriage return
- \\ -- backslash \
- \: -- colon symbol :
- \# -- hash tag #
* Comments
We support # style comments. Comments can appear at the ending
part of a line.
* Statements
A statement is of the form option_name = value.
Each statement contains a '=', where extra white-spaces
are supported. However, we don't support multi-lined statement.
Furthermore, each line can only contain at most one statement.
* Section
Sections are of the form [SecitonTitle "SectionArgument"],
where section argument is optional.
* List
We use colon-separated string to represent a list.
For instance, n1:n2:n3:n4 is a list containing four values.
Below is an example of a RocksDB options file:
[Version]
rocksdb_version=4.0.0
options_file_version=1.0
[DBOptions]
max_open_files=12345
max_background_flushes=301
[CFOptions "default"]
[CFOptions "the second column family"]
[CFOptions "the third column family"]
Test Plan: Added many tests in options_test.cc
Reviewers: igor, IslamAbdelRahman, sdong, anthony
Reviewed By: anthony
Subscribers: maykov, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D46059
9 years ago
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
|
|
|
|
} // namespace ROCKSDB_NAMESPACE
|