|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
|
|
//
|
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
//
|
|
|
|
|
Support using ZDICT_finalizeDictionary to generate zstd dictionary (#9857)
Summary:
An untrained dictionary is currently simply the concatenation of several samples. The ZSTD API, ZDICT_finalizeDictionary(), can improve such a dictionary's effectiveness at low cost. This PR changes how dictionary is created by calling the ZSTD ZDICT_finalizeDictionary() API instead of creating raw content dictionary (when max_dict_buffer_bytes > 0), and pass in all buffered uncompressed data blocks as samples.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9857
Test Plan:
#### db_bench test for cpu/memory of compression+decompression and space saving on synthetic data:
Set up: change the parameter [here](https://github.com/facebook/rocksdb/blob/fb9a167a55e0970b1ef6f67c1600c8d9c4c6114f/tools/db_bench_tool.cc#L1766) to 16384 to make synthetic data more compressible.
```
# linked local ZSTD with version 1.5.2
# DEBUG_LEVEL=0 ROCKSDB_NO_FBCODE=1 ROCKSDB_DISABLE_ZSTD=1 EXTRA_CXXFLAGS="-DZSTD_STATIC_LINKING_ONLY -DZSTD -I/data/users/changyubi/install/include/" EXTRA_LDFLAGS="-L/data/users/changyubi/install/lib/ -l:libzstd.a" make -j32 db_bench
dict_bytes=16384
train_bytes=1048576
echo "========== No Dictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=0 -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=0 -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total
echo "========== Raw Content Dictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench_main -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench_main -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total
echo "========== FinalizeDictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total
echo "========== TrainDictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total
# Result: TrainDictionary is much better on space saving, but FinalizeDictionary seems to use less memory.
# before compression data size: 1.2GB
dict_bytes=16384
max_dict_buffer_bytes = 1048576
space cpu/memory
No Dictionary 468M 14.93user 1.00system 0:15.92elapsed 100%CPU (0avgtext+0avgdata 23904maxresident)k
Raw Dictionary 251M 15.81user 0.80system 0:16.56elapsed 100%CPU (0avgtext+0avgdata 156808maxresident)k
FinalizeDictionary 236M 11.93user 0.64system 0:12.56elapsed 100%CPU (0avgtext+0avgdata 89548maxresident)k
TrainDictionary 84M 7.29user 0.45system 0:07.75elapsed 100%CPU (0avgtext+0avgdata 97288maxresident)k
```
#### Benchmark on 10 sample SST files for spacing saving and CPU time on compression:
FinalizeDictionary is comparable to TrainDictionary in terms of space saving, and takes less time in compression.
```
dict_bytes=16384
train_bytes=1048576
for sst_file in `ls ../temp/myrock-sst/`
do
echo "********** $sst_file **********"
echo "========== No Dictionary =========="
./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD
echo "========== Raw Content Dictionary =========="
./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD --compression_max_dict_bytes=$dict_bytes
echo "========== FinalizeDictionary =========="
./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD --compression_max_dict_bytes=$dict_bytes --compression_zstd_max_train_bytes=$train_bytes --compression_use_zstd_finalize_dict
echo "========== TrainDictionary =========="
./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD --compression_max_dict_bytes=$dict_bytes --compression_zstd_max_train_bytes=$train_bytes
done
010240.sst (Size/Time) 011029.sst 013184.sst 021552.sst 185054.sst 185137.sst 191666.sst 7560381.sst 7604174.sst 7635312.sst
No Dictionary 28165569 / 2614419 32899411 / 2976832 32977848 / 3055542 31966329 / 2004590 33614351 / 1755877 33429029 / 1717042 33611933 / 1776936 33634045 / 2771417 33789721 / 2205414 33592194 / 388254
Raw Content Dictionary 28019950 / 2697961 33748665 / 3572422 33896373 / 3534701 26418431 / 2259658 28560825 / 1839168 28455030 / 1846039 28494319 / 1861349 32391599 / 3095649 33772142 / 2407843 33592230 / 474523
FinalizeDictionary 27896012 / 2650029 33763886 / 3719427 33904283 / 3552793 26008225 / 2198033 28111872 / 1869530 28014374 / 1789771 28047706 / 1848300 32296254 / 3204027 33698698 / 2381468 33592344 / 517433
TrainDictionary 28046089 / 2740037 33706480 / 3679019 33885741 / 3629351 25087123 / 2204558 27194353 / 1970207 27234229 / 1896811 27166710 / 1903119 32011041 / 3322315 32730692 / 2406146 33608631 / 570593
```
#### Decompression/Read test:
With FinalizeDictionary/TrainDictionary, some data structure used for decompression are in stored in dictionary, so they are expected to be faster in terms of decompression/reads.
```
dict_bytes=16384
train_bytes=1048576
echo "No Dictionary"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=0 > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=0 2>&1 | grep MB/s
echo "Raw Dictionary"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes 2>&1 | grep MB/s
echo "FinalizeDict"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false 2>&1 | grep MB/s
echo "Train Dictionary"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes 2>&1 | grep MB/s
No Dictionary
readrandom : 12.183 micros/op 82082 ops/sec 12.183 seconds 1000000 operations; 9.1 MB/s (1000000 of 1000000 found)
Raw Dictionary
readrandom : 12.314 micros/op 81205 ops/sec 12.314 seconds 1000000 operations; 9.0 MB/s (1000000 of 1000000 found)
FinalizeDict
readrandom : 9.787 micros/op 102180 ops/sec 9.787 seconds 1000000 operations; 11.3 MB/s (1000000 of 1000000 found)
Train Dictionary
readrandom : 9.698 micros/op 103108 ops/sec 9.699 seconds 1000000 operations; 11.4 MB/s (1000000 of 1000000 found)
```
Reviewed By: ajkr
Differential Revision: D35720026
Pulled By: cbi42
fbshipit-source-id: 24d230fdff0fd28a1bb650658798f00dfcfb2a1f
3 years ago
|
|
|
#include "util/compression.h"
|
|
|
|
#ifdef GFLAGS
|
|
|
|
#include "cache/fast_lru_cache.h"
|
|
|
|
#include "db_stress_tool/db_stress_common.h"
|
|
|
|
#include "db_stress_tool/db_stress_compaction_filter.h"
|
|
|
|
#include "db_stress_tool/db_stress_driver.h"
|
|
|
|
#include "db_stress_tool/db_stress_table_properties_collector.h"
|
|
|
|
#include "rocksdb/convenience.h"
|
|
|
|
#include "rocksdb/filter_policy.h"
|
|
|
|
#include "rocksdb/secondary_cache.h"
|
|
|
|
#include "rocksdb/sst_file_manager.h"
|
|
|
|
#include "rocksdb/types.h"
|
|
|
|
#include "rocksdb/utilities/object_registry.h"
|
|
|
|
#include "test_util/testutil.h"
|
|
|
|
#include "util/cast_util.h"
|
|
|
|
#include "utilities/backup/backup_engine_impl.h"
|
|
|
|
#include "utilities/fault_injection_fs.h"
|
|
|
|
#include "utilities/fault_injection_secondary_cache.h"
|
|
|
|
|
|
|
|
namespace ROCKSDB_NAMESPACE {
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
|
|
|
|
std::shared_ptr<const FilterPolicy> CreateFilterPolicy() {
|
|
|
|
if (FLAGS_bloom_bits < 0) {
|
|
|
|
return BlockBasedTableOptions().filter_policy;
|
|
|
|
}
|
|
|
|
const FilterPolicy* new_policy;
|
Add Bloom/Ribbon hybrid API support (#8679)
Summary:
This is essentially resurrection and fixing of the part of
https://github.com/facebook/rocksdb/issues/8198 that was reverted in https://github.com/facebook/rocksdb/issues/8212, using data added in https://github.com/facebook/rocksdb/issues/8246. Basically,
when configuring Ribbon filter, you can specify an LSM level before which
Bloom will be used instead of Ribbon. But Bloom is only considered for
Leveled and Universal compaction styles and file going into a known LSM
level. This way, SST file writer, FIFO compaction, etc. use Ribbon filter as
you would expect with NewRibbonFilterPolicy.
So that this can be controlled with a single int value and so that flushes
can be distinguished from intra-L0, we consider flush to go to level -1 for
the purposes of this option. (Explained in API comment.)
I also expect the most common and recommended Ribbon configuration to
use Bloom during flush, to minimize slowing down writes and because according
to my estimates, Ribbon only pays off if the structure lives in memory for
more than an hour. Thus, I have changed the default for NewRibbonFilterPolicy
to be this mild hybrid configuration. I don't really want to add something like
NewHybridFilterPolicy because at least the mild hybrid configuration (Bloom for
flush, Ribbon otherwise) should be considered a natural choice.
C APIs also updated, but because they don't support overloading,
rocksdb_filterpolicy_create_ribbon is kept pure ribbon for clarity and
rocksdb_filterpolicy_create_ribbon_hybrid must be called for a hybrid
configuration. While touching C API, I changed bits per key options from
int to double.
BuiltinFilterPolicy is needed so that LevelThresholdFilterPolicy doesn't inherit
unused fields from BloomFilterPolicy.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8679
Test Plan: new + updated tests, including crash test
Reviewed By: jay-zhuang
Differential Revision: D30445797
Pulled By: pdillinger
fbshipit-source-id: 6f5aeddfd6d79f7e55493b563c2d1d2d568892e1
3 years ago
|
|
|
if (FLAGS_use_block_based_filter) {
|
|
|
|
if (FLAGS_ribbon_starting_level < 999) {
|
|
|
|
fprintf(
|
|
|
|
stderr,
|
|
|
|
"Cannot combine use_block_based_filter and ribbon_starting_level\n");
|
|
|
|
exit(1);
|
|
|
|
} else {
|
|
|
|
new_policy = NewBloomFilterPolicy(FLAGS_bloom_bits, true);
|
|
|
|
}
|
Add Bloom/Ribbon hybrid API support (#8679)
Summary:
This is essentially resurrection and fixing of the part of
https://github.com/facebook/rocksdb/issues/8198 that was reverted in https://github.com/facebook/rocksdb/issues/8212, using data added in https://github.com/facebook/rocksdb/issues/8246. Basically,
when configuring Ribbon filter, you can specify an LSM level before which
Bloom will be used instead of Ribbon. But Bloom is only considered for
Leveled and Universal compaction styles and file going into a known LSM
level. This way, SST file writer, FIFO compaction, etc. use Ribbon filter as
you would expect with NewRibbonFilterPolicy.
So that this can be controlled with a single int value and so that flushes
can be distinguished from intra-L0, we consider flush to go to level -1 for
the purposes of this option. (Explained in API comment.)
I also expect the most common and recommended Ribbon configuration to
use Bloom during flush, to minimize slowing down writes and because according
to my estimates, Ribbon only pays off if the structure lives in memory for
more than an hour. Thus, I have changed the default for NewRibbonFilterPolicy
to be this mild hybrid configuration. I don't really want to add something like
NewHybridFilterPolicy because at least the mild hybrid configuration (Bloom for
flush, Ribbon otherwise) should be considered a natural choice.
C APIs also updated, but because they don't support overloading,
rocksdb_filterpolicy_create_ribbon is kept pure ribbon for clarity and
rocksdb_filterpolicy_create_ribbon_hybrid must be called for a hybrid
configuration. While touching C API, I changed bits per key options from
int to double.
BuiltinFilterPolicy is needed so that LevelThresholdFilterPolicy doesn't inherit
unused fields from BloomFilterPolicy.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8679
Test Plan: new + updated tests, including crash test
Reviewed By: jay-zhuang
Differential Revision: D30445797
Pulled By: pdillinger
fbshipit-source-id: 6f5aeddfd6d79f7e55493b563c2d1d2d568892e1
3 years ago
|
|
|
} else if (FLAGS_ribbon_starting_level >= 999) {
|
|
|
|
// Use Bloom API
|
|
|
|
new_policy = NewBloomFilterPolicy(FLAGS_bloom_bits, false);
|
|
|
|
} else {
|
|
|
|
new_policy = NewRibbonFilterPolicy(
|
|
|
|
FLAGS_bloom_bits, /* bloom_before_level */ FLAGS_ribbon_starting_level);
|
|
|
|
}
|
|
|
|
return std::shared_ptr<const FilterPolicy>(new_policy);
|
|
|
|
}
|
|
|
|
|
|
|
|
} // namespace
|
|
|
|
|
|
|
|
StressTest::StressTest()
|
|
|
|
: cache_(NewCache(FLAGS_cache_size, FLAGS_cache_numshardbits)),
|
Avoid overwriting options loaded from OPTIONS (#9943)
Summary:
This is similar to https://github.com/facebook/rocksdb/issues/9862, including the following fixes/refactoring:
1. If OPTIONS file is specified via `-options_file`, majority of options will be loaded from the file. We should not
overwrite options that have been loaded from the file. Instead, we configure only fields of options which are
shared objects and not set by the OPTIONS file. We also configure a few fields, e.g. `create_if_missing` necessary
for stress test to run.
2. Refactor options initialization into three functions, `InitializeOptionsFromFile()`, `InitializeOptionsFromFlags()`
and `InitializeOptionsGeneral()` similar to db_bench. I hope they can be shared in the future. The high-level logic is
as follows:
```cpp
if (!InitializeOptionsFromFile(...)) {
InitializeOptionsFromFlags(...);
}
InitializeOptionsGeneral(...);
```
3. Currently, the setting for `block_cache_compressed` does not seem correct because it by default specifies a
size of `numeric_limits<size_t>::max()` ((size_t)-1). According to code comments, `-1` indicates default value,
which should be referring to `num_shard_bits` argument.
4. Clarify `fail_if_options_file_error`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9943
Test Plan:
1. make check
2. Run stress tests, and manually check generated OPTIONS file and compare them with input OPTIONS files
Reviewed By: jay-zhuang
Differential Revision: D36133769
Pulled By: riversand963
fbshipit-source-id: 35dacdc090a0a72c922907170cd132b9ecaa073e
3 years ago
|
|
|
compressed_cache_(NewLRUCache(FLAGS_compressed_cache_size,
|
|
|
|
FLAGS_compressed_cache_numshardbits)),
|
|
|
|
filter_policy_(CreateFilterPolicy()),
|
|
|
|
db_(nullptr),
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
txn_db_(nullptr),
|
|
|
|
#endif
|
|
|
|
db_aptr_(nullptr),
|
|
|
|
clock_(db_stress_env->GetSystemClock().get()),
|
|
|
|
new_column_family_name_(1),
|
|
|
|
num_times_reopened_(0),
|
|
|
|
db_preload_finished_(false),
|
|
|
|
cmp_db_(nullptr),
|
|
|
|
is_db_stopped_(false) {
|
|
|
|
if (FLAGS_destroy_db_initially) {
|
|
|
|
std::vector<std::string> files;
|
|
|
|
db_stress_env->GetChildren(FLAGS_db, &files);
|
|
|
|
for (unsigned int i = 0; i < files.size(); i++) {
|
|
|
|
if (Slice(files[i]).starts_with("heap-")) {
|
|
|
|
db_stress_env->DeleteFile(FLAGS_db + "/" + files[i]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Options options;
|
|
|
|
options.env = db_stress_env;
|
|
|
|
// Remove files without preserving manfiest files
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
const Status s = !FLAGS_use_blob_db
|
|
|
|
? DestroyDB(FLAGS_db, options)
|
|
|
|
: blob_db::DestroyBlobDB(FLAGS_db, options,
|
|
|
|
blob_db::BlobDBOptions());
|
|
|
|
#else
|
|
|
|
const Status s = DestroyDB(FLAGS_db, options);
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
|
|
|
|
if (!s.ok()) {
|
|
|
|
fprintf(stderr, "Cannot destroy original db: %s\n", s.ToString().c_str());
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
StressTest::~StressTest() {
|
|
|
|
for (auto cf : column_families_) {
|
|
|
|
delete cf;
|
|
|
|
}
|
|
|
|
column_families_.clear();
|
|
|
|
delete db_;
|
|
|
|
|
|
|
|
for (auto* cf : cmp_cfhs_) {
|
|
|
|
delete cf;
|
|
|
|
}
|
|
|
|
cmp_cfhs_.clear();
|
|
|
|
delete cmp_db_;
|
|
|
|
}
|
|
|
|
|
|
|
|
std::shared_ptr<Cache> StressTest::NewCache(size_t capacity,
|
|
|
|
int32_t num_shard_bits) {
|
|
|
|
ConfigOptions config_options;
|
|
|
|
if (capacity <= 0) {
|
|
|
|
return nullptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (FLAGS_cache_type == "clock_cache") {
|
|
|
|
auto cache = NewClockCache((size_t)capacity);
|
|
|
|
if (!cache) {
|
|
|
|
fprintf(stderr, "Clock cache not supported.");
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
return cache;
|
|
|
|
} else if (FLAGS_cache_type == "fast_lru_cache") {
|
|
|
|
return NewFastLRUCache(static_cast<size_t>(capacity), FLAGS_block_size,
|
|
|
|
num_shard_bits, false /*strict_capacity_limit*/,
|
|
|
|
kDefaultCacheMetadataChargePolicy);
|
|
|
|
} else if (FLAGS_cache_type == "lru_cache") {
|
|
|
|
LRUCacheOptions opts;
|
|
|
|
opts.capacity = capacity;
|
|
|
|
opts.num_shard_bits = num_shard_bits;
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
std::shared_ptr<SecondaryCache> secondary_cache;
|
|
|
|
if (!FLAGS_secondary_cache_uri.empty()) {
|
|
|
|
Status s = SecondaryCache::CreateFromString(
|
|
|
|
config_options, FLAGS_secondary_cache_uri, &secondary_cache);
|
|
|
|
if (secondary_cache == nullptr) {
|
|
|
|
fprintf(stderr,
|
|
|
|
"No secondary cache registered matching string: %s status=%s\n",
|
|
|
|
FLAGS_secondary_cache_uri.c_str(), s.ToString().c_str());
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
if (FLAGS_secondary_cache_fault_one_in > 0) {
|
|
|
|
secondary_cache = std::make_shared<FaultInjectionSecondaryCache>(
|
|
|
|
secondary_cache, static_cast<uint32_t>(FLAGS_seed),
|
|
|
|
FLAGS_secondary_cache_fault_one_in);
|
|
|
|
}
|
|
|
|
opts.secondary_cache = secondary_cache;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
return NewLRUCache(opts);
|
|
|
|
} else {
|
|
|
|
fprintf(stderr, "Cache type not supported.");
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
std::vector<std::string> StressTest::GetBlobCompressionTags() {
|
|
|
|
std::vector<std::string> compression_tags{"kNoCompression"};
|
|
|
|
|
|
|
|
if (Snappy_Supported()) {
|
|
|
|
compression_tags.emplace_back("kSnappyCompression");
|
|
|
|
}
|
|
|
|
if (LZ4_Supported()) {
|
|
|
|
compression_tags.emplace_back("kLZ4Compression");
|
|
|
|
}
|
|
|
|
if (ZSTD_Supported()) {
|
|
|
|
compression_tags.emplace_back("kZSTD");
|
|
|
|
}
|
|
|
|
|
|
|
|
return compression_tags;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool StressTest::BuildOptionsTable() {
|
|
|
|
if (FLAGS_set_options_one_in <= 0) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
std::unordered_map<std::string, std::vector<std::string>> options_tbl = {
|
|
|
|
{"write_buffer_size",
|
|
|
|
{std::to_string(options_.write_buffer_size),
|
|
|
|
std::to_string(options_.write_buffer_size * 2),
|
|
|
|
std::to_string(options_.write_buffer_size * 4)}},
|
|
|
|
{"max_write_buffer_number",
|
|
|
|
{std::to_string(options_.max_write_buffer_number),
|
|
|
|
std::to_string(options_.max_write_buffer_number * 2),
|
|
|
|
std::to_string(options_.max_write_buffer_number * 4)}},
|
|
|
|
{"arena_block_size",
|
|
|
|
{
|
|
|
|
std::to_string(options_.arena_block_size),
|
|
|
|
std::to_string(options_.write_buffer_size / 4),
|
|
|
|
std::to_string(options_.write_buffer_size / 8),
|
|
|
|
}},
|
|
|
|
{"memtable_huge_page_size", {"0", std::to_string(2 * 1024 * 1024)}},
|
|
|
|
{"max_successive_merges", {"0", "2", "4"}},
|
|
|
|
{"inplace_update_num_locks", {"100", "200", "300"}},
|
|
|
|
// TODO(ljin): enable test for this option
|
|
|
|
// {"disable_auto_compactions", {"100", "200", "300"}},
|
|
|
|
{"level0_file_num_compaction_trigger",
|
|
|
|
{
|
|
|
|
std::to_string(options_.level0_file_num_compaction_trigger),
|
|
|
|
std::to_string(options_.level0_file_num_compaction_trigger + 2),
|
|
|
|
std::to_string(options_.level0_file_num_compaction_trigger + 4),
|
|
|
|
}},
|
|
|
|
{"level0_slowdown_writes_trigger",
|
|
|
|
{
|
|
|
|
std::to_string(options_.level0_slowdown_writes_trigger),
|
|
|
|
std::to_string(options_.level0_slowdown_writes_trigger + 2),
|
|
|
|
std::to_string(options_.level0_slowdown_writes_trigger + 4),
|
|
|
|
}},
|
|
|
|
{"level0_stop_writes_trigger",
|
|
|
|
{
|
|
|
|
std::to_string(options_.level0_stop_writes_trigger),
|
|
|
|
std::to_string(options_.level0_stop_writes_trigger + 2),
|
|
|
|
std::to_string(options_.level0_stop_writes_trigger + 4),
|
|
|
|
}},
|
|
|
|
{"max_compaction_bytes",
|
|
|
|
{
|
|
|
|
std::to_string(options_.target_file_size_base * 5),
|
|
|
|
std::to_string(options_.target_file_size_base * 15),
|
|
|
|
std::to_string(options_.target_file_size_base * 100),
|
|
|
|
}},
|
|
|
|
{"target_file_size_base",
|
|
|
|
{
|
|
|
|
std::to_string(options_.target_file_size_base),
|
|
|
|
std::to_string(options_.target_file_size_base * 2),
|
|
|
|
std::to_string(options_.target_file_size_base * 4),
|
|
|
|
}},
|
|
|
|
{"target_file_size_multiplier",
|
|
|
|
{
|
|
|
|
std::to_string(options_.target_file_size_multiplier),
|
|
|
|
"1",
|
|
|
|
"2",
|
|
|
|
}},
|
|
|
|
{"max_bytes_for_level_base",
|
|
|
|
{
|
|
|
|
std::to_string(options_.max_bytes_for_level_base / 2),
|
|
|
|
std::to_string(options_.max_bytes_for_level_base),
|
|
|
|
std::to_string(options_.max_bytes_for_level_base * 2),
|
|
|
|
}},
|
|
|
|
{"max_bytes_for_level_multiplier",
|
|
|
|
{
|
|
|
|
std::to_string(options_.max_bytes_for_level_multiplier),
|
|
|
|
"1",
|
|
|
|
"2",
|
|
|
|
}},
|
|
|
|
{"max_sequential_skip_in_iterations", {"4", "8", "12"}},
|
|
|
|
};
|
|
|
|
|
|
|
|
if (FLAGS_allow_setting_blob_options_dynamically) {
|
|
|
|
options_tbl.emplace("enable_blob_files",
|
|
|
|
std::vector<std::string>{"false", "true"});
|
|
|
|
options_tbl.emplace("min_blob_size",
|
|
|
|
std::vector<std::string>{"0", "8", "16"});
|
|
|
|
options_tbl.emplace("blob_file_size",
|
|
|
|
std::vector<std::string>{"1M", "16M", "256M", "1G"});
|
|
|
|
options_tbl.emplace("blob_compression_type", GetBlobCompressionTags());
|
|
|
|
options_tbl.emplace("enable_blob_garbage_collection",
|
|
|
|
std::vector<std::string>{"false", "true"});
|
|
|
|
options_tbl.emplace(
|
|
|
|
"blob_garbage_collection_age_cutoff",
|
|
|
|
std::vector<std::string>{"0.0", "0.25", "0.5", "0.75", "1.0"});
|
Make it possible to force the garbage collection of the oldest blob files (#8994)
Summary:
The current BlobDB garbage collection logic works by relocating the valid
blobs from the oldest blob files as they are encountered during compaction,
and cleaning up blob files once they contain nothing but garbage. However,
with sufficiently skewed workloads, it is theoretically possible to end up in a
situation when few or no compactions get scheduled for the SST files that contain
references to the oldest blob files, which can lead to increased space amp due
to the lack of GC.
In order to efficiently handle such workloads, the patch adds a new BlobDB
configuration option called `blob_garbage_collection_force_threshold`,
which signals to BlobDB to schedule targeted compactions for the SST files
that keep alive the oldest batch of blob files if the overall ratio of garbage in
the given blob files meets the threshold *and* all the given blob files are
eligible for GC based on `blob_garbage_collection_age_cutoff`. (For example,
if the new option is set to 0.9, targeted compactions will get scheduled if the
sum of garbage bytes meets or exceeds 90% of the sum of total bytes in the
oldest blob files, assuming all affected blob files are below the age-based cutoff.)
The net result of these targeted compactions is that the valid blobs in the oldest
blob files are relocated and the oldest blob files themselves cleaned up (since
*all* SST files that rely on them get compacted away).
These targeted compactions are similar to periodic compactions in the sense
that they force certain SST files that otherwise would not get picked up to undergo
compaction and also in the sense that instead of merging files from multiple levels,
they target a single file. (Note: such compactions might still include neighboring files
from the same level due to the need of having a "clean cut" boundary but they never
include any files from any other level.)
This functionality is currently only supported with the leveled compaction style
and is inactive by default (since the default value is set to 1.0, i.e. 100%).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8994
Test Plan: Ran `make check` and tested using `db_bench` and the stress/crash tests.
Reviewed By: riversand963
Differential Revision: D31489850
Pulled By: ltamasi
fbshipit-source-id: 44057d511726a0e2a03c5d9313d7511b3f0c4eab
3 years ago
|
|
|
options_tbl.emplace("blob_garbage_collection_force_threshold",
|
|
|
|
std::vector<std::string>{"0.5", "0.75", "1.0"});
|
|
|
|
options_tbl.emplace("blob_compaction_readahead_size",
|
|
|
|
std::vector<std::string>{"0", "1M", "4M"});
|
|
|
|
options_tbl.emplace("blob_file_starting_level",
|
|
|
|
std::vector<std::string>{"0", "1", "2"});
|
|
|
|
}
|
|
|
|
|
|
|
|
options_table_ = std::move(options_tbl);
|
|
|
|
|
|
|
|
for (const auto& iter : options_table_) {
|
|
|
|
options_index_.push_back(iter.first);
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
void StressTest::InitDb(SharedState* shared) {
|
|
|
|
uint64_t now = clock_->NowMicros();
|
|
|
|
fprintf(stdout, "%s Initializing db_stress\n",
|
|
|
|
clock_->TimeToString(now / 1000000).c_str());
|
|
|
|
PrintEnv();
|
|
|
|
Open(shared);
|
|
|
|
BuildOptionsTable();
|
|
|
|
}
|
|
|
|
|
|
|
|
void StressTest::FinishInitDb(SharedState* shared) {
|
|
|
|
if (FLAGS_read_only) {
|
|
|
|
uint64_t now = clock_->NowMicros();
|
|
|
|
fprintf(stdout, "%s Preloading db with %" PRIu64 " KVs\n",
|
|
|
|
clock_->TimeToString(now / 1000000).c_str(), FLAGS_max_key);
|
|
|
|
PreloadDbAndReopenAsReadOnly(FLAGS_max_key, shared);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (shared->HasHistory()) {
|
|
|
|
// The way it works right now is, if there's any history, that means the
|
|
|
|
// previous run mutating the DB had all its operations traced, in which case
|
|
|
|
// we should always be able to `Restore()` the expected values to match the
|
|
|
|
// `db_`'s current seqno.
|
|
|
|
Status s = shared->Restore(db_);
|
|
|
|
if (!s.ok()) {
|
|
|
|
fprintf(stderr, "Error restoring historical expected values: %s\n",
|
|
|
|
s.ToString().c_str());
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (FLAGS_enable_compaction_filter) {
|
|
|
|
auto* compaction_filter_factory =
|
|
|
|
reinterpret_cast<DbStressCompactionFilterFactory*>(
|
|
|
|
options_.compaction_filter_factory.get());
|
|
|
|
assert(compaction_filter_factory);
|
|
|
|
// This must be called only after any potential `SharedState::Restore()` has
|
|
|
|
// completed in order for the `compaction_filter_factory` to operate on the
|
|
|
|
// correct latest values file.
|
|
|
|
compaction_filter_factory->SetSharedState(shared);
|
|
|
|
fprintf(stdout, "Compaction filter factory: %s\n",
|
|
|
|
compaction_filter_factory->Name());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void StressTest::TrackExpectedState(SharedState* shared) {
|
|
|
|
if ((FLAGS_sync_fault_injection || FLAGS_disable_wal) && IsStateTracked()) {
|
|
|
|
Status s = shared->SaveAtAndAfter(db_);
|
|
|
|
if (!s.ok()) {
|
|
|
|
fprintf(stderr, "Error enabling history tracing: %s\n",
|
|
|
|
s.ToString().c_str());
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Status StressTest::AssertSame(DB* db, ColumnFamilyHandle* cf,
|
|
|
|
ThreadState::SnapshotState& snap_state) {
|
|
|
|
Status s;
|
|
|
|
if (cf->GetName() != snap_state.cf_at_name) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
// This `ReadOptions` is for validation purposes. Ignore
|
|
|
|
// `FLAGS_rate_limit_user_ops` to avoid slowing any validation.
|
|
|
|
ReadOptions ropt;
|
|
|
|
ropt.snapshot = snap_state.snapshot;
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
Slice ts;
|
|
|
|
if (!snap_state.timestamp.empty()) {
|
|
|
|
ts = snap_state.timestamp;
|
|
|
|
ropt.timestamp = &ts;
|
|
|
|
}
|
|
|
|
PinnableSlice exp_v(&snap_state.value);
|
|
|
|
exp_v.PinSelf();
|
|
|
|
PinnableSlice v;
|
|
|
|
s = db->Get(ropt, cf, snap_state.key, &v);
|
|
|
|
if (!s.ok() && !s.IsNotFound()) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
if (snap_state.status != s) {
|
|
|
|
return Status::Corruption(
|
|
|
|
"The snapshot gave inconsistent results for key " +
|
|
|
|
std::to_string(Hash(snap_state.key.c_str(), snap_state.key.size(), 0)) +
|
|
|
|
" in cf " + cf->GetName() + ": (" + snap_state.status.ToString() +
|
|
|
|
") vs. (" + s.ToString() + ")");
|
|
|
|
}
|
|
|
|
if (s.ok()) {
|
|
|
|
if (exp_v != v) {
|
|
|
|
return Status::Corruption("The snapshot gave inconsistent values: (" +
|
|
|
|
exp_v.ToString() + ") vs. (" + v.ToString() +
|
|
|
|
")");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (snap_state.key_vec != nullptr) {
|
|
|
|
// When `prefix_extractor` is set, seeking to beginning and scanning
|
|
|
|
// across prefixes are only supported with `total_order_seek` set.
|
|
|
|
ropt.total_order_seek = true;
|
|
|
|
std::unique_ptr<Iterator> iterator(db->NewIterator(ropt));
|
|
|
|
std::unique_ptr<std::vector<bool>> tmp_bitvec(
|
|
|
|
new std::vector<bool>(FLAGS_max_key));
|
|
|
|
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
|
|
|
|
uint64_t key_val;
|
|
|
|
if (GetIntVal(iterator->key().ToString(), &key_val)) {
|
|
|
|
(*tmp_bitvec.get())[key_val] = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!std::equal(snap_state.key_vec->begin(), snap_state.key_vec->end(),
|
|
|
|
tmp_bitvec.get()->begin())) {
|
|
|
|
return Status::Corruption("Found inconsistent keys at this snapshot");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
void StressTest::VerificationAbort(SharedState* shared, std::string msg,
|
|
|
|
Status s) const {
|
|
|
|
fprintf(stderr, "Verification failed: %s. Status is %s\n", msg.c_str(),
|
|
|
|
s.ToString().c_str());
|
|
|
|
shared->SetVerificationFailure();
|
|
|
|
}
|
|
|
|
|
|
|
|
void StressTest::VerificationAbort(SharedState* shared, std::string msg, int cf,
|
|
|
|
int64_t key) const {
|
|
|
|
auto key_str = Key(key);
|
|
|
|
Slice key_slice = key_str;
|
|
|
|
fprintf(stderr,
|
|
|
|
"Verification failed for column family %d key %s (%" PRIi64 "): %s\n",
|
|
|
|
cf, key_slice.ToString(true).c_str(), key, msg.c_str());
|
|
|
|
shared->SetVerificationFailure();
|
|
|
|
}
|
|
|
|
|
|
|
|
void StressTest::PrintStatistics() {
|
|
|
|
if (dbstats) {
|
|
|
|
fprintf(stdout, "STATISTICS:\n%s\n", dbstats->ToString().c_str());
|
|
|
|
}
|
|
|
|
if (dbstats_secondaries) {
|
|
|
|
fprintf(stdout, "Secondary instances STATISTICS:\n%s\n",
|
|
|
|
dbstats_secondaries->ToString().c_str());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
Snapshots with user-specified timestamps (#9879)
Summary:
In RocksDB, keys are associated with (internal) sequence numbers which denote when the keys are written
to the database. Sequence numbers in different RocksDB instances are unrelated, thus not comparable.
It is nice if we can associate sequence numbers with their corresponding actual timestamps. One thing we can
do is to support user-defined timestamp, which allows the applications to specify the format of custom timestamps
and encode a timestamp with each key. More details can be found at https://github.com/facebook/rocksdb/wiki/User-defined-Timestamp-%28Experimental%29.
This PR provides a different but complementary approach. We can associate rocksdb snapshots (defined in
https://github.com/facebook/rocksdb/blob/7.2.fb/include/rocksdb/snapshot.h#L20) with **user-specified** timestamps.
Since a snapshot is essentially an object representing a sequence number, this PR establishes a bi-directional mapping between sequence numbers and timestamps.
In the past, snapshots are usually taken by readers. The current super-version is grabbed, and a `rocksdb::Snapshot`
object is created with the last published sequence number of the super-version. You can see that the reader actually
has no good idea of what timestamp to assign to this snapshot, because by the time the `GetSnapshot()` is called,
an arbitrarily long period of time may have already elapsed since the last write, which is when the last published
sequence number is written.
This observation motivates the creation of "timestamped" snapshots on the write path. Currently, this functionality is
exposed only to the layer of `TransactionDB`. Application can tell RocksDB to create a snapshot when a transaction
commits, effectively associating the last sequence number with a timestamp. It is also assumed that application will
ensure any two snapshots with timestamps should satisfy the following:
```
snapshot1.seq < snapshot2.seq iff. snapshot1.ts < snapshot2.ts
```
If the application can guarantee that when a reader takes a timestamped snapshot, there is no active writes going on
in the database, then we also allow the user to use a new API `TransactionDB::CreateTimestampedSnapshot()` to create
a snapshot with associated timestamp.
Code example
```cpp
// Create a timestamped snapshot when committing transaction.
txn->SetCommitTimestamp(100);
txn->SetSnapshotOnNextOperation();
txn->Commit();
// A wrapper API for convenience
Status Transaction::CommitAndTryCreateSnapshot(
std::shared_ptr<TransactionNotifier> notifier,
TxnTimestamp ts,
std::shared_ptr<const Snapshot>* ret);
// Create a timestamped snapshot if caller guarantees no concurrent writes
std::pair<Status, std::shared_ptr<const Snapshot>> snapshot = txn_db->CreateTimestampedSnapshot(100);
```
The snapshots created in this way will be managed by RocksDB with ref-counting and potentially shared with
other readers. We provide the following APIs for readers to retrieve a snapshot given a timestamp.
```cpp
// Return the timestamped snapshot correponding to given timestamp. If ts is
// kMaxTxnTimestamp, then we return the latest timestamped snapshot if present.
// Othersise, we return the snapshot whose timestamp is equal to `ts`. If no
// such snapshot exists, then we return null.
std::shared_ptr<const Snapshot> TransactionDB::GetTimestampedSnapshot(TxnTimestamp ts) const;
// Return the latest timestamped snapshot if present.
std::shared_ptr<const Snapshot> TransactionDB::GetLatestTimestampedSnapshot() const;
```
We also provide two additional APIs for stats collection and reporting purposes.
```cpp
Status TransactionDB::GetAllTimestampedSnapshots(
std::vector<std::shared_ptr<const Snapshot>>& snapshots) const;
// Return timestamped snapshots whose timestamps fall in [ts_lb, ts_ub) and store them in `snapshots`.
Status TransactionDB::GetTimestampedSnapshots(
TxnTimestamp ts_lb,
TxnTimestamp ts_ub,
std::vector<std::shared_ptr<const Snapshot>>& snapshots) const;
```
To prevent the number of timestamped snapshots from growing infinitely, we provide the following API to release
timestamped snapshots whose timestamps are older than or equal to a given threshold.
```cpp
void TransactionDB::ReleaseTimestampedSnapshotsOlderThan(TxnTimestamp ts);
```
Before shutdown, RocksDB will release all timestamped snapshots.
Comparison with user-defined timestamp and how they can be combined:
User-defined timestamp persists every key with a timestamp, while timestamped snapshots maintain a volatile
mapping between snapshots (sequence numbers) and timestamps.
Different internal keys with the same user key but different timestamps will be treated as different by compaction,
thus a newer version will not hide older versions (with smaller timestamps) unless they are eligible for garbage collection.
In contrast, taking a timestamped snapshot at a certain sequence number and timestamp prevents all the keys visible in
this snapshot from been dropped by compaction. Here, visible means (seq < snapshot and most recent).
The timestamped snapshot supports the semantics of reading at an exact point in time.
Timestamped snapshots can also be used with user-defined timestamp.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9879
Test Plan:
```
make check
TEST_TMPDIR=/dev/shm make crash_test_with_txn
```
Reviewed By: siying
Differential Revision: D35783919
Pulled By: riversand963
fbshipit-source-id: 586ad905e169189e19d3bfc0cb0177a7239d1bd4
3 years ago
|
|
|
void StressTest::ReleaseOldTimestampedSnapshots(uint64_t ts) {
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
if (!txn_db_) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
assert(txn_db_);
|
|
|
|
txn_db_->ReleaseTimestampedSnapshotsOlderThan(ts);
|
|
|
|
#else
|
|
|
|
(void)ts;
|
|
|
|
fprintf(stderr, "timestamped snapshots not supported in LITE mode\n");
|
|
|
|
exit(1);
|
|
|
|
#endif // ROCKSDB_LITE
|
|
|
|
}
|
|
|
|
|
|
|
|
// Currently PreloadDb has to be single-threaded.
|
|
|
|
void StressTest::PreloadDbAndReopenAsReadOnly(int64_t number_of_keys,
|
|
|
|
SharedState* shared) {
|
|
|
|
WriteOptions write_opts;
|
|
|
|
write_opts.disableWAL = FLAGS_disable_wal;
|
|
|
|
if (FLAGS_sync) {
|
|
|
|
write_opts.sync = true;
|
|
|
|
}
|
Rate-limit automatic WAL flush after each user write (#9607)
Summary:
**Context:**
WAL flush is currently not rate-limited by `Options::rate_limiter`. This PR is to provide rate-limiting to auto WAL flush, the one that automatically happen after each user write operation (i.e, `Options::manual_wal_flush == false`), by adding `WriteOptions::rate_limiter_options`.
Note that we are NOT rate-limiting WAL flush that do NOT automatically happen after each user write, such as `Options::manual_wal_flush == true + manual FlushWAL()` (rate-limiting multiple WAL flushes), for the benefits of:
- being consistent with [ReadOptions::rate_limiter_priority](https://github.com/facebook/rocksdb/blob/7.0.fb/include/rocksdb/options.h#L515)
- being able to turn off some WAL flush's rate-limiting but not all (e.g, turn off specific the WAL flush of a critical user write like a service's heartbeat)
`WriteOptions::rate_limiter_options` only accept `Env::IO_USER` and `Env::IO_TOTAL` currently due to an implementation constraint.
- The constraint is that we currently queue parallel writes (including WAL writes) based on FIFO policy which does not factor rate limiter priority into this layer's scheduling. If we allow lower priorities such as `Env::IO_HIGH/MID/LOW` and such writes specified with lower priorities occurs before ones specified with higher priorities (even just by a tiny bit in arrival time), the former would have blocked the latter, leading to a "priority inversion" issue and contradictory to what we promise for rate-limiting priority. Therefore we only allow `Env::IO_USER` and `Env::IO_TOTAL` right now before improving that scheduling.
A pre-requisite to this feature is to support operation-level rate limiting in `WritableFileWriter`, which is also included in this PR.
**Summary:**
- Renamed test suite `DBRateLimiterTest to DBRateLimiterOnReadTest` for adding a new test suite
- Accept `rate_limiter_priority` in `WritableFileWriter`'s private and public write functions
- Passed `WriteOptions::rate_limiter_options` to `WritableFileWriter` in the path of automatic WAL flush.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9607
Test Plan:
- Added new unit test to verify existing flush/compaction rate-limiting does not break, since `DBTest, RateLimitingTest` is disabled and current db-level rate-limiting tests focus on read only (e.g, `db_rate_limiter_test`, `DBTest2, RateLimitedCompactionReads`).
- Added new unit test `DBRateLimiterOnWriteWALTest, AutoWalFlush`
- `strace -ftt -e trace=write ./db_bench -benchmarks=fillseq -db=/dev/shm/testdb -rate_limit_auto_wal_flush=1 -rate_limiter_bytes_per_sec=15 -rate_limiter_refill_period_us=1000000 -write_buffer_size=100000000 -disable_auto_compactions=1 -num=100`
- verified that WAL flush(i.e, system-call _write_) were chunked into 15 bytes and each _write_ was roughly 1 second apart
- verified the chunking disappeared when `-rate_limit_auto_wal_flush=0`
- crash test: `python3 tools/db_crashtest.py blackbox --disable_wal=0 --rate_limit_auto_wal_flush=1 --rate_limiter_bytes_per_sec=10485760 --interval=10` killed as normal
**Benchmarked on flush/compaction to ensure no performance regression:**
- compaction with rate-limiting (see table 1, avg over 1280-run): pre-change: **915635 micros/op**; post-change:
**907350 micros/op (improved by 0.106%)**
```
#!/bin/bash
TEST_TMPDIR=/dev/shm/testdb
START=1
NUM_DATA_ENTRY=8
N=10
rm -f compact_bmk_output.txt compact_bmk_output_2.txt dont_care_output.txt
for i in $(eval echo "{$START..$NUM_DATA_ENTRY}")
do
NUM_RUN=$(($N*(2**($i-1))))
for j in $(eval echo "{$START..$NUM_RUN}")
do
./db_bench --benchmarks=fillrandom -db=$TEST_TMPDIR -disable_auto_compactions=1 -write_buffer_size=6710886 > dont_care_output.txt && ./db_bench --benchmarks=compact -use_existing_db=1 -db=$TEST_TMPDIR -level0_file_num_compaction_trigger=1 -rate_limiter_bytes_per_sec=100000000 | egrep 'compact'
done > compact_bmk_output.txt && awk -v NUM_RUN=$NUM_RUN '{sum+=$3;sum_sqrt+=$3^2}END{print sum/NUM_RUN, sqrt(sum_sqrt/NUM_RUN-(sum/NUM_RUN)^2)}' compact_bmk_output.txt >> compact_bmk_output_2.txt
done
```
- compaction w/o rate-limiting (see table 2, avg over 640-run): pre-change: **822197 micros/op**; post-change: **823148 micros/op (regressed by 0.12%)**
```
Same as above script, except that -rate_limiter_bytes_per_sec=0
```
- flush with rate-limiting (see table 3, avg over 320-run, run on the [patch](https://github.com/hx235/rocksdb/commit/ee5c6023a9f6533fab9afdc681568daa21da4953) to augment current db_bench ): pre-change: **745752 micros/op**; post-change: **745331 micros/op (regressed by 0.06 %)**
```
#!/bin/bash
TEST_TMPDIR=/dev/shm/testdb
START=1
NUM_DATA_ENTRY=8
N=10
rm -f flush_bmk_output.txt flush_bmk_output_2.txt
for i in $(eval echo "{$START..$NUM_DATA_ENTRY}")
do
NUM_RUN=$(($N*(2**($i-1))))
for j in $(eval echo "{$START..$NUM_RUN}")
do
./db_bench -db=$TEST_TMPDIR -write_buffer_size=1048576000 -num=1000000 -rate_limiter_bytes_per_sec=100000000 -benchmarks=fillseq,flush | egrep 'flush'
done > flush_bmk_output.txt && awk -v NUM_RUN=$NUM_RUN '{sum+=$3;sum_sqrt+=$3^2}END{print sum/NUM_RUN, sqrt(sum_sqrt/NUM_RUN-(sum/NUM_RUN)^2)}' flush_bmk_output.txt >> flush_bmk_output_2.txt
done
```
- flush w/o rate-limiting (see table 4, avg over 320-run, run on the [patch](https://github.com/hx235/rocksdb/commit/ee5c6023a9f6533fab9afdc681568daa21da4953) to augment current db_bench): pre-change: **487512 micros/op**, post-change: **485856 micors/ops (improved by 0.34%)**
```
Same as above script, except that -rate_limiter_bytes_per_sec=0
```
| table 1 - compact with rate-limiting|
#-run | (pre-change) avg micros/op | std micros/op | (post-change) avg micros/op | std micros/op | change in avg micros/op (%)
-- | -- | -- | -- | -- | --
10 | 896978 | 16046.9 | 901242 | 15670.9 | 0.475373978
20 | 893718 | 15813 | 886505 | 17544.7 | -0.8070778478
40 | 900426 | 23882.2 | 894958 | 15104.5 | -0.6072681153
80 | 906635 | 21761.5 | 903332 | 23948.3 | -0.3643141948
160 | 898632 | 21098.9 | 907583 | 21145 | 0.9960695813
3.20E+02 | 905252 | 22785.5 | 908106 | 25325.5 | 0.3152713278
6.40E+02 | 905213 | 23598.6 | 906741 | 21370.5 | 0.1688000504
**1.28E+03** | **908316** | **23533.1** | **907350** | **24626.8** | **-0.1063506533**
average over #-run | 901896.25 | 21064.9625 | 901977.125 | 20592.025 | 0.008967217682
| table 2 - compact w/o rate-limiting|
#-run | (pre-change) avg micros/op | std micros/op | (post-change) avg micros/op | std micros/op | change in avg micros/op (%)
-- | -- | -- | -- | -- | --
10 | 811211 | 26996.7 | 807586 | 28456.4 | -0.4468627768
20 | 815465 | 14803.7 | 814608 | 28719.7 | -0.105093413
40 | 809203 | 26187.1 | 797835 | 25492.1 | -1.404839082
80 | 822088 | 28765.3 | 822192 | 32840.4 | 0.01265071379
160 | 821719 | 36344.7 | 821664 | 29544.9 | -0.006693285661
3.20E+02 | 820921 | 27756.4 | 821403 | 28347.7 | 0.05871454135
**6.40E+02** | **822197** | **28960.6** | **823148** | **30055.1** | **0.1156657103**
average over #-run | 8.18E+05 | 2.71E+04 | 8.15E+05 | 2.91E+04 | -0.25
| table 3 - flush with rate-limiting|
#-run | (pre-change) avg micros/op | std micros/op | (post-change) avg micros/op | std micros/op | change in avg micros/op (%)
-- | -- | -- | -- | -- | --
10 | 741721 | 11770.8 | 740345 | 5949.76 | -0.1855144994
20 | 735169 | 3561.83 | 743199 | 9755.77 | 1.09226586
40 | 743368 | 8891.03 | 742102 | 8683.22 | -0.1703059588
80 | 742129 | 8148.51 | 743417 | 9631.58| 0.1735547324
160 | 749045 | 9757.21 | 746256 | 9191.86 | -0.3723407806
**3.20E+02** | **745752** | **9819.65** | **745331** | **9840.62** | **-0.0564530836**
6.40E+02 | 749006 | 11080.5 | 748173 | 10578.7 | -0.1112140624
average over #-run | 743741.4286 | 9004.218571 | 744117.5714 | 9090.215714 | 0.05057441238
| table 4 - flush w/o rate-limiting|
#-run | (pre-change) avg micros/op | std micros/op | (post-change) avg micros/op | std micros/op | change in avg micros/op (%)
-- | -- | -- | -- | -- | --
10 | 477283 | 24719.6 | 473864 | 12379 | -0.7163464863
20 | 486743 | 20175.2 | 502296 | 23931.3 | 3.195320734
40 | 482846 | 15309.2 | 489820 | 22259.5 | 1.444352858
80 | 491490 | 21883.1 | 490071 | 23085.7 | -0.2887139108
160 | 493347 | 28074.3 | 483609 | 21211.7 | -1.973864238
**3.20E+02** | **487512** | **21401.5** | **485856** | **22195.2** | **-0.3396839462**
6.40E+02 | 490307 | 25418.6 | 485435 | 22405.2 | -0.9936631539
average over #-run | 4.87E+05 | 2.24E+04 | 4.87E+05 | 2.11E+04 | 0.00E+00
Reviewed By: ajkr
Differential Revision: D34442441
Pulled By: hx235
fbshipit-source-id: 4790f13e1e5c0a95ae1d1cc93ffcf69dc6e78bdd
3 years ago
|
|
|
if (FLAGS_rate_limit_auto_wal_flush) {
|
|
|
|
write_opts.rate_limiter_priority = Env::IO_USER;
|
|
|
|
}
|
|
|
|
char value[100];
|
|
|
|
int cf_idx = 0;
|
|
|
|
Status s;
|
|
|
|
for (auto cfh : column_families_) {
|
|
|
|
for (int64_t k = 0; k != number_of_keys; ++k) {
|
|
|
|
std::string key_str = Key(k);
|
|
|
|
Slice key = key_str;
|
|
|
|
size_t sz = GenerateValue(0 /*value_base*/, value, sizeof(value));
|
|
|
|
Slice v(value, sz);
|
|
|
|
shared->Put(cf_idx, k, 0, true /* pending */);
|
|
|
|
|
|
|
|
if (FLAGS_use_merge) {
|
|
|
|
if (!FLAGS_use_txn) {
|
|
|
|
s = db_->Merge(write_opts, cfh, key, v);
|
|
|
|
} else {
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
Transaction* txn;
|
|
|
|
s = NewTxn(write_opts, &txn);
|
|
|
|
if (s.ok()) {
|
|
|
|
s = txn->Merge(cfh, key, v);
|
|
|
|
if (s.ok()) {
|
|
|
|
s = CommitTxn(txn);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if (!FLAGS_use_txn) {
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
std::string ts_str;
|
|
|
|
Slice ts;
|
|
|
|
if (FLAGS_user_timestamp_size > 0) {
|
|
|
|
ts_str = NowNanosStr();
|
|
|
|
ts = ts_str;
|
Revise APIs related to user-defined timestamp (#8946)
Summary:
ajkr reminded me that we have a rule of not including per-kv related data in `WriteOptions`.
Namely, `WriteOptions` should not include information about "what-to-write", but should just
include information about "how-to-write".
According to this rule, `WriteOptions::timestamp` (experimental) is clearly a violation. Therefore,
this PR removes `WriteOptions::timestamp` for compliance.
After the removal, we need to pass timestamp info via another set of APIs. This PR proposes a set
of overloaded functions `Put(write_opts, key, value, ts)`, `Delete(write_opts, key, ts)`, and
`SingleDelete(write_opts, key, ts)`. Planned to add `Write(write_opts, batch, ts)`, but its complexity
made me reconsider doing it in another PR (maybe).
For better checking and returning error early, we also add a new set of APIs to `WriteBatch` that take
extra `timestamp` information when writing to `WriteBatch`es.
These set of APIs in `WriteBatchWithIndex` are currently not supported, and are on our TODO list.
Removed `WriteBatch::AssignTimestamps()` and renamed `WriteBatch::AssignTimestamp()` to
`WriteBatch::UpdateTimestamps()` since this method require that all keys have space for timestamps
allocated already and multiple timestamps can be updated.
The constructor of `WriteBatch` now takes a fourth argument `default_cf_ts_sz` which is the timestamp
size of the default column family. This will be used to allocate space when calling APIs that do not
specify a column family handle.
Also, updated `DB::Get()`, `DB::MultiGet()`, `DB::NewIterator()`, `DB::NewIterators()` methods, replacing
some assertions about timestamp to returning Status code.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8946
Test Plan:
make check
./db_bench -benchmarks=fillseq,fillrandom,readrandom,readseq,deleterandom -user_timestamp_size=8
./db_stress --user_timestamp_size=8 -nooverwritepercent=0 -test_secondary=0 -secondary_catch_up_one_in=0 -continuous_verification_interval=0
Make sure there is no perf regression by running the following
```
./db_bench_opt -db=/dev/shm/rocksdb -use_existing_db=0 -level0_stop_writes_trigger=256 -level0_slowdown_writes_trigger=256 -level0_file_num_compaction_trigger=256 -disable_wal=1 -duration=10 -benchmarks=fillrandom
```
Before this PR
```
DB path: [/dev/shm/rocksdb]
fillrandom : 1.831 micros/op 546235 ops/sec; 60.4 MB/s
```
After this PR
```
DB path: [/dev/shm/rocksdb]
fillrandom : 1.820 micros/op 549404 ops/sec; 60.8 MB/s
```
Reviewed By: ltamasi
Differential Revision: D33721359
Pulled By: riversand963
fbshipit-source-id: c131561534272c120ffb80711d42748d21badf09
3 years ago
|
|
|
s = db_->Put(write_opts, cfh, key, ts, v);
|
|
|
|
} else {
|
|
|
|
s = db_->Put(write_opts, cfh, key, v);
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
}
|
|
|
|
} else {
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
Transaction* txn;
|
|
|
|
s = NewTxn(write_opts, &txn);
|
|
|
|
if (s.ok()) {
|
|
|
|
s = txn->Put(cfh, key, v);
|
|
|
|
if (s.ok()) {
|
|
|
|
s = CommitTxn(txn);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
shared->Put(cf_idx, k, 0, false /* pending */);
|
|
|
|
if (!s.ok()) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!s.ok()) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
++cf_idx;
|
|
|
|
}
|
|
|
|
if (s.ok()) {
|
|
|
|
s = db_->Flush(FlushOptions(), column_families_);
|
|
|
|
}
|
|
|
|
if (s.ok()) {
|
|
|
|
for (auto cf : column_families_) {
|
|
|
|
delete cf;
|
|
|
|
}
|
|
|
|
column_families_.clear();
|
|
|
|
delete db_;
|
|
|
|
db_ = nullptr;
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
txn_db_ = nullptr;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
db_preload_finished_.store(true);
|
|
|
|
auto now = clock_->NowMicros();
|
|
|
|
fprintf(stdout, "%s Reopening database in read-only\n",
|
|
|
|
clock_->TimeToString(now / 1000000).c_str());
|
|
|
|
// Reopen as read-only, can ignore all options related to updates
|
|
|
|
Open(shared);
|
|
|
|
} else {
|
|
|
|
fprintf(stderr, "Failed to preload db");
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Status StressTest::SetOptions(ThreadState* thread) {
|
|
|
|
assert(FLAGS_set_options_one_in > 0);
|
|
|
|
std::unordered_map<std::string, std::string> opts;
|
|
|
|
std::string name =
|
|
|
|
options_index_[thread->rand.Next() % options_index_.size()];
|
|
|
|
int value_idx = thread->rand.Next() % options_table_[name].size();
|
|
|
|
if (name == "level0_file_num_compaction_trigger" ||
|
|
|
|
name == "level0_slowdown_writes_trigger" ||
|
|
|
|
name == "level0_stop_writes_trigger") {
|
|
|
|
opts["level0_file_num_compaction_trigger"] =
|
|
|
|
options_table_["level0_file_num_compaction_trigger"][value_idx];
|
|
|
|
opts["level0_slowdown_writes_trigger"] =
|
|
|
|
options_table_["level0_slowdown_writes_trigger"][value_idx];
|
|
|
|
opts["level0_stop_writes_trigger"] =
|
|
|
|
options_table_["level0_stop_writes_trigger"][value_idx];
|
|
|
|
} else {
|
|
|
|
opts[name] = options_table_[name][value_idx];
|
|
|
|
}
|
|
|
|
|
|
|
|
int rand_cf_idx = thread->rand.Next() % FLAGS_column_families;
|
|
|
|
auto cfh = column_families_[rand_cf_idx];
|
|
|
|
return db_->SetOptions(cfh, opts);
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
Status StressTest::NewTxn(WriteOptions& write_opts, Transaction** txn) {
|
|
|
|
if (!FLAGS_use_txn) {
|
|
|
|
return Status::InvalidArgument("NewTxn when FLAGS_use_txn is not set");
|
|
|
|
}
|
|
|
|
write_opts.disableWAL = FLAGS_disable_wal;
|
|
|
|
static std::atomic<uint64_t> txn_id = {0};
|
|
|
|
TransactionOptions txn_options;
|
|
|
|
txn_options.use_only_the_last_commit_time_batch_for_recovery =
|
|
|
|
FLAGS_use_only_the_last_commit_time_batch_for_recovery;
|
|
|
|
txn_options.lock_timeout = 600000; // 10 min
|
|
|
|
txn_options.deadlock_detect = true;
|
|
|
|
*txn = txn_db_->BeginTransaction(write_opts, txn_options);
|
|
|
|
auto istr = std::to_string(txn_id.fetch_add(1));
|
|
|
|
Status s = (*txn)->SetName("xid" + istr);
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
Snapshots with user-specified timestamps (#9879)
Summary:
In RocksDB, keys are associated with (internal) sequence numbers which denote when the keys are written
to the database. Sequence numbers in different RocksDB instances are unrelated, thus not comparable.
It is nice if we can associate sequence numbers with their corresponding actual timestamps. One thing we can
do is to support user-defined timestamp, which allows the applications to specify the format of custom timestamps
and encode a timestamp with each key. More details can be found at https://github.com/facebook/rocksdb/wiki/User-defined-Timestamp-%28Experimental%29.
This PR provides a different but complementary approach. We can associate rocksdb snapshots (defined in
https://github.com/facebook/rocksdb/blob/7.2.fb/include/rocksdb/snapshot.h#L20) with **user-specified** timestamps.
Since a snapshot is essentially an object representing a sequence number, this PR establishes a bi-directional mapping between sequence numbers and timestamps.
In the past, snapshots are usually taken by readers. The current super-version is grabbed, and a `rocksdb::Snapshot`
object is created with the last published sequence number of the super-version. You can see that the reader actually
has no good idea of what timestamp to assign to this snapshot, because by the time the `GetSnapshot()` is called,
an arbitrarily long period of time may have already elapsed since the last write, which is when the last published
sequence number is written.
This observation motivates the creation of "timestamped" snapshots on the write path. Currently, this functionality is
exposed only to the layer of `TransactionDB`. Application can tell RocksDB to create a snapshot when a transaction
commits, effectively associating the last sequence number with a timestamp. It is also assumed that application will
ensure any two snapshots with timestamps should satisfy the following:
```
snapshot1.seq < snapshot2.seq iff. snapshot1.ts < snapshot2.ts
```
If the application can guarantee that when a reader takes a timestamped snapshot, there is no active writes going on
in the database, then we also allow the user to use a new API `TransactionDB::CreateTimestampedSnapshot()` to create
a snapshot with associated timestamp.
Code example
```cpp
// Create a timestamped snapshot when committing transaction.
txn->SetCommitTimestamp(100);
txn->SetSnapshotOnNextOperation();
txn->Commit();
// A wrapper API for convenience
Status Transaction::CommitAndTryCreateSnapshot(
std::shared_ptr<TransactionNotifier> notifier,
TxnTimestamp ts,
std::shared_ptr<const Snapshot>* ret);
// Create a timestamped snapshot if caller guarantees no concurrent writes
std::pair<Status, std::shared_ptr<const Snapshot>> snapshot = txn_db->CreateTimestampedSnapshot(100);
```
The snapshots created in this way will be managed by RocksDB with ref-counting and potentially shared with
other readers. We provide the following APIs for readers to retrieve a snapshot given a timestamp.
```cpp
// Return the timestamped snapshot correponding to given timestamp. If ts is
// kMaxTxnTimestamp, then we return the latest timestamped snapshot if present.
// Othersise, we return the snapshot whose timestamp is equal to `ts`. If no
// such snapshot exists, then we return null.
std::shared_ptr<const Snapshot> TransactionDB::GetTimestampedSnapshot(TxnTimestamp ts) const;
// Return the latest timestamped snapshot if present.
std::shared_ptr<const Snapshot> TransactionDB::GetLatestTimestampedSnapshot() const;
```
We also provide two additional APIs for stats collection and reporting purposes.
```cpp
Status TransactionDB::GetAllTimestampedSnapshots(
std::vector<std::shared_ptr<const Snapshot>>& snapshots) const;
// Return timestamped snapshots whose timestamps fall in [ts_lb, ts_ub) and store them in `snapshots`.
Status TransactionDB::GetTimestampedSnapshots(
TxnTimestamp ts_lb,
TxnTimestamp ts_ub,
std::vector<std::shared_ptr<const Snapshot>>& snapshots) const;
```
To prevent the number of timestamped snapshots from growing infinitely, we provide the following API to release
timestamped snapshots whose timestamps are older than or equal to a given threshold.
```cpp
void TransactionDB::ReleaseTimestampedSnapshotsOlderThan(TxnTimestamp ts);
```
Before shutdown, RocksDB will release all timestamped snapshots.
Comparison with user-defined timestamp and how they can be combined:
User-defined timestamp persists every key with a timestamp, while timestamped snapshots maintain a volatile
mapping between snapshots (sequence numbers) and timestamps.
Different internal keys with the same user key but different timestamps will be treated as different by compaction,
thus a newer version will not hide older versions (with smaller timestamps) unless they are eligible for garbage collection.
In contrast, taking a timestamped snapshot at a certain sequence number and timestamp prevents all the keys visible in
this snapshot from been dropped by compaction. Here, visible means (seq < snapshot and most recent).
The timestamped snapshot supports the semantics of reading at an exact point in time.
Timestamped snapshots can also be used with user-defined timestamp.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9879
Test Plan:
```
make check
TEST_TMPDIR=/dev/shm make crash_test_with_txn
```
Reviewed By: siying
Differential Revision: D35783919
Pulled By: riversand963
fbshipit-source-id: 586ad905e169189e19d3bfc0cb0177a7239d1bd4
3 years ago
|
|
|
Status StressTest::CommitTxn(Transaction* txn, ThreadState* thread) {
|
|
|
|
if (!FLAGS_use_txn) {
|
|
|
|
return Status::InvalidArgument("CommitTxn when FLAGS_use_txn is not set");
|
|
|
|
}
|
|
|
|
Status s = txn->Prepare();
|
Snapshots with user-specified timestamps (#9879)
Summary:
In RocksDB, keys are associated with (internal) sequence numbers which denote when the keys are written
to the database. Sequence numbers in different RocksDB instances are unrelated, thus not comparable.
It is nice if we can associate sequence numbers with their corresponding actual timestamps. One thing we can
do is to support user-defined timestamp, which allows the applications to specify the format of custom timestamps
and encode a timestamp with each key. More details can be found at https://github.com/facebook/rocksdb/wiki/User-defined-Timestamp-%28Experimental%29.
This PR provides a different but complementary approach. We can associate rocksdb snapshots (defined in
https://github.com/facebook/rocksdb/blob/7.2.fb/include/rocksdb/snapshot.h#L20) with **user-specified** timestamps.
Since a snapshot is essentially an object representing a sequence number, this PR establishes a bi-directional mapping between sequence numbers and timestamps.
In the past, snapshots are usually taken by readers. The current super-version is grabbed, and a `rocksdb::Snapshot`
object is created with the last published sequence number of the super-version. You can see that the reader actually
has no good idea of what timestamp to assign to this snapshot, because by the time the `GetSnapshot()` is called,
an arbitrarily long period of time may have already elapsed since the last write, which is when the last published
sequence number is written.
This observation motivates the creation of "timestamped" snapshots on the write path. Currently, this functionality is
exposed only to the layer of `TransactionDB`. Application can tell RocksDB to create a snapshot when a transaction
commits, effectively associating the last sequence number with a timestamp. It is also assumed that application will
ensure any two snapshots with timestamps should satisfy the following:
```
snapshot1.seq < snapshot2.seq iff. snapshot1.ts < snapshot2.ts
```
If the application can guarantee that when a reader takes a timestamped snapshot, there is no active writes going on
in the database, then we also allow the user to use a new API `TransactionDB::CreateTimestampedSnapshot()` to create
a snapshot with associated timestamp.
Code example
```cpp
// Create a timestamped snapshot when committing transaction.
txn->SetCommitTimestamp(100);
txn->SetSnapshotOnNextOperation();
txn->Commit();
// A wrapper API for convenience
Status Transaction::CommitAndTryCreateSnapshot(
std::shared_ptr<TransactionNotifier> notifier,
TxnTimestamp ts,
std::shared_ptr<const Snapshot>* ret);
// Create a timestamped snapshot if caller guarantees no concurrent writes
std::pair<Status, std::shared_ptr<const Snapshot>> snapshot = txn_db->CreateTimestampedSnapshot(100);
```
The snapshots created in this way will be managed by RocksDB with ref-counting and potentially shared with
other readers. We provide the following APIs for readers to retrieve a snapshot given a timestamp.
```cpp
// Return the timestamped snapshot correponding to given timestamp. If ts is
// kMaxTxnTimestamp, then we return the latest timestamped snapshot if present.
// Othersise, we return the snapshot whose timestamp is equal to `ts`. If no
// such snapshot exists, then we return null.
std::shared_ptr<const Snapshot> TransactionDB::GetTimestampedSnapshot(TxnTimestamp ts) const;
// Return the latest timestamped snapshot if present.
std::shared_ptr<const Snapshot> TransactionDB::GetLatestTimestampedSnapshot() const;
```
We also provide two additional APIs for stats collection and reporting purposes.
```cpp
Status TransactionDB::GetAllTimestampedSnapshots(
std::vector<std::shared_ptr<const Snapshot>>& snapshots) const;
// Return timestamped snapshots whose timestamps fall in [ts_lb, ts_ub) and store them in `snapshots`.
Status TransactionDB::GetTimestampedSnapshots(
TxnTimestamp ts_lb,
TxnTimestamp ts_ub,
std::vector<std::shared_ptr<const Snapshot>>& snapshots) const;
```
To prevent the number of timestamped snapshots from growing infinitely, we provide the following API to release
timestamped snapshots whose timestamps are older than or equal to a given threshold.
```cpp
void TransactionDB::ReleaseTimestampedSnapshotsOlderThan(TxnTimestamp ts);
```
Before shutdown, RocksDB will release all timestamped snapshots.
Comparison with user-defined timestamp and how they can be combined:
User-defined timestamp persists every key with a timestamp, while timestamped snapshots maintain a volatile
mapping between snapshots (sequence numbers) and timestamps.
Different internal keys with the same user key but different timestamps will be treated as different by compaction,
thus a newer version will not hide older versions (with smaller timestamps) unless they are eligible for garbage collection.
In contrast, taking a timestamped snapshot at a certain sequence number and timestamp prevents all the keys visible in
this snapshot from been dropped by compaction. Here, visible means (seq < snapshot and most recent).
The timestamped snapshot supports the semantics of reading at an exact point in time.
Timestamped snapshots can also be used with user-defined timestamp.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9879
Test Plan:
```
make check
TEST_TMPDIR=/dev/shm make crash_test_with_txn
```
Reviewed By: siying
Differential Revision: D35783919
Pulled By: riversand963
fbshipit-source-id: 586ad905e169189e19d3bfc0cb0177a7239d1bd4
3 years ago
|
|
|
std::shared_ptr<const Snapshot> timestamped_snapshot;
|
|
|
|
if (s.ok()) {
|
Snapshots with user-specified timestamps (#9879)
Summary:
In RocksDB, keys are associated with (internal) sequence numbers which denote when the keys are written
to the database. Sequence numbers in different RocksDB instances are unrelated, thus not comparable.
It is nice if we can associate sequence numbers with their corresponding actual timestamps. One thing we can
do is to support user-defined timestamp, which allows the applications to specify the format of custom timestamps
and encode a timestamp with each key. More details can be found at https://github.com/facebook/rocksdb/wiki/User-defined-Timestamp-%28Experimental%29.
This PR provides a different but complementary approach. We can associate rocksdb snapshots (defined in
https://github.com/facebook/rocksdb/blob/7.2.fb/include/rocksdb/snapshot.h#L20) with **user-specified** timestamps.
Since a snapshot is essentially an object representing a sequence number, this PR establishes a bi-directional mapping between sequence numbers and timestamps.
In the past, snapshots are usually taken by readers. The current super-version is grabbed, and a `rocksdb::Snapshot`
object is created with the last published sequence number of the super-version. You can see that the reader actually
has no good idea of what timestamp to assign to this snapshot, because by the time the `GetSnapshot()` is called,
an arbitrarily long period of time may have already elapsed since the last write, which is when the last published
sequence number is written.
This observation motivates the creation of "timestamped" snapshots on the write path. Currently, this functionality is
exposed only to the layer of `TransactionDB`. Application can tell RocksDB to create a snapshot when a transaction
commits, effectively associating the last sequence number with a timestamp. It is also assumed that application will
ensure any two snapshots with timestamps should satisfy the following:
```
snapshot1.seq < snapshot2.seq iff. snapshot1.ts < snapshot2.ts
```
If the application can guarantee that when a reader takes a timestamped snapshot, there is no active writes going on
in the database, then we also allow the user to use a new API `TransactionDB::CreateTimestampedSnapshot()` to create
a snapshot with associated timestamp.
Code example
```cpp
// Create a timestamped snapshot when committing transaction.
txn->SetCommitTimestamp(100);
txn->SetSnapshotOnNextOperation();
txn->Commit();
// A wrapper API for convenience
Status Transaction::CommitAndTryCreateSnapshot(
std::shared_ptr<TransactionNotifier> notifier,
TxnTimestamp ts,
std::shared_ptr<const Snapshot>* ret);
// Create a timestamped snapshot if caller guarantees no concurrent writes
std::pair<Status, std::shared_ptr<const Snapshot>> snapshot = txn_db->CreateTimestampedSnapshot(100);
```
The snapshots created in this way will be managed by RocksDB with ref-counting and potentially shared with
other readers. We provide the following APIs for readers to retrieve a snapshot given a timestamp.
```cpp
// Return the timestamped snapshot correponding to given timestamp. If ts is
// kMaxTxnTimestamp, then we return the latest timestamped snapshot if present.
// Othersise, we return the snapshot whose timestamp is equal to `ts`. If no
// such snapshot exists, then we return null.
std::shared_ptr<const Snapshot> TransactionDB::GetTimestampedSnapshot(TxnTimestamp ts) const;
// Return the latest timestamped snapshot if present.
std::shared_ptr<const Snapshot> TransactionDB::GetLatestTimestampedSnapshot() const;
```
We also provide two additional APIs for stats collection and reporting purposes.
```cpp
Status TransactionDB::GetAllTimestampedSnapshots(
std::vector<std::shared_ptr<const Snapshot>>& snapshots) const;
// Return timestamped snapshots whose timestamps fall in [ts_lb, ts_ub) and store them in `snapshots`.
Status TransactionDB::GetTimestampedSnapshots(
TxnTimestamp ts_lb,
TxnTimestamp ts_ub,
std::vector<std::shared_ptr<const Snapshot>>& snapshots) const;
```
To prevent the number of timestamped snapshots from growing infinitely, we provide the following API to release
timestamped snapshots whose timestamps are older than or equal to a given threshold.
```cpp
void TransactionDB::ReleaseTimestampedSnapshotsOlderThan(TxnTimestamp ts);
```
Before shutdown, RocksDB will release all timestamped snapshots.
Comparison with user-defined timestamp and how they can be combined:
User-defined timestamp persists every key with a timestamp, while timestamped snapshots maintain a volatile
mapping between snapshots (sequence numbers) and timestamps.
Different internal keys with the same user key but different timestamps will be treated as different by compaction,
thus a newer version will not hide older versions (with smaller timestamps) unless they are eligible for garbage collection.
In contrast, taking a timestamped snapshot at a certain sequence number and timestamp prevents all the keys visible in
this snapshot from been dropped by compaction. Here, visible means (seq < snapshot and most recent).
The timestamped snapshot supports the semantics of reading at an exact point in time.
Timestamped snapshots can also be used with user-defined timestamp.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9879
Test Plan:
```
make check
TEST_TMPDIR=/dev/shm make crash_test_with_txn
```
Reviewed By: siying
Differential Revision: D35783919
Pulled By: riversand963
fbshipit-source-id: 586ad905e169189e19d3bfc0cb0177a7239d1bd4
3 years ago
|
|
|
if (thread && FLAGS_create_timestamped_snapshot_one_in &&
|
|
|
|
thread->rand.OneIn(FLAGS_create_timestamped_snapshot_one_in)) {
|
|
|
|
uint64_t ts = db_stress_env->NowNanos();
|
|
|
|
s = txn->CommitAndTryCreateSnapshot(/*notifier=*/nullptr, ts,
|
|
|
|
×tamped_snapshot);
|
|
|
|
} else {
|
|
|
|
s = txn->Commit();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
delete txn;
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
Status StressTest::RollbackTxn(Transaction* txn) {
|
|
|
|
if (!FLAGS_use_txn) {
|
|
|
|
return Status::InvalidArgument(
|
|
|
|
"RollbackTxn when FLAGS_use_txn is not"
|
|
|
|
" set");
|
|
|
|
}
|
|
|
|
Status s = txn->Rollback();
|
|
|
|
delete txn;
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
void StressTest::OperateDb(ThreadState* thread) {
|
|
|
|
ReadOptions read_opts(FLAGS_verify_checksum, true);
|
|
|
|
read_opts.rate_limiter_priority =
|
|
|
|
FLAGS_rate_limit_user_ops ? Env::IO_USER : Env::IO_TOTAL;
|
|
|
|
read_opts.async_io = FLAGS_async_io;
|
|
|
|
read_opts.adaptive_readahead = FLAGS_adaptive_readahead;
|
|
|
|
WriteOptions write_opts;
|
Rate-limit automatic WAL flush after each user write (#9607)
Summary:
**Context:**
WAL flush is currently not rate-limited by `Options::rate_limiter`. This PR is to provide rate-limiting to auto WAL flush, the one that automatically happen after each user write operation (i.e, `Options::manual_wal_flush == false`), by adding `WriteOptions::rate_limiter_options`.
Note that we are NOT rate-limiting WAL flush that do NOT automatically happen after each user write, such as `Options::manual_wal_flush == true + manual FlushWAL()` (rate-limiting multiple WAL flushes), for the benefits of:
- being consistent with [ReadOptions::rate_limiter_priority](https://github.com/facebook/rocksdb/blob/7.0.fb/include/rocksdb/options.h#L515)
- being able to turn off some WAL flush's rate-limiting but not all (e.g, turn off specific the WAL flush of a critical user write like a service's heartbeat)
`WriteOptions::rate_limiter_options` only accept `Env::IO_USER` and `Env::IO_TOTAL` currently due to an implementation constraint.
- The constraint is that we currently queue parallel writes (including WAL writes) based on FIFO policy which does not factor rate limiter priority into this layer's scheduling. If we allow lower priorities such as `Env::IO_HIGH/MID/LOW` and such writes specified with lower priorities occurs before ones specified with higher priorities (even just by a tiny bit in arrival time), the former would have blocked the latter, leading to a "priority inversion" issue and contradictory to what we promise for rate-limiting priority. Therefore we only allow `Env::IO_USER` and `Env::IO_TOTAL` right now before improving that scheduling.
A pre-requisite to this feature is to support operation-level rate limiting in `WritableFileWriter`, which is also included in this PR.
**Summary:**
- Renamed test suite `DBRateLimiterTest to DBRateLimiterOnReadTest` for adding a new test suite
- Accept `rate_limiter_priority` in `WritableFileWriter`'s private and public write functions
- Passed `WriteOptions::rate_limiter_options` to `WritableFileWriter` in the path of automatic WAL flush.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9607
Test Plan:
- Added new unit test to verify existing flush/compaction rate-limiting does not break, since `DBTest, RateLimitingTest` is disabled and current db-level rate-limiting tests focus on read only (e.g, `db_rate_limiter_test`, `DBTest2, RateLimitedCompactionReads`).
- Added new unit test `DBRateLimiterOnWriteWALTest, AutoWalFlush`
- `strace -ftt -e trace=write ./db_bench -benchmarks=fillseq -db=/dev/shm/testdb -rate_limit_auto_wal_flush=1 -rate_limiter_bytes_per_sec=15 -rate_limiter_refill_period_us=1000000 -write_buffer_size=100000000 -disable_auto_compactions=1 -num=100`
- verified that WAL flush(i.e, system-call _write_) were chunked into 15 bytes and each _write_ was roughly 1 second apart
- verified the chunking disappeared when `-rate_limit_auto_wal_flush=0`
- crash test: `python3 tools/db_crashtest.py blackbox --disable_wal=0 --rate_limit_auto_wal_flush=1 --rate_limiter_bytes_per_sec=10485760 --interval=10` killed as normal
**Benchmarked on flush/compaction to ensure no performance regression:**
- compaction with rate-limiting (see table 1, avg over 1280-run): pre-change: **915635 micros/op**; post-change:
**907350 micros/op (improved by 0.106%)**
```
#!/bin/bash
TEST_TMPDIR=/dev/shm/testdb
START=1
NUM_DATA_ENTRY=8
N=10
rm -f compact_bmk_output.txt compact_bmk_output_2.txt dont_care_output.txt
for i in $(eval echo "{$START..$NUM_DATA_ENTRY}")
do
NUM_RUN=$(($N*(2**($i-1))))
for j in $(eval echo "{$START..$NUM_RUN}")
do
./db_bench --benchmarks=fillrandom -db=$TEST_TMPDIR -disable_auto_compactions=1 -write_buffer_size=6710886 > dont_care_output.txt && ./db_bench --benchmarks=compact -use_existing_db=1 -db=$TEST_TMPDIR -level0_file_num_compaction_trigger=1 -rate_limiter_bytes_per_sec=100000000 | egrep 'compact'
done > compact_bmk_output.txt && awk -v NUM_RUN=$NUM_RUN '{sum+=$3;sum_sqrt+=$3^2}END{print sum/NUM_RUN, sqrt(sum_sqrt/NUM_RUN-(sum/NUM_RUN)^2)}' compact_bmk_output.txt >> compact_bmk_output_2.txt
done
```
- compaction w/o rate-limiting (see table 2, avg over 640-run): pre-change: **822197 micros/op**; post-change: **823148 micros/op (regressed by 0.12%)**
```
Same as above script, except that -rate_limiter_bytes_per_sec=0
```
- flush with rate-limiting (see table 3, avg over 320-run, run on the [patch](https://github.com/hx235/rocksdb/commit/ee5c6023a9f6533fab9afdc681568daa21da4953) to augment current db_bench ): pre-change: **745752 micros/op**; post-change: **745331 micros/op (regressed by 0.06 %)**
```
#!/bin/bash
TEST_TMPDIR=/dev/shm/testdb
START=1
NUM_DATA_ENTRY=8
N=10
rm -f flush_bmk_output.txt flush_bmk_output_2.txt
for i in $(eval echo "{$START..$NUM_DATA_ENTRY}")
do
NUM_RUN=$(($N*(2**($i-1))))
for j in $(eval echo "{$START..$NUM_RUN}")
do
./db_bench -db=$TEST_TMPDIR -write_buffer_size=1048576000 -num=1000000 -rate_limiter_bytes_per_sec=100000000 -benchmarks=fillseq,flush | egrep 'flush'
done > flush_bmk_output.txt && awk -v NUM_RUN=$NUM_RUN '{sum+=$3;sum_sqrt+=$3^2}END{print sum/NUM_RUN, sqrt(sum_sqrt/NUM_RUN-(sum/NUM_RUN)^2)}' flush_bmk_output.txt >> flush_bmk_output_2.txt
done
```
- flush w/o rate-limiting (see table 4, avg over 320-run, run on the [patch](https://github.com/hx235/rocksdb/commit/ee5c6023a9f6533fab9afdc681568daa21da4953) to augment current db_bench): pre-change: **487512 micros/op**, post-change: **485856 micors/ops (improved by 0.34%)**
```
Same as above script, except that -rate_limiter_bytes_per_sec=0
```
| table 1 - compact with rate-limiting|
#-run | (pre-change) avg micros/op | std micros/op | (post-change) avg micros/op | std micros/op | change in avg micros/op (%)
-- | -- | -- | -- | -- | --
10 | 896978 | 16046.9 | 901242 | 15670.9 | 0.475373978
20 | 893718 | 15813 | 886505 | 17544.7 | -0.8070778478
40 | 900426 | 23882.2 | 894958 | 15104.5 | -0.6072681153
80 | 906635 | 21761.5 | 903332 | 23948.3 | -0.3643141948
160 | 898632 | 21098.9 | 907583 | 21145 | 0.9960695813
3.20E+02 | 905252 | 22785.5 | 908106 | 25325.5 | 0.3152713278
6.40E+02 | 905213 | 23598.6 | 906741 | 21370.5 | 0.1688000504
**1.28E+03** | **908316** | **23533.1** | **907350** | **24626.8** | **-0.1063506533**
average over #-run | 901896.25 | 21064.9625 | 901977.125 | 20592.025 | 0.008967217682
| table 2 - compact w/o rate-limiting|
#-run | (pre-change) avg micros/op | std micros/op | (post-change) avg micros/op | std micros/op | change in avg micros/op (%)
-- | -- | -- | -- | -- | --
10 | 811211 | 26996.7 | 807586 | 28456.4 | -0.4468627768
20 | 815465 | 14803.7 | 814608 | 28719.7 | -0.105093413
40 | 809203 | 26187.1 | 797835 | 25492.1 | -1.404839082
80 | 822088 | 28765.3 | 822192 | 32840.4 | 0.01265071379
160 | 821719 | 36344.7 | 821664 | 29544.9 | -0.006693285661
3.20E+02 | 820921 | 27756.4 | 821403 | 28347.7 | 0.05871454135
**6.40E+02** | **822197** | **28960.6** | **823148** | **30055.1** | **0.1156657103**
average over #-run | 8.18E+05 | 2.71E+04 | 8.15E+05 | 2.91E+04 | -0.25
| table 3 - flush with rate-limiting|
#-run | (pre-change) avg micros/op | std micros/op | (post-change) avg micros/op | std micros/op | change in avg micros/op (%)
-- | -- | -- | -- | -- | --
10 | 741721 | 11770.8 | 740345 | 5949.76 | -0.1855144994
20 | 735169 | 3561.83 | 743199 | 9755.77 | 1.09226586
40 | 743368 | 8891.03 | 742102 | 8683.22 | -0.1703059588
80 | 742129 | 8148.51 | 743417 | 9631.58| 0.1735547324
160 | 749045 | 9757.21 | 746256 | 9191.86 | -0.3723407806
**3.20E+02** | **745752** | **9819.65** | **745331** | **9840.62** | **-0.0564530836**
6.40E+02 | 749006 | 11080.5 | 748173 | 10578.7 | -0.1112140624
average over #-run | 743741.4286 | 9004.218571 | 744117.5714 | 9090.215714 | 0.05057441238
| table 4 - flush w/o rate-limiting|
#-run | (pre-change) avg micros/op | std micros/op | (post-change) avg micros/op | std micros/op | change in avg micros/op (%)
-- | -- | -- | -- | -- | --
10 | 477283 | 24719.6 | 473864 | 12379 | -0.7163464863
20 | 486743 | 20175.2 | 502296 | 23931.3 | 3.195320734
40 | 482846 | 15309.2 | 489820 | 22259.5 | 1.444352858
80 | 491490 | 21883.1 | 490071 | 23085.7 | -0.2887139108
160 | 493347 | 28074.3 | 483609 | 21211.7 | -1.973864238
**3.20E+02** | **487512** | **21401.5** | **485856** | **22195.2** | **-0.3396839462**
6.40E+02 | 490307 | 25418.6 | 485435 | 22405.2 | -0.9936631539
average over #-run | 4.87E+05 | 2.24E+04 | 4.87E+05 | 2.11E+04 | 0.00E+00
Reviewed By: ajkr
Differential Revision: D34442441
Pulled By: hx235
fbshipit-source-id: 4790f13e1e5c0a95ae1d1cc93ffcf69dc6e78bdd
3 years ago
|
|
|
if (FLAGS_rate_limit_auto_wal_flush) {
|
|
|
|
write_opts.rate_limiter_priority = Env::IO_USER;
|
|
|
|
}
|
|
|
|
auto shared = thread->shared;
|
|
|
|
char value[100];
|
|
|
|
std::string from_db;
|
|
|
|
if (FLAGS_sync) {
|
|
|
|
write_opts.sync = true;
|
|
|
|
}
|
|
|
|
write_opts.disableWAL = FLAGS_disable_wal;
|
|
|
|
const int prefix_bound = static_cast<int>(FLAGS_readpercent) +
|
|
|
|
static_cast<int>(FLAGS_prefixpercent);
|
|
|
|
const int write_bound = prefix_bound + static_cast<int>(FLAGS_writepercent);
|
|
|
|
const int del_bound = write_bound + static_cast<int>(FLAGS_delpercent);
|
|
|
|
const int delrange_bound =
|
|
|
|
del_bound + static_cast<int>(FLAGS_delrangepercent);
|
|
|
|
const int iterate_bound =
|
|
|
|
delrange_bound + static_cast<int>(FLAGS_iterpercent);
|
|
|
|
|
|
|
|
const uint64_t ops_per_open = FLAGS_ops_per_thread / (FLAGS_reopen + 1);
|
|
|
|
|
|
|
|
#ifndef NDEBUG
|
|
|
|
if (FLAGS_read_fault_one_in) {
|
|
|
|
fault_fs_guard->SetThreadLocalReadErrorContext(thread->shared->GetSeed(),
|
|
|
|
FLAGS_read_fault_one_in);
|
|
|
|
}
|
|
|
|
#endif // NDEBUG
|
|
|
|
if (FLAGS_write_fault_one_in) {
|
|
|
|
IOStatus error_msg;
|
|
|
|
if (FLAGS_injest_error_severity <= 1 || FLAGS_injest_error_severity > 2) {
|
|
|
|
error_msg = IOStatus::IOError("Retryable IO Error");
|
|
|
|
error_msg.SetRetryable(true);
|
|
|
|
} else if (FLAGS_injest_error_severity == 2) {
|
|
|
|
// Ingest the fatal error
|
|
|
|
error_msg = IOStatus::IOError("Fatal IO Error");
|
|
|
|
error_msg.SetDataLoss(true);
|
|
|
|
}
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
std::vector<FileType> types = {FileType::kTableFile,
|
|
|
|
FileType::kDescriptorFile,
|
|
|
|
FileType::kCurrentFile};
|
|
|
|
fault_fs_guard->SetRandomWriteError(
|
|
|
|
thread->shared->GetSeed(), FLAGS_write_fault_one_in, error_msg,
|
|
|
|
/*inject_for_all_file_types=*/false, types);
|
|
|
|
}
|
|
|
|
thread->stats.Start();
|
|
|
|
for (int open_cnt = 0; open_cnt <= FLAGS_reopen; ++open_cnt) {
|
|
|
|
if (thread->shared->HasVerificationFailedYet() ||
|
|
|
|
thread->shared->ShouldStopTest()) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (open_cnt != 0) {
|
|
|
|
thread->stats.FinishedSingleOp();
|
|
|
|
MutexLock l(thread->shared->GetMutex());
|
|
|
|
while (!thread->snapshot_queue.empty()) {
|
|
|
|
db_->ReleaseSnapshot(thread->snapshot_queue.front().second.snapshot);
|
|
|
|
delete thread->snapshot_queue.front().second.key_vec;
|
|
|
|
thread->snapshot_queue.pop();
|
|
|
|
}
|
|
|
|
thread->shared->IncVotedReopen();
|
|
|
|
if (thread->shared->AllVotedReopen()) {
|
|
|
|
thread->shared->GetStressTest()->Reopen(thread);
|
|
|
|
thread->shared->GetCondVar()->SignalAll();
|
|
|
|
} else {
|
|
|
|
thread->shared->GetCondVar()->Wait();
|
|
|
|
}
|
|
|
|
// Commenting this out as we don't want to reset stats on each open.
|
|
|
|
// thread->stats.Start();
|
|
|
|
}
|
|
|
|
|
|
|
|
for (uint64_t i = 0; i < ops_per_open; i++) {
|
|
|
|
if (thread->shared->HasVerificationFailedYet()) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Change Options
|
|
|
|
if (thread->rand.OneInOpt(FLAGS_set_options_one_in)) {
|
|
|
|
SetOptions(thread);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (thread->rand.OneInOpt(FLAGS_set_in_place_one_in)) {
|
|
|
|
options_.inplace_update_support ^= options_.inplace_update_support;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (thread->tid == 0 && FLAGS_verify_db_one_in > 0 &&
|
|
|
|
thread->rand.OneIn(FLAGS_verify_db_one_in)) {
|
|
|
|
ContinuouslyVerifyDb(thread);
|
|
|
|
if (thread->shared->ShouldStopTest()) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
MaybeClearOneColumnFamily(thread);
|
|
|
|
|
|
|
|
if (thread->rand.OneInOpt(FLAGS_sync_wal_one_in)) {
|
|
|
|
Status s = db_->SyncWAL();
|
|
|
|
if (!s.ok() && !s.IsNotSupported()) {
|
|
|
|
fprintf(stderr, "SyncWAL() failed: %s\n", s.ToString().c_str());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
int rand_column_family = thread->rand.Next() % FLAGS_column_families;
|
|
|
|
ColumnFamilyHandle* column_family = column_families_[rand_column_family];
|
|
|
|
|
|
|
|
if (thread->rand.OneInOpt(FLAGS_compact_files_one_in)) {
|
|
|
|
TestCompactFiles(thread, column_family);
|
|
|
|
}
|
|
|
|
|
|
|
|
int64_t rand_key = GenerateOneKey(thread, i);
|
|
|
|
std::string keystr = Key(rand_key);
|
|
|
|
Slice key = keystr;
|
|
|
|
std::unique_ptr<MutexLock> lock;
|
|
|
|
if (ShouldAcquireMutexOnKey()) {
|
|
|
|
lock.reset(new MutexLock(
|
|
|
|
shared->GetMutexForKey(rand_column_family, rand_key)));
|
|
|
|
}
|
|
|
|
|
|
|
|
if (thread->rand.OneInOpt(FLAGS_compact_range_one_in)) {
|
|
|
|
TestCompactRange(thread, rand_key, key, column_family);
|
|
|
|
if (thread->shared->HasVerificationFailedYet()) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
std::vector<int> rand_column_families =
|
|
|
|
GenerateColumnFamilies(FLAGS_column_families, rand_column_family);
|
|
|
|
|
|
|
|
if (thread->rand.OneInOpt(FLAGS_flush_one_in)) {
|
|
|
|
Status status = TestFlush(rand_column_families);
|
|
|
|
if (!status.ok()) {
|
|
|
|
fprintf(stdout, "Unable to perform Flush(): %s\n",
|
|
|
|
status.ToString().c_str());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
// Verify GetLiveFiles with a 1 in N chance.
|
|
|
|
if (thread->rand.OneInOpt(FLAGS_get_live_files_one_in) &&
|
|
|
|
!FLAGS_write_fault_one_in) {
|
|
|
|
Status status = VerifyGetLiveFiles();
|
|
|
|
if (!status.ok()) {
|
|
|
|
VerificationAbort(shared, "VerifyGetLiveFiles status not OK", status);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Verify GetSortedWalFiles with a 1 in N chance.
|
|
|
|
if (thread->rand.OneInOpt(FLAGS_get_sorted_wal_files_one_in)) {
|
|
|
|
Status status = VerifyGetSortedWalFiles();
|
|
|
|
if (!status.ok()) {
|
|
|
|
VerificationAbort(shared, "VerifyGetSortedWalFiles status not OK",
|
|
|
|
status);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Verify GetCurrentWalFile with a 1 in N chance.
|
|
|
|
if (thread->rand.OneInOpt(FLAGS_get_current_wal_file_one_in)) {
|
|
|
|
Status status = VerifyGetCurrentWalFile();
|
|
|
|
if (!status.ok()) {
|
|
|
|
VerificationAbort(shared, "VerifyGetCurrentWalFile status not OK",
|
|
|
|
status);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
|
|
|
|
if (thread->rand.OneInOpt(FLAGS_pause_background_one_in)) {
|
|
|
|
Status status = TestPauseBackground(thread);
|
|
|
|
if (!status.ok()) {
|
|
|
|
VerificationAbort(
|
|
|
|
shared, "Pause/ContinueBackgroundWork status not OK", status);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
if (thread->rand.OneInOpt(FLAGS_verify_checksum_one_in)) {
|
|
|
|
Status status = db_->VerifyChecksum();
|
|
|
|
if (!status.ok()) {
|
|
|
|
VerificationAbort(shared, "VerifyChecksum status not OK", status);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (thread->rand.OneInOpt(FLAGS_get_property_one_in)) {
|
|
|
|
TestGetProperty(thread);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
std::vector<int64_t> rand_keys = GenerateKeys(rand_key);
|
|
|
|
|
|
|
|
if (thread->rand.OneInOpt(FLAGS_ingest_external_file_one_in)) {
|
|
|
|
TestIngestExternalFile(thread, rand_column_families, rand_keys, lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (thread->rand.OneInOpt(FLAGS_backup_one_in)) {
|
Fix, enable, and enhance backup/restore in db_stress (#7348)
Summary:
Although added to db_stress, testing of backup/restore
was never integrated into the crash test, originally concerned about
performance. I've enabled it now and to address the peformance concern,
testing backup/restore is always skipped once the db exceeds a certain
size threshold, default 100MB. This should provide sufficient
opportunity for testing BackupEngine without bogging down everything
else with heavier and heavier operations.
Also fixed backup/restore in db_stress by making sure PurgeOldBackups
can remove manifest files, which are normally kept around for db_stress.
Added more coverage of backup options, and up to three backups being
saved in one backup directory (in some cases).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7348
Test Plan:
ran 'make blackbox_crash_test' for a while, with heightened
probabilitly of taking backups (1/10k). Also confirmed with some debug
output that the code is being covered, TestBackupRestore only takes
a few seconds to complete when triggered, and even at 1/10k and ~50MB
database, there's <,~ 1 thread testing backups at any time.
Reviewed By: ajkr
Differential Revision: D23510835
Pulled By: pdillinger
fbshipit-source-id: b6b8735591808141f81f10773ac31634cf03b6c0
4 years ago
|
|
|
// Beyond a certain DB size threshold, this test becomes heavier than
|
|
|
|
// it's worth.
|
|
|
|
uint64_t total_size = 0;
|
|
|
|
if (FLAGS_backup_max_size > 0) {
|
|
|
|
std::vector<FileAttributes> files;
|
|
|
|
db_stress_env->GetChildrenFileAttributes(FLAGS_db, &files);
|
|
|
|
for (auto& file : files) {
|
|
|
|
total_size += file.size_bytes;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (total_size <= FLAGS_backup_max_size) {
|
|
|
|
Status s = TestBackupRestore(thread, rand_column_families, rand_keys);
|
|
|
|
if (!s.ok()) {
|
|
|
|
VerificationAbort(shared, "Backup/restore gave inconsistent state",
|
|
|
|
s);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (thread->rand.OneInOpt(FLAGS_checkpoint_one_in)) {
|
|
|
|
Status s = TestCheckpoint(thread, rand_column_families, rand_keys);
|
|
|
|
if (!s.ok()) {
|
|
|
|
VerificationAbort(shared, "Checkpoint gave inconsistent state", s);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
if (thread->rand.OneInOpt(FLAGS_approximate_size_one_in)) {
|
|
|
|
Status s =
|
|
|
|
TestApproximateSize(thread, i, rand_column_families, rand_keys);
|
|
|
|
if (!s.ok()) {
|
|
|
|
VerificationAbort(shared, "ApproximateSize Failed", s);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
if (thread->rand.OneInOpt(FLAGS_acquire_snapshot_one_in)) {
|
|
|
|
TestAcquireSnapshot(thread, rand_column_family, keystr, i);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*always*/ {
|
|
|
|
Status s = MaybeReleaseSnapshots(thread, i);
|
|
|
|
if (!s.ok()) {
|
|
|
|
VerificationAbort(shared, "Snapshot gave inconsistent state", s);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
// Assign timestamps if necessary.
|
|
|
|
std::string read_ts_str;
|
|
|
|
std::string write_ts_str;
|
|
|
|
Slice read_ts;
|
|
|
|
Slice write_ts;
|
|
|
|
if (ShouldAcquireMutexOnKey() && FLAGS_user_timestamp_size > 0) {
|
|
|
|
read_ts_str = GenerateTimestampForRead();
|
|
|
|
read_ts = read_ts_str;
|
|
|
|
read_opts.timestamp = &read_ts;
|
|
|
|
write_ts_str = NowNanosStr();
|
|
|
|
write_ts = write_ts_str;
|
|
|
|
}
|
|
|
|
|
|
|
|
int prob_op = thread->rand.Uniform(100);
|
|
|
|
// Reset this in case we pick something other than a read op. We don't
|
|
|
|
// want to use a stale value when deciding at the beginning of the loop
|
|
|
|
// whether to vote to reopen
|
|
|
|
if (prob_op >= 0 && prob_op < static_cast<int>(FLAGS_readpercent)) {
|
|
|
|
assert(0 <= prob_op);
|
|
|
|
// OPERATION read
|
|
|
|
if (FLAGS_use_multiget) {
|
|
|
|
// Leave room for one more iteration of the loop with a single key
|
|
|
|
// batch. This is to ensure that each thread does exactly the same
|
|
|
|
// number of ops
|
|
|
|
int multiget_batch_size = static_cast<int>(
|
|
|
|
std::min(static_cast<uint64_t>(thread->rand.Uniform(64)),
|
|
|
|
FLAGS_ops_per_thread - i - 1));
|
|
|
|
// If its the last iteration, ensure that multiget_batch_size is 1
|
|
|
|
multiget_batch_size = std::max(multiget_batch_size, 1);
|
|
|
|
rand_keys = GenerateNKeys(thread, multiget_batch_size, i);
|
|
|
|
TestMultiGet(thread, read_opts, rand_column_families, rand_keys);
|
|
|
|
i += multiget_batch_size - 1;
|
|
|
|
} else {
|
|
|
|
TestGet(thread, read_opts, rand_column_families, rand_keys);
|
|
|
|
}
|
|
|
|
} else if (prob_op < prefix_bound) {
|
|
|
|
assert(static_cast<int>(FLAGS_readpercent) <= prob_op);
|
|
|
|
// OPERATION prefix scan
|
|
|
|
// keys are 8 bytes long, prefix size is FLAGS_prefix_size. There are
|
|
|
|
// (8 - FLAGS_prefix_size) bytes besides the prefix. So there will
|
|
|
|
// be 2 ^ ((8 - FLAGS_prefix_size) * 8) possible keys with the same
|
|
|
|
// prefix
|
|
|
|
TestPrefixScan(thread, read_opts, rand_column_families, rand_keys);
|
|
|
|
} else if (prob_op < write_bound) {
|
|
|
|
assert(prefix_bound <= prob_op);
|
|
|
|
// OPERATION write
|
|
|
|
TestPut(thread, write_opts, read_opts, rand_column_families, rand_keys,
|
|
|
|
value, lock);
|
|
|
|
} else if (prob_op < del_bound) {
|
|
|
|
assert(write_bound <= prob_op);
|
|
|
|
// OPERATION delete
|
|
|
|
TestDelete(thread, write_opts, rand_column_families, rand_keys, lock);
|
|
|
|
} else if (prob_op < delrange_bound) {
|
|
|
|
assert(del_bound <= prob_op);
|
|
|
|
// OPERATION delete range
|
|
|
|
TestDeleteRange(thread, write_opts, rand_column_families, rand_keys,
|
|
|
|
lock);
|
|
|
|
} else if (prob_op < iterate_bound) {
|
|
|
|
assert(delrange_bound <= prob_op);
|
|
|
|
// OPERATION iterate
|
|
|
|
int num_seeks = static_cast<int>(
|
|
|
|
std::min(static_cast<uint64_t>(thread->rand.Uniform(4)),
|
|
|
|
FLAGS_ops_per_thread - i - 1));
|
|
|
|
rand_keys = GenerateNKeys(thread, num_seeks, i);
|
|
|
|
i += num_seeks - 1;
|
|
|
|
TestIterate(thread, read_opts, rand_column_families, rand_keys);
|
|
|
|
} else {
|
|
|
|
assert(iterate_bound <= prob_op);
|
|
|
|
TestCustomOperations(thread, rand_column_families);
|
|
|
|
}
|
|
|
|
thread->stats.FinishedSingleOp();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
while (!thread->snapshot_queue.empty()) {
|
|
|
|
db_->ReleaseSnapshot(thread->snapshot_queue.front().second.snapshot);
|
|
|
|
delete thread->snapshot_queue.front().second.key_vec;
|
|
|
|
thread->snapshot_queue.pop();
|
|
|
|
}
|
|
|
|
|
|
|
|
thread->stats.Stop();
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
// Generated a list of keys that close to boundaries of SST keys.
|
|
|
|
// If there isn't any SST file in the DB, return empty list.
|
|
|
|
std::vector<std::string> StressTest::GetWhiteBoxKeys(ThreadState* thread,
|
|
|
|
DB* db,
|
|
|
|
ColumnFamilyHandle* cfh,
|
|
|
|
size_t num_keys) {
|
|
|
|
ColumnFamilyMetaData cfmd;
|
|
|
|
db->GetColumnFamilyMetaData(cfh, &cfmd);
|
|
|
|
std::vector<std::string> boundaries;
|
|
|
|
for (const LevelMetaData& lmd : cfmd.levels) {
|
|
|
|
for (const SstFileMetaData& sfmd : lmd.files) {
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
// If FLAGS_user_timestamp_size > 0, then both smallestkey and largestkey
|
|
|
|
// have timestamps.
|
|
|
|
const auto& skey = sfmd.smallestkey;
|
|
|
|
const auto& lkey = sfmd.largestkey;
|
|
|
|
assert(skey.size() >= FLAGS_user_timestamp_size);
|
|
|
|
assert(lkey.size() >= FLAGS_user_timestamp_size);
|
|
|
|
boundaries.push_back(
|
|
|
|
skey.substr(0, skey.size() - FLAGS_user_timestamp_size));
|
|
|
|
boundaries.push_back(
|
|
|
|
lkey.substr(0, lkey.size() - FLAGS_user_timestamp_size));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (boundaries.empty()) {
|
|
|
|
return {};
|
|
|
|
}
|
|
|
|
|
|
|
|
std::vector<std::string> ret;
|
|
|
|
for (size_t j = 0; j < num_keys; j++) {
|
|
|
|
std::string k =
|
|
|
|
boundaries[thread->rand.Uniform(static_cast<int>(boundaries.size()))];
|
|
|
|
if (thread->rand.OneIn(3)) {
|
|
|
|
// Reduce one byte from the string
|
|
|
|
for (int i = static_cast<int>(k.length()) - 1; i >= 0; i--) {
|
|
|
|
uint8_t cur = k[i];
|
|
|
|
if (cur > 0) {
|
|
|
|
k[i] = static_cast<char>(cur - 1);
|
|
|
|
break;
|
|
|
|
} else if (i > 0) {
|
|
|
|
k[i] = 0xFFu;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else if (thread->rand.OneIn(2)) {
|
|
|
|
// Add one byte to the string
|
|
|
|
for (int i = static_cast<int>(k.length()) - 1; i >= 0; i--) {
|
|
|
|
uint8_t cur = k[i];
|
|
|
|
if (cur < 255) {
|
|
|
|
k[i] = static_cast<char>(cur + 1);
|
|
|
|
break;
|
|
|
|
} else if (i > 0) {
|
|
|
|
k[i] = 0x00;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
ret.push_back(k);
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
|
|
|
|
// Given a key K, this creates an iterator which scans to K and then
|
|
|
|
// does a random sequence of Next/Prev operations.
|
|
|
|
Status StressTest::TestIterate(ThreadState* thread,
|
|
|
|
const ReadOptions& read_opts,
|
|
|
|
const std::vector<int>& rand_column_families,
|
|
|
|
const std::vector<int64_t>& rand_keys) {
|
|
|
|
Status s;
|
|
|
|
const Snapshot* snapshot = db_->GetSnapshot();
|
|
|
|
ReadOptions readoptionscopy = read_opts;
|
|
|
|
readoptionscopy.snapshot = snapshot;
|
|
|
|
|
|
|
|
bool expect_total_order = false;
|
|
|
|
if (thread->rand.OneIn(16)) {
|
|
|
|
// When prefix extractor is used, it's useful to cover total order seek.
|
|
|
|
readoptionscopy.total_order_seek = true;
|
|
|
|
expect_total_order = true;
|
|
|
|
} else if (thread->rand.OneIn(4)) {
|
|
|
|
readoptionscopy.total_order_seek = false;
|
|
|
|
readoptionscopy.auto_prefix_mode = true;
|
|
|
|
expect_total_order = true;
|
|
|
|
} else if (options_.prefix_extractor.get() == nullptr) {
|
|
|
|
expect_total_order = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
std::string upper_bound_str;
|
|
|
|
Slice upper_bound;
|
|
|
|
if (thread->rand.OneIn(16)) {
|
|
|
|
// in 1/16 chance, set a iterator upper bound
|
|
|
|
int64_t rand_upper_key = GenerateOneKey(thread, FLAGS_ops_per_thread);
|
|
|
|
upper_bound_str = Key(rand_upper_key);
|
|
|
|
upper_bound = Slice(upper_bound_str);
|
|
|
|
// uppder_bound can be smaller than seek key, but the query itself
|
|
|
|
// should not crash either.
|
|
|
|
readoptionscopy.iterate_upper_bound = &upper_bound;
|
|
|
|
}
|
|
|
|
std::string lower_bound_str;
|
|
|
|
Slice lower_bound;
|
|
|
|
if (thread->rand.OneIn(16)) {
|
|
|
|
// in 1/16 chance, enable iterator lower bound
|
|
|
|
int64_t rand_lower_key = GenerateOneKey(thread, FLAGS_ops_per_thread);
|
|
|
|
lower_bound_str = Key(rand_lower_key);
|
|
|
|
lower_bound = Slice(lower_bound_str);
|
|
|
|
// uppder_bound can be smaller than seek key, but the query itself
|
|
|
|
// should not crash either.
|
|
|
|
readoptionscopy.iterate_lower_bound = &lower_bound;
|
|
|
|
}
|
|
|
|
|
|
|
|
auto cfh = column_families_[rand_column_families[0]];
|
|
|
|
std::unique_ptr<Iterator> iter(db_->NewIterator(readoptionscopy, cfh));
|
|
|
|
|
|
|
|
std::vector<std::string> key_str;
|
|
|
|
if (thread->rand.OneIn(16)) {
|
|
|
|
// Generate keys close to lower or upper bound of SST files.
|
|
|
|
key_str = GetWhiteBoxKeys(thread, db_, cfh, rand_keys.size());
|
|
|
|
}
|
|
|
|
if (key_str.empty()) {
|
|
|
|
// If key string is not geneerated using white block keys,
|
|
|
|
// Use randomized key passe in.
|
|
|
|
for (int64_t rkey : rand_keys) {
|
|
|
|
key_str.push_back(Key(rkey));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
std::string op_logs;
|
|
|
|
const size_t kOpLogsLimit = 10000;
|
|
|
|
|
|
|
|
for (const std::string& skey : key_str) {
|
|
|
|
if (op_logs.size() > kOpLogsLimit) {
|
|
|
|
// Shouldn't take too much memory for the history log. Clear it.
|
|
|
|
op_logs = "(cleared...)\n";
|
|
|
|
}
|
|
|
|
|
|
|
|
Slice key = skey;
|
|
|
|
|
|
|
|
if (readoptionscopy.iterate_upper_bound != nullptr &&
|
|
|
|
thread->rand.OneIn(2)) {
|
|
|
|
// 1/2 chance, change the upper bound.
|
|
|
|
// It is possible that it is changed without first use, but there is no
|
|
|
|
// problem with that.
|
|
|
|
int64_t rand_upper_key = GenerateOneKey(thread, FLAGS_ops_per_thread);
|
|
|
|
upper_bound_str = Key(rand_upper_key);
|
|
|
|
upper_bound = Slice(upper_bound_str);
|
|
|
|
} else if (readoptionscopy.iterate_lower_bound != nullptr &&
|
|
|
|
thread->rand.OneIn(4)) {
|
|
|
|
// 1/4 chance, change the lower bound.
|
|
|
|
// It is possible that it is changed without first use, but there is no
|
|
|
|
// problem with that.
|
|
|
|
int64_t rand_lower_key = GenerateOneKey(thread, FLAGS_ops_per_thread);
|
|
|
|
lower_bound_str = Key(rand_lower_key);
|
|
|
|
lower_bound = Slice(lower_bound_str);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Record some options to op_logs;
|
|
|
|
op_logs += "total_order_seek: ";
|
|
|
|
op_logs += (readoptionscopy.total_order_seek ? "1 " : "0 ");
|
|
|
|
op_logs += "auto_prefix_mode: ";
|
|
|
|
op_logs += (readoptionscopy.auto_prefix_mode ? "1 " : "0 ");
|
|
|
|
if (readoptionscopy.iterate_upper_bound != nullptr) {
|
|
|
|
op_logs += "ub: " + upper_bound.ToString(true) + " ";
|
|
|
|
}
|
|
|
|
if (readoptionscopy.iterate_lower_bound != nullptr) {
|
|
|
|
op_logs += "lb: " + lower_bound.ToString(true) + " ";
|
|
|
|
}
|
|
|
|
|
|
|
|
// Set up an iterator and does the same without bounds and with total
|
|
|
|
// order seek and compare the results. This is to identify bugs related
|
|
|
|
// to bounds, prefix extractor or reseeking. Sometimes we are comparing
|
|
|
|
// iterators with the same set-up, and it doesn't hurt to check them
|
|
|
|
// to be equal.
|
|
|
|
// This `ReadOptions` is for validation purposes. Ignore
|
|
|
|
// `FLAGS_rate_limit_user_ops` to avoid slowing any validation.
|
|
|
|
ReadOptions cmp_ro;
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
cmp_ro.timestamp = readoptionscopy.timestamp;
|
|
|
|
cmp_ro.snapshot = snapshot;
|
|
|
|
cmp_ro.total_order_seek = true;
|
|
|
|
ColumnFamilyHandle* cmp_cfh =
|
|
|
|
GetControlCfh(thread, rand_column_families[0]);
|
|
|
|
std::unique_ptr<Iterator> cmp_iter(db_->NewIterator(cmp_ro, cmp_cfh));
|
|
|
|
bool diverged = false;
|
|
|
|
|
|
|
|
bool support_seek_first_or_last = expect_total_order;
|
|
|
|
|
|
|
|
LastIterateOp last_op;
|
|
|
|
if (support_seek_first_or_last && thread->rand.OneIn(100)) {
|
|
|
|
iter->SeekToFirst();
|
|
|
|
cmp_iter->SeekToFirst();
|
|
|
|
last_op = kLastOpSeekToFirst;
|
|
|
|
op_logs += "STF ";
|
|
|
|
} else if (support_seek_first_or_last && thread->rand.OneIn(100)) {
|
|
|
|
iter->SeekToLast();
|
|
|
|
cmp_iter->SeekToLast();
|
|
|
|
last_op = kLastOpSeekToLast;
|
|
|
|
op_logs += "STL ";
|
|
|
|
} else if (thread->rand.OneIn(8)) {
|
|
|
|
iter->SeekForPrev(key);
|
|
|
|
cmp_iter->SeekForPrev(key);
|
|
|
|
last_op = kLastOpSeekForPrev;
|
|
|
|
op_logs += "SFP " + key.ToString(true) + " ";
|
|
|
|
} else {
|
|
|
|
iter->Seek(key);
|
|
|
|
cmp_iter->Seek(key);
|
|
|
|
last_op = kLastOpSeek;
|
|
|
|
op_logs += "S " + key.ToString(true) + " ";
|
|
|
|
}
|
|
|
|
VerifyIterator(thread, cmp_cfh, readoptionscopy, iter.get(), cmp_iter.get(),
|
|
|
|
last_op, key, op_logs, &diverged);
|
|
|
|
|
|
|
|
bool no_reverse =
|
|
|
|
(FLAGS_memtablerep == "prefix_hash" && !expect_total_order);
|
|
|
|
for (uint64_t i = 0; i < FLAGS_num_iterations && iter->Valid(); i++) {
|
|
|
|
if (no_reverse || thread->rand.OneIn(2)) {
|
|
|
|
iter->Next();
|
|
|
|
if (!diverged) {
|
|
|
|
assert(cmp_iter->Valid());
|
|
|
|
cmp_iter->Next();
|
|
|
|
}
|
|
|
|
op_logs += "N";
|
|
|
|
} else {
|
|
|
|
iter->Prev();
|
|
|
|
if (!diverged) {
|
|
|
|
assert(cmp_iter->Valid());
|
|
|
|
cmp_iter->Prev();
|
|
|
|
}
|
|
|
|
op_logs += "P";
|
|
|
|
}
|
|
|
|
last_op = kLastOpNextOrPrev;
|
|
|
|
VerifyIterator(thread, cmp_cfh, readoptionscopy, iter.get(),
|
|
|
|
cmp_iter.get(), last_op, key, op_logs, &diverged);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (s.ok()) {
|
|
|
|
thread->stats.AddIterations(1);
|
|
|
|
} else {
|
|
|
|
fprintf(stderr, "TestIterate error: %s\n", s.ToString().c_str());
|
|
|
|
thread->stats.AddErrors(1);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
op_logs += "; ";
|
|
|
|
}
|
|
|
|
|
|
|
|
db_->ReleaseSnapshot(snapshot);
|
|
|
|
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
// Test the return status of GetLiveFiles.
|
|
|
|
Status StressTest::VerifyGetLiveFiles() const {
|
|
|
|
std::vector<std::string> live_file;
|
|
|
|
uint64_t manifest_size = 0;
|
|
|
|
return db_->GetLiveFiles(live_file, &manifest_size);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Test the return status of GetSortedWalFiles.
|
|
|
|
Status StressTest::VerifyGetSortedWalFiles() const {
|
|
|
|
VectorLogPtr log_ptr;
|
|
|
|
return db_->GetSortedWalFiles(log_ptr);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Test the return status of GetCurrentWalFile.
|
|
|
|
Status StressTest::VerifyGetCurrentWalFile() const {
|
|
|
|
std::unique_ptr<LogFile> cur_wal_file;
|
|
|
|
return db_->GetCurrentWalFile(&cur_wal_file);
|
|
|
|
}
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
|
|
|
|
// Compare the two iterator, iter and cmp_iter are in the same position,
|
|
|
|
// unless iter might be made invalidate or undefined because of
|
|
|
|
// upper or lower bounds, or prefix extractor.
|
|
|
|
// Will flag failure if the verification fails.
|
|
|
|
// diverged = true if the two iterator is already diverged.
|
|
|
|
// True if verification passed, false if not.
|
|
|
|
void StressTest::VerifyIterator(ThreadState* thread,
|
|
|
|
ColumnFamilyHandle* cmp_cfh,
|
|
|
|
const ReadOptions& ro, Iterator* iter,
|
|
|
|
Iterator* cmp_iter, LastIterateOp op,
|
|
|
|
const Slice& seek_key,
|
|
|
|
const std::string& op_logs, bool* diverged) {
|
|
|
|
if (*diverged) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (op == kLastOpSeekToFirst && ro.iterate_lower_bound != nullptr) {
|
|
|
|
// SeekToFirst() with lower bound is not well defined.
|
|
|
|
*diverged = true;
|
|
|
|
return;
|
|
|
|
} else if (op == kLastOpSeekToLast && ro.iterate_upper_bound != nullptr) {
|
|
|
|
// SeekToLast() with higher bound is not well defined.
|
|
|
|
*diverged = true;
|
|
|
|
return;
|
|
|
|
} else if (op == kLastOpSeek && ro.iterate_lower_bound != nullptr &&
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
(options_.comparator->CompareWithoutTimestamp(
|
|
|
|
*ro.iterate_lower_bound, /*a_has_ts=*/false, seek_key,
|
|
|
|
/*b_has_ts=*/false) >= 0 ||
|
|
|
|
(ro.iterate_upper_bound != nullptr &&
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
options_.comparator->CompareWithoutTimestamp(
|
|
|
|
*ro.iterate_lower_bound, /*a_has_ts=*/false,
|
|
|
|
*ro.iterate_upper_bound, /*b_has_ts*/ false) >= 0))) {
|
|
|
|
// Lower bound behavior is not well defined if it is larger than
|
|
|
|
// seek key or upper bound. Disable the check for now.
|
|
|
|
*diverged = true;
|
|
|
|
return;
|
|
|
|
} else if (op == kLastOpSeekForPrev && ro.iterate_upper_bound != nullptr &&
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
(options_.comparator->CompareWithoutTimestamp(
|
|
|
|
*ro.iterate_upper_bound, /*a_has_ts=*/false, seek_key,
|
|
|
|
/*b_has_ts=*/false) <= 0 ||
|
|
|
|
(ro.iterate_lower_bound != nullptr &&
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
options_.comparator->CompareWithoutTimestamp(
|
|
|
|
*ro.iterate_lower_bound, /*a_has_ts=*/false,
|
|
|
|
*ro.iterate_upper_bound, /*b_has_ts=*/false) >= 0))) {
|
|
|
|
// Uppder bound behavior is not well defined if it is smaller than
|
|
|
|
// seek key or lower bound. Disable the check for now.
|
|
|
|
*diverged = true;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
const SliceTransform* pe = (ro.total_order_seek || ro.auto_prefix_mode)
|
|
|
|
? nullptr
|
|
|
|
: options_.prefix_extractor.get();
|
|
|
|
const Comparator* cmp = options_.comparator;
|
|
|
|
|
|
|
|
if (iter->Valid() && !cmp_iter->Valid()) {
|
|
|
|
if (pe != nullptr) {
|
|
|
|
if (!pe->InDomain(seek_key)) {
|
|
|
|
// Prefix seek a non-in-domain key is undefined. Skip checking for
|
|
|
|
// this scenario.
|
|
|
|
*diverged = true;
|
|
|
|
return;
|
|
|
|
} else if (!pe->InDomain(iter->key())) {
|
|
|
|
// out of range is iterator key is not in domain anymore.
|
|
|
|
*diverged = true;
|
|
|
|
return;
|
|
|
|
} else if (pe->Transform(iter->key()) != pe->Transform(seek_key)) {
|
|
|
|
*diverged = true;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
fprintf(stderr,
|
|
|
|
"Control interator is invalid but iterator has key %s "
|
|
|
|
"%s\n",
|
|
|
|
iter->key().ToString(true).c_str(), op_logs.c_str());
|
|
|
|
|
|
|
|
*diverged = true;
|
|
|
|
} else if (cmp_iter->Valid()) {
|
|
|
|
// Iterator is not valid. It can be legimate if it has already been
|
|
|
|
// out of upper or lower bound, or filtered out by prefix iterator.
|
|
|
|
const Slice& total_order_key = cmp_iter->key();
|
|
|
|
|
|
|
|
if (pe != nullptr) {
|
|
|
|
if (!pe->InDomain(seek_key)) {
|
|
|
|
// Prefix seek a non-in-domain key is undefined. Skip checking for
|
|
|
|
// this scenario.
|
|
|
|
*diverged = true;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!pe->InDomain(total_order_key) ||
|
|
|
|
pe->Transform(total_order_key) != pe->Transform(seek_key)) {
|
|
|
|
// If the prefix is exhausted, the only thing needs to check
|
|
|
|
// is the iterator isn't return a position in prefix.
|
|
|
|
// Either way, checking can stop from here.
|
|
|
|
*diverged = true;
|
|
|
|
if (!iter->Valid() || !pe->InDomain(iter->key()) ||
|
|
|
|
pe->Transform(iter->key()) != pe->Transform(seek_key)) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
fprintf(stderr,
|
|
|
|
"Iterator stays in prefix but contol doesn't"
|
|
|
|
" iterator key %s control iterator key %s %s\n",
|
|
|
|
iter->key().ToString(true).c_str(),
|
|
|
|
cmp_iter->key().ToString(true).c_str(), op_logs.c_str());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Check upper or lower bounds.
|
|
|
|
if (!*diverged) {
|
|
|
|
if ((iter->Valid() && iter->key() != cmp_iter->key()) ||
|
|
|
|
(!iter->Valid() &&
|
|
|
|
(ro.iterate_upper_bound == nullptr ||
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
cmp->CompareWithoutTimestamp(total_order_key, /*a_has_ts=*/false,
|
|
|
|
*ro.iterate_upper_bound,
|
|
|
|
/*b_has_ts=*/false) < 0) &&
|
|
|
|
(ro.iterate_lower_bound == nullptr ||
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
cmp->CompareWithoutTimestamp(total_order_key, /*a_has_ts=*/false,
|
|
|
|
*ro.iterate_lower_bound,
|
|
|
|
/*b_has_ts=*/false) > 0))) {
|
|
|
|
fprintf(stderr,
|
|
|
|
"Iterator diverged from control iterator which"
|
|
|
|
" has value %s %s\n",
|
|
|
|
total_order_key.ToString(true).c_str(), op_logs.c_str());
|
|
|
|
if (iter->Valid()) {
|
|
|
|
fprintf(stderr, "iterator has value %s\n",
|
|
|
|
iter->key().ToString(true).c_str());
|
|
|
|
} else {
|
|
|
|
fprintf(stderr, "iterator is not valid\n");
|
|
|
|
}
|
|
|
|
*diverged = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (*diverged) {
|
|
|
|
fprintf(stderr, "Control CF %s\n", cmp_cfh->GetName().c_str());
|
|
|
|
thread->stats.AddErrors(1);
|
|
|
|
// Fail fast to preserve the DB state.
|
|
|
|
thread->shared->SetVerificationFailure();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef ROCKSDB_LITE
|
|
|
|
Status StressTest::TestBackupRestore(
|
|
|
|
ThreadState* /* thread */,
|
|
|
|
const std::vector<int>& /* rand_column_families */,
|
|
|
|
const std::vector<int64_t>& /* rand_keys */) {
|
|
|
|
assert(false);
|
|
|
|
fprintf(stderr,
|
|
|
|
"RocksDB lite does not support "
|
|
|
|
"TestBackupRestore\n");
|
|
|
|
std::terminate();
|
|
|
|
}
|
|
|
|
|
|
|
|
Status StressTest::TestCheckpoint(
|
|
|
|
ThreadState* /* thread */,
|
|
|
|
const std::vector<int>& /* rand_column_families */,
|
|
|
|
const std::vector<int64_t>& /* rand_keys */) {
|
|
|
|
assert(false);
|
|
|
|
fprintf(stderr,
|
|
|
|
"RocksDB lite does not support "
|
|
|
|
"TestCheckpoint\n");
|
|
|
|
std::terminate();
|
|
|
|
}
|
|
|
|
|
|
|
|
void StressTest::TestCompactFiles(ThreadState* /* thread */,
|
|
|
|
ColumnFamilyHandle* /* column_family */) {
|
|
|
|
assert(false);
|
|
|
|
fprintf(stderr,
|
|
|
|
"RocksDB lite does not support "
|
|
|
|
"CompactFiles\n");
|
|
|
|
std::terminate();
|
|
|
|
}
|
|
|
|
#else // ROCKSDB_LITE
|
|
|
|
Status StressTest::TestBackupRestore(
|
|
|
|
ThreadState* thread, const std::vector<int>& rand_column_families,
|
|
|
|
const std::vector<int64_t>& rand_keys) {
|
|
|
|
const std::string backup_dir =
|
|
|
|
FLAGS_db + "/.backup" + std::to_string(thread->tid);
|
|
|
|
const std::string restore_dir =
|
|
|
|
FLAGS_db + "/.restore" + std::to_string(thread->tid);
|
|
|
|
BackupEngineOptions backup_opts(backup_dir);
|
Fix, enable, and enhance backup/restore in db_stress (#7348)
Summary:
Although added to db_stress, testing of backup/restore
was never integrated into the crash test, originally concerned about
performance. I've enabled it now and to address the peformance concern,
testing backup/restore is always skipped once the db exceeds a certain
size threshold, default 100MB. This should provide sufficient
opportunity for testing BackupEngine without bogging down everything
else with heavier and heavier operations.
Also fixed backup/restore in db_stress by making sure PurgeOldBackups
can remove manifest files, which are normally kept around for db_stress.
Added more coverage of backup options, and up to three backups being
saved in one backup directory (in some cases).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7348
Test Plan:
ran 'make blackbox_crash_test' for a while, with heightened
probabilitly of taking backups (1/10k). Also confirmed with some debug
output that the code is being covered, TestBackupRestore only takes
a few seconds to complete when triggered, and even at 1/10k and ~50MB
database, there's <,~ 1 thread testing backups at any time.
Reviewed By: ajkr
Differential Revision: D23510835
Pulled By: pdillinger
fbshipit-source-id: b6b8735591808141f81f10773ac31634cf03b6c0
4 years ago
|
|
|
// For debugging, get info_log from live options
|
|
|
|
backup_opts.info_log = db_->GetDBOptions().info_log.get();
|
|
|
|
if (thread->rand.OneIn(10)) {
|
|
|
|
backup_opts.share_table_files = false;
|
|
|
|
} else {
|
|
|
|
backup_opts.share_table_files = true;
|
|
|
|
if (thread->rand.OneIn(5)) {
|
|
|
|
backup_opts.share_files_with_checksum = false;
|
|
|
|
} else {
|
|
|
|
backup_opts.share_files_with_checksum = true;
|
|
|
|
if (thread->rand.OneIn(2)) {
|
|
|
|
// old
|
Restore file size in backup table file names (and other cleanup) (#7400)
Summary:
Prior to 6.12, backup files using share_files_with_checksum had
the file size encoded in the file name, after the last '\_' and before
the last '.'. We considered this an implementation detail subject to
change, and indeed removed this information from the file name (with an
option to use old behavior) because it was considered
ineffective/inefficient for file name uniqueness. However, some
downstream RocksDB users were relying on this information since the file
size is not explicitly in the backup manifest file.
This primary purpose of this change is "retrofitting" the 6.12 release
(not yet a public release) to simultaneously support the benefits of the
new naming scheme (I/O performance and data correctness at scale) and
preserve the file size information, both as default behaviors. With this
change, we are essentially making the file size information encoded in
the file name an official, though obscure, extension of the backup meta
file format.
We preserve an option (kLegacyCrc32cAndFileSize) to use the original
"legacy" naming scheme, with its caveats, and make it easy to omit the
file size information (no kFlagIncludeFileSize), for more compact file
names. But note that changing the naming scheme used on an existing db
and backup directory can lead to transient space amplification, as some
files will be stored under two names in the shared_checksum directory.
Because some backups were saved using the original 6.12 naming scheme,
we offer two ways of dealing with those files: SST files generated by
older 6.12 versions can either use the default naming scheme in effect
when the SST files were generated (kFlagMatchInterimNaming, default, no
transient space amplification) or can use a new naming scheme (no
kFlagMatchInterimNaming, potential space amplification because some
already stored files getting a new name).
We don't have a natural way to detect which files were generated by
previous 6.12 versions, but this change hacks one in by changing DB
session ids to now use a more concise encoding, reducing file name
length, saving ~dozen bytes from SST files, and making them visually
distinct from DB ids so that they are less likely to be mixed up.
Two final auxiliary notes:
Recognizing that the backup file names have become a de facto part of
the backup meta schema, this change makes them easier to parse and
extend by putting a distinct marker, 's', before DB session ids embedded
in the name. When we extend this to allow custom checksums in the name,
they can get their own marker to ensure safe parsing. For backward
compatibility, file size does not get a marker but is assumed for
`_[0-9]+[.]`
Another change from initial 6.12 default behavior is never including
file custom checksum in the file name. Looking ahead to 6.13, we do not
want the default behavior to cause backup space amplification for
someone turning on file custom checksum checking in BackupEngine; we
want that to be an easy decision. When implemented, including file
custom checksums in backup file names will be a non-default option.
Actual file name patterns and priorities, as regexes:
kLegacyCrc32cAndFileSize OR pre-6.12 SST file ->
[0-9]+_[0-9]+_[0-9]+[.]sst
kFlagMatchInterimNaming set (default) AND early 6.12 SST file ->
[0-9]+_[0-9a-fA-F-]+[.]sst
kUseDbSessionId AND NOT kFlagIncludeFileSize ->
[0-9]+_s[0-9A-Z]{20}[.]sst
kUseDbSessionId AND kFlagIncludeFileSize (default) ->
[0-9]+_s[0-9A-Z]{20}_[0-9]+[.]sst
We might add opt-in options for more '\_' separated data in the name,
but embedded file size, if present, will always be after last '\_' and
before '.sst'.
This change was originally applied to version 6.12. (See https://github.com/facebook/rocksdb/issues/7390)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7400
Test Plan:
unit tests included. Sync point callbacks are used to mimic
previous version SST files.
Reviewed By: ajkr
Differential Revision: D23759587
Pulled By: pdillinger
fbshipit-source-id: f62d8af4e0978de0a34f26288cfbe66049b70025
4 years ago
|
|
|
backup_opts.share_files_with_checksum_naming =
|
|
|
|
BackupEngineOptions::kLegacyCrc32cAndFileSize;
|
Fix, enable, and enhance backup/restore in db_stress (#7348)
Summary:
Although added to db_stress, testing of backup/restore
was never integrated into the crash test, originally concerned about
performance. I've enabled it now and to address the peformance concern,
testing backup/restore is always skipped once the db exceeds a certain
size threshold, default 100MB. This should provide sufficient
opportunity for testing BackupEngine without bogging down everything
else with heavier and heavier operations.
Also fixed backup/restore in db_stress by making sure PurgeOldBackups
can remove manifest files, which are normally kept around for db_stress.
Added more coverage of backup options, and up to three backups being
saved in one backup directory (in some cases).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7348
Test Plan:
ran 'make blackbox_crash_test' for a while, with heightened
probabilitly of taking backups (1/10k). Also confirmed with some debug
output that the code is being covered, TestBackupRestore only takes
a few seconds to complete when triggered, and even at 1/10k and ~50MB
database, there's <,~ 1 thread testing backups at any time.
Reviewed By: ajkr
Differential Revision: D23510835
Pulled By: pdillinger
fbshipit-source-id: b6b8735591808141f81f10773ac31634cf03b6c0
4 years ago
|
|
|
} else {
|
|
|
|
// new
|
|
|
|
backup_opts.share_files_with_checksum_naming =
|
|
|
|
BackupEngineOptions::kUseDbSessionId;
|
Restore file size in backup table file names (and other cleanup) (#7400)
Summary:
Prior to 6.12, backup files using share_files_with_checksum had
the file size encoded in the file name, after the last '\_' and before
the last '.'. We considered this an implementation detail subject to
change, and indeed removed this information from the file name (with an
option to use old behavior) because it was considered
ineffective/inefficient for file name uniqueness. However, some
downstream RocksDB users were relying on this information since the file
size is not explicitly in the backup manifest file.
This primary purpose of this change is "retrofitting" the 6.12 release
(not yet a public release) to simultaneously support the benefits of the
new naming scheme (I/O performance and data correctness at scale) and
preserve the file size information, both as default behaviors. With this
change, we are essentially making the file size information encoded in
the file name an official, though obscure, extension of the backup meta
file format.
We preserve an option (kLegacyCrc32cAndFileSize) to use the original
"legacy" naming scheme, with its caveats, and make it easy to omit the
file size information (no kFlagIncludeFileSize), for more compact file
names. But note that changing the naming scheme used on an existing db
and backup directory can lead to transient space amplification, as some
files will be stored under two names in the shared_checksum directory.
Because some backups were saved using the original 6.12 naming scheme,
we offer two ways of dealing with those files: SST files generated by
older 6.12 versions can either use the default naming scheme in effect
when the SST files were generated (kFlagMatchInterimNaming, default, no
transient space amplification) or can use a new naming scheme (no
kFlagMatchInterimNaming, potential space amplification because some
already stored files getting a new name).
We don't have a natural way to detect which files were generated by
previous 6.12 versions, but this change hacks one in by changing DB
session ids to now use a more concise encoding, reducing file name
length, saving ~dozen bytes from SST files, and making them visually
distinct from DB ids so that they are less likely to be mixed up.
Two final auxiliary notes:
Recognizing that the backup file names have become a de facto part of
the backup meta schema, this change makes them easier to parse and
extend by putting a distinct marker, 's', before DB session ids embedded
in the name. When we extend this to allow custom checksums in the name,
they can get their own marker to ensure safe parsing. For backward
compatibility, file size does not get a marker but is assumed for
`_[0-9]+[.]`
Another change from initial 6.12 default behavior is never including
file custom checksum in the file name. Looking ahead to 6.13, we do not
want the default behavior to cause backup space amplification for
someone turning on file custom checksum checking in BackupEngine; we
want that to be an easy decision. When implemented, including file
custom checksums in backup file names will be a non-default option.
Actual file name patterns and priorities, as regexes:
kLegacyCrc32cAndFileSize OR pre-6.12 SST file ->
[0-9]+_[0-9]+_[0-9]+[.]sst
kFlagMatchInterimNaming set (default) AND early 6.12 SST file ->
[0-9]+_[0-9a-fA-F-]+[.]sst
kUseDbSessionId AND NOT kFlagIncludeFileSize ->
[0-9]+_s[0-9A-Z]{20}[.]sst
kUseDbSessionId AND kFlagIncludeFileSize (default) ->
[0-9]+_s[0-9A-Z]{20}_[0-9]+[.]sst
We might add opt-in options for more '\_' separated data in the name,
but embedded file size, if present, will always be after last '\_' and
before '.sst'.
This change was originally applied to version 6.12. (See https://github.com/facebook/rocksdb/issues/7390)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7400
Test Plan:
unit tests included. Sync point callbacks are used to mimic
previous version SST files.
Reviewed By: ajkr
Differential Revision: D23759587
Pulled By: pdillinger
fbshipit-source-id: f62d8af4e0978de0a34f26288cfbe66049b70025
4 years ago
|
|
|
}
|
|
|
|
if (thread->rand.OneIn(2)) {
|
|
|
|
backup_opts.share_files_with_checksum_naming =
|
|
|
|
backup_opts.share_files_with_checksum_naming |
|
|
|
|
BackupEngineOptions::kFlagIncludeFileSize;
|
Restore file size in backup table file names (and other cleanup) (#7400)
Summary:
Prior to 6.12, backup files using share_files_with_checksum had
the file size encoded in the file name, after the last '\_' and before
the last '.'. We considered this an implementation detail subject to
change, and indeed removed this information from the file name (with an
option to use old behavior) because it was considered
ineffective/inefficient for file name uniqueness. However, some
downstream RocksDB users were relying on this information since the file
size is not explicitly in the backup manifest file.
This primary purpose of this change is "retrofitting" the 6.12 release
(not yet a public release) to simultaneously support the benefits of the
new naming scheme (I/O performance and data correctness at scale) and
preserve the file size information, both as default behaviors. With this
change, we are essentially making the file size information encoded in
the file name an official, though obscure, extension of the backup meta
file format.
We preserve an option (kLegacyCrc32cAndFileSize) to use the original
"legacy" naming scheme, with its caveats, and make it easy to omit the
file size information (no kFlagIncludeFileSize), for more compact file
names. But note that changing the naming scheme used on an existing db
and backup directory can lead to transient space amplification, as some
files will be stored under two names in the shared_checksum directory.
Because some backups were saved using the original 6.12 naming scheme,
we offer two ways of dealing with those files: SST files generated by
older 6.12 versions can either use the default naming scheme in effect
when the SST files were generated (kFlagMatchInterimNaming, default, no
transient space amplification) or can use a new naming scheme (no
kFlagMatchInterimNaming, potential space amplification because some
already stored files getting a new name).
We don't have a natural way to detect which files were generated by
previous 6.12 versions, but this change hacks one in by changing DB
session ids to now use a more concise encoding, reducing file name
length, saving ~dozen bytes from SST files, and making them visually
distinct from DB ids so that they are less likely to be mixed up.
Two final auxiliary notes:
Recognizing that the backup file names have become a de facto part of
the backup meta schema, this change makes them easier to parse and
extend by putting a distinct marker, 's', before DB session ids embedded
in the name. When we extend this to allow custom checksums in the name,
they can get their own marker to ensure safe parsing. For backward
compatibility, file size does not get a marker but is assumed for
`_[0-9]+[.]`
Another change from initial 6.12 default behavior is never including
file custom checksum in the file name. Looking ahead to 6.13, we do not
want the default behavior to cause backup space amplification for
someone turning on file custom checksum checking in BackupEngine; we
want that to be an easy decision. When implemented, including file
custom checksums in backup file names will be a non-default option.
Actual file name patterns and priorities, as regexes:
kLegacyCrc32cAndFileSize OR pre-6.12 SST file ->
[0-9]+_[0-9]+_[0-9]+[.]sst
kFlagMatchInterimNaming set (default) AND early 6.12 SST file ->
[0-9]+_[0-9a-fA-F-]+[.]sst
kUseDbSessionId AND NOT kFlagIncludeFileSize ->
[0-9]+_s[0-9A-Z]{20}[.]sst
kUseDbSessionId AND kFlagIncludeFileSize (default) ->
[0-9]+_s[0-9A-Z]{20}_[0-9]+[.]sst
We might add opt-in options for more '\_' separated data in the name,
but embedded file size, if present, will always be after last '\_' and
before '.sst'.
This change was originally applied to version 6.12. (See https://github.com/facebook/rocksdb/issues/7390)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7400
Test Plan:
unit tests included. Sync point callbacks are used to mimic
previous version SST files.
Reviewed By: ajkr
Differential Revision: D23759587
Pulled By: pdillinger
fbshipit-source-id: f62d8af4e0978de0a34f26288cfbe66049b70025
4 years ago
|
|
|
}
|
Fix, enable, and enhance backup/restore in db_stress (#7348)
Summary:
Although added to db_stress, testing of backup/restore
was never integrated into the crash test, originally concerned about
performance. I've enabled it now and to address the peformance concern,
testing backup/restore is always skipped once the db exceeds a certain
size threshold, default 100MB. This should provide sufficient
opportunity for testing BackupEngine without bogging down everything
else with heavier and heavier operations.
Also fixed backup/restore in db_stress by making sure PurgeOldBackups
can remove manifest files, which are normally kept around for db_stress.
Added more coverage of backup options, and up to three backups being
saved in one backup directory (in some cases).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7348
Test Plan:
ran 'make blackbox_crash_test' for a while, with heightened
probabilitly of taking backups (1/10k). Also confirmed with some debug
output that the code is being covered, TestBackupRestore only takes
a few seconds to complete when triggered, and even at 1/10k and ~50MB
database, there's <,~ 1 thread testing backups at any time.
Reviewed By: ajkr
Differential Revision: D23510835
Pulled By: pdillinger
fbshipit-source-id: b6b8735591808141f81f10773ac31634cf03b6c0
4 years ago
|
|
|
}
|
|
|
|
}
|
New backup meta schema, with file temperatures (#9660)
Summary:
The primary goal of this change is to add support for backing up and
restoring (applying on restore) file temperature metadata, without
committing to either the DB manifest or the FS reported "current"
temperatures being exclusive "source of truth".
To achieve this goal, we need to add temperature information to backup
metadata, which requires updated backup meta schema. Fortunately I
prepared for this in https://github.com/facebook/rocksdb/issues/8069, which began forward compatibility in version
6.19.0 for this kind of schema update. (Previously, backup meta schema
was not extensible! Making this schema update public will allow some
other "nice to have" features like taking backups with hard links, and
avoiding crc32c checksum computation when another checksum is already
available.) While schema version 2 is newly public, the default schema
version is still 1. Until we change the default, users will need to set
to 2 to enable features like temperature data backup+restore. New
metadata like temperature information will be ignored with a warning
in versions before this change and since 6.19.0. The metadata is
considered ignorable because a functioning DB can be restored without
it.
Some detail:
* Some renaming because "future schema" is now just public schema 2.
* Initialize some atomics in TestFs (linter reported)
* Add temperature hint support to SstFileDumper (used by BackupEngine)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9660
Test Plan:
related unit test majorly updated for the new functionality,
including some shared testing support for tracking temperatures in a FS.
Some other tests and testing hooks into production code also updated for
making the backup meta schema change public.
Reviewed By: ajkr
Differential Revision: D34686968
Pulled By: pdillinger
fbshipit-source-id: 3ac1fa3e67ee97ca8a5103d79cc87d872c1d862a
3 years ago
|
|
|
if (thread->rand.OneIn(2)) {
|
|
|
|
backup_opts.schema_version = 1;
|
|
|
|
} else {
|
|
|
|
backup_opts.schema_version = 2;
|
|
|
|
}
|
|
|
|
BackupEngine* backup_engine = nullptr;
|
|
|
|
std::string from = "a backup/restore operation";
|
|
|
|
Status s = BackupEngine::Open(db_stress_env, backup_opts, &backup_engine);
|
|
|
|
if (!s.ok()) {
|
|
|
|
from = "BackupEngine::Open";
|
|
|
|
}
|
|
|
|
if (s.ok()) {
|
New backup meta schema, with file temperatures (#9660)
Summary:
The primary goal of this change is to add support for backing up and
restoring (applying on restore) file temperature metadata, without
committing to either the DB manifest or the FS reported "current"
temperatures being exclusive "source of truth".
To achieve this goal, we need to add temperature information to backup
metadata, which requires updated backup meta schema. Fortunately I
prepared for this in https://github.com/facebook/rocksdb/issues/8069, which began forward compatibility in version
6.19.0 for this kind of schema update. (Previously, backup meta schema
was not extensible! Making this schema update public will allow some
other "nice to have" features like taking backups with hard links, and
avoiding crc32c checksum computation when another checksum is already
available.) While schema version 2 is newly public, the default schema
version is still 1. Until we change the default, users will need to set
to 2 to enable features like temperature data backup+restore. New
metadata like temperature information will be ignored with a warning
in versions before this change and since 6.19.0. The metadata is
considered ignorable because a functioning DB can be restored without
it.
Some detail:
* Some renaming because "future schema" is now just public schema 2.
* Initialize some atomics in TestFs (linter reported)
* Add temperature hint support to SstFileDumper (used by BackupEngine)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9660
Test Plan:
related unit test majorly updated for the new functionality,
including some shared testing support for tracking temperatures in a FS.
Some other tests and testing hooks into production code also updated for
making the backup meta schema change public.
Reviewed By: ajkr
Differential Revision: D34686968
Pulled By: pdillinger
fbshipit-source-id: 3ac1fa3e67ee97ca8a5103d79cc87d872c1d862a
3 years ago
|
|
|
if (backup_opts.schema_version >= 2 && thread->rand.OneIn(2)) {
|
|
|
|
TEST_BackupMetaSchemaOptions test_opts;
|
Begin forward compatibility for new backup meta schema (#8069)
Summary:
This does not add any new public APIs or published
functionality, but adds the ability to read and use (and in tests,
write) backups with a new meta file schema, based on the old schema
but not forward-compatible (before this change). The new schema enables
some capabilities not in the old:
* Explicit versioning, so that users get clean error messages the next
time we want to break forward compatibility.
* Ignoring unrecognized fields (with warning), so that new non-critical
features can be added without breaking forward compatibility.
* Rejecting future "non-ignorable" fields, so that new features critical
to some use-cases could potentially be added outside of linear schema
versions, with broken forward compatibility.
* Fields at the end of the meta file, such as for checksum of the meta
file's contents (up to that point)
* New optional 'size' field for each file, which is checked when present
* Optionally omitting 'crc32' field, so that we aren't required to have
a crc32c checksum for files to take a backup. (E.g. to support backup
via hard links and to better support file custom checksums.)
Because we do not have a JSON parser and to share code, the new schema
is simply derived from the old schema.
BackupEngine code is updated to allow missing checksums in some places,
and to make that easier, `has_checksum` and `verify_checksum_after_work`
are eliminated. Empty `checksum_hex` indicates checksum is unknown. I'm
not too afraid of regressing on data integrity, because
(a) we have pretty good test coverage of corruption detection in backups, and
(b) we are increasingly relying on the DB itself for data integrity rather than
it being an exclusive feature of backups.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8069
Test Plan:
new unit tests, added to crash test (some local run with
boosted backup probability)
Reviewed By: ajkr
Differential Revision: D27139824
Pulled By: pdillinger
fbshipit-source-id: 9e0e4decfb42bb84783d64d2d246456d97e8e8c5
4 years ago
|
|
|
test_opts.crc32c_checksums = thread->rand.OneIn(2) == 0;
|
|
|
|
test_opts.file_sizes = thread->rand.OneIn(2) == 0;
|
New backup meta schema, with file temperatures (#9660)
Summary:
The primary goal of this change is to add support for backing up and
restoring (applying on restore) file temperature metadata, without
committing to either the DB manifest or the FS reported "current"
temperatures being exclusive "source of truth".
To achieve this goal, we need to add temperature information to backup
metadata, which requires updated backup meta schema. Fortunately I
prepared for this in https://github.com/facebook/rocksdb/issues/8069, which began forward compatibility in version
6.19.0 for this kind of schema update. (Previously, backup meta schema
was not extensible! Making this schema update public will allow some
other "nice to have" features like taking backups with hard links, and
avoiding crc32c checksum computation when another checksum is already
available.) While schema version 2 is newly public, the default schema
version is still 1. Until we change the default, users will need to set
to 2 to enable features like temperature data backup+restore. New
metadata like temperature information will be ignored with a warning
in versions before this change and since 6.19.0. The metadata is
considered ignorable because a functioning DB can be restored without
it.
Some detail:
* Some renaming because "future schema" is now just public schema 2.
* Initialize some atomics in TestFs (linter reported)
* Add temperature hint support to SstFileDumper (used by BackupEngine)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9660
Test Plan:
related unit test majorly updated for the new functionality,
including some shared testing support for tracking temperatures in a FS.
Some other tests and testing hooks into production code also updated for
making the backup meta schema change public.
Reviewed By: ajkr
Differential Revision: D34686968
Pulled By: pdillinger
fbshipit-source-id: 3ac1fa3e67ee97ca8a5103d79cc87d872c1d862a
3 years ago
|
|
|
TEST_SetBackupMetaSchemaOptions(backup_engine, test_opts);
|
Begin forward compatibility for new backup meta schema (#8069)
Summary:
This does not add any new public APIs or published
functionality, but adds the ability to read and use (and in tests,
write) backups with a new meta file schema, based on the old schema
but not forward-compatible (before this change). The new schema enables
some capabilities not in the old:
* Explicit versioning, so that users get clean error messages the next
time we want to break forward compatibility.
* Ignoring unrecognized fields (with warning), so that new non-critical
features can be added without breaking forward compatibility.
* Rejecting future "non-ignorable" fields, so that new features critical
to some use-cases could potentially be added outside of linear schema
versions, with broken forward compatibility.
* Fields at the end of the meta file, such as for checksum of the meta
file's contents (up to that point)
* New optional 'size' field for each file, which is checked when present
* Optionally omitting 'crc32' field, so that we aren't required to have
a crc32c checksum for files to take a backup. (E.g. to support backup
via hard links and to better support file custom checksums.)
Because we do not have a JSON parser and to share code, the new schema
is simply derived from the old schema.
BackupEngine code is updated to allow missing checksums in some places,
and to make that easier, `has_checksum` and `verify_checksum_after_work`
are eliminated. Empty `checksum_hex` indicates checksum is unknown. I'm
not too afraid of regressing on data integrity, because
(a) we have pretty good test coverage of corruption detection in backups, and
(b) we are increasingly relying on the DB itself for data integrity rather than
it being an exclusive feature of backups.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8069
Test Plan:
new unit tests, added to crash test (some local run with
boosted backup probability)
Reviewed By: ajkr
Differential Revision: D27139824
Pulled By: pdillinger
fbshipit-source-id: 9e0e4decfb42bb84783d64d2d246456d97e8e8c5
4 years ago
|
|
|
}
|
|
|
|
CreateBackupOptions create_opts;
|
|
|
|
if (FLAGS_disable_wal) {
|
|
|
|
// The verification can only work when latest value of `key` is backed up,
|
|
|
|
// which requires flushing in case of WAL disabled.
|
|
|
|
//
|
|
|
|
// Note this triggers a flush with a key lock held. Meanwhile, operations
|
|
|
|
// like flush/compaction may attempt to grab key locks like in
|
|
|
|
// `DbStressCompactionFilter`. The philosophy around preventing deadlock
|
|
|
|
// is the background operation key lock acquisition only tries but does
|
|
|
|
// not wait for the lock. So here in the foreground it is OK to hold the
|
|
|
|
// lock and wait on a background operation (flush).
|
|
|
|
create_opts.flush_before_backup = true;
|
|
|
|
}
|
|
|
|
s = backup_engine->CreateNewBackup(create_opts, db_);
|
|
|
|
if (!s.ok()) {
|
|
|
|
from = "BackupEngine::CreateNewBackup";
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (s.ok()) {
|
|
|
|
delete backup_engine;
|
|
|
|
backup_engine = nullptr;
|
|
|
|
s = BackupEngine::Open(db_stress_env, backup_opts, &backup_engine);
|
|
|
|
if (!s.ok()) {
|
|
|
|
from = "BackupEngine::Open (again)";
|
|
|
|
}
|
|
|
|
}
|
Fix, enable, and enhance backup/restore in db_stress (#7348)
Summary:
Although added to db_stress, testing of backup/restore
was never integrated into the crash test, originally concerned about
performance. I've enabled it now and to address the peformance concern,
testing backup/restore is always skipped once the db exceeds a certain
size threshold, default 100MB. This should provide sufficient
opportunity for testing BackupEngine without bogging down everything
else with heavier and heavier operations.
Also fixed backup/restore in db_stress by making sure PurgeOldBackups
can remove manifest files, which are normally kept around for db_stress.
Added more coverage of backup options, and up to three backups being
saved in one backup directory (in some cases).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7348
Test Plan:
ran 'make blackbox_crash_test' for a while, with heightened
probabilitly of taking backups (1/10k). Also confirmed with some debug
output that the code is being covered, TestBackupRestore only takes
a few seconds to complete when triggered, and even at 1/10k and ~50MB
database, there's <,~ 1 thread testing backups at any time.
Reviewed By: ajkr
Differential Revision: D23510835
Pulled By: pdillinger
fbshipit-source-id: b6b8735591808141f81f10773ac31634cf03b6c0
4 years ago
|
|
|
std::vector<BackupInfo> backup_info;
|
Make backups openable as read-only DBs (#8142)
Summary:
A current limitation of backups is that you don't know the
exact database state of when the backup was taken. With this new
feature, you can at least inspect the backup's DB state without
restoring it by opening it as a read-only DB.
Rather than add something like OpenAsReadOnlyDB to the BackupEngine API,
which would inhibit opening stackable DB implementations read-only
(if/when their APIs support it), we instead provide a DB name and Env
that can be used to open as a read-only DB.
Possible follow-up work:
* Add a version of GetBackupInfo for a single backup.
* Let CreateNewBackup return the BackupID of the newly-created backup.
Implementation details:
Refactored ChrootFileSystem to split off new base class RemapFileSystem,
which allows more general remapping of files. We use this base class to
implement BackupEngineImpl::RemapSharedFileSystem.
To minimize API impact, I decided to just add these fields `name_for_open`
and `env_for_open` to those set by GetBackupInfo when
include_file_details=true. Creating the RemapSharedFileSystem adds a bit
to the memory consumption, perhaps unnecessarily in some cases, but this
has been mitigated by (a) only initialize the RemapSharedFileSystem
lazily when GetBackupInfo with include_file_details=true is called, and
(b) using the existing `shared_ptr<FileInfo>` objects to hold most of the
mapping data.
To enhance API safety, RemapSharedFileSystem is wrapped by new
ReadOnlyFileSystem which rejects any attempts to write. This uncovered a
couple of places in which DB::OpenForReadOnly would write to the
filesystem, so I fixed these. Added a release note because this affects
logging.
Additional minor refactoring in backupable_db.cc to support the new
functionality.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8142
Test Plan:
new test (run with ASAN and UBSAN), added to stress test and
ran it for a while with amplified backup_one_in
Reviewed By: ajkr
Differential Revision: D27535408
Pulled By: pdillinger
fbshipit-source-id: 04666d310aa0261ef6b2385c43ca793ce1dfd148
4 years ago
|
|
|
// If inplace_not_restore, we verify the backup by opening it as a
|
|
|
|
// read-only DB. If !inplace_not_restore, we restore it to a temporary
|
|
|
|
// directory for verification.
|
|
|
|
bool inplace_not_restore = thread->rand.OneIn(3);
|
Fix, enable, and enhance backup/restore in db_stress (#7348)
Summary:
Although added to db_stress, testing of backup/restore
was never integrated into the crash test, originally concerned about
performance. I've enabled it now and to address the peformance concern,
testing backup/restore is always skipped once the db exceeds a certain
size threshold, default 100MB. This should provide sufficient
opportunity for testing BackupEngine without bogging down everything
else with heavier and heavier operations.
Also fixed backup/restore in db_stress by making sure PurgeOldBackups
can remove manifest files, which are normally kept around for db_stress.
Added more coverage of backup options, and up to three backups being
saved in one backup directory (in some cases).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7348
Test Plan:
ran 'make blackbox_crash_test' for a while, with heightened
probabilitly of taking backups (1/10k). Also confirmed with some debug
output that the code is being covered, TestBackupRestore only takes
a few seconds to complete when triggered, and even at 1/10k and ~50MB
database, there's <,~ 1 thread testing backups at any time.
Reviewed By: ajkr
Differential Revision: D23510835
Pulled By: pdillinger
fbshipit-source-id: b6b8735591808141f81f10773ac31634cf03b6c0
4 years ago
|
|
|
if (s.ok()) {
|
Make backups openable as read-only DBs (#8142)
Summary:
A current limitation of backups is that you don't know the
exact database state of when the backup was taken. With this new
feature, you can at least inspect the backup's DB state without
restoring it by opening it as a read-only DB.
Rather than add something like OpenAsReadOnlyDB to the BackupEngine API,
which would inhibit opening stackable DB implementations read-only
(if/when their APIs support it), we instead provide a DB name and Env
that can be used to open as a read-only DB.
Possible follow-up work:
* Add a version of GetBackupInfo for a single backup.
* Let CreateNewBackup return the BackupID of the newly-created backup.
Implementation details:
Refactored ChrootFileSystem to split off new base class RemapFileSystem,
which allows more general remapping of files. We use this base class to
implement BackupEngineImpl::RemapSharedFileSystem.
To minimize API impact, I decided to just add these fields `name_for_open`
and `env_for_open` to those set by GetBackupInfo when
include_file_details=true. Creating the RemapSharedFileSystem adds a bit
to the memory consumption, perhaps unnecessarily in some cases, but this
has been mitigated by (a) only initialize the RemapSharedFileSystem
lazily when GetBackupInfo with include_file_details=true is called, and
(b) using the existing `shared_ptr<FileInfo>` objects to hold most of the
mapping data.
To enhance API safety, RemapSharedFileSystem is wrapped by new
ReadOnlyFileSystem which rejects any attempts to write. This uncovered a
couple of places in which DB::OpenForReadOnly would write to the
filesystem, so I fixed these. Added a release note because this affects
logging.
Additional minor refactoring in backupable_db.cc to support the new
functionality.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8142
Test Plan:
new test (run with ASAN and UBSAN), added to stress test and
ran it for a while with amplified backup_one_in
Reviewed By: ajkr
Differential Revision: D27535408
Pulled By: pdillinger
fbshipit-source-id: 04666d310aa0261ef6b2385c43ca793ce1dfd148
4 years ago
|
|
|
backup_engine->GetBackupInfo(&backup_info,
|
|
|
|
/*include_file_details*/ inplace_not_restore);
|
Fix, enable, and enhance backup/restore in db_stress (#7348)
Summary:
Although added to db_stress, testing of backup/restore
was never integrated into the crash test, originally concerned about
performance. I've enabled it now and to address the peformance concern,
testing backup/restore is always skipped once the db exceeds a certain
size threshold, default 100MB. This should provide sufficient
opportunity for testing BackupEngine without bogging down everything
else with heavier and heavier operations.
Also fixed backup/restore in db_stress by making sure PurgeOldBackups
can remove manifest files, which are normally kept around for db_stress.
Added more coverage of backup options, and up to three backups being
saved in one backup directory (in some cases).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7348
Test Plan:
ran 'make blackbox_crash_test' for a while, with heightened
probabilitly of taking backups (1/10k). Also confirmed with some debug
output that the code is being covered, TestBackupRestore only takes
a few seconds to complete when triggered, and even at 1/10k and ~50MB
database, there's <,~ 1 thread testing backups at any time.
Reviewed By: ajkr
Differential Revision: D23510835
Pulled By: pdillinger
fbshipit-source-id: b6b8735591808141f81f10773ac31634cf03b6c0
4 years ago
|
|
|
if (backup_info.empty()) {
|
|
|
|
s = Status::NotFound("no backups found");
|
|
|
|
from = "BackupEngine::GetBackupInfo";
|
Fix, enable, and enhance backup/restore in db_stress (#7348)
Summary:
Although added to db_stress, testing of backup/restore
was never integrated into the crash test, originally concerned about
performance. I've enabled it now and to address the peformance concern,
testing backup/restore is always skipped once the db exceeds a certain
size threshold, default 100MB. This should provide sufficient
opportunity for testing BackupEngine without bogging down everything
else with heavier and heavier operations.
Also fixed backup/restore in db_stress by making sure PurgeOldBackups
can remove manifest files, which are normally kept around for db_stress.
Added more coverage of backup options, and up to three backups being
saved in one backup directory (in some cases).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7348
Test Plan:
ran 'make blackbox_crash_test' for a while, with heightened
probabilitly of taking backups (1/10k). Also confirmed with some debug
output that the code is being covered, TestBackupRestore only takes
a few seconds to complete when triggered, and even at 1/10k and ~50MB
database, there's <,~ 1 thread testing backups at any time.
Reviewed By: ajkr
Differential Revision: D23510835
Pulled By: pdillinger
fbshipit-source-id: b6b8735591808141f81f10773ac31634cf03b6c0
4 years ago
|
|
|
}
|
|
|
|
}
|
|
|
|
if (s.ok() && thread->rand.OneIn(2)) {
|
|
|
|
s = backup_engine->VerifyBackup(
|
|
|
|
backup_info.front().backup_id,
|
|
|
|
thread->rand.OneIn(2) /* verify_with_checksum */);
|
|
|
|
if (!s.ok()) {
|
|
|
|
from = "BackupEngine::VerifyBackup";
|
|
|
|
}
|
Fix, enable, and enhance backup/restore in db_stress (#7348)
Summary:
Although added to db_stress, testing of backup/restore
was never integrated into the crash test, originally concerned about
performance. I've enabled it now and to address the peformance concern,
testing backup/restore is always skipped once the db exceeds a certain
size threshold, default 100MB. This should provide sufficient
opportunity for testing BackupEngine without bogging down everything
else with heavier and heavier operations.
Also fixed backup/restore in db_stress by making sure PurgeOldBackups
can remove manifest files, which are normally kept around for db_stress.
Added more coverage of backup options, and up to three backups being
saved in one backup directory (in some cases).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7348
Test Plan:
ran 'make blackbox_crash_test' for a while, with heightened
probabilitly of taking backups (1/10k). Also confirmed with some debug
output that the code is being covered, TestBackupRestore only takes
a few seconds to complete when triggered, and even at 1/10k and ~50MB
database, there's <,~ 1 thread testing backups at any time.
Reviewed By: ajkr
Differential Revision: D23510835
Pulled By: pdillinger
fbshipit-source-id: b6b8735591808141f81f10773ac31634cf03b6c0
4 years ago
|
|
|
}
|
|
|
|
const bool allow_persistent = thread->tid == 0; // not too many
|
|
|
|
bool from_latest = false;
|
Make backups openable as read-only DBs (#8142)
Summary:
A current limitation of backups is that you don't know the
exact database state of when the backup was taken. With this new
feature, you can at least inspect the backup's DB state without
restoring it by opening it as a read-only DB.
Rather than add something like OpenAsReadOnlyDB to the BackupEngine API,
which would inhibit opening stackable DB implementations read-only
(if/when their APIs support it), we instead provide a DB name and Env
that can be used to open as a read-only DB.
Possible follow-up work:
* Add a version of GetBackupInfo for a single backup.
* Let CreateNewBackup return the BackupID of the newly-created backup.
Implementation details:
Refactored ChrootFileSystem to split off new base class RemapFileSystem,
which allows more general remapping of files. We use this base class to
implement BackupEngineImpl::RemapSharedFileSystem.
To minimize API impact, I decided to just add these fields `name_for_open`
and `env_for_open` to those set by GetBackupInfo when
include_file_details=true. Creating the RemapSharedFileSystem adds a bit
to the memory consumption, perhaps unnecessarily in some cases, but this
has been mitigated by (a) only initialize the RemapSharedFileSystem
lazily when GetBackupInfo with include_file_details=true is called, and
(b) using the existing `shared_ptr<FileInfo>` objects to hold most of the
mapping data.
To enhance API safety, RemapSharedFileSystem is wrapped by new
ReadOnlyFileSystem which rejects any attempts to write. This uncovered a
couple of places in which DB::OpenForReadOnly would write to the
filesystem, so I fixed these. Added a release note because this affects
logging.
Additional minor refactoring in backupable_db.cc to support the new
functionality.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8142
Test Plan:
new test (run with ASAN and UBSAN), added to stress test and
ran it for a while with amplified backup_one_in
Reviewed By: ajkr
Differential Revision: D27535408
Pulled By: pdillinger
fbshipit-source-id: 04666d310aa0261ef6b2385c43ca793ce1dfd148
4 years ago
|
|
|
int count = static_cast<int>(backup_info.size());
|
|
|
|
if (s.ok() && !inplace_not_restore) {
|
|
|
|
if (count > 1) {
|
|
|
|
s = backup_engine->RestoreDBFromBackup(
|
|
|
|
RestoreOptions(), backup_info[thread->rand.Uniform(count)].backup_id,
|
|
|
|
restore_dir /* db_dir */, restore_dir /* wal_dir */);
|
|
|
|
if (!s.ok()) {
|
|
|
|
from = "BackupEngine::RestoreDBFromBackup";
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
from_latest = true;
|
|
|
|
s = backup_engine->RestoreDBFromLatestBackup(RestoreOptions(),
|
|
|
|
restore_dir /* db_dir */,
|
|
|
|
restore_dir /* wal_dir */);
|
|
|
|
if (!s.ok()) {
|
|
|
|
from = "BackupEngine::RestoreDBFromLatestBackup";
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
Make backups openable as read-only DBs (#8142)
Summary:
A current limitation of backups is that you don't know the
exact database state of when the backup was taken. With this new
feature, you can at least inspect the backup's DB state without
restoring it by opening it as a read-only DB.
Rather than add something like OpenAsReadOnlyDB to the BackupEngine API,
which would inhibit opening stackable DB implementations read-only
(if/when their APIs support it), we instead provide a DB name and Env
that can be used to open as a read-only DB.
Possible follow-up work:
* Add a version of GetBackupInfo for a single backup.
* Let CreateNewBackup return the BackupID of the newly-created backup.
Implementation details:
Refactored ChrootFileSystem to split off new base class RemapFileSystem,
which allows more general remapping of files. We use this base class to
implement BackupEngineImpl::RemapSharedFileSystem.
To minimize API impact, I decided to just add these fields `name_for_open`
and `env_for_open` to those set by GetBackupInfo when
include_file_details=true. Creating the RemapSharedFileSystem adds a bit
to the memory consumption, perhaps unnecessarily in some cases, but this
has been mitigated by (a) only initialize the RemapSharedFileSystem
lazily when GetBackupInfo with include_file_details=true is called, and
(b) using the existing `shared_ptr<FileInfo>` objects to hold most of the
mapping data.
To enhance API safety, RemapSharedFileSystem is wrapped by new
ReadOnlyFileSystem which rejects any attempts to write. This uncovered a
couple of places in which DB::OpenForReadOnly would write to the
filesystem, so I fixed these. Added a release note because this affects
logging.
Additional minor refactoring in backupable_db.cc to support the new
functionality.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8142
Test Plan:
new test (run with ASAN and UBSAN), added to stress test and
ran it for a while with amplified backup_one_in
Reviewed By: ajkr
Differential Revision: D27535408
Pulled By: pdillinger
fbshipit-source-id: 04666d310aa0261ef6b2385c43ca793ce1dfd148
4 years ago
|
|
|
if (s.ok() && !inplace_not_restore) {
|
|
|
|
// Purge early if restoring, to ensure the restored directory doesn't
|
|
|
|
// have some secret dependency on the backup directory.
|
Fix, enable, and enhance backup/restore in db_stress (#7348)
Summary:
Although added to db_stress, testing of backup/restore
was never integrated into the crash test, originally concerned about
performance. I've enabled it now and to address the peformance concern,
testing backup/restore is always skipped once the db exceeds a certain
size threshold, default 100MB. This should provide sufficient
opportunity for testing BackupEngine without bogging down everything
else with heavier and heavier operations.
Also fixed backup/restore in db_stress by making sure PurgeOldBackups
can remove manifest files, which are normally kept around for db_stress.
Added more coverage of backup options, and up to three backups being
saved in one backup directory (in some cases).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7348
Test Plan:
ran 'make blackbox_crash_test' for a while, with heightened
probabilitly of taking backups (1/10k). Also confirmed with some debug
output that the code is being covered, TestBackupRestore only takes
a few seconds to complete when triggered, and even at 1/10k and ~50MB
database, there's <,~ 1 thread testing backups at any time.
Reviewed By: ajkr
Differential Revision: D23510835
Pulled By: pdillinger
fbshipit-source-id: b6b8735591808141f81f10773ac31634cf03b6c0
4 years ago
|
|
|
uint32_t to_keep = 0;
|
|
|
|
if (allow_persistent) {
|
Fix, enable, and enhance backup/restore in db_stress (#7348)
Summary:
Although added to db_stress, testing of backup/restore
was never integrated into the crash test, originally concerned about
performance. I've enabled it now and to address the peformance concern,
testing backup/restore is always skipped once the db exceeds a certain
size threshold, default 100MB. This should provide sufficient
opportunity for testing BackupEngine without bogging down everything
else with heavier and heavier operations.
Also fixed backup/restore in db_stress by making sure PurgeOldBackups
can remove manifest files, which are normally kept around for db_stress.
Added more coverage of backup options, and up to three backups being
saved in one backup directory (in some cases).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7348
Test Plan:
ran 'make blackbox_crash_test' for a while, with heightened
probabilitly of taking backups (1/10k). Also confirmed with some debug
output that the code is being covered, TestBackupRestore only takes
a few seconds to complete when triggered, and even at 1/10k and ~50MB
database, there's <,~ 1 thread testing backups at any time.
Reviewed By: ajkr
Differential Revision: D23510835
Pulled By: pdillinger
fbshipit-source-id: b6b8735591808141f81f10773ac31634cf03b6c0
4 years ago
|
|
|
// allow one thread to keep up to 2 backups
|
|
|
|
to_keep = thread->rand.Uniform(3);
|
|
|
|
}
|
|
|
|
s = backup_engine->PurgeOldBackups(to_keep);
|
|
|
|
if (!s.ok()) {
|
|
|
|
from = "BackupEngine::PurgeOldBackups";
|
|
|
|
}
|
|
|
|
}
|
|
|
|
DB* restored_db = nullptr;
|
|
|
|
std::vector<ColumnFamilyHandle*> restored_cf_handles;
|
|
|
|
// Not yet implemented: opening restored BlobDB or TransactionDB
|
|
|
|
if (s.ok() && !FLAGS_use_txn && !FLAGS_use_blob_db) {
|
|
|
|
Options restore_options(options_);
|
|
|
|
restore_options.best_efforts_recovery = false;
|
|
|
|
restore_options.listeners.clear();
|
|
|
|
// Avoid dangling/shared file descriptors, for reliable destroy
|
|
|
|
restore_options.sst_file_manager = nullptr;
|
|
|
|
std::vector<ColumnFamilyDescriptor> cf_descriptors;
|
|
|
|
// TODO(ajkr): `column_family_names_` is not safe to access here when
|
|
|
|
// `clear_column_family_one_in != 0`. But we can't easily switch to
|
|
|
|
// `ListColumnFamilies` to get names because it won't necessarily give
|
|
|
|
// the same order as `column_family_names_`.
|
|
|
|
assert(FLAGS_clear_column_family_one_in == 0);
|
|
|
|
for (auto name : column_family_names_) {
|
|
|
|
cf_descriptors.emplace_back(name, ColumnFamilyOptions(restore_options));
|
|
|
|
}
|
Make backups openable as read-only DBs (#8142)
Summary:
A current limitation of backups is that you don't know the
exact database state of when the backup was taken. With this new
feature, you can at least inspect the backup's DB state without
restoring it by opening it as a read-only DB.
Rather than add something like OpenAsReadOnlyDB to the BackupEngine API,
which would inhibit opening stackable DB implementations read-only
(if/when their APIs support it), we instead provide a DB name and Env
that can be used to open as a read-only DB.
Possible follow-up work:
* Add a version of GetBackupInfo for a single backup.
* Let CreateNewBackup return the BackupID of the newly-created backup.
Implementation details:
Refactored ChrootFileSystem to split off new base class RemapFileSystem,
which allows more general remapping of files. We use this base class to
implement BackupEngineImpl::RemapSharedFileSystem.
To minimize API impact, I decided to just add these fields `name_for_open`
and `env_for_open` to those set by GetBackupInfo when
include_file_details=true. Creating the RemapSharedFileSystem adds a bit
to the memory consumption, perhaps unnecessarily in some cases, but this
has been mitigated by (a) only initialize the RemapSharedFileSystem
lazily when GetBackupInfo with include_file_details=true is called, and
(b) using the existing `shared_ptr<FileInfo>` objects to hold most of the
mapping data.
To enhance API safety, RemapSharedFileSystem is wrapped by new
ReadOnlyFileSystem which rejects any attempts to write. This uncovered a
couple of places in which DB::OpenForReadOnly would write to the
filesystem, so I fixed these. Added a release note because this affects
logging.
Additional minor refactoring in backupable_db.cc to support the new
functionality.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8142
Test Plan:
new test (run with ASAN and UBSAN), added to stress test and
ran it for a while with amplified backup_one_in
Reviewed By: ajkr
Differential Revision: D27535408
Pulled By: pdillinger
fbshipit-source-id: 04666d310aa0261ef6b2385c43ca793ce1dfd148
4 years ago
|
|
|
if (inplace_not_restore) {
|
|
|
|
BackupInfo& info = backup_info[thread->rand.Uniform(count)];
|
|
|
|
restore_options.env = info.env_for_open.get();
|
|
|
|
s = DB::OpenForReadOnly(DBOptions(restore_options), info.name_for_open,
|
|
|
|
cf_descriptors, &restored_cf_handles,
|
|
|
|
&restored_db);
|
|
|
|
if (!s.ok()) {
|
|
|
|
from = "DB::OpenForReadOnly in backup/restore";
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
s = DB::Open(DBOptions(restore_options), restore_dir, cf_descriptors,
|
|
|
|
&restored_cf_handles, &restored_db);
|
|
|
|
if (!s.ok()) {
|
|
|
|
from = "DB::Open in backup/restore";
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Note the column families chosen by `rand_column_families` cannot be
|
|
|
|
// dropped while the locks for `rand_keys` are held. So we should not have
|
|
|
|
// to worry about accessing those column families throughout this function.
|
|
|
|
//
|
|
|
|
// For simplicity, currently only verifies existence/non-existence of a
|
|
|
|
// single key
|
|
|
|
for (size_t i = 0; restored_db && s.ok() && i < rand_column_families.size();
|
|
|
|
++i) {
|
|
|
|
std::string key_str = Key(rand_keys[0]);
|
|
|
|
Slice key = key_str;
|
|
|
|
std::string restored_value;
|
|
|
|
// This `ReadOptions` is for validation purposes. Ignore
|
|
|
|
// `FLAGS_rate_limit_user_ops` to avoid slowing any validation.
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
ReadOptions read_opts;
|
|
|
|
std::string ts_str;
|
|
|
|
Slice ts;
|
|
|
|
if (FLAGS_user_timestamp_size > 0) {
|
|
|
|
ts_str = GenerateTimestampForRead();
|
|
|
|
ts = ts_str;
|
|
|
|
read_opts.timestamp = &ts;
|
|
|
|
}
|
|
|
|
Status get_status = restored_db->Get(
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
read_opts, restored_cf_handles[rand_column_families[i]], key,
|
|
|
|
&restored_value);
|
|
|
|
bool exists = thread->shared->Exists(rand_column_families[i], rand_keys[0]);
|
|
|
|
if (get_status.ok()) {
|
|
|
|
if (!exists && from_latest && ShouldAcquireMutexOnKey()) {
|
|
|
|
std::ostringstream oss;
|
|
|
|
oss << "0x" << key.ToString(true)
|
|
|
|
<< " exists in restore but not in original db";
|
|
|
|
s = Status::Corruption(oss.str());
|
|
|
|
}
|
|
|
|
} else if (get_status.IsNotFound()) {
|
|
|
|
if (exists && from_latest && ShouldAcquireMutexOnKey()) {
|
|
|
|
std::ostringstream oss;
|
|
|
|
oss << "0x" << key.ToString(true)
|
|
|
|
<< " exists in original db but not in restore";
|
|
|
|
s = Status::Corruption(oss.str());
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
s = get_status;
|
|
|
|
if (!s.ok()) {
|
|
|
|
from = "DB::Get in backup/restore";
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (restored_db != nullptr) {
|
|
|
|
for (auto* cf_handle : restored_cf_handles) {
|
|
|
|
restored_db->DestroyColumnFamilyHandle(cf_handle);
|
|
|
|
}
|
|
|
|
delete restored_db;
|
|
|
|
restored_db = nullptr;
|
|
|
|
}
|
Make backups openable as read-only DBs (#8142)
Summary:
A current limitation of backups is that you don't know the
exact database state of when the backup was taken. With this new
feature, you can at least inspect the backup's DB state without
restoring it by opening it as a read-only DB.
Rather than add something like OpenAsReadOnlyDB to the BackupEngine API,
which would inhibit opening stackable DB implementations read-only
(if/when their APIs support it), we instead provide a DB name and Env
that can be used to open as a read-only DB.
Possible follow-up work:
* Add a version of GetBackupInfo for a single backup.
* Let CreateNewBackup return the BackupID of the newly-created backup.
Implementation details:
Refactored ChrootFileSystem to split off new base class RemapFileSystem,
which allows more general remapping of files. We use this base class to
implement BackupEngineImpl::RemapSharedFileSystem.
To minimize API impact, I decided to just add these fields `name_for_open`
and `env_for_open` to those set by GetBackupInfo when
include_file_details=true. Creating the RemapSharedFileSystem adds a bit
to the memory consumption, perhaps unnecessarily in some cases, but this
has been mitigated by (a) only initialize the RemapSharedFileSystem
lazily when GetBackupInfo with include_file_details=true is called, and
(b) using the existing `shared_ptr<FileInfo>` objects to hold most of the
mapping data.
To enhance API safety, RemapSharedFileSystem is wrapped by new
ReadOnlyFileSystem which rejects any attempts to write. This uncovered a
couple of places in which DB::OpenForReadOnly would write to the
filesystem, so I fixed these. Added a release note because this affects
logging.
Additional minor refactoring in backupable_db.cc to support the new
functionality.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8142
Test Plan:
new test (run with ASAN and UBSAN), added to stress test and
ran it for a while with amplified backup_one_in
Reviewed By: ajkr
Differential Revision: D27535408
Pulled By: pdillinger
fbshipit-source-id: 04666d310aa0261ef6b2385c43ca793ce1dfd148
4 years ago
|
|
|
if (s.ok() && inplace_not_restore) {
|
|
|
|
// Purge late if inplace open read-only
|
|
|
|
uint32_t to_keep = 0;
|
|
|
|
if (allow_persistent) {
|
|
|
|
// allow one thread to keep up to 2 backups
|
|
|
|
to_keep = thread->rand.Uniform(3);
|
|
|
|
}
|
|
|
|
s = backup_engine->PurgeOldBackups(to_keep);
|
|
|
|
if (!s.ok()) {
|
|
|
|
from = "BackupEngine::PurgeOldBackups";
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (backup_engine != nullptr) {
|
|
|
|
delete backup_engine;
|
|
|
|
backup_engine = nullptr;
|
|
|
|
}
|
|
|
|
if (s.ok()) {
|
|
|
|
// Preserve directories on failure, or allowed persistent backup
|
|
|
|
if (!allow_persistent) {
|
|
|
|
s = DestroyDir(db_stress_env, backup_dir);
|
|
|
|
if (!s.ok()) {
|
|
|
|
from = "Destroy backup dir";
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (s.ok()) {
|
|
|
|
s = DestroyDir(db_stress_env, restore_dir);
|
|
|
|
if (!s.ok()) {
|
|
|
|
from = "Destroy restore dir";
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!s.ok()) {
|
|
|
|
fprintf(stderr, "Failure in %s with: %s\n", from.c_str(),
|
|
|
|
s.ToString().c_str());
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
Status StressTest::TestApproximateSize(
|
|
|
|
ThreadState* thread, uint64_t iteration,
|
|
|
|
const std::vector<int>& rand_column_families,
|
|
|
|
const std::vector<int64_t>& rand_keys) {
|
|
|
|
// rand_keys likely only has one key. Just use the first one.
|
|
|
|
assert(!rand_keys.empty());
|
|
|
|
assert(!rand_column_families.empty());
|
|
|
|
int64_t key1 = rand_keys[0];
|
|
|
|
int64_t key2;
|
|
|
|
if (thread->rand.OneIn(2)) {
|
|
|
|
// Two totally random keys. This tends to cover large ranges.
|
|
|
|
key2 = GenerateOneKey(thread, iteration);
|
|
|
|
if (key2 < key1) {
|
|
|
|
std::swap(key1, key2);
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
// Unless users pass a very large FLAGS_max_key, it we should not worry
|
|
|
|
// about overflow. It is for testing, so we skip the overflow checking
|
|
|
|
// for simplicity.
|
|
|
|
key2 = key1 + static_cast<int64_t>(thread->rand.Uniform(1000));
|
|
|
|
}
|
|
|
|
std::string key1_str = Key(key1);
|
|
|
|
std::string key2_str = Key(key2);
|
|
|
|
Range range{Slice(key1_str), Slice(key2_str)};
|
|
|
|
SizeApproximationOptions sao;
|
|
|
|
sao.include_memtables = thread->rand.OneIn(2);
|
|
|
|
if (sao.include_memtables) {
|
|
|
|
sao.include_files = thread->rand.OneIn(2);
|
|
|
|
}
|
|
|
|
if (thread->rand.OneIn(2)) {
|
|
|
|
if (thread->rand.OneIn(2)) {
|
|
|
|
sao.files_size_error_margin = 0.0;
|
|
|
|
} else {
|
|
|
|
sao.files_size_error_margin =
|
|
|
|
static_cast<double>(thread->rand.Uniform(3));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
uint64_t result;
|
|
|
|
return db_->GetApproximateSizes(
|
|
|
|
sao, column_families_[rand_column_families[0]], &range, 1, &result);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status StressTest::TestCheckpoint(ThreadState* thread,
|
|
|
|
const std::vector<int>& rand_column_families,
|
|
|
|
const std::vector<int64_t>& rand_keys) {
|
|
|
|
std::string checkpoint_dir =
|
|
|
|
FLAGS_db + "/.checkpoint" + std::to_string(thread->tid);
|
|
|
|
Options tmp_opts(options_);
|
|
|
|
tmp_opts.listeners.clear();
|
|
|
|
tmp_opts.env = db_stress_env;
|
|
|
|
|
|
|
|
DestroyDB(checkpoint_dir, tmp_opts);
|
|
|
|
|
|
|
|
if (db_stress_env->FileExists(checkpoint_dir).ok()) {
|
|
|
|
// If the directory might still exist, try to delete the files one by one.
|
|
|
|
// Likely a trash file is still there.
|
|
|
|
Status my_s = DestroyDir(db_stress_env, checkpoint_dir);
|
|
|
|
if (!my_s.ok()) {
|
|
|
|
fprintf(stderr, "Fail to destory directory before checkpoint: %s",
|
|
|
|
my_s.ToString().c_str());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Checkpoint* checkpoint = nullptr;
|
|
|
|
Status s = Checkpoint::Create(db_, &checkpoint);
|
|
|
|
if (s.ok()) {
|
|
|
|
s = checkpoint->CreateCheckpoint(checkpoint_dir);
|
|
|
|
if (!s.ok()) {
|
|
|
|
fprintf(stderr, "Fail to create checkpoint to %s\n",
|
|
|
|
checkpoint_dir.c_str());
|
|
|
|
std::vector<std::string> files;
|
|
|
|
Status my_s = db_stress_env->GetChildren(checkpoint_dir, &files);
|
|
|
|
if (my_s.ok()) {
|
|
|
|
for (const auto& f : files) {
|
|
|
|
fprintf(stderr, " %s\n", f.c_str());
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
fprintf(stderr, "Fail to get files under the directory to %s\n",
|
|
|
|
my_s.ToString().c_str());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
delete checkpoint;
|
|
|
|
checkpoint = nullptr;
|
|
|
|
std::vector<ColumnFamilyHandle*> cf_handles;
|
|
|
|
DB* checkpoint_db = nullptr;
|
|
|
|
if (s.ok()) {
|
|
|
|
Options options(options_);
|
|
|
|
options.best_efforts_recovery = false;
|
|
|
|
options.listeners.clear();
|
|
|
|
// Avoid race condition in trash handling after delete checkpoint_db
|
|
|
|
options.sst_file_manager.reset();
|
|
|
|
std::vector<ColumnFamilyDescriptor> cf_descs;
|
|
|
|
// TODO(ajkr): `column_family_names_` is not safe to access here when
|
|
|
|
// `clear_column_family_one_in != 0`. But we can't easily switch to
|
|
|
|
// `ListColumnFamilies` to get names because it won't necessarily give
|
|
|
|
// the same order as `column_family_names_`.
|
|
|
|
assert(FLAGS_clear_column_family_one_in == 0);
|
|
|
|
if (FLAGS_clear_column_family_one_in == 0) {
|
|
|
|
for (const auto& name : column_family_names_) {
|
|
|
|
cf_descs.emplace_back(name, ColumnFamilyOptions(options));
|
|
|
|
}
|
|
|
|
s = DB::OpenForReadOnly(DBOptions(options), checkpoint_dir, cf_descs,
|
|
|
|
&cf_handles, &checkpoint_db);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (checkpoint_db != nullptr) {
|
|
|
|
// Note the column families chosen by `rand_column_families` cannot be
|
|
|
|
// dropped while the locks for `rand_keys` are held. So we should not have
|
|
|
|
// to worry about accessing those column families throughout this function.
|
|
|
|
for (size_t i = 0; s.ok() && i < rand_column_families.size(); ++i) {
|
|
|
|
std::string key_str = Key(rand_keys[0]);
|
|
|
|
Slice key = key_str;
|
|
|
|
std::string ts_str;
|
|
|
|
Slice ts;
|
|
|
|
ReadOptions read_opts;
|
|
|
|
if (FLAGS_user_timestamp_size > 0) {
|
|
|
|
ts_str = GenerateTimestampForRead();
|
|
|
|
ts = ts_str;
|
|
|
|
read_opts.timestamp = &ts;
|
|
|
|
}
|
|
|
|
std::string value;
|
|
|
|
Status get_status = checkpoint_db->Get(
|
|
|
|
read_opts, cf_handles[rand_column_families[i]], key, &value);
|
|
|
|
bool exists =
|
|
|
|
thread->shared->Exists(rand_column_families[i], rand_keys[0]);
|
|
|
|
if (get_status.ok()) {
|
|
|
|
if (!exists && ShouldAcquireMutexOnKey()) {
|
|
|
|
std::ostringstream oss;
|
|
|
|
oss << "0x" << key.ToString(true) << " exists in checkpoint "
|
|
|
|
<< checkpoint_dir << " but not in original db";
|
|
|
|
s = Status::Corruption(oss.str());
|
|
|
|
}
|
|
|
|
} else if (get_status.IsNotFound()) {
|
|
|
|
if (exists && ShouldAcquireMutexOnKey()) {
|
|
|
|
std::ostringstream oss;
|
|
|
|
oss << "0x" << key.ToString(true)
|
|
|
|
<< " exists in original db but not in checkpoint "
|
|
|
|
<< checkpoint_dir;
|
|
|
|
s = Status::Corruption(oss.str());
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
s = get_status;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for (auto cfh : cf_handles) {
|
|
|
|
delete cfh;
|
|
|
|
}
|
|
|
|
cf_handles.clear();
|
|
|
|
delete checkpoint_db;
|
|
|
|
checkpoint_db = nullptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!s.ok()) {
|
|
|
|
fprintf(stderr, "A checkpoint operation failed with: %s\n",
|
|
|
|
s.ToString().c_str());
|
|
|
|
} else {
|
|
|
|
DestroyDB(checkpoint_dir, tmp_opts);
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
void StressTest::TestGetProperty(ThreadState* thread) const {
|
|
|
|
std::unordered_set<std::string> levelPropertyNames = {
|
|
|
|
DB::Properties::kAggregatedTablePropertiesAtLevel,
|
|
|
|
DB::Properties::kCompressionRatioAtLevelPrefix,
|
|
|
|
DB::Properties::kNumFilesAtLevelPrefix,
|
|
|
|
};
|
|
|
|
std::unordered_set<std::string> unknownPropertyNames = {
|
|
|
|
DB::Properties::kEstimateOldestKeyTime,
|
|
|
|
DB::Properties::kOptionsStatistics,
|
|
|
|
DB::Properties::
|
|
|
|
kLiveSstFilesSizeAtTemperature, // similar to levelPropertyNames, it
|
|
|
|
// requires a number suffix
|
|
|
|
};
|
|
|
|
unknownPropertyNames.insert(levelPropertyNames.begin(),
|
|
|
|
levelPropertyNames.end());
|
|
|
|
|
|
|
|
std::string prop;
|
|
|
|
for (const auto& ppt_name_and_info : InternalStats::ppt_name_to_info) {
|
|
|
|
bool res = db_->GetProperty(ppt_name_and_info.first, &prop);
|
|
|
|
if (unknownPropertyNames.find(ppt_name_and_info.first) ==
|
|
|
|
unknownPropertyNames.end()) {
|
|
|
|
if (!res) {
|
|
|
|
fprintf(stderr, "Failed to get DB property: %s\n",
|
|
|
|
ppt_name_and_info.first.c_str());
|
|
|
|
thread->shared->SetVerificationFailure();
|
|
|
|
}
|
|
|
|
if (ppt_name_and_info.second.handle_int != nullptr) {
|
|
|
|
uint64_t prop_int;
|
|
|
|
if (!db_->GetIntProperty(ppt_name_and_info.first, &prop_int)) {
|
|
|
|
fprintf(stderr, "Failed to get Int property: %s\n",
|
|
|
|
ppt_name_and_info.first.c_str());
|
|
|
|
thread->shared->SetVerificationFailure();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (ppt_name_and_info.second.handle_map != nullptr) {
|
|
|
|
std::map<std::string, std::string> prop_map;
|
|
|
|
if (!db_->GetMapProperty(ppt_name_and_info.first, &prop_map)) {
|
|
|
|
fprintf(stderr, "Failed to get Map property: %s\n",
|
|
|
|
ppt_name_and_info.first.c_str());
|
|
|
|
thread->shared->SetVerificationFailure();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
ROCKSDB_NAMESPACE::ColumnFamilyMetaData cf_meta_data;
|
|
|
|
db_->GetColumnFamilyMetaData(&cf_meta_data);
|
|
|
|
int level_size = static_cast<int>(cf_meta_data.levels.size());
|
|
|
|
for (int level = 0; level < level_size; level++) {
|
|
|
|
for (const auto& ppt_name : levelPropertyNames) {
|
|
|
|
bool res = db_->GetProperty(ppt_name + std::to_string(level), &prop);
|
|
|
|
if (!res) {
|
|
|
|
fprintf(stderr, "Failed to get DB property: %s\n",
|
|
|
|
(ppt_name + std::to_string(level)).c_str());
|
|
|
|
thread->shared->SetVerificationFailure();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Test for an invalid property name
|
|
|
|
if (thread->rand.OneIn(100)) {
|
|
|
|
if (db_->GetProperty("rocksdb.invalid_property_name", &prop)) {
|
|
|
|
fprintf(stderr, "Failed to return false for invalid property name\n");
|
|
|
|
thread->shared->SetVerificationFailure();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void StressTest::TestCompactFiles(ThreadState* thread,
|
|
|
|
ColumnFamilyHandle* column_family) {
|
|
|
|
ROCKSDB_NAMESPACE::ColumnFamilyMetaData cf_meta_data;
|
|
|
|
db_->GetColumnFamilyMetaData(column_family, &cf_meta_data);
|
|
|
|
|
|
|
|
if (cf_meta_data.levels.empty()) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Randomly compact up to three consecutive files from a level
|
|
|
|
const int kMaxRetry = 3;
|
|
|
|
for (int attempt = 0; attempt < kMaxRetry; ++attempt) {
|
|
|
|
size_t random_level =
|
|
|
|
thread->rand.Uniform(static_cast<int>(cf_meta_data.levels.size()));
|
|
|
|
|
|
|
|
const auto& files = cf_meta_data.levels[random_level].files;
|
|
|
|
if (files.size() > 0) {
|
|
|
|
size_t random_file_index =
|
|
|
|
thread->rand.Uniform(static_cast<int>(files.size()));
|
|
|
|
if (files[random_file_index].being_compacted) {
|
|
|
|
// Retry as the selected file is currently being compacted
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
std::vector<std::string> input_files;
|
|
|
|
input_files.push_back(files[random_file_index].name);
|
|
|
|
if (random_file_index > 0 &&
|
|
|
|
!files[random_file_index - 1].being_compacted) {
|
|
|
|
input_files.push_back(files[random_file_index - 1].name);
|
|
|
|
}
|
|
|
|
if (random_file_index + 1 < files.size() &&
|
|
|
|
!files[random_file_index + 1].being_compacted) {
|
|
|
|
input_files.push_back(files[random_file_index + 1].name);
|
|
|
|
}
|
|
|
|
|
|
|
|
size_t output_level =
|
|
|
|
std::min(random_level + 1, cf_meta_data.levels.size() - 1);
|
|
|
|
auto s = db_->CompactFiles(CompactionOptions(), column_family,
|
|
|
|
input_files, static_cast<int>(output_level));
|
|
|
|
if (!s.ok()) {
|
|
|
|
fprintf(stdout, "Unable to perform CompactFiles(): %s\n",
|
|
|
|
s.ToString().c_str());
|
|
|
|
thread->stats.AddNumCompactFilesFailed(1);
|
|
|
|
} else {
|
|
|
|
thread->stats.AddNumCompactFilesSucceed(1);
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif // ROCKSDB_LITE
|
|
|
|
|
|
|
|
Status StressTest::TestFlush(const std::vector<int>& rand_column_families) {
|
|
|
|
FlushOptions flush_opts;
|
|
|
|
if (FLAGS_atomic_flush) {
|
|
|
|
return db_->Flush(flush_opts, column_families_);
|
|
|
|
}
|
|
|
|
std::vector<ColumnFamilyHandle*> cfhs;
|
|
|
|
std::for_each(rand_column_families.begin(), rand_column_families.end(),
|
|
|
|
[this, &cfhs](int k) { cfhs.push_back(column_families_[k]); });
|
|
|
|
return db_->Flush(flush_opts, cfhs);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status StressTest::TestPauseBackground(ThreadState* thread) {
|
|
|
|
Status status = db_->PauseBackgroundWork();
|
|
|
|
if (!status.ok()) {
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
// To avoid stalling/deadlocking ourself in this thread, just
|
|
|
|
// sleep here during pause and let other threads do db operations.
|
|
|
|
// Sleep up to ~16 seconds (2**24 microseconds), but very skewed
|
|
|
|
// toward short pause. (1 chance in 25 of pausing >= 1s;
|
|
|
|
// 1 chance in 625 of pausing full 16s.)
|
|
|
|
int pwr2_micros =
|
|
|
|
std::min(thread->rand.Uniform(25), thread->rand.Uniform(25));
|
|
|
|
clock_->SleepForMicroseconds(1 << pwr2_micros);
|
|
|
|
return db_->ContinueBackgroundWork();
|
|
|
|
}
|
|
|
|
|
|
|
|
void StressTest::TestAcquireSnapshot(ThreadState* thread,
|
|
|
|
int rand_column_family,
|
|
|
|
const std::string& keystr, uint64_t i) {
|
|
|
|
Slice key = keystr;
|
|
|
|
ColumnFamilyHandle* column_family = column_families_[rand_column_family];
|
|
|
|
// This `ReadOptions` is for validation purposes. Ignore
|
|
|
|
// `FLAGS_rate_limit_user_ops` to avoid slowing any validation.
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
ReadOptions ropt;
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
auto db_impl = static_cast_with_check<DBImpl>(db_->GetRootDB());
|
|
|
|
const bool ww_snapshot = thread->rand.OneIn(10);
|
|
|
|
const Snapshot* snapshot =
|
|
|
|
ww_snapshot ? db_impl->GetSnapshotForWriteConflictBoundary()
|
|
|
|
: db_->GetSnapshot();
|
|
|
|
#else
|
|
|
|
const Snapshot* snapshot = db_->GetSnapshot();
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
ropt.snapshot = snapshot;
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
|
|
|
|
// Ideally, we want snapshot taking and timestamp generation to be atomic
|
|
|
|
// here, so that the snapshot corresponds to the timestamp. However, it is
|
|
|
|
// not possible with current GetSnapshot() API.
|
|
|
|
std::string ts_str;
|
|
|
|
Slice ts;
|
|
|
|
if (FLAGS_user_timestamp_size > 0) {
|
|
|
|
ts_str = GenerateTimestampForRead();
|
|
|
|
ts = ts_str;
|
|
|
|
ropt.timestamp = &ts;
|
|
|
|
}
|
|
|
|
|
|
|
|
std::string value_at;
|
|
|
|
// When taking a snapshot, we also read a key from that snapshot. We
|
|
|
|
// will later read the same key before releasing the snapshot and
|
|
|
|
// verify that the results are the same.
|
|
|
|
auto status_at = db_->Get(ropt, column_family, key, &value_at);
|
|
|
|
std::vector<bool>* key_vec = nullptr;
|
|
|
|
|
|
|
|
if (FLAGS_compare_full_db_state_snapshot && (thread->tid == 0)) {
|
|
|
|
key_vec = new std::vector<bool>(FLAGS_max_key);
|
|
|
|
// When `prefix_extractor` is set, seeking to beginning and scanning
|
|
|
|
// across prefixes are only supported with `total_order_seek` set.
|
|
|
|
ropt.total_order_seek = true;
|
|
|
|
std::unique_ptr<Iterator> iterator(db_->NewIterator(ropt));
|
|
|
|
for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
|
|
|
|
uint64_t key_val;
|
|
|
|
if (GetIntVal(iterator->key().ToString(), &key_val)) {
|
|
|
|
(*key_vec)[key_val] = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
ThreadState::SnapshotState snap_state = {snapshot,
|
|
|
|
rand_column_family,
|
|
|
|
column_family->GetName(),
|
|
|
|
keystr,
|
|
|
|
status_at,
|
|
|
|
value_at,
|
|
|
|
key_vec,
|
|
|
|
ts_str};
|
|
|
|
uint64_t hold_for = FLAGS_snapshot_hold_ops;
|
|
|
|
if (FLAGS_long_running_snapshots) {
|
|
|
|
// Hold 10% of snapshots for 10x more
|
|
|
|
if (thread->rand.OneIn(10)) {
|
|
|
|
assert(hold_for < std::numeric_limits<uint64_t>::max() / 10);
|
|
|
|
hold_for *= 10;
|
|
|
|
// Hold 1% of snapshots for 100x more
|
|
|
|
if (thread->rand.OneIn(10)) {
|
|
|
|
assert(hold_for < std::numeric_limits<uint64_t>::max() / 10);
|
|
|
|
hold_for *= 10;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
uint64_t release_at = std::min(FLAGS_ops_per_thread - 1, i + hold_for);
|
|
|
|
thread->snapshot_queue.emplace(release_at, snap_state);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status StressTest::MaybeReleaseSnapshots(ThreadState* thread, uint64_t i) {
|
|
|
|
while (!thread->snapshot_queue.empty() &&
|
|
|
|
i >= thread->snapshot_queue.front().first) {
|
|
|
|
auto snap_state = thread->snapshot_queue.front().second;
|
|
|
|
assert(snap_state.snapshot);
|
|
|
|
// Note: this is unsafe as the cf might be dropped concurrently. But
|
|
|
|
// it is ok since unclean cf drop is cunnrently not supported by write
|
|
|
|
// prepared transactions.
|
|
|
|
Status s = AssertSame(db_, column_families_[snap_state.cf_at], snap_state);
|
|
|
|
db_->ReleaseSnapshot(snap_state.snapshot);
|
|
|
|
delete snap_state.key_vec;
|
|
|
|
thread->snapshot_queue.pop();
|
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
|
|
|
void StressTest::TestCompactRange(ThreadState* thread, int64_t rand_key,
|
|
|
|
const Slice& start_key,
|
|
|
|
ColumnFamilyHandle* column_family) {
|
|
|
|
int64_t end_key_num;
|
|
|
|
if (std::numeric_limits<int64_t>::max() - rand_key <
|
|
|
|
FLAGS_compact_range_width) {
|
|
|
|
end_key_num = std::numeric_limits<int64_t>::max();
|
|
|
|
} else {
|
|
|
|
end_key_num = FLAGS_compact_range_width + rand_key;
|
|
|
|
}
|
|
|
|
std::string end_key_buf = Key(end_key_num);
|
|
|
|
Slice end_key(end_key_buf);
|
|
|
|
|
|
|
|
CompactRangeOptions cro;
|
|
|
|
cro.exclusive_manual_compaction = static_cast<bool>(thread->rand.Next() % 2);
|
|
|
|
cro.change_level = static_cast<bool>(thread->rand.Next() % 2);
|
|
|
|
std::vector<BottommostLevelCompaction> bottom_level_styles = {
|
|
|
|
BottommostLevelCompaction::kSkip,
|
|
|
|
BottommostLevelCompaction::kIfHaveCompactionFilter,
|
|
|
|
BottommostLevelCompaction::kForce,
|
|
|
|
BottommostLevelCompaction::kForceOptimized};
|
|
|
|
cro.bottommost_level_compaction =
|
|
|
|
bottom_level_styles[thread->rand.Next() %
|
|
|
|
static_cast<uint32_t>(bottom_level_styles.size())];
|
|
|
|
cro.allow_write_stall = static_cast<bool>(thread->rand.Next() % 2);
|
|
|
|
cro.max_subcompactions = static_cast<uint32_t>(thread->rand.Next() % 4);
|
|
|
|
std::vector<BlobGarbageCollectionPolicy> blob_gc_policies = {
|
|
|
|
BlobGarbageCollectionPolicy::kForce,
|
|
|
|
BlobGarbageCollectionPolicy::kDisable,
|
|
|
|
BlobGarbageCollectionPolicy::kUseDefault};
|
|
|
|
cro.blob_garbage_collection_policy =
|
|
|
|
blob_gc_policies[thread->rand.Next() %
|
|
|
|
static_cast<uint32_t>(blob_gc_policies.size())];
|
|
|
|
cro.blob_garbage_collection_age_cutoff =
|
|
|
|
static_cast<double>(thread->rand.Next() % 100) / 100.0;
|
|
|
|
|
|
|
|
const Snapshot* pre_snapshot = nullptr;
|
|
|
|
uint32_t pre_hash = 0;
|
|
|
|
if (thread->rand.OneIn(2)) {
|
|
|
|
// Do some validation by declaring a snapshot and compare the data before
|
|
|
|
// and after the compaction
|
|
|
|
pre_snapshot = db_->GetSnapshot();
|
|
|
|
pre_hash =
|
|
|
|
GetRangeHash(thread, pre_snapshot, column_family, start_key, end_key);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status status = db_->CompactRange(cro, column_family, &start_key, &end_key);
|
|
|
|
|
|
|
|
if (!status.ok()) {
|
|
|
|
fprintf(stdout, "Unable to perform CompactRange(): %s\n",
|
|
|
|
status.ToString().c_str());
|
|
|
|
}
|
|
|
|
|
|
|
|
if (pre_snapshot != nullptr) {
|
|
|
|
uint32_t post_hash =
|
|
|
|
GetRangeHash(thread, pre_snapshot, column_family, start_key, end_key);
|
|
|
|
if (pre_hash != post_hash) {
|
|
|
|
fprintf(stderr,
|
|
|
|
"Data hash different before and after compact range "
|
|
|
|
"start_key %s end_key %s\n",
|
|
|
|
start_key.ToString(true).c_str(), end_key.ToString(true).c_str());
|
|
|
|
thread->stats.AddErrors(1);
|
|
|
|
// Fail fast to preserve the DB state.
|
|
|
|
thread->shared->SetVerificationFailure();
|
|
|
|
}
|
|
|
|
db_->ReleaseSnapshot(pre_snapshot);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
uint32_t StressTest::GetRangeHash(ThreadState* thread, const Snapshot* snapshot,
|
|
|
|
ColumnFamilyHandle* column_family,
|
|
|
|
const Slice& start_key,
|
|
|
|
const Slice& end_key) {
|
|
|
|
const std::string kCrcCalculatorSepearator = ";";
|
|
|
|
uint32_t crc = 0;
|
|
|
|
// This `ReadOptions` is for validation purposes. Ignore
|
|
|
|
// `FLAGS_rate_limit_user_ops` to avoid slowing any validation.
|
|
|
|
ReadOptions ro;
|
|
|
|
ro.snapshot = snapshot;
|
|
|
|
ro.total_order_seek = true;
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
std::string ts_str;
|
|
|
|
Slice ts;
|
|
|
|
if (FLAGS_user_timestamp_size > 0) {
|
|
|
|
ts_str = GenerateTimestampForRead();
|
|
|
|
ts = ts_str;
|
|
|
|
ro.timestamp = &ts;
|
|
|
|
}
|
|
|
|
std::unique_ptr<Iterator> it(db_->NewIterator(ro, column_family));
|
|
|
|
for (it->Seek(start_key);
|
|
|
|
it->Valid() && options_.comparator->Compare(it->key(), end_key) <= 0;
|
|
|
|
it->Next()) {
|
|
|
|
crc = crc32c::Extend(crc, it->key().data(), it->key().size());
|
|
|
|
crc = crc32c::Extend(crc, kCrcCalculatorSepearator.data(), 1);
|
|
|
|
crc = crc32c::Extend(crc, it->value().data(), it->value().size());
|
|
|
|
crc = crc32c::Extend(crc, kCrcCalculatorSepearator.data(), 1);
|
|
|
|
}
|
|
|
|
if (!it->status().ok()) {
|
|
|
|
fprintf(stderr, "Iterator non-OK when calculating range CRC: %s\n",
|
|
|
|
it->status().ToString().c_str());
|
|
|
|
thread->stats.AddErrors(1);
|
|
|
|
// Fail fast to preserve the DB state.
|
|
|
|
thread->shared->SetVerificationFailure();
|
|
|
|
}
|
|
|
|
return crc;
|
|
|
|
}
|
|
|
|
|
|
|
|
void StressTest::PrintEnv() const {
|
|
|
|
fprintf(stdout, "RocksDB version : %d.%d\n", kMajorVersion,
|
|
|
|
kMinorVersion);
|
|
|
|
fprintf(stdout, "Format version : %d\n", FLAGS_format_version);
|
|
|
|
fprintf(stdout, "TransactionDB : %s\n",
|
|
|
|
FLAGS_use_txn ? "true" : "false");
|
|
|
|
|
|
|
|
if (FLAGS_use_txn) {
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
fprintf(stdout, "Two write queues: : %s\n",
|
|
|
|
FLAGS_two_write_queues ? "true" : "false");
|
|
|
|
fprintf(stdout, "Write policy : %d\n",
|
|
|
|
static_cast<int>(FLAGS_txn_write_policy));
|
|
|
|
if (static_cast<uint64_t>(TxnDBWritePolicy::WRITE_PREPARED) ==
|
|
|
|
FLAGS_txn_write_policy ||
|
|
|
|
static_cast<uint64_t>(TxnDBWritePolicy::WRITE_UNPREPARED) ==
|
|
|
|
FLAGS_txn_write_policy) {
|
|
|
|
fprintf(stdout, "Snapshot cache bits : %d\n",
|
|
|
|
static_cast<int>(FLAGS_wp_snapshot_cache_bits));
|
|
|
|
fprintf(stdout, "Commit cache bits : %d\n",
|
|
|
|
static_cast<int>(FLAGS_wp_commit_cache_bits));
|
|
|
|
}
|
|
|
|
fprintf(stdout, "last cwb for recovery : %s\n",
|
|
|
|
FLAGS_use_only_the_last_commit_time_batch_for_recovery ? "true"
|
|
|
|
: "false");
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
fprintf(stdout, "Stacked BlobDB : %s\n",
|
|
|
|
FLAGS_use_blob_db ? "true" : "false");
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
fprintf(stdout, "Read only mode : %s\n",
|
|
|
|
FLAGS_read_only ? "true" : "false");
|
|
|
|
fprintf(stdout, "Atomic flush : %s\n",
|
|
|
|
FLAGS_atomic_flush ? "true" : "false");
|
|
|
|
fprintf(stdout, "Column families : %d\n", FLAGS_column_families);
|
|
|
|
if (!FLAGS_test_batches_snapshots) {
|
|
|
|
fprintf(stdout, "Clear CFs one in : %d\n",
|
|
|
|
FLAGS_clear_column_family_one_in);
|
|
|
|
}
|
|
|
|
fprintf(stdout, "Number of threads : %d\n", FLAGS_threads);
|
|
|
|
fprintf(stdout, "Ops per thread : %lu\n",
|
|
|
|
(unsigned long)FLAGS_ops_per_thread);
|
|
|
|
std::string ttl_state("unused");
|
|
|
|
if (FLAGS_ttl > 0) {
|
|
|
|
ttl_state = std::to_string(FLAGS_ttl);
|
|
|
|
}
|
|
|
|
fprintf(stdout, "Time to live(sec) : %s\n", ttl_state.c_str());
|
|
|
|
fprintf(stdout, "Read percentage : %d%%\n", FLAGS_readpercent);
|
|
|
|
fprintf(stdout, "Prefix percentage : %d%%\n", FLAGS_prefixpercent);
|
|
|
|
fprintf(stdout, "Write percentage : %d%%\n", FLAGS_writepercent);
|
|
|
|
fprintf(stdout, "Delete percentage : %d%%\n", FLAGS_delpercent);
|
|
|
|
fprintf(stdout, "Delete range percentage : %d%%\n", FLAGS_delrangepercent);
|
|
|
|
fprintf(stdout, "No overwrite percentage : %d%%\n",
|
|
|
|
FLAGS_nooverwritepercent);
|
|
|
|
fprintf(stdout, "Iterate percentage : %d%%\n", FLAGS_iterpercent);
|
|
|
|
fprintf(stdout, "Custom ops percentage : %d%%\n", FLAGS_customopspercent);
|
|
|
|
fprintf(stdout, "DB-write-buffer-size : %" PRIu64 "\n",
|
|
|
|
FLAGS_db_write_buffer_size);
|
|
|
|
fprintf(stdout, "Write-buffer-size : %d\n", FLAGS_write_buffer_size);
|
|
|
|
fprintf(stdout, "Iterations : %lu\n",
|
|
|
|
(unsigned long)FLAGS_num_iterations);
|
|
|
|
fprintf(stdout, "Max key : %lu\n",
|
|
|
|
(unsigned long)FLAGS_max_key);
|
|
|
|
fprintf(stdout, "Ratio #ops/#keys : %f\n",
|
|
|
|
(1.0 * FLAGS_ops_per_thread * FLAGS_threads) / FLAGS_max_key);
|
|
|
|
fprintf(stdout, "Num times DB reopens : %d\n", FLAGS_reopen);
|
|
|
|
fprintf(stdout, "Batches/snapshots : %d\n",
|
|
|
|
FLAGS_test_batches_snapshots);
|
|
|
|
fprintf(stdout, "Do update in place : %d\n", FLAGS_in_place_update);
|
|
|
|
fprintf(stdout, "Num keys per lock : %d\n",
|
|
|
|
1 << FLAGS_log2_keys_per_lock);
|
|
|
|
std::string compression = CompressionTypeToString(compression_type_e);
|
|
|
|
fprintf(stdout, "Compression : %s\n", compression.c_str());
|
|
|
|
std::string bottommost_compression =
|
|
|
|
CompressionTypeToString(bottommost_compression_type_e);
|
|
|
|
fprintf(stdout, "Bottommost Compression : %s\n",
|
|
|
|
bottommost_compression.c_str());
|
|
|
|
std::string checksum = ChecksumTypeToString(checksum_type_e);
|
|
|
|
fprintf(stdout, "Checksum type : %s\n", checksum.c_str());
|
|
|
|
fprintf(stdout, "File checksum impl : %s\n",
|
|
|
|
FLAGS_file_checksum_impl.c_str());
|
|
|
|
fprintf(stdout, "Bloom bits / key : %s\n",
|
|
|
|
FormatDoubleParam(FLAGS_bloom_bits).c_str());
|
|
|
|
fprintf(stdout, "Max subcompactions : %" PRIu64 "\n",
|
|
|
|
FLAGS_subcompactions);
|
|
|
|
fprintf(stdout, "Use MultiGet : %s\n",
|
|
|
|
FLAGS_use_multiget ? "true" : "false");
|
|
|
|
|
|
|
|
const char* memtablerep = "";
|
|
|
|
switch (FLAGS_rep_factory) {
|
|
|
|
case kSkipList:
|
|
|
|
memtablerep = "skip_list";
|
|
|
|
break;
|
|
|
|
case kHashSkipList:
|
|
|
|
memtablerep = "prefix_hash";
|
|
|
|
break;
|
|
|
|
case kVectorRep:
|
|
|
|
memtablerep = "vector";
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
fprintf(stdout, "Memtablerep : %s\n", memtablerep);
|
|
|
|
|
|
|
|
#ifndef NDEBUG
|
|
|
|
KillPoint* kp = KillPoint::GetInstance();
|
|
|
|
fprintf(stdout, "Test kill odd : %d\n", kp->rocksdb_kill_odds);
|
|
|
|
if (!kp->rocksdb_kill_exclude_prefixes.empty()) {
|
|
|
|
fprintf(stdout, "Skipping kill points prefixes:\n");
|
|
|
|
for (auto& p : kp->rocksdb_kill_exclude_prefixes) {
|
|
|
|
fprintf(stdout, " %s\n", p.c_str());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
fprintf(stdout, "Periodic Compaction Secs : %" PRIu64 "\n",
|
|
|
|
FLAGS_periodic_compaction_seconds);
|
|
|
|
fprintf(stdout, "Compaction TTL : %" PRIu64 "\n",
|
|
|
|
FLAGS_compaction_ttl);
|
|
|
|
fprintf(stdout, "Background Purge : %d\n",
|
|
|
|
static_cast<int>(FLAGS_avoid_unnecessary_blocking_io));
|
|
|
|
fprintf(stdout, "Write DB ID to manifest : %d\n",
|
|
|
|
static_cast<int>(FLAGS_write_dbid_to_manifest));
|
|
|
|
fprintf(stdout, "Max Write Batch Group Size: %" PRIu64 "\n",
|
|
|
|
FLAGS_max_write_batch_group_size_bytes);
|
|
|
|
fprintf(stdout, "Use dynamic level : %d\n",
|
|
|
|
static_cast<int>(FLAGS_level_compaction_dynamic_level_bytes));
|
|
|
|
fprintf(stdout, "Read fault one in : %d\n", FLAGS_read_fault_one_in);
|
|
|
|
fprintf(stdout, "Write fault one in : %d\n", FLAGS_write_fault_one_in);
|
|
|
|
fprintf(stdout, "Open metadata write fault one in:\n");
|
|
|
|
fprintf(stdout, " %d\n",
|
|
|
|
FLAGS_open_metadata_write_fault_one_in);
|
|
|
|
fprintf(stdout, "Sync fault injection : %d\n",
|
|
|
|
FLAGS_sync_fault_injection);
|
|
|
|
fprintf(stdout, "Best efforts recovery : %d\n",
|
|
|
|
static_cast<int>(FLAGS_best_efforts_recovery));
|
|
|
|
fprintf(stdout, "Fail if OPTIONS file error: %d\n",
|
|
|
|
static_cast<int>(FLAGS_fail_if_options_file_error));
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
fprintf(stdout, "User timestamp size bytes : %d\n",
|
|
|
|
static_cast<int>(FLAGS_user_timestamp_size));
|
|
|
|
fprintf(stdout, "WAL compression : %s\n",
|
|
|
|
FLAGS_wal_compression.c_str());
|
|
|
|
fprintf(stdout, "Try verify sst unique id : %d\n",
|
|
|
|
static_cast<int>(FLAGS_verify_sst_unique_id_in_manifest));
|
|
|
|
|
|
|
|
fprintf(stdout, "------------------------------------------------\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
void StressTest::Open(SharedState* shared) {
|
|
|
|
assert(db_ == nullptr);
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
assert(txn_db_ == nullptr);
|
|
|
|
#else
|
|
|
|
(void)shared;
|
|
|
|
#endif
|
Avoid overwriting options loaded from OPTIONS (#9943)
Summary:
This is similar to https://github.com/facebook/rocksdb/issues/9862, including the following fixes/refactoring:
1. If OPTIONS file is specified via `-options_file`, majority of options will be loaded from the file. We should not
overwrite options that have been loaded from the file. Instead, we configure only fields of options which are
shared objects and not set by the OPTIONS file. We also configure a few fields, e.g. `create_if_missing` necessary
for stress test to run.
2. Refactor options initialization into three functions, `InitializeOptionsFromFile()`, `InitializeOptionsFromFlags()`
and `InitializeOptionsGeneral()` similar to db_bench. I hope they can be shared in the future. The high-level logic is
as follows:
```cpp
if (!InitializeOptionsFromFile(...)) {
InitializeOptionsFromFlags(...);
}
InitializeOptionsGeneral(...);
```
3. Currently, the setting for `block_cache_compressed` does not seem correct because it by default specifies a
size of `numeric_limits<size_t>::max()` ((size_t)-1). According to code comments, `-1` indicates default value,
which should be referring to `num_shard_bits` argument.
4. Clarify `fail_if_options_file_error`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9943
Test Plan:
1. make check
2. Run stress tests, and manually check generated OPTIONS file and compare them with input OPTIONS files
Reviewed By: jay-zhuang
Differential Revision: D36133769
Pulled By: riversand963
fbshipit-source-id: 35dacdc090a0a72c922907170cd132b9ecaa073e
3 years ago
|
|
|
if (!InitializeOptionsFromFile(options_)) {
|
|
|
|
InitializeOptionsFromFlags(cache_, compressed_cache_, filter_policy_,
|
|
|
|
options_);
|
|
|
|
}
|
Avoid overwriting options loaded from OPTIONS (#9943)
Summary:
This is similar to https://github.com/facebook/rocksdb/issues/9862, including the following fixes/refactoring:
1. If OPTIONS file is specified via `-options_file`, majority of options will be loaded from the file. We should not
overwrite options that have been loaded from the file. Instead, we configure only fields of options which are
shared objects and not set by the OPTIONS file. We also configure a few fields, e.g. `create_if_missing` necessary
for stress test to run.
2. Refactor options initialization into three functions, `InitializeOptionsFromFile()`, `InitializeOptionsFromFlags()`
and `InitializeOptionsGeneral()` similar to db_bench. I hope they can be shared in the future. The high-level logic is
as follows:
```cpp
if (!InitializeOptionsFromFile(...)) {
InitializeOptionsFromFlags(...);
}
InitializeOptionsGeneral(...);
```
3. Currently, the setting for `block_cache_compressed` does not seem correct because it by default specifies a
size of `numeric_limits<size_t>::max()` ((size_t)-1). According to code comments, `-1` indicates default value,
which should be referring to `num_shard_bits` argument.
4. Clarify `fail_if_options_file_error`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9943
Test Plan:
1. make check
2. Run stress tests, and manually check generated OPTIONS file and compare them with input OPTIONS files
Reviewed By: jay-zhuang
Differential Revision: D36133769
Pulled By: riversand963
fbshipit-source-id: 35dacdc090a0a72c922907170cd132b9ecaa073e
3 years ago
|
|
|
InitializeOptionsGeneral(cache_, compressed_cache_, filter_policy_, options_);
|
|
|
|
|
|
|
|
if (FLAGS_prefix_size == 0 && FLAGS_rep_factory == kHashSkipList) {
|
|
|
|
fprintf(stderr,
|
|
|
|
"prefeix_size cannot be zero if memtablerep == prefix_hash\n");
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
if (FLAGS_prefix_size != 0 && FLAGS_rep_factory != kHashSkipList) {
|
|
|
|
fprintf(stderr,
|
|
|
|
"WARNING: prefix_size is non-zero but "
|
|
|
|
"memtablerep != prefix_hash\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
if ((options_.enable_blob_files || options_.enable_blob_garbage_collection ||
|
|
|
|
FLAGS_allow_setting_blob_options_dynamically) &&
|
|
|
|
FLAGS_best_efforts_recovery) {
|
|
|
|
fprintf(stderr,
|
|
|
|
"Integrated BlobDB is currently incompatible with best-effort "
|
|
|
|
"recovery\n");
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
|
|
|
|
fprintf(stdout,
|
|
|
|
"Integrated BlobDB: blob files enabled %d, min blob size %" PRIu64
|
|
|
|
", blob file size %" PRIu64
|
|
|
|
", blob compression type %s, blob GC enabled %d, cutoff %f, force "
|
|
|
|
"threshold %f, blob compaction readahead size %" PRIu64
|
|
|
|
", blob file starting level %d\n",
|
|
|
|
options_.enable_blob_files, options_.min_blob_size,
|
|
|
|
options_.blob_file_size,
|
|
|
|
CompressionTypeToString(options_.blob_compression_type).c_str(),
|
|
|
|
options_.enable_blob_garbage_collection,
|
|
|
|
options_.blob_garbage_collection_age_cutoff,
|
|
|
|
options_.blob_garbage_collection_force_threshold,
|
|
|
|
options_.blob_compaction_readahead_size,
|
|
|
|
options_.blob_file_starting_level);
|
|
|
|
|
|
|
|
fprintf(stdout, "DB path: [%s]\n", FLAGS_db.c_str());
|
|
|
|
|
|
|
|
Status s;
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
|
|
|
|
if (FLAGS_ttl == -1) {
|
|
|
|
std::vector<std::string> existing_column_families;
|
|
|
|
s = DB::ListColumnFamilies(DBOptions(options_), FLAGS_db,
|
|
|
|
&existing_column_families); // ignore errors
|
|
|
|
if (!s.ok()) {
|
|
|
|
// DB doesn't exist
|
|
|
|
assert(existing_column_families.empty());
|
|
|
|
assert(column_family_names_.empty());
|
|
|
|
column_family_names_.push_back(kDefaultColumnFamilyName);
|
|
|
|
} else if (column_family_names_.empty()) {
|
|
|
|
// this is the first call to the function Open()
|
|
|
|
column_family_names_ = existing_column_families;
|
|
|
|
} else {
|
|
|
|
// this is a reopen. just assert that existing column_family_names are
|
|
|
|
// equivalent to what we remember
|
|
|
|
auto sorted_cfn = column_family_names_;
|
|
|
|
std::sort(sorted_cfn.begin(), sorted_cfn.end());
|
|
|
|
std::sort(existing_column_families.begin(),
|
|
|
|
existing_column_families.end());
|
|
|
|
if (sorted_cfn != existing_column_families) {
|
|
|
|
fprintf(stderr, "Expected column families differ from the existing:\n");
|
|
|
|
fprintf(stderr, "Expected: {");
|
|
|
|
for (auto cf : sorted_cfn) {
|
|
|
|
fprintf(stderr, "%s ", cf.c_str());
|
|
|
|
}
|
|
|
|
fprintf(stderr, "}\n");
|
|
|
|
fprintf(stderr, "Existing: {");
|
|
|
|
for (auto cf : existing_column_families) {
|
|
|
|
fprintf(stderr, "%s ", cf.c_str());
|
|
|
|
}
|
|
|
|
fprintf(stderr, "}\n");
|
|
|
|
}
|
|
|
|
assert(sorted_cfn == existing_column_families);
|
|
|
|
}
|
|
|
|
std::vector<ColumnFamilyDescriptor> cf_descriptors;
|
|
|
|
for (auto name : column_family_names_) {
|
|
|
|
if (name != kDefaultColumnFamilyName) {
|
|
|
|
new_column_family_name_ =
|
|
|
|
std::max(new_column_family_name_.load(), std::stoi(name) + 1);
|
|
|
|
}
|
|
|
|
cf_descriptors.emplace_back(name, ColumnFamilyOptions(options_));
|
|
|
|
}
|
|
|
|
while (cf_descriptors.size() < (size_t)FLAGS_column_families) {
|
|
|
|
std::string name = std::to_string(new_column_family_name_.load());
|
|
|
|
new_column_family_name_++;
|
|
|
|
cf_descriptors.emplace_back(name, ColumnFamilyOptions(options_));
|
|
|
|
column_family_names_.push_back(name);
|
|
|
|
}
|
Avoid overwriting options loaded from OPTIONS (#9943)
Summary:
This is similar to https://github.com/facebook/rocksdb/issues/9862, including the following fixes/refactoring:
1. If OPTIONS file is specified via `-options_file`, majority of options will be loaded from the file. We should not
overwrite options that have been loaded from the file. Instead, we configure only fields of options which are
shared objects and not set by the OPTIONS file. We also configure a few fields, e.g. `create_if_missing` necessary
for stress test to run.
2. Refactor options initialization into three functions, `InitializeOptionsFromFile()`, `InitializeOptionsFromFlags()`
and `InitializeOptionsGeneral()` similar to db_bench. I hope they can be shared in the future. The high-level logic is
as follows:
```cpp
if (!InitializeOptionsFromFile(...)) {
InitializeOptionsFromFlags(...);
}
InitializeOptionsGeneral(...);
```
3. Currently, the setting for `block_cache_compressed` does not seem correct because it by default specifies a
size of `numeric_limits<size_t>::max()` ((size_t)-1). According to code comments, `-1` indicates default value,
which should be referring to `num_shard_bits` argument.
4. Clarify `fail_if_options_file_error`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9943
Test Plan:
1. make check
2. Run stress tests, and manually check generated OPTIONS file and compare them with input OPTIONS files
Reviewed By: jay-zhuang
Differential Revision: D36133769
Pulled By: riversand963
fbshipit-source-id: 35dacdc090a0a72c922907170cd132b9ecaa073e
3 years ago
|
|
|
|
|
|
|
options_.listeners.clear();
|
Experimental support for SST unique IDs (#8990)
Summary:
* New public header unique_id.h and function GetUniqueIdFromTableProperties
which computes a universally unique identifier based on table properties
of table files from recent RocksDB versions.
* Generation of DB session IDs is refactored so that they are
guaranteed unique in the lifetime of a process running RocksDB.
(SemiStructuredUniqueIdGen, new test included.) Along with file numbers,
this enables SST unique IDs to be guaranteed unique among SSTs generated
in a single process, and "better than random" between processes.
See https://github.com/pdillinger/unique_id
* In addition to public API producing 'external' unique IDs, there is a function
for producing 'internal' unique IDs, with functions for converting between the
two. In short, the external ID is "safe" for things people might do with it, and
the internal ID enables more "power user" features for the future. Specifically,
the external ID goes through a hashing layer so that any subset of bits in the
external ID can be used as a hash of the full ID, while also preserving
uniqueness guarantees in the first 128 bits (bijective both on first 128 bits
and on full 192 bits).
Intended follow-up:
* Use the internal unique IDs in cache keys. (Avoid conflicts with https://github.com/facebook/rocksdb/issues/8912) (The file offset can be XORed into
the third 64-bit value of the unique ID.)
* Publish the external unique IDs in FileStorageInfo (https://github.com/facebook/rocksdb/issues/8968)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8990
Test Plan:
Unit tests added, and checking of unique ids in stress test.
NOTE in stress test we do not generate nearly enough files to thoroughly
stress uniqueness, but the test trims off pieces of the ID to check for
uniqueness so that we can infer (with some assumptions) stronger
properties in the aggregate.
Reviewed By: zhichao-cao, mrambacher
Differential Revision: D31582865
Pulled By: pdillinger
fbshipit-source-id: 1f620c4c86af9abe2a8d177b9ccf2ad2b9f48243
3 years ago
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
options_.listeners.emplace_back(new DbStressListener(
|
|
|
|
FLAGS_db, options_.db_paths, cf_descriptors, db_stress_listener_env));
|
Experimental support for SST unique IDs (#8990)
Summary:
* New public header unique_id.h and function GetUniqueIdFromTableProperties
which computes a universally unique identifier based on table properties
of table files from recent RocksDB versions.
* Generation of DB session IDs is refactored so that they are
guaranteed unique in the lifetime of a process running RocksDB.
(SemiStructuredUniqueIdGen, new test included.) Along with file numbers,
this enables SST unique IDs to be guaranteed unique among SSTs generated
in a single process, and "better than random" between processes.
See https://github.com/pdillinger/unique_id
* In addition to public API producing 'external' unique IDs, there is a function
for producing 'internal' unique IDs, with functions for converting between the
two. In short, the external ID is "safe" for things people might do with it, and
the internal ID enables more "power user" features for the future. Specifically,
the external ID goes through a hashing layer so that any subset of bits in the
external ID can be used as a hash of the full ID, while also preserving
uniqueness guarantees in the first 128 bits (bijective both on first 128 bits
and on full 192 bits).
Intended follow-up:
* Use the internal unique IDs in cache keys. (Avoid conflicts with https://github.com/facebook/rocksdb/issues/8912) (The file offset can be XORed into
the third 64-bit value of the unique ID.)
* Publish the external unique IDs in FileStorageInfo (https://github.com/facebook/rocksdb/issues/8968)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8990
Test Plan:
Unit tests added, and checking of unique ids in stress test.
NOTE in stress test we do not generate nearly enough files to thoroughly
stress uniqueness, but the test trims off pieces of the ID to check for
uniqueness so that we can infer (with some assumptions) stronger
properties in the aggregate.
Reviewed By: zhichao-cao, mrambacher
Differential Revision: D31582865
Pulled By: pdillinger
fbshipit-source-id: 1f620c4c86af9abe2a8d177b9ccf2ad2b9f48243
3 years ago
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
RegisterAdditionalListeners();
|
Avoid overwriting options loaded from OPTIONS (#9943)
Summary:
This is similar to https://github.com/facebook/rocksdb/issues/9862, including the following fixes/refactoring:
1. If OPTIONS file is specified via `-options_file`, majority of options will be loaded from the file. We should not
overwrite options that have been loaded from the file. Instead, we configure only fields of options which are
shared objects and not set by the OPTIONS file. We also configure a few fields, e.g. `create_if_missing` necessary
for stress test to run.
2. Refactor options initialization into three functions, `InitializeOptionsFromFile()`, `InitializeOptionsFromFlags()`
and `InitializeOptionsGeneral()` similar to db_bench. I hope they can be shared in the future. The high-level logic is
as follows:
```cpp
if (!InitializeOptionsFromFile(...)) {
InitializeOptionsFromFlags(...);
}
InitializeOptionsGeneral(...);
```
3. Currently, the setting for `block_cache_compressed` does not seem correct because it by default specifies a
size of `numeric_limits<size_t>::max()` ((size_t)-1). According to code comments, `-1` indicates default value,
which should be referring to `num_shard_bits` argument.
4. Clarify `fail_if_options_file_error`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9943
Test Plan:
1. make check
2. Run stress tests, and manually check generated OPTIONS file and compare them with input OPTIONS files
Reviewed By: jay-zhuang
Differential Revision: D36133769
Pulled By: riversand963
fbshipit-source-id: 35dacdc090a0a72c922907170cd132b9ecaa073e
3 years ago
|
|
|
|
|
|
|
if (!FLAGS_use_txn) {
|
|
|
|
// Determine whether we need to ingest file metadata write failures
|
|
|
|
// during DB reopen. If it does, enable it.
|
|
|
|
// Only ingest metadata error if it is reopening, as initial open
|
|
|
|
// failure doesn't need to be handled.
|
|
|
|
// TODO cover transaction DB is not covered in this fault test too.
|
|
|
|
bool ingest_meta_error = false;
|
|
|
|
bool ingest_write_error = false;
|
|
|
|
bool ingest_read_error = false;
|
|
|
|
if ((FLAGS_open_metadata_write_fault_one_in ||
|
|
|
|
FLAGS_open_write_fault_one_in || FLAGS_open_read_fault_one_in) &&
|
|
|
|
fault_fs_guard
|
|
|
|
->FileExists(FLAGS_db + "/CURRENT", IOOptions(), nullptr)
|
|
|
|
.ok()) {
|
|
|
|
if (!FLAGS_sync) {
|
|
|
|
// When DB Stress is not sync mode, we expect all WAL writes to
|
|
|
|
// WAL is durable. Buffering unsynced writes will cause false
|
|
|
|
// positive in crash tests. Before we figure out a way to
|
|
|
|
// solve it, skip WAL from failure injection.
|
|
|
|
fault_fs_guard->SetSkipDirectWritableTypes({kWalFile});
|
|
|
|
}
|
|
|
|
ingest_meta_error = FLAGS_open_metadata_write_fault_one_in;
|
|
|
|
ingest_write_error = FLAGS_open_write_fault_one_in;
|
|
|
|
ingest_read_error = FLAGS_open_read_fault_one_in;
|
|
|
|
if (ingest_meta_error) {
|
|
|
|
fault_fs_guard->EnableMetadataWriteErrorInjection();
|
|
|
|
fault_fs_guard->SetRandomMetadataWriteError(
|
|
|
|
FLAGS_open_metadata_write_fault_one_in);
|
|
|
|
}
|
|
|
|
if (ingest_write_error) {
|
|
|
|
fault_fs_guard->SetFilesystemDirectWritable(false);
|
|
|
|
fault_fs_guard->EnableWriteErrorInjection();
|
|
|
|
fault_fs_guard->SetRandomWriteError(
|
|
|
|
static_cast<uint32_t>(FLAGS_seed), FLAGS_open_write_fault_one_in,
|
|
|
|
IOStatus::IOError("Injected Open Error"),
|
|
|
|
/*inject_for_all_file_types=*/true, /*types=*/{});
|
|
|
|
}
|
|
|
|
if (ingest_read_error) {
|
|
|
|
fault_fs_guard->SetRandomReadError(FLAGS_open_read_fault_one_in);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
while (true) {
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
// StackableDB-based BlobDB
|
|
|
|
if (FLAGS_use_blob_db) {
|
|
|
|
blob_db::BlobDBOptions blob_db_options;
|
|
|
|
blob_db_options.min_blob_size = FLAGS_blob_db_min_blob_size;
|
|
|
|
blob_db_options.bytes_per_sync = FLAGS_blob_db_bytes_per_sync;
|
|
|
|
blob_db_options.blob_file_size = FLAGS_blob_db_file_size;
|
|
|
|
blob_db_options.enable_garbage_collection = FLAGS_blob_db_enable_gc;
|
|
|
|
blob_db_options.garbage_collection_cutoff = FLAGS_blob_db_gc_cutoff;
|
|
|
|
|
|
|
|
blob_db::BlobDB* blob_db = nullptr;
|
|
|
|
s = blob_db::BlobDB::Open(options_, blob_db_options, FLAGS_db,
|
|
|
|
cf_descriptors, &column_families_,
|
|
|
|
&blob_db);
|
|
|
|
if (s.ok()) {
|
|
|
|
db_ = blob_db;
|
|
|
|
}
|
|
|
|
} else
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
{
|
|
|
|
if (db_preload_finished_.load() && FLAGS_read_only) {
|
|
|
|
s = DB::OpenForReadOnly(DBOptions(options_), FLAGS_db,
|
|
|
|
cf_descriptors, &column_families_, &db_);
|
|
|
|
} else {
|
|
|
|
s = DB::Open(DBOptions(options_), FLAGS_db, cf_descriptors,
|
|
|
|
&column_families_, &db_);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (ingest_meta_error || ingest_write_error || ingest_read_error) {
|
|
|
|
fault_fs_guard->SetFilesystemDirectWritable(true);
|
|
|
|
fault_fs_guard->DisableMetadataWriteErrorInjection();
|
|
|
|
fault_fs_guard->DisableWriteErrorInjection();
|
|
|
|
fault_fs_guard->SetSkipDirectWritableTypes({});
|
|
|
|
fault_fs_guard->SetRandomReadError(0);
|
|
|
|
if (s.ok()) {
|
|
|
|
// Ingested errors might happen in background compactions. We
|
|
|
|
// wait for all compactions to finish to make sure DB is in
|
|
|
|
// clean state before executing queries.
|
|
|
|
s = static_cast_with_check<DBImpl>(db_->GetRootDB())
|
|
|
|
->WaitForCompact(true /* wait_unscheduled */);
|
|
|
|
if (!s.ok()) {
|
|
|
|
for (auto cf : column_families_) {
|
|
|
|
delete cf;
|
|
|
|
}
|
|
|
|
column_families_.clear();
|
|
|
|
delete db_;
|
|
|
|
db_ = nullptr;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!s.ok()) {
|
|
|
|
// After failure to opening a DB due to IO error, retry should
|
|
|
|
// successfully open the DB with correct data if no IO error shows
|
|
|
|
// up.
|
|
|
|
ingest_meta_error = false;
|
|
|
|
ingest_write_error = false;
|
|
|
|
ingest_read_error = false;
|
|
|
|
|
|
|
|
Random rand(static_cast<uint32_t>(FLAGS_seed));
|
|
|
|
if (rand.OneIn(2)) {
|
|
|
|
fault_fs_guard->DeleteFilesCreatedAfterLastDirSync(IOOptions(),
|
|
|
|
nullptr);
|
|
|
|
}
|
|
|
|
if (rand.OneIn(3)) {
|
|
|
|
fault_fs_guard->DropUnsyncedFileData();
|
|
|
|
} else if (rand.OneIn(2)) {
|
|
|
|
fault_fs_guard->DropRandomUnsyncedFileData(&rand);
|
|
|
|
}
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
TransactionDBOptions txn_db_options;
|
|
|
|
assert(FLAGS_txn_write_policy <= TxnDBWritePolicy::WRITE_UNPREPARED);
|
|
|
|
txn_db_options.write_policy =
|
|
|
|
static_cast<TxnDBWritePolicy>(FLAGS_txn_write_policy);
|
|
|
|
if (FLAGS_unordered_write) {
|
|
|
|
assert(txn_db_options.write_policy == TxnDBWritePolicy::WRITE_PREPARED);
|
|
|
|
options_.unordered_write = true;
|
|
|
|
options_.two_write_queues = true;
|
|
|
|
txn_db_options.skip_concurrency_control = true;
|
|
|
|
} else {
|
|
|
|
options_.two_write_queues = FLAGS_two_write_queues;
|
|
|
|
}
|
|
|
|
txn_db_options.wp_snapshot_cache_bits =
|
|
|
|
static_cast<size_t>(FLAGS_wp_snapshot_cache_bits);
|
|
|
|
txn_db_options.wp_commit_cache_bits =
|
|
|
|
static_cast<size_t>(FLAGS_wp_commit_cache_bits);
|
|
|
|
PrepareTxnDbOptions(shared, txn_db_options);
|
|
|
|
s = TransactionDB::Open(options_, txn_db_options, FLAGS_db,
|
|
|
|
cf_descriptors, &column_families_, &txn_db_);
|
|
|
|
if (!s.ok()) {
|
|
|
|
fprintf(stderr, "Error in opening the TransactionDB [%s]\n",
|
|
|
|
s.ToString().c_str());
|
|
|
|
fflush(stderr);
|
|
|
|
}
|
|
|
|
assert(s.ok());
|
|
|
|
|
|
|
|
// Do not swap the order of the following.
|
|
|
|
{
|
|
|
|
db_ = txn_db_;
|
|
|
|
db_aptr_.store(txn_db_, std::memory_order_release);
|
|
|
|
}
|
|
|
|
|
|
|
|
// after a crash, rollback to commit recovered transactions
|
|
|
|
std::vector<Transaction*> trans;
|
|
|
|
txn_db_->GetAllPreparedTransactions(&trans);
|
|
|
|
Random rand(static_cast<uint32_t>(FLAGS_seed));
|
|
|
|
for (auto txn : trans) {
|
|
|
|
if (rand.OneIn(2)) {
|
|
|
|
s = txn->Commit();
|
|
|
|
assert(s.ok());
|
|
|
|
} else {
|
|
|
|
s = txn->Rollback();
|
|
|
|
assert(s.ok());
|
|
|
|
}
|
|
|
|
delete txn;
|
|
|
|
}
|
|
|
|
trans.clear();
|
|
|
|
txn_db_->GetAllPreparedTransactions(&trans);
|
|
|
|
assert(trans.size() == 0);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
assert(s.ok());
|
|
|
|
assert(column_families_.size() ==
|
|
|
|
static_cast<size_t>(FLAGS_column_families));
|
|
|
|
|
|
|
|
// Secondary instance does not support write-prepared/write-unprepared
|
|
|
|
// transactions, thus just disable secondary instance if we use
|
|
|
|
// transaction.
|
|
|
|
if (s.ok() && FLAGS_test_secondary && !FLAGS_use_txn) {
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
Options tmp_opts;
|
|
|
|
// TODO(yanqin) support max_open_files != -1 for secondary instance.
|
|
|
|
tmp_opts.max_open_files = -1;
|
|
|
|
tmp_opts.env = db_stress_env;
|
|
|
|
const std::string& secondary_path = FLAGS_secondaries_base;
|
|
|
|
s = DB::OpenAsSecondary(tmp_opts, FLAGS_db, secondary_path,
|
|
|
|
cf_descriptors, &cmp_cfhs_, &cmp_db_);
|
|
|
|
assert(s.ok());
|
|
|
|
assert(cmp_cfhs_.size() == static_cast<size_t>(FLAGS_column_families));
|
|
|
|
#else
|
|
|
|
fprintf(stderr, "Secondary is not supported in RocksDBLite\n");
|
|
|
|
exit(1);
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
DBWithTTL* db_with_ttl;
|
|
|
|
s = DBWithTTL::Open(options_, FLAGS_db, &db_with_ttl, FLAGS_ttl);
|
|
|
|
db_ = db_with_ttl;
|
|
|
|
#else
|
|
|
|
fprintf(stderr, "TTL is not supported in RocksDBLite\n");
|
|
|
|
exit(1);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
if (!s.ok()) {
|
|
|
|
fprintf(stderr, "open error: %s\n", s.ToString().c_str());
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void StressTest::Reopen(ThreadState* thread) {
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
// BG jobs in WritePrepared must be canceled first because i) they can access
|
|
|
|
// the db via a callbac ii) they hold on to a snapshot and the upcoming
|
|
|
|
// ::Close would complain about it.
|
|
|
|
const bool write_prepared = FLAGS_use_txn && FLAGS_txn_write_policy != 0;
|
|
|
|
bool bg_canceled __attribute__((unused)) = false;
|
|
|
|
if (write_prepared || thread->rand.OneIn(2)) {
|
|
|
|
const bool wait =
|
|
|
|
write_prepared || static_cast<bool>(thread->rand.OneIn(2));
|
|
|
|
CancelAllBackgroundWork(db_, wait);
|
|
|
|
bg_canceled = wait;
|
|
|
|
}
|
|
|
|
assert(!write_prepared || bg_canceled);
|
|
|
|
#else
|
|
|
|
(void)thread;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
for (auto cf : column_families_) {
|
|
|
|
delete cf;
|
|
|
|
}
|
|
|
|
column_families_.clear();
|
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
if (thread->rand.OneIn(2)) {
|
|
|
|
Status s = db_->Close();
|
|
|
|
if (!s.ok()) {
|
|
|
|
fprintf(stderr, "Non-ok close status: %s\n", s.ToString().c_str());
|
|
|
|
fflush(stderr);
|
|
|
|
}
|
|
|
|
assert(s.ok());
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
delete db_;
|
|
|
|
db_ = nullptr;
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
txn_db_ = nullptr;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
num_times_reopened_++;
|
|
|
|
auto now = clock_->NowMicros();
|
|
|
|
fprintf(stdout, "%s Reopening database for the %dth time\n",
|
|
|
|
clock_->TimeToString(now / 1000000).c_str(), num_times_reopened_);
|
|
|
|
Open(thread->shared);
|
|
|
|
|
|
|
|
if ((FLAGS_sync_fault_injection || FLAGS_disable_wal) && IsStateTracked()) {
|
|
|
|
Status s = thread->shared->SaveAtAndAfter(db_);
|
|
|
|
if (!s.ok()) {
|
|
|
|
fprintf(stderr, "Error enabling history tracing: %s\n",
|
|
|
|
s.ToString().c_str());
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
|
Avoid overwriting options loaded from OPTIONS (#9943)
Summary:
This is similar to https://github.com/facebook/rocksdb/issues/9862, including the following fixes/refactoring:
1. If OPTIONS file is specified via `-options_file`, majority of options will be loaded from the file. We should not
overwrite options that have been loaded from the file. Instead, we configure only fields of options which are
shared objects and not set by the OPTIONS file. We also configure a few fields, e.g. `create_if_missing` necessary
for stress test to run.
2. Refactor options initialization into three functions, `InitializeOptionsFromFile()`, `InitializeOptionsFromFlags()`
and `InitializeOptionsGeneral()` similar to db_bench. I hope they can be shared in the future. The high-level logic is
as follows:
```cpp
if (!InitializeOptionsFromFile(...)) {
InitializeOptionsFromFlags(...);
}
InitializeOptionsGeneral(...);
```
3. Currently, the setting for `block_cache_compressed` does not seem correct because it by default specifies a
size of `numeric_limits<size_t>::max()` ((size_t)-1). According to code comments, `-1` indicates default value,
which should be referring to `num_shard_bits` argument.
4. Clarify `fail_if_options_file_error`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9943
Test Plan:
1. make check
2. Run stress tests, and manually check generated OPTIONS file and compare them with input OPTIONS files
Reviewed By: jay-zhuang
Differential Revision: D36133769
Pulled By: riversand963
fbshipit-source-id: 35dacdc090a0a72c922907170cd132b9ecaa073e
3 years ago
|
|
|
void CheckAndSetOptionsForUserTimestamp(Options& options) {
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
assert(FLAGS_user_timestamp_size > 0);
|
|
|
|
const Comparator* const cmp = test::BytewiseComparatorWithU64TsWrapper();
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
assert(cmp);
|
|
|
|
if (FLAGS_user_timestamp_size != cmp->timestamp_size()) {
|
|
|
|
fprintf(stderr,
|
|
|
|
"Only -user_timestamp_size=%d is supported in stress test.\n",
|
|
|
|
static_cast<int>(cmp->timestamp_size()));
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
if (FLAGS_use_merge || FLAGS_use_full_merge_v1) {
|
|
|
|
fprintf(stderr, "Merge does not support timestamp yet.\n");
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
if (FLAGS_delrangepercent > 0) {
|
|
|
|
fprintf(stderr, "DeleteRange does not support timestamp yet.\n");
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
if (FLAGS_use_txn) {
|
|
|
|
fprintf(stderr, "TransactionDB does not support timestamp yet.\n");
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
if (FLAGS_enable_blob_files || FLAGS_use_blob_db) {
|
|
|
|
fprintf(stderr, "BlobDB not supported with timestamp.\n");
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
if (FLAGS_enable_compaction_filter) {
|
|
|
|
fprintf(stderr, "CompactionFilter not supported with timestamp.\n");
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
if (FLAGS_test_cf_consistency || FLAGS_test_batches_snapshots) {
|
|
|
|
fprintf(stderr,
|
|
|
|
"Due to per-key ts-seq ordering constraint, only the (default) "
|
|
|
|
"non-batched test is supported with timestamp.\n");
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
if (FLAGS_ingest_external_file_one_in > 0) {
|
|
|
|
fprintf(stderr, "Bulk loading may not support timestamp yet.\n");
|
|
|
|
exit(1);
|
|
|
|
}
|
Avoid overwriting options loaded from OPTIONS (#9943)
Summary:
This is similar to https://github.com/facebook/rocksdb/issues/9862, including the following fixes/refactoring:
1. If OPTIONS file is specified via `-options_file`, majority of options will be loaded from the file. We should not
overwrite options that have been loaded from the file. Instead, we configure only fields of options which are
shared objects and not set by the OPTIONS file. We also configure a few fields, e.g. `create_if_missing` necessary
for stress test to run.
2. Refactor options initialization into three functions, `InitializeOptionsFromFile()`, `InitializeOptionsFromFlags()`
and `InitializeOptionsGeneral()` similar to db_bench. I hope they can be shared in the future. The high-level logic is
as follows:
```cpp
if (!InitializeOptionsFromFile(...)) {
InitializeOptionsFromFlags(...);
}
InitializeOptionsGeneral(...);
```
3. Currently, the setting for `block_cache_compressed` does not seem correct because it by default specifies a
size of `numeric_limits<size_t>::max()` ((size_t)-1). According to code comments, `-1` indicates default value,
which should be referring to `num_shard_bits` argument.
4. Clarify `fail_if_options_file_error`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9943
Test Plan:
1. make check
2. Run stress tests, and manually check generated OPTIONS file and compare them with input OPTIONS files
Reviewed By: jay-zhuang
Differential Revision: D36133769
Pulled By: riversand963
fbshipit-source-id: 35dacdc090a0a72c922907170cd132b9ecaa073e
3 years ago
|
|
|
options.comparator = cmp;
|
Add user-defined timestamps to db_stress (#8061)
Summary:
Add some basic test for user-defined timestamp to db_stress. Currently,
read with timestamp always tries to read using the current timestamp.
Due to the per-key timestamp-sequence ordering constraint, we only add timestamp-
related tests to the `NonBatchedOpsStressTest` since this test serializes accesses
to the same key and uses a file to cross-check data correctness.
The timestamp feature is not supported in a number of components, e.g. Merge, SingleDelete,
DeleteRange, CompactionFilter, Readonly instance, secondary instance, SST file ingestion, transaction,
etc. Therefore, db_stress should exit if user enables both timestamp and these features at the same
time. The (currently) incompatible features can be found in
`CheckAndSetOptionsForUserTimestamp`.
This PR also fixes a bug triggered when timestamp is enabled together with
`index_type=kBinarySearchWithFirstKey`. This bug fix will also be in another separate PR
with more unit tests coverage. Fixing it here because I do not want to exclude the index type
from crash test.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8061
Test Plan: make crash_test_with_ts
Reviewed By: jay-zhuang
Differential Revision: D27056282
Pulled By: riversand963
fbshipit-source-id: c3e00ad1023fdb9ebbdf9601ec18270c5e2925a9
4 years ago
|
|
|
}
|
Avoid overwriting options loaded from OPTIONS (#9943)
Summary:
This is similar to https://github.com/facebook/rocksdb/issues/9862, including the following fixes/refactoring:
1. If OPTIONS file is specified via `-options_file`, majority of options will be loaded from the file. We should not
overwrite options that have been loaded from the file. Instead, we configure only fields of options which are
shared objects and not set by the OPTIONS file. We also configure a few fields, e.g. `create_if_missing` necessary
for stress test to run.
2. Refactor options initialization into three functions, `InitializeOptionsFromFile()`, `InitializeOptionsFromFlags()`
and `InitializeOptionsGeneral()` similar to db_bench. I hope they can be shared in the future. The high-level logic is
as follows:
```cpp
if (!InitializeOptionsFromFile(...)) {
InitializeOptionsFromFlags(...);
}
InitializeOptionsGeneral(...);
```
3. Currently, the setting for `block_cache_compressed` does not seem correct because it by default specifies a
size of `numeric_limits<size_t>::max()` ((size_t)-1). According to code comments, `-1` indicates default value,
which should be referring to `num_shard_bits` argument.
4. Clarify `fail_if_options_file_error`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9943
Test Plan:
1. make check
2. Run stress tests, and manually check generated OPTIONS file and compare them with input OPTIONS files
Reviewed By: jay-zhuang
Differential Revision: D36133769
Pulled By: riversand963
fbshipit-source-id: 35dacdc090a0a72c922907170cd132b9ecaa073e
3 years ago
|
|
|
|
|
|
|
bool InitializeOptionsFromFile(Options& options) {
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
DBOptions db_options;
|
|
|
|
std::vector<ColumnFamilyDescriptor> cf_descriptors;
|
|
|
|
if (!FLAGS_options_file.empty()) {
|
|
|
|
Status s = LoadOptionsFromFile(FLAGS_options_file, db_stress_env,
|
|
|
|
&db_options, &cf_descriptors);
|
|
|
|
if (!s.ok()) {
|
|
|
|
fprintf(stderr, "Unable to load options file %s --- %s\n",
|
|
|
|
FLAGS_options_file.c_str(), s.ToString().c_str());
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
db_options.env = new DbStressEnvWrapper(db_stress_env);
|
|
|
|
options = Options(db_options, cf_descriptors[0].options);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
(void)options;
|
|
|
|
fprintf(stderr, "--options_file not supported in lite mode\n");
|
|
|
|
exit(1);
|
|
|
|
#endif //! ROCKSDB_LITE
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
void InitializeOptionsFromFlags(
|
|
|
|
const std::shared_ptr<Cache>& cache,
|
|
|
|
const std::shared_ptr<Cache>& block_cache_compressed,
|
|
|
|
const std::shared_ptr<const FilterPolicy>& filter_policy,
|
|
|
|
Options& options) {
|
|
|
|
BlockBasedTableOptions block_based_options;
|
|
|
|
block_based_options.block_cache = cache;
|
|
|
|
block_based_options.cache_index_and_filter_blocks =
|
|
|
|
FLAGS_cache_index_and_filter_blocks;
|
|
|
|
block_based_options.metadata_cache_options.top_level_index_pinning =
|
|
|
|
static_cast<PinningTier>(FLAGS_top_level_index_pinning);
|
|
|
|
block_based_options.metadata_cache_options.partition_pinning =
|
|
|
|
static_cast<PinningTier>(FLAGS_partition_pinning);
|
|
|
|
block_based_options.metadata_cache_options.unpartitioned_pinning =
|
|
|
|
static_cast<PinningTier>(FLAGS_unpartitioned_pinning);
|
|
|
|
block_based_options.block_cache_compressed = block_cache_compressed;
|
|
|
|
block_based_options.checksum = checksum_type_e;
|
|
|
|
block_based_options.block_size = FLAGS_block_size;
|
|
|
|
block_based_options.cache_usage_options.options_overrides.insert(
|
|
|
|
{CacheEntryRole::kCompressionDictionaryBuildingBuffer,
|
|
|
|
{/*.charged = */ FLAGS_charge_compression_dictionary_building_buffer
|
|
|
|
? CacheEntryRoleOptions::Decision::kEnabled
|
|
|
|
: CacheEntryRoleOptions::Decision::kDisabled}});
|
|
|
|
block_based_options.cache_usage_options.options_overrides.insert(
|
|
|
|
{CacheEntryRole::kFilterConstruction,
|
|
|
|
{/*.charged = */ FLAGS_charge_filter_construction
|
|
|
|
? CacheEntryRoleOptions::Decision::kEnabled
|
|
|
|
: CacheEntryRoleOptions::Decision::kDisabled}});
|
|
|
|
block_based_options.cache_usage_options.options_overrides.insert(
|
|
|
|
{CacheEntryRole::kBlockBasedTableReader,
|
|
|
|
{/*.charged = */ FLAGS_charge_table_reader
|
|
|
|
? CacheEntryRoleOptions::Decision::kEnabled
|
|
|
|
: CacheEntryRoleOptions::Decision::kDisabled}});
|
Account memory of FileMetaData in global memory limit (#9924)
Summary:
**Context/Summary:**
As revealed by heap profiling, allocation of `FileMetaData` for [newly created file added to a Version](https://github.com/facebook/rocksdb/pull/9924/files#diff-a6aa385940793f95a2c5b39cc670bd440c4547fa54fd44622f756382d5e47e43R774) can consume significant heap memory. This PR is to account that toward our global memory limit based on block cache capacity.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9924
Test Plan:
- Previous `make check` verified there are only 2 places where the memory of the allocated `FileMetaData` can be released
- New unit test `TEST_P(ChargeFileMetadataTestWithParam, Basic)`
- db bench (CPU cost of `charge_file_metadata` in write and compact)
- **write micros/op: -0.24%** : `TEST_TMPDIR=/dev/shm/testdb ./db_bench -benchmarks=fillseq -db=$TEST_TMPDIR -charge_file_metadata=1 (remove this option for pre-PR) -disable_auto_compactions=1 -write_buffer_size=100000 -num=4000000 | egrep 'fillseq'`
- **compact micros/op -0.87%** : `TEST_TMPDIR=/dev/shm/testdb ./db_bench -benchmarks=fillseq -db=$TEST_TMPDIR -charge_file_metadata=1 -disable_auto_compactions=1 -write_buffer_size=100000 -num=4000000 -numdistinct=1000 && ./db_bench -benchmarks=compact -db=$TEST_TMPDIR -use_existing_db=1 -charge_file_metadata=1 -disable_auto_compactions=1 | egrep 'compact'`
table 1 - write
#-run | (pre-PR) avg micros/op | std micros/op | (post-PR) micros/op | std micros/op | change (%)
-- | -- | -- | -- | -- | --
10 | 3.9711 | 0.264408 | 3.9914 | 0.254563 | 0.5111933721
20 | 3.83905 | 0.0664488 | 3.8251 | 0.0695456 | -0.3633711465
40 | 3.86625 | 0.136669 | 3.8867 | 0.143765 | 0.5289363078
80 | 3.87828 | 0.119007 | 3.86791 | 0.115674 | **-0.2673865734**
160 | 3.87677 | 0.162231 | 3.86739 | 0.16663 | **-0.2419539978**
table 2 - compact
#-run | (pre-PR) avg micros/op | std micros/op | (post-PR) micros/op | std micros/op | change (%)
-- | -- | -- | -- | -- | --
10 | 2,399,650.00 | 96,375.80 | 2,359,537.00 | 53,243.60 | -1.67
20 | 2,410,480.00 | 89,988.00 | 2,433,580.00 | 91,121.20 | 0.96
40 | 2.41E+06 | 121811 | 2.39E+06 | 131525 | **-0.96**
80 | 2.40E+06 | 134503 | 2.39E+06 | 108799 | **-0.78**
- stress test: `python3 tools/db_crashtest.py blackbox --charge_file_metadata=1 --cache_size=1` killed as normal
Reviewed By: ajkr
Differential Revision: D36055583
Pulled By: hx235
fbshipit-source-id: b60eab94707103cb1322cf815f05810ef0232625
3 years ago
|
|
|
block_based_options.cache_usage_options.options_overrides.insert(
|
|
|
|
{CacheEntryRole::kFileMetadata,
|
|
|
|
{/*.charged = */ FLAGS_charge_file_metadata
|
|
|
|
? CacheEntryRoleOptions::Decision::kEnabled
|
|
|
|
: CacheEntryRoleOptions::Decision::kDisabled}});
|
Avoid overwriting options loaded from OPTIONS (#9943)
Summary:
This is similar to https://github.com/facebook/rocksdb/issues/9862, including the following fixes/refactoring:
1. If OPTIONS file is specified via `-options_file`, majority of options will be loaded from the file. We should not
overwrite options that have been loaded from the file. Instead, we configure only fields of options which are
shared objects and not set by the OPTIONS file. We also configure a few fields, e.g. `create_if_missing` necessary
for stress test to run.
2. Refactor options initialization into three functions, `InitializeOptionsFromFile()`, `InitializeOptionsFromFlags()`
and `InitializeOptionsGeneral()` similar to db_bench. I hope they can be shared in the future. The high-level logic is
as follows:
```cpp
if (!InitializeOptionsFromFile(...)) {
InitializeOptionsFromFlags(...);
}
InitializeOptionsGeneral(...);
```
3. Currently, the setting for `block_cache_compressed` does not seem correct because it by default specifies a
size of `numeric_limits<size_t>::max()` ((size_t)-1). According to code comments, `-1` indicates default value,
which should be referring to `num_shard_bits` argument.
4. Clarify `fail_if_options_file_error`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9943
Test Plan:
1. make check
2. Run stress tests, and manually check generated OPTIONS file and compare them with input OPTIONS files
Reviewed By: jay-zhuang
Differential Revision: D36133769
Pulled By: riversand963
fbshipit-source-id: 35dacdc090a0a72c922907170cd132b9ecaa073e
3 years ago
|
|
|
block_based_options.format_version =
|
|
|
|
static_cast<uint32_t>(FLAGS_format_version);
|
|
|
|
block_based_options.index_block_restart_interval =
|
|
|
|
static_cast<int32_t>(FLAGS_index_block_restart_interval);
|
|
|
|
block_based_options.filter_policy = filter_policy;
|
|
|
|
block_based_options.partition_filters = FLAGS_partition_filters;
|
|
|
|
block_based_options.optimize_filters_for_memory =
|
|
|
|
FLAGS_optimize_filters_for_memory;
|
|
|
|
block_based_options.detect_filter_construct_corruption =
|
|
|
|
FLAGS_detect_filter_construct_corruption;
|
|
|
|
block_based_options.index_type =
|
|
|
|
static_cast<BlockBasedTableOptions::IndexType>(FLAGS_index_type);
|
|
|
|
block_based_options.prepopulate_block_cache =
|
|
|
|
static_cast<BlockBasedTableOptions::PrepopulateBlockCache>(
|
|
|
|
FLAGS_prepopulate_block_cache);
|
|
|
|
options.table_factory.reset(NewBlockBasedTableFactory(block_based_options));
|
|
|
|
options.db_write_buffer_size = FLAGS_db_write_buffer_size;
|
|
|
|
options.write_buffer_size = FLAGS_write_buffer_size;
|
|
|
|
options.max_write_buffer_number = FLAGS_max_write_buffer_number;
|
|
|
|
options.min_write_buffer_number_to_merge =
|
|
|
|
FLAGS_min_write_buffer_number_to_merge;
|
|
|
|
options.max_write_buffer_number_to_maintain =
|
|
|
|
FLAGS_max_write_buffer_number_to_maintain;
|
|
|
|
options.max_write_buffer_size_to_maintain =
|
|
|
|
FLAGS_max_write_buffer_size_to_maintain;
|
|
|
|
options.memtable_prefix_bloom_size_ratio =
|
|
|
|
FLAGS_memtable_prefix_bloom_size_ratio;
|
|
|
|
options.memtable_whole_key_filtering = FLAGS_memtable_whole_key_filtering;
|
|
|
|
options.disable_auto_compactions = FLAGS_disable_auto_compactions;
|
|
|
|
options.max_background_compactions = FLAGS_max_background_compactions;
|
|
|
|
options.max_background_flushes = FLAGS_max_background_flushes;
|
|
|
|
options.compaction_style =
|
|
|
|
static_cast<ROCKSDB_NAMESPACE::CompactionStyle>(FLAGS_compaction_style);
|
|
|
|
if (FLAGS_prefix_size >= 0) {
|
|
|
|
options.prefix_extractor.reset(NewFixedPrefixTransform(FLAGS_prefix_size));
|
|
|
|
}
|
|
|
|
options.max_open_files = FLAGS_open_files;
|
|
|
|
options.statistics = dbstats;
|
|
|
|
options.env = db_stress_env;
|
|
|
|
options.use_fsync = FLAGS_use_fsync;
|
|
|
|
options.compaction_readahead_size = FLAGS_compaction_readahead_size;
|
|
|
|
options.allow_mmap_reads = FLAGS_mmap_read;
|
|
|
|
options.allow_mmap_writes = FLAGS_mmap_write;
|
|
|
|
options.use_direct_reads = FLAGS_use_direct_reads;
|
|
|
|
options.use_direct_io_for_flush_and_compaction =
|
|
|
|
FLAGS_use_direct_io_for_flush_and_compaction;
|
|
|
|
options.recycle_log_file_num =
|
|
|
|
static_cast<size_t>(FLAGS_recycle_log_file_num);
|
|
|
|
options.target_file_size_base = FLAGS_target_file_size_base;
|
|
|
|
options.target_file_size_multiplier = FLAGS_target_file_size_multiplier;
|
|
|
|
options.max_bytes_for_level_base = FLAGS_max_bytes_for_level_base;
|
|
|
|
options.max_bytes_for_level_multiplier = FLAGS_max_bytes_for_level_multiplier;
|
|
|
|
options.level0_stop_writes_trigger = FLAGS_level0_stop_writes_trigger;
|
|
|
|
options.level0_slowdown_writes_trigger = FLAGS_level0_slowdown_writes_trigger;
|
|
|
|
options.level0_file_num_compaction_trigger =
|
|
|
|
FLAGS_level0_file_num_compaction_trigger;
|
|
|
|
options.compression = compression_type_e;
|
|
|
|
options.bottommost_compression = bottommost_compression_type_e;
|
|
|
|
options.compression_opts.max_dict_bytes = FLAGS_compression_max_dict_bytes;
|
|
|
|
options.compression_opts.zstd_max_train_bytes =
|
|
|
|
FLAGS_compression_zstd_max_train_bytes;
|
|
|
|
options.compression_opts.parallel_threads =
|
|
|
|
FLAGS_compression_parallel_threads;
|
|
|
|
options.compression_opts.max_dict_buffer_bytes =
|
|
|
|
FLAGS_compression_max_dict_buffer_bytes;
|
Support using ZDICT_finalizeDictionary to generate zstd dictionary (#9857)
Summary:
An untrained dictionary is currently simply the concatenation of several samples. The ZSTD API, ZDICT_finalizeDictionary(), can improve such a dictionary's effectiveness at low cost. This PR changes how dictionary is created by calling the ZSTD ZDICT_finalizeDictionary() API instead of creating raw content dictionary (when max_dict_buffer_bytes > 0), and pass in all buffered uncompressed data blocks as samples.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9857
Test Plan:
#### db_bench test for cpu/memory of compression+decompression and space saving on synthetic data:
Set up: change the parameter [here](https://github.com/facebook/rocksdb/blob/fb9a167a55e0970b1ef6f67c1600c8d9c4c6114f/tools/db_bench_tool.cc#L1766) to 16384 to make synthetic data more compressible.
```
# linked local ZSTD with version 1.5.2
# DEBUG_LEVEL=0 ROCKSDB_NO_FBCODE=1 ROCKSDB_DISABLE_ZSTD=1 EXTRA_CXXFLAGS="-DZSTD_STATIC_LINKING_ONLY -DZSTD -I/data/users/changyubi/install/include/" EXTRA_LDFLAGS="-L/data/users/changyubi/install/lib/ -l:libzstd.a" make -j32 db_bench
dict_bytes=16384
train_bytes=1048576
echo "========== No Dictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=0 -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=0 -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total
echo "========== Raw Content Dictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench_main -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench_main -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total
echo "========== FinalizeDictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total
echo "========== TrainDictionary =========="
TEST_TMPDIR=/dev/shm ./db_bench -benchmarks=filluniquerandom,compact -num=10000000 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -block_size=4096 -max_background_jobs=24 -memtablerep=vector -allow_concurrent_memtable_write=false -disable_wal=true -max_write_buffer_number=8 >/dev/null 2>&1
TEST_TMPDIR=/dev/shm /usr/bin/time ./db_bench -use_existing_db=true -benchmarks=compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -block_size=4096 2>&1 | grep elapsed
du -hc /dev/shm/dbbench/*sst | grep total
# Result: TrainDictionary is much better on space saving, but FinalizeDictionary seems to use less memory.
# before compression data size: 1.2GB
dict_bytes=16384
max_dict_buffer_bytes = 1048576
space cpu/memory
No Dictionary 468M 14.93user 1.00system 0:15.92elapsed 100%CPU (0avgtext+0avgdata 23904maxresident)k
Raw Dictionary 251M 15.81user 0.80system 0:16.56elapsed 100%CPU (0avgtext+0avgdata 156808maxresident)k
FinalizeDictionary 236M 11.93user 0.64system 0:12.56elapsed 100%CPU (0avgtext+0avgdata 89548maxresident)k
TrainDictionary 84M 7.29user 0.45system 0:07.75elapsed 100%CPU (0avgtext+0avgdata 97288maxresident)k
```
#### Benchmark on 10 sample SST files for spacing saving and CPU time on compression:
FinalizeDictionary is comparable to TrainDictionary in terms of space saving, and takes less time in compression.
```
dict_bytes=16384
train_bytes=1048576
for sst_file in `ls ../temp/myrock-sst/`
do
echo "********** $sst_file **********"
echo "========== No Dictionary =========="
./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD
echo "========== Raw Content Dictionary =========="
./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD --compression_max_dict_bytes=$dict_bytes
echo "========== FinalizeDictionary =========="
./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD --compression_max_dict_bytes=$dict_bytes --compression_zstd_max_train_bytes=$train_bytes --compression_use_zstd_finalize_dict
echo "========== TrainDictionary =========="
./sst_dump --file="../temp/myrock-sst/$sst_file" --command=recompress --compression_level_from=6 --compression_level_to=6 --compression_types=kZSTD --compression_max_dict_bytes=$dict_bytes --compression_zstd_max_train_bytes=$train_bytes
done
010240.sst (Size/Time) 011029.sst 013184.sst 021552.sst 185054.sst 185137.sst 191666.sst 7560381.sst 7604174.sst 7635312.sst
No Dictionary 28165569 / 2614419 32899411 / 2976832 32977848 / 3055542 31966329 / 2004590 33614351 / 1755877 33429029 / 1717042 33611933 / 1776936 33634045 / 2771417 33789721 / 2205414 33592194 / 388254
Raw Content Dictionary 28019950 / 2697961 33748665 / 3572422 33896373 / 3534701 26418431 / 2259658 28560825 / 1839168 28455030 / 1846039 28494319 / 1861349 32391599 / 3095649 33772142 / 2407843 33592230 / 474523
FinalizeDictionary 27896012 / 2650029 33763886 / 3719427 33904283 / 3552793 26008225 / 2198033 28111872 / 1869530 28014374 / 1789771 28047706 / 1848300 32296254 / 3204027 33698698 / 2381468 33592344 / 517433
TrainDictionary 28046089 / 2740037 33706480 / 3679019 33885741 / 3629351 25087123 / 2204558 27194353 / 1970207 27234229 / 1896811 27166710 / 1903119 32011041 / 3322315 32730692 / 2406146 33608631 / 570593
```
#### Decompression/Read test:
With FinalizeDictionary/TrainDictionary, some data structure used for decompression are in stored in dictionary, so they are expected to be faster in terms of decompression/reads.
```
dict_bytes=16384
train_bytes=1048576
echo "No Dictionary"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=0 > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=0 2>&1 | grep MB/s
echo "Raw Dictionary"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes 2>&1 | grep MB/s
echo "FinalizeDict"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes -compression_use_zstd_dict_trainer=false 2>&1 | grep MB/s
echo "Train Dictionary"
TEST_TMPDIR=/dev/shm/ ./db_bench -benchmarks=filluniquerandom,compact -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes > /dev/null 2>&1
TEST_TMPDIR=/dev/shm/ ./db_bench -use_existing_db=true -benchmarks=readrandom -cache_size=0 -compression_type=zstd -compression_max_dict_bytes=$dict_bytes -compression_zstd_max_train_bytes=$train_bytes 2>&1 | grep MB/s
No Dictionary
readrandom : 12.183 micros/op 82082 ops/sec 12.183 seconds 1000000 operations; 9.1 MB/s (1000000 of 1000000 found)
Raw Dictionary
readrandom : 12.314 micros/op 81205 ops/sec 12.314 seconds 1000000 operations; 9.0 MB/s (1000000 of 1000000 found)
FinalizeDict
readrandom : 9.787 micros/op 102180 ops/sec 9.787 seconds 1000000 operations; 11.3 MB/s (1000000 of 1000000 found)
Train Dictionary
readrandom : 9.698 micros/op 103108 ops/sec 9.699 seconds 1000000 operations; 11.4 MB/s (1000000 of 1000000 found)
```
Reviewed By: ajkr
Differential Revision: D35720026
Pulled By: cbi42
fbshipit-source-id: 24d230fdff0fd28a1bb650658798f00dfcfb2a1f
3 years ago
|
|
|
if (ZSTD_FinalizeDictionarySupported()) {
|
|
|
|
options.compression_opts.use_zstd_dict_trainer =
|
|
|
|
FLAGS_compression_use_zstd_dict_trainer;
|
|
|
|
} else if (!FLAGS_compression_use_zstd_dict_trainer) {
|
|
|
|
fprintf(
|
|
|
|
stderr,
|
|
|
|
"WARNING: use_zstd_dict_trainer is false but zstd finalizeDictionary "
|
|
|
|
"cannot be used because ZSTD 1.4.5+ is not linked with the binary."
|
|
|
|
" zstd dictionary trainer will be used.\n");
|
|
|
|
}
|
Avoid overwriting options loaded from OPTIONS (#9943)
Summary:
This is similar to https://github.com/facebook/rocksdb/issues/9862, including the following fixes/refactoring:
1. If OPTIONS file is specified via `-options_file`, majority of options will be loaded from the file. We should not
overwrite options that have been loaded from the file. Instead, we configure only fields of options which are
shared objects and not set by the OPTIONS file. We also configure a few fields, e.g. `create_if_missing` necessary
for stress test to run.
2. Refactor options initialization into three functions, `InitializeOptionsFromFile()`, `InitializeOptionsFromFlags()`
and `InitializeOptionsGeneral()` similar to db_bench. I hope they can be shared in the future. The high-level logic is
as follows:
```cpp
if (!InitializeOptionsFromFile(...)) {
InitializeOptionsFromFlags(...);
}
InitializeOptionsGeneral(...);
```
3. Currently, the setting for `block_cache_compressed` does not seem correct because it by default specifies a
size of `numeric_limits<size_t>::max()` ((size_t)-1). According to code comments, `-1` indicates default value,
which should be referring to `num_shard_bits` argument.
4. Clarify `fail_if_options_file_error`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9943
Test Plan:
1. make check
2. Run stress tests, and manually check generated OPTIONS file and compare them with input OPTIONS files
Reviewed By: jay-zhuang
Differential Revision: D36133769
Pulled By: riversand963
fbshipit-source-id: 35dacdc090a0a72c922907170cd132b9ecaa073e
3 years ago
|
|
|
options.max_manifest_file_size = FLAGS_max_manifest_file_size;
|
|
|
|
options.inplace_update_support = FLAGS_in_place_update;
|
|
|
|
options.max_subcompactions = static_cast<uint32_t>(FLAGS_subcompactions);
|
|
|
|
options.allow_concurrent_memtable_write =
|
|
|
|
FLAGS_allow_concurrent_memtable_write;
|
|
|
|
options.experimental_mempurge_threshold =
|
|
|
|
FLAGS_experimental_mempurge_threshold;
|
|
|
|
options.periodic_compaction_seconds = FLAGS_periodic_compaction_seconds;
|
|
|
|
options.ttl = FLAGS_compaction_ttl;
|
|
|
|
options.enable_pipelined_write = FLAGS_enable_pipelined_write;
|
|
|
|
options.enable_write_thread_adaptive_yield =
|
|
|
|
FLAGS_enable_write_thread_adaptive_yield;
|
|
|
|
options.compaction_options_universal.size_ratio = FLAGS_universal_size_ratio;
|
|
|
|
options.compaction_options_universal.min_merge_width =
|
|
|
|
FLAGS_universal_min_merge_width;
|
|
|
|
options.compaction_options_universal.max_merge_width =
|
|
|
|
FLAGS_universal_max_merge_width;
|
|
|
|
options.compaction_options_universal.max_size_amplification_percent =
|
|
|
|
FLAGS_universal_max_size_amplification_percent;
|
|
|
|
options.atomic_flush = FLAGS_atomic_flush;
|
|
|
|
options.avoid_unnecessary_blocking_io = FLAGS_avoid_unnecessary_blocking_io;
|
|
|
|
options.write_dbid_to_manifest = FLAGS_write_dbid_to_manifest;
|
|
|
|
options.avoid_flush_during_recovery = FLAGS_avoid_flush_during_recovery;
|
|
|
|
options.max_write_batch_group_size_bytes =
|
|
|
|
FLAGS_max_write_batch_group_size_bytes;
|
|
|
|
options.level_compaction_dynamic_level_bytes =
|
|
|
|
FLAGS_level_compaction_dynamic_level_bytes;
|
|
|
|
options.track_and_verify_wals_in_manifest = true;
|
|
|
|
options.verify_sst_unique_id_in_manifest =
|
|
|
|
FLAGS_verify_sst_unique_id_in_manifest;
|
Avoid overwriting options loaded from OPTIONS (#9943)
Summary:
This is similar to https://github.com/facebook/rocksdb/issues/9862, including the following fixes/refactoring:
1. If OPTIONS file is specified via `-options_file`, majority of options will be loaded from the file. We should not
overwrite options that have been loaded from the file. Instead, we configure only fields of options which are
shared objects and not set by the OPTIONS file. We also configure a few fields, e.g. `create_if_missing` necessary
for stress test to run.
2. Refactor options initialization into three functions, `InitializeOptionsFromFile()`, `InitializeOptionsFromFlags()`
and `InitializeOptionsGeneral()` similar to db_bench. I hope they can be shared in the future. The high-level logic is
as follows:
```cpp
if (!InitializeOptionsFromFile(...)) {
InitializeOptionsFromFlags(...);
}
InitializeOptionsGeneral(...);
```
3. Currently, the setting for `block_cache_compressed` does not seem correct because it by default specifies a
size of `numeric_limits<size_t>::max()` ((size_t)-1). According to code comments, `-1` indicates default value,
which should be referring to `num_shard_bits` argument.
4. Clarify `fail_if_options_file_error`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9943
Test Plan:
1. make check
2. Run stress tests, and manually check generated OPTIONS file and compare them with input OPTIONS files
Reviewed By: jay-zhuang
Differential Revision: D36133769
Pulled By: riversand963
fbshipit-source-id: 35dacdc090a0a72c922907170cd132b9ecaa073e
3 years ago
|
|
|
|
|
|
|
// Integrated BlobDB
|
|
|
|
options.enable_blob_files = FLAGS_enable_blob_files;
|
|
|
|
options.min_blob_size = FLAGS_min_blob_size;
|
|
|
|
options.blob_file_size = FLAGS_blob_file_size;
|
|
|
|
options.blob_compression_type =
|
|
|
|
StringToCompressionType(FLAGS_blob_compression_type.c_str());
|
|
|
|
options.enable_blob_garbage_collection = FLAGS_enable_blob_garbage_collection;
|
|
|
|
options.blob_garbage_collection_age_cutoff =
|
|
|
|
FLAGS_blob_garbage_collection_age_cutoff;
|
|
|
|
options.blob_garbage_collection_force_threshold =
|
|
|
|
FLAGS_blob_garbage_collection_force_threshold;
|
|
|
|
options.blob_compaction_readahead_size = FLAGS_blob_compaction_readahead_size;
|
|
|
|
options.blob_file_starting_level = FLAGS_blob_file_starting_level;
|
Avoid overwriting options loaded from OPTIONS (#9943)
Summary:
This is similar to https://github.com/facebook/rocksdb/issues/9862, including the following fixes/refactoring:
1. If OPTIONS file is specified via `-options_file`, majority of options will be loaded from the file. We should not
overwrite options that have been loaded from the file. Instead, we configure only fields of options which are
shared objects and not set by the OPTIONS file. We also configure a few fields, e.g. `create_if_missing` necessary
for stress test to run.
2. Refactor options initialization into three functions, `InitializeOptionsFromFile()`, `InitializeOptionsFromFlags()`
and `InitializeOptionsGeneral()` similar to db_bench. I hope they can be shared in the future. The high-level logic is
as follows:
```cpp
if (!InitializeOptionsFromFile(...)) {
InitializeOptionsFromFlags(...);
}
InitializeOptionsGeneral(...);
```
3. Currently, the setting for `block_cache_compressed` does not seem correct because it by default specifies a
size of `numeric_limits<size_t>::max()` ((size_t)-1). According to code comments, `-1` indicates default value,
which should be referring to `num_shard_bits` argument.
4. Clarify `fail_if_options_file_error`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9943
Test Plan:
1. make check
2. Run stress tests, and manually check generated OPTIONS file and compare them with input OPTIONS files
Reviewed By: jay-zhuang
Differential Revision: D36133769
Pulled By: riversand963
fbshipit-source-id: 35dacdc090a0a72c922907170cd132b9ecaa073e
3 years ago
|
|
|
|
|
|
|
options.wal_compression =
|
|
|
|
StringToCompressionType(FLAGS_wal_compression.c_str());
|
|
|
|
|
|
|
|
switch (FLAGS_rep_factory) {
|
|
|
|
case kSkipList:
|
|
|
|
// no need to do anything
|
|
|
|
break;
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
case kHashSkipList:
|
|
|
|
options.memtable_factory.reset(NewHashSkipListRepFactory(10000));
|
|
|
|
break;
|
|
|
|
case kVectorRep:
|
|
|
|
options.memtable_factory.reset(new VectorRepFactory());
|
|
|
|
break;
|
|
|
|
#else
|
|
|
|
default:
|
|
|
|
fprintf(stderr,
|
|
|
|
"RocksdbLite only supports skip list mem table. Skip "
|
|
|
|
"--rep_factory\n");
|
|
|
|
#endif // ROCKSDB_LITE
|
|
|
|
}
|
|
|
|
|
|
|
|
if (FLAGS_use_full_merge_v1) {
|
|
|
|
options.merge_operator = MergeOperators::CreateDeprecatedPutOperator();
|
|
|
|
} else {
|
|
|
|
options.merge_operator = MergeOperators::CreatePutOperator();
|
|
|
|
}
|
|
|
|
|
|
|
|
if (FLAGS_enable_compaction_filter) {
|
|
|
|
options.compaction_filter_factory =
|
|
|
|
std::make_shared<DbStressCompactionFilterFactory>();
|
|
|
|
}
|
|
|
|
|
|
|
|
options.best_efforts_recovery = FLAGS_best_efforts_recovery;
|
|
|
|
options.paranoid_file_checks = FLAGS_paranoid_file_checks;
|
|
|
|
options.fail_if_options_file_error = FLAGS_fail_if_options_file_error;
|
|
|
|
|
|
|
|
if (FLAGS_user_timestamp_size > 0) {
|
|
|
|
CheckAndSetOptionsForUserTimestamp(options);
|
|
|
|
}
|
|
|
|
|
|
|
|
options.allow_data_in_errors = FLAGS_allow_data_in_errors;
|
Avoid overwriting options loaded from OPTIONS (#9943)
Summary:
This is similar to https://github.com/facebook/rocksdb/issues/9862, including the following fixes/refactoring:
1. If OPTIONS file is specified via `-options_file`, majority of options will be loaded from the file. We should not
overwrite options that have been loaded from the file. Instead, we configure only fields of options which are
shared objects and not set by the OPTIONS file. We also configure a few fields, e.g. `create_if_missing` necessary
for stress test to run.
2. Refactor options initialization into three functions, `InitializeOptionsFromFile()`, `InitializeOptionsFromFlags()`
and `InitializeOptionsGeneral()` similar to db_bench. I hope they can be shared in the future. The high-level logic is
as follows:
```cpp
if (!InitializeOptionsFromFile(...)) {
InitializeOptionsFromFlags(...);
}
InitializeOptionsGeneral(...);
```
3. Currently, the setting for `block_cache_compressed` does not seem correct because it by default specifies a
size of `numeric_limits<size_t>::max()` ((size_t)-1). According to code comments, `-1` indicates default value,
which should be referring to `num_shard_bits` argument.
4. Clarify `fail_if_options_file_error`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/9943
Test Plan:
1. make check
2. Run stress tests, and manually check generated OPTIONS file and compare them with input OPTIONS files
Reviewed By: jay-zhuang
Differential Revision: D36133769
Pulled By: riversand963
fbshipit-source-id: 35dacdc090a0a72c922907170cd132b9ecaa073e
3 years ago
|
|
|
}
|
|
|
|
|
|
|
|
void InitializeOptionsGeneral(
|
|
|
|
const std::shared_ptr<Cache>& cache,
|
|
|
|
const std::shared_ptr<Cache>& block_cache_compressed,
|
|
|
|
const std::shared_ptr<const FilterPolicy>& filter_policy,
|
|
|
|
Options& options) {
|
|
|
|
options.create_missing_column_families = true;
|
|
|
|
options.create_if_missing = true;
|
|
|
|
|
|
|
|
if (!options.statistics) {
|
|
|
|
options.statistics = dbstats;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (options.env == Options().env) {
|
|
|
|
options.env = db_stress_env;
|
|
|
|
}
|
|
|
|
|
|
|
|
assert(options.table_factory);
|
|
|
|
auto table_options =
|
|
|
|
options.table_factory->GetOptions<BlockBasedTableOptions>();
|
|
|
|
if (table_options) {
|
|
|
|
if (FLAGS_cache_size > 0) {
|
|
|
|
table_options->block_cache = cache;
|
|
|
|
}
|
|
|
|
if (!table_options->block_cache_compressed &&
|
|
|
|
FLAGS_compressed_cache_size > 0) {
|
|
|
|
table_options->block_cache_compressed = block_cache_compressed;
|
|
|
|
}
|
|
|
|
if (!table_options->filter_policy) {
|
|
|
|
table_options->filter_policy = filter_policy;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// TODO: row_cache, thread-pool IO priority, CPU priority.
|
|
|
|
|
|
|
|
if (!options.rate_limiter) {
|
|
|
|
if (FLAGS_rate_limiter_bytes_per_sec > 0) {
|
|
|
|
options.rate_limiter.reset(NewGenericRateLimiter(
|
|
|
|
FLAGS_rate_limiter_bytes_per_sec, 1000 /* refill_period_us */,
|
|
|
|
10 /* fairness */,
|
|
|
|
FLAGS_rate_limit_bg_reads ? RateLimiter::Mode::kReadsOnly
|
|
|
|
: RateLimiter::Mode::kWritesOnly));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!options.file_checksum_gen_factory) {
|
|
|
|
options.file_checksum_gen_factory =
|
|
|
|
GetFileChecksumImpl(FLAGS_file_checksum_impl);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (FLAGS_sst_file_manager_bytes_per_sec > 0 ||
|
|
|
|
FLAGS_sst_file_manager_bytes_per_truncate > 0) {
|
|
|
|
Status status;
|
|
|
|
options.sst_file_manager.reset(NewSstFileManager(
|
|
|
|
db_stress_env, options.info_log, "" /* trash_dir */,
|
|
|
|
static_cast<int64_t>(FLAGS_sst_file_manager_bytes_per_sec),
|
|
|
|
true /* delete_existing_trash */, &status,
|
|
|
|
0.25 /* max_trash_db_ratio */,
|
|
|
|
FLAGS_sst_file_manager_bytes_per_truncate));
|
|
|
|
if (!status.ok()) {
|
|
|
|
fprintf(stderr, "SstFileManager creation failed: %s\n",
|
|
|
|
status.ToString().c_str());
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
options.table_properties_collector_factories.emplace_back(
|
|
|
|
std::make_shared<DbStressTablePropertiesCollectorFactory>());
|
|
|
|
}
|
|
|
|
|
|
|
|
} // namespace ROCKSDB_NAMESPACE
|
|
|
|
#endif // GFLAGS
|