From 37fd2b9694052323d5867f1dc56174202023c016 Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Mon, 23 Dec 2019 16:31:08 -0800 Subject: [PATCH] Revert "Generate variable length keys in db_stress (#6165)" and follow-ups (#6243) Summary: This commit is suspected in some crash test failures such as Verification failed for column family 0 key 78438077: Value not found: NotFound: Pull Request resolved: https://github.com/facebook/rocksdb/pull/6243 Test Plan: 'make check' and start 'make crash_test' Differential Revision: D19220495 Pulled By: pdillinger fbshipit-source-id: 6c4709cee80ab4344e06ce360f51e947d79fb3fa --- db_stress_tool/cf_consistency_stress.cc | 8 +- db_stress_tool/db_stress_common.h | 110 ++---------------------- db_stress_tool/db_stress_gflags.cc | 11 --- db_stress_tool/db_stress_tool.cc | 34 -------- db_stress_tool/no_batched_ops_stress.cc | 16 +--- tools/db_crashtest.py | 4 +- 6 files changed, 15 insertions(+), 168 deletions(-) diff --git a/db_stress_tool/cf_consistency_stress.cc b/db_stress_tool/cf_consistency_stress.cc index ecacaf548..0a8e1b7f6 100644 --- a/db_stress_tool/cf_consistency_stress.cc +++ b/db_stress_tool/cf_consistency_stress.cc @@ -244,9 +244,7 @@ class CfConsistencyStressTest : public StressTest { std::string upper_bound; Slice ub_slice; ReadOptions ro_copy = readoptions; - // Get the next prefix first and then see if we want to set upper bound. - // We'll use the next prefix in an assertion later on - if (GetNextPrefix(prefix, &upper_bound) && thread->rand.OneIn(2)) { + if (thread->rand.OneIn(2) && GetNextPrefix(prefix, &upper_bound)) { ub_slice = Slice(upper_bound); ro_copy.iterate_upper_bound = &ub_slice; } @@ -254,13 +252,13 @@ class CfConsistencyStressTest : public StressTest { column_families_[rand_column_families[thread->rand.Next() % rand_column_families.size()]]; Iterator* iter = db_->NewIterator(ro_copy, cfh); - unsigned long count = 0; + long count = 0; for (iter->Seek(prefix); iter->Valid() && iter->key().starts_with(prefix); iter->Next()) { ++count; } assert(prefix_to_use == 0 || - count <= GetPrefixKeyCount(prefix.ToString(), upper_bound)); + count <= (static_cast(1) << ((8 - prefix_to_use) * 8))); Status s = iter->status(); if (s.ok()) { thread->stats.AddPrefixes(1, count); diff --git a/db_stress_tool/db_stress_common.h b/db_stress_tool/db_stress_common.h index 26523db31..8d739f8da 100644 --- a/db_stress_tool/db_stress_common.h +++ b/db_stress_tool/db_stress_common.h @@ -82,9 +82,6 @@ DECLARE_uint64(seed); DECLARE_bool(read_only); DECLARE_int64(max_key); DECLARE_double(hot_key_alpha); -DECLARE_int32(max_key_len); -DECLARE_string(key_len_percent_dist); -DECLARE_int32(key_window_scale_factor); DECLARE_int32(column_families); DECLARE_string(options_file); DECLARE_int64(active_width); @@ -355,7 +352,7 @@ inline bool GetNextPrefix(const rocksdb::Slice& src, std::string* v) { #endif // convert long to a big-endian slice key -extern inline std::string GetStringFromInt(int64_t val) { +extern inline std::string Key(int64_t val) { std::string little_endian_key; std::string big_endian_key; PutFixed64(&little_endian_key, val); @@ -367,109 +364,16 @@ extern inline std::string GetStringFromInt(int64_t val) { return big_endian_key; } -// A struct for maintaining the parameters for generating variable length keys -struct KeyGenContext { - // Number of adjacent keys in one cycle of key lengths - uint64_t window; - // Number of keys of each possible length in a given window - std::vector weights; -}; -extern KeyGenContext key_gen_ctx; - -// Generate a variable length key string from the given int64 val. The -// order of the keys is preserved. The key could be anywhere from 8 to -// max_key_len * 8 bytes. -// The algorithm picks the length based on the -// offset of the val within a configured window and the distribution of the -// number of keys of various lengths in that window. For example, if x, y, x are -// the weights assigned to each possible key length, the keys generated would be -// - {0}...{x-1} -// {(x-1),0}..{(x-1),(y-1)},{(x-1),(y-1),0}..{(x-1),(y-1),(z-1)} and so on. -// Additionally, a trailer of 0-7 bytes could be appended. -extern inline std::string Key(int64_t val) { - uint64_t window = key_gen_ctx.window; - size_t levels = key_gen_ctx.weights.size(); - std::string key; - - for (size_t level = 0; level < levels; ++level) { - uint64_t weight = key_gen_ctx.weights[level]; - uint64_t offset = static_cast(val) % window; - uint64_t mult = static_cast(val) / window; - uint64_t pfx = mult * weight + (offset >= weight ? weight - 1 : offset); - key.append(GetStringFromInt(pfx)); - if (offset < weight) { - // Use the bottom 3 bits of offset as the number of trailing 'x's in the - // key. If the next key is going to be of the next level, then skip the - // trailer as it would break ordering. If the key length is already at max, - // skip the trailer. - if (offset < weight - 1 && level < levels - 1) { - size_t trailer_len = offset & 0x7; - key.append(trailer_len, 'x'); - } - break; - } - val = offset - weight; - window -= weight; - } - - return key; -} - -// Given a string key, map it to an index into the expected values buffer extern inline bool GetIntVal(std::string big_endian_key, uint64_t* key_p) { - size_t size_key = big_endian_key.size(); - std::vector prefixes; - - // Trim the key to multiple of 8 bytes - size_key &= ~7; - assert(size_key <= key_gen_ctx.weights.size() * sizeof(uint64_t)); - - // Pad with zeros to make it a multiple of 8. This function may be called - // with a prefix, in which case we return the first index that falls - // inside or outside that prefix, dependeing on whether the prefix is - // the start of upper bound of a scan - unsigned int pad = sizeof(uint64_t) - (size_key % sizeof(uint64_t)); - if (pad < sizeof(uint64_t)) { - big_endian_key.append(pad, '\0'); - } - + unsigned int size_key = sizeof(*key_p); + assert(big_endian_key.size() == size_key); std::string little_endian_key; little_endian_key.resize(size_key); - for (size_t level = 0; level < size_key; level += sizeof(int64_t)) { - size_t start = level * sizeof(int64_t); - size_t end = (level + 1) * sizeof(int64_t); - for (size_t i = start; i < end; ++i) { - little_endian_key[i] = big_endian_key[end - 1 - i]; - } - Slice little_endian_slice = - Slice(&little_endian_key[start], sizeof(int64_t)); - uint64_t pfx; - if (!GetFixed64(&little_endian_slice, &pfx)) { - return false; - } - prefixes.emplace_back(pfx); - } - - uint64_t key = 0; - for (size_t i = 0; i < prefixes.size(); ++i) { - uint64_t pfx = prefixes[i]; - key += (pfx / key_gen_ctx.weights[i]) * key_gen_ctx.window + - pfx % key_gen_ctx.weights[i]; - } - *key_p = key; - return true; -} - -extern inline uint64_t GetPrefixKeyCount(const std::string& prefix, - const std::string& ub) { - uint64_t start = 0; - uint64_t end = 0; - - if (!GetIntVal(prefix, &start) || !GetIntVal(ub, &end)) { - return 0; + for (size_t i = 0; i < size_key; ++i) { + little_endian_key[i] = big_endian_key[size_key - 1 - i]; } - - return end - start; + Slice little_endian_slice = Slice(little_endian_key); + return GetFixed64(&little_endian_slice, key_p); } extern inline std::string StringToHex(const std::string& str) { diff --git a/db_stress_tool/db_stress_gflags.cc b/db_stress_tool/db_stress_gflags.cc index 1c7390c3f..27a808456 100644 --- a/db_stress_tool/db_stress_gflags.cc +++ b/db_stress_tool/db_stress_gflags.cc @@ -28,17 +28,6 @@ DEFINE_bool(read_only, false, "True if open DB in read-only mode during tests"); DEFINE_int64(max_key, 1 * KB * KB, "Max number of key/values to place in database"); -DEFINE_int32(max_key_len, 3, "Maximum length of a key in 8-byte units"); - -DEFINE_string(key_len_percent_dist, "", - "Percentages of keys of various lengths. For example, 1,30,69 " - "means 1% of keys are 8 bytes, 30% are 16 bytes, and 69% are " - "24 bytes. If not specified, it will be evenly distributed"); - -DEFINE_int32(key_window_scale_factor, 10, - "This value will be multiplied by 100 to come up with a window " - "size for varying the key length"); - DEFINE_int32(column_families, 10, "Number of column families"); DEFINE_double( diff --git a/db_stress_tool/db_stress_tool.cc b/db_stress_tool/db_stress_tool.cc index 3ed1cf838..61198c6f4 100644 --- a/db_stress_tool/db_stress_tool.cc +++ b/db_stress_tool/db_stress_tool.cc @@ -30,8 +30,6 @@ static std::shared_ptr env_guard; static std::shared_ptr env_wrapper_guard; } // namespace -KeyGenContext key_gen_ctx; - int db_stress_tool(int argc, char** argv) { SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) + " [OPTIONS]..."); @@ -199,38 +197,6 @@ int db_stress_tool(int argc, char** argv) { rocksdb_kill_odds = FLAGS_kill_random_test; rocksdb_kill_prefix_blacklist = SplitString(FLAGS_kill_prefix_blacklist); - unsigned int levels = FLAGS_max_key_len; - std::vector weights; - uint64_t scale_factor = FLAGS_key_window_scale_factor; - key_gen_ctx.window = scale_factor * 100; - if (!FLAGS_key_len_percent_dist.empty()) { - weights = SplitString(FLAGS_key_len_percent_dist); - if (weights.size() != levels) { - fprintf(stderr, - "Number of weights in key_len_dist should be equal to" - " max_key_len"); - exit(1); - } - - uint64_t total_weight = 0; - for (std::string& weight : weights) { - uint64_t val = std::stoull(weight); - key_gen_ctx.weights.emplace_back(val * scale_factor); - total_weight += val; - } - if (total_weight != 100) { - fprintf(stderr, "Sum of all weights in key_len_dist should be 100"); - exit(1); - } - } else { - uint64_t keys_per_level = key_gen_ctx.window / levels; - for (unsigned int level = 0; level < levels - 1; ++level) { - key_gen_ctx.weights.emplace_back(keys_per_level); - } - key_gen_ctx.weights.emplace_back(key_gen_ctx.window - - keys_per_level * (levels - 1)); - } - std::unique_ptr stress; if (FLAGS_test_cf_consistency) { stress.reset(CreateCfConsistencyStressTest()); diff --git a/db_stress_tool/no_batched_ops_stress.cc b/db_stress_tool/no_batched_ops_stress.cc index 8d4b891de..0523ae0fe 100644 --- a/db_stress_tool/no_batched_ops_stress.cc +++ b/db_stress_tool/no_batched_ops_stress.cc @@ -52,13 +52,12 @@ class NonBatchedOpsStressTest : public StressTest { Slice k = keystr; Status s = iter->status(); if (iter->Valid()) { - Slice iter_key = iter->key(); if (iter->key().compare(k) > 0) { s = Status::NotFound(Slice()); } else if (iter->key().compare(k) == 0) { from_db = iter->value().ToString(); iter->Next(); - } else if (iter_key.compare(k) < 0) { + } else if (iter->key().compare(k) < 0) { VerificationAbort(shared, "An out of range key was found", static_cast(cf), i); } @@ -254,26 +253,19 @@ class NonBatchedOpsStressTest : public StressTest { std::string upper_bound; Slice ub_slice; ReadOptions ro_copy = read_opts; - // Get the next prefix first and then see if we want to set upper bound. - // We'll use the next prefix in an assertion later on - if (GetNextPrefix(prefix, &upper_bound) && thread->rand.OneIn(2)) { + if (thread->rand.OneIn(2) && GetNextPrefix(prefix, &upper_bound)) { // For half of the time, set the upper bound to the next prefix ub_slice = Slice(upper_bound); ro_copy.iterate_upper_bound = &ub_slice; } Iterator* iter = db_->NewIterator(ro_copy, cfh); - unsigned long count = 0; + long count = 0; for (iter->Seek(prefix); iter->Valid() && iter->key().starts_with(prefix); iter->Next()) { ++count; } - - // FIXME: This was an assertion but was failing on occasion - if (count > GetPrefixKeyCount(prefix.ToString(), upper_bound)) { - fprintf(stdout, "FIXME: count > GetPrefixKeyCount\n"); - } - + assert(count <= (static_cast(1) << ((8 - FLAGS_prefix_size) * 8))); Status s = iter->status(); if (iter->status().ok()) { thread->stats.AddPrefixes(1, count); diff --git a/tools/db_crashtest.py b/tools/db_crashtest.py index 3fe72982e..3319d81f6 100644 --- a/tools/db_crashtest.py +++ b/tools/db_crashtest.py @@ -107,9 +107,7 @@ default_params = { # "level_compaction_dynamic_level_bytes" : True, "verify_checksum_one_in": 1000000, "verify_db_one_in": 100000, - "continuous_verification_interval" : 0, - "max_key_len": 3, - "key_len_percent_dist": "1,30,69" + "continuous_verification_interval" : 0 } _TEST_DIR_ENV_VAR = 'TEST_TMPDIR'