Consider wide columns when checksumming in the stress tests (#10788)

Summary:
There are two places in the stress test code where we compute the CRC
for a range of KVs for the purposes of checking consistency, namely in the
CF consistency test (to make sure CFs contain the same data), and when
performing `CompactRange` (to make sure the pre- and post-compaction
states are equivalent). The patch extends the logic so that wide columns
are also considered in both cases.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/10788

Test Plan: Tested using some simple blackbox crash test runs.

Reviewed By: riversand963

Differential Revision: D40191134

Pulled By: ltamasi

fbshipit-source-id: 542c21cac9077c6d225780deb210319bb5eee955
main
Levi Tamasi 2 years ago committed by Facebook GitHub Bot
parent 5a5f21c489
commit 85399b14f7
  1. 8
      db_stress_tool/cf_consistency_stress.cc
  2. 21
      db_stress_tool/db_stress_test_base.cc

@ -510,11 +510,19 @@ class CfConsistencyStressTest : public StressTest {
const auto checksum_column_family = [](Iterator* iter, const auto checksum_column_family = [](Iterator* iter,
uint32_t* checksum) -> Status { uint32_t* checksum) -> Status {
assert(nullptr != checksum); assert(nullptr != checksum);
uint32_t ret = 0; uint32_t ret = 0;
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
ret = crc32c::Extend(ret, iter->key().data(), iter->key().size()); ret = crc32c::Extend(ret, iter->key().data(), iter->key().size());
ret = crc32c::Extend(ret, iter->value().data(), iter->value().size()); ret = crc32c::Extend(ret, iter->value().data(), iter->value().size());
for (const auto& column : iter->columns()) {
ret = crc32c::Extend(ret, column.name().data(), column.name().size());
ret =
crc32c::Extend(ret, column.value().data(), column.value().size());
}
} }
*checksum = ret; *checksum = ret;
return iter->status(); return iter->status();
}; };

@ -2285,8 +2285,6 @@ uint32_t StressTest::GetRangeHash(ThreadState* thread, const Snapshot* snapshot,
ColumnFamilyHandle* column_family, ColumnFamilyHandle* column_family,
const Slice& start_key, const Slice& start_key,
const Slice& end_key) { const Slice& end_key) {
const std::string kCrcCalculatorSepearator = ";";
uint32_t crc = 0;
// This `ReadOptions` is for validation purposes. Ignore // This `ReadOptions` is for validation purposes. Ignore
// `FLAGS_rate_limit_user_ops` to avoid slowing any validation. // `FLAGS_rate_limit_user_ops` to avoid slowing any validation.
ReadOptions ro; ReadOptions ro;
@ -2299,15 +2297,29 @@ uint32_t StressTest::GetRangeHash(ThreadState* thread, const Snapshot* snapshot,
ts = ts_str; ts = ts_str;
ro.timestamp = &ts; ro.timestamp = &ts;
} }
std::unique_ptr<Iterator> it(db_->NewIterator(ro, column_family)); std::unique_ptr<Iterator> it(db_->NewIterator(ro, column_family));
constexpr char kCrcCalculatorSepearator = ';';
uint32_t crc = 0;
for (it->Seek(start_key); for (it->Seek(start_key);
it->Valid() && options_.comparator->Compare(it->key(), end_key) <= 0; it->Valid() && options_.comparator->Compare(it->key(), end_key) <= 0;
it->Next()) { it->Next()) {
crc = crc32c::Extend(crc, it->key().data(), it->key().size()); crc = crc32c::Extend(crc, it->key().data(), it->key().size());
crc = crc32c::Extend(crc, kCrcCalculatorSepearator.data(), 1); crc = crc32c::Extend(crc, &kCrcCalculatorSepearator, sizeof(char));
crc = crc32c::Extend(crc, it->value().data(), it->value().size()); crc = crc32c::Extend(crc, it->value().data(), it->value().size());
crc = crc32c::Extend(crc, kCrcCalculatorSepearator.data(), 1); crc = crc32c::Extend(crc, &kCrcCalculatorSepearator, sizeof(char));
for (const auto& column : it->columns()) {
crc = crc32c::Extend(crc, column.name().data(), column.name().size());
crc = crc32c::Extend(crc, &kCrcCalculatorSepearator, sizeof(char));
crc = crc32c::Extend(crc, column.value().data(), column.value().size());
crc = crc32c::Extend(crc, &kCrcCalculatorSepearator, sizeof(char));
}
} }
if (!it->status().ok()) { if (!it->status().ok()) {
fprintf(stderr, "Iterator non-OK when calculating range CRC: %s\n", fprintf(stderr, "Iterator non-OK when calculating range CRC: %s\n",
it->status().ToString().c_str()); it->status().ToString().c_str());
@ -2315,6 +2327,7 @@ uint32_t StressTest::GetRangeHash(ThreadState* thread, const Snapshot* snapshot,
// Fail fast to preserve the DB state. // Fail fast to preserve the DB state.
thread->shared->SetVerificationFailure(); thread->shared->SetVerificationFailure();
} }
return crc; return crc;
} }

Loading…
Cancel
Save