From a72d55c99de6852377f54b26e13501c48d15c915 Mon Sep 17 00:00:00 2001 From: Levi Tamasi Date: Fri, 17 Mar 2023 14:47:29 -0700 Subject: [PATCH] Increase the stress test coverage of GetEntity (#11303) Summary: The `GetEntity` API is currently used in the stress tests for verification purposes; this patch extends the coverage by adding a mode where all point lookups in the non-batched, batched, and CF consistency stress tests are done using this API. The PR also includes a bit of refactoring to eliminate some boilerplate code around the wide-column consistency checks. Pull Request resolved: https://github.com/facebook/rocksdb/pull/11303 Test Plan: Ran stress tests of the batched, non-batched, and CF consistency varieties. Reviewed By: akankshamahajan15 Differential Revision: D44148503 Pulled By: ltamasi fbshipit-source-id: fecdbfd3e65a459bbf16ab7aa7b9173e19240077 --- db_stress_tool/batched_ops_stress.cc | 116 +++++++++++++++-- db_stress_tool/cf_consistency_stress.cc | 159 ++++++++++++++++++++++-- db_stress_tool/db_stress_common.cc | 31 +++++ db_stress_tool/db_stress_common.h | 21 ++++ db_stress_tool/db_stress_gflags.cc | 2 + db_stress_tool/db_stress_test_base.cc | 52 +++----- db_stress_tool/db_stress_test_base.h | 11 +- db_stress_tool/expected_state.cc | 16 +-- db_stress_tool/multi_ops_txns_stress.cc | 6 + db_stress_tool/multi_ops_txns_stress.h | 4 + db_stress_tool/no_batched_ops_stress.cc | 156 +++++++++++++++++------ tools/db_crashtest.py | 1 + 12 files changed, 473 insertions(+), 102 deletions(-) diff --git a/db_stress_tool/batched_ops_stress.cc b/db_stress_tool/batched_ops_stress.cc index 3f3446076..62a8290e9 100644 --- a/db_stress_tool/batched_ops_stress.cc +++ b/db_stress_tool/batched_ops_stress.cc @@ -268,6 +268,110 @@ class BatchedOpsStressTest : public StressTest { return ret_status; } + void TestGetEntity(ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + assert(thread); + + ManagedSnapshot snapshot_guard(db_); + + ReadOptions read_opts_copy(read_opts); + read_opts_copy.snapshot = snapshot_guard.snapshot(); + + assert(!rand_keys.empty()); + + const std::string key_suffix = Key(rand_keys[0]); + + assert(!rand_column_families.empty()); + assert(rand_column_families[0] >= 0); + assert(rand_column_families[0] < static_cast(column_families_.size())); + + ColumnFamilyHandle* const cfh = column_families_[rand_column_families[0]]; + assert(cfh); + + constexpr size_t num_keys = 10; + + std::array results; + + for (size_t i = 0; i < num_keys; ++i) { + const std::string key = std::to_string(i) + key_suffix; + + const Status s = db_->GetEntity(read_opts_copy, cfh, key, &results[i]); + + if (!s.ok() && !s.IsNotFound()) { + fprintf(stderr, "GetEntity error: %s\n", s.ToString().c_str()); + thread->stats.AddErrors(1); + } else if (s.IsNotFound()) { + thread->stats.AddGets(1, 0); + } else { + thread->stats.AddGets(1, 1); + } + } + + // Compare columns ignoring the last character of column values + auto compare = [](const WideColumns& lhs, const WideColumns& rhs) { + if (lhs.size() != rhs.size()) { + return false; + } + + for (size_t i = 0; i < lhs.size(); ++i) { + if (lhs[i].name() != rhs[i].name()) { + return false; + } + + if (lhs[i].value().size() != rhs[i].value().size()) { + return false; + } + + if (lhs[i].value().difference_offset(rhs[i].value()) < + lhs[i].value().size() - 1) { + return false; + } + } + + return true; + }; + + for (size_t i = 0; i < num_keys; ++i) { + const WideColumns& columns = results[i].columns(); + + if (!compare(results[0].columns(), columns)) { + fprintf(stderr, + "GetEntity error: inconsistent entities for key %s: %s, %s\n", + StringToHex(key_suffix).c_str(), + WideColumnsToHex(results[0].columns()).c_str(), + WideColumnsToHex(columns).c_str()); + } + + if (!columns.empty()) { + // The last character of each column value should be 'i' as a decimal + // digit + const char expected = static_cast('0' + i); + + for (const auto& column : columns) { + const Slice& value = column.value(); + + if (value.empty() || value[value.size() - 1] != expected) { + fprintf(stderr, + "GetEntity error: incorrect column value for key " + "%s, entity %s, column value %s, expected %c\n", + StringToHex(key_suffix).c_str(), + WideColumnsToHex(columns).c_str(), + value.ToString(/* hex */ true).c_str(), expected); + } + } + + if (!VerifyWideColumns(columns)) { + fprintf( + stderr, + "GetEntity error: inconsistent columns for key %s, entity %s\n", + StringToHex(key_suffix).c_str(), + WideColumnsToHex(columns).c_str()); + } + } + } + } + // Given a key, this does prefix scans for "0"+P, "1"+P, ..., "9"+P // in the same snapshot where P is the first FLAGS_prefix_size - 1 bytes // of the key. Each of these 10 scans returns a series of values; @@ -357,16 +461,14 @@ class BatchedOpsStressTest : public StressTest { } // make sure value() and columns() are consistent - const WideColumns expected_columns = GenerateExpectedWideColumns( - GetValueBase(iters[i]->value()), iters[i]->value()); - if (iters[i]->columns() != expected_columns) { + if (!VerifyWideColumns(iters[i]->value(), iters[i]->columns())) { fprintf(stderr, "prefix scan error : %" ROCKSDB_PRIszt - ", value and columns inconsistent for prefix %s: %s\n", + ", value and columns inconsistent for prefix %s: value: %s, " + "columns: %s\n", i, prefix_slices[i].ToString(/* hex */ true).c_str(), - DebugString(iters[i]->value(), iters[i]->columns(), - expected_columns) - .c_str()); + iters[i]->value().ToString(/* hex */ true).c_str(), + WideColumnsToHex(iters[i]->columns()).c_str()); } iters[i]->Next(); diff --git a/db_stress_tool/cf_consistency_stress.cc b/db_stress_tool/cf_consistency_stress.cc index 411f38d59..883a17b6e 100644 --- a/db_stress_tool/cf_consistency_stress.cc +++ b/db_stress_tool/cf_consistency_stress.cc @@ -251,6 +251,146 @@ class CfConsistencyStressTest : public StressTest { return statuses; } + void TestGetEntity(ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + assert(thread); + assert(!rand_column_families.empty()); + assert(!rand_keys.empty()); + + const std::string key = Key(rand_keys[0]); + + Status s; + bool is_consistent = true; + + if (thread->rand.OneIn(2)) { + // With a 1/2 chance, do a random read from a random CF + const size_t cf_id = thread->rand.Next() % rand_column_families.size(); + + assert(rand_column_families[cf_id] >= 0); + assert(rand_column_families[cf_id] < + static_cast(column_families_.size())); + + ColumnFamilyHandle* const cfh = + column_families_[rand_column_families[cf_id]]; + assert(cfh); + + PinnableWideColumns result; + s = db_->GetEntity(read_opts, cfh, key, &result); + + if (s.ok()) { + if (!VerifyWideColumns(result.columns())) { + fprintf( + stderr, + "GetEntity error: inconsistent columns for key %s, entity %s\n", + StringToHex(key).c_str(), + WideColumnsToHex(result.columns()).c_str()); + is_consistent = false; + } + } + } else { + // With a 1/2 chance, compare one key across all CFs + ManagedSnapshot snapshot_guard(db_); + + ReadOptions read_opts_copy = read_opts; + read_opts_copy.snapshot = snapshot_guard.snapshot(); + + assert(rand_column_families[0] >= 0); + assert(rand_column_families[0] < + static_cast(column_families_.size())); + + PinnableWideColumns cmp_result; + s = db_->GetEntity(read_opts_copy, + column_families_[rand_column_families[0]], key, + &cmp_result); + + if (s.ok() || s.IsNotFound()) { + const bool cmp_found = s.ok(); + + if (cmp_found) { + if (!VerifyWideColumns(cmp_result.columns())) { + fprintf(stderr, + "GetEntity error: inconsistent columns for key %s, " + "entity %s\n", + StringToHex(key).c_str(), + WideColumnsToHex(cmp_result.columns()).c_str()); + is_consistent = false; + } + } + + if (is_consistent) { + for (size_t i = 1; i < rand_column_families.size(); ++i) { + assert(rand_column_families[i] >= 0); + assert(rand_column_families[i] < + static_cast(column_families_.size())); + + PinnableWideColumns result; + s = db_->GetEntity(read_opts_copy, + column_families_[rand_column_families[i]], key, + &result); + + if (!s.ok() && !s.IsNotFound()) { + break; + } + + const bool found = s.ok(); + + assert(!column_family_names_.empty()); + assert(i < column_family_names_.size()); + + if (!cmp_found && found) { + fprintf(stderr, + "GetEntity returns different results for key %s: CF %s " + "returns not found, CF %s returns entity %s\n", + StringToHex(key).c_str(), column_family_names_[0].c_str(), + column_family_names_[i].c_str(), + WideColumnsToHex(result.columns()).c_str()); + is_consistent = false; + break; + } + + if (cmp_found && !found) { + fprintf(stderr, + "GetEntity returns different results for key %s: CF %s " + "returns entity %s, CF %s returns not found\n", + StringToHex(key).c_str(), column_family_names_[0].c_str(), + WideColumnsToHex(cmp_result.columns()).c_str(), + column_family_names_[i].c_str()); + is_consistent = false; + break; + } + + if (found && result != cmp_result) { + fprintf(stderr, + "GetEntity returns different results for key %s: CF %s " + "returns entity %s, CF %s returns entity %s\n", + StringToHex(key).c_str(), column_family_names_[0].c_str(), + WideColumnsToHex(cmp_result.columns()).c_str(), + column_family_names_[i].c_str(), + WideColumnsToHex(result.columns()).c_str()); + is_consistent = false; + break; + } + } + } + } + } + + if (!is_consistent) { + fprintf(stderr, "TestGetEntity error: results are not consistent\n"); + thread->stats.AddErrors(1); + // Fail fast to preserve the DB state. + thread->shared->SetVerificationFailure(); + } else if (s.ok()) { + thread->stats.AddGets(1, 1); + } else if (s.IsNotFound()) { + thread->stats.AddGets(1, 0); + } else { + fprintf(stderr, "TestGetEntity error: %s\n", s.ToString().c_str()); + thread->stats.AddErrors(1); + } + } + Status TestPrefixScan(ThreadState* thread, const ReadOptions& readoptions, const std::vector& rand_column_families, const std::vector& rand_keys) override { @@ -290,12 +430,9 @@ class CfConsistencyStressTest : public StressTest { iter->Next()) { ++count; - const WideColumns expected_columns = GenerateExpectedWideColumns( - GetValueBase(iter->value()), iter->value()); - if (iter->columns() != expected_columns) { - s = Status::Corruption( - "Value and columns inconsistent", - DebugString(iter->value(), iter->columns(), expected_columns)); + if (!VerifyWideColumns(iter->value(), iter->columns())) { + s = Status::Corruption("Value and columns inconsistent", + DebugString(iter->value(), iter->columns())); break; } } @@ -372,12 +509,10 @@ class CfConsistencyStressTest : public StressTest { assert(iter); if (iter->Valid()) { - const WideColumns expected_columns = GenerateExpectedWideColumns( - GetValueBase(iter->value()), iter->value()); - if (iter->columns() != expected_columns) { - statuses[i] = Status::Corruption( - "Value and columns inconsistent", - DebugString(iter->value(), iter->columns(), expected_columns)); + if (!VerifyWideColumns(iter->value(), iter->columns())) { + statuses[i] = + Status::Corruption("Value and columns inconsistent", + DebugString(iter->value(), iter->columns())); } else { ++valid_cnt; } diff --git a/db_stress_tool/db_stress_common.cc b/db_stress_tool/db_stress_common.cc index af8db9e2f..93436d0f8 100644 --- a/db_stress_tool/db_stress_common.cc +++ b/db_stress_tool/db_stress_common.cc @@ -278,6 +278,37 @@ WideColumns GenerateExpectedWideColumns(uint32_t value_base, return columns; } +bool VerifyWideColumns(const Slice& value, const WideColumns& columns) { + if (value.size() < sizeof(uint32_t)) { + return false; + } + + const uint32_t value_base = GetValueBase(value); + + const WideColumns expected_columns = + GenerateExpectedWideColumns(value_base, value); + + if (columns != expected_columns) { + return false; + } + + return true; +} + +bool VerifyWideColumns(const WideColumns& columns) { + if (columns.empty()) { + return false; + } + + if (columns.front().name() != kDefaultWideColumnName) { + return false; + } + + const Slice& value_of_default = columns.front().value(); + + return VerifyWideColumns(value_of_default, columns); +} + std::string GetNowNanos() { uint64_t t = db_stress_env->NowNanos(); std::string ret; diff --git a/db_stress_tool/db_stress_common.h b/db_stress_tool/db_stress_common.h index e029216bb..062b6b98c 100644 --- a/db_stress_tool/db_stress_common.h +++ b/db_stress_tool/db_stress_common.h @@ -213,6 +213,7 @@ DECLARE_bool(compare_full_db_state_snapshot); DECLARE_uint64(snapshot_hold_ops); DECLARE_bool(long_running_snapshots); DECLARE_bool(use_multiget); +DECLARE_bool(use_get_entity); DECLARE_int32(readpercent); DECLARE_int32(prefixpercent); DECLARE_int32(writepercent); @@ -596,6 +597,24 @@ extern inline std::string StringToHex(const std::string& str) { return result; } +inline std::string WideColumnsToHex(const WideColumns& columns) { + if (columns.empty()) { + return std::string(); + } + + std::ostringstream oss; + + oss << std::hex; + + auto it = columns.begin(); + oss << *it; + for (++it; it != columns.end(); ++it) { + oss << ' ' << *it; + } + + return oss.str(); +} + // Unified output format for double parameters extern inline std::string FormatDoubleParam(double param) { return std::to_string(param); @@ -626,6 +645,8 @@ extern uint32_t GetValueBase(Slice s); extern WideColumns GenerateWideColumns(uint32_t value_base, const Slice& slice); extern WideColumns GenerateExpectedWideColumns(uint32_t value_base, const Slice& slice); +extern bool VerifyWideColumns(const Slice& value, const WideColumns& columns); +extern bool VerifyWideColumns(const WideColumns& columns); extern StressTest* CreateCfConsistencyStressTest(); extern StressTest* CreateBatchedOpsStressTest(); diff --git a/db_stress_tool/db_stress_gflags.cc b/db_stress_tool/db_stress_gflags.cc index 8b7354bf2..d7cf8b10f 100644 --- a/db_stress_tool/db_stress_gflags.cc +++ b/db_stress_tool/db_stress_gflags.cc @@ -747,6 +747,8 @@ DEFINE_bool(long_running_snapshots, false, DEFINE_bool(use_multiget, false, "If set, use the batched MultiGet API for reads"); +DEFINE_bool(use_get_entity, false, "If set, use the GetEntity API for reads"); + static bool ValidateInt32Percent(const char* flagname, int32_t value) { if (value < 0 || value > 100) { fprintf(stderr, "Invalid value for --%s: %d, 0<= pct <=100 \n", flagname, diff --git a/db_stress_tool/db_stress_test_base.cc b/db_stress_tool/db_stress_test_base.cc index b04b29b8c..215367760 100644 --- a/db_stress_tool/db_stress_test_base.cc +++ b/db_stress_tool/db_stress_test_base.cc @@ -429,47 +429,27 @@ void StressTest::VerificationAbort(SharedState* shared, std::string msg, int cf, void StressTest::VerificationAbort(SharedState* shared, int cf, int64_t key, const Slice& value, - const WideColumns& columns, - const WideColumns& expected_columns) const { + const WideColumns& columns) const { assert(shared); auto key_str = Key(key); fprintf(stderr, "Verification failed for column family %d key %s (%" PRIi64 - "): Value and columns inconsistent: %s\n", + "): Value and columns inconsistent: value: %s, columns: %s\n", cf, Slice(key_str).ToString(/* hex */ true).c_str(), key, - DebugString(value, columns, expected_columns).c_str()); + value.ToString(/* hex */ true).c_str(), + WideColumnsToHex(columns).c_str()); shared->SetVerificationFailure(); } std::string StressTest::DebugString(const Slice& value, - const WideColumns& columns, - const WideColumns& expected_columns) { + const WideColumns& columns) { std::ostringstream oss; - oss << "value: " << value.ToString(/* hex */ true); - - auto dump = [](const WideColumns& cols, std::ostream& os) { - if (cols.empty()) { - return; - } - - os << std::hex; - - auto it = cols.begin(); - os << *it; - for (++it; it != cols.end(); ++it) { - os << ' ' << *it; - } - }; - - oss << ", columns: "; - dump(columns, oss); - - oss << ", expected_columns: "; - dump(expected_columns, oss); + oss << "value: " << value.ToString(/* hex */ true) + << ", columns: " << WideColumnsToHex(columns); return oss.str(); } @@ -1004,7 +984,9 @@ void StressTest::OperateDb(ThreadState* thread) { if (prob_op >= 0 && prob_op < static_cast(FLAGS_readpercent)) { assert(0 <= prob_op); // OPERATION read - if (FLAGS_use_multiget) { + if (FLAGS_use_get_entity) { + TestGetEntity(thread, read_opts, rand_column_families, rand_keys); + } else if (FLAGS_use_multiget) { // Leave room for one more iteration of the loop with a single key // batch. This is to ensure that each thread does exactly the same // number of ops @@ -1491,12 +1473,12 @@ void StressTest::VerifyIterator(ThreadState* thread, } if (!*diverged && iter->Valid()) { - const WideColumns expected_columns = - GenerateExpectedWideColumns(GetValueBase(iter->value()), iter->value()); - if (iter->columns() != expected_columns) { - fprintf(stderr, "Value and columns inconsistent for iterator: %s\n", - DebugString(iter->value(), iter->columns(), expected_columns) - .c_str()); + if (!VerifyWideColumns(iter->value(), iter->columns())) { + fprintf(stderr, + "Value and columns inconsistent for iterator: value: %s, " + "columns: %s\n", + iter->value().ToString(/* hex */ true).c_str(), + WideColumnsToHex(iter->columns()).c_str()); *diverged = true; } @@ -2402,6 +2384,8 @@ void StressTest::PrintEnv() const { FLAGS_subcompactions); fprintf(stdout, "Use MultiGet : %s\n", FLAGS_use_multiget ? "true" : "false"); + fprintf(stdout, "Use GetEntity : %s\n", + FLAGS_use_get_entity ? "true" : "false"); const char* memtablerep = ""; switch (FLAGS_rep_factory) { diff --git a/db_stress_tool/db_stress_test_base.h b/db_stress_tool/db_stress_test_base.h index 83f9bb1f1..e6de74d7b 100644 --- a/db_stress_tool/db_stress_test_base.h +++ b/db_stress_tool/db_stress_test_base.h @@ -94,6 +94,10 @@ class StressTest { const std::vector& rand_column_families, const std::vector& rand_keys) = 0; + virtual void TestGetEntity(ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) = 0; + virtual Status TestPrefixScan(ThreadState* thread, const ReadOptions& read_opts, const std::vector& rand_column_families, @@ -221,11 +225,10 @@ class StressTest { Slice value_from_expected) const; void VerificationAbort(SharedState* shared, int cf, int64_t key, - const Slice& value, const WideColumns& columns, - const WideColumns& expected_columns) const; + const Slice& value, const WideColumns& columns) const; - static std::string DebugString(const Slice& value, const WideColumns& columns, - const WideColumns& expected_columns); + static std::string DebugString(const Slice& value, + const WideColumns& columns); void PrintEnv() const; diff --git a/db_stress_tool/expected_state.cc b/db_stress_tool/expected_state.cc index d07c2efb0..0d921c712 100644 --- a/db_stress_tool/expected_state.cc +++ b/db_stress_tool/expected_state.cc @@ -416,16 +416,7 @@ class ExpectedStateTraceRecordHandler : public TraceRecord::Handler, entity.ToString(/* hex */ true)); } - if (columns.empty() || columns[0].name() != kDefaultWideColumnName) { - return Status::Corruption("Cannot find default column in entity", - entity.ToString(/* hex */ true)); - } - - const Slice& value_of_default = columns[0].value(); - - const uint32_t value_base = GetValueBase(value_of_default); - - if (columns != GenerateExpectedWideColumns(value_base, value_of_default)) { + if (!VerifyWideColumns(columns)) { return Status::Corruption("Wide columns in entity inconsistent", entity.ToString(/* hex */ true)); } @@ -435,6 +426,11 @@ class ExpectedStateTraceRecordHandler : public TraceRecord::Handler, column_family_id, key, columns); } + assert(!columns.empty()); + assert(columns.front().name() == kDefaultWideColumnName); + + const uint32_t value_base = GetValueBase(columns.front().value()); + state_->Put(column_family_id, static_cast(key_id), value_base, false /* pending */); diff --git a/db_stress_tool/multi_ops_txns_stress.cc b/db_stress_tool/multi_ops_txns_stress.cc index 3a87f7716..b543b0246 100644 --- a/db_stress_tool/multi_ops_txns_stress.cc +++ b/db_stress_tool/multi_ops_txns_stress.cc @@ -387,6 +387,12 @@ std::vector MultiOpsTxnsStressTest::TestMultiGet( return std::vector{Status::NotSupported()}; } +// Wide columns are currently not supported by transactions. +void MultiOpsTxnsStressTest::TestGetEntity( + ThreadState* /* thread */, const ReadOptions& /* read_opts */, + const std::vector& /* rand_column_families */, + const std::vector& /* rand_keys */) {} + Status MultiOpsTxnsStressTest::TestPrefixScan( ThreadState* thread, const ReadOptions& read_opts, const std::vector& rand_column_families, diff --git a/db_stress_tool/multi_ops_txns_stress.h b/db_stress_tool/multi_ops_txns_stress.h index cd19a264b..479344643 100644 --- a/db_stress_tool/multi_ops_txns_stress.h +++ b/db_stress_tool/multi_ops_txns_stress.h @@ -210,6 +210,10 @@ class MultiOpsTxnsStressTest : public StressTest { const std::vector& rand_column_families, const std::vector& rand_keys) override; + void TestGetEntity(ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override; + Status TestPrefixScan(ThreadState* thread, const ReadOptions& read_opts, const std::vector& rand_column_families, const std::vector& rand_keys) override; diff --git a/db_stress_tool/no_batched_ops_stress.cc b/db_stress_tool/no_batched_ops_stress.cc index ad9971012..716ea3802 100644 --- a/db_stress_tool/no_batched_ops_stress.cc +++ b/db_stress_tool/no_batched_ops_stress.cc @@ -101,12 +101,9 @@ class NonBatchedOpsStressTest : public StressTest { if (diff > 0) { s = Status::NotFound(); } else if (diff == 0) { - const WideColumns expected_columns = GenerateExpectedWideColumns( - GetValueBase(iter->value()), iter->value()); - if (iter->columns() != expected_columns) { + if (!VerifyWideColumns(iter->value(), iter->columns())) { VerificationAbort(shared, static_cast(cf), i, - iter->value(), iter->columns(), - expected_columns); + iter->value(), iter->columns()); } from_db = iter->value().ToString(); @@ -159,26 +156,24 @@ class NonBatchedOpsStressTest : public StressTest { } const std::string key = Key(i); - PinnableWideColumns columns; + PinnableWideColumns result; Status s = - db_->GetEntity(options, column_families_[cf], key, &columns); + db_->GetEntity(options, column_families_[cf], key, &result); std::string from_db; if (s.ok()) { - const WideColumns& columns_from_db = columns.columns(); + const WideColumns& columns = result.columns(); - if (!columns_from_db.empty() && - columns_from_db[0].name() == kDefaultWideColumnName) { - from_db = columns_from_db[0].value().ToString(); + if (!columns.empty() && + columns.front().name() == kDefaultWideColumnName) { + from_db = columns.front().value().ToString(); } - const WideColumns expected_columns = - GenerateExpectedWideColumns(GetValueBase(from_db), from_db); - if (columns_from_db != expected_columns) { + if (!VerifyWideColumns(columns)) { VerificationAbort(shared, static_cast(cf), i, from_db, - columns_from_db, expected_columns); + columns); } } @@ -256,18 +251,16 @@ class NonBatchedOpsStressTest : public StressTest { std::string from_db; if (statuses[j].ok()) { - const WideColumns& columns_from_db = results[j].columns(); + const WideColumns& columns = results[j].columns(); - if (!columns_from_db.empty() && - columns_from_db[0].name() == kDefaultWideColumnName) { - from_db = columns_from_db[0].value().ToString(); + if (!columns.empty() && + columns.front().name() == kDefaultWideColumnName) { + from_db = columns.front().value().ToString(); } - const WideColumns expected_columns = - GenerateExpectedWideColumns(GetValueBase(from_db), from_db); - if (columns_from_db != expected_columns) { + if (!VerifyWideColumns(columns)) { VerificationAbort(shared, static_cast(cf), i, from_db, - columns_from_db, expected_columns); + columns); } } @@ -756,6 +749,104 @@ class NonBatchedOpsStressTest : public StressTest { return statuses; } + void TestGetEntity(ThreadState* thread, const ReadOptions& read_opts, + const std::vector& rand_column_families, + const std::vector& rand_keys) override { + if (fault_fs_guard) { + fault_fs_guard->EnableErrorInjection(); + SharedState::ignore_read_error = false; + } + + assert(thread); + + SharedState* const shared = thread->shared; + assert(shared); + + assert(!rand_column_families.empty()); + assert(!rand_keys.empty()); + + std::unique_ptr lock(new MutexLock( + shared->GetMutexForKey(rand_column_families[0], rand_keys[0]))); + + assert(rand_column_families[0] >= 0); + assert(rand_column_families[0] < static_cast(column_families_.size())); + + ColumnFamilyHandle* const cfh = column_families_[rand_column_families[0]]; + assert(cfh); + + const std::string key = Key(rand_keys[0]); + + PinnableWideColumns from_db; + + const Status s = db_->GetEntity(read_opts, cfh, key, &from_db); + + int error_count = 0; + + if (fault_fs_guard) { + error_count = fault_fs_guard->GetAndResetErrorCount(); + } + + if (s.ok()) { + if (fault_fs_guard) { + if (error_count && !SharedState::ignore_read_error) { + // Grab mutex so multiple threads don't try to print the + // stack trace at the same time + MutexLock l(shared->GetMutex()); + fprintf(stderr, "Didn't get expected error from GetEntity\n"); + fprintf(stderr, "Call stack that injected the fault\n"); + fault_fs_guard->PrintFaultBacktrace(); + std::terminate(); + } + } + + thread->stats.AddGets(1, 1); + + if (!FLAGS_skip_verifydb) { + const WideColumns& columns = from_db.columns(); + + if (!VerifyWideColumns(columns)) { + shared->SetVerificationFailure(); + fprintf(stderr, + "error : inconsistent columns returned by GetEntity for key " + "%s: %s\n", + StringToHex(key).c_str(), WideColumnsToHex(columns).c_str()); + } else if (shared->Get(rand_column_families[0], rand_keys[0]) == + SharedState::DELETION_SENTINEL) { + shared->SetVerificationFailure(); + fprintf( + stderr, + "error : inconsistent values for key %s: GetEntity returns %s, " + "expected state does not have the key.\n", + StringToHex(key).c_str(), WideColumnsToHex(columns).c_str()); + } + } + } else if (s.IsNotFound()) { + thread->stats.AddGets(1, 0); + + if (!FLAGS_skip_verifydb) { + auto expected = shared->Get(rand_column_families[0], rand_keys[0]); + if (expected != SharedState::DELETION_SENTINEL && + expected != SharedState::UNKNOWN_SENTINEL) { + shared->SetVerificationFailure(); + fprintf(stderr, + "error : inconsistent values for key %s: expected state has " + "the key, GetEntity returns NotFound.\n", + StringToHex(key).c_str()); + } + } + } else { + if (error_count == 0) { + thread->stats.AddErrors(1); + } else { + thread->stats.AddVerifiedErrors(1); + } + } + + if (fault_fs_guard) { + fault_fs_guard->DisableErrorInjection(); + } + } + Status TestPrefixScan(ThreadState* thread, const ReadOptions& read_opts, const std::vector& rand_column_families, const std::vector& rand_keys) override { @@ -810,12 +901,9 @@ class NonBatchedOpsStressTest : public StressTest { } } - const WideColumns expected_columns = GenerateExpectedWideColumns( - GetValueBase(iter->value()), iter->value()); - if (iter->columns() != expected_columns) { - s = Status::Corruption( - "Value and columns inconsistent", - DebugString(iter->value(), iter->columns(), expected_columns)); + if (!VerifyWideColumns(iter->value(), iter->columns())) { + s = Status::Corruption("Value and columns inconsistent", + DebugString(iter->value(), iter->columns())); break; } } @@ -1268,17 +1356,15 @@ class NonBatchedOpsStressTest : public StressTest { assert(iter); assert(iter->Valid()); - const WideColumns expected_columns = GenerateExpectedWideColumns( - GetValueBase(iter->value()), iter->value()); - if (iter->columns() != expected_columns) { + if (!VerifyWideColumns(iter->value(), iter->columns())) { shared->SetVerificationFailure(); fprintf(stderr, "Verification failed for key %s: " - "Value and columns inconsistent: %s\n", + "Value and columns inconsistent: value: %s, columns: %s\n", Slice(iter->key()).ToString(/* hex */ true).c_str(), - DebugString(iter->value(), iter->columns(), expected_columns) - .c_str()); + iter->value().ToString(/* hex */ true).c_str(), + WideColumnsToHex(iter->columns()).c_str()); fprintf(stderr, "Column family: %s, op_logs: %s\n", cfh->GetName().c_str(), op_logs.c_str()); diff --git a/tools/db_crashtest.py b/tools/db_crashtest.py index a13199794..fd1213b54 100644 --- a/tools/db_crashtest.py +++ b/tools/db_crashtest.py @@ -136,6 +136,7 @@ default_params = { "format_version": lambda: random.choice([2, 3, 4, 5, 5]), "index_block_restart_interval": lambda: random.choice(range(1, 16)), "use_multiget": lambda: random.randint(0, 1), + "use_get_entity": lambda: random.choice([0] * 7 + [1]), "periodic_compaction_seconds": lambda: random.choice([0, 0, 1, 2, 10, 100, 1000]), # 0 = never (used by some), 10 = often (for threading bugs), 600 = default "stats_dump_period_sec": lambda: random.choice([0, 10, 600]),