Increase the stress test coverage of GetEntity (#11303)

Summary:
The `GetEntity` API is currently used in the stress tests for verification purposes;
this patch extends the coverage by adding a mode where all point lookups in
the non-batched, batched, and CF consistency stress tests are done using this API.
The PR also includes a bit of refactoring to eliminate some boilerplate code around
the wide-column consistency checks.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/11303

Test Plan: Ran stress tests of the batched, non-batched, and CF consistency varieties.

Reviewed By: akankshamahajan15

Differential Revision: D44148503

Pulled By: ltamasi

fbshipit-source-id: fecdbfd3e65a459bbf16ab7aa7b9173e19240077

db_stress_tool/batched_ops_stress.cc

@@ -268,6 +268,110 @@ class BatchedOpsStressTest : public StressTest {
     return ret_status;
   }
 
+  void TestGetEntity(ThreadState* thread, const ReadOptions& read_opts,
+                     const std::vector<int>& rand_column_families,
+                     const std::vector<int64_t>& rand_keys) override {
+    assert(thread);
+
+    ManagedSnapshot snapshot_guard(db_);
+
+    ReadOptions read_opts_copy(read_opts);
+    read_opts_copy.snapshot = snapshot_guard.snapshot();
+
+    assert(!rand_keys.empty());
+
+    const std::string key_suffix = Key(rand_keys[0]);
+
+    assert(!rand_column_families.empty());
+    assert(rand_column_families[0] >= 0);
+    assert(rand_column_families[0] < static_cast<int>(column_families_.size()));
+
+    ColumnFamilyHandle* const cfh = column_families_[rand_column_families[0]];
+    assert(cfh);
+
+    constexpr size_t num_keys = 10;
+
+    std::array<PinnableWideColumns, num_keys> results;
+
+    for (size_t i = 0; i < num_keys; ++i) {
+      const std::string key = std::to_string(i) + key_suffix;
+
+      const Status s = db_->GetEntity(read_opts_copy, cfh, key, &results[i]);
+
+      if (!s.ok() && !s.IsNotFound()) {
+        fprintf(stderr, "GetEntity error: %s\n", s.ToString().c_str());
+        thread->stats.AddErrors(1);
+      } else if (s.IsNotFound()) {
+        thread->stats.AddGets(1, 0);
+      } else {
+        thread->stats.AddGets(1, 1);
+      }
+    }
+
+    // Compare columns ignoring the last character of column values
+    auto compare = [](const WideColumns& lhs, const WideColumns& rhs) {
+      if (lhs.size() != rhs.size()) {
+        return false;
+      }
+
+      for (size_t i = 0; i < lhs.size(); ++i) {
+        if (lhs[i].name() != rhs[i].name()) {
+          return false;
+        }
+
+        if (lhs[i].value().size() != rhs[i].value().size()) {
+          return false;
+        }
+
+        if (lhs[i].value().difference_offset(rhs[i].value()) <
+            lhs[i].value().size() - 1) {
+          return false;
+        }
+      }
+
+      return true;
+    };
+
+    for (size_t i = 0; i < num_keys; ++i) {
+      const WideColumns& columns = results[i].columns();
+
+      if (!compare(results[0].columns(), columns)) {
+        fprintf(stderr,
+                "GetEntity error: inconsistent entities for key %s: %s, %s\n",
+                StringToHex(key_suffix).c_str(),
+                WideColumnsToHex(results[0].columns()).c_str(),
+                WideColumnsToHex(columns).c_str());
+      }
+
+      if (!columns.empty()) {
+        // The last character of each column value should be 'i' as a decimal
+        // digit
+        const char expected = static_cast<char>('0' + i);
+
+        for (const auto& column : columns) {
+          const Slice& value = column.value();
+
+          if (value.empty() || value[value.size() - 1] != expected) {
+            fprintf(stderr,
+                    "GetEntity error: incorrect column value for key "
+                    "%s, entity %s, column value %s, expected %c\n",
+                    StringToHex(key_suffix).c_str(),
+                    WideColumnsToHex(columns).c_str(),
+                    value.ToString(/* hex */ true).c_str(), expected);
+          }
+        }
+
+        if (!VerifyWideColumns(columns)) {
+          fprintf(
+              stderr,
+              "GetEntity error: inconsistent columns for key %s, entity %s\n",
+              StringToHex(key_suffix).c_str(),
+              WideColumnsToHex(columns).c_str());
+        }
+      }
+    }
+  }
+
   // Given a key, this does prefix scans for "0"+P, "1"+P, ..., "9"+P
   // in the same snapshot where P is the first FLAGS_prefix_size - 1 bytes
   // of the key. Each of these 10 scans returns a series of values;
@@ -357,16 +461,14 @@ class BatchedOpsStressTest : public StressTest {
         }
 
         // make sure value() and columns() are consistent
-        const WideColumns expected_columns = GenerateExpectedWideColumns(
-            GetValueBase(iters[i]->value()), iters[i]->value());
-        if (iters[i]->columns() != expected_columns) {
+        if (!VerifyWideColumns(iters[i]->value(), iters[i]->columns())) {
           fprintf(stderr,
                   "prefix scan error : %" ROCKSDB_PRIszt
-                  ", value and columns inconsistent for prefix %s: %s\n",
+                  ", value and columns inconsistent for prefix %s: value: %s, "
+                  "columns: %s\n",
                   i, prefix_slices[i].ToString(/* hex */ true).c_str(),
-                  DebugString(iters[i]->value(), iters[i]->columns(),
-                              expected_columns)
-                      .c_str());
+                  iters[i]->value().ToString(/* hex */ true).c_str(),
+                  WideColumnsToHex(iters[i]->columns()).c_str());
         }
 
         iters[i]->Next();

db_stress_tool/cf_consistency_stress.cc

@@ -251,6 +251,146 @@ class CfConsistencyStressTest : public StressTest {
     return statuses;
   }
 
+  void TestGetEntity(ThreadState* thread, const ReadOptions& read_opts,
+                     const std::vector<int>& rand_column_families,
+                     const std::vector<int64_t>& rand_keys) override {
+    assert(thread);
+    assert(!rand_column_families.empty());
+    assert(!rand_keys.empty());
+
+    const std::string key = Key(rand_keys[0]);
+
+    Status s;
+    bool is_consistent = true;
+
+    if (thread->rand.OneIn(2)) {
+      // With a 1/2 chance, do a random read from a random CF
+      const size_t cf_id = thread->rand.Next() % rand_column_families.size();
+
+      assert(rand_column_families[cf_id] >= 0);
+      assert(rand_column_families[cf_id] <
+             static_cast<int>(column_families_.size()));
+
+      ColumnFamilyHandle* const cfh =
+          column_families_[rand_column_families[cf_id]];
+      assert(cfh);
+
+      PinnableWideColumns result;
+      s = db_->GetEntity(read_opts, cfh, key, &result);
+
+      if (s.ok()) {
+        if (!VerifyWideColumns(result.columns())) {
+          fprintf(
+              stderr,
+              "GetEntity error: inconsistent columns for key %s, entity %s\n",
+              StringToHex(key).c_str(),
+              WideColumnsToHex(result.columns()).c_str());
+          is_consistent = false;
+        }
+      }
+    } else {
+      // With a 1/2 chance, compare one key across all CFs
+      ManagedSnapshot snapshot_guard(db_);
+
+      ReadOptions read_opts_copy = read_opts;
+      read_opts_copy.snapshot = snapshot_guard.snapshot();
+
+      assert(rand_column_families[0] >= 0);
+      assert(rand_column_families[0] <
+             static_cast<int>(column_families_.size()));
+
+      PinnableWideColumns cmp_result;
+      s = db_->GetEntity(read_opts_copy,
+                         column_families_[rand_column_families[0]], key,
+                         &cmp_result);
+
+      if (s.ok() || s.IsNotFound()) {
+        const bool cmp_found = s.ok();
+
+        if (cmp_found) {
+          if (!VerifyWideColumns(cmp_result.columns())) {
+            fprintf(stderr,
+                    "GetEntity error: inconsistent columns for key %s, "
+                    "entity %s\n",
+                    StringToHex(key).c_str(),
+                    WideColumnsToHex(cmp_result.columns()).c_str());
+            is_consistent = false;
+          }
+        }
+
+        if (is_consistent) {
+          for (size_t i = 1; i < rand_column_families.size(); ++i) {
+            assert(rand_column_families[i] >= 0);
+            assert(rand_column_families[i] <
+                   static_cast<int>(column_families_.size()));
+
+            PinnableWideColumns result;
+            s = db_->GetEntity(read_opts_copy,
+                               column_families_[rand_column_families[i]], key,
+                               &result);
+
+            if (!s.ok() && !s.IsNotFound()) {
+              break;
+            }
+
+            const bool found = s.ok();
+
+            assert(!column_family_names_.empty());
+            assert(i < column_family_names_.size());
+
+            if (!cmp_found && found) {
+              fprintf(stderr,
+                      "GetEntity returns different results for key %s: CF %s "
+                      "returns not found, CF %s returns entity %s\n",
+                      StringToHex(key).c_str(), column_family_names_[0].c_str(),
+                      column_family_names_[i].c_str(),
+                      WideColumnsToHex(result.columns()).c_str());
+              is_consistent = false;
+              break;
+            }
+
+            if (cmp_found && !found) {
+              fprintf(stderr,
+                      "GetEntity returns different results for key %s: CF %s "
+                      "returns entity %s, CF %s returns not found\n",
+                      StringToHex(key).c_str(), column_family_names_[0].c_str(),
+                      WideColumnsToHex(cmp_result.columns()).c_str(),
+                      column_family_names_[i].c_str());
+              is_consistent = false;
+              break;
+            }
+
+            if (found && result != cmp_result) {
+              fprintf(stderr,
+                      "GetEntity returns different results for key %s: CF %s "
+                      "returns entity %s, CF %s returns entity %s\n",
+                      StringToHex(key).c_str(), column_family_names_[0].c_str(),
+                      WideColumnsToHex(cmp_result.columns()).c_str(),
+                      column_family_names_[i].c_str(),
+                      WideColumnsToHex(result.columns()).c_str());
+              is_consistent = false;
+              break;
+            }
+          }
+        }
+      }
+    }
+
+    if (!is_consistent) {
+      fprintf(stderr, "TestGetEntity error: results are not consistent\n");
+      thread->stats.AddErrors(1);
+      // Fail fast to preserve the DB state.
+      thread->shared->SetVerificationFailure();
+    } else if (s.ok()) {
+      thread->stats.AddGets(1, 1);
+    } else if (s.IsNotFound()) {
+      thread->stats.AddGets(1, 0);
+    } else {
+      fprintf(stderr, "TestGetEntity error: %s\n", s.ToString().c_str());
+      thread->stats.AddErrors(1);
+    }
+  }
+
   Status TestPrefixScan(ThreadState* thread, const ReadOptions& readoptions,
                         const std::vector<int>& rand_column_families,
                         const std::vector<int64_t>& rand_keys) override {
@@ -290,12 +430,9 @@ class CfConsistencyStressTest : public StressTest {
          iter->Next()) {
       ++count;
 
-      const WideColumns expected_columns = GenerateExpectedWideColumns(
-          GetValueBase(iter->value()), iter->value());
-      if (iter->columns() != expected_columns) {
-        s = Status::Corruption(
-            "Value and columns inconsistent",
-            DebugString(iter->value(), iter->columns(), expected_columns));
+      if (!VerifyWideColumns(iter->value(), iter->columns())) {
+        s = Status::Corruption("Value and columns inconsistent",
+                               DebugString(iter->value(), iter->columns()));
         break;
       }
     }
@@ -372,12 +509,10 @@ class CfConsistencyStressTest : public StressTest {
         assert(iter);
 
         if (iter->Valid()) {
-          const WideColumns expected_columns = GenerateExpectedWideColumns(
-              GetValueBase(iter->value()), iter->value());
-          if (iter->columns() != expected_columns) {
-            statuses[i] = Status::Corruption(
-                "Value and columns inconsistent",
-                DebugString(iter->value(), iter->columns(), expected_columns));
+          if (!VerifyWideColumns(iter->value(), iter->columns())) {
+            statuses[i] =
+                Status::Corruption("Value and columns inconsistent",
+                                   DebugString(iter->value(), iter->columns()));
           } else {
             ++valid_cnt;
           }

db_stress_tool/db_stress_common.cc

@@ -278,6 +278,37 @@ WideColumns GenerateExpectedWideColumns(uint32_t value_base,
   return columns;
 }
 
+bool VerifyWideColumns(const Slice& value, const WideColumns& columns) {
+  if (value.size() < sizeof(uint32_t)) {
+    return false;
+  }
+
+  const uint32_t value_base = GetValueBase(value);
+
+  const WideColumns expected_columns =
+      GenerateExpectedWideColumns(value_base, value);
+
+  if (columns != expected_columns) {
+    return false;
+  }
+
+  return true;
+}
+
+bool VerifyWideColumns(const WideColumns& columns) {
+  if (columns.empty()) {
+    return false;
+  }
+
+  if (columns.front().name() != kDefaultWideColumnName) {
+    return false;
+  }
+
+  const Slice& value_of_default = columns.front().value();
+
+  return VerifyWideColumns(value_of_default, columns);
+}
+
 std::string GetNowNanos() {
   uint64_t t = db_stress_env->NowNanos();
   std::string ret;

db_stress_tool/db_stress_common.h

@@ -213,6 +213,7 @@ DECLARE_bool(compare_full_db_state_snapshot);
 DECLARE_uint64(snapshot_hold_ops);
 DECLARE_bool(long_running_snapshots);
 DECLARE_bool(use_multiget);
+DECLARE_bool(use_get_entity);
 DECLARE_int32(readpercent);
 DECLARE_int32(prefixpercent);
 DECLARE_int32(writepercent);
@@ -596,6 +597,24 @@ extern inline std::string StringToHex(const std::string& str) {
   return result;
 }
 
+inline std::string WideColumnsToHex(const WideColumns& columns) {
+  if (columns.empty()) {
+    return std::string();
+  }
+
+  std::ostringstream oss;
+
+  oss << std::hex;
+
+  auto it = columns.begin();
+  oss << *it;
+  for (++it; it != columns.end(); ++it) {
+    oss << ' ' << *it;
+  }
+
+  return oss.str();
+}
+
 // Unified output format for double parameters
 extern inline std::string FormatDoubleParam(double param) {
   return std::to_string(param);
@@ -626,6 +645,8 @@ extern uint32_t GetValueBase(Slice s);
 extern WideColumns GenerateWideColumns(uint32_t value_base, const Slice& slice);
 extern WideColumns GenerateExpectedWideColumns(uint32_t value_base,
                                                const Slice& slice);
+extern bool VerifyWideColumns(const Slice& value, const WideColumns& columns);
+extern bool VerifyWideColumns(const WideColumns& columns);
 
 extern StressTest* CreateCfConsistencyStressTest();
 extern StressTest* CreateBatchedOpsStressTest();

db_stress_tool/db_stress_gflags.cc

@@ -747,6 +747,8 @@ DEFINE_bool(long_running_snapshots, false,
 DEFINE_bool(use_multiget, false,
             "If set, use the batched MultiGet API for reads");
 
+DEFINE_bool(use_get_entity, false, "If set, use the GetEntity API for reads");
+
 static bool ValidateInt32Percent(const char* flagname, int32_t value) {
   if (value < 0 || value > 100) {
     fprintf(stderr, "Invalid value for --%s: %d, 0<= pct <=100 \n", flagname,

db_stress_tool/db_stress_test_base.cc

@@ -429,47 +429,27 @@ void StressTest::VerificationAbort(SharedState* shared, std::string msg, int cf,
 
 void StressTest::VerificationAbort(SharedState* shared, int cf, int64_t key,
                                    const Slice& value,
-                                   const WideColumns& columns,
-                                   const WideColumns& expected_columns) const {
+                                   const WideColumns& columns) const {
   assert(shared);
 
   auto key_str = Key(key);
 
   fprintf(stderr,
           "Verification failed for column family %d key %s (%" PRIi64
-          "): Value and columns inconsistent: %s\n",
+          "): Value and columns inconsistent: value: %s, columns: %s\n",
           cf, Slice(key_str).ToString(/* hex */ true).c_str(), key,
-          DebugString(value, columns, expected_columns).c_str());
+          value.ToString(/* hex */ true).c_str(),
+          WideColumnsToHex(columns).c_str());
 
   shared->SetVerificationFailure();
 }
 
 std::string StressTest::DebugString(const Slice& value,
-                                    const WideColumns& columns,
-                                    const WideColumns& expected_columns) {
+                                    const WideColumns& columns) {
   std::ostringstream oss;
 
-  oss << "value: " << value.ToString(/* hex */ true);
-
-  auto dump = [](const WideColumns& cols, std::ostream& os) {
-    if (cols.empty()) {
-      return;
-    }
-
-    os << std::hex;
-
-    auto it = cols.begin();
-    os << *it;
-    for (++it; it != cols.end(); ++it) {
-      os << ' ' << *it;
-    }
-  };
-
-  oss << ", columns: ";
-  dump(columns, oss);
-
-  oss << ", expected_columns: ";
-  dump(expected_columns, oss);
+  oss << "value: " << value.ToString(/* hex */ true)
+      << ", columns: " << WideColumnsToHex(columns);
 
   return oss.str();
 }
@@ -1004,7 +984,9 @@ void StressTest::OperateDb(ThreadState* thread) {
       if (prob_op >= 0 && prob_op < static_cast<int>(FLAGS_readpercent)) {
         assert(0 <= prob_op);
         // OPERATION read
-        if (FLAGS_use_multiget) {
+        if (FLAGS_use_get_entity) {
+          TestGetEntity(thread, read_opts, rand_column_families, rand_keys);
+        } else if (FLAGS_use_multiget) {
           // Leave room for one more iteration of the loop with a single key
           // batch. This is to ensure that each thread does exactly the same
           // number of ops
@@ -1491,12 +1473,12 @@ void StressTest::VerifyIterator(ThreadState* thread,
   }
 
   if (!*diverged && iter->Valid()) {
-    const WideColumns expected_columns =
-        GenerateExpectedWideColumns(GetValueBase(iter->value()), iter->value());
-    if (iter->columns() != expected_columns) {
-      fprintf(stderr, "Value and columns inconsistent for iterator: %s\n",
-              DebugString(iter->value(), iter->columns(), expected_columns)
-                  .c_str());
+    if (!VerifyWideColumns(iter->value(), iter->columns())) {
+      fprintf(stderr,
+              "Value and columns inconsistent for iterator: value: %s, "
+              "columns: %s\n",
+              iter->value().ToString(/* hex */ true).c_str(),
+              WideColumnsToHex(iter->columns()).c_str());
 
       *diverged = true;
     }
@@ -2402,6 +2384,8 @@ void StressTest::PrintEnv() const {
           FLAGS_subcompactions);
   fprintf(stdout, "Use MultiGet              : %s\n",
           FLAGS_use_multiget ? "true" : "false");
+  fprintf(stdout, "Use GetEntity             : %s\n",
+          FLAGS_use_get_entity ? "true" : "false");
 
   const char* memtablerep = "";
   switch (FLAGS_rep_factory) {

db_stress_tool/db_stress_test_base.h

@@ -94,6 +94,10 @@ class StressTest {
       const std::vector<int>& rand_column_families,
       const std::vector<int64_t>& rand_keys) = 0;
 
+  virtual void TestGetEntity(ThreadState* thread, const ReadOptions& read_opts,
+                             const std::vector<int>& rand_column_families,
+                             const std::vector<int64_t>& rand_keys) = 0;
+
   virtual Status TestPrefixScan(ThreadState* thread,
                                 const ReadOptions& read_opts,
                                 const std::vector<int>& rand_column_families,
@@ -221,11 +225,10 @@ class StressTest {
                          Slice value_from_expected) const;
 
   void VerificationAbort(SharedState* shared, int cf, int64_t key,
-                         const Slice& value, const WideColumns& columns,
-                         const WideColumns& expected_columns) const;
+                         const Slice& value, const WideColumns& columns) const;
 
-  static std::string DebugString(const Slice& value, const WideColumns& columns,
-                                 const WideColumns& expected_columns);
+  static std::string DebugString(const Slice& value,
+                                 const WideColumns& columns);
 
   void PrintEnv() const;
 

db_stress_tool/expected_state.cc

@@ -416,16 +416,7 @@ class ExpectedStateTraceRecordHandler : public TraceRecord::Handler,
                                 entity.ToString(/* hex */ true));
     }
 
-    if (columns.empty() || columns[0].name() != kDefaultWideColumnName) {
-      return Status::Corruption("Cannot find default column in entity",
-                                entity.ToString(/* hex */ true));
-    }
-
-    const Slice& value_of_default = columns[0].value();
-
-    const uint32_t value_base = GetValueBase(value_of_default);
-
-    if (columns != GenerateExpectedWideColumns(value_base, value_of_default)) {
+    if (!VerifyWideColumns(columns)) {
       return Status::Corruption("Wide columns in entity inconsistent",
                                 entity.ToString(/* hex */ true));
     }
@@ -435,6 +426,11 @@ class ExpectedStateTraceRecordHandler : public TraceRecord::Handler,
                                            column_family_id, key, columns);
     }
 
+    assert(!columns.empty());
+    assert(columns.front().name() == kDefaultWideColumnName);
+
+    const uint32_t value_base = GetValueBase(columns.front().value());
+
     state_->Put(column_family_id, static_cast<int64_t>(key_id), value_base,
                 false /* pending */);
 

db_stress_tool/multi_ops_txns_stress.cc

@@ -387,6 +387,12 @@ std::vector<Status> MultiOpsTxnsStressTest::TestMultiGet(
   return std::vector<Status>{Status::NotSupported()};
 }
 
+// Wide columns are currently not supported by transactions.
+void MultiOpsTxnsStressTest::TestGetEntity(
+    ThreadState* /* thread */, const ReadOptions& /* read_opts */,
+    const std::vector<int>& /* rand_column_families */,
+    const std::vector<int64_t>& /* rand_keys */) {}
+
 Status MultiOpsTxnsStressTest::TestPrefixScan(
     ThreadState* thread, const ReadOptions& read_opts,
     const std::vector<int>& rand_column_families,

db_stress_tool/multi_ops_txns_stress.h

@@ -210,6 +210,10 @@ class MultiOpsTxnsStressTest : public StressTest {
       const std::vector<int>& rand_column_families,
       const std::vector<int64_t>& rand_keys) override;
 
+  void TestGetEntity(ThreadState* thread, const ReadOptions& read_opts,
+                     const std::vector<int>& rand_column_families,
+                     const std::vector<int64_t>& rand_keys) override;
+
   Status TestPrefixScan(ThreadState* thread, const ReadOptions& read_opts,
                         const std::vector<int>& rand_column_families,
                         const std::vector<int64_t>& rand_keys) override;

db_stress_tool/no_batched_ops_stress.cc

@@ -101,12 +101,9 @@ class NonBatchedOpsStressTest : public StressTest {
             if (diff > 0) {
               s = Status::NotFound();
             } else if (diff == 0) {
-              const WideColumns expected_columns = GenerateExpectedWideColumns(
-                  GetValueBase(iter->value()), iter->value());
-              if (iter->columns() != expected_columns) {
+              if (!VerifyWideColumns(iter->value(), iter->columns())) {
                 VerificationAbort(shared, static_cast<int>(cf), i,
-                                  iter->value(), iter->columns(),
-                                  expected_columns);
+                                  iter->value(), iter->columns());
               }
 
               from_db = iter->value().ToString();
@@ -159,26 +156,24 @@ class NonBatchedOpsStressTest : public StressTest {
           }
 
           const std::string key = Key(i);
-          PinnableWideColumns columns;
+          PinnableWideColumns result;
 
           Status s =
-              db_->GetEntity(options, column_families_[cf], key, &columns);
+              db_->GetEntity(options, column_families_[cf], key, &result);
 
           std::string from_db;
 
           if (s.ok()) {
-            const WideColumns& columns_from_db = columns.columns();
+            const WideColumns& columns = result.columns();
 
-            if (!columns_from_db.empty() &&
-                columns_from_db[0].name() == kDefaultWideColumnName) {
-              from_db = columns_from_db[0].value().ToString();
+            if (!columns.empty() &&
+                columns.front().name() == kDefaultWideColumnName) {
+              from_db = columns.front().value().ToString();
             }
 
-            const WideColumns expected_columns =
-                GenerateExpectedWideColumns(GetValueBase(from_db), from_db);
-            if (columns_from_db != expected_columns) {
+            if (!VerifyWideColumns(columns)) {
               VerificationAbort(shared, static_cast<int>(cf), i, from_db,
-                                columns_from_db, expected_columns);
+                                columns);
             }
           }
 
@@ -256,18 +251,16 @@ class NonBatchedOpsStressTest : public StressTest {
             std::string from_db;
 
             if (statuses[j].ok()) {
-              const WideColumns& columns_from_db = results[j].columns();
+              const WideColumns& columns = results[j].columns();
 
-              if (!columns_from_db.empty() &&
-                  columns_from_db[0].name() == kDefaultWideColumnName) {
-                from_db = columns_from_db[0].value().ToString();
+              if (!columns.empty() &&
+                  columns.front().name() == kDefaultWideColumnName) {
+                from_db = columns.front().value().ToString();
               }
 
-              const WideColumns expected_columns =
-                  GenerateExpectedWideColumns(GetValueBase(from_db), from_db);
-              if (columns_from_db != expected_columns) {
+              if (!VerifyWideColumns(columns)) {
                 VerificationAbort(shared, static_cast<int>(cf), i, from_db,
-                                  columns_from_db, expected_columns);
+                                  columns);
               }
             }
 
@@ -756,6 +749,104 @@ class NonBatchedOpsStressTest : public StressTest {
     return statuses;
   }
 
+  void TestGetEntity(ThreadState* thread, const ReadOptions& read_opts,
+                     const std::vector<int>& rand_column_families,
+                     const std::vector<int64_t>& rand_keys) override {
+    if (fault_fs_guard) {
+      fault_fs_guard->EnableErrorInjection();
+      SharedState::ignore_read_error = false;
+    }
+
+    assert(thread);
+
+    SharedState* const shared = thread->shared;
+    assert(shared);
+
+    assert(!rand_column_families.empty());
+    assert(!rand_keys.empty());
+
+    std::unique_ptr<MutexLock> lock(new MutexLock(
+        shared->GetMutexForKey(rand_column_families[0], rand_keys[0])));
+
+    assert(rand_column_families[0] >= 0);
+    assert(rand_column_families[0] < static_cast<int>(column_families_.size()));
+
+    ColumnFamilyHandle* const cfh = column_families_[rand_column_families[0]];
+    assert(cfh);
+
+    const std::string key = Key(rand_keys[0]);
+
+    PinnableWideColumns from_db;
+
+    const Status s = db_->GetEntity(read_opts, cfh, key, &from_db);
+
+    int error_count = 0;
+
+    if (fault_fs_guard) {
+      error_count = fault_fs_guard->GetAndResetErrorCount();
+    }
+
+    if (s.ok()) {
+      if (fault_fs_guard) {
+        if (error_count && !SharedState::ignore_read_error) {
+          // Grab mutex so multiple threads don't try to print the
+          // stack trace at the same time
+          MutexLock l(shared->GetMutex());
+          fprintf(stderr, "Didn't get expected error from GetEntity\n");
+          fprintf(stderr, "Call stack that injected the fault\n");
+          fault_fs_guard->PrintFaultBacktrace();
+          std::terminate();
+        }
+      }
+
+      thread->stats.AddGets(1, 1);
+
+      if (!FLAGS_skip_verifydb) {
+        const WideColumns& columns = from_db.columns();
+
+        if (!VerifyWideColumns(columns)) {
+          shared->SetVerificationFailure();
+          fprintf(stderr,
+                  "error : inconsistent columns returned by GetEntity for key "
+                  "%s: %s\n",
+                  StringToHex(key).c_str(), WideColumnsToHex(columns).c_str());
+        } else if (shared->Get(rand_column_families[0], rand_keys[0]) ==
+                   SharedState::DELETION_SENTINEL) {
+          shared->SetVerificationFailure();
+          fprintf(
+              stderr,
+              "error : inconsistent values for key %s: GetEntity returns %s, "
+              "expected state does not have the key.\n",
+              StringToHex(key).c_str(), WideColumnsToHex(columns).c_str());
+        }
+      }
+    } else if (s.IsNotFound()) {
+      thread->stats.AddGets(1, 0);
+
+      if (!FLAGS_skip_verifydb) {
+        auto expected = shared->Get(rand_column_families[0], rand_keys[0]);
+        if (expected != SharedState::DELETION_SENTINEL &&
+            expected != SharedState::UNKNOWN_SENTINEL) {
+          shared->SetVerificationFailure();
+          fprintf(stderr,
+                  "error : inconsistent values for key %s: expected state has "
+                  "the key, GetEntity returns NotFound.\n",
+                  StringToHex(key).c_str());
+        }
+      }
+    } else {
+      if (error_count == 0) {
+        thread->stats.AddErrors(1);
+      } else {
+        thread->stats.AddVerifiedErrors(1);
+      }
+    }
+
+    if (fault_fs_guard) {
+      fault_fs_guard->DisableErrorInjection();
+    }
+  }
+
   Status TestPrefixScan(ThreadState* thread, const ReadOptions& read_opts,
                         const std::vector<int>& rand_column_families,
                         const std::vector<int64_t>& rand_keys) override {
@@ -810,12 +901,9 @@ class NonBatchedOpsStressTest : public StressTest {
         }
       }
 
-      const WideColumns expected_columns = GenerateExpectedWideColumns(
-          GetValueBase(iter->value()), iter->value());
-      if (iter->columns() != expected_columns) {
-        s = Status::Corruption(
-            "Value and columns inconsistent",
-            DebugString(iter->value(), iter->columns(), expected_columns));
+      if (!VerifyWideColumns(iter->value(), iter->columns())) {
+        s = Status::Corruption("Value and columns inconsistent",
+                               DebugString(iter->value(), iter->columns()));
         break;
       }
     }
@@ -1268,17 +1356,15 @@ class NonBatchedOpsStressTest : public StressTest {
       assert(iter);
       assert(iter->Valid());
 
-      const WideColumns expected_columns = GenerateExpectedWideColumns(
-          GetValueBase(iter->value()), iter->value());
-      if (iter->columns() != expected_columns) {
+      if (!VerifyWideColumns(iter->value(), iter->columns())) {
         shared->SetVerificationFailure();
 
         fprintf(stderr,
                 "Verification failed for key %s: "
-                "Value and columns inconsistent: %s\n",
+                "Value and columns inconsistent: value: %s, columns: %s\n",
                 Slice(iter->key()).ToString(/* hex */ true).c_str(),
-                DebugString(iter->value(), iter->columns(), expected_columns)
-                    .c_str());
+                iter->value().ToString(/* hex */ true).c_str(),
+                WideColumnsToHex(iter->columns()).c_str());
         fprintf(stderr, "Column family: %s, op_logs: %s\n",
                 cfh->GetName().c_str(), op_logs.c_str());
 

tools/db_crashtest.py

@@ -136,6 +136,7 @@ default_params = {
     "format_version": lambda: random.choice([2, 3, 4, 5, 5]),
     "index_block_restart_interval": lambda: random.choice(range(1, 16)),
     "use_multiget": lambda: random.randint(0, 1),
+    "use_get_entity": lambda: random.choice([0] * 7 + [1]),
     "periodic_compaction_seconds": lambda: random.choice([0, 0, 1, 2, 10, 100, 1000]),
     # 0 = never (used by some), 10 = often (for threading bugs), 600 = default
     "stats_dump_period_sec": lambda: random.choice([0, 10, 600]),