Support injecting read errors for RandomAccessFile when using FaultInjectionTestEnv (#7447)

Summary:
The patch adds support for injecting errors when reading from `RandomAccessFile`
using `FaultInjectionTestEnv`. (This functionality was curiously missing
w/r/t `RandomAccessFile`, even though it was implemented for `RandomRWFile`.)
The patch also fixes up a test case in `blob_db_test` which uses `FaultInjectionTestEnv`
but has so far relied on reads from `RandomAccessFile`s succeeding even after
deactivating the filesystem.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7447

Test Plan: `make check`

Reviewed By: zhichao-cao

Differential Revision: D23971740

Pulled By: ltamasi

fbshipit-source-id: 8492736cb64b1ee138c658822535f3ff4fe560c6

utilities/blob_db/blob_db_test.cc

@@ -1469,12 +1469,14 @@ TEST_F(BlobDBTest, UserCompactionFilter_BlobIOError) {
     auto s = blob_db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
     ASSERT_TRUE(s.IsIOError());
 
+    // Reactivate file system to allow test to verify and close DB.
+    fault_injection_env_->SetFilesystemActive(true);
+    SyncPoint::GetInstance()->DisableProcessing();
+    SyncPoint::GetInstance()->ClearAllCallBacks();
+
     // Verify full data set after compaction failure
     VerifyDB(data);
 
-    // Reactivate file system to allow test to close DB.
-    fault_injection_env_->SetFilesystemActive(true);
-    SyncPoint::GetInstance()->ClearAllCallBacks();
     delete options.compaction_filter;
     Destroy();
   }

utilities/fault_injection_env.cc

@@ -107,6 +107,51 @@ Status TestDirectory::Fsync() {
   return dir_->Fsync();
 }
 
+TestRandomAccessFile::TestRandomAccessFile(
+    std::unique_ptr<RandomAccessFile>&& target, FaultInjectionTestEnv* env)
+    : target_(std::move(target)), env_(env) {
+  assert(target_);
+  assert(env_);
+}
+
+Status TestRandomAccessFile::Read(uint64_t offset, size_t n, Slice* result,
+                                  char* scratch) const {
+  assert(env_);
+  if (!env_->IsFilesystemActive()) {
+    return env_->GetError();
+  }
+
+  assert(target_);
+  return target_->Read(offset, n, result, scratch);
+}
+
+Status TestRandomAccessFile::Prefetch(uint64_t offset, size_t n) {
+  assert(env_);
+  if (!env_->IsFilesystemActive()) {
+    return env_->GetError();
+  }
+
+  assert(target_);
+  return target_->Prefetch(offset, n);
+}
+
+Status TestRandomAccessFile::MultiRead(ReadRequest* reqs, size_t num_reqs) {
+  assert(env_);
+  if (!env_->IsFilesystemActive()) {
+    const Status s = env_->GetError();
+
+    assert(reqs);
+    for (size_t i = 0; i < num_reqs; ++i) {
+      reqs[i].status = s;
+    }
+
+    return s;
+  }
+
+  assert(target_);
+  return target_->MultiRead(reqs, num_reqs);
+}
+
 TestWritableFile::TestWritableFile(const std::string& fname,
                                    std::unique_ptr<WritableFile>&& f,
                                    FaultInjectionTestEnv* env)
@@ -299,7 +344,17 @@ Status FaultInjectionTestEnv::NewRandomAccessFile(
   if (!IsFilesystemActive()) {
     return GetError();
   }
-  return target()->NewRandomAccessFile(fname, result, soptions);
+
+  assert(target());
+  const Status s = target()->NewRandomAccessFile(fname, result, soptions);
+  if (!s.ok()) {
+    return s;
+  }
+
+  assert(result);
+  result->reset(new TestRandomAccessFile(std::move(*result), this));
+
+  return Status::OK();
 }
 
 Status FaultInjectionTestEnv::DeleteFile(const std::string& f) {

utilities/fault_injection_env.h

@@ -47,6 +47,23 @@ struct FileState {
   Status DropRandomUnsyncedData(Env* env, Random* rand) const;
 };
 
+class TestRandomAccessFile : public RandomAccessFile {
+ public:
+  TestRandomAccessFile(std::unique_ptr<RandomAccessFile>&& target,
+                       FaultInjectionTestEnv* env);
+
+  Status Read(uint64_t offset, size_t n, Slice* result,
+              char* scratch) const override;
+
+  Status Prefetch(uint64_t offset, size_t n) override;
+
+  Status MultiRead(ReadRequest* reqs, size_t num_reqs) override;
+
+ private:
+  std::unique_ptr<RandomAccessFile> target_;
+  FaultInjectionTestEnv* env_;
+};
+
 // A wrapper around WritableFileWriter* file
 // is written to or sync'ed.
 class TestWritableFile : public WritableFile {