Implement TestFSRandomAccessFile::MultiRead() (#8925)

Summary:
Right now, the failure injection test for MultiGet() is not sufficient. Improve it with TestFSRandomAccessFile::MultiRead() injecting failures.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/8925

Test Plan: Run crash test locally for a while.

Reviewed By: anand1976

Differential Revision: D31000529

fbshipit-source-id: 439c7e02cf7440ac5af82deb609e202abdca3e1f

utilities/fault_injection_fs.cc

@@ -330,8 +330,9 @@ TestFSRandomAccessFile::TestFSRandomAccessFile(const std::string& /*fname*/,
 }
 
 IOStatus TestFSRandomAccessFile::Read(uint64_t offset, size_t n,
-                                  const IOOptions& options, Slice* result,
+                                      const IOOptions& options, Slice* result,
-                                  char* scratch, IODebugContext* dbg) const {
+                                      char* scratch,
+                                      IODebugContext* dbg) const {
   if (!fs_->IsFilesystemActive()) {
     return fs_->GetError();
   }
@@ -347,6 +348,33 @@ IOStatus TestFSRandomAccessFile::Read(uint64_t offset, size_t n,
   return s;
 }
 
+IOStatus TestFSRandomAccessFile::MultiRead(FSReadRequest* reqs, size_t num_reqs,
+                                           const IOOptions& options,
+                                           IODebugContext* dbg) {
+  if (!fs_->IsFilesystemActive()) {
+    return fs_->GetError();
+  }
+  IOStatus s = target_->MultiRead(reqs, num_reqs, options, dbg);
+  for (size_t i = 0; i < num_reqs; i++) {
+    if (!reqs[i].status.ok()) {
+      // Already seeing an error.
+      break;
+    }
+    reqs[i].status = fs_->InjectThreadSpecificReadError(
+        FaultInjectionTestFS::ErrorOperation::kRead, &reqs[i].result,
+        use_direct_io(), reqs[i].scratch);
+  }
+  if (s.ok()) {
+    s = fs_->InjectThreadSpecificReadError(
+        FaultInjectionTestFS::ErrorOperation::kRead, nullptr, use_direct_io(),
+        nullptr);
+  }
+  if (s.ok() && fs_->ShouldInjectRandomReadError()) {
+    return IOStatus::IOError("Injected read error");
+  }
+  return s;
+}
+
 size_t TestFSRandomAccessFile::GetUniqueId(char* id, size_t max_size) const {
   if (fs_->ShouldFailGetUniqueId()) {
     return 0;

utilities/fault_injection_fs.h

@@ -138,6 +138,8 @@ class TestFSRandomAccessFile : public FSRandomAccessFile {
   IOStatus Read(uint64_t offset, size_t n, const IOOptions& options,
                 Slice* result, char* scratch,
                 IODebugContext* dbg) const override;
+  IOStatus MultiRead(FSReadRequest* reqs, size_t num_reqs,
+                     const IOOptions& options, IODebugContext* dbg) override;
   size_t GetRequiredBufferAlignment() const override {
     return target_->GetRequiredBufferAlignment();
   }