From 485fd9d9db4e444e5dca89f95700ec804ba010b3 Mon Sep 17 00:00:00 2001
From: Zhichao Cao <zhichao@fb.com>
Date: Sat, 19 Sep 2020 17:56:39 -0700
Subject: [PATCH] fix the flaky test failure (#7415)

Summary:
Fix the flaky test failure in error_handler_fs_test. Add the sync point, solve the dependency.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7415

Test Plan: make asan_check, ~/gtest-parallel/gtest-parallel -r 100 ./error_handler_fs_test

Reviewed By: siying

Differential Revision: D23804330

Pulled By: zhichao-cao

fbshipit-source-id: 5175108651f7652e47e15978f2a9c1669ef59d80
---
 db/builder.cc               |  2 +
 db/error_handler_fs_test.cc | 89 +++++++++++++++++++++++++++++--------
 2 files changed, 72 insertions(+), 19 deletions(-)

diff --git a/db/builder.cc b/db/builder.cc
index 9e530bfca..551016077 100644
--- a/db/builder.cc
+++ b/db/builder.cc
@@ -245,10 +245,12 @@ Status BuildTable(
     delete builder;
 
     // Finish and check for file errors
+    TEST_SYNC_POINT("BuildTable:BeforeSyncTable");
     if (s.ok() && !empty) {
       StopWatch sw(env, ioptions.statistics, TABLE_SYNC_MICROS);
       *io_status = file_writer->Sync(ioptions.use_fsync);
     }
+    TEST_SYNC_POINT("BuildTable:BeforeCloseTableFile");
     if (s.ok() && io_status->ok() && !empty) {
       *io_status = file_writer->Close();
     }
diff --git a/db/error_handler_fs_test.cc b/db/error_handler_fs_test.cc
index 24291cca1..4336043af 100644
--- a/db/error_handler_fs_test.cc
+++ b/db/error_handler_fs_test.cc
@@ -247,7 +247,7 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableError) {
   Destroy(options);
 }
 
-TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError) {
+TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError1) {
   std::shared_ptr<FaultInjectionTestFS> fault_fs(
       new FaultInjectionTestFS(FileSystem::Default()));
   std::unique_ptr<Env> fault_fs_env(NewCompositeEnv(fault_fs));
@@ -289,47 +289,98 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError) {
   ASSERT_OK(s);
   ASSERT_EQ("val3", Get(Key(3)));
 
-  Put(Key(4), "val4", wo);
+  Destroy(options);
+}
+
+TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError2) {
+  std::shared_ptr<FaultInjectionTestFS> fault_fs(
+      new FaultInjectionTestFS(FileSystem::Default()));
+  std::unique_ptr<Env> fault_fs_env(NewCompositeEnv(fault_fs));
+  std::shared_ptr<ErrorHandlerFSListener> listener(
+      new ErrorHandlerFSListener());
+  Options options = GetDefaultOptions();
+  options.env = fault_fs_env.get();
+  options.create_if_missing = true;
+  options.listeners.emplace_back(listener);
+  options.max_bgerror_resume_count = 0;
+  Status s;
+
+  listener->EnableAutoRecovery(false);
+  DestroyAndReopen(options);
+
+  IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
+  error_msg.SetRetryable(true);
+
+  WriteOptions wo = WriteOptions();
+  wo.disableWAL = true;
+
+  Put(Key(1), "val1", wo);
   SyncPoint::GetInstance()->SetCallBack(
-      "BuildTable:BeforeCloseTableFile",
+      "BuildTable:BeforeSyncTable",
       [&](void*) { fault_fs->SetFilesystemActive(false, error_msg); });
   SyncPoint::GetInstance()->EnableProcessing();
   s = Flush();
-  Put(Key(5), "val5", wo);
+  Put(Key(2), "val2", wo);
   ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
-  ASSERT_EQ("val5", Get(Key(5)));
+  ASSERT_EQ("val2", Get(Key(2)));
   SyncPoint::GetInstance()->DisableProcessing();
   fault_fs->SetFilesystemActive(true);
   s = dbfull()->Resume();
   ASSERT_EQ(s, Status::OK());
-  ASSERT_EQ("val4", Get(Key(4)));
-  ASSERT_EQ("val5", Get(Key(5)));
-  Put(Key(6), "val6", wo);
-  ASSERT_EQ("val6", Get(Key(6)));
+  ASSERT_EQ("val1", Get(Key(1)));
+  ASSERT_EQ("val2", Get(Key(2)));
+  Put(Key(3), "val3", wo);
+  ASSERT_EQ("val3", Get(Key(3)));
   s = Flush();
   ASSERT_OK(s);
-  ASSERT_EQ("val6", Get(Key(6)));
+  ASSERT_EQ("val3", Get(Key(3)));
 
-  Put(Key(7), "val7", wo);
+  Destroy(options);
+}
+
+TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError3) {
+  std::shared_ptr<FaultInjectionTestFS> fault_fs(
+      new FaultInjectionTestFS(FileSystem::Default()));
+  std::unique_ptr<Env> fault_fs_env(NewCompositeEnv(fault_fs));
+  std::shared_ptr<ErrorHandlerFSListener> listener(
+      new ErrorHandlerFSListener());
+  Options options = GetDefaultOptions();
+  options.env = fault_fs_env.get();
+  options.create_if_missing = true;
+  options.listeners.emplace_back(listener);
+  options.max_bgerror_resume_count = 0;
+  Status s;
+
+  listener->EnableAutoRecovery(false);
+  DestroyAndReopen(options);
+
+  IOStatus error_msg = IOStatus::IOError("Retryable IO Error");
+  error_msg.SetRetryable(true);
+
+  WriteOptions wo = WriteOptions();
+  wo.disableWAL = true;
+
+  Put(Key(1), "val1", wo);
   SyncPoint::GetInstance()->SetCallBack(
-      "BuildTable:BeforeSyncTable",
+      "BuildTable:BeforeCloseTableFile",
       [&](void*) { fault_fs->SetFilesystemActive(false, error_msg); });
   SyncPoint::GetInstance()->EnableProcessing();
   s = Flush();
-  Put(Key(8), "val8", wo);
+  Put(Key(2), "val2", wo);
   ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kSoftError);
-  ASSERT_EQ("val8", Get(Key(8)));
+  ASSERT_EQ("val2", Get(Key(2)));
   SyncPoint::GetInstance()->DisableProcessing();
   fault_fs->SetFilesystemActive(true);
   s = dbfull()->Resume();
   ASSERT_EQ(s, Status::OK());
-  ASSERT_EQ("val7", Get(Key(7)));
-  ASSERT_EQ("val8", Get(Key(8)));
-  Put(Key(9), "val9", wo);
-  ASSERT_EQ("val9", Get(Key(9)));
+  ASSERT_EQ("val1", Get(Key(1)));
+  ASSERT_EQ("val2", Get(Key(2)));
+  Put(Key(3), "val3", wo);
+  ASSERT_EQ("val3", Get(Key(3)));
   s = Flush();
   ASSERT_OK(s);
-  ASSERT_EQ("val9", Get(Key(9)));
+  ASSERT_EQ("val3", Get(Key(3)));
+
   Destroy(options);
 }