cf_consistency_stress (crash_test_with_atomic_flush) checkpoint clean (#7103)

Summary:
Delicious copy-pasta from https://github.com/facebook/rocksdb/issues/7039

Also fixing DestroyDir to allow files to go missing while it is operating. This seems to fix failures I got with test plan reproducer.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7103

Test Plan:
make blackbox_crash_test_with_atomic_flush for a while with
checkpoint_one_in=100

Reviewed By: siying

Differential Revision: D22435315

Pulled By: pdillinger

fbshipit-source-id: 0ec0538402493887aeda43ecc03f32979cb84ced

db_stress_tool/cf_consistency_stress.cc

@@ -304,10 +304,34 @@ class CfConsistencyStressTest : public StressTest {
     opt_copy.env = db_stress_env->target();
     DestroyDB(checkpoint_dir, opt_copy);
 
+    if (db_stress_env->FileExists(checkpoint_dir).ok()) {
+      // If the directory might still exist, try to delete the files one by one.
+      // Likely a trash file is still there.
+      Status my_s = test::DestroyDir(db_stress_env, checkpoint_dir);
+      if (!my_s.ok()) {
+        fprintf(stderr, "Fail to destory directory before checkpoint: %s",
+                my_s.ToString().c_str());
+      }
+    }
+
     Checkpoint* checkpoint = nullptr;
     Status s = Checkpoint::Create(db_, &checkpoint);
     if (s.ok()) {
       s = checkpoint->CreateCheckpoint(checkpoint_dir);
+      if (!s.ok()) {
+        fprintf(stderr, "Fail to create checkpoint to %s\n",
+                checkpoint_dir.c_str());
+        std::vector<std::string> files;
+        Status my_s = db_stress_env->GetChildren(checkpoint_dir, &files);
+        if (my_s.ok()) {
+          for (const auto& f : files) {
+            fprintf(stderr, " %s\n", f.c_str());
+          }
+        } else {
+          fprintf(stderr, "Fail to get files under the directory to %s\n",
+                  my_s.ToString().c_str());
+        }
+      }
     }
     std::vector<ColumnFamilyHandle*> cf_handles;
     DB* checkpoint_db = nullptr;
@@ -337,10 +361,12 @@ class CfConsistencyStressTest : public StressTest {
       delete checkpoint_db;
       checkpoint_db = nullptr;
     }
-    DestroyDB(checkpoint_dir, opt_copy);
+
     if (!s.ok()) {
       fprintf(stderr, "A checkpoint operation failed with: %s\n",
               s.ToString().c_str());
+    } else {
+      DestroyDB(checkpoint_dir, opt_copy);
     }
     return s;
   }

test_util/testutil.cc

@@ -476,13 +476,24 @@ Status DestroyDir(Env* env, const std::string& dir) {
         }
       }
       if (!s.ok()) {
+        // IsDirectory, etc. might not report NotFound
+        if (s.IsNotFound() || env->FileExists(path).IsNotFound()) {
+          // Allow files to be deleted externally
+          s = Status::OK();
+        } else {
           break;
         }
       }
     }
+  }
 
   if (s.ok()) {
     s = env->DeleteDir(dir);
+    // DeleteDir might or might not report NotFound
+    if (!s.ok() && (s.IsNotFound() || env->FileExists(dir).IsNotFound())) {
+      // Allow to be deleted externally
+      s = Status::OK();
+    }
   }
   return s;
 }