diff --git a/db_stress_tool/cf_consistency_stress.cc b/db_stress_tool/cf_consistency_stress.cc
index ac9c014f7..c6ba5d699 100644
--- a/db_stress_tool/cf_consistency_stress.cc
+++ b/db_stress_tool/cf_consistency_stress.cc
@@ -304,10 +304,34 @@ class CfConsistencyStressTest : public StressTest {
     opt_copy.env = db_stress_env->target();
     DestroyDB(checkpoint_dir, opt_copy);
 
+    if (db_stress_env->FileExists(checkpoint_dir).ok()) {
+      // If the directory might still exist, try to delete the files one by one.
+      // Likely a trash file is still there.
+      Status my_s = test::DestroyDir(db_stress_env, checkpoint_dir);
+      if (!my_s.ok()) {
+        fprintf(stderr, "Fail to destory directory before checkpoint: %s",
+                my_s.ToString().c_str());
+      }
+    }
+
     Checkpoint* checkpoint = nullptr;
     Status s = Checkpoint::Create(db_, &checkpoint);
     if (s.ok()) {
       s = checkpoint->CreateCheckpoint(checkpoint_dir);
+      if (!s.ok()) {
+        fprintf(stderr, "Fail to create checkpoint to %s\n",
+                checkpoint_dir.c_str());
+        std::vector<std::string> files;
+        Status my_s = db_stress_env->GetChildren(checkpoint_dir, &files);
+        if (my_s.ok()) {
+          for (const auto& f : files) {
+            fprintf(stderr, " %s\n", f.c_str());
+          }
+        } else {
+          fprintf(stderr, "Fail to get files under the directory to %s\n",
+                  my_s.ToString().c_str());
+        }
+      }
     }
     std::vector<ColumnFamilyHandle*> cf_handles;
     DB* checkpoint_db = nullptr;
@@ -337,10 +361,12 @@ class CfConsistencyStressTest : public StressTest {
       delete checkpoint_db;
       checkpoint_db = nullptr;
     }
-    DestroyDB(checkpoint_dir, opt_copy);
+
     if (!s.ok()) {
       fprintf(stderr, "A checkpoint operation failed with: %s\n",
               s.ToString().c_str());
+    } else {
+      DestroyDB(checkpoint_dir, opt_copy);
     }
     return s;
   }
diff --git a/test_util/testutil.cc b/test_util/testutil.cc
index f981fa7ec..0d95e3feb 100644
--- a/test_util/testutil.cc
+++ b/test_util/testutil.cc
@@ -476,13 +476,24 @@ Status DestroyDir(Env* env, const std::string& dir) {
         }
       }
       if (!s.ok()) {
-        break;
+        // IsDirectory, etc. might not report NotFound
+        if (s.IsNotFound() || env->FileExists(path).IsNotFound()) {
+          // Allow files to be deleted externally
+          s = Status::OK();
+        } else {
+          break;
+        }
       }
     }
   }
 
   if (s.ok()) {
     s = env->DeleteDir(dir);
+    // DeleteDir might or might not report NotFound
+    if (!s.ok() && (s.IsNotFound() || env->FileExists(dir).IsNotFound())) {
+      // Allow to be deleted externally
+      s = Status::OK();
+    }
   }
   return s;
 }