From 90fd6b0cc865d0f1f722b6f780a5561253df7687 Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Wed, 8 Jul 2020 13:03:17 -0700 Subject: [PATCH] cf_consistency_stress (crash_test_with_atomic_flush) checkpoint clean (#7103) Summary: Delicious copy-pasta from https://github.com/facebook/rocksdb/issues/7039 Also fixing DestroyDir to allow files to go missing while it is operating. This seems to fix failures I got with test plan reproducer. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7103 Test Plan: make blackbox_crash_test_with_atomic_flush for a while with checkpoint_one_in=100 Reviewed By: siying Differential Revision: D22435315 Pulled By: pdillinger fbshipit-source-id: 0ec0538402493887aeda43ecc03f32979cb84ced --- db_stress_tool/cf_consistency_stress.cc | 28 ++++++++++++++++++++++++- test_util/testutil.cc | 13 +++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/db_stress_tool/cf_consistency_stress.cc b/db_stress_tool/cf_consistency_stress.cc index ac9c014f7..c6ba5d699 100644 --- a/db_stress_tool/cf_consistency_stress.cc +++ b/db_stress_tool/cf_consistency_stress.cc @@ -304,10 +304,34 @@ class CfConsistencyStressTest : public StressTest { opt_copy.env = db_stress_env->target(); DestroyDB(checkpoint_dir, opt_copy); + if (db_stress_env->FileExists(checkpoint_dir).ok()) { + // If the directory might still exist, try to delete the files one by one. + // Likely a trash file is still there. + Status my_s = test::DestroyDir(db_stress_env, checkpoint_dir); + if (!my_s.ok()) { + fprintf(stderr, "Fail to destory directory before checkpoint: %s", + my_s.ToString().c_str()); + } + } + Checkpoint* checkpoint = nullptr; Status s = Checkpoint::Create(db_, &checkpoint); if (s.ok()) { s = checkpoint->CreateCheckpoint(checkpoint_dir); + if (!s.ok()) { + fprintf(stderr, "Fail to create checkpoint to %s\n", + checkpoint_dir.c_str()); + std::vector files; + Status my_s = db_stress_env->GetChildren(checkpoint_dir, &files); + if (my_s.ok()) { + for (const auto& f : files) { + fprintf(stderr, " %s\n", f.c_str()); + } + } else { + fprintf(stderr, "Fail to get files under the directory to %s\n", + my_s.ToString().c_str()); + } + } } std::vector cf_handles; DB* checkpoint_db = nullptr; @@ -337,10 +361,12 @@ class CfConsistencyStressTest : public StressTest { delete checkpoint_db; checkpoint_db = nullptr; } - DestroyDB(checkpoint_dir, opt_copy); + if (!s.ok()) { fprintf(stderr, "A checkpoint operation failed with: %s\n", s.ToString().c_str()); + } else { + DestroyDB(checkpoint_dir, opt_copy); } return s; } diff --git a/test_util/testutil.cc b/test_util/testutil.cc index f981fa7ec..0d95e3feb 100644 --- a/test_util/testutil.cc +++ b/test_util/testutil.cc @@ -476,13 +476,24 @@ Status DestroyDir(Env* env, const std::string& dir) { } } if (!s.ok()) { - break; + // IsDirectory, etc. might not report NotFound + if (s.IsNotFound() || env->FileExists(path).IsNotFound()) { + // Allow files to be deleted externally + s = Status::OK(); + } else { + break; + } } } } if (s.ok()) { s = env->DeleteDir(dir); + // DeleteDir might or might not report NotFound + if (!s.ok() && (s.IsNotFound() || env->FileExists(dir).IsNotFound())) { + // Allow to be deleted externally + s = Status::OK(); + } } return s; }