diff --git a/db_stress_tool/db_stress_common.cc b/db_stress_tool/db_stress_common.cc index 9798ccca3..1b989de3a 100644 --- a/db_stress_tool/db_stress_common.cc +++ b/db_stress_tool/db_stress_common.cc @@ -346,5 +346,76 @@ std::shared_ptr GetFileChecksumImpl( return std::make_shared(internal_name); } +Status DeleteFilesInDirectory(const std::string& dirname) { + std::vector filenames; + Status s = Env::Default()->GetChildren(dirname, &filenames); + for (size_t i = 0; s.ok() && i < filenames.size(); ++i) { + s = Env::Default()->DeleteFile(dirname + "/" + filenames[i]); + } + return s; +} + +Status SaveFilesInDirectory(const std::string& src_dirname, + const std::string& dst_dirname) { + std::vector filenames; + Status s = Env::Default()->GetChildren(src_dirname, &filenames); + for (size_t i = 0; s.ok() && i < filenames.size(); ++i) { + bool is_dir = false; + s = Env::Default()->IsDirectory(src_dirname + "/" + filenames[i], &is_dir); + if (s.ok()) { + if (is_dir) { + continue; + } + s = Env::Default()->LinkFile(src_dirname + "/" + filenames[i], + dst_dirname + "/" + filenames[i]); + } + } + return s; +} + +Status InitUnverifiedSubdir(const std::string& dirname) { + Status s = Env::Default()->FileExists(dirname); + if (s.IsNotFound()) { + return Status::OK(); + } + + const std::string kUnverifiedDirname = dirname + "/unverified"; + if (s.ok()) { + s = Env::Default()->CreateDirIfMissing(kUnverifiedDirname); + } + if (s.ok()) { + // It might already exist with some stale contents. Delete any such + // contents. + s = DeleteFilesInDirectory(kUnverifiedDirname); + } + if (s.ok()) { + s = SaveFilesInDirectory(dirname, kUnverifiedDirname); + } + return s; +} + +Status DestroyUnverifiedSubdir(const std::string& dirname) { + Status s = Env::Default()->FileExists(dirname); + if (s.IsNotFound()) { + return Status::OK(); + } + + const std::string kUnverifiedDirname = dirname + "/unverified"; + if (s.ok()) { + s = Env::Default()->FileExists(kUnverifiedDirname); + } + if (s.IsNotFound()) { + return Status::OK(); + } + + if (s.ok()) { + s = DeleteFilesInDirectory(kUnverifiedDirname); + } + if (s.ok()) { + s = Env::Default()->DeleteDir(kUnverifiedDirname); + } + return s; +} + } // namespace ROCKSDB_NAMESPACE #endif // GFLAGS diff --git a/db_stress_tool/db_stress_common.h b/db_stress_tool/db_stress_common.h index 17a2072dd..9f8d78960 100644 --- a/db_stress_tool/db_stress_common.h +++ b/db_stress_tool/db_stress_common.h @@ -313,6 +313,7 @@ DECLARE_bool(enable_tiered_storage); // set last_level_temperature DECLARE_int64(preclude_last_level_data_seconds); DECLARE_int32(verify_iterator_with_expected_state_one_in); +DECLARE_bool(preserve_unverified_changes); DECLARE_uint64(readahead_size); DECLARE_uint64(initial_auto_readahead_size); @@ -631,5 +632,11 @@ extern std::string GetNowNanos(); std::shared_ptr GetFileChecksumImpl( const std::string& name); + +Status DeleteFilesInDirectory(const std::string& dirname); +Status SaveFilesInDirectory(const std::string& src_dirname, + const std::string& dst_dirname); +Status DestroyUnverifiedSubdir(const std::string& dirname); +Status InitUnverifiedSubdir(const std::string& dirname); } // namespace ROCKSDB_NAMESPACE #endif // GFLAGS diff --git a/db_stress_tool/db_stress_driver.cc b/db_stress_tool/db_stress_driver.cc index 009168ae3..b4856b347 100644 --- a/db_stress_tool/db_stress_driver.cc +++ b/db_stress_tool/db_stress_driver.cc @@ -58,7 +58,21 @@ void ThreadBody(void* v) { bool RunStressTest(StressTest* stress) { SystemClock* clock = db_stress_env->GetSystemClock().get(); + SharedState shared(db_stress_env, stress); + + if (shared.ShouldVerifyAtBeginning() && FLAGS_preserve_unverified_changes) { + Status s = InitUnverifiedSubdir(FLAGS_db); + if (s.ok() && !FLAGS_expected_values_dir.empty()) { + s = InitUnverifiedSubdir(FLAGS_expected_values_dir); + } + if (!s.ok()) { + fprintf(stderr, "Failed to setup unverified state dir: %s\n", + s.ToString().c_str()); + exit(1); + } + } + stress->InitDb(&shared); stress->FinishInitDb(&shared); @@ -115,6 +129,15 @@ bool RunStressTest(StressTest* stress) { fprintf(stderr, "Crash-recovery verification failed :(\n"); } else { fprintf(stdout, "Crash-recovery verification passed :)\n"); + Status s = DestroyUnverifiedSubdir(FLAGS_db); + if (s.ok() && !FLAGS_expected_values_dir.empty()) { + s = DestroyUnverifiedSubdir(FLAGS_expected_values_dir); + } + if (!s.ok()) { + fprintf(stderr, "Failed to cleanup unverified state dir: %s\n", + s.ToString().c_str()); + exit(1); + } } } diff --git a/db_stress_tool/db_stress_gflags.cc b/db_stress_tool/db_stress_gflags.cc index 654e022b6..3639895e4 100644 --- a/db_stress_tool/db_stress_gflags.cc +++ b/db_stress_tool/db_stress_gflags.cc @@ -1044,4 +1044,12 @@ DEFINE_uint64( num_file_reads_for_auto_readahead, 0, "Num of sequential reads to enable auto prefetching during Iteration"); +DEFINE_bool( + preserve_unverified_changes, false, + "DB files of the current run will all be preserved in `FLAGS_db`. DB files " + "from the last run will be preserved in `FLAGS_db/unverified` until the " + "first verification succeeds. Expected state files from the last run will " + "be preserved similarly under `FLAGS_expected_values_dir/unverified` when " + "`--expected_values_dir` is nonempty."); + #endif // GFLAGS diff --git a/db_stress_tool/db_stress_test_base.cc b/db_stress_tool/db_stress_test_base.cc index e2e0c05c1..93ffee7ea 100644 --- a/db_stress_tool/db_stress_test_base.cc +++ b/db_stress_tool/db_stress_test_base.cc @@ -2695,6 +2695,21 @@ void StressTest::Open(SharedState* shared) { exit(1); #endif } + + if (FLAGS_preserve_unverified_changes) { + // Up until now, no live file should have become obsolete due to these + // options. After `DisableFileDeletions()` we can reenable auto compactions + // since, even if live files become obsolete, they won't be deleted. + assert(options_.avoid_flush_during_recovery); + assert(options_.disable_auto_compactions); + if (s.ok()) { + s = db_->DisableFileDeletions(); + } + if (s.ok()) { + s = db_->EnableAutoCompaction(column_families_); + } + } + if (!s.ok()) { fprintf(stderr, "open error: %s\n", s.ToString().c_str()); exit(1); @@ -3209,6 +3224,20 @@ void InitializeOptionsGeneral( } } + if (FLAGS_preserve_unverified_changes) { + if (!options.avoid_flush_during_recovery) { + fprintf(stderr, + "WARNING: flipping `avoid_flush_during_recovery` to true for " + "`preserve_unverified_changes` to keep all files\n"); + options.avoid_flush_during_recovery = true; + } + // Together with `avoid_flush_during_recovery == true`, this will prevent + // live files from becoming obsolete and deleted between `DB::Open()` and + // `DisableFileDeletions()` due to flush or compaction. We do not need to + // warn the user since we will reenable compaction soon. + options.disable_auto_compactions = true; + } + options.table_properties_collector_factories.emplace_back( std::make_shared()); } diff --git a/db_stress_tool/db_stress_tool.cc b/db_stress_tool/db_stress_tool.cc index 3e8490ccc..8aaab31c0 100644 --- a/db_stress_tool/db_stress_tool.cc +++ b/db_stress_tool/db_stress_tool.cc @@ -280,6 +280,12 @@ int db_stress_tool(int argc, char** argv) { } } + if (FLAGS_preserve_unverified_changes && FLAGS_reopen != 0) { + fprintf(stderr, + "Reopen DB is incompatible with preserving unverified changes\n"); + exit(1); + } + #ifndef NDEBUG KillPoint* kp = KillPoint::GetInstance(); kp->rocksdb_kill_odds = FLAGS_kill_random_test;