From c3911f1a72add1ec7d86ad337c9f39182ccdb16a Mon Sep 17 00:00:00 2001 From: Cheng Chang Date: Mon, 9 Nov 2020 08:42:41 -0800 Subject: [PATCH] Track WAL in MANIFEST: Track deleted WALs in MANIFEST after recovering from the WALs (#7649) Summary: After replaying the WALs, the memtables are flushed synchronously to L0 instead of being flushed in background. Currently, we only track WAL obsoletion events in the code path of background flush jobs. This PR tracks these events in RecoverLogFiles. After this change, we can enable `track_and_verify_wal_in_manifest` in `db_stress`. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7649 Test Plan: `python tools/db_crashtest.py whitebox` Reviewed By: riversand963 Differential Revision: D24824501 Pulled By: cheng-chang fbshipit-source-id: 207129f7b845c50b333680ce6818a68a2fad54b9 --- db/db_impl/db_impl_open.cc | 10 +++++++++- db_stress_tool/db_stress_test_base.cc | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/db/db_impl/db_impl_open.cc b/db/db_impl/db_impl_open.cc index 247985d00..43a26da9c 100644 --- a/db/db_impl/db_impl_open.cc +++ b/db/db_impl/db_impl_open.cc @@ -1143,7 +1143,7 @@ Status DBImpl::RecoverLogFiles(const std::vector& wal_numbers, if (!read_only) { // no need to refcount since client still doesn't have access // to the DB and can not drop column families while we iterate - auto max_wal_number = wal_numbers.back(); + const WalNumber max_wal_number = wal_numbers.back(); for (auto cfd : *versions_->GetColumnFamilySet()) { auto iter = version_edits.find(cfd->GetID()); assert(iter != version_edits.end()); @@ -1210,6 +1210,14 @@ Status DBImpl::RecoverLogFiles(const std::vector& wal_numbers, assert(iter != version_edits.end()); edit_lists.push_back({&iter->second}); } + + std::unique_ptr wal_deletion; + if (immutable_db_options_.track_and_verify_wals_in_manifest) { + wal_deletion.reset(new VersionEdit); + wal_deletion->DeleteWalsBefore(max_wal_number + 1); + edit_lists.back().push_back(wal_deletion.get()); + } + // write MANIFEST with update status = versions_->LogAndApply(cfds, cf_opts, edit_lists, &mutex_, directories_.GetDbDir(), diff --git a/db_stress_tool/db_stress_test_base.cc b/db_stress_tool/db_stress_test_base.cc index fe90a0796..da1098e27 100644 --- a/db_stress_tool/db_stress_test_base.cc +++ b/db_stress_tool/db_stress_test_base.cc @@ -2066,6 +2066,7 @@ void StressTest::Open() { FLAGS_level_compaction_dynamic_level_bytes; options_.file_checksum_gen_factory = GetFileChecksumImpl(FLAGS_file_checksum_impl); + options_.track_and_verify_wals_in_manifest = true; } else { #ifdef ROCKSDB_LITE fprintf(stderr, "--options_file not supported in lite mode\n");