From 9fa5c146d7d5ba9aea2ede329c0bc78309d9787b Mon Sep 17 00:00:00 2001 From: Peter Dillinger Date: Fri, 12 Aug 2022 13:08:50 -0700 Subject: [PATCH] LOG more info on oldest snapshot and sequence numbers (#10454) Summary: The info LOG file does not currently give any direct information about the existence of old, live snapshots, nor how to estimate wall time from a sequence number within the scope of LOG history. This change addresses both with: * Logging smallest and largest seqnos for generated SST files, which can help associate sequence numbers with write time (based on flushes). * Logging oldest_snapshot_seqno for each compaction, which (along with that seqno info) helps us to determine how much old data might be kept around for old (leaked?) snapshots. Including the date here I thought might be excessive. I wanted to log the date and seqno of the oldest snapshot with periodic stats, but the current structure of the code doesn't really support that because `DumpDBStats` doesn't have access to the DB object. Pull Request resolved: https://github.com/facebook/rocksdb/pull/10454 Test Plan: manual inspect LOG from `KEEP_DB=1 ./db_basic_test --gtest_filter=*CompactBetweenSnapshots*` Reviewed By: ajkr Differential Revision: D38326948 Pulled By: pdillinger fbshipit-source-id: 294918ffc04a419844146cd826045321b4d5c038 --- db/compaction/compaction_job.cc | 5 ++++- db/event_helpers.cc | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/db/compaction/compaction_job.cc b/db/compaction/compaction_job.cc index 35e80e7c6..a76e91696 100644 --- a/db/compaction/compaction_job.cc +++ b/db/compaction/compaction_job.cc @@ -1961,7 +1961,10 @@ void CompactionJob::LogCompaction() { stream.EndArray(); } stream << "score" << compaction->score() << "input_data_size" - << compaction->CalculateTotalInputSize(); + << compaction->CalculateTotalInputSize() << "oldest_snapshot_seqno" + << (existing_snapshots_.empty() + ? int64_t{-1} // Use -1 for "none" + : static_cast(existing_snapshots_[0])); } } diff --git a/db/event_helpers.cc b/db/event_helpers.cc index 3f290cc05..6a5c93661 100644 --- a/db/event_helpers.cc +++ b/db/event_helpers.cc @@ -97,7 +97,8 @@ void EventHelpers::LogAndNotifyTableFileCreationFinished( << "file_number" << fd.GetNumber() << "file_size" << fd.GetFileSize() << "file_checksum" << Slice(file_checksum).ToString(true) << "file_checksum_func_name" - << file_checksum_func_name; + << file_checksum_func_name << "smallest_seqno" << fd.smallest_seqno + << "largest_seqno" << fd.largest_seqno; // table_properties {