From 963314ffd681596ef2738a95249fe4c1163ef87a Mon Sep 17 00:00:00 2001 From: Akanksha Mahajan Date: Fri, 28 Aug 2020 17:36:27 -0700 Subject: [PATCH] Add unit test for max_write_buffer_size_to_maintain (#7311) Summary: Add a unit test case to check memory usage when max_write_buffer_size_to_maintain is set if flushed immutable memtables are trimmed timely or not. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7311 Test Plan: Compared the results with before bug fix. Reviewed By: ltamasi Differential Revision: D23321702 Pulled By: akankshamahajan15 fbshipit-source-id: da04ee21137d641a07fd499a9e2749eb036fcb1e --- HISTORY.md | 2 +- db/db_test.cc | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 70afd8e44..70c70e801 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -6,8 +6,8 @@ * Sanitize `recycle_log_file_num` to zero when the user attempts to enable it in combination with `WALRecoveryMode::kTolerateCorruptedTailRecords`. Previously the two features were allowed together, which compromised the user's configured crash-recovery guarantees. * Fix a bug where a level refitting in CompactRange() might race with an automatic compaction that puts the data to the target level of the refitting. The bug has been there for years. * Fixed a bug in version 6.12 in which BackupEngine::CreateNewBackup could fail intermittently with non-OK status when backing up a read-write DB configured with a DBOptions::file_checksum_gen_factory. -* Fix a bug where immutable flushed memtable is never destroyed because a memtable is not added to delete list because of refernce hold by super version and super version doesn't switch because of empty delete list. So memory usage increases beyond write_buffer_size + max_write_buffer_size_to_maintain. * Fix useless no-op compactions scheduled upon snapshot release when options.disable-auto-compactions = true. +* Fix a bug when max_write_buffer_size_to_maintain is set, immutable flushed memtable destruction is delayed until the next super version is installed. A memtable is not added to delete list because of its reference hold by super version and super version doesn't switch because of empt delete list. So memory usage keeps on increasing beyond write_buffer_size + max_write_buffer_size_to_maintain. ### New Features * A new option `std::shared_ptr file_checksum_gen_factory` is added to `BackupableDBOptions`. The default value for this option is `nullptr`. If this option is null, the default backup engine checksum function (crc32c) will be used for creating, verifying, or restoring backups. If it is not null and is set to the DB custom checksum factory, the custom checksum function used in DB will also be used for creating, verifying, or restoring backups, in addition to the default checksum function (crc32c). If it is not null and is set to a custom checksum factory different than the DB custom checksum factory (which may be null), BackupEngine will return `Status::InvalidArgument()`. diff --git a/db/db_test.cc b/db/db_test.cc index 24a7ef65b..41f1de8e9 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -6677,6 +6677,46 @@ TEST_F(DBTest, CreationTimeOfOldestFile) { ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing(); } +TEST_F(DBTest, MemoryUsageWithMaxWriteBufferSizeToMaintain) { + Options options = CurrentOptions(); + options.max_write_buffer_size_to_maintain = 10000; + options.write_buffer_size = 160000; + Reopen(options); + Random rnd(301); + bool memory_limit_exceeded = false; + uint64_t size_all_mem_table = 0; + uint64_t cur_active_mem = 0; + for (int i = 0; i < 1000; i++) { + std::string value = rnd.RandomString(1000); + ASSERT_OK(Put("keykey_" + std::to_string(i), value)); + + dbfull()->TEST_WaitForFlushMemTable(); + + ASSERT_TRUE(db_->GetIntProperty(db_->DefaultColumnFamily(), + DB::Properties::kSizeAllMemTables, + &size_all_mem_table)); + ASSERT_TRUE(db_->GetIntProperty(db_->DefaultColumnFamily(), + DB::Properties::kCurSizeActiveMemTable, + &cur_active_mem)); + + // Errors out if memory usage keeps on increasing beyond the limit. + // Once memory limit exceeds, memory_limit_exceeded is set and if + // size_all_mem_table doesn't drop out in the next write then it errors out + // (not expected behaviour). If memory usage drops then + // memory_limit_exceeded is set to false. + if ((size_all_mem_table > cur_active_mem) && + (cur_active_mem >= + static_cast(options.max_write_buffer_size_to_maintain)) && + (size_all_mem_table > options.max_write_buffer_size_to_maintain + + options.write_buffer_size)) { + ASSERT_FALSE(memory_limit_exceeded); + memory_limit_exceeded = true; + } else { + memory_limit_exceeded = false; + } + } +} + #endif } // namespace ROCKSDB_NAMESPACE