From b7220077789b8b1ebab1d2c52c849065d8a4c827 Mon Sep 17 00:00:00 2001 From: Andres Notzli Date: Mon, 31 Aug 2015 23:11:12 -0700 Subject: [PATCH] Fix listener_test when using ROCKSDB_MALLOC_USABLE_SIZE Summary: Flushes in listener_test happened to early when ROCKSDB_MALLOC_USABLE_SIZE was active (e.g. when compiling with ROCKSDB_FBCODE_BUILD_WITH_481=1) due to malloc_usable_size() reporting a better estimate (similar to https://reviews.facebook.net/D43317 ). This patch grows the write buffer size slightly to compensate for this. Test Plan: ROCKSDB_FBCODE_BUILD_WITH_481=1 make listener_test && ./listener_test Reviewers: rven, anthony, yhchiang, igor, sdong Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D45921 --- db/db_compaction_test.cc | 6 +++--- db/listener_test.cc | 23 +++++++++++++---------- db/memtable.cc | 2 +- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/db/db_compaction_test.cc b/db/db_compaction_test.cc index 5d74a81b5..7c277a08a 100644 --- a/db/db_compaction_test.cc +++ b/db/db_compaction_test.cc @@ -505,7 +505,7 @@ TEST_F(DBCompactionTest, DisableStatsUpdateReopen) { TEST_P(DBCompactionTestWithParam, CompactionTrigger) { Options options; - options.write_buffer_size = 110 << 10; // 100KB + options.write_buffer_size = 110 << 10; // 110KB options.arena_block_size = 4 << 10; options.num_levels = 3; options.level0_file_num_compaction_trigger = 3; @@ -1008,7 +1008,7 @@ TEST_P(DBCompactionTestWithParam, LevelCompactionPathUse) { options.db_paths.emplace_back(dbname_ + "_2", 4 * 1024 * 1024); options.db_paths.emplace_back(dbname_ + "_3", 1024 * 1024 * 1024); options.compaction_style = kCompactionStyleLevel; - options.write_buffer_size = 110 << 10; // 100KB + options.write_buffer_size = 110 << 10; // 110KB options.arena_block_size = 4 << 10; options.level0_file_num_compaction_trigger = 2; options.num_levels = 4; @@ -1639,7 +1639,7 @@ TEST_P(DBCompactionTestWithParam, CompressLevelCompaction) { } Options options = CurrentOptions(); options.compaction_style = kCompactionStyleLevel; - options.write_buffer_size = 110 << 10; // 100KB + options.write_buffer_size = 110 << 10; // 110KB options.arena_block_size = 4 << 10; options.level0_file_num_compaction_trigger = 2; options.num_levels = 4; diff --git a/db/listener_test.cc b/db/listener_test.cc index f15f74b2d..ce683a5b3 100644 --- a/db/listener_test.cc +++ b/db/listener_test.cc @@ -2,8 +2,9 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. An additional grant // of patent rights can be found in the PATENTS file in the same directory. -#include "db/dbformat.h" + #include "db/db_impl.h" +#include "db/dbformat.h" #include "db/filename.h" #include "db/version_set.h" #include "db/write_batch_internal.h" @@ -12,25 +13,25 @@ #include "rocksdb/db.h" #include "rocksdb/env.h" #include "rocksdb/filter_policy.h" +#include "rocksdb/options.h" #include "rocksdb/perf_context.h" #include "rocksdb/slice.h" #include "rocksdb/slice_transform.h" #include "rocksdb/table.h" -#include "rocksdb/options.h" #include "rocksdb/table_properties.h" #include "table/block_based_table_factory.h" #include "table/plain_table_factory.h" #include "util/hash.h" #include "util/hash_linklist_rep.h" -#include "utilities/merge_operators.h" #include "util/logging.h" #include "util/mutexlock.h" #include "util/rate_limiter.h" #include "util/statistics.h" #include "util/string_util.h" -#include "util/testharness.h" #include "util/sync_point.h" +#include "util/testharness.h" #include "util/testutil.h" +#include "utilities/merge_operators.h" #ifndef ROCKSDB_LITE @@ -134,7 +135,7 @@ class EventListenerTest : public testing::Test { return db_->Put(wo, handles_[cf], k, v); } - Status Flush(int cf = 0) { + Status Flush(size_t cf = 0) { FlushOptions opt = FlushOptions(); opt.wait = true; if (cf == 0) { @@ -144,6 +145,8 @@ class EventListenerTest : public testing::Test { } } + const size_t k110KB = 110 << 10; + DB* db_; std::string dbname_; std::vector handles_; @@ -198,7 +201,7 @@ TEST_F(EventListenerTest, OnSingleDBCompactionTest) { ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a'))); ASSERT_OK(Put(7, "popovich", std::string(90000, 'p'))); for (size_t i = 1; i < 8; ++i) { - ASSERT_OK(Flush(static_cast(i))); + ASSERT_OK(Flush(i)); const Slice kStart = "a"; const Slice kEnd = "z"; ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), handles_[i], @@ -287,7 +290,7 @@ class TestFlushListener : public EventListener { TEST_F(EventListenerTest, OnSingleDBFlushTest) { Options options; - options.write_buffer_size = 100000; + options.write_buffer_size = k110KB; #if ROCKSDB_USING_THREAD_STATUS options.enable_thread_tracking = true; #endif // ROCKSDB_USING_THREAD_STATUS @@ -306,7 +309,7 @@ TEST_F(EventListenerTest, OnSingleDBFlushTest) { ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a'))); ASSERT_OK(Put(7, "popovich", std::string(90000, 'p'))); for (size_t i = 1; i < 8; ++i) { - ASSERT_OK(Flush(static_cast(i))); + ASSERT_OK(Flush(i)); dbfull()->TEST_WaitForFlushMemTable(); ASSERT_EQ(listener->flushed_dbs_.size(), i); ASSERT_EQ(listener->flushed_column_family_names_.size(), i); @@ -321,7 +324,7 @@ TEST_F(EventListenerTest, OnSingleDBFlushTest) { TEST_F(EventListenerTest, MultiCF) { Options options; - options.write_buffer_size = 100000; + options.write_buffer_size = k110KB; #if ROCKSDB_USING_THREAD_STATUS options.enable_thread_tracking = true; #endif // ROCKSDB_USING_THREAD_STATUS @@ -340,7 +343,7 @@ TEST_F(EventListenerTest, MultiCF) { ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a'))); ASSERT_OK(Put(7, "popovich", std::string(90000, 'p'))); for (size_t i = 1; i < 8; ++i) { - ASSERT_OK(Flush(static_cast(i))); + ASSERT_OK(Flush(i)); ASSERT_EQ(listener->flushed_dbs_.size(), i); ASSERT_EQ(listener->flushed_column_family_names_.size(), i); } diff --git a/db/memtable.cc b/db/memtable.cc index fded51e67..d1bbd3960 100644 --- a/db/memtable.cc +++ b/db/memtable.cc @@ -112,7 +112,7 @@ bool MemTable::ShouldFlushNow() const { // In a lot of times, we cannot allocate arena blocks that exactly matches the // buffer size. Thus we have to decide if we should over-allocate or // under-allocate. - // This constant avariable can be interpreted as: if we still have more than + // This constant variable can be interpreted as: if we still have more than // "kAllowOverAllocationRatio * kArenaBlockSize" space left, we'd try to over // allocate one more block. const double kAllowOverAllocationRatio = 0.6;