Fix listener_test when using ROCKSDB_MALLOC_USABLE_SIZE

Summary:
Flushes in listener_test happened to early when ROCKSDB_MALLOC_USABLE_SIZE was
active (e.g. when compiling with ROCKSDB_FBCODE_BUILD_WITH_481=1) due to
malloc_usable_size() reporting a better estimate (similar to
https://reviews.facebook.net/D43317 ). This patch grows the write buffer size
slightly to compensate for this.

Test Plan: ROCKSDB_FBCODE_BUILD_WITH_481=1 make listener_test && ./listener_test

Reviewers: rven, anthony, yhchiang, igor, sdong

Reviewed By: igor

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D45921
main
Andres Notzli 9 years ago
parent 40cd91b7fe
commit b722007778
  1. 6
      db/db_compaction_test.cc
  2. 23
      db/listener_test.cc
  3. 2
      db/memtable.cc

@ -505,7 +505,7 @@ TEST_F(DBCompactionTest, DisableStatsUpdateReopen) {
TEST_P(DBCompactionTestWithParam, CompactionTrigger) { TEST_P(DBCompactionTestWithParam, CompactionTrigger) {
Options options; Options options;
options.write_buffer_size = 110 << 10; // 100KB options.write_buffer_size = 110 << 10; // 110KB
options.arena_block_size = 4 << 10; options.arena_block_size = 4 << 10;
options.num_levels = 3; options.num_levels = 3;
options.level0_file_num_compaction_trigger = 3; options.level0_file_num_compaction_trigger = 3;
@ -1008,7 +1008,7 @@ TEST_P(DBCompactionTestWithParam, LevelCompactionPathUse) {
options.db_paths.emplace_back(dbname_ + "_2", 4 * 1024 * 1024); options.db_paths.emplace_back(dbname_ + "_2", 4 * 1024 * 1024);
options.db_paths.emplace_back(dbname_ + "_3", 1024 * 1024 * 1024); options.db_paths.emplace_back(dbname_ + "_3", 1024 * 1024 * 1024);
options.compaction_style = kCompactionStyleLevel; options.compaction_style = kCompactionStyleLevel;
options.write_buffer_size = 110 << 10; // 100KB options.write_buffer_size = 110 << 10; // 110KB
options.arena_block_size = 4 << 10; options.arena_block_size = 4 << 10;
options.level0_file_num_compaction_trigger = 2; options.level0_file_num_compaction_trigger = 2;
options.num_levels = 4; options.num_levels = 4;
@ -1639,7 +1639,7 @@ TEST_P(DBCompactionTestWithParam, CompressLevelCompaction) {
} }
Options options = CurrentOptions(); Options options = CurrentOptions();
options.compaction_style = kCompactionStyleLevel; options.compaction_style = kCompactionStyleLevel;
options.write_buffer_size = 110 << 10; // 100KB options.write_buffer_size = 110 << 10; // 110KB
options.arena_block_size = 4 << 10; options.arena_block_size = 4 << 10;
options.level0_file_num_compaction_trigger = 2; options.level0_file_num_compaction_trigger = 2;
options.num_levels = 4; options.num_levels = 4;

@ -2,8 +2,9 @@
// This source code is licensed under the BSD-style license found in the // This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant // LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory. // of patent rights can be found in the PATENTS file in the same directory.
#include "db/dbformat.h"
#include "db/db_impl.h" #include "db/db_impl.h"
#include "db/dbformat.h"
#include "db/filename.h" #include "db/filename.h"
#include "db/version_set.h" #include "db/version_set.h"
#include "db/write_batch_internal.h" #include "db/write_batch_internal.h"
@ -12,25 +13,25 @@
#include "rocksdb/db.h" #include "rocksdb/db.h"
#include "rocksdb/env.h" #include "rocksdb/env.h"
#include "rocksdb/filter_policy.h" #include "rocksdb/filter_policy.h"
#include "rocksdb/options.h"
#include "rocksdb/perf_context.h" #include "rocksdb/perf_context.h"
#include "rocksdb/slice.h" #include "rocksdb/slice.h"
#include "rocksdb/slice_transform.h" #include "rocksdb/slice_transform.h"
#include "rocksdb/table.h" #include "rocksdb/table.h"
#include "rocksdb/options.h"
#include "rocksdb/table_properties.h" #include "rocksdb/table_properties.h"
#include "table/block_based_table_factory.h" #include "table/block_based_table_factory.h"
#include "table/plain_table_factory.h" #include "table/plain_table_factory.h"
#include "util/hash.h" #include "util/hash.h"
#include "util/hash_linklist_rep.h" #include "util/hash_linklist_rep.h"
#include "utilities/merge_operators.h"
#include "util/logging.h" #include "util/logging.h"
#include "util/mutexlock.h" #include "util/mutexlock.h"
#include "util/rate_limiter.h" #include "util/rate_limiter.h"
#include "util/statistics.h" #include "util/statistics.h"
#include "util/string_util.h" #include "util/string_util.h"
#include "util/testharness.h"
#include "util/sync_point.h" #include "util/sync_point.h"
#include "util/testharness.h"
#include "util/testutil.h" #include "util/testutil.h"
#include "utilities/merge_operators.h"
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
@ -134,7 +135,7 @@ class EventListenerTest : public testing::Test {
return db_->Put(wo, handles_[cf], k, v); return db_->Put(wo, handles_[cf], k, v);
} }
Status Flush(int cf = 0) { Status Flush(size_t cf = 0) {
FlushOptions opt = FlushOptions(); FlushOptions opt = FlushOptions();
opt.wait = true; opt.wait = true;
if (cf == 0) { if (cf == 0) {
@ -144,6 +145,8 @@ class EventListenerTest : public testing::Test {
} }
} }
const size_t k110KB = 110 << 10;
DB* db_; DB* db_;
std::string dbname_; std::string dbname_;
std::vector<ColumnFamilyHandle*> handles_; std::vector<ColumnFamilyHandle*> handles_;
@ -198,7 +201,7 @@ TEST_F(EventListenerTest, OnSingleDBCompactionTest) {
ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a'))); ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a')));
ASSERT_OK(Put(7, "popovich", std::string(90000, 'p'))); ASSERT_OK(Put(7, "popovich", std::string(90000, 'p')));
for (size_t i = 1; i < 8; ++i) { for (size_t i = 1; i < 8; ++i) {
ASSERT_OK(Flush(static_cast<int>(i))); ASSERT_OK(Flush(i));
const Slice kStart = "a"; const Slice kStart = "a";
const Slice kEnd = "z"; const Slice kEnd = "z";
ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), handles_[i], ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), handles_[i],
@ -287,7 +290,7 @@ class TestFlushListener : public EventListener {
TEST_F(EventListenerTest, OnSingleDBFlushTest) { TEST_F(EventListenerTest, OnSingleDBFlushTest) {
Options options; Options options;
options.write_buffer_size = 100000; options.write_buffer_size = k110KB;
#if ROCKSDB_USING_THREAD_STATUS #if ROCKSDB_USING_THREAD_STATUS
options.enable_thread_tracking = true; options.enable_thread_tracking = true;
#endif // ROCKSDB_USING_THREAD_STATUS #endif // ROCKSDB_USING_THREAD_STATUS
@ -306,7 +309,7 @@ TEST_F(EventListenerTest, OnSingleDBFlushTest) {
ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a'))); ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a')));
ASSERT_OK(Put(7, "popovich", std::string(90000, 'p'))); ASSERT_OK(Put(7, "popovich", std::string(90000, 'p')));
for (size_t i = 1; i < 8; ++i) { for (size_t i = 1; i < 8; ++i) {
ASSERT_OK(Flush(static_cast<int>(i))); ASSERT_OK(Flush(i));
dbfull()->TEST_WaitForFlushMemTable(); dbfull()->TEST_WaitForFlushMemTable();
ASSERT_EQ(listener->flushed_dbs_.size(), i); ASSERT_EQ(listener->flushed_dbs_.size(), i);
ASSERT_EQ(listener->flushed_column_family_names_.size(), i); ASSERT_EQ(listener->flushed_column_family_names_.size(), i);
@ -321,7 +324,7 @@ TEST_F(EventListenerTest, OnSingleDBFlushTest) {
TEST_F(EventListenerTest, MultiCF) { TEST_F(EventListenerTest, MultiCF) {
Options options; Options options;
options.write_buffer_size = 100000; options.write_buffer_size = k110KB;
#if ROCKSDB_USING_THREAD_STATUS #if ROCKSDB_USING_THREAD_STATUS
options.enable_thread_tracking = true; options.enable_thread_tracking = true;
#endif // ROCKSDB_USING_THREAD_STATUS #endif // ROCKSDB_USING_THREAD_STATUS
@ -340,7 +343,7 @@ TEST_F(EventListenerTest, MultiCF) {
ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a'))); ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a')));
ASSERT_OK(Put(7, "popovich", std::string(90000, 'p'))); ASSERT_OK(Put(7, "popovich", std::string(90000, 'p')));
for (size_t i = 1; i < 8; ++i) { for (size_t i = 1; i < 8; ++i) {
ASSERT_OK(Flush(static_cast<int>(i))); ASSERT_OK(Flush(i));
ASSERT_EQ(listener->flushed_dbs_.size(), i); ASSERT_EQ(listener->flushed_dbs_.size(), i);
ASSERT_EQ(listener->flushed_column_family_names_.size(), i); ASSERT_EQ(listener->flushed_column_family_names_.size(), i);
} }

@ -112,7 +112,7 @@ bool MemTable::ShouldFlushNow() const {
// In a lot of times, we cannot allocate arena blocks that exactly matches the // In a lot of times, we cannot allocate arena blocks that exactly matches the
// buffer size. Thus we have to decide if we should over-allocate or // buffer size. Thus we have to decide if we should over-allocate or
// under-allocate. // under-allocate.
// This constant avariable can be interpreted as: if we still have more than // This constant variable can be interpreted as: if we still have more than
// "kAllowOverAllocationRatio * kArenaBlockSize" space left, we'd try to over // "kAllowOverAllocationRatio * kArenaBlockSize" space left, we'd try to over
// allocate one more block. // allocate one more block.
const double kAllowOverAllocationRatio = 0.6; const double kAllowOverAllocationRatio = 0.6;

Loading…
Cancel
Save