Summary: Introduces a new class for managing write buffer memory across column families. We supplement ColumnFamilyOptions::write_buffer_size with ColumnFamilyOptions::write_buffer, a shared pointer to a WriteBuffer instance that enforces memory limits before flushing out to disk. Test Plan: Added SharedWriteBuffer unit test to db_test.cc Reviewers: sdong, rven, ljin, igor Reviewed By: igor Subscribers: tnovak, yhchiang, dhruba, xjin, MarkCallaghan, yoshinorim Differential Revision: https://reviews.facebook.net/D22581main
Jonah Cohen
10 years ago
parent
37d73d597e
commit
a14b7873ee
@ -0,0 +1,52 @@ |
||||
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include <assert.h> |
||||
|
||||
#include "db/memtable_allocator.h" |
||||
#include "db/writebuffer.h" |
||||
#include "util/arena.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
MemTableAllocator::MemTableAllocator(Arena* arena, WriteBuffer* write_buffer) |
||||
: arena_(arena), write_buffer_(write_buffer), bytes_allocated_(0) { |
||||
} |
||||
|
||||
MemTableAllocator::~MemTableAllocator() { |
||||
DoneAllocating(); |
||||
} |
||||
|
||||
char* MemTableAllocator::Allocate(size_t bytes) { |
||||
assert(write_buffer_ != nullptr); |
||||
bytes_allocated_ += bytes; |
||||
write_buffer_->ReserveMem(bytes); |
||||
return arena_->Allocate(bytes); |
||||
} |
||||
|
||||
char* MemTableAllocator::AllocateAligned(size_t bytes, size_t huge_page_size, |
||||
Logger* logger) { |
||||
assert(write_buffer_ != nullptr); |
||||
bytes_allocated_ += bytes; |
||||
write_buffer_->ReserveMem(bytes); |
||||
return arena_->AllocateAligned(bytes, huge_page_size, logger); |
||||
} |
||||
|
||||
void MemTableAllocator::DoneAllocating() { |
||||
if (write_buffer_ != nullptr) { |
||||
write_buffer_->FreeMem(bytes_allocated_); |
||||
write_buffer_ = nullptr; |
||||
} |
||||
} |
||||
|
||||
size_t MemTableAllocator::BlockSize() const { |
||||
return arena_->BlockSize(); |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,47 @@ |
||||
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
//
|
||||
// This is used by the MemTable to allocate write buffer memory. It connects
|
||||
// to WriteBuffer so we can track and enforce overall write buffer limits.
|
||||
|
||||
#pragma once |
||||
#include "util/allocator.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class Arena; |
||||
class Logger; |
||||
class WriteBuffer; |
||||
|
||||
class MemTableAllocator : public Allocator { |
||||
public: |
||||
explicit MemTableAllocator(Arena* arena, WriteBuffer* write_buffer); |
||||
~MemTableAllocator(); |
||||
|
||||
// Allocator interface
|
||||
char* Allocate(size_t bytes) override; |
||||
char* AllocateAligned(size_t bytes, size_t huge_page_size = 0, |
||||
Logger* logger = nullptr) override; |
||||
size_t BlockSize() const override; |
||||
|
||||
// Call when we're finished allocating memory so we can free it from
|
||||
// the write buffer's limit.
|
||||
void DoneAllocating(); |
||||
|
||||
private: |
||||
Arena* arena_; |
||||
WriteBuffer* write_buffer_; |
||||
size_t bytes_allocated_; |
||||
|
||||
// No copying allowed
|
||||
MemTableAllocator(const MemTableAllocator&); |
||||
void operator=(const MemTableAllocator&); |
||||
}; |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,44 @@ |
||||
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
//
|
||||
// WriteBuffer is for managing memory allocation for one or more MemTables.
|
||||
|
||||
#pragma once |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class WriteBuffer { |
||||
public: |
||||
explicit WriteBuffer(size_t _buffer_size) |
||||
: buffer_size_(_buffer_size), memory_used_(0) {} |
||||
|
||||
~WriteBuffer() {} |
||||
|
||||
size_t memory_usage() const { return memory_used_; } |
||||
size_t buffer_size() const { return buffer_size_; } |
||||
|
||||
// Should only be called from write thread
|
||||
bool ShouldFlush() const { |
||||
return buffer_size() > 0 && memory_usage() >= buffer_size(); |
||||
} |
||||
|
||||
// Should only be called from write thread
|
||||
void ReserveMem(size_t mem) { memory_used_ += mem; } |
||||
void FreeMem(size_t mem) { memory_used_ -= mem; } |
||||
|
||||
private: |
||||
const size_t buffer_size_; |
||||
size_t memory_used_; |
||||
|
||||
// No copying allowed
|
||||
WriteBuffer(const WriteBuffer&); |
||||
void operator=(const WriteBuffer&); |
||||
}; |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,32 @@ |
||||
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
//
|
||||
// Abstract interface for allocating memory in blocks. This memory is freed
|
||||
// when the allocator object is destroyed. See the Arena class for more info.
|
||||
|
||||
#pragma once |
||||
#include <cstddef> |
||||
#include <cerrno> |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class Logger; |
||||
|
||||
class Allocator { |
||||
public: |
||||
virtual ~Allocator() {} |
||||
|
||||
virtual char* Allocate(size_t bytes) = 0; |
||||
virtual char* AllocateAligned(size_t bytes, size_t huge_page_size = 0, |
||||
Logger* logger = nullptr) = 0; |
||||
|
||||
virtual size_t BlockSize() const = 0; |
||||
}; |
||||
|
||||
} // namespace rocksdb
|
||||
Loading…
Reference in new issue