Summary: Introduces a new class for managing write buffer memory across column families. We supplement ColumnFamilyOptions::write_buffer_size with ColumnFamilyOptions::write_buffer, a shared pointer to a WriteBuffer instance that enforces memory limits before flushing out to disk. Test Plan: Added SharedWriteBuffer unit test to db_test.cc Reviewers: sdong, rven, ljin, igor Reviewed By: igor Subscribers: tnovak, yhchiang, dhruba, xjin, MarkCallaghan, yoshinorim Differential Revision: https://reviews.facebook.net/D22581main
Jonah Cohen
10 years ago
parent
37d73d597e
commit
a14b7873ee
@ -0,0 +1,52 @@ |
|||||||
|
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
|
#include <assert.h> |
||||||
|
|
||||||
|
#include "db/memtable_allocator.h" |
||||||
|
#include "db/writebuffer.h" |
||||||
|
#include "util/arena.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
MemTableAllocator::MemTableAllocator(Arena* arena, WriteBuffer* write_buffer) |
||||||
|
: arena_(arena), write_buffer_(write_buffer), bytes_allocated_(0) { |
||||||
|
} |
||||||
|
|
||||||
|
MemTableAllocator::~MemTableAllocator() { |
||||||
|
DoneAllocating(); |
||||||
|
} |
||||||
|
|
||||||
|
char* MemTableAllocator::Allocate(size_t bytes) { |
||||||
|
assert(write_buffer_ != nullptr); |
||||||
|
bytes_allocated_ += bytes; |
||||||
|
write_buffer_->ReserveMem(bytes); |
||||||
|
return arena_->Allocate(bytes); |
||||||
|
} |
||||||
|
|
||||||
|
char* MemTableAllocator::AllocateAligned(size_t bytes, size_t huge_page_size, |
||||||
|
Logger* logger) { |
||||||
|
assert(write_buffer_ != nullptr); |
||||||
|
bytes_allocated_ += bytes; |
||||||
|
write_buffer_->ReserveMem(bytes); |
||||||
|
return arena_->AllocateAligned(bytes, huge_page_size, logger); |
||||||
|
} |
||||||
|
|
||||||
|
void MemTableAllocator::DoneAllocating() { |
||||||
|
if (write_buffer_ != nullptr) { |
||||||
|
write_buffer_->FreeMem(bytes_allocated_); |
||||||
|
write_buffer_ = nullptr; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
size_t MemTableAllocator::BlockSize() const { |
||||||
|
return arena_->BlockSize(); |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,47 @@ |
|||||||
|
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
//
|
||||||
|
// This is used by the MemTable to allocate write buffer memory. It connects
|
||||||
|
// to WriteBuffer so we can track and enforce overall write buffer limits.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
#include "util/allocator.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class Arena; |
||||||
|
class Logger; |
||||||
|
class WriteBuffer; |
||||||
|
|
||||||
|
class MemTableAllocator : public Allocator { |
||||||
|
public: |
||||||
|
explicit MemTableAllocator(Arena* arena, WriteBuffer* write_buffer); |
||||||
|
~MemTableAllocator(); |
||||||
|
|
||||||
|
// Allocator interface
|
||||||
|
char* Allocate(size_t bytes) override; |
||||||
|
char* AllocateAligned(size_t bytes, size_t huge_page_size = 0, |
||||||
|
Logger* logger = nullptr) override; |
||||||
|
size_t BlockSize() const override; |
||||||
|
|
||||||
|
// Call when we're finished allocating memory so we can free it from
|
||||||
|
// the write buffer's limit.
|
||||||
|
void DoneAllocating(); |
||||||
|
|
||||||
|
private: |
||||||
|
Arena* arena_; |
||||||
|
WriteBuffer* write_buffer_; |
||||||
|
size_t bytes_allocated_; |
||||||
|
|
||||||
|
// No copying allowed
|
||||||
|
MemTableAllocator(const MemTableAllocator&); |
||||||
|
void operator=(const MemTableAllocator&); |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,44 @@ |
|||||||
|
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
//
|
||||||
|
// WriteBuffer is for managing memory allocation for one or more MemTables.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class WriteBuffer { |
||||||
|
public: |
||||||
|
explicit WriteBuffer(size_t _buffer_size) |
||||||
|
: buffer_size_(_buffer_size), memory_used_(0) {} |
||||||
|
|
||||||
|
~WriteBuffer() {} |
||||||
|
|
||||||
|
size_t memory_usage() const { return memory_used_; } |
||||||
|
size_t buffer_size() const { return buffer_size_; } |
||||||
|
|
||||||
|
// Should only be called from write thread
|
||||||
|
bool ShouldFlush() const { |
||||||
|
return buffer_size() > 0 && memory_usage() >= buffer_size(); |
||||||
|
} |
||||||
|
|
||||||
|
// Should only be called from write thread
|
||||||
|
void ReserveMem(size_t mem) { memory_used_ += mem; } |
||||||
|
void FreeMem(size_t mem) { memory_used_ -= mem; } |
||||||
|
|
||||||
|
private: |
||||||
|
const size_t buffer_size_; |
||||||
|
size_t memory_used_; |
||||||
|
|
||||||
|
// No copying allowed
|
||||||
|
WriteBuffer(const WriteBuffer&); |
||||||
|
void operator=(const WriteBuffer&); |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,32 @@ |
|||||||
|
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
//
|
||||||
|
// Abstract interface for allocating memory in blocks. This memory is freed
|
||||||
|
// when the allocator object is destroyed. See the Arena class for more info.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
#include <cstddef> |
||||||
|
#include <cerrno> |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class Logger; |
||||||
|
|
||||||
|
class Allocator { |
||||||
|
public: |
||||||
|
virtual ~Allocator() {} |
||||||
|
|
||||||
|
virtual char* Allocate(size_t bytes) = 0; |
||||||
|
virtual char* AllocateAligned(size_t bytes, size_t huge_page_size = 0, |
||||||
|
Logger* logger = nullptr) = 0; |
||||||
|
|
||||||
|
virtual size_t BlockSize() const = 0; |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
Loading…
Reference in new issue