Naveen
10 years ago
commit
ba6d660f6d
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,62 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#include "db/flush_scheduler.h" |
||||||
|
|
||||||
|
#include <cassert> |
||||||
|
|
||||||
|
#include "db/column_family.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
void FlushScheduler::ScheduleFlush(ColumnFamilyData* cfd) { |
||||||
|
#ifndef NDEBUG |
||||||
|
assert(column_families_set_.find(cfd) == column_families_set_.end()); |
||||||
|
column_families_set_.insert(cfd); |
||||||
|
#endif // NDEBUG
|
||||||
|
cfd->Ref(); |
||||||
|
column_families_.push_back(cfd); |
||||||
|
} |
||||||
|
|
||||||
|
ColumnFamilyData* FlushScheduler::GetNextColumnFamily() { |
||||||
|
ColumnFamilyData* cfd = nullptr; |
||||||
|
while (column_families_.size() > 0) { |
||||||
|
cfd = column_families_.front(); |
||||||
|
column_families_.pop_front(); |
||||||
|
if (cfd->IsDropped()) { |
||||||
|
if (cfd->Unref()) { |
||||||
|
delete cfd; |
||||||
|
} |
||||||
|
} else { |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
#ifndef NDEBUG |
||||||
|
if (cfd != nullptr) { |
||||||
|
auto itr = column_families_set_.find(cfd); |
||||||
|
assert(itr != column_families_set_.end()); |
||||||
|
column_families_set_.erase(itr); |
||||||
|
} |
||||||
|
#endif // NDEBUG
|
||||||
|
return cfd; |
||||||
|
} |
||||||
|
|
||||||
|
bool FlushScheduler::Empty() { return column_families_.empty(); } |
||||||
|
|
||||||
|
void FlushScheduler::Clear() { |
||||||
|
for (auto cfd : column_families_) { |
||||||
|
#ifndef NDEBUG |
||||||
|
auto itr = column_families_set_.find(cfd); |
||||||
|
assert(itr != column_families_set_.end()); |
||||||
|
column_families_set_.erase(itr); |
||||||
|
#endif // NDEBUG
|
||||||
|
if (cfd->Unref()) { |
||||||
|
delete cfd; |
||||||
|
} |
||||||
|
} |
||||||
|
column_families_.clear(); |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,39 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <stdint.h> |
||||||
|
#include <deque> |
||||||
|
#include <set> |
||||||
|
#include <vector> |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class ColumnFamilyData; |
||||||
|
|
||||||
|
// This class is thread-compatible. It's should only be accessed from single
|
||||||
|
// write thread (between BeginWrite() and EndWrite())
|
||||||
|
class FlushScheduler { |
||||||
|
public: |
||||||
|
FlushScheduler() = default; |
||||||
|
~FlushScheduler() = default; |
||||||
|
|
||||||
|
void ScheduleFlush(ColumnFamilyData* cfd); |
||||||
|
// Returns Ref()-ed column family. Client needs to Unref()
|
||||||
|
ColumnFamilyData* GetNextColumnFamily(); |
||||||
|
|
||||||
|
bool Empty(); |
||||||
|
|
||||||
|
void Clear(); |
||||||
|
|
||||||
|
private: |
||||||
|
std::deque<ColumnFamilyData*> column_families_; |
||||||
|
#ifndef NDEBUG |
||||||
|
std::set<ColumnFamilyData*> column_families_set_; |
||||||
|
#endif // NDEBUG
|
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,37 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#include "db/write_controller.h" |
||||||
|
|
||||||
|
#include <cassert> |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
std::unique_ptr<WriteControllerToken> WriteController::GetStopToken() { |
||||||
|
++total_stopped_; |
||||||
|
return std::unique_ptr<WriteControllerToken>(new StopWriteToken(this)); |
||||||
|
} |
||||||
|
|
||||||
|
std::unique_ptr<WriteControllerToken> WriteController::GetDelayToken( |
||||||
|
uint64_t delay_us) { |
||||||
|
total_delay_us_ += delay_us; |
||||||
|
return std::unique_ptr<WriteControllerToken>( |
||||||
|
new DelayWriteToken(this, delay_us)); |
||||||
|
} |
||||||
|
|
||||||
|
bool WriteController::IsStopped() const { return total_stopped_ > 0; } |
||||||
|
uint64_t WriteController::GetDelay() const { return total_delay_us_; } |
||||||
|
|
||||||
|
StopWriteToken::~StopWriteToken() { |
||||||
|
assert(controller_->total_stopped_ >= 1); |
||||||
|
--controller_->total_stopped_; |
||||||
|
} |
||||||
|
|
||||||
|
DelayWriteToken::~DelayWriteToken() { |
||||||
|
assert(controller_->total_delay_us_ >= delay_us_); |
||||||
|
controller_->total_delay_us_ -= delay_us_; |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,78 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <stdint.h> |
||||||
|
|
||||||
|
#include <memory> |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class WriteControllerToken; |
||||||
|
|
||||||
|
// WriteController is controlling write stalls in our write code-path. Write
|
||||||
|
// stalls happen when compaction can't keep up with write rate.
|
||||||
|
// All of the methods here (including WriteControllerToken's destructors) need
|
||||||
|
// to be called while holding DB mutex
|
||||||
|
class WriteController { |
||||||
|
public: |
||||||
|
WriteController() : total_stopped_(0), total_delay_us_(0) {} |
||||||
|
~WriteController() = default; |
||||||
|
|
||||||
|
// When an actor (column family) requests a stop token, all writes will be
|
||||||
|
// stopped until the stop token is released (deleted)
|
||||||
|
std::unique_ptr<WriteControllerToken> GetStopToken(); |
||||||
|
// When an actor (column family) requests a delay token, total delay for all
|
||||||
|
// writes will be increased by delay_us. The delay will last until delay token
|
||||||
|
// is released
|
||||||
|
std::unique_ptr<WriteControllerToken> GetDelayToken(uint64_t delay_us); |
||||||
|
|
||||||
|
// these two metods are querying the state of the WriteController
|
||||||
|
bool IsStopped() const; |
||||||
|
uint64_t GetDelay() const; |
||||||
|
|
||||||
|
private: |
||||||
|
friend class WriteControllerToken; |
||||||
|
friend class StopWriteToken; |
||||||
|
friend class DelayWriteToken; |
||||||
|
|
||||||
|
int total_stopped_; |
||||||
|
uint64_t total_delay_us_; |
||||||
|
}; |
||||||
|
|
||||||
|
class WriteControllerToken { |
||||||
|
public: |
||||||
|
explicit WriteControllerToken(WriteController* controller) |
||||||
|
: controller_(controller) {} |
||||||
|
virtual ~WriteControllerToken() {} |
||||||
|
|
||||||
|
protected: |
||||||
|
WriteController* controller_; |
||||||
|
|
||||||
|
private: |
||||||
|
// no copying allowed
|
||||||
|
WriteControllerToken(const WriteControllerToken&) = delete; |
||||||
|
void operator=(const WriteControllerToken&) = delete; |
||||||
|
}; |
||||||
|
|
||||||
|
class StopWriteToken : public WriteControllerToken { |
||||||
|
public: |
||||||
|
explicit StopWriteToken(WriteController* controller) |
||||||
|
: WriteControllerToken(controller) {} |
||||||
|
virtual ~StopWriteToken(); |
||||||
|
}; |
||||||
|
|
||||||
|
class DelayWriteToken : public WriteControllerToken { |
||||||
|
public: |
||||||
|
DelayWriteToken(WriteController* controller, uint64_t delay_us) |
||||||
|
: WriteControllerToken(controller), delay_us_(delay_us) {} |
||||||
|
virtual ~DelayWriteToken(); |
||||||
|
|
||||||
|
private: |
||||||
|
uint64_t delay_us_; |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,40 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
#include "db/write_controller.h" |
||||||
|
|
||||||
|
#include "util/testharness.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class WriteControllerTest {}; |
||||||
|
|
||||||
|
TEST(WriteControllerTest, SanityTest) { |
||||||
|
WriteController controller; |
||||||
|
auto stop_token_1 = controller.GetStopToken(); |
||||||
|
auto stop_token_2 = controller.GetStopToken(); |
||||||
|
|
||||||
|
ASSERT_EQ(true, controller.IsStopped()); |
||||||
|
stop_token_1.reset(); |
||||||
|
ASSERT_EQ(true, controller.IsStopped()); |
||||||
|
stop_token_2.reset(); |
||||||
|
ASSERT_EQ(false, controller.IsStopped()); |
||||||
|
|
||||||
|
auto delay_token_1 = controller.GetDelayToken(5); |
||||||
|
ASSERT_EQ(static_cast<uint64_t>(5), controller.GetDelay()); |
||||||
|
auto delay_token_2 = controller.GetDelayToken(8); |
||||||
|
ASSERT_EQ(static_cast<uint64_t>(13), controller.GetDelay()); |
||||||
|
|
||||||
|
delay_token_2.reset(); |
||||||
|
ASSERT_EQ(static_cast<uint64_t>(5), controller.GetDelay()); |
||||||
|
delay_token_1.reset(); |
||||||
|
ASSERT_EQ(static_cast<uint64_t>(0), controller.GetDelay()); |
||||||
|
delay_token_1.reset(); |
||||||
|
ASSERT_EQ(false, controller.IsStopped()); |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
|
||||||
|
int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); } |
||||||
@ -0,0 +1,147 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#include "db/write_thread.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
Status WriteThread::EnterWriteThread(WriteThread::Writer* w, |
||||||
|
uint64_t expiration_time) { |
||||||
|
// the following code block pushes the current writer "w" into the writer
|
||||||
|
// queue "writers_" and wait until one of the following conditions met:
|
||||||
|
// 1. the job of "w" has been done by some other writers.
|
||||||
|
// 2. "w" becomes the first writer in "writers_"
|
||||||
|
// 3. "w" timed-out.
|
||||||
|
writers_.push_back(w); |
||||||
|
|
||||||
|
bool timed_out = false; |
||||||
|
while (!w->done && w != writers_.front()) { |
||||||
|
if (expiration_time == 0) { |
||||||
|
w->cv.Wait(); |
||||||
|
} else if (w->cv.TimedWait(expiration_time)) { |
||||||
|
if (w->in_batch_group) { |
||||||
|
// then it means the front writer is currently doing the
|
||||||
|
// write on behalf of this "timed-out" writer. Then it
|
||||||
|
// should wait until the write completes.
|
||||||
|
expiration_time = 0; |
||||||
|
} else { |
||||||
|
timed_out = true; |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if (timed_out) { |
||||||
|
#ifndef NDEBUG |
||||||
|
bool found = false; |
||||||
|
#endif |
||||||
|
for (auto iter = writers_.begin(); iter != writers_.end(); iter++) { |
||||||
|
if (*iter == w) { |
||||||
|
writers_.erase(iter); |
||||||
|
#ifndef NDEBUG |
||||||
|
found = true; |
||||||
|
#endif |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
#ifndef NDEBUG |
||||||
|
assert(found); |
||||||
|
#endif |
||||||
|
// writers_.front() might still be in cond_wait without a time-out.
|
||||||
|
// As a result, we need to signal it to wake it up. Otherwise no
|
||||||
|
// one else will wake him up, and RocksDB will hang.
|
||||||
|
if (!writers_.empty()) { |
||||||
|
writers_.front()->cv.Signal(); |
||||||
|
} |
||||||
|
return Status::TimedOut(); |
||||||
|
} |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
void WriteThread::ExitWriteThread(WriteThread::Writer* w, |
||||||
|
WriteThread::Writer* last_writer, |
||||||
|
Status status) { |
||||||
|
// Pop out the current writer and all writers being pushed before the
|
||||||
|
// current writer from the writer queue.
|
||||||
|
while (!writers_.empty()) { |
||||||
|
Writer* ready = writers_.front(); |
||||||
|
writers_.pop_front(); |
||||||
|
if (ready != w) { |
||||||
|
ready->status = status; |
||||||
|
ready->done = true; |
||||||
|
ready->cv.Signal(); |
||||||
|
} |
||||||
|
if (ready == last_writer) break; |
||||||
|
} |
||||||
|
|
||||||
|
// Notify new head of write queue
|
||||||
|
if (!writers_.empty()) { |
||||||
|
writers_.front()->cv.Signal(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// This function will be called only when the first writer succeeds.
|
||||||
|
// All writers in the to-be-built batch group will be processed.
|
||||||
|
//
|
||||||
|
// REQUIRES: Writer list must be non-empty
|
||||||
|
// REQUIRES: First writer must have a non-nullptr batch
|
||||||
|
void WriteThread::BuildBatchGroup(WriteThread::Writer** last_writer, |
||||||
|
autovector<WriteBatch*>* write_batch_group) { |
||||||
|
assert(!writers_.empty()); |
||||||
|
Writer* first = writers_.front(); |
||||||
|
assert(first->batch != nullptr); |
||||||
|
|
||||||
|
size_t size = WriteBatchInternal::ByteSize(first->batch); |
||||||
|
write_batch_group->push_back(first->batch); |
||||||
|
|
||||||
|
// Allow the group to grow up to a maximum size, but if the
|
||||||
|
// original write is small, limit the growth so we do not slow
|
||||||
|
// down the small write too much.
|
||||||
|
size_t max_size = 1 << 20; |
||||||
|
if (size <= (128<<10)) { |
||||||
|
max_size = size + (128<<10); |
||||||
|
} |
||||||
|
|
||||||
|
*last_writer = first; |
||||||
|
std::deque<Writer*>::iterator iter = writers_.begin(); |
||||||
|
++iter; // Advance past "first"
|
||||||
|
for (; iter != writers_.end(); ++iter) { |
||||||
|
Writer* w = *iter; |
||||||
|
if (w->sync && !first->sync) { |
||||||
|
// Do not include a sync write into a batch handled by a non-sync write.
|
||||||
|
break; |
||||||
|
} |
||||||
|
|
||||||
|
if (!w->disableWAL && first->disableWAL) { |
||||||
|
// Do not include a write that needs WAL into a batch that has
|
||||||
|
// WAL disabled.
|
||||||
|
break; |
||||||
|
} |
||||||
|
|
||||||
|
if (w->timeout_hint_us < first->timeout_hint_us) { |
||||||
|
// Do not include those writes with shorter timeout. Otherwise, we might
|
||||||
|
// execute a write that should instead be aborted because of timeout.
|
||||||
|
break; |
||||||
|
} |
||||||
|
|
||||||
|
if (w->batch == nullptr) { |
||||||
|
// Do not include those writes with nullptr batch. Those are not writes,
|
||||||
|
// those are something else. They want to be alone
|
||||||
|
break; |
||||||
|
} |
||||||
|
|
||||||
|
size += WriteBatchInternal::ByteSize(w->batch); |
||||||
|
if (size > max_size) { |
||||||
|
// Do not make batch too big
|
||||||
|
break; |
||||||
|
} |
||||||
|
|
||||||
|
write_batch_group->push_back(w->batch); |
||||||
|
w->in_batch_group = true; |
||||||
|
*last_writer = w; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,80 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <stdint.h> |
||||||
|
#include <deque> |
||||||
|
#include <limits> |
||||||
|
#include "rocksdb/status.h" |
||||||
|
#include "db/write_batch_internal.h" |
||||||
|
#include "util/autovector.h" |
||||||
|
#include "port/port.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class WriteThread { |
||||||
|
public: |
||||||
|
static const uint64_t kNoTimeOut = std::numeric_limits<uint64_t>::max(); |
||||||
|
// Information kept for every waiting writer
|
||||||
|
struct Writer { |
||||||
|
Status status; |
||||||
|
WriteBatch* batch; |
||||||
|
bool sync; |
||||||
|
bool disableWAL; |
||||||
|
bool in_batch_group; |
||||||
|
bool done; |
||||||
|
uint64_t timeout_hint_us; |
||||||
|
port::CondVar cv; |
||||||
|
|
||||||
|
explicit Writer(port::Mutex* mu) |
||||||
|
: batch(nullptr), |
||||||
|
sync(false), |
||||||
|
disableWAL(false), |
||||||
|
in_batch_group(false), |
||||||
|
done(false), |
||||||
|
timeout_hint_us(kNoTimeOut), |
||||||
|
cv(mu) {} |
||||||
|
}; |
||||||
|
|
||||||
|
WriteThread() = default; |
||||||
|
~WriteThread() = default; |
||||||
|
|
||||||
|
// Before applying write operation (such as DBImpl::Write, DBImpl::Flush)
|
||||||
|
// thread should grab the mutex_ and be the first on writers queue.
|
||||||
|
// EnterWriteThread is used for it.
|
||||||
|
// Be aware! Writer's job can be done by other thread (see DBImpl::Write
|
||||||
|
// for examples), so check it via w.done before applying changes.
|
||||||
|
//
|
||||||
|
// Writer* w: writer to be placed in the queue
|
||||||
|
// uint64_t expiration_time: maximum time to be in the queue
|
||||||
|
// See also: ExitWriteThread
|
||||||
|
// REQUIRES: db mutex held
|
||||||
|
Status EnterWriteThread(Writer* w, uint64_t expiration_time); |
||||||
|
|
||||||
|
// After doing write job, we need to remove already used writers from
|
||||||
|
// writers_ queue and notify head of the queue about it.
|
||||||
|
// ExitWriteThread is used for this.
|
||||||
|
//
|
||||||
|
// Writer* w: Writer, that was added by EnterWriteThread function
|
||||||
|
// Writer* last_writer: Since we can join a few Writers (as DBImpl::Write
|
||||||
|
// does)
|
||||||
|
// we should pass last_writer as a parameter to
|
||||||
|
// ExitWriteThread
|
||||||
|
// (if you don't touch other writers, just pass w)
|
||||||
|
// Status status: Status of write operation
|
||||||
|
// See also: EnterWriteThread
|
||||||
|
// REQUIRES: db mutex held
|
||||||
|
void ExitWriteThread(Writer* w, Writer* last_writer, Status status); |
||||||
|
|
||||||
|
void BuildBatchGroup(Writer** last_writer, |
||||||
|
autovector<WriteBatch*>* write_batch_group); |
||||||
|
|
||||||
|
private: |
||||||
|
// Queue of writers.
|
||||||
|
std::deque<Writer*> writers_; |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,84 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <vector> |
||||||
|
#include "rocksdb/options.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
// ImmutableCFOptions is a data struct used by RocksDB internal. It contains a
|
||||||
|
// subset of Options that should not be changed during the entire lifetime
|
||||||
|
// of DB. You shouldn't need to access this data structure unless you are
|
||||||
|
// implementing a new TableFactory. Raw pointers defined in this struct do
|
||||||
|
// not have ownership to the data they point to. Options contains shared_ptr
|
||||||
|
// to these data.
|
||||||
|
struct ImmutableCFOptions { |
||||||
|
explicit ImmutableCFOptions(const Options& options); |
||||||
|
|
||||||
|
CompactionStyle compaction_style; |
||||||
|
|
||||||
|
CompactionOptionsUniversal compaction_options_universal; |
||||||
|
|
||||||
|
const SliceTransform* prefix_extractor; |
||||||
|
|
||||||
|
const Comparator* comparator; |
||||||
|
|
||||||
|
MergeOperator* merge_operator; |
||||||
|
|
||||||
|
const CompactionFilter* compaction_filter; |
||||||
|
|
||||||
|
CompactionFilterFactory* compaction_filter_factory; |
||||||
|
|
||||||
|
CompactionFilterFactoryV2* compaction_filter_factory_v2; |
||||||
|
|
||||||
|
Logger* info_log; |
||||||
|
|
||||||
|
Statistics* statistics; |
||||||
|
|
||||||
|
InfoLogLevel info_log_level; |
||||||
|
|
||||||
|
Env* env; |
||||||
|
|
||||||
|
// Allow the OS to mmap file for reading sst tables. Default: false
|
||||||
|
bool allow_mmap_reads; |
||||||
|
|
||||||
|
// Allow the OS to mmap file for writing. Default: false
|
||||||
|
bool allow_mmap_writes; |
||||||
|
|
||||||
|
std::vector<DbPath> db_paths; |
||||||
|
|
||||||
|
MemTableRepFactory* memtable_factory; |
||||||
|
|
||||||
|
TableFactory* table_factory; |
||||||
|
|
||||||
|
Options::TablePropertiesCollectorFactories |
||||||
|
table_properties_collector_factories; |
||||||
|
|
||||||
|
bool advise_random_on_open; |
||||||
|
|
||||||
|
// This options is required by PlainTableReader. May need to move it
|
||||||
|
// to PlainTalbeOptions just like bloom_bits_per_key
|
||||||
|
uint32_t bloom_locality; |
||||||
|
|
||||||
|
bool purge_redundant_kvs_while_flush; |
||||||
|
|
||||||
|
uint32_t min_partial_merge_operands; |
||||||
|
|
||||||
|
bool disable_data_sync; |
||||||
|
|
||||||
|
bool use_fsync; |
||||||
|
|
||||||
|
CompressionType compression; |
||||||
|
|
||||||
|
std::vector<CompressionType> compression_per_level; |
||||||
|
|
||||||
|
CompressionOptions compression_opts; |
||||||
|
|
||||||
|
Options::AccessHint access_hint_on_compaction_start; |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,105 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
//
|
||||||
|
// A WriteBatchWithIndex with a binary searchable index built for all the keys
|
||||||
|
// inserted.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include "rocksdb/comparator.h" |
||||||
|
#include "rocksdb/slice.h" |
||||||
|
#include "rocksdb/status.h" |
||||||
|
#include "rocksdb/write_batch.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class ColumnFamilyHandle; |
||||||
|
struct SliceParts; |
||||||
|
class Comparator; |
||||||
|
|
||||||
|
enum WriteType { kPutRecord, kMergeRecord, kDeleteRecord, kLogDataRecord }; |
||||||
|
|
||||||
|
// an entry for Put, Merge or Delete entry for write batches. Used in
|
||||||
|
// WBWIIterator.
|
||||||
|
struct WriteEntry { |
||||||
|
WriteType type; |
||||||
|
Slice key; |
||||||
|
Slice value; |
||||||
|
}; |
||||||
|
|
||||||
|
// Iterator of one column family out of a WriteBatchWithIndex.
|
||||||
|
class WBWIIterator { |
||||||
|
public: |
||||||
|
virtual ~WBWIIterator() {} |
||||||
|
|
||||||
|
virtual bool Valid() const = 0; |
||||||
|
|
||||||
|
virtual void Seek(const Slice& key) = 0; |
||||||
|
|
||||||
|
virtual void Next() = 0; |
||||||
|
|
||||||
|
virtual const WriteEntry& Entry() const = 0; |
||||||
|
|
||||||
|
virtual Status status() const = 0; |
||||||
|
}; |
||||||
|
|
||||||
|
// A WriteBatchWithIndex with a binary searchable index built for all the keys
|
||||||
|
// inserted.
|
||||||
|
// In Put(), Merge() or Delete(), the same function of the wrapped will be
|
||||||
|
// called. At the same time, indexes will be built.
|
||||||
|
// By calling GetWriteBatch(), a user will get the WriteBatch for the data
|
||||||
|
// they inserted, which can be used for DB::Write().
|
||||||
|
// A user can call NewIterator() to create an iterator.
|
||||||
|
class WriteBatchWithIndex { |
||||||
|
public: |
||||||
|
// backup_index_comparator: the backup comparator used to compare keys
|
||||||
|
// within the same column family, if column family is not given in the
|
||||||
|
// interface, or we can't find a column family from the column family handle
|
||||||
|
// passed in, backup_index_comparator will be used for the column family.
|
||||||
|
// reserved_bytes: reserved bytes in underlying WriteBatch
|
||||||
|
explicit WriteBatchWithIndex( |
||||||
|
const Comparator* backup_index_comparator = BytewiseComparator(), |
||||||
|
size_t reserved_bytes = 0); |
||||||
|
virtual ~WriteBatchWithIndex(); |
||||||
|
|
||||||
|
WriteBatch* GetWriteBatch(); |
||||||
|
|
||||||
|
virtual void Put(ColumnFamilyHandle* column_family, const Slice& key, |
||||||
|
const Slice& value); |
||||||
|
|
||||||
|
virtual void Put(const Slice& key, const Slice& value); |
||||||
|
|
||||||
|
virtual void Merge(ColumnFamilyHandle* column_family, const Slice& key, |
||||||
|
const Slice& value); |
||||||
|
|
||||||
|
virtual void Merge(const Slice& key, const Slice& value); |
||||||
|
|
||||||
|
virtual void PutLogData(const Slice& blob); |
||||||
|
|
||||||
|
virtual void Delete(ColumnFamilyHandle* column_family, const Slice& key); |
||||||
|
virtual void Delete(const Slice& key); |
||||||
|
|
||||||
|
virtual void Delete(ColumnFamilyHandle* column_family, const SliceParts& key); |
||||||
|
|
||||||
|
virtual void Delete(const SliceParts& key); |
||||||
|
|
||||||
|
// Create an iterator of a column family. User can call iterator.Seek() to
|
||||||
|
// search to the next entry of or after a key. Keys will be iterated in the
|
||||||
|
// order given by index_comparator. For multiple updates on the same key,
|
||||||
|
// each update will be returned as a separate entry, in the order of update
|
||||||
|
// time.
|
||||||
|
virtual WBWIIterator* NewIterator(ColumnFamilyHandle* column_family); |
||||||
|
// Create an iterator of the default column family.
|
||||||
|
virtual WBWIIterator* NewIterator(); |
||||||
|
|
||||||
|
private: |
||||||
|
struct Rep; |
||||||
|
Rep* rep; |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,210 @@ |
|||||||
|
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
package org.rocksdb; |
||||||
|
|
||||||
|
/** |
||||||
|
* The config for plain table sst format. |
||||||
|
* |
||||||
|
* BlockBasedTable is a RocksDB's default SST file format. |
||||||
|
*/ |
||||||
|
public class BlockBasedTableConfig extends TableFormatConfig { |
||||||
|
|
||||||
|
public BlockBasedTableConfig() { |
||||||
|
noBlockCache_ = false; |
||||||
|
blockCacheSize_ = 8 * 1024 * 1024; |
||||||
|
blockSize_ = 4 * 1024; |
||||||
|
blockSizeDeviation_ =10; |
||||||
|
blockRestartInterval_ =16; |
||||||
|
wholeKeyFiltering_ = true; |
||||||
|
bitsPerKey_ = 0; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Disable block cache. If this is set to true, |
||||||
|
* then no block cache should be used, and the block_cache should |
||||||
|
* point to a nullptr object. |
||||||
|
* Default: false |
||||||
|
* |
||||||
|
* @param noBlockCache if use block cache |
||||||
|
* @return the reference to the current config. |
||||||
|
*/ |
||||||
|
public BlockBasedTableConfig setNoBlockCache(boolean noBlockCache) { |
||||||
|
noBlockCache_ = noBlockCache; |
||||||
|
return this; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @return if block cache is disabled |
||||||
|
*/ |
||||||
|
public boolean noBlockCache() { |
||||||
|
return noBlockCache_; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Set the amount of cache in bytes that will be used by RocksDB. |
||||||
|
* If cacheSize is non-positive, then cache will not be used. |
||||||
|
* DEFAULT: 8M |
||||||
|
* |
||||||
|
* @param blockCacheSize block cache size in bytes |
||||||
|
* @return the reference to the current config. |
||||||
|
*/ |
||||||
|
public BlockBasedTableConfig setBlockCacheSize(long blockCacheSize) { |
||||||
|
blockCacheSize_ = blockCacheSize; |
||||||
|
return this; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @return block cache size in bytes |
||||||
|
*/ |
||||||
|
public long blockCacheSize() { |
||||||
|
return blockCacheSize_; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Controls the number of shards for the block cache. |
||||||
|
* This is applied only if cacheSize is set to non-negative. |
||||||
|
* |
||||||
|
* @param numShardBits the number of shard bits. The resulting |
||||||
|
* number of shards would be 2 ^ numShardBits. Any negative |
||||||
|
* number means use default settings." |
||||||
|
* @return the reference to the current option. |
||||||
|
*/ |
||||||
|
public BlockBasedTableConfig setCacheNumShardBits(int numShardBits) { |
||||||
|
numShardBits_ = numShardBits; |
||||||
|
return this; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Returns the number of shard bits used in the block cache. |
||||||
|
* The resulting number of shards would be 2 ^ (returned value). |
||||||
|
* Any negative number means use default settings. |
||||||
|
* |
||||||
|
* @return the number of shard bits used in the block cache. |
||||||
|
*/ |
||||||
|
public int cacheNumShardBits() { |
||||||
|
return numShardBits_; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Approximate size of user data packed per block. Note that the |
||||||
|
* block size specified here corresponds to uncompressed data. The |
||||||
|
* actual size of the unit read from disk may be smaller if |
||||||
|
* compression is enabled. This parameter can be changed dynamically. |
||||||
|
* Default: 4K |
||||||
|
* |
||||||
|
* @param blockSize block size in bytes |
||||||
|
* @return the reference to the current config. |
||||||
|
*/ |
||||||
|
public BlockBasedTableConfig setBlockSize(long blockSize) { |
||||||
|
blockSize_ = blockSize; |
||||||
|
return this; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @return block size in bytes |
||||||
|
*/ |
||||||
|
public long blockSize() { |
||||||
|
return blockSize_; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* This is used to close a block before it reaches the configured |
||||||
|
* 'block_size'. If the percentage of free space in the current block is less |
||||||
|
* than this specified number and adding a new record to the block will |
||||||
|
* exceed the configured block size, then this block will be closed and the |
||||||
|
* new record will be written to the next block. |
||||||
|
* Default is 10. |
||||||
|
* |
||||||
|
* @param blockSizeDeviation the deviation to block size allowed |
||||||
|
* @return the reference to the current config. |
||||||
|
*/ |
||||||
|
public BlockBasedTableConfig setBlockSizeDeviation(int blockSizeDeviation) { |
||||||
|
blockSizeDeviation_ = blockSizeDeviation; |
||||||
|
return this; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @return the hash table ratio. |
||||||
|
*/ |
||||||
|
public int blockSizeDeviation() { |
||||||
|
return blockSizeDeviation_; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Set block restart interval |
||||||
|
* |
||||||
|
* @param restartInterval block restart interval. |
||||||
|
* @return the reference to the current config. |
||||||
|
*/ |
||||||
|
public BlockBasedTableConfig setBlockRestartInterval(int restartInterval) { |
||||||
|
blockRestartInterval_ = restartInterval; |
||||||
|
return this; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @return block restart interval |
||||||
|
*/ |
||||||
|
public int blockRestartInterval() { |
||||||
|
return blockRestartInterval_; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* If true, place whole keys in the filter (not just prefixes). |
||||||
|
* This must generally be true for gets to be efficient. |
||||||
|
* Default: true |
||||||
|
* |
||||||
|
* @param wholeKeyFiltering if enable whole key filtering |
||||||
|
* @return the reference to the current config. |
||||||
|
*/ |
||||||
|
public BlockBasedTableConfig setWholeKeyFiltering(boolean wholeKeyFiltering) { |
||||||
|
wholeKeyFiltering_ = wholeKeyFiltering; |
||||||
|
return this; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* @return if whole key filtering is enabled |
||||||
|
*/ |
||||||
|
public boolean wholeKeyFiltering() { |
||||||
|
return wholeKeyFiltering_; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Use the specified filter policy to reduce disk reads. |
||||||
|
* |
||||||
|
* Filter should not be disposed before options instances using this filter is |
||||||
|
* disposed. If dispose() function is not called, then filter object will be |
||||||
|
* GC'd automatically. |
||||||
|
* |
||||||
|
* Filter instance can be re-used in multiple options instances. |
||||||
|
* |
||||||
|
* @param Filter policy java instance. |
||||||
|
* @return the reference to the current config. |
||||||
|
*/ |
||||||
|
public BlockBasedTableConfig setFilterBitsPerKey(int bitsPerKey) { |
||||||
|
bitsPerKey_ = bitsPerKey; |
||||||
|
return this; |
||||||
|
} |
||||||
|
|
||||||
|
@Override protected long newTableFactoryHandle() { |
||||||
|
return newTableFactoryHandle(noBlockCache_, blockCacheSize_, numShardBits_, |
||||||
|
blockSize_, blockSizeDeviation_, blockRestartInterval_, |
||||||
|
wholeKeyFiltering_, bitsPerKey_); |
||||||
|
} |
||||||
|
|
||||||
|
private native long newTableFactoryHandle( |
||||||
|
boolean noBlockCache, long blockCacheSize, int numShardbits, |
||||||
|
long blockSize, int blockSizeDeviation, int blockRestartInterval, |
||||||
|
boolean wholeKeyFiltering, int bitsPerKey); |
||||||
|
|
||||||
|
private boolean noBlockCache_; |
||||||
|
private long blockCacheSize_; |
||||||
|
private int numShardBits_; |
||||||
|
private long shard; |
||||||
|
private long blockSize_; |
||||||
|
private int blockSizeDeviation_; |
||||||
|
private int blockRestartInterval_; |
||||||
|
private boolean wholeKeyFiltering_; |
||||||
|
private int bitsPerKey_; |
||||||
|
} |
||||||
@ -0,0 +1,36 @@ |
|||||||
|
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
package org.rocksdb; |
||||||
|
|
||||||
|
/** |
||||||
|
* Config for rate limiter, which is used to control write rate of flush and |
||||||
|
* compaction. |
||||||
|
*/ |
||||||
|
public class GenericRateLimiterConfig extends RateLimiterConfig { |
||||||
|
private static final long DEFAULT_REFILL_PERIOD_MICROS = (100 * 1000); |
||||||
|
private static final int DEFAULT_FAIRNESS = 10; |
||||||
|
|
||||||
|
public GenericRateLimiterConfig(long rateBytesPerSecond, |
||||||
|
long refillPeriodMicros, int fairness) { |
||||||
|
rateBytesPerSecond_ = rateBytesPerSecond; |
||||||
|
refillPeriodMicros_ = refillPeriodMicros; |
||||||
|
fairness_ = fairness; |
||||||
|
} |
||||||
|
|
||||||
|
public GenericRateLimiterConfig(long rateBytesPerSecond) { |
||||||
|
this(rateBytesPerSecond, DEFAULT_REFILL_PERIOD_MICROS, DEFAULT_FAIRNESS); |
||||||
|
} |
||||||
|
|
||||||
|
@Override protected long newRateLimiterHandle() { |
||||||
|
return newRateLimiterHandle(rateBytesPerSecond_, refillPeriodMicros_, |
||||||
|
fairness_); |
||||||
|
} |
||||||
|
|
||||||
|
private native long newRateLimiterHandle(long rateBytesPerSecond, |
||||||
|
long refillPeriodMicros, int fairness); |
||||||
|
private final long rateBytesPerSecond_; |
||||||
|
private final long refillPeriodMicros_; |
||||||
|
private final int fairness_; |
||||||
|
} |
||||||
@ -0,0 +1,20 @@ |
|||||||
|
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
package org.rocksdb; |
||||||
|
|
||||||
|
/** |
||||||
|
* Config for rate limiter, which is used to control write rate of flush and |
||||||
|
* compaction. |
||||||
|
*/ |
||||||
|
public abstract class RateLimiterConfig { |
||||||
|
/** |
||||||
|
* This function should only be called by Options.setRateLimiter(), |
||||||
|
* which will create a c++ shared-pointer to the c++ RateLimiter |
||||||
|
* that is associated with the Java RateLimtierConifg. |
||||||
|
* |
||||||
|
* @see Options.setRateLimiter() |
||||||
|
*/ |
||||||
|
abstract protected long newRateLimiterHandle(); |
||||||
|
} |
||||||
@ -0,0 +1,24 @@ |
|||||||
|
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// This file implements the "bridge" between Java and C++ for RateLimiter.
|
||||||
|
|
||||||
|
#include "rocksjni/portal.h" |
||||||
|
#include "include/org_rocksdb_GenericRateLimiterConfig.h" |
||||||
|
#include "rocksdb/rate_limiter.h" |
||||||
|
|
||||||
|
/*
|
||||||
|
* Class: org_rocksdb_GenericRateLimiterConfig |
||||||
|
* Method: newRateLimiterHandle |
||||||
|
* Signature: (JJI)J |
||||||
|
*/ |
||||||
|
jlong Java_org_rocksdb_GenericRateLimiterConfig_newRateLimiterHandle( |
||||||
|
JNIEnv* env, jobject jobj, jlong jrate_bytes_per_second, |
||||||
|
jlong jrefill_period_micros, jint jfairness) { |
||||||
|
return reinterpret_cast<jlong>(rocksdb::NewGenericRateLimiter( |
||||||
|
rocksdb::jlong_to_size_t(jrate_bytes_per_second), |
||||||
|
rocksdb::jlong_to_size_t(jrefill_period_micros), |
||||||
|
static_cast<int32_t>(jfairness))); |
||||||
|
} |
||||||
@ -0,0 +1,101 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
//
|
||||||
|
// A filter block is stored near the end of a Table file. It contains
|
||||||
|
// filters (e.g., bloom filters) for all data blocks in the table combined
|
||||||
|
// into a single filter block.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <stddef.h> |
||||||
|
#include <stdint.h> |
||||||
|
#include <string> |
||||||
|
#include <memory> |
||||||
|
#include <vector> |
||||||
|
#include "rocksdb/options.h" |
||||||
|
#include "rocksdb/slice.h" |
||||||
|
#include "rocksdb/slice_transform.h" |
||||||
|
#include "table/filter_block.h" |
||||||
|
#include "util/hash.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
|
||||||
|
// A BlockBasedFilterBlockBuilder is used to construct all of the filters for a
|
||||||
|
// particular Table. It generates a single string which is stored as
|
||||||
|
// a special block in the Table.
|
||||||
|
//
|
||||||
|
// The sequence of calls to BlockBasedFilterBlockBuilder must match the regexp:
|
||||||
|
// (StartBlock Add*)* Finish
|
||||||
|
class BlockBasedFilterBlockBuilder : public FilterBlockBuilder { |
||||||
|
public: |
||||||
|
BlockBasedFilterBlockBuilder(const SliceTransform* prefix_extractor, |
||||||
|
const BlockBasedTableOptions& table_opt); |
||||||
|
|
||||||
|
virtual bool IsBlockBased() override { return true; } |
||||||
|
virtual void StartBlock(uint64_t block_offset) override; |
||||||
|
virtual void Add(const Slice& key) override; |
||||||
|
virtual Slice Finish() override; |
||||||
|
|
||||||
|
private: |
||||||
|
void AddKey(const Slice& key); |
||||||
|
void AddPrefix(const Slice& key); |
||||||
|
void GenerateFilter(); |
||||||
|
|
||||||
|
// important: all of these might point to invalid addresses
|
||||||
|
// at the time of destruction of this filter block. destructor
|
||||||
|
// should NOT dereference them.
|
||||||
|
const FilterPolicy* policy_; |
||||||
|
const SliceTransform* prefix_extractor_; |
||||||
|
bool whole_key_filtering_; |
||||||
|
|
||||||
|
std::string entries_; // Flattened entry contents
|
||||||
|
std::vector<size_t> start_; // Starting index in entries_ of each entry
|
||||||
|
uint32_t added_to_start_; // To indicate if key is added
|
||||||
|
std::string result_; // Filter data computed so far
|
||||||
|
std::vector<Slice> tmp_entries_; // policy_->CreateFilter() argument
|
||||||
|
std::vector<uint32_t> filter_offsets_; |
||||||
|
|
||||||
|
// No copying allowed
|
||||||
|
BlockBasedFilterBlockBuilder(const BlockBasedFilterBlockBuilder&); |
||||||
|
void operator=(const BlockBasedFilterBlockBuilder&); |
||||||
|
}; |
||||||
|
|
||||||
|
// A FilterBlockReader is used to parse filter from SST table.
|
||||||
|
// KeyMayMatch and PrefixMayMatch would trigger filter checking
|
||||||
|
class BlockBasedFilterBlockReader : public FilterBlockReader { |
||||||
|
public: |
||||||
|
// REQUIRES: "contents" and *policy must stay live while *this is live.
|
||||||
|
BlockBasedFilterBlockReader(const SliceTransform* prefix_extractor, |
||||||
|
const BlockBasedTableOptions& table_opt, |
||||||
|
BlockContents&& contents); |
||||||
|
virtual bool IsBlockBased() override { return true; } |
||||||
|
virtual bool KeyMayMatch(const Slice& key, |
||||||
|
uint64_t block_offset = kNotValid) override; |
||||||
|
virtual bool PrefixMayMatch(const Slice& prefix, |
||||||
|
uint64_t block_offset = kNotValid) override; |
||||||
|
virtual size_t ApproximateMemoryUsage() const override; |
||||||
|
|
||||||
|
private: |
||||||
|
const FilterPolicy* policy_; |
||||||
|
const SliceTransform* prefix_extractor_; |
||||||
|
bool whole_key_filtering_; |
||||||
|
const char* data_; // Pointer to filter data (at block-start)
|
||||||
|
const char* offset_; // Pointer to beginning of offset array (at block-end)
|
||||||
|
size_t num_; // Number of entries in offset array
|
||||||
|
size_t base_lg_; // Encoding parameter (see kFilterBaseLg in .cc file)
|
||||||
|
BlockContents contents_; |
||||||
|
|
||||||
|
bool MayMatch(const Slice& entry, uint64_t block_offset); |
||||||
|
|
||||||
|
// No copying allowed
|
||||||
|
BlockBasedFilterBlockReader(const BlockBasedFilterBlockReader&); |
||||||
|
void operator=(const BlockBasedFilterBlockReader&); |
||||||
|
}; |
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,242 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
|
#include "table/block_based_filter_block.h" |
||||||
|
|
||||||
|
#include "rocksdb/filter_policy.h" |
||||||
|
#include "util/coding.h" |
||||||
|
#include "util/hash.h" |
||||||
|
#include "util/logging.h" |
||||||
|
#include "util/testharness.h" |
||||||
|
#include "util/testutil.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
// For testing: emit an array with one hash value per key
|
||||||
|
class TestHashFilter : public FilterPolicy { |
||||||
|
public: |
||||||
|
virtual const char* Name() const { |
||||||
|
return "TestHashFilter"; |
||||||
|
} |
||||||
|
|
||||||
|
virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const { |
||||||
|
for (int i = 0; i < n; i++) { |
||||||
|
uint32_t h = Hash(keys[i].data(), keys[i].size(), 1); |
||||||
|
PutFixed32(dst, h); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const { |
||||||
|
uint32_t h = Hash(key.data(), key.size(), 1); |
||||||
|
for (unsigned int i = 0; i + 4 <= filter.size(); i += 4) { |
||||||
|
if (h == DecodeFixed32(filter.data() + i)) { |
||||||
|
return true; |
||||||
|
} |
||||||
|
} |
||||||
|
return false; |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
class FilterBlockTest { |
||||||
|
public: |
||||||
|
TestHashFilter policy_; |
||||||
|
BlockBasedTableOptions table_options_; |
||||||
|
|
||||||
|
FilterBlockTest() { |
||||||
|
table_options_.filter_policy.reset(new TestHashFilter()); |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
TEST(FilterBlockTest, EmptyBuilder) { |
||||||
|
BlockBasedFilterBlockBuilder builder(nullptr, table_options_); |
||||||
|
BlockContents block(builder.Finish(), false, kNoCompression); |
||||||
|
ASSERT_EQ("\\x00\\x00\\x00\\x00\\x0b", EscapeString(block.data)); |
||||||
|
BlockBasedFilterBlockReader reader(nullptr, table_options_, std::move(block)); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("foo", 0)); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("foo", 100000)); |
||||||
|
} |
||||||
|
|
||||||
|
TEST(FilterBlockTest, SingleChunk) { |
||||||
|
BlockBasedFilterBlockBuilder builder(nullptr, table_options_); |
||||||
|
builder.StartBlock(100); |
||||||
|
builder.Add("foo"); |
||||||
|
builder.Add("bar"); |
||||||
|
builder.Add("box"); |
||||||
|
builder.StartBlock(200); |
||||||
|
builder.Add("box"); |
||||||
|
builder.StartBlock(300); |
||||||
|
builder.Add("hello"); |
||||||
|
BlockContents block(builder.Finish(), false, kNoCompression); |
||||||
|
BlockBasedFilterBlockReader reader(nullptr, table_options_, std::move(block)); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("foo", 100)); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("bar", 100)); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("box", 100)); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("hello", 100)); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("foo", 100)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("missing", 100)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("other", 100)); |
||||||
|
} |
||||||
|
|
||||||
|
TEST(FilterBlockTest, MultiChunk) { |
||||||
|
BlockBasedFilterBlockBuilder builder(nullptr, table_options_); |
||||||
|
|
||||||
|
// First filter
|
||||||
|
builder.StartBlock(0); |
||||||
|
builder.Add("foo"); |
||||||
|
builder.StartBlock(2000); |
||||||
|
builder.Add("bar"); |
||||||
|
|
||||||
|
// Second filter
|
||||||
|
builder.StartBlock(3100); |
||||||
|
builder.Add("box"); |
||||||
|
|
||||||
|
// Third filter is empty
|
||||||
|
|
||||||
|
// Last filter
|
||||||
|
builder.StartBlock(9000); |
||||||
|
builder.Add("box"); |
||||||
|
builder.Add("hello"); |
||||||
|
|
||||||
|
BlockContents block(builder.Finish(), false, kNoCompression); |
||||||
|
BlockBasedFilterBlockReader reader(nullptr, table_options_, std::move(block)); |
||||||
|
|
||||||
|
// Check first filter
|
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("foo", 0)); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("bar", 2000)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("box", 0)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("hello", 0)); |
||||||
|
|
||||||
|
// Check second filter
|
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("box", 3100)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("foo", 3100)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("bar", 3100)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("hello", 3100)); |
||||||
|
|
||||||
|
// Check third filter (empty)
|
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("foo", 4100)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("bar", 4100)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("box", 4100)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("hello", 4100)); |
||||||
|
|
||||||
|
// Check last filter
|
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("box", 9000)); |
||||||
|
ASSERT_TRUE(reader.KeyMayMatch("hello", 9000)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("foo", 9000)); |
||||||
|
ASSERT_TRUE(!reader.KeyMayMatch("bar", 9000)); |
||||||
|
} |
||||||
|
|
||||||
|
// Test for block based filter block
|
||||||
|
// use new interface in FilterPolicy to create filter builder/reader
|
||||||
|
class BlockBasedFilterBlockTest { |
||||||
|
public: |
||||||
|
BlockBasedTableOptions table_options_; |
||||||
|
|
||||||
|
BlockBasedFilterBlockTest() { |
||||||
|
table_options_.filter_policy.reset(NewBloomFilterPolicy(10)); |
||||||
|
} |
||||||
|
|
||||||
|
~BlockBasedFilterBlockTest() {} |
||||||
|
}; |
||||||
|
|
||||||
|
TEST(BlockBasedFilterBlockTest, BlockBasedEmptyBuilder) { |
||||||
|
FilterBlockBuilder* builder = new BlockBasedFilterBlockBuilder( |
||||||
|
nullptr, table_options_); |
||||||
|
BlockContents block(builder->Finish(), false, kNoCompression); |
||||||
|
ASSERT_EQ("\\x00\\x00\\x00\\x00\\x0b", EscapeString(block.data)); |
||||||
|
FilterBlockReader* reader = new BlockBasedFilterBlockReader( |
||||||
|
nullptr, table_options_, std::move(block)); |
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("foo", 0)); |
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("foo", 100000)); |
||||||
|
|
||||||
|
delete builder; |
||||||
|
delete reader; |
||||||
|
} |
||||||
|
|
||||||
|
TEST(BlockBasedFilterBlockTest, BlockBasedSingleChunk) { |
||||||
|
FilterBlockBuilder* builder = new BlockBasedFilterBlockBuilder( |
||||||
|
nullptr, table_options_); |
||||||
|
builder->StartBlock(100); |
||||||
|
builder->Add("foo"); |
||||||
|
builder->Add("bar"); |
||||||
|
builder->Add("box"); |
||||||
|
builder->StartBlock(200); |
||||||
|
builder->Add("box"); |
||||||
|
builder->StartBlock(300); |
||||||
|
builder->Add("hello"); |
||||||
|
BlockContents block(builder->Finish(), false, kNoCompression); |
||||||
|
FilterBlockReader* reader = new BlockBasedFilterBlockReader( |
||||||
|
nullptr, table_options_, std::move(block)); |
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("foo", 100)); |
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("bar", 100)); |
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("box", 100)); |
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("hello", 100)); |
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("foo", 100)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("missing", 100)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("other", 100)); |
||||||
|
|
||||||
|
delete builder; |
||||||
|
delete reader; |
||||||
|
} |
||||||
|
|
||||||
|
TEST(BlockBasedFilterBlockTest, BlockBasedMultiChunk) { |
||||||
|
FilterBlockBuilder* builder = new BlockBasedFilterBlockBuilder( |
||||||
|
nullptr, table_options_); |
||||||
|
|
||||||
|
// First filter
|
||||||
|
builder->StartBlock(0); |
||||||
|
builder->Add("foo"); |
||||||
|
builder->StartBlock(2000); |
||||||
|
builder->Add("bar"); |
||||||
|
|
||||||
|
// Second filter
|
||||||
|
builder->StartBlock(3100); |
||||||
|
builder->Add("box"); |
||||||
|
|
||||||
|
// Third filter is empty
|
||||||
|
|
||||||
|
// Last filter
|
||||||
|
builder->StartBlock(9000); |
||||||
|
builder->Add("box"); |
||||||
|
builder->Add("hello"); |
||||||
|
|
||||||
|
BlockContents block(builder->Finish(), false, kNoCompression); |
||||||
|
FilterBlockReader* reader = new BlockBasedFilterBlockReader( |
||||||
|
nullptr, table_options_, std::move(block)); |
||||||
|
|
||||||
|
// Check first filter
|
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("foo", 0)); |
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("bar", 2000)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("box", 0)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("hello", 0)); |
||||||
|
|
||||||
|
// Check second filter
|
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("box", 3100)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("foo", 3100)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("bar", 3100)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("hello", 3100)); |
||||||
|
|
||||||
|
// Check third filter (empty)
|
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("foo", 4100)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("bar", 4100)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("box", 4100)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("hello", 4100)); |
||||||
|
|
||||||
|
// Check last filter
|
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("box", 9000)); |
||||||
|
ASSERT_TRUE(reader->KeyMayMatch("hello", 9000)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("foo", 9000)); |
||||||
|
ASSERT_TRUE(!reader->KeyMayMatch("bar", 9000)); |
||||||
|
|
||||||
|
delete builder; |
||||||
|
delete reader; |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
|
||||||
|
int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); } |
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue