NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Consolidate SliceTransform object ownership

Browse Source 
Summary:
(1) Fix SanitizeOptions() to also check HashLinkList. The current
dynamic case just happens to work because the 2 classes have the same
layout.
(2) Do not delete SliceTransform object in HashSkipListFactory and
HashLinkListFactory destructor. Reason: SanitizeOptions() enforces
prefix_extractor and SliceTransform to be the same object when
Hash**Factory is used. This makes the behavior strange: when
Hash**Factory is used, prefix_extractor will be released by RocksDB. If
other memtable factory is used, prefix_extractor should be released by
user.

Test Plan: db_bench && make asan_check

Reviewers: haobo, igor, sdong

Reviewed By: igor

CC: leveldb, dhruba

Differential Revision: https://reviews.facebook.net/D16587
main
Lei Jin 11 years ago
parent 9e0e6aa7f6
commit 8d007b4aaf
23 changed files with 137 additions and 170 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      HISTORY.md
  2. 28
      db/c.cc
  3. 9
      db/db_bench.cc
  4. 24
      db/db_impl.cc
  5. 29
      db/db_test.cc
  6. 5
      db/memtable.cc
  7. 6
      db/plain_table_db_test.cc
  8. 11
      db/prefix_test.cc
  9. 6
      db/skiplist.h
  10. 2
      db/tailing_iter.cc
  11. 20
      include/rocksdb/memtablerep.h
  12. 2
      include/rocksdb/options.h
  13. 4
      table/filter_block.cc
  14. 8
      table/table_reader_bench.cc
  15. 11
      table/table_test.cc
  16. 14
      tools/db_stress.cc
  17. 2
      tools/sst_dump.cc
  18. 18
      util/hash_linklist_rep.cc
  19. 17
      util/hash_linklist_rep.h
  20. 21
      util/hash_skiplist_rep.cc
  21. 14
      util/hash_skiplist_rep.h
  22. 11
      util/skiplistrep.cc
  23. 3
      util/vectorrep.cc

2
HISTORY.md
Unescape View File

@ -10,6 +10,8 @@
* Added "virtual void WaitForJoin() = 0" in class Env * Added "virtual void WaitForJoin() = 0" in class Env
* Removed BackupEngine::DeleteBackupsNewerThan() function * Removed BackupEngine::DeleteBackupsNewerThan() function
* Added new option -- verify_checksums_in_compaction * Added new option -- verify_checksums_in_compaction
* Chagned Options.prefix_extractor from raw pointer to shared_ptr (take ownership)
Changed HashSkipListRepFactory and HashLinkListRepFactory constructor to not take SliceTransform object (use Options.prefix_extractor implicitly)
### New Features ### New Features
* If we find one truncated record at the end of the MANIFEST or WAL files, * If we find one truncated record at the end of the MANIFEST or WAL files,

28
db/c.cc
Unescape View File

@ -294,10 +294,10 @@ struct rocksdb_universal_compaction_options_t {
}; };
static bool SaveError(char** errptr, const Status& s) { static bool SaveError(char** errptr, const Status& s) {
assert(errptr != NULL); assert(errptr != nullptr);
if (s.ok()) { if (s.ok()) {
return false; return false;
} else if (*errptr == NULL) { } else if (*errptr == nullptr) {
*errptr = strdup(s.ToString().c_str()); *errptr = strdup(s.ToString().c_str());
} else { } else {
// TODO(sanjay): Merge with existing error? // TODO(sanjay): Merge with existing error?
@ -319,7 +319,7 @@ rocksdb_t* rocksdb_open(
char** errptr) { char** errptr) {
DB* db; DB* db;
if (SaveError(errptr, DB::Open(options->rep, std::string(name), &db))) { if (SaveError(errptr, DB::Open(options->rep, std::string(name), &db))) {
return NULL; return nullptr;
} }
rocksdb_t* result = new rocksdb_t; rocksdb_t* result = new rocksdb_t;
result->rep = db; result->rep = db;
@ -373,7 +373,7 @@ char* rocksdb_get(
const char* key, size_t keylen, const char* key, size_t keylen,
size_t* vallen, size_t* vallen,
char** errptr) { char** errptr) {
char* result = NULL; char* result = nullptr;
std::string tmp; std::string tmp;
Status s = db->rep->Get(options->rep, Slice(key, keylen), &tmp); Status s = db->rep->Get(options->rep, Slice(key, keylen), &tmp);
if (s.ok()) { if (s.ok()) {
@ -418,7 +418,7 @@ char* rocksdb_property_value(
// We use strdup() since we expect human readable output. // We use strdup() since we expect human readable output.
return strdup(tmp.c_str()); return strdup(tmp.c_str());
} else { } else {
return NULL; return nullptr;
} }
} }
@ -456,9 +456,9 @@ void rocksdb_compact_range(
const char* limit_key, size_t limit_key_len) { const char* limit_key, size_t limit_key_len) {
Slice a, b; Slice a, b;
db->rep->CompactRange( db->rep->CompactRange(
// Pass NULL Slice if corresponding "const char*" is NULL // Pass nullptr Slice if corresponding "const char*" is nullptr
(start_key ? (a = Slice(start_key, start_key_len), &a) : NULL), (start_key ? (a = Slice(start_key, start_key_len), &a) : nullptr),
(limit_key ? (b = Slice(limit_key, limit_key_len), &b) : NULL)); (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : nullptr));
} }
void rocksdb_flush( void rocksdb_flush(
@ -647,7 +647,7 @@ void rocksdb_options_set_paranoid_checks(
} }
void rocksdb_options_set_env(rocksdb_options_t* opt, rocksdb_env_t* env) { void rocksdb_options_set_env(rocksdb_options_t* opt, rocksdb_env_t* env) {
opt->rep.env = (env ? env->rep : NULL); opt->rep.env = (env ? env->rep : nullptr);
} }
void rocksdb_options_set_info_log(rocksdb_options_t* opt, rocksdb_logger_t* l) { void rocksdb_options_set_info_log(rocksdb_options_t* opt, rocksdb_logger_t* l) {
@ -765,7 +765,7 @@ void rocksdb_options_set_compression_options(
void rocksdb_options_set_prefix_extractor( void rocksdb_options_set_prefix_extractor(
rocksdb_options_t* opt, rocksdb_slicetransform_t* prefix_extractor) { rocksdb_options_t* opt, rocksdb_slicetransform_t* prefix_extractor) {
opt->rep.prefix_extractor = prefix_extractor; opt->rep.prefix_extractor.reset(prefix_extractor);
} }
void rocksdb_options_set_whole_key_filtering( void rocksdb_options_set_whole_key_filtering(
@ -1087,8 +1087,8 @@ rocksdb_filterpolicy_t* rocksdb_filterpolicy_create_bloom(int bits_per_key) {
}; };
Wrapper* wrapper = new Wrapper; Wrapper* wrapper = new Wrapper;
wrapper->rep_ = NewBloomFilterPolicy(bits_per_key); wrapper->rep_ = NewBloomFilterPolicy(bits_per_key);
wrapper->state_ = NULL; wrapper->state_ = nullptr;
wrapper->delete_filter_ = NULL; wrapper->delete_filter_ = nullptr;
wrapper->destructor_ = &Wrapper::DoNothing; wrapper->destructor_ = &Wrapper::DoNothing;
return wrapper; return wrapper;
} }
@ -1154,7 +1154,7 @@ void rocksdb_readoptions_set_prefix_seek(
void rocksdb_readoptions_set_snapshot( void rocksdb_readoptions_set_snapshot(
rocksdb_readoptions_t* opt, rocksdb_readoptions_t* opt,
const rocksdb_snapshot_t* snap) { const rocksdb_snapshot_t* snap) {
opt->rep.snapshot = (snap ? snap->rep : NULL); opt->rep.snapshot = (snap ? snap->rep : nullptr);
} }
void rocksdb_readoptions_set_prefix( void rocksdb_readoptions_set_prefix(
@ -1280,7 +1280,7 @@ rocksdb_slicetransform_t* rocksdb_slicetransform_create_fixed_prefix(size_t pref
}; };
Wrapper* wrapper = new Wrapper; Wrapper* wrapper = new Wrapper;
wrapper->rep_ = rocksdb::NewFixedPrefixTransform(prefixLen); wrapper->rep_ = rocksdb::NewFixedPrefixTransform(prefixLen);
wrapper->state_ = NULL; wrapper->state_ = nullptr;
wrapper->destructor_ = &Wrapper::DoNothing; wrapper->destructor_ = &Wrapper::DoNothing;
return wrapper; return wrapper;
} }

9
db/db_bench.cc
Unescape View File

@ -1538,9 +1538,10 @@ class Benchmark {
options.compaction_style = FLAGS_compaction_style_e; options.compaction_style = FLAGS_compaction_style_e;
options.block_size = FLAGS_block_size; options.block_size = FLAGS_block_size;
options.filter_policy = filter_policy_; options.filter_policy = filter_policy_;
options.prefix_extractor = if (FLAGS_use_plain_table || FLAGS_use_prefix_blooms) {
(FLAGS_use_plain_table || FLAGS_use_prefix_blooms) ? prefix_extractor_ options.prefix_extractor.reset(
: nullptr; NewFixedPrefixTransform(FLAGS_prefix_size));
}
options.memtable_prefix_bloom_bits = FLAGS_memtable_bloom_bits; options.memtable_prefix_bloom_bits = FLAGS_memtable_bloom_bits;
options.max_open_files = FLAGS_open_files; options.max_open_files = FLAGS_open_files;
options.statistics = dbstats; options.statistics = dbstats;
@ -1564,7 +1565,6 @@ class Benchmark {
switch (FLAGS_rep_factory) { switch (FLAGS_rep_factory) {
case kPrefixHash: case kPrefixHash:
options.memtable_factory.reset(NewHashSkipListRepFactory( options.memtable_factory.reset(NewHashSkipListRepFactory(
prefix_extractor_,
FLAGS_hash_bucket_count)); FLAGS_hash_bucket_count));
break; break;
case kSkipList: case kSkipList:
@ -1572,7 +1572,6 @@ class Benchmark {
break; break;
case kHashLinkedList: case kHashLinkedList:
options.memtable_factory.reset(NewHashLinkListRepFactory( options.memtable_factory.reset(NewHashLinkListRepFactory(
prefix_extractor_,
FLAGS_hash_bucket_count)); FLAGS_hash_bucket_count));
break; break;
case kVectorRep: case kVectorRep:

24
db/db_impl.cc
Unescape View File

@ -56,6 +56,7 @@
#include "util/build_version.h" #include "util/build_version.h"
#include "util/coding.h" #include "util/coding.h"
#include "util/hash_skiplist_rep.h" #include "util/hash_skiplist_rep.h"
#include "util/hash_linklist_rep.h"
#include "util/logging.h" #include "util/logging.h"
#include "util/log_buffer.h" #include "util/log_buffer.h"
#include "util/mutexlock.h" #include "util/mutexlock.h"
@ -176,19 +177,16 @@ Options SanitizeOptions(const std::string& dbname,
Log(result.info_log, "Compaction filter specified, ignore factory"); Log(result.info_log, "Compaction filter specified, ignore factory");
} }
if (result.prefix_extractor) { if (result.prefix_extractor) {
// If a prefix extractor has been supplied and a HashSkipListRepFactory is Log(result.info_log, "prefix extractor %s in use.",
// being used, make sure that the latter uses the former as its transform result.prefix_extractor->Name());
// function. } else {
auto factory = dynamic_cast<HashSkipListRepFactory*>( assert(result.memtable_factory);
result.memtable_factory.get()); Slice name = result.memtable_factory->Name();
if (factory && if (name.compare("HashSkipListRepFactory") == 0 ||
factory->GetTransform() != result.prefix_extractor) { name.compare("HashLinkListRepFactory") == 0) {
Log(result.info_log, "A prefix hash representation factory was supplied " Log(result.info_log, "prefix extractor is not provided while using %s. "
"whose prefix extractor does not match options.prefix_extractor. " "fallback to skiplist", name.ToString().c_str());
"Falling back to skip list representation factory");
result.memtable_factory = std::make_shared<SkipListFactory>(); result.memtable_factory = std::make_shared<SkipListFactory>();
} else if (factory) {
Log(result.info_log, "Prefix hash memtable rep is in use.");
} }
} }
@ -3207,7 +3205,7 @@ Iterator* DBImpl::NewIterator(const ReadOptions& options) {
// use extra wrapper to exclude any keys from the results which // use extra wrapper to exclude any keys from the results which
// don't begin with the prefix // don't begin with the prefix
iter = new PrefixFilterIterator(iter, *options.prefix, iter = new PrefixFilterIterator(iter, *options.prefix,
options_.prefix_extractor); options_.prefix_extractor.get());
} }
return iter; return iter;
} }

29
db/db_test.cc
Unescape View File

@ -259,8 +259,6 @@ class SpecialEnv : public EnvWrapper {
class DBTest { class DBTest {
private: private:
const FilterPolicy* filter_policy_; const FilterPolicy* filter_policy_;
static std::unique_ptr<const SliceTransform> prefix_1_transform;
static std::unique_ptr<const SliceTransform> noop_transform;
protected: protected:
// Sequence of option configurations to try // Sequence of option configurations to try
@ -374,18 +372,18 @@ class DBTest {
Options options; Options options;
switch (option_config_) { switch (option_config_) {
case kHashSkipList: case kHashSkipList:
options.memtable_factory.reset( options.prefix_extractor.reset(NewFixedPrefixTransform(1));
NewHashSkipListRepFactory(NewFixedPrefixTransform(1))); options.memtable_factory.reset(NewHashSkipListRepFactory());
break; break;
case kPlainTableFirstBytePrefix: case kPlainTableFirstBytePrefix:
options.table_factory.reset(new PlainTableFactory()); options.table_factory.reset(new PlainTableFactory());
options.prefix_extractor = prefix_1_transform.get(); options.prefix_extractor.reset(NewFixedPrefixTransform(1));
options.allow_mmap_reads = true; options.allow_mmap_reads = true;
options.max_sequential_skip_in_iterations = 999999; options.max_sequential_skip_in_iterations = 999999;
break; break;
case kPlainTableAllBytesPrefix: case kPlainTableAllBytesPrefix:
options.table_factory.reset(new PlainTableFactory()); options.table_factory.reset(new PlainTableFactory());
options.prefix_extractor = noop_transform.get(); options.prefix_extractor.reset(NewNoopTransform());
options.allow_mmap_reads = true; options.allow_mmap_reads = true;
options.max_sequential_skip_in_iterations = 999999; options.max_sequential_skip_in_iterations = 999999;
break; break;
@ -425,8 +423,8 @@ class DBTest {
options.memtable_factory.reset(new VectorRepFactory(100)); options.memtable_factory.reset(new VectorRepFactory(100));
break; break;
case kHashLinkList: case kHashLinkList:
options.memtable_factory.reset( options.prefix_extractor.reset(NewFixedPrefixTransform(1));
NewHashLinkListRepFactory(NewFixedPrefixTransform(1), 4)); options.memtable_factory.reset(NewHashLinkListRepFactory(4));
break; break;
case kUniversalCompaction: case kUniversalCompaction:
options.compaction_style = kCompactionStyleUniversal; options.compaction_style = kCompactionStyleUniversal;
@ -819,10 +817,6 @@ class DBTest {
} }
}; };
std::unique_ptr<const SliceTransform> DBTest::prefix_1_transform(
NewFixedPrefixTransform(1));
std::unique_ptr<const SliceTransform> DBTest::noop_transform(
NewNoopTransform());
static std::string Key(int i) { static std::string Key(int i) {
char buf[100]; char buf[100];
@ -5404,7 +5398,7 @@ TEST(DBTest, PrefixScan) {
options.env = env_; options.env = env_;
options.no_block_cache = true; options.no_block_cache = true;
options.filter_policy = NewBloomFilterPolicy(10); options.filter_policy = NewBloomFilterPolicy(10);
options.prefix_extractor = NewFixedPrefixTransform(8); options.prefix_extractor.reset(NewFixedPrefixTransform(8));
options.whole_key_filtering = false; options.whole_key_filtering = false;
options.disable_auto_compactions = true; options.disable_auto_compactions = true;
options.max_background_compactions = 2; options.max_background_compactions = 2;
@ -5412,8 +5406,7 @@ TEST(DBTest, PrefixScan) {
options.disable_seek_compaction = true; options.disable_seek_compaction = true;
// Tricky: options.prefix_extractor will be released by // Tricky: options.prefix_extractor will be released by
// NewHashSkipListRepFactory after use. // NewHashSkipListRepFactory after use.
options.memtable_factory.reset( options.memtable_factory.reset(NewHashSkipListRepFactory());
NewHashSkipListRepFactory(options.prefix_extractor));
// prefix specified, with blooms: 2 RAND I/Os // prefix specified, with blooms: 2 RAND I/Os
// SeekToFirst // SeekToFirst
@ -5609,14 +5602,12 @@ TEST(DBTest, TailingIteratorPrefixSeek) {
read_options.tailing = true; read_options.tailing = true;
read_options.prefix_seek = true; read_options.prefix_seek = true;
auto prefix_extractor = NewFixedPrefixTransform(2);
Options options = CurrentOptions(); Options options = CurrentOptions();
options.env = env_; options.env = env_;
options.create_if_missing = true; options.create_if_missing = true;
options.disable_auto_compactions = true; options.disable_auto_compactions = true;
options.prefix_extractor = prefix_extractor; options.prefix_extractor.reset(NewFixedPrefixTransform(2));
options.memtable_factory.reset(NewHashSkipListRepFactory(prefix_extractor)); options.memtable_factory.reset(NewHashSkipListRepFactory());
DestroyAndReopen(&options); DestroyAndReopen(&options);
std::unique_ptr<Iterator> iter(db_->NewIterator(read_options)); std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));

5
db/memtable.cc
Unescape View File

@ -32,7 +32,8 @@ MemTable::MemTable(const InternalKeyComparator& cmp, const Options& options)
: comparator_(cmp), : comparator_(cmp),
refs_(0), refs_(0),
arena_(options.arena_block_size), arena_(options.arena_block_size),
table_(options.memtable_factory->CreateMemTableRep(comparator_, &arena_)), table_(options.memtable_factory->CreateMemTableRep(
comparator_, &arena_, options.prefix_extractor.get())),
flush_in_progress_(false), flush_in_progress_(false),
flush_completed_(false), flush_completed_(false),
file_number_(0), file_number_(0),
@ -41,7 +42,7 @@ MemTable::MemTable(const InternalKeyComparator& cmp, const Options& options)
mem_logfile_number_(0), mem_logfile_number_(0),
locks_(options.inplace_update_support ? options.inplace_update_num_locks locks_(options.inplace_update_support ? options.inplace_update_num_locks
: 0), : 0),
prefix_extractor_(options.prefix_extractor) { prefix_extractor_(options.prefix_extractor.get()) {
if (prefix_extractor_ && options.memtable_prefix_bloom_bits > 0) { if (prefix_extractor_ && options.memtable_prefix_bloom_bits > 0) {
prefix_bloom_.reset(new DynamicBloom(options.memtable_prefix_bloom_bits, prefix_bloom_.reset(new DynamicBloom(options.memtable_prefix_bloom_bits,
options.memtable_prefix_bloom_probes)); options.memtable_prefix_bloom_probes));

6
db/plain_table_db_test.cc
Unescape View File

@ -44,7 +44,6 @@ class PlainTableDBTest {
DB* db_; DB* db_;
Options last_options_; Options last_options_;
static std::unique_ptr<const SliceTransform> prefix_transform;
public: public:
PlainTableDBTest() : env_(Env::Default()) { PlainTableDBTest() : env_(Env::Default()) {
@ -66,7 +65,7 @@ class PlainTableDBTest {
Options CurrentOptions() { Options CurrentOptions() {
Options options; Options options;
options.table_factory.reset(NewPlainTableFactory(16, 2, 0.8, 3)); options.table_factory.reset(NewPlainTableFactory(16, 2, 0.8, 3));
options.prefix_extractor = prefix_transform.get(); options.prefix_extractor.reset(NewFixedPrefixTransform(8));
options.allow_mmap_reads = true; options.allow_mmap_reads = true;
return options; return options;
} }
@ -173,9 +172,6 @@ class PlainTableDBTest {
} }
}; };
std::unique_ptr<const SliceTransform> PlainTableDBTest::prefix_transform(
NewFixedPrefixTransform(8));
TEST(PlainTableDBTest, Empty) { TEST(PlainTableDBTest, Empty) {
ASSERT_TRUE(dbfull() != nullptr); ASSERT_TRUE(dbfull() != nullptr);
ASSERT_EQ("NOT_FOUND", Get("0000000000000foo")); ASSERT_EQ("NOT_FOUND", Get("0000000000000foo"));

11
db/prefix_test.cc
Unescape View File

@ -161,16 +161,15 @@ class PrefixTest {
// skip some options // skip some options
option_config_++; option_config_++;
if (option_config_ < kEnd) { if (option_config_ < kEnd) {
auto prefix_extractor = NewFixedPrefixTransform(8); options.prefix_extractor.reset(NewFixedPrefixTransform(8));
options.prefix_extractor = prefix_extractor;
switch(option_config_) { switch(option_config_) {
case kHashSkipList: case kHashSkipList:
options.memtable_factory.reset(NewHashSkipListRepFactory( options.memtable_factory.reset(
options.prefix_extractor, bucket_count, FLAGS_skiplist_height)); NewHashSkipListRepFactory(bucket_count, FLAGS_skiplist_height));
return true; return true;
case kHashLinkList: case kHashLinkList:
options.memtable_factory.reset(NewHashLinkListRepFactory( options.memtable_factory.reset(
options.prefix_extractor, bucket_count)); NewHashLinkListRepFactory(bucket_count));
return true; return true;
default: default:
return false; return false;

6
db/skiplist.h
Unescape View File

@ -277,8 +277,8 @@ bool SkipList<Key,Comparator>::KeyIsAfterNode(const Key& key, Node* n) const {
} }
template<typename Key, class Comparator> template<typename Key, class Comparator>
typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindGreaterOrEqual(const Key& key, Node** prev) typename SkipList<Key, Comparator>::Node* SkipList<Key, Comparator>::
const { FindGreaterOrEqual(const Key& key, Node** prev) const {
// Use prev as an optimization hint and fallback to slow path // Use prev as an optimization hint and fallback to slow path
if (prev && !KeyIsAfterNode(key, prev[0]->Next(0))) { if (prev && !KeyIsAfterNode(key, prev[0]->Next(0))) {
Node* x = prev[0]; Node* x = prev[0];
@ -356,7 +356,7 @@ typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindLast()
} }
template<typename Key, class Comparator> template<typename Key, class Comparator>
SkipList<Key,Comparator>::SkipList(Comparator cmp, Arena* arena, SkipList<Key, Comparator>::SkipList(const Comparator cmp, Arena* arena,
int32_t max_height, int32_t max_height,
int32_t branching_factor) int32_t branching_factor)
: kMaxHeight_(max_height), : kMaxHeight_(max_height),

2
db/tailing_iter.cc
Unescape View File

@ -155,7 +155,7 @@ bool TailingIterator::IsCurrentVersion() const {
} }
bool TailingIterator::IsSamePrefix(const Slice& target) const { bool TailingIterator::IsSamePrefix(const Slice& target) const {
const SliceTransform* extractor = db_->options_.prefix_extractor; const SliceTransform* extractor = db_->options_.prefix_extractor.get();
assert(extractor); assert(extractor);
assert(is_prev_set_); assert(is_prev_set_);

20
include/rocksdb/memtablerep.h
Unescape View File

@ -160,8 +160,8 @@ class MemTableRep {
class MemTableRepFactory { class MemTableRepFactory {
public: public:
virtual ~MemTableRepFactory() {} virtual ~MemTableRepFactory() {}
virtual MemTableRep* CreateMemTableRep(MemTableRep::KeyComparator&, virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&,
Arena*) = 0; Arena*, const SliceTransform*) = 0;
virtual const char* Name() const = 0; virtual const char* Name() const = 0;
}; };
@ -178,8 +178,9 @@ class VectorRepFactory : public MemTableRepFactory {
public: public:
explicit VectorRepFactory(size_t count = 0) : count_(count) { } explicit VectorRepFactory(size_t count = 0) : count_(count) { }
virtual MemTableRep* CreateMemTableRep(MemTableRep::KeyComparator&, virtual MemTableRep* CreateMemTableRep(
Arena*) override; const MemTableRep::KeyComparator&, Arena*,
const SliceTransform*) override;
virtual const char* Name() const override { virtual const char* Name() const override {
return "VectorRepFactory"; return "VectorRepFactory";
} }
@ -188,8 +189,9 @@ class VectorRepFactory : public MemTableRepFactory {
// This uses a skip list to store keys. It is the default. // This uses a skip list to store keys. It is the default.
class SkipListFactory : public MemTableRepFactory { class SkipListFactory : public MemTableRepFactory {
public: public:
virtual MemTableRep* CreateMemTableRep(MemTableRep::KeyComparator&, virtual MemTableRep* CreateMemTableRep(
Arena*) override; const MemTableRep::KeyComparator&, Arena*,
const SliceTransform*) override;
virtual const char* Name() const override { virtual const char* Name() const override {
return "SkipListFactory"; return "SkipListFactory";
} }
@ -202,8 +204,8 @@ class SkipListFactory : public MemTableRepFactory {
// skiplist_branching_factor: probabilistic size ratio between adjacent // skiplist_branching_factor: probabilistic size ratio between adjacent
// link lists in the skiplist // link lists in the skiplist
extern MemTableRepFactory* NewHashSkipListRepFactory( extern MemTableRepFactory* NewHashSkipListRepFactory(
const SliceTransform* transform, size_t bucket_count = 1000000, size_t bucket_count = 1000000, int32_t skiplist_height = 4,
int32_t skiplist_height = 4, int32_t skiplist_branching_factor = 4 int32_t skiplist_branching_factor = 4
); );
// The factory is to create memtables with a hashed linked list: // The factory is to create memtables with a hashed linked list:
@ -211,6 +213,6 @@ extern MemTableRepFactory* NewHashSkipListRepFactory(
// linked list (null if the bucket is empty). // linked list (null if the bucket is empty).
// bucket_count: number of fixed array buckets // bucket_count: number of fixed array buckets
extern MemTableRepFactory* NewHashLinkListRepFactory( extern MemTableRepFactory* NewHashLinkListRepFactory(
const SliceTransform* transform, size_t bucket_count = 50000); size_t bucket_count = 50000);
} // namespace rocksdb } // namespace rocksdb

2
include/rocksdb/options.h
Unescape View File

@ -277,7 +277,7 @@ struct Options {
// 4) prefix(prefix(key)) == prefix(key) // 4) prefix(prefix(key)) == prefix(key)
// //
// Default: nullptr // Default: nullptr
const SliceTransform* prefix_extractor; std::shared_ptr<const SliceTransform> prefix_extractor;
// If true, place whole keys in the filter (not just prefixes). // If true, place whole keys in the filter (not just prefixes).
// This must generally be true for gets to be efficient. // This must generally be true for gets to be efficient.

4
table/filter_block.cc
Unescape View File

@ -24,7 +24,7 @@ static const size_t kFilterBase = 1 << kFilterBaseLg;
FilterBlockBuilder::FilterBlockBuilder(const Options& opt, FilterBlockBuilder::FilterBlockBuilder(const Options& opt,
const Comparator* internal_comparator) const Comparator* internal_comparator)
: policy_(opt.filter_policy), : policy_(opt.filter_policy),
prefix_extractor_(opt.prefix_extractor), prefix_extractor_(opt.prefix_extractor.get()),
whole_key_filtering_(opt.whole_key_filtering), whole_key_filtering_(opt.whole_key_filtering),
comparator_(internal_comparator) {} comparator_(internal_comparator) {}
@ -133,7 +133,7 @@ void FilterBlockBuilder::GenerateFilter() {
FilterBlockReader::FilterBlockReader( FilterBlockReader::FilterBlockReader(
const Options& opt, const Slice& contents, bool delete_contents_after_use) const Options& opt, const Slice& contents, bool delete_contents_after_use)
: policy_(opt.filter_policy), : policy_(opt.filter_policy),
prefix_extractor_(opt.prefix_extractor), prefix_extractor_(opt.prefix_extractor.get()),
whole_key_filtering_(opt.whole_key_filtering), whole_key_filtering_(opt.whole_key_filtering),
data_(nullptr), data_(nullptr),
offset_(nullptr), offset_(nullptr),

8
table/table_reader_bench.cc
Unescape View File

@ -240,8 +240,8 @@ int main(int argc, char** argv) {
rocksdb::TableFactory* tf = new rocksdb::BlockBasedTableFactory(); rocksdb::TableFactory* tf = new rocksdb::BlockBasedTableFactory();
rocksdb::Options options; rocksdb::Options options;
if (FLAGS_prefix_len < 16) { if (FLAGS_prefix_len < 16) {
options.prefix_extractor = rocksdb::NewFixedPrefixTransform( options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(
FLAGS_prefix_len); FLAGS_prefix_len));
} }
rocksdb::ReadOptions ro; rocksdb::ReadOptions ro;
rocksdb::EnvOptions env_options; rocksdb::EnvOptions env_options;
@ -254,8 +254,8 @@ int main(int argc, char** argv) {
env_options.use_mmap_reads = true; env_options.use_mmap_reads = true;
tf = new rocksdb::PlainTableFactory(16, (FLAGS_prefix_len == 16) ? 0 : 8, tf = new rocksdb::PlainTableFactory(16, (FLAGS_prefix_len == 16) ? 0 : 8,
0.75); 0.75);
options.prefix_extractor = rocksdb::NewFixedPrefixTransform( options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(
FLAGS_prefix_len); FLAGS_prefix_len));
} else { } else {
tf = new rocksdb::BlockBasedTableFactory(); tf = new rocksdb::BlockBasedTableFactory();
} }

11
table/table_test.cc
Unescape View File

@ -696,7 +696,7 @@ class Harness {
case PLAIN_TABLE_SEMI_FIXED_PREFIX: case PLAIN_TABLE_SEMI_FIXED_PREFIX:
support_prev_ = false; support_prev_ = false;
only_support_prefix_seek_ = true; only_support_prefix_seek_ = true;
options_.prefix_extractor = prefix_transform.get(); options_.prefix_extractor.reset(new FixedOrLessPrefixTransform(2));
options_.allow_mmap_reads = true; options_.allow_mmap_reads = true;
options_.table_factory.reset(NewPlainTableFactory()); options_.table_factory.reset(NewPlainTableFactory());
constructor_ = new TableConstructor(options_.comparator, true, true); constructor_ = new TableConstructor(options_.comparator, true, true);
@ -706,7 +706,7 @@ class Harness {
case PLAIN_TABLE_FULL_STR_PREFIX: case PLAIN_TABLE_FULL_STR_PREFIX:
support_prev_ = false; support_prev_ = false;
only_support_prefix_seek_ = true; only_support_prefix_seek_ = true;
options_.prefix_extractor = noop_transform.get(); options_.prefix_extractor.reset(NewNoopTransform());
options_.allow_mmap_reads = true; options_.allow_mmap_reads = true;
options_.table_factory.reset(NewPlainTableFactory()); options_.table_factory.reset(NewPlainTableFactory());
constructor_ = new TableConstructor(options_.comparator, true, true); constructor_ = new TableConstructor(options_.comparator, true, true);
@ -919,15 +919,8 @@ class Harness {
bool support_prev_; bool support_prev_;
bool only_support_prefix_seek_; bool only_support_prefix_seek_;
shared_ptr<InternalKeyComparator> internal_comparator_; shared_ptr<InternalKeyComparator> internal_comparator_;
static std::unique_ptr<const SliceTransform> noop_transform;
static std::unique_ptr<const SliceTransform> prefix_transform;
}; };
std::unique_ptr<const SliceTransform> Harness::noop_transform(
NewNoopTransform());
std::unique_ptr<const SliceTransform> Harness::prefix_transform(
new FixedOrLessPrefixTransform(2));
static bool Between(uint64_t val, uint64_t low, uint64_t high) { static bool Between(uint64_t val, uint64_t low, uint64_t high) {
bool result = (val >= low) && (val <= high); bool result = (val >= low) && (val <= high);
if (!result) { if (!result) {

14
tools/db_stress.cc
Unescape View File

@ -688,9 +688,6 @@ class StressTest {
filter_policy_(FLAGS_bloom_bits >= 0 filter_policy_(FLAGS_bloom_bits >= 0
? NewBloomFilterPolicy(FLAGS_bloom_bits) ? NewBloomFilterPolicy(FLAGS_bloom_bits)
: nullptr), : nullptr),
prefix_extractor_(NewFixedPrefixTransform(
FLAGS_test_batches_snapshots ?
sizeof(long) : sizeof(long)-1)),
db_(nullptr), db_(nullptr),
num_times_reopened_(0) { num_times_reopened_(0) {
if (FLAGS_destroy_db_initially) { if (FLAGS_destroy_db_initially) {
@ -708,7 +705,6 @@ class StressTest {
~StressTest() { ~StressTest() {
delete db_; delete db_;
delete filter_policy_; delete filter_policy_;
delete prefix_extractor_;
} }
void Run() { void Run() {
@ -1373,7 +1369,7 @@ class StressTest {
static_cast<rocksdb::CompactionStyle>(FLAGS_compaction_style); static_cast<rocksdb::CompactionStyle>(FLAGS_compaction_style);
options.block_size = FLAGS_block_size; options.block_size = FLAGS_block_size;
options.filter_policy = filter_policy_; options.filter_policy = filter_policy_;
options.prefix_extractor = prefix_extractor_; options.prefix_extractor.reset(NewFixedPrefixTransform(FLAGS_prefix_size));
options.max_open_files = FLAGS_open_files; options.max_open_files = FLAGS_open_files;
options.statistics = dbstats; options.statistics = dbstats;
options.env = FLAGS_env; options.env = FLAGS_env;
@ -1405,16 +1401,13 @@ class StressTest {
} }
switch (FLAGS_rep_factory) { switch (FLAGS_rep_factory) {
case kHashSkipList: case kHashSkipList:
options.memtable_factory.reset(NewHashSkipListRepFactory( options.memtable_factory.reset(NewHashSkipListRepFactory());
NewFixedPrefixTransform(FLAGS_prefix_size)));
break; break;
case kSkipList: case kSkipList:
// no need to do anything // no need to do anything
break; break;
case kVectorRep: case kVectorRep:
options.memtable_factory.reset( options.memtable_factory.reset(new VectorRepFactory());
new VectorRepFactory()
);
break; break;
} }
static Random purge_percent(1000); // no benefit from non-determinism here static Random purge_percent(1000); // no benefit from non-determinism here
@ -1488,7 +1481,6 @@ class StressTest {
shared_ptr<Cache> cache_; shared_ptr<Cache> cache_;
shared_ptr<Cache> compressed_cache_; shared_ptr<Cache> compressed_cache_;
const FilterPolicy* filter_policy_; const FilterPolicy* filter_policy_;
const SliceTransform* prefix_extractor_;
DB* db_; DB* db_;
StackableDB* sdb_; StackableDB* sdb_;
int num_times_reopened_; int num_times_reopened_;

2
tools/sst_dump.cc
Unescape View File

@ -130,7 +130,7 @@ Status SstFileReader::SetTableOptionsByMagicNumber(uint64_t table_magic_number,
options_.allow_mmap_reads = true; options_.allow_mmap_reads = true;
options_.table_factory = std::make_shared<PlainTableFactory>( options_.table_factory = std::make_shared<PlainTableFactory>(
table_properties->fixed_key_len, 2, 0.8); table_properties->fixed_key_len, 2, 0.8);
options_.prefix_extractor = NewNoopTransform(); options_.prefix_extractor.reset(NewNoopTransform());
fprintf(stdout, "Sst file format: plain table\n"); fprintf(stdout, "Sst file format: plain table\n");
} else { } else {
char error_msg_buffer[80]; char error_msg_buffer[80];

18
util/hash_linklist_rep.cc
Unescape View File

@ -55,7 +55,7 @@ private:
class HashLinkListRep : public MemTableRep { class HashLinkListRep : public MemTableRep {
public: public:
HashLinkListRep(MemTableRep::KeyComparator& compare, Arena* arena, HashLinkListRep(const MemTableRep::KeyComparator& compare, Arena* arena,
const SliceTransform* transform, size_t bucket_size); const SliceTransform* transform, size_t bucket_size);
virtual void Insert(const char* key) override; virtual void Insert(const char* key) override;
@ -81,7 +81,7 @@ class HashLinkListRep : public MemTableRep {
private: private:
friend class DynamicIterator; friend class DynamicIterator;
typedef SkipList<const char*, MemTableRep::KeyComparator&> FullList; typedef SkipList<const char*, const MemTableRep::KeyComparator&> FullList;
size_t bucket_size_; size_t bucket_size_;
@ -92,7 +92,7 @@ class HashLinkListRep : public MemTableRep {
// The user-supplied transform whose domain is the user keys. // The user-supplied transform whose domain is the user keys.
const SliceTransform* transform_; const SliceTransform* transform_;
MemTableRep::KeyComparator& compare_; const MemTableRep::KeyComparator& compare_;
// immutable after construction // immutable after construction
Arena* const arena_; Arena* const arena_;
@ -314,7 +314,7 @@ class HashLinkListRep : public MemTableRep {
}; };
}; };
HashLinkListRep::HashLinkListRep(MemTableRep::KeyComparator& compare, HashLinkListRep::HashLinkListRep(const MemTableRep::KeyComparator& compare,
Arena* arena, const SliceTransform* transform, Arena* arena, const SliceTransform* transform,
size_t bucket_size) size_t bucket_size)
: bucket_size_(bucket_size), : bucket_size_(bucket_size),
@ -475,13 +475,13 @@ Node* HashLinkListRep::FindGreaterOrEqualInBucket(Node* head,
} // anon namespace } // anon namespace
MemTableRep* HashLinkListRepFactory::CreateMemTableRep( MemTableRep* HashLinkListRepFactory::CreateMemTableRep(
MemTableRep::KeyComparator& compare, Arena* arena) { const MemTableRep::KeyComparator& compare, Arena* arena,
return new HashLinkListRep(compare, arena, transform_, bucket_count_); const SliceTransform* transform) {
return new HashLinkListRep(compare, arena, transform, bucket_count_);
} }
MemTableRepFactory* NewHashLinkListRepFactory( MemTableRepFactory* NewHashLinkListRepFactory(size_t bucket_count) {
const SliceTransform* transform, size_t bucket_count) { return new HashLinkListRepFactory(bucket_count);
return new HashLinkListRepFactory(transform, bucket_count);
} }
} // namespace rocksdb } // namespace rocksdb

17
util/hash_linklist_rep.h
Unescape View File

@ -14,25 +14,20 @@ namespace rocksdb {
class HashLinkListRepFactory : public MemTableRepFactory { class HashLinkListRepFactory : public MemTableRepFactory {
public: public:
explicit HashLinkListRepFactory( explicit HashLinkListRepFactory(size_t bucket_count)
const SliceTransform* transform, : bucket_count_(bucket_count) { }
size_t bucket_count)
: transform_(transform),
bucket_count_(bucket_count) { }
virtual ~HashLinkListRepFactory() { delete transform_; } virtual ~HashLinkListRepFactory() {}
virtual MemTableRep* CreateMemTableRep(MemTableRep::KeyComparator& compare, virtual MemTableRep* CreateMemTableRep(
Arena* arena) override; const MemTableRep::KeyComparator& compare, Arena* arena,
const SliceTransform* transform) override;
virtual const char* Name() const override { virtual const char* Name() const override {
return "HashLinkListRepFactory"; return "HashLinkListRepFactory";
} }
const SliceTransform* GetTransform() { return transform_; }
private: private:
const SliceTransform* transform_;
const size_t bucket_count_; const size_t bucket_count_;
}; };

21
util/hash_skiplist_rep.cc
Unescape View File

@ -21,7 +21,7 @@ namespace {
class HashSkipListRep : public MemTableRep { class HashSkipListRep : public MemTableRep {
public: public:
HashSkipListRep(MemTableRep::KeyComparator& compare, Arena* arena, HashSkipListRep(const MemTableRep::KeyComparator& compare, Arena* arena,
const SliceTransform* transform, size_t bucket_size, const SliceTransform* transform, size_t bucket_size,
int32_t skiplist_height, int32_t skiplist_branching_factor); int32_t skiplist_height, int32_t skiplist_branching_factor);
@ -48,7 +48,7 @@ class HashSkipListRep : public MemTableRep {
private: private:
friend class DynamicIterator; friend class DynamicIterator;
typedef SkipList<const char*, MemTableRep::KeyComparator&> Bucket; typedef SkipList<const char*, const MemTableRep::KeyComparator&> Bucket;
size_t bucket_size_; size_t bucket_size_;
@ -62,7 +62,7 @@ class HashSkipListRep : public MemTableRep {
// The user-supplied transform whose domain is the user keys. // The user-supplied transform whose domain is the user keys.
const SliceTransform* transform_; const SliceTransform* transform_;
MemTableRep::KeyComparator& compare_; const MemTableRep::KeyComparator& compare_;
// immutable after construction // immutable after construction
Arena* const arena_; Arena* const arena_;
@ -221,7 +221,7 @@ class HashSkipListRep : public MemTableRep {
}; };
}; };
HashSkipListRep::HashSkipListRep(MemTableRep::KeyComparator& compare, HashSkipListRep::HashSkipListRep(const MemTableRep::KeyComparator& compare,
Arena* arena, const SliceTransform* transform, Arena* arena, const SliceTransform* transform,
size_t bucket_size, int32_t skiplist_height, size_t bucket_size, int32_t skiplist_height,
int32_t skiplist_branching_factor) int32_t skiplist_branching_factor)
@ -321,16 +321,17 @@ MemTableRep::Iterator* HashSkipListRep::GetDynamicPrefixIterator() {
} // anon namespace } // anon namespace
MemTableRep* HashSkipListRepFactory::CreateMemTableRep( MemTableRep* HashSkipListRepFactory::CreateMemTableRep(
MemTableRep::KeyComparator& compare, Arena* arena) { const MemTableRep::KeyComparator& compare, Arena* arena,
return new HashSkipListRep(compare, arena, transform_, bucket_count_, const SliceTransform* transform) {
return new HashSkipListRep(compare, arena, transform, bucket_count_,
skiplist_height_, skiplist_branching_factor_); skiplist_height_, skiplist_branching_factor_);
} }
MemTableRepFactory* NewHashSkipListRepFactory( MemTableRepFactory* NewHashSkipListRepFactory(
const SliceTransform* transform, size_t bucket_count, size_t bucket_count, int32_t skiplist_height,
int32_t skiplist_height, int32_t skiplist_branching_factor) { int32_t skiplist_branching_factor) {
return new HashSkipListRepFactory(transform, bucket_count, return new HashSkipListRepFactory(bucket_count, skiplist_height,
skiplist_height, skiplist_branching_factor); skiplist_branching_factor);
} }
} // namespace rocksdb } // namespace rocksdb

14
util/hash_skiplist_rep.h
Unescape View File

@ -15,28 +15,24 @@ namespace rocksdb {
class HashSkipListRepFactory : public MemTableRepFactory { class HashSkipListRepFactory : public MemTableRepFactory {
public: public:
explicit HashSkipListRepFactory( explicit HashSkipListRepFactory(
const SliceTransform* transform,
size_t bucket_count, size_t bucket_count,
int32_t skiplist_height, int32_t skiplist_height,
int32_t skiplist_branching_factor) int32_t skiplist_branching_factor)
: transform_(transform), : bucket_count_(bucket_count),
bucket_count_(bucket_count),
skiplist_height_(skiplist_height), skiplist_height_(skiplist_height),
skiplist_branching_factor_(skiplist_branching_factor) { } skiplist_branching_factor_(skiplist_branching_factor) { }
virtual ~HashSkipListRepFactory() { delete transform_; } virtual ~HashSkipListRepFactory() {}
virtual MemTableRep* CreateMemTableRep(MemTableRep::KeyComparator& compare, virtual MemTableRep* CreateMemTableRep(
Arena* arena) override; const MemTableRep::KeyComparator& compare, Arena* arena,
const SliceTransform* transform) override;
virtual const char* Name() const override { virtual const char* Name() const override {
return "HashSkipListRepFactory"; return "HashSkipListRepFactory";
} }
const SliceTransform* GetTransform() { return transform_; }
private: private:
const SliceTransform* transform_;
const size_t bucket_count_; const size_t bucket_count_;
const int32_t skiplist_height_; const int32_t skiplist_height_;
const int32_t skiplist_branching_factor_; const int32_t skiplist_branching_factor_;

11
util/skiplistrep.cc
Unescape View File

@ -10,9 +10,9 @@
namespace rocksdb { namespace rocksdb {
namespace { namespace {
class SkipListRep : public MemTableRep { class SkipListRep : public MemTableRep {
SkipList<const char*, MemTableRep::KeyComparator&> skip_list_; SkipList<const char*, const MemTableRep::KeyComparator&> skip_list_;
public: public:
explicit SkipListRep(MemTableRep::KeyComparator& compare, Arena* arena) explicit SkipListRep(const MemTableRep::KeyComparator& compare, Arena* arena)
: skip_list_(compare, arena) { : skip_list_(compare, arena) {
} }
@ -47,12 +47,12 @@ public:
// Iteration over the contents of a skip list // Iteration over the contents of a skip list
class Iterator : public MemTableRep::Iterator { class Iterator : public MemTableRep::Iterator {
SkipList<const char*, MemTableRep::KeyComparator&>::Iterator iter_; SkipList<const char*, const MemTableRep::KeyComparator&>::Iterator iter_;
public: public:
// Initialize an iterator over the specified list. // Initialize an iterator over the specified list.
// The returned iterator is not valid. // The returned iterator is not valid.
explicit Iterator( explicit Iterator(
const SkipList<const char*, MemTableRep::KeyComparator&>* list const SkipList<const char*, const MemTableRep::KeyComparator&>* list
) : iter_(list) { } ) : iter_(list) { }
virtual ~Iterator() override { } virtual ~Iterator() override { }
@ -115,7 +115,8 @@ public:
} }
MemTableRep* SkipListFactory::CreateMemTableRep( MemTableRep* SkipListFactory::CreateMemTableRep(
MemTableRep::KeyComparator& compare, Arena* arena) { const MemTableRep::KeyComparator& compare, Arena* arena,
const SliceTransform*) {
return new SkipListRep(compare, arena); return new SkipListRep(compare, arena);
} }

3
util/vectorrep.cc
Unescape View File

@ -271,7 +271,8 @@ MemTableRep::Iterator* VectorRep::GetIterator() {
} // anon namespace } // anon namespace
MemTableRep* VectorRepFactory::CreateMemTableRep( MemTableRep* VectorRepFactory::CreateMemTableRep(
MemTableRep::KeyComparator& compare, Arena* arena) { const MemTableRep::KeyComparator& compare, Arena* arena,
const SliceTransform*) {
return new VectorRep(compare, arena, count_); return new VectorRep(compare, arena, count_);
} }
} // namespace rocksdb } // namespace rocksdb

Write Preview
Loading…
Cancel
Save