NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

use FileLevel in LevelFileNumIterator

Browse Source 
Summary:
  Use FileLevel in LevelFileNumIterator, thus use new version of findFile.
  Old version of findFile function is deleted.
  Write a function in version_set.cc to generate FileLevel from files_.
  Add GenerateFileLevelTest in version_set_test.cc

Test Plan:
  make all check

Reviewers: ljin, haobo, yhchiang, sdong

Reviewed By: sdong

Subscribers: igor, dhruba

Differential Revision: https://reviews.facebook.net/D19659
main
Feng Zhu 10 years ago
parent 3b97ee96c4
commit 178fd6f9db
7 changed files with 133 additions and 132 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 7
      db/compaction.cc
  2. 19
      db/compaction.h
  3. 2
      db/db_impl.cc
  4. 2
      db/version_edit.h
  5. 141
      db/version_set.cc
  6. 16
      db/version_set.h
  7. 78
      db/version_set_test.cc

7
db/compaction.cc
Unescape View File

@ -75,6 +75,13 @@ Compaction::~Compaction() {
}
}
void Compaction::GenerateFileLevels() {
input_levels_.resize(2);
for (int which = 0; which < 2; which++) {
DoGenerateFileLevel(&input_levels_[which], inputs_[which], &arena_);
}
}
bool Compaction::IsTrivialMove() const {
// Avoid a move if there is lots of overlapping grandparent data.
// Otherwise, the move could create a parent file that will require

19
db/compaction.h
Unescape View File

@ -8,6 +8,8 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#pragma once
#include "util/arena.h"
#include "util/autovector.h"
#include "db/version_set.h"
namespace rocksdb {
@ -18,6 +20,10 @@ class ColumnFamilyData;
// A Compaction encapsulates information about a compaction.
class Compaction {
public:
// No copying allowed
Compaction(const Compaction&) = delete;
void operator=(const Compaction&) = delete;
~Compaction();
// Return the level that is being compacted. Inputs from "level"
@ -44,6 +50,9 @@ class Compaction {
std::vector<FileMetaData*>* inputs(int which) { return &inputs_[which]; }
// Return the input_level file
FileLevel* input_levels(int which) { return &input_levels_[which]; }
// Maximum size of files to build during this compaction.
uint64_t MaxOutputFileSize() const { return max_output_file_size_; }
@ -53,7 +62,11 @@ class Compaction {
// Whether need to write output file to second DB path.
uint32_t GetOutputPathId() const { return output_path_id_; }
// Is this a trivial compaction that can be implemented by just
// Generate input_levels_ from inputs_
// Should be called when inputs_ is stable
void GenerateFileLevels();
// Is this a trivial compaction that can be implemented by just
// moving a single input file to the next level (no merging or splitting)
bool IsTrivialMove() const;
@ -118,6 +131,7 @@ class Compaction {
VersionEdit* edit_;
int number_levels_;
ColumnFamilyData* cfd_;
Arena arena_; // Arena used to allocate space for file_levels_
uint32_t output_path_id_;
CompressionType output_compression_;
@ -128,6 +142,9 @@ class Compaction {
// Each compaction reads inputs from "level_" and "level_+1"
std::vector<FileMetaData*> inputs_[2]; // The two sets of inputs
// A copy of inputs_, organized more closely in memory
autovector<FileLevel, 2> input_levels_;
// State used to check for number of of overlapping grandparent files
// (parent == level_ + 1, grandparent == level_ + 2)
std::vector<FileMetaData*> grandparents_;

2
db/db_impl.cc
Unescape View File

@ -2903,6 +2903,8 @@ Status DBImpl::DoCompactionWork(CompactionState* compact,
compact->CleanupMergedBuffer();
bool prefix_initialized = false;
// Generate file_levels_ for compaction berfore making Iterator
compact->compaction->GenerateFileLevels();
int64_t imm_micros = 0; // Micros spent doing imm_ compactions
ColumnFamilyData* cfd = compact->compaction->column_family_data();
LogToBuffer(

2
db/version_edit.h
Unescape View File

@ -14,6 +14,8 @@
#include <string>
#include "rocksdb/cache.h"
#include "db/dbformat.h"
#include "util/arena.h"
#include "util/autovector.h"
namespace rocksdb {

141
db/version_set.cc
Unescape View File

@ -82,33 +82,6 @@ Version::~Version() {
delete[] files_;
}
int FindFileInRange(const InternalKeyComparator& icmp,
const std::vector<FileMetaData*>& files,
const Slice& key,
uint32_t left,
uint32_t right) {
while (left < right) {
uint32_t mid = (left + right) / 2;
const FileMetaData* f = files[mid];
if (icmp.InternalKeyComparator::Compare(f->largest.Encode(), key) < 0) {
// Key at "mid.largest" is < "target". Therefore all
// files at or before "mid" are uninteresting.
left = mid + 1;
} else {
// Key at "mid.largest" is >= "target". Therefore all files
// after "mid" are uninteresting.
right = mid;
}
}
return right;
}
int FindFile(const InternalKeyComparator& icmp,
const std::vector<FileMetaData*>& files,
const Slice& key) {
return FindFileInRange(icmp, files, key, 0, files.size());
}
// Find File in FileLevel data structure
// Within an index range defined by left and right
int FindFileInRange(const InternalKeyComparator& icmp,
@ -138,6 +111,36 @@ int FindFile(const InternalKeyComparator& icmp,
return FindFileInRange(icmp, file_level, key, 0, file_level.num_files);
}
void DoGenerateFileLevel(FileLevel* file_level,
const std::vector<FileMetaData*>& files,
Arena* arena) {
assert(file_level);
assert(files.size() >= 0);
assert(arena);
size_t num = files.size();
file_level->num_files = num;
char* mem = arena->AllocateAligned(num * sizeof(FdWithKeyRange));
file_level->files = new (mem)FdWithKeyRange[num];
for (size_t i = 0; i < num; i++) {
Slice smallest_key = files[i]->smallest.Encode();
Slice largest_key = files[i]->largest.Encode();
// Copy key slice to sequential memory
size_t smallest_size = smallest_key.size();
size_t largest_size = largest_key.size();
mem = arena->AllocateAligned(smallest_size + largest_size);
memcpy(mem, smallest_key.data(), smallest_size);
memcpy(mem + smallest_size, largest_key.data(), largest_size);
FdWithKeyRange& f = file_level->files[i];
f.fd = files[i]->fd;
f.smallest_key = Slice(mem, smallest_size);
f.largest_key = Slice(mem + smallest_size, largest_size);
}
}
static bool AfterFile(const Comparator* ucmp,
const Slice* user_key, const FdWithKeyRange* f) {
// nullptr user_key occurs before all keys and is therefore never after *f
@ -152,7 +155,6 @@ static bool BeforeFile(const Comparator* ucmp,
ucmp->Compare(*user_key, ExtractUserKey(f->smallest_key)) < 0);
}
bool SomeFileOverlapsRange(
const InternalKeyComparator& icmp,
bool disjoint_sorted_files,
@ -198,21 +200,21 @@ bool SomeFileOverlapsRange(
class Version::LevelFileNumIterator : public Iterator {
public:
LevelFileNumIterator(const InternalKeyComparator& icmp,
const std::vector<FileMetaData*>* flist)
const FileLevel* flevel)
: icmp_(icmp),
flist_(flist),
index_(flist->size()),
flevel_(flevel),
index_(flevel->num_files),
current_value_(0, 0, 0) { // Marks as invalid
}
virtual bool Valid() const {
return index_ < flist_->size();
return index_ < flevel_->num_files;
}
virtual void Seek(const Slice& target) {
index_ = FindFile(icmp_, *flist_, target);
index_ = FindFile(icmp_, *flevel_, target);
}
virtual void SeekToFirst() { index_ = 0; }
virtual void SeekToLast() {
index_ = flist_->empty() ? 0 : flist_->size() - 1;
index_ = (flevel_->num_files == 0) ? 0 : flevel_->num_files - 1;
}
virtual void Next() {
assert(Valid());
@ -221,26 +223,27 @@ class Version::LevelFileNumIterator : public Iterator {
virtual void Prev() {
assert(Valid());
if (index_ == 0) {
index_ = flist_->size(); // Marks as invalid
index_ = flevel_->num_files; // Marks as invalid
} else {
index_--;
}
}
Slice key() const {
assert(Valid());
return (*flist_)[index_]->largest.Encode();
return flevel_->files[index_].largest_key;
}
Slice value() const {
assert(Valid());
auto* file_meta = (*flist_)[index_];
current_value_ = file_meta->fd;
auto file_meta = flevel_->files[index_];
current_value_ = file_meta.fd;
return Slice(reinterpret_cast<const char*>(&current_value_),
sizeof(FileDescriptor));
}
virtual Status status() const { return Status::OK(); }
private:
const InternalKeyComparator icmp_;
const std::vector<FileMetaData*>* const flist_;
const FileLevel* flevel_;
uint32_t index_;
mutable FileDescriptor current_value_;
};
@ -357,21 +360,23 @@ void Version::AddIterators(const ReadOptions& read_options,
const EnvOptions& soptions,
std::vector<Iterator*>* iters) {
// Merge all level zero files together since they may overlap
for (const FileMetaData* file : files_[0]) {
for (size_t i = 0; i < file_levels_[0].num_files; i++) {
const auto& file = file_levels_[0].files[i];
iters->push_back(cfd_->table_cache()->NewIterator(
read_options, soptions, cfd_->internal_comparator(), file->fd));
read_options, soptions, cfd_->internal_comparator(), file.fd));
}
// For levels > 0, we can use a concatenating iterator that sequentially
// walks through the non-overlapping files in the level, opening them
// lazily.
for (int level = 1; level < num_levels_; level++) {
if (!files_[level].empty()) {
if (file_levels_[level].num_files != 0) {
iters->push_back(NewTwoLevelIterator(new LevelFileIteratorState(
cfd_->table_cache(), read_options, soptions,
cfd_->internal_comparator(), false /* for_compaction */,
cfd_->options()->prefix_extractor != nullptr),
new LevelFileNumIterator(cfd_->internal_comparator(), &files_[level])));
new LevelFileNumIterator(cfd_->internal_comparator(),
&file_levels_[level])));
}
}
}
@ -380,9 +385,10 @@ void Version::AddIterators(const ReadOptions& read_options,
const EnvOptions& soptions,
MergeIteratorBuilder* merge_iter_builder) {
// Merge all level zero files together since they may overlap
for (const FileMetaData* file : files_[0]) {
for (size_t i = 0; i < file_levels_[0].num_files; i++) {
const auto& file = file_levels_[0].files[i];
merge_iter_builder->AddIterator(cfd_->table_cache()->NewIterator(
read_options, soptions, cfd_->internal_comparator(), file->fd, nullptr,
read_options, soptions, cfd_->internal_comparator(), file.fd, nullptr,
false, merge_iter_builder->GetArena()));
}
@ -390,14 +396,14 @@ void Version::AddIterators(const ReadOptions& read_options,
// walks through the non-overlapping files in the level, opening them
// lazily.
for (int level = 1; level < num_levels_; level++) {
if (!files_[level].empty()) {
if (file_levels_[level].num_files != 0) {
merge_iter_builder->AddIterator(NewTwoLevelIterator(
new LevelFileIteratorState(
cfd_->table_cache(), read_options, soptions,
cfd_->internal_comparator(), false /* for_compaction */,
cfd_->options()->prefix_extractor != nullptr),
new LevelFileNumIterator(cfd_->internal_comparator(), &files_[level]),
merge_iter_builder->GetArena()));
new LevelFileNumIterator(cfd_->internal_comparator(),
&file_levels_[level]), merge_iter_builder->GetArena()));
}
}
}
@ -659,6 +665,7 @@ void Version::Get(const ReadOptions& options,
for (int32_t i = start_index; i < num_files;) {
FdWithKeyRange* f = &files[i];
assert(f->fd.GetNumber() == files_[level][i]->fd.GetNumber());
int cmp_largest = -1;
// Do key range filtering of files or/and fractional cascading if:
@ -746,7 +753,6 @@ void Version::Get(const ReadOptions& options,
}
}
if (kMerge == saver.state) {
// merge_operands are in saver and we hit the beginning of the key history
// do a final merge of nullptr and operands;
@ -767,29 +773,7 @@ void Version::Get(const ReadOptions& options,
void Version::GenerateFileLevels() {
file_levels_.resize(num_non_empty_levels_);
for (int level = 0; level < num_non_empty_levels_; level++) {
const auto& files = files_[level];
auto& file_level = file_levels_[level];
size_t num = files.size();
file_level.num_files = num;
char* mem = arena_.AllocateAligned(num * sizeof(FdWithKeyRange));
file_level.files = new (mem)FdWithKeyRange[num];
for (size_t i = 0; i < files.size(); i++) {
Slice smallest_key = files[i]->smallest.Encode();
Slice largest_key = files[i]->largest.Encode();
// Copy key slice to sequential memory
size_t smallest_size = smallest_key.size();
size_t largest_size = largest_key.size();
mem = arena_.AllocateAligned(smallest_size + largest_size);
memcpy(mem, smallest_key.data(), smallest_size);
memcpy(mem + smallest_size, largest_key.data(), largest_size);
file_level.files[i].fd = files[i]->fd;
file_level.files[i].smallest_key = Slice(mem, smallest_size);
file_level.files[i].largest_key = Slice(mem+smallest_size, largest_size);
}
DoGenerateFileLevel(&file_levels_[level], files_[level], &arena_);
}
}
@ -1181,7 +1165,7 @@ void Version::GetOverlappingInputsBinarySearch(
// The midIndex specifies the index of at least one file that
// overlaps the specified range. From that file, iterate backward
// and forward to find all overlapping files.
// Use compressed file meda data, make search faster
// Use FileLevel in searching, make it faster
void Version::ExtendOverlappingInputs(
int level,
const Slice& user_begin,
@ -2764,16 +2748,17 @@ Iterator* VersionSet::MakeInputIterator(Compaction* c) {
// Level-0 files have to be merged together. For other levels,
// we will make a concatenating iterator per level.
// TODO(opt): use concatenating iterator for level-0 if there is no overlap
const int space = (c->level() == 0 ? c->inputs(0)->size() + 1 : 2);
const int space = (c->level() == 0 ? c->input_levels(0)->num_files + 1 : 2);
Iterator** list = new Iterator*[space];
int num = 0;
for (int which = 0; which < 2; which++) {
if (!c->inputs(which)->empty()) {
if (c->input_levels(which)->num_files != 0) {
if (c->level() + which == 0) {
for (const auto& file : *c->inputs(which)) {
const FileLevel* flevel = c->input_levels(which);
for (size_t i = 0; i < flevel->num_files; i++) {
list[num++] = cfd->table_cache()->NewIterator(
read_options, storage_options_compactions_,
cfd->internal_comparator(), file->fd, nullptr,
cfd->internal_comparator(), flevel->files[i].fd, nullptr,
true /* for compaction */);
}
} else {
@ -2783,7 +2768,7 @@ Iterator* VersionSet::MakeInputIterator(Compaction* c) {
cfd->internal_comparator(), true /* for_compaction */,
false /* prefix enabled */),
new Version::LevelFileNumIterator(cfd->internal_comparator(),
c->inputs(which)));
c->input_levels(which)));
}
}
}

16
db/version_set.h
Unescape View File

@ -53,14 +53,6 @@ class ColumnFamilySet;
class TableCache;
class MergeIteratorBuilder;
// Return the smallest index i such that files[i]->largest >= key.
// Return files.size() if there is no such file.
// REQUIRES: "files" contains a sorted list of non-overlapping files.
extern int FindFile(const InternalKeyComparator& icmp,
const std::vector<FileMetaData*>& files,
const Slice& key);
// Return the smallest index i such that file_level.files[i]->largest >= key.
// Return file_level.num_files if there is no such file.
// REQUIRES: "file_level.files" contains a sorted list of
@ -82,6 +74,13 @@ extern bool SomeFileOverlapsRange(
const Slice* smallest_user_key,
const Slice* largest_user_key);
// Generate FileLevel from vector<FdWithKeyRange*>
// Would copy smallest_key and largest_key data to sequential memory
// arena: Arena used to allocate the memory
extern void DoGenerateFileLevel(FileLevel* file_level,
const std::vector<FileMetaData*>& files,
Arena* arena);
class Version {
public:
// Append to *iters a sequence of iterators that will
@ -294,7 +293,6 @@ class Version {
// in increasing order of keys
std::vector<FileMetaData*>* files_;
// A list for the same set of files that are stored in files_,
// but files in each level are now sorted based on file
// size. The file with the largest size is at the front.

78
db/version_set_test.cc
Unescape View File

@ -14,14 +14,15 @@
namespace rocksdb {
class FindFileTest {
class GenerateFileLevelTest {
public:
std::vector<FileMetaData*> files_;
bool disjoint_sorted_files_;
FileLevel file_level_;
Arena arena_;
FindFileTest() : disjoint_sorted_files_(true) { }
GenerateFileLevelTest() { }
~FindFileTest() {
~GenerateFileLevelTest() {
for (unsigned int i = 0; i < files_.size(); i++) {
delete files_[i];
}
@ -37,55 +38,44 @@ class FindFileTest {
files_.push_back(f);
}
int Find(const char* key) {
InternalKey target(key, 100, kTypeValue);
InternalKeyComparator cmp(BytewiseComparator());
return FindFile(cmp, files_, target.Encode());
int Compare() {
int diff = 0;
for (size_t i = 0; i < files_.size(); i++) {
if (file_level_.files[i].fd.GetNumber() != files_[i]->fd.GetNumber()) {
diff++;
}
}
return diff;
}
};
TEST(FindFileTest, Empty) {
ASSERT_EQ(0, Find("foo"));
TEST(GenerateFileLevelTest, Empty) {
DoGenerateFileLevel(&file_level_, files_, &arena_);
ASSERT_EQ(0, file_level_.num_files);
ASSERT_EQ(0, Compare());
}
TEST(FindFileTest, Single) {
TEST(GenerateFileLevelTest, Single) {
Add("p", "q");
ASSERT_EQ(0, Find("a"));
ASSERT_EQ(0, Find("p"));
ASSERT_EQ(0, Find("p1"));
ASSERT_EQ(0, Find("q"));
ASSERT_EQ(1, Find("q1"));
ASSERT_EQ(1, Find("z"));
DoGenerateFileLevel(&file_level_, files_, &arena_);
ASSERT_EQ(1, file_level_.num_files);
ASSERT_EQ(0, Compare());
}
TEST(FindFileTest, Multiple) {
TEST(GenerateFileLevelTest, Multiple) {
Add("150", "200");
Add("200", "250");
Add("300", "350");
Add("400", "450");
ASSERT_EQ(0, Find("100"));
ASSERT_EQ(0, Find("150"));
ASSERT_EQ(0, Find("151"));
ASSERT_EQ(0, Find("199"));
ASSERT_EQ(0, Find("200"));
ASSERT_EQ(1, Find("201"));
ASSERT_EQ(1, Find("249"));
ASSERT_EQ(1, Find("250"));
ASSERT_EQ(2, Find("251"));
ASSERT_EQ(2, Find("299"));
ASSERT_EQ(2, Find("300"));
ASSERT_EQ(2, Find("349"));
ASSERT_EQ(2, Find("350"));
ASSERT_EQ(3, Find("351"));
ASSERT_EQ(3, Find("400"));
ASSERT_EQ(3, Find("450"));
ASSERT_EQ(4, Find("451"));
DoGenerateFileLevel(&file_level_, files_, &arena_);
ASSERT_EQ(4, file_level_.num_files);
ASSERT_EQ(0, Compare());
}
class FindLevelFileTest {
public:
FileLevel level_files_;
FileLevel file_level_;
bool disjoint_sorted_files_;
Arena arena_;
@ -96,8 +86,8 @@ class FindLevelFileTest {
void LevelFileInit(size_t num = 0) {
char* mem = arena_.AllocateAligned(num * sizeof(FdWithKeyRange));
level_files_.files = new (mem)FdWithKeyRange[num];
level_files_.num_files = 0;
file_level_.files = new (mem)FdWithKeyRange[num];
file_level_.num_files = 0;
}
void Add(const char* smallest, const char* largest,
@ -115,27 +105,27 @@ class FindLevelFileTest {
memcpy(mem + smallest_slice.size(), largest_slice.data(),
largest_slice.size());
// add compressd_level_
size_t num = level_files_.num_files;
auto& file = level_files_.files[num];
// add to file_level_
size_t num = file_level_.num_files;
auto& file = file_level_.files[num];
file.fd = FileDescriptor(num + 1, 0, 0);
file.smallest_key = Slice(mem, smallest_slice.size());
file.largest_key = Slice(mem + smallest_slice.size(),
largest_slice.size());
level_files_.num_files++;
file_level_.num_files++;
}
int Find(const char* key) {
InternalKey target(key, 100, kTypeValue);
InternalKeyComparator cmp(BytewiseComparator());
return FindFile(cmp, level_files_, target.Encode());
return FindFile(cmp, file_level_, target.Encode());
}
bool Overlaps(const char* smallest, const char* largest) {
InternalKeyComparator cmp(BytewiseComparator());
Slice s(smallest != nullptr ? smallest : "");
Slice l(largest != nullptr ? largest : "");
return SomeFileOverlapsRange(cmp, disjoint_sorted_files_, level_files_,
return SomeFileOverlapsRange(cmp, disjoint_sorted_files_, file_level_,
(smallest != nullptr ? &s : nullptr),
(largest != nullptr ? &l : nullptr));
}

Write Preview
Loading…
Cancel
Save