|
|
|
@ -455,11 +455,16 @@ int Version::PickLevelForMemTableOutput( |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
// Store in "*inputs" all files in "level" that overlap [begin,end]
|
|
|
|
// Store in "*inputs" all files in "level" that overlap [begin,end]
|
|
|
|
|
|
|
|
// If hint_index is specified, then it points to a file in the
|
|
|
|
|
|
|
|
// overlapping range.
|
|
|
|
|
|
|
|
// The file_index returns a pointer to any file in an overlapping range.
|
|
|
|
void Version::GetOverlappingInputs( |
|
|
|
void Version::GetOverlappingInputs( |
|
|
|
int level, |
|
|
|
int level, |
|
|
|
const InternalKey* begin, |
|
|
|
const InternalKey* begin, |
|
|
|
const InternalKey* end, |
|
|
|
const InternalKey* end, |
|
|
|
std::vector<FileMetaData*>* inputs) { |
|
|
|
std::vector<FileMetaData*>* inputs, |
|
|
|
|
|
|
|
int hint_index, |
|
|
|
|
|
|
|
int* file_index) { |
|
|
|
inputs->clear(); |
|
|
|
inputs->clear(); |
|
|
|
Slice user_begin, user_end; |
|
|
|
Slice user_begin, user_end; |
|
|
|
if (begin != NULL) { |
|
|
|
if (begin != NULL) { |
|
|
|
@ -470,7 +475,8 @@ void Version::GetOverlappingInputs( |
|
|
|
} |
|
|
|
} |
|
|
|
const Comparator* user_cmp = vset_->icmp_.user_comparator(); |
|
|
|
const Comparator* user_cmp = vset_->icmp_.user_comparator(); |
|
|
|
if (begin != NULL && end != NULL && level > 0) { |
|
|
|
if (begin != NULL && end != NULL && level > 0) { |
|
|
|
GetOverlappingInputsBinarySearch(level, user_begin, user_end, inputs); |
|
|
|
GetOverlappingInputsBinarySearch(level, user_begin, user_end, inputs, |
|
|
|
|
|
|
|
hint_index, file_index); |
|
|
|
return; |
|
|
|
return; |
|
|
|
} |
|
|
|
} |
|
|
|
for (size_t i = 0; i < files_[level].size(); ) { |
|
|
|
for (size_t i = 0; i < files_[level].size(); ) { |
|
|
|
@ -495,6 +501,8 @@ void Version::GetOverlappingInputs( |
|
|
|
inputs->clear(); |
|
|
|
inputs->clear(); |
|
|
|
i = 0; |
|
|
|
i = 0; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
} else if (file_index) { |
|
|
|
|
|
|
|
*file_index = i; |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
@ -508,14 +516,24 @@ void Version::GetOverlappingInputsBinarySearch( |
|
|
|
int level, |
|
|
|
int level, |
|
|
|
const Slice& user_begin, |
|
|
|
const Slice& user_begin, |
|
|
|
const Slice& user_end, |
|
|
|
const Slice& user_end, |
|
|
|
std::vector<FileMetaData*>* inputs) { |
|
|
|
std::vector<FileMetaData*>* inputs, |
|
|
|
|
|
|
|
int hint_index, |
|
|
|
|
|
|
|
int* file_index) { |
|
|
|
assert(level > 0); |
|
|
|
assert(level > 0); |
|
|
|
int min = 0; |
|
|
|
int min = 0; |
|
|
|
int mid = 0; |
|
|
|
int mid = 0; |
|
|
|
int max = files_[level].size() -1; |
|
|
|
int max = files_[level].size() -1; |
|
|
|
bool foundOverlap = false; |
|
|
|
bool foundOverlap = false; |
|
|
|
const Comparator* user_cmp = vset_->icmp_.user_comparator(); |
|
|
|
const Comparator* user_cmp = vset_->icmp_.user_comparator(); |
|
|
|
while (min <= max) { |
|
|
|
|
|
|
|
|
|
|
|
// if the caller already knows the index of a file that has overlap,
|
|
|
|
|
|
|
|
// then we can skip the binary search.
|
|
|
|
|
|
|
|
if (hint_index != -1) { |
|
|
|
|
|
|
|
mid = hint_index; |
|
|
|
|
|
|
|
foundOverlap = true; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
while (!foundOverlap && min <= max) { |
|
|
|
mid = (min + max)/2; |
|
|
|
mid = (min + max)/2; |
|
|
|
FileMetaData* f = files_[level][mid]; |
|
|
|
FileMetaData* f = files_[level][mid]; |
|
|
|
const Slice file_start = f->smallest.user_key(); |
|
|
|
const Slice file_start = f->smallest.user_key(); |
|
|
|
@ -534,6 +552,10 @@ void Version::GetOverlappingInputsBinarySearch( |
|
|
|
if (!foundOverlap) { |
|
|
|
if (!foundOverlap) { |
|
|
|
return; |
|
|
|
return; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
// returns the index where an overlap is found
|
|
|
|
|
|
|
|
if (file_index) { |
|
|
|
|
|
|
|
*file_index = mid; |
|
|
|
|
|
|
|
} |
|
|
|
ExtendOverlappingInputs(level, user_begin, user_end, inputs, mid); |
|
|
|
ExtendOverlappingInputs(level, user_begin, user_end, inputs, mid); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@ -548,13 +570,21 @@ void Version::ExtendOverlappingInputs( |
|
|
|
std::vector<FileMetaData*>* inputs, |
|
|
|
std::vector<FileMetaData*>* inputs, |
|
|
|
int midIndex) { |
|
|
|
int midIndex) { |
|
|
|
|
|
|
|
|
|
|
|
// assert that the file at midIndex overlaps with the range
|
|
|
|
|
|
|
|
const Comparator* user_cmp = vset_->icmp_.user_comparator(); |
|
|
|
const Comparator* user_cmp = vset_->icmp_.user_comparator(); |
|
|
|
assert(midIndex < files_[level].size()); |
|
|
|
#ifndef NDEBUG |
|
|
|
assert((user_cmp->Compare(files_[level][midIndex]->largest.user_key(), |
|
|
|
{ |
|
|
|
user_begin) >= 0) || |
|
|
|
// assert that the file at midIndex overlaps with the range
|
|
|
|
(user_cmp->Compare(files_[level][midIndex]->smallest.user_key(), |
|
|
|
assert(midIndex < files_[level].size()); |
|
|
|
user_end) <= 0)); |
|
|
|
FileMetaData* f = files_[level][midIndex]; |
|
|
|
|
|
|
|
const Slice fstart = f->smallest.user_key(); |
|
|
|
|
|
|
|
const Slice flimit = f->largest.user_key(); |
|
|
|
|
|
|
|
if (user_cmp->Compare(fstart, user_begin) >= 0) { |
|
|
|
|
|
|
|
assert(user_cmp->Compare(fstart, user_end) <= 0); |
|
|
|
|
|
|
|
} else { |
|
|
|
|
|
|
|
assert(user_cmp->Compare(flimit, user_begin) >= 0); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
#endif |
|
|
|
|
|
|
|
|
|
|
|
// check backwards from 'mid' to lower indices
|
|
|
|
// check backwards from 'mid' to lower indices
|
|
|
|
for (size_t i = midIndex; i < files_[level].size(); i--) { |
|
|
|
for (size_t i = midIndex; i < files_[level].size(); i--) { |
|
|
|
@ -864,11 +894,11 @@ VersionSet::VersionSet(const std::string& dbname, |
|
|
|
last_sequence_(0), |
|
|
|
last_sequence_(0), |
|
|
|
log_number_(0), |
|
|
|
log_number_(0), |
|
|
|
prev_log_number_(0), |
|
|
|
prev_log_number_(0), |
|
|
|
|
|
|
|
num_levels_(options_->num_levels), |
|
|
|
descriptor_file_(NULL), |
|
|
|
descriptor_file_(NULL), |
|
|
|
descriptor_log_(NULL), |
|
|
|
descriptor_log_(NULL), |
|
|
|
dummy_versions_(this), |
|
|
|
dummy_versions_(this), |
|
|
|
current_(NULL), |
|
|
|
current_(NULL), |
|
|
|
num_levels_(options_->num_levels), |
|
|
|
|
|
|
|
compactions_in_progress_(options_->num_levels), |
|
|
|
compactions_in_progress_(options_->num_levels), |
|
|
|
current_version_number_(0) { |
|
|
|
current_version_number_(0) { |
|
|
|
compact_pointer_ = new std::string[options_->num_levels]; |
|
|
|
compact_pointer_ = new std::string[options_->num_levels]; |
|
|
|
@ -940,9 +970,8 @@ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu, |
|
|
|
|
|
|
|
|
|
|
|
// process all requests in the queue
|
|
|
|
// process all requests in the queue
|
|
|
|
ManifestWriter* last_writer = &w; |
|
|
|
ManifestWriter* last_writer = &w; |
|
|
|
ManifestWriter* first = manifest_writers_.front(); |
|
|
|
|
|
|
|
assert(!manifest_writers_.empty()); |
|
|
|
assert(!manifest_writers_.empty()); |
|
|
|
assert(first == &w); |
|
|
|
assert(manifest_writers_.front() == &w); |
|
|
|
std::deque<ManifestWriter*>::iterator iter = manifest_writers_.begin(); |
|
|
|
std::deque<ManifestWriter*>::iterator iter = manifest_writers_.begin(); |
|
|
|
for (; iter != manifest_writers_.end(); ++iter) { |
|
|
|
for (; iter != manifest_writers_.end(); ++iter) { |
|
|
|
last_writer = *iter; |
|
|
|
last_writer = *iter; |
|
|
|
@ -1014,7 +1043,7 @@ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu, |
|
|
|
prev_log_number_ = edit->prev_log_number_; |
|
|
|
prev_log_number_ = edit->prev_log_number_; |
|
|
|
|
|
|
|
|
|
|
|
} else { |
|
|
|
} else { |
|
|
|
Log(options_->info_log, "Error in committing version %d",
|
|
|
|
Log(options_->info_log, "Error in committing version %ld",
|
|
|
|
v->GetVersionNumber()); |
|
|
|
v->GetVersionNumber()); |
|
|
|
delete v; |
|
|
|
delete v; |
|
|
|
if (!new_manifest_file.empty()) { |
|
|
|
if (!new_manifest_file.empty()) { |
|
|
|
@ -1732,10 +1761,13 @@ Compaction* VersionSet::PickCompactionBySize(int level) { |
|
|
|
} |
|
|
|
} |
|
|
|
// Do not pick this file if its parents at level+1 are being compacted.
|
|
|
|
// Do not pick this file if its parents at level+1 are being compacted.
|
|
|
|
// Maybe we can avoid redoing this work in SetupOtherInputs
|
|
|
|
// Maybe we can avoid redoing this work in SetupOtherInputs
|
|
|
|
if (ParentFilesInCompaction(f, level)) { |
|
|
|
int parent_index = -1; |
|
|
|
|
|
|
|
if (ParentFilesInCompaction(f, level, &parent_index)) { |
|
|
|
continue; |
|
|
|
continue; |
|
|
|
} |
|
|
|
} |
|
|
|
c->inputs_[0].push_back(f); |
|
|
|
c->inputs_[0].push_back(f); |
|
|
|
|
|
|
|
c->base_index_ = i; |
|
|
|
|
|
|
|
c->parent_index_ = parent_index; |
|
|
|
break; |
|
|
|
break; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@ -1748,7 +1780,7 @@ Compaction* VersionSet::PickCompactionBySize(int level) { |
|
|
|
|
|
|
|
|
|
|
|
Compaction* VersionSet::PickCompaction() { |
|
|
|
Compaction* VersionSet::PickCompaction() { |
|
|
|
Compaction* c = NULL; |
|
|
|
Compaction* c = NULL; |
|
|
|
int level; |
|
|
|
int level = -1; |
|
|
|
|
|
|
|
|
|
|
|
// compute the compactions needed. It is better to do it here
|
|
|
|
// compute the compactions needed. It is better to do it here
|
|
|
|
// and also in LogAndApply(), otherwise the values could be stale.
|
|
|
|
// and also in LogAndApply(), otherwise the values could be stale.
|
|
|
|
@ -1795,7 +1827,8 @@ Compaction* VersionSet::PickCompaction() { |
|
|
|
current_->GetOverlappingInputs(0, &smallest, &largest, &more); |
|
|
|
current_->GetOverlappingInputs(0, &smallest, &largest, &more); |
|
|
|
for (unsigned int i = 0; i < more.size(); i++) { |
|
|
|
for (unsigned int i = 0; i < more.size(); i++) { |
|
|
|
FileMetaData* f = more[i]; |
|
|
|
FileMetaData* f = more[i]; |
|
|
|
if (!f->being_compacted && !ParentFilesInCompaction(f, level)) { |
|
|
|
if (!f->being_compacted && |
|
|
|
|
|
|
|
!ParentFilesInCompaction(f, level, &c->parent_index_)) { |
|
|
|
c->inputs_[0].push_back(f); |
|
|
|
c->inputs_[0].push_back(f); |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
@ -1814,9 +1847,11 @@ Compaction* VersionSet::PickCompaction() { |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
// Returns true if any one of the parent files are being compacted
|
|
|
|
// Returns true if any one of the parent files are being compacted
|
|
|
|
bool VersionSet::ParentFilesInCompaction(FileMetaData* f, int level) { |
|
|
|
bool VersionSet::ParentFilesInCompaction(FileMetaData* f, int level, |
|
|
|
|
|
|
|
int* parent_index) { |
|
|
|
std::vector<FileMetaData*> inputs; |
|
|
|
std::vector<FileMetaData*> inputs; |
|
|
|
current_->GetOverlappingInputs(level+1, &f->smallest, &f->largest, &inputs); |
|
|
|
current_->GetOverlappingInputs(level+1, &f->smallest, &f->largest, |
|
|
|
|
|
|
|
&inputs, *parent_index, parent_index); |
|
|
|
return FilesInCompaction(inputs); |
|
|
|
return FilesInCompaction(inputs); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@ -1835,7 +1870,8 @@ void VersionSet::SetupOtherInputs(Compaction* c) { |
|
|
|
InternalKey smallest, largest; |
|
|
|
InternalKey smallest, largest; |
|
|
|
GetRange(c->inputs_[0], &smallest, &largest); |
|
|
|
GetRange(c->inputs_[0], &smallest, &largest); |
|
|
|
|
|
|
|
|
|
|
|
current_->GetOverlappingInputs(level+1, &smallest, &largest, &c->inputs_[1]); |
|
|
|
current_->GetOverlappingInputs(level+1, &smallest, &largest, &c->inputs_[1], |
|
|
|
|
|
|
|
c->parent_index_, &c->parent_index_); |
|
|
|
|
|
|
|
|
|
|
|
// Get entire range covered by compaction
|
|
|
|
// Get entire range covered by compaction
|
|
|
|
InternalKey all_start, all_limit; |
|
|
|
InternalKey all_start, all_limit; |
|
|
|
@ -1845,7 +1881,8 @@ void VersionSet::SetupOtherInputs(Compaction* c) { |
|
|
|
// changing the number of "level+1" files we pick up.
|
|
|
|
// changing the number of "level+1" files we pick up.
|
|
|
|
if (!c->inputs_[1].empty()) { |
|
|
|
if (!c->inputs_[1].empty()) { |
|
|
|
std::vector<FileMetaData*> expanded0; |
|
|
|
std::vector<FileMetaData*> expanded0; |
|
|
|
current_->GetOverlappingInputs(level, &all_start, &all_limit, &expanded0); |
|
|
|
current_->GetOverlappingInputs(level, &all_start, &all_limit, &expanded0, |
|
|
|
|
|
|
|
c->base_index_, NULL); |
|
|
|
const int64_t inputs0_size = TotalFileSize(c->inputs_[0]); |
|
|
|
const int64_t inputs0_size = TotalFileSize(c->inputs_[0]); |
|
|
|
const int64_t inputs1_size = TotalFileSize(c->inputs_[1]); |
|
|
|
const int64_t inputs1_size = TotalFileSize(c->inputs_[1]); |
|
|
|
const int64_t expanded0_size = TotalFileSize(expanded0); |
|
|
|
const int64_t expanded0_size = TotalFileSize(expanded0); |
|
|
|
@ -1857,7 +1894,8 @@ void VersionSet::SetupOtherInputs(Compaction* c) { |
|
|
|
GetRange(expanded0, &new_start, &new_limit); |
|
|
|
GetRange(expanded0, &new_start, &new_limit); |
|
|
|
std::vector<FileMetaData*> expanded1; |
|
|
|
std::vector<FileMetaData*> expanded1; |
|
|
|
current_->GetOverlappingInputs(level+1, &new_start, &new_limit, |
|
|
|
current_->GetOverlappingInputs(level+1, &new_start, &new_limit, |
|
|
|
&expanded1); |
|
|
|
&expanded1, c->parent_index_, |
|
|
|
|
|
|
|
&c->parent_index_); |
|
|
|
if (expanded1.size() == c->inputs_[1].size() &&
|
|
|
|
if (expanded1.size() == c->inputs_[1].size() &&
|
|
|
|
!FilesInCompaction(expanded1)) { |
|
|
|
!FilesInCompaction(expanded1)) { |
|
|
|
Log(options_->info_log, |
|
|
|
Log(options_->info_log, |
|
|
|
@ -1947,7 +1985,9 @@ Compaction::Compaction(int level, uint64_t target_file_size, |
|
|
|
seek_compaction_(seek_compaction), |
|
|
|
seek_compaction_(seek_compaction), |
|
|
|
grandparent_index_(0), |
|
|
|
grandparent_index_(0), |
|
|
|
seen_key_(false), |
|
|
|
seen_key_(false), |
|
|
|
overlapped_bytes_(0) { |
|
|
|
overlapped_bytes_(0), |
|
|
|
|
|
|
|
base_index_(-1), |
|
|
|
|
|
|
|
parent_index_(-1) { |
|
|
|
edit_ = new VersionEdit(number_levels_); |
|
|
|
edit_ = new VersionEdit(number_levels_); |
|
|
|
level_ptrs_ = new size_t[number_levels_]; |
|
|
|
level_ptrs_ = new size_t[number_levels_]; |
|
|
|
for (int i = 0; i < number_levels_; i++) { |
|
|
|
for (int i = 0; i < number_levels_; i++) { |
|
|
|
|
Dhruba Borthakur
12 years ago