Conflicts: Makefile db/db_impl.cc db/db_impl.h db/db_test.cc db/memtable.cc db/memtable.h db/version_edit.h db/version_set.cc include/rocksdb/options.h util/hash_skiplist_rep.cc util/options.ccmain
kailiu
11 years ago
commit
1304d8c8ce
@ -0,0 +1,83 @@ |
||||
#!/bin/bash |
||||
set -e |
||||
# If clang_format_diff.py command is not specfied, we assume we are able to |
||||
# access directly without any path. |
||||
if [ -z $CLANG_FORMAT_DIFF ] |
||||
then |
||||
CLANG_FORMAT_DIFF="clang-format-diff.py" |
||||
fi |
||||
|
||||
# Check clang-format-diff.py |
||||
if ! which $CLANG_FORMAT_DIFF &> /dev/null |
||||
then |
||||
echo "You didn't have clang-format-diff.py available in your computer!" |
||||
echo "You can download it by running: " |
||||
echo " curl https://fburl.com/clang-format-diff" |
||||
exit 128 |
||||
fi |
||||
|
||||
# Check argparse, a library that clang-format-diff.py requires. |
||||
python 2>/dev/null << EOF |
||||
import argparse |
||||
EOF |
||||
|
||||
if [ "$?" != 0 ] |
||||
then |
||||
echo "To run clang-format-diff.py, we'll need the library "argparse" to be" |
||||
echo "installed. You can try either of the follow ways to install it:" |
||||
echo " 1. Manually download argparse: https://pypi.python.org/pypi/argparse" |
||||
echo " 2. easy_install argparse (if you have easy_install)" |
||||
echo " 3. pip install argparse (if you have pip)" |
||||
exit 129 |
||||
fi |
||||
|
||||
# TODO(kailiu) following work is not complete since we still need to figure |
||||
# out how to add the modified files done pre-commit hook to git's commit index. |
||||
# |
||||
# Check if this script has already been added to pre-commit hook. |
||||
# Will suggest user to add this script to pre-commit hook if their pre-commit |
||||
# is empty. |
||||
# PRE_COMMIT_SCRIPT_PATH="`git rev-parse --show-toplevel`/.git/hooks/pre-commit" |
||||
# if ! ls $PRE_COMMIT_SCRIPT_PATH &> /dev/null |
||||
# then |
||||
# echo "Would you like to add this script to pre-commit hook, which will do " |
||||
# echo -n "the format check for all the affected lines before you check in (y/n):" |
||||
# read add_to_hook |
||||
# if [ "$add_to_hook" == "y" ] |
||||
# then |
||||
# ln -s `git rev-parse --show-toplevel`/build_tools/format-diff.sh $PRE_COMMIT_SCRIPT_PATH |
||||
# fi |
||||
# fi |
||||
|
||||
# Check the format of recently changed lines, |
||||
diffs=$(git diff -U0 HEAD^ | $CLANG_FORMAT_DIFF -p 1) |
||||
|
||||
if [ -z "$diffs" ] |
||||
then |
||||
echo "Nothing needs to be reformatted!" |
||||
exit 0 |
||||
fi |
||||
|
||||
# Highlight the insertion/deletion from the clang-format-diff.py's output |
||||
COLOR_END="\033[0m" |
||||
COLOR_RED="\033[0;31m" |
||||
COLOR_GREEN="\033[0;32m" |
||||
|
||||
echo -e "Detect lines that doesn't follow the format rules:\r" |
||||
# Add the color to the diff. lines added will be green; lines removed will be red. |
||||
echo "$diffs" | |
||||
sed -e "s/\(^-.*$\)/`echo -e \"$COLOR_RED\1$COLOR_END\"`/" | |
||||
sed -e "s/\(^+.*$\)/`echo -e \"$COLOR_GREEN\1$COLOR_END\"`/" |
||||
echo -e "Would you like to fix the format automatically (y/n): \c" |
||||
|
||||
# Make sure under any mode, we can read user input. |
||||
exec < /dev/tty |
||||
read to_fix |
||||
|
||||
if [ "$to_fix" != "y" ] |
||||
then |
||||
exit 1 |
||||
fi |
||||
|
||||
# Do in-place format adjustment. |
||||
git diff -U0 HEAD^ | $CLANG_FORMAT_DIFF -i -p 1 |
||||
@ -0,0 +1,214 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "db/compaction.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
static uint64_t TotalFileSize(const std::vector<FileMetaData*>& files) { |
||||
uint64_t sum = 0; |
||||
for (size_t i = 0; i < files.size() && files[i]; i++) { |
||||
sum += files[i]->file_size; |
||||
} |
||||
return sum; |
||||
} |
||||
|
||||
Compaction::Compaction(Version* input_version, int level, int out_level, |
||||
uint64_t target_file_size, |
||||
uint64_t max_grandparent_overlap_bytes, |
||||
bool seek_compaction, bool enable_compression) |
||||
: level_(level), |
||||
out_level_(out_level), |
||||
max_output_file_size_(target_file_size), |
||||
maxGrandParentOverlapBytes_(max_grandparent_overlap_bytes), |
||||
input_version_(input_version), |
||||
number_levels_(input_version_->NumberLevels()), |
||||
seek_compaction_(seek_compaction), |
||||
enable_compression_(enable_compression), |
||||
grandparent_index_(0), |
||||
seen_key_(false), |
||||
overlapped_bytes_(0), |
||||
base_index_(-1), |
||||
parent_index_(-1), |
||||
score_(0), |
||||
bottommost_level_(false), |
||||
is_full_compaction_(false), |
||||
level_ptrs_(std::vector<size_t>(number_levels_)) { |
||||
|
||||
input_version_->Ref(); |
||||
edit_ = new VersionEdit(); |
||||
for (int i = 0; i < number_levels_; i++) { |
||||
level_ptrs_[i] = 0; |
||||
} |
||||
} |
||||
|
||||
Compaction::~Compaction() { |
||||
delete edit_; |
||||
if (input_version_ != nullptr) { |
||||
input_version_->Unref(); |
||||
} |
||||
} |
||||
|
||||
bool Compaction::IsTrivialMove() const { |
||||
// Avoid a move if there is lots of overlapping grandparent data.
|
||||
// Otherwise, the move could create a parent file that will require
|
||||
// a very expensive merge later on.
|
||||
// If level_== out_level_, the purpose is to force compaction filter to be
|
||||
// applied to that level, and thus cannot be a trivia move.
|
||||
return (level_ != out_level_ && |
||||
num_input_files(0) == 1 && |
||||
num_input_files(1) == 0 && |
||||
TotalFileSize(grandparents_) <= maxGrandParentOverlapBytes_); |
||||
} |
||||
|
||||
void Compaction::AddInputDeletions(VersionEdit* edit) { |
||||
for (int which = 0; which < 2; which++) { |
||||
for (size_t i = 0; i < inputs_[which].size(); i++) { |
||||
edit->DeleteFile(level_ + which, inputs_[which][i]->number); |
||||
} |
||||
} |
||||
} |
||||
|
||||
bool Compaction::IsBaseLevelForKey(const Slice& user_key) { |
||||
if (input_version_->vset_->options_->compaction_style == |
||||
kCompactionStyleUniversal) { |
||||
return bottommost_level_; |
||||
} |
||||
// Maybe use binary search to find right entry instead of linear search?
|
||||
const Comparator* user_cmp = input_version_->vset_->icmp_.user_comparator(); |
||||
for (int lvl = level_ + 2; lvl < number_levels_; lvl++) { |
||||
const std::vector<FileMetaData*>& files = input_version_->files_[lvl]; |
||||
for (; level_ptrs_[lvl] < files.size(); ) { |
||||
FileMetaData* f = files[level_ptrs_[lvl]]; |
||||
if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) { |
||||
// We've advanced far enough
|
||||
if (user_cmp->Compare(user_key, f->smallest.user_key()) >= 0) { |
||||
// Key falls in this file's range, so definitely not base level
|
||||
return false; |
||||
} |
||||
break; |
||||
} |
||||
level_ptrs_[lvl]++; |
||||
} |
||||
} |
||||
return true; |
||||
} |
||||
|
||||
bool Compaction::ShouldStopBefore(const Slice& internal_key) { |
||||
// Scan to find earliest grandparent file that contains key.
|
||||
const InternalKeyComparator* icmp = &input_version_->vset_->icmp_; |
||||
while (grandparent_index_ < grandparents_.size() && |
||||
icmp->Compare(internal_key, |
||||
grandparents_[grandparent_index_]->largest.Encode()) > 0) { |
||||
if (seen_key_) { |
||||
overlapped_bytes_ += grandparents_[grandparent_index_]->file_size; |
||||
} |
||||
assert(grandparent_index_ + 1 >= grandparents_.size() || |
||||
icmp->Compare(grandparents_[grandparent_index_]->largest.Encode(), |
||||
grandparents_[grandparent_index_+1]->smallest.Encode()) |
||||
< 0); |
||||
grandparent_index_++; |
||||
} |
||||
seen_key_ = true; |
||||
|
||||
if (overlapped_bytes_ > maxGrandParentOverlapBytes_) { |
||||
// Too much overlap for current output; start new output
|
||||
overlapped_bytes_ = 0; |
||||
return true; |
||||
} else { |
||||
return false; |
||||
} |
||||
} |
||||
|
||||
// Mark (or clear) each file that is being compacted
|
||||
void Compaction::MarkFilesBeingCompacted(bool value) { |
||||
for (int i = 0; i < 2; i++) { |
||||
std::vector<FileMetaData*> v = inputs_[i]; |
||||
for (unsigned int j = 0; j < inputs_[i].size(); j++) { |
||||
assert(value ? !inputs_[i][j]->being_compacted : |
||||
inputs_[i][j]->being_compacted); |
||||
inputs_[i][j]->being_compacted = value; |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Is this compaction producing files at the bottommost level?
|
||||
void Compaction::SetupBottomMostLevel(bool isManual) { |
||||
if (input_version_->vset_->options_->compaction_style == |
||||
kCompactionStyleUniversal) { |
||||
// If universal compaction style is used and manual
|
||||
// compaction is occuring, then we are guaranteed that
|
||||
// all files will be picked in a single compaction
|
||||
// run. We can safely set bottommost_level_ = true.
|
||||
// If it is not manual compaction, then bottommost_level_
|
||||
// is already set when the Compaction was created.
|
||||
if (isManual) { |
||||
bottommost_level_ = true; |
||||
} |
||||
return; |
||||
} |
||||
bottommost_level_ = true; |
||||
int num_levels = input_version_->vset_->NumberLevels(); |
||||
for (int i = output_level() + 1; i < num_levels; i++) { |
||||
if (input_version_->NumLevelFiles(i) > 0) { |
||||
bottommost_level_ = false; |
||||
break; |
||||
} |
||||
} |
||||
} |
||||
|
||||
void Compaction::ReleaseInputs() { |
||||
if (input_version_ != nullptr) { |
||||
input_version_->Unref(); |
||||
input_version_ = nullptr; |
||||
} |
||||
} |
||||
|
||||
void Compaction::ResetNextCompactionIndex() { |
||||
input_version_->ResetNextCompactionIndex(level_); |
||||
} |
||||
|
||||
static void InputSummary(std::vector<FileMetaData*>& files, char* output, |
||||
int len) { |
||||
int write = 0; |
||||
for (unsigned int i = 0; i < files.size(); i++) { |
||||
int sz = len - write; |
||||
int ret = snprintf(output + write, sz, "%lu(%lu) ", |
||||
(unsigned long)files.at(i)->number, |
||||
(unsigned long)files.at(i)->file_size); |
||||
if (ret < 0 || ret >= sz) |
||||
break; |
||||
write += ret; |
||||
} |
||||
} |
||||
|
||||
void Compaction::Summary(char* output, int len) { |
||||
int write = snprintf(output, len, |
||||
"Base version %lu Base level %d, seek compaction:%d, inputs:", |
||||
(unsigned long)input_version_->GetVersionNumber(), |
||||
level_, |
||||
seek_compaction_); |
||||
if (write < 0 || write > len) { |
||||
return; |
||||
} |
||||
|
||||
char level_low_summary[100]; |
||||
InputSummary(inputs_[0], level_low_summary, sizeof(level_low_summary)); |
||||
char level_up_summary[100]; |
||||
if (inputs_[1].size()) { |
||||
InputSummary(inputs_[1], level_up_summary, sizeof(level_up_summary)); |
||||
} else { |
||||
level_up_summary[0] = '\0'; |
||||
} |
||||
|
||||
snprintf(output + write, len - write, "[%s],[%s]", |
||||
level_low_summary, level_up_summary); |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,131 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#pragma once |
||||
#include "db/version_set.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class Version; |
||||
|
||||
// A Compaction encapsulates information about a compaction.
|
||||
class Compaction { |
||||
public: |
||||
~Compaction(); |
||||
|
||||
// Return the level that is being compacted. Inputs from "level"
|
||||
// will be merged.
|
||||
int level() const { return level_; } |
||||
|
||||
// Outputs will go to this level
|
||||
int output_level() const { return out_level_; } |
||||
|
||||
// Return the object that holds the edits to the descriptor done
|
||||
// by this compaction.
|
||||
VersionEdit* edit() { return edit_; } |
||||
|
||||
// "which" must be either 0 or 1
|
||||
int num_input_files(int which) const { return inputs_[which].size(); } |
||||
|
||||
// Return the ith input file at "level()+which" ("which" must be 0 or 1).
|
||||
FileMetaData* input(int which, int i) const { return inputs_[which][i]; } |
||||
|
||||
// Maximum size of files to build during this compaction.
|
||||
uint64_t MaxOutputFileSize() const { return max_output_file_size_; } |
||||
|
||||
// Whether compression will be enabled for compaction outputs
|
||||
bool enable_compression() const { return enable_compression_; } |
||||
|
||||
// Is this a trivial compaction that can be implemented by just
|
||||
// moving a single input file to the next level (no merging or splitting)
|
||||
bool IsTrivialMove() const; |
||||
|
||||
// Add all inputs to this compaction as delete operations to *edit.
|
||||
void AddInputDeletions(VersionEdit* edit); |
||||
|
||||
// Returns true if the information we have available guarantees that
|
||||
// the compaction is producing data in "level+1" for which no data exists
|
||||
// in levels greater than "level+1".
|
||||
bool IsBaseLevelForKey(const Slice& user_key); |
||||
|
||||
// Returns true iff we should stop building the current output
|
||||
// before processing "internal_key".
|
||||
bool ShouldStopBefore(const Slice& internal_key); |
||||
|
||||
// Release the input version for the compaction, once the compaction
|
||||
// is successful.
|
||||
void ReleaseInputs(); |
||||
|
||||
void Summary(char* output, int len); |
||||
|
||||
// Return the score that was used to pick this compaction run.
|
||||
double score() const { return score_; } |
||||
|
||||
// Is this compaction creating a file in the bottom most level?
|
||||
bool BottomMostLevel() { return bottommost_level_; } |
||||
|
||||
// Does this compaction include all sst files?
|
||||
bool IsFullCompaction() { return is_full_compaction_; } |
||||
|
||||
private: |
||||
friend class Version; |
||||
friend class VersionSet; |
||||
|
||||
Compaction(Version* input_version, int level, int out_level, |
||||
uint64_t target_file_size, uint64_t max_grandparent_overlap_bytes, |
||||
bool seek_compaction = false, bool enable_compression = true); |
||||
|
||||
int level_; |
||||
int out_level_; // levels to which output files are stored
|
||||
uint64_t max_output_file_size_; |
||||
uint64_t maxGrandParentOverlapBytes_; |
||||
Version* input_version_; |
||||
VersionEdit* edit_; |
||||
int number_levels_; |
||||
|
||||
bool seek_compaction_; |
||||
bool enable_compression_; |
||||
|
||||
// Each compaction reads inputs from "level_" and "level_+1"
|
||||
std::vector<FileMetaData*> inputs_[2]; // The two sets of inputs
|
||||
|
||||
// State used to check for number of of overlapping grandparent files
|
||||
// (parent == level_ + 1, grandparent == level_ + 2)
|
||||
std::vector<FileMetaData*> grandparents_; |
||||
size_t grandparent_index_; // Index in grandparent_starts_
|
||||
bool seen_key_; // Some output key has been seen
|
||||
uint64_t overlapped_bytes_; // Bytes of overlap between current output
|
||||
// and grandparent files
|
||||
int base_index_; // index of the file in files_[level_]
|
||||
int parent_index_; // index of some file with same range in files_[level_+1]
|
||||
double score_; // score that was used to pick this compaction.
|
||||
|
||||
// Is this compaction creating a file in the bottom most level?
|
||||
bool bottommost_level_; |
||||
// Does this compaction include all sst files?
|
||||
bool is_full_compaction_; |
||||
|
||||
// level_ptrs_ holds indices into input_version_->levels_: our state
|
||||
// is that we are positioned at one of the file ranges for each
|
||||
// higher level than the ones involved in this compaction (i.e. for
|
||||
// all L >= level_ + 2).
|
||||
std::vector<size_t> level_ptrs_; |
||||
|
||||
// mark (or clear) all files that are being compacted
|
||||
void MarkFilesBeingCompacted(bool); |
||||
|
||||
// Initialize whether compaction producing files at the bottommost level
|
||||
void SetupBottomMostLevel(bool isManual); |
||||
|
||||
// In case of compaction error, reset the nextIndex that is used
|
||||
// to pick up the next file to be compacted from files_by_size_
|
||||
void ResetNextCompactionIndex(); |
||||
}; |
||||
|
||||
} // namespace rocksdb
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in new issue