Summary: This change will allow other table to reuse the code for meta blocks. Test Plan: all existing unit tests passed Reviewers: dhruba, haobo, sdong CC: leveldb Differential Revision: https://reviews.facebook.net/D14475main
kailiu
11 years ago
parent
e1d92dfd2e
commit
90729f8b23
@ -0,0 +1,134 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#include "table/meta_blocks.h" |
||||||
|
|
||||||
|
#include <map> |
||||||
|
|
||||||
|
#include "rocksdb/table_properties.h" |
||||||
|
#include "table/format.h" |
||||||
|
#include "util/coding.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
MetaIndexBuilder::MetaIndexBuilder() |
||||||
|
: meta_index_block_( |
||||||
|
new BlockBuilder(1 /* restart interval */, BytewiseComparator())) { |
||||||
|
} |
||||||
|
|
||||||
|
void MetaIndexBuilder::Add(const std::string& key, |
||||||
|
const BlockHandle& handle) { |
||||||
|
std::string handle_encoding; |
||||||
|
handle.EncodeTo(&handle_encoding); |
||||||
|
meta_block_handles_.insert({key, handle_encoding}); |
||||||
|
} |
||||||
|
|
||||||
|
Slice MetaIndexBuilder::Finish() { |
||||||
|
for (const auto& metablock : meta_block_handles_) { |
||||||
|
meta_index_block_->Add(metablock.first, metablock.second); |
||||||
|
} |
||||||
|
return meta_index_block_->Finish(); |
||||||
|
} |
||||||
|
|
||||||
|
PropertyBlockBuilder::PropertyBlockBuilder() |
||||||
|
: properties_block_( |
||||||
|
new BlockBuilder(1 /* restart interval */, BytewiseComparator())) { |
||||||
|
} |
||||||
|
|
||||||
|
void PropertyBlockBuilder::Add(const std::string& name, |
||||||
|
const std::string& val) { |
||||||
|
props_.insert({name, val}); |
||||||
|
} |
||||||
|
|
||||||
|
void PropertyBlockBuilder::Add(const std::string& name, uint64_t val) { |
||||||
|
assert(props_.find(name) == props_.end()); |
||||||
|
|
||||||
|
std::string dst; |
||||||
|
PutVarint64(&dst, val); |
||||||
|
|
||||||
|
Add(name, dst); |
||||||
|
} |
||||||
|
|
||||||
|
void PropertyBlockBuilder::Add( |
||||||
|
const UserCollectedProperties& user_collected_properties) { |
||||||
|
for (const auto& prop : user_collected_properties) { |
||||||
|
Add(prop.first, prop.second); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
void PropertyBlockBuilder::AddTableProperty(const TableProperties& props) { |
||||||
|
Add(TablePropertiesNames::kRawKeySize, props.raw_key_size); |
||||||
|
Add(TablePropertiesNames::kRawValueSize, props.raw_value_size); |
||||||
|
Add(TablePropertiesNames::kDataSize, props.data_size); |
||||||
|
Add(TablePropertiesNames::kIndexSize, props.index_size); |
||||||
|
Add(TablePropertiesNames::kNumEntries, props.num_entries); |
||||||
|
Add(TablePropertiesNames::kNumDataBlocks, props.num_data_blocks); |
||||||
|
Add(TablePropertiesNames::kFilterSize, props.filter_size); |
||||||
|
|
||||||
|
if (!props.filter_policy_name.empty()) { |
||||||
|
Add(TablePropertiesNames::kFilterPolicy, |
||||||
|
props.filter_policy_name); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
Slice PropertyBlockBuilder::Finish() { |
||||||
|
for (const auto& prop : props_) { |
||||||
|
properties_block_->Add(prop.first, prop.second); |
||||||
|
} |
||||||
|
|
||||||
|
return properties_block_->Finish(); |
||||||
|
} |
||||||
|
|
||||||
|
void LogPropertiesCollectionError( |
||||||
|
Logger* info_log, const std::string& method, const std::string& name) { |
||||||
|
assert(method == "Add" || method == "Finish"); |
||||||
|
|
||||||
|
std::string msg = |
||||||
|
"[Warning] encountered error when calling TablePropertiesCollector::" + |
||||||
|
method + "() with collector name: " + name; |
||||||
|
Log(info_log, "%s", msg.c_str()); |
||||||
|
} |
||||||
|
|
||||||
|
bool NotifyCollectTableCollectorsOnAdd( |
||||||
|
const Slice& key, |
||||||
|
const Slice& value, |
||||||
|
const Options::TablePropertiesCollectors& collectors, |
||||||
|
Logger* info_log) { |
||||||
|
bool all_succeeded = true; |
||||||
|
for (auto collector : collectors) { |
||||||
|
Status s = collector->Add(key, value); |
||||||
|
all_succeeded = all_succeeded && s.ok(); |
||||||
|
if (!s.ok()) { |
||||||
|
LogPropertiesCollectionError( |
||||||
|
info_log, "Add", /* method */ collector->Name() |
||||||
|
); |
||||||
|
} |
||||||
|
} |
||||||
|
return all_succeeded; |
||||||
|
} |
||||||
|
|
||||||
|
bool NotifyCollectTableCollectorsOnFinish( |
||||||
|
const Options::TablePropertiesCollectors& collectors, |
||||||
|
Logger* info_log, |
||||||
|
PropertyBlockBuilder* builder) { |
||||||
|
bool all_succeeded = true; |
||||||
|
for (auto collector : collectors) { |
||||||
|
UserCollectedProperties user_collected_properties; |
||||||
|
Status s = collector->Finish(&user_collected_properties); |
||||||
|
|
||||||
|
all_succeeded = all_succeeded && s.ok(); |
||||||
|
if (!s.ok()) { |
||||||
|
LogPropertiesCollectionError( |
||||||
|
info_log, "Finish", /* method */ collector->Name() |
||||||
|
); |
||||||
|
} else { |
||||||
|
builder->Add(user_collected_properties); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return all_succeeded; |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,106 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <map> |
||||||
|
#include <memory> |
||||||
|
#include <string> |
||||||
|
|
||||||
|
#include "rocksdb/comparator.h" |
||||||
|
#include "rocksdb/options.h" |
||||||
|
#include "rocksdb/slice.h" |
||||||
|
#include "table/block_builder.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class BlockHandle; |
||||||
|
class BlockBuilder; |
||||||
|
class Logger; |
||||||
|
struct TableProperties; |
||||||
|
|
||||||
|
// An STL style comparator that does the bytewise comparator comparasion
|
||||||
|
// internally.
|
||||||
|
struct BytewiseLessThan { |
||||||
|
bool operator()(const std::string& key1, const std::string& key2) const { |
||||||
|
// smaller entries will be placed in front.
|
||||||
|
return comparator->Compare(key1, key2) <= 0; |
||||||
|
} |
||||||
|
|
||||||
|
const Comparator* comparator = BytewiseComparator(); |
||||||
|
}; |
||||||
|
|
||||||
|
// When writing to a block that requires entries to be sorted by
|
||||||
|
// `BytewiseComparator`, we can buffer the content to `BytewiseSortedMap`
|
||||||
|
// before writng to store.
|
||||||
|
typedef std::map<std::string, std::string, BytewiseLessThan> BytewiseSortedMap; |
||||||
|
|
||||||
|
class MetaIndexBuilder { |
||||||
|
public: |
||||||
|
MetaIndexBuilder(const MetaIndexBuilder&) = delete; |
||||||
|
MetaIndexBuilder& operator=(const MetaIndexBuilder&) = delete; |
||||||
|
|
||||||
|
MetaIndexBuilder(); |
||||||
|
void Add(const std::string& key, const BlockHandle& handle); |
||||||
|
|
||||||
|
// Write all the added key/value pairs to the block and return the contents
|
||||||
|
// of the block.
|
||||||
|
Slice Finish(); |
||||||
|
|
||||||
|
private: |
||||||
|
// * Key: meta block name
|
||||||
|
// * Value: block handle to that meta block
|
||||||
|
struct Rep; |
||||||
|
Rep* rep_; |
||||||
|
|
||||||
|
// store the sorted key/handle of the metablocks.
|
||||||
|
BytewiseSortedMap meta_block_handles_; |
||||||
|
std::unique_ptr<BlockBuilder> meta_index_block_; |
||||||
|
}; |
||||||
|
|
||||||
|
class PropertyBlockBuilder { |
||||||
|
public: |
||||||
|
PropertyBlockBuilder(const PropertyBlockBuilder&) = delete; |
||||||
|
PropertyBlockBuilder& operator=(const PropertyBlockBuilder&) = delete; |
||||||
|
|
||||||
|
PropertyBlockBuilder(); |
||||||
|
|
||||||
|
void AddTableProperty(const TableProperties& props); |
||||||
|
void Add(const std::string& key, uint64_t value); |
||||||
|
void Add(const std::string& key, const std::string& value); |
||||||
|
void Add(const UserCollectedProperties& user_collected_properties); |
||||||
|
|
||||||
|
// Write all the added entries to the block and return the block contents
|
||||||
|
Slice Finish(); |
||||||
|
|
||||||
|
private: |
||||||
|
std::unique_ptr<BlockBuilder> properties_block_; |
||||||
|
BytewiseSortedMap props_; |
||||||
|
}; |
||||||
|
|
||||||
|
// Were we encounter any error occurs during user-defined statistics collection,
|
||||||
|
// we'll write the warning message to info log.
|
||||||
|
void LogPropertiesCollectionError( |
||||||
|
Logger* info_log, const std::string& method, const std::string& name); |
||||||
|
|
||||||
|
// Utility functions help table builder to trigger batch events for user
|
||||||
|
// defined property collectors.
|
||||||
|
// Return value indicates if there is any error occurred; if error occurred,
|
||||||
|
// the warning message will be logged.
|
||||||
|
// NotifyCollectTableCollectorsOnAdd() triggers the `Add` event for all
|
||||||
|
// property collectors.
|
||||||
|
bool NotifyCollectTableCollectorsOnAdd( |
||||||
|
const Slice& key, |
||||||
|
const Slice& value, |
||||||
|
const Options::TablePropertiesCollectors& collectors, |
||||||
|
Logger* info_log); |
||||||
|
|
||||||
|
// NotifyCollectTableCollectorsOnAdd() triggers the `Finish` event for all
|
||||||
|
// property collectors. The collected properties will be added to `builder`.
|
||||||
|
bool NotifyCollectTableCollectorsOnFinish( |
||||||
|
const Options::TablePropertiesCollectors& collectors, |
||||||
|
Logger* info_log, |
||||||
|
PropertyBlockBuilder* builder); |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,108 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#include "rocksdb/table_properties.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
namespace { |
||||||
|
void AppendProperty( |
||||||
|
std::string& props, |
||||||
|
const std::string& key, |
||||||
|
const std::string& value, |
||||||
|
const std::string& prop_delim, |
||||||
|
const std::string& kv_delim) { |
||||||
|
props.append(key); |
||||||
|
props.append(kv_delim); |
||||||
|
props.append(value); |
||||||
|
props.append(prop_delim); |
||||||
|
} |
||||||
|
|
||||||
|
template <class TValue> |
||||||
|
void AppendProperty( |
||||||
|
std::string& props, |
||||||
|
const std::string& key, |
||||||
|
const TValue& value, |
||||||
|
const std::string& prop_delim, |
||||||
|
const std::string& kv_delim) { |
||||||
|
AppendProperty( |
||||||
|
props, key, std::to_string(value), prop_delim, kv_delim |
||||||
|
); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
std::string TableProperties::ToString( |
||||||
|
const std::string& prop_delim, |
||||||
|
const std::string& kv_delim) const { |
||||||
|
std::string result; |
||||||
|
result.reserve(1024); |
||||||
|
|
||||||
|
// Basic Info
|
||||||
|
AppendProperty( |
||||||
|
result, "# data blocks", num_data_blocks, prop_delim, kv_delim |
||||||
|
); |
||||||
|
AppendProperty(result, "# entries", num_entries, prop_delim, kv_delim); |
||||||
|
|
||||||
|
AppendProperty(result, "raw key size", raw_key_size, prop_delim, kv_delim); |
||||||
|
AppendProperty( |
||||||
|
result, |
||||||
|
"raw average key size", |
||||||
|
num_entries != 0 ? 1.0 * raw_key_size / num_entries : 0.0, |
||||||
|
prop_delim, |
||||||
|
kv_delim |
||||||
|
); |
||||||
|
AppendProperty( |
||||||
|
result, "raw value size", raw_value_size, prop_delim, kv_delim |
||||||
|
); |
||||||
|
AppendProperty( |
||||||
|
result, |
||||||
|
"raw average value size", |
||||||
|
num_entries != 0 ? 1.0 * raw_value_size / num_entries : 0.0, |
||||||
|
prop_delim, |
||||||
|
kv_delim |
||||||
|
); |
||||||
|
|
||||||
|
AppendProperty(result, "data block size", data_size, prop_delim, kv_delim); |
||||||
|
AppendProperty(result, "index block size", index_size, prop_delim, kv_delim); |
||||||
|
AppendProperty( |
||||||
|
result, "filter block size", filter_size, prop_delim, kv_delim |
||||||
|
); |
||||||
|
AppendProperty( |
||||||
|
result, |
||||||
|
"(estimated) table size", |
||||||
|
data_size + index_size + filter_size, |
||||||
|
prop_delim, |
||||||
|
kv_delim |
||||||
|
); |
||||||
|
|
||||||
|
AppendProperty( |
||||||
|
result, |
||||||
|
"filter policy name", |
||||||
|
filter_policy_name.empty() ? std::string("N/A") : filter_policy_name, |
||||||
|
prop_delim, |
||||||
|
kv_delim |
||||||
|
); |
||||||
|
|
||||||
|
return result; |
||||||
|
} |
||||||
|
|
||||||
|
const std::string TablePropertiesNames::kDataSize = |
||||||
|
"rocksdb.data.size"; |
||||||
|
const std::string TablePropertiesNames::kIndexSize = |
||||||
|
"rocksdb.index.size"; |
||||||
|
const std::string TablePropertiesNames::kFilterSize = |
||||||
|
"rocksdb.filter.size"; |
||||||
|
const std::string TablePropertiesNames::kRawKeySize = |
||||||
|
"rocksdb.raw.key.size"; |
||||||
|
const std::string TablePropertiesNames::kRawValueSize = |
||||||
|
"rocksdb.raw.value.size"; |
||||||
|
const std::string TablePropertiesNames::kNumDataBlocks = |
||||||
|
"rocksdb.num.data.blocks"; |
||||||
|
const std::string TablePropertiesNames::kNumEntries = |
||||||
|
"rocksdb.num.entries"; |
||||||
|
const std::string TablePropertiesNames::kFilterPolicy = |
||||||
|
"rocksdb.filter.policy"; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
Loading…
Reference in new issue