Summary: This change will allow other table to reuse the code for meta blocks. Test Plan: all existing unit tests passed Reviewers: dhruba, haobo, sdong CC: leveldb Differential Revision: https://reviews.facebook.net/D14475main
kailiu
11 years ago
parent
e1d92dfd2e
commit
90729f8b23
@ -0,0 +1,134 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#include "table/meta_blocks.h" |
||||
|
||||
#include <map> |
||||
|
||||
#include "rocksdb/table_properties.h" |
||||
#include "table/format.h" |
||||
#include "util/coding.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
MetaIndexBuilder::MetaIndexBuilder() |
||||
: meta_index_block_( |
||||
new BlockBuilder(1 /* restart interval */, BytewiseComparator())) { |
||||
} |
||||
|
||||
void MetaIndexBuilder::Add(const std::string& key, |
||||
const BlockHandle& handle) { |
||||
std::string handle_encoding; |
||||
handle.EncodeTo(&handle_encoding); |
||||
meta_block_handles_.insert({key, handle_encoding}); |
||||
} |
||||
|
||||
Slice MetaIndexBuilder::Finish() { |
||||
for (const auto& metablock : meta_block_handles_) { |
||||
meta_index_block_->Add(metablock.first, metablock.second); |
||||
} |
||||
return meta_index_block_->Finish(); |
||||
} |
||||
|
||||
PropertyBlockBuilder::PropertyBlockBuilder() |
||||
: properties_block_( |
||||
new BlockBuilder(1 /* restart interval */, BytewiseComparator())) { |
||||
} |
||||
|
||||
void PropertyBlockBuilder::Add(const std::string& name, |
||||
const std::string& val) { |
||||
props_.insert({name, val}); |
||||
} |
||||
|
||||
void PropertyBlockBuilder::Add(const std::string& name, uint64_t val) { |
||||
assert(props_.find(name) == props_.end()); |
||||
|
||||
std::string dst; |
||||
PutVarint64(&dst, val); |
||||
|
||||
Add(name, dst); |
||||
} |
||||
|
||||
void PropertyBlockBuilder::Add( |
||||
const UserCollectedProperties& user_collected_properties) { |
||||
for (const auto& prop : user_collected_properties) { |
||||
Add(prop.first, prop.second); |
||||
} |
||||
} |
||||
|
||||
void PropertyBlockBuilder::AddTableProperty(const TableProperties& props) { |
||||
Add(TablePropertiesNames::kRawKeySize, props.raw_key_size); |
||||
Add(TablePropertiesNames::kRawValueSize, props.raw_value_size); |
||||
Add(TablePropertiesNames::kDataSize, props.data_size); |
||||
Add(TablePropertiesNames::kIndexSize, props.index_size); |
||||
Add(TablePropertiesNames::kNumEntries, props.num_entries); |
||||
Add(TablePropertiesNames::kNumDataBlocks, props.num_data_blocks); |
||||
Add(TablePropertiesNames::kFilterSize, props.filter_size); |
||||
|
||||
if (!props.filter_policy_name.empty()) { |
||||
Add(TablePropertiesNames::kFilterPolicy, |
||||
props.filter_policy_name); |
||||
} |
||||
} |
||||
|
||||
Slice PropertyBlockBuilder::Finish() { |
||||
for (const auto& prop : props_) { |
||||
properties_block_->Add(prop.first, prop.second); |
||||
} |
||||
|
||||
return properties_block_->Finish(); |
||||
} |
||||
|
||||
void LogPropertiesCollectionError( |
||||
Logger* info_log, const std::string& method, const std::string& name) { |
||||
assert(method == "Add" || method == "Finish"); |
||||
|
||||
std::string msg = |
||||
"[Warning] encountered error when calling TablePropertiesCollector::" + |
||||
method + "() with collector name: " + name; |
||||
Log(info_log, "%s", msg.c_str()); |
||||
} |
||||
|
||||
bool NotifyCollectTableCollectorsOnAdd( |
||||
const Slice& key, |
||||
const Slice& value, |
||||
const Options::TablePropertiesCollectors& collectors, |
||||
Logger* info_log) { |
||||
bool all_succeeded = true; |
||||
for (auto collector : collectors) { |
||||
Status s = collector->Add(key, value); |
||||
all_succeeded = all_succeeded && s.ok(); |
||||
if (!s.ok()) { |
||||
LogPropertiesCollectionError( |
||||
info_log, "Add", /* method */ collector->Name() |
||||
); |
||||
} |
||||
} |
||||
return all_succeeded; |
||||
} |
||||
|
||||
bool NotifyCollectTableCollectorsOnFinish( |
||||
const Options::TablePropertiesCollectors& collectors, |
||||
Logger* info_log, |
||||
PropertyBlockBuilder* builder) { |
||||
bool all_succeeded = true; |
||||
for (auto collector : collectors) { |
||||
UserCollectedProperties user_collected_properties; |
||||
Status s = collector->Finish(&user_collected_properties); |
||||
|
||||
all_succeeded = all_succeeded && s.ok(); |
||||
if (!s.ok()) { |
||||
LogPropertiesCollectionError( |
||||
info_log, "Finish", /* method */ collector->Name() |
||||
); |
||||
} else { |
||||
builder->Add(user_collected_properties); |
||||
} |
||||
} |
||||
|
||||
return all_succeeded; |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,106 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
#pragma once |
||||
|
||||
#include <map> |
||||
#include <memory> |
||||
#include <string> |
||||
|
||||
#include "rocksdb/comparator.h" |
||||
#include "rocksdb/options.h" |
||||
#include "rocksdb/slice.h" |
||||
#include "table/block_builder.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class BlockHandle; |
||||
class BlockBuilder; |
||||
class Logger; |
||||
struct TableProperties; |
||||
|
||||
// An STL style comparator that does the bytewise comparator comparasion
|
||||
// internally.
|
||||
struct BytewiseLessThan { |
||||
bool operator()(const std::string& key1, const std::string& key2) const { |
||||
// smaller entries will be placed in front.
|
||||
return comparator->Compare(key1, key2) <= 0; |
||||
} |
||||
|
||||
const Comparator* comparator = BytewiseComparator(); |
||||
}; |
||||
|
||||
// When writing to a block that requires entries to be sorted by
|
||||
// `BytewiseComparator`, we can buffer the content to `BytewiseSortedMap`
|
||||
// before writng to store.
|
||||
typedef std::map<std::string, std::string, BytewiseLessThan> BytewiseSortedMap; |
||||
|
||||
class MetaIndexBuilder { |
||||
public: |
||||
MetaIndexBuilder(const MetaIndexBuilder&) = delete; |
||||
MetaIndexBuilder& operator=(const MetaIndexBuilder&) = delete; |
||||
|
||||
MetaIndexBuilder(); |
||||
void Add(const std::string& key, const BlockHandle& handle); |
||||
|
||||
// Write all the added key/value pairs to the block and return the contents
|
||||
// of the block.
|
||||
Slice Finish(); |
||||
|
||||
private: |
||||
// * Key: meta block name
|
||||
// * Value: block handle to that meta block
|
||||
struct Rep; |
||||
Rep* rep_; |
||||
|
||||
// store the sorted key/handle of the metablocks.
|
||||
BytewiseSortedMap meta_block_handles_; |
||||
std::unique_ptr<BlockBuilder> meta_index_block_; |
||||
}; |
||||
|
||||
class PropertyBlockBuilder { |
||||
public: |
||||
PropertyBlockBuilder(const PropertyBlockBuilder&) = delete; |
||||
PropertyBlockBuilder& operator=(const PropertyBlockBuilder&) = delete; |
||||
|
||||
PropertyBlockBuilder(); |
||||
|
||||
void AddTableProperty(const TableProperties& props); |
||||
void Add(const std::string& key, uint64_t value); |
||||
void Add(const std::string& key, const std::string& value); |
||||
void Add(const UserCollectedProperties& user_collected_properties); |
||||
|
||||
// Write all the added entries to the block and return the block contents
|
||||
Slice Finish(); |
||||
|
||||
private: |
||||
std::unique_ptr<BlockBuilder> properties_block_; |
||||
BytewiseSortedMap props_; |
||||
}; |
||||
|
||||
// Were we encounter any error occurs during user-defined statistics collection,
|
||||
// we'll write the warning message to info log.
|
||||
void LogPropertiesCollectionError( |
||||
Logger* info_log, const std::string& method, const std::string& name); |
||||
|
||||
// Utility functions help table builder to trigger batch events for user
|
||||
// defined property collectors.
|
||||
// Return value indicates if there is any error occurred; if error occurred,
|
||||
// the warning message will be logged.
|
||||
// NotifyCollectTableCollectorsOnAdd() triggers the `Add` event for all
|
||||
// property collectors.
|
||||
bool NotifyCollectTableCollectorsOnAdd( |
||||
const Slice& key, |
||||
const Slice& value, |
||||
const Options::TablePropertiesCollectors& collectors, |
||||
Logger* info_log); |
||||
|
||||
// NotifyCollectTableCollectorsOnAdd() triggers the `Finish` event for all
|
||||
// property collectors. The collected properties will be added to `builder`.
|
||||
bool NotifyCollectTableCollectorsOnFinish( |
||||
const Options::TablePropertiesCollectors& collectors, |
||||
Logger* info_log, |
||||
PropertyBlockBuilder* builder); |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,108 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#include "rocksdb/table_properties.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
namespace { |
||||
void AppendProperty( |
||||
std::string& props, |
||||
const std::string& key, |
||||
const std::string& value, |
||||
const std::string& prop_delim, |
||||
const std::string& kv_delim) { |
||||
props.append(key); |
||||
props.append(kv_delim); |
||||
props.append(value); |
||||
props.append(prop_delim); |
||||
} |
||||
|
||||
template <class TValue> |
||||
void AppendProperty( |
||||
std::string& props, |
||||
const std::string& key, |
||||
const TValue& value, |
||||
const std::string& prop_delim, |
||||
const std::string& kv_delim) { |
||||
AppendProperty( |
||||
props, key, std::to_string(value), prop_delim, kv_delim |
||||
); |
||||
} |
||||
} |
||||
|
||||
std::string TableProperties::ToString( |
||||
const std::string& prop_delim, |
||||
const std::string& kv_delim) const { |
||||
std::string result; |
||||
result.reserve(1024); |
||||
|
||||
// Basic Info
|
||||
AppendProperty( |
||||
result, "# data blocks", num_data_blocks, prop_delim, kv_delim |
||||
); |
||||
AppendProperty(result, "# entries", num_entries, prop_delim, kv_delim); |
||||
|
||||
AppendProperty(result, "raw key size", raw_key_size, prop_delim, kv_delim); |
||||
AppendProperty( |
||||
result, |
||||
"raw average key size", |
||||
num_entries != 0 ? 1.0 * raw_key_size / num_entries : 0.0, |
||||
prop_delim, |
||||
kv_delim |
||||
); |
||||
AppendProperty( |
||||
result, "raw value size", raw_value_size, prop_delim, kv_delim |
||||
); |
||||
AppendProperty( |
||||
result, |
||||
"raw average value size", |
||||
num_entries != 0 ? 1.0 * raw_value_size / num_entries : 0.0, |
||||
prop_delim, |
||||
kv_delim |
||||
); |
||||
|
||||
AppendProperty(result, "data block size", data_size, prop_delim, kv_delim); |
||||
AppendProperty(result, "index block size", index_size, prop_delim, kv_delim); |
||||
AppendProperty( |
||||
result, "filter block size", filter_size, prop_delim, kv_delim |
||||
); |
||||
AppendProperty( |
||||
result, |
||||
"(estimated) table size", |
||||
data_size + index_size + filter_size, |
||||
prop_delim, |
||||
kv_delim |
||||
); |
||||
|
||||
AppendProperty( |
||||
result, |
||||
"filter policy name", |
||||
filter_policy_name.empty() ? std::string("N/A") : filter_policy_name, |
||||
prop_delim, |
||||
kv_delim |
||||
); |
||||
|
||||
return result; |
||||
} |
||||
|
||||
const std::string TablePropertiesNames::kDataSize = |
||||
"rocksdb.data.size"; |
||||
const std::string TablePropertiesNames::kIndexSize = |
||||
"rocksdb.index.size"; |
||||
const std::string TablePropertiesNames::kFilterSize = |
||||
"rocksdb.filter.size"; |
||||
const std::string TablePropertiesNames::kRawKeySize = |
||||
"rocksdb.raw.key.size"; |
||||
const std::string TablePropertiesNames::kRawValueSize = |
||||
"rocksdb.raw.value.size"; |
||||
const std::string TablePropertiesNames::kNumDataBlocks = |
||||
"rocksdb.num.data.blocks"; |
||||
const std::string TablePropertiesNames::kNumEntries = |
||||
"rocksdb.num.entries"; |
||||
const std::string TablePropertiesNames::kFilterPolicy = |
||||
"rocksdb.filter.policy"; |
||||
|
||||
} // namespace rocksdb
|
||||
Loading…
Reference in new issue