From 90729f8b23918d8448d15abd8f6ef216a7aa641a Mon Sep 17 00:00:00 2001 From: kailiu Date: Thu, 5 Dec 2013 13:09:13 -0800 Subject: [PATCH] Extract metaindex block from block-based table Summary: This change will allow other table to reuse the code for meta blocks. Test Plan: all existing unit tests passed Reviewers: dhruba, haobo, sdong CC: leveldb Differential Revision: https://reviews.facebook.net/D14475 --- db/table_properties_collector.cc | 91 +------------ db/table_properties_collector.h | 12 +- db/table_properties_collector_test.cc | 7 +- include/rocksdb/options.h | 5 +- include/rocksdb/table_properties.h | 38 ++++-- table/block_based_table_builder.cc | 176 ++++++-------------------- table/block_based_table_reader.cc | 37 ++---- table/block_based_table_reader.h | 11 -- table/meta_blocks.cc | 134 ++++++++++++++++++++ table/meta_blocks.h | 106 ++++++++++++++++ table/table_properties.cc | 108 ++++++++++++++++ 11 files changed, 432 insertions(+), 293 deletions(-) create mode 100644 table/meta_blocks.cc create mode 100644 table/meta_blocks.h create mode 100644 table/table_properties.cc diff --git a/db/table_properties_collector.cc b/db/table_properties_collector.cc index 3654663c1..25bd70036 100644 --- a/db/table_properties_collector.cc +++ b/db/table_properties_collector.cc @@ -10,87 +10,6 @@ namespace rocksdb { -namespace { - void AppendProperty( - std::string& props, - const std::string& key, - const std::string& value, - const std::string& prop_delim, - const std::string& kv_delim) { - props.append(key); - props.append(kv_delim); - props.append(value); - props.append(prop_delim); - } - - template - void AppendProperty( - std::string& props, - const std::string& key, - const TValue& value, - const std::string& prop_delim, - const std::string& kv_delim) { - AppendProperty( - props, key, std::to_string(value), prop_delim, kv_delim - ); - } -} - -std::string TableProperties::ToString( - const std::string& prop_delim, - const std::string& kv_delim) const { - std::string result; - result.reserve(1024); - - // Basic Info - AppendProperty( - result, "# data blocks", num_data_blocks, prop_delim, kv_delim - ); - AppendProperty(result, "# entries", num_entries, prop_delim, kv_delim); - - AppendProperty(result, "raw key size", raw_key_size, prop_delim, kv_delim); - AppendProperty( - result, - "raw average key size", - num_entries != 0 ? 1.0 * raw_key_size / num_entries : 0.0, - prop_delim, - kv_delim - ); - AppendProperty( - result, "raw value size", raw_value_size, prop_delim, kv_delim - ); - AppendProperty( - result, - "raw average value size", - num_entries != 0 ? 1.0 * raw_value_size / num_entries : 0.0, - prop_delim, - kv_delim - ); - - AppendProperty(result, "data block size", data_size, prop_delim, kv_delim); - AppendProperty(result, "index block size", index_size, prop_delim, kv_delim); - AppendProperty( - result, "filter block size", filter_size, prop_delim, kv_delim - ); - AppendProperty( - result, - "(estimated) table size", - data_size + index_size + filter_size, - prop_delim, - kv_delim - ); - - AppendProperty( - result, - "filter policy name", - filter_policy_name.empty() ? std::string("N/A") : filter_policy_name, - prop_delim, - kv_delim - ); - - return result; -} - Status InternalKeyPropertiesCollector::Add( const Slice& key, const Slice& value) { ParsedInternalKey ikey; @@ -106,7 +25,7 @@ Status InternalKeyPropertiesCollector::Add( } Status InternalKeyPropertiesCollector::Finish( - TableProperties::UserCollectedProperties* properties) { + UserCollectedProperties* properties) { assert(properties); assert(properties->find( InternalKeyTablePropertiesNames::kDeletedKeys) == properties->end()); @@ -118,7 +37,7 @@ Status InternalKeyPropertiesCollector::Finish( return Status::OK(); } -TableProperties::UserCollectedProperties +UserCollectedProperties InternalKeyPropertiesCollector::GetReadableProperties() const { return { { "kDeletedKeys", std::to_string(deleted_keys_) } @@ -137,11 +56,11 @@ Status UserKeyTablePropertiesCollector::Add( } Status UserKeyTablePropertiesCollector::Finish( - TableProperties::UserCollectedProperties* properties) { + UserCollectedProperties* properties) { return collector_->Finish(properties); } -TableProperties::UserCollectedProperties +UserCollectedProperties UserKeyTablePropertiesCollector::GetReadableProperties() const { return collector_->GetReadableProperties(); } @@ -151,7 +70,7 @@ const std::string InternalKeyTablePropertiesNames::kDeletedKeys = "rocksdb.deleted.keys"; uint64_t GetDeletedKeys( - const TableProperties::UserCollectedProperties& props) { + const UserCollectedProperties& props) { auto pos = props.find(InternalKeyTablePropertiesNames::kDeletedKeys); if (pos == props.end()) { return 0; diff --git a/db/table_properties_collector.h b/db/table_properties_collector.h index 533130db7..6cf56291a 100644 --- a/db/table_properties_collector.h +++ b/db/table_properties_collector.h @@ -24,15 +24,13 @@ class InternalKeyPropertiesCollector : public TablePropertiesCollector { public: virtual Status Add(const Slice& key, const Slice& value) override; - virtual Status Finish( - TableProperties::UserCollectedProperties* properties) override; + virtual Status Finish(UserCollectedProperties* properties) override; virtual const char* Name() const override { return "InternalKeyPropertiesCollector"; } - TableProperties::UserCollectedProperties - GetReadableProperties() const override; + UserCollectedProperties GetReadableProperties() const override; private: uint64_t deleted_keys_ = 0; @@ -61,13 +59,11 @@ class UserKeyTablePropertiesCollector : public TablePropertiesCollector { virtual Status Add(const Slice& key, const Slice& value) override; - virtual Status Finish( - TableProperties::UserCollectedProperties* properties) override; + virtual Status Finish(UserCollectedProperties* properties) override; virtual const char* Name() const override { return collector_->Name(); } - TableProperties::UserCollectedProperties - GetReadableProperties() const override; + UserCollectedProperties GetReadableProperties() const override; protected: std::shared_ptr collector_; diff --git a/db/table_properties_collector_test.cc b/db/table_properties_collector_test.cc index 6f405b28a..bbac4aa64 100644 --- a/db/table_properties_collector_test.cc +++ b/db/table_properties_collector_test.cc @@ -114,10 +114,10 @@ class RegularKeysStartWithA: public TablePropertiesCollector { public: const char* Name() const { return "RegularKeysStartWithA"; } - Status Finish(TableProperties::UserCollectedProperties* properties) { + Status Finish(UserCollectedProperties* properties) { std::string encoded; PutVarint32(&encoded, count_); - *properties = TableProperties::UserCollectedProperties { + *properties = UserCollectedProperties { { "TablePropertiesTest", "Rocksdb" }, { "Count", encoded } }; @@ -132,8 +132,7 @@ class RegularKeysStartWithA: public TablePropertiesCollector { return Status::OK(); } - virtual TableProperties::UserCollectedProperties - GetReadableProperties() const { + virtual UserCollectedProperties GetReadableProperties() const { return {}; } diff --git a/include/rocksdb/options.h b/include/rocksdb/options.h index 0cc33be68..d5f671ebe 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -611,8 +611,9 @@ struct Options { // the tables. // Default: emtpy vector -- no user-defined statistics collection will be // performed. - std::vector> - table_properties_collectors; + typedef std::vector> + TablePropertiesCollectors; + TablePropertiesCollectors table_properties_collectors; // Allows thread-safe inplace updates. Requires Updates iff // * key exists in current memtable diff --git a/include/rocksdb/table_properties.h b/include/rocksdb/table_properties.h index 8824ca13c..75c8bcc16 100644 --- a/include/rocksdb/table_properties.h +++ b/include/rocksdb/table_properties.h @@ -11,18 +11,18 @@ namespace rocksdb { +// Other than basic table properties, each table may also have the user +// collected properties. +// The value of the user-collected properties are encoded as raw bytes -- +// users have to interprete these values by themselves. +typedef + std::unordered_map + UserCollectedProperties; + // TableProperties contains a bunch of read-only properties of its associated // table. struct TableProperties { public: - // Other than basic table properties, each table may also have the user - // collected properties. - // The value of the user-collected properties are encoded as raw bytes -- - // users have to interprete these values by themselves. - typedef - std::unordered_map - UserCollectedProperties; - // the total size of all data blocks. uint64_t data_size = 0; // the size of index block. @@ -52,6 +52,19 @@ struct TableProperties { const std::string& kv_delim = "=") const; }; +// table properties' human-readable names in the property block. +struct TablePropertiesNames { + static const std::string kDataSize; + static const std::string kIndexSize; + static const std::string kFilterSize; + static const std::string kRawKeySize; + static const std::string kRawValueSize; + static const std::string kNumDataBlocks; + static const std::string kNumEntries; + static const std::string kFilterPolicy; +}; + + // `TablePropertiesCollector` provides the mechanism for users to collect // their own interested properties. This class is essentially a collection // of callback functions that will be invoked during table building. @@ -68,23 +81,20 @@ class TablePropertiesCollector { // for writing the properties block. // @params properties User will add their collected statistics to // `properties`. - virtual Status Finish( - TableProperties::UserCollectedProperties* properties) = 0; + virtual Status Finish(UserCollectedProperties* properties) = 0; // The name of the properties collector can be used for debugging purpose. virtual const char* Name() const = 0; // Return the human-readable properties, where the key is property name and // the value is the human-readable form of value. - virtual TableProperties::UserCollectedProperties - GetReadableProperties() const = 0; + virtual UserCollectedProperties GetReadableProperties() const = 0; }; // Extra properties // Below is a list of non-basic properties that are collected by database // itself. Especially some properties regarding to the internal keys (which // is unknown to `table`). -extern uint64_t GetDeletedKeys( - const TableProperties::UserCollectedProperties& props); +extern uint64_t GetDeletedKeys(const UserCollectedProperties& props); } // namespace rocksdb diff --git a/table/block_based_table_builder.cc b/table/block_based_table_builder.cc index 4cba3934f..a6dbe3519 100644 --- a/table/block_based_table_builder.cc +++ b/table/block_based_table_builder.cc @@ -26,6 +26,7 @@ #include "table/block_builder.h" #include "table/filter_block.h" #include "table/format.h" +#include "table/meta_blocks.h" #include "util/coding.h" #include "util/crc32c.h" #include "util/stop_watch.h" @@ -34,47 +35,11 @@ namespace rocksdb { namespace { -struct BytewiseLessThan { - bool operator()(const std::string& key1, const std::string& key2) const { - // smaller entries will be placed in front. - return comparator->Compare(key1, key2) <= 0; - } - const Comparator* comparator = BytewiseComparator(); -}; - -// When writing to a block that requires entries to be sorted by -// `BytewiseComparator`, we can buffer the content to `BytewiseSortedMap` -// before writng to store. -typedef std::map BytewiseSortedMap; - -void AddProperties(BytewiseSortedMap& props, std::string name, uint64_t val) { - assert(props.find(name) == props.end()); - - std::string dst; - PutVarint64(&dst, val); - - props.insert( - std::make_pair(name, dst) - ); -} - static bool GoodCompressionRatio(size_t compressed_size, size_t raw_size) { // Check to see if compressed less than 12.5% return compressed_size < raw_size - (raw_size / 8u); } -// Were we encounter any error occurs during user-defined statistics collection, -// we'll write the warning message to info log. -void LogPropertiesCollectionError( - Logger* info_log, const std::string& method, const std::string& name) { - assert(method == "Add" || method == "Finish"); - - std::string msg = - "[Warning] encountered error when calling TablePropertiesCollector::" + - method + "() with collector name: " + name; - Log(info_log, "%s", msg.c_str()); -} - } // anonymous namespace // kBlockBasedTableMagicNumber was picked by running @@ -186,16 +151,12 @@ void BlockBasedTableBuilder::Add(const Slice& key, const Slice& value) { r->props.raw_key_size += key.size(); r->props.raw_value_size += value.size(); - for (auto collector : r->options.table_properties_collectors) { - Status s = collector->Add(key, value); - if (!s.ok()) { - LogPropertiesCollectionError( - r->options.info_log.get(), - "Add", /* method */ - collector->Name() - ); - } - } + NotifyCollectTableCollectorsOnAdd( + key, + value, + r->options.table_properties_collectors, + r->options.info_log.get() + ); } void BlockBasedTableBuilder::Flush() { @@ -389,14 +350,7 @@ Status BlockBasedTableBuilder::Finish() { // 2. [meta block: properties] // 3. [metaindex block] if (ok()) { - // We use `BytewiseComparator` as the comparator for meta block. - BlockBuilder meta_index_block( - r->options.block_restart_interval, - BytewiseComparator() - ); - // Key: meta block name - // Value: block handle to that meta block - BytewiseSortedMap meta_block_handles; + MetaIndexBuilder meta_index_builer; // Write filter block. if (r->filter_block != nullptr) { @@ -404,104 +358,43 @@ Status BlockBasedTableBuilder::Finish() { // of filter data. std::string key = BlockBasedTable::kFilterBlockPrefix; key.append(r->options.filter_policy->Name()); - std::string handle_encoding; - filter_block_handle.EncodeTo(&handle_encoding); - meta_block_handles.insert( - std::make_pair(key, handle_encoding) - ); + meta_index_builer.Add(key, filter_block_handle); } // Write properties block. { - BlockBuilder properties_block( - r->options.block_restart_interval, - BytewiseComparator() - ); - - BytewiseSortedMap properties; - - // Add basic properties - AddProperties( - properties, - BlockBasedTablePropertiesNames::kRawKeySize, - r->props.raw_key_size - ); - AddProperties( - properties, - BlockBasedTablePropertiesNames::kRawValueSize, - r->props.raw_value_size - ); - AddProperties( - properties, - BlockBasedTablePropertiesNames::kDataSize, - r->props.data_size - ); + PropertyBlockBuilder property_block_builder; + std::vector failed_user_prop_collectors; + r->props.filter_policy_name = r->options.filter_policy != nullptr ? + r->options.filter_policy->Name() : ""; r->props.index_size = r->index_block.CurrentSizeEstimate() + kBlockTrailerSize; - AddProperties( - properties, - BlockBasedTablePropertiesNames::kIndexSize, - r->props.index_size - ); - AddProperties( - properties, - BlockBasedTablePropertiesNames::kNumEntries, - r->props.num_entries - ); - AddProperties( - properties, - BlockBasedTablePropertiesNames::kNumDataBlocks, - r->props.num_data_blocks); - if (r->filter_block != nullptr) { - properties.insert({ - BlockBasedTablePropertiesNames::kFilterPolicy, - r->options.filter_policy->Name() - }); - } - AddProperties( - properties, - BlockBasedTablePropertiesNames::kFilterSize, - r->props.filter_size - ); - for (auto collector : r->options.table_properties_collectors) { - TableProperties::UserCollectedProperties user_collected_properties; - Status s = - collector->Finish(&user_collected_properties); - - if (!s.ok()) { - LogPropertiesCollectionError( - r->options.info_log.get(), - "Finish", /* method */ - collector->Name() - ); - } else { - properties.insert( - user_collected_properties.begin(), - user_collected_properties.end() - ); - } - } + // Add basic properties + property_block_builder.AddTableProperty(r->props); - for (const auto& stat : properties) { - properties_block.Add(stat.first, stat.second); - } + NotifyCollectTableCollectorsOnFinish( + r->options.table_properties_collectors, + r->options.info_log.get(), + &property_block_builder + ); BlockHandle properties_block_handle; - WriteBlock(&properties_block, &properties_block_handle); - - std::string handle_encoding; - properties_block_handle.EncodeTo(&handle_encoding); - meta_block_handles.insert( - { BlockBasedTable::kPropertiesBlock, handle_encoding } + WriteRawBlock( + property_block_builder.Finish(), + kNoCompression, + &properties_block_handle ); - } // end of properties block writing - for (const auto& metablock : meta_block_handles) { - meta_index_block.Add(metablock.first, metablock.second); - } + meta_index_builer.Add(BlockBasedTable::kPropertiesBlock, + properties_block_handle); + } // end of properties block writing - WriteBlock(&meta_index_block, &metaindex_block_handle); + WriteRawBlock( + meta_index_builer.Finish(), + kNoCompression, + &metaindex_block_handle + ); } // meta blocks and metaindex block. // Write index block @@ -563,4 +456,9 @@ uint64_t BlockBasedTableBuilder::FileSize() const { return rep_->offset; } +const std::string BlockBasedTable::kFilterBlockPrefix = + "filter."; +const std::string BlockBasedTable::kPropertiesBlock = + "rocksdb.properties"; + } // namespace rocksdb diff --git a/table/block_based_table_reader.cc b/table/block_based_table_reader.cc index 11b8f6ca8..e69bef679 100644 --- a/table/block_based_table_reader.cc +++ b/table/block_based_table_reader.cc @@ -432,19 +432,19 @@ Status BlockBasedTable::ReadProperties( // All pre-defined properties of type uint64_t std::unordered_map predefined_uint64_properties = { - { BlockBasedTablePropertiesNames::kDataSize, + { TablePropertiesNames::kDataSize, &table_properties->data_size }, - { BlockBasedTablePropertiesNames::kIndexSize, + { TablePropertiesNames::kIndexSize, &table_properties->index_size }, - { BlockBasedTablePropertiesNames::kFilterSize, + { TablePropertiesNames::kFilterSize, &table_properties->filter_size }, - { BlockBasedTablePropertiesNames::kRawKeySize, + { TablePropertiesNames::kRawKeySize, &table_properties->raw_key_size }, - { BlockBasedTablePropertiesNames::kRawValueSize, + { TablePropertiesNames::kRawValueSize, &table_properties->raw_value_size }, - { BlockBasedTablePropertiesNames::kNumDataBlocks, + { TablePropertiesNames::kNumDataBlocks, &table_properties->num_data_blocks }, - { BlockBasedTablePropertiesNames::kNumEntries, + { TablePropertiesNames::kNumEntries, &table_properties->num_entries }, }; @@ -478,7 +478,7 @@ Status BlockBasedTable::ReadProperties( continue; } *(pos->second) = val; - } else if (key == BlockBasedTablePropertiesNames::kFilterPolicy) { + } else if (key == TablePropertiesNames::kFilterPolicy) { table_properties->filter_policy_name = raw_val.ToString(); } else { // handle user-collected @@ -1062,25 +1062,4 @@ uint64_t BlockBasedTable::ApproximateOffsetOf(const Slice& key) { return result; } -const std::string BlockBasedTable::kFilterBlockPrefix = - "filter."; -const std::string BlockBasedTable::kPropertiesBlock = - "rocksdb.properties"; -const std::string BlockBasedTablePropertiesNames::kDataSize = - "rocksdb.data.size"; -const std::string BlockBasedTablePropertiesNames::kIndexSize = - "rocksdb.index.size"; -const std::string BlockBasedTablePropertiesNames::kFilterSize = - "rocksdb.filter.size"; -const std::string BlockBasedTablePropertiesNames::kRawKeySize = - "rocksdb.raw.key.size"; -const std::string BlockBasedTablePropertiesNames::kRawValueSize = - "rocksdb.raw.value.size"; -const std::string BlockBasedTablePropertiesNames::kNumDataBlocks = - "rocksdb.num.data.blocks"; -const std::string BlockBasedTablePropertiesNames::kNumEntries = - "rocksdb.num.entries"; -const std::string BlockBasedTablePropertiesNames::kFilterPolicy = - "rocksdb.filter.policy"; - } // namespace rocksdb diff --git a/table/block_based_table_reader.h b/table/block_based_table_reader.h index 02bbfd74c..72fb35fa5 100644 --- a/table/block_based_table_reader.h +++ b/table/block_based_table_reader.h @@ -181,15 +181,4 @@ class BlockBasedTable : public TableReader { void operator=(const TableReader&) = delete; }; -struct BlockBasedTablePropertiesNames { - static const std::string kDataSize; - static const std::string kIndexSize; - static const std::string kFilterSize; - static const std::string kRawKeySize; - static const std::string kRawValueSize; - static const std::string kNumDataBlocks; - static const std::string kNumEntries; - static const std::string kFilterPolicy; -}; - } // namespace rocksdb diff --git a/table/meta_blocks.cc b/table/meta_blocks.cc new file mode 100644 index 000000000..df3ee5dae --- /dev/null +++ b/table/meta_blocks.cc @@ -0,0 +1,134 @@ +// Copyright (c) 2013, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "table/meta_blocks.h" + +#include + +#include "rocksdb/table_properties.h" +#include "table/format.h" +#include "util/coding.h" + +namespace rocksdb { + +MetaIndexBuilder::MetaIndexBuilder() + : meta_index_block_( + new BlockBuilder(1 /* restart interval */, BytewiseComparator())) { +} + +void MetaIndexBuilder::Add(const std::string& key, + const BlockHandle& handle) { + std::string handle_encoding; + handle.EncodeTo(&handle_encoding); + meta_block_handles_.insert({key, handle_encoding}); +} + +Slice MetaIndexBuilder::Finish() { + for (const auto& metablock : meta_block_handles_) { + meta_index_block_->Add(metablock.first, metablock.second); + } + return meta_index_block_->Finish(); +} + +PropertyBlockBuilder::PropertyBlockBuilder() + : properties_block_( + new BlockBuilder(1 /* restart interval */, BytewiseComparator())) { +} + +void PropertyBlockBuilder::Add(const std::string& name, + const std::string& val) { + props_.insert({name, val}); +} + +void PropertyBlockBuilder::Add(const std::string& name, uint64_t val) { + assert(props_.find(name) == props_.end()); + + std::string dst; + PutVarint64(&dst, val); + + Add(name, dst); +} + +void PropertyBlockBuilder::Add( + const UserCollectedProperties& user_collected_properties) { + for (const auto& prop : user_collected_properties) { + Add(prop.first, prop.second); + } +} + +void PropertyBlockBuilder::AddTableProperty(const TableProperties& props) { + Add(TablePropertiesNames::kRawKeySize, props.raw_key_size); + Add(TablePropertiesNames::kRawValueSize, props.raw_value_size); + Add(TablePropertiesNames::kDataSize, props.data_size); + Add(TablePropertiesNames::kIndexSize, props.index_size); + Add(TablePropertiesNames::kNumEntries, props.num_entries); + Add(TablePropertiesNames::kNumDataBlocks, props.num_data_blocks); + Add(TablePropertiesNames::kFilterSize, props.filter_size); + + if (!props.filter_policy_name.empty()) { + Add(TablePropertiesNames::kFilterPolicy, + props.filter_policy_name); + } +} + +Slice PropertyBlockBuilder::Finish() { + for (const auto& prop : props_) { + properties_block_->Add(prop.first, prop.second); + } + + return properties_block_->Finish(); +} + +void LogPropertiesCollectionError( + Logger* info_log, const std::string& method, const std::string& name) { + assert(method == "Add" || method == "Finish"); + + std::string msg = + "[Warning] encountered error when calling TablePropertiesCollector::" + + method + "() with collector name: " + name; + Log(info_log, "%s", msg.c_str()); +} + +bool NotifyCollectTableCollectorsOnAdd( + const Slice& key, + const Slice& value, + const Options::TablePropertiesCollectors& collectors, + Logger* info_log) { + bool all_succeeded = true; + for (auto collector : collectors) { + Status s = collector->Add(key, value); + all_succeeded = all_succeeded && s.ok(); + if (!s.ok()) { + LogPropertiesCollectionError( + info_log, "Add", /* method */ collector->Name() + ); + } + } + return all_succeeded; +} + +bool NotifyCollectTableCollectorsOnFinish( + const Options::TablePropertiesCollectors& collectors, + Logger* info_log, + PropertyBlockBuilder* builder) { + bool all_succeeded = true; + for (auto collector : collectors) { + UserCollectedProperties user_collected_properties; + Status s = collector->Finish(&user_collected_properties); + + all_succeeded = all_succeeded && s.ok(); + if (!s.ok()) { + LogPropertiesCollectionError( + info_log, "Finish", /* method */ collector->Name() + ); + } else { + builder->Add(user_collected_properties); + } + } + + return all_succeeded; +} + +} // namespace rocksdb diff --git a/table/meta_blocks.h b/table/meta_blocks.h new file mode 100644 index 000000000..d0718ec07 --- /dev/null +++ b/table/meta_blocks.h @@ -0,0 +1,106 @@ +// Copyright (c) 2013, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +#pragma once + +#include +#include +#include + +#include "rocksdb/comparator.h" +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "table/block_builder.h" + +namespace rocksdb { + +class BlockHandle; +class BlockBuilder; +class Logger; +struct TableProperties; + +// An STL style comparator that does the bytewise comparator comparasion +// internally. +struct BytewiseLessThan { + bool operator()(const std::string& key1, const std::string& key2) const { + // smaller entries will be placed in front. + return comparator->Compare(key1, key2) <= 0; + } + + const Comparator* comparator = BytewiseComparator(); +}; + +// When writing to a block that requires entries to be sorted by +// `BytewiseComparator`, we can buffer the content to `BytewiseSortedMap` +// before writng to store. +typedef std::map BytewiseSortedMap; + +class MetaIndexBuilder { + public: + MetaIndexBuilder(const MetaIndexBuilder&) = delete; + MetaIndexBuilder& operator=(const MetaIndexBuilder&) = delete; + + MetaIndexBuilder(); + void Add(const std::string& key, const BlockHandle& handle); + + // Write all the added key/value pairs to the block and return the contents + // of the block. + Slice Finish(); + + private: + // * Key: meta block name + // * Value: block handle to that meta block + struct Rep; + Rep* rep_; + + // store the sorted key/handle of the metablocks. + BytewiseSortedMap meta_block_handles_; + std::unique_ptr meta_index_block_; +}; + +class PropertyBlockBuilder { + public: + PropertyBlockBuilder(const PropertyBlockBuilder&) = delete; + PropertyBlockBuilder& operator=(const PropertyBlockBuilder&) = delete; + + PropertyBlockBuilder(); + + void AddTableProperty(const TableProperties& props); + void Add(const std::string& key, uint64_t value); + void Add(const std::string& key, const std::string& value); + void Add(const UserCollectedProperties& user_collected_properties); + + // Write all the added entries to the block and return the block contents + Slice Finish(); + + private: + std::unique_ptr properties_block_; + BytewiseSortedMap props_; +}; + +// Were we encounter any error occurs during user-defined statistics collection, +// we'll write the warning message to info log. +void LogPropertiesCollectionError( + Logger* info_log, const std::string& method, const std::string& name); + +// Utility functions help table builder to trigger batch events for user +// defined property collectors. +// Return value indicates if there is any error occurred; if error occurred, +// the warning message will be logged. +// NotifyCollectTableCollectorsOnAdd() triggers the `Add` event for all +// property collectors. +bool NotifyCollectTableCollectorsOnAdd( + const Slice& key, + const Slice& value, + const Options::TablePropertiesCollectors& collectors, + Logger* info_log); + +// NotifyCollectTableCollectorsOnAdd() triggers the `Finish` event for all +// property collectors. The collected properties will be added to `builder`. +bool NotifyCollectTableCollectorsOnFinish( + const Options::TablePropertiesCollectors& collectors, + Logger* info_log, + PropertyBlockBuilder* builder); + +} // namespace rocksdb diff --git a/table/table_properties.cc b/table/table_properties.cc new file mode 100644 index 000000000..2c9905884 --- /dev/null +++ b/table/table_properties.cc @@ -0,0 +1,108 @@ +// Copyright (c) 2013, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "rocksdb/table_properties.h" + +namespace rocksdb { + +namespace { + void AppendProperty( + std::string& props, + const std::string& key, + const std::string& value, + const std::string& prop_delim, + const std::string& kv_delim) { + props.append(key); + props.append(kv_delim); + props.append(value); + props.append(prop_delim); + } + + template + void AppendProperty( + std::string& props, + const std::string& key, + const TValue& value, + const std::string& prop_delim, + const std::string& kv_delim) { + AppendProperty( + props, key, std::to_string(value), prop_delim, kv_delim + ); + } +} + +std::string TableProperties::ToString( + const std::string& prop_delim, + const std::string& kv_delim) const { + std::string result; + result.reserve(1024); + + // Basic Info + AppendProperty( + result, "# data blocks", num_data_blocks, prop_delim, kv_delim + ); + AppendProperty(result, "# entries", num_entries, prop_delim, kv_delim); + + AppendProperty(result, "raw key size", raw_key_size, prop_delim, kv_delim); + AppendProperty( + result, + "raw average key size", + num_entries != 0 ? 1.0 * raw_key_size / num_entries : 0.0, + prop_delim, + kv_delim + ); + AppendProperty( + result, "raw value size", raw_value_size, prop_delim, kv_delim + ); + AppendProperty( + result, + "raw average value size", + num_entries != 0 ? 1.0 * raw_value_size / num_entries : 0.0, + prop_delim, + kv_delim + ); + + AppendProperty(result, "data block size", data_size, prop_delim, kv_delim); + AppendProperty(result, "index block size", index_size, prop_delim, kv_delim); + AppendProperty( + result, "filter block size", filter_size, prop_delim, kv_delim + ); + AppendProperty( + result, + "(estimated) table size", + data_size + index_size + filter_size, + prop_delim, + kv_delim + ); + + AppendProperty( + result, + "filter policy name", + filter_policy_name.empty() ? std::string("N/A") : filter_policy_name, + prop_delim, + kv_delim + ); + + return result; +} + +const std::string TablePropertiesNames::kDataSize = + "rocksdb.data.size"; +const std::string TablePropertiesNames::kIndexSize = + "rocksdb.index.size"; +const std::string TablePropertiesNames::kFilterSize = + "rocksdb.filter.size"; +const std::string TablePropertiesNames::kRawKeySize = + "rocksdb.raw.key.size"; +const std::string TablePropertiesNames::kRawValueSize = + "rocksdb.raw.value.size"; +const std::string TablePropertiesNames::kNumDataBlocks = + "rocksdb.num.data.blocks"; +const std::string TablePropertiesNames::kNumEntries = + "rocksdb.num.entries"; +const std::string TablePropertiesNames::kFilterPolicy = + "rocksdb.filter.policy"; + +} // namespace rocksdb