Summary:
The primary motivation of the changes is to make it easier to figure out the inside of the tables.
* rename "table stats" to "table properties" since now we have more than "integers" to store in the property block.
* Add filter block size to the basic table properties.
* Whenever a table is built, we'll log the table properties (the sample output is in Test Plan).
* Make an api to expose deleted keys.
Test Plan:
Passed all existing test. and the sample output of table stats:
==================================================================
Basic Properties
------------------------------------------------------------------
# data blocks: 1
# entries: 1
raw key size: 9
raw average key size: 9
raw value size: 9
raw average value size: 0
data block size: 25
index block size: 27
filter block size: 18
(estimated) table size: 70
filter policy: rocksdb.BuiltinBloomFilter
==================================================================
User collected properties: InternalKeyPropertiesCollector
------------------------------------------------------------------
kDeletedKeys: 1
==================================================================
Reviewers: dhruba, haobo
Reviewed By: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14187
main
kailiu
11 years ago
parent
f045871f1c
commit
1415f8820d
@ -0,0 +1,164 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#include "db/table_properties_collector.h" |
||||
|
||||
#include "db/dbformat.h" |
||||
#include "util/coding.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
namespace { |
||||
void AppendProperty( |
||||
std::string& props, |
||||
const std::string& key, |
||||
const std::string& value, |
||||
const std::string& prop_delim, |
||||
const std::string& kv_delim) { |
||||
props.append(key); |
||||
props.append(kv_delim); |
||||
props.append(value); |
||||
props.append(prop_delim); |
||||
} |
||||
|
||||
template <class TValue> |
||||
void AppendProperty( |
||||
std::string& props, |
||||
const std::string& key, |
||||
const TValue& value, |
||||
const std::string& prop_delim, |
||||
const std::string& kv_delim) { |
||||
AppendProperty( |
||||
props, key, std::to_string(value), prop_delim, kv_delim |
||||
); |
||||
} |
||||
} |
||||
|
||||
std::string TableProperties::ToString( |
||||
const std::string& prop_delim, |
||||
const std::string& kv_delim) const { |
||||
std::string result; |
||||
result.reserve(1024); |
||||
|
||||
// Basic Info
|
||||
AppendProperty( |
||||
result, "# data blocks", num_data_blocks, prop_delim, kv_delim |
||||
); |
||||
AppendProperty(result, "# entries", num_entries, prop_delim, kv_delim); |
||||
|
||||
AppendProperty(result, "raw key size", raw_key_size, prop_delim, kv_delim); |
||||
AppendProperty( |
||||
result, |
||||
"raw average key size", |
||||
num_entries != 0 ? 1.0 * raw_key_size / num_entries : 0.0, |
||||
prop_delim, |
||||
kv_delim |
||||
); |
||||
AppendProperty( |
||||
result, "raw value size", raw_value_size, prop_delim, kv_delim |
||||
); |
||||
AppendProperty( |
||||
result, |
||||
"raw average value size", |
||||
num_entries != 0 ? 1.0 * raw_value_size / num_entries : 0.0, |
||||
prop_delim, |
||||
kv_delim |
||||
); |
||||
|
||||
AppendProperty(result, "data block size", data_size, prop_delim, kv_delim); |
||||
AppendProperty(result, "index block size", index_size, prop_delim, kv_delim); |
||||
AppendProperty( |
||||
result, "filter block size", filter_size, prop_delim, kv_delim |
||||
); |
||||
AppendProperty( |
||||
result, |
||||
"(estimated) table size=", |
||||
data_size + index_size + filter_size, |
||||
prop_delim, |
||||
kv_delim |
||||
); |
||||
|
||||
AppendProperty( |
||||
result, |
||||
"filter policy name", |
||||
filter_policy_name.empty() ? std::string("N/A") : filter_policy_name, |
||||
prop_delim, |
||||
kv_delim |
||||
); |
||||
|
||||
return result; |
||||
} |
||||
|
||||
Status InternalKeyPropertiesCollector::Add( |
||||
const Slice& key, const Slice& value) { |
||||
ParsedInternalKey ikey; |
||||
if (!ParseInternalKey(key, &ikey)) { |
||||
return Status::InvalidArgument("Invalid internal key"); |
||||
} |
||||
|
||||
if (ikey.type == ValueType::kTypeDeletion) { |
||||
++deleted_keys_; |
||||
} |
||||
|
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status InternalKeyPropertiesCollector::Finish( |
||||
TableProperties::UserCollectedProperties* properties) { |
||||
assert(properties); |
||||
assert(properties->find( |
||||
InternalKeyTablePropertiesNames::kDeletedKeys) == properties->end()); |
||||
std::string val; |
||||
|
||||
PutVarint64(&val, deleted_keys_); |
||||
properties->insert({ InternalKeyTablePropertiesNames::kDeletedKeys, val }); |
||||
|
||||
return Status::OK(); |
||||
} |
||||
|
||||
TableProperties::UserCollectedProperties |
||||
InternalKeyPropertiesCollector::GetReadableProperties() const { |
||||
return { |
||||
{ "kDeletedKeys", std::to_string(deleted_keys_) } |
||||
}; |
||||
} |
||||
|
||||
|
||||
Status UserKeyTablePropertiesCollector::Add( |
||||
const Slice& key, const Slice& value) { |
||||
ParsedInternalKey ikey; |
||||
if (!ParseInternalKey(key, &ikey)) { |
||||
return Status::InvalidArgument("Invalid internal key"); |
||||
} |
||||
|
||||
return collector_->Add(ikey.user_key, value); |
||||
} |
||||
|
||||
Status UserKeyTablePropertiesCollector::Finish( |
||||
TableProperties::UserCollectedProperties* properties) { |
||||
return collector_->Finish(properties); |
||||
} |
||||
|
||||
TableProperties::UserCollectedProperties |
||||
UserKeyTablePropertiesCollector::GetReadableProperties() const { |
||||
return collector_->GetReadableProperties(); |
||||
} |
||||
|
||||
|
||||
const std::string InternalKeyTablePropertiesNames::kDeletedKeys |
||||
= "rocksdb.deleted.keys"; |
||||
|
||||
uint64_t GetDeletedKeys( |
||||
const TableProperties::UserCollectedProperties& props) { |
||||
auto pos = props.find(InternalKeyTablePropertiesNames::kDeletedKeys); |
||||
if (pos == props.end()) { |
||||
return 0; |
||||
} |
||||
Slice raw = pos->second; |
||||
uint64_t val = 0; |
||||
return GetVarint64(&raw, &val) ? val : 0; |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,76 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// This file defines a collection of statistics collectors.
|
||||
#pragma once |
||||
|
||||
#include "rocksdb/table_properties.h" |
||||
|
||||
#include <memory> |
||||
#include <string> |
||||
#include <vector> |
||||
|
||||
namespace rocksdb { |
||||
|
||||
struct InternalKeyTablePropertiesNames { |
||||
static const std::string kDeletedKeys; |
||||
}; |
||||
|
||||
// Collecting the statistics for internal keys. Visible only by internal
|
||||
// rocksdb modules.
|
||||
class InternalKeyPropertiesCollector : public TablePropertiesCollector { |
||||
public: |
||||
virtual Status Add(const Slice& key, const Slice& value) override; |
||||
|
||||
virtual Status Finish( |
||||
TableProperties::UserCollectedProperties* properties) override; |
||||
|
||||
virtual const char* Name() const override { |
||||
return "InternalKeyPropertiesCollector"; |
||||
} |
||||
|
||||
TableProperties::UserCollectedProperties |
||||
GetReadableProperties() const override; |
||||
|
||||
private: |
||||
uint64_t deleted_keys_ = 0; |
||||
}; |
||||
|
||||
// When rocksdb creates a new table, it will encode all "user keys" into
|
||||
// "internal keys", which contains meta information of a given entry.
|
||||
//
|
||||
// This class extracts user key from the encoded internal key when Add() is
|
||||
// invoked.
|
||||
class UserKeyTablePropertiesCollector : public TablePropertiesCollector { |
||||
public: |
||||
explicit UserKeyTablePropertiesCollector( |
||||
TablePropertiesCollector* collector) : |
||||
UserKeyTablePropertiesCollector( |
||||
std::shared_ptr<TablePropertiesCollector>(collector) |
||||
) { |
||||
} |
||||
|
||||
explicit UserKeyTablePropertiesCollector( |
||||
std::shared_ptr<TablePropertiesCollector> collector) : |
||||
collector_(collector) { |
||||
} |
||||
|
||||
virtual ~UserKeyTablePropertiesCollector() { } |
||||
|
||||
virtual Status Add(const Slice& key, const Slice& value) override; |
||||
|
||||
virtual Status Finish( |
||||
TableProperties::UserCollectedProperties* properties) override; |
||||
|
||||
virtual const char* Name() const override { return collector_->Name(); } |
||||
|
||||
TableProperties::UserCollectedProperties |
||||
GetReadableProperties() const override; |
||||
|
||||
protected: |
||||
std::shared_ptr<TablePropertiesCollector> collector_; |
||||
}; |
||||
|
||||
} // namespace rocksdb
|
||||
@ -1,55 +0,0 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#include "db/table_stats_collector.h" |
||||
|
||||
#include "db/dbformat.h" |
||||
#include "util/coding.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
Status InternalKeyStatsCollector::Add(const Slice& key, const Slice& value) { |
||||
ParsedInternalKey ikey; |
||||
if (!ParseInternalKey(key, &ikey)) { |
||||
return Status::InvalidArgument("Invalid internal key"); |
||||
} |
||||
|
||||
if (ikey.type == ValueType::kTypeDeletion) { |
||||
++deleted_keys_; |
||||
} |
||||
|
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status InternalKeyStatsCollector::Finish( |
||||
TableStats::UserCollectedStats* stats) { |
||||
assert(stats); |
||||
assert(stats->find(InternalKeyTableStatsNames::kDeletedKeys) == stats->end()); |
||||
std::string val; |
||||
|
||||
PutVarint64(&val, deleted_keys_); |
||||
stats->insert(std::make_pair(InternalKeyTableStatsNames::kDeletedKeys, val)); |
||||
|
||||
return Status::OK(); |
||||
} |
||||
|
||||
Status UserKeyTableStatsCollector::Add(const Slice& key, const Slice& value) { |
||||
ParsedInternalKey ikey; |
||||
if (!ParseInternalKey(key, &ikey)) { |
||||
return Status::InvalidArgument("Invalid internal key"); |
||||
} |
||||
|
||||
return collector_->Add(ikey.user_key, value); |
||||
} |
||||
|
||||
Status UserKeyTableStatsCollector::Finish( |
||||
TableStats::UserCollectedStats* stats) { |
||||
return collector_->Finish(stats); |
||||
} |
||||
|
||||
const std::string InternalKeyTableStatsNames::kDeletedKeys |
||||
= "rocksdb.deleted.keys"; |
||||
|
||||
} // namespace rocksdb
|
||||
@ -1,58 +0,0 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// This file defines a collection of statistics collectors.
|
||||
#pragma once |
||||
|
||||
#include "rocksdb/table_stats.h" |
||||
|
||||
#include <memory> |
||||
#include <string> |
||||
#include <vector> |
||||
|
||||
namespace rocksdb { |
||||
|
||||
struct InternalKeyTableStatsNames { |
||||
static const std::string kDeletedKeys; |
||||
}; |
||||
|
||||
// Collecting the statistics for internal keys. Visible only by internal
|
||||
// rocksdb modules.
|
||||
class InternalKeyStatsCollector : public TableStatsCollector { |
||||
public: |
||||
virtual Status Add(const Slice& key, const Slice& value); |
||||
virtual Status Finish(TableStats::UserCollectedStats* stats); |
||||
virtual const char* Name() const { return "InternalKeyStatsCollector"; } |
||||
|
||||
private: |
||||
uint64_t deleted_keys_ = 0; |
||||
}; |
||||
|
||||
// When rocksdb creates a new table, it will encode all "user keys" into
|
||||
// "internal keys", which contains meta information of a given entry.
|
||||
//
|
||||
// This class extracts user key from the encoded internal key when Add() is
|
||||
// invoked.
|
||||
class UserKeyTableStatsCollector : public TableStatsCollector { |
||||
public: |
||||
explicit UserKeyTableStatsCollector(TableStatsCollector* collector): |
||||
UserKeyTableStatsCollector( |
||||
std::shared_ptr<TableStatsCollector>(collector) |
||||
) { |
||||
} |
||||
|
||||
explicit UserKeyTableStatsCollector( |
||||
std::shared_ptr<TableStatsCollector> collector) : collector_(collector) { |
||||
} |
||||
virtual ~UserKeyTableStatsCollector() { } |
||||
virtual Status Add(const Slice& key, const Slice& value); |
||||
virtual Status Finish(TableStats::UserCollectedStats* stats); |
||||
virtual const char* Name() const { return collector_->Name(); } |
||||
|
||||
protected: |
||||
std::shared_ptr<TableStatsCollector> collector_; |
||||
}; |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,90 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
#pragma once |
||||
|
||||
#include <string> |
||||
#include <unordered_map> |
||||
|
||||
#include "rocksdb/status.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
// TableProperties contains a bunch of read-only properties of its associated
|
||||
// table.
|
||||
struct TableProperties { |
||||
public: |
||||
// Other than basic table properties, each table may also have the user
|
||||
// collected properties.
|
||||
// The value of the user-collected properties are encoded as raw bytes --
|
||||
// users have to interprete these values by themselves.
|
||||
typedef |
||||
std::unordered_map<std::string, std::string> |
||||
UserCollectedProperties; |
||||
|
||||
// the total size of all data blocks.
|
||||
uint64_t data_size = 0; |
||||
// the size of index block.
|
||||
uint64_t index_size = 0; |
||||
// the size of filter block.
|
||||
uint64_t filter_size = 0; |
||||
// total raw key size
|
||||
uint64_t raw_key_size = 0; |
||||
// total raw value size
|
||||
uint64_t raw_value_size = 0; |
||||
// the number of blocks in this table
|
||||
uint64_t num_data_blocks = 0; |
||||
// the number of entries in this table
|
||||
uint64_t num_entries = 0; |
||||
|
||||
// The name of the filter policy used in this table.
|
||||
// If no filter policy is used, `filter_policy_name` will be an empty string.
|
||||
std::string filter_policy_name; |
||||
|
||||
// user collected properties
|
||||
UserCollectedProperties user_collected_properties; |
||||
|
||||
// convert this object to a human readable form
|
||||
// @prop_delim: delimiter for each property.
|
||||
std::string ToString( |
||||
const std::string& prop_delim = "; ", |
||||
const std::string& kv_delim = "=") const; |
||||
}; |
||||
|
||||
// `TablePropertiesCollector` provides the mechanism for users to collect
|
||||
// their own interested properties. This class is essentially a collection
|
||||
// of callback functions that will be invoked during table building.
|
||||
class TablePropertiesCollector { |
||||
public: |
||||
virtual ~TablePropertiesCollector() { } |
||||
|
||||
// Add() will be called when a new key/value pair is inserted into the table.
|
||||
// @params key the original key that is inserted into the table.
|
||||
// @params value the original value that is inserted into the table.
|
||||
virtual Status Add(const Slice& key, const Slice& value) = 0; |
||||
|
||||
// Finish() will be called when a table has already been built and is ready
|
||||
// for writing the properties block.
|
||||
// @params properties User will add their collected statistics to
|
||||
// `properties`.
|
||||
virtual Status Finish( |
||||
TableProperties::UserCollectedProperties* properties) = 0; |
||||
|
||||
// The name of the properties collector can be used for debugging purpose.
|
||||
virtual const char* Name() const = 0; |
||||
|
||||
// Return the human-readable properties, where the key is property name and
|
||||
// the value is the human-readable form of value.
|
||||
virtual TableProperties::UserCollectedProperties |
||||
GetReadableProperties() const = 0; |
||||
}; |
||||
|
||||
// Extra properties
|
||||
// Below is a list of non-basic properties that are collected by database
|
||||
// itself. Especially some properties regarding to the internal keys (which
|
||||
// is unknown to `table`).
|
||||
extern uint64_t GetDeletedKeys( |
||||
const TableProperties::UserCollectedProperties& props); |
||||
|
||||
} // namespace rocksdb
|
||||
@ -1,67 +0,0 @@ |
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
#pragma once |
||||
|
||||
#include <string> |
||||
#include <unordered_map> |
||||
|
||||
#include "rocksdb/status.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
// TableStats contains a bunch of read-only stats of its associated
|
||||
// table.
|
||||
struct TableStats { |
||||
public: |
||||
// Other than basic table stats, each table may also have the user
|
||||
// collected stats.
|
||||
// The value of the user-collected stats are encoded as raw bytes --
|
||||
// users have to interprete these values by themselves.
|
||||
typedef |
||||
std::unordered_map<std::string, std::string> |
||||
UserCollectedStats; |
||||
|
||||
// the total size of all data blocks.
|
||||
uint64_t data_size = 0; |
||||
// the total size of all index blocks.
|
||||
uint64_t index_size = 0; |
||||
// total raw key size
|
||||
uint64_t raw_key_size = 0; |
||||
// total raw value size
|
||||
uint64_t raw_value_size = 0; |
||||
// the number of blocks in this table
|
||||
uint64_t num_data_blocks = 0; |
||||
// the number of entries in this table
|
||||
uint64_t num_entries = 0; |
||||
|
||||
// The name of the filter policy used in this table.
|
||||
// If no filter policy is used, `filter_policy_name` will be an empty string.
|
||||
std::string filter_policy_name; |
||||
|
||||
// user collected stats
|
||||
UserCollectedStats user_collected_stats; |
||||
}; |
||||
|
||||
// `TableStatsCollector` provides the mechanism for users to collect their own
|
||||
// interested stats. This class is essentially a collection of callback
|
||||
// functions that will be invoked during table building.
|
||||
class TableStatsCollector { |
||||
public: |
||||
virtual ~TableStatsCollector() { } |
||||
// Add() will be called when a new key/value pair is inserted into the table.
|
||||
// @params key the original key that is inserted into the table.
|
||||
// @params value the original value that is inserted into the table.
|
||||
virtual Status Add(const Slice& key, const Slice& value) = 0; |
||||
|
||||
// Finish() will be called when a table has already been built and is ready
|
||||
// for writing the stats block.
|
||||
// @params stats User will add their collected statistics to `stats`.
|
||||
virtual Status Finish(TableStats::UserCollectedStats* stats) = 0; |
||||
|
||||
// The name of the stats collector can be used for debugging purpose.
|
||||
virtual const char* Name() const = 0; |
||||
}; |
||||
|
||||
} // namespace rocksdb
|
||||
Loading…
Reference in new issue