Summary:
The primary motivation of the changes is to make it easier to figure out the inside of the tables.
* rename "table stats" to "table properties" since now we have more than "integers" to store in the property block.
* Add filter block size to the basic table properties.
* Whenever a table is built, we'll log the table properties (the sample output is in Test Plan).
* Make an api to expose deleted keys.
Test Plan:
Passed all existing test. and the sample output of table stats:
==================================================================
Basic Properties
------------------------------------------------------------------
# data blocks: 1
# entries: 1
raw key size: 9
raw average key size: 9
raw value size: 9
raw average value size: 0
data block size: 25
index block size: 27
filter block size: 18
(estimated) table size: 70
filter policy: rocksdb.BuiltinBloomFilter
==================================================================
User collected properties: InternalKeyPropertiesCollector
------------------------------------------------------------------
kDeletedKeys: 1
==================================================================
Reviewers: dhruba, haobo
Reviewed By: dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14187
main
kailiu
11 years ago
parent
f045871f1c
commit
1415f8820d
@ -0,0 +1,164 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#include "db/table_properties_collector.h" |
||||||
|
|
||||||
|
#include "db/dbformat.h" |
||||||
|
#include "util/coding.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
namespace { |
||||||
|
void AppendProperty( |
||||||
|
std::string& props, |
||||||
|
const std::string& key, |
||||||
|
const std::string& value, |
||||||
|
const std::string& prop_delim, |
||||||
|
const std::string& kv_delim) { |
||||||
|
props.append(key); |
||||||
|
props.append(kv_delim); |
||||||
|
props.append(value); |
||||||
|
props.append(prop_delim); |
||||||
|
} |
||||||
|
|
||||||
|
template <class TValue> |
||||||
|
void AppendProperty( |
||||||
|
std::string& props, |
||||||
|
const std::string& key, |
||||||
|
const TValue& value, |
||||||
|
const std::string& prop_delim, |
||||||
|
const std::string& kv_delim) { |
||||||
|
AppendProperty( |
||||||
|
props, key, std::to_string(value), prop_delim, kv_delim |
||||||
|
); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
std::string TableProperties::ToString( |
||||||
|
const std::string& prop_delim, |
||||||
|
const std::string& kv_delim) const { |
||||||
|
std::string result; |
||||||
|
result.reserve(1024); |
||||||
|
|
||||||
|
// Basic Info
|
||||||
|
AppendProperty( |
||||||
|
result, "# data blocks", num_data_blocks, prop_delim, kv_delim |
||||||
|
); |
||||||
|
AppendProperty(result, "# entries", num_entries, prop_delim, kv_delim); |
||||||
|
|
||||||
|
AppendProperty(result, "raw key size", raw_key_size, prop_delim, kv_delim); |
||||||
|
AppendProperty( |
||||||
|
result, |
||||||
|
"raw average key size", |
||||||
|
num_entries != 0 ? 1.0 * raw_key_size / num_entries : 0.0, |
||||||
|
prop_delim, |
||||||
|
kv_delim |
||||||
|
); |
||||||
|
AppendProperty( |
||||||
|
result, "raw value size", raw_value_size, prop_delim, kv_delim |
||||||
|
); |
||||||
|
AppendProperty( |
||||||
|
result, |
||||||
|
"raw average value size", |
||||||
|
num_entries != 0 ? 1.0 * raw_value_size / num_entries : 0.0, |
||||||
|
prop_delim, |
||||||
|
kv_delim |
||||||
|
); |
||||||
|
|
||||||
|
AppendProperty(result, "data block size", data_size, prop_delim, kv_delim); |
||||||
|
AppendProperty(result, "index block size", index_size, prop_delim, kv_delim); |
||||||
|
AppendProperty( |
||||||
|
result, "filter block size", filter_size, prop_delim, kv_delim |
||||||
|
); |
||||||
|
AppendProperty( |
||||||
|
result, |
||||||
|
"(estimated) table size=", |
||||||
|
data_size + index_size + filter_size, |
||||||
|
prop_delim, |
||||||
|
kv_delim |
||||||
|
); |
||||||
|
|
||||||
|
AppendProperty( |
||||||
|
result, |
||||||
|
"filter policy name", |
||||||
|
filter_policy_name.empty() ? std::string("N/A") : filter_policy_name, |
||||||
|
prop_delim, |
||||||
|
kv_delim |
||||||
|
); |
||||||
|
|
||||||
|
return result; |
||||||
|
} |
||||||
|
|
||||||
|
Status InternalKeyPropertiesCollector::Add( |
||||||
|
const Slice& key, const Slice& value) { |
||||||
|
ParsedInternalKey ikey; |
||||||
|
if (!ParseInternalKey(key, &ikey)) { |
||||||
|
return Status::InvalidArgument("Invalid internal key"); |
||||||
|
} |
||||||
|
|
||||||
|
if (ikey.type == ValueType::kTypeDeletion) { |
||||||
|
++deleted_keys_; |
||||||
|
} |
||||||
|
|
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status InternalKeyPropertiesCollector::Finish( |
||||||
|
TableProperties::UserCollectedProperties* properties) { |
||||||
|
assert(properties); |
||||||
|
assert(properties->find( |
||||||
|
InternalKeyTablePropertiesNames::kDeletedKeys) == properties->end()); |
||||||
|
std::string val; |
||||||
|
|
||||||
|
PutVarint64(&val, deleted_keys_); |
||||||
|
properties->insert({ InternalKeyTablePropertiesNames::kDeletedKeys, val }); |
||||||
|
|
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
TableProperties::UserCollectedProperties |
||||||
|
InternalKeyPropertiesCollector::GetReadableProperties() const { |
||||||
|
return { |
||||||
|
{ "kDeletedKeys", std::to_string(deleted_keys_) } |
||||||
|
}; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
Status UserKeyTablePropertiesCollector::Add( |
||||||
|
const Slice& key, const Slice& value) { |
||||||
|
ParsedInternalKey ikey; |
||||||
|
if (!ParseInternalKey(key, &ikey)) { |
||||||
|
return Status::InvalidArgument("Invalid internal key"); |
||||||
|
} |
||||||
|
|
||||||
|
return collector_->Add(ikey.user_key, value); |
||||||
|
} |
||||||
|
|
||||||
|
Status UserKeyTablePropertiesCollector::Finish( |
||||||
|
TableProperties::UserCollectedProperties* properties) { |
||||||
|
return collector_->Finish(properties); |
||||||
|
} |
||||||
|
|
||||||
|
TableProperties::UserCollectedProperties |
||||||
|
UserKeyTablePropertiesCollector::GetReadableProperties() const { |
||||||
|
return collector_->GetReadableProperties(); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
const std::string InternalKeyTablePropertiesNames::kDeletedKeys |
||||||
|
= "rocksdb.deleted.keys"; |
||||||
|
|
||||||
|
uint64_t GetDeletedKeys( |
||||||
|
const TableProperties::UserCollectedProperties& props) { |
||||||
|
auto pos = props.find(InternalKeyTablePropertiesNames::kDeletedKeys); |
||||||
|
if (pos == props.end()) { |
||||||
|
return 0; |
||||||
|
} |
||||||
|
Slice raw = pos->second; |
||||||
|
uint64_t val = 0; |
||||||
|
return GetVarint64(&raw, &val) ? val : 0; |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,76 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// This file defines a collection of statistics collectors.
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include "rocksdb/table_properties.h" |
||||||
|
|
||||||
|
#include <memory> |
||||||
|
#include <string> |
||||||
|
#include <vector> |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
struct InternalKeyTablePropertiesNames { |
||||||
|
static const std::string kDeletedKeys; |
||||||
|
}; |
||||||
|
|
||||||
|
// Collecting the statistics for internal keys. Visible only by internal
|
||||||
|
// rocksdb modules.
|
||||||
|
class InternalKeyPropertiesCollector : public TablePropertiesCollector { |
||||||
|
public: |
||||||
|
virtual Status Add(const Slice& key, const Slice& value) override; |
||||||
|
|
||||||
|
virtual Status Finish( |
||||||
|
TableProperties::UserCollectedProperties* properties) override; |
||||||
|
|
||||||
|
virtual const char* Name() const override { |
||||||
|
return "InternalKeyPropertiesCollector"; |
||||||
|
} |
||||||
|
|
||||||
|
TableProperties::UserCollectedProperties |
||||||
|
GetReadableProperties() const override; |
||||||
|
|
||||||
|
private: |
||||||
|
uint64_t deleted_keys_ = 0; |
||||||
|
}; |
||||||
|
|
||||||
|
// When rocksdb creates a new table, it will encode all "user keys" into
|
||||||
|
// "internal keys", which contains meta information of a given entry.
|
||||||
|
//
|
||||||
|
// This class extracts user key from the encoded internal key when Add() is
|
||||||
|
// invoked.
|
||||||
|
class UserKeyTablePropertiesCollector : public TablePropertiesCollector { |
||||||
|
public: |
||||||
|
explicit UserKeyTablePropertiesCollector( |
||||||
|
TablePropertiesCollector* collector) : |
||||||
|
UserKeyTablePropertiesCollector( |
||||||
|
std::shared_ptr<TablePropertiesCollector>(collector) |
||||||
|
) { |
||||||
|
} |
||||||
|
|
||||||
|
explicit UserKeyTablePropertiesCollector( |
||||||
|
std::shared_ptr<TablePropertiesCollector> collector) : |
||||||
|
collector_(collector) { |
||||||
|
} |
||||||
|
|
||||||
|
virtual ~UserKeyTablePropertiesCollector() { } |
||||||
|
|
||||||
|
virtual Status Add(const Slice& key, const Slice& value) override; |
||||||
|
|
||||||
|
virtual Status Finish( |
||||||
|
TableProperties::UserCollectedProperties* properties) override; |
||||||
|
|
||||||
|
virtual const char* Name() const override { return collector_->Name(); } |
||||||
|
|
||||||
|
TableProperties::UserCollectedProperties |
||||||
|
GetReadableProperties() const override; |
||||||
|
|
||||||
|
protected: |
||||||
|
std::shared_ptr<TablePropertiesCollector> collector_; |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -1,55 +0,0 @@ |
|||||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
||||||
// This source code is licensed under the BSD-style license found in the
|
|
||||||
// LICENSE file in the root directory of this source tree. An additional grant
|
|
||||||
// of patent rights can be found in the PATENTS file in the same directory.
|
|
||||||
|
|
||||||
#include "db/table_stats_collector.h" |
|
||||||
|
|
||||||
#include "db/dbformat.h" |
|
||||||
#include "util/coding.h" |
|
||||||
|
|
||||||
namespace rocksdb { |
|
||||||
|
|
||||||
Status InternalKeyStatsCollector::Add(const Slice& key, const Slice& value) { |
|
||||||
ParsedInternalKey ikey; |
|
||||||
if (!ParseInternalKey(key, &ikey)) { |
|
||||||
return Status::InvalidArgument("Invalid internal key"); |
|
||||||
} |
|
||||||
|
|
||||||
if (ikey.type == ValueType::kTypeDeletion) { |
|
||||||
++deleted_keys_; |
|
||||||
} |
|
||||||
|
|
||||||
return Status::OK(); |
|
||||||
} |
|
||||||
|
|
||||||
Status InternalKeyStatsCollector::Finish( |
|
||||||
TableStats::UserCollectedStats* stats) { |
|
||||||
assert(stats); |
|
||||||
assert(stats->find(InternalKeyTableStatsNames::kDeletedKeys) == stats->end()); |
|
||||||
std::string val; |
|
||||||
|
|
||||||
PutVarint64(&val, deleted_keys_); |
|
||||||
stats->insert(std::make_pair(InternalKeyTableStatsNames::kDeletedKeys, val)); |
|
||||||
|
|
||||||
return Status::OK(); |
|
||||||
} |
|
||||||
|
|
||||||
Status UserKeyTableStatsCollector::Add(const Slice& key, const Slice& value) { |
|
||||||
ParsedInternalKey ikey; |
|
||||||
if (!ParseInternalKey(key, &ikey)) { |
|
||||||
return Status::InvalidArgument("Invalid internal key"); |
|
||||||
} |
|
||||||
|
|
||||||
return collector_->Add(ikey.user_key, value); |
|
||||||
} |
|
||||||
|
|
||||||
Status UserKeyTableStatsCollector::Finish( |
|
||||||
TableStats::UserCollectedStats* stats) { |
|
||||||
return collector_->Finish(stats); |
|
||||||
} |
|
||||||
|
|
||||||
const std::string InternalKeyTableStatsNames::kDeletedKeys |
|
||||||
= "rocksdb.deleted.keys"; |
|
||||||
|
|
||||||
} // namespace rocksdb
|
|
||||||
@ -1,58 +0,0 @@ |
|||||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
||||||
// This source code is licensed under the BSD-style license found in the
|
|
||||||
// LICENSE file in the root directory of this source tree. An additional grant
|
|
||||||
// of patent rights can be found in the PATENTS file in the same directory.
|
|
||||||
//
|
|
||||||
// This file defines a collection of statistics collectors.
|
|
||||||
#pragma once |
|
||||||
|
|
||||||
#include "rocksdb/table_stats.h" |
|
||||||
|
|
||||||
#include <memory> |
|
||||||
#include <string> |
|
||||||
#include <vector> |
|
||||||
|
|
||||||
namespace rocksdb { |
|
||||||
|
|
||||||
struct InternalKeyTableStatsNames { |
|
||||||
static const std::string kDeletedKeys; |
|
||||||
}; |
|
||||||
|
|
||||||
// Collecting the statistics for internal keys. Visible only by internal
|
|
||||||
// rocksdb modules.
|
|
||||||
class InternalKeyStatsCollector : public TableStatsCollector { |
|
||||||
public: |
|
||||||
virtual Status Add(const Slice& key, const Slice& value); |
|
||||||
virtual Status Finish(TableStats::UserCollectedStats* stats); |
|
||||||
virtual const char* Name() const { return "InternalKeyStatsCollector"; } |
|
||||||
|
|
||||||
private: |
|
||||||
uint64_t deleted_keys_ = 0; |
|
||||||
}; |
|
||||||
|
|
||||||
// When rocksdb creates a new table, it will encode all "user keys" into
|
|
||||||
// "internal keys", which contains meta information of a given entry.
|
|
||||||
//
|
|
||||||
// This class extracts user key from the encoded internal key when Add() is
|
|
||||||
// invoked.
|
|
||||||
class UserKeyTableStatsCollector : public TableStatsCollector { |
|
||||||
public: |
|
||||||
explicit UserKeyTableStatsCollector(TableStatsCollector* collector): |
|
||||||
UserKeyTableStatsCollector( |
|
||||||
std::shared_ptr<TableStatsCollector>(collector) |
|
||||||
) { |
|
||||||
} |
|
||||||
|
|
||||||
explicit UserKeyTableStatsCollector( |
|
||||||
std::shared_ptr<TableStatsCollector> collector) : collector_(collector) { |
|
||||||
} |
|
||||||
virtual ~UserKeyTableStatsCollector() { } |
|
||||||
virtual Status Add(const Slice& key, const Slice& value); |
|
||||||
virtual Status Finish(TableStats::UserCollectedStats* stats); |
|
||||||
virtual const char* Name() const { return collector_->Name(); } |
|
||||||
|
|
||||||
protected: |
|
||||||
std::shared_ptr<TableStatsCollector> collector_; |
|
||||||
}; |
|
||||||
|
|
||||||
} // namespace rocksdb
|
|
||||||
@ -0,0 +1,90 @@ |
|||||||
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <string> |
||||||
|
#include <unordered_map> |
||||||
|
|
||||||
|
#include "rocksdb/status.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
// TableProperties contains a bunch of read-only properties of its associated
|
||||||
|
// table.
|
||||||
|
struct TableProperties { |
||||||
|
public: |
||||||
|
// Other than basic table properties, each table may also have the user
|
||||||
|
// collected properties.
|
||||||
|
// The value of the user-collected properties are encoded as raw bytes --
|
||||||
|
// users have to interprete these values by themselves.
|
||||||
|
typedef |
||||||
|
std::unordered_map<std::string, std::string> |
||||||
|
UserCollectedProperties; |
||||||
|
|
||||||
|
// the total size of all data blocks.
|
||||||
|
uint64_t data_size = 0; |
||||||
|
// the size of index block.
|
||||||
|
uint64_t index_size = 0; |
||||||
|
// the size of filter block.
|
||||||
|
uint64_t filter_size = 0; |
||||||
|
// total raw key size
|
||||||
|
uint64_t raw_key_size = 0; |
||||||
|
// total raw value size
|
||||||
|
uint64_t raw_value_size = 0; |
||||||
|
// the number of blocks in this table
|
||||||
|
uint64_t num_data_blocks = 0; |
||||||
|
// the number of entries in this table
|
||||||
|
uint64_t num_entries = 0; |
||||||
|
|
||||||
|
// The name of the filter policy used in this table.
|
||||||
|
// If no filter policy is used, `filter_policy_name` will be an empty string.
|
||||||
|
std::string filter_policy_name; |
||||||
|
|
||||||
|
// user collected properties
|
||||||
|
UserCollectedProperties user_collected_properties; |
||||||
|
|
||||||
|
// convert this object to a human readable form
|
||||||
|
// @prop_delim: delimiter for each property.
|
||||||
|
std::string ToString( |
||||||
|
const std::string& prop_delim = "; ", |
||||||
|
const std::string& kv_delim = "=") const; |
||||||
|
}; |
||||||
|
|
||||||
|
// `TablePropertiesCollector` provides the mechanism for users to collect
|
||||||
|
// their own interested properties. This class is essentially a collection
|
||||||
|
// of callback functions that will be invoked during table building.
|
||||||
|
class TablePropertiesCollector { |
||||||
|
public: |
||||||
|
virtual ~TablePropertiesCollector() { } |
||||||
|
|
||||||
|
// Add() will be called when a new key/value pair is inserted into the table.
|
||||||
|
// @params key the original key that is inserted into the table.
|
||||||
|
// @params value the original value that is inserted into the table.
|
||||||
|
virtual Status Add(const Slice& key, const Slice& value) = 0; |
||||||
|
|
||||||
|
// Finish() will be called when a table has already been built and is ready
|
||||||
|
// for writing the properties block.
|
||||||
|
// @params properties User will add their collected statistics to
|
||||||
|
// `properties`.
|
||||||
|
virtual Status Finish( |
||||||
|
TableProperties::UserCollectedProperties* properties) = 0; |
||||||
|
|
||||||
|
// The name of the properties collector can be used for debugging purpose.
|
||||||
|
virtual const char* Name() const = 0; |
||||||
|
|
||||||
|
// Return the human-readable properties, where the key is property name and
|
||||||
|
// the value is the human-readable form of value.
|
||||||
|
virtual TableProperties::UserCollectedProperties |
||||||
|
GetReadableProperties() const = 0; |
||||||
|
}; |
||||||
|
|
||||||
|
// Extra properties
|
||||||
|
// Below is a list of non-basic properties that are collected by database
|
||||||
|
// itself. Especially some properties regarding to the internal keys (which
|
||||||
|
// is unknown to `table`).
|
||||||
|
extern uint64_t GetDeletedKeys( |
||||||
|
const TableProperties::UserCollectedProperties& props); |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -1,67 +0,0 @@ |
|||||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
||||||
// This source code is licensed under the BSD-style license found in the
|
|
||||||
// LICENSE file in the root directory of this source tree. An additional grant
|
|
||||||
// of patent rights can be found in the PATENTS file in the same directory.
|
|
||||||
#pragma once |
|
||||||
|
|
||||||
#include <string> |
|
||||||
#include <unordered_map> |
|
||||||
|
|
||||||
#include "rocksdb/status.h" |
|
||||||
|
|
||||||
namespace rocksdb { |
|
||||||
|
|
||||||
// TableStats contains a bunch of read-only stats of its associated
|
|
||||||
// table.
|
|
||||||
struct TableStats { |
|
||||||
public: |
|
||||||
// Other than basic table stats, each table may also have the user
|
|
||||||
// collected stats.
|
|
||||||
// The value of the user-collected stats are encoded as raw bytes --
|
|
||||||
// users have to interprete these values by themselves.
|
|
||||||
typedef |
|
||||||
std::unordered_map<std::string, std::string> |
|
||||||
UserCollectedStats; |
|
||||||
|
|
||||||
// the total size of all data blocks.
|
|
||||||
uint64_t data_size = 0; |
|
||||||
// the total size of all index blocks.
|
|
||||||
uint64_t index_size = 0; |
|
||||||
// total raw key size
|
|
||||||
uint64_t raw_key_size = 0; |
|
||||||
// total raw value size
|
|
||||||
uint64_t raw_value_size = 0; |
|
||||||
// the number of blocks in this table
|
|
||||||
uint64_t num_data_blocks = 0; |
|
||||||
// the number of entries in this table
|
|
||||||
uint64_t num_entries = 0; |
|
||||||
|
|
||||||
// The name of the filter policy used in this table.
|
|
||||||
// If no filter policy is used, `filter_policy_name` will be an empty string.
|
|
||||||
std::string filter_policy_name; |
|
||||||
|
|
||||||
// user collected stats
|
|
||||||
UserCollectedStats user_collected_stats; |
|
||||||
}; |
|
||||||
|
|
||||||
// `TableStatsCollector` provides the mechanism for users to collect their own
|
|
||||||
// interested stats. This class is essentially a collection of callback
|
|
||||||
// functions that will be invoked during table building.
|
|
||||||
class TableStatsCollector { |
|
||||||
public: |
|
||||||
virtual ~TableStatsCollector() { } |
|
||||||
// Add() will be called when a new key/value pair is inserted into the table.
|
|
||||||
// @params key the original key that is inserted into the table.
|
|
||||||
// @params value the original value that is inserted into the table.
|
|
||||||
virtual Status Add(const Slice& key, const Slice& value) = 0; |
|
||||||
|
|
||||||
// Finish() will be called when a table has already been built and is ready
|
|
||||||
// for writing the stats block.
|
|
||||||
// @params stats User will add their collected statistics to `stats`.
|
|
||||||
virtual Status Finish(TableStats::UserCollectedStats* stats) = 0; |
|
||||||
|
|
||||||
// The name of the stats collector can be used for debugging purpose.
|
|
||||||
virtual const char* Name() const = 0; |
|
||||||
}; |
|
||||||
|
|
||||||
} // namespace rocksdb
|
|
||||||
Loading…
Reference in new issue