Add comparator, merge operator, property collectors to SST file properties (again)

Summary:
This is the original diff that I have landed and reverted and now I want to land again https://reviews.facebook.net/D34269

For old SST files we will show
```
  comparator name: N/A
  merge operator name: N/A
  property collectors names: N/A
```

For new SST files with no merge operator name and with no property collectors
```
  comparator name: leveldb.BytewiseComparator
  merge operator name: nullptr
  property collectors names: []
```

for new SST files with these properties
```
  comparator name: leveldb.BytewiseComparator
  merge operator name: UInt64AddOperator
  property collectors names: [DummyPropertiesCollector1,DummyPropertiesCollector2]
```

Test Plan: unittests

Reviewers: andrewkr, yhchiang, sdong

Reviewed By: sdong

Subscribers: andrewkr, dhruba

Differential Revision: https://reviews.facebook.net/D56487
main
Islam AbdelRahman 9 years ago
parent 7a6045a3c7
commit 5bd4022fec
  1. 15
      include/rocksdb/table_properties.h
  2. 20
      table/block_based_table_builder.cc
  3. 20
      table/meta_blocks.cc
  4. 19
      table/table_properties.cc
  5. 88
      table/table_test.cc

@ -41,6 +41,9 @@ struct TablePropertiesNames {
static const std::string kFilterPolicy;
static const std::string kColumnFamilyName;
static const std::string kColumnFamilyId;
static const std::string kComparator;
static const std::string kMergeOperator;
static const std::string kPropertyCollectors;
};
extern const std::string kPropertiesBlock;
@ -154,6 +157,18 @@ struct TableProperties {
// If no filter policy is used, `filter_policy_name` will be an empty string.
std::string filter_policy_name;
// The name of the comparator used in this table.
std::string comparator_name;
// The name of the merge operator used in this table.
// If no merge operator is used, `merge_operator_name` will be "nullptr".
std::string merge_operator_name;
// The names of the property collectors factories used in this table
// separated by commas
// {collector_name[1]},{collector_name[2]},{collector_name[3]} ..
std::string property_collectors_names;
// user collected properties
UserCollectedProperties user_collected_properties;
UserCollectedProperties readable_properties;

@ -26,6 +26,7 @@
#include "rocksdb/env.h"
#include "rocksdb/filter_policy.h"
#include "rocksdb/flush_block_policy.h"
#include "rocksdb/merge_operator.h"
#include "rocksdb/table.h"
#include "table/block.h"
@ -810,6 +811,25 @@ Status BlockBasedTableBuilder::Finish() {
r->table_options.filter_policy->Name() : "";
r->props.index_size =
r->index_builder->EstimatedSize() + kBlockTrailerSize;
r->props.comparator_name = r->ioptions.comparator != nullptr
? r->ioptions.comparator->Name()
: "nullptr";
r->props.merge_operator_name = r->ioptions.merge_operator != nullptr
? r->ioptions.merge_operator->Name()
: "nullptr";
std::string property_collectors_names = "[";
property_collectors_names = "[";
for (size_t i = 0;
i < r->ioptions.table_properties_collector_factories.size(); ++i) {
if (i != 0) {
property_collectors_names += ",";
}
property_collectors_names +=
r->ioptions.table_properties_collector_factories[i]->Name();
}
property_collectors_names += "]";
r->props.property_collectors_names = property_collectors_names;
// Add basic properties
property_block_builder.AddTableProperty(r->props);

@ -72,8 +72,18 @@ void PropertyBlockBuilder::AddTableProperty(const TableProperties& props) {
Add(TablePropertiesNames::kColumnFamilyId, props.column_family_id);
if (!props.filter_policy_name.empty()) {
Add(TablePropertiesNames::kFilterPolicy,
props.filter_policy_name);
Add(TablePropertiesNames::kFilterPolicy, props.filter_policy_name);
}
if (!props.comparator_name.empty()) {
Add(TablePropertiesNames::kComparator, props.comparator_name);
}
if (!props.merge_operator_name.empty()) {
Add(TablePropertiesNames::kMergeOperator, props.merge_operator_name);
}
if (!props.property_collectors_names.empty()) {
Add(TablePropertiesNames::kPropertyCollectors,
props.property_collectors_names);
}
if (!props.column_family_name.empty()) {
Add(TablePropertiesNames::kColumnFamilyName, props.column_family_name);
@ -212,6 +222,12 @@ Status ReadProperties(const Slice& handle_value, RandomAccessFileReader* file,
new_table_properties->filter_policy_name = raw_val.ToString();
} else if (key == TablePropertiesNames::kColumnFamilyName) {
new_table_properties->column_family_name = raw_val.ToString();
} else if (key == TablePropertiesNames::kComparator) {
new_table_properties->comparator_name = raw_val.ToString();
} else if (key == TablePropertiesNames::kMergeOperator) {
new_table_properties->merge_operator_name = raw_val.ToString();
} else if (key == TablePropertiesNames::kPropertyCollectors) {
new_table_properties->property_collectors_names = raw_val.ToString();
} else {
// handle user-collected properties
new_table_properties->user_collected_properties.insert(

@ -86,6 +86,20 @@ std::string TableProperties::ToString(
column_family_name.empty() ? std::string("N/A") : column_family_name,
prop_delim, kv_delim);
AppendProperty(result, "comparator name",
comparator_name.empty() ? std::string("N/A") : comparator_name,
prop_delim, kv_delim);
AppendProperty(
result, "merge operator name",
merge_operator_name.empty() ? std::string("N/A") : merge_operator_name,
prop_delim, kv_delim);
AppendProperty(result, "property collectors names",
property_collectors_names.empty() ? std::string("N/A")
: property_collectors_names,
prop_delim, kv_delim);
return result;
}
@ -123,6 +137,11 @@ const std::string TablePropertiesNames::kColumnFamilyId =
"rocksdb.column.family.id";
const std::string TablePropertiesNames::kColumnFamilyName =
"rocksdb.column.family.name";
const std::string TablePropertiesNames::kComparator = "rocksdb.comparator";
const std::string TablePropertiesNames::kMergeOperator =
"rocksdb.merge.operator";
const std::string TablePropertiesNames::kPropertyCollectors =
"rocksdb.property.collectors";
extern const std::string kPropertiesBlock = "rocksdb.properties";
// Old property block name for backward compatibility

@ -47,6 +47,7 @@
#include "util/string_util.h"
#include "util/testharness.h"
#include "util/testutil.h"
#include "utilities/merge_operators.h"
namespace rocksdb {
@ -57,6 +58,40 @@ extern const uint64_t kPlainTableMagicNumber;
namespace {
// DummyPropertiesCollector used to test BlockBasedTableProperties
class DummyPropertiesCollector : public TablePropertiesCollector {
public:
const char* Name() const { return ""; }
Status Finish(UserCollectedProperties* properties) { return Status::OK(); }
Status Add(const Slice& user_key, const Slice& value) { return Status::OK(); }
virtual UserCollectedProperties GetReadableProperties() const {
return UserCollectedProperties{};
}
};
class DummyPropertiesCollectorFactory1
: public TablePropertiesCollectorFactory {
public:
virtual TablePropertiesCollector* CreateTablePropertiesCollector(
TablePropertiesCollectorFactory::Context context) {
return new DummyPropertiesCollector();
}
const char* Name() const { return "DummyPropertiesCollector1"; }
};
class DummyPropertiesCollectorFactory2
: public TablePropertiesCollectorFactory {
public:
virtual TablePropertiesCollector* CreateTablePropertiesCollector(
TablePropertiesCollectorFactory::Context context) {
return new DummyPropertiesCollector();
}
const char* Name() const { return "DummyPropertiesCollector2"; }
};
// Return reverse of "key".
// Used to test non-lexicographic comparators.
std::string Reverse(const Slice& key) {
@ -1028,6 +1063,59 @@ TEST_F(BlockBasedTableTest, BasicBlockBasedTableProperties) {
c.ResetTableReader();
}
TEST_F(BlockBasedTableTest, BlockBasedTableProperties2) {
TableConstructor c(&reverse_key_comparator);
std::vector<std::string> keys;
stl_wrappers::KVMap kvmap;
{
Options options;
BlockBasedTableOptions table_options;
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
const ImmutableCFOptions ioptions(options);
c.Finish(options, ioptions, table_options,
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
auto& props = *c.GetTableReader()->GetTableProperties();
// Default comparator
ASSERT_EQ("leveldb.BytewiseComparator", props.comparator_name);
// No merge operator
ASSERT_EQ("nullptr", props.merge_operator_name);
// No property collectors
ASSERT_EQ("[]", props.property_collectors_names);
// No filter policy is used
ASSERT_EQ("", props.filter_policy_name);
c.ResetTableReader();
}
{
Options options;
BlockBasedTableOptions table_options;
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
options.comparator = &reverse_key_comparator;
options.merge_operator = MergeOperators::CreateUInt64AddOperator();
options.table_properties_collector_factories.emplace_back(
new DummyPropertiesCollectorFactory1());
options.table_properties_collector_factories.emplace_back(
new DummyPropertiesCollectorFactory2());
const ImmutableCFOptions ioptions(options);
c.Finish(options, ioptions, table_options,
GetPlainInternalComparator(options.comparator), &keys, &kvmap);
auto& props = *c.GetTableReader()->GetTableProperties();
ASSERT_EQ("rocksdb.ReverseBytewiseComparator", props.comparator_name);
ASSERT_EQ("UInt64AddOperator", props.merge_operator_name);
ASSERT_EQ("[DummyPropertiesCollector1,DummyPropertiesCollector2]",
props.property_collectors_names);
ASSERT_EQ("", props.filter_policy_name); // no filter policy is used
c.ResetTableReader();
}
}
TEST_F(BlockBasedTableTest, FilterPolicyNameProperties) {
TableConstructor c(BytewiseComparator(), true);
c.Add("a1", "val1");

Loading…
Cancel
Save