Make MergeOperator+CompactionFilter/Factory into Customizable Classes (#8481)
Summary: - Changed MergeOperator, CompactionFilter, and CompactionFilterFactory into Customizable classes. - Added Options/Configurable/Object Registration for TTL and Cassandra variants - Changed the StringAppend MergeOperators to accept a string delimiter rather than a simple char. Made the delimiter into a configurable option - Added tests for new functionality Pull Request resolved: https://github.com/facebook/rocksdb/pull/8481 Reviewed By: zhichao-cao Differential Revision: D30136050 Pulled By: mrambacher fbshipit-source-id: 271d1772835935b6773abaf018ee71e42f9491afmain
mrambacher
3 years ago
committed by
Facebook GitHub Bot
parent
fd2079938d
commit
d057e8326d
@ -0,0 +1,40 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
#pragma once |
||||
#include <cinttypes> |
||||
|
||||
#include "rocksdb/rocksdb_namespace.h" |
||||
|
||||
namespace ROCKSDB_NAMESPACE { |
||||
class ObjectLibrary; |
||||
namespace cassandra { |
||||
struct CassandraOptions { |
||||
static const char* kName() { return "CassandraOptions"; } |
||||
CassandraOptions(int32_t _gc_grace_period_in_seconds, size_t _operands_limit, |
||||
bool _purge_ttl_on_expiration = false) |
||||
: operands_limit(_operands_limit), |
||||
gc_grace_period_in_seconds(_gc_grace_period_in_seconds), |
||||
purge_ttl_on_expiration(_purge_ttl_on_expiration) {} |
||||
// Limit on the number of merge operands.
|
||||
size_t operands_limit; |
||||
|
||||
// How long (in seconds) tombstoned data remains before it is purged
|
||||
int32_t gc_grace_period_in_seconds; |
||||
|
||||
// If is set to true, expired data will be directly purged.
|
||||
// Otherwise expired data will be converted tombstones first,
|
||||
// then be eventually removed after gc grace period. This value should
|
||||
// only true if all writes have same ttl setting, otherwise it could bring old
|
||||
// data back.
|
||||
bool purge_ttl_on_expiration; |
||||
}; |
||||
#ifndef ROCKSDB_LITE |
||||
extern "C" { |
||||
int RegisterCassandraObjects(ObjectLibrary& library, const std::string& arg); |
||||
} // extern "C"
|
||||
#endif // ROCKSDB_LITE
|
||||
} // namespace cassandra
|
||||
} // namespace ROCKSDB_NAMESPACE
|
||||
@ -0,0 +1,56 @@ |
||||
// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include <memory> |
||||
|
||||
#include "rocksdb/compaction_filter.h" |
||||
#include "rocksdb/options.h" |
||||
#include "rocksdb/utilities/customizable_util.h" |
||||
#include "rocksdb/utilities/options_type.h" |
||||
#include "utilities/compaction_filters/layered_compaction_filter_base.h" |
||||
#include "utilities/compaction_filters/remove_emptyvalue_compactionfilter.h" |
||||
|
||||
namespace ROCKSDB_NAMESPACE { |
||||
#ifndef ROCKSDB_LITE |
||||
static int RegisterBuiltinCompactionFilters(ObjectLibrary& library, |
||||
const std::string& /*arg*/) { |
||||
library.Register<CompactionFilter>( |
||||
RemoveEmptyValueCompactionFilter::kClassName(), |
||||
[](const std::string& /*uri*/, |
||||
std::unique_ptr<CompactionFilter>* /*guard*/, |
||||
std::string* /*errmsg*/) { |
||||
return new RemoveEmptyValueCompactionFilter(); |
||||
}); |
||||
return 1; |
||||
} |
||||
#endif // ROCKSDB_LITE
|
||||
Status CompactionFilter::CreateFromString(const ConfigOptions& config_options, |
||||
const std::string& value, |
||||
const CompactionFilter** result) { |
||||
#ifndef ROCKSDB_LITE |
||||
static std::once_flag once; |
||||
std::call_once(once, [&]() { |
||||
RegisterBuiltinCompactionFilters(*(ObjectLibrary::Default().get()), ""); |
||||
}); |
||||
#endif // ROCKSDB_LITE
|
||||
CompactionFilter* filter = const_cast<CompactionFilter*>(*result); |
||||
Status status = LoadStaticObject<CompactionFilter>(config_options, value, |
||||
nullptr, &filter); |
||||
if (status.ok()) { |
||||
*result = const_cast<CompactionFilter*>(filter); |
||||
} |
||||
return status; |
||||
} |
||||
|
||||
Status CompactionFilterFactory::CreateFromString( |
||||
const ConfigOptions& config_options, const std::string& value, |
||||
std::shared_ptr<CompactionFilterFactory>* result) { |
||||
// Currently there are no builtin CompactionFilterFactories.
|
||||
// If any are introduced, they need to be registered here.
|
||||
Status status = LoadSharedObject<CompactionFilterFactory>( |
||||
config_options, value, nullptr, result); |
||||
return status; |
||||
} |
||||
} // namespace ROCKSDB_NAMESPACE
|
||||
@ -0,0 +1,125 @@ |
||||
// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "utilities/merge_operators.h" |
||||
|
||||
#include <memory> |
||||
|
||||
#include "rocksdb/merge_operator.h" |
||||
#include "rocksdb/options.h" |
||||
#include "rocksdb/utilities/customizable_util.h" |
||||
#include "rocksdb/utilities/object_registry.h" |
||||
#include "utilities/merge_operators/bytesxor.h" |
||||
#include "utilities/merge_operators/sortlist.h" |
||||
#include "utilities/merge_operators/string_append/stringappend.h" |
||||
#include "utilities/merge_operators/string_append/stringappend2.h" |
||||
|
||||
namespace ROCKSDB_NAMESPACE { |
||||
static bool LoadMergeOperator(const std::string& id, |
||||
std::shared_ptr<MergeOperator>* result) { |
||||
bool success = true; |
||||
// TODO: Hook the "name" up to the actual Name() of the MergeOperators?
|
||||
// Requires these classes be moved into a header file...
|
||||
if (id == "put" || id == "PutOperator") { |
||||
*result = MergeOperators::CreatePutOperator(); |
||||
} else if (id == "put_v1") { |
||||
*result = MergeOperators::CreateDeprecatedPutOperator(); |
||||
} else if (id == "uint64add" || id == "UInt64AddOperator") { |
||||
*result = MergeOperators::CreateUInt64AddOperator(); |
||||
} else if (id == "max" || id == "MaxOperator") { |
||||
*result = MergeOperators::CreateMaxOperator(); |
||||
#ifdef ROCKSDB_LITE |
||||
// The remainder of the classes are handled by the ObjectRegistry in
|
||||
// non-LITE mode
|
||||
} else if (id == StringAppendOperator::kNickName() || |
||||
id == StringAppendOperator::kClassName()) { |
||||
*result = MergeOperators::CreateStringAppendOperator(); |
||||
} else if (id == StringAppendTESTOperator::kNickName() || |
||||
id == StringAppendTESTOperator::kClassName()) { |
||||
*result = MergeOperators::CreateStringAppendTESTOperator(); |
||||
} else if (id == BytesXOROperator::kNickName() || |
||||
id == BytesXOROperator::kClassName()) { |
||||
*result = MergeOperators::CreateBytesXOROperator(); |
||||
} else if (id == SortList::kNickName() || id == SortList::kClassName()) { |
||||
*result = MergeOperators::CreateSortOperator(); |
||||
#endif // ROCKSDB_LITE
|
||||
} else { |
||||
success = false; |
||||
} |
||||
return success; |
||||
} |
||||
|
||||
#ifndef ROCKSDB_LITE |
||||
static int RegisterBuiltinMergeOperators(ObjectLibrary& library, |
||||
const std::string& /*arg*/) { |
||||
size_t num_types; |
||||
auto AsRegex = [](const std::string& name, const std::string& alt) { |
||||
std::string regex; |
||||
regex.append("(").append(name); |
||||
regex.append("|").append(alt).append(")"); |
||||
return regex; |
||||
}; |
||||
|
||||
library.Register<MergeOperator>( |
||||
AsRegex(StringAppendOperator::kClassName(), |
||||
StringAppendOperator::kNickName()), |
||||
[](const std::string& /*uri*/, std::unique_ptr<MergeOperator>* guard, |
||||
std::string* /*errmsg*/) { |
||||
guard->reset(new StringAppendOperator(",")); |
||||
return guard->get(); |
||||
}); |
||||
library.Register<MergeOperator>( |
||||
AsRegex(StringAppendTESTOperator::kClassName(), |
||||
StringAppendTESTOperator::kNickName()), |
||||
[](const std::string& /*uri*/, std::unique_ptr<MergeOperator>* guard, |
||||
std::string* /*errmsg*/) { |
||||
guard->reset(new StringAppendTESTOperator(",")); |
||||
return guard->get(); |
||||
}); |
||||
library.Register<MergeOperator>( |
||||
AsRegex(SortList::kClassName(), SortList::kNickName()), |
||||
[](const std::string& /*uri*/, std::unique_ptr<MergeOperator>* guard, |
||||
std::string* /*errmsg*/) { |
||||
guard->reset(new SortList()); |
||||
return guard->get(); |
||||
}); |
||||
library.Register<MergeOperator>( |
||||
AsRegex(BytesXOROperator::kClassName(), BytesXOROperator::kNickName()), |
||||
[](const std::string& /*uri*/, std::unique_ptr<MergeOperator>* guard, |
||||
std::string* /*errmsg*/) { |
||||
guard->reset(new BytesXOROperator()); |
||||
return guard->get(); |
||||
}); |
||||
|
||||
return static_cast<int>(library.GetFactoryCount(&num_types)); |
||||
} |
||||
#endif // ROCKSDB_LITE
|
||||
|
||||
Status MergeOperator::CreateFromString(const ConfigOptions& config_options, |
||||
const std::string& value, |
||||
std::shared_ptr<MergeOperator>* result) { |
||||
#ifndef ROCKSDB_LITE |
||||
static std::once_flag once; |
||||
std::call_once(once, [&]() { |
||||
RegisterBuiltinMergeOperators(*(ObjectLibrary::Default().get()), ""); |
||||
}); |
||||
#endif // ROCKSDB_LITE
|
||||
return LoadSharedObject<MergeOperator>(config_options, value, |
||||
LoadMergeOperator, result); |
||||
} |
||||
|
||||
std::shared_ptr<MergeOperator> MergeOperators::CreateFromStringId( |
||||
const std::string& id) { |
||||
std::shared_ptr<MergeOperator> result; |
||||
Status s = MergeOperator::CreateFromString(ConfigOptions(), id, &result); |
||||
if (s.ok()) { |
||||
return result; |
||||
} else { |
||||
// Empty or unknown, just return nullptr
|
||||
return nullptr; |
||||
} |
||||
} |
||||
|
||||
} // namespace ROCKSDB_NAMESPACE
|
||||
Loading…
Reference in new issue