Make MergeOperator+CompactionFilter/Factory into Customizable Classes (#8481)
Summary: - Changed MergeOperator, CompactionFilter, and CompactionFilterFactory into Customizable classes. - Added Options/Configurable/Object Registration for TTL and Cassandra variants - Changed the StringAppend MergeOperators to accept a string delimiter rather than a simple char. Made the delimiter into a configurable option - Added tests for new functionality Pull Request resolved: https://github.com/facebook/rocksdb/pull/8481 Reviewed By: zhichao-cao Differential Revision: D30136050 Pulled By: mrambacher fbshipit-source-id: 271d1772835935b6773abaf018ee71e42f9491afmain
mrambacher
3 years ago
committed by
Facebook GitHub Bot
parent
fd2079938d
commit
d057e8326d
@ -0,0 +1,40 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
#include <cinttypes> |
||||||
|
|
||||||
|
#include "rocksdb/rocksdb_namespace.h" |
||||||
|
|
||||||
|
namespace ROCKSDB_NAMESPACE { |
||||||
|
class ObjectLibrary; |
||||||
|
namespace cassandra { |
||||||
|
struct CassandraOptions { |
||||||
|
static const char* kName() { return "CassandraOptions"; } |
||||||
|
CassandraOptions(int32_t _gc_grace_period_in_seconds, size_t _operands_limit, |
||||||
|
bool _purge_ttl_on_expiration = false) |
||||||
|
: operands_limit(_operands_limit), |
||||||
|
gc_grace_period_in_seconds(_gc_grace_period_in_seconds), |
||||||
|
purge_ttl_on_expiration(_purge_ttl_on_expiration) {} |
||||||
|
// Limit on the number of merge operands.
|
||||||
|
size_t operands_limit; |
||||||
|
|
||||||
|
// How long (in seconds) tombstoned data remains before it is purged
|
||||||
|
int32_t gc_grace_period_in_seconds; |
||||||
|
|
||||||
|
// If is set to true, expired data will be directly purged.
|
||||||
|
// Otherwise expired data will be converted tombstones first,
|
||||||
|
// then be eventually removed after gc grace period. This value should
|
||||||
|
// only true if all writes have same ttl setting, otherwise it could bring old
|
||||||
|
// data back.
|
||||||
|
bool purge_ttl_on_expiration; |
||||||
|
}; |
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
extern "C" { |
||||||
|
int RegisterCassandraObjects(ObjectLibrary& library, const std::string& arg); |
||||||
|
} // extern "C"
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
|
} // namespace cassandra
|
||||||
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
@ -0,0 +1,56 @@ |
|||||||
|
// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
|
||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
|
#include <memory> |
||||||
|
|
||||||
|
#include "rocksdb/compaction_filter.h" |
||||||
|
#include "rocksdb/options.h" |
||||||
|
#include "rocksdb/utilities/customizable_util.h" |
||||||
|
#include "rocksdb/utilities/options_type.h" |
||||||
|
#include "utilities/compaction_filters/layered_compaction_filter_base.h" |
||||||
|
#include "utilities/compaction_filters/remove_emptyvalue_compactionfilter.h" |
||||||
|
|
||||||
|
namespace ROCKSDB_NAMESPACE { |
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
static int RegisterBuiltinCompactionFilters(ObjectLibrary& library, |
||||||
|
const std::string& /*arg*/) { |
||||||
|
library.Register<CompactionFilter>( |
||||||
|
RemoveEmptyValueCompactionFilter::kClassName(), |
||||||
|
[](const std::string& /*uri*/, |
||||||
|
std::unique_ptr<CompactionFilter>* /*guard*/, |
||||||
|
std::string* /*errmsg*/) { |
||||||
|
return new RemoveEmptyValueCompactionFilter(); |
||||||
|
}); |
||||||
|
return 1; |
||||||
|
} |
||||||
|
#endif // ROCKSDB_LITE
|
||||||
|
Status CompactionFilter::CreateFromString(const ConfigOptions& config_options, |
||||||
|
const std::string& value, |
||||||
|
const CompactionFilter** result) { |
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
static std::once_flag once; |
||||||
|
std::call_once(once, [&]() { |
||||||
|
RegisterBuiltinCompactionFilters(*(ObjectLibrary::Default().get()), ""); |
||||||
|
}); |
||||||
|
#endif // ROCKSDB_LITE
|
||||||
|
CompactionFilter* filter = const_cast<CompactionFilter*>(*result); |
||||||
|
Status status = LoadStaticObject<CompactionFilter>(config_options, value, |
||||||
|
nullptr, &filter); |
||||||
|
if (status.ok()) { |
||||||
|
*result = const_cast<CompactionFilter*>(filter); |
||||||
|
} |
||||||
|
return status; |
||||||
|
} |
||||||
|
|
||||||
|
Status CompactionFilterFactory::CreateFromString( |
||||||
|
const ConfigOptions& config_options, const std::string& value, |
||||||
|
std::shared_ptr<CompactionFilterFactory>* result) { |
||||||
|
// Currently there are no builtin CompactionFilterFactories.
|
||||||
|
// If any are introduced, they need to be registered here.
|
||||||
|
Status status = LoadSharedObject<CompactionFilterFactory>( |
||||||
|
config_options, value, nullptr, result); |
||||||
|
return status; |
||||||
|
} |
||||||
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
@ -0,0 +1,125 @@ |
|||||||
|
// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
|
||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
|
#include "utilities/merge_operators.h" |
||||||
|
|
||||||
|
#include <memory> |
||||||
|
|
||||||
|
#include "rocksdb/merge_operator.h" |
||||||
|
#include "rocksdb/options.h" |
||||||
|
#include "rocksdb/utilities/customizable_util.h" |
||||||
|
#include "rocksdb/utilities/object_registry.h" |
||||||
|
#include "utilities/merge_operators/bytesxor.h" |
||||||
|
#include "utilities/merge_operators/sortlist.h" |
||||||
|
#include "utilities/merge_operators/string_append/stringappend.h" |
||||||
|
#include "utilities/merge_operators/string_append/stringappend2.h" |
||||||
|
|
||||||
|
namespace ROCKSDB_NAMESPACE { |
||||||
|
static bool LoadMergeOperator(const std::string& id, |
||||||
|
std::shared_ptr<MergeOperator>* result) { |
||||||
|
bool success = true; |
||||||
|
// TODO: Hook the "name" up to the actual Name() of the MergeOperators?
|
||||||
|
// Requires these classes be moved into a header file...
|
||||||
|
if (id == "put" || id == "PutOperator") { |
||||||
|
*result = MergeOperators::CreatePutOperator(); |
||||||
|
} else if (id == "put_v1") { |
||||||
|
*result = MergeOperators::CreateDeprecatedPutOperator(); |
||||||
|
} else if (id == "uint64add" || id == "UInt64AddOperator") { |
||||||
|
*result = MergeOperators::CreateUInt64AddOperator(); |
||||||
|
} else if (id == "max" || id == "MaxOperator") { |
||||||
|
*result = MergeOperators::CreateMaxOperator(); |
||||||
|
#ifdef ROCKSDB_LITE |
||||||
|
// The remainder of the classes are handled by the ObjectRegistry in
|
||||||
|
// non-LITE mode
|
||||||
|
} else if (id == StringAppendOperator::kNickName() || |
||||||
|
id == StringAppendOperator::kClassName()) { |
||||||
|
*result = MergeOperators::CreateStringAppendOperator(); |
||||||
|
} else if (id == StringAppendTESTOperator::kNickName() || |
||||||
|
id == StringAppendTESTOperator::kClassName()) { |
||||||
|
*result = MergeOperators::CreateStringAppendTESTOperator(); |
||||||
|
} else if (id == BytesXOROperator::kNickName() || |
||||||
|
id == BytesXOROperator::kClassName()) { |
||||||
|
*result = MergeOperators::CreateBytesXOROperator(); |
||||||
|
} else if (id == SortList::kNickName() || id == SortList::kClassName()) { |
||||||
|
*result = MergeOperators::CreateSortOperator(); |
||||||
|
#endif // ROCKSDB_LITE
|
||||||
|
} else { |
||||||
|
success = false; |
||||||
|
} |
||||||
|
return success; |
||||||
|
} |
||||||
|
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
static int RegisterBuiltinMergeOperators(ObjectLibrary& library, |
||||||
|
const std::string& /*arg*/) { |
||||||
|
size_t num_types; |
||||||
|
auto AsRegex = [](const std::string& name, const std::string& alt) { |
||||||
|
std::string regex; |
||||||
|
regex.append("(").append(name); |
||||||
|
regex.append("|").append(alt).append(")"); |
||||||
|
return regex; |
||||||
|
}; |
||||||
|
|
||||||
|
library.Register<MergeOperator>( |
||||||
|
AsRegex(StringAppendOperator::kClassName(), |
||||||
|
StringAppendOperator::kNickName()), |
||||||
|
[](const std::string& /*uri*/, std::unique_ptr<MergeOperator>* guard, |
||||||
|
std::string* /*errmsg*/) { |
||||||
|
guard->reset(new StringAppendOperator(",")); |
||||||
|
return guard->get(); |
||||||
|
}); |
||||||
|
library.Register<MergeOperator>( |
||||||
|
AsRegex(StringAppendTESTOperator::kClassName(), |
||||||
|
StringAppendTESTOperator::kNickName()), |
||||||
|
[](const std::string& /*uri*/, std::unique_ptr<MergeOperator>* guard, |
||||||
|
std::string* /*errmsg*/) { |
||||||
|
guard->reset(new StringAppendTESTOperator(",")); |
||||||
|
return guard->get(); |
||||||
|
}); |
||||||
|
library.Register<MergeOperator>( |
||||||
|
AsRegex(SortList::kClassName(), SortList::kNickName()), |
||||||
|
[](const std::string& /*uri*/, std::unique_ptr<MergeOperator>* guard, |
||||||
|
std::string* /*errmsg*/) { |
||||||
|
guard->reset(new SortList()); |
||||||
|
return guard->get(); |
||||||
|
}); |
||||||
|
library.Register<MergeOperator>( |
||||||
|
AsRegex(BytesXOROperator::kClassName(), BytesXOROperator::kNickName()), |
||||||
|
[](const std::string& /*uri*/, std::unique_ptr<MergeOperator>* guard, |
||||||
|
std::string* /*errmsg*/) { |
||||||
|
guard->reset(new BytesXOROperator()); |
||||||
|
return guard->get(); |
||||||
|
}); |
||||||
|
|
||||||
|
return static_cast<int>(library.GetFactoryCount(&num_types)); |
||||||
|
} |
||||||
|
#endif // ROCKSDB_LITE
|
||||||
|
|
||||||
|
Status MergeOperator::CreateFromString(const ConfigOptions& config_options, |
||||||
|
const std::string& value, |
||||||
|
std::shared_ptr<MergeOperator>* result) { |
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
static std::once_flag once; |
||||||
|
std::call_once(once, [&]() { |
||||||
|
RegisterBuiltinMergeOperators(*(ObjectLibrary::Default().get()), ""); |
||||||
|
}); |
||||||
|
#endif // ROCKSDB_LITE
|
||||||
|
return LoadSharedObject<MergeOperator>(config_options, value, |
||||||
|
LoadMergeOperator, result); |
||||||
|
} |
||||||
|
|
||||||
|
std::shared_ptr<MergeOperator> MergeOperators::CreateFromStringId( |
||||||
|
const std::string& id) { |
||||||
|
std::shared_ptr<MergeOperator> result; |
||||||
|
Status s = MergeOperator::CreateFromString(ConfigOptions(), id, &result); |
||||||
|
if (s.ok()) { |
||||||
|
return result; |
||||||
|
} else { |
||||||
|
// Empty or unknown, just return nullptr
|
||||||
|
return nullptr; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
Loading…
Reference in new issue