Remove Lua compaction filter from RocksDB main repo (#4971)
Summary: as title. For people who continue to need Lua compaction filter, you can copy the include/rocksdb/utilities/rocks_lua/lua_compaction_filter.h and utilities/lua/rocks_lua_compaction_filter.cc to your own codebase. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4971 Differential Revision: D14047468 Pulled By: riversand963 fbshipit-source-id: 9ad1a6484a7c94e478f1e108127a3184e4069f70main
parent
a69d4deefb
commit
5af9446ee6
@ -1,189 +0,0 @@ |
||||
// Copyright (c) 2016, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
#pragma once |
||||
|
||||
#if defined(LUA) && !defined(ROCKSDB_LITE) |
||||
// lua headers
|
||||
extern "C" { |
||||
#include <lauxlib.h> |
||||
#include <lua.h> |
||||
#include <lualib.h> |
||||
} |
||||
|
||||
#include <mutex> |
||||
#include <string> |
||||
#include <vector> |
||||
|
||||
#include "rocksdb/compaction_filter.h" |
||||
#include "rocksdb/env.h" |
||||
#include "rocksdb/slice.h" |
||||
#include "rocksdb/utilities/lua/rocks_lua_custom_library.h" |
||||
#include "rocksdb/utilities/lua/rocks_lua_util.h" |
||||
|
||||
namespace rocksdb { |
||||
namespace lua { |
||||
|
||||
struct RocksLuaCompactionFilterOptions { |
||||
// The lua script in string that implements all necessary CompactionFilter
|
||||
// virtual functions. The specified lua_script must implement the following
|
||||
// functions, which are Name and Filter, as described below.
|
||||
//
|
||||
// 0. The Name function simply returns a string representing the name of
|
||||
// the lua script. If there's any erorr in the Name function, an
|
||||
// empty string will be used.
|
||||
// --- Example
|
||||
// function Name()
|
||||
// return "DefaultLuaCompactionFilter"
|
||||
// end
|
||||
//
|
||||
//
|
||||
// 1. The script must contains a function called Filter, which implements
|
||||
// CompactionFilter::Filter() , takes three input arguments, and returns
|
||||
// three values as the following API:
|
||||
//
|
||||
// function Filter(level, key, existing_value)
|
||||
// ...
|
||||
// return is_filtered, is_changed, new_value
|
||||
// end
|
||||
//
|
||||
// Note that if ignore_value is set to true, then Filter should implement
|
||||
// the following API:
|
||||
//
|
||||
// function Filter(level, key)
|
||||
// ...
|
||||
// return is_filtered
|
||||
// end
|
||||
//
|
||||
// If there're any error in the Filter() function, then it will keep
|
||||
// the input key / value pair.
|
||||
//
|
||||
// -- Input
|
||||
// The function must take three arguments (integer, string, string),
|
||||
// which map to "level", "key", and "existing_value" passed from
|
||||
// RocksDB.
|
||||
//
|
||||
// -- Output
|
||||
// The function must return three values (boolean, boolean, string).
|
||||
// - is_filtered: if the first return value is true, then it indicates
|
||||
// the input key / value pair should be filtered.
|
||||
// - is_changed: if the second return value is true, then it indicates
|
||||
// the existing_value needs to be changed, and the resulting value
|
||||
// is stored in the third return value.
|
||||
// - new_value: if the second return value is true, then this third
|
||||
// return value stores the new value of the input key / value pair.
|
||||
//
|
||||
// -- Examples
|
||||
// -- a filter that keeps all key-value pairs
|
||||
// function Filter(level, key, existing_value)
|
||||
// return false, false, ""
|
||||
// end
|
||||
//
|
||||
// -- a filter that keeps all keys and change their values to "Rocks"
|
||||
// function Filter(level, key, existing_value)
|
||||
// return false, true, "Rocks"
|
||||
// end
|
||||
|
||||
std::string lua_script; |
||||
|
||||
// If set to true, then existing_value will not be passed to the Filter
|
||||
// function, and the Filter function only needs to return a single boolean
|
||||
// flag indicating whether to filter out this key or not.
|
||||
//
|
||||
// function Filter(level, key)
|
||||
// ...
|
||||
// return is_filtered
|
||||
// end
|
||||
bool ignore_value = false; |
||||
|
||||
// A boolean flag to determine whether to ignore snapshots.
|
||||
bool ignore_snapshots = true; |
||||
|
||||
// When specified a non-null pointer, the first "error_limit_per_filter"
|
||||
// errors of each CompactionFilter that is lua related will be included
|
||||
// in this log.
|
||||
std::shared_ptr<Logger> error_log; |
||||
|
||||
// The number of errors per CompactionFilter will be printed
|
||||
// to error_log.
|
||||
int error_limit_per_filter = 1; |
||||
|
||||
// A string to luaL_reg array map that allows the Lua CompactionFilter
|
||||
// to use custom C library. The string will be used as the library
|
||||
// name in Lua.
|
||||
std::vector<std::shared_ptr<RocksLuaCustomLibrary>> libraries; |
||||
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
// NOT YET SUPPORTED
|
||||
// The name of the Lua function in "lua_script" that implements
|
||||
// CompactionFilter::FilterMergeOperand(). The function must take
|
||||
// three input arguments (integer, string, string), which map to "level",
|
||||
// "key", and "operand" passed from the RocksDB. In addition, the
|
||||
// function must return a single boolean value, indicating whether
|
||||
// to filter the input key / operand.
|
||||
//
|
||||
// DEFAULT: the default implementation always returns false.
|
||||
// @see CompactionFilter::FilterMergeOperand
|
||||
}; |
||||
|
||||
class RocksLuaCompactionFilterFactory : public CompactionFilterFactory { |
||||
public: |
||||
explicit RocksLuaCompactionFilterFactory( |
||||
const RocksLuaCompactionFilterOptions opt); |
||||
|
||||
virtual ~RocksLuaCompactionFilterFactory() {} |
||||
|
||||
std::unique_ptr<CompactionFilter> CreateCompactionFilter( |
||||
const CompactionFilter::Context& context) override; |
||||
|
||||
// Change the Lua script so that the next compaction after this
|
||||
// function call will use the new Lua script.
|
||||
void SetScript(const std::string& new_script); |
||||
|
||||
// Obtain the current Lua script
|
||||
std::string GetScript(); |
||||
|
||||
const char* Name() const override; |
||||
|
||||
private: |
||||
RocksLuaCompactionFilterOptions opt_; |
||||
std::string name_; |
||||
// A lock to protect "opt_" to make it dynamically changeable.
|
||||
std::mutex opt_mutex_; |
||||
}; |
||||
|
||||
// A wrapper class that invokes Lua script to perform CompactionFilter
|
||||
// functions.
|
||||
class RocksLuaCompactionFilter : public rocksdb::CompactionFilter { |
||||
public: |
||||
explicit RocksLuaCompactionFilter(const RocksLuaCompactionFilterOptions& opt) |
||||
: options_(opt), |
||||
lua_state_wrapper_(opt.lua_script, opt.libraries), |
||||
error_count_(0), |
||||
name_("") {} |
||||
|
||||
virtual bool Filter(int level, const Slice& key, const Slice& existing_value, |
||||
std::string* new_value, |
||||
bool* value_changed) const override; |
||||
// Not yet supported
|
||||
virtual bool FilterMergeOperand(int /*level*/, const Slice& /*key*/, |
||||
const Slice& /*operand*/) const override { |
||||
return false; |
||||
} |
||||
virtual bool IgnoreSnapshots() const override; |
||||
virtual const char* Name() const override; |
||||
|
||||
protected: |
||||
void LogLuaError(const char* format, ...) const; |
||||
|
||||
RocksLuaCompactionFilterOptions options_; |
||||
LuaStateWrapper lua_state_wrapper_; |
||||
mutable int error_count_; |
||||
mutable std::string name_; |
||||
}; |
||||
|
||||
} // namespace lua
|
||||
} // namespace rocksdb
|
||||
#endif // defined(LUA) && !defined(ROCKSDB_LITE)
|
@ -1,242 +0,0 @@ |
||||
// Copyright (c) 2016, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
#if defined(LUA) && !defined(ROCKSDB_LITE) |
||||
#include "rocksdb/utilities/lua/rocks_lua_compaction_filter.h" |
||||
|
||||
extern "C" { |
||||
#include <luaconf.h> |
||||
} |
||||
|
||||
#include "rocksdb/compaction_filter.h" |
||||
|
||||
namespace rocksdb { |
||||
namespace lua { |
||||
|
||||
const std::string kFilterFunctionName = "Filter"; |
||||
const std::string kNameFunctionName = "Name"; |
||||
|
||||
void RocksLuaCompactionFilter::LogLuaError(const char* format, ...) const { |
||||
if (options_.error_log.get() != nullptr && |
||||
error_count_ < options_.error_limit_per_filter) { |
||||
error_count_++; |
||||
|
||||
va_list ap; |
||||
va_start(ap, format); |
||||
options_.error_log->Logv(InfoLogLevel::ERROR_LEVEL, format, ap); |
||||
va_end(ap); |
||||
} |
||||
} |
||||
|
||||
bool RocksLuaCompactionFilter::Filter(int level, const Slice& key, |
||||
const Slice& existing_value, |
||||
std::string* new_value, |
||||
bool* value_changed) const { |
||||
auto* lua_state = lua_state_wrapper_.GetLuaState(); |
||||
// push the right function into the lua stack
|
||||
lua_getglobal(lua_state, kFilterFunctionName.c_str()); |
||||
|
||||
int error_no = 0; |
||||
int num_input_values; |
||||
int num_return_values; |
||||
if (options_.ignore_value == false) { |
||||
// push input arguments into the lua stack
|
||||
lua_pushnumber(lua_state, level); |
||||
lua_pushlstring(lua_state, key.data(), key.size()); |
||||
lua_pushlstring(lua_state, existing_value.data(), existing_value.size()); |
||||
num_input_values = 3; |
||||
num_return_values = 3; |
||||
} else { |
||||
// If ignore_value is set to true, then we only put two arguments
|
||||
// and expect one return value
|
||||
lua_pushnumber(lua_state, level); |
||||
lua_pushlstring(lua_state, key.data(), key.size()); |
||||
num_input_values = 2; |
||||
num_return_values = 1; |
||||
} |
||||
|
||||
// perform the lua call
|
||||
if ((error_no = |
||||
lua_pcall(lua_state, num_input_values, num_return_values, 0)) != 0) { |
||||
LogLuaError("[Lua] Error(%d) in Filter function --- %s", error_no, |
||||
lua_tostring(lua_state, -1)); |
||||
// pops out the lua error from stack
|
||||
lua_pop(lua_state, 1); |
||||
return false; |
||||
} |
||||
|
||||
// As lua_pcall went successfully, it can be guaranteed that the top
|
||||
// three elements in the Lua stack are the three returned values.
|
||||
|
||||
bool has_error = false; |
||||
const int kIndexIsFiltered = -num_return_values; |
||||
const int kIndexValueChanged = -num_return_values + 1; |
||||
const int kIndexNewValue = -num_return_values + 2; |
||||
|
||||
// check the types of three return values
|
||||
// is_filtered
|
||||
if (!lua_isboolean(lua_state, kIndexIsFiltered)) { |
||||
LogLuaError( |
||||
"[Lua] Error in Filter function -- " |
||||
"1st return value (is_filtered) is not a boolean " |
||||
"while a boolean is expected."); |
||||
has_error = true; |
||||
} |
||||
|
||||
if (options_.ignore_value == false) { |
||||
// value_changed
|
||||
if (!lua_isboolean(lua_state, kIndexValueChanged)) { |
||||
LogLuaError( |
||||
"[Lua] Error in Filter function -- " |
||||
"2nd return value (value_changed) is not a boolean " |
||||
"while a boolean is expected."); |
||||
has_error = true; |
||||
} |
||||
// new_value
|
||||
if (!lua_isstring(lua_state, kIndexNewValue)) { |
||||
LogLuaError( |
||||
"[Lua] Error in Filter function -- " |
||||
"3rd return value (new_value) is not a string " |
||||
"while a string is expected."); |
||||
has_error = true; |
||||
} |
||||
} |
||||
|
||||
if (has_error) { |
||||
lua_pop(lua_state, num_return_values); |
||||
return false; |
||||
} |
||||
|
||||
// Fetch the return values
|
||||
bool is_filtered = false; |
||||
if (!has_error) { |
||||
is_filtered = lua_toboolean(lua_state, kIndexIsFiltered); |
||||
if (options_.ignore_value == false) { |
||||
*value_changed = lua_toboolean(lua_state, kIndexValueChanged); |
||||
if (*value_changed) { |
||||
const char* new_value_buf = lua_tostring(lua_state, kIndexNewValue); |
||||
const size_t new_value_size = lua_strlen(lua_state, kIndexNewValue); |
||||
// Note that any string that lua_tostring returns always has a zero at
|
||||
// its end, bu/t it can have other zeros inside it
|
||||
assert(new_value_buf[new_value_size] == '\0'); |
||||
assert(strlen(new_value_buf) <= new_value_size); |
||||
new_value->assign(new_value_buf, new_value_size); |
||||
} |
||||
} else { |
||||
*value_changed = false; |
||||
} |
||||
} |
||||
// pops the three return values.
|
||||
lua_pop(lua_state, num_return_values); |
||||
return is_filtered; |
||||
} |
||||
|
||||
const char* RocksLuaCompactionFilter::Name() const { |
||||
if (name_ != "") { |
||||
return name_.c_str(); |
||||
} |
||||
auto* lua_state = lua_state_wrapper_.GetLuaState(); |
||||
// push the right function into the lua stack
|
||||
lua_getglobal(lua_state, kNameFunctionName.c_str()); |
||||
|
||||
// perform the call (0 arguments, 1 result)
|
||||
int error_no; |
||||
if ((error_no = lua_pcall(lua_state, 0, 1, 0)) != 0) { |
||||
LogLuaError("[Lua] Error(%d) in Name function --- %s", error_no, |
||||
lua_tostring(lua_state, -1)); |
||||
// pops out the lua error from stack
|
||||
lua_pop(lua_state, 1); |
||||
return name_.c_str(); |
||||
} |
||||
|
||||
// check the return value
|
||||
if (!lua_isstring(lua_state, -1)) { |
||||
LogLuaError( |
||||
"[Lua] Error in Name function -- " |
||||
"return value is not a string while string is expected"); |
||||
} else { |
||||
const char* name_buf = lua_tostring(lua_state, -1); |
||||
const size_t name_size __attribute__((__unused__)) = lua_strlen(lua_state, -1); |
||||
assert(name_buf[name_size] == '\0'); |
||||
assert(strlen(name_buf) <= name_size); |
||||
name_ = name_buf; |
||||
} |
||||
lua_pop(lua_state, 1); |
||||
return name_.c_str(); |
||||
} |
||||
|
||||
/* Not yet supported
|
||||
bool RocksLuaCompactionFilter::FilterMergeOperand( |
||||
int level, const Slice& key, const Slice& operand) const { |
||||
auto* lua_state = lua_state_wrapper_.GetLuaState(); |
||||
// push the right function into the lua stack
|
||||
lua_getglobal(lua_state, "FilterMergeOperand"); |
||||
|
||||
// push input arguments into the lua stack
|
||||
lua_pushnumber(lua_state, level); |
||||
lua_pushlstring(lua_state, key.data(), key.size()); |
||||
lua_pushlstring(lua_state, operand.data(), operand.size()); |
||||
|
||||
// perform the call (3 arguments, 1 result)
|
||||
int error_no; |
||||
if ((error_no = lua_pcall(lua_state, 3, 1, 0)) != 0) { |
||||
LogLuaError("[Lua] Error(%d) in FilterMergeOperand function --- %s", |
||||
error_no, lua_tostring(lua_state, -1)); |
||||
// pops out the lua error from stack
|
||||
lua_pop(lua_state, 1); |
||||
return false; |
||||
} |
||||
|
||||
bool is_filtered = false; |
||||
// check the return value
|
||||
if (!lua_isboolean(lua_state, -1)) { |
||||
LogLuaError("[Lua] Error in FilterMergeOperand function -- " |
||||
"return value is not a boolean while boolean is expected"); |
||||
} else { |
||||
is_filtered = lua_toboolean(lua_state, -1); |
||||
} |
||||
|
||||
lua_pop(lua_state, 1); |
||||
|
||||
return is_filtered; |
||||
} |
||||
*/ |
||||
|
||||
bool RocksLuaCompactionFilter::IgnoreSnapshots() const { |
||||
return options_.ignore_snapshots; |
||||
} |
||||
|
||||
RocksLuaCompactionFilterFactory::RocksLuaCompactionFilterFactory( |
||||
const RocksLuaCompactionFilterOptions opt) |
||||
: opt_(opt) { |
||||
auto filter = CreateCompactionFilter(CompactionFilter::Context()); |
||||
name_ = std::string("RocksLuaCompactionFilterFactory::") + |
||||
std::string(filter->Name()); |
||||
} |
||||
|
||||
std::unique_ptr<CompactionFilter> |
||||
RocksLuaCompactionFilterFactory::CreateCompactionFilter( |
||||
const CompactionFilter::Context& /*context*/) { |
||||
std::lock_guard<std::mutex> lock(opt_mutex_); |
||||
return std::unique_ptr<CompactionFilter>(new RocksLuaCompactionFilter(opt_)); |
||||
} |
||||
|
||||
std::string RocksLuaCompactionFilterFactory::GetScript() { |
||||
std::lock_guard<std::mutex> lock(opt_mutex_); |
||||
return opt_.lua_script; |
||||
} |
||||
|
||||
void RocksLuaCompactionFilterFactory::SetScript(const std::string& new_script) { |
||||
std::lock_guard<std::mutex> lock(opt_mutex_); |
||||
opt_.lua_script = new_script; |
||||
} |
||||
|
||||
const char* RocksLuaCompactionFilterFactory::Name() const { |
||||
return name_.c_str(); |
||||
} |
||||
|
||||
} // namespace lua
|
||||
} // namespace rocksdb
|
||||
#endif // defined(LUA) && !defined(ROCKSDB_LITE)
|
@ -1,498 +0,0 @@ |
||||
// Copyright (c) 2016, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
#include <stdio.h> |
||||
|
||||
#if !defined(ROCKSDB_LITE) |
||||
|
||||
#if defined(LUA) |
||||
|
||||
#include <string> |
||||
|
||||
#include "db/db_test_util.h" |
||||
#include "port/stack_trace.h" |
||||
#include "rocksdb/compaction_filter.h" |
||||
#include "rocksdb/db.h" |
||||
#include "rocksdb/utilities/lua/rocks_lua_compaction_filter.h" |
||||
#include "util/testharness.h" |
||||
|
||||
namespace rocksdb { |
||||
|
||||
class StopOnErrorLogger : public Logger { |
||||
public: |
||||
using Logger::Logv; |
||||
virtual void Logv(const char* format, va_list ap) override { |
||||
vfprintf(stderr, format, ap); |
||||
fprintf(stderr, "\n"); |
||||
FAIL(); |
||||
} |
||||
}; |
||||
|
||||
|
||||
class RocksLuaTest : public testing::Test { |
||||
public: |
||||
RocksLuaTest() : rnd_(301) { |
||||
temp_dir_ = test::TmpDir(Env::Default()); |
||||
db_ = nullptr; |
||||
} |
||||
|
||||
std::string RandomString(int len) { |
||||
std::string res; |
||||
for (int i = 0; i < len; ++i) { |
||||
res += rnd_.Uniform(26) + 'a'; |
||||
} |
||||
return res; |
||||
} |
||||
|
||||
void CreateDBWithLuaCompactionFilter( |
||||
const lua::RocksLuaCompactionFilterOptions& lua_opt, |
||||
const std::string& db_path, |
||||
std::unordered_map<std::string, std::string>* kvs, |
||||
const int kNumFlushes = 5, |
||||
std::shared_ptr<rocksdb::lua::RocksLuaCompactionFilterFactory>* |
||||
output_factory = nullptr) { |
||||
const int kKeySize = 10; |
||||
const int kValueSize = 50; |
||||
const int kKeysPerFlush = 2; |
||||
auto factory = |
||||
std::make_shared<rocksdb::lua::RocksLuaCompactionFilterFactory>( |
||||
lua_opt); |
||||
if (output_factory != nullptr) { |
||||
*output_factory = factory; |
||||
} |
||||
|
||||
options_ = Options(); |
||||
options_.create_if_missing = true; |
||||
options_.compaction_filter_factory = factory; |
||||
options_.disable_auto_compactions = true; |
||||
options_.max_bytes_for_level_base = |
||||
(kKeySize + kValueSize) * kKeysPerFlush * 2; |
||||
options_.max_bytes_for_level_multiplier = 2; |
||||
options_.target_file_size_base = (kKeySize + kValueSize) * kKeysPerFlush; |
||||
options_.level0_file_num_compaction_trigger = 2; |
||||
DestroyDB(db_path, options_); |
||||
ASSERT_OK(DB::Open(options_, db_path, &db_)); |
||||
|
||||
for (int f = 0; f < kNumFlushes; ++f) { |
||||
for (int i = 0; i < kKeysPerFlush; ++i) { |
||||
std::string key = RandomString(kKeySize); |
||||
std::string value = RandomString(kValueSize); |
||||
kvs->insert({key, value}); |
||||
ASSERT_OK(db_->Put(WriteOptions(), key, value)); |
||||
} |
||||
db_->Flush(FlushOptions()); |
||||
} |
||||
} |
||||
|
||||
~RocksLuaTest() { |
||||
if (db_) { |
||||
delete db_; |
||||
} |
||||
} |
||||
std::string temp_dir_; |
||||
DB* db_; |
||||
Random rnd_; |
||||
Options options_; |
||||
}; |
||||
|
||||
TEST_F(RocksLuaTest, Default) { |
||||
// If nothing is set in the LuaCompactionFilterOptions, then
|
||||
// RocksDB will keep all the key / value pairs, but it will also
|
||||
// print our error log indicating failure.
|
||||
std::string db_path = test::PerThreadDBPath(temp_dir_, "rocks_lua_test"); |
||||
|
||||
lua::RocksLuaCompactionFilterOptions lua_opt; |
||||
|
||||
std::unordered_map<std::string, std::string> kvs; |
||||
CreateDBWithLuaCompactionFilter(lua_opt, db_path, &kvs); |
||||
|
||||
for (auto const& entry : kvs) { |
||||
std::string value; |
||||
ASSERT_OK(db_->Get(ReadOptions(), entry.first, &value)); |
||||
ASSERT_EQ(value, entry.second); |
||||
} |
||||
} |
||||
|
||||
TEST_F(RocksLuaTest, KeepsAll) { |
||||
std::string db_path = test::PerThreadDBPath(temp_dir_, "rocks_lua_test"); |
||||
|
||||
lua::RocksLuaCompactionFilterOptions lua_opt; |
||||
lua_opt.error_log = std::make_shared<StopOnErrorLogger>(); |
||||
// keeps all the key value pairs
|
||||
lua_opt.lua_script = |
||||
"function Filter(level, key, existing_value)\n" |
||||
" return false, false, \"\"\n" |
||||
"end\n" |
||||
"\n" |
||||
"function FilterMergeOperand(level, key, operand)\n" |
||||
" return false\n" |
||||
"end\n" |
||||
"function Name()\n" |
||||
" return \"KeepsAll\"\n" |
||||
"end\n" |
||||
"\n"; |
||||
|
||||
std::unordered_map<std::string, std::string> kvs; |
||||
CreateDBWithLuaCompactionFilter(lua_opt, db_path, &kvs); |
||||
|
||||
for (auto const& entry : kvs) { |
||||
std::string value; |
||||
ASSERT_OK(db_->Get(ReadOptions(), entry.first, &value)); |
||||
ASSERT_EQ(value, entry.second); |
||||
} |
||||
} |
||||
|
||||
TEST_F(RocksLuaTest, GetName) { |
||||
std::string db_path = test::PerThreadDBPath(temp_dir_, "rocks_lua_test"); |
||||
|
||||
lua::RocksLuaCompactionFilterOptions lua_opt; |
||||
lua_opt.error_log = std::make_shared<StopOnErrorLogger>(); |
||||
const std::string kScriptName = "SimpleLuaCompactionFilter"; |
||||
lua_opt.lua_script = |
||||
std::string( |
||||
"function Filter(level, key, existing_value)\n" |
||||
" return false, false, \"\"\n" |
||||
"end\n" |
||||
"\n" |
||||
"function FilterMergeOperand(level, key, operand)\n" |
||||
" return false\n" |
||||
"end\n" |
||||
"function Name()\n" |
||||
" return \"") + kScriptName + "\"\n" |
||||
"end\n" |
||||
"\n"; |
||||
|
||||
std::shared_ptr<CompactionFilterFactory> factory = |
||||
std::make_shared<lua::RocksLuaCompactionFilterFactory>(lua_opt); |
||||
std::string factory_name(factory->Name()); |
||||
ASSERT_NE(factory_name.find(kScriptName), std::string::npos); |
||||
} |
||||
|
||||
TEST_F(RocksLuaTest, RemovesAll) { |
||||
std::string db_path = test::PerThreadDBPath(temp_dir_, "rocks_lua_test"); |
||||
|
||||
lua::RocksLuaCompactionFilterOptions lua_opt; |
||||
lua_opt.error_log = std::make_shared<StopOnErrorLogger>(); |
||||
// removes all the key value pairs
|
||||
lua_opt.lua_script = |
||||
"function Filter(level, key, existing_value)\n" |
||||
" return true, false, \"\"\n" |
||||
"end\n" |
||||
"\n" |
||||
"function FilterMergeOperand(level, key, operand)\n" |
||||
" return false\n" |
||||
"end\n" |
||||
"function Name()\n" |
||||
" return \"RemovesAll\"\n" |
||||
"end\n" |
||||
"\n"; |
||||
|
||||
std::unordered_map<std::string, std::string> kvs; |
||||
CreateDBWithLuaCompactionFilter(lua_opt, db_path, &kvs); |
||||
// Issue full compaction and expect nothing is in the DB.
|
||||
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); |
||||
|
||||
for (auto const& entry : kvs) { |
||||
std::string value; |
||||
auto s = db_->Get(ReadOptions(), entry.first, &value); |
||||
ASSERT_TRUE(s.IsNotFound()); |
||||
} |
||||
} |
||||
|
||||
TEST_F(RocksLuaTest, FilterByKey) { |
||||
std::string db_path = test::PerThreadDBPath(temp_dir_, "rocks_lua_test"); |
||||
|
||||
lua::RocksLuaCompactionFilterOptions lua_opt; |
||||
lua_opt.error_log = std::make_shared<StopOnErrorLogger>(); |
||||
// removes all keys whose initial is less than 'r'
|
||||
lua_opt.lua_script = |
||||
"function Filter(level, key, existing_value)\n" |
||||
" if key:sub(1,1) < 'r' then\n" |
||||
" return true, false, \"\"\n" |
||||
" end\n" |
||||
" return false, false, \"\"\n" |
||||
"end\n" |
||||
"\n" |
||||
"function FilterMergeOperand(level, key, operand)\n" |
||||
" return false\n" |
||||
"end\n" |
||||
"function Name()\n" |
||||
" return \"KeepsAll\"\n" |
||||
"end\n"; |
||||
|
||||
std::unordered_map<std::string, std::string> kvs; |
||||
CreateDBWithLuaCompactionFilter(lua_opt, db_path, &kvs); |
||||
// Issue full compaction and expect nothing is in the DB.
|
||||
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); |
||||
|
||||
for (auto const& entry : kvs) { |
||||
std::string value; |
||||
auto s = db_->Get(ReadOptions(), entry.first, &value); |
||||
if (entry.first[0] < 'r') { |
||||
ASSERT_TRUE(s.IsNotFound()); |
||||
} else { |
||||
ASSERT_TRUE(s.ok()); |
||||
ASSERT_TRUE(value == entry.second); |
||||
} |
||||
} |
||||
} |
||||
|
||||
TEST_F(RocksLuaTest, FilterByValue) { |
||||
std::string db_path = test::PerThreadDBPath(temp_dir_, "rocks_lua_test"); |
||||
|
||||
lua::RocksLuaCompactionFilterOptions lua_opt; |
||||
lua_opt.error_log = std::make_shared<StopOnErrorLogger>(); |
||||
// removes all values whose initial is less than 'r'
|
||||
lua_opt.lua_script = |
||||
"function Filter(level, key, existing_value)\n" |
||||
" if existing_value:sub(1,1) < 'r' then\n" |
||||
" return true, false, \"\"\n" |
||||
" end\n" |
||||
" return false, false, \"\"\n" |
||||
"end\n" |
||||
"\n" |
||||
"function FilterMergeOperand(level, key, operand)\n" |
||||
" return false\n" |
||||
"end\n" |
||||
"function Name()\n" |
||||
" return \"FilterByValue\"\n" |
||||
"end\n" |
||||
"\n"; |
||||
|
||||
std::unordered_map<std::string, std::string> kvs; |
||||
CreateDBWithLuaCompactionFilter(lua_opt, db_path, &kvs); |
||||
// Issue full compaction and expect nothing is in the DB.
|
||||
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); |
||||
|
||||
for (auto const& entry : kvs) { |
||||
std::string value; |
||||
auto s = db_->Get(ReadOptions(), entry.first, &value); |
||||
if (entry.second[0] < 'r') { |
||||
ASSERT_TRUE(s.IsNotFound()); |
||||
} else { |
||||
ASSERT_TRUE(s.ok()); |
||||
ASSERT_EQ(value, entry.second); |
||||
} |
||||
} |
||||
} |
||||
|
||||
TEST_F(RocksLuaTest, ChangeValue) { |
||||
std::string db_path = test::PerThreadDBPath(temp_dir_, "rocks_lua_test"); |
||||
|
||||
lua::RocksLuaCompactionFilterOptions lua_opt; |
||||
lua_opt.error_log = std::make_shared<StopOnErrorLogger>(); |
||||
// Replace all values by their reversed key
|
||||
lua_opt.lua_script = |
||||
"function Filter(level, key, existing_value)\n" |
||||
" return false, true, key:reverse()\n" |
||||
"end\n" |
||||
"\n" |
||||
"function FilterMergeOperand(level, key, operand)\n" |
||||
" return false\n" |
||||
"end\n" |
||||
"function Name()\n" |
||||
" return \"ChangeValue\"\n" |
||||
"end\n" |
||||
"\n"; |
||||
|
||||
std::unordered_map<std::string, std::string> kvs; |
||||
CreateDBWithLuaCompactionFilter(lua_opt, db_path, &kvs); |
||||
// Issue full compaction and expect nothing is in the DB.
|
||||
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); |
||||
|
||||
for (auto const& entry : kvs) { |
||||
std::string value; |
||||
ASSERT_OK(db_->Get(ReadOptions(), entry.first, &value)); |
||||
std::string new_value = entry.first; |
||||
std::reverse(new_value.begin(), new_value.end()); |
||||
ASSERT_EQ(value, new_value); |
||||
} |
||||
} |
||||
|
||||
TEST_F(RocksLuaTest, ConditionallyChangeAndFilterValue) { |
||||
std::string db_path = test::PerThreadDBPath(temp_dir_, "rocks_lua_test"); |
||||
|
||||
lua::RocksLuaCompactionFilterOptions lua_opt; |
||||
lua_opt.error_log = std::make_shared<StopOnErrorLogger>(); |
||||
// Performs the following logic:
|
||||
// If key[0] < 'h' --> replace value by reverse key
|
||||
// If key[0] >= 'r' --> keep the original key value
|
||||
// Otherwise, filter the key value
|
||||
lua_opt.lua_script = |
||||
"function Filter(level, key, existing_value)\n" |
||||
" if key:sub(1,1) < 'h' then\n" |
||||
" return false, true, key:reverse()\n" |
||||
" elseif key:sub(1,1) < 'r' then\n" |
||||
" return true, false, \"\"\n" |
||||
" end\n" |
||||
" return false, false, \"\"\n" |
||||
"end\n" |
||||
"function Name()\n" |
||||
" return \"ConditionallyChangeAndFilterValue\"\n" |
||||
"end\n" |
||||
"\n"; |
||||
|
||||
std::unordered_map<std::string, std::string> kvs; |
||||
CreateDBWithLuaCompactionFilter(lua_opt, db_path, &kvs); |
||||
// Issue full compaction and expect nothing is in the DB.
|
||||
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); |
||||
|
||||
for (auto const& entry : kvs) { |
||||
std::string value; |
||||
auto s = db_->Get(ReadOptions(), entry.first, &value); |
||||
if (entry.first[0] < 'h') { |
||||
ASSERT_TRUE(s.ok()); |
||||
std::string new_value = entry.first; |
||||
std::reverse(new_value.begin(), new_value.end()); |
||||
ASSERT_EQ(value, new_value); |
||||
} else if (entry.first[0] < 'r') { |
||||
ASSERT_TRUE(s.IsNotFound()); |
||||
} else { |
||||
ASSERT_TRUE(s.ok()); |
||||
ASSERT_EQ(value, entry.second); |
||||
} |
||||
} |
||||
} |
||||
|
||||
TEST_F(RocksLuaTest, DynamicChangeScript) { |
||||
std::string db_path = test::PerThreadDBPath(temp_dir_, "rocks_lua_test"); |
||||
|
||||
lua::RocksLuaCompactionFilterOptions lua_opt; |
||||
lua_opt.error_log = std::make_shared<StopOnErrorLogger>(); |
||||
// keeps all the key value pairs
|
||||
lua_opt.lua_script = |
||||
"function Filter(level, key, existing_value)\n" |
||||
" return false, false, \"\"\n" |
||||
"end\n" |
||||
"\n" |
||||
"function FilterMergeOperand(level, key, operand)\n" |
||||
" return false\n" |
||||
"end\n" |
||||
"function Name()\n" |
||||
" return \"KeepsAll\"\n" |
||||
"end\n" |
||||
"\n"; |
||||
|
||||
std::unordered_map<std::string, std::string> kvs; |
||||
std::shared_ptr<rocksdb::lua::RocksLuaCompactionFilterFactory> factory; |
||||
CreateDBWithLuaCompactionFilter(lua_opt, db_path, &kvs, 30, &factory); |
||||
uint64_t count = 0; |
||||
ASSERT_TRUE(db_->GetIntProperty( |
||||
rocksdb::DB::Properties::kNumEntriesActiveMemTable, &count)); |
||||
ASSERT_EQ(count, 0); |
||||
ASSERT_TRUE(db_->GetIntProperty( |
||||
rocksdb::DB::Properties::kNumEntriesImmMemTables, &count)); |
||||
ASSERT_EQ(count, 0); |
||||
|
||||
CompactRangeOptions cr_opt; |
||||
cr_opt.bottommost_level_compaction = |
||||
rocksdb::BottommostLevelCompaction::kForce; |
||||
|
||||
// Issue full compaction and expect everything is in the DB.
|
||||
ASSERT_OK(db_->CompactRange(cr_opt, nullptr, nullptr)); |
||||
|
||||
for (auto const& entry : kvs) { |
||||
std::string value; |
||||
ASSERT_OK(db_->Get(ReadOptions(), entry.first, &value)); |
||||
ASSERT_EQ(value, entry.second); |
||||
} |
||||
|
||||
// change the lua script to removes all the key value pairs
|
||||
factory->SetScript( |
||||
"function Filter(level, key, existing_value)\n" |
||||
" return true, false, \"\"\n" |
||||
"end\n" |
||||
"\n" |
||||
"function FilterMergeOperand(level, key, operand)\n" |
||||
" return false\n" |
||||
"end\n" |
||||
"function Name()\n" |
||||
" return \"RemovesAll\"\n" |
||||
"end\n" |
||||
"\n"); |
||||
{ |
||||
std::string key = "another-key"; |
||||
std::string value = "another-value"; |
||||
kvs.insert({key, value}); |
||||
ASSERT_OK(db_->Put(WriteOptions(), key, value)); |
||||
db_->Flush(FlushOptions()); |
||||
} |
||||
|
||||
cr_opt.change_level = true; |
||||
cr_opt.target_level = 5; |
||||
// Issue full compaction and expect nothing is in the DB.
|
||||
ASSERT_OK(db_->CompactRange(cr_opt, nullptr, nullptr)); |
||||
|
||||
for (auto const& entry : kvs) { |
||||
std::string value; |
||||
auto s = db_->Get(ReadOptions(), entry.first, &value); |
||||
ASSERT_TRUE(s.IsNotFound()); |
||||
} |
||||
} |
||||
|
||||
TEST_F(RocksLuaTest, LuaConditionalTypeError) { |
||||
std::string db_path = test::PerThreadDBPath(temp_dir_, "rocks_lua_test"); |
||||
|
||||
lua::RocksLuaCompactionFilterOptions lua_opt; |
||||
// Filter() error when input key's initial >= 'r'
|
||||
lua_opt.lua_script = |
||||
"function Filter(level, key, existing_value)\n" |
||||
" if existing_value:sub(1,1) >= 'r' then\n" |
||||
" return true, 2, \"\" -- incorrect type of 2nd return value\n" |
||||
" end\n" |
||||
" return true, false, \"\"\n" |
||||
"end\n" |
||||
"\n" |
||||
"function FilterMergeOperand(level, key, operand)\n" |
||||
" return false\n" |
||||
"end\n" |
||||
"function Name()\n" |
||||
" return \"BuggyCode\"\n" |
||||
"end\n" |
||||
"\n"; |
||||
|
||||
std::unordered_map<std::string, std::string> kvs; |
||||
// Create DB with 10 files
|
||||
CreateDBWithLuaCompactionFilter(lua_opt, db_path, &kvs, 10); |
||||
|
||||
// Issue full compaction and expect all keys which initial is < 'r'
|
||||
// will be deleted as we keep the key value when we hit an error.
|
||||
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); |
||||
|
||||
for (auto const& entry : kvs) { |
||||
std::string value; |
||||
auto s = db_->Get(ReadOptions(), entry.first, &value); |
||||
if (entry.second[0] < 'r') { |
||||
ASSERT_TRUE(s.IsNotFound()); |
||||
} else { |
||||
ASSERT_TRUE(s.ok()); |
||||
ASSERT_EQ(value, entry.second); |
||||
} |
||||
} |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) { |
||||
rocksdb::port::InstallStackTraceHandler(); |
||||
::testing::InitGoogleTest(&argc, argv); |
||||
return RUN_ALL_TESTS(); |
||||
} |
||||
|
||||
#else |
||||
|
||||
int main(int /*argc*/, char** /*argv*/) { |
||||
printf("LUA_PATH is not set. Ignoring the test.\n"); |
||||
} |
||||
|
||||
#endif // defined(LUA)
|
||||
|
||||
#else |
||||
|
||||
int main(int /*argc*/, char** /*argv*/) { |
||||
printf("Lua is not supported in RocksDBLite. Ignoring the test.\n"); |
||||
} |
||||
|
||||
#endif // !defined(ROCKSDB_LITE)
|
Loading…
Reference in new issue