Summary: Major changes in this PR: * Implement CassandraCompactionFilter to remove expired columns and rows (if all column expired) * Move cassandra related code from utilities/merge_operators/cassandra to utilities/cassandra/* * Switch to use shared_ptr<> from uniqu_ptr for Column membership management in RowValue. Since columns do have multiple owners in Merge and GC process, use shared_ptr helps make RowValue immutable. * Rename cassandra_merge_test to cassandra_functional_test and add two TTL compaction related tests there. Closes https://github.com/facebook/rocksdb/pull/2588 Differential Revision: D5430010 Pulled By: wpc fbshipit-source-id: 9566c21e06de17491d486a68c70f52d501f27687main
Pengchao Wang
7 years ago
committed by
Facebook Github Bot
parent
63163a8c6e
commit
534c255c7a
@ -0,0 +1,22 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
#include <jni.h> |
||||
|
||||
#include "include/org_rocksdb_CassandraCompactionFilter.h" |
||||
#include "utilities/cassandra/cassandra_compaction_filter.h" |
||||
|
||||
/*
|
||||
* Class: org_rocksdb_CassandraCompactionFilter |
||||
* Method: createNewCassandraCompactionFilter0 |
||||
* Signature: ()J |
||||
*/ |
||||
jlong Java_org_rocksdb_CassandraCompactionFilter_createNewCassandraCompactionFilter0( |
||||
JNIEnv* env, jclass jcls, jboolean purge_ttl_on_expiration) { |
||||
auto* compaction_filter = |
||||
new rocksdb::cassandra::CassandraCompactionFilter(purge_ttl_on_expiration); |
||||
// set the native handle to our native compaction filter
|
||||
return reinterpret_cast<jlong>(compaction_filter); |
||||
} |
||||
@ -0,0 +1,18 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
package org.rocksdb; |
||||
|
||||
/** |
||||
* Just a Java wrapper around CassandraCompactionFilter implemented in C++ |
||||
*/ |
||||
public class CassandraCompactionFilter |
||||
extends AbstractCompactionFilter<Slice> { |
||||
public CassandraCompactionFilter(boolean purgeTtlOnExpiration) { |
||||
super(createNewCassandraCompactionFilter0(purgeTtlOnExpiration)); |
||||
} |
||||
|
||||
private native static long createNewCassandraCompactionFilter0(boolean purgeTtlOnExpiration); |
||||
} |
||||
@ -0,0 +1,47 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
#include "utilities/cassandra/cassandra_compaction_filter.h" |
||||
#include <string> |
||||
#include "rocksdb/slice.h" |
||||
#include "utilities/cassandra/format.h" |
||||
|
||||
|
||||
namespace rocksdb { |
||||
namespace cassandra { |
||||
|
||||
const char* CassandraCompactionFilter::Name() const { |
||||
return "CassandraCompactionFilter"; |
||||
} |
||||
|
||||
CompactionFilter::Decision CassandraCompactionFilter::FilterV2( |
||||
int level, |
||||
const Slice& key, |
||||
ValueType value_type, |
||||
const Slice& existing_value, |
||||
std::string* new_value, |
||||
std::string* skip_until) const { |
||||
|
||||
bool value_changed = false; |
||||
RowValue row_value = RowValue::Deserialize( |
||||
existing_value.data(), existing_value.size()); |
||||
RowValue compacted = purge_ttl_on_expiration_ ? |
||||
row_value.PurgeTtl(&value_changed) : |
||||
row_value.ExpireTtl(&value_changed); |
||||
|
||||
if(compacted.Empty()) { |
||||
return Decision::kRemove; |
||||
} |
||||
|
||||
if (value_changed) { |
||||
compacted.Serialize(new_value); |
||||
return Decision::kChangeValue; |
||||
} |
||||
|
||||
return Decision::kKeep; |
||||
} |
||||
|
||||
} // namespace cassandra
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,39 @@ |
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
#pragma once |
||||
#include <string> |
||||
#include "rocksdb/compaction_filter.h" |
||||
#include "rocksdb/slice.h" |
||||
|
||||
namespace rocksdb { |
||||
namespace cassandra { |
||||
|
||||
/**
|
||||
* Compaction filter for removing expired Cassandra data with ttl. |
||||
* If option `purge_ttl_on_expiration` is set to true, expired data |
||||
* will be directly purged. Otherwise expired data will be converted |
||||
* tombstones first, then be eventally removed after gc grace period.
|
||||
* `purge_ttl_on_expiration` should only be on in the case all the
|
||||
* writes have same ttl setting, otherwise it could bring old data back. |
||||
*/ |
||||
class CassandraCompactionFilter : public CompactionFilter { |
||||
public: |
||||
explicit CassandraCompactionFilter(bool purge_ttl_on_expiration) |
||||
: purge_ttl_on_expiration_(purge_ttl_on_expiration) {} |
||||
|
||||
const char* Name() const override; |
||||
virtual Decision FilterV2(int level, |
||||
const Slice& key, |
||||
ValueType value_type, |
||||
const Slice& existing_value, |
||||
std::string* new_value, |
||||
std::string* skip_until) const override; |
||||
|
||||
private: |
||||
bool purge_ttl_on_expiration_; |
||||
}; |
||||
} // namespace cassandra
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,251 @@ |
||||
// Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
#include <iostream> |
||||
#include "rocksdb/db.h" |
||||
#include "db/db_impl.h" |
||||
#include "rocksdb/merge_operator.h" |
||||
#include "rocksdb/utilities/db_ttl.h" |
||||
#include "util/testharness.h" |
||||
#include "util/random.h" |
||||
#include "utilities/merge_operators.h" |
||||
#include "utilities/cassandra/cassandra_compaction_filter.h" |
||||
#include "utilities/cassandra/merge_operator.h" |
||||
#include "utilities/cassandra/test_utils.h" |
||||
|
||||
using namespace rocksdb; |
||||
|
||||
namespace rocksdb { |
||||
namespace cassandra { |
||||
|
||||
// Path to the database on file system
|
||||
const std::string kDbName = test::TmpDir() + "/cassandra_functional_test"; |
||||
|
||||
class CassandraStore { |
||||
public: |
||||
explicit CassandraStore(std::shared_ptr<DB> db) |
||||
: db_(db), |
||||
merge_option_(), |
||||
get_option_() { |
||||
assert(db); |
||||
} |
||||
|
||||
bool Append(const std::string& key, const RowValue& val){ |
||||
std::string result; |
||||
val.Serialize(&result); |
||||
Slice valSlice(result.data(), result.size()); |
||||
auto s = db_->Merge(merge_option_, key, valSlice); |
||||
|
||||
if (s.ok()) { |
||||
return true; |
||||
} else { |
||||
std::cerr << "ERROR " << s.ToString() << std::endl; |
||||
return false; |
||||
} |
||||
} |
||||
|
||||
void Flush() { |
||||
dbfull()->TEST_FlushMemTable(); |
||||
dbfull()->TEST_WaitForCompact(); |
||||
} |
||||
|
||||
void Compact() { |
||||
dbfull()->TEST_CompactRange( |
||||
0, nullptr, nullptr, db_->DefaultColumnFamily()); |
||||
} |
||||
|
||||
std::tuple<bool, RowValue> Get(const std::string& key){ |
||||
std::string result; |
||||
auto s = db_->Get(get_option_, key, &result); |
||||
|
||||
if (s.ok()) { |
||||
return std::make_tuple(true, |
||||
RowValue::Deserialize(result.data(), |
||||
result.size())); |
||||
} |
||||
|
||||
if (!s.IsNotFound()) { |
||||
std::cerr << "ERROR " << s.ToString() << std::endl; |
||||
} |
||||
|
||||
return std::make_tuple(false, RowValue(0, 0)); |
||||
} |
||||
|
||||
private: |
||||
std::shared_ptr<DB> db_; |
||||
WriteOptions merge_option_; |
||||
ReadOptions get_option_; |
||||
|
||||
DBImpl* dbfull() { return reinterpret_cast<DBImpl*>(db_.get()); } |
||||
|
||||
}; |
||||
|
||||
class TestCompactionFilterFactory : public CompactionFilterFactory { |
||||
public: |
||||
explicit TestCompactionFilterFactory(bool purge_ttl_on_expiration) |
||||
: purge_ttl_on_expiration_(purge_ttl_on_expiration) {} |
||||
|
||||
virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter( |
||||
const CompactionFilter::Context& context) override { |
||||
return unique_ptr<CompactionFilter>(new CassandraCompactionFilter(purge_ttl_on_expiration_)); |
||||
} |
||||
|
||||
virtual const char* Name() const override { |
||||
return "TestCompactionFilterFactory"; |
||||
} |
||||
|
||||
private: |
||||
bool purge_ttl_on_expiration_; |
||||
}; |
||||
|
||||
|
||||
// The class for unit-testing
|
||||
class CassandraFunctionalTest : public testing::Test { |
||||
public: |
||||
CassandraFunctionalTest() { |
||||
DestroyDB(kDbName, Options()); // Start each test with a fresh DB
|
||||
} |
||||
|
||||
std::shared_ptr<DB> OpenDb() { |
||||
DB* db; |
||||
Options options; |
||||
options.create_if_missing = true; |
||||
options.merge_operator.reset(new CassandraValueMergeOperator()); |
||||
auto* cf_factory = new TestCompactionFilterFactory(purge_ttl_on_expiration_); |
||||
options.compaction_filter_factory.reset(cf_factory); |
||||
EXPECT_OK(DB::Open(options, kDbName, &db)); |
||||
return std::shared_ptr<DB>(db); |
||||
} |
||||
|
||||
bool purge_ttl_on_expiration_ = false; |
||||
}; |
||||
|
||||
// THE TEST CASES BEGIN HERE
|
||||
|
||||
TEST_F(CassandraFunctionalTest, SimpleMergeTest) { |
||||
CassandraStore store(OpenDb()); |
||||
|
||||
store.Append("k1", CreateTestRowValue({ |
||||
std::make_tuple(kTombstone, 0, 5), |
||||
std::make_tuple(kColumn, 1, 8), |
||||
std::make_tuple(kExpiringColumn, 2, 5), |
||||
})); |
||||
store.Append("k1",CreateTestRowValue({ |
||||
std::make_tuple(kColumn, 0, 2), |
||||
std::make_tuple(kExpiringColumn, 1, 5), |
||||
std::make_tuple(kTombstone, 2, 7), |
||||
std::make_tuple(kExpiringColumn, 7, 17), |
||||
})); |
||||
store.Append("k1", CreateTestRowValue({ |
||||
std::make_tuple(kExpiringColumn, 0, 6), |
||||
std::make_tuple(kTombstone, 1, 5), |
||||
std::make_tuple(kColumn, 2, 4), |
||||
std::make_tuple(kTombstone, 11, 11), |
||||
})); |
||||
|
||||
auto ret = store.Get("k1"); |
||||
|
||||
ASSERT_TRUE(std::get<0>(ret)); |
||||
RowValue& merged = std::get<1>(ret); |
||||
EXPECT_EQ(merged.columns_.size(), 5); |
||||
VerifyRowValueColumns(merged.columns_, 0, kExpiringColumn, 0, 6); |
||||
VerifyRowValueColumns(merged.columns_, 1, kColumn, 1, 8); |
||||
VerifyRowValueColumns(merged.columns_, 2, kTombstone, 2, 7); |
||||
VerifyRowValueColumns(merged.columns_, 3, kExpiringColumn, 7, 17); |
||||
VerifyRowValueColumns(merged.columns_, 4, kTombstone, 11, 11); |
||||
} |
||||
|
||||
TEST_F(CassandraFunctionalTest, |
||||
CompactionShouldConvertExpiredColumnsToTombstone) { |
||||
CassandraStore store(OpenDb()); |
||||
int64_t now= time(nullptr); |
||||
|
||||
store.Append("k1", CreateTestRowValue({ |
||||
std::make_tuple(kExpiringColumn, 0, ToMicroSeconds(now - kTtl - 20)), //expired
|
||||
std::make_tuple(kExpiringColumn, 1, ToMicroSeconds(now - kTtl + 10)), // not expired
|
||||
std::make_tuple(kTombstone, 3, ToMicroSeconds(now)) |
||||
})); |
||||
|
||||
store.Flush(); |
||||
|
||||
store.Append("k1",CreateTestRowValue({ |
||||
std::make_tuple(kExpiringColumn, 0, ToMicroSeconds(now - kTtl - 10)), //expired
|
||||
std::make_tuple(kColumn, 2, ToMicroSeconds(now)) |
||||
})); |
||||
|
||||
store.Flush(); |
||||
store.Compact(); |
||||
|
||||
auto ret = store.Get("k1"); |
||||
ASSERT_TRUE(std::get<0>(ret)); |
||||
RowValue& merged = std::get<1>(ret); |
||||
EXPECT_EQ(merged.columns_.size(), 4); |
||||
VerifyRowValueColumns(merged.columns_, 0, kTombstone, 0, ToMicroSeconds(now - 10)); |
||||
VerifyRowValueColumns(merged.columns_, 1, kExpiringColumn, 1, ToMicroSeconds(now - kTtl + 10)); |
||||
VerifyRowValueColumns(merged.columns_, 2, kColumn, 2, ToMicroSeconds(now)); |
||||
VerifyRowValueColumns(merged.columns_, 3, kTombstone, 3, ToMicroSeconds(now)); |
||||
} |
||||
|
||||
|
||||
TEST_F(CassandraFunctionalTest, |
||||
CompactionShouldPurgeExpiredColumnsIfPurgeTtlIsOn) { |
||||
purge_ttl_on_expiration_ = true; |
||||
CassandraStore store(OpenDb()); |
||||
int64_t now = time(nullptr); |
||||
|
||||
store.Append("k1", CreateTestRowValue({ |
||||
std::make_tuple(kExpiringColumn, 0, ToMicroSeconds(now - kTtl - 20)), //expired
|
||||
std::make_tuple(kExpiringColumn, 1, ToMicroSeconds(now)), // not expired
|
||||
std::make_tuple(kTombstone, 3, ToMicroSeconds(now)) |
||||
})); |
||||
|
||||
store.Flush(); |
||||
|
||||
store.Append("k1",CreateTestRowValue({ |
||||
std::make_tuple(kExpiringColumn, 0, ToMicroSeconds(now - kTtl - 10)), //expired
|
||||
std::make_tuple(kColumn, 2, ToMicroSeconds(now)) |
||||
})); |
||||
|
||||
store.Flush(); |
||||
store.Compact(); |
||||
|
||||
auto ret = store.Get("k1"); |
||||
ASSERT_TRUE(std::get<0>(ret)); |
||||
RowValue& merged = std::get<1>(ret); |
||||
EXPECT_EQ(merged.columns_.size(), 3); |
||||
VerifyRowValueColumns(merged.columns_, 0, kExpiringColumn, 1, ToMicroSeconds(now)); |
||||
VerifyRowValueColumns(merged.columns_, 1, kColumn, 2, ToMicroSeconds(now)); |
||||
VerifyRowValueColumns(merged.columns_, 2, kTombstone, 3, ToMicroSeconds(now)); |
||||
} |
||||
|
||||
TEST_F(CassandraFunctionalTest, |
||||
CompactionShouldRemoveRowWhenAllColumnsExpiredIfPurgeTtlIsOn) { |
||||
purge_ttl_on_expiration_ = true; |
||||
CassandraStore store(OpenDb()); |
||||
int64_t now = time(nullptr); |
||||
|
||||
store.Append("k1", CreateTestRowValue({ |
||||
std::make_tuple(kExpiringColumn, 0, ToMicroSeconds(now - kTtl - 20)), |
||||
std::make_tuple(kExpiringColumn, 1, ToMicroSeconds(now - kTtl - 20)), |
||||
})); |
||||
|
||||
store.Flush(); |
||||
|
||||
store.Append("k1",CreateTestRowValue({ |
||||
std::make_tuple(kExpiringColumn, 0, ToMicroSeconds(now - kTtl - 10)), |
||||
})); |
||||
|
||||
store.Flush(); |
||||
store.Compact(); |
||||
ASSERT_FALSE(std::get<0>(store.Get("k1"))); |
||||
} |
||||
|
||||
} // namespace cassandra
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) { |
||||
::testing::InitGoogleTest(&argc, argv); |
||||
return RUN_ALL_TESTS(); |
||||
} |
||||
@ -1,134 +0,0 @@ |
||||
// Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
// This source code is also licensed under the GPLv2 license found in the
|
||||
// COPYING file in the root directory of this source tree.
|
||||
|
||||
#include <iostream> |
||||
|
||||
#include "rocksdb/db.h" |
||||
#include "rocksdb/merge_operator.h" |
||||
#include "rocksdb/utilities/db_ttl.h" |
||||
#include "util/testharness.h" |
||||
#include "util/random.h" |
||||
#include "utilities/merge_operators.h" |
||||
#include "utilities/merge_operators/cassandra/merge_operator.h" |
||||
#include "utilities/merge_operators/cassandra/test_utils.h" |
||||
|
||||
using namespace rocksdb; |
||||
|
||||
namespace rocksdb { |
||||
namespace cassandra { |
||||
|
||||
// Path to the database on file system
|
||||
const std::string kDbName = test::TmpDir() + "/cassandramerge_test"; |
||||
|
||||
class CassandraStore { |
||||
public: |
||||
explicit CassandraStore(std::shared_ptr<DB> db) |
||||
: db_(db), |
||||
merge_option_(), |
||||
get_option_() { |
||||
assert(db); |
||||
} |
||||
|
||||
bool Append(const std::string& key, const RowValue& val){ |
||||
std::string result; |
||||
val.Serialize(&result); |
||||
Slice valSlice(result.data(), result.size()); |
||||
auto s = db_->Merge(merge_option_, key, valSlice); |
||||
|
||||
if (s.ok()) { |
||||
return true; |
||||
} else { |
||||
std::cerr << "ERROR " << s.ToString() << std::endl; |
||||
return false; |
||||
} |
||||
} |
||||
|
||||
std::tuple<bool, RowValue> Get(const std::string& key){ |
||||
std::string result; |
||||
auto s = db_->Get(get_option_, key, &result); |
||||
|
||||
if (s.ok()) { |
||||
return std::make_tuple(true, |
||||
RowValue::Deserialize(result.data(), |
||||
result.size())); |
||||
} |
||||
|
||||
if (!s.IsNotFound()) { |
||||
std::cerr << "ERROR " << s.ToString() << std::endl; |
||||
} |
||||
|
||||
return std::make_tuple(false, RowValue(0, 0)); |
||||
} |
||||
|
||||
private: |
||||
std::shared_ptr<DB> db_; |
||||
WriteOptions merge_option_; |
||||
ReadOptions get_option_; |
||||
}; |
||||
|
||||
|
||||
// The class for unit-testing
|
||||
class CassandraMergeTest : public testing::Test { |
||||
public: |
||||
CassandraMergeTest() { |
||||
DestroyDB(kDbName, Options()); // Start each test with a fresh DB
|
||||
} |
||||
|
||||
std::shared_ptr<DB> OpenDb() { |
||||
DB* db; |
||||
Options options; |
||||
options.create_if_missing = true; |
||||
options.merge_operator.reset(new CassandraValueMergeOperator()); |
||||
EXPECT_OK(DB::Open(options, kDbName, &db)); |
||||
return std::shared_ptr<DB>(db); |
||||
} |
||||
}; |
||||
|
||||
// THE TEST CASES BEGIN HERE
|
||||
|
||||
TEST_F(CassandraMergeTest, SimpleTest) { |
||||
auto db = OpenDb(); |
||||
CassandraStore store(db); |
||||
|
||||
store.Append("k1", CreateTestRowValue({ |
||||
std::make_tuple(kTombstone, 0, 5), |
||||
std::make_tuple(kColumn, 1, 8), |
||||
std::make_tuple(kExpiringColumn, 2, 5), |
||||
})); |
||||
store.Append("k1",CreateTestRowValue({ |
||||
std::make_tuple(kColumn, 0, 2), |
||||
std::make_tuple(kExpiringColumn, 1, 5), |
||||
std::make_tuple(kTombstone, 2, 7), |
||||
std::make_tuple(kExpiringColumn, 7, 17), |
||||
})); |
||||
store.Append("k1", CreateTestRowValue({ |
||||
std::make_tuple(kExpiringColumn, 0, 6), |
||||
std::make_tuple(kTombstone, 1, 5), |
||||
std::make_tuple(kColumn, 2, 4), |
||||
std::make_tuple(kTombstone, 11, 11), |
||||
})); |
||||
|
||||
auto ret = store.Get("k1"); |
||||
|
||||
ASSERT_TRUE(std::get<0>(ret)); |
||||
RowValue& merged = std::get<1>(ret); |
||||
EXPECT_EQ(merged.columns_.size(), 5); |
||||
VerifyRowValueColumns(merged.columns_, 0, kExpiringColumn, 0, 6); |
||||
VerifyRowValueColumns(merged.columns_, 1, kColumn, 1, 8); |
||||
VerifyRowValueColumns(merged.columns_, 2, kTombstone, 2, 7); |
||||
VerifyRowValueColumns(merged.columns_, 3, kExpiringColumn, 7, 17); |
||||
VerifyRowValueColumns(merged.columns_, 4, kTombstone, 11, 11); |
||||
} |
||||
|
||||
|
||||
} // namespace cassandra
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) { |
||||
::testing::InitGoogleTest(&argc, argv); |
||||
return RUN_ALL_TESTS(); |
||||
} |
||||
Loading…
Reference in new issue